From 80b3dc4806b9beb13ed4b009c1a471387aea912a Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 11:30:42 +0200 Subject: [PATCH 01/20] Add ICE dust removal toggle to Linear Output Fourth correction toggle that applies IR-based dust and scratch correction before writing. Uses the file's retouch config (method and threshold). Visible only when the source has an IR channel. Supports both NegPy and OpenICE methods. --- negpy/desktop/controller.py | 2 + negpy/desktop/session.py | 4 +- negpy/desktop/view/sidebar/export.py | 28 +++++++++++-- negpy/services/export/linear_output.py | 57 ++++++++++++++++++++++++-- 4 files changed, 83 insertions(+), 8 deletions(-) diff --git a/negpy/desktop/controller.py b/negpy/desktop/controller.py index bcb3ac32..2537ee26 100644 --- a/negpy/desktop/controller.py +++ b/negpy/desktop/controller.py @@ -3505,6 +3505,8 @@ def request_linear_output_export(self, files: list[dict] | None = None) -> None: apply_wb=self.state.linear_apply_wb, apply_flatfield=self.state.linear_apply_flatfield, apply_sensor=self.state.linear_apply_sensor, + apply_ice=self.state.linear_apply_ice, + retouch=params.retouch, ) exported += 1 except Exception as e: diff --git a/negpy/desktop/session.py b/negpy/desktop/session.py index ecf2fa88..c5de5bf4 100644 --- a/negpy/desktop/session.py +++ b/negpy/desktop/session.py @@ -172,6 +172,7 @@ class AppState: linear_apply_wb: bool = False linear_apply_flatfield: bool = False linear_apply_sensor: bool = False + linear_apply_ice: bool = False @property def local_hidden_masks(self) -> set: @@ -500,7 +501,7 @@ def __init__(self, repo: StorageRepository): saved_linear_output = self.repo.get_global_setting("linear_output") if saved_linear_output is not None: self.state.linear_output = bool(saved_linear_output) - for key in ("linear_apply_wb", "linear_apply_flatfield", "linear_apply_sensor"): + for key in ("linear_apply_wb", "linear_apply_flatfield", "linear_apply_sensor", "linear_apply_ice"): val = self.repo.get_global_setting(key) if val is not None: setattr(self.state, key, bool(val)) @@ -585,6 +586,7 @@ def save_flat_output_prefs(self) -> None: self.repo.save_global_setting("linear_apply_wb", self.state.linear_apply_wb) self.repo.save_global_setting("linear_apply_flatfield", self.state.linear_apply_flatfield) self.repo.save_global_setting("linear_apply_sensor", self.state.linear_apply_sensor) + self.repo.save_global_setting("linear_apply_ice", self.state.linear_apply_ice) def _apply_sticky_settings(self, config: WorkspaceConfig, only_global: bool = False) -> WorkspaceConfig: """ diff --git a/negpy/desktop/view/sidebar/export.py b/negpy/desktop/view/sidebar/export.py index e79393b6..5e3cff70 100644 --- a/negpy/desktop/view/sidebar/export.py +++ b/negpy/desktop/view/sidebar/export.py @@ -594,6 +594,13 @@ def _add_flat_master_section(self) -> None: self.linear_sensor_checkbox.toggled.connect(self._on_linear_correction_changed) box.addWidget(self.linear_sensor_checkbox) + self.linear_ice_checkbox = QCheckBox("Apply ICE dust removal") + self.linear_ice_checkbox.setToolTip("Apply IR-based dust and scratch correction") + self.linear_ice_checkbox.setChecked(self.state.linear_apply_ice) + self.linear_ice_checkbox.setVisible(False) + self.linear_ice_checkbox.toggled.connect(self._on_linear_correction_changed) + box.addWidget(self.linear_ice_checkbox) + self.linear_corrections_hint = hint_label( "Corrections are baked in and cannot be undone from the exported file. Re-export from the original RAW to get uncorrected data." ) @@ -621,6 +628,7 @@ def _sync_flat_enabled(self) -> None: self.linear_wb_checkbox.setVisible(False) self.linear_flatfield_checkbox.setVisible(False) self.linear_sensor_checkbox.setVisible(False) + self.linear_ice_checkbox.setVisible(False) self.linear_corrections_hint.setVisible(False) if hasattr(self, "_presets_section"): self._presets_section.setVisible(not linear_on) @@ -704,10 +712,18 @@ def _refresh_linear_expansion_combo(self) -> None: self._current_expansion_source_type = source_type is_camera = source_type == "camera" - self.linear_corrections_label.setVisible(is_camera) + has_ir = self.state.has_ir + show_corrections = is_camera or has_ir + self.linear_corrections_label.setVisible(show_corrections) self.linear_wb_checkbox.setVisible(is_camera) self.linear_flatfield_checkbox.setVisible(is_camera) self.linear_sensor_checkbox.setVisible(is_camera) + self.linear_ice_checkbox.setVisible(has_ir) + self.linear_ice_checkbox.setEnabled(has_ir) + if not has_ir: + self.linear_ice_checkbox.setToolTip("Source has no IR channel") + else: + self.linear_ice_checkbox.setToolTip("Apply IR-based dust and scratch correction") has_flatfield = bool(self.state.config.flatfield.apply and self.state.config.flatfield.profile_id) self.linear_flatfield_checkbox.setEnabled(has_flatfield) @@ -727,8 +743,9 @@ def _refresh_linear_expansion_combo(self) -> None: self.state.linear_apply_wb or (self.state.linear_apply_flatfield and has_flatfield) or (self.state.linear_apply_sensor and has_matrix) + or (self.state.linear_apply_ice and has_ir) ) - self.linear_corrections_hint.setVisible(is_camera and any_on) + self.linear_corrections_hint.setVisible(show_corrections and any_on) def _on_linear_expansion_changed(self, index: int) -> None: source_type = getattr(self, "_current_expansion_source_type", "unsupported") @@ -740,8 +757,11 @@ def _on_linear_correction_changed(self, _checked: bool) -> None: self.state.linear_apply_wb = self.linear_wb_checkbox.isChecked() self.state.linear_apply_flatfield = self.linear_flatfield_checkbox.isChecked() self.state.linear_apply_sensor = self.linear_sensor_checkbox.isChecked() + self.state.linear_apply_ice = self.linear_ice_checkbox.isChecked() self.controller.session.save_flat_output_prefs() - any_on = self.state.linear_apply_wb or self.state.linear_apply_flatfield or self.state.linear_apply_sensor + any_on = ( + self.state.linear_apply_wb or self.state.linear_apply_flatfield or self.state.linear_apply_sensor or self.state.linear_apply_ice + ) self.linear_corrections_hint.setVisible(any_on) def _on_flat_peek_changed(self, active: bool) -> None: @@ -1179,6 +1199,7 @@ def sync_ui(self) -> None: self.linear_wb_checkbox.setChecked(self.state.linear_apply_wb) self.linear_flatfield_checkbox.setChecked(self.state.linear_apply_flatfield) self.linear_sensor_checkbox.setChecked(self.state.linear_apply_sensor) + self.linear_ice_checkbox.setChecked(self.state.linear_apply_ice) finally: self.block_signals(False) @@ -1205,6 +1226,7 @@ def block_signals(self, blocked: bool) -> None: self.linear_wb_checkbox, self.linear_flatfield_checkbox, self.linear_sensor_checkbox, + self.linear_ice_checkbox, ] for w in widgets: w.blockSignals(blocked) diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index f5034620..9edf72aa 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -18,6 +18,7 @@ import tifffile as _tifffile from negpy.features.flatfield.logic import apply_flatfield as _apply_flatfield_correction +from negpy.features.retouch.models import IR_METHOD_OPENICE, RetouchConfig from negpy.features.flatfield.models import FlatFieldConfig from negpy.features.geometry.models import GeometryConfig from negpy.features.process.models import ProcessConfig @@ -170,6 +171,43 @@ def _apply_white_balance(f32: np.ndarray, wb: _CameraWB) -> np.ndarray: return f32 +def _apply_ice(rgb: np.ndarray, ir: np.ndarray, retouch: RetouchConfig) -> np.ndarray: + """Apply IR dust correction to a linear RGB buffer using the IR channel.""" + if retouch.ir_method == IR_METHOD_OPENICE: + from negpy.features.retouch import openice + + corrected, _, _, _ = openice.run(rgb, ir, float(retouch.ir_threshold), None) + return corrected + + from negpy.features.retouch.logic import ( + apply_ir_attenuation, + apply_ir_reconstruction, + downsample_ir, + ir_defect_score, + ir_detect_cutoff, + ir_detect_target, + ir_ratio_and_gain, + ) + + target = ir_detect_target(max(rgb.shape[:2]), max(rgb.shape[:2])) + ir_det = downsample_ir(np.ascontiguousarray(ir, dtype=np.float32), target) + h, w = rgb.shape[:2] + if max(h, w) > target: + import cv2 + + s = target / max(h, w) + rgb_det = cv2.resize(rgb, (max(1, round(w * s)), max(1, round(h * s))), interpolation=cv2.INTER_AREA) + else: + rgb_det = rgb + ratio_det, gain_det, degenerate, _ = ir_ratio_and_gain(ir_det, rgb_det) + if degenerate: + return rgb + score_det = ir_defect_score(ratio_det, ir_detect_cutoff(retouch.ir_threshold, retouch.ir_attenuation)) + out = apply_ir_attenuation(rgb, gain_det) if retouch.ir_attenuation else rgb + out = apply_ir_reconstruction(out, score_det) + return out + + def _decode_linear( file_path: str, geometry: Optional[GeometryConfig] = None, @@ -530,6 +568,7 @@ def _write_tiff( wb_applied: bool = False, flatfield_applied: bool = False, sensor_applied: bool = False, + ice_applied: bool = False, ) -> None: """Write a float32 buffer as an untagged 16-bit TIFF to *dest* (path or file-like).""" u16 = _to_uint16_jit(np.ascontiguousarray(f32, dtype=np.float32)) @@ -547,7 +586,7 @@ def _write_tiff( parts.append(f"no WB applied (as-shot: {r:.3f} {g:.3f} {b:.3f})") else: parts.append("no WB applied") - corrections = [s for s, on in (("flatfield", flatfield_applied), ("sensor", sensor_applied)) if on] + corrections = [s for s, on in (("flatfield", flatfield_applied), ("sensor", sensor_applied), ("ICE", ice_applied)) if on] if corrections: parts.append(f"corrections: {', '.join(corrections)}") parts.append("no color management") @@ -608,6 +647,8 @@ def export_linear_output( apply_wb: bool = False, apply_flatfield: bool = False, apply_sensor: bool = False, + apply_ice: bool = False, + retouch: Optional[RetouchConfig] = None, ) -> None: """Decode *file_path* and write an untagged linear 16-bit TIFF to *output_path*. @@ -624,9 +665,11 @@ def export_linear_output( applicable), applies flatfield and sensor correction per-part, then assembles via stitch_composite. - *apply_wb*, *apply_flatfield*, *apply_sensor*: optional per-step corrections. - When False (default), the raw dump is written unchanged. When True, the - corresponding correction is applied before writing. + *apply_wb*, *apply_flatfield*, *apply_sensor*, *apply_ice*: optional per-step + corrections. When False (default), the raw dump is written unchanged. When + True, the corresponding correction is applied before writing. *apply_ice* + requires an IR channel in the source and a *retouch* config; it uses the + configured IR method and threshold. If the source has an IR channel, it is written as a separate grayscale TIFF with an ``_ir`` suffix next to the RGB output. @@ -645,6 +688,11 @@ def export_linear_output( apply_flatfield=apply_flatfield, apply_sensor=apply_sensor, ) + ice_applied = False + if apply_ice and ir is not None: + ret = retouch if retouch is not None else RetouchConfig() + f32 = _apply_ice(f32, ir, ret) + ice_applied = True is_stitch = stitch is not None and stitch.stitch_enabled and stitch.stitch_paths os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True) _write_tiff( @@ -659,6 +707,7 @@ def export_linear_output( wb_applied=apply_wb, flatfield_applied=apply_flatfield or is_stitch, sensor_applied=apply_sensor or is_stitch, + ice_applied=ice_applied, ) if ir is not None: From 72dda4171c18a7a7130111e564bb6ba026c3d864 Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 12:41:49 +0200 Subject: [PATCH 02/20] Add TIFF linear output with manual gamma linearization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TIFF files are now a supported Linear Output source. A manual "Input gamma" dropdown (Linear, 1.8, 2.2, 2.4, 2.6, sRGB, L*, Rec.709) lets the user declare the actual encoding so it can be inverted to linear before export — apps often tag wrong ICC profiles, so auto-detection is unreliable. The chosen gamma is persisted across sessions and recorded in the exported TIFF description metadata. Also fixes a KeyError when exporting a single file (missing hash in the asset dict). --- negpy/desktop/controller.py | 3 +- negpy/desktop/session.py | 5 + negpy/desktop/view/sidebar/export.py | 52 ++++++++++ negpy/services/export/linear_output.py | 94 ++++++++++++++++- tests/test_linear_output.py | 137 ++++++++++++++++++++++++- 5 files changed, 285 insertions(+), 6 deletions(-) diff --git a/negpy/desktop/controller.py b/negpy/desktop/controller.py index 2537ee26..d5f549aa 100644 --- a/negpy/desktop/controller.py +++ b/negpy/desktop/controller.py @@ -3471,7 +3471,7 @@ def request_linear_output_export(self, files: list[dict] | None = None) -> None: if not is_linear_output_supported(file_path): self.set_status("Linear Output is not supported for this file type", 4000) return - files = [{"path": file_path, "name": os.path.basename(file_path)}] + files = [{"path": file_path, "name": os.path.basename(file_path), "hash": self.state.current_file_hash}] supported = [f for f in files if is_linear_output_supported(f["path"])] if not supported: @@ -3507,6 +3507,7 @@ def request_linear_output_export(self, files: list[dict] | None = None) -> None: apply_sensor=self.state.linear_apply_sensor, apply_ice=self.state.linear_apply_ice, retouch=params.retouch, + gamma_key=self.state.linear_gamma_key, ) exported += 1 except Exception as e: diff --git a/negpy/desktop/session.py b/negpy/desktop/session.py index c5de5bf4..ea8e0a25 100644 --- a/negpy/desktop/session.py +++ b/negpy/desktop/session.py @@ -173,6 +173,7 @@ class AppState: linear_apply_flatfield: bool = False linear_apply_sensor: bool = False linear_apply_ice: bool = False + linear_gamma_key: str = "linear" @property def local_hidden_masks(self) -> set: @@ -505,6 +506,9 @@ def __init__(self, repo: StorageRepository): val = self.repo.get_global_setting(key) if val is not None: setattr(self.state, key, bool(val)) + saved_gamma = self.repo.get_global_setting("linear_gamma_key") + if saved_gamma is not None: + self.state.linear_gamma_key = str(saved_gamma) self.state.export_presets = self.repo.load_export_presets() @@ -587,6 +591,7 @@ def save_flat_output_prefs(self) -> None: self.repo.save_global_setting("linear_apply_flatfield", self.state.linear_apply_flatfield) self.repo.save_global_setting("linear_apply_sensor", self.state.linear_apply_sensor) self.repo.save_global_setting("linear_apply_ice", self.state.linear_apply_ice) + self.repo.save_global_setting("linear_gamma_key", self.state.linear_gamma_key) def _apply_sticky_settings(self, config: WorkspaceConfig, only_global: bool = False) -> WorkspaceConfig: """ diff --git a/negpy/desktop/view/sidebar/export.py b/negpy/desktop/view/sidebar/export.py index 5e3cff70..abb6ab3e 100644 --- a/negpy/desktop/view/sidebar/export.py +++ b/negpy/desktop/view/sidebar/export.py @@ -601,6 +601,22 @@ def _add_flat_master_section(self) -> None: self.linear_ice_checkbox.toggled.connect(self._on_linear_correction_changed) box.addWidget(self.linear_ice_checkbox) + gamma_row = QHBoxLayout() + gamma_row.setContentsMargins(0, 0, 0, 0) + self.linear_gamma_label = field_label("Input gamma") + gamma_row.addWidget(self.linear_gamma_label) + self.linear_gamma_combo = QComboBox() + self.linear_gamma_combo.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed) + gamma_row.addWidget(self.linear_gamma_combo) + self.linear_gamma_row = QWidget() + self.linear_gamma_row.setLayout(gamma_row) + self.linear_gamma_row.setVisible(False) + box.addWidget(self.linear_gamma_row) + self.linear_gamma_hint = hint_label("Select the gamma encoding of the input TIFF so it can be linearized before export.") + self.linear_gamma_hint.setVisible(False) + box.addWidget(self.linear_gamma_hint) + self.linear_gamma_combo.currentIndexChanged.connect(self._on_linear_gamma_changed) + self.linear_corrections_hint = hint_label( "Corrections are baked in and cannot be undone from the exported file. Re-export from the original RAW to get uncorrected data." ) @@ -623,6 +639,9 @@ def _sync_flat_enabled(self) -> None: self.linear_expansion_hint.setVisible(linear_on) if linear_on: self._refresh_linear_expansion_combo() + if hasattr(self, "linear_gamma_row") and not linear_on: + self.linear_gamma_row.setVisible(False) + self.linear_gamma_hint.setVisible(False) if hasattr(self, "linear_corrections_label") and not linear_on: self.linear_corrections_label.setVisible(False) self.linear_wb_checkbox.setVisible(False) @@ -681,6 +700,7 @@ def _on_linear_output_changed(self, enabled: bool) -> None: "pakon_f335": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "dng": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "camera": [], + "tiff": [], "unsupported": [], } @@ -711,6 +731,12 @@ def _refresh_linear_expansion_combo(self) -> None: combo.blockSignals(False) self._current_expansion_source_type = source_type + is_tiff = source_type == "tiff" + self.linear_gamma_row.setVisible(is_tiff) + self.linear_gamma_hint.setVisible(is_tiff) + if is_tiff: + self._refresh_linear_gamma_combo() + is_camera = source_type == "camera" has_ir = self.state.has_ir show_corrections = is_camera or has_ir @@ -753,6 +779,30 @@ def _on_linear_expansion_changed(self, index: int) -> None: if 0 <= index < len(options): self.state.linear_expansion = options[index][1] + def _refresh_linear_gamma_combo(self) -> None: + from negpy.services.export.linear_output import TIFF_GAMMA_OPTIONS + + combo = self.linear_gamma_combo + combo.blockSignals(True) + combo.clear() + for key, label in TIFF_GAMMA_OPTIONS: + combo.addItem(label, key) + current = self.state.linear_gamma_key + for i, (key, _label) in enumerate(TIFF_GAMMA_OPTIONS): + if key == current: + combo.setCurrentIndex(i) + break + else: + combo.setCurrentIndex(0) + combo.blockSignals(False) + + def _on_linear_gamma_changed(self, index: int) -> None: + from negpy.services.export.linear_output import TIFF_GAMMA_OPTIONS + + if 0 <= index < len(TIFF_GAMMA_OPTIONS): + self.state.linear_gamma_key = TIFF_GAMMA_OPTIONS[index][0] + self.controller.session.save_flat_output_prefs() + def _on_linear_correction_changed(self, _checked: bool) -> None: self.state.linear_apply_wb = self.linear_wb_checkbox.isChecked() self.state.linear_apply_flatfield = self.linear_flatfield_checkbox.isChecked() @@ -1200,6 +1250,7 @@ def sync_ui(self) -> None: self.linear_flatfield_checkbox.setChecked(self.state.linear_apply_flatfield) self.linear_sensor_checkbox.setChecked(self.state.linear_apply_sensor) self.linear_ice_checkbox.setChecked(self.state.linear_apply_ice) + self._refresh_linear_gamma_combo() finally: self.block_signals(False) @@ -1227,6 +1278,7 @@ def block_signals(self, blocked: bool) -> None: self.linear_flatfield_checkbox, self.linear_sensor_checkbox, self.linear_ice_checkbox, + self.linear_gamma_combo, ] for w in widgets: w.blockSignals(blocked) diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index 9edf72aa..bce20157 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -111,6 +111,10 @@ def _is_camera_raw(file_path: str) -> bool: return ext in SUPPORTED_RAW_EXTENSIONS +def _is_tiff(file_path: str) -> bool: + return os.path.splitext(file_path)[1].lower() in SUPPORTED_TIFF_EXTENSIONS + + def is_linear_output_supported(file_path: str) -> bool: if PakonLoader.can_handle(file_path): return True @@ -118,13 +122,15 @@ def is_linear_output_supported(file_path: str) -> bool: return _is_linearraw_dng(file_path) or _is_camera_raw(file_path) if _is_camera_raw(file_path): return True + if _is_tiff(file_path): + return True return False def linear_output_source_type(file_path: str) -> str: """Classify a file for Linear Output expansion options. - Returns ``"pakon"``, ``"dng"``, ``"camera"``, or ``"unsupported"``. + Returns ``"pakon"``, ``"dng"``, ``"camera"``, ``"tiff"``, or ``"unsupported"``. """ if PakonLoader.can_handle(file_path): return "pakon_f335" if _is_pakon_f335(file_path) else "pakon" @@ -132,6 +138,8 @@ def linear_output_source_type(file_path: str) -> str: return "dng" if _is_camera_raw(file_path): return "camera" + if _is_tiff(file_path): + return "tiff" return "unsupported" @@ -161,6 +169,41 @@ def _apply_geometry(f32: np.ndarray, orientation: int, geometry: Optional[Geomet return f32 +TIFF_GAMMA_OPTIONS: list[tuple[str, str]] = [ + ("linear", "Linear (1.0)"), + ("1.8", "Gamma 1.8"), + ("2.2", "Gamma 2.2"), + ("2.4", "Gamma 2.4"), + ("2.6", "Gamma 2.6"), + ("srgb", "sRGB"), + ("lstar", "L*"), + ("rec709", "Rec.709"), +] + + +def _linearize(f32: np.ndarray, gamma_key: str) -> np.ndarray: + """Reverse a gamma encoding to recover linear-light values.""" + if gamma_key == "linear": + return f32 + f32 = np.clip(f32, 0.0, 1.0) + if gamma_key in ("1.8", "2.2", "2.4", "2.6"): + g = float(gamma_key) + return np.power(f32, g, dtype=np.float32) + if gamma_key == "srgb": + lo = f32 / 12.92 + hi = np.power((f32 + 0.055) / 1.055, 2.4, dtype=np.float32) + return np.where(f32 <= 0.04045, lo, hi).astype(np.float32) + if gamma_key == "rec709": + lo = f32 / 4.5 + hi = np.power((f32 + 0.099) / 1.099, 1.0 / 0.45, dtype=np.float32) + return np.where(f32 <= 0.081, lo, hi).astype(np.float32) + if gamma_key == "lstar": + lo = f32 / 9.0329 + hi = np.power((f32 + 0.16) / 1.16, 3.0, dtype=np.float32) + return np.where(f32 <= 0.08, lo, hi).astype(np.float32) + return f32 + + def _apply_white_balance(f32: np.ndarray, wb: _CameraWB) -> np.ndarray: """Multiply a linear RGB buffer by the as-shot white-balance gains.""" r, _g, b = _normalize_wb_rgb(wb.as_shot) @@ -219,6 +262,7 @@ def _decode_linear( apply_wb: bool = False, apply_flatfield: bool = False, apply_sensor: bool = False, + gamma_key: str = "linear", ) -> tuple[np.ndarray, Optional[np.ndarray], Optional[_CameraWB], _SourceMeta]: """Decode to an oriented float32 buffer. Returns (rgb, ir_or_none, camera_wb_or_none, source_meta).""" if stitch is not None and stitch.stitch_enabled and stitch.stitch_paths: @@ -260,6 +304,10 @@ def _decode_linear( if apply_wb and wb is not None: rgb = _apply_white_balance(rgb, wb) return rgb, ir, wb, merged + if _is_tiff(file_path): + meta = _read_source_meta_tiff(file_path) + rgb, ir = _decode_tiff(file_path, geometry, gamma_key=gamma_key) + return rgb, ir, None, meta raise ValueError(f"Linear Output is not supported for this file type: {file_path}") @@ -288,6 +336,41 @@ def _pakon_spec_desc(file_path: str) -> str: return "Unknown" +def _decode_tiff( + file_path: str, + geometry: Optional[GeometryConfig] = None, + gamma_key: str = "linear", +) -> tuple[np.ndarray, Optional[np.ndarray]]: + """Read a TIFF, optionally linearize, strip everything. Returns (rgb, ir_or_none).""" + with _tifffile.TiffFile(file_path) as tif: + page = tif.pages[0] + arr = page.asarray() + samples = page.samplesperpixel + if arr.dtype == np.uint16: + scale = 1.0 / 65535.0 + elif arr.dtype == np.uint8: + scale = 1.0 / 255.0 + elif arr.dtype == np.float32: + scale = 1.0 + else: + scale = 1.0 / float(np.iinfo(arr.dtype).max) if np.issubdtype(arr.dtype, np.integer) else 1.0 + f32 = arr.astype(np.float32) * scale + ir = None + if samples == 4 and f32.ndim == 3 and f32.shape[2] == 4: + ir = f32[:, :, 3] + f32 = f32[:, :, :3] + elif samples == 1 and f32.ndim == 2: + f32 = np.stack([f32, f32, f32], axis=2) + f32 = np.clip(f32, 0.0, 1.0) + if gamma_key != "linear": + f32 = _linearize(f32, gamma_key) + orientation = read_orientation(file_path) + f32 = _apply_geometry(f32, orientation, geometry) + if ir is not None: + ir = _apply_geometry(ir, orientation, geometry) + return f32, ir + + def _decode_pakon(file_path: str, geometry: Optional[GeometryConfig] = None, expansion: Optional[float] = None) -> tuple[np.ndarray, None]: loader = PakonLoader() ctx_mgr, metadata = loader.load(file_path) @@ -553,6 +636,8 @@ def _source_format_label( if rgbscan is not None and is_rgb_triplet(rgbscan): return "camera RAW (RGB triplet)" return "camera RAW" + if _is_tiff(file_path): + return "TIFF" return "unknown" @@ -569,6 +654,7 @@ def _write_tiff( flatfield_applied: bool = False, sensor_applied: bool = False, ice_applied: bool = False, + gamma_key: str = "linear", ) -> None: """Write a float32 buffer as an untagged 16-bit TIFF to *dest* (path or file-like).""" u16 = _to_uint16_jit(np.ascontiguousarray(f32, dtype=np.float32)) @@ -578,6 +664,9 @@ def _write_tiff( parts.append(f"expansion: x{expansion:g}") else: parts.append("no scaling") + if gamma_key != "linear": + gamma_labels = dict(TIFF_GAMMA_OPTIONS) + parts.append(f"linearized from {gamma_labels.get(gamma_key, gamma_key)}") if camera_wb is not None: r, g, b = _normalize_wb_rgb(camera_wb.as_shot) if wb_applied: @@ -649,6 +738,7 @@ def export_linear_output( apply_sensor: bool = False, apply_ice: bool = False, retouch: Optional[RetouchConfig] = None, + gamma_key: str = "linear", ) -> None: """Decode *file_path* and write an untagged linear 16-bit TIFF to *output_path*. @@ -687,6 +777,7 @@ def export_linear_output( apply_wb=apply_wb, apply_flatfield=apply_flatfield, apply_sensor=apply_sensor, + gamma_key=gamma_key, ) ice_applied = False if apply_ice and ir is not None: @@ -708,6 +799,7 @@ def export_linear_output( flatfield_applied=apply_flatfield or is_stitch, sensor_applied=apply_sensor or is_stitch, ice_applied=ice_applied, + gamma_key=gamma_key, ) if ir is not None: diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index a1d4e6d8..a7ce0758 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -13,6 +13,7 @@ from negpy.features.stitch.models import StitchConfig from negpy.kernel.image.logic import apply_exif_orientation from negpy.services.export.linear_output import ( + TIFF_GAMMA_OPTIONS, _CameraWB, _SourceMeta, _apply_white_balance, @@ -20,6 +21,8 @@ _default_pakon_expansion, _effective_expansion, _is_camera_raw, + _is_tiff, + _linearize, _normalize_wb_rgb, _source_format_label, _write_tiff, @@ -76,11 +79,11 @@ def test_pakon_raw_supported(self, tmp_path: str) -> None: path = _make_pakon_raw(str(tmp_path)) assert is_linear_output_supported(path) - def test_regular_tiff_not_supported(self, tmp_path: str) -> None: + def test_regular_tiff_supported(self, tmp_path: str) -> None: path = os.path.join(str(tmp_path), "photo.tiff") arr = np.zeros((10, 10, 3), dtype=np.uint16) tifffile.imwrite(path, arr) - assert not is_linear_output_supported(path) + assert is_linear_output_supported(path) def test_nonexistent_raw_supported_by_extension(self) -> None: """A .raw extension is in SUPPORTED_RAW_EXTENSIONS; support is a format check.""" @@ -152,8 +155,9 @@ def test_pixel_values_roundtrip(self, tmp_path: str) -> None: np.testing.assert_allclose(actual_u16.astype(np.int32), expected_u16.astype(np.int32), atol=1) def test_rejects_unsupported_file(self, tmp_path: str) -> None: - path = os.path.join(str(tmp_path), "photo.tiff") - tifffile.imwrite(path, np.zeros((10, 10, 3), dtype=np.uint16)) + path = os.path.join(str(tmp_path), "photo.jpeg") + with open(path, "wb") as fh: + fh.write(b"\xff\xd8\xff\xe0") out = os.path.join(str(tmp_path), "out.tiff") with pytest.raises(ValueError, match="not supported"): export_linear_output(path, out) @@ -1055,3 +1059,128 @@ def test_description_no_corrections_by_default(self, tmp_path: str) -> None: with tifffile.TiffFile(out) as tf: desc = tf.pages[0].description assert "corrections:" not in desc + + def test_description_includes_gamma_linearization(self, tmp_path: str) -> None: + f32 = np.full((4, 4, 3), 0.5, dtype=np.float32) + out = os.path.join(str(tmp_path), "out.tiff") + _write_tiff(f32, out, "test.tif", gamma_key="2.2") + with tifffile.TiffFile(out) as tf: + desc = tf.pages[0].description + assert "linearized from Gamma 2.2" in desc + + def test_description_no_gamma_for_linear(self, tmp_path: str) -> None: + f32 = np.full((4, 4, 3), 0.5, dtype=np.float32) + out = os.path.join(str(tmp_path), "out.tiff") + _write_tiff(f32, out, "test.tif", gamma_key="linear") + with tifffile.TiffFile(out) as tf: + desc = tf.pages[0].description + assert "linearized" not in desc + + +class TestTiffLinearOutput: + def test_is_tiff_extensions(self) -> None: + assert _is_tiff("scan.tif") + assert _is_tiff("scan.tiff") + assert _is_tiff("scan.TIF") + assert not _is_tiff("scan.dng") + assert not _is_tiff("scan.nef") + + def test_linearize_identity(self) -> None: + data = np.array([0.0, 0.25, 0.5, 0.75, 1.0], dtype=np.float32) + result = _linearize(data, "linear") + np.testing.assert_array_equal(result, data) + + def test_linearize_gamma_22(self) -> None: + data = np.array([0.0, 0.5, 1.0], dtype=np.float32) + result = _linearize(data, "2.2") + np.testing.assert_allclose(result[0], 0.0, atol=1e-7) + np.testing.assert_allclose(result[1], 0.5**2.2, rtol=1e-5) + np.testing.assert_allclose(result[2], 1.0, atol=1e-7) + + def test_linearize_srgb(self) -> None: + result = _linearize(np.array([0.0, 0.04045, 0.5, 1.0], dtype=np.float32), "srgb") + np.testing.assert_allclose(result[0], 0.0, atol=1e-7) + np.testing.assert_allclose(result[1], 0.04045 / 12.92, rtol=1e-5) + np.testing.assert_allclose(result[3], 1.0, atol=1e-7) + + def test_linearize_lstar(self) -> None: + result = _linearize(np.array([0.0, 1.0], dtype=np.float32), "lstar") + np.testing.assert_allclose(result[0], 0.0, atol=1e-7) + np.testing.assert_allclose(result[1], 1.0, atol=1e-7) + + def test_linearize_rec709(self) -> None: + result = _linearize(np.array([0.0, 0.081, 1.0], dtype=np.float32), "rec709") + np.testing.assert_allclose(result[0], 0.0, atol=1e-7) + np.testing.assert_allclose(result[1], 0.081 / 4.5, rtol=1e-5) + np.testing.assert_allclose(result[2], 1.0, atol=1e-7) + + def test_linearize_clamps_input(self) -> None: + data = np.array([-0.1, 1.5], dtype=np.float32) + result = _linearize(data, "2.2") + assert result[0] >= 0.0 + assert result[1] <= 1.0 + + def test_linearize_all_gamma_options_have_keys(self) -> None: + keys = [k for k, _ in TIFF_GAMMA_OPTIONS] + data = np.array([0.5], dtype=np.float32) + for key in keys: + result = _linearize(data, key) + assert result.shape == data.shape + + def test_source_type_tiff(self, tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "scan.tif") + tifffile.imwrite(path, np.zeros((4, 4, 3), dtype=np.uint16)) + assert linear_output_source_type(path) == "tiff" + + def test_tiff_supported(self, tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "scan.tif") + tifffile.imwrite(path, np.zeros((4, 4, 3), dtype=np.uint16)) + assert is_linear_output_supported(path) + + def test_source_format_label_tiff(self) -> None: + assert _source_format_label("scan.tif") == "TIFF" + assert _source_format_label("scan.tiff") == "TIFF" + + def test_decode_tiff_rgb(self, tmp_path: str) -> None: + from negpy.services.export.linear_output import _decode_tiff + + rng = np.random.RandomState(42) + data = rng.randint(0, 65535, size=(10, 10, 3), dtype=np.uint16) + path = os.path.join(str(tmp_path), "rgb.tif") + tifffile.imwrite(path, data) + rgb, ir = _decode_tiff(path) + assert rgb.shape == (10, 10, 3) + assert rgb.dtype == np.float32 + assert ir is None + + def test_decode_tiff_4ch_splits_ir(self, tmp_path: str) -> None: + from negpy.services.export.linear_output import _decode_tiff + + data = np.ones((8, 8, 4), dtype=np.uint16) * 32768 + path = os.path.join(str(tmp_path), "4ch.tif") + tifffile.imwrite(path, data) + rgb, ir = _decode_tiff(path) + assert rgb.shape == (8, 8, 3) + assert ir is not None + assert ir.shape[:2] == (8, 8) + + def test_decode_tiff_applies_gamma(self, tmp_path: str) -> None: + from negpy.services.export.linear_output import _decode_tiff + + data = np.full((4, 4, 3), 32768, dtype=np.uint16) + path = os.path.join(str(tmp_path), "gamma.tif") + tifffile.imwrite(path, data) + rgb_lin, _ = _decode_tiff(path, gamma_key="linear") + rgb_22, _ = _decode_tiff(path, gamma_key="2.2") + assert np.all(rgb_22 < rgb_lin) + + def test_export_tiff_with_gamma(self, tmp_path: str) -> None: + data = np.full((4, 4, 3), 32768, dtype=np.uint16) + src = os.path.join(str(tmp_path), "input.tif") + tifffile.imwrite(src, data) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(src, out, gamma_key="2.2") + with tifffile.TiffFile(out) as tf: + desc = tf.pages[0].description + assert "linearized from Gamma 2.2" in desc + assert "source: TIFF" in desc From fe024bf3b459cddb95430e6bb50cf17d1d596d19 Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 12:52:41 +0200 Subject: [PATCH 03/20] Add TIFF expansion and strip color profiles options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expansion (Off/2×/4×, default off) scales the normalized pixel data before writing — useful when the source TIFF doesn't use the full bit range. Strip color profiles drops all XMP from the exported TIFF so downstream tools see a completely unmanaged file (no ICC profile, no color-space XMP, no EXIF color tags). Make/model/datetime are kept. Noted in the TIFF description as "profiles stripped". --- negpy/desktop/controller.py | 1 + negpy/desktop/session.py | 5 +++ negpy/desktop/view/sidebar/export.py | 17 ++++++++- negpy/services/export/linear_output.py | 17 +++++++-- tests/test_linear_output.py | 52 ++++++++++++++++++++++++++ 5 files changed, 88 insertions(+), 4 deletions(-) diff --git a/negpy/desktop/controller.py b/negpy/desktop/controller.py index d5f549aa..f46c6cb0 100644 --- a/negpy/desktop/controller.py +++ b/negpy/desktop/controller.py @@ -3508,6 +3508,7 @@ def request_linear_output_export(self, files: list[dict] | None = None) -> None: apply_ice=self.state.linear_apply_ice, retouch=params.retouch, gamma_key=self.state.linear_gamma_key, + strip_profiles=self.state.linear_strip_profiles, ) exported += 1 except Exception as e: diff --git a/negpy/desktop/session.py b/negpy/desktop/session.py index ea8e0a25..ef7c3bfc 100644 --- a/negpy/desktop/session.py +++ b/negpy/desktop/session.py @@ -174,6 +174,7 @@ class AppState: linear_apply_sensor: bool = False linear_apply_ice: bool = False linear_gamma_key: str = "linear" + linear_strip_profiles: bool = False @property def local_hidden_masks(self) -> set: @@ -509,6 +510,9 @@ def __init__(self, repo: StorageRepository): saved_gamma = self.repo.get_global_setting("linear_gamma_key") if saved_gamma is not None: self.state.linear_gamma_key = str(saved_gamma) + saved_strip = self.repo.get_global_setting("linear_strip_profiles") + if saved_strip is not None: + self.state.linear_strip_profiles = bool(saved_strip) self.state.export_presets = self.repo.load_export_presets() @@ -592,6 +596,7 @@ def save_flat_output_prefs(self) -> None: self.repo.save_global_setting("linear_apply_sensor", self.state.linear_apply_sensor) self.repo.save_global_setting("linear_apply_ice", self.state.linear_apply_ice) self.repo.save_global_setting("linear_gamma_key", self.state.linear_gamma_key) + self.repo.save_global_setting("linear_strip_profiles", self.state.linear_strip_profiles) def _apply_sticky_settings(self, config: WorkspaceConfig, only_global: bool = False) -> WorkspaceConfig: """ diff --git a/negpy/desktop/view/sidebar/export.py b/negpy/desktop/view/sidebar/export.py index abb6ab3e..e1fc3913 100644 --- a/negpy/desktop/view/sidebar/export.py +++ b/negpy/desktop/view/sidebar/export.py @@ -617,6 +617,13 @@ def _add_flat_master_section(self) -> None: box.addWidget(self.linear_gamma_hint) self.linear_gamma_combo.currentIndexChanged.connect(self._on_linear_gamma_changed) + self.linear_strip_checkbox = QCheckBox("Strip color profiles") + self.linear_strip_checkbox.setToolTip("Remove all ICC profiles, color space tags, and XMP color metadata from the exported TIFF") + self.linear_strip_checkbox.setChecked(self.state.linear_strip_profiles) + self.linear_strip_checkbox.setVisible(False) + self.linear_strip_checkbox.toggled.connect(self._on_linear_strip_changed) + box.addWidget(self.linear_strip_checkbox) + self.linear_corrections_hint = hint_label( "Corrections are baked in and cannot be undone from the exported file. Re-export from the original RAW to get uncorrected data." ) @@ -642,6 +649,7 @@ def _sync_flat_enabled(self) -> None: if hasattr(self, "linear_gamma_row") and not linear_on: self.linear_gamma_row.setVisible(False) self.linear_gamma_hint.setVisible(False) + self.linear_strip_checkbox.setVisible(False) if hasattr(self, "linear_corrections_label") and not linear_on: self.linear_corrections_label.setVisible(False) self.linear_wb_checkbox.setVisible(False) @@ -700,7 +708,7 @@ def _on_linear_output_changed(self, enabled: bool) -> None: "pakon_f335": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "dng": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "camera": [], - "tiff": [], + "tiff": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "unsupported": [], } @@ -734,6 +742,7 @@ def _refresh_linear_expansion_combo(self) -> None: is_tiff = source_type == "tiff" self.linear_gamma_row.setVisible(is_tiff) self.linear_gamma_hint.setVisible(is_tiff) + self.linear_strip_checkbox.setVisible(is_tiff) if is_tiff: self._refresh_linear_gamma_combo() @@ -803,6 +812,10 @@ def _on_linear_gamma_changed(self, index: int) -> None: self.state.linear_gamma_key = TIFF_GAMMA_OPTIONS[index][0] self.controller.session.save_flat_output_prefs() + def _on_linear_strip_changed(self, checked: bool) -> None: + self.state.linear_strip_profiles = checked + self.controller.session.save_flat_output_prefs() + def _on_linear_correction_changed(self, _checked: bool) -> None: self.state.linear_apply_wb = self.linear_wb_checkbox.isChecked() self.state.linear_apply_flatfield = self.linear_flatfield_checkbox.isChecked() @@ -1250,6 +1263,7 @@ def sync_ui(self) -> None: self.linear_flatfield_checkbox.setChecked(self.state.linear_apply_flatfield) self.linear_sensor_checkbox.setChecked(self.state.linear_apply_sensor) self.linear_ice_checkbox.setChecked(self.state.linear_apply_ice) + self.linear_strip_checkbox.setChecked(self.state.linear_strip_profiles) self._refresh_linear_gamma_combo() finally: self.block_signals(False) @@ -1279,6 +1293,7 @@ def block_signals(self, blocked: bool) -> None: self.linear_sensor_checkbox, self.linear_ice_checkbox, self.linear_gamma_combo, + self.linear_strip_checkbox, ] for w in widgets: w.blockSignals(blocked) diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index bce20157..56efa3da 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -306,7 +306,7 @@ def _decode_linear( return rgb, ir, wb, merged if _is_tiff(file_path): meta = _read_source_meta_tiff(file_path) - rgb, ir = _decode_tiff(file_path, geometry, gamma_key=gamma_key) + rgb, ir = _decode_tiff(file_path, geometry, gamma_key=gamma_key, expansion=expansion) return rgb, ir, None, meta raise ValueError(f"Linear Output is not supported for this file type: {file_path}") @@ -340,6 +340,7 @@ def _decode_tiff( file_path: str, geometry: Optional[GeometryConfig] = None, gamma_key: str = "linear", + expansion: Optional[float] = None, ) -> tuple[np.ndarray, Optional[np.ndarray]]: """Read a TIFF, optionally linearize, strip everything. Returns (rgb, ir_or_none).""" with _tifffile.TiffFile(file_path) as tif: @@ -364,6 +365,8 @@ def _decode_tiff( f32 = np.clip(f32, 0.0, 1.0) if gamma_key != "linear": f32 = _linearize(f32, gamma_key) + if expansion is not None and expansion > 1.0: + f32 = np.clip(f32 * expansion, 0.0, 1.0) orientation = read_orientation(file_path) f32 = _apply_geometry(f32, orientation, geometry) if ir is not None: @@ -613,6 +616,8 @@ def _effective_expansion(file_path: str, expansion: Optional[float]) -> float: return factor if factor > 1.0 else 1.0 if _is_dng(file_path) and _is_linearraw_dng(file_path): return expansion if (expansion is not None and expansion > 1.0) else 1.0 + if _is_tiff(file_path): + return expansion if (expansion is not None and expansion > 1.0) else 1.0 return 1.0 @@ -655,6 +660,7 @@ def _write_tiff( sensor_applied: bool = False, ice_applied: bool = False, gamma_key: str = "linear", + strip_profiles: bool = False, ) -> None: """Write a float32 buffer as an untagged 16-bit TIFF to *dest* (path or file-like).""" u16 = _to_uint16_jit(np.ascontiguousarray(f32, dtype=np.float32)) @@ -678,11 +684,14 @@ def _write_tiff( corrections = [s for s, on in (("flatfield", flatfield_applied), ("sensor", sensor_applied), ("ICE", ice_applied)) if on] if corrections: parts.append(f"corrections: {', '.join(corrections)}") - parts.append("no color management") + if strip_profiles: + parts.append("no color management (profiles stripped)") + else: + parts.append("no color management") description = f"NegPy Linear Output -- {', '.join(parts)}." extratags: list[tuple] = [] - if camera_wb is not None and source_path is not None: + if not strip_profiles and camera_wb is not None and source_path is not None: xmp_bytes = _build_xmp(source_path, camera_wb, title=description, wb_applied=wb_applied) extratags.append((700, 1, len(xmp_bytes), xmp_bytes, True)) @@ -739,6 +748,7 @@ def export_linear_output( apply_ice: bool = False, retouch: Optional[RetouchConfig] = None, gamma_key: str = "linear", + strip_profiles: bool = False, ) -> None: """Decode *file_path* and write an untagged linear 16-bit TIFF to *output_path*. @@ -800,6 +810,7 @@ def export_linear_output( sensor_applied=apply_sensor or is_stitch, ice_applied=ice_applied, gamma_key=gamma_key, + strip_profiles=strip_profiles, ) if ir is not None: diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index a7ce0758..f0719097 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -1184,3 +1184,55 @@ def test_export_tiff_with_gamma(self, tmp_path: str) -> None: desc = tf.pages[0].description assert "linearized from Gamma 2.2" in desc assert "source: TIFF" in desc + + def test_decode_tiff_with_expansion(self, tmp_path: str) -> None: + from negpy.services.export.linear_output import _decode_tiff + + data = np.full((4, 4, 3), 16384, dtype=np.uint16) + path = os.path.join(str(tmp_path), "dim.tif") + tifffile.imwrite(path, data) + rgb_no_exp, _ = _decode_tiff(path) + rgb_2x, _ = _decode_tiff(path, expansion=2.0) + np.testing.assert_allclose(rgb_2x, np.clip(rgb_no_exp * 2.0, 0.0, 1.0), atol=1e-6) + + def test_export_tiff_with_expansion(self, tmp_path: str) -> None: + data = np.full((4, 4, 3), 16384, dtype=np.uint16) + src = os.path.join(str(tmp_path), "input.tif") + tifffile.imwrite(src, data) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(src, out, expansion=2.0) + with tifffile.TiffFile(out) as tf: + desc = tf.pages[0].description + assert "expansion: x2" in desc + + def test_strip_profiles_removes_xmp(self, tmp_path: str) -> None: + f32 = np.full((4, 4, 3), 0.5, dtype=np.float32) + wb = _CameraWB(as_shot=(512.0, 256.0, 304.0, 700.0), daylight=(1.0, 1.0, 1.0, 1.0)) + out_with = os.path.join(str(tmp_path), "with.tiff") + _write_tiff(f32, out_with, "test.nef", camera_wb=wb, source_path="/fake/test.nef") + out_stripped = os.path.join(str(tmp_path), "stripped.tiff") + _write_tiff(f32, out_stripped, "test.nef", camera_wb=wb, source_path="/fake/test.nef", strip_profiles=True) + with tifffile.TiffFile(out_with) as tf: + assert 700 in [t.code for t in tf.pages[0].tags.values()] + with tifffile.TiffFile(out_stripped) as tf: + tag_codes = [t.code for t in tf.pages[0].tags.values()] + assert 700 not in tag_codes # no XMP + assert 34675 not in tag_codes # no ICC profile + + def test_strip_profiles_description(self, tmp_path: str) -> None: + f32 = np.full((4, 4, 3), 0.5, dtype=np.float32) + out = os.path.join(str(tmp_path), "out.tiff") + _write_tiff(f32, out, "test.tif", strip_profiles=True) + with tifffile.TiffFile(out) as tf: + desc = tf.pages[0].description + assert "profiles stripped" in desc + + def test_strip_profiles_keeps_make_model(self, tmp_path: str) -> None: + f32 = np.full((4, 4, 3), 0.5, dtype=np.float32) + meta = _SourceMeta(make="Nikon", model="D750") + out = os.path.join(str(tmp_path), "out.tiff") + _write_tiff(f32, out, "test.nef", source_meta=meta, strip_profiles=True) + with tifffile.TiffFile(out) as tf: + tags = {t.code: t.value for t in tf.pages[0].tags.values()} + assert tags.get(271) == "Nikon" + assert tags.get(272) == "D750" From 6a5eac48efa353e384ccb80df23e94449a9dcfd7 Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 12:59:25 +0200 Subject: [PATCH 04/20] =?UTF-8?q?Remove=20strip-profiles=20checkbox=20?= =?UTF-8?q?=E2=80=94=20TIFF=20output=20is=20always=20clean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linear Output writes the TIFF from scratch: only raw pixels plus Make/Model/DateTime from the source. ICC profiles, EXIF color space, and XMP color metadata from scanner software or editors are never copied through, so a separate toggle was unnecessary. --- negpy/desktop/controller.py | 1 - negpy/desktop/session.py | 5 ---- negpy/desktop/view/sidebar/export.py | 15 ----------- negpy/services/export/linear_output.py | 10 ++------ tests/test_linear_output.py | 35 +++++++++----------------- 5 files changed, 14 insertions(+), 52 deletions(-) diff --git a/negpy/desktop/controller.py b/negpy/desktop/controller.py index f46c6cb0..d5f549aa 100644 --- a/negpy/desktop/controller.py +++ b/negpy/desktop/controller.py @@ -3508,7 +3508,6 @@ def request_linear_output_export(self, files: list[dict] | None = None) -> None: apply_ice=self.state.linear_apply_ice, retouch=params.retouch, gamma_key=self.state.linear_gamma_key, - strip_profiles=self.state.linear_strip_profiles, ) exported += 1 except Exception as e: diff --git a/negpy/desktop/session.py b/negpy/desktop/session.py index ef7c3bfc..ea8e0a25 100644 --- a/negpy/desktop/session.py +++ b/negpy/desktop/session.py @@ -174,7 +174,6 @@ class AppState: linear_apply_sensor: bool = False linear_apply_ice: bool = False linear_gamma_key: str = "linear" - linear_strip_profiles: bool = False @property def local_hidden_masks(self) -> set: @@ -510,9 +509,6 @@ def __init__(self, repo: StorageRepository): saved_gamma = self.repo.get_global_setting("linear_gamma_key") if saved_gamma is not None: self.state.linear_gamma_key = str(saved_gamma) - saved_strip = self.repo.get_global_setting("linear_strip_profiles") - if saved_strip is not None: - self.state.linear_strip_profiles = bool(saved_strip) self.state.export_presets = self.repo.load_export_presets() @@ -596,7 +592,6 @@ def save_flat_output_prefs(self) -> None: self.repo.save_global_setting("linear_apply_sensor", self.state.linear_apply_sensor) self.repo.save_global_setting("linear_apply_ice", self.state.linear_apply_ice) self.repo.save_global_setting("linear_gamma_key", self.state.linear_gamma_key) - self.repo.save_global_setting("linear_strip_profiles", self.state.linear_strip_profiles) def _apply_sticky_settings(self, config: WorkspaceConfig, only_global: bool = False) -> WorkspaceConfig: """ diff --git a/negpy/desktop/view/sidebar/export.py b/negpy/desktop/view/sidebar/export.py index e1fc3913..38b770c4 100644 --- a/negpy/desktop/view/sidebar/export.py +++ b/negpy/desktop/view/sidebar/export.py @@ -617,13 +617,6 @@ def _add_flat_master_section(self) -> None: box.addWidget(self.linear_gamma_hint) self.linear_gamma_combo.currentIndexChanged.connect(self._on_linear_gamma_changed) - self.linear_strip_checkbox = QCheckBox("Strip color profiles") - self.linear_strip_checkbox.setToolTip("Remove all ICC profiles, color space tags, and XMP color metadata from the exported TIFF") - self.linear_strip_checkbox.setChecked(self.state.linear_strip_profiles) - self.linear_strip_checkbox.setVisible(False) - self.linear_strip_checkbox.toggled.connect(self._on_linear_strip_changed) - box.addWidget(self.linear_strip_checkbox) - self.linear_corrections_hint = hint_label( "Corrections are baked in and cannot be undone from the exported file. Re-export from the original RAW to get uncorrected data." ) @@ -649,7 +642,6 @@ def _sync_flat_enabled(self) -> None: if hasattr(self, "linear_gamma_row") and not linear_on: self.linear_gamma_row.setVisible(False) self.linear_gamma_hint.setVisible(False) - self.linear_strip_checkbox.setVisible(False) if hasattr(self, "linear_corrections_label") and not linear_on: self.linear_corrections_label.setVisible(False) self.linear_wb_checkbox.setVisible(False) @@ -742,7 +734,6 @@ def _refresh_linear_expansion_combo(self) -> None: is_tiff = source_type == "tiff" self.linear_gamma_row.setVisible(is_tiff) self.linear_gamma_hint.setVisible(is_tiff) - self.linear_strip_checkbox.setVisible(is_tiff) if is_tiff: self._refresh_linear_gamma_combo() @@ -812,10 +803,6 @@ def _on_linear_gamma_changed(self, index: int) -> None: self.state.linear_gamma_key = TIFF_GAMMA_OPTIONS[index][0] self.controller.session.save_flat_output_prefs() - def _on_linear_strip_changed(self, checked: bool) -> None: - self.state.linear_strip_profiles = checked - self.controller.session.save_flat_output_prefs() - def _on_linear_correction_changed(self, _checked: bool) -> None: self.state.linear_apply_wb = self.linear_wb_checkbox.isChecked() self.state.linear_apply_flatfield = self.linear_flatfield_checkbox.isChecked() @@ -1263,7 +1250,6 @@ def sync_ui(self) -> None: self.linear_flatfield_checkbox.setChecked(self.state.linear_apply_flatfield) self.linear_sensor_checkbox.setChecked(self.state.linear_apply_sensor) self.linear_ice_checkbox.setChecked(self.state.linear_apply_ice) - self.linear_strip_checkbox.setChecked(self.state.linear_strip_profiles) self._refresh_linear_gamma_combo() finally: self.block_signals(False) @@ -1293,7 +1279,6 @@ def block_signals(self, blocked: bool) -> None: self.linear_sensor_checkbox, self.linear_ice_checkbox, self.linear_gamma_combo, - self.linear_strip_checkbox, ] for w in widgets: w.blockSignals(blocked) diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index 56efa3da..89d01b46 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -660,7 +660,6 @@ def _write_tiff( sensor_applied: bool = False, ice_applied: bool = False, gamma_key: str = "linear", - strip_profiles: bool = False, ) -> None: """Write a float32 buffer as an untagged 16-bit TIFF to *dest* (path or file-like).""" u16 = _to_uint16_jit(np.ascontiguousarray(f32, dtype=np.float32)) @@ -684,14 +683,11 @@ def _write_tiff( corrections = [s for s, on in (("flatfield", flatfield_applied), ("sensor", sensor_applied), ("ICE", ice_applied)) if on] if corrections: parts.append(f"corrections: {', '.join(corrections)}") - if strip_profiles: - parts.append("no color management (profiles stripped)") - else: - parts.append("no color management") + parts.append("no color management") description = f"NegPy Linear Output -- {', '.join(parts)}." extratags: list[tuple] = [] - if not strip_profiles and camera_wb is not None and source_path is not None: + if camera_wb is not None and source_path is not None: xmp_bytes = _build_xmp(source_path, camera_wb, title=description, wb_applied=wb_applied) extratags.append((700, 1, len(xmp_bytes), xmp_bytes, True)) @@ -748,7 +744,6 @@ def export_linear_output( apply_ice: bool = False, retouch: Optional[RetouchConfig] = None, gamma_key: str = "linear", - strip_profiles: bool = False, ) -> None: """Decode *file_path* and write an untagged linear 16-bit TIFF to *output_path*. @@ -810,7 +805,6 @@ def export_linear_output( sensor_applied=apply_sensor or is_stitch, ice_applied=ice_applied, gamma_key=gamma_key, - strip_profiles=strip_profiles, ) if ir is not None: diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index f0719097..ab6db164 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -1205,34 +1205,23 @@ def test_export_tiff_with_expansion(self, tmp_path: str) -> None: desc = tf.pages[0].description assert "expansion: x2" in desc - def test_strip_profiles_removes_xmp(self, tmp_path: str) -> None: - f32 = np.full((4, 4, 3), 0.5, dtype=np.float32) - wb = _CameraWB(as_shot=(512.0, 256.0, 304.0, 700.0), daylight=(1.0, 1.0, 1.0, 1.0)) - out_with = os.path.join(str(tmp_path), "with.tiff") - _write_tiff(f32, out_with, "test.nef", camera_wb=wb, source_path="/fake/test.nef") - out_stripped = os.path.join(str(tmp_path), "stripped.tiff") - _write_tiff(f32, out_stripped, "test.nef", camera_wb=wb, source_path="/fake/test.nef", strip_profiles=True) - with tifffile.TiffFile(out_with) as tf: - assert 700 in [t.code for t in tf.pages[0].tags.values()] - with tifffile.TiffFile(out_stripped) as tf: + def test_tiff_output_has_no_icc_profile(self, tmp_path: str) -> None: + data = np.full((4, 4, 3), 32768, dtype=np.uint16) + src = os.path.join(str(tmp_path), "input.tif") + tifffile.imwrite(src, data) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(src, out) + with tifffile.TiffFile(out) as tf: tag_codes = [t.code for t in tf.pages[0].tags.values()] - assert 700 not in tag_codes # no XMP assert 34675 not in tag_codes # no ICC profile + assert 34665 not in tag_codes # no EXIF IFD - def test_strip_profiles_description(self, tmp_path: str) -> None: - f32 = np.full((4, 4, 3), 0.5, dtype=np.float32) - out = os.path.join(str(tmp_path), "out.tiff") - _write_tiff(f32, out, "test.tif", strip_profiles=True) - with tifffile.TiffFile(out) as tf: - desc = tf.pages[0].description - assert "profiles stripped" in desc - - def test_strip_profiles_keeps_make_model(self, tmp_path: str) -> None: + def test_tiff_output_keeps_make_model(self, tmp_path: str) -> None: f32 = np.full((4, 4, 3), 0.5, dtype=np.float32) - meta = _SourceMeta(make="Nikon", model="D750") + meta = _SourceMeta(make="Nikon", model="CoolScan 5000") out = os.path.join(str(tmp_path), "out.tiff") - _write_tiff(f32, out, "test.nef", source_meta=meta, strip_profiles=True) + _write_tiff(f32, out, "test.tif", source_meta=meta) with tifffile.TiffFile(out) as tf: tags = {t.code: t.value for t in tf.pages[0].tags.values()} assert tags.get(271) == "Nikon" - assert tags.get(272) == "D750" + assert tags.get(272) == "CoolScan 5000" From f0210cf5e899b37f8448a43c373133a3bd0ebe4a Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 15:35:50 +0200 Subject: [PATCH 05/20] Fix linear output crash on non-standard TIFF DateTime tags Parse and normalise variant DateTime formats (dots, dashes) to TIFF- standard YYYY:MM:DD HH:MM:SS. Drop unparseable datetimes rather than failing the export. Wrap all metadata assembly in _write_tiff so any unexpected tag issue degrades gracefully instead of aborting the write. --- negpy/services/export/linear_output.py | 40 +++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index 89d01b46..4670be51 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -646,6 +646,21 @@ def _source_format_label( return "unknown" +def _parse_tiff_datetime(dt_str: Optional[str]) -> Optional[str]: + """Validate/normalise a TIFF DateTime string to ``YYYY:MM:DD HH:MM:SS``.""" + if not dt_str: + return None + from datetime import datetime + + for fmt in ("%Y:%m:%d %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%Y.%m.%d %H.%M.%S", "%Y:%m:%d", "%Y-%m-%d"): + try: + parsed = datetime.strptime(dt_str.strip(), fmt) + return parsed.strftime("%Y:%m:%d %H:%M:%S") + except ValueError: + continue + return None + + def _write_tiff( f32: np.ndarray, dest, @@ -687,17 +702,20 @@ def _write_tiff( description = f"NegPy Linear Output -- {', '.join(parts)}." extratags: list[tuple] = [] - if camera_wb is not None and source_path is not None: - xmp_bytes = _build_xmp(source_path, camera_wb, title=description, wb_applied=wb_applied) - extratags.append((700, 1, len(xmp_bytes), xmp_bytes, True)) - - if source_meta is not None: - if source_meta.make: - extratags.append((271, 2, 0, source_meta.make, True)) - if source_meta.model: - extratags.append((272, 2, 0, source_meta.model, True)) - - dt = (source_meta.datetime if source_meta else None) or None + dt: Optional[str] = None + try: + if camera_wb is not None and source_path is not None: + xmp_bytes = _build_xmp(source_path, camera_wb, title=description, wb_applied=wb_applied) + extratags.append((700, 1, len(xmp_bytes), xmp_bytes, True)) + if source_meta is not None: + if source_meta.make: + extratags.append((271, 2, 0, source_meta.make, True)) + if source_meta.model: + extratags.append((272, 2, 0, source_meta.model, True)) + dt = _parse_tiff_datetime((source_meta.datetime if source_meta else None) or None) + except Exception: + extratags = [] + dt = None _tifffile.imwrite( dest, From d479baff9003ecea439037df944b814e395a1e56 Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 16:14:08 +0200 Subject: [PATCH 06/20] Add Coolscan NEF loader as a first-class scanner format Detect scanner NEFs by checking for an RGB SubIFD (vs Bayer CFA in camera NEFs). Load the full-res 16-bit RGB data via tifffile, handle ICC profiles and IR channels, apply color space logic identical to TiffLoader. Camera NEFs continue to go through rawpy. Wire into the loader factory, linear output (source type "nef", no expansion), and the export sidebar. 13 new tests cover detection, classification, export roundtrip, and IR extraction. --- negpy/desktop/view/sidebar/export.py | 1 + negpy/infrastructure/loaders/factory.py | 5 + negpy/infrastructure/loaders/nef_loader.py | 109 ++++++++++++++++++++ negpy/services/export/linear_output.py | 53 +++++++++- tests/test_linear_output.py | 110 +++++++++++++++++++++ 5 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 negpy/infrastructure/loaders/nef_loader.py diff --git a/negpy/desktop/view/sidebar/export.py b/negpy/desktop/view/sidebar/export.py index 38b770c4..14dfcbef 100644 --- a/negpy/desktop/view/sidebar/export.py +++ b/negpy/desktop/view/sidebar/export.py @@ -699,6 +699,7 @@ def _on_linear_output_changed(self, enabled: bool) -> None: "pakon": [("4× (default)", None), ("2×", 2.0), ("Off", 1.0)], "pakon_f335": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "dng": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], + "nef": [], "camera": [], "tiff": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "unsupported": [], diff --git a/negpy/infrastructure/loaders/factory.py b/negpy/infrastructure/loaders/factory.py index c2ed87d9..75892e0c 100644 --- a/negpy/infrastructure/loaders/factory.py +++ b/negpy/infrastructure/loaders/factory.py @@ -3,6 +3,7 @@ from negpy.infrastructure.loaders.pakon_loader import PakonLoader from negpy.infrastructure.loaders.tiff_loader import TiffLoader from negpy.infrastructure.loaders.jpeg_loader import JpegLoader +from negpy.infrastructure.loaders.nef_loader import NefLoader, is_coolscan_nef from negpy.infrastructure.loaders.rawpy_loader import RawpyLoader from negpy.infrastructure.loaders.constants import ( SUPPORTED_TIFF_EXTENSIONS, @@ -19,6 +20,7 @@ def __init__(self) -> None: self._pakon = PakonLoader() self._tiff = TiffLoader() self._jpeg = JpegLoader() + self._nef = NefLoader() self._rawpy = RawpyLoader() def get_loader(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager[Any], dict]: @@ -33,6 +35,9 @@ def get_loader(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextM if PakonLoader.can_handle(file_path): return self._pakon.load(file_path) + if is_coolscan_nef(file_path): + return self._nef.load(file_path, linear_raw=linear_raw) + return self._rawpy.load(file_path) diff --git a/negpy/infrastructure/loaders/nef_loader.py b/negpy/infrastructure/loaders/nef_loader.py new file mode 100644 index 00000000..1d96d87f --- /dev/null +++ b/negpy/infrastructure/loaders/nef_loader.py @@ -0,0 +1,109 @@ +import os +from typing import Any, ContextManager, Optional, Tuple + +import numpy as np +import tifffile + +from negpy.domain.interfaces import IImageLoader +from negpy.domain.models import ColorSpace +from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, identify_color_space_from_icc, read_orientation +from negpy.infrastructure.loaders.ir_planes import normalize_ir_to_float32 +from negpy.kernel.image.logic import srgb_to_linear, uint8_to_float32, uint16_to_float32 +from negpy.kernel.system.logging import get_logger + +logger = get_logger(__name__) + + +def _find_rgb_subifd(tif: tifffile.TiffFile) -> Optional[Any]: + """Return the largest RGB SubIFD (Coolscan NEF stores full-res here via tag 0x014A).""" + page0 = tif.pages[0] + best = None + best_pixels = 0 + for sub in page0.pages or []: + tags = getattr(sub, "tags", None) + if tags is None: + continue + spp_tag = tags.get("SamplesPerPixel") + photo_tag = tags.get("PhotometricInterpretation") + if spp_tag is None or photo_tag is None: + continue + spp = int(spp_tag.value) + photo = int(photo_tag.value) + if spp < 3 or photo != 2: + continue + pixels = sub.shape[0] * sub.shape[1] + if pixels > best_pixels: + best = sub + best_pixels = pixels + return best + + +def is_coolscan_nef(file_path: str) -> bool: + """True if this NEF is a Nikon Coolscan scanner file (already-processed RGB in SubIFDs).""" + if os.path.splitext(file_path)[1].lower() != ".nef": + return False + try: + with tifffile.TiffFile(file_path) as tif: + return _find_rgb_subifd(tif) is not None + except Exception: + return False + + +class NefLoader(IImageLoader): + """Loader for Nikon Coolscan scanner NEF files. + + These are TIFF-structured files with the full-res processed RGB image in a + SubIFD chain (tag 0x014A). The data is Nikon Scan's output — curves, gain, + and optionally DigitalICE are already applied — not raw sensor data. + + Color space handling follows TiffLoader: ICC profile → identify space → + linearise if sRGB. Untagged 16-bit is assumed linear; untagged 8-bit is + assumed sRGB. + """ + + def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager[Any], dict]: + with tifffile.TiffFile(file_path) as tif: + sub = _find_rgb_subifd(tif) + if sub is None: + raise ValueError(f"No RGB SubIFD in {file_path}") + arr = sub.asarray() + + icc_bytes: Optional[bytes] = None + for page in (sub, tif.pages[0]): + tags = getattr(page, "tags", None) + if tags is None: + continue + tag = tags.get("InterColorProfile") + if tag is not None and tag.value: + icc_bytes = bytes(tag.value) + break + + ir: Optional[np.ndarray] = None + if arr.ndim == 3 and arr.shape[2] == 4: + ir = normalize_ir_to_float32(arr[:, :, 3]) + arr = np.ascontiguousarray(arr[:, :, :3]) + elif arr.ndim == 2: + arr = np.stack([arr] * 3, axis=-1) + + if arr.dtype == np.uint8: + f32 = uint8_to_float32(np.ascontiguousarray(arr)) + elif arr.dtype == np.uint16: + f32 = uint16_to_float32(np.ascontiguousarray(arr)) + else: + f32 = np.clip(arr.astype(np.float32), 0, 1) + + color_space = None + if not linear_raw: + color_space = identify_color_space_from_icc(icc_bytes) + if color_space is None and arr.dtype == np.uint8: + color_space = ColorSpace.SRGB.value + if color_space == ColorSpace.SRGB.value: + f32 = srgb_to_linear(f32) + + metadata = { + "orientation": read_orientation(file_path), + "color_space": color_space, + "icc_profile": icc_bytes, + "ir": ir, + } + return NonStandardFileWrapper(f32), metadata diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index 4670be51..146b4ab2 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -30,6 +30,7 @@ from negpy.infrastructure.loaders.constants import SUPPORTED_JPEG_EXTENSIONS, SUPPORTED_RAW_EXTENSIONS, SUPPORTED_TIFF_EXTENSIONS from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, get_best_demosaic_algorithm, read_orientation from negpy.infrastructure.loaders.pakon_loader import PakonLoader +from negpy.infrastructure.loaders.nef_loader import is_coolscan_nef from negpy.infrastructure.loaders.rawpy_loader import ( _find_linearraw_page, _is_dng, @@ -108,6 +109,8 @@ def _is_camera_raw(file_path: str) -> bool: return False if PakonLoader.can_handle(file_path): return False + if is_coolscan_nef(file_path): + return False return ext in SUPPORTED_RAW_EXTENSIONS @@ -120,6 +123,8 @@ def is_linear_output_supported(file_path: str) -> bool: return True if _is_dng(file_path): return _is_linearraw_dng(file_path) or _is_camera_raw(file_path) + if is_coolscan_nef(file_path): + return True if _is_camera_raw(file_path): return True if _is_tiff(file_path): @@ -130,12 +135,14 @@ def is_linear_output_supported(file_path: str) -> bool: def linear_output_source_type(file_path: str) -> str: """Classify a file for Linear Output expansion options. - Returns ``"pakon"``, ``"dng"``, ``"camera"``, ``"tiff"``, or ``"unsupported"``. + Returns ``"pakon"``, ``"dng"``, ``"camera"``, ``"nef"``, ``"tiff"``, or ``"unsupported"``. """ if PakonLoader.can_handle(file_path): return "pakon_f335" if _is_pakon_f335(file_path) else "pakon" if _is_dng(file_path) and _is_linearraw_dng(file_path): return "dng" + if is_coolscan_nef(file_path): + return "nef" if _is_camera_raw(file_path): return "camera" if _is_tiff(file_path): @@ -282,6 +289,10 @@ def _decode_linear( if _is_camera_raw(file_path): rgb, ir, wb = _decode_camera_raw(file_path, geometry) return rgb, ir, wb, meta + if is_coolscan_nef(file_path): + meta = _read_source_meta_tiff(file_path) + rgb, ir = _decode_nef(file_path, geometry) + return rgb, ir, None, meta if _is_camera_raw(file_path): if rgbscan is not None and is_rgb_triplet(rgbscan): rgb, ir, wb, meta = _decode_camera_raw_triplet(file_path, rgbscan, geometry) @@ -388,6 +399,44 @@ def _decode_pakon(file_path: str, geometry: Optional[GeometryConfig] = None, exp return f32, None +def _decode_nef( + file_path: str, + geometry: Optional[GeometryConfig] = None, +) -> tuple[np.ndarray, Optional[np.ndarray]]: + """Read a Coolscan NEF via SubIFDs. Returns (rgb, ir_or_none).""" + from negpy.infrastructure.loaders.nef_loader import _find_rgb_subifd + + with _tifffile.TiffFile(file_path) as tif: + sub = _find_rgb_subifd(tif) + if sub is None: + raise ValueError(f"No RGB SubIFD in {file_path}") + arr = sub.asarray() + + if arr.dtype == np.uint16: + scale = 1.0 / 65535.0 + elif arr.dtype == np.uint8: + scale = 1.0 / 255.0 + elif arr.dtype == np.float32: + scale = 1.0 + else: + scale = 1.0 / float(np.iinfo(arr.dtype).max) if np.issubdtype(arr.dtype, np.integer) else 1.0 + f32 = arr.astype(np.float32) * scale + + ir = None + if f32.ndim == 3 and f32.shape[2] == 4: + ir = f32[:, :, 3] + f32 = f32[:, :, :3] + elif f32.ndim == 2: + f32 = np.stack([f32, f32, f32], axis=2) + f32 = np.clip(f32, 0.0, 1.0) + + orientation = read_orientation(file_path) + f32 = _apply_geometry(f32, orientation, geometry) + if ir is not None: + ir = _apply_geometry(ir, orientation, geometry) + return f32, ir + + def _decode_dng( file_path: str, geometry: Optional[GeometryConfig] = None, expansion: Optional[float] = None ) -> tuple[np.ndarray, Optional[np.ndarray]]: @@ -631,6 +680,8 @@ def _source_format_label( return f"Pakon {_pakon_spec_desc(file_path)}" if _is_dng(file_path) and _is_linearraw_dng(file_path): return "DNG LinearRaw" + if is_coolscan_nef(file_path): + return "Coolscan NEF" if _is_camera_raw(file_path): if is_stitch and stitch_has_triplets(stitch): n = 1 + len(stitch.stitch_paths) diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index ab6db164..9ab546b3 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -12,6 +12,7 @@ from negpy.features.rgbscan.models import RgbScanConfig from negpy.features.stitch.models import StitchConfig from negpy.kernel.image.logic import apply_exif_orientation +from negpy.infrastructure.loaders.nef_loader import is_coolscan_nef from negpy.services.export.linear_output import ( TIFF_GAMMA_OPTIONS, _CameraWB, @@ -1225,3 +1226,112 @@ def test_tiff_output_keeps_make_model(self, tmp_path: str) -> None: tags = {t.code: t.value for t in tf.pages[0].tags.values()} assert tags.get(271) == "Nikon" assert tags.get(272) == "CoolScan 5000" + + +def _make_coolscan_nef(tmp_dir: str, h: int = 200, w: int = 300, channels: int = 3) -> str: + """Create a synthetic Coolscan-style NEF: thumbnail in IFD0, full-res RGB in SubIFD.""" + thumb = np.zeros((50, 75, 3), dtype=np.uint8) + rng = np.random.RandomState(42) + fullres = rng.randint(0, 65535, (h, w, channels), dtype=np.uint16) + path = os.path.join(tmp_dir, "coolscan.nef") + with tifffile.TiffWriter(path) as tw: + tw.write(thumb, photometric="rgb", subifds=1) + tw.write(fullres, photometric="rgb") + return path + + +def _make_camera_nef(tmp_dir: str) -> str: + """Create a synthetic camera-style NEF: single-channel Bayer, no RGB SubIFD.""" + bayer = np.zeros((200, 300), dtype=np.uint16) + path = os.path.join(tmp_dir, "camera.nef") + with tifffile.TiffWriter(path) as tw: + tw.write(bayer, photometric="minisblack") + return path + + +class TestCoolscanNef: + def test_detect_coolscan_nef(self, tmp_path: str) -> None: + path = _make_coolscan_nef(str(tmp_path)) + assert is_coolscan_nef(path) + + def test_camera_nef_not_detected(self, tmp_path: str) -> None: + path = _make_camera_nef(str(tmp_path)) + assert not is_coolscan_nef(path) + + def test_non_nef_not_detected(self, tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "photo.tiff") + tifffile.imwrite(path, np.zeros((10, 10, 3), dtype=np.uint16)) + assert not is_coolscan_nef(path) + + def test_coolscan_nef_not_camera_raw(self, tmp_path: str) -> None: + path = _make_coolscan_nef(str(tmp_path)) + assert not _is_camera_raw(path) + + def test_camera_nef_is_camera_raw(self, tmp_path: str) -> None: + path = _make_camera_nef(str(tmp_path)) + assert _is_camera_raw(path) + + def test_linear_output_supported(self, tmp_path: str) -> None: + path = _make_coolscan_nef(str(tmp_path)) + assert is_linear_output_supported(path) + + def test_source_type_nef(self, tmp_path: str) -> None: + path = _make_coolscan_nef(str(tmp_path)) + assert linear_output_source_type(path) == "nef" + + def test_source_format_label(self, tmp_path: str) -> None: + path = _make_coolscan_nef(str(tmp_path)) + assert _source_format_label(path) == "Coolscan NEF" + + def test_no_expansion(self, tmp_path: str) -> None: + path = _make_coolscan_nef(str(tmp_path)) + assert _effective_expansion(path, None) == 1.0 + assert _effective_expansion(path, 4.0) == 1.0 + + def test_export_roundtrip(self, tmp_path: str) -> None: + path = _make_coolscan_nef(str(tmp_path)) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(path, out) + with tifffile.TiffFile(out) as tf: + arr = tf.pages[0].asarray() + assert arr.dtype == np.uint16 + assert arr.shape == (200, 300, 3) + desc = tf.pages[0].description + assert "Coolscan NEF" in desc + assert "no scaling" in desc + + def test_export_4ch_splits_ir(self, tmp_path: str) -> None: + path = _make_coolscan_nef(str(tmp_path), channels=4) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(path, out) + ir_path = os.path.join(str(tmp_path), "output_ir.tiff") + assert os.path.exists(ir_path) + with tifffile.TiffFile(out) as tf: + assert tf.pages[0].asarray().shape == (200, 300, 3) + with tifffile.TiffFile(ir_path) as tf: + assert tf.pages[0].asarray().shape == (200, 300) + + def test_loader_returns_float32(self, tmp_path: str) -> None: + from negpy.infrastructure.loaders.nef_loader import NefLoader + + path = _make_coolscan_nef(str(tmp_path)) + loader = NefLoader() + wrapper, metadata = loader.load(path) + with wrapper as w: + assert w.data.dtype == np.float32 + assert w.data.shape == (200, 300, 3) + assert w.data.min() >= 0.0 + assert w.data.max() <= 1.0 + assert "orientation" in metadata + assert "ir" in metadata + + def test_loader_extracts_ir(self, tmp_path: str) -> None: + from negpy.infrastructure.loaders.nef_loader import NefLoader + + path = _make_coolscan_nef(str(tmp_path), channels=4) + loader = NefLoader() + wrapper, metadata = loader.load(path) + with wrapper as w: + assert w.data.shape == (200, 300, 3) + assert metadata["ir"] is not None + assert metadata["ir"].shape == (200, 300) From 1229f679099546746a4d4e613c23c1d4867df0ac Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 16:24:11 +0200 Subject: [PATCH 07/20] Add Flextight FFF loader as a first-class scanner format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Detect FFF scanner files by checking for a 16-bit RGB IFD (vs Bayer in .3fr camera-back files). Pick the full-res image by pixel count, not SubfileType tag — the spec confirms that tag is unreliable on real samples. Data is always linear, no expansion needed. Wire into the loader factory, linear output (source type "fff", no expansion), and the export sidebar. 10 new tests cover detection, classification, largest-IFD selection, export roundtrip, and loader data integrity. --- negpy/desktop/view/sidebar/export.py | 1 + negpy/infrastructure/loaders/factory.py | 5 + negpy/infrastructure/loaders/fff_loader.py | 117 +++++++++++++++++++++ negpy/services/export/linear_output.py | 53 +++++++++- tests/test_linear_output.py | 85 +++++++++++++++ 5 files changed, 260 insertions(+), 1 deletion(-) create mode 100644 negpy/infrastructure/loaders/fff_loader.py diff --git a/negpy/desktop/view/sidebar/export.py b/negpy/desktop/view/sidebar/export.py index 14dfcbef..e894f8ac 100644 --- a/negpy/desktop/view/sidebar/export.py +++ b/negpy/desktop/view/sidebar/export.py @@ -700,6 +700,7 @@ def _on_linear_output_changed(self, enabled: bool) -> None: "pakon_f335": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "dng": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "nef": [], + "fff": [], "camera": [], "tiff": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "unsupported": [], diff --git a/negpy/infrastructure/loaders/factory.py b/negpy/infrastructure/loaders/factory.py index 75892e0c..75fe5305 100644 --- a/negpy/infrastructure/loaders/factory.py +++ b/negpy/infrastructure/loaders/factory.py @@ -3,6 +3,7 @@ from negpy.infrastructure.loaders.pakon_loader import PakonLoader from negpy.infrastructure.loaders.tiff_loader import TiffLoader from negpy.infrastructure.loaders.jpeg_loader import JpegLoader +from negpy.infrastructure.loaders.fff_loader import FffLoader, is_flextight_fff from negpy.infrastructure.loaders.nef_loader import NefLoader, is_coolscan_nef from negpy.infrastructure.loaders.rawpy_loader import RawpyLoader from negpy.infrastructure.loaders.constants import ( @@ -21,6 +22,7 @@ def __init__(self) -> None: self._tiff = TiffLoader() self._jpeg = JpegLoader() self._nef = NefLoader() + self._fff = FffLoader() self._rawpy = RawpyLoader() def get_loader(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager[Any], dict]: @@ -38,6 +40,9 @@ def get_loader(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextM if is_coolscan_nef(file_path): return self._nef.load(file_path, linear_raw=linear_raw) + if is_flextight_fff(file_path): + return self._fff.load(file_path, linear_raw=linear_raw) + return self._rawpy.load(file_path) diff --git a/negpy/infrastructure/loaders/fff_loader.py b/negpy/infrastructure/loaders/fff_loader.py new file mode 100644 index 00000000..ece727aa --- /dev/null +++ b/negpy/infrastructure/loaders/fff_loader.py @@ -0,0 +1,117 @@ +import os +from typing import Any, ContextManager, Optional, Tuple + +import numpy as np +import tifffile + +from negpy.domain.interfaces import IImageLoader +from negpy.domain.models import ColorSpace +from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, identify_color_space_from_icc, read_orientation +from negpy.infrastructure.loaders.ir_planes import normalize_ir_to_float32 +from negpy.kernel.image.logic import srgb_to_linear, uint8_to_float32, uint16_to_float32 +from negpy.kernel.system.logging import get_logger + +logger = get_logger(__name__) + + +def _find_full_res_ifd(tif: tifffile.TiffFile) -> Optional[Any]: + """Return the largest RGB IFD by pixel count. + + FFF files can have multiple IFDs flagged as full-resolution (the SubfileType + tag is unreliable — e.g. a small secondary image tagged full-res). Pixel + count is the reliable signal, matching the approach in flexcolor-tool and + the reference loader. + """ + best = None + best_pixels = 0 + for page in tif.pages: + tags = getattr(page, "tags", None) + if tags is None: + continue + spp_tag = tags.get("SamplesPerPixel") + photo_tag = tags.get("PhotometricInterpretation") + bps_tag = tags.get("BitsPerSample") + if spp_tag is None or photo_tag is None: + continue + spp = int(spp_tag.value) if not hasattr(spp_tag.value, "__len__") else int(spp_tag.value[0]) + photo = int(photo_tag.value) + if spp < 3 or photo != 2: + continue + bits = int(bps_tag.value) if bps_tag and not hasattr(bps_tag.value, "__len__") else (int(bps_tag.value[0]) if bps_tag else 8) + if bits < 16: + continue + pixels = page.shape[0] * page.shape[1] + if pixels > best_pixels: + best = page + best_pixels = pixels + return best + + +def is_flextight_fff(file_path: str) -> bool: + """True if this FFF is an Imacon/Hasselblad Flextight scanner file (16-bit RGB in a top-level IFD).""" + if os.path.splitext(file_path)[1].lower() != ".fff": + return False + try: + with tifffile.TiffFile(file_path) as tif: + return _find_full_res_ifd(tif) is not None + except Exception: + return False + + +class FffLoader(IImageLoader): + """Loader for Imacon/Hasselblad Flextight FFF scanner files. + + These are big-endian TIFFs with the full-res 16-bit linear RGB image in a + top-level IFD (picked by pixel count, not SubfileType tag). The data is + uninverted scanner output — linear, no gamma applied. + + Color space handling follows TiffLoader: ICC profile → identify space → + linearise if sRGB. Untagged 16-bit is assumed linear. + """ + + def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager[Any], dict]: + with tifffile.TiffFile(file_path) as tif: + page = _find_full_res_ifd(tif) + if page is None: + raise ValueError(f"No full-res RGB IFD in {file_path}") + arr = page.asarray() + + icc_bytes: Optional[bytes] = None + for p in (page, tif.pages[0]): + tags = getattr(p, "tags", None) + if tags is None: + continue + tag = tags.get("InterColorProfile") + if tag is not None and tag.value: + icc_bytes = bytes(tag.value) + break + + ir: Optional[np.ndarray] = None + if arr.ndim == 3 and arr.shape[2] == 4: + ir = normalize_ir_to_float32(arr[:, :, 3]) + arr = np.ascontiguousarray(arr[:, :, :3]) + elif arr.ndim == 2: + arr = np.stack([arr] * 3, axis=-1) + + if arr.dtype == np.uint8: + f32 = uint8_to_float32(np.ascontiguousarray(arr)) + elif arr.dtype == np.uint16: + f32 = uint16_to_float32(np.ascontiguousarray(arr)) + else: + f32 = np.clip(arr.astype(np.float32), 0, 1) + + color_space = None + if not linear_raw: + color_space = identify_color_space_from_icc(icc_bytes) + if color_space is None and arr.dtype == np.uint8: + color_space = ColorSpace.SRGB.value + if color_space == ColorSpace.SRGB.value: + f32 = srgb_to_linear(f32) + + metadata = { + "orientation": read_orientation(file_path), + "color_space": color_space, + "icc_profile": icc_bytes, + "ir": ir, + } + return NonStandardFileWrapper(f32), metadata diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index 146b4ab2..2ed3eeb1 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -30,6 +30,7 @@ from negpy.infrastructure.loaders.constants import SUPPORTED_JPEG_EXTENSIONS, SUPPORTED_RAW_EXTENSIONS, SUPPORTED_TIFF_EXTENSIONS from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, get_best_demosaic_algorithm, read_orientation from negpy.infrastructure.loaders.pakon_loader import PakonLoader +from negpy.infrastructure.loaders.fff_loader import is_flextight_fff from negpy.infrastructure.loaders.nef_loader import is_coolscan_nef from negpy.infrastructure.loaders.rawpy_loader import ( _find_linearraw_page, @@ -111,6 +112,8 @@ def _is_camera_raw(file_path: str) -> bool: return False if is_coolscan_nef(file_path): return False + if is_flextight_fff(file_path): + return False return ext in SUPPORTED_RAW_EXTENSIONS @@ -125,6 +128,8 @@ def is_linear_output_supported(file_path: str) -> bool: return _is_linearraw_dng(file_path) or _is_camera_raw(file_path) if is_coolscan_nef(file_path): return True + if is_flextight_fff(file_path): + return True if _is_camera_raw(file_path): return True if _is_tiff(file_path): @@ -135,7 +140,7 @@ def is_linear_output_supported(file_path: str) -> bool: def linear_output_source_type(file_path: str) -> str: """Classify a file for Linear Output expansion options. - Returns ``"pakon"``, ``"dng"``, ``"camera"``, ``"nef"``, ``"tiff"``, or ``"unsupported"``. + Returns ``"pakon"``, ``"dng"``, ``"camera"``, ``"nef"``, ``"fff"``, ``"tiff"``, or ``"unsupported"``. """ if PakonLoader.can_handle(file_path): return "pakon_f335" if _is_pakon_f335(file_path) else "pakon" @@ -143,6 +148,8 @@ def linear_output_source_type(file_path: str) -> str: return "dng" if is_coolscan_nef(file_path): return "nef" + if is_flextight_fff(file_path): + return "fff" if _is_camera_raw(file_path): return "camera" if _is_tiff(file_path): @@ -293,6 +300,10 @@ def _decode_linear( meta = _read_source_meta_tiff(file_path) rgb, ir = _decode_nef(file_path, geometry) return rgb, ir, None, meta + if is_flextight_fff(file_path): + meta = _read_source_meta_tiff(file_path) + rgb, ir = _decode_fff(file_path, geometry) + return rgb, ir, None, meta if _is_camera_raw(file_path): if rgbscan is not None and is_rgb_triplet(rgbscan): rgb, ir, wb, meta = _decode_camera_raw_triplet(file_path, rgbscan, geometry) @@ -437,6 +448,44 @@ def _decode_nef( return f32, ir +def _decode_fff( + file_path: str, + geometry: Optional[GeometryConfig] = None, +) -> tuple[np.ndarray, Optional[np.ndarray]]: + """Read a Flextight FFF via largest RGB IFD. Returns (rgb, ir_or_none).""" + from negpy.infrastructure.loaders.fff_loader import _find_full_res_ifd + + with _tifffile.TiffFile(file_path) as tif: + page = _find_full_res_ifd(tif) + if page is None: + raise ValueError(f"No full-res RGB IFD in {file_path}") + arr = page.asarray() + + if arr.dtype == np.uint16: + scale = 1.0 / 65535.0 + elif arr.dtype == np.uint8: + scale = 1.0 / 255.0 + elif arr.dtype == np.float32: + scale = 1.0 + else: + scale = 1.0 / float(np.iinfo(arr.dtype).max) if np.issubdtype(arr.dtype, np.integer) else 1.0 + f32 = arr.astype(np.float32) * scale + + ir = None + if f32.ndim == 3 and f32.shape[2] == 4: + ir = f32[:, :, 3] + f32 = f32[:, :, :3] + elif f32.ndim == 2: + f32 = np.stack([f32, f32, f32], axis=2) + f32 = np.clip(f32, 0.0, 1.0) + + orientation = read_orientation(file_path) + f32 = _apply_geometry(f32, orientation, geometry) + if ir is not None: + ir = _apply_geometry(ir, orientation, geometry) + return f32, ir + + def _decode_dng( file_path: str, geometry: Optional[GeometryConfig] = None, expansion: Optional[float] = None ) -> tuple[np.ndarray, Optional[np.ndarray]]: @@ -682,6 +731,8 @@ def _source_format_label( return "DNG LinearRaw" if is_coolscan_nef(file_path): return "Coolscan NEF" + if is_flextight_fff(file_path): + return "Flextight FFF" if _is_camera_raw(file_path): if is_stitch and stitch_has_triplets(stitch): n = 1 + len(stitch.stitch_paths) diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index 9ab546b3..0411c128 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -12,6 +12,7 @@ from negpy.features.rgbscan.models import RgbScanConfig from negpy.features.stitch.models import StitchConfig from negpy.kernel.image.logic import apply_exif_orientation +from negpy.infrastructure.loaders.fff_loader import is_flextight_fff from negpy.infrastructure.loaders.nef_loader import is_coolscan_nef from negpy.services.export.linear_output import ( TIFF_GAMMA_OPTIONS, @@ -1335,3 +1336,87 @@ def test_loader_extracts_ir(self, tmp_path: str) -> None: assert w.data.shape == (200, 300, 3) assert metadata["ir"] is not None assert metadata["ir"].shape == (200, 300) + + +def _make_flextight_fff(tmp_dir: str, h: int = 400, w: int = 600, channels: int = 3) -> str: + """Create a synthetic Flextight FFF: big-endian TIFF, full-res RGB in IFD0, small preview in IFD1.""" + rng = np.random.RandomState(42) + fullres = rng.randint(0, 65535, (h, w, channels), dtype=np.uint16) + preview = np.zeros((50, 75, 3), dtype=np.uint8) + path = os.path.join(tmp_dir, "scan.fff") + with tifffile.TiffWriter(path, byteorder=">") as tw: + tw.write(fullres, photometric="rgb") + tw.write(preview, photometric="rgb") + return path + + +class TestFlextightFff: + def test_detect_flextight_fff(self, tmp_path: str) -> None: + path = _make_flextight_fff(str(tmp_path)) + assert is_flextight_fff(path) + + def test_non_fff_not_detected(self, tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "photo.tiff") + tifffile.imwrite(path, np.zeros((10, 10, 3), dtype=np.uint16)) + assert not is_flextight_fff(path) + + def test_fff_not_camera_raw(self, tmp_path: str) -> None: + path = _make_flextight_fff(str(tmp_path)) + assert not _is_camera_raw(path) + + def test_linear_output_supported(self, tmp_path: str) -> None: + path = _make_flextight_fff(str(tmp_path)) + assert is_linear_output_supported(path) + + def test_source_type_fff(self, tmp_path: str) -> None: + path = _make_flextight_fff(str(tmp_path)) + assert linear_output_source_type(path) == "fff" + + def test_source_format_label(self, tmp_path: str) -> None: + path = _make_flextight_fff(str(tmp_path)) + assert _source_format_label(path) == "Flextight FFF" + + def test_no_expansion(self, tmp_path: str) -> None: + path = _make_flextight_fff(str(tmp_path)) + assert _effective_expansion(path, None) == 1.0 + assert _effective_expansion(path, 4.0) == 1.0 + + def test_export_roundtrip(self, tmp_path: str) -> None: + path = _make_flextight_fff(str(tmp_path)) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(path, out) + with tifffile.TiffFile(out) as tf: + arr = tf.pages[0].asarray() + assert arr.dtype == np.uint16 + assert arr.shape == (400, 600, 3) + desc = tf.pages[0].description + assert "Flextight FFF" in desc + assert "no scaling" in desc + + def test_picks_largest_ifd(self, tmp_path: str) -> None: + """When multiple IFDs exist, the largest by pixel count is used.""" + rng = np.random.RandomState(42) + fullres = rng.randint(0, 65535, (500, 750, 3), dtype=np.uint16) + small = np.zeros((50, 75, 3), dtype=np.uint16) + path = os.path.join(str(tmp_path), "multi.fff") + with tifffile.TiffWriter(path, byteorder=">") as tw: + tw.write(fullres, photometric="rgb") + tw.write(small, photometric="rgb") + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(path, out) + with tifffile.TiffFile(out) as tf: + assert tf.pages[0].asarray().shape == (500, 750, 3) + + def test_loader_returns_float32(self, tmp_path: str) -> None: + from negpy.infrastructure.loaders.fff_loader import FffLoader + + path = _make_flextight_fff(str(tmp_path)) + loader = FffLoader() + wrapper, metadata = loader.load(path) + with wrapper as w: + assert w.data.dtype == np.float32 + assert w.data.shape == (400, 600, 3) + assert w.data.min() >= 0.0 + assert w.data.max() <= 1.0 + assert "orientation" in metadata + assert "ir" in metadata From 58209f517f905320663f66ca4b580c0e07dc9bde Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 16:57:36 +0200 Subject: [PATCH 08/20] Add Noritsu RAW loader as a first-class scanner format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Headerless BGR16 LE files from Noritsu EZController (FULL*.RAW). Tiered dimension detection: exact table match (16 known pairs from Negmaster + confirmed samples), then known-height solve (4502/5028/6391). BGR→RGB swap, 12-bit data in 16-bit container, default 16× expansion for linear output (matches chemvert's confirmed x16 bit-shift). Pakon exclusion by file size prevents .raw collision — Pakon sizes (9M/24M/36M/48M/72M) never overlap with Noritsu sizes. --- negpy/desktop/view/sidebar/export.py | 1 + negpy/infrastructure/loaders/factory.py | 5 + .../infrastructure/loaders/noritsu_loader.py | 99 +++++++++++ negpy/services/export/linear_output.py | 38 +++++ tests/test_linear_output.py | 154 ++++++++++++++++++ 5 files changed, 297 insertions(+) create mode 100644 negpy/infrastructure/loaders/noritsu_loader.py diff --git a/negpy/desktop/view/sidebar/export.py b/negpy/desktop/view/sidebar/export.py index e894f8ac..1ae4f26b 100644 --- a/negpy/desktop/view/sidebar/export.py +++ b/negpy/desktop/view/sidebar/export.py @@ -701,6 +701,7 @@ def _on_linear_output_changed(self, enabled: bool) -> None: "dng": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "nef": [], "fff": [], + "noritsu": [("16× (default)", None), ("8×", 8.0), ("Off", 1.0)], "camera": [], "tiff": [("Off (default)", None), ("2×", 2.0), ("4×", 4.0)], "unsupported": [], diff --git a/negpy/infrastructure/loaders/factory.py b/negpy/infrastructure/loaders/factory.py index 75fe5305..10b26f2c 100644 --- a/negpy/infrastructure/loaders/factory.py +++ b/negpy/infrastructure/loaders/factory.py @@ -5,6 +5,7 @@ from negpy.infrastructure.loaders.jpeg_loader import JpegLoader from negpy.infrastructure.loaders.fff_loader import FffLoader, is_flextight_fff from negpy.infrastructure.loaders.nef_loader import NefLoader, is_coolscan_nef +from negpy.infrastructure.loaders.noritsu_loader import NoritsuLoader, is_noritsu_raw from negpy.infrastructure.loaders.rawpy_loader import RawpyLoader from negpy.infrastructure.loaders.constants import ( SUPPORTED_TIFF_EXTENSIONS, @@ -23,6 +24,7 @@ def __init__(self) -> None: self._jpeg = JpegLoader() self._nef = NefLoader() self._fff = FffLoader() + self._noritsu = NoritsuLoader() self._rawpy = RawpyLoader() def get_loader(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager[Any], dict]: @@ -37,6 +39,9 @@ def get_loader(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextM if PakonLoader.can_handle(file_path): return self._pakon.load(file_path) + if is_noritsu_raw(file_path): + return self._noritsu.load(file_path) + if is_coolscan_nef(file_path): return self._nef.load(file_path, linear_raw=linear_raw) diff --git a/negpy/infrastructure/loaders/noritsu_loader.py b/negpy/infrastructure/loaders/noritsu_loader.py new file mode 100644 index 00000000..2c9a802a --- /dev/null +++ b/negpy/infrastructure/loaders/noritsu_loader.py @@ -0,0 +1,99 @@ +import os +from typing import Any, ContextManager, Optional, Tuple + +import numpy as np + +from negpy.domain.interfaces import IImageLoader +from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper +from negpy.infrastructure.loaders.pakon_loader import PakonLoader +from negpy.kernel.image.logic import uint16_to_float32 + +KNOWN_NORITSU_DIMS: list[tuple[int, int]] = [ + (3711, 5028), + (4937, 5028), + (6079, 5028), + (7317, 5028), + (5010, 5028), + (4036, 5028), + (5185, 5028), + (5256, 5028), + (9972, 5028), + (10379, 5028), + (3859, 5028), + (7158, 4502), + (3551, 4502), + (12681, 4502), + (11348, 4502), + (4042, 6391), +] + +KNOWN_NORITSU_HEIGHTS: list[int] = sorted({h for _, h in KNOWN_NORITSU_DIMS}) + + +def detect_noritsu_dims(file_path: str) -> Optional[tuple[int, int]]: + """Tiered dimension detection for a headerless Noritsu RAW file. + + Tier 1: exact match against KNOWN_NORITSU_DIMS. + Tier 2: try each known height — if exactly one divides the file size + evenly, use it to solve for width. A new width under a known height + is expected (widths vary per frame length). + + Returns (width, height) or None if ambiguous/unknown. + """ + try: + size = os.path.getsize(file_path) + except OSError: + return None + for w, h in KNOWN_NORITSU_DIMS: + if w * h * 6 == size: + return (w, h) + height_matches: list[tuple[int, int]] = [] + for h in KNOWN_NORITSU_HEIGHTS: + stride = h * 6 + if size % stride == 0: + height_matches.append((size // stride, h)) + if len(height_matches) == 1: + return height_matches[0] + return None + + +def is_noritsu_raw(file_path: str) -> bool: + """True if this is a headerless Noritsu EZController RAW file. + + Detection: .raw extension + file size resolves to dimensions via the + known-dims table or a known-height match + NOT a Pakon spec. + """ + ext = os.path.splitext(file_path)[1].lower() + if ext != ".raw": + return False + if PakonLoader.can_handle(file_path): + return False + return detect_noritsu_dims(file_path) is not None + + +class NoritsuLoader(IImageLoader): + """Loader for headerless Noritsu EZController RAW files. + + Format: flat BGR16 little-endian, chunky, no header, no compression. + 12-bit sensor data in 16-bit container (max value 4095). + Dimensions resolved from file size via tiered detection. + """ + + def load(self, file_path: str) -> Tuple[ContextManager[Any], dict]: + dims = detect_noritsu_dims(file_path) + if dims is None: + raise ValueError(f"Unknown Noritsu dimensions for {file_path}") + w, h = dims + expected_pixels = w * h * 3 + + with open(file_path, "rb") as f: + data = np.fromfile(f, dtype=" bool: return False if is_flextight_fff(file_path): return False + if is_noritsu_raw(file_path): + return False return ext in SUPPORTED_RAW_EXTENSIONS @@ -130,6 +133,8 @@ def is_linear_output_supported(file_path: str) -> bool: return True if is_flextight_fff(file_path): return True + if is_noritsu_raw(file_path): + return True if _is_camera_raw(file_path): return True if _is_tiff(file_path): @@ -150,6 +155,8 @@ def linear_output_source_type(file_path: str) -> str: return "nef" if is_flextight_fff(file_path): return "fff" + if is_noritsu_raw(file_path): + return "noritsu" if _is_camera_raw(file_path): return "camera" if _is_tiff(file_path): @@ -304,6 +311,10 @@ def _decode_linear( meta = _read_source_meta_tiff(file_path) rgb, ir = _decode_fff(file_path, geometry) return rgb, ir, None, meta + if is_noritsu_raw(file_path): + rgb, ir = _decode_noritsu(file_path, geometry, expansion=expansion) + meta = _SourceMeta(make="Noritsu") + return rgb, ir, None, meta if _is_camera_raw(file_path): if rgbscan is not None and is_rgb_triplet(rgbscan): rgb, ir, wb, meta = _decode_camera_raw_triplet(file_path, rgbscan, geometry) @@ -334,6 +345,7 @@ def _decode_linear( PAKON_EXPANSION = 4.0 +NORITSU_EXPANSION = 16.0 _F335_SIZE = 72000000 @@ -486,6 +498,27 @@ def _decode_fff( return f32, ir +def _decode_noritsu( + file_path: str, + geometry: Optional[GeometryConfig] = None, + expansion: Optional[float] = None, +) -> tuple[np.ndarray, None]: + """Read a headerless Noritsu EZController RAW. Returns (rgb, None).""" + dims = detect_noritsu_dims(file_path) + if dims is None: + raise ValueError(f"Unknown Noritsu dimensions for {file_path}") + w, h = dims + with open(file_path, "rb") as f: + data = np.fromfile(f, dtype=" 1.0: + f32 = np.clip(f32 * factor, 0.0, 1.0) + f32 = _apply_geometry(f32, 0, geometry) + return f32, None + + def _decode_dng( file_path: str, geometry: Optional[GeometryConfig] = None, expansion: Optional[float] = None ) -> tuple[np.ndarray, Optional[np.ndarray]]: @@ -712,6 +745,9 @@ def _effective_expansion(file_path: str, expansion: Optional[float]) -> float: if PakonLoader.can_handle(file_path): factor = expansion if expansion is not None else _default_pakon_expansion(file_path) return factor if factor > 1.0 else 1.0 + if is_noritsu_raw(file_path): + factor = expansion if expansion is not None else NORITSU_EXPANSION + return factor if factor > 1.0 else 1.0 if _is_dng(file_path) and _is_linearraw_dng(file_path): return expansion if (expansion is not None and expansion > 1.0) else 1.0 if _is_tiff(file_path): @@ -733,6 +769,8 @@ def _source_format_label( return "Coolscan NEF" if is_flextight_fff(file_path): return "Flextight FFF" + if is_noritsu_raw(file_path): + return "Noritsu RAW" if _is_camera_raw(file_path): if is_stitch and stitch_has_triplets(stitch): n = 1 + len(stitch.stitch_paths) diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index 0411c128..3a7ce234 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -14,6 +14,7 @@ from negpy.kernel.image.logic import apply_exif_orientation from negpy.infrastructure.loaders.fff_loader import is_flextight_fff from negpy.infrastructure.loaders.nef_loader import is_coolscan_nef +from negpy.infrastructure.loaders.noritsu_loader import is_noritsu_raw, KNOWN_NORITSU_DIMS, KNOWN_NORITSU_HEIGHTS, detect_noritsu_dims from negpy.services.export.linear_output import ( TIFF_GAMMA_OPTIONS, _CameraWB, @@ -1420,3 +1421,156 @@ def test_loader_returns_float32(self, tmp_path: str) -> None: assert w.data.max() <= 1.0 assert "orientation" in metadata assert "ir" in metadata + + +def _make_noritsu_raw(tmp_dir: str, w: int = 4042, h: int = 6391) -> str: + """Create a synthetic Noritsu RAW file: headerless BGR16 LE, 12-bit data.""" + rng = np.random.RandomState(42) + bgr = rng.randint(0, 4096, size=(h, w, 3), dtype=np.uint16) + path = os.path.join(tmp_dir, "FULL000000020000.RAW") + bgr.astype(" str: + """Create a Noritsu RAW with the smallest known dims (3551×4502).""" + w, h = 3551, 4502 + rng = np.random.RandomState(77) + bgr = rng.randint(0, 4096, size=(h, w, 3), dtype=np.uint16) + path = os.path.join(tmp_dir, "FULL_small.raw") + bgr.astype(" None: + path = _make_noritsu_raw(str(tmp_path)) + assert is_noritsu_raw(path) + + def test_pakon_not_detected_as_noritsu(self, tmp_path: str) -> None: + path = _make_pakon_raw(str(tmp_path)) + assert not is_noritsu_raw(path) + + def test_unknown_size_not_detected(self, tmp_path: str) -> None: + data = np.zeros(12345678, dtype=np.uint8) + path = os.path.join(str(tmp_path), "mystery.raw") + data.tofile(path) + assert not is_noritsu_raw(path) + + def test_non_raw_ext_not_detected(self, tmp_path: str) -> None: + path = os.path.join(str(tmp_path), "photo.tiff") + tifffile.imwrite(path, np.zeros((10, 10, 3), dtype=np.uint16)) + assert not is_noritsu_raw(path) + + def test_noritsu_not_camera_raw(self, tmp_path: str) -> None: + path = _make_noritsu_raw(str(tmp_path)) + assert not _is_camera_raw(path) + + def test_linear_output_supported(self, tmp_path: str) -> None: + path = _make_noritsu_raw(str(tmp_path)) + assert is_linear_output_supported(path) + + def test_source_type_noritsu(self, tmp_path: str) -> None: + path = _make_noritsu_raw(str(tmp_path)) + assert linear_output_source_type(path) == "noritsu" + + def test_source_format_label(self, tmp_path: str) -> None: + path = _make_noritsu_raw(str(tmp_path)) + assert _source_format_label(path) == "Noritsu RAW" + + def test_default_expansion(self, tmp_path: str) -> None: + path = _make_noritsu_raw(str(tmp_path)) + assert _effective_expansion(path, None) == 16.0 + + def test_custom_expansion(self, tmp_path: str) -> None: + path = _make_noritsu_raw(str(tmp_path)) + assert _effective_expansion(path, 8.0) == 8.0 + + def test_export_roundtrip(self, tmp_path: str) -> None: + path = _make_noritsu_raw(str(tmp_path)) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(path, out) + with tifffile.TiffFile(out) as tf: + arr = tf.pages[0].asarray() + assert arr.dtype == np.uint16 + assert arr.shape == (6391, 4042, 3) + desc = tf.pages[0].description + assert "Noritsu RAW" in desc + assert "expansion: x16" in desc + + def test_export_custom_expansion(self, tmp_path: str) -> None: + path = _make_noritsu_raw(str(tmp_path)) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(path, out, expansion=8.0) + with tifffile.TiffFile(out) as tf: + desc = tf.pages[0].description + assert "expansion: x8" in desc + + def test_bgr_to_rgb_swap(self, tmp_path: str) -> None: + """Verify the loader performs BGR to RGB channel swap.""" + w, h = 4042, 6391 + bgr = np.zeros((h, w, 3), dtype=np.uint16) + bgr[:, :, 0] = 100 # B channel + bgr[:, :, 1] = 200 # G channel + bgr[:, :, 2] = 300 # R channel + path = os.path.join(str(tmp_path), "swap_test.raw") + bgr.astype(" g_val > b_val + + def test_loader_returns_float32(self, tmp_path: str) -> None: + from negpy.infrastructure.loaders.noritsu_loader import NoritsuLoader + + path = _make_noritsu_raw(str(tmp_path)) + loader = NoritsuLoader() + wrapper, metadata = loader.load(path) + with wrapper as w: + assert w.data.dtype == np.float32 + assert w.data.shape == (6391, 4042, 3) + assert w.data.min() >= 0.0 + assert w.data.max() <= 1.0 + assert metadata["orientation"] == 0 + assert metadata["ir"] is None + + def test_all_known_dims_unique_sizes(self) -> None: + """Every known dimension pair must produce a unique file size.""" + sizes = [w * h * 6 for w, h in KNOWN_NORITSU_DIMS] + assert len(sizes) == len(set(sizes)) + + def test_small_dims_detected(self, tmp_path: str) -> None: + path = _make_noritsu_raw_small(str(tmp_path)) + assert is_noritsu_raw(path) + + def test_novel_width_known_height_detected(self, tmp_path: str) -> None: + """Tier 2: a new width (not in the table) under a known height resolves.""" + novel_w, h = 5555, 5028 + assert (novel_w, h) not in KNOWN_NORITSU_DIMS + rng = np.random.RandomState(99) + bgr = rng.randint(0, 4096, size=(h, novel_w, 3), dtype=np.uint16) + path = os.path.join(str(tmp_path), "novel.raw") + bgr.astype(" None: + """If a file size divides evenly by multiple known heights, reject it.""" + from math import lcm + + common = lcm(KNOWN_NORITSU_HEIGHTS[0], KNOWN_NORITSU_HEIGHTS[1]) + size = common * 6 + data = np.zeros(size, dtype=np.uint8) + path = os.path.join(str(tmp_path), "ambiguous.raw") + data.tofile(path) + assert detect_noritsu_dims(path) is None + assert not is_noritsu_raw(path) From 6f9b3f8fb78539af001a7c5184c72493abbaeac7 Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 20:26:11 +0200 Subject: [PATCH 09/20] Harden scanner loaders: CFA rejection, NEF gamma, Noritsu tier-3, overlap tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - NEF: add _has_cfa_subifd() to reject camera NEFs that have RGB preview SubIFDs alongside Bayer data (false-positive fix) - NEF: wire gamma_key through _decode_nef() so Linear Output gamma combo applies to Coolscan NEFs, not just TIFFs - Noritsu: add tier-3 open divisor search for files that match no known height — accepts only if exactly one film-plausible (w,h) pair exists - Add Pakon/Noritsu overlap tests documenting the theoretical 1777×4502 collision (below any real scan width, harmless) - Document dead 4-channel branches in NEF and FFF loaders --- negpy/desktop/view/sidebar/export.py | 8 +- negpy/infrastructure/loaders/fff_loader.py | 2 + negpy/infrastructure/loaders/nef_loader.py | 31 +++++- .../infrastructure/loaders/noritsu_loader.py | 27 +++++- negpy/services/export/linear_output.py | 5 +- tests/test_linear_output.py | 96 +++++++++++++++++++ 6 files changed, 161 insertions(+), 8 deletions(-) diff --git a/negpy/desktop/view/sidebar/export.py b/negpy/desktop/view/sidebar/export.py index 1ae4f26b..c05099eb 100644 --- a/negpy/desktop/view/sidebar/export.py +++ b/negpy/desktop/view/sidebar/export.py @@ -734,10 +734,10 @@ def _refresh_linear_expansion_combo(self) -> None: combo.blockSignals(False) self._current_expansion_source_type = source_type - is_tiff = source_type == "tiff" - self.linear_gamma_row.setVisible(is_tiff) - self.linear_gamma_hint.setVisible(is_tiff) - if is_tiff: + needs_gamma = source_type in ("tiff", "nef") + self.linear_gamma_row.setVisible(needs_gamma) + self.linear_gamma_hint.setVisible(needs_gamma) + if needs_gamma: self._refresh_linear_gamma_combo() is_camera = source_type == "camera" diff --git a/negpy/infrastructure/loaders/fff_loader.py b/negpy/infrastructure/loaders/fff_loader.py index ece727aa..3660c5bd 100644 --- a/negpy/infrastructure/loaders/fff_loader.py +++ b/negpy/infrastructure/loaders/fff_loader.py @@ -86,6 +86,8 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager icc_bytes = bytes(tag.value) break + # Imacon/Flextight scanners have no IR hardware — no 4th channel exists. + # 4-channel branch kept for defensive consistency with TiffLoader. ir: Optional[np.ndarray] = None if arr.ndim == 3 and arr.shape[2] == 4: ir = normalize_ir_to_float32(arr[:, :, 3]) diff --git a/negpy/infrastructure/loaders/nef_loader.py b/negpy/infrastructure/loaders/nef_loader.py index 1d96d87f..799d0947 100644 --- a/negpy/infrastructure/loaders/nef_loader.py +++ b/negpy/infrastructure/loaders/nef_loader.py @@ -38,12 +38,39 @@ def _find_rgb_subifd(tif: tifffile.TiffFile) -> Optional[Any]: return best +def _has_cfa_subifd(tif: tifffile.TiffFile) -> bool: + """True if any SubIFD carries Bayer/CFA data (camera NEF indicator). + + Camera NEFs embed an RGB preview SubIFD alongside the Bayer mosaic data, + so "has RGB SubIFD" alone is not enough to detect a scanner NEF — we must + also confirm there's no CFA data. + """ + page0 = tif.pages[0] + for sub in page0.pages or []: + tags = getattr(sub, "tags", None) + if tags is None: + continue + photo_tag = tags.get("PhotometricInterpretation") + comp_tag = tags.get("Compression") + spp_tag = tags.get("SamplesPerPixel") + if photo_tag is not None and int(photo_tag.value) == 32803: + return True + if comp_tag is not None and int(comp_tag.value) == 34713: + return True + spp = int(spp_tag.value) if spp_tag is not None else 1 + if spp == 1 and sub.shape[0] * sub.shape[1] > 100_000: + return True + return False + + def is_coolscan_nef(file_path: str) -> bool: - """True if this NEF is a Nikon Coolscan scanner file (already-processed RGB in SubIFDs).""" + """True if this NEF is a Nikon Coolscan scanner file (RGB SubIFDs, no CFA data).""" if os.path.splitext(file_path)[1].lower() != ".nef": return False try: with tifffile.TiffFile(file_path) as tif: + if _has_cfa_subifd(tif): + return False return _find_rgb_subifd(tif) is not None except Exception: return False @@ -78,6 +105,8 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager icc_bytes = bytes(tag.value) break + # Coolscan NEFs have no separate IR channel (ICE is baked at scan time). + # 4-channel branch kept for defensive consistency with TiffLoader. ir: Optional[np.ndarray] = None if arr.ndim == 3 and arr.shape[2] == 4: ir = normalize_ir_to_float32(arr[:, :, 3]) diff --git a/negpy/infrastructure/loaders/noritsu_loader.py b/negpy/infrastructure/loaders/noritsu_loader.py index 2c9a802a..473a455b 100644 --- a/negpy/infrastructure/loaders/noritsu_loader.py +++ b/negpy/infrastructure/loaders/noritsu_loader.py @@ -30,13 +30,21 @@ KNOWN_NORITSU_HEIGHTS: list[int] = sorted({h for _, h in KNOWN_NORITSU_DIMS}) +_MIN_SCAN_WIDTH = 2000 +_MAX_SCAN_WIDTH = 15000 +_MIN_ASPECT = 1.0 +_MAX_ASPECT = 4.5 + + def detect_noritsu_dims(file_path: str) -> Optional[tuple[int, int]]: """Tiered dimension detection for a headerless Noritsu RAW file. Tier 1: exact match against KNOWN_NORITSU_DIMS. Tier 2: try each known height — if exactly one divides the file size - evenly, use it to solve for width. A new width under a known height - is expected (widths vary per frame length). + evenly, use it to solve for width. + Tier 3: open divisor search — enumerate all (w, h) where w*h*6 == size, + both dimensions in a sane scanner range, and aspect ratio is film-plausible. + Accept only if exactly one candidate survives. Returns (width, height) or None if ambiguous/unknown. """ @@ -54,6 +62,21 @@ def detect_noritsu_dims(file_path: str) -> Optional[tuple[int, int]]: height_matches.append((size // stride, h)) if len(height_matches) == 1: return height_matches[0] + + total_pixels = size // 6 + if total_pixels * 6 != size: + return None + candidates: list[tuple[int, int]] = [] + for w in range(_MIN_SCAN_WIDTH, _MAX_SCAN_WIDTH + 1): + if total_pixels % w == 0: + h = total_pixels // w + if h < _MIN_SCAN_WIDTH: + continue + aspect = max(w, h) / min(w, h) + if _MIN_ASPECT <= aspect <= _MAX_ASPECT: + candidates.append((w, h)) + if len(candidates) == 1: + return candidates[0] return None diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index 5bad31e1..1dfee57a 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -305,7 +305,7 @@ def _decode_linear( return rgb, ir, wb, meta if is_coolscan_nef(file_path): meta = _read_source_meta_tiff(file_path) - rgb, ir = _decode_nef(file_path, geometry) + rgb, ir = _decode_nef(file_path, geometry, gamma_key=gamma_key) return rgb, ir, None, meta if is_flextight_fff(file_path): meta = _read_source_meta_tiff(file_path) @@ -425,6 +425,7 @@ def _decode_pakon(file_path: str, geometry: Optional[GeometryConfig] = None, exp def _decode_nef( file_path: str, geometry: Optional[GeometryConfig] = None, + gamma_key: str = "linear", ) -> tuple[np.ndarray, Optional[np.ndarray]]: """Read a Coolscan NEF via SubIFDs. Returns (rgb, ir_or_none).""" from negpy.infrastructure.loaders.nef_loader import _find_rgb_subifd @@ -452,6 +453,8 @@ def _decode_nef( elif f32.ndim == 2: f32 = np.stack([f32, f32, f32], axis=2) f32 = np.clip(f32, 0.0, 1.0) + if gamma_key != "linear": + f32 = _linearize(f32, gamma_key) orientation = read_orientation(file_path) f32 = _apply_geometry(f32, orientation, geometry) diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index 3a7ce234..12b5d342 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -1251,6 +1251,23 @@ def _make_camera_nef(tmp_dir: str) -> str: return path +def _make_camera_nef_with_preview(tmp_dir: str) -> str: + """Camera NEF with both a Bayer SubIFD and an RGB preview SubIFD. + + Real Nikon cameras routinely embed a full-res RGB preview alongside + the CFA data. This must NOT match the scanner detector. + """ + thumb = np.zeros((50, 75, 3), dtype=np.uint8) + bayer = np.zeros((4000, 6000), dtype=np.uint16) + preview = np.zeros((4000, 6000, 3), dtype=np.uint8) + path = os.path.join(tmp_dir, "camera_preview.nef") + with tifffile.TiffWriter(path) as tw: + tw.write(thumb, photometric="rgb", subifds=2) + tw.write(bayer, photometric="minisblack") + tw.write(preview, photometric="rgb") + return path + + class TestCoolscanNef: def test_detect_coolscan_nef(self, tmp_path: str) -> None: path = _make_coolscan_nef(str(tmp_path)) @@ -1260,6 +1277,15 @@ def test_camera_nef_not_detected(self, tmp_path: str) -> None: path = _make_camera_nef(str(tmp_path)) assert not is_coolscan_nef(path) + def test_camera_nef_with_preview_not_detected(self, tmp_path: str) -> None: + """Camera NEF with RGB preview SubIFD alongside Bayer data must not match.""" + path = _make_camera_nef_with_preview(str(tmp_path)) + assert not is_coolscan_nef(path) + + def test_camera_nef_with_preview_is_camera_raw(self, tmp_path: str) -> None: + path = _make_camera_nef_with_preview(str(tmp_path)) + assert _is_camera_raw(path) + def test_non_nef_not_detected(self, tmp_path: str) -> None: path = os.path.join(str(tmp_path), "photo.tiff") tifffile.imwrite(path, np.zeros((10, 10, 3), dtype=np.uint16)) @@ -1574,3 +1600,73 @@ def test_ambiguous_heights_not_detected(self, tmp_path: str) -> None: data.tofile(path) assert detect_noritsu_dims(path) is None assert not is_noritsu_raw(path) + + def test_tier3_novel_height_detected(self, tmp_path: str) -> None: + """Tier 3: a file whose dimensions match no known height but has + exactly one film-plausible divisor pair resolves. + + 4001 is prime, so 4001×4001 is the only (w,h) where both are in + the scan-width range — the open divisor search finds it uniquely. + """ + w, h = 4001, 4001 + assert h not in KNOWN_NORITSU_HEIGHTS + assert (w, h) not in KNOWN_NORITSU_DIMS + data = np.zeros(w * h * 3, dtype=np.uint16) + path = os.path.join(str(tmp_path), "tier3.raw") + data.astype(" None: + """Tier 3: if multiple film-plausible divisor pairs exist, reject.""" + w, h = 4000, 6000 + assert h not in KNOWN_NORITSU_HEIGHTS + data = np.zeros(w * h * 3, dtype=np.uint16) + path = os.path.join(str(tmp_path), "ambiguous_t3.raw") + data.astype(" None: + """No known Noritsu dimension pair produces a file size within Pakon's tolerance.""" + from negpy.infrastructure.loaders.pakon_loader import PakonLoader + + for w, h in KNOWN_NORITSU_DIMS: + noritsu_size = w * h * 6 + for spec in PakonLoader.PAKON_SPECS: + assert abs(noritsu_size - spec["size"]) >= 1024, ( + f"Noritsu {w}x{h} ({noritsu_size}) collides with Pakon {spec['desc']} ({spec['size']})" + ) + + def test_pakon_noritsu_tier2_no_collision_above_min_scan_width(self) -> None: + """No Noritsu tier-2 file at a real scan width lands in Pakon's tolerance. + + The narrowest known Noritsu scan width is 3551. Checks every known height + x every width from 3000-15000 (generous margin below the minimum). + Widths below 3000 are implausible for any real film scan. + """ + from negpy.infrastructure.loaders.pakon_loader import PakonLoader + + min_scan_width = 3000 + collisions: list[str] = [] + for h in KNOWN_NORITSU_HEIGHTS: + for w in range(min_scan_width, 15001): + noritsu_size = w * h * 6 + for spec in PakonLoader.PAKON_SPECS: + if abs(noritsu_size - spec["size"]) < 1024: + collisions.append(f"{w}x{h} ({noritsu_size}) vs Pakon {spec['desc']} ({spec['size']})") + assert not collisions, f"Collisions at real scan widths: {collisions}" + + def test_pakon_noritsu_theoretical_collision_documented(self) -> None: + """Document the known theoretical collision at width 1777 (not a real scan width). + + Height 4502, width 1777 produces 48,000,324 bytes — within Pakon's + 48M ± 1024 window. This is harmless because: (a) 1777 is far below + any real Noritsu scan width (min known: 3551), and (b) Pakon checks + first in the factory, so this size would be claimed as Pakon. + """ + from negpy.infrastructure.loaders.pakon_loader import PakonLoader + + collision_size = 1777 * 4502 * 6 + pakon_48m = next(s for s in PakonLoader.PAKON_SPECS if s["size"] == 48000000) + assert abs(collision_size - pakon_48m["size"]) < 1024 + assert 1777 < 3000 # well below any real scan width From aad23b3016f63c99fad4eeab22ccdde0d7ac7c30 Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 21:03:52 +0200 Subject: [PATCH 10/20] Fix Linear Output ICE for extra-page and sidecar IR sources _decode_tiff only checked for IR as a 4th sample in the main TIFF page, missing SilverFast HDRi TIFFs (IR stored as a separate full-res grayscale page with SubfileType=4) and _ir.tif sidecars. Add fallback searches matching what TiffLoader already does. Skip writing the IR sidecar when ICE is baked into the RGB output. --- negpy/services/export/linear_output.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index 1dfee57a..38f5e022 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -33,6 +33,8 @@ from negpy.infrastructure.loaders.fff_loader import is_flextight_fff from negpy.infrastructure.loaders.nef_loader import is_coolscan_nef from negpy.infrastructure.loaders.noritsu_loader import is_noritsu_raw, detect_noritsu_dims +from negpy.infrastructure.loaders.ir_planes import find_ir_plane +from negpy.infrastructure.loaders.tiff_loader import _read_sidecar_ir from negpy.infrastructure.loaders.rawpy_loader import ( _find_linearraw_page, _is_dng, @@ -396,6 +398,17 @@ def _decode_tiff( f32 = f32[:, :, :3] elif samples == 1 and f32.ndim == 2: f32 = np.stack([f32, f32, f32], axis=2) + + if ir is None: + try: + with _tifffile.TiffFile(file_path) as tif: + ir = find_ir_plane(tif.pages[1:], f32.shape[0], f32.shape[1]) + except Exception: + pass + + if ir is None: + ir, _ = _read_sidecar_ir(file_path) + f32 = np.clip(f32, 0.0, 1.0) if gamma_key != "linear": f32 = _linearize(f32, gamma_key) @@ -968,7 +981,7 @@ def export_linear_output( gamma_key=gamma_key, ) - if ir is not None: + if ir is not None and not ice_applied: stem, ext = os.path.splitext(output_path) ir_path = f"{stem}_ir{ext}" _write_ir_tiff(ir, ir_path, os.path.basename(file_path)) From 6d9937f1389d4425c4875ffe6e43ce8b9c2acaca Mon Sep 17 00:00:00 2001 From: Mats Date: Wed, 5 Aug 2026 22:36:37 +0200 Subject: [PATCH 11/20] Parse FFF plist metadata, fix Linear Output IR bugs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FFF loader: parse tag 50457 (FlexColor plist) and tag 46279 (firmware) to extract film_stock, film_type, flexcolor_gamma, scan_dpi, scan_date, flexcolor_version, and scanner_serial into the metadata dict. Linear Output exports now carry Make/Model/DateTime from these fields. Linear Output: fix two IR handling bugs found by auditing against the main loader path: - 4-channel TIFFs now check ExtraSamples (via _extract_ir_from_extrasamples) instead of blindly treating the 4th channel as IR — alpha channels are correctly dropped instead of being fed to ICE as fake dust data. - Sidecar IR validity mask is no longer discarded — _read_sidecar_ir already applies the mask internally (invalid pixels set to 1.0). --- negpy/infrastructure/loaders/fff_loader.py | 69 ++++++++++++++++++++++ negpy/services/export/linear_output.py | 40 +++++++++++-- tests/test_linear_output.py | 12 +++- 3 files changed, 115 insertions(+), 6 deletions(-) diff --git a/negpy/infrastructure/loaders/fff_loader.py b/negpy/infrastructure/loaders/fff_loader.py index 3660c5bd..0b2c3c46 100644 --- a/negpy/infrastructure/loaders/fff_loader.py +++ b/negpy/infrastructure/loaders/fff_loader.py @@ -1,4 +1,6 @@ import os +import plistlib +import re from typing import Any, ContextManager, Optional, Tuple import numpy as np @@ -14,6 +16,63 @@ logger = get_logger(__name__) +_FILM_TYPES = {0: "positive", 1: "negative", 2: "b&w"} + + +def _parse_fff_plist(raw: bytes) -> dict: + """Extract FlexColor metadata from tag 50457. + + Returns a flat dict with scanner-relevant fields or {} on failure. + The plist may have a 4-byte length prefix (FlexColor 4.8.10+) and is + always null-padded to a fixed block size. + """ + try: + xml_start = raw.find(b"") + if xml_start < 0 or end < 0: + return {} + plist = plistlib.loads(raw[xml_start : end + len(b"")]) + settings = plist.get("ImageSettings", [{}])[0] + ic = settings.get("ImageCorrection", {}) + desc = settings.get("ImageDescription", {}) + created = settings.get("Created", {}) + + result: dict = {} + film_name = settings.get("Name") + if film_name: + result["film_stock"] = film_name + film_type = ic.get("FilmType") + if film_type is not None: + result["film_type"] = _FILM_TYPES.get(film_type, str(film_type)) + gamma = ic.get("Gamma") + if gamma is not None: + result["flexcolor_gamma"] = round(float(gamma), 2) + res = desc.get("Resolution") + if res: + result["scan_dpi"] = int(res) + if created.get("Year"): + result["scan_date"] = f"{created['Year']:04d}-{created.get('Month', 0):02d}-{created.get('Day', 0):02d}" + return result + except Exception: + return {} + + +def _parse_fff_firmware(raw: bytes) -> dict: + """Extract FlexColor version and scanner serial from tag 46279.""" + try: + text = raw.decode("latin1", errors="replace") + result: dict = {} + ver = re.search(r"(\d+\.\d+[\.\d]* \w+)", text) + if ver: + result["flexcolor_version"] = ver.group(1) + ser = re.search(r"(FX\d+)", text) + if ser: + result["scanner_serial"] = ser.group(1) + return result + except Exception: + return {} + + def _find_full_res_ifd(tif: tifffile.TiffFile) -> Optional[Any]: """Return the largest RGB IFD by pixel count. @@ -77,6 +136,8 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager arr = page.asarray() icc_bytes: Optional[bytes] = None + fff_meta: dict = {} + p0_tags = getattr(tif.pages[0], "tags", None) for p in (page, tif.pages[0]): tags = getattr(p, "tags", None) if tags is None: @@ -85,6 +146,13 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager if tag is not None and tag.value: icc_bytes = bytes(tag.value) break + if p0_tags is not None: + plist_tag = p0_tags.get(50457) + if plist_tag is not None and isinstance(plist_tag.value, bytes): + fff_meta.update(_parse_fff_plist(plist_tag.value)) + fw_tag = p0_tags.get(46279) + if fw_tag is not None and isinstance(fw_tag.value, bytes): + fff_meta.update(_parse_fff_firmware(fw_tag.value)) # Imacon/Flextight scanners have no IR hardware — no 4th channel exists. # 4-channel branch kept for defensive consistency with TiffLoader. @@ -115,5 +183,6 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager "color_space": color_space, "icc_profile": icc_bytes, "ir": ir, + **fff_meta, } return NonStandardFileWrapper(f32), metadata diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index 38f5e022..c944666e 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -34,7 +34,7 @@ from negpy.infrastructure.loaders.nef_loader import is_coolscan_nef from negpy.infrastructure.loaders.noritsu_loader import is_noritsu_raw, detect_noritsu_dims from negpy.infrastructure.loaders.ir_planes import find_ir_plane -from negpy.infrastructure.loaders.tiff_loader import _read_sidecar_ir +from negpy.infrastructure.loaders.tiff_loader import _extract_ir_from_extrasamples, _read_sidecar_ir from negpy.infrastructure.loaders.rawpy_loader import ( _find_linearraw_page, _is_dng, @@ -107,6 +107,37 @@ def _read_source_meta_exif(file_path: str) -> _SourceMeta: return _SourceMeta() +def _read_fff_meta(file_path: str) -> _SourceMeta: + """Build _SourceMeta from FFF proprietary tags (plist + firmware).""" + try: + from negpy.infrastructure.loaders.fff_loader import _parse_fff_firmware, _parse_fff_plist + + with _tifffile.TiffFile(file_path) as tif: + p0_tags = getattr(tif.pages[0], "tags", None) + if p0_tags is None: + return _SourceMeta() + parts: dict = {} + plist_tag = p0_tags.get(50457) + if plist_tag is not None and isinstance(plist_tag.value, bytes): + parts.update(_parse_fff_plist(plist_tag.value)) + fw_tag = p0_tags.get(46279) + if fw_tag is not None and isinstance(fw_tag.value, bytes): + parts.update(_parse_fff_firmware(fw_tag.value)) + make = "Imacon/Hasselblad" + serial = parts.get("scanner_serial", "") + model_parts = [s for s in ("Flextight", serial) if s] + model = " ".join(model_parts) if model_parts else None + film = parts.get("film_stock") + film_type = parts.get("film_type") + if film and film_type: + make = f"{make} ({film}, {film_type})" + elif film: + make = f"{make} ({film})" + return _SourceMeta(make=make, model=model, datetime=parts.get("scan_date")) + except Exception: + return _SourceMeta(make="Imacon/Hasselblad") + + def _is_camera_raw(file_path: str) -> bool: ext = os.path.splitext(file_path)[1].lower() if ext in SUPPORTED_TIFF_EXTENSIONS | SUPPORTED_JPEG_EXTENSIONS: @@ -310,7 +341,7 @@ def _decode_linear( rgb, ir = _decode_nef(file_path, geometry, gamma_key=gamma_key) return rgb, ir, None, meta if is_flextight_fff(file_path): - meta = _read_source_meta_tiff(file_path) + meta = _read_fff_meta(file_path) rgb, ir = _decode_fff(file_path, geometry) return rgb, ir, None, meta if is_noritsu_raw(file_path): @@ -394,8 +425,7 @@ def _decode_tiff( f32 = arr.astype(np.float32) * scale ir = None if samples == 4 and f32.ndim == 3 and f32.shape[2] == 4: - ir = f32[:, :, 3] - f32 = f32[:, :, :3] + f32, ir = _extract_ir_from_extrasamples(file_path, f32) elif samples == 1 and f32.ndim == 2: f32 = np.stack([f32, f32, f32], axis=2) @@ -407,7 +437,7 @@ def _decode_tiff( pass if ir is None: - ir, _ = _read_sidecar_ir(file_path) + ir, _mask = _read_sidecar_ir(file_path) f32 = np.clip(f32, 0.0, 1.0) if gamma_key != "linear": diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index 12b5d342..b0d9a30b 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -1161,12 +1161,22 @@ def test_decode_tiff_4ch_splits_ir(self, tmp_path: str) -> None: data = np.ones((8, 8, 4), dtype=np.uint16) * 32768 path = os.path.join(str(tmp_path), "4ch.tif") - tifffile.imwrite(path, data) + tifffile.imwrite(path, data, extrasamples=[0]) rgb, ir = _decode_tiff(path) assert rgb.shape == (8, 8, 3) assert ir is not None assert ir.shape[:2] == (8, 8) + def test_decode_tiff_4ch_alpha_not_ir(self, tmp_path: str) -> None: + from negpy.services.export.linear_output import _decode_tiff + + data = np.ones((8, 8, 4), dtype=np.uint16) * 32768 + path = os.path.join(str(tmp_path), "4ch_alpha.tif") + tifffile.imwrite(path, data, extrasamples=[2]) + rgb, ir = _decode_tiff(path) + assert rgb.shape == (8, 8, 3) + assert ir is None + def test_decode_tiff_applies_gamma(self, tmp_path: str) -> None: from negpy.services.export.linear_output import _decode_tiff From c9ca59976221e83b3f1af85703b9d7861c2fdd32 Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 01:25:06 +0200 Subject: [PATCH 12/20] Consolidate Linear Output decode functions to use main loaders _decode_tiff, _decode_nef, _decode_fff replaced their hand-rolled dtype-scaling, IR-extraction and geometry logic with a single _decode_via_loader helper that calls the main-path loader with linear_raw=True. Removes ~100 lines of duplicated code that was drifting from the loaders (ExtraSamples handling, sidecar IR, etc.). --- negpy/services/export/linear_output.py | 126 ++++++------------------- 1 file changed, 27 insertions(+), 99 deletions(-) diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index c944666e..baf5ef75 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -33,8 +33,6 @@ from negpy.infrastructure.loaders.fff_loader import is_flextight_fff from negpy.infrastructure.loaders.nef_loader import is_coolscan_nef from negpy.infrastructure.loaders.noritsu_loader import is_noritsu_raw, detect_noritsu_dims -from negpy.infrastructure.loaders.ir_planes import find_ir_plane -from negpy.infrastructure.loaders.tiff_loader import _extract_ir_from_extrasamples, _read_sidecar_ir from negpy.infrastructure.loaders.rawpy_loader import ( _find_linearraw_page, _is_dng, @@ -409,46 +407,10 @@ def _decode_tiff( gamma_key: str = "linear", expansion: Optional[float] = None, ) -> tuple[np.ndarray, Optional[np.ndarray]]: - """Read a TIFF, optionally linearize, strip everything. Returns (rgb, ir_or_none).""" - with _tifffile.TiffFile(file_path) as tif: - page = tif.pages[0] - arr = page.asarray() - samples = page.samplesperpixel - if arr.dtype == np.uint16: - scale = 1.0 / 65535.0 - elif arr.dtype == np.uint8: - scale = 1.0 / 255.0 - elif arr.dtype == np.float32: - scale = 1.0 - else: - scale = 1.0 / float(np.iinfo(arr.dtype).max) if np.issubdtype(arr.dtype, np.integer) else 1.0 - f32 = arr.astype(np.float32) * scale - ir = None - if samples == 4 and f32.ndim == 3 and f32.shape[2] == 4: - f32, ir = _extract_ir_from_extrasamples(file_path, f32) - elif samples == 1 and f32.ndim == 2: - f32 = np.stack([f32, f32, f32], axis=2) - - if ir is None: - try: - with _tifffile.TiffFile(file_path) as tif: - ir = find_ir_plane(tif.pages[1:], f32.shape[0], f32.shape[1]) - except Exception: - pass + """Read a TIFF via the main loader, optionally linearize. Returns (rgb, ir_or_none).""" + from negpy.infrastructure.loaders.tiff_loader import TiffLoader - if ir is None: - ir, _mask = _read_sidecar_ir(file_path) - - f32 = np.clip(f32, 0.0, 1.0) - if gamma_key != "linear": - f32 = _linearize(f32, gamma_key) - if expansion is not None and expansion > 1.0: - f32 = np.clip(f32 * expansion, 0.0, 1.0) - orientation = read_orientation(file_path) - f32 = _apply_geometry(f32, orientation, geometry) - if ir is not None: - ir = _apply_geometry(ir, orientation, geometry) - return f32, ir + return _decode_via_loader(TiffLoader(), file_path, geometry, gamma_key, expansion) def _decode_pakon(file_path: str, geometry: Optional[GeometryConfig] = None, expansion: Optional[float] = None) -> tuple[np.ndarray, None]: @@ -465,83 +427,49 @@ def _decode_pakon(file_path: str, geometry: Optional[GeometryConfig] = None, exp return f32, None -def _decode_nef( +def _decode_via_loader( + loader: "IImageLoader", file_path: str, geometry: Optional[GeometryConfig] = None, gamma_key: str = "linear", + expansion: Optional[float] = None, ) -> tuple[np.ndarray, Optional[np.ndarray]]: - """Read a Coolscan NEF via SubIFDs. Returns (rgb, ir_or_none).""" - from negpy.infrastructure.loaders.nef_loader import _find_rgb_subifd - - with _tifffile.TiffFile(file_path) as tif: - sub = _find_rgb_subifd(tif) - if sub is None: - raise ValueError(f"No RGB SubIFD in {file_path}") - arr = sub.asarray() - - if arr.dtype == np.uint16: - scale = 1.0 / 65535.0 - elif arr.dtype == np.uint8: - scale = 1.0 / 255.0 - elif arr.dtype == np.float32: - scale = 1.0 - else: - scale = 1.0 / float(np.iinfo(arr.dtype).max) if np.issubdtype(arr.dtype, np.integer) else 1.0 - f32 = arr.astype(np.float32) * scale - - ir = None - if f32.ndim == 3 and f32.shape[2] == 4: - ir = f32[:, :, 3] - f32 = f32[:, :, :3] - elif f32.ndim == 2: - f32 = np.stack([f32, f32, f32], axis=2) + """Decode through a main-path loader with linear_raw=True, then apply geometry.""" + ctx_mgr, metadata = loader.load(file_path, linear_raw=True) + with ctx_mgr as wrapper: + f32 = wrapper.data if isinstance(wrapper, NonStandardFileWrapper) else np.asarray(wrapper) + ir = metadata.get("ir") f32 = np.clip(f32, 0.0, 1.0) if gamma_key != "linear": f32 = _linearize(f32, gamma_key) - - orientation = read_orientation(file_path) + if expansion is not None and expansion > 1.0: + f32 = np.clip(f32 * expansion, 0.0, 1.0) + orientation = metadata.get("orientation", 0) f32 = _apply_geometry(f32, orientation, geometry) if ir is not None: ir = _apply_geometry(ir, orientation, geometry) return f32, ir -def _decode_fff( +def _decode_nef( file_path: str, geometry: Optional[GeometryConfig] = None, + gamma_key: str = "linear", ) -> tuple[np.ndarray, Optional[np.ndarray]]: - """Read a Flextight FFF via largest RGB IFD. Returns (rgb, ir_or_none).""" - from negpy.infrastructure.loaders.fff_loader import _find_full_res_ifd + """Read a Coolscan NEF via the main loader. Returns (rgb, ir_or_none).""" + from negpy.infrastructure.loaders.nef_loader import NefLoader - with _tifffile.TiffFile(file_path) as tif: - page = _find_full_res_ifd(tif) - if page is None: - raise ValueError(f"No full-res RGB IFD in {file_path}") - arr = page.asarray() + return _decode_via_loader(NefLoader(), file_path, geometry, gamma_key) - if arr.dtype == np.uint16: - scale = 1.0 / 65535.0 - elif arr.dtype == np.uint8: - scale = 1.0 / 255.0 - elif arr.dtype == np.float32: - scale = 1.0 - else: - scale = 1.0 / float(np.iinfo(arr.dtype).max) if np.issubdtype(arr.dtype, np.integer) else 1.0 - f32 = arr.astype(np.float32) * scale - - ir = None - if f32.ndim == 3 and f32.shape[2] == 4: - ir = f32[:, :, 3] - f32 = f32[:, :, :3] - elif f32.ndim == 2: - f32 = np.stack([f32, f32, f32], axis=2) - f32 = np.clip(f32, 0.0, 1.0) - orientation = read_orientation(file_path) - f32 = _apply_geometry(f32, orientation, geometry) - if ir is not None: - ir = _apply_geometry(ir, orientation, geometry) - return f32, ir +def _decode_fff( + file_path: str, + geometry: Optional[GeometryConfig] = None, +) -> tuple[np.ndarray, Optional[np.ndarray]]: + """Read a Flextight FFF via the main loader. Returns (rgb, ir_or_none).""" + from negpy.infrastructure.loaders.fff_loader import FffLoader + + return _decode_via_loader(FffLoader(), file_path, geometry) def _decode_noritsu( From b763a20728edd1820661b782051549ae54e0ca59 Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 01:42:17 +0200 Subject: [PATCH 13/20] Strip sRGB linearization from NEF and FFF loaders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scanner loaders should return data as-is without color-space assumptions. The sRGB→linear conversion was copied from TiffLoader but compromises the ability to recover the original pixel values. --- negpy/infrastructure/loaders/fff_loader.py | 27 +++----------------- negpy/infrastructure/loaders/nef_loader.py | 29 +++------------------- 2 files changed, 6 insertions(+), 50 deletions(-) diff --git a/negpy/infrastructure/loaders/fff_loader.py b/negpy/infrastructure/loaders/fff_loader.py index 0b2c3c46..f4d28605 100644 --- a/negpy/infrastructure/loaders/fff_loader.py +++ b/negpy/infrastructure/loaders/fff_loader.py @@ -7,10 +7,9 @@ import tifffile from negpy.domain.interfaces import IImageLoader -from negpy.domain.models import ColorSpace -from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, identify_color_space_from_icc, read_orientation +from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, read_orientation from negpy.infrastructure.loaders.ir_planes import normalize_ir_to_float32 -from negpy.kernel.image.logic import srgb_to_linear, uint8_to_float32, uint16_to_float32 +from negpy.kernel.image.logic import uint8_to_float32, uint16_to_float32 from negpy.kernel.system.logging import get_logger logger = get_logger(__name__) @@ -124,8 +123,7 @@ class FffLoader(IImageLoader): top-level IFD (picked by pixel count, not SubfileType tag). The data is uninverted scanner output — linear, no gamma applied. - Color space handling follows TiffLoader: ICC profile → identify space → - linearise if sRGB. Untagged 16-bit is assumed linear. + Data is returned as-is — no color-space assumptions or linearization. """ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager[Any], dict]: @@ -135,17 +133,8 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager raise ValueError(f"No full-res RGB IFD in {file_path}") arr = page.asarray() - icc_bytes: Optional[bytes] = None fff_meta: dict = {} p0_tags = getattr(tif.pages[0], "tags", None) - for p in (page, tif.pages[0]): - tags = getattr(p, "tags", None) - if tags is None: - continue - tag = tags.get("InterColorProfile") - if tag is not None and tag.value: - icc_bytes = bytes(tag.value) - break if p0_tags is not None: plist_tag = p0_tags.get(50457) if plist_tag is not None and isinstance(plist_tag.value, bytes): @@ -170,18 +159,8 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager else: f32 = np.clip(arr.astype(np.float32), 0, 1) - color_space = None - if not linear_raw: - color_space = identify_color_space_from_icc(icc_bytes) - if color_space is None and arr.dtype == np.uint8: - color_space = ColorSpace.SRGB.value - if color_space == ColorSpace.SRGB.value: - f32 = srgb_to_linear(f32) - metadata = { "orientation": read_orientation(file_path), - "color_space": color_space, - "icc_profile": icc_bytes, "ir": ir, **fff_meta, } diff --git a/negpy/infrastructure/loaders/nef_loader.py b/negpy/infrastructure/loaders/nef_loader.py index 799d0947..7d03d3c9 100644 --- a/negpy/infrastructure/loaders/nef_loader.py +++ b/negpy/infrastructure/loaders/nef_loader.py @@ -5,10 +5,9 @@ import tifffile from negpy.domain.interfaces import IImageLoader -from negpy.domain.models import ColorSpace -from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, identify_color_space_from_icc, read_orientation +from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, read_orientation from negpy.infrastructure.loaders.ir_planes import normalize_ir_to_float32 -from negpy.kernel.image.logic import srgb_to_linear, uint8_to_float32, uint16_to_float32 +from negpy.kernel.image.logic import uint8_to_float32, uint16_to_float32 from negpy.kernel.system.logging import get_logger logger = get_logger(__name__) @@ -83,9 +82,7 @@ class NefLoader(IImageLoader): SubIFD chain (tag 0x014A). The data is Nikon Scan's output — curves, gain, and optionally DigitalICE are already applied — not raw sensor data. - Color space handling follows TiffLoader: ICC profile → identify space → - linearise if sRGB. Untagged 16-bit is assumed linear; untagged 8-bit is - assumed sRGB. + Data is returned as-is — no color-space assumptions or linearization. """ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager[Any], dict]: @@ -95,16 +92,6 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager raise ValueError(f"No RGB SubIFD in {file_path}") arr = sub.asarray() - icc_bytes: Optional[bytes] = None - for page in (sub, tif.pages[0]): - tags = getattr(page, "tags", None) - if tags is None: - continue - tag = tags.get("InterColorProfile") - if tag is not None and tag.value: - icc_bytes = bytes(tag.value) - break - # Coolscan NEFs have no separate IR channel (ICE is baked at scan time). # 4-channel branch kept for defensive consistency with TiffLoader. ir: Optional[np.ndarray] = None @@ -121,18 +108,8 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager else: f32 = np.clip(arr.astype(np.float32), 0, 1) - color_space = None - if not linear_raw: - color_space = identify_color_space_from_icc(icc_bytes) - if color_space is None and arr.dtype == np.uint8: - color_space = ColorSpace.SRGB.value - if color_space == ColorSpace.SRGB.value: - f32 = srgb_to_linear(f32) - metadata = { "orientation": read_orientation(file_path), - "color_space": color_space, - "icc_profile": icc_bytes, "ir": ir, } return NonStandardFileWrapper(f32), metadata From 4bfe2f9ff7648f7509921ddbd18e527a12768090 Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 01:52:25 +0200 Subject: [PATCH 14/20] Revert _decode_tiff to independent implementation with IR fixes Keep _decode_tiff decoupled from TiffLoader to avoid inheriting its sRGB linearization assumptions. Ports the three IR sources that the old inline version was missing: ExtraSamples check, extra-page search, and sidecar IR with mask. --- negpy/services/export/linear_output.py | 50 +++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/negpy/services/export/linear_output.py b/negpy/services/export/linear_output.py index baf5ef75..198a367f 100644 --- a/negpy/services/export/linear_output.py +++ b/negpy/services/export/linear_output.py @@ -11,7 +11,7 @@ import io import os from dataclasses import dataclass -from typing import Optional +from typing import Any, Optional import numpy as np import rawpy @@ -407,10 +407,50 @@ def _decode_tiff( gamma_key: str = "linear", expansion: Optional[float] = None, ) -> tuple[np.ndarray, Optional[np.ndarray]]: - """Read a TIFF via the main loader, optionally linearize. Returns (rgb, ir_or_none).""" - from negpy.infrastructure.loaders.tiff_loader import TiffLoader + """Read a TIFF, optionally linearize. Returns (rgb, ir_or_none).""" + from negpy.infrastructure.loaders.ir_planes import find_ir_plane + from negpy.infrastructure.loaders.tiff_loader import _extract_ir_from_extrasamples, _read_sidecar_ir - return _decode_via_loader(TiffLoader(), file_path, geometry, gamma_key, expansion) + with _tifffile.TiffFile(file_path) as tif: + page = tif.pages[0] + arr = page.asarray() + if arr.dtype == np.uint16: + scale = 1.0 / 65535.0 + elif arr.dtype == np.uint8: + scale = 1.0 / 255.0 + elif arr.dtype == np.float32: + scale = 1.0 + else: + scale = 1.0 / float(np.iinfo(arr.dtype).max) if np.issubdtype(arr.dtype, np.integer) else 1.0 + f32 = arr.astype(np.float32) * scale + + ir: Optional[np.ndarray] = None + if f32.ndim == 3 and f32.shape[2] == 4: + f32, ir = _extract_ir_from_extrasamples(file_path, f32) + elif f32.ndim == 2: + f32 = np.stack([f32, f32, f32], axis=2) + + if ir is None: + try: + with _tifffile.TiffFile(file_path) as tif: + ir = find_ir_plane(tif.pages[1:], f32.shape[0], f32.shape[1]) + except Exception: + pass + + if ir is None: + ir_result, _mask = _read_sidecar_ir(file_path) + ir = ir_result + + f32 = np.clip(f32, 0.0, 1.0) + if gamma_key != "linear": + f32 = _linearize(f32, gamma_key) + if expansion is not None and expansion > 1.0: + f32 = np.clip(f32 * expansion, 0.0, 1.0) + orientation = read_orientation(file_path) + f32 = _apply_geometry(f32, orientation, geometry) + if ir is not None: + ir = _apply_geometry(ir, orientation, geometry) + return f32, ir def _decode_pakon(file_path: str, geometry: Optional[GeometryConfig] = None, expansion: Optional[float] = None) -> tuple[np.ndarray, None]: @@ -428,7 +468,7 @@ def _decode_pakon(file_path: str, geometry: Optional[GeometryConfig] = None, exp def _decode_via_loader( - loader: "IImageLoader", + loader: Any, file_path: str, geometry: Optional[GeometryConfig] = None, gamma_key: str = "linear", From 43574b6aef4188d0f678a1982862bbc1d2df14b7 Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 02:28:21 +0200 Subject: [PATCH 15/20] Remove dead IR logic from NEF and FFF loaders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither format supports a separate IR channel — Coolscan bakes ICE into pixel data, Flextight has no IR hardware. Extra channels beyond 3 are now silently dropped instead of being misidentified as IR. --- negpy/infrastructure/loaders/fff_loader.py | 8 +------- negpy/infrastructure/loaders/nef_loader.py | 8 +------- tests/test_linear_output.py | 13 ++++--------- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/negpy/infrastructure/loaders/fff_loader.py b/negpy/infrastructure/loaders/fff_loader.py index f4d28605..f4af01eb 100644 --- a/negpy/infrastructure/loaders/fff_loader.py +++ b/negpy/infrastructure/loaders/fff_loader.py @@ -8,7 +8,6 @@ from negpy.domain.interfaces import IImageLoader from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, read_orientation -from negpy.infrastructure.loaders.ir_planes import normalize_ir_to_float32 from negpy.kernel.image.logic import uint8_to_float32, uint16_to_float32 from negpy.kernel.system.logging import get_logger @@ -143,11 +142,7 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager if fw_tag is not None and isinstance(fw_tag.value, bytes): fff_meta.update(_parse_fff_firmware(fw_tag.value)) - # Imacon/Flextight scanners have no IR hardware — no 4th channel exists. - # 4-channel branch kept for defensive consistency with TiffLoader. - ir: Optional[np.ndarray] = None - if arr.ndim == 3 and arr.shape[2] == 4: - ir = normalize_ir_to_float32(arr[:, :, 3]) + if arr.ndim == 3 and arr.shape[2] > 3: arr = np.ascontiguousarray(arr[:, :, :3]) elif arr.ndim == 2: arr = np.stack([arr] * 3, axis=-1) @@ -161,7 +156,6 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager metadata = { "orientation": read_orientation(file_path), - "ir": ir, **fff_meta, } return NonStandardFileWrapper(f32), metadata diff --git a/negpy/infrastructure/loaders/nef_loader.py b/negpy/infrastructure/loaders/nef_loader.py index 7d03d3c9..389151a7 100644 --- a/negpy/infrastructure/loaders/nef_loader.py +++ b/negpy/infrastructure/loaders/nef_loader.py @@ -6,7 +6,6 @@ from negpy.domain.interfaces import IImageLoader from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, read_orientation -from negpy.infrastructure.loaders.ir_planes import normalize_ir_to_float32 from negpy.kernel.image.logic import uint8_to_float32, uint16_to_float32 from negpy.kernel.system.logging import get_logger @@ -92,11 +91,7 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager raise ValueError(f"No RGB SubIFD in {file_path}") arr = sub.asarray() - # Coolscan NEFs have no separate IR channel (ICE is baked at scan time). - # 4-channel branch kept for defensive consistency with TiffLoader. - ir: Optional[np.ndarray] = None - if arr.ndim == 3 and arr.shape[2] == 4: - ir = normalize_ir_to_float32(arr[:, :, 3]) + if arr.ndim == 3 and arr.shape[2] > 3: arr = np.ascontiguousarray(arr[:, :, :3]) elif arr.ndim == 2: arr = np.stack([arr] * 3, axis=-1) @@ -110,6 +105,5 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager metadata = { "orientation": read_orientation(file_path), - "ir": ir, } return NonStandardFileWrapper(f32), metadata diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index b0d9a30b..9a2ee2ce 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -1338,16 +1338,14 @@ def test_export_roundtrip(self, tmp_path: str) -> None: assert "Coolscan NEF" in desc assert "no scaling" in desc - def test_export_4ch_splits_ir(self, tmp_path: str) -> None: + def test_export_4ch_drops_extra_channel(self, tmp_path: str) -> None: path = _make_coolscan_nef(str(tmp_path), channels=4) out = os.path.join(str(tmp_path), "output.tiff") export_linear_output(path, out) ir_path = os.path.join(str(tmp_path), "output_ir.tiff") - assert os.path.exists(ir_path) + assert not os.path.exists(ir_path) with tifffile.TiffFile(out) as tf: assert tf.pages[0].asarray().shape == (200, 300, 3) - with tifffile.TiffFile(ir_path) as tf: - assert tf.pages[0].asarray().shape == (200, 300) def test_loader_returns_float32(self, tmp_path: str) -> None: from negpy.infrastructure.loaders.nef_loader import NefLoader @@ -1361,9 +1359,8 @@ def test_loader_returns_float32(self, tmp_path: str) -> None: assert w.data.min() >= 0.0 assert w.data.max() <= 1.0 assert "orientation" in metadata - assert "ir" in metadata - def test_loader_extracts_ir(self, tmp_path: str) -> None: + def test_loader_drops_extra_channel(self, tmp_path: str) -> None: from negpy.infrastructure.loaders.nef_loader import NefLoader path = _make_coolscan_nef(str(tmp_path), channels=4) @@ -1371,8 +1368,7 @@ def test_loader_extracts_ir(self, tmp_path: str) -> None: wrapper, metadata = loader.load(path) with wrapper as w: assert w.data.shape == (200, 300, 3) - assert metadata["ir"] is not None - assert metadata["ir"].shape == (200, 300) + assert metadata.get("ir") is None def _make_flextight_fff(tmp_dir: str, h: int = 400, w: int = 600, channels: int = 3) -> str: @@ -1456,7 +1452,6 @@ def test_loader_returns_float32(self, tmp_path: str) -> None: assert w.data.min() >= 0.0 assert w.data.max() <= 1.0 assert "orientation" in metadata - assert "ir" in metadata def _make_noritsu_raw(tmp_dir: str, w: int = 4042, h: int = 6391) -> str: From 007778ce740b1af997e5d9da437ce8df645c484b Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 02:44:12 +0200 Subject: [PATCH 16/20] Document new Linear Output sources: NEF, FFF, Noritsu, TIFF, ICE Add Coolscan NEF, Flextight FFF, Noritsu RAW, and generic TIFF to the supported sources list in both USER_GUIDE.md and PIPELINE.md. Document ICE dust removal toggle and input gamma selector. Replace MakeTiff/ColorPerfect references with neutral phrasing. --- docs/PIPELINE.md | 12 +++++++++--- docs/USER_GUIDE.md | 9 +++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index e748d104..a0efc000 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -157,15 +157,21 @@ Both are fixed (no per-frame metering) so an evenly-exposed roll renders identic When the render intent is **Linear**, the entire darkroom pipeline is bypassed. The source file is decoded to its native linear buffer, lossless geometry (EXIF orientation + user rotation/flip) is applied, and the result is written as an untagged 16-bit TIFF with zlib compression. No normalization, exposure, colour management, flatfield, or sensor correction runs. -* **Pakon RAW**: the uint16 scanner data is scaled by an expansion factor to use more of the 16-bit output range. F135 (14-bit sensor, confirmed) defaults to 4× (`PAKON_EXPANSION`); F335 (16-bit sensor, detected by file size) defaults to 1× (off). The 2k Square and Panoram specs are assumed 14-bit (same default as F135) but this has not been verified with real samples — override manually if needed. MakeTiff uses 2×; 4× places the typical F135 negative peak around 50–55 % of the range. The user can override via the Expansion combo. The applied expansion factor is recorded in the output TIFF's ImageDescription tag. +* **Pakon RAW**: the uint16 scanner data is scaled by an expansion factor to use more of the 16-bit output range. F135 (14-bit sensor, confirmed) defaults to 4× (`PAKON_EXPANSION`); F335 (16-bit sensor, detected by file size) defaults to 1× (off). The 2k Square and Panoram specs are assumed 14-bit (same default as F135) but this has not been verified with real samples — override manually if needed. Some external Pakon tools use 2×; 4× places the typical F135 negative peak around 50–55 % of the range. The user can override via the Expansion combo. The applied expansion factor is recorded in the output TIFF's ImageDescription tag. * **LinearRaw DNG**: 4-channel VueScan (RGB+IR) and 3-channel SilverFast HDRi files are read directly via tifffile, bypassing rawpy. The IR channel, when present, is written as a separate grayscale TIFF with an `_ir` suffix. Expansion defaults to off; 2× and 4× are available. -* **Camera RAW**: demosaiced by rawpy with `user_wb=[1,1,1,1]` (unity), `output_color=raw` (sensor-native), `gamma=(1,1)` (linear), `no_auto_bright=True`. The camera's as-shot white balance multipliers (green-normalized to three RGB values) are embedded in XMP as `RAW-WB: R G B` inside `dc:description`, matching the MakeTiff/ColorPerfect convention. The source filename is stored in `crs:RawFileName`. No expansion option (camera sensors use the full bit depth). +* **Camera RAW**: demosaiced by rawpy with `user_wb=[1,1,1,1]` (unity), `output_color=raw` (sensor-native), `gamma=(1,1)` (linear), `no_auto_bright=True`. The camera's as-shot white balance multipliers (green-normalized to three RGB values) are embedded in XMP as `RAW-WB: R G B` inside `dc:description`, following the `RAW-WB` XMP convention used by external linear-workflow tools. The source filename is stored in `crs:RawFileName`. No expansion option (camera sensors use the full bit depth). +* **Coolscan NEF** (`negpy.infrastructure.loaders.nef_loader`): Nikon Coolscan scanner files are TIFF-structured with the full-res RGB image in a SubIFD chain (tag 0x014A). The loader picks the largest RGB SubIFD by pixel count and rejects files containing CFA/Bayer SubIFDs (camera NEFs). Despite the name, scanner NEFs are not raw sensor data — they are processed images whose content depends on the Nikon Scan settings used at scan time (curves, gain, DigitalICE, etc.). Getting linear, unprocessed output requires the right Nikon Scan settings before scanning. The loader makes no assumptions about linearity or colour space. No separate IR channel exists; any extra channels beyond RGB are dropped. No expansion. +* **Flextight FFF** (`negpy.infrastructure.loaders.fff_loader`): Imacon/Hasselblad Flextight scanner files are big-endian TIFFs with the full-res 16-bit linear RGB image in a top-level IFD (selected by pixel count, not SubfileType — the tag is unreliable). The data is gain-normalized linear transmittance with no gamma applied. Embedded FlexColor metadata is parsed from two proprietary tags: tag 50457 (Apple plist with film stock, film type, gamma, DPI, scan date) and tag 46279 (firmware version, scanner serial). No IR hardware exists on Flextight scanners; extra channels beyond RGB are dropped. No expansion. +* **Noritsu RAW** (`negpy.infrastructure.loaders.noritsu_loader`): headerless BGR 16-bit little-endian scanner dumps. Frame dimensions are auto-detected from file size against a table of known Noritsu scan dimensions (three tiers: exact match, known-width with novel height, and novel-width fallback). The channel order is BGR, swapped to RGB on load. 12-bit sensor data in 16-bit range; default expansion is 16× (`NORITSU_EXPANSION`). +* **TIFF** (standalone `_decode_tiff`, not routed through `TiffLoader` — kept independent to avoid inheriting TiffLoader's sRGB linearization assumptions): the 4th channel is treated as IR only when the ExtraSamples tag is 0 (UNSPECIFIED) or missing; values 1/2 (associated/unassociated alpha) are dropped. Sidecar IR files (`_ir.tif` next to the source, with optional `_ir_valid` mask) and IR stored in secondary TIFF pages (SilverFast iSRD convention) are also detected. An **Input gamma** selector lets the user declare the source encoding (linear, 1.8, 2.2, or sRGB) so the data can be linearized before export. Expansion is available (off by default). * **RGB-scan triplets**: when the current frame is an RGB-scan composite (three narrowband exposures), all three are decoded and merged via `merge_rgb_triplet()` into a single combined TIFF. The red exposure is the primary file; green and blue paths come from the frame's `RgbScanConfig`. No sensor correction is applied (narrowband exposures have no cross-channel leakage). WB and device metadata are taken from the primary (red) exposure. * **Stitch composites**: when the frame is a multi-part stitch, each part is decoded and corrected (flatfield + sensor correction for single-shot parts, flatfield only for triplet parts — triplets have no cross-channel leakage), then assembled via `stitch_composite()` with gain compensation and feather blending. Stitch + triplet combinations are supported: each stitch part can be an RGB-scan triplet, producing one combined TIFF from all source files (e.g. a 4-part stitch of triplets = 12 RAW files → 1 TIFF). +**ICE dust removal** (visible when an IR channel is available): when enabled, IR-based dust and scratch correction is applied to the linear buffer before writing. Off by default — the raw dump philosophy applies. + **Optional corrections** (camera RAW only): three toggles let you bake corrections into the linear output before writing. All default to off (raw dump philosophy — the output is unchanged sensor data). *Apply white balance* multiplies the buffer by the as-shot WB gains (green-normalized). *Apply flatfield* applies the configured flatfield gain correction. *Apply sensor correction* applies the crosstalk unmixing matrix. For stitch composites, flatfield and sensor correction are always applied per-part regardless of these toggles — without them, vignetting and crosstalk differences create visible seams at part boundaries. -Source device metadata (Make, Model, DateTime) is carried through to the output TIFF when available from the source file. +Source device metadata (Make, Model, DateTime) is carried through to the output TIFF when available from the source file. For Flextight FFF files, the Make field includes film stock and type from the embedded plist, and the Model field includes the scanner serial. --- diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 46de88fd..29d9639f 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -545,8 +545,13 @@ A scrollable list of every edit step (last 100 kept), newest on top; the current * **Linear**: bypass the entire darkroom pipeline and dump the scanner's or camera's decoded buffer as an untagged linear 16-bit TIFF. No normalization, exposure, colour management, flatfield, or sensor correction — just the raw data with lossless geometry (rotation/flip) applied. Supported sources: * **Pakon RAW** — 4× expansion by default (14-bit sensor range scaled into 16-bit). F335 files (16-bit sensor) default to no expansion. * **LinearRaw DNG** — SilverFast HDRi (3-channel) and VueScan (4-channel RGB+IR). IR is written as a separate grayscale TIFF with an `_ir` suffix. - * **Camera RAW** — demosaiced with unity white balance (1,1,1,1). The camera's as-shot WB is written into XMP (`RAW-WB: R G B`, MakeTiff-compatible) so it can be applied downstream. Source device and timestamp are preserved. RGB-scan triplets (narrowband R/G/B exposures) are merged into a single combined TIFF. Stitch composites are assembled with flatfield and sensor correction applied per-part for clean seams; stitch + triplet combinations are also supported. - * **Expansion**: scales the linear data before writing. The combo box shows source-appropriate options: Pakon F135/F235 default to 4×, F335 and LinearRaw DNG default to off. Camera RAW files have no expansion option. Leave at the default unless you know why you need to change it. + * **Camera RAW** — demosaiced with unity white balance (1,1,1,1). The camera's as-shot WB is written into XMP (`RAW-WB: R G B`) so it can be applied by downstream tools. Source device and timestamp are preserved. RGB-scan triplets (narrowband R/G/B exposures) are merged into a single combined TIFF. Stitch composites are assembled with flatfield and sensor correction applied per-part for clean seams; stitch + triplet combinations are also supported. + * **Coolscan NEF** — Nikon Coolscan scanner files. Despite the name, these are not raw sensor data — the content depends on the Nikon Scan settings used at scan time. Getting linear, unprocessed output requires the right settings before scanning. The full-res RGB SubIFD is read directly; any extra channels beyond RGB are dropped (Coolscan has no separate IR channel). No expansion. + * **Flextight FFF** — Imacon/Hasselblad Flextight scanner files. The largest 16-bit RGB IFD is selected by pixel count. Data is linear scanner transmittance (see the linearity findings in the project). Embedded FlexColor metadata (film stock, film type, scan date, scanner serial) from the proprietary plist (tag 50457) and firmware blob (tag 46279) is carried through to the output TIFF headers. No expansion. + * **Noritsu RAW** — headerless BGR 16-bit scanner dumps. Frame dimensions are auto-detected from file size against known Noritsu scan dimensions. 16× expansion by default (12-bit sensor data in 16-bit range). + * **TIFF** — generic scanner TIFFs. If the file has a 4th channel tagged as IR (ExtraSamples = UNSPECIFIED or missing), it is written as a separate `_ir` TIFF. Sidecar IR files (`_ir.tif` next to the source) and IR stored in secondary TIFF pages are also detected. **Input gamma** lets you select the gamma encoding of the source (linear, 1.8, 2.2, or sRGB) so the data can be linearized before export. Expansion is available (off by default). + * **Expansion**: scales the linear data before writing. The combo box shows source-appropriate options: Pakon F135/F235 default to 4×, Noritsu defaults to 16×, F335 and LinearRaw DNG default to off. Camera RAW, Coolscan NEF, and Flextight FFF files have no expansion option. Leave at the default unless you know why you need to change it. + * **Apply ICE dust removal** (visible when an IR channel is available): applies IR-based dust and scratch correction to the linear output before writing. Off by default. * **Corrections** (camera RAW only): three optional toggles that bake corrections into the linear output before writing. All default to off (raw dump philosophy). **Apply white balance** multiplies by the as-shot WB gains. **Apply flatfield** applies the flatfield gain correction. **Apply sensor correction** applies the sensor crosstalk unmixing matrix. For stitch composites, flatfield and sensor correction are always applied per-part regardless of these toggles (required for clean seams). ### Export button From 6719040824fde9c6db8ec0b9c0d91ee0e94b07a5 Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 03:06:02 +0200 Subject: [PATCH 17/20] Detect SGI LogLuv FFF files and error clearly LogLuv-encoded FFF files (compression 34676/34677) are not yet supported. Detect them early and raise a clear error instead of silently falling through to the generic TIFF path. --- negpy/infrastructure/loaders/fff_loader.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/negpy/infrastructure/loaders/fff_loader.py b/negpy/infrastructure/loaders/fff_loader.py index f4af01eb..d95c172b 100644 --- a/negpy/infrastructure/loaders/fff_loader.py +++ b/negpy/infrastructure/loaders/fff_loader.py @@ -104,13 +104,31 @@ def _find_full_res_ifd(tif: tifffile.TiffFile) -> Optional[Any]: return best +_SGILOG_COMPRESSIONS = {34676, 34677} + + +def _has_sgilog_ifd(tif: tifffile.TiffFile) -> bool: + for page in tif.pages: + tags = getattr(page, "tags", None) + if tags is None: + continue + comp_tag = tags.get("Compression") + if comp_tag is not None and int(comp_tag.value) in _SGILOG_COMPRESSIONS: + return True + return False + + def is_flextight_fff(file_path: str) -> bool: """True if this FFF is an Imacon/Hasselblad Flextight scanner file (16-bit RGB in a top-level IFD).""" if os.path.splitext(file_path)[1].lower() != ".fff": return False try: with tifffile.TiffFile(file_path) as tif: - return _find_full_res_ifd(tif) is not None + if _find_full_res_ifd(tif) is not None: + return True + if _has_sgilog_ifd(tif): + logger.warning(f"SGI LogLuv FFF detected but not supported: {file_path}") + return False except Exception: return False @@ -129,6 +147,8 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager with tifffile.TiffFile(file_path) as tif: page = _find_full_res_ifd(tif) if page is None: + if _has_sgilog_ifd(tif): + raise ValueError(f"SGI LogLuv encoded FFF files are not supported: {file_path}") raise ValueError(f"No full-res RGB IFD in {file_path}") arr = page.asarray() From 42c8182743ad354ba91da33c4cf3d527640d77d6 Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 03:35:34 +0200 Subject: [PATCH 18/20] Add SGI LogLuv decoder for Flextight FFF raw files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode LogLuv32 (RLE compressed) and LogLuv24 FFF files instead of rejecting them. The LogLuv → XYZ → linear sRGB pipeline is ported from flexcolor-tool (MIT, attributed in source). --- docs/PIPELINE.md | 2 +- docs/USER_GUIDE.md | 2 +- negpy/infrastructure/loaders/fff_loader.py | 81 +++-- negpy/infrastructure/loaders/logluv.py | 396 +++++++++++++++++++++ tests/test_linear_output.py | 124 +++++++ tests/test_logluv.py | 97 +++++ 6 files changed, 672 insertions(+), 30 deletions(-) create mode 100644 negpy/infrastructure/loaders/logluv.py create mode 100644 tests/test_logluv.py diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index a0efc000..ea25e6d6 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -161,7 +161,7 @@ When the render intent is **Linear**, the entire darkroom pipeline is bypassed. * **LinearRaw DNG**: 4-channel VueScan (RGB+IR) and 3-channel SilverFast HDRi files are read directly via tifffile, bypassing rawpy. The IR channel, when present, is written as a separate grayscale TIFF with an `_ir` suffix. Expansion defaults to off; 2× and 4× are available. * **Camera RAW**: demosaiced by rawpy with `user_wb=[1,1,1,1]` (unity), `output_color=raw` (sensor-native), `gamma=(1,1)` (linear), `no_auto_bright=True`. The camera's as-shot white balance multipliers (green-normalized to three RGB values) are embedded in XMP as `RAW-WB: R G B` inside `dc:description`, following the `RAW-WB` XMP convention used by external linear-workflow tools. The source filename is stored in `crs:RawFileName`. No expansion option (camera sensors use the full bit depth). * **Coolscan NEF** (`negpy.infrastructure.loaders.nef_loader`): Nikon Coolscan scanner files are TIFF-structured with the full-res RGB image in a SubIFD chain (tag 0x014A). The loader picks the largest RGB SubIFD by pixel count and rejects files containing CFA/Bayer SubIFDs (camera NEFs). Despite the name, scanner NEFs are not raw sensor data — they are processed images whose content depends on the Nikon Scan settings used at scan time (curves, gain, DigitalICE, etc.). Getting linear, unprocessed output requires the right Nikon Scan settings before scanning. The loader makes no assumptions about linearity or colour space. No separate IR channel exists; any extra channels beyond RGB are dropped. No expansion. -* **Flextight FFF** (`negpy.infrastructure.loaders.fff_loader`): Imacon/Hasselblad Flextight scanner files are big-endian TIFFs with the full-res 16-bit linear RGB image in a top-level IFD (selected by pixel count, not SubfileType — the tag is unreliable). The data is gain-normalized linear transmittance with no gamma applied. Embedded FlexColor metadata is parsed from two proprietary tags: tag 50457 (Apple plist with film stock, film type, gamma, DPI, scan date) and tag 46279 (firmware version, scanner serial). No IR hardware exists on Flextight scanners; extra channels beyond RGB are dropped. No expansion. +* **Flextight FFF** (`negpy.infrastructure.loaders.fff_loader`): Imacon/Hasselblad Flextight scanner files in two variants: uncompressed 16-bit RGB (standard FlexColor export), and SGI LogLuv compressed (raw `.3fr`/`.fff` from the scanner hardware). Uncompressed files are big-endian TIFFs with the full-res image in a top-level IFD (selected by pixel count, not SubfileType — the tag is unreliable). LogLuv files (compression tags 34676/34677) are decoded via `negpy.infrastructure.loaders.logluv` — a LogLuv32/24 → CIE XYZ → linear sRGB pipeline ported from [flexcolor-tool](https://github.com/rohanpandula/flexcolor-tool) (MIT). The data is gain-normalized linear transmittance with no gamma applied. Embedded FlexColor metadata is parsed from two proprietary tags: tag 50457 (Apple plist with film stock, film type, gamma, DPI, scan date) and tag 46279 (firmware version, scanner serial). No IR hardware exists on Flextight scanners; extra channels beyond RGB are dropped. No expansion. * **Noritsu RAW** (`negpy.infrastructure.loaders.noritsu_loader`): headerless BGR 16-bit little-endian scanner dumps. Frame dimensions are auto-detected from file size against a table of known Noritsu scan dimensions (three tiers: exact match, known-width with novel height, and novel-width fallback). The channel order is BGR, swapped to RGB on load. 12-bit sensor data in 16-bit range; default expansion is 16× (`NORITSU_EXPANSION`). * **TIFF** (standalone `_decode_tiff`, not routed through `TiffLoader` — kept independent to avoid inheriting TiffLoader's sRGB linearization assumptions): the 4th channel is treated as IR only when the ExtraSamples tag is 0 (UNSPECIFIED) or missing; values 1/2 (associated/unassociated alpha) are dropped. Sidecar IR files (`_ir.tif` next to the source, with optional `_ir_valid` mask) and IR stored in secondary TIFF pages (SilverFast iSRD convention) are also detected. An **Input gamma** selector lets the user declare the source encoding (linear, 1.8, 2.2, or sRGB) so the data can be linearized before export. Expansion is available (off by default). * **RGB-scan triplets**: when the current frame is an RGB-scan composite (three narrowband exposures), all three are decoded and merged via `merge_rgb_triplet()` into a single combined TIFF. The red exposure is the primary file; green and blue paths come from the frame's `RgbScanConfig`. No sensor correction is applied (narrowband exposures have no cross-channel leakage). WB and device metadata are taken from the primary (red) exposure. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 29d9639f..5ffcd2e1 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -547,7 +547,7 @@ A scrollable list of every edit step (last 100 kept), newest on top; the current * **LinearRaw DNG** — SilverFast HDRi (3-channel) and VueScan (4-channel RGB+IR). IR is written as a separate grayscale TIFF with an `_ir` suffix. * **Camera RAW** — demosaiced with unity white balance (1,1,1,1). The camera's as-shot WB is written into XMP (`RAW-WB: R G B`) so it can be applied by downstream tools. Source device and timestamp are preserved. RGB-scan triplets (narrowband R/G/B exposures) are merged into a single combined TIFF. Stitch composites are assembled with flatfield and sensor correction applied per-part for clean seams; stitch + triplet combinations are also supported. * **Coolscan NEF** — Nikon Coolscan scanner files. Despite the name, these are not raw sensor data — the content depends on the Nikon Scan settings used at scan time. Getting linear, unprocessed output requires the right settings before scanning. The full-res RGB SubIFD is read directly; any extra channels beyond RGB are dropped (Coolscan has no separate IR channel). No expansion. - * **Flextight FFF** — Imacon/Hasselblad Flextight scanner files. The largest 16-bit RGB IFD is selected by pixel count. Data is linear scanner transmittance (see the linearity findings in the project). Embedded FlexColor metadata (film stock, film type, scan date, scanner serial) from the proprietary plist (tag 50457) and firmware blob (tag 46279) is carried through to the output TIFF headers. No expansion. + * **Flextight FFF** — Imacon/Hasselblad Flextight scanner files, including both standard uncompressed 16-bit RGB exports and SGI LogLuv compressed raw files (`.3fr`/`.fff`). LogLuv files are decoded through a LogLuv → XYZ → linear sRGB pipeline. The largest image IFD is selected by pixel count. Data is linear scanner transmittance. Embedded FlexColor metadata (film stock, film type, scan date, scanner serial) from the proprietary plist (tag 50457) and firmware blob (tag 46279) is carried through to the output TIFF headers. No expansion. * **Noritsu RAW** — headerless BGR 16-bit scanner dumps. Frame dimensions are auto-detected from file size against known Noritsu scan dimensions. 16× expansion by default (12-bit sensor data in 16-bit range). * **TIFF** — generic scanner TIFFs. If the file has a 4th channel tagged as IR (ExtraSamples = UNSPECIFIED or missing), it is written as a separate `_ir` TIFF. Sidecar IR files (`_ir.tif` next to the source) and IR stored in secondary TIFF pages are also detected. **Input gamma** lets you select the gamma encoding of the source (linear, 1.8, 2.2, or sRGB) so the data can be linearized before export. Expansion is available (off by default). * **Expansion**: scales the linear data before writing. The combo box shows source-appropriate options: Pakon F135/F235 default to 4×, Noritsu defaults to 16×, F335 and LinearRaw DNG default to off. Camera RAW, Coolscan NEF, and Flextight FFF files have no expansion option. Leave at the default unless you know why you need to change it. diff --git a/negpy/infrastructure/loaders/fff_loader.py b/negpy/infrastructure/loaders/fff_loader.py index d95c172b..69ac7acd 100644 --- a/negpy/infrastructure/loaders/fff_loader.py +++ b/negpy/infrastructure/loaders/fff_loader.py @@ -8,6 +8,9 @@ from negpy.domain.interfaces import IImageLoader from negpy.infrastructure.loaders.helpers import NonStandardFileWrapper, read_orientation +from negpy.infrastructure.loaders.logluv import ( + decode_logluv_strips, +) from negpy.kernel.image.logic import uint8_to_float32, uint16_to_float32 from negpy.kernel.system.logging import get_logger @@ -71,13 +74,17 @@ def _parse_fff_firmware(raw: bytes) -> dict: return {} +_PHOTOMETRIC_RGB = 2 +_PHOTOMETRIC_LOGLUV = 32845 + + def _find_full_res_ifd(tif: tifffile.TiffFile) -> Optional[Any]: - """Return the largest RGB IFD by pixel count. + """Return the largest image IFD by pixel count. FFF files can have multiple IFDs flagged as full-resolution (the SubfileType tag is unreliable — e.g. a small secondary image tagged full-res). Pixel count is the reliable signal, matching the approach in flexcolor-tool and - the reference loader. + the reference loader. Accepts both RGB and LogLuv photometric. """ best = None best_pixels = 0 @@ -92,7 +99,18 @@ def _find_full_res_ifd(tif: tifffile.TiffFile) -> Optional[Any]: continue spp = int(spp_tag.value) if not hasattr(spp_tag.value, "__len__") else int(spp_tag.value[0]) photo = int(photo_tag.value) - if spp < 3 or photo != 2: + if photo == _PHOTOMETRIC_LOGLUV: + pixels = page.shape[0] * page.shape[1] if hasattr(page, "shape") else 0 + if pixels == 0: + w_tag = tags.get("ImageWidth") + h_tag = tags.get("ImageLength") + if w_tag and h_tag: + pixels = int(w_tag.value) * int(h_tag.value) + if pixels > best_pixels: + best = page + best_pixels = pixels + continue + if spp < 3 or photo != _PHOTOMETRIC_RGB: continue bits = int(bps_tag.value) if bps_tag and not hasattr(bps_tag.value, "__len__") else (int(bps_tag.value[0]) if bps_tag else 8) if bits < 16: @@ -119,16 +137,12 @@ def _has_sgilog_ifd(tif: tifffile.TiffFile) -> bool: def is_flextight_fff(file_path: str) -> bool: - """True if this FFF is an Imacon/Hasselblad Flextight scanner file (16-bit RGB in a top-level IFD).""" + """True if this FFF is an Imacon/Hasselblad Flextight scanner file.""" if os.path.splitext(file_path)[1].lower() != ".fff": return False try: with tifffile.TiffFile(file_path) as tif: - if _find_full_res_ifd(tif) is not None: - return True - if _has_sgilog_ifd(tif): - logger.warning(f"SGI LogLuv FFF detected but not supported: {file_path}") - return False + return _find_full_res_ifd(tif) is not None except Exception: return False @@ -136,9 +150,9 @@ def is_flextight_fff(file_path: str) -> bool: class FffLoader(IImageLoader): """Loader for Imacon/Hasselblad Flextight FFF scanner files. - These are big-endian TIFFs with the full-res 16-bit linear RGB image in a - top-level IFD (picked by pixel count, not SubfileType tag). The data is - uninverted scanner output — linear, no gamma applied. + Handles two variants: + - Uncompressed 16-bit RGB (standard FFF from FlexColor export) + - SGI LogLuv compressed (raw .3fr/.fff from the scanner hardware) Data is returned as-is — no color-space assumptions or linearization. """ @@ -147,10 +161,33 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager with tifffile.TiffFile(file_path) as tif: page = _find_full_res_ifd(tif) if page is None: - if _has_sgilog_ifd(tif): - raise ValueError(f"SGI LogLuv encoded FFF files are not supported: {file_path}") - raise ValueError(f"No full-res RGB IFD in {file_path}") - arr = page.asarray() + raise ValueError(f"No full-res image IFD in {file_path}") + + page_tags = getattr(page, "tags", None) + comp_tag = page_tags.get("Compression") if page_tags else None + is_logluv = comp_tag is not None and int(comp_tag.value) in _SGILOG_COMPRESSIONS + + if is_logluv: + w_tag = page_tags.get("ImageWidth") + h_tag = page_tags.get("ImageLength") + w = int(w_tag.value) if w_tag else page.shape[1] + h = int(h_tag.value) if h_tag else page.shape[0] + with open(file_path, "rb") as fh: + raw_data = fh.read() + byte_order = "<" if tif.byteorder == "<" else ">" + f32 = decode_logluv_strips([page], w, h, raw_data, byte_order) + else: + arr = page.asarray() + if arr.ndim == 3 and arr.shape[2] > 3: + arr = np.ascontiguousarray(arr[:, :, :3]) + elif arr.ndim == 2: + arr = np.stack([arr] * 3, axis=-1) + if arr.dtype == np.uint8: + f32 = uint8_to_float32(np.ascontiguousarray(arr)) + elif arr.dtype == np.uint16: + f32 = uint16_to_float32(np.ascontiguousarray(arr)) + else: + f32 = np.clip(arr.astype(np.float32), 0, 1) fff_meta: dict = {} p0_tags = getattr(tif.pages[0], "tags", None) @@ -162,18 +199,6 @@ def load(self, file_path: str, linear_raw: bool = False) -> Tuple[ContextManager if fw_tag is not None and isinstance(fw_tag.value, bytes): fff_meta.update(_parse_fff_firmware(fw_tag.value)) - if arr.ndim == 3 and arr.shape[2] > 3: - arr = np.ascontiguousarray(arr[:, :, :3]) - elif arr.ndim == 2: - arr = np.stack([arr] * 3, axis=-1) - - if arr.dtype == np.uint8: - f32 = uint8_to_float32(np.ascontiguousarray(arr)) - elif arr.dtype == np.uint16: - f32 = uint16_to_float32(np.ascontiguousarray(arr)) - else: - f32 = np.clip(arr.astype(np.float32), 0, 1) - metadata = { "orientation": read_orientation(file_path), **fff_meta, diff --git a/negpy/infrastructure/loaders/logluv.py b/negpy/infrastructure/loaders/logluv.py new file mode 100644 index 00000000..b33eb67d --- /dev/null +++ b/negpy/infrastructure/loaders/logluv.py @@ -0,0 +1,396 @@ +# LogLuv32/24 decoder — SGI LogLuv (Greg Ward) to CIE XYZ to linear sRGB. +# +# Ported from flexcolor-tool (MIT License): +# Copyright (c) 2026 flexcolor-tool contributors +# https://github.com/rohanpandula/flexcolor-tool +# +# The decode math mirrors libtiff's tif_luv.c reference implementation. +# Constants (UVSCALE, U_NEU, V_NEU, uv_row table) are from the published +# LogLuv spec and libtiff's uvcode.h (Greg Ward, v1.0). + +from typing import Tuple + +import numpy as np + +_M_LN2 = 0.69314718055994530942 +_UVSCALE = 410.0 +_U_NEU = 0.210526316 +_V_NEU = 0.473684211 + +COMPRESSION_SGILOG = 34676 +COMPRESSION_SGILOG24 = 34677 + + +def _logl16_to_y(ptop: np.ndarray) -> np.ndarray: + """LogL16 top-16 bits → luminance Y. Matches libtiff LogL16toY.""" + top = ptop.astype(np.int32) + le = top & 0x7FFF + sign = (top & 0x8000) != 0 + y = np.exp(_M_LN2 / 256.0 * (le.astype(np.float64) + 0.5) - _M_LN2 * 64.0) + y = np.where(sign, -y, y) + y = np.where(le != 0, y, 0.0) + return y + + +def logluv32_to_xyz(packed: np.ndarray) -> np.ndarray: + """LogLuv32 uint32 → XYZ float64 (..., 3).""" + p = np.asarray(packed, dtype=np.uint32) + out = np.empty(p.shape + (3,), dtype=np.float64) + top = p.view(np.int32) >> 16 + L = _logl16_to_y(top) + u = ((p >> 8) & 0xFF).astype(np.float64) + 0.5 + v = (p & 0xFF).astype(np.float64) + 0.5 + u = u / _UVSCALE + v = v / _UVSCALE + s = 1.0 / (6.0 * u - 16.0 * v + 12.0) + x = 9.0 * u * s + yv = 4.0 * v * s + out[..., 0] = x / yv * L + out[..., 1] = L + out[..., 2] = (1.0 - x - yv) / yv * L + ok = L > 0.0 + out[~ok] = 0.0 + return out + + +def decode_strip_logluv32(compressed: bytes, npixels: int) -> np.ndarray: + """Decode a LogLuv32 RLE strip → npixels uint32 values. + + Four byte-planes (shifts 24,16,8,0) are run-length coded into one + stream; runs ≥128 are count+value, else literals. + """ + buf = np.frombuffer(compressed, dtype=np.uint8) + n = buf.shape[0] + pixels = np.zeros(npixels, dtype=np.uint32) + pos = 0 + for shft in (24, 16, 8, 0): + i = 0 + while i < npixels and pos < n: + cnt = buf[pos] + if cnt >= 128: + if pos + 1 >= n: + break + rc = int(cnt) - 126 + val = np.uint32(buf[pos + 1]) << shft + pos += 2 + seg = min(rc, npixels - i) + pixels[i : i + seg] |= val + i += seg + else: + pos += 1 + seg = min(int(cnt), npixels - i, n - pos) + pixels[i : i + seg] |= buf[pos : pos + seg].astype(np.uint32) << shft + pos += seg + i += seg + return pixels + + +# LogLuv24: 10-bit log luminance + 14-bit uv index (libtiff uvcode.h table) +_UV_SQSIZ = 0.003500 +_UV_VSTART = 0.016940 +_UV_NVS = 163 +_UV_NDIVS = 16289 + +_uv_row = [ + (0.247663, 4, 0), + (0.243779, 6, 4), + (0.241684, 7, 10), + (0.237874, 9, 17), + (0.235906, 10, 26), + (0.232153, 12, 36), + (0.228352, 14, 48), + (0.226259, 15, 62), + (0.222371, 17, 77), + (0.220410, 18, 94), + (0.214710, 21, 112), + (0.212714, 22, 133), + (0.210721, 23, 155), + (0.204976, 26, 178), + (0.202986, 27, 204), + (0.199245, 29, 231), + (0.195525, 31, 260), + (0.193560, 32, 291), + (0.189878, 34, 323), + (0.186216, 36, 357), + (0.186216, 36, 393), + (0.182592, 38, 429), + (0.179003, 40, 467), + (0.175466, 42, 507), + (0.172001, 44, 549), + (0.172001, 44, 593), + (0.168612, 46, 637), + (0.168612, 46, 683), + (0.163575, 49, 729), + (0.158642, 52, 778), + (0.158642, 52, 830), + (0.158642, 52, 882), + (0.153815, 55, 934), + (0.153815, 55, 989), + (0.149097, 58, 1044), + (0.149097, 58, 1102), + (0.142746, 62, 1160), + (0.142746, 62, 1222), + (0.142746, 62, 1284), + (0.138270, 65, 1346), + (0.138270, 65, 1411), + (0.138270, 65, 1476), + (0.132166, 69, 1541), + (0.132166, 69, 1610), + (0.126204, 73, 1679), + (0.126204, 73, 1752), + (0.126204, 73, 1825), + (0.120381, 77, 1898), + (0.120381, 77, 1975), + (0.120381, 77, 2052), + (0.120381, 77, 2129), + (0.112962, 82, 2206), + (0.112962, 82, 2288), + (0.112962, 82, 2370), + (0.107450, 86, 2452), + (0.107450, 86, 2538), + (0.107450, 86, 2624), + (0.107450, 86, 2710), + (0.100343, 91, 2796), + (0.100343, 91, 2887), + (0.100343, 91, 2978), + (0.095126, 95, 3069), + (0.095126, 95, 3164), + (0.095126, 95, 3259), + (0.095126, 95, 3354), + (0.088276, 100, 3449), + (0.088276, 100, 3549), + (0.088276, 100, 3649), + (0.088276, 100, 3749), + (0.081523, 105, 3849), + (0.081523, 105, 3954), + (0.081523, 105, 4059), + (0.081523, 105, 4164), + (0.074861, 110, 4269), + (0.074861, 110, 4379), + (0.074861, 110, 4489), + (0.074861, 110, 4599), + (0.068290, 115, 4709), + (0.068290, 115, 4824), + (0.068290, 115, 4939), + (0.068290, 115, 5054), + (0.063573, 119, 5169), + (0.063573, 119, 5288), + (0.063573, 119, 5407), + (0.063573, 119, 5526), + (0.057219, 124, 5645), + (0.057219, 124, 5769), + (0.057219, 124, 5893), + (0.057219, 124, 6017), + (0.050985, 129, 6141), + (0.050985, 129, 6270), + (0.050985, 129, 6399), + (0.050985, 129, 6528), + (0.050985, 129, 6657), + (0.044859, 134, 6786), + (0.044859, 134, 6920), + (0.044859, 134, 7054), + (0.044859, 134, 7188), + (0.040571, 138, 7322), + (0.040571, 138, 7460), + (0.040571, 138, 7598), + (0.040571, 138, 7736), + (0.036339, 142, 7874), + (0.036339, 142, 8016), + (0.036339, 142, 8158), + (0.036339, 142, 8300), + (0.032139, 146, 8442), + (0.032139, 146, 8588), + (0.032139, 146, 8734), + (0.032139, 146, 8880), + (0.027947, 150, 9026), + (0.027947, 150, 9176), + (0.027947, 150, 9326), + (0.023739, 154, 9476), + (0.023739, 154, 9630), + (0.023739, 154, 9784), + (0.023739, 154, 9938), + (0.019504, 158, 10092), + (0.019504, 158, 10250), + (0.019504, 158, 10408), + (0.016976, 161, 10566), + (0.016976, 161, 10727), + (0.016976, 161, 10888), + (0.016976, 161, 11049), + (0.012639, 165, 11210), + (0.012639, 165, 11375), + (0.012639, 165, 11540), + (0.009991, 168, 11705), + (0.009991, 168, 11873), + (0.009991, 168, 12041), + (0.009016, 170, 12209), + (0.009016, 170, 12379), + (0.009016, 170, 12549), + (0.006217, 173, 12719), + (0.006217, 173, 12892), + (0.005097, 175, 13065), + (0.005097, 175, 13240), + (0.005097, 175, 13415), + (0.003909, 177, 13590), + (0.003909, 177, 13767), + (0.002340, 177, 13944), + (0.002389, 170, 14121), + (0.001068, 164, 14291), + (0.001653, 157, 14455), + (0.000717, 150, 14612), + (0.001614, 143, 14762), + (0.000270, 136, 14905), + (0.000484, 129, 15041), + (0.001103, 123, 15170), + (0.001242, 115, 15293), + (0.001188, 109, 15408), + (0.001011, 103, 15517), + (0.000709, 97, 15620), + (0.000301, 89, 15717), + (0.002416, 82, 15806), + (0.003251, 76, 15888), + (0.003246, 69, 15964), + (0.004141, 62, 16033), + (0.005963, 55, 16095), + (0.008839, 47, 16150), + (0.010490, 40, 16197), + (0.016994, 31, 16237), + (0.023659, 21, 16268), +] +_u_start = np.array([r[0] for r in _uv_row], dtype=np.float64) +_u_nus = np.array([r[1] for r in _uv_row], dtype=np.int64) +_u_ncum = np.array([r[2] for r in _uv_row], dtype=np.int64) + + +def _logl10_to_y(p10: np.ndarray) -> np.ndarray: + """LogL10 10-bit → luminance Y. Matches libtiff LogL10toY.""" + p10 = np.asarray(p10, dtype=np.int64) + y = np.exp(_M_LN2 / 64.0 * (p10 + 0.5) - _M_LN2 * 12.0) + return np.where(p10 != 0, y, 0.0) + + +def _uv_decode_index(ce: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """UV index (0..16288) → (u', v'). Vectorized libtiff uv_decode.""" + ce = np.asarray(ce, dtype=np.int64) + vi = np.searchsorted(_u_ncum, ce, side="right") - 1 + vi = np.clip(vi, 0, _UV_NVS - 1) + ui = ce - _u_ncum[vi] + u = _u_start[vi] + (ui + 0.5) * _UV_SQSIZ + v = _UV_VSTART + (vi + 0.5) * _UV_SQSIZ + bad = (ce < 0) | (ce >= _UV_NDIVS) + u = np.where(bad, _U_NEU, u) + v = np.where(bad, _V_NEU, v) + return u, v + + +def logluv24_to_xyz(packed: np.ndarray) -> np.ndarray: + """LogLuv24 uint32 (24-bit in low bits) → XYZ float64 (..., 3).""" + p = np.asarray(packed, dtype=np.uint32) + le10 = (p >> 14) & 0x3FF + ce = p & 0x3FFF + L = _logl10_to_y(le10) + u, v = _uv_decode_index(ce) + s = 1.0 / (6.0 * u - 16.0 * v + 12.0) + x = 9.0 * u * s + yv = 4.0 * v * s + out = np.empty(p.shape + (3,), dtype=np.float64) + out[..., 0] = x / yv * L + out[..., 1] = L + out[..., 2] = (1.0 - x - yv) / yv * L + ok = L > 0.0 + out[~ok] = 0.0 + return out + + +# XYZ → linear sRGB (Rec.709/sRGB primaries, D65) — libtiff XYZtoRGB24 matrix +_XYZ_TO_LRGB = np.array( + [[2.690, -1.276, -0.414], [-1.022, 1.978, 0.044], [0.061, -0.224, 1.163]], + dtype=np.float64, +) + + +def xyz_to_linear_rgb(xyz: np.ndarray) -> np.ndarray: + """CIE XYZ (..., 3) → linear sRGB float64 (..., 3), unbounded.""" + xyz = np.asarray(xyz, dtype=np.float64) + v = np.moveaxis(xyz, -1, 0).reshape(3, -1) + rgb = _XYZ_TO_LRGB @ v + return np.moveaxis(rgb.reshape(3, *xyz.shape[:-1]), 0, -1) + + +def decode_logluv_strips( + pages: list, + width: int, + height: int, + data: bytes, + byte_order: str, +) -> np.ndarray: + """Decode LogLuv32/24 strips from raw TIFF pages → float32 RGB (H, W, 3). + + Reads strip offsets/byte counts from the TIFF page tags, decompresses, + converts LogLuv → XYZ → linear sRGB, and normalizes to [0, 1]. + + Parameters + ---------- + pages : list of tifffile pages with LogLuv compression + width, height : image dimensions + data : raw file bytes + byte_order : '<' for little-endian, '>' for big-endian + + Returns float32 array in [0, 1], clipped. + """ + page = pages[0] + tags = page.tags + comp_val = int(tags["Compression"].value) + + if comp_val == COMPRESSION_SGILOG: + px_to_xyz = logluv32_to_xyz + bpp = 4 + use_rle = True + elif comp_val == COMPRESSION_SGILOG24: + px_to_xyz = logluv24_to_xyz + bpp = 3 + use_rle = False + else: + raise ValueError(f"Not a LogLuv compression: {comp_val}") + + rps_tag = tags.get("RowsPerStrip") + rps = int(rps_tag.value) if rps_tag else height + + offsets = _tag_to_list(tags.get("StripOffsets")) + counts = _tag_to_list(tags.get("StripByteCounts")) + if not offsets: + raise ValueError("No StripOffsets in LogLuv IFD") + + npixels_total = width * height + packed = np.empty(npixels_total, dtype=np.uint32) + filled = 0 + n_strips = len(offsets) + + for si, (off, cnt) in enumerate(zip(offsets, counts)): + rows = rps if si < n_strips - 1 else (height - (n_strips - 1) * rps) + npix = rows * width + chunk = data[off : off + cnt] + if len(chunk) < cnt: + raise ValueError(f"Strip {si} underrun (need {cnt}, got {len(chunk)})") + + if use_rle: + packed[filled : filled + npix] = decode_strip_logluv32(chunk, npix) + else: + b = np.frombuffer(chunk, dtype=np.uint8, count=npix * bpp) + p = b.reshape(npix, 3).astype(np.uint32) + packed[filled : filled + npix] = (p[:, 0] << 16) | (p[:, 1] << 8) | p[:, 2] + filled += npix + + packed = packed[:npixels_total].reshape(height, width) + xyz = px_to_xyz(packed) + lin = xyz_to_linear_rgb(xyz) + f32 = np.clip(lin, 0.0, 1.0).astype(np.float32) + return f32 + + +def _tag_to_list(tag) -> list: + if tag is None: + return [] + v = tag.value + if hasattr(v, "__len__") and not isinstance(v, (str, bytes)): + return [int(x) for x in v] + return [int(v)] diff --git a/tests/test_linear_output.py b/tests/test_linear_output.py index 9a2ee2ce..af41bf7c 100644 --- a/tests/test_linear_output.py +++ b/tests/test_linear_output.py @@ -1454,6 +1454,130 @@ def test_loader_returns_float32(self, tmp_path: str) -> None: assert "orientation" in metadata +def _make_logluv_fff(tmp_dir: str, h: int = 4, w: int = 6) -> str: + """Create a synthetic SGI LogLuv32 FFF file (minimal hand-built TIFF).""" + import struct + + from negpy.infrastructure.loaders.logluv import ( + COMPRESSION_SGILOG, + ) + + UVSCALE = 410.0 + U_NEU = 0.210526316 + V_NEU = 0.473684211 + PHOTOMETRIC_LOGLUV = 32845 + + rng = np.random.RandomState(123) + luminance = rng.uniform(0.01, 1.0, (h, w)) + + def logl16_from_y(y): + le = np.floor(256.0 * (np.log2(np.abs(y)) + 64.0)).astype(np.int64) + le = np.clip(le, 0, 0x7FFF) + le = np.where(y <= 0, 0, le) + return le.astype(np.uint32) + + le = logl16_from_y(luminance) + ue = np.clip(np.trunc(UVSCALE * U_NEU), 0, 255).astype(np.uint32) + ve = np.clip(np.trunc(UVSCALE * V_NEU), 0, 255).astype(np.uint32) + packed = (le << 16) | (ue << 8) | ve + + pixel_bytes = packed.astype(">u4").tobytes() + + byte_order = b"MM" + ifd_offset = 8 + n_tags = 8 + ifd_size = 2 + n_tags * 12 + 4 + strip_offset = ifd_offset + ifd_size + + ifd = struct.pack(">H", n_tags) + + def tag(t, typ, cnt, val): + if typ == 3: + return struct.pack(">HHIH2x", t, typ, cnt, val) + return struct.pack(">HHII", t, typ, cnt, val) + + ifd += tag(256, 4, 1, w) # ImageWidth + ifd += tag(257, 4, 1, h) # ImageLength + ifd += tag(258, 3, 1, 32) # BitsPerSample + ifd += tag(259, 3, 1, COMPRESSION_SGILOG) # Compression + ifd += tag(262, 3, 1, PHOTOMETRIC_LOGLUV) # PhotometricInterpretation + ifd += tag(273, 4, 1, strip_offset) # StripOffsets + ifd += tag(277, 3, 1, 3) # SamplesPerPixel + ifd += tag(278, 4, 1, h) # RowsPerStrip + ifd += struct.pack(">I", 0) # next IFD + + header = byte_order + struct.pack(">HI", 42, ifd_offset) + + # StripByteCounts: we omit it — the decoder should handle this + # Actually we need it. Rebuild with 9 tags. + n_tags = 9 + ifd_size = 2 + n_tags * 12 + 4 + strip_offset = ifd_offset + ifd_size + + ifd = struct.pack(">H", n_tags) + ifd += tag(256, 4, 1, w) + ifd += tag(257, 4, 1, h) + ifd += tag(258, 3, 1, 32) + ifd += tag(259, 3, 1, COMPRESSION_SGILOG) + ifd += tag(262, 3, 1, PHOTOMETRIC_LOGLUV) + ifd += tag(273, 4, 1, strip_offset) + ifd += tag(277, 3, 1, 3) + ifd += tag(278, 4, 1, h) + ifd += tag(279, 4, 1, len(pixel_bytes)) # StripByteCounts + ifd += struct.pack(">I", 0) + + header = byte_order + struct.pack(">HI", 42, ifd_offset) + data = header + ifd + pixel_bytes + + path = os.path.join(tmp_dir, "logluv.fff") + with open(path, "wb") as f: + f.write(data) + return path + + +class TestFlextightLogLuv: + def test_detect_logluv_fff(self, tmp_path: str) -> None: + path = _make_logluv_fff(str(tmp_path)) + assert is_flextight_fff(path) + + def test_loader_decodes_logluv(self, tmp_path: str) -> None: + from negpy.infrastructure.loaders.fff_loader import FffLoader + + path = _make_logluv_fff(str(tmp_path), h=4, w=6) + loader = FffLoader() + wrapper, metadata = loader.load(path) + with wrapper as w: + assert w.data.dtype == np.float32 + assert w.data.shape == (4, 6, 3) + assert w.data.min() >= 0.0 + assert w.data.max() <= 1.0 + assert "orientation" in metadata + + def test_logluv_export_roundtrip(self, tmp_path: str) -> None: + path = _make_logluv_fff(str(tmp_path), h=10, w=15) + out = os.path.join(str(tmp_path), "output.tiff") + export_linear_output(path, out) + with tifffile.TiffFile(out) as tf: + arr = tf.pages[0].asarray() + assert arr.dtype == np.uint16 + assert arr.shape == (10, 15, 3) + desc = tf.pages[0].description + assert "Flextight FFF" in desc + + def test_logluv_produces_nonzero_rgb(self, tmp_path: str) -> None: + from negpy.infrastructure.loaders.fff_loader import FffLoader + + path = _make_logluv_fff(str(tmp_path), h=4, w=6) + loader = FffLoader() + wrapper, _meta = loader.load(path) + with wrapper as w: + assert w.data.mean() > 0.01, "LogLuv decode produced near-zero output" + + def test_logluv_source_type(self, tmp_path: str) -> None: + path = _make_logluv_fff(str(tmp_path)) + assert linear_output_source_type(path) == "fff" + + def _make_noritsu_raw(tmp_dir: str, w: int = 4042, h: int = 6391) -> str: """Create a synthetic Noritsu RAW file: headerless BGR16 LE, 12-bit data.""" rng = np.random.RandomState(42) diff --git a/tests/test_logluv.py b/tests/test_logluv.py new file mode 100644 index 00000000..83c3d81c --- /dev/null +++ b/tests/test_logluv.py @@ -0,0 +1,97 @@ +"""Tests for the LogLuv decoder (ported from flexcolor-tool).""" + +import numpy as np + +from negpy.infrastructure.loaders.logluv import ( + decode_strip_logluv32, + logluv24_to_xyz, + logluv32_to_xyz, + xyz_to_linear_rgb, +) + + +_M_LN2 = 0.69314718055994530942 +_UVSCALE = 410.0 +_U_NEU = 0.210526316 +_V_NEU = 0.473684211 + + +def _logl16_from_y(y): + y = np.asarray(y, dtype=np.float64) + le = np.floor(256.0 * (np.log2(np.abs(y)) + 64.0)).astype(np.int64) + le = np.clip(le, 0, 0x7FFF) + le = np.where(y <= 0, 0, le) + return le.astype(np.uint32) + + +def _pack_logluv32(luminance, u_prime=_U_NEU, v_prime=_V_NEU): + le = _logl16_from_y(luminance) + ue = np.clip(np.trunc(_UVSCALE * u_prime), 0, 255).astype(np.uint32) + ve = np.clip(np.trunc(_UVSCALE * v_prime), 0, 255).astype(np.uint32) + return (le << 16) | (ue << 8) | ve + + +class TestLogLuv32: + def test_roundtrip_luminance(self): + Y_in = np.array([0.001, 0.01, 0.1, 0.5, 1.0, 5.0]) + packed = _pack_logluv32(Y_in) + xyz = logluv32_to_xyz(packed) + Y_out = xyz[..., 1] + np.testing.assert_allclose(Y_out, Y_in, rtol=0.02) + + def test_black_pixel(self): + packed = np.array([0], dtype=np.uint32) + xyz = logluv32_to_xyz(packed) + assert np.all(xyz == 0.0) + + def test_xyz_to_linear_rgb_d65_white(self): + xyz = np.array([[[0.9505, 1.0, 1.089]]], dtype=np.float64) + rgb = xyz_to_linear_rgb(xyz) + np.testing.assert_allclose(rgb, 1.0, atol=0.2) + + def test_xyz_to_linear_rgb_shape(self): + xyz = np.ones((10, 20, 3), dtype=np.float64) + rgb = xyz_to_linear_rgb(xyz) + assert rgb.shape == (10, 20, 3) + + def test_decode_strip_roundtrip(self): + packed = _pack_logluv32(np.array([0.5, 1.0, 0.1, 2.0])) + + def encode_strip_simple(pixels): + pixels = np.asarray(pixels, dtype=np.uint32).ravel() + n = pixels.size + out = bytearray() + for shft in (24, 16, 8, 0): + plane = ((pixels >> shft) & 0xFF).astype(np.uint8) + for start in range(0, n, 127): + chunk = plane[start : start + 127] + out.append(len(chunk)) + out += bytes(chunk) + return bytes(out) + + compressed = encode_strip_simple(packed) + decoded = decode_strip_logluv32(compressed, len(packed)) + np.testing.assert_array_equal(decoded, packed) + + +class TestLogLuv24: + def test_zero_luminance(self): + packed = np.array([0], dtype=np.uint32) + xyz = logluv24_to_xyz(packed) + assert np.all(xyz == 0.0) + + def test_nonzero_produces_xyz(self): + le10 = 500 + ce = 8000 + packed = np.array([(le10 << 14) | ce], dtype=np.uint32) + xyz = logluv24_to_xyz(packed) + assert xyz[0, 1] > 0.0 + + +class TestXyzToLinearRgb: + def test_pure_luminance_maps_neutral(self): + xyz = np.array([[[0.9505, 1.0, 1.089]]], dtype=np.float64) + rgb = xyz_to_linear_rgb(xyz) + assert np.all(rgb > 0.8) + spread = rgb.max() - rgb.min() + assert spread < 0.3 From 32b1ad7efb2db699e116cec405f7750ed1fb6b17 Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 04:35:29 +0200 Subject: [PATCH 19/20] Document that Linear Output TIFF is always written clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No ICC profiles, EXIF color space tags, or XMP color metadata from the source are copied through — only raw pixels plus device metadata. --- docs/PIPELINE.md | 2 +- docs/USER_GUIDE.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index ea25e6d6..c38a59ac 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -171,7 +171,7 @@ When the render intent is **Linear**, the entire darkroom pipeline is bypassed. **Optional corrections** (camera RAW only): three toggles let you bake corrections into the linear output before writing. All default to off (raw dump philosophy — the output is unchanged sensor data). *Apply white balance* multiplies the buffer by the as-shot WB gains (green-normalized). *Apply flatfield* applies the configured flatfield gain correction. *Apply sensor correction* applies the crosstalk unmixing matrix. For stitch composites, flatfield and sensor correction are always applied per-part regardless of these toggles — without them, vignetting and crosstalk differences create visible seams at part boundaries. -Source device metadata (Make, Model, DateTime) is carried through to the output TIFF when available from the source file. For Flextight FFF files, the Make field includes film stock and type from the embedded plist, and the Model field includes the scanner serial. +**Output is always clean.** The TIFF is written from scratch — only raw pixels plus Make/Model/DateTime from the source. ICC profiles, EXIF color space tags, and XMP color metadata from scanner software or editors are never copied through. The description field records the source format, expansion, white balance, and any applied corrections, and ends with "no color management". For Flextight FFF files, the Make field includes film stock and type from the embedded plist, and the Model field includes the scanner serial. --- diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 5ffcd2e1..8bb68399 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -554,6 +554,8 @@ A scrollable list of every edit step (last 100 kept), newest on top; the current * **Apply ICE dust removal** (visible when an IR channel is available): applies IR-based dust and scratch correction to the linear output before writing. Off by default. * **Corrections** (camera RAW only): three optional toggles that bake corrections into the linear output before writing. All default to off (raw dump philosophy). **Apply white balance** multiplies by the as-shot WB gains. **Apply flatfield** applies the flatfield gain correction. **Apply sensor correction** applies the sensor crosstalk unmixing matrix. For stitch composites, flatfield and sensor correction are always applied per-part regardless of these toggles (required for clean seams). + The output TIFF is always written clean — no ICC profiles, no EXIF color space tags, and no XMP color metadata from the source are carried through. Only raw pixels plus device metadata (Make, Model, DateTime) from the source file. + ### Export button The primary **Export** action. Its chevron menu picks the scope: current frame (Ctrl+E), selected frames, all visible with current settings, or all visible with each frame's saved settings. From 738f5e8d4d931a2c86fbeeb44e58cd23dc21a03a Mon Sep 17 00:00:00 2001 From: Mats Date: Thu, 6 Aug 2026 05:53:35 +0200 Subject: [PATCH 20/20] Add per-channel percentile normalization to LogLuv decoder LogLuv is an HDR encoding whose raw linear values routinely exceed 1.0; the bare clip(0,1) was silently truncating data instead of normalizing it. Port the normalize_linear step from the flexcolor-tool reference (0.2th/99.8th percentile per channel), which also corrects the per-channel black/gain offset inherent in Flextight CCD data. --- docs/PIPELINE.md | 2 +- docs/USER_GUIDE.md | 2 +- negpy/infrastructure/loaders/logluv.py | 19 ++++++++++++++-- tests/test_logluv.py | 31 ++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index c38a59ac..e4d7e9ee 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -161,7 +161,7 @@ When the render intent is **Linear**, the entire darkroom pipeline is bypassed. * **LinearRaw DNG**: 4-channel VueScan (RGB+IR) and 3-channel SilverFast HDRi files are read directly via tifffile, bypassing rawpy. The IR channel, when present, is written as a separate grayscale TIFF with an `_ir` suffix. Expansion defaults to off; 2× and 4× are available. * **Camera RAW**: demosaiced by rawpy with `user_wb=[1,1,1,1]` (unity), `output_color=raw` (sensor-native), `gamma=(1,1)` (linear), `no_auto_bright=True`. The camera's as-shot white balance multipliers (green-normalized to three RGB values) are embedded in XMP as `RAW-WB: R G B` inside `dc:description`, following the `RAW-WB` XMP convention used by external linear-workflow tools. The source filename is stored in `crs:RawFileName`. No expansion option (camera sensors use the full bit depth). * **Coolscan NEF** (`negpy.infrastructure.loaders.nef_loader`): Nikon Coolscan scanner files are TIFF-structured with the full-res RGB image in a SubIFD chain (tag 0x014A). The loader picks the largest RGB SubIFD by pixel count and rejects files containing CFA/Bayer SubIFDs (camera NEFs). Despite the name, scanner NEFs are not raw sensor data — they are processed images whose content depends on the Nikon Scan settings used at scan time (curves, gain, DigitalICE, etc.). Getting linear, unprocessed output requires the right Nikon Scan settings before scanning. The loader makes no assumptions about linearity or colour space. No separate IR channel exists; any extra channels beyond RGB are dropped. No expansion. -* **Flextight FFF** (`negpy.infrastructure.loaders.fff_loader`): Imacon/Hasselblad Flextight scanner files in two variants: uncompressed 16-bit RGB (standard FlexColor export), and SGI LogLuv compressed (raw `.3fr`/`.fff` from the scanner hardware). Uncompressed files are big-endian TIFFs with the full-res image in a top-level IFD (selected by pixel count, not SubfileType — the tag is unreliable). LogLuv files (compression tags 34676/34677) are decoded via `negpy.infrastructure.loaders.logluv` — a LogLuv32/24 → CIE XYZ → linear sRGB pipeline ported from [flexcolor-tool](https://github.com/rohanpandula/flexcolor-tool) (MIT). The data is gain-normalized linear transmittance with no gamma applied. Embedded FlexColor metadata is parsed from two proprietary tags: tag 50457 (Apple plist with film stock, film type, gamma, DPI, scan date) and tag 46279 (firmware version, scanner serial). No IR hardware exists on Flextight scanners; extra channels beyond RGB are dropped. No expansion. +* **Flextight FFF** (`negpy.infrastructure.loaders.fff_loader`): Imacon/Hasselblad Flextight scanner files in two variants: uncompressed 16-bit RGB (standard FlexColor export), and SGI LogLuv compressed (raw `.3fr`/`.fff` from the scanner hardware). Uncompressed files are big-endian TIFFs with the full-res image in a top-level IFD (selected by pixel count, not SubfileType — the tag is unreliable). LogLuv files (compression tags 34676/34677) are decoded via `negpy.infrastructure.loaders.logluv` — a LogLuv32/24 → CIE XYZ → linear sRGB pipeline ported from [flexcolor-tool](https://github.com/rohanpandula/flexcolor-tool) (MIT). Because LogLuv is an HDR encoding whose raw linear values routinely exceed 1.0, the decoder applies per-channel percentile normalization (0.2th/99.8th) after the XYZ → linear RGB conversion — this is part of the decode recipe (matching the flexcolor-tool reference), not a creative edit; without it the data would be silently truncated, not "raw." The data is gain-normalized linear transmittance with no gamma applied. Embedded FlexColor metadata is parsed from two proprietary tags: tag 50457 (Apple plist with film stock, film type, gamma, DPI, scan date) and tag 46279 (firmware version, scanner serial). No IR hardware exists on Flextight scanners; extra channels beyond RGB are dropped. No expansion. * **Noritsu RAW** (`negpy.infrastructure.loaders.noritsu_loader`): headerless BGR 16-bit little-endian scanner dumps. Frame dimensions are auto-detected from file size against a table of known Noritsu scan dimensions (three tiers: exact match, known-width with novel height, and novel-width fallback). The channel order is BGR, swapped to RGB on load. 12-bit sensor data in 16-bit range; default expansion is 16× (`NORITSU_EXPANSION`). * **TIFF** (standalone `_decode_tiff`, not routed through `TiffLoader` — kept independent to avoid inheriting TiffLoader's sRGB linearization assumptions): the 4th channel is treated as IR only when the ExtraSamples tag is 0 (UNSPECIFIED) or missing; values 1/2 (associated/unassociated alpha) are dropped. Sidecar IR files (`_ir.tif` next to the source, with optional `_ir_valid` mask) and IR stored in secondary TIFF pages (SilverFast iSRD convention) are also detected. An **Input gamma** selector lets the user declare the source encoding (linear, 1.8, 2.2, or sRGB) so the data can be linearized before export. Expansion is available (off by default). * **RGB-scan triplets**: when the current frame is an RGB-scan composite (three narrowband exposures), all three are decoded and merged via `merge_rgb_triplet()` into a single combined TIFF. The red exposure is the primary file; green and blue paths come from the frame's `RgbScanConfig`. No sensor correction is applied (narrowband exposures have no cross-channel leakage). WB and device metadata are taken from the primary (red) exposure. diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 8bb68399..d8a0c5e7 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -547,7 +547,7 @@ A scrollable list of every edit step (last 100 kept), newest on top; the current * **LinearRaw DNG** — SilverFast HDRi (3-channel) and VueScan (4-channel RGB+IR). IR is written as a separate grayscale TIFF with an `_ir` suffix. * **Camera RAW** — demosaiced with unity white balance (1,1,1,1). The camera's as-shot WB is written into XMP (`RAW-WB: R G B`) so it can be applied by downstream tools. Source device and timestamp are preserved. RGB-scan triplets (narrowband R/G/B exposures) are merged into a single combined TIFF. Stitch composites are assembled with flatfield and sensor correction applied per-part for clean seams; stitch + triplet combinations are also supported. * **Coolscan NEF** — Nikon Coolscan scanner files. Despite the name, these are not raw sensor data — the content depends on the Nikon Scan settings used at scan time. Getting linear, unprocessed output requires the right settings before scanning. The full-res RGB SubIFD is read directly; any extra channels beyond RGB are dropped (Coolscan has no separate IR channel). No expansion. - * **Flextight FFF** — Imacon/Hasselblad Flextight scanner files, including both standard uncompressed 16-bit RGB exports and SGI LogLuv compressed raw files (`.3fr`/`.fff`). LogLuv files are decoded through a LogLuv → XYZ → linear sRGB pipeline. The largest image IFD is selected by pixel count. Data is linear scanner transmittance. Embedded FlexColor metadata (film stock, film type, scan date, scanner serial) from the proprietary plist (tag 50457) and firmware blob (tag 46279) is carried through to the output TIFF headers. No expansion. + * **Flextight FFF** — Imacon/Hasselblad Flextight scanner files, including both standard uncompressed 16-bit RGB exports and SGI LogLuv compressed raw files (`.3fr`/`.fff`). LogLuv files are decoded through a LogLuv → XYZ → linear sRGB pipeline with per-channel percentile normalization (LogLuv is HDR, so normalization is part of the decode — without it the data would be truncated, not raw). The largest image IFD is selected by pixel count. Data is linear scanner transmittance. Embedded FlexColor metadata (film stock, film type, scan date, scanner serial) from the proprietary plist (tag 50457) and firmware blob (tag 46279) is carried through to the output TIFF headers. No expansion. * **Noritsu RAW** — headerless BGR 16-bit scanner dumps. Frame dimensions are auto-detected from file size against known Noritsu scan dimensions. 16× expansion by default (12-bit sensor data in 16-bit range). * **TIFF** — generic scanner TIFFs. If the file has a 4th channel tagged as IR (ExtraSamples = UNSPECIFIED or missing), it is written as a separate `_ir` TIFF. Sidecar IR files (`_ir.tif` next to the source) and IR stored in secondary TIFF pages are also detected. **Input gamma** lets you select the gamma encoding of the source (linear, 1.8, 2.2, or sRGB) so the data can be linearized before export. Expansion is available (off by default). * **Expansion**: scales the linear data before writing. The combo box shows source-appropriate options: Pakon F135/F235 default to 4×, Noritsu defaults to 16×, F335 and LinearRaw DNG default to off. Camera RAW, Coolscan NEF, and Flextight FFF files have no expansion option. Leave at the default unless you know why you need to change it. diff --git a/negpy/infrastructure/loaders/logluv.py b/negpy/infrastructure/loaders/logluv.py index b33eb67d..64080973 100644 --- a/negpy/infrastructure/loaders/logluv.py +++ b/negpy/infrastructure/loaders/logluv.py @@ -308,6 +308,21 @@ def logluv24_to_xyz(packed: np.ndarray) -> np.ndarray: ) +def normalize_linear(lin: np.ndarray) -> np.ndarray: + """Per-channel percentile black/white normalization. + + Maps raw HDR linear RGB into [0, 1] using the 0.2th/99.8th percentile + per channel, correcting the per-channel black/gain offset inherent in + Flextight CCD data (worst on red — weak CCD response, shallow C-41 cyan). + """ + flat = lin.reshape(-1, lin.shape[-1]) + lo = np.percentile(flat, 0.2, axis=0) + hi = np.percentile(flat, 99.8, axis=0) + denom = hi - lo + denom = np.where(denom > 0, denom, 1.0) + return np.clip((lin - lo) / denom, 0.0, 1.0) + + def xyz_to_linear_rgb(xyz: np.ndarray) -> np.ndarray: """CIE XYZ (..., 3) → linear sRGB float64 (..., 3), unbounded.""" xyz = np.asarray(xyz, dtype=np.float64) @@ -383,8 +398,8 @@ def decode_logluv_strips( packed = packed[:npixels_total].reshape(height, width) xyz = px_to_xyz(packed) lin = xyz_to_linear_rgb(xyz) - f32 = np.clip(lin, 0.0, 1.0).astype(np.float32) - return f32 + normed = normalize_linear(lin) + return normed.astype(np.float32) def _tag_to_list(tag) -> list: diff --git a/tests/test_logluv.py b/tests/test_logluv.py index 83c3d81c..c917638d 100644 --- a/tests/test_logluv.py +++ b/tests/test_logluv.py @@ -6,6 +6,7 @@ decode_strip_logluv32, logluv24_to_xyz, logluv32_to_xyz, + normalize_linear, xyz_to_linear_rgb, ) @@ -88,6 +89,36 @@ def test_nonzero_produces_xyz(self): assert xyz[0, 1] > 0.0 +class TestNormalizeLinear: + def test_hdr_values_normalized_to_unit_range(self): + lin = np.array( + [[[0.1, 0.05, 0.2], [0.5, 0.4, 0.6], [2.0, 1.8, 3.0], [4.0, 3.5, 5.0]]], + dtype=np.float64, + ) + normed = normalize_linear(lin) + assert normed.min() >= 0.0 + assert normed.max() <= 1.0 + + def test_per_channel_offset_correction(self): + rng = np.random.default_rng(42) + n = 1000 + r = rng.uniform(0.5, 2.0, n) + g = rng.uniform(0.1, 1.5, n) + b = rng.uniform(0.3, 1.8, n) + lin = np.stack([r, g, b], axis=-1).reshape(1, n, 3) + normed = normalize_linear(lin) + for ch in range(3): + vals = normed[0, :, ch] + assert vals.min() >= 0.0 + assert vals.max() <= 1.0 + assert vals.max() > 0.9 + + def test_uniform_channel_not_divided_by_zero(self): + lin = np.full((1, 10, 3), 0.5, dtype=np.float64) + normed = normalize_linear(lin) + assert np.all(np.isfinite(normed)) + + class TestXyzToLinearRgb: def test_pure_luminance_maps_neutral(self): xyz = np.array([[[0.9505, 1.0, 1.089]]], dtype=np.float64)