diff --git a/crates/compositor/src/linux_decode.rs b/crates/compositor/src/linux_decode.rs index e8bb81bc..1255898e 100644 --- a/crates/compositor/src/linux_decode.rs +++ b/crates/compositor/src/linux_decode.rs @@ -16,6 +16,8 @@ use anyhow::{bail, Context, Result}; use std::ffi::CString; use std::ptr; +use crate::timeline_walk::NextFrameTime; + use crate::ffi::{ av_frame_alloc, av_frame_free, av_frame_move_ref, av_frame_unref, av_packet_alloc, av_packet_free, av_packet_unref, av_read_frame, av_seek_frame, avcodec_alloc_context3, @@ -58,6 +60,11 @@ pub struct SwDecoder { frame: *mut AVFrame, sent_eof: bool, cur_pts: Option, + /// Buffer de lookahead pour `peek_next_time_sec` : symétrique de + /// `pipeline_macos::Decoder::peek_frame` — cf. là-bas pour la justification. + peek_frame: *mut AVFrame, + /// `true` si `peek_frame` porte une frame décodée en attente de `commit_peek`. + has_peek: bool, } /// Libère toutes les ressources ffmpeg. `Drop` ne peut pas faillir ; on @@ -78,6 +85,9 @@ impl Drop for SwDecoder { if !self.pkt.is_null() { av_packet_free(&mut self.pkt); } + if !self.peek_frame.is_null() { + av_frame_free(&mut self.peek_frame); + } } } } @@ -177,7 +187,8 @@ impl SwDecoder { }; let pkt = av_packet_alloc(); let frame = av_frame_alloc(); - if pkt.is_null() || frame.is_null() { + let peek_frame = av_frame_alloc(); + if pkt.is_null() || frame.is_null() || peek_frame.is_null() { avcodec_free_context(&mut dec); avformat_close_input(&mut fmt); bail!("av_packet_alloc/av_frame_alloc (pompage sequentiel)"); @@ -192,6 +203,8 @@ impl SwDecoder { frame, sent_eof: false, cur_pts: None, + peek_frame, + has_peek: false, }) } @@ -201,21 +214,33 @@ impl SwDecoder { /// seek PAS : le decodeur garde son etat, donc une lecture sequentielle coute /// UN packet par frame au lieu d'un re-parcours de demi-GOP. pub unsafe fn next_frame(&mut self) -> Result<*mut AVFrame> { + if self.has_peek { + return self.commit_peek(); + } + if !self.receive_into(self.frame)? { + return Ok(ptr::null_mut()); + } + let pts = (*self.frame).best_effort_timestamp; + self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + Ok(self.frame) + } + + /// Décode dans `into` (buffer courant ou de lookahead) jusqu'à obtenir une frame ou + /// l'EOF — cf. `pipeline_macos::Decoder::receive_into` pour la justification. + unsafe fn receive_into(&mut self, into: *mut AVFrame) -> Result { loop { - let r = avcodec_receive_frame(self.dec, self.frame); + let r = avcodec_receive_frame(self.dec, into); if r == 0 { - let pts = (*self.frame).best_effort_timestamp; - self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; - return Ok(self.frame); + return Ok(true); } if r == AVERROR_EOF { - return Ok(ptr::null_mut()); + return Ok(false); } if r != AVERROR_EAGAIN { bail!("avcodec_receive_frame: {r}"); } if self.sent_eof { - return Ok(ptr::null_mut()); + return Ok(false); } let rr = av_read_frame(self.fmt, self.pkt); if rr < 0 { @@ -240,6 +265,41 @@ impl SwDecoder { } } + /// Décode la prochaine frame dans le buffer de lookahead et renvoie son temps. + /// Cf. `pipeline_macos::Decoder::peek_next_time_sec`. + pub(crate) unsafe fn peek_next_time_sec(&mut self) -> Result { + if !self.has_peek { + if !self.receive_into(self.peek_frame)? { + return Ok(NextFrameTime::Eof); + } + self.has_peek = true; + } + let pts = (*self.peek_frame).best_effort_timestamp; + // Sans pts ni time_base exploitables on ne PEUT pas dire si la frame est due : + // `Unknown`, et non `0.0` — qui passait pour « due » à tous les coups. + Ok(if pts == i64::MIN || self.stream_timebase <= 0.0 { + NextFrameTime::Unknown + } else { + NextFrameTime::At(pts as f64 * self.stream_timebase) + }) + } + + /// Promeut la frame de lookahead au rang de frame courante. Cf. + /// `pipeline_macos::Decoder::commit_peek`. + pub(crate) unsafe fn commit_peek(&mut self) -> Result<*mut AVFrame> { + // `bail!` et non `debug_assert!` : compilée en release, l'assertion disparaissait + // et l'échange promouvait un `AVFrame` jamais rempli, avec un + // `best_effort_timestamp` indéterminé, jusque dans le chemin de présentation. + if !self.has_peek { + bail!("commit_peek sans peek_next_time_sec préalable"); + } + std::mem::swap(&mut self.frame, &mut self.peek_frame); + self.has_peek = false; + let pts = (*self.frame).best_effort_timestamp; + self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + Ok(self.frame) + } + /// Temps source (secondes) de la derniere frame rendue par `next_frame` / /// `decode_at`, tire du pts REEL et non d'un compteur d'index. pub fn cur_time_sec(&self) -> Option { @@ -263,6 +323,8 @@ impl SwDecoder { /// `AVERROR_INVALIDDATA` plutôt que de paniquer : la prochaine itération /// lira le packet complet suivant. pub unsafe fn decode_at(&mut self, frame_idx: u32) -> Result<*mut AVFrame> { + // Tout seek invalide un éventuel peek en attente — cf. pipeline_macos::Decoder::seek_to. + self.has_peek = false; let fps = self.fps; let target_ts = (frame_idx as f64 / fps) * 1_000_000.0; // AV_TIME_BASE = µs // `AVSEEK_FLAG_BACKWARD` vaut 1, pas 4 — 4 est `AVSEEK_FLAG_ANY`. La constante diff --git a/crates/compositor/src/live.rs b/crates/compositor/src/live.rs index 49199a69..9c1c8f38 100644 --- a/crates/compositor/src/live.rs +++ b/crates/compositor/src/live.rs @@ -29,6 +29,7 @@ use crate::config::{self, Cfg}; use crate::cursor::CursorTrack; use crate::d3d::Gpu; use crate::pipeline::Decoder; +use crate::timeline_walk::{frame_step, FrameStep, NextFrameTime}; use anyhow::Result; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -338,42 +339,73 @@ impl Player { } /// Temps source courant du décodeur écran — utilisé par `render_thread` pour détecter le - /// franchissement de la fin de fenêtre du clip actif pendant la lecture libre. - pub(crate) unsafe fn screen_time_sec(&self) -> f64 { + /// franchissement de la fin de fenêtre du clip actif pendant la lecture libre, et pour + /// calculer la cible de `step` en lecture libre. `pub` (pas `pub(crate)`) : le harnais + /// `poc-d3d` (crate externe) en a besoin pour piloter sa propre boucle de lecture libre. + pub unsafe fn screen_time_sec(&self) -> f64 { self.sdec.cur_time_sec() } - /// Compose la frame suivante (→ `comp.rt`). Boucle sur EOF. `false` si fixture vide. + /// Compose la PROCHAINE frame due (→ `comp.rt`), au plus une, si `target_source_time` + /// (temps écran) est atteint. Sémantique de "hold" : `false` sans rien composer quand la + /// frame suivante n'est pas encore due — l'appelant garde alors l'image déjà affichée, + /// au lieu d'avancer aveuglément. Boucle sur EOF réel. `false` aussi si fixture vide. /// - /// L'écran pilote la cadence (1 frame/tick) ; la webcam suit son PROPRE temps source - /// (`screen_time - webcam_offset_sec`), pas un pas 1:1 avec l'écran — BUG corrigé : les - /// deux décodeurs avançaient d'exactement une frame par tick chacun, quelle que soit leur - /// cadence réelle. Écran et webcam sont capturés par des pipelines indépendants (souvent - /// à des fps différents), donc la webcam jouait 2× trop vite dès que sa cadence était - /// inférieure à celle de l'écran. Même logique que `advance_decoder_to` (pipeline.rs), - /// déjà correcte côté export — la preview live ne l'avait jamais reprise. La webcam boucle - /// aussi de façon INDÉPENDANTE à son propre EOF (un clip webcam plus court que l'écran ne - /// doit pas réinitialiser le décodeur écran). - pub unsafe fn step(&mut self, comp: &Compositor, cfg: &Cfg) -> Result { + /// BUG corrigé : cette fonction consommait auparavant EXACTEMENT une frame réelle par + /// appel (un `next()` inconditionnel), et `render_thread` l'appelait une fois par tranche + /// de 1/60s de temps réel écoulé — une hypothèse de vidéo à ~60 fps constant. Or + /// ScreenCaptureKit (et les captures équivalentes) ne livre une frame que quand l'écran + /// change : un enregistrement de 26s avec de longs plans fixes peut ne contenir que + /// quelques centaines de frames RÉELLES. Consommer 1 frame/tick épuisait alors le flux + /// bien avant que le temps réel écoulé n'atteigne la durée de l'enregistrement — le + /// décodeur retombait sur l'EOF, rebouclait sur `seek_to(0.0)`, et la preview semblait + /// « accélérer puis sauter au début » en boucle. Le curseur/zoom, eux, suivent le pts réel + /// (`sync_time`) et se retrouvaient donc en avance sur ce que l'œil voyait défiler. + /// + /// Le correctif : ne décoder/adopter (`commit_peek`) la frame suivante QUE si son pts a + /// réellement été atteint par `target_source_time` (le temps réel écoulé, mis à l'échelle + /// par la vitesse active — cf. `render_thread`) ; sinon on continue de tenir la frame + /// courante, aussi longtemps qu'il le faut. Même principe que `advance_decoder_to` + /// (`timeline_walk.rs`), déjà correct côté export. + /// + /// La webcam suit le MÊME principe indépendamment (son propre temps source = + /// `screen_time - webcam_offset_sec`, pas un pas 1:1 avec l'écran) : deux pipelines de + /// capture indépendants n'ont pas la même cadence ni les mêmes trous. Arrivée à son + /// propre EOF — un clip webcam plus court que l'écran, cas normal quand la caméra + /// s'arrête avant la capture — elle TIENT sa dernière image et laisse l'écran + /// continuer seul, plutôt que de reboucler au début. + pub unsafe fn step(&mut self, comp: &Compositor, cfg: &Cfg, target_source_time: f64) -> Result { let use_current = self.use_current_on_next_step; self.use_current_on_next_step = false; - let mut sf = if use_current { + let sf = if use_current { self.sdec.cur_frame() } else { - self.sdec.next()? + // Match direct sur `NextFrameTime` plutôt que via `frame_step` : la lecture + // live a une politique d'EOF PROPRE (reboucler au début), là où l'export tient + // la dernière frame. Le reste — « due » vs « pas encore due » — est la même + // règle, `>`/`<=` compris. + match self.sdec.peek_next_time_sec()? { + NextFrameTime::At(t) if t <= target_source_time => self.sdec.commit_peek()?, + NextFrameTime::At(_) => return Ok(false), // pas encore due : on tient la courante. + // pts inexploitable : impossible de dire si elle est due. `step()` n'adopte + // qu'une frame écran par appel, donc l'adopter revient exactement à + // l'ancien « une frame par tick » — le repli correct pour un flux cassé. + NextFrameTime::Unknown => self.sdec.commit_peek()?, + NextFrameTime::Eof => { + // EOF réel (plus aucune frame à décoder) : reboucle sur le début. + self.idx = 0; + self.sdec.seek_to(0.0)? + } + } }; - if sf.is_null() { - sf = self.sdec.seek_to(0.0)?; - self.idx = 0; - } if sf.is_null() { self.has_current_frame = false; return Ok(false); } let target_webcam_t = (self.sdec.cur_time_sec() - self.webcam_offset_sec).max(0.0); - let mut wf = if use_current { + let wf = if use_current { self.wdec.cur_frame() } else { let cur = self.wdec.cur_frame(); @@ -381,19 +413,30 @@ impl Player { // Jamais décodée (nouvelle ouverture) : on saute directement au temps synchronisé. self.wdec.seek_to(target_webcam_t)? } else { - // Rattrape la webcam vers `target_webcam_t`, au pire une poignée de frames par - // tick (fps proches) — le garde-fou n'existe que contre un cas pathologique. + // Rattrape la webcam vers `target_webcam_t` par pts réel, jamais au-delà — + // même sémantique de hold que l'écran ci-dessus (et que `advance_decoder_to`) : + // adopter une frame webcam dont le pts dépasse `target_webcam_t` l'afficherait + // en avance sur son heure. Le garde-fou ne joue que contre un cas pathologique. + // + // BUG corrigé : à son EOF la webcam était reseekée à 0 alors que + // `target_webcam_t` continue de croître avec le temps écran. Au tick + // suivant, le rattrapage repartait donc de 0 et réavalait le fichier + // entier vers une cible toujours aussi lointaine — 1000 frames par tick + // (le plafond du garde-fou), en boucle, pour l'éternité. Un décodage + // permanent à fond, pour afficher une webcam qui n'a plus rien à montrer. + // Une fois l'EOF traité comme un hold, la décision webcam est EXACTEMENT + // celle de l'écran à l'export : `frame_step` couvre les quatre cas sans + // rien de spécifique, et ses tests couvrent donc aussi ce chemin. let mut wf = cur; let mut guard = 0u32; - while self.wdec.cur_time_sec() < target_webcam_t { - match self.wdec.next()? { - f if f.is_null() => { - // Fin de la webcam avant l'écran : elle boucle SEULE — l'écran - // garde sa propre position, inchangée. - wf = self.wdec.seek_to(0.0)?; + loop { + match frame_step(self.wdec.peek_next_time_sec()?, 0.0, target_webcam_t) { + FrameStep::Commit => wf = self.wdec.commit_peek()?, + FrameStep::CommitAndStop => { + wf = self.wdec.commit_peek()?; break; } - f => wf = f, + FrameStep::Hold => break, } guard += 1; if guard > 1000 { @@ -469,6 +512,24 @@ impl Player { } } +/// Retranche de l'accumulateur le temps source RÉELLEMENT consommé par la frame qui vient +/// d'être adoptée. C'est ce qui fait jouer une source à sa propre cadence : une frame de +/// 1/24 s consomme 1/24 s d'accumulateur, donc 24 frames par seconde réelle — là où +/// l'ancien pas fixe de 1/60 s en décodait 60, soit 2,5× trop vite sur du 24 fps. +/// +/// `after < before` : `step()` a rebouclé sur l'EOF (le temps recule) — le delta n'a plus +/// de sens, on repart d'un accumulateur propre. +/// +/// Fonction à part pour être testable : c'est l'arithmétique dont dépend la vitesse de +/// lecture, et elle vivait au milieu de la boucle de rendu. +pub(crate) fn consume_acc(acc: f64, before: f64, after: f64) -> f64 { + if after >= before { + (acc - (after - before)).max(0.0) + } else { + 0.0 + } +} + /// Paramètres inspector pilotés depuis l'UI (setParam). Le thread de rendu les applique : /// booléens/taps → reconstruits dans le `Cfg` ; valeurs continues → `set_live_params`. #[derive(Clone, Copy, PartialEq)] @@ -1450,26 +1511,34 @@ unsafe fn render_thread( } acc = 0.0; // resynchronise l'accumulateur de lecture libre après un seek } else if shared.playing.load(Ordering::Relaxed) { - // BUG corrigé : la lecture libre décodait toujours exactement 1 frame par tick de - // 1/60s réel, quelle que soit la speed region active au temps source courant — ni - // l'écran ni la webcam n'accéléraient/ralentissaient jamais en preview live (seul - // l'export, via `speed_segments_for_window`/`advance_decoder_to` dans pipeline.rs, - // retimait correctement). Mod 3 corrige déjà le fps-mismatch webcam/écran (la webcam - // suit le temps source RÉEL de l'écran, pas un pas 1:1) — reprend ici la même idée : - // l'accumulateur de temps réel est mis à l'échelle par le multiplicateur de vitesse - // actif, donc `step()` (qui resynchronise la webcam sur le temps écran courant, - // cf. plus haut) décode plus/moins de frames par seconde réelle selon la région. + // BUG corrigé : la lecture libre décodait auparavant exactement 1 frame RÉELLE par + // tranche de 1/60s de temps réel écoulé, quelle que soit la densité effective de + // frames de la source. ScreenCaptureKit (et les captures équivalentes) ne livre une + // frame que quand l'écran change : un enregistrement avec de longs plans fixes peut + // ne contenir que quelques centaines de frames RÉELLES sur toute sa durée. Consommer + // 1 frame/tick épuisait alors le flux bien avant que le temps réel écoulé n'atteigne + // la durée de l'enregistrement — `step()` retombait sur l'EOF et rebouclait sur + // `seek_to(0.0)`, d'où la preview qui semblait « accélérer puis sauter au début » en + // boucle (cf. doc de `step()` pour le détail). + // + // Le correctif : `acc` (mis à l'échelle par la speed region active, cf. mod 3 plus + // bas pour le fps-mismatch webcam/écran) n'est plus consommé par tranches fixes de + // 1/60s — c'est une CIBLE de temps source (`target = temps courant + acc`) que + // `step()` n'atteint qu'en adoptant une frame dont le pts est réellement dû (hold + // sinon). `acc` n'est décrémenté que du temps source RÉELLEMENT consommé par chaque + // frame adoptée, jamais d'un pas fixe — donc les plans fixes ne consomment aucune + // frame et n'avancent le décodeur que quand une frame due existe vraiment. let speed = full_scene .as_ref() .map(|scene| speed_at(&scene.speed_regions, active_clip_index, player.screen_time_sec())) .unwrap_or(1.0); acc += dt * speed; - let step = 1.0 / 60.0; let mut n = 0; - // Cap proportionnel à la vitesse (borné) : à vitesse élevée, plus de frames doivent - // être décodées par tick réel pour ne pas prendre du retard sur l'accumulateur. + // Cap sur le nombre de frames RÉELLEMENT adoptées par tick (pas sur le nombre de + // ticks) : à vitesse élevée sur du contenu dense, plus de frames doivent être + // décodées par tick réel pour ne pas prendre du retard sur l'accumulateur. let max_steps = ((3.0 * speed.max(1.0)).ceil() as i32).min(64); - while acc >= step && n < max_steps { + loop { // Timeline = niveau d'abstraction AU-DESSUS des clips : dès que le décodeur // écran atteint la fin de fenêtre du clip actif, on enchaîne nous-mêmes sur // le clip suivant (ou on reboucle sur le premier après le dernier) — sans @@ -1507,9 +1576,9 @@ unsafe fn render_thread( } } let screen_time_before_step = full_scene.as_ref().map(|_| player.screen_time_sec()); - if player.step(&comp, &cfg)? { - stepped = true; - } + let before = player.screen_time_sec(); + let target = before + acc; + let committed = player.step(&comp, &cfg, target)?; // Filet de sécurité : un clip NON trimmé (source_end_sec == durée totale du // fichier) peut ne jamais franchir le seuil ci-dessus si la dernière frame // réelle a un PTS strictement inférieur à `source_end_sec` déclaré — `step()` @@ -1532,11 +1601,22 @@ unsafe fn render_thread( ); } } - acc -= step; + if !committed { + // Rien n'est dû pour l'instant (hold) : `acc` reste tel quel — il continue + // de s'accumuler aux ticks suivants jusqu'à ce qu'une vraie frame arrive. + break; + } + stepped = true; + let after = player.screen_time_sec(); + acc = consume_acc(acc, before, after); n += 1; - } - if acc > step { - acc = 0.0; + if n >= max_steps { + // Rattrapage plafonné : le contenu dû est plus dense que ce qu'on peut décoder + // en un tick réel. On laisse tomber le reliquat plutôt que de creuser une + // dette qui s'accumulerait indéfiniment d'un tick à l'autre. + acc = 0.0; + break; + } } } else if first || ip_changed || scene_changed || clip_changed || resized { // pause : recompose la frame courante (param / scène / clip / résolution changés). @@ -1777,6 +1857,29 @@ mod tests { }"##).expect("multiclip scene") } + #[test] + fn acc_is_reduced_by_the_source_time_the_frame_actually_consumed() { + // 1/24 s d'accumulateur par frame de 24 fps : c'est ce qui fait jouer une source à + // SA cadence. L'ancien pas fixe retranchait 1/60 s quelle que soit la source, d'où + // 60 frames décodées par seconde réelle — 2,5× trop vite sur du 24 fps. + let acc = consume_acc(0.05, 10.0, 10.0 + 1.0 / 24.0); + assert!((acc - (0.05 - 1.0 / 24.0)).abs() < 1e-12); + } + + #[test] + fn acc_never_goes_negative() { + // Une frame plus longue que le retard accumulé ne doit pas creuser une dette + // négative qui ferait sauter la frame suivante. + assert_eq!(consume_acc(0.01, 10.0, 10.5), 0.0); + } + + #[test] + fn acc_resets_when_playback_loops_back_to_the_start() { + // `after < before` : `step()` a rebouclé sur l'EOF. Le delta serait négatif et + // gonflerait l'accumulateur — lecture emballée juste après la boucle. + assert_eq!(consume_acc(0.02, 1950.0, 0.0), 0.0); + } + #[test] fn explicit_index_disambiguates_clips_sharing_sources() { let scene = multiclip_scene(); diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 2e8af625..0d0d9804 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -28,6 +28,7 @@ use crate::config::Cfg; use crate::d3d::Gpu; use crate::ffi::AVFrame; use crate::linux_decode::SwDecoder; +use crate::timeline_walk::NextFrameTime; use crate::linux_frames::CpuFrames; /// `SWS_POINT` (plus proche voisin). Bindgen ne genere pas les `SWS_*` (macros), @@ -139,6 +140,24 @@ impl Decoder { Ok(carrier) } + /// Décode la prochaine frame dans le buffer de lookahead du décodeur sous-jacent et + /// renvoie son temps, sans la présenter (donc sans toucher `self.cur`). + /// Cf. `pipeline_macos::Decoder::peek_next_time_sec` pour la sémantique "hold". + pub(crate) unsafe fn peek_next_time_sec(&mut self) -> Result { + self.sw.peek_next_time_sec() + } + + /// Promeut la frame de lookahead au rang de frame courante ET la présente (upload NV12 + /// vers la texture carrier), contrairement au chemin macOS/Windows où la promotion est + /// un pur échange de pointeurs — ici la présentation est le pas qui manque. + pub(crate) unsafe fn commit_peek(&mut self) -> Result<*mut AVFrame> { + let raw = self.sw.commit_peek()?; + let carrier = self.frames.present(raw)?; + self.cur = carrier; + self.next_idx = self.next_idx.saturating_add(1); + Ok(carrier) + } + pub unsafe fn cur_frame(&self) -> *mut AVFrame { self.cur } diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 8fec0fb2..2ad12b66 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -35,6 +35,7 @@ use crate::audio::{ }; use crate::compositor::Compositor; use crate::d3d::Gpu; +use crate::timeline_walk::NextFrameTime; use anyhow::{anyhow, bail, Result}; use std::ffi::{c_void, CString}; use std::ptr; @@ -85,6 +86,14 @@ pub struct Decoder { /// qu'on pose dans `data[0]`). `None` quand VideoToolbox couvre le codec — le décodeur /// rend alors directement la frame VideoToolbox. cpu: Option, + /// Buffer de lookahead pour `peek_next_time_sec` : une frame décodée à l'avance, pas + /// encore promue en frame courante. Sépare "voir le pts de la frame suivante" de + /// "l'adopter" — condition de la sémantique "hold" (cf. `timeline_walk::advance_decoder_to` + /// et `live::Player::step`) : sans ce second buffer, `avcodec_receive_frame` écraserait + /// `frame` avant qu'on ait pu décider si son pts est déjà dû. + peek_frame: *mut crate::ffi::AVFrame, + /// `true` si `peek_frame` porte une frame décodée en attente de `commit_peek`. + has_peek: bool, } impl Decoder { @@ -201,11 +210,18 @@ impl Decoder { sent_eof: false, cur_pts: None, cpu, + peek_frame: crate::ffi::av_frame_alloc(), + has_peek: false, }) } } pub unsafe fn rewind(&mut self) -> Result<()> { + // Même règle que `seek_to` : tout repositionnement invalide le peek en attente. + // Il portait sur « la frame d'après l'ancienne position », qui n'a plus de sens + // ici — sans ça le `next()` suivant promouvait une frame décodée avant le rewind, + // avec son ancien `cur_pts`. + self.has_peek = false; crate::ffi::averr( crate::ffi::av_seek_frame( self.fmt, @@ -235,6 +251,9 @@ impl Decoder { /// rapide compris : mêmes seuils, même critère d'arrêt (`decode_forward_to`), pour /// que les deux moteurs rendent la même frame au même coût relatif. pub unsafe fn seek_to(&mut self, seconds: f64) -> Result<*mut crate::ffi::AVFrame> { + // Tout seek invalide un éventuel peek en attente : il portait sur "la frame après + // l'ancienne position courante", qui n'a plus de sens une fois qu'on a sauté ailleurs. + self.has_peek = false; let tb_sec = self.tb_sec(); if tb_sec > 0.0 { @@ -309,24 +328,42 @@ impl Decoder { /// Windows, juste sans le dispatch D3D11VA (le GPU hand-off est déjà fait par /// `av_hwdevice_ctx_create`). pub unsafe fn next(&mut self) -> Result<*mut crate::ffi::AVFrame> { + // Un peek déjà décodé en attente : l'appelant n'est pas passé par `commit_peek` + // (chemins qui ne raisonnent pas en hold, ex. `seek_to`/`decode_forward_to` après + // qu'aucun peek n'ait été posé) — le promouvoir reste correct dans tous les cas : + // c'est bien la prochaine frame du flux. + if self.has_peek { + return self.commit_peek(); + } + if !self.receive_into(self.frame)? { + return Ok(ptr::null_mut()); + } + let pts = (*self.frame).best_effort_timestamp; + self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + match &mut self.cpu { + Some(cpu) => cpu.present(self.frame), + None => Ok(self.frame), + } + } + + /// Décode dans `into` (buffer courant ou de lookahead) jusqu'à obtenir une frame ou + /// l'EOF — pompage `avcodec_receive_frame`/`av_read_frame` brut, indépendant du buffer + /// cible. Factorisé pour que `next()` et `peek_next_time_sec()` partagent exactement la + /// même mécanique de décodage, seul le buffer destinataire changeant. + unsafe fn receive_into(&mut self, into: *mut crate::ffi::AVFrame) -> Result { loop { - let r = crate::ffi::avcodec_receive_frame(self.dctx, self.frame); + let r = crate::ffi::avcodec_receive_frame(self.dctx, into); if r == 0 { - let pts = (*self.frame).best_effort_timestamp; - self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; - return match &mut self.cpu { - Some(cpu) => cpu.present(self.frame), - None => Ok(self.frame), - }; + return Ok(true); } if r == crate::ffi::AVERROR_EOF { - return Ok(ptr::null_mut()); + return Ok(false); } if r != crate::ffi::AVERROR_EAGAIN { crate::ffi::averr(r, "receive_frame")?; } if self.sent_eof { - return Ok(ptr::null_mut()); + return Ok(false); } let rr = crate::ffi::av_read_frame(self.fmt, self.pkt); if rr == crate::ffi::AVERROR_EOF { @@ -345,6 +382,49 @@ impl Decoder { } } + /// Décode la PROCHAINE frame dans le buffer de lookahead (si aucun peek n'est déjà en + /// attente) et renvoie son temps. Ne touche pas au buffer courant : l'appelant peut + /// ainsi comparer ce pts à une cible avant de décider d'adopter la frame + /// (`commit_peek`) ou de continuer à tenir la frame courante (hold). + /// + /// `NextFrameTime::Unknown` — et non `0.0` — quand le pts est inexploitable : `0.0` + /// satisfait toujours la condition d'adoption, ce qui vidait le flux jusqu'à l'EOF. + pub(crate) unsafe fn peek_next_time_sec(&mut self) -> Result { + if !self.has_peek { + if !self.receive_into(self.peek_frame)? { + return Ok(NextFrameTime::Eof); + } + self.has_peek = true; + } + let pts = (*self.peek_frame).best_effort_timestamp; + let tb_sec = self.tb_sec(); + Ok(if pts == i64::MIN || tb_sec <= 0.0 { + NextFrameTime::Unknown + } else { + NextFrameTime::At(pts as f64 * tb_sec) + }) + } + + /// Promeut la frame de lookahead (décodée par un `peek_next_time_sec` précédent) au + /// rang de frame courante — échange de pointeurs, aucune E/S. Ne doit être appelé + /// qu'après un `peek_next_time_sec` ayant renvoyé une frame. + pub(crate) unsafe fn commit_peek(&mut self) -> Result<*mut crate::ffi::AVFrame> { + // `bail!` et non `debug_assert!` : compilée en release, l'assertion disparaissait + // et l'échange promouvait un `AVFrame` jamais rempli, avec un + // `best_effort_timestamp` indéterminé, jusque dans le chemin de présentation. + if !self.has_peek { + anyhow::bail!("commit_peek sans peek_next_time_sec préalable"); + } + std::mem::swap(&mut self.frame, &mut self.peek_frame); + self.has_peek = false; + let pts = (*self.frame).best_effort_timestamp; + self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + match &mut self.cpu { + Some(cpu) => cpu.present(self.frame), + None => Ok(self.frame), + } + } + pub unsafe fn cur_frame(&self) -> *mut crate::ffi::AVFrame { match &self.cpu { Some(cpu) => cpu.current(), @@ -405,6 +485,7 @@ impl Drop for Decoder { fn drop(&mut self) { unsafe { crate::ffi::av_frame_free(&mut self.frame); + crate::ffi::av_frame_free(&mut self.peek_frame); crate::ffi::av_packet_free(&mut self.pkt); crate::ffi::avcodec_free_context(&mut self.dctx); if !self.hwdev.is_null() { diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index bbdab264..f5b02ac2 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -17,7 +17,7 @@ use crate::scene::Scene; // `walk_composited_timeline` / `advance_decoder_to` vivaient ici ; ils sont // portables et servent aussi au pipeline macOS et à `gif_export` — voir // `timeline_walk.rs` pour le pourquoi du déplacement. -use crate::timeline_walk::walk_composited_timeline; +use crate::timeline_walk::{walk_composited_timeline, NextFrameTime}; use anyhow::{anyhow, bail, Result}; use std::collections::HashMap; use std::ffi::{c_void, CString}; @@ -486,6 +486,11 @@ pub(crate) struct Decoder { /// sous le même contrat que D3D11VA (voir `cpu_frames`). `None` en matériel — le /// décodeur rend alors directement la texture du pool D3D11VA, sans copie. cpu: Option, + /// Buffer de lookahead pour `peek_next_time_sec` : symétrique de + /// `pipeline_macos::Decoder::peek_frame`. Cf. là-bas pour la justification. + peek_frame: *mut AVFrame, + /// `true` si `peek_frame` porte une frame décodée en attente de `commit_peek`. + has_peek: bool, } // SAFETY: `Decoder` only owns FFI pointers into FFmpeg's own heap-allocated state, which @@ -555,6 +560,8 @@ impl Decoder { sent_eof: false, cur_pts: None, cpu, + peek_frame: av_frame_alloc(), + has_peek: false, }) } @@ -575,6 +582,11 @@ impl Decoder { /// Repositionne le flux à la première keyframe (t=0) et vide le codec — pour boucler /// la playback sans réallouer les décodeurs. La fixture démarre sur un IDR (§11). pub(crate) unsafe fn rewind(&mut self) -> Result<()> { + // Même règle que `seek_to` : tout repositionnement invalide le peek en attente. + // Il portait sur « la frame d'après l'ancienne position », qui n'a plus de sens + // ici — sans ça le `next()` suivant promouvait une frame décodée avant le rewind, + // avec son ancien `cur_pts`. + self.has_peek = false; averr(av_seek_frame(self.fmt, self.vidx, 0, AVSEEK_FLAG_BACKWARD), "seek")?; avcodec_flush_buffers(self.dctx); self.sent_eof = false; @@ -592,6 +604,8 @@ impl Decoder { /// perf multiclip — un seul seek par frontière de clip, décodage séquentiel ensuite, /// donc le débit par frame ne change pas. Renvoie la frame (ou null à EOF). pub(crate) unsafe fn seek_to(&mut self, seconds: f64) -> Result<*mut AVFrame> { + // Tout seek invalide un éventuel peek en attente — cf. pipeline_macos::Decoder::seek_to. + self.has_peek = false; let tb_sec = self.tb_sec(); // Chemin rapide. Le seek complet ci-dessous jette TOUT l'état du décodeur et repart @@ -711,24 +725,36 @@ impl Decoder { /// Rend la prochaine frame (valide jusqu'au prochain appel), ou null à EOF. pub(crate) unsafe fn next(&mut self) -> Result<*mut AVFrame> { + if self.has_peek { + return self.commit_peek(); + } + if !self.receive_into(self.frame)? { + return Ok(ptr::null_mut()); + } + let pts = (*self.frame).best_effort_timestamp; + self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + match &mut self.cpu { + Some(cpu) => cpu.present(self.frame), + None => Ok(self.frame), + } + } + + /// Décode dans `into` (buffer courant ou de lookahead) jusqu'à obtenir une frame ou + /// l'EOF — cf. `pipeline_macos::Decoder::receive_into` pour la justification. + unsafe fn receive_into(&mut self, into: *mut AVFrame) -> Result { loop { - let r = avcodec_receive_frame(self.dctx, self.frame); + let r = avcodec_receive_frame(self.dctx, into); if r == 0 { - let pts = (*self.frame).best_effort_timestamp; - self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; - return match &mut self.cpu { - Some(cpu) => cpu.present(self.frame), - None => Ok(self.frame), - }; + return Ok(true); } if r == AVERROR_EOF { - return Ok(ptr::null_mut()); + return Ok(false); } if r != AVERROR_EAGAIN { averr(r, "receive_frame")?; } if self.sent_eof { - return Ok(ptr::null_mut()); + return Ok(false); } let rr = av_read_frame(self.fmt, self.pkt); if rr == AVERROR_EOF { @@ -743,12 +769,52 @@ impl Decoder { } } } + + /// Décode la prochaine frame dans le buffer de lookahead et renvoie son temps. + /// Cf. `pipeline_macos::Decoder::peek_next_time_sec`. + pub(crate) unsafe fn peek_next_time_sec(&mut self) -> Result { + if !self.has_peek { + if !self.receive_into(self.peek_frame)? { + return Ok(NextFrameTime::Eof); + } + self.has_peek = true; + } + let pts = (*self.peek_frame).best_effort_timestamp; + let tb_sec = self.tb_sec(); + // Sans pts ni time_base exploitables on ne PEUT pas dire si la frame est due : + // `Unknown`, et non `0.0` — qui passait pour « due » à tous les coups. + Ok(if pts == i64::MIN || tb_sec <= 0.0 { + NextFrameTime::Unknown + } else { + NextFrameTime::At(pts as f64 * tb_sec) + }) + } + + /// Promeut la frame de lookahead au rang de frame courante. Cf. + /// `pipeline_macos::Decoder::commit_peek`. + pub(crate) unsafe fn commit_peek(&mut self) -> Result<*mut AVFrame> { + // `bail!` et non `debug_assert!` : compilée en release, l'assertion disparaissait + // et l'échange promouvait un `AVFrame` jamais rempli, avec un + // `best_effort_timestamp` indéterminé, jusque dans le chemin de présentation. + if !self.has_peek { + anyhow::bail!("commit_peek sans peek_next_time_sec préalable"); + } + std::mem::swap(&mut self.frame, &mut self.peek_frame); + self.has_peek = false; + let pts = (*self.frame).best_effort_timestamp; + self.cur_pts = if pts == i64::MIN { None } else { Some(pts) }; + match &mut self.cpu { + Some(cpu) => cpu.present(self.frame), + None => Ok(self.frame), + } + } } impl Drop for Decoder { fn drop(&mut self) { unsafe { av_frame_free(&mut self.frame); + av_frame_free(&mut self.peek_frame); av_packet_free(&mut self.pkt); avcodec_free_context(&mut self.dctx); av_buffer_unref(&mut self.hwdev); diff --git a/crates/compositor/src/timeline_walk.rs b/crates/compositor/src/timeline_walk.rs index e5146e6c..0e0efdad 100644 --- a/crates/compositor/src/timeline_walk.rs +++ b/crates/compositor/src/timeline_walk.rs @@ -23,23 +23,108 @@ use crate::scene::Scene; use anyhow::Result; use std::collections::HashMap; -/// Avance un décodeur jusqu'au premier pts dans le référentiel écran qui atteint la cible. -/// `timeline_offset_sec` remet les pts webcam dans ce référentiel (`webcam + offset = screen`) : -/// chaque source garde ainsi sa cadence propre au lieu d'être consommée 1:1 avec l'autre. +/// Ce que le décodeur sait de la PROCHAINE frame, sans l'adopter. +/// +/// Un simple `Option` ne suffisait pas : il confondait « pts inexploitable » et +/// « pts = 0 ». `peek_next_time_sec` renvoyait `0.0` quand `best_effort_timestamp` vaut +/// `i64::MIN` (ou que la time_base est nulle), et `0.0` satisfait TOUJOURS la condition +/// d'adoption — un flux sans pts fiable se faisait donc vider frame après frame jusqu'à +/// l'EOF, exactement le défaut que la sémantique de hold est censée corriger, en pire. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum NextFrameTime { + /// pts exploitable de la frame en attente, en secondes. + At(f64), + /// Une frame attend, mais son pts est inexploitable : impossible de dire si elle est + /// due. Le seul repli honnête est d'avancer d'UNE frame puis de rendre la main — + /// c'est le comportement d'avant la sémantique de hold, restreint au flux cassé qui + /// le mérite au lieu d'être la règle générale. + Unknown, + /// Plus aucune frame à décoder. + Eof, +} + +/// Ce que la boucle d'avance doit faire de la frame en attente. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) enum FrameStep { + /// Adopter la frame, puis continuer à chercher. + Commit, + /// Adopter la frame puis s'arrêter (cas `Unknown`, cf. ci-dessus). + CommitAndStop, + /// Ne rien adopter : on tient la frame courante. + Hold, +} + +/// Décision pure de la sémantique de hold — extraite pour être testable sans ffmpeg ni +/// fichier, parce que c'est ici que vivent les cas limites (EOF, pts inconnu, et la +/// frontière exacte « due » vs « pas encore due »). +/// +/// `offset_sec` remet le pts dans le référentiel de la cible (`webcam + offset = écran`). +pub(crate) fn frame_step(next: NextFrameTime, offset_sec: f64, target_sec: f64) -> FrameStep { + match next { + // Plus rien à décoder : on tient la dernière frame connue. C'est ce qui permet à + // un clip dont la dernière frame RÉELLE précède la fin déclarée d'occuper quand + // même toute sa fenêtre — cf. `walk_composited_timeline`. + NextFrameTime::Eof => FrameStep::Hold, + NextFrameTime::Unknown => FrameStep::CommitAndStop, + // `>` et non `>=` : une frame dont le pts vaut EXACTEMENT la cible est due + // (« la dernière frame dont le pts est ≤ t »). + NextFrameTime::At(t) if t + offset_sec > target_sec => FrameStep::Hold, + NextFrameTime::At(_) => FrameStep::Commit, + } +} + +/// Avance un décodeur vers `target_source_time`, sémantique de "hold" : à l'instant t on +/// affiche la DERNIÈRE frame dont le pts est ≤ t, jamais une frame dont le pts est encore à +/// venir. `timeline_offset_sec` remet les pts webcam dans le référentiel écran +/// (`webcam + offset = screen`) : chaque source garde ainsi sa cadence propre au lieu +/// d'être consommée 1:1 avec l'autre. +/// +/// BUG corrigé : l'ancienne version avançait tant que `cur_time_sec() < target`, un pas de +/// `next()` à la fois, et s'arrêtait dès que la frame COURANTE dépassait la cible — mais +/// `next()` saute à la prochaine frame RÉELLEMENT capturée, qui peut être très en avance +/// sur `target` quand la source a un trou (ex. ScreenCaptureKit qui ne livre rien tant que +/// l'écran ne change pas). Un seul `next()` pouvait alors faire passer le décodeur d'un pts +/// proche de la cible à un pts bien après elle, et la condition d'arrêt considérait ça comme +/// "atteint" — la frame FUTURE se retrouvait affichée bien avant son heure. Ici, `next()` +/// n'est plus appelé à l'aveugle : on regarde d'abord le pts de la frame suivante +/// (`peek_next_time_sec`, décodée dans un buffer séparé) et on ne l'adopte +/// (`commit_peek`) que si elle est réellement due ; sinon on continue de tenir la frame +/// courante, aussi longtemps qu'il le faut. +/// +/// CHANGEMENT DE COMPORTEMENT À L'EXPORT, délibéré : à l'EOF cette fonction renvoie +/// désormais `true` (on tient la dernière frame) là où elle renvoyait `false`, ce qui +/// coupait la boucle du clip (`break 'clip_frames`). Un clip dont la fenêtre déclarée +/// dépasse le dernier pts réel — dernière frame légèrement avant `source_end_sec`, ou +/// piste webcam plus courte que l'écran — ne se termine donc plus en avance : il occupe +/// toute sa fenêtre en tenant sa dernière image. C'est ce que l'audio suppose déjà : +/// `on_clip_end` reçoit le nombre de frames du clip et l'audio est étiré sur la durée +/// DÉCLARÉE (`stretch_clip_pcm_by_speed`), donc une vidéo qui s'arrêtait tôt décalait la +/// jonction audio/vidéo du clip suivant. La contrepartie assumée : une source réellement +/// tronquée produit maintenant une image figée jusqu'au bout de sa fenêtre au lieu de +/// s'arrêter net. pub(crate) unsafe fn advance_decoder_to( decoder: &mut Decoder, target_source_time: f64, timeline_offset_sec: f64, ) -> Result { + if decoder.cur_frame().is_null() { + return Ok(false); + } loop { - if decoder.cur_frame().is_null() { - return Ok(false); - } - if decoder.cur_time_sec() + timeline_offset_sec >= target_source_time { - return Ok(true); - } - if decoder.next()?.is_null() { - return Ok(false); + let next = decoder.peek_next_time_sec()?; + match frame_step(next, timeline_offset_sec, target_source_time) { + FrameStep::Hold => return Ok(true), + // La frame adoptée devient la frame courante : si la présentation n'a rien + // produit, la boucle n'a plus d'invariant (elle tournerait sur une frame + // nulle jusqu'à l'EOF) — on rend la main comme le faisait le garde d'entrée. + FrameStep::Commit => { + if decoder.commit_peek()?.is_null() { + return Ok(false); + } + } + FrameStep::CommitAndStop => { + return Ok(!decoder.commit_peek()?.is_null()); + } } } } @@ -217,3 +302,97 @@ pub(crate) unsafe fn walk_composited_timeline( comp.set_timeline_time(None); Ok(frames) } + +#[cfg(test)] +mod tests { + use super::{frame_step, FrameStep, NextFrameTime}; + + /// La cadence de lecture, en une phrase : à 24 fps, une seconde réelle doit adopter 24 + /// frames et pas une de plus. Le bug d'origine (un pas fixe de 1/60 s, une frame par + /// pas) en consommait 60, soit 2,5× trop vite — c'est ce que ce test verrouille. + fn frames_committed_over(fps: f64, window_sec: f64) -> usize { + let mut committed = 0usize; + // La frame 0 est déjà la frame courante : on compte ce qui est ADOPTÉ ensuite. + // Le pts est recalculé depuis un index entier plutôt qu'accumulé, sinon la dérive + // flottante fausse le compte au bout de quelques dizaines de frames. + let mut index = 1u64; + // Une cible qui avance au temps réel, échantillonnée à 60 Hz comme le thread de rendu. + let ticks = (window_sec * 60.0).round() as usize; + for tick in 1..=ticks { + let target = tick as f64 / 60.0; + loop { + let pts = index as f64 / fps; + match frame_step(NextFrameTime::At(pts), 0.0, target) { + FrameStep::Commit => { + committed += 1; + index += 1; + } + _ => break, + } + } + } + committed + } + + #[test] + fn plays_a_24fps_source_at_24_frames_per_second() { + assert_eq!(frames_committed_over(24.0, 1.0), 24); + assert_eq!(frames_committed_over(24.0, 2.0), 48); + } + + #[test] + fn plays_a_60fps_source_at_60_frames_per_second() { + // Le cas qui tombait juste par hasard avant le correctif. + assert_eq!(frames_committed_over(60.0, 1.0), 60); + } + + #[test] + fn plays_a_30fps_source_at_30_frames_per_second() { + assert_eq!(frames_committed_over(30.0, 1.0), 30); + } + + #[test] + fn holds_a_frame_that_is_not_due_yet() { + assert_eq!(frame_step(NextFrameTime::At(0.5), 0.0, 0.4), FrameStep::Hold); + } + + #[test] + fn adopts_a_frame_whose_pts_is_exactly_the_target() { + // « la DERNIÈRE frame dont le pts est ≤ t » : l'égalité est due. + assert_eq!(frame_step(NextFrameTime::At(0.4), 0.0, 0.4), FrameStep::Commit); + } + + #[test] + fn a_sparse_source_holds_across_its_gap() { + // ScreenCaptureKit ne livre rien tant que l'écran ne bouge pas : la frame suivante + // peut être 10 s plus loin. Elle ne doit surtout pas être adoptée à la seconde 1. + assert_eq!(frame_step(NextFrameTime::At(10.0), 0.0, 1.0), FrameStep::Hold); + assert_eq!(frame_step(NextFrameTime::At(10.0), 0.0, 10.0), FrameStep::Commit); + } + + #[test] + fn offset_moves_the_webcam_into_the_screen_clock() { + // webcam + offset = écran : à offset 2 s, une frame webcam à 0.5 s vaut 2.5 s écran. + assert_eq!(frame_step(NextFrameTime::At(0.5), 2.0, 2.4), FrameStep::Hold); + assert_eq!(frame_step(NextFrameTime::At(0.5), 2.0, 2.5), FrameStep::Commit); + } + + #[test] + fn eof_holds_the_last_frame_instead_of_ending_the_clip() { + // Le changement de comportement à l'export, verrouillé : un clip dont la fenêtre + // déclarée dépasse le dernier pts réel occupe toute sa fenêtre en tenant sa + // dernière image, au lieu de s'arrêter net et de décaler l'audio du clip suivant. + assert_eq!(frame_step(NextFrameTime::Eof, 0.0, 1_000.0), FrameStep::Hold); + } + + #[test] + fn an_unusable_pts_advances_exactly_one_frame() { + // Régression : `peek_next_time_sec` renvoyait `0.0` pour un pts inexploitable, et + // `0.0` est toujours ≤ à la cible — le flux se vidait jusqu'à l'EOF d'un seul coup. + assert_eq!(frame_step(NextFrameTime::Unknown, 0.0, 0.0), FrameStep::CommitAndStop); + assert_eq!( + frame_step(NextFrameTime::Unknown, 0.0, 1_000.0), + FrameStep::CommitAndStop + ); + } +} diff --git a/crates/poc-d3d/src/app.rs b/crates/poc-d3d/src/app.rs index 6c49b7d2..5822a615 100644 --- a/crates/poc-d3d/src/app.rs +++ b/crates/poc-d3d/src/app.rs @@ -100,16 +100,24 @@ struct App { } impl App { - /// Compose + affiche la 1re frame, avant l'ouverture de la fenêtre. + /// Compose + affiche la 1re frame, avant l'ouverture de la fenêtre. Cible `INFINITY` : + /// on veut cette toute première frame quel que soit son pts, la notion de "due" n'a pas + /// encore de sens avant le premier tick réel (cf. `Player::step`, sémantique de hold). unsafe fn init_first_frame(&mut self) { let cfg = self.cfgs[self.cur].clone(); - let _ = self.player.step(&self.comp, &cfg); + let _ = self.player.step(&self.comp, &cfg, f64::INFINITY); let _ = self.render(); self.update_ready_status(); self.last = Instant::now(); } /// Cadence 60 fps par horloge murale (accumulateur), avec garde anti-spirale. + /// + /// `self.acc` est une CIBLE de temps source (mis à l'échelle par le temps réel écoulé), + /// pas un compte de frames à décoder — `Player::step` n'adopte une frame que si son pts + /// est réellement dû, sinon il tient la frame courante (hold). Sans ça, ce harnais + /// consommerait une frame réelle par 1/60s de temps réel même quand la source n'en livre + /// pas autant (cf. la doc de `Player::step` côté lib, même bug que la preview Electron). unsafe fn on_tick(&mut self) -> Result<()> { if self.exporting || !self.playing { return Ok(()); @@ -118,19 +126,27 @@ impl App { let dt = (now - self.last).as_secs_f64().min(0.1); self.last = now; self.acc += dt; - let step = 1.0 / 60.0; let cfg = self.cfgs[self.cur].clone(); let mut stepped = false; let mut n = 0; - while self.acc >= step && n < 3 { - if self.player.step(&self.comp, &cfg)? { - stepped = true; + loop { + let before = self.player.screen_time_sec(); + let target = before + self.acc; + if !self.player.step(&self.comp, &cfg, target)? { + break; // rien de dû pour l'instant : `self.acc` reste tel quel. } - self.acc -= step; + stepped = true; + let after = self.player.screen_time_sec(); + self.acc = if after >= before { + (self.acc - (after - before)).max(0.0) + } else { + 0.0 // reboucle sur l'EOF (temps qui recule) : accumulateur remis à zéro. + }; n += 1; - } - if self.acc > step { - self.acc = 0.0; // largue le retard accumulé (fenêtre masquée, etc.) + if n >= 3 { + self.acc = 0.0; // largue le retard accumulé (fenêtre masquée, etc.) + break; + } } if stepped { self.render()?; diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index bbac9765..8c4920fd 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -295,6 +295,10 @@ interface Window { message?: string; error?: string; }>; + getAudioPeaks: ( + filePath: string, + durationSec: number, + ) => Promise; readFileChunk: ( filePath: string, offset: number, @@ -390,6 +394,7 @@ interface Window { transcribe: ( request: import("./stt/transcriptionContract").SttTranscribeRequest, ) => Promise; + cancel: () => Promise; onStatus: ( callback: (event: import("./stt/transcriptionContract").SttStatusEvent) => void, ) => () => void; diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 4baa9ee8..0000ec0e 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -55,6 +55,7 @@ import { LlmConfigStore } from "../ai-edition/llm-config-store"; import { mainLogBuffer } from "../diagnostics/main-log-buffer"; import { mainT } from "../i18n"; import { RECORDINGS_DIR } from "../main"; +import { type AudioPeaksResult, getAudioPeaks } from "../media/audioPeaks"; import { readCursorRecordingFile as readCursorRecordingFileFrom, readCursorSidecar, @@ -3120,6 +3121,31 @@ export function registerIpcHandlers( } }); + // Waveform peaks for a timeline clip, decoded natively (see media/audioPeaks). + // The renderer's own pipelines take ~12s on a 32-minute recording because they + // decode the whole track in Chromium; ffmpeg does the same work in ~2s off the + // UI process, and the result is cached on disk so it is paid once per file. + // `peaks: null` means "no native path available" — the caller falls back to + // its own decoding rather than losing the waveform. + ipcMain.handle( + "get-audio-peaks", + async (_, filePath: string, durationSec: number): Promise => { + try { + // Same approval gate as every other read of a renderer-supplied path. + const normalizedPath = await approveReadableVideoPath(filePath); + if (!normalizedPath) { + return { success: false, message: "File path is not approved" }; + } + const peaks = await getAudioPeaks(normalizedPath, durationSec); + return { success: true, peaks }; + } catch (error) { + // A clip with no audio track lands here. Degrade quietly: the renderer + // draws no waveform, which is correct, and logs its own warning. + return { success: false, message: String(error) }; + } + }, + ); + // Cap renderer-requested chunk sizes so a buggy or compromised renderer // cannot make the main process allocate an arbitrarily large buffer. const MAX_IPC_CHUNK_BYTES = 64 * 1024 * 1024; diff --git a/electron/media/__fixtures__/peaks-sample.m4a b/electron/media/__fixtures__/peaks-sample.m4a new file mode 100644 index 00000000..431172aa Binary files /dev/null and b/electron/media/__fixtures__/peaks-sample.m4a differ diff --git a/electron/media/audioPeaks.test.ts b/electron/media/audioPeaks.test.ts new file mode 100644 index 00000000..fb20b267 --- /dev/null +++ b/electron/media/audioPeaks.test.ts @@ -0,0 +1,100 @@ +// @vitest-environment node +import { existsSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { ffmpegCandidates, peakBlockCount, resolveFfmpeg } from "./audioPeaks"; + +const ROOT = path.resolve(__dirname, "..", ".."); + +describe("peakBlockCount", () => { + it("matches the browser pipelines' block maths", () => { + // Same formula as audioPeaksWorker.ts / streamingAudioPeaks.ts: a clip must + // not change shape depending on which pipeline drew it. + expect(peakBlockCount(10)).toBe(2000); + expect(peakBlockCount(60)).toBe(12000); + // Capped, so a 30-minute recording costs the same DOM/array budget as a + // 2-minute one. + expect(peakBlockCount(1951)).toBe(24000); + expect(peakBlockCount(99999)).toBe(24000); + }); + + it("never returns zero blocks for a sliver of audio", () => { + expect(peakBlockCount(0.001)).toBe(1); + }); +}); + +describe("ffmpeg resolution", () => { + it("prefers the shared build the installer actually ships", () => { + const candidates = ffmpegCandidates(ROOT); + const shared = candidates.findIndex((c) => c.endsWith("ffmpeg-shared.exe")); + const vendorTree = candidates.findIndex((c) => c.includes("lgpl-shared")); + if (process.platform === "win32") { + expect(shared).toBeGreaterThanOrEqual(0); + // The static ffmpeg.exe is excluded from the Windows installer + // ("!win32-*/ffmpeg.exe"), so resolving to it would work in dev and fail + // in production. It must not be a candidate at all. + expect( + candidates.some((c) => c.endsWith(`bin${path.sep}win32-x64${path.sep}ffmpeg.exe`)), + ).toBe(false); + expect(shared).toBeLessThan(vendorTree); + } + }); + + it("honours the env override first", () => { + process.env.OPENSCREEN_FFMPEG_PATH = "/custom/ffmpeg"; + try { + expect(ffmpegCandidates(ROOT)[0]).toBe("/custom/ffmpeg"); + } finally { + process.env.OPENSCREEN_FFMPEG_PATH = undefined; + } + }); + + it("returns null rather than throwing when nothing is staged", () => { + expect(resolveFfmpeg(path.join(ROOT, "does", "not", "exist"))).toBeNull(); + }); +}); + +// Only runs where the binary is actually staged; skipped elsewhere rather than +// failing a checkout that has not run scripts/fetch-ffmpeg.mjs. +const staged = resolveFfmpeg(ROOT); +describe.runIf(staged)("decoding a real file", () => { + it("produces peaks in range, with real signal in them", async () => { + // A synthetic 5s 440 Hz tone from ffmpeg's own lavfi source — no user + // recording in the repo, and a signal whose shape is known rather than + // "whatever this capture happened to contain". + const fixture = path.join(ROOT, "electron", "media", "__fixtures__", "peaks-sample.m4a"); + if (!existsSync(fixture)) return; + const { getAudioPeaks } = await import("./audioPeaks"); + const peaks = await getAudioPeaks(fixture, 5); + expect(peaks).not.toBeNull(); + if (!peaks) return; + expect(peaks.length).toBe(peakBlockCount(5) * 2); + // [min, max] pairs, both inside [-1, 1], min <= 0 <= max (the folder starts + // each block at the silence baseline, like the worker does). + for (let i = 0; i < peaks.length; i += 2) { + expect(peaks[i]).toBeLessThanOrEqual(0); + expect(peaks[i + 1]).toBeGreaterThanOrEqual(0); + expect(peaks[i]).toBeGreaterThanOrEqual(-1); + expect(peaks[i + 1]).toBeLessThanOrEqual(1); + } + // Not all silence — otherwise everything above would pass on a pipeline + // that returned a zeroed array. + // + // The bound is tight rather than "> 0" because loose is the same as + // absent here: the mistakes worth catching are all scale errors — int16 + // divided by 65536 instead of 32768, a stereo downmix halving the signal, + // a block whose samples never get compared — and every one of them is a + // factor of two. 0.214 is what ffmpeg itself decodes this fixture to + // (verified with `-f s16le` straight to a file), so this asserts the fold + // agrees with the decoder rather than restating the fixture's nominal + // amplitude, which lavfi's volume filter does not actually deliver. + let mn = 0; + let mx = 0; + for (const v of peaks) { + if (v < mn) mn = v; + if (v > mx) mx = v; + } + expect(mx).toBeCloseTo(0.214, 1); + expect(mn).toBeCloseTo(-0.205, 1); + }, 120_000); +}); diff --git a/electron/media/audioPeaks.ts b/electron/media/audioPeaks.ts new file mode 100644 index 00000000..056cab54 --- /dev/null +++ b/electron/media/audioPeaks.ts @@ -0,0 +1,288 @@ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { app } from "electron"; + +/** + * Waveform peaks for the timeline, computed in the main process with ffmpeg. + * + * WHY THIS EXISTS. The renderer had two pipelines and both decode the whole + * audio track in Chromium, which is the entire cost. Measured head-to-head on a + * 32-minute screen recording (68 MB): + * + * decodeAudioData (whole track) 12003 ms, 714 MB resident + * WebCodecs chunk-by-chunk streaming 12259 ms, ~192 kB resident + * ffmpeg -vn -ac 1 -ar 16000 ~2000 ms, nothing resident + * + * The two browser paths differ only in memory; ffmpeg is ~6x faster than both + * because a native AAC decoder is simply faster than Chromium's, and it runs + * off the UI process entirely. The peaks then get cached on disk, so the cost + * is paid once per recording rather than once per session. + * + * ponytail: the CLI, not libav bindings in the compositor addon. The addon + * would avoid a process spawn — worth ~20 ms against a ~2000 ms decode — for a + * new Rust surface, an N-API entry point and a build story on three platforms. + * Revisit only if peaks ever need to share a decode with something else. + */ + +/** IPC reply. `peaks: null` on success means "no native ffmpeg here" — a + * fallback signal, not a failure. */ +export interface AudioPeaksResult { + success: boolean; + peaks?: Float32Array | null; + message?: string; +} + +/** PCM the peaks are computed from. Mono (ffmpeg downmixes) so channels are + * already averaged, and 16 kHz because peak buckets are at most 200/s: that + * still leaves 80 samples per bucket, far more than a min/max needs. */ +const PCM_RATE = 16_000; + +/** Matches `audioPeaksWorker.ts` and `streamingAudioPeaks.ts` so all three + * render identically — a clip must not change shape with the pipeline. */ +const MAX_PEAK_BLOCKS = 24_000; +const PEAK_BLOCKS_PER_SEC = 200; + +/** A recording whose audio takes longer than this to decode is not a recording, + * it is a wedged ffmpeg. ~30x the worst measured case. */ +const DECODE_TIMEOUT_MS = 60_000; + +/** + * Where to find an ffmpeg that actually exists at runtime, in priority order. + * + * Note the Windows shape: `electron-builder.json5` deliberately excludes the + * STATIC `ffmpeg.exe` (109 MB) from the installer, so resolving to it would + * work in dev and fail in production — the exact class of bug that is invisible + * until someone runs the packaged app. The SHARED build is 1 MB and links the + * same `av*.dll` set the compositor already ships, so that is the one that gets + * packaged (see the `filter` in electron-builder.json5) and the one preferred + * here. + */ +export function ffmpegCandidates(here: string = process.cwd()): string[] { + const tag = `${process.platform}-${process.arch}`; + const exe = process.platform === "win32" ? "ffmpeg.exe" : "ffmpeg"; + const env = process.env.OPENSCREEN_FFMPEG_PATH?.trim(); + const roots: string[] = []; + // `app` is absent when this module is imported by a test. + const appPath = (() => { + try { + return typeof app?.getAppPath === "function" ? app.getAppPath() : null; + } catch { + return null; + } + })(); + if (appPath) roots.push(appPath); + if (process.resourcesPath) roots.push(process.resourcesPath); + roots.push(here); + + const names = + process.platform === "win32" + ? [ + // Staged flat by fetch-ffmpeg.mjs, beside the av*.dll set it links + // against — the only ffmpeg the Windows installer carries. + "ffmpeg-shared.exe", + // The unpacked vendor tree, present in a dev checkout that has not + // re-run the fetch script. + path.join("ffmpeg-n8.1.2-win64-lgpl-shared", "bin", exe), + ] + : [exe]; + return [ + ...(env ? [env] : []), + ...roots.flatMap((root) => + names.map((n) => path.join(root, "electron", "native", "bin", tag, n)), + ), + ]; +} + +let cachedFfmpeg: string | null | undefined; + +/** First candidate that exists, or null when none does (callers fall back). */ +export function resolveFfmpeg(here?: string): string | null { + if (cachedFfmpeg !== undefined && here === undefined) return cachedFfmpeg; + const found = ffmpegCandidates(here).find((p) => existsSync(p)) ?? null; + if (here === undefined) cachedFfmpeg = found; + return found; +} + +/** Number of min/max blocks for a clip of `durationSec`. */ +export function peakBlockCount(durationSec: number): number { + return Math.min(MAX_PEAK_BLOCKS, Math.max(1, Math.ceil(durationSec * PEAK_BLOCKS_PER_SEC))); +} + +/** + * Folds a stream of mono int16 samples into `[min0, max0, min1, max1, ...]`. + * + * Incremental on purpose: the PCM for a 32-minute recording is 62 MB and never + * needs to exist all at once. Kept as a class rather than a closure so it holds + * only the counters it needs, not an enclosing scope. + */ +class PeakFolder { + private readonly peaks: Float32Array; + private readonly samplesPerBlock: number; + private sampleIndex = 0; + /** int16 straddling a chunk boundary: its low byte arrived, its high byte did not. */ + private pendingLowByte: number | null = null; + + constructor( + private readonly blocks: number, + totalSamples: number, + ) { + this.peaks = new Float32Array(blocks * 2); + this.samplesPerBlock = Math.max(1, totalSamples / blocks); + } + + push(chunk: Buffer): void { + let offset = 0; + if (this.pendingLowByte !== null && chunk.length > 0) { + this.addSample((chunk[0] << 8) | this.pendingLowByte); + this.pendingLowByte = null; + offset = 1; + } + const end = chunk.length - ((chunk.length - offset) % 2); + for (let i = offset; i < end; i += 2) { + this.addSample(chunk.readInt16LE(i)); + } + if (end < chunk.length) this.pendingLowByte = chunk[end]; + } + + private addSample(raw: number): void { + // readInt16LE is signed; the hand-assembled straddling sample is not. + const signed = raw > 32767 ? raw - 65536 : raw; + const value = signed / 32768; + const block = Math.min(this.blocks - 1, Math.floor(this.sampleIndex / this.samplesPerBlock)); + const lo = block * 2; + if (value < this.peaks[lo]) this.peaks[lo] = value; + if (value > this.peaks[lo + 1]) this.peaks[lo + 1] = value; + this.sampleIndex++; + } + + result(): Float32Array { + return this.peaks; + } +} + +/** Runs ffmpeg and folds its PCM straight into peaks. Never buffers the audio. */ +async function decodePeaks( + ffmpeg: string, + filePath: string, + durationSec: number, +): Promise { + const blocks = peakBlockCount(durationSec); + const folder = new PeakFolder(blocks, durationSec * PCM_RATE); + const child = spawn( + ffmpeg, + [ + "-hide_banner", + "-loglevel", + "error", + "-i", + filePath, + "-vn", + "-ac", + "1", + "-ar", + String(PCM_RATE), + "-f", + "s16le", + "-", + ], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + + return new Promise((resolve, reject) => { + let stderr = ""; + const timer = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`ffmpeg timed out after ${DECODE_TIMEOUT_MS}ms on ${filePath}`)); + }, DECODE_TIMEOUT_MS); + + child.stdout.on("data", (c: Buffer) => folder.push(c)); + child.stderr.on("data", (c: Buffer) => { + stderr = (stderr + c.toString()).slice(-2048); + }); + child.once("error", (err) => { + clearTimeout(timer); + reject(err); + }); + child.once("close", (code) => { + clearTimeout(timer); + // A file with no audio track exits non-zero. That is not an error worth + // surfacing — it is a clip that legitimately has no waveform — but it is + // the caller's job to decide, so it still rejects, with the reason. + if (code !== 0) { + reject(new Error(`ffmpeg exited ${code}${stderr ? `: ${stderr.trim()}` : ""}`)); + return; + } + resolve(folder.result()); + }); + }); +} + +/** + * Cache key: path plus size plus mtime. A recording is immutable in practice, + * but keying on identity alone would serve stale peaks for a re-encoded or + * replaced file, and that failure is silent and confusing. + */ +async function cacheKey(filePath: string): Promise { + const info = await stat(filePath); + return createHash("sha1") + .update(`${filePath}:${info.size}:${info.mtimeMs}`) + .digest("hex") + .slice(0, 32); +} + +/** Null outside Electron (tests, any headless use): decoding still works, it + * just is not cached, rather than the whole call failing on a missing `app`. */ +function cacheDir(): string | null { + try { + return typeof app?.getPath === "function" + ? path.join(app.getPath("userData"), "audio-peaks") + : null; + } catch { + return null; + } +} + +/** + * Peaks for `filePath`, from disk when they have been computed before. + * + * The cache is what makes this feel instant: peaks for a given recording never + * change, so the ~2s decode is paid once ever rather than once per session. + * Returns null when no ffmpeg is available, so the renderer can fall back to + * its own pipelines instead of losing the waveform. + */ +export async function getAudioPeaks( + filePath: string, + durationSec: number, +): Promise { + const ffmpeg = resolveFfmpeg(); + if (!ffmpeg || !durationSec || durationSec <= 0) return null; + + const dir = cacheDir(); + const cachePath = dir ? path.join(dir, `${await cacheKey(filePath)}.f32`) : null; + if (cachePath) { + try { + const cached = await readFile(cachePath); + // A Buffer's memory may not be 4-byte aligned and its byteOffset is + // almost never 0 — copy rather than viewing it in place. + return new Float32Array( + cached.buffer.slice(cached.byteOffset, cached.byteOffset + cached.byteLength), + ); + } catch { + // Not cached yet. + } + } + + const peaks = await decodePeaks(ffmpeg, filePath, durationSec); + if (cachePath && dir) { + try { + await mkdir(dir, { recursive: true }); + await writeFile(cachePath, Buffer.from(peaks.buffer)); + } catch { + // A cache we cannot write is a slower next launch, not a failure. + } + } + return peaks; +} diff --git a/electron/preload.ts b/electron/preload.ts index f3b44d57..82dd2b40 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -284,6 +284,10 @@ contextBridge.exposeInMainWorld("electronAPI", { getReadableFileInfo: (filePath: string) => { return ipcRenderer.invoke("get-readable-file-info", filePath); }, + /** Native waveform peaks, disk-cached. See electron/media/audioPeaks.ts. */ + getAudioPeaks: (filePath: string, durationSec: number) => { + return ipcRenderer.invoke("get-audio-peaks", filePath, durationSec); + }, readFileChunk: (filePath: string, offset: number, length: number) => { return ipcRenderer.invoke("read-file-chunk", filePath, offset, length); }, @@ -422,6 +426,8 @@ contextBridge.exposeInMainWorld("electronAPI", { transcribe: (request: SttTranscribeRequest): Promise => { return ipcRenderer.invoke("stt:transcribe", request) as Promise; }, + /** Stop the running transcription at its next chunk boundary. */ + cancel: (): Promise => ipcRenderer.invoke("stt:cancel") as Promise, onStatus: (callback: (event: SttStatusEvent) => void) => { const listener = (_event: unknown, payload: SttStatusEvent) => callback(payload); ipcRenderer.on("stt:status", listener); diff --git a/electron/stt/chunking.test.ts b/electron/stt/chunking.test.ts new file mode 100644 index 00000000..db19cbcf --- /dev/null +++ b/electron/stt/chunking.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { planChunks } from "./chunking"; + +const RATE = 16_000; + +/** Loud tone with silent gaps punched in at the given [startSec, endSec) ranges. */ +function toneWithSilences(durationSec: number, silences: [number, number][]): Float32Array { + const samples = new Float32Array(Math.round(durationSec * RATE)); + for (let i = 0; i < samples.length; i++) { + samples[i] = Math.sin((i / RATE) * 2 * Math.PI * 440); + } + for (const [from, to] of silences) { + samples.fill(0, Math.round(from * RATE), Math.round(to * RATE)); + } + return samples; +} + +describe("planChunks", () => { + it("covers the whole buffer with contiguous chunks", () => { + const samples = toneWithSilences(25, []); + const chunks = planChunks(samples, RATE, { targetSec: 10, searchSec: 1 }); + expect(chunks[0].startSample).toBe(0); + expect(chunks[chunks.length - 1].endSample).toBe(samples.length); + for (let i = 1; i < chunks.length; i++) { + expect(chunks[i].startSample).toBe(chunks[i - 1].endSample); + } + }); + + it("cuts inside a pause rather than on the fixed grid", () => { + // Pause at 9.5-9.9s: the ideal 10s boundary should be pulled back into it. + const samples = toneWithSilences(25, [ + [9.5, 9.9], + [19.4, 19.8], + ]); + const chunks = planChunks(samples, RATE, { targetSec: 10, searchSec: 1 }); + const cutSec = chunks[0].endSample / RATE; + expect(cutSec).toBeGreaterThanOrEqual(9.5); + expect(cutSec).toBeLessThan(9.9); + }); + + it("returns a single chunk when the recording is shorter than the target", () => { + const samples = toneWithSilences(5, []); + expect(planChunks(samples, RATE, { targetSec: 120 })).toEqual([ + { startSample: 0, endSample: samples.length }, + ]); + }); + + it("keeps the target when every frame ties, on a fully silent buffer", () => { + // Every frame scores exactly 0, so the tie-break is what decides. Keeping + // the earliest one pulled every cut back to `ideal - searchSec` — 5s chunks + // here instead of 10s, i.e. twice the requests and twice the seams, on the + // audio most likely to tie (a muted track, a gap between takes). + const samples = new Float32Array(60 * RATE); + const chunks = planChunks(samples, RATE, { targetSec: 10, searchSec: 5 }); + expect(chunks.map((c) => c.endSample / RATE)).toEqual([10, 20, 30, 40, 50, 60]); + for (const chunk of chunks) { + expect(chunk.endSample).toBeGreaterThan(chunk.startSample); + } + }); + + it("handles an empty buffer", () => { + expect(planChunks(new Float32Array(0), RATE)).toEqual([]); + }); + + it("makes progress even when the target is shorter than one energy frame", () => { + // Degenerate but reachable through the options: below one frame the scan has + // nothing to measure, and the boundary must still move or the loop spins. + const chunks = planChunks(new Float32Array(RATE), RATE, { targetSec: 0.001 }); + for (const chunk of chunks) expect(chunk.endSample).toBeGreaterThan(chunk.startSample); + expect(chunks[chunks.length - 1].endSample).toBe(RATE); + }); +}); diff --git a/electron/stt/chunking.ts b/electron/stt/chunking.ts new file mode 100644 index 00000000..5f36251d --- /dev/null +++ b/electron/stt/chunking.ts @@ -0,0 +1,132 @@ +/** + * Splits a long recording into inference-sized chunks for the STT pipeline. + * + * Why chunk at all: whisper-stt-server answers a `/inference` request only once + * it has transcribed the WHOLE upload, so a 30-minute recording was one ~10 + * minute request with no progress and no recovery — one hiccup lost everything + * (and undici's 300s `headersTimeout` killed it outright before it ever + * finished). Per-chunk requests give the caller a progress signal, a retry unit, + * and requests short enough that no transport timeout is in play. + * + * Where the cut lands matters: slicing on a fixed grid cuts mid-word, and + * whisper then mis-transcribes both halves. So the boundary is nudged to the + * quietest 20ms frame within a search window around the ideal position — a + * pause between words in practice. + * + * ponytail: energy minimum, not a real VAD. whisper.cpp ships a Silero VAD, but + * it lives behind the server's own `--vad` flag and would run per REQUEST — it + * can't tell us where to cut BEFORE we upload. A plain RMS scan over a few + * seconds is enough to find a pause and costs nothing. If a recording is so + * dense that no pause exists in the window, the cut lands at the quietest point + * anyway and one word may be split; upgrade path is an overlap + de-duplication + * pass on the seam, which is a lot more code than it is worth today. + */ + +/** One chunk of the source buffer: `[startSample, endSample)`. */ +export interface SttChunkPlan { + startSample: number; + /** Exclusive. */ + endSample: number; +} + +/** Energy is measured over frames this long; a cut lands on a frame boundary. */ +const FRAME_MS = 20; + +export interface PlanChunksOptions { + /** Ideal chunk length. Shorter = smoother progress, more per-request overhead. */ + targetSec?: number; + /** How far on either side of the ideal boundary to hunt for a pause. */ + searchSec?: number; +} + +/** + * 90s is a compromise between three pressures: progress granularity (the bar + * only moves once a chunk lands), whisper's own quality (it decodes in 30s + * windows and loses cross-chunk context at every seam, so more seams is worse), + * and `whisperServer`'s 280s per-request ceiling — 90s of audio has to + * transcribe in under that on the SLOWEST machine we care about (~0.3x realtime; + * this Vulkan box does 3.1x, i.e. ~29s per chunk). + */ +const DEFAULT_TARGET_SEC = 90; + +/** + * Index of the quietest frame start in `[from, to)`, breaking ties toward the + * frame nearest `preferred`. Both bounds are clamped by the caller; returns + * `from` when the range holds less than one full frame. + * + * The tie-break is not decoration. Digital silence — a muted track, a gap + * between takes — makes every frame in the window score exactly 0, and keeping + * the first one would pull every boundary back to `from`, i.e. shorten every + * chunk by the whole search window (90s → 87s by default, and 10s → 5s in the + * silent test case). That is extra requests and extra seams bought for nothing, + * against the very context loss `DEFAULT_TARGET_SEC` is sized to limit. + */ +function quietestFrameStart( + samples: Float32Array, + from: number, + to: number, + frameSamples: number, + preferred: number, +): number { + let bestStart = from; + let bestEnergy = Number.POSITIVE_INFINITY; + for (let start = from; start + frameSamples <= to; start += frameSamples) { + let energy = 0; + for (let i = start; i < start + frameSamples; i++) { + energy += samples[i] * samples[i]; + } + if ( + energy < bestEnergy || + (energy === bestEnergy && Math.abs(start - preferred) < Math.abs(bestStart - preferred)) + ) { + bestEnergy = energy; + bestStart = start; + } + } + return bestStart; +} + +/** + * Plan the chunk boundaries for `samples`. Chunks are contiguous and cover the + * whole buffer: `chunks[0].startSample === 0`, each `endSample` is the next + * `startSample`, and the last one ends at `samples.length`. + */ +export function planChunks( + samples: Float32Array, + sampleRate: number, + options: PlanChunksOptions = {}, +): SttChunkPlan[] { + if (samples.length === 0 || sampleRate <= 0) return []; + const frameSamples = Math.max(1, Math.round((FRAME_MS / 1000) * sampleRate)); + // One frame is the floor: a target shorter than the unit the scan works in + // would put the ideal boundary BEFORE the earliest legal cut, which is the + // only way the loop below could fail to make progress. + const targetSamples = Math.max( + frameSamples, + Math.round((options.targetSec ?? DEFAULT_TARGET_SEC) * sampleRate), + ); + const searchSamples = Math.max(0, Math.round((options.searchSec ?? 3) * sampleRate)); + + const chunks: SttChunkPlan[] = []; + let start = 0; + while (start < samples.length) { + const ideal = start + targetSamples; + // Last chunk: what's left is at most one target long, so there's nothing to cut. + if (ideal >= samples.length) { + chunks.push({ startSample: start, endSample: samples.length }); + break; + } + // The search window never reaches back to `start` (a zero-length chunk would + // loop forever) and never past the end of the buffer. + const from = Math.max(start + frameSamples, ideal - searchSamples); + const to = Math.min(samples.length, ideal + searchSamples); + // No clamping needed on either side: `ideal >= from` because `targetSamples` + // is at least one frame, and every value the scan can return lies in + // `[from, to)` — so the cut is always past `start` and inside the buffer. + const endSample = + to > from ? quietestFrameStart(samples, from, to, frameSamples, ideal) : ideal; + chunks.push({ startSample: start, endSample }); + start = endSample; + } + return chunks; +} diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts index df94fc10..ad0eaf97 100644 --- a/electron/stt/index.test.ts +++ b/electron/stt/index.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { planChunks } from "./chunking"; import { _resetSttManagerForTests, SttManager } from "./index"; import type { SttStatusEvent, SttTranscribeResponse } from "./transcriptionContract"; @@ -87,6 +88,112 @@ describe("SttManager", () => { expect(fakeWhisperServer.transcribe).toHaveBeenCalledOnce(); }); + it("splits a long recording and shifts each chunk's timestamps to absolute time", async () => { + // Every chunk reports the same relative segment at 1.0s; correct merging + // turns those into one absolute timestamp per chunk start. + fakeWhisperServer.transcribe.mockResolvedValue({ + segments: [{ text: "hello", startSec: 1, endSec: 1.5 }], + wordSegments: [{ word: "hello", startSec: 1, endSec: 1.5 }], + detectedLanguage: "en", + backend: "whispercpp-cpu", + }); + const samples = new Float32Array(200 * 16000); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + const result = await mgr.transcribe({ samples, language: "en" }); + + const expectedOffsets = planChunks(samples, 16000).map((c) => c.startSample / 16000); + expect(expectedOffsets.length).toBeGreaterThan(1); + expect(fakeWhisperServer.transcribe).toHaveBeenCalledTimes(expectedOffsets.length); + expect(result.segments.map((s) => s.startSec)).toEqual(expectedOffsets.map((o) => o + 1)); + expect(result.wordSegments.map((w) => w.startSec)).toEqual(expectedOffsets.map((o) => o + 1)); + }); + + it("reports monotonic progress that ends on the full duration", async () => { + const sink = vi.fn<(e: SttStatusEvent) => void>(); + const samples = new Float32Array(200 * 16000); + const mgr = new SttManager(); + await mgr.init({ statusSink: sink, modelsBaseDir: "/tmp/fake-stt-models" }); + sink.mockClear(); + await mgr.transcribe({ samples, language: "en" }); + + const progress = sink.mock.calls + .map(([event]) => event) + .filter((event) => event.completedSec !== undefined); + expect(progress[0].completedSec).toBe(0); + expect(progress[progress.length - 1].completedSec).toBe(200); + for (const event of progress) expect(event.totalSec).toBe(200); + for (let i = 1; i < progress.length; i++) { + expect(progress[i].completedSec).toBeGreaterThan(progress[i - 1].completedSec ?? -1); + } + }); + + // Both spellings of "detect it for me". `"auto"` is the one the request + // contract documents, and it is truthy — which is exactly how it used to slip + // past the pin and let every chunk detect its own language. + it.each([ + ["omitted", undefined] as const, + ["auto", "auto"] as const, + ])("pins later chunks to the language detected on the first one (%s)", async (_label, language) => { + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + await mgr.transcribe({ samples: new Float32Array(200 * 16000), language }); + const languages = fakeWhisperServer.transcribe.mock.calls.map(([req]) => req.language); + expect(languages[0]).toBeUndefined(); + expect(languages.slice(1).every((l) => l === "en")).toBe(true); + }); + + it("retries a failed chunk instead of losing the whole transcription", async () => { + fakeWhisperServer.transcribe.mockRejectedValueOnce(new Error("helper died")).mockResolvedValue({ + segments: [{ text: "hello", startSec: 0, endSec: 0.5 }], + wordSegments: [{ word: "hello", startSec: 0, endSec: 0.5 }], + detectedLanguage: "en", + backend: "whispercpp-cpu", + }); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + const result = await mgr.transcribe({ samples: new Float32Array(16000), language: "en" }); + expect(result.segments).toHaveLength(1); + // start() again on the retry — the usual cause is a dead helper. + expect(fakeWhisperServer.start.mock.calls.length).toBeGreaterThan(1); + }); + + it("fails the request when a chunk never succeeds, saying how far it got", async () => { + fakeWhisperServer.transcribe.mockRejectedValue(new Error("helper wedged")); + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + // 200s in, the failure is on chunk 2 of 3 — "Transcription failed" alone + // tells the user nothing about a recording this long. + await expect(mgr.transcribe({ samples: new Float32Array(200 * 16000) })).rejects.toThrow( + /transcription failed \d+s into a 200s recording \(chunk 1\/3\).*helper wedged/, + ); + }); + + it("stops at the next chunk boundary when cancelled", async () => { + const mgr = new SttManager(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + // Cancel lands while the first chunk is in flight — the loop must not go on + // to the remaining ones, which is what left "regenerate" waiting on a run + // nobody wanted any more. + fakeWhisperServer.transcribe.mockImplementation(async () => { + mgr.cancel(); + return { + segments: [], + wordSegments: [], + detectedLanguage: "en", + backend: "whispercpp-cpu" as const, + }; + }); + const samples = new Float32Array(300 * 16000); + expect(planChunks(samples, 16000).length).toBeGreaterThan(1); + + const error = await mgr.transcribe({ samples }).catch((e: unknown) => e); + // `AbortError` by name, so the renderer treats it as "the user asked" and + // drops the job silently instead of toasting an engine failure. + expect((error as Error).name).toBe("AbortError"); + expect(fakeWhisperServer.transcribe).toHaveBeenCalledOnce(); + }); + it("shutdown() stops whisper-stt-server", async () => { const mgr = new SttManager(); await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); @@ -114,12 +221,25 @@ describe("SttManager", () => { expect(fakeWhisperServer.start).toHaveBeenCalledOnce(); }); - it("setStatusSink replaces the previous sink (last call wins)", () => { + it("fans status out to every sink, and detaching one leaves the others", async () => { const mgr = new SttManager(); - const a = vi.fn(); - const b = vi.fn(); - mgr.setStatusSink(a); - mgr.setStatusSink(b); - expect(mgr.getStatusSink()).toBe(b); + const a = vi.fn<(e: SttStatusEvent) => void>(); + const b = vi.fn<(e: SttStatusEvent) => void>(); + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + const detachA = mgr.addStatusSink(a); + mgr.addStatusSink(b); + await mgr.transcribe({ samples: new Float32Array(16000), language: "en" }); + expect(a).toHaveBeenCalled(); + expect(b).toHaveBeenCalled(); + + // The whole point of the Set. Two overlapping IPC requests each attach a + // sink; when the first finishes and detaches, the second must keep getting + // its own progress instead of falling silent for the rest of its run. + detachA(); + a.mockClear(); + b.mockClear(); + await mgr.transcribe({ samples: new Float32Array(16000), language: "en" }); + expect(a).not.toHaveBeenCalled(); + expect(b).toHaveBeenCalled(); }); }); diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 641a3abe..aaa59718 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -1,10 +1,13 @@ import path from "node:path"; import { app, type IpcMain } from "electron"; +import { planChunks } from "./chunking"; import { ensureModels, modelPaths } from "./modelManager"; import type { + SttPhraseSegment, SttStatusEvent, SttTranscribeRequest, SttTranscribeResponse, + SttWordSegment, } from "./transcriptionContract"; import { WhisperServerManager } from "./whisperServer"; @@ -13,17 +16,50 @@ import { WhisperServerManager } from "./whisperServer"; * * Workflow: * 1. `init()` spawns `whisper-stt-server` (or queues the call if it's busy). - * 2. `transcribe()` proxies the renderer's `Float32Array` through - * whisper-stt-server's HTTP `/inference`, which returns both phrase- and - * word-level segments in one pass (see whisperServer.ts). Word - * timestamps come from whisper.cpp's native DTW token timestamps - * (`t_dtw`, SMALL aheads preset, `flash_attn = false`), see + * 2. `transcribe()` splits the renderer's `Float32Array` into chunks + * (`chunking.ts`) and runs each through whisper-stt-server's HTTP + * `/inference`, which returns both phrase- and word-level segments in one + * pass (see whisperServer.ts). Word timestamps come from whisper.cpp's + * native DTW token timestamps (`t_dtw`, SMALL aheads preset, + * `flash_attn = false`), see * technical-documentation/architecture/transcription-and-captions.md § Decision rationale. * 3. `shutdown()` tears down on app quit. * - * Status events fan out via `statusSink` so the renderer can drive its + * Status events fan out to every attached sink so the renderer can drive its * "loading model" / "transcribing" indicator. + * + * Why chunked rather than one request: a 30-minute recording took ~10 minutes + * in a single `/inference` call — no progress to show, no way to recover from a + * transient failure without redoing everything, and long enough that the HTTP + * client's own header timeout killed it before whisper ever answered. Chunks + * turn that into a progress signal, a retry unit, and — via `cancel()` — the + * only point where a run in flight can be stopped at all. + * + * Chunks run SEQUENTIALLY, and that is a measured choice, not an omission: + * whisper-stt-server holds a single model context, so concurrent `/inference` + * calls don't just serialize — they get SLOWER. Two 120s chunks took 76.9s one + * after the other and 144.1s fired together (0.53x, i.e. ~1.9x slower) on this + * Vulkan backend. A client-side worker pool is therefore a pessimisation. Real + * parallelism would need several server processes, each with its own copy of + * the model resident on the GPU; that trade (VRAM + spawn cost per worker) is + * worth revisiting only if a much smaller model ever becomes the default. + */ + +/** The renderer always sends mono 16 kHz (see `extractMono16kFromVideoUrl`). */ +const SAMPLE_RATE = 16_000; + +/** Attempts per chunk before the whole transcription fails. */ +const CHUNK_ATTEMPTS = 3; + +/** + * `AbortError` by name so the renderer's `isAbortError` recognizes it as "the + * user asked for this" rather than an engine failure worth a toast. */ +function cancelledError(): Error { + const error = new Error("Transcription cancelled"); + error.name = "AbortError"; + return error; +} export interface SttManagerInitOptions { statusSink?: (event: SttStatusEvent) => void; @@ -34,21 +70,48 @@ export interface SttManagerInitOptions { export class SttManager { private readonly server = new WhisperServerManager(); private modelsBaseDir: string | null = null; - private statusSink: ((event: SttStatusEvent) => void) | null = null; + private readonly statusSinks = new Set<(event: SttStatusEvent) => void>(); private initPromise: Promise | null = null; + /** Kept from `prepare()` so a chunk retry can respawn a helper that died mid-run. */ + private modelPath: string | null = null; + /** + * Bumped by `cancel()`. The chunk loop compares it against the value it + * captured on entry, so a cancel that lands after a new run started cannot + * kill that new run. + */ + private cancelEpoch = 0; - /** Wire a sink for the renderer status channel. */ - setStatusSink(sink: ((event: SttStatusEvent) => void) | null): void { - this.statusSink = sink; + /** + * Attach a sink for the renderer status channel; returns its detach function. + * + * A SET rather than one slot: this used to be a single field that each IPC + * invocation saved and restored, so with two overlapping transcriptions the + * first to finish restored the sink captured at ITS start and left the other + * one emitting into nothing — no progress for the rest of its run, which + * reads as a hang. + */ + addStatusSink(sink: (event: SttStatusEvent) => void): () => void { + this.statusSinks.add(sink); + return () => { + this.statusSinks.delete(sink); + }; } - /** Read the currently-installed status sink (mostly for tests). */ - getStatusSink(): ((event: SttStatusEvent) => void) | null { - return this.statusSink; + private emit(event: SttStatusEvent): void { + for (const sink of this.statusSinks) sink(event); } - private emit(event: SttStatusEvent): void { - this.statusSink?.(event); + /** + * Stop the in-flight transcription at the next chunk boundary. + * + * ponytail: one epoch for the whole manager, not a handle per request. The + * pipeline runs one transcription at a time by construction (the renderer's + * queue serializes, and `WhisperServerManager` single-flights on top), so + * "cancel what is running" is the only question anyone can ask. Per-request + * tokens the day two recordings can transcribe at once. + */ + cancel(): void { + this.cancelEpoch++; } /** @@ -56,7 +119,7 @@ export class SttManager { * means the second caller just awaits the same completion. */ init(options: SttManagerInitOptions = {}): Promise { - if (options.statusSink) this.statusSink = options.statusSink; + if (options.statusSink) this.addStatusSink(options.statusSink); if (options.modelsBaseDir) this.modelsBaseDir = options.modelsBaseDir; if (!this.initPromise) { // A REJECTED init must not be cached. `prepare()` downloads a 253 MB @@ -97,23 +160,138 @@ export class SttManager { }); const paths = modelPaths(modelsDir); + this.modelPath = paths.whisper; await this.server.start({ modelPath: paths.whisper }); this.emit({ phase: "transcribe" }); } - /** Run one transcription request through whisper-stt-server. */ + /** + * Run one chunk, retrying a few times before giving up on the whole request. + * + * A failure here is usually the helper process dying (OOM, driver reset) + * rather than a bad chunk, so each retry first re-runs `server.start()` — + * idempotent when the helper is alive, a respawn when it isn't. That is what + * makes a 30-minute transcription survive a helper that dies once mid-run. + * + * What it does NOT do is salvage a chunk that fails all three attempts: the + * request fails whole and the chunks that already succeeded go with it. A + * transcript silently missing 90 seconds in the middle is worse than no + * transcript, since nothing downstream (captions, trims, the transcript + * editor) could tell the gap from a silence. The caller is told how far it + * got instead — see the wrapper in `transcribe()`. + */ + private async transcribeChunk( + samples: Float32Array, + language: string | undefined, + ): Promise>> { + let lastError: unknown; + for (let attempt = 1; attempt <= CHUNK_ATTEMPTS; attempt++) { + try { + return await this.server.transcribe({ samples, language }); + } catch (error) { + lastError = error; + if (attempt === CHUNK_ATTEMPTS) break; + if (this.modelPath) { + await this.server.start({ modelPath: this.modelPath }).catch(() => undefined); + } + await new Promise((resolve) => setTimeout(resolve, 500 * attempt)); + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); + } + + /** Transcribe a whole recording, chunk by chunk, reporting progress as it goes. */ async transcribe(req: SttTranscribeRequest): Promise { await this.init(); - this.emit({ phase: "transcribe" }); - const phrase = await this.server.transcribe({ - samples: req.samples, - language: req.language, - }); - const backend = phrase.backend ?? this.server.status.backend ?? "whispercpp-cpu"; + + const epoch = this.cancelEpoch; + const totalSec = req.samples.length / SAMPLE_RATE; + const chunks = planChunks(req.samples, SAMPLE_RATE); + this.emit({ phase: "transcribe", completedSec: 0, totalSec }); + + const segments: SttPhraseSegment[] = []; + const wordSegments: SttWordSegment[] = []; + let detectedLanguage: string | null = null; + let backend = this.server.status.backend ?? "whispercpp-cpu"; + // Only the first chunk auto-detects; every later chunk is forced onto the + // language it resolved, so whisper cannot flip mid-recording on a chunk + // that opens with a proper noun or a silence and "transcribe" the rest as + // another language. + // + // `"auto"` must collapse to `undefined` here rather than merely falsy + // values: `SttTranscribeRequest` documents it as the explicit way to ask + // for detection, and it is TRUTHY — left in place it makes the pin below + // unreachable for every caller that spells its intent out. + // + // This depends on the helper reporting what it RESOLVED rather than + // echoing the request, which it only does since cc781806 (30/07/2026, + // `whisper_full_lang_id()` in electron/native/whisper-stt/src/main.cpp). + // A stale `electron/native/bin//whisper-stt-server` — the directory + // is gitignored, so a dev tree keeps whatever was last staged there — + // silently reverts this to "every chunk detects on its own": the echo + // comes back as the literal "auto", the guard below rejects it, and + // nothing anywhere says why. `scripts/stage-whisper-stt.sh` refuses to + // overwrite a local binary by design, so it will not rescue you either. + // Verified end-to-end on 352s of real speech: `[undefined,"en","en","en"]`. + let language = req.language && req.language !== "auto" ? req.language : undefined; + + for (const [index, chunk] of chunks.entries()) { + // Between chunks is the only place this loop can be interrupted, and it + // is enough: a chunk is bounded by `whisperServer`'s own request ceiling. + if (this.cancelEpoch !== epoch) throw cancelledError(); + const offsetSec = chunk.startSample / SAMPLE_RATE; + const result = await this.transcribeChunk( + req.samples.subarray(chunk.startSample, chunk.endSample), + language, + ).catch((error) => { + // Say where it died. Without this the user gets "Transcription + // failed" for a 30-minute recording with no hint that 18 of those + // minutes were fine and the helper fell over at one specific spot. + // `Object.assign` rather than the `{ cause }` constructor option: the + // project targets ES2020, where that overload does not exist (see + // `BackgroundLoadError` in src/lib/wallpaper.ts for the same dance). + throw Object.assign( + new Error( + `transcription failed ${Math.round(offsetSec)}s into a ${Math.round(totalSec)}s ` + + `recording (chunk ${index + 1}/${chunks.length}): ` + + `${error instanceof Error ? error.message : String(error)}`, + ), + { cause: error }, + ); + }); + // Chunk-relative timestamps → absolute, the only thing every consumer + // (captions, transcript editor, trims) reads. + for (const segment of result.segments) { + segments.push({ + text: segment.text, + startSec: segment.startSec + offsetSec, + endSec: segment.endSec + offsetSec, + }); + } + for (const word of result.wordSegments) { + wordSegments.push({ + word: word.word, + startSec: word.startSec + offsetSec, + endSec: word.endSec + offsetSec, + confidence: word.confidence, + }); + } + if (!detectedLanguage && result.detectedLanguage && result.detectedLanguage !== "auto") { + detectedLanguage = result.detectedLanguage; + if (!language) language = detectedLanguage; + } + backend = result.backend ?? backend; + this.emit({ + phase: "transcribe", + completedSec: chunk.endSample / SAMPLE_RATE, + totalSec, + }); + } + return { - segments: phrase.segments, - wordSegments: phrase.wordSegments, - detectedLanguage: phrase.detectedLanguage, + segments, + wordSegments, + detectedLanguage: detectedLanguage ?? language ?? "auto", backend, }; } @@ -138,10 +316,11 @@ export function _resetSttManagerForTests(): void { } /** - * Wire the IPC channel. Call this from `registerIpcHandlers` so the renderer - * can `invoke("stt:transcribe", request)` and receive `SttTranscribeResponse`. - * Status events fan out on `"stt:status"` (main → renderer push), scoped to - * the calling `webContents` so two windows don't cross-talk. + * Wire the IPC channels. Call this from `registerIpcHandlers` so the renderer + * can `invoke("stt:transcribe", request)` and receive `SttTranscribeResponse`, + * and `invoke("stt:cancel")` to stop a run it no longer wants. Status events + * fan out on `"stt:status"` (main → renderer push), scoped to the calling + * `webContents` so two windows don't cross-talk. */ export function registerSttIpc(ipcMain: IpcMain): void { const manager = getSttManager(); @@ -149,8 +328,9 @@ export function registerSttIpc(ipcMain: IpcMain): void { "stt:transcribe", async (event, req: SttTranscribeRequest): Promise => { const senderId = event.sender.id; - const previous = manager.getStatusSink(); - manager.setStatusSink((statusEvent) => { + // Attach for the life of THIS request only. Overlapping requests each + // own their own sink, so neither can silence the other on the way out. + const detach = manager.addStatusSink((statusEvent) => { if (event.sender.id === senderId && !event.sender.isDestroyed()) { event.sender.send("stt:status", statusEvent); } @@ -158,8 +338,11 @@ export function registerSttIpc(ipcMain: IpcMain): void { try { return await manager.transcribe(req); } finally { - manager.setStatusSink(previous); + detach(); } }, ); + ipcMain.handle("stt:cancel", () => { + manager.cancel(); + }); } diff --git a/electron/stt/transcriptionContract.ts b/electron/stt/transcriptionContract.ts index 85b6c715..2fa8a88e 100644 --- a/electron/stt/transcriptionContract.ts +++ b/electron/stt/transcriptionContract.ts @@ -50,6 +50,14 @@ export interface SttStatusEvent { totalBytes?: number; /** Which model is downloading. */ model?: "whisper"; + /** + * Seconds of audio transcribed so far, and the total for this request. Only + * when `phase === "transcribe"`. Progress is reported per CHUNK (see + * `chunking.ts`), so it steps rather than sweeps — whisper gives no + * sub-request progress signal to interpolate from. + */ + completedSec?: number; + totalSec?: number; } /** IPC request: renderer → main. */ diff --git a/electron/stt/whisperServer.ts b/electron/stt/whisperServer.ts index 4762ebf6..087073a2 100644 --- a/electron/stt/whisperServer.ts +++ b/electron/stt/whisperServer.ts @@ -14,6 +14,24 @@ import { cleanupWav, writeSamplesAsWav } from "./wav"; /** whisper.cpp helper is stdio-shaped: stdin ignored, stdout/stderr captured. */ type WhisperChild = ChildProcessByStdio; +/** + * Per-request ceiling. whisper answers `/inference` only once the WHOLE upload + * is transcribed, so this bounds one chunk, not one recording. + * + * ponytail: 280s because Node's global fetch (undici) applies its own + * undocumented 300s `headersTimeout` that we cannot configure without taking a + * direct dependency on `undici` — going over it produces an opaque + * "TypeError: fetch failed" instead of anything actionable (this is exactly how + * a single 30-minute request died: killed at 300s while whisper needed 574s). + * Aborting at 280s keeps the failure OURS: named, logged with the helper's + * stderr, and retried by SttManager. The real ceiling this leaves is a machine + * so slow that one 90s chunk needs more than 280s (~0.3x realtime); the upgrade + * path is a direct `undici` dependency and + * `new Agent({ headersTimeout: 0, bodyTimeout: 0 })` as the fetch dispatcher, + * which removes the cliff entirely. + */ +const REQUEST_TIMEOUT_MS = 280_000; + /** * Owns the long-lived `whisper-stt-server` process used to recognize speech. * @@ -291,12 +309,55 @@ export class WhisperServerManager { form.set("file", blob, path.basename(opts.wavPath)); form.set("response_format", "verbose_json"); form.set("language", opts.language && opts.language !== "auto" ? opts.language : "auto"); - const res = await fetch(url, { method: "POST", body: form }); + let res: Response; + try { + res = await fetch(url, { + method: "POST", + body: form, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + } catch (error) { + // Name the failure. Node's global fetch reports BOTH a real transport + // error and its own header timeout as a bare "fetch failed", which is + // how a too-long request used to reach the user as an unactionable + // "Transcription failed" toast. + // + // The timeout wording is reserved for an ACTUAL timeout: a helper that + // died a moment ago rejects in a millisecond, and telling the reader it + // spent 280s on an over-long chunk sends them to the wrong problem + // entirely. Everything else carries its own message, plus `cause` so the + // errno survives. + // `Object.assign` rather than the `{ cause }` constructor option: the + // project targets ES2020, where that overload does not exist. + throw Object.assign( + new Error( + error instanceof Error && error.name === "TimeoutError" + ? `whisper-stt-server /inference timed out after ${Math.round(REQUEST_TIMEOUT_MS / 1000)}s ` + + `(audio chunk too long for this machine, or the helper is wedged); ` + + `stderr=${this.stderrTail.slice(-256)}` + : `whisper-stt-server /inference failed: ` + + `${error instanceof Error ? error.message : String(error)}; ` + + `stderr=${this.stderrTail.slice(-256)}`, + ), + { cause: error }, + ); + } if (!res.ok) { const text = await res.text().catch(() => ""); throw new Error(`whisper-stt-server /inference HTTP ${res.status}: ${text.slice(0, 512)}`); } - return (await res.json()) as WhisperJsonResponse; + // The same timeout still covers the body: it can fire between the headers + // and the last byte, and an unnamed `AbortError` here would read exactly + // like the "fetch failed" this whole block exists to replace. + return (await res.json().catch((error: unknown) => { + throw Object.assign( + new Error( + `whisper-stt-server /inference response was unreadable: ` + + `${error instanceof Error ? error.message : String(error)}`, + ), + { cause: error }, + ); + })) as WhisperJsonResponse; } /** Defensive number parse for `verbose_json` values that may arrive as strings. */ diff --git a/nix/package.nix b/nix/package.nix index 620e610b..6245dcb3 100644 --- a/nix/package.nix +++ b/nix/package.nix @@ -11,7 +11,11 @@ buildNpmPackage { nodejs = nodejs_22; pname = "openscreen"; - version = "1.7.0"; + # Read, not restated. A hand-copied version is one more thing to remember at + # release time and it had already drifted two minors behind the app it names. + # (`npmDepsHash` below still has to be updated by hand — that is Nix, not a + # choice — but it fails loudly, where a stale version number never does.) + version = (lib.importJSON ../package.json).version; src = let diff --git a/package.json b/package.json index 2adc681f..9fdf8bf9 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "openscreen", "private": true, - "version": "1.8.0-rc.5", + "version": "1.8.0", "type": "module", "packageManager": "npm@10.9.4", "engines": { diff --git a/scripts/fetch-ffmpeg-macos.mjs b/scripts/fetch-ffmpeg-macos.mjs index 14330ac2..d611aa62 100644 --- a/scripts/fetch-ffmpeg-macos.mjs +++ b/scripts/fetch-ffmpeg-macos.mjs @@ -40,6 +40,106 @@ function run(cmd, args, opts = {}) { } } +/** + * Where the pinned tarball can be fetched, in order of preference. + * + * ffmpeg.org is canonical and stays first, but it cannot be the only one: + * GitHub's `macos-15-intel` runner pool cannot reach it. On the v1.8.0-rc.6 + * build the x64 leg died twice, twenty minutes apart, on + * `curl: (35) Recv failure: Connection reset by peer` about a second after + * starting — while the arm64 leg on `macos-latest` fetched the same tarball + * from the same host in the same runs and succeeded both times. An immediate + * reset that reproduces on one runner pool and never on the other is an + * egress-level block, not congestion, so retrying alone does not clear it. + * + * Debian's `.orig.tar.xz` is the upstream tarball unmodified — verified + * byte-identical to TARBALL_SHA256 below — and deb.debian.org is CDN-backed. + * It is a fallback, not a replacement: the checksum is what makes trusting a + * second origin safe, and it gates every source equally. + * + * When the pin moves, a mirror may not carry the new version yet. That is not + * a failure mode to design around — the list is tried in order and a source + * that 404s falls through to the next, with every attempt reported if none + * works. (`--retry-all-errors` does not except a 404, so a missing mirror costs + * its retries — a few seconds, once per pin bump. Narrowing that would mean + * giving up the retry on connection resets, which is the whole point.) + */ +const TARBALL_URLS = [ + `https://ffmpeg.org/releases/ffmpeg-${VERSION}.tar.xz`, + `https://deb.debian.org/debian/pool/main/f/ffmpeg/ffmpeg_${VERSION}.orig.tar.xz`, +]; + +/** + * Fetches the pinned tarball to `dest`, trying each source until one both + * downloads and matches the checksum. + * + * `--retry-all-errors` rather than a plain `--retry`: curl only auto-retries + * what it classes as transient (timeouts, 429, 5xx), which does not include a + * connection reset or a TLS handshake failure — precisely the errors seen here. + * `-f` keeps an HTTP error page from being written to the tarball and + * resurfacing as a checksum mismatch, which reads like a moved pin. + */ +function downloadTarball(dest) { + const failures = []; + for (const url of TARBALL_URLS) { + console.log(`Downloading ffmpeg ${VERSION} from ${new URL(url).host}…`); + // Two ceilings, because a stuck download stalls in two different ways. + // --connect-timeout covers the handshake: a throttled origin does not + // refuse, it hangs — measured against ffmpeg.org after a few rapid + // fetches, a single connect sat for 75s before failing, and times four + // attempts that is five minutes of a build spent before the second source + // is even tried. 20s is far above any healthy handshake. + // --speed-limit/--speed-time covers everything AFTER the handshake, which + // --connect-timeout does not reach at all: an origin that accepts the + // connection and then trickles (or simply stops sending) never errors, so + // nothing here would retry or fall through — the job would sit until the + // runner's own six-hour limit. Aborting below 1 KiB/s sustained for 30s + // cannot fire on a link that is merely slow: the tarball is ~10 MB. + const r = spawnSync( + "curl", + [ + "-fsSL", + "--connect-timeout", + "20", + "--speed-limit", + "1024", + "--speed-time", + "30", + "--retry", + "3", + "--retry-delay", + "2", + "--retry-all-errors", + "-o", + dest, + url, + ], + { stdio: "inherit" }, + ); + if (r.status !== 0) { + // `r.error` is the case `status` cannot express: no curl on PATH gives + // `{ status: null, error: ENOENT }`, and reporting only "exited with + // null" twice sends the reader hunting a network problem. + failures.push(` ${url}\n ${r.error ? r.error.message : `curl exited with ${r.status}`}`); + continue; + } + const actual = crypto.createHash("sha256").update(fs.readFileSync(dest)).digest("hex"); + if (actual !== TARBALL_SHA256) { + // Not fatal on its own — a mirror may carry a repacked tarball. It is + // reported in full, so a genuinely moved pin is still legible as + // "every source disagreed the same way" rather than "network down". + failures.push(` ${url}\n checksum ${actual}`); + continue; + } + console.log("Checksum OK."); + return; + } + throw new Error( + `Could not obtain ffmpeg ${VERSION} from any source.\n` + + `Expected sha256 ${TARBALL_SHA256}\n${failures.join("\n")}`, + ); +} + /** The binary's own licence banner — the only claim worth trusting. */ function isLgpl(dir) { const bin = path.join(dir, "bin", "ffmpeg"); @@ -68,16 +168,7 @@ if (fs.existsSync(path.join(DEST, "include"))) { const work = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-ffmpeg-")); const tarball = path.join(work, `ffmpeg-${VERSION}.tar.xz`); -console.log(`Downloading ffmpeg ${VERSION}…`); -run("curl", ["-sSL", "-o", tarball, `https://ffmpeg.org/releases/ffmpeg-${VERSION}.tar.xz`]); - -const actual = crypto.createHash("sha256").update(fs.readFileSync(tarball)).digest("hex"); -if (actual !== TARBALL_SHA256) { - throw new Error( - `Checksum mismatch for the ffmpeg tarball.\n expected ${TARBALL_SHA256}\n got ${actual}`, - ); -} -console.log("Checksum OK."); +downloadTarball(tarball); run("tar", ["-xJf", tarball, "-C", work]); const src = path.join(work, `ffmpeg-${VERSION}`); diff --git a/scripts/fetch-ffmpeg.mjs b/scripts/fetch-ffmpeg.mjs index 1c42a92c..9a98bcf1 100644 --- a/scripts/fetch-ffmpeg.mjs +++ b/scripts/fetch-ffmpeg.mjs @@ -457,7 +457,15 @@ async function fetchSharedDlls(tag, binDir) { fs.copyFileSync(lib, dest); } } - console.log(`Vendored ${libs.length} shared librar(ies) -> ${binDir}`); + // The shared CLI too, beside the DLLs it links against. 1 MB, against + // the 109 MB of the static exe the installer excludes — and unlike that + // one, this is spawned at runtime: electron/media/audioPeaks.ts decodes + // waveform peaks with it, ~6x faster than either browser pipeline and + // off the UI process. Named apart from `ffmpeg.exe` on purpose, so the + // packager's `!win32-*/ffmpeg.exe` rule keeps dropping the static build + // while this one ships under the plain `win32-*/*` include. + fs.copyFileSync(exe, path.join(binDir, "ffmpeg-shared.exe")); + console.log(`Vendored ${libs.length} shared librar(ies) + ffmpeg-shared.exe -> ${binDir}`); } if (sdkDest) vendorFfmpegSdk(tmp, sdkDest); console.log("LGPL verified: safe to ship with an MIT app."); diff --git a/src/components/ai-edition/ExportDialog.tsx b/src/components/ai-edition/ExportDialog.tsx index 363c55d0..97ae481c 100644 --- a/src/components/ai-edition/ExportDialog.tsx +++ b/src/components/ai-edition/ExportDialog.tsx @@ -31,7 +31,7 @@ import { type GifFrameRate, type GifSizePreset, } from "@/lib/exporter"; -import { calculateMp4ExportSettings } from "@/lib/exporter/mp4ExportSettings"; +import { calculateMp4ExportSettings, wouldUpscale } from "@/lib/exporter/mp4ExportSettings"; import { exportGifNative, exportMultiNative, useIsCpuCompositor } from "@/native"; import type { CompositorClipInput } from "@/native/contracts"; import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; @@ -86,20 +86,12 @@ function buildNativeClipList(document: AxcutDocument): CompositorClipInput[] { }); } -// Target short side (px) for the two fixed quality tiers -- mirrors the legacy -// editor's SettingsPanel (MP4_EXPORT_SHORT_SIDES), used only to decide whether -// picking that tier would upscale past the source's actual resolution. -const MEDIUM_SHORT_SIDE = 720; -const HIGH_SHORT_SIDE = 1080; - const QUALITY_OPTIONS: Array<{ value: ExportQuality; labelKey: string; - /** Target short side for the upscale check; undefined for "source" (no fixed target). */ - targetShortSide?: number; }> = [ - { value: "medium", labelKey: "exportQuality.low", targetShortSide: MEDIUM_SHORT_SIDE }, - { value: "good", labelKey: "exportQuality.medium", targetShortSide: HIGH_SHORT_SIDE }, + { value: "medium", labelKey: "exportQuality.low" }, + { value: "good", labelKey: "exportQuality.medium" }, { value: "source", labelKey: "exportQuality.high" }, ]; @@ -162,17 +154,12 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { // Smallest clip's true (cropped) footprint on the timeline — a multiclip timeline can mix // crops/resolutions, so this is what "Source" quality actually targets: sizing to the // SMALLEST clip's own resolution means no clip on the timeline is ever upscaled past its - // true footprint by picking Source, which is why Source shows no upscale/downscale badge - // at all any more (see the tier badges below) — it's upscale-proof by construction. The - // fixed 720p/1080p tiers don't have that property (they can still upscale a small clip), - // so `smallestShortSide` below still feeds their upscale badge. + // true footprint by picking Source. It also feeds the upscale badge on the fixed + // 720p/1080p tiers, which can still genuinely upscale a small clip. const smallestSource = useMemo( () => pickExtremeDims(effectiveClipDims, "smallest"), [effectiveClipDims], ); - const smallestShortSide = smallestSource - ? Math.min(smallestSource.width, smallestSource.height) - : null; // Aspect the export normalizes to: the timeline's selected ratio (mirrors documentExporter), // so the sizes shown match what the export produces. Read through `getEditorSettings` — the @@ -419,16 +406,15 @@ export function ExportDialog({ open, onClose, document }: ExportDialogProps) { const dims = tierOutputDims(q.value); if (!dims) return null; // Downscale badge removed everywhere — restated what picking a lower - // tier already means, not actionable. Upscale badge removed only for - // "Source": it's upscale-proof by construction now (targets the - // smallest clip), so the warning never meant anything there. The fixed - // 720p/1080p tiers can still genuinely upscale a small clip, so that - // badge stays relevant for them. - const outShortSide = Math.min(dims.width, dims.height); - const isUpscale = - q.value !== "source" && - smallestShortSide !== null && - outShortSide > smallestShortSide; + // tier already means, not actionable. The upscale badge asks whether + // the clip has to be STRETCHED to fill this frame (`wouldUpscale`), + // which is a contain-fit question: a short-side compare read the + // letterbox rows a non-16:9 source gets in a 16:9 project as if they + // were stretched pixels, and flagged "1080p" on the very frame + // "Source" produced unflagged. No "Source" special case any more — + // its frame is the source's long side at the project ratio, so its + // contain scale is never above 1 and the general test covers it. + const isUpscale = smallestSource !== null && wouldUpscale(dims, smallestSource); return ( > })?.speedRegions ?? []; - const speed = region as unknown as (typeof speedRegions)[number]; - const duration = Number(speed.endMs) - Number(speed.startMs); - speed.startMs = timeMs; - speed.endMs = timeMs + duration; + const src = snapshot.region as { startMs: number; endMs: number }; + const prefix = snapshot.kind === "annotation" ? "ann" : snapshot.kind; + const pasted = { + ...snapshot.region, + id: createId(prefix), + startMs: timeMs, + endMs: timeMs + (Number(src.endMs) - Number(src.startMs)), + }; + // Anchor to the clip(s) it covers, exactly like every add* does. Pasting + // used to store a bare startMs/endMs, so the region survived until the + // first clip reorder or trim and then drifted off its content — see + // technical-documentation/architecture/timeline-model.md. + const anchored = anchorRegionsWithDerivedMs( + [pasted as unknown as { id: string; startMs: number; endMs: number }], + doc.timeline.clips, + () => createId(prefix), + ); + + if (snapshot.kind === "zoom") { await saveDocument({ ...doc, - legacyEditor: { - ...doc.legacyEditor, - speedRegions: [...speedRegions, speed], - }, + zoomRanges: [...doc.zoomRanges, ...anchored] as typeof doc.zoomRanges, + }); + } else if (snapshot.kind === "annotation") { + await saveDocument({ + ...doc, + annotations: [...doc.annotations, ...anchored] as typeof doc.annotations, + }); + } else { + // speed and cameraFullscreen are both plain spans on legacyEditor. + const key = snapshot.kind === "speed" ? "speedRegions" : "cameraFullscreenRegions"; + const legacy = (doc.legacyEditor as Record) ?? {}; + const prev = (legacy[key] as unknown[]) ?? []; + await saveDocument({ + ...doc, + legacyEditor: { ...legacy, [key]: [...prev, ...anchored] }, }); } toast.success("Region pasted"); - }, [saveDocument]); + // `tl` belongs here now that the trim branch calls tl.addTrim: useTimeline + // returns a fresh object each render, so memoizing on saveDocument alone + // would paste through a callback holding a stale document. + }, [saveDocument, tl]); + // Copy the SELECTED pill. Reads the same arrays the lanes render, so what gets + // copied is what the user is looking at — the old version dug into the raw + // document with a ternary chain that mapped a trim to "zoom" and sent + // cameraFullscreen down the speed branch, where neither could ever be found. const handleCopyRegion = useCallback(async () => { - const doc = useProjectStore.getState().document; - if (!doc || !tl.selection) return; + const sel = tl.selection; + if (!sel) return; const { copyRegion } = await import("@/lib/ai-edition/store/regionClipboard"); - const region = - tl.selection.kind === "zoom" - ? doc.zoomRanges.find((z) => z.id === tl.selection?.id) - : tl.selection.kind === "annotation" - ? (doc.annotations as unknown[]).find( - (a) => (a as { id: string }).id === tl.selection?.id, - ) - : ((doc.legacyEditor as { speedRegions?: unknown[] } | null)?.speedRegions ?? []).find( - (s) => (s as { id: string }).id === tl.selection?.id, - ); - if (region) { - copyRegion({ - kind: tl.selection.kind === "trim" ? "zoom" : tl.selection.kind, - region: region as Record, - }); + + // A trim is stored in SOURCE time against a clip anchor, so there is no + // row to clone — but there is nothing to clone either: what the user means + // by copying a cut is its LENGTH. Paste then makes a fresh trim of that + // length at the playhead, which is the same deal every other kind gets + // (properties kept, position taken from the playhead). + if (sel.kind === "trim") { + const { coalescedTrimGroups } = await import("@/lib/ai-edition/timeline/trim-mapping"); + const group = coalescedTrimGroups(tl.trimRanges, tl.clips).find((g) => + g.ids.includes(sel.id), + ); + if (!group) return; + copyRegion({ kind: "trim", region: { durationSec: group.end - group.start } }); + setCopiedClipId(null); toast.success("Region copied"); + return; } + + const source = + sel.kind === "zoom" + ? tl.zoomRegions + : sel.kind === "annotation" + ? tl.annotationRegions + : sel.kind === "speed" + ? tl.speedRegions + : tl.cameraFullscreenRegions; + const region = (source as Array<{ id: string }>).find((r) => r.id === sel.id); + if (!region) return; + copyRegion({ kind: sel.kind, region: region as unknown as Record }); + // One clipboard wins at a time: a copied pill retires the copied clip. + setCopiedClipId(null); + toast.success("Region copied"); }, [tl]); useEffect(() => { @@ -836,20 +883,28 @@ export function NewEditorShell() { // of hardcoded keys, so rebinding in the shortcuts dialog actually // changes runtime behavior. if (matchesShortcut(e, shortcuts.copySelected, isMac)) { - if (tl.clipSelection) { + // A pill and a clip can no longer both be selected (see selectRegion / + // selectClip), so this reads the one the user actually picked instead + // of preferring clips whatever was clicked last. + if (tl.selection) { e.preventDefault(); - setCopiedClipId(tl.clipSelection); + void handleCopyRegion(); return; } - if (tl.selection) { + if (tl.clipSelection) { e.preventDefault(); - void handleCopyRegion(); + setCopiedClipId(tl.clipSelection); + // A copied clip retires the copied pill — see clearRegionClipboard. + void import("@/lib/ai-edition/store/regionClipboard").then((m) => + m.clearRegionClipboard(), + ); return; } } if (ctrl && e.key.toLowerCase() === "x") { // F2.8 — cut: remember the region in the clipboard, then remove it. - if (tl.selection && tl.selection.kind !== "trim") { + // Trims included now that copying one means copying its length. + if (tl.selection) { e.preventDefault(); const cut = tl.selection; void handleCopyRegion().then(() => tl.removeRegion(cut.kind, cut.id)); @@ -858,12 +913,13 @@ export function NewEditorShell() { } if (matchesShortcut(e, shortcuts.paste, isMac)) { e.preventDefault(); - // A selected/copied clip takes priority — pasting with a clip in - // hand is unambiguously "duplicate this clip", even if a region - // was copied earlier in the session. - const clipToDuplicate = copiedClipId ?? tl.clipSelection; - if (clipToDuplicate) { - void tl.duplicateClip(clipToDuplicate); + // Paste what was COPIED. It used to fall back to `tl.clipSelection`, + // so a clip merely being selected hijacked the paste — and since + // `copiedClipId` was never cleared, one Ctrl+C on a clip turned every + // later Ctrl+V into a clip duplication for the rest of the session, + // whatever the user copied afterwards. + if (copiedClipId) { + void tl.duplicateClip(copiedClipId); return; } void pasteRegion(); @@ -884,29 +940,34 @@ export function NewEditorShell() { deleteSelection(); return; } + // Same size on screen as the toolbar buttons produce — these shortcuts are + // what the empty lanes advertise ("Press Z to add zoom"), so they are the + // way most regions get created. Left on the flat default they came out + // under two pixels on a 30-minute recording, hidden behind the playhead + // they were created at. See timeline/newRegionDuration. if (matchesShortcut(e, shortcuts.addZoom, isMac)) { e.preventDefault(); - void tl.addZoom(); + void tl.addZoom(newRegionDurationSec()); return; } if (matchesShortcut(e, shortcuts.addTrim, isMac)) { e.preventDefault(); - void tl.addTrim(); + void tl.addTrim(newRegionDurationSec()); return; } if (matchesShortcut(e, shortcuts.addAnnotation, isMac)) { e.preventDefault(); - void tl.addAnnotation(); + void tl.addAnnotation(newRegionDurationSec()); return; } if (matchesShortcut(e, shortcuts.addSpeed, isMac)) { e.preventDefault(); - void tl.addSpeed(); + void tl.addSpeed(newRegionDurationSec()); return; } if (matchesShortcut(e, shortcuts.addCameraFullscreen, isMac)) { e.preventDefault(); - void tl.addCameraFullscreen(); + void tl.addCameraFullscreen(newRegionDurationSec()); return; } diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 0490f310..5ac70471 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -1042,7 +1042,22 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ // words (inside a skip range) render red+strikethrough with a hover bin. // `isCue` highlights the word the playback head is currently inside with // an accent underline (matches axcut's `word.transcript-word.cue` rule). -function TranscriptWord({ +// +// `memo` for the same reason as `TranscriptClipBlock`, one level down — and it +// is the level that actually decides the cost. The block's memo assumes +// `cueWordId` moves "a few times per second, not sixty", which holds for +// playback at 1x and NOT for a scrub: dragging the playhead crosses many words +// per frame, so `cueWordId` changes on essentially every frame and the block +// re-renders. Without a memo here that meant re-rendering one component per +// transcript word, every frame. Measured over a 40-frame scrub in jsdom: +// 19.6 ms/frame at 100 words, 132.6 ms at 4501 (a real 30-minute recording) — +// the cost was simply proportional to transcript length. With the memo only +// the two words whose `isCue` actually flipped re-render. +// +// This holds because every other prop is referentially stable across a +// playhead tick: `cw` comes from the memoised `sections`, `target` from a +// `useMemo`, and both callbacks from `useCallback`s that do not depend on time. +const TranscriptWord = memo(function TranscriptWord({ cw, isCue, target, @@ -1198,7 +1213,7 @@ function TranscriptWord({ ) : null} ); -} +}); // ─── Caret / selection helpers ──────────────────────────────────── // Ponytail port of axcut's findCollapsedDeletionWordId. The non-collapsed diff --git a/src/components/ai-edition/TranscriptionStatus.tsx b/src/components/ai-edition/TranscriptionStatus.tsx index 5e17bd27..5d74dd55 100644 --- a/src/components/ai-edition/TranscriptionStatus.tsx +++ b/src/components/ai-edition/TranscriptionStatus.tsx @@ -9,7 +9,10 @@ import { Loader2 } from "lucide-react"; import { useScopedT } from "@/contexts/I18nContext"; -import type { AssetTranscriptionView } from "@/lib/ai-edition/transcription/status"; +import { + type AssetTranscriptionView, + progressFraction, +} from "@/lib/ai-edition/transcription/status"; /** Human-readable state of one asset's transcript, in the user's language. */ export function useTranscriptionLabel(): (view: AssetTranscriptionView) => string { @@ -20,8 +23,21 @@ export function useTranscriptionLabel(): (view: AssetTranscriptionView) => strin return t("mediaStage.transcriptReady"); case "queued": return t("mediaStage.pendingTranscription"); - case "running": - return t("mediaStage.transcribing"); + case "running": { + // The first-run model download is a 253 MB wait with nothing else on + // screen to explain it, so it gets its own words rather than being + // labelled "Transcribing" — this is the phase most often mistaken for + // a hang, and the one `phase` was carried through the store for. + if (view.phase === "loading-model") return t("mediaStage.downloadingModel"); + // Transcribing a long recording runs for minutes. A bare + // "Transcribing…" for that whole time is indistinguishable from a + // hang, so append the percentage as soon as the main process reports + // chunk progress — and only then (see `TranscriptionProgressBar`). + const fraction = progressFraction(view.progress); + return fraction === null + ? t("mediaStage.transcribing") + : `${t("mediaStage.transcribing")} ${Math.round(fraction * 100)}%`; + } case "empty": return t("mediaStage.noSpeechDetected"); case "failed": @@ -79,3 +95,45 @@ export function TranscriptionStatusDot({ /> ); } + +/** + * Determinate progress bar for a running transcription. Renders nothing unless + * the job actually reports measurable progress — a job that is queued, + * extracting audio or downloading the model has no meaningful fraction, and a + * bar pinned at 0% reads as "stuck" where the spinner reads as "working". + * + * It owns its own spacing on purpose. A wrapper in the caller cannot render + * itself away with the bar, and in a flex column (where margins don't collapse) + * an empty one still takes its margins — 8px of dead gap under every media card + * that isn't transcribing. + */ +export function TranscriptionProgressBar({ view }: { view: AssetTranscriptionView }) { + const label = useTranscriptionLabel(); + const fraction = view.status === "running" ? progressFraction(view.progress) : null; + if (fraction === null) return null; + return ( +
+
+
+ ); +} diff --git a/src/components/ai-edition/v4/MediaStage.tsx b/src/components/ai-edition/v4/MediaStage.tsx index 8c4bca65..a9933bc8 100644 --- a/src/components/ai-edition/v4/MediaStage.tsx +++ b/src/components/ai-edition/v4/MediaStage.tsx @@ -14,7 +14,11 @@ import type { AssetTranscriptionView, } from "@/lib/ai-edition/transcription/status"; import { formatBytes } from "@/utils/formatBytes"; -import { TranscriptionStatusDot, useTranscriptionLabel } from "../TranscriptionStatus"; +import { + TranscriptionProgressBar, + TranscriptionStatusDot, + useTranscriptionLabel, +} from "../TranscriptionStatus"; import styles from "./EditorShellV4.module.css"; const ASSET_MIME = "application/x-axcut-asset"; @@ -266,8 +270,36 @@ export function MediaStage() { {transcriptionLabel(selectedTranscription)} + {/* The language whisper resolved on the first chunk, which every later + chunk was then pinned to. It had a pill in SourceTranscriptModal, + but that lives under LeftPanel's `MediaPane` — and the only mount + site is ``, a literal, so it renders + `ChatStripPanel` and nothing else. The value was reaching the + document and being displayed nowhere. It belongs next to + "Regenerate as" below in any case: that selector is the control + you set BECAUSE of what was detected. */} + {transcript?.language && transcript.language !== "auto" ? ( + + {t("mediaStage.detectedLanguage", { language: transcript.language })} + + ) : null}
+ {/* Renders itself away — spacing included — unless the run reports progress. */} + + {selectedTranscription.failure ? (

{ /* the drag only awaits it */ }), + addZoom: vi.fn(async () => { + /* the toolbar only awaits it */ + }), }; render( { }); }); +describe("V4Timeline create-from-toolbar", () => { + // The button asks for a DURATION worth a fixed number of pixels at the current + // zoom, so the pill you get is always the same size on screen — which is what + // the flat 2 s could not do: on this 30-minute fixture zoomed out it is one + // pixel. (It used to look fine only because the removed 1.5% minimum width + // inflated it in the rendering.) + const durationOf = (tl: { addZoom: ReturnType }) => + tl.addZoom.mock.calls.at(-1)?.[0] as number; + + it("scales the new region's duration with the zoom", () => { + const { tl } = renderTimeline(); + fireEvent.click(screen.getByTitle("buttons.addZoom")); + // 900px viewport / 1800 s = 0.5 px per second, so a 96px pill is 192 s. + expect(durationOf(tl)).toBeCloseTo(192, 3); + + // Zoomed to the 50x ceiling the same 96px is worth 3.84 s: same pill on + // screen, a region 50x shorter. + zoomIn(40); + fireEvent.click(screen.getByTitle("buttons.addZoom")); + expect(durationOf(tl)).toBeCloseTo(3.84, 3); + }); + + it("never asks for a slice too short to be worth creating", () => { + // Past ~30x on a short timeline the pixels are worth hundredths of a + // second; the region would be born unusable, so the duration floors. + const { tl } = renderTimeline([clip(0, 3)]); + zoomIn(40); + fireEvent.click(screen.getByTitle("buttons.addZoom")); + expect(durationOf(tl)).toBeCloseTo(0.25, 3); + }); +}); + describe("V4Timeline clip row", () => { // Three clips = two junctions. As a flex row with `gap: 6px`, each junction // added 6px while every clip shrank proportionally to pay for it, so a clip's diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index 7cee2378..22b79aa8 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -39,6 +39,10 @@ import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; import { formatSec } from "@/lib/ai-edition/timeline/format"; +import { + newRegionDurationSec, + setTimelineScale, +} from "@/lib/ai-edition/timeline/newRegionDuration"; import { ventilateSpanAcrossClips } from "@/lib/ai-edition/timeline/region-ventilation"; import { coalesceRegionsForRuler } from "@/lib/ai-edition/timeline/timelineMap"; import { @@ -125,6 +129,9 @@ const PILL_HANDLE_OUT_PX = PILL_HANDLE_PX + PILL_MOVE_GAP_PX; const PILL_CONTENT_MIN_PX = 34; /** Edge-snap radius while dragging a pill, in screen px. */ const PILL_SNAP_PX = 8; +// The size a newly created pill aims for (PILL_CREATE_PX) lives in +// timeline/newRegionDuration, because the keyboard shortcuts create regions too +// and they are handled in NewEditorShell, outside this component. /** Visual separation between two clip cards. Taken off each clip's own width * (see .tlClip) rather than inserted between them, so it cannot displace the * clips that follow — which is what a flex `gap` did, once per junction. */ @@ -257,7 +264,9 @@ const ClipWaveform = memo(function ClipWaveform({ sourceStartSec: number; sourceEndSec: number; }) { - const peaks = useAudioPeaks(videoUrl); + // The duration is what tells `useAudioPeaks` whether this recording is small + // enough to decode whole — the file's byte size does not, on compressed video. + const peaks = useAudioPeaks(videoUrl, assetDurationSec); const bars = useMemo(() => { if (!peaks || peaks.length === 0 || !assetDurationSec) return null; const totalBlocks = Math.floor(peaks.length / 2); @@ -449,6 +458,13 @@ export function V4Timeline({ // `total`, which is a duration in disguise and so scales with the recording. const navSpan = Math.max(0.02, nav.end - nav.start); const pxPerSec = viewportWidthPx / navSpan / total; + // Publish the scale so the keyboard shortcuts (NewEditorShell) size a new + // region exactly like the buttons below do — `nav` never leaves this + // component, so without this they fall back to a flat default and a pill + // created with `Z` comes out invisible on a long recording. + useEffect(() => { + setTimelineScale(pxPerSec); + }, [pxPerSec]); // ── region lanes ──────────────────────────────────────────────── // zoom/speed/annotation: one pill per row, never coalesced — each carries @@ -597,6 +613,11 @@ export function V4Timeline({ const startScrub = useCallback( (e: ReactPointerEvent) => { if (e.button !== 0) return; + // Media has no playhead rendered, so there is nothing to scrub. Guarded + // here rather than at the three call sites: seeking an invisible cursor + // would still move `currentTimeSec`, i.e. silently reposition the Edit + // tab's preview from a screen that shows no time at all. + if (!showLanes) return; const target = e.target as HTMLElement; if (target.closest("[data-clip-id]") || target.closest(`.${styles.lanePill}`)) return; tl.clearSelection(); @@ -622,7 +643,7 @@ export function V4Timeline({ window.addEventListener("pointermove", move); window.addEventListener("pointerup", up); }, - [seekToClientX, tl, setCurrentTime], + [seekToClientX, tl, setCurrentTime, showLanes], ); const [activePillDrag, setActivePillDrag] = useState<{ @@ -803,6 +824,9 @@ export function V4Timeline({ useEffect(() => { const el = tracksRef.current; if (!el) return; + // Media shows no zoom window, so leave the wheel alone there: a zoom with + // no control to undo it and no ruler reading to explain it is a trap. + if (!showLanes) return; const onWheelNative = (e: WheelEvent) => { const r = el.getBoundingClientRect(); const viewportPct = Math.min(1, Math.max(0, (e.clientX - r.left) / r.width)); @@ -836,7 +860,7 @@ export function V4Timeline({ }; el.addEventListener("wheel", onWheelNative, { passive: false }); return () => el.removeEventListener("wheel", onWheelNative); - }, []); + }, [showLanes]); // Track the tracks' content width for the ruler. .tlTracks and .tlRulerRow // carry the same horizontal padding and the tracks' scrollbar is hidden, so @@ -1303,9 +1327,12 @@ export function V4Timeline({ title={tool.label} aria-label={tool.label} onClick={() => { - if (tool.id === "speed") void tl.addSpeed(); - if (tool.id === "comment") void tl.addAnnotation(); - if (tool.id === "cut") void tl.addTrim(); + // Read at CLICK time: a render-time value would be one zoom + // notch stale when the user zooms and immediately creates. + const dur = newRegionDurationSec(); + if (tool.id === "speed") void tl.addSpeed(dur); + if (tool.id === "comment") void tl.addAnnotation(dur); + if (tool.id === "cut") void tl.addTrim(dur); }} > {tool.icon} @@ -1316,7 +1343,7 @@ export function V4Timeline({ className={styles.tlToolBtn} title={t("buttons.addZoom")} aria-label={t("buttons.addZoom")} - onClick={() => void tl.addZoom()} + onClick={() => void tl.addZoom(newRegionDurationSec())} > @@ -1339,7 +1366,7 @@ export function V4Timeline({ className={styles.tlToolBtn} title={t("buttons.addCameraFullscreen")} aria-label={t("buttons.addCameraFullscreen")} - onClick={() => void tl.addCameraFullscreen()} + onClick={() => void tl.addCameraFullscreen(newRegionDurationSec())} > @@ -1410,7 +1437,19 @@ export function V4Timeline({ ) : ( -

+ // Media is an ARRANGING surface: add, remove, reorder. Nothing here + // plays or edits, so the transport, the scroll hints, the zoom nav and + // the playhead are absent rather than inert — this caption is the whole + // header, and it centres because it is alone in the row. +
{t("toolbar.arrangeClips")} @@ -1419,23 +1458,27 @@ export function V4Timeline({
)} - -
- - Shift+Scroll {t("labels.pan")} - - - Ctrl+Scroll {t("labels.zoom")} - -
+ {showLanes ? ( + <> + +
+ + Shift+Scroll {t("labels.pan")} + + + Ctrl+Scroll {t("labels.zoom")} + +
+ + ) : null}
{/* Ruler + tracks share one relative wrapper so a single playhead overlay @@ -1611,41 +1654,48 @@ export function V4Timeline({ {/* Single playhead overlay spanning the ruler + tracks: fixed vertically (a cursor, so it doesn't scroll with the lanes) and sharing the exact same zoom/pan transform + width as the canvases, so its line stays - continuous from the ruler down through the clips and its head aligns. */} - + continuous from the ruler down through the clips and its head aligns. + Edit only: there is no playback to follow on the Media surface. */} + {showLanes ? ( + + ) : null} -
-
-
startNavDrag("pan", e)} - /> -
startNavDrag("left", e)} - > - -
-
startNavDrag("right", e)} - > - + {/* Zoom/pan window. Edit only: arranging clips needs the whole timeline + on screen at once, and there is nothing to zoom INTO without lanes. */} + {showLanes ? ( +
+
+
startNavDrag("pan", e)} + /> +
startNavDrag("left", e)} + > + +
+
startNavDrag("right", e)} + > + +
-
+ ) : null}
); } diff --git a/src/hooks/useAudioPeaks.test.ts b/src/hooks/useAudioPeaks.test.ts new file mode 100644 index 00000000..77a850b7 --- /dev/null +++ b/src/hooks/useAudioPeaks.test.ts @@ -0,0 +1,87 @@ +// Two properties that decide whether a long recording's waveform appears +// quickly or not at all: which pipeline a file is routed to, and how many times +// it is decoded. +import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAudioPeaks } from "./useAudioPeaks"; + +const streamingCalls = vi.fn(); +const inMemoryCalls = vi.fn(); + +vi.mock("./streamingAudioPeaks", () => ({ + computePeaksFromFileStreaming: async () => { + streamingCalls(); + return new Float32Array([0, 1]); + }, +})); + +vi.mock("@/lib/exporter/localSourceFile", () => ({ + materializeLocalSourceFile: async (_url: string, name: string) => ({ name }), + releaseLocalSourceFile: () => {}, +})); + +vi.mock("@/lib/exporter/streamingDecoder", () => ({ + loadFileAsArrayBuffer: async () => { + inMemoryCalls(); + return { data: new ArrayBuffer(8) }; + }, +})); + +// A 68 MB file — comfortably under the 256 MB in-memory threshold, which is +// exactly why routing on file size sent a 32-minute recording down the +// decode-everything path. +const FILE_BYTES = 68 * 1024 * 1024; +const THIRTY_TWO_MINUTES = 1951; + +beforeEach(() => { + streamingCalls.mockClear(); + inMemoryCalls.mockClear(); + (window as unknown as { electronAPI: unknown }).electronAPI = { + getReadableFileInfo: async () => ({ success: true, size: FILE_BYTES }), + }; +}); + +afterEach(cleanup); + +describe("useAudioPeaks", () => { + it("streams a long recording instead of decoding it whole", async () => { + const { result } = renderHook(() => useAudioPeaks("/tmp/long-a.mp4", THIRTY_TWO_MINUTES)); + await waitFor(() => expect(result.current).not.toBeNull()); + expect(streamingCalls).toHaveBeenCalledOnce(); + // The whole point: 68 MB on disk is 656 MB decoded, so this must NOT be + // the path that reads the file and hands it to decodeAudioData. + expect(inMemoryCalls).not.toHaveBeenCalled(); + }); + + it("keeps decoding short clips in memory", async () => { + // Only the ROUTE is asserted: the in-memory path then needs a real + // AudioContext and a Worker, neither of which jsdom has. + renderHook(() => useAudioPeaks("/tmp/short-a.mp4", 20)); + await waitFor(() => expect(inMemoryCalls).toHaveBeenCalled()); + expect(streamingCalls).not.toHaveBeenCalled(); + }); + + it("decodes a file once, however many clips mount it and however often", async () => { + const url = "/tmp/long-b.mp4"; + // Three clips of the same asset, mounted together. + const a = renderHook(() => useAudioPeaks(url, THIRTY_TWO_MINUTES)); + const b = renderHook(() => useAudioPeaks(url, THIRTY_TWO_MINUTES)); + const c = renderHook(() => useAudioPeaks(url, THIRTY_TWO_MINUTES)); + await waitFor(() => expect(a.result.current).not.toBeNull()); + await waitFor(() => expect(b.result.current).not.toBeNull()); + await waitFor(() => expect(c.result.current).not.toBeNull()); + expect(streamingCalls).toHaveBeenCalledOnce(); + + // Unmount everything — this is a Media↔Edit tab switch — and come back. + // With the cache scoped to a component ref, this re-decoded the whole + // recording every single time. + act(() => { + a.unmount(); + b.unmount(); + c.unmount(); + }); + const again = renderHook(() => useAudioPeaks(url, THIRTY_TWO_MINUTES)); + await waitFor(() => expect(again.result.current).not.toBeNull()); + expect(streamingCalls).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/hooks/useAudioPeaks.ts b/src/hooks/useAudioPeaks.ts index daa0abf0..da7512e2 100644 --- a/src/hooks/useAudioPeaks.ts +++ b/src/hooks/useAudioPeaks.ts @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState } from "react"; +import { useEffect, useState } from "react"; import { materializeLocalSourceFile, releaseLocalSourceFile } from "@/lib/exporter/localSourceFile"; import { MAX_IN_MEMORY_SOURCE_BYTES } from "@/lib/exporter/sourceFileLimits"; import { loadFileAsArrayBuffer } from "@/lib/exporter/streamingDecoder"; @@ -62,17 +62,59 @@ function computePeaksInWorker( } /** - * Routes to the right peaks pipeline for the source size. Small/remote files - * use the original decodeAudioData → worker path. Local recordings above the - * in-memory limit stream instead: the file is materialized into OPFS (reused by - * the export afterwards) and its audio is decoded chunk-by-chunk into peaks, so - * the whole recording is never held in memory. + * Bytes one second of decoded audio occupies in an `AudioBuffer`: Float32, + * stereo, 44.1 kHz. An estimate on purpose — it picks the pipeline, before + * anything has been decoded and while the real rate is still unknown. */ -async function computePeaksForUrl(videoUrl: string, signal?: AbortSignal): Promise { +const DECODED_BYTES_PER_SEC = 44_100 * 2 * 4; + +/** + * Routes to the right peaks pipeline. Small/remote files use the original + * decodeAudioData → worker path. Recordings too big to hold decoded stream + * instead: the file is materialized into OPFS (reused by the export afterwards) + * and its audio is decoded chunk-by-chunk into peaks, so the whole recording is + * never held in memory. + * + * "Too big" is measured on the DECODED size, estimated from duration — not on + * the file's bytes, which is close to meaningless here and is what this used to + * compare. Compression ratio is the entire point of a screen recording: a + * 32-minute capture is 68 MB on disk and 656 MB decoded, and the in-memory path + * then `slice()`s every channel again for the worker transfer. That is ~1.4 GB + * of transient allocation to draw 400 bars, and it sat comfortably under a + * 256 MB *file* threshold — so the streaming path built for exactly this case + * never ran. (`ffmpeg -vn -f null` decodes the same track in 2.1s: that is the + * floor all that allocation was being piled onto.) + */ +async function computePeaksForUrl( + videoUrl: string, + signal?: AbortSignal, + durationSec?: number, +): Promise { const isRemoteUrl = /^(https?:|blob:|data:)/i.test(videoUrl); + + // Native first. Both browser pipelines below decode the whole track in + // Chromium — 12s on a 32-minute recording, whichever one runs — where ffmpeg + // in the main process takes ~2s and caches the result on disk, so the second + // time it is free. Anything that stops this from working (no ffmpeg staged, + // an unapproved path, a clip with no audio) falls through rather than + // dropping the waveform. + if (!isRemoteUrl && durationSec && window.electronAPI?.getAudioPeaks) { + try { + const native = await window.electronAPI.getAudioPeaks(videoUrl, durationSec); + if (native.success && native.peaks && native.peaks.length > 0) return native.peaks; + } catch { + // Fall through to the browser pipelines. + } + } + if (!isRemoteUrl && window.electronAPI?.getReadableFileInfo) { const info = await window.electronAPI.getReadableFileInfo(videoUrl); - if (info.success && typeof info.size === "number" && info.size > MAX_IN_MEMORY_SOURCE_BYTES) { + const decodedBytes = (durationSec ?? 0) * DECODED_BYTES_PER_SEC; + if ( + info.success && + ((typeof info.size === "number" && info.size > MAX_IN_MEMORY_SOURCE_BYTES) || + decodedBytes > MAX_IN_MEMORY_SOURCE_BYTES) + ) { const filename = (videoUrl.split(/[\\/]/).pop() || "video").replace(/^file:/, ""); // signal also aborts the OPFS copy (unless the export shares it). const file = await materializeLocalSourceFile(videoUrl, filename, { signal }); @@ -89,16 +131,49 @@ async function computePeaksForUrl(videoUrl: string, signal?: AbortSignal): Promi return computePeaksInWorker(audioBuffer, signal); } +/** + * Peaks describe a FILE, so they are cached per file, at module scope. + * + * This used to be a `useRef` Map, i.e. one cache per mounted component. Peaks + * for a 32-minute recording cost seconds and (before the routing fix above) a + * gigabyte-plus of transient allocation, and that was paid again for every clip + * of the same asset, and again from scratch on every remount — switching + * Media↔Edit re-decoded the whole recording, which is what "the waveform takes + * ages to appear" actually was. + * + * `inFlight` is the other half: N clips of one asset mounting together must + * share a single decode instead of racing N of them. + */ +const peaksCache = new Map(); +const peaksInFlight = new Map>(); + +function loadPeaks(videoUrl: string, durationSec?: number): Promise { + const existing = peaksInFlight.get(videoUrl); + if (existing) return existing; + // Deliberately NOT wired to any component's AbortSignal: the work is shared, + // so one subscriber unmounting must not cancel it for the others. An unmount + // drops the result instead — and the cache means the next mount is free. + const promise = computePeaksForUrl(videoUrl, undefined, durationSec) + .then((p) => { + peaksCache.set(videoUrl, p); + return p; + }) + .finally(() => { + peaksInFlight.delete(videoUrl); + }); + peaksInFlight.set(videoUrl, promise); + return promise; +} + /** * Decodes audio from `videoUrl` into paired [min, max] peaks (length = 2 * N * blocks). Returns `null` while decoding, and stays `null` on no audio track or - * decode failure (silent degradation). Results are cached in a ref scoped to the - * hook instance, so they survive re-renders and waveform toggles but not unmount. + * decode failure (silent degradation). `durationSec` only picks the pipeline + * (see `computePeaksForUrl`); omitting it is safe, just slower on long files. */ -export function useAudioPeaks(videoUrl?: string): Float32Array | null { - const cacheRef = useRef>(new Map()); +export function useAudioPeaks(videoUrl?: string, durationSec?: number): Float32Array | null { const [peaks, setPeaks] = useState(() => - videoUrl ? (cacheRef.current.get(videoUrl) ?? null) : null, + videoUrl ? (peaksCache.get(videoUrl) ?? null) : null, ); useEffect(() => { @@ -107,7 +182,7 @@ export function useAudioPeaks(videoUrl?: string): Float32Array | null { return; } - const cached = cacheRef.current.get(videoUrl); + const cached = peaksCache.get(videoUrl); if (cached) { setPeaks(cached); return; @@ -115,27 +190,21 @@ export function useAudioPeaks(videoUrl?: string): Float32Array | null { setPeaks(null); let cancelled = false; - const controller = new AbortController(); - (async () => { - try { - const p = await computePeaksForUrl(videoUrl, controller.signal); - if (cancelled) return; - cacheRef.current.set(videoUrl, p); - setPeaks(p); - } catch (err) { - // AbortError means the effect cleaned up, so no state update needed. + loadPeaks(videoUrl, durationSec) + .then((p) => { + if (!cancelled) setPeaks(p); + }) + .catch((err: unknown) => { if (err instanceof DOMException && err.name === "AbortError") return; // No audio track or unsupported format: degrade to no waveform, but log // so an unexpectedly-missing waveform is diagnosable. console.warn("useAudioPeaks: could not decode audio for waveform:", err); if (!cancelled) setPeaks(null); - } - })(); + }); return () => { cancelled = true; - controller.abort(); }; }, [videoUrl]); diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 255bad4c..81f5fe6c 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "النص فارغ.", "notGeneratedHint": "لم يُنشأ بعد — اختر لغة وانقر على إعادة الإنشاء.", "transcribing": "جارٍ النسخ", + "downloadingModel": "جارٍ تنزيل نموذج الكلام", "transcribingEllipsis": "جارٍ النسخ…", "pendingTranscription": "بانتظار النسخ", "transcriptionFailed": "فشل النسخ", diff --git a/src/i18n/locales/ar/timeline.json b/src/i18n/locales/ar/timeline.json index a47328e8..011483f8 100644 --- a/src/i18n/locales/ar/timeline.json +++ b/src/i18n/locales/ar/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "أدوات المخطط الزمني", "arrangeClips": "ترتيب المقاطع", "arrangeClipsHint": "اسحب المقاطع أدناه لإعادة ترتيبها أو إسقاط مقاطع جديدة", - "newAnnotation": "شرح جديد", + "newAnnotation": "شرح", "dragToReorderHint": "اسحب لإعادة الترتيب · انقر نقرًا مزدوجًا لتعديل نقطتي البداية والنهاية", "editInOutPoints": "تعديل نقطتي البداية والنهاية", "deleteClip": "حذف المقطع", diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 2e270ffb..ae85f095 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -134,6 +134,7 @@ "close": "Close", "transcriptReady": "Transcript ready", "transcribing": "Transcribing", + "downloadingModel": "Downloading speech model", "transcribingEllipsis": "Transcribing…", "pendingTranscription": "Pending transcription", "transcriptionFailed": "Transcription failed", diff --git a/src/i18n/locales/en/timeline.json b/src/i18n/locales/en/timeline.json index d1940b2e..517024fc 100644 --- a/src/i18n/locales/en/timeline.json +++ b/src/i18n/locales/en/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "Timeline tools", "arrangeClips": "Arrange clips", "arrangeClipsHint": "Drag clips below to reorder or drop new ones in", - "newAnnotation": "New annotation", + "newAnnotation": "Annotation", "dragToReorderHint": "Drag to reorder · double-click to edit in/out points", "editInOutPoints": "Edit in/out points", "deleteClip": "Delete clip", diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index 8f5ba5be..a5773c4b 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "La transcripción está vacía.", "notGeneratedHint": "Aún no generada — elige un idioma y haz clic en regenerar.", "transcribing": "Transcribiendo", + "downloadingModel": "Descargando modelo de voz", "transcribingEllipsis": "Transcribiendo…", "pendingTranscription": "Transcripción pendiente", "transcriptionFailed": "Error de transcripción", diff --git a/src/i18n/locales/es/timeline.json b/src/i18n/locales/es/timeline.json index 4226ead0..a4b4124b 100644 --- a/src/i18n/locales/es/timeline.json +++ b/src/i18n/locales/es/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "Herramientas de la línea de tiempo", "arrangeClips": "Organizar clips", "arrangeClipsHint": "Arrastra los clips de abajo para reordenarlos o suelta otros nuevos", - "newAnnotation": "Nueva anotación", + "newAnnotation": "Anotación", "dragToReorderHint": "Arrastra para reordenar · doble clic para editar los puntos de entrada/salida", "editInOutPoints": "Editar puntos de entrada/salida", "deleteClip": "Eliminar clip", diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 5cda995b..ba0afda6 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "La transcription est vide.", "notGeneratedHint": "Pas encore générée — choisissez une langue et cliquez sur régénérer.", "transcribing": "Transcription en cours", + "downloadingModel": "Téléchargement du modèle vocal", "transcribingEllipsis": "Transcription en cours…", "pendingTranscription": "Transcription en attente", "transcriptionFailed": "Échec de la transcription", diff --git a/src/i18n/locales/fr/timeline.json b/src/i18n/locales/fr/timeline.json index 210fbfe4..8e871599 100644 --- a/src/i18n/locales/fr/timeline.json +++ b/src/i18n/locales/fr/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "Outils de la timeline", "arrangeClips": "Organiser les clips", "arrangeClipsHint": "Glissez les clips ci-dessous pour les réorganiser ou en déposer de nouveaux", - "newAnnotation": "Nouvelle annotation", + "newAnnotation": "Annotation", "dragToReorderHint": "Glissez pour réorganiser · double-cliquez pour modifier les points d'entrée/sortie", "editInOutPoints": "Modifier les points d'entrée/sortie", "deleteClip": "Supprimer le clip", diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index ecc998b7..926b5638 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "La trascrizione è vuota.", "notGeneratedHint": "Non ancora generata — scegli una lingua e clicca su rigenera.", "transcribing": "Trascrizione in corso", + "downloadingModel": "Download del modello vocale", "transcribingEllipsis": "Trascrizione in corso…", "pendingTranscription": "Trascrizione in attesa", "transcriptionFailed": "Trascrizione non riuscita", diff --git a/src/i18n/locales/it/timeline.json b/src/i18n/locales/it/timeline.json index ed9f2f9f..8390254c 100644 --- a/src/i18n/locales/it/timeline.json +++ b/src/i18n/locales/it/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "Strumenti della timeline", "arrangeClips": "Organizza clip", "arrangeClipsHint": "Trascina le clip qui sotto per riordinarle o rilasciane di nuove", - "newAnnotation": "Nuova annotazione", + "newAnnotation": "Annotazione", "dragToReorderHint": "Trascina per riordinare · doppio clic per modificare i punti di entrata/uscita", "editInOutPoints": "Modifica punti di entrata/uscita", "deleteClip": "Elimina clip", diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 615cf15a..7f83fcc5 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "文字起こしは空です。", "notGeneratedHint": "まだ生成されていません — 言語を選んで再生成をクリックしてください。", "transcribing": "文字起こし中", + "downloadingModel": "音声モデルをダウンロード中", "transcribingEllipsis": "文字起こし中…", "pendingTranscription": "文字起こし待機中", "transcriptionFailed": "文字起こしに失敗しました", diff --git a/src/i18n/locales/ja-JP/timeline.json b/src/i18n/locales/ja-JP/timeline.json index 2f12a3b9..a2c9a88e 100644 --- a/src/i18n/locales/ja-JP/timeline.json +++ b/src/i18n/locales/ja-JP/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "タイムラインツール", "arrangeClips": "クリップを配置", "arrangeClipsHint": "下のクリップをドラッグして並べ替えるか、新しいクリップをドロップします", - "newAnnotation": "新しい注釈", + "newAnnotation": "注釈", "dragToReorderHint": "ドラッグして並べ替え・ダブルクリックでイン/アウトポイントを編集", "editInOutPoints": "イン/アウトポイントを編集", "deleteClip": "クリップを削除", diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index 190b9af0..6e2bc465 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "대본이 비어 있습니다.", "notGeneratedHint": "아직 생성되지 않음 — 언어를 선택하고 재생성을 클릭하세요.", "transcribing": "받아쓰는 중", + "downloadingModel": "음성 모델 다운로드 중", "transcribingEllipsis": "받아쓰는 중…", "pendingTranscription": "받아쓰기 대기 중", "transcriptionFailed": "받아쓰기 실패", diff --git a/src/i18n/locales/ko-KR/timeline.json b/src/i18n/locales/ko-KR/timeline.json index d0503564..75417443 100644 --- a/src/i18n/locales/ko-KR/timeline.json +++ b/src/i18n/locales/ko-KR/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "타임라인 도구", "arrangeClips": "클립 정리", "arrangeClipsHint": "아래 클립을 드래그하여 순서를 바꾸거나 새 클립을 놓으세요", - "newAnnotation": "새 주석", + "newAnnotation": "주석", "dragToReorderHint": "드래그하여 순서 변경 · 더블클릭하여 시작/종료 지점 편집", "editInOutPoints": "시작/종료 지점 편집", "deleteClip": "클립 삭제", diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index b7d90914..bf26c473 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "A transcrição está vazia.", "notGeneratedHint": "Ainda não gerada — escolha um idioma e clique em regenerar.", "transcribing": "Transcrevendo", + "downloadingModel": "Baixando modelo de voz", "transcribingEllipsis": "Transcrevendo…", "pendingTranscription": "Transcrição pendente", "transcriptionFailed": "Falha na transcrição", diff --git a/src/i18n/locales/pt-BR/timeline.json b/src/i18n/locales/pt-BR/timeline.json index 4a70046d..4cb43bc0 100644 --- a/src/i18n/locales/pt-BR/timeline.json +++ b/src/i18n/locales/pt-BR/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "Ferramentas da linha do tempo", "arrangeClips": "Organizar clipes", "arrangeClipsHint": "Arraste os clipes abaixo para reordená-los ou solte novos", - "newAnnotation": "Nova anotação", + "newAnnotation": "Anotação", "dragToReorderHint": "Arraste para reordenar · clique duas vezes para editar os pontos de entrada/saída", "editInOutPoints": "Editar pontos de entrada/saída", "deleteClip": "Excluir clipe", diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index b50d5a98..ccd84010 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "Транскрипт пуст.", "notGeneratedHint": "Ещё не создан — выберите язык и нажмите «Пересоздать».", "transcribing": "Расшифровка", + "downloadingModel": "Загрузка речевой модели", "transcribingEllipsis": "Расшифровка…", "pendingTranscription": "Ожидает расшифровки", "transcriptionFailed": "Ошибка расшифровки", diff --git a/src/i18n/locales/ru/timeline.json b/src/i18n/locales/ru/timeline.json index ebd77431..eb679791 100644 --- a/src/i18n/locales/ru/timeline.json +++ b/src/i18n/locales/ru/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "Инструменты таймлайна", "arrangeClips": "Упорядочить клипы", "arrangeClipsHint": "Перетащите клипы ниже, чтобы изменить порядок, или добавьте новые", - "newAnnotation": "Новая аннотация", + "newAnnotation": "Аннотация", "dragToReorderHint": "Перетащите для изменения порядка · дважды щёлкните для редактирования точек входа/выхода", "editInOutPoints": "Редактировать точки входа/выхода", "deleteClip": "Удалить клип", diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index 0c1d7734..cc11d3c0 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "Metin dökümü boş.", "notGeneratedHint": "Henüz oluşturulmadı — bir dil seçin ve yeniden oluştur'a tıklayın.", "transcribing": "Metne dökülüyor", + "downloadingModel": "Konuşma modeli indiriliyor", "transcribingEllipsis": "Metne dökülüyor…", "pendingTranscription": "Metne dökme bekliyor", "transcriptionFailed": "Metne dökme başarısız oldu", diff --git a/src/i18n/locales/tr/timeline.json b/src/i18n/locales/tr/timeline.json index d8f4431c..1b36c8e4 100644 --- a/src/i18n/locales/tr/timeline.json +++ b/src/i18n/locales/tr/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "Zaman çizelgesi araçları", "arrangeClips": "Klipleri düzenle", "arrangeClipsHint": "Yeniden sıralamak için aşağıdaki klipleri sürükleyin veya yenilerini bırakın", - "newAnnotation": "Yeni açıklama", + "newAnnotation": "Açıklama", "dragToReorderHint": "Yeniden sıralamak için sürükleyin · giriş/çıkış noktalarını düzenlemek için çift tıklayın", "editInOutPoints": "Giriş/çıkış noktalarını düzenle", "deleteClip": "Klibi sil", diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index 0ad0a044..ba7acfc6 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "Bản ghi lời thoại trống.", "notGeneratedHint": "Chưa được tạo — chọn ngôn ngữ và nhấp vào tạo lại.", "transcribing": "Đang phiên âm", + "downloadingModel": "Đang tải mô hình giọng nói", "transcribingEllipsis": "Đang phiên âm…", "pendingTranscription": "Đang chờ phiên âm", "transcriptionFailed": "Phiên âm thất bại", diff --git a/src/i18n/locales/vi/timeline.json b/src/i18n/locales/vi/timeline.json index 4b9218ba..9162539f 100644 --- a/src/i18n/locales/vi/timeline.json +++ b/src/i18n/locales/vi/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "Công cụ dòng thời gian", "arrangeClips": "Sắp xếp clip", "arrangeClipsHint": "Kéo các clip bên dưới để sắp xếp lại hoặc thả clip mới vào", - "newAnnotation": "Chú thích mới", + "newAnnotation": "Chú thích", "dragToReorderHint": "Kéo để sắp xếp lại · nhấp đúp để chỉnh sửa điểm vào/ra", "editInOutPoints": "Chỉnh sửa điểm vào/ra", "deleteClip": "Xóa clip", diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 35610a0b..d4dfe179 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "转录内容为空。", "notGeneratedHint": "尚未生成 — 选择语言并点击重新生成。", "transcribing": "正在转录", + "downloadingModel": "正在下载语音模型", "transcribingEllipsis": "正在转录…", "pendingTranscription": "等待转录", "transcriptionFailed": "转录失败", diff --git a/src/i18n/locales/zh-CN/timeline.json b/src/i18n/locales/zh-CN/timeline.json index f1065037..9a278d0f 100644 --- a/src/i18n/locales/zh-CN/timeline.json +++ b/src/i18n/locales/zh-CN/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "时间轴工具", "arrangeClips": "排列片段", "arrangeClipsHint": "拖动下方片段以重新排序，或拖入新片段", - "newAnnotation": "新建标注", + "newAnnotation": "标注", "dragToReorderHint": "拖动以重新排序 · 双击以编辑入点/出点", "editInOutPoints": "编辑入点/出点", "deleteClip": "删除片段", diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index c3e7e55e..46017c8a 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -140,6 +140,7 @@ "transcriptEmpty": "逐字稿為空。", "notGeneratedHint": "尚未產生 — 選擇語言並點擊重新產生。", "transcribing": "轉錄中", + "downloadingModel": "正在下載語音模型", "transcribingEllipsis": "轉錄中…", "pendingTranscription": "等待轉錄", "transcriptionFailed": "轉錄失敗", diff --git a/src/i18n/locales/zh-TW/timeline.json b/src/i18n/locales/zh-TW/timeline.json index 94fce3f4..036e4658 100644 --- a/src/i18n/locales/zh-TW/timeline.json +++ b/src/i18n/locales/zh-TW/timeline.json @@ -71,7 +71,7 @@ "timelineTools": "時間軸工具", "arrangeClips": "排列片段", "arrangeClipsHint": "拖曳下方片段以重新排序，或拖曳新片段至此", - "newAnnotation": "新增註解", + "newAnnotation": "註解", "dragToReorderHint": "拖曳以重新排序 · 按兩下以編輯入點/出點", "editInOutPoints": "編輯入點/出點", "deleteClip": "刪除片段", diff --git a/src/lib/ai-edition/document/transcribe.ts b/src/lib/ai-edition/document/transcribe.ts index 460828b6..0c0a23aa 100644 --- a/src/lib/ai-edition/document/transcribe.ts +++ b/src/lib/ai-edition/document/transcribe.ts @@ -9,9 +9,20 @@ import { toFileUrl } from "@/components/video-editor/projectPersistence"; import { extractMono16kFromVideoUrl, transcribeMono16kToSegments } from "@/lib/captioning"; import type { AxcutDocument, AxcutTranscript, AxcutTranscriptSegment, AxcutWord } from "../schema"; +/** + * What the caller can show while a transcription runs. `completedSec` / + * `totalSec` arrive only during `"transcribing"`, once the main process starts + * landing chunks — until then the phase alone is all there is to show. + */ +export interface TranscribeStatus { + phase: "extracting-audio" | "loading-model" | "transcribing"; + completedSec?: number; + totalSec?: number; +} + export interface TranscribeAssetOptions { language?: string; - onStatus?: (status: string) => void; + onStatus?: (status: TranscribeStatus) => void; signal?: AbortSignal; } @@ -27,12 +38,12 @@ export async function transcribeAsset( const videoUrl = toFileUrl(asset.originalPath); - options.onStatus?.("extracting-audio"); + options.onStatus?.({ phase: "extracting-audio" }); const audioResult = await extractMono16kFromVideoUrl(videoUrl, { signal: options.signal, }); - options.onStatus?.("transcribing"); + options.onStatus?.({ phase: "transcribing" }); // Only pass `language` to the worker when the caller forced a specific // code. `"auto"` (or any falsy value) leaves Whisper to detect from // the audio. The pipeline tags every chunk with the language it used @@ -44,6 +55,15 @@ export async function transcribeAsset( trimRegions: [], signal: options.signal, language: forcedLanguage, + // Forward the main process's per-chunk progress. Without this the status + // callback only ever fired the two coarse phases above, so a 30-minute + // recording showed one static "transcribing" for ten minutes. + onStatus: (status) => + options.onStatus?.({ + phase: status.phase === "model" ? "loading-model" : "transcribing", + completedSec: status.completedSec, + totalSec: status.totalSec, + }), }); const segments: AxcutTranscriptSegment[] = []; diff --git a/src/lib/ai-edition/store/regionClipboard.ts b/src/lib/ai-edition/store/regionClipboard.ts index 96758f65..ef1116ff 100644 --- a/src/lib/ai-edition/store/regionClipboard.ts +++ b/src/lib/ai-edition/store/regionClipboard.ts @@ -10,7 +10,13 @@ export type RegionSnapshot = | { kind: "zoom"; region: Record } | { kind: "annotation"; region: Record } | { kind: "speed"; region: Record } - | { kind: "cameraFullscreen"; region: Record }; + | { kind: "cameraFullscreen"; region: Record } + // A trim carries no user-visible properties, so all there is to copy is how + // LONG it was — `{ durationSec }`. That is not a special case so much as the + // general one made obvious: every paste keeps the copied properties and takes + // its start from the playhead, so a zoom's start/end already change too. A + // trim just has nothing left once you remove position. + | { kind: "trim"; region: { durationSec: number } }; let clipboard: RegionSnapshot | null = null; const listeners = new Set<() => void>(); @@ -24,6 +30,13 @@ export function copyRegion(snap: RegionSnapshot) { export function pasteClipboard(): RegionSnapshot | null { return clipboard; } +/** Empty it. What the user copied LAST is what Ctrl+V must paste, so copying a + * clip has to retire whatever region sat here — otherwise both clipboards stay + * loaded at once and paste is left guessing between them. */ +export function clearRegionClipboard() { + clipboard = null; + notify(); +} export function useRegionClipboard() { const [, force] = useState(0); useEffect(() => { diff --git a/src/lib/ai-edition/store/transcriptionStore.ts b/src/lib/ai-edition/store/transcriptionStore.ts index 9d765afc..1cee92d1 100644 --- a/src/lib/ai-edition/store/transcriptionStore.ts +++ b/src/lib/ai-edition/store/transcriptionStore.ts @@ -40,6 +40,7 @@ import { type TranscriptGate, type TranscriptionFailure, type TranscriptionPhase, + type TranscriptionProgress, transcriptRelevantAssetIds, } from "../transcription/status"; import { useProjectStore } from "./projectStore"; @@ -50,6 +51,8 @@ export interface TranscriptionJob { * finishes after the user asked for another one cannot clear its successor. */ runId?: number; phase?: TranscriptionPhase; + /** Chunk progress while transcribing; absent until the first chunk lands. */ + progress?: TranscriptionProgress; /** `"auto"` unless the user forced a language from the media card. */ language: string; failure?: TranscriptionFailure; @@ -273,7 +276,7 @@ function failRemainingQueue(projectId: string, failure: TranscriptionFailure): v for (const assetId of queued) { const job = jobs[assetId]; if (job?.status !== "queued") continue; - jobs[assetId] = { ...job, status: "failed", phase: undefined, failure }; + jobs[assetId] = { ...job, status: "failed", phase: undefined, progress: undefined, failure }; } return { jobs }; }); @@ -365,7 +368,19 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise { const transcript = await transcribeAsset(doc, assetId, { language: job.language, signal: controller.signal, - onStatus: (phase) => patchJob(assetId, runId, { phase: phase as TranscriptionPhase }), + // `TranscribeStatus` and `TranscriptionPhase` are the same vocabulary on + // purpose (see status.ts), so this no longer needs a cast. `progress` + // only arrives during "transcribing"; carrying it through undefined the + // rest of the time is what lets the UI fall back to a spinner instead + // of a bar frozen at 0%. + onStatus: (status) => + patchJob(assetId, runId, { + phase: status.phase, + progress: + status.completedSec !== undefined && status.totalSec !== undefined + ? { completedSec: status.completedSec, totalSec: status.totalSec } + : undefined, + }), }); if (controller.signal.aborted) { dropJob(assetId, runId); @@ -400,7 +415,7 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise { } if (!isCurrentRun(assetId, runId)) return; // superseded by a newer request const failure = classifyTranscriptionError(error); - patchJob(assetId, runId, { status: "failed", phase: undefined, failure }); + patchJob(assetId, runId, { status: "failed", phase: undefined, progress: undefined, failure }); flushSettleWaiters(assetId); await persistPermanentFailure(projectId, assetId, failure); // A transient failure is about the ENGINE, not about this media: the model diff --git a/src/lib/ai-edition/store/useTimeline.test.ts b/src/lib/ai-edition/store/useTimeline.test.ts index 74ad15de..ca1edf89 100644 --- a/src/lib/ai-edition/store/useTimeline.test.ts +++ b/src/lib/ai-edition/store/useTimeline.test.ts @@ -611,4 +611,60 @@ describe("useTimeline is not re-rendered by playhead ticks", () => { endMs: 6200, }); }); + + // Pasting a copied trim is exactly this call: a trim carries no properties, so + // all a copy holds is its length, and paste recreates one that long at the + // playhead. Same primitive the toolbar's cut button uses. + it("creates a trim of the requested length", async () => { + const { result } = renderTimeline(); + act(() => { + useProjectStore.getState().setCurrentTime(3); + }); + await act(async () => { + await result.current.addTrim(1.25); + }); + const trim = useProjectStore.getState().document?.timeline.trimRanges.at(-1); + expect((trim?.endSec ?? 0) - (trim?.startSec ?? 0)).toBeCloseTo(1.25, 6); + }); + + // The timeline's toolbar passes a duration worth a fixed number of pixels at + // the current zoom; every other entry point keeps the 2 s above. + it("honours a caller-supplied duration", async () => { + const { result } = renderTimeline(); + act(() => { + useProjectStore.getState().setCurrentTime(4.2); + }); + await act(async () => { + await result.current.addZoom(0.4); + }); + expect(useProjectStore.getState().document?.zoomRanges.at(-1)).toMatchObject({ + startMs: 4200, + endMs: 4600, + }); + }); +}); + +describe("useTimeline selection", () => { + // A pill and a clip are one selection, not two. While both could be set at + // once, copy/paste keyed off "is a clip selected?" and so acted on the clip + // whatever the user had actually clicked. + it("lets a clip and a pill cancel each other", () => { + const { result } = renderTimeline(); + + act(() => result.current.selectClip("clip_1")); + expect(result.current.clipSelection).toBe("clip_1"); + + act(() => result.current.selectRegion("zoom", "z1")); + expect(result.current.selection).toMatchObject({ kind: "zoom", id: "z1" }); + expect(result.current.clipSelection).toBeNull(); + + act(() => result.current.selectClip("clip_2")); + expect(result.current.clipSelection).toBe("clip_2"); + expect(result.current.selection).toBeNull(); + expect(result.current.multiSelection).toEqual([]); + + act(() => result.current.clearSelection()); + expect(result.current.selection).toBeNull(); + expect(result.current.clipSelection).toBeNull(); + }); }); diff --git a/src/lib/ai-edition/store/useTimeline.ts b/src/lib/ai-edition/store/useTimeline.ts index ba20250d..28c499ed 100644 --- a/src/lib/ai-edition/store/useTimeline.ts +++ b/src/lib/ai-edition/store/useTimeline.ts @@ -31,6 +31,13 @@ import { dropTrimPillsByIds, resolveTimelineSpanToTrim } from "../timeline/trim- import type { AutoZoomSuggestion } from "../timeline/zoom-suggestions"; import { useProjectStore } from "./projectStore"; +// How long a region lasts when the caller doesn't say. The timeline's toolbar +// passes its own duration instead, derived from the current zoom so the new pill +// always comes out the same WIDTH on screen (see PILL_CREATE_PX in V4Timeline). +// Every other entry point — keyboard shortcuts, the agent, auto-zooms — gets +// these 2 s, which is what all five add* used to hardcode. +const DEFAULT_NEW_REGION_SEC = 2; + // NaN-guarded floors. Timeline inputs arrive from drag deltas and persisted // documents, both of which can carry NaN; every action needs the same guard. const finiteSec = (n: number) => (Number.isFinite(n) ? Math.max(0, n) : 0); @@ -155,29 +162,33 @@ export function useTimeline() { // technical-documentation/architecture/timeline-model.md) — writing only startMs/endMs // would strand it. A region created across a clip boundary becomes one fragment per // clip; the ruler renders them as one pill because their properties are equal. - const addZoom = useCallback(async () => { - if (!document) return; - const timeMs = Math.round(playheadSec() * 1000); - const anchored = anchorRegionsWithDerivedMs( - [ - { - id: createId("zoom"), - startMs: timeMs, - endMs: timeMs + 2000, - depth: 3, - focus: { cx: 0.5, cy: 0.5 }, - focusMode: "manual" as const, - }, - ], - document.timeline.clips, - () => createId("zoom"), - ); - const next: AxcutDocument = { - ...document, - zoomRanges: [...document.zoomRanges, ...anchored] as AxcutDocument["zoomRanges"], - }; - await saveDocument(next); - }, [document, saveDocument]); + const addZoom = useCallback( + async (durationSec = DEFAULT_NEW_REGION_SEC) => { + if (!document) return; + const timeMs = Math.round(playheadSec() * 1000); + const endMs = timeMs + Math.round(durationSec * 1000); + const anchored = anchorRegionsWithDerivedMs( + [ + { + id: createId("zoom"), + startMs: timeMs, + endMs, + depth: 3, + focus: { cx: 0.5, cy: 0.5 }, + focusMode: "manual" as const, + }, + ], + document.timeline.clips, + () => createId("zoom"), + ); + const next: AxcutDocument = { + ...document, + zoomRanges: [...document.zoomRanges, ...anchored] as AxcutDocument["zoomRanges"], + }; + await saveDocument(next); + }, + [document, saveDocument], + ); // Append several auto-generated zoom regions in one save (auto-enhance). // Suggestions come from buildAutoZoomSuggestions, which already reserves @@ -212,135 +223,153 @@ export function useTimeline() { [document, saveDocument], ); - const addTrim = useCallback(async () => { - if (!document) return; - // Insert a 2s trim at the playhead in *timeline* time, then resolve it - // down to the correct clip's asset + source-time. Writing currentTimeSec - // straight into startSec (as before) only happened to be right for an - // identity single-clip project — for trimmed/reordered clips it landed - // the trim at the wrong source position. - const playhead = playheadSec(); - const resolved = resolveTimelineSpanToTrim(playhead, playhead + 2, document.timeline.clips); - const asset = - document.assets.find((a) => a.id === document.project.primaryAssetId) ?? document.assets[0]; - if (!resolved && !asset) return; - const next: AxcutDocument = { - ...document, - timeline: { - ...document.timeline, - trimRanges: [ - ...document.timeline.trimRanges, - { - id: createId("trim"), - assetId: resolved?.assetId ?? asset!.id, - // The carrier clip, so the cut lands on THAT clip and not on every clip - // sharing its media (see `trimAppliesToClip`). Absent only in the - // no-clip fallback below, where there is no clip to name. - ...(resolved ? { clipId: resolved.clipId } : {}), - startSec: resolved?.sourceStartSec ?? playhead, - endSec: resolved?.sourceEndSec ?? playhead + 2, - reason: "manual", - origin: "user" as const, - }, - ], - }, - }; - await saveDocument(next); - }, [document, saveDocument]); + const addTrim = useCallback( + async (durationSec = DEFAULT_NEW_REGION_SEC) => { + if (!document) return; + // Insert a 2s trim at the playhead in *timeline* time, then resolve it + // down to the correct clip's asset + source-time. Writing currentTimeSec + // straight into startSec (as before) only happened to be right for an + // identity single-clip project — for trimmed/reordered clips it landed + // the trim at the wrong source position. + const playhead = playheadSec(); + const end = playhead + durationSec; + const resolved = resolveTimelineSpanToTrim(playhead, end, document.timeline.clips); + const asset = + document.assets.find((a) => a.id === document.project.primaryAssetId) ?? document.assets[0]; + if (!resolved && !asset) return; + const next: AxcutDocument = { + ...document, + timeline: { + ...document.timeline, + trimRanges: [ + ...document.timeline.trimRanges, + { + id: createId("trim"), + assetId: resolved?.assetId ?? asset!.id, + // The carrier clip, so the cut lands on THAT clip and not on every clip + // sharing its media (see `trimAppliesToClip`). Absent only in the + // no-clip fallback below, where there is no clip to name. + ...(resolved ? { clipId: resolved.clipId } : {}), + startSec: resolved?.sourceStartSec ?? playhead, + endSec: resolved?.sourceEndSec ?? end, + reason: "manual", + origin: "user" as const, + }, + ], + }, + }; + await saveDocument(next); + }, + [document, saveDocument], + ); - const addAnnotation = useCallback(async () => { - if (!document) return; - const timeMs = Math.round(playheadSec() * 1000); - const ann: AnnotationRegion = { - id: createId("ann"), - startMs: timeMs, - endMs: timeMs + 2000, - type: "text" as AnnotationType, - // Real, localised text rather than an empty field. An empty annotation - // renders nothing at all, so the user added a region and saw no change - // on the canvas; the inspector's placeholder is CSS ghost text that - // never reaches `content`, so it never reached the compositor either. - // `textContent` stays empty because the render path reads - // `content || textContent` and seeding both would just duplicate it. - content: ts("annotation.defaultText"), - textContent: "", - position: { x: 50, y: 50 }, - size: { width: 30, height: 20 }, - style: { - color: "#ffffff", - backgroundColor: "transparent", - fontSize: 32, - fontFamily: "Inter", - fontWeight: "bold", - fontStyle: "normal", - textDecoration: "none", - textAlign: "center", - textAnimation: "none", - }, - zIndex: document.annotations.length + 1, - }; - const created = anchorRegionsWithDerivedMs([ann], document.timeline.clips, () => - createId("ann"), - ); - const next: AxcutDocument = { - ...document, - annotations: [...document.annotations, ...created] as unknown as AxcutDocument["annotations"], - }; - await saveDocument(next); - // Select the freshly added annotation so its inspector opens and it shows a - // selection box on the canvas, ready to be retyped over. - const newId = created[0]?.id ?? ann.id; - setMultiSelection([{ kind: "annotation", id: newId }]); - setSelection({ kind: "annotation", id: newId }); - // `ts` is memoised on [locale, namespace] by useScopedT, so this does not - // churn the callback identity between renders. - }, [document, saveDocument, ts]); - - const addSpeed = useCallback(async () => { - if (!document) return; - const timeMs = Math.round(playheadSec() * 1000); - const legacy = (document.legacyEditor as Record) ?? {}; - const prev = (legacy.speedRegions as unknown[]) ?? []; - const next: AxcutDocument = { - ...document, - legacyEditor: { - ...legacy, - speedRegions: [ - ...prev, - ...anchorRegionsWithDerivedMs( - [{ id: createId("speed"), startMs: timeMs, endMs: timeMs + 2000, speed: 1.5 as const }], - document.timeline.clips, - () => createId("speed"), - ), - ], - }, - }; - await saveDocument(next); - }, [document, saveDocument]); + const addAnnotation = useCallback( + async (durationSec = DEFAULT_NEW_REGION_SEC) => { + if (!document) return; + const timeMs = Math.round(playheadSec() * 1000); + const ann: AnnotationRegion = { + id: createId("ann"), + startMs: timeMs, + endMs: timeMs + Math.round(durationSec * 1000), + type: "text" as AnnotationType, + // Real, localised text rather than an empty field. An empty annotation + // renders nothing at all, so the user added a region and saw no change + // on the canvas; the inspector's placeholder is CSS ghost text that + // never reaches `content`, so it never reached the compositor either. + // `textContent` stays empty because the render path reads + // `content || textContent` and seeding both would just duplicate it. + content: ts("annotation.defaultText"), + textContent: "", + position: { x: 50, y: 50 }, + size: { width: 30, height: 20 }, + style: { + color: "#ffffff", + backgroundColor: "transparent", + fontSize: 32, + fontFamily: "Inter", + fontWeight: "bold", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", + textAnimation: "none", + }, + zIndex: document.annotations.length + 1, + }; + const created = anchorRegionsWithDerivedMs([ann], document.timeline.clips, () => + createId("ann"), + ); + const next: AxcutDocument = { + ...document, + annotations: [ + ...document.annotations, + ...created, + ] as unknown as AxcutDocument["annotations"], + }; + await saveDocument(next); + // Select the freshly added annotation so its inspector opens and it shows a + // selection box on the canvas, ready to be retyped over. + const newId = created[0]?.id ?? ann.id; + setMultiSelection([{ kind: "annotation", id: newId }]); + setSelection({ kind: "annotation", id: newId }); + // `ts` is memoised on [locale, namespace] by useScopedT, so this does not + // churn the callback identity between renders. + }, + [document, saveDocument, ts], + ); + + const addSpeed = useCallback( + async (durationSec = DEFAULT_NEW_REGION_SEC) => { + if (!document) return; + const timeMs = Math.round(playheadSec() * 1000); + const endMs = timeMs + Math.round(durationSec * 1000); + const legacy = (document.legacyEditor as Record) ?? {}; + const prev = (legacy.speedRegions as unknown[]) ?? []; + const next: AxcutDocument = { + ...document, + legacyEditor: { + ...legacy, + speedRegions: [ + ...prev, + ...anchorRegionsWithDerivedMs( + [{ id: createId("speed"), startMs: timeMs, endMs, speed: 1.5 as const }], + document.timeline.clips, + () => createId("speed"), + ), + ], + }, + }; + await saveDocument(next); + }, + [document, saveDocument], + ); // Full Camera: a plain time span (no value) during which the preview/export // grows the webcam overlay to (almost) fill the canvas and eases it back. - const addCameraFullscreen = useCallback(async () => { - if (!document) return; - const timeMs = Math.round(playheadSec() * 1000); - const legacy = (document.legacyEditor as Record) ?? {}; - const prev = (legacy.cameraFullscreenRegions as unknown[]) ?? []; - const next: AxcutDocument = { - ...document, - legacyEditor: { - ...legacy, - cameraFullscreenRegions: [ - ...prev, - ...anchorRegionsWithDerivedMs( - [{ id: createId("camfull"), startMs: timeMs, endMs: timeMs + 2000 }], - document.timeline.clips, - () => createId("camfull"), - ), - ], - }, - }; - await saveDocument(next); - }, [document, saveDocument]); + const addCameraFullscreen = useCallback( + async (durationSec = DEFAULT_NEW_REGION_SEC) => { + if (!document) return; + const timeMs = Math.round(playheadSec() * 1000); + const endMs = timeMs + Math.round(durationSec * 1000); + const legacy = (document.legacyEditor as Record) ?? {}; + const prev = (legacy.cameraFullscreenRegions as unknown[]) ?? []; + const next: AxcutDocument = { + ...document, + legacyEditor: { + ...legacy, + cameraFullscreenRegions: [ + ...prev, + ...anchorRegionsWithDerivedMs( + [{ id: createId("camfull"), startMs: timeMs, endMs }], + document.timeline.clips, + () => createId("camfull"), + ), + ], + }, + }; + await saveDocument(next); + }, + [document, saveDocument], + ); // Like updateTrimRange but also re-attaches the trim to a (possibly different) CLIP — // needed when a trim is dragged across a clip boundary, whether or not the landing clip @@ -721,9 +750,16 @@ export function useTimeline() { [document, saveDocument], ); + // Selecting a pill and selecting a clip are the SAME act — "this is the thing + // I mean" — so they cancel each other. They used to be two states that could + // both be set: the user saw one highlighted element while the app still held + // the other, and everything keyed off "is a clip selected?" (copy, paste, + // delete) silently acted on the invisible one. Copy/paste is where it showed: + // it always operated on the clip, whatever the user had just clicked. const selectRegion = useCallback( (kind: RegionKind, id: string, opts?: { additive?: boolean }) => { const handle = { kind, id }; + setClipSelection(null); if (opts?.additive) { // Shift-click toggles membership; the focused region follows the click. setMultiSelection((prev) => { @@ -742,6 +778,7 @@ export function useTimeline() { const clearSelection = useCallback(() => { setSelection(null); setMultiSelection([]); + setClipSelection(null); }, []); // Axcut-consistent clip trim: only the source range is user-editable (the @@ -943,7 +980,12 @@ export function useTimeline() { [document, clipSelection, saveDocument], ); - const selectClip = useCallback((id: string) => setClipSelection(id), []); + // Mirror of selectRegion: picking a clip retires the pill selection. + const selectClip = useCallback((id: string) => { + setClipSelection(id); + setSelection(null); + setMultiSelection([]); + }, []); const speedRegions = hasDoc ? (((document.legacyEditor as Record | null)?.speedRegions as Array<{ diff --git a/src/lib/ai-edition/timeline/newRegionDuration.test.ts b/src/lib/ai-edition/timeline/newRegionDuration.test.ts new file mode 100644 index 00000000..ed68dbc1 --- /dev/null +++ b/src/lib/ai-edition/timeline/newRegionDuration.test.ts @@ -0,0 +1,50 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + newRegionDurationSec, + PILL_CREATE_MIN_SEC, + PILL_CREATE_PX, + setTimelineScale, +} from "./newRegionDuration"; + +describe("newRegionDurationSec", () => { + beforeEach(() => setTimelineScale(0)); + + it("trades duration for a constant width", () => { + // The reported case: 30 minutes across a ~760px panel is 0.42 px/s, where a + // flat 2 s region is under a pixel — invisible behind the playhead it was + // created at. At that scale a readable pill is worth nearly four minutes. + setTimelineScale(760 / 1800); + expect(newRegionDurationSec()).toBeCloseTo(227.4, 1); + + // Zoom in 50x and the same gesture creates a region 50x shorter — the pill + // on screen is the same size either way, which is the whole point. + setTimelineScale((760 / 1800) * 50); + expect(newRegionDurationSec()).toBeCloseTo(4.55, 2); + }); + + it("stays at the floor when the pixels are worth almost nothing", () => { + // 40px of a 3-second clip zoomed to the ceiling: without the floor the + // region would be born a few hundredths of a second long. + setTimelineScale(10_000); + expect(newRegionDurationSec()).toBe(PILL_CREATE_MIN_SEC); + }); + + it("says nothing at all before the timeline has been measured", () => { + // First paint, or no timeline mounted: callers fall back to their own + // default rather than deriving a length from a width of zero. + expect(newRegionDurationSec()).toBeUndefined(); + setTimelineScale(Number.NaN); + expect(newRegionDurationSec()).toBeUndefined(); + setTimelineScale(Number.POSITIVE_INFINITY); + expect(newRegionDurationSec()).toBeUndefined(); + }); + + it("keeps the width it promises", () => { + for (const pxPerSec of [0.2, 1, 7.5, 120]) { + setTimelineScale(pxPerSec); + const width = (newRegionDurationSec() as number) * pxPerSec; + // Above the floor the duration is exactly PILL_CREATE_PX worth of time. + expect(width).toBeCloseTo(PILL_CREATE_PX, 6); + } + }); +}); diff --git a/src/lib/ai-edition/timeline/newRegionDuration.ts b/src/lib/ai-edition/timeline/newRegionDuration.ts new file mode 100644 index 00000000..e45f77a2 --- /dev/null +++ b/src/lib/ai-edition/timeline/newRegionDuration.ts @@ -0,0 +1,63 @@ +/** + * How long a region should be when the USER creates one, so the pill they get is + * always the same comfortable size on screen. + * + * A 30-minute recording has to fit the panel's width, so the default view is + * heavily dezoomed — around 0.4 px per second. A fixed 2 s region is under a + * pixel there: invisible, and hidden behind the very playhead it was created at. + * What has to stay constant is the WIDTH; the duration is whatever that width is + * worth at the current zoom, which is why it can be 95 s zoomed out and 2 s + * zoomed in for the same gesture. + * + * (A flat 2 s only ever looked right because the old 1.5%-of-the-timeline + * minimum pill width inflated it in the RENDERING — the lie removed in #233.) + * + * This lives outside the timeline component because BOTH ways of creating a + * region must agree: the toolbar buttons in V4Timeline and the keyboard + * shortcuts in NewEditorShell, which the empty lanes advertise ("Press Z to add + * zoom") and which have no other access to the zoom — `nav` is local state + * inside V4Timeline. Paths that are not a user placing a pill by hand (the + * agent, auto-zooms) don't call this and keep useTimeline's flat default. + */ + +/** + * On-screen width a freshly created pill aims for: wide enough to READ, not just + * to see. Measured in a browser, the width each label needs before the ellipsis + * bites — icon + gap + text + padding — is "Full Camera" 93px, "Annotation" + * 90px, "1.80×" 61px, "1.5×" 55px. 96 covers the longest with a couple of px to + * spare; longer translations of those two still ellipsize, which is what the + * ellipsis is for. + * + * The cost of a wide default is a long region: at full zoom-out on a 30-minute + * recording (~0.42 px/s) one click creates about 3 min 45 s. That is the trade + * the constant width implies — the duration is the variable, and the pill is + * immediately draggable by either edge. + */ +export const PILL_CREATE_PX = 96; +/** Floor on the duration. Only bites past ~30x zoom, where 40px is worth a few + * hundredths of a second and the region would be born unusable. */ +export const PILL_CREATE_MIN_SEC = 0.25; + +/** + * The timeline's current scale, in px per timeline-second. + * + * Module state, written by the one timeline that exists and read IMPERATIVELY at + * the instant a region is created — deliberately not a store subscription. The + * value changes on every zoom notch and nothing renders it, so subscribing would + * re-render the whole editor shell for a number only a click ever reads. Same + * reasoning as `playheadSec()` in useTimeline. + */ +let pxPerSec = 0; + +export function setTimelineScale(value: number): void { + pxPerSec = Number.isFinite(value) && value > 0 ? value : 0; +} + +/** + * Duration to create a region with, or `undefined` while the timeline has not + * been measured yet (first paint, or no timeline mounted) — callers then fall + * back to their own default rather than inventing a length from a width of zero. + */ +export function newRegionDurationSec(): number | undefined { + return pxPerSec > 0 ? Math.max(PILL_CREATE_MIN_SEC, PILL_CREATE_PX / pxPerSec) : undefined; +} diff --git a/src/lib/ai-edition/transcription/status.test.ts b/src/lib/ai-edition/transcription/status.test.ts index adc21cd3..00d8864e 100644 --- a/src/lib/ai-edition/transcription/status.test.ts +++ b/src/lib/ai-edition/transcription/status.test.ts @@ -5,6 +5,7 @@ import { classifyTranscriptionError, deriveAssetStatus, isPermanentFailure, + progressFraction, resolveTranscriptGate, transcriptHasSpeech, transcriptRelevantAssetIds, @@ -93,6 +94,30 @@ describe("deriveAssetStatus", () => { expect(derived).toEqual({ assetId: "asset_1", status: "running", phase: "transcribing" }); }); + it("carries chunk progress through to the view", () => { + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { + status: "running", + phase: "transcribing", + progress: { completedSec: 90, totalSec: 300 }, + }, + }); + expect(derived.progress).toEqual({ completedSec: 90, totalSec: 300 }); + expect(progressFraction(derived.progress)).toBeCloseTo(0.3); + }); + + it("leaves progress absent while nothing measurable is running", () => { + // Audio extraction and the model download have no fraction to report; the + // UI must get `undefined` so it keeps the spinner instead of a 0% bar. + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { status: "running", phase: "extracting-audio" }, + }); + expect(derived.progress).toBeUndefined(); + expect(progressFraction(derived.progress)).toBeNull(); + }); + it("reports ready from the document, with no job at all", () => { expect( deriveAssetStatus({ assetId: "asset_1", transcript: transcript("asset_1", ["hello"]) }) diff --git a/src/lib/ai-edition/transcription/status.ts b/src/lib/ai-edition/transcription/status.ts index fced8390..f05454b6 100644 --- a/src/lib/ai-edition/transcription/status.ts +++ b/src/lib/ai-edition/transcription/status.ts @@ -17,8 +17,28 @@ export interface TranscriptionFailure { message: string; } -/** Which half of the pipeline a running job is in (mirrors `TranscribeAssetOptions.onStatus`). */ -export type TranscriptionPhase = "extracting-audio" | "transcribing"; +/** Which part of the pipeline a running job is in (mirrors `TranscribeAssetOptions.onStatus`). */ +export type TranscriptionPhase = "extracting-audio" | "loading-model" | "transcribing"; + +/** + * How far a running transcription has got, in seconds of audio. + * + * Only the `"transcribing"` phase reports this, and only once the main process + * starts landing chunks — audio extraction and the first-run model download + * have nothing to measure. Absent means "running, no measurable progress", not + * "zero": the UI must fall back to an indeterminate spinner rather than render + * a bar stuck at 0%. + */ +export interface TranscriptionProgress { + completedSec: number; + totalSec: number; +} + +/** `0..1`, or null when the job reports no measurable progress. */ +export function progressFraction(progress: TranscriptionProgress | undefined): number | null { + if (!progress || !(progress.totalSec > 0)) return null; + return Math.min(1, Math.max(0, progress.completedSec / progress.totalSec)); +} /** * A media that has no audio track (or one Whisper cannot read) will fail the @@ -72,6 +92,7 @@ export interface AssetTranscriptionView { assetId: string; status: AssetTranscriptionStatus; phase?: TranscriptionPhase; + progress?: TranscriptionProgress; failure?: TranscriptionFailure; } @@ -79,6 +100,7 @@ export interface AssetTranscriptionView { export interface TranscriptionJobLike { status: "queued" | "running" | "failed"; phase?: TranscriptionPhase; + progress?: TranscriptionProgress; failure?: TranscriptionFailure; } @@ -122,7 +144,7 @@ export function deriveAssetStatus(input: { }): AssetTranscriptionView { const { assetId, job, transcript, persistedFailure } = input; if (job && job.status !== "failed") { - return { assetId, status: job.status, phase: job.phase }; + return { assetId, status: job.status, phase: job.phase, progress: job.progress }; } if (transcript) { return { diff --git a/src/lib/captioning/transcribe.test.ts b/src/lib/captioning/transcribe.test.ts index 18c6cfa7..82e630b9 100644 --- a/src/lib/captioning/transcribe.test.ts +++ b/src/lib/captioning/transcribe.test.ts @@ -23,7 +23,13 @@ import { transcribeMono16kToSegments } from "./transcribe"; * worker that the previous Web-Worker pipeline owned, so they run in any env. */ -type Listener = (event: { phase: "model" | "transcribe" }) => void; +// Mirrors `SttRendererStatus` — the mock must accept the progress fields, since +// carrying them across the IPC hop is exactly what this file asserts. +type Listener = (event: { + phase: "model" | "transcribe"; + completedSec?: number; + totalSec?: number; +}) => void; type RendererSttApi = { transcribe: (request: { samples: Float32Array; language?: string }) => Promise<{ @@ -115,12 +121,12 @@ describe("transcribeMono16kToSegments", () => { expect(result.segments).toEqual([{ text: "hello world", startSec: 0, endSec: 0.65 }]); }); - it("forwards 'model' / 'transcribe' phases to onStatus and tears the listener down", async () => { + it("forwards the whole status event to onStatus and tears the listener down", async () => { const onStatus = vi.fn(); mockApi.transcribe.mockImplementationOnce(async () => { // Simulate the IPC handler emitting a status event mid-flight. lastStatusCb?.({ phase: "model" }); - lastStatusCb?.({ phase: "transcribe" }); + lastStatusCb?.({ phase: "transcribe", completedSec: 90, totalSec: 300 }); return { segments: [], wordSegments: [{ word: "ok", startSec: 0, endSec: 0.1 }], @@ -130,8 +136,14 @@ describe("transcribeMono16kToSegments", () => { }); await transcribeMono16kToSegments(new Float32Array(1600), { onStatus }); - expect(onStatus).toHaveBeenCalledWith("model"); - expect(onStatus).toHaveBeenCalledWith("transcribe"); + expect(onStatus).toHaveBeenCalledWith({ phase: "model" }); + // The chunk progress must survive the hop, not just the phase — it is what + // drives the progress bar. + expect(onStatus).toHaveBeenCalledWith({ + phase: "transcribe", + completedSec: 90, + totalSec: 300, + }); // onStatus listener is detached once the promise settles. expect(lastStatusCb).toBeNull(); }); diff --git a/src/lib/captioning/transcribe.ts b/src/lib/captioning/transcribe.ts index a1b39d43..0ac43da2 100644 --- a/src/lib/captioning/transcribe.ts +++ b/src/lib/captioning/transcribe.ts @@ -28,6 +28,18 @@ export interface TranscribeMono16kResult { export type SttRendererStatusPhase = "model" | "transcribe"; +/** + * Progress the main process reports while a transcription runs. `completedSec` / + * `totalSec` are present only during `"transcribe"`, and only once chunking has + * started — they let the UI show a real bar instead of an indeterminate spinner + * for what can be several minutes of work. + */ +export interface SttRendererStatus { + phase: SttRendererStatusPhase; + completedSec?: number; + totalSec?: number; +} + interface RendererSttApi { transcribe: (request: { samples: Float32Array; language?: string }) => Promise<{ segments: CaptionSegment[]; @@ -35,7 +47,8 @@ interface RendererSttApi { detectedLanguage: string; backend: string; }>; - onStatus?: (callback: (event: { phase: SttRendererStatusPhase }) => void) => () => void; + cancel?: () => Promise; + onStatus?: (callback: (event: SttRendererStatus) => void) => () => void; } /** @@ -52,7 +65,7 @@ export function transcribeMono16kToSegments( samples: Float32Array, options?: { trimRegions?: TrimRegion[]; - onStatus?: (phase: SttRendererStatusPhase) => void; + onStatus?: (status: SttRendererStatus) => void; signal?: AbortSignal; language?: string; }, @@ -67,8 +80,13 @@ export function transcribeMono16kToSegments( return Promise.resolve({ segments: [], granularity: "word" }); } - const unsubscribe = - options?.onStatus && api.onStatus?.((event) => options.onStatus?.(event.phase)); + const unsubscribe = options?.onStatus && api.onStatus?.((event) => options.onStatus?.(event)); + // Aborting has to reach the MAIN process: the work is a chunk loop over there, + // and a renderer that merely stops awaiting still leaves the helper busy for + // minutes — with the replacement request queued behind it, which is what made + // "regenerate in another language" look dead. + const onAbort = () => void api.cancel?.(); + options?.signal?.addEventListener("abort", onAbort, { once: true }); const forcedLanguage = options?.language && options.language !== "auto" ? options.language : undefined; // ponytail: word timestamps come back already absolute from whisper.cpp @@ -104,7 +122,15 @@ export function transcribeMono16kToSegments( } return { segments, granularity, detectedLanguage: result.detectedLanguage }; }) + .catch((error: unknown) => { + // A run the caller cancelled surfaces as an abort, not as an engine + // failure: the store drops it silently instead of toasting the user + // about something they asked for. + if (options?.signal?.aborted) throw new DOMException("Aborted", "AbortError"); + throw error; + }) .finally(() => { + options?.signal?.removeEventListener("abort", onAbort); unsubscribe?.(); }); } diff --git a/src/lib/exporter/mp4ExportSettings.test.ts b/src/lib/exporter/mp4ExportSettings.test.ts index 2a941547..c66029ee 100644 --- a/src/lib/exporter/mp4ExportSettings.test.ts +++ b/src/lib/exporter/mp4ExportSettings.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { calculateEffectiveSourceDimensions, calculateMp4ExportSettings, + wouldUpscale, } from "./mp4ExportSettings"; describe("calculateMp4ExportSettings", () => { @@ -108,6 +109,40 @@ describe("calculateMp4ExportSettings", () => { }); }); + it("does not call letterbox rows an upscale (1920x1032 window capture, 16:9 project)", () => { + const source = { width: 1920, height: 1032 }; + const tier = (quality: "medium" | "good" | "source") => + calculateMp4ExportSettings({ + quality, + sourceWidth: source.width, + sourceHeight: source.height, + aspectRatioValue: 16 / 9, + }); + + // The reported bug: both tiers resolve to the exact same 1920x1080 frame, so they must + // carry the same badge. The old short-side test flagged only one of them. + expect(tier("good")).toMatchObject({ width: 1920, height: 1080 }); + expect(tier("source")).toMatchObject({ width: 1920, height: 1080 }); + expect(wouldUpscale(tier("good"), source)).toBe(false); + expect(wouldUpscale(tier("source"), source)).toBe(false); + expect(wouldUpscale(tier("medium"), source)).toBe(false); + }); + + it("still flags a tier that genuinely stretches the source", () => { + const source = { width: 1280, height: 720 }; + expect( + wouldUpscale( + calculateMp4ExportSettings({ + quality: "good", + sourceWidth: source.width, + sourceHeight: source.height, + aspectRatioValue: 16 / 9, + }), + source, + ), + ).toBe(true); + }); + it("uses the cropped area as the effective source size", () => { const effectiveSource = calculateEffectiveSourceDimensions(3840, 2160, { width: 854 / 3840, diff --git a/src/lib/exporter/mp4ExportSettings.ts b/src/lib/exporter/mp4ExportSettings.ts index b75193fc..f2acdfeb 100644 --- a/src/lib/exporter/mp4ExportSettings.ts +++ b/src/lib/exporter/mp4ExportSettings.ts @@ -11,6 +11,25 @@ interface SourceCropRegion { height: number; } +interface Dims { + width: number; + height: number; +} + +/** + * Would rendering a `source`-sized clip into an `output`-sized frame stretch it past its own + * resolution? + * + * The clip is CONTAIN-fitted into the output frame, so the answer is that fit scale — not a + * comparison of short sides, which counts letterbox rows as if they were stretched pixels. A + * 1920x1032 window capture in a 16:9 project gives a 1920x1080 frame whose 48 extra rows are + * wallpaper, at scale 1.0: nothing is upscaled. The short-side test called that frame an + * upscale under the "1080p" tier while "Source" produced the exact same frame unflagged. + */ +export function wouldUpscale(output: Dims, source: Dims): boolean { + return Math.min(output.width / source.width, output.height / source.height) > 1; +} + const MEDIUM_SHORT_SIDE = 720; const HIGH_SHORT_SIDE = 1080; diff --git a/technical-documentation/architecture/editor-shell.md b/technical-documentation/architecture/editor-shell.md index b6c79db9..eed79f8f 100644 --- a/technical-documentation/architecture/editor-shell.md +++ b/technical-documentation/architecture/editor-shell.md @@ -129,9 +129,9 @@ switches on it is checked below. Each path has been verified on this branch. (`document/timeline.ts:591`) for batch / single deletes. 4. **Lane in `V4Timeline`** — `src/components/ai-edition/v4/V4Timeline.tsx`. Compute the pills at the same call site as the four existing lanes - (`coalesceRegionsForRuler(tl.xxxRegions).map(...)` near `:461-509`), render + (`coalesceRegionsForRuler(tl.xxxRegions).map(...)` near `:463-511`), render them through `renderPills` inside a `
` - block (`:1478-1488`), and extend the `kind` union at `:332` so drag, + block (`:1504-1512`), and extend the `kind` union at `:334` so drag, resize, and delete handler switches route correctly. **Coordinates on the timeline canvas obey one rule**: position and size are