diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index e89dee55090..046a18b1f47 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -49,6 +49,7 @@ use std::{ use cap_cursor_info::CursorFamily; use cap_editor::{EditorFrameOutput, EditorInstance, EditorState}; +use cap_project::caption_words::caption_text_from_words; use cap_project::{ ClipSpeedAudioMode, Cursors, ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, TimelineConfiguration, XY, @@ -5346,44 +5347,6 @@ pub(crate) struct EditorNotice { /// How long a notice stays up. const EDITOR_NOTICE_DURATION: Duration = Duration::from_millis(2500); -fn caption_text_from_words(words: &[cap_project::CaptionWord]) -> String { - let mut text = String::new(); - for word in words { - let value = word.text.trim(); - if value.is_empty() { - continue; - } - let attaches = value.chars().next().is_some_and(|value| { - matches!( - value, - ',' | '.' - | '!' - | '?' - | ';' - | ':' - | '%' - | ')' - | ']' - | '}' - | '\'' - | '’' - | '、' - | '。' - | '！' - | '？' - | '；' - | '：' - | '，' - ) - }); - if !text.is_empty() && !attaches { - text.push(' '); - } - text.push_str(value); - } - text -} - fn fresh_caption_source_id(project: &ProjectConfiguration) -> String { use std::hash::{BuildHasher, Hasher}; diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index 5ea9ccdaa1d..92fa942a9cc 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -1856,167 +1856,10 @@ fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(), // Caption words & chunking // --------------------------------------------------------------------------- -/// `MAX_CAPTION_WORD_DURATION` (`captions.rs:75`): Whisper/Parakeet sometimes -/// stretch a trailing word's end across a following silence, which leaves the -/// rendered caption stuck on screen and duplicates the word across timeline -/// cuts once projected. -pub const MAX_CAPTION_WORD_DURATION: f32 = 2.5; -/// `captions.rs:67-69`. -const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6; -const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8; -const MIN_FINAL_CAPTION_WORDS: usize = 3; - -/// `caption_char_attaches_to_previous` (`captions.rs:745-767`). -fn caption_char_attaches_to_previous(value: char) -> bool { - matches!( - value, - ',' | '.' - | '!' - | '?' - | ';' - | ':' - | '%' - | ')' - | ']' - | '}' - | '\'' - | '’' - | '、' - | '。' - | '！' - | '？' - | '；' - | '：' - | '，' - ) -} - -fn caption_token_attaches_to_previous(text: &str) -> bool { - text.trim() - .chars() - .next() - .is_some_and(caption_char_attaches_to_previous) -} - -/// `caption_boundary_word_is_weak` (`captions.rs:769-801`). -fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool { - let normalized = word - .text - .trim() - .chars() - .filter(|c| c.is_alphanumeric()) - .collect::() - .to_lowercase(); - - normalized.chars().count() <= 1 - || matches!( - normalized.as_str(), - "an" | "as" - | "at" - | "be" - | "by" - | "do" - | "he" - | "if" - | "in" - | "is" - | "it" - | "me" - | "my" - | "of" - | "on" - | "or" - | "so" - | "to" - | "up" - | "we" - ) -} - -/// `normalize_caption_words` (`captions.rs:803-831`). -fn normalize_caption_words(words: Vec) -> Vec { - let mut normalized: Vec = Vec::with_capacity(words.len()); - - for word in words { - let text = word.text.trim(); - if text.is_empty() { - continue; - } - - if caption_token_attaches_to_previous(text) - && let Some(previous) = normalized.last_mut() - { - previous.text.push_str(text); - previous.end = word.end; - } else { - normalized.push(CaptionWord { - text: text.to_string(), - start: word.start, - end: word.end, - }); - } - } - - for word in &mut normalized { - word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION); - } - - normalized -} - -/// `caption_text_from_words` (`captions.rs:833-849`). -fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { - let mut text = String::new(); - - for word in words { - let word_text = word.text.trim(); - if word_text.is_empty() { - continue; - } - - if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { - text.push(' '); - } - text.push_str(word_text); - } - - text -} - -/// `caption_word_chunks` (`captions.rs:851-883`). -fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> { - let mut chunks = Vec::new(); - let mut start = 0; - - while start < words.len() { - let remaining = words.len() - start; - if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT { - chunks.push(&words[start..]); - break; - } - - let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len()); - while end < words.len() - && caption_boundary_word_is_weak(&words[end - 1]) - && end - start < MAX_CAPTION_WORDS_PER_SEGMENT - { - end += 1; - } - - let remaining_after = words.len() - end; - if remaining_after > 0 - && remaining_after < MIN_FINAL_CAPTION_WORDS - && caption_boundary_word_is_weak(&words[end]) - { - end = words.len(); - } - - chunks.push(&words[start..end]); - start = end; - } - - chunks -} +use cap_project::caption_words::{ + caption_text_from_words, caption_word_chunks, normalize_caption_words, + MAX_CAPTION_WORD_DURATION, +}; // --------------------------------------------------------------------------- // Track derivation -- deriveCaptionTrackSegments, in Rust diff --git a/apps/desktop/src-tauri/src/captions.rs b/apps/desktop/src-tauri/src/captions.rs index b64305aa478..90d6712ab13 100644 --- a/apps/desktop/src-tauri/src/captions.rs +++ b/apps/desktop/src-tauri/src/captions.rs @@ -74,15 +74,11 @@ lazy_static::lazy_static! { } const WHISPER_SAMPLE_RATE: u32 = 16000; -const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6; -const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8; -const MIN_FINAL_CAPTION_WORDS: usize = 3; -// Whisper/Parakeet sometimes stretch a trailing word's end across a following -// silence (e.g. a 16s "seconds."), which leaves the rendered caption stuck on -// screen and duplicates the word across timeline cuts once projected. Real -// spoken words never approach this, so cap each word's duration to keep timing -// tied to speech rather than silence. -const MAX_CAPTION_WORD_DURATION: f32 = 2.5; + +use cap_project::caption_words::{ + caption_text_from_words, caption_word_chunks, normalize_caption_words, + MAX_CAPTION_WORD_DURATION, +}; #[cfg(not(all(target_os = "macos", target_arch = "x86_64")))] struct CachedParakeetContext { @@ -769,154 +765,6 @@ fn is_special_token(token_text: &str) -> bool { is_special } -fn caption_token_attaches_to_previous(text: &str) -> bool { - let Some(first_char) = text.trim().chars().next() else { - return false; - }; - - caption_char_attaches_to_previous(first_char) -} - -fn caption_char_attaches_to_previous(value: char) -> bool { - matches!( - value, - ',' | '.' - | '!' - | '?' - | ';' - | ':' - | '%' - | ')' - | ']' - | '}' - | '\'' - | '’' - | '、' - | '。' - | '！' - | '？' - | '；' - | '：' - | '，' - ) -} - -fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool { - let normalized = word - .text - .trim() - .chars() - .filter(|c| c.is_alphanumeric()) - .collect::() - .to_lowercase(); - - normalized.len() <= 1 - || matches!( - normalized.as_str(), - "an" | "as" - | "at" - | "be" - | "by" - | "do" - | "he" - | "if" - | "in" - | "is" - | "it" - | "me" - | "my" - | "of" - | "on" - | "or" - | "so" - | "to" - | "up" - | "we" - ) -} - -fn normalize_caption_words(words: Vec) -> Vec { - let mut normalized: Vec = Vec::with_capacity(words.len()); - - for word in words { - let text = word.text.trim(); - if text.is_empty() { - continue; - } - - if caption_token_attaches_to_previous(text) - && let Some(previous) = normalized.last_mut() - { - previous.text.push_str(text); - previous.end = word.end; - } else { - normalized.push(CaptionWord { - text: text.to_string(), - start: word.start, - end: word.end, - }); - } - } - - for word in &mut normalized { - word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION); - } - - normalized -} - -fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { - let mut text = String::new(); - - for word in words { - let word_text = word.text.trim(); - if word_text.is_empty() { - continue; - } - - if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { - text.push(' '); - } - text.push_str(word_text); - } - - text -} - -fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> { - let mut chunks = Vec::new(); - let mut start = 0; - - while start < words.len() { - let remaining = words.len() - start; - if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT { - chunks.push(&words[start..]); - break; - } - - let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len()); - while end < words.len() - && caption_boundary_word_is_weak(&words[end - 1]) - && end - start < MAX_CAPTION_WORDS_PER_SEGMENT - { - end += 1; - } - - let remaining_after = words.len() - end; - if remaining_after > 0 - && remaining_after < MIN_FINAL_CAPTION_WORDS - && caption_boundary_word_is_weak(&words[end]) - { - end = words.len(); - } - - chunks.push(&words[start..end]); - start = end; - } - - chunks -} - fn normalize_audio_for_transcription(samples: &mut [f32]) -> f32 { if samples.is_empty() { return 1.0; diff --git a/crates/project/src/caption_timing.rs b/crates/project/src/caption_timing.rs index b938177af4d..5f9fa655b1f 100644 --- a/crates/project/src/caption_timing.rs +++ b/crates/project/src/caption_timing.rs @@ -1,10 +1,9 @@ +use crate::caption_words::{MAX_CAPTION_WORD_DURATION, caption_text_from_words}; use crate::{ CaptionSegment, CaptionTrackSegment, CaptionWord, ProjectConfiguration, TimelineConfiguration, }; use std::collections::HashMap; -const MAX_CAPTION_WORD_DURATION: f32 = 2.5; - pub(crate) fn clip_timeline_offsets(timeline: &TimelineConfiguration) -> Vec { let mut offset = 0.0; timeline @@ -22,55 +21,6 @@ pub(crate) fn clip_timeline_offsets(timeline: &TimelineConfiguration) -> Vec bool { - matches!( - value, - ',' | '.' - | '!' - | '?' - | ';' - | ':' - | '%' - | ')' - | ']' - | '}' - | '\'' - | '’' - | '、' - | '。' - | '！' - | '？' - | '；' - | '：' - | '，' - ) -} - -fn caption_token_attaches_to_previous(text: &str) -> bool { - text.trim() - .chars() - .next() - .is_some_and(caption_char_attaches_to_previous) -} - -fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { - let mut text = String::new(); - - for word in words { - let word_text = word.text.trim(); - if word_text.is_empty() { - continue; - } - - if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { - text.push(' '); - } - text.push_str(word_text); - } - - text -} - const CAPTION_EDL_SEPARATOR: &str = "::edl"; pub fn source_caption_id(track_id: &str) -> &str { diff --git a/crates/project/src/caption_words.rs b/crates/project/src/caption_words.rs new file mode 100644 index 00000000000..767ed15ff9e --- /dev/null +++ b/crates/project/src/caption_words.rs @@ -0,0 +1,284 @@ +//! Shared caption word normalization and segmentation. +//! +//! Single source of truth for how transcribed words are cleaned up and grouped +//! into on-screen caption segments. This used to be copy-pasted across the +//! Tauri captions pipeline (`apps/desktop/src-tauri/src/captions.rs`), the gpui +//! transcription pipeline (`apps/desktop-gpui/src/transcription.rs`), +//! `caption_timing`, and `editor_window`, and the copies drifted apart. + +use crate::CaptionWord; + +/// Whisper/Parakeet sometimes stretch a trailing word's end across a following +/// silence (e.g. a 16s "seconds."), which leaves the rendered caption stuck on +/// screen and duplicates the word across timeline cuts once projected. Real +/// spoken words never approach this, so cap each word's duration to keep timing +/// tied to speech rather than silence. +pub const MAX_CAPTION_WORD_DURATION: f32 = 2.5; +pub const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6; +pub const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8; +pub const MIN_FINAL_CAPTION_WORDS: usize = 3; + +fn caption_char_attaches_to_previous(value: char) -> bool { + matches!( + value, + ',' | '.' + | '!' + | '?' + | ';' + | ':' + | '%' + | ')' + | ']' + | '}' + | '\'' + | '’' + | '、' + | '。' + | '！' + | '？' + | '；' + | '：' + | '，' + ) +} + +fn caption_token_attaches_to_previous(text: &str) -> bool { + text.trim() + .chars() + .next() + .is_some_and(caption_char_attaches_to_previous) +} + +/// A caption segment should not end on a one letter word or a short connector +/// when more words follow, since that reads badly on screen. Comparing +/// `chars().count()` matters: a byte length check counts non-ASCII letters like +/// "à" as two and treats them as strong boundary words. +pub fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool { + let normalized = word + .text + .trim() + .chars() + .filter(|c| c.is_alphanumeric()) + .collect::() + .to_lowercase(); + + normalized.chars().count() <= 1 + || matches!( + normalized.as_str(), + "an" | "as" + | "at" + | "be" + | "by" + | "do" + | "he" + | "if" + | "in" + | "is" + | "it" + | "me" + | "my" + | "of" + | "on" + | "or" + | "so" + | "to" + | "up" + | "we" + ) +} + +/// Drops empty tokens, attaches punctuation-only tokens to the previous word, +/// and clamps inflated trailing word durations. +pub fn normalize_caption_words(words: Vec) -> Vec { + let mut normalized: Vec = Vec::with_capacity(words.len()); + + for word in words { + let text = word.text.trim(); + if text.is_empty() { + continue; + } + + if caption_token_attaches_to_previous(text) + && let Some(previous) = normalized.last_mut() + { + previous.text.push_str(text); + previous.end = word.end; + } else { + normalized.push(CaptionWord { + text: text.to_string(), + start: word.start, + end: word.end, + }); + } + } + + for word in &mut normalized { + word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION); + } + + normalized +} + +pub fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { + let mut text = String::new(); + + for word in words { + let word_text = word.text.trim(); + if word_text.is_empty() { + continue; + } + + if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { + text.push(' '); + } + text.push_str(word_text); + } + + text +} + +/// Groups normalized words into caption segments of at most +/// [`MAX_CAPTION_WORDS_PER_SEGMENT`] words, aiming for +/// [`TARGET_CAPTION_WORDS_PER_SEGMENT`]. The boundary word may move past the +/// target to avoid ending on a weak word, and a trailing fragment shorter than +/// [`MIN_FINAL_CAPTION_WORDS`] merges into the previous segment when it starts +/// with a weak word. +pub fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> { + let mut chunks = Vec::new(); + let mut start = 0; + + while start < words.len() { + let remaining = words.len() - start; + if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT { + chunks.push(&words[start..]); + break; + } + + let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len()); + while end < words.len() + && caption_boundary_word_is_weak(&words[end - 1]) + && end - start < MAX_CAPTION_WORDS_PER_SEGMENT + { + end += 1; + } + + let remaining_after = words.len() - end; + if remaining_after > 0 + && remaining_after < MIN_FINAL_CAPTION_WORDS + && caption_boundary_word_is_weak(&words[end]) + { + end = words.len(); + } + + chunks.push(&words[start..end]); + start = end; + } + + chunks +} + +#[cfg(test)] +mod tests { + use super::*; + + fn word(text: &str, index: usize) -> CaptionWord { + CaptionWord { + text: text.to_string(), + start: index as f32, + end: index as f32 + 0.5, + } + } + + #[test] + fn normalize_caption_words_attaches_punctuation() { + let words = normalize_caption_words(vec![ + word("test", 0), + word(",", 1), + word("test", 2), + word(".", 3), + ]); + + assert_eq!(caption_text_from_words(&words), "test, test."); + assert_eq!(words.len(), 2); + } + + #[test] + fn normalize_caption_words_clamps_inflated_trailing_word() { + let words = normalize_caption_words(vec![CaptionWord { + text: "seconds.".to_string(), + start: 53.92, + end: 70.16, + }]); + + assert_eq!(words.len(), 1); + assert!((words[0].end - (53.92 + MAX_CAPTION_WORD_DURATION)).abs() < 1e-4); + } + + #[test] + fn normalize_caption_words_keeps_normal_word_durations() { + let words = normalize_caption_words(vec![CaptionWord { + text: "hello".to_string(), + start: 1.0, + end: 1.4, + }]); + + assert_eq!(words.len(), 1); + assert!((words[0].end - 1.4).abs() < 1e-4); + } + + #[test] + fn caption_word_chunks_do_not_end_on_short_connector_when_more_words_follow() { + for connector in ["I", "à", "à,", "a\u{300}", "日"] { + let words = [ + "This", "is", "where", "we", "record", connector, "want", "clean", "captions", + ] + .iter() + .enumerate() + .map(|(index, text)| word(text, index)) + .collect::>(); + + let chunks = caption_word_chunks(&words); + + assert_eq!( + caption_text_from_words(chunks[0]), + format!("This is where we record {connector} want"), + "connector {connector}" + ); + assert_eq!(caption_text_from_words(chunks[1]), "clean captions"); + } + } + + #[test] + fn caption_word_chunks_merge_short_tail_starting_with_connector() { + for connector in ["I", "à", "à,", "a\u{300}", "日"] { + let words = [ + "These", "captions", "should", "stay", "grouped", "together", connector, "Paris", + ] + .iter() + .enumerate() + .map(|(index, text)| word(text, index)) + .collect::>(); + + let chunks = caption_word_chunks(&words); + + assert_eq!(chunks.len(), 1, "connector {connector}"); + assert_eq!( + caption_text_from_words(chunks[0]), + format!("These captions should stay grouped together {connector} Paris") + ); + } + } + + #[test] + fn caption_word_chunks_target_size_groups_strong_boundaries() { + let words = (0..12) + .map(|index| word(&format!("w{index}"), index)) + .collect::>(); + + let chunks = caption_word_chunks(&words); + + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].len(), 6); + assert_eq!(chunks[1].len(), 6); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 080355b0578..f4bdcf5869d 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -1,5 +1,6 @@ mod animated_gradient; mod caption_timing; +pub mod caption_words; #[cfg(test)] mod config_write_tests; mod configuration; @@ -9,6 +10,11 @@ mod meta; pub use animated_gradient::*; pub use caption_timing::{derive_caption_track_segments, source_caption_id, synchronize_captions}; +pub use caption_words::{ + MAX_CAPTION_WORD_DURATION, MAX_CAPTION_WORDS_PER_SEGMENT, MIN_FINAL_CAPTION_WORDS, + TARGET_CAPTION_WORDS_PER_SEGMENT, caption_boundary_word_is_weak, caption_text_from_words, + caption_word_chunks, normalize_caption_words, +}; pub use configuration::*; pub use cursor::*; pub use keyboard::*; diff --git a/crates/rendering/src/layers/captions.rs b/crates/rendering/src/layers/captions.rs index 5a6d10bbef7..c26e715b3e6 100644 --- a/crates/rendering/src/layers/captions.rs +++ b/crates/rendering/src/layers/captions.rs @@ -139,9 +139,9 @@ const BASE_TEXT_OPACITY: f32 = 0.8; const BOUNCE_OFFSET_PIXELS: f32 = 8.0; // Safety net for caption segments whose trailing word end was stretched across a // silence by transcription (e.g. a 16s "seconds."). Without this, such a segment -// stays on screen for the whole inflated duration. Kept in sync with -// MAX_CAPTION_WORD_DURATION in the desktop transcription/projection layers. -const MAX_CAPTION_WORD_DURATION: f64 = 2.5; +// stays on screen for the whole inflated duration. Value comes from the shared +// caption word utilities so it stays in sync with the transcription layers. +const MAX_CAPTION_WORD_DURATION: f64 = cap_project::caption_words::MAX_CAPTION_WORD_DURATION as f64; fn ease_out_cubic(t: f32) -> f32 { let t = t.clamp(0.0, 1.0);