diff options
Diffstat (limited to 'rust/crates/audio')
| -rw-r--r-- | rust/crates/audio/Cargo.toml | 11 | ||||
| -rw-r--r-- | rust/crates/audio/src/buckets.rs | 105 | ||||
| -rw-r--r-- | rust/crates/audio/src/clips.rs | 169 | ||||
| -rw-r--r-- | rust/crates/audio/src/lib.rs | 29 | ||||
| -rw-r--r-- | rust/crates/audio/src/mix.rs | 181 | ||||
| -rw-r--r-- | rust/crates/audio/tests/collection.rs | 263 | ||||
| -rw-r--r-- | rust/crates/audio/tests/fixtures/sample.mp4 | bin | 0 -> 81284 bytes | |||
| -rw-r--r-- | rust/crates/audio/tests/fixtures/tone.wav | bin | 0 -> 88278 bytes | |||
| -rw-r--r-- | rust/crates/audio/tests/mix.rs | 183 |
9 files changed, 941 insertions, 0 deletions
diff --git a/rust/crates/audio/Cargo.toml b/rust/crates/audio/Cargo.toml new file mode 100644 index 00000000..a8d897cc --- /dev/null +++ b/rust/crates/audio/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "audio" +version = "0.1.0" +edition = "2024" + +[dependencies] +animation = { version = "0.1.0", path = "../animation" } +scene = { version = "0.1.0", path = "../scene" } +time = { version = "0.1.0", path = "../time" } +media = { version = "0.1.0", path = "../media" } +thiserror = "2" diff --git a/rust/crates/audio/src/buckets.rs b/rust/crates/audio/src/buckets.rs new file mode 100644 index 00000000..261c2373 --- /dev/null +++ b/rust/crates/audio/src/buckets.rs @@ -0,0 +1,105 @@ +pub const RMS_ANALYSIS_WINDOW_SECONDS: f64 = 0.02; + +pub fn sample_bucket_range( + start_sample: usize, + end_sample: usize, + bucket_index: usize, + bucket_count: usize, +) -> (usize, usize) { + let range_length = end_sample.saturating_sub(start_sample); + let bucket_start = start_sample + bucket_index * range_length / bucket_count; + let bucket_end = start_sample + (bucket_index + 1) * range_length / bucket_count; + (bucket_start, bucket_end.max(bucket_start)) +} + +pub fn extract_peak_range( + samples: &[f32], + channels: usize, + count: usize, + start_sample: usize, + end_sample: usize, +) -> Vec<f32> { + let channels = channels.max(1); + let frames = samples.len() / channels; + let end = end_sample.min(frames); + let start = start_sample.min(end); + let mut peaks = vec![0.0_f32; count]; + for (index, peak) in peaks.iter_mut().enumerate() { + let (bucket_start, bucket_end) = sample_bucket_range(start, end, index, count); + let mut bucket_peak = 0.0_f32; + for frame in bucket_start..bucket_end { + for channel in 0..channels { + let magnitude = samples[frame * channels + channel].abs(); + if magnitude > bucket_peak { + bucket_peak = magnitude; + } + } + } + *peak = bucket_peak; + } + peaks +} + +pub fn extract_rms_windowed( + samples: &[f32], + channels: usize, + sample_rate: u32, + buckets: &[(usize, usize)], +) -> Vec<f32> { + let channels = channels.max(1); + let max_window = + (f64::from(sample_rate) * RMS_ANALYSIS_WINDOW_SECONDS).floor().max(1.0) as usize; + buckets + .iter() + .map(|&(bucket_start, bucket_end)| { + let bucket_length = bucket_end.saturating_sub(bucket_start); + if bucket_length == 0 { + return 0.0; + } + let window_length = bucket_length.min(max_window).max(1); + let mut max_mean_square = 0.0_f64; + let mut window_start = bucket_start; + while window_start < bucket_end { + let window_end = (window_start + window_length).min(bucket_end); + let n = window_end - window_start; + let mut sum = 0.0_f64; + for frame in window_start..window_end { + for channel in 0..channels { + let value = f64::from(samples[frame * channels + channel]); + sum += value * value; + } + } + let mean_square = sum / (n * channels) as f64; + if mean_square > max_mean_square { + max_mean_square = mean_square; + } + window_start = window_end; + } + max_mean_square.sqrt() as f32 + }) + .collect() +} + +pub fn extract_rms_range( + samples: &[f32], + channels: usize, + sample_rate: u32, + count: usize, + start_sample: usize, + end_sample: usize, +) -> Vec<f32> { + let buckets: Vec<(usize, usize)> = (0..count) + .map(|index| sample_bucket_range(start_sample, end_sample, index, count)) + .collect(); + extract_rms_windowed(samples, channels, sample_rate, &buckets) +} + +pub fn extract_rms_buckets( + samples: &[f32], + channels: usize, + sample_rate: u32, + bucket_count: usize, +) -> Vec<f32> { + let frames = samples.len() / channels.max(1); + extract_rms_range(samples, channels, sample_rate, bucket_count, 0, frames) +} diff --git a/rust/crates/audio/src/clips.rs b/rust/crates/audio/src/clips.rs new file mode 100644 index 00000000..1f4c839c --- /dev/null +++ b/rust/crates/audio/src/clips.rs @@ -0,0 +1,169 @@ +use std::path::PathBuf; + +use animation::{has_keyframes_for_path, resolve_number_at_time}; +use scene::{ + AudioElement, BaseTimelineElement, Element, ElementAnimations, ParamValue, RetimeConfig, + SceneTracks, Track, +}; +use time::MediaTime; + +pub const MIN_RETIME_RATE: f64 = 0.01; +pub const MAX_RETIME_RATE: f64 = 5.0; +pub const VOLUME_DB_MIN: f64 = -60.0; +pub const VOLUME_DB_MAX: f64 = 20.0; + +pub fn clamp_retime_rate(rate: f64) -> f64 { + if !rate.is_finite() || rate <= 0.0 { + return 1.0; + } + rate.clamp(MIN_RETIME_RATE, MAX_RETIME_RATE) +} + +pub fn source_time_at_clip_time(clip_time: f64, retime: Option<&RetimeConfig>) -> f64 { + clip_time * clamp_retime_rate(retime.map(|config| config.rate).unwrap_or(1.0)) +} + +pub fn clamp_db(value: f64) -> f64 { + if !value.is_finite() { + return 0.0; + } + value.clamp(VOLUME_DB_MIN, VOLUME_DB_MAX) +} + +pub fn db_to_linear(db: f64) -> f64 { + 10_f64.powf(clamp_db(db) / 20.0) +} + +#[derive(Debug, Clone)] +pub struct AudibleClip { + pub element_id: String, + pub path: PathBuf, + pub start_time: MediaTime, + pub duration: MediaTime, + pub trim_start: MediaTime, + pub trim_end: MediaTime, + pub volume_db: f64, + pub retime: Option<RetimeConfig>, + pub animations: Option<ElementAnimations>, +} + +impl AudibleClip { + pub fn end_time(&self) -> MediaTime { + self.start_time + self.duration + } + + pub fn effective_rate(&self) -> f64 { + clamp_retime_rate(self.retime.map(|config| config.rate).unwrap_or(1.0)) + } + + pub fn requires_maintain_pitch(&self) -> bool { + self.retime.is_some_and(|config| { + config.maintain_pitch == Some(true) + && config.rate.is_finite() + && config.rate > 0.0 + && clamp_retime_rate(config.rate) != 1.0 + }) + } + + pub fn has_animated_volume(&self) -> bool { + has_keyframes_for_path(self.animations.as_ref(), "volume") + } + + pub fn gain_at(&self, local_time: MediaTime) -> f64 { + db_to_linear(resolve_number_at_time( + self.volume_db, + self.animations.as_ref(), + "volume", + local_time, + )) + } + + pub fn static_gain(&self) -> f64 { + db_to_linear(self.volume_db) + } +} + +fn track_audio_muted(track: &Track) -> bool { + match track { + Track::Video { muted, .. } | Track::Audio { muted, .. } => *muted, + _ => false, + } +} + +fn element_source(element: &Element) -> Option<(&BaseTimelineElement, &str, Option<RetimeConfig>)> { + match element { + Element::Audio(AudioElement::Upload(upload)) => { + Some((&upload.base, upload.media_id.as_str(), upload.retime)) + } + Element::Audio(AudioElement::Library(library)) => { + Some((&library.base, library.source_url.as_str(), library.retime)) + } + Element::Video(video) => { + if video.hidden == Some(true) || video.is_source_audio_enabled == Some(false) { + return None; + } + Some((&video.base, video.media_id.as_str(), video.retime)) + } + _ => None, + } +} + +fn collect_from_track( + track: &Track, + media: &dyn Fn(&str) -> Option<PathBuf>, + range: (MediaTime, MediaTime), + clips: &mut Vec<AudibleClip>, +) { + if track_audio_muted(track) { + return; + } + for element in track.elements() { + let Some((base, source_key, retime)) = element_source(element) else { + continue; + }; + if base.duration <= MediaTime::ZERO { + continue; + } + let end_time = base.start_time + base.duration; + if end_time <= range.0 || base.start_time >= range.1 { + continue; + } + if base.params.get("muted") == Some(&ParamValue::Bool(true)) { + continue; + } + let volume_db = match base.params.get("volume") { + Some(ParamValue::Number(value)) => *value, + _ => 0.0, + }; + let Some(path) = media(source_key) else { + continue; + }; + clips.push(AudibleClip { + element_id: base.id.clone(), + path, + start_time: base.start_time, + duration: base.duration, + trim_start: base.trim_start, + trim_end: base.trim_end, + volume_db, + retime, + animations: base.animations.clone(), + }); + } +} + +pub fn collect_audible_elements( + tracks: &SceneTracks, + media: &dyn Fn(&str) -> Option<PathBuf>, + range: (MediaTime, MediaTime), +) -> Vec<AudibleClip> { + let mut clips = Vec::new(); + for track in &tracks.overlay { + collect_from_track(track, media, range, &mut clips); + } + collect_from_track(&tracks.main, media, range, &mut clips); + for track in &tracks.audio { + collect_from_track(track, media, range, &mut clips); + } + clips +} diff --git a/rust/crates/audio/src/lib.rs b/rust/crates/audio/src/lib.rs new file mode 100644 index 00000000..0bce5647 --- /dev/null +++ b/rust/crates/audio/src/lib.rs @@ -0,0 +1,29 @@ +mod buckets; +mod clips; +mod mix; + +pub use buckets::{ + extract_peak_range, extract_rms_buckets, extract_rms_range, extract_rms_windowed, + sample_bucket_range, +}; +pub use clips::{ + AudibleClip, clamp_db, clamp_retime_rate, collect_audible_elements, db_to_linear, + source_time_at_clip_time, +}; +pub use mix::{apply_mastering, mix_range, render_export_buffer}; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error( + "element {element_id} requires pitch-preserving retime at rate {rate}, which is not supported" + )] + MaintainPitchUnsupported { element_id: String, rate: f64 }, + #[error("invalid time range: end must be after start")] + InvalidTimeRange, + #[error("sample rate must be at least 1")] + InvalidSampleRate, + #[error("channel count must be at least 1")] + InvalidChannelCount, +} + +pub type Result<T> = std::result::Result<T, Error>; diff --git a/rust/crates/audio/src/mix.rs b/rust/crates/audio/src/mix.rs new file mode 100644 index 00000000..887111eb --- /dev/null +++ b/rust/crates/audio/src/mix.rs @@ -0,0 +1,181 @@ +use media::{AudioSamples, Decoder}; +use time::MediaTime; + +use crate::Error; +use crate::clips::AudibleClip; + +pub const MASTER_LIMITER_THRESHOLD_DB: f64 = -1.0; +pub const MASTER_LIMITER_RATIO: f64 = 20.0; +pub const MASTER_LIMITER_ATTACK_SECONDS: f64 = 0.001; +pub const MASTER_LIMITER_RELEASE_SECONDS: f64 = 0.12; +pub const MASTER_OUTPUT_HEADROOM: f64 = 0.98; +pub const MASTER_OUTPUT_HEADROOM_F32: f32 = 0.98; + +fn decode_clip_source(clip: &AudibleClip, source_start: f64, source_end: f64) -> Option<AudioSamples> { + let start = MediaTime::from_seconds_f64(source_start.max(0.0))?; + let end = MediaTime::from_seconds_f64(source_end.max(0.0))?; + if end <= start { + return None; + } + let mut decoder = Decoder::open(&clip.path).ok()?; + let stream = decoder.info().audio_streams.first().cloned()?; + decoder.decode_audio_range(stream.index, start, end).ok() +} + +fn mix_clip( + clip: &AudibleClip, + range_start_seconds: f64, + out_frames: usize, + sample_rate: u32, + channel_count: usize, + output: &mut [f32], +) { + let clip_start_seconds = clip.start_time.to_seconds_f64(); + let clip_end_seconds = clip_start_seconds + clip.duration.to_seconds_f64(); + let rate = clip.effective_rate(); + let trim_start_seconds = clip.trim_start.to_seconds_f64(); + let rate_f64 = f64::from(sample_rate); + + let first_frame = ((clip_start_seconds - range_start_seconds) * rate_f64) + .ceil() + .max(0.0) as usize; + let last_frame = (((clip_end_seconds - range_start_seconds) * rate_f64) + .ceil() + .max(0.0) as usize) + .min(out_frames); + if last_frame <= first_frame { + return; + } + + let source_start = trim_start_seconds + + (first_frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds) * rate; + let source_end = trim_start_seconds + + (last_frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds) * rate; + let Some(decoded) = decode_clip_source(clip, source_start, source_end) else { + return; + }; + let decoded_channels = decoded.channels as usize; + let decoded_frames = decoded.samples.len() / decoded_channels; + let decoded_rate = f64::from(decoded.sample_rate); + let static_gain = clip.static_gain() as f32; + let animated_volume = clip.has_animated_volume(); + + for frame in first_frame..last_frame { + let clip_time = frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds; + let source_time = trim_start_seconds + clip_time * rate; + let source_position = (source_time - source_start) * decoded_rate; + if source_position >= decoded_frames as f64 { + break; + } + let source_position = source_position.max(0.0); + let lower = source_position.floor() as usize; + let upper = (lower + 1).min(decoded_frames.saturating_sub(1)); + let fraction = (source_position - lower as f64) as f32; + let gain = if animated_volume { + clip.gain_at(MediaTime::from_seconds_f64(clip_time).unwrap_or(MediaTime::ZERO)) as f32 + } else { + static_gain + }; + for channel in 0..channel_count { + let source_channel = channel.min(decoded_channels - 1); + let a = decoded.samples[lower * decoded_channels + source_channel]; + let b = decoded.samples[upper * decoded_channels + source_channel]; + output[frame * channel_count + channel] += + (a * (1.0 - fraction) + b * fraction) * gain; + } + } +} + +pub fn mix_range( + clips: &[AudibleClip], + range: (MediaTime, MediaTime), + sample_rate: u32, + channels: u32, +) -> Result<Vec<f32>, Error> { + if range.1 <= range.0 { + return Err(Error::InvalidTimeRange); + } + if sample_rate == 0 { + return Err(Error::InvalidSampleRate); + } + if channels == 0 { + return Err(Error::InvalidChannelCount); + } + for clip in clips { + if clip.requires_maintain_pitch() + && clip.end_time() > range.0 + && clip.start_time < range.1 + { + return Err(Error::MaintainPitchUnsupported { + element_id: clip.element_id.clone(), + rate: clip.effective_rate(), + }); + } + } + + let range_seconds = (range.1 - range.0).to_seconds_f64(); + let out_frames = (range_seconds * f64::from(sample_rate)).ceil() as usize; + let channel_count = channels as usize; + let mut output = vec![0.0_f32; out_frames * channel_count]; + let range_start_seconds = range.0.to_seconds_f64(); + for clip in clips { + mix_clip( + clip, + range_start_seconds, + out_frames, + sample_rate, + channel_count, + &mut output, + ); + } + Ok(output) +} + +pub fn apply_mastering(samples: &mut [f32], channels: usize, sample_rate: u32) { + let peak = samples.iter().fold(0.0_f32, |peak, s| peak.max(s.abs())); + if f64::from(peak) <= MASTER_OUTPUT_HEADROOM { + return; + } + let attack_coeff = + (-1.0 / (MASTER_LIMITER_ATTACK_SECONDS * f64::from(sample_rate))).exp(); + let release_coeff = + (-1.0 / (MASTER_LIMITER_RELEASE_SECONDS * f64::from(sample_rate))).exp(); + let mut envelope_db = 0.0_f64; + for frame in samples.chunks_mut(channels) { + let level = frame.iter().fold(0.0_f32, |level, s| level.max(s.abs())); + let level_db = 20.0 * f64::from(level.max(1e-6)).log10(); + let over = level_db - MASTER_LIMITER_THRESHOLD_DB; + let target_db = if over > 0.0 { + -over * (1.0 - 1.0 / MASTER_LIMITER_RATIO) + } else { + 0.0 + }; + let coeff = if target_db < envelope_db { + attack_coeff + } else { + release_coeff + }; + envelope_db = target_db + (envelope_db - target_db) * coeff; + let gain = 10_f64.powf(envelope_db / 20.0) * MASTER_OUTPUT_HEADROOM; + for sample in frame.iter_mut() { + *sample = (f64::from(*sample) * gain) as f32; + } + } + for sample in samples.iter_mut() { + *sample = sample.clamp(-MASTER_OUTPUT_HEADROOM_F32, MASTER_OUTPUT_HEADROOM_F32); + } +} + +pub fn render_export_buffer( + clips: &[AudibleClip], + total_duration: MediaTime, + sample_rate: u32, + channels: u32, +) -> Result<Vec<f32>, Error> { + if clips.is_empty() || total_duration <= MediaTime::ZERO { + return Ok(Vec::new()); + } + let mut buffer = mix_range(clips, (MediaTime::ZERO, total_duration), sample_rate, channels)?; + apply_mastering(&mut buffer, channels as usize, sample_rate); + Ok(buffer) +} diff --git a/rust/crates/audio/tests/collection.rs b/rust/crates/audio/tests/collection.rs new file mode 100644 index 00000000..6c1f8a9f --- /dev/null +++ b/rust/crates/audio/tests/collection.rs @@ -0,0 +1,263 @@ +use std::collections::BTreeMap; +use std::path::PathBuf; + +use audio::collect_audible_elements; +use scene::{ + AudioElement, BaseTimelineElement, Element, LibraryAudioElement, ParamValue, RetimeConfig, + SceneTracks, Track, UploadAudioElement, VideoElement, +}; +use time::MediaTime; + +fn seconds(value: f64) -> MediaTime { + MediaTime::from_seconds_f64(value).expect("finite seconds") +} + +fn base(id: &str, start: f64, duration: f64) -> BaseTimelineElement { + BaseTimelineElement { + id: id.to_string(), + name: id.to_string(), + duration: seconds(duration), + start_time: seconds(start), + trim_start: MediaTime::ZERO, + trim_end: MediaTime::ZERO, + source_duration: None, + animations: None, + params: BTreeMap::new(), + } +} + +fn upload_audio(id: &str, start: f64, duration: f64) -> Element { + Element::Audio(AudioElement::Upload(UploadAudioElement { + base: base(id, start, duration), + retime: None, + media_id: format!("media-{id}"), + })) +} + +fn library_audio(id: &str, start: f64, duration: f64) -> Element { + Element::Audio(AudioElement::Library(LibraryAudioElement { + base: base(id, start, duration), + retime: None, + source_url: format!("https://cdn.test/{id}.mp3"), + })) +} + +fn video(id: &str, start: f64, duration: f64) -> Element { + Element::Video(VideoElement { + base: base(id, start, duration), + media_id: format!("media-{id}"), + is_source_audio_enabled: None, + hidden: None, + retime: None, + effects: None, + masks: None, + }) +} + +fn main_track(elements: Vec<Element>) -> Track { + Track::Video { + id: "main".to_string(), + name: "main".to_string(), + elements, + muted: false, + hidden: false, + } +} + +fn audio_track(id: &str, muted: bool, elements: Vec<Element>) -> Track { + Track::Audio { + id: id.to_string(), + name: id.to_string(), + elements, + muted, + } +} + +fn tracks(main: Vec<Element>, audio: Vec<Track>) -> SceneTracks { + SceneTracks { + overlay: vec![], + main: main_track(main), + audio, + } +} + +fn resolve_any(key: &str) -> Option<PathBuf> { + Some(PathBuf::from(key)) +} + +fn full_range() -> (MediaTime, MediaTime) { + (MediaTime::ZERO, seconds(60.0)) +} + +#[test] +fn collects_upload_audio_and_video_with_source_audio() { + let scene = tracks( + vec![video("v1", 0.0, 2.0)], + vec![audio_track("a", false, vec![upload_audio("u1", 1.0, 1.0)])], + ); + let clips = collect_audible_elements(&scene, &resolve_any, full_range()); + assert_eq!(clips.len(), 2); + assert_eq!(clips[0].element_id, "v1"); + assert_eq!(clips[1].element_id, "u1"); + assert_eq!(clips[0].path, PathBuf::from("media-v1")); + assert_eq!(clips[1].start_time, seconds(1.0)); + assert_eq!(clips[1].duration, seconds(1.0)); +} + +#[test] +fn library_audio_resolves_by_source_url() { + let scene = tracks( + vec![], + vec![audio_track("a", false, vec![library_audio("lib", 0.0, 1.0)])], + ); + let clips = collect_audible_elements(&scene, &resolve_any, full_range()); + assert_eq!(clips.len(), 1); + assert_eq!(clips[0].path, PathBuf::from("https://cdn.test/lib.mp3")); +} + +#[test] +fn muted_track_excludes_its_elements() { + let scene = tracks( + vec![], + vec![ + audio_track("muted-track", true, vec![upload_audio("u1", 0.0, 1.0)]), + audio_track("live-track", false, vec![upload_audio("u2", 0.0, 1.0)]), + ], + ); + let clips = collect_audible_elements(&scene, &resolve_any, full_range()); + assert_eq!(clips.len(), 1); + assert_eq!(clips[0].element_id, "u2"); +} + +#[test] +fn muted_element_is_excluded() { + let mut element = match upload_audio("u1", 0.0, 1.0) { + Element::Audio(AudioElement::Upload(mut upload)) => { + upload + .base + .params + .insert("muted".to_string(), ParamValue::Bool(true)); + Element::Audio(AudioElement::Upload(upload)) + } + other => other, + }; + let scene = tracks(vec![], vec![audio_track("a", false, vec![element])]); + let clips = collect_audible_elements(&scene, &resolve_any, full_range()); + assert!(clips.is_empty()); + + element = match upload_audio("u2", 0.0, 1.0) { + Element::Audio(AudioElement::Upload(mut upload)) => { + upload + .base + .params + .insert("muted".to_string(), ParamValue::Bool(false)); + Element::Audio(AudioElement::Upload(upload)) + } + other => other, + }; + let scene = tracks(vec![], vec![audio_track("a", false, vec![element])]); + assert_eq!(collect_audible_elements(&scene, &resolve_any, full_range()).len(), 1); +} + +#[test] +fn hidden_video_is_excluded() { + let hidden = Element::Video(VideoElement { + hidden: Some(true), + ..match video("v1", 0.0, 2.0) { + Element::Video(v) => v, + _ => unreachable!(), + } + }); + let scene = tracks(vec![hidden], vec![]); + assert!(collect_audible_elements(&scene, &resolve_any, full_range()).is_empty()); +} + +#[test] +fn video_with_source_audio_disabled_is_excluded() { + let disabled = Element::Video(VideoElement { + is_source_audio_enabled: Some(false), + ..match video("v1", 0.0, 2.0) { + Element::Video(v) => v, + _ => unreachable!(), + } + }); + let enabled = Element::Video(VideoElement { + is_source_audio_enabled: Some(true), + ..match video("v2", 0.0, 2.0) { + Element::Video(v) => v, + _ => unreachable!(), + } + }); + let scene = tracks(vec![disabled, enabled], vec![]); + let clips = collect_audible_elements(&scene, &resolve_any, full_range()); + assert_eq!(clips.len(), 1); + assert_eq!(clips[0].element_id, "v2"); +} + +#[test] +fn range_overlap_trims_collection() { + let scene = tracks( + vec![video("v1", 2.0, 2.0), video("v2", 10.0, 2.0)], + vec![], + ); + let clips = collect_audible_elements(&scene, &resolve_any, (seconds(3.0), seconds(4.5))); + assert_eq!(clips.len(), 1); + assert_eq!(clips[0].element_id, "v1"); + + let clips = collect_audible_elements(&scene, &resolve_any, (seconds(4.0), seconds(10.0))); + assert!(clips.is_empty()); + + let clips = collect_audible_elements(&scene, &resolve_any, (seconds(4.0), seconds(10.5))); + assert_eq!(clips.len(), 1); + assert_eq!(clips[0].element_id, "v2"); +} + +#[test] +fn volume_param_is_decibels_with_zero_default() { + let quiet = match upload_audio("u1", 0.0, 1.0) { + Element::Audio(AudioElement::Upload(mut upload)) => { + upload + .base + .params + .insert("volume".to_string(), ParamValue::Number(-6.0)); + Element::Audio(AudioElement::Upload(upload)) + } + other => other, + }; + let plain = upload_audio("u2", 0.0, 1.0); + let scene = tracks(vec![], vec![audio_track("a", false, vec![quiet, plain])]); + let clips = collect_audible_elements(&scene, &resolve_any, full_range()); + assert_eq!(clips[0].volume_db, -6.0); + assert_eq!(clips[1].volume_db, 0.0); + assert!((clips[0].static_gain() - 0.5012).abs() < 0.001); + assert_eq!(clips[1].static_gain(), 1.0); +} + +#[test] +fn retime_and_trim_are_carried() { + let element = match upload_audio("u1", 0.0, 0.5) { + Element::Audio(AudioElement::Upload(mut upload)) => { + upload.base.trim_start = seconds(0.25); + upload.base.trim_end = seconds(0.25); + upload.retime = Some(RetimeConfig { + rate: 2.0, + maintain_pitch: None, + }); + Element::Audio(AudioElement::Upload(upload)) + } + other => other, + }; + let scene = tracks(vec![], vec![audio_track("a", false, vec![element])]); + let clips = collect_audible_elements(&scene, &resolve_any, full_range()); + assert_eq!(clips.len(), 1); + assert_eq!(clips[0].trim_start, seconds(0.25)); + assert_eq!(clips[0].effective_rate(), 2.0); + assert!(!clips[0].requires_maintain_pitch()); +} + +#[test] +fn unresolvable_media_is_excluded() { + let scene = tracks(vec![video("v1", 0.0, 1.0)], vec![]); + let clips = collect_audible_elements(&scene, &|_| None, full_range()); + assert!(clips.is_empty()); +} diff --git a/rust/crates/audio/tests/fixtures/sample.mp4 b/rust/crates/audio/tests/fixtures/sample.mp4 Binary files differnew file mode 100644 index 00000000..13c113e5 --- /dev/null +++ b/rust/crates/audio/tests/fixtures/sample.mp4 diff --git a/rust/crates/audio/tests/fixtures/tone.wav b/rust/crates/audio/tests/fixtures/tone.wav Binary files differnew file mode 100644 index 00000000..8d5f7886 --- /dev/null +++ b/rust/crates/audio/tests/fixtures/tone.wav diff --git a/rust/crates/audio/tests/mix.rs b/rust/crates/audio/tests/mix.rs new file mode 100644 index 00000000..6d0ed0d1 --- /dev/null +++ b/rust/crates/audio/tests/mix.rs @@ -0,0 +1,183 @@ +use std::path::PathBuf; + +use audio::{ + AudibleClip, Error, apply_mastering, extract_rms_buckets, mix_range, render_export_buffer, +}; +use media::Decoder; +use scene::RetimeConfig; +use time::MediaTime; + +const TONE_WAV: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/tone.wav"); + +fn seconds(value: f64) -> MediaTime { + MediaTime::from_seconds_f64(value).expect("finite seconds") +} + +fn tone_clip(id: &str, start: f64, duration: f64) -> AudibleClip { + AudibleClip { + element_id: id.to_string(), + path: PathBuf::from(TONE_WAV), + start_time: seconds(start), + duration: seconds(duration), + trim_start: MediaTime::ZERO, + trim_end: MediaTime::ZERO, + volume_db: 0.0, + retime: None, + animations: None, + } +} + +fn decode_tone(start: f64, end: f64) -> (Vec<f32>, u32, usize) { + let mut decoder = Decoder::open(TONE_WAV).expect("open tone.wav"); + let stream = decoder.info().audio_streams.first().cloned().expect("audio stream"); + let decoded = decoder + .decode_audio_range(stream.index, seconds(start), seconds(end)) + .expect("decode tone"); + (decoded.samples, decoded.sample_rate, decoded.channels as usize) +} + +#[test] +fn two_overlapping_tones_sum() { + let clip_a = tone_clip("a", 0.0, 1.0); + let clip_b = tone_clip("b", 0.0, 1.0); + let single = mix_range(&[clip_a.clone()], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap(); + let doubled = mix_range(&[clip_a, clip_b], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap(); + assert_eq!(single.len(), doubled.len()); + for (one, two) in single.iter().zip(doubled.iter()) { + assert!((two - 2.0 * one).abs() < 1e-6, "{two} != 2 * {one}"); + } +} + +#[test] +fn volume_param_scales_samples() { + let full = tone_clip("full", 0.0, 1.0); + let mut half = tone_clip("half", 0.0, 1.0); + half.volume_db = -6.020599913279624; + let full_mix = mix_range(&[full], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap(); + let half_mix = mix_range(&[half], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap(); + for (f, h) in full_mix.iter().zip(half_mix.iter()) { + assert!((h - f * 0.5).abs() < 1e-3, "{h} != 0.5 * {f}"); + } +} + +#[test] +fn retime_2x_consumes_source_twice_as_fast() { + let normal = tone_clip("normal", 0.0, 1.0); + let mut fast = tone_clip("fast", 0.0, 0.5); + fast.retime = Some(RetimeConfig { + rate: 2.0, + maintain_pitch: None, + }); + let normal_mix = mix_range(&[normal], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap(); + let fast_mix = mix_range(&[fast], (seconds(0.0), seconds(0.5)), 44100, 2).unwrap(); + let frames = fast_mix.len() / 2; + for i in 0..frames { + for channel in 0..2 { + let expected = normal_mix[2 * i * 2 + channel]; + let actual = fast_mix[i * 2 + channel]; + assert!((actual - expected).abs() < 1e-6, "frame {i}: {actual} != {expected}"); + } + } +} + +#[test] +fn trim_start_offsets_source() { + let mut clip = tone_clip("trimmed", 0.0, 0.5); + clip.trim_start = seconds(0.5); + let mix = mix_range(&[clip], (seconds(0.0), seconds(0.5)), 44100, 2).unwrap(); + let (decoded, _rate, channels) = decode_tone(0.5, 1.0); + assert_eq!(channels, 1); + let frames = mix.len() / 2; + for i in 0..frames { + for channel in 0..2 { + let actual = mix[i * 2 + channel]; + assert!((actual - decoded[i]).abs() < 1e-6, "frame {i}: {actual} != {}", decoded[i]); + } + } +} + +#[test] +fn rms_buckets_of_tone_match_hand_computed() { + let (samples, rate, channels) = decode_tone(0.0, 1.0); + let peak = samples.iter().fold(0.0_f32, |p, s| p.max(s.abs())); + let expected_rms = peak / 2.0_f32.sqrt(); + let buckets = extract_rms_buckets(&samples, channels, rate, 10); + assert_eq!(buckets.len(), 10); + for bucket in &buckets { + let relative = (bucket - expected_rms).abs() / expected_rms; + assert!(relative < 0.05, "bucket {bucket} vs expected {expected_rms}"); + } +} + +#[test] +fn mastering_limits_peaks_on_hot_input() { + let frames = 4096; + let mut samples = vec![1.5_f32; frames * 2]; + apply_mastering(&mut samples, 2, 44100); + let peak = samples.iter().fold(0.0_f32, |p, s| p.max(s.abs())); + assert!(peak <= 0.98, "peak was {peak}"); + assert!(peak > 0.9, "limiter over-attenuated: peak {peak}"); +} + +#[test] +fn mastering_leaves_quiet_input_untouched() { + let mut samples: Vec<f32> = (0..1000).map(|i| (i as f32 / 1000.0) * 0.5).collect(); + let original = samples.clone(); + apply_mastering(&mut samples, 1, 44100); + assert_eq!(samples, original); +} + +#[test] +fn maintain_pitch_true_returns_typed_error() { + let mut clip = tone_clip("pitched", 0.0, 0.5); + clip.retime = Some(RetimeConfig { + rate: 2.0, + maintain_pitch: Some(true), + }); + let result = mix_range(&[clip], (seconds(0.0), seconds(1.0)), 44100, 2); + match result { + Err(Error::MaintainPitchUnsupported { element_id, rate }) => { + assert_eq!(element_id, "pitched"); + assert_eq!(rate, 2.0); + } + other => panic!("expected MaintainPitchUnsupported, got {other:?}"), + } +} + +#[test] +fn maintain_pitch_at_unity_rate_is_allowed() { + let mut clip = tone_clip("unity", 0.0, 0.5); + clip.retime = Some(RetimeConfig { + rate: 1.0, + maintain_pitch: Some(true), + }); + assert!(mix_range(&[clip], (seconds(0.0), seconds(0.5)), 44100, 2).is_ok()); +} + +#[test] +fn render_export_buffer_mixes_and_masters() { + let clip = tone_clip("a", 0.0, 1.0); + let buffer = render_export_buffer(&[clip], seconds(1.0), 44100, 2).unwrap(); + assert_eq!(buffer.len(), 44100 * 2); + let peak = buffer.iter().fold(0.0_f32, |p, s| p.max(s.abs())); + assert!(peak > 0.05, "peak was {peak}"); + assert!(render_export_buffer(&[], seconds(1.0), 44100, 2).unwrap().is_empty()); +} + +#[test] +fn range_render_only_covers_requested_window() { + let clip = tone_clip("a", 1.0, 1.0); + let buffer = mix_range(&[clip], (seconds(0.0), seconds(2.0)), 44100, 2).unwrap(); + let first_half = &buffer[..44100 * 2]; + assert!(first_half.iter().all(|s| *s == 0.0)); + let second_peak = buffer[44100 * 2..].iter().fold(0.0_f32, |p, s| p.max(s.abs())); + assert!(second_peak > 0.05); +} + +#[test] +fn undecodable_clip_contributes_silence() { + let mut clip = tone_clip("missing", 0.0, 1.0); + clip.path = PathBuf::from("/nonexistent/nope.wav"); + let buffer = mix_range(&[clip], (seconds(0.0), seconds(0.5)), 44100, 2).unwrap(); + assert!(buffer.iter().all(|s| *s == 0.0)); +} |
