summaryrefslogtreecommitdiff
path: root/rust/crates/audio
diff options
context:
space:
mode:
Diffstat (limited to 'rust/crates/audio')
-rw-r--r--rust/crates/audio/Cargo.toml11
-rw-r--r--rust/crates/audio/src/buckets.rs105
-rw-r--r--rust/crates/audio/src/clips.rs169
-rw-r--r--rust/crates/audio/src/lib.rs29
-rw-r--r--rust/crates/audio/src/mix.rs181
-rw-r--r--rust/crates/audio/tests/collection.rs263
-rw-r--r--rust/crates/audio/tests/fixtures/sample.mp4bin0 -> 81284 bytes
-rw-r--r--rust/crates/audio/tests/fixtures/tone.wavbin0 -> 88278 bytes
-rw-r--r--rust/crates/audio/tests/mix.rs183
9 files changed, 941 insertions, 0 deletions
diff --git a/rust/crates/audio/Cargo.toml b/rust/crates/audio/Cargo.toml
new file mode 100644
index 00000000..a8d897cc
--- /dev/null
+++ b/rust/crates/audio/Cargo.toml
@@ -0,0 +1,11 @@
+[package]
+name = "audio"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+animation = { version = "0.1.0", path = "../animation" }
+scene = { version = "0.1.0", path = "../scene" }
+time = { version = "0.1.0", path = "../time" }
+media = { version = "0.1.0", path = "../media" }
+thiserror = "2"
diff --git a/rust/crates/audio/src/buckets.rs b/rust/crates/audio/src/buckets.rs
new file mode 100644
index 00000000..261c2373
--- /dev/null
+++ b/rust/crates/audio/src/buckets.rs
@@ -0,0 +1,105 @@
+pub const RMS_ANALYSIS_WINDOW_SECONDS: f64 = 0.02;
+
+pub fn sample_bucket_range(
+ start_sample: usize,
+ end_sample: usize,
+ bucket_index: usize,
+ bucket_count: usize,
+) -> (usize, usize) {
+ let range_length = end_sample.saturating_sub(start_sample);
+ let bucket_start = start_sample + bucket_index * range_length / bucket_count;
+ let bucket_end = start_sample + (bucket_index + 1) * range_length / bucket_count;
+ (bucket_start, bucket_end.max(bucket_start))
+}
+
+pub fn extract_peak_range(
+ samples: &[f32],
+ channels: usize,
+ count: usize,
+ start_sample: usize,
+ end_sample: usize,
+) -> Vec<f32> {
+ let channels = channels.max(1);
+ let frames = samples.len() / channels;
+ let end = end_sample.min(frames);
+ let start = start_sample.min(end);
+ let mut peaks = vec![0.0_f32; count];
+ for (index, peak) in peaks.iter_mut().enumerate() {
+ let (bucket_start, bucket_end) = sample_bucket_range(start, end, index, count);
+ let mut bucket_peak = 0.0_f32;
+ for frame in bucket_start..bucket_end {
+ for channel in 0..channels {
+ let magnitude = samples[frame * channels + channel].abs();
+ if magnitude > bucket_peak {
+ bucket_peak = magnitude;
+ }
+ }
+ }
+ *peak = bucket_peak;
+ }
+ peaks
+}
+
+pub fn extract_rms_windowed(
+ samples: &[f32],
+ channels: usize,
+ sample_rate: u32,
+ buckets: &[(usize, usize)],
+) -> Vec<f32> {
+ let channels = channels.max(1);
+ let max_window =
+ (f64::from(sample_rate) * RMS_ANALYSIS_WINDOW_SECONDS).floor().max(1.0) as usize;
+ buckets
+ .iter()
+ .map(|&(bucket_start, bucket_end)| {
+ let bucket_length = bucket_end.saturating_sub(bucket_start);
+ if bucket_length == 0 {
+ return 0.0;
+ }
+ let window_length = bucket_length.min(max_window).max(1);
+ let mut max_mean_square = 0.0_f64;
+ let mut window_start = bucket_start;
+ while window_start < bucket_end {
+ let window_end = (window_start + window_length).min(bucket_end);
+ let n = window_end - window_start;
+ let mut sum = 0.0_f64;
+ for frame in window_start..window_end {
+ for channel in 0..channels {
+ let value = f64::from(samples[frame * channels + channel]);
+ sum += value * value;
+ }
+ }
+ let mean_square = sum / (n * channels) as f64;
+ if mean_square > max_mean_square {
+ max_mean_square = mean_square;
+ }
+ window_start = window_end;
+ }
+ max_mean_square.sqrt() as f32
+ })
+ .collect()
+}
+
+pub fn extract_rms_range(
+ samples: &[f32],
+ channels: usize,
+ sample_rate: u32,
+ count: usize,
+ start_sample: usize,
+ end_sample: usize,
+) -> Vec<f32> {
+ let buckets: Vec<(usize, usize)> = (0..count)
+ .map(|index| sample_bucket_range(start_sample, end_sample, index, count))
+ .collect();
+ extract_rms_windowed(samples, channels, sample_rate, &buckets)
+}
+
+pub fn extract_rms_buckets(
+ samples: &[f32],
+ channels: usize,
+ sample_rate: u32,
+ bucket_count: usize,
+) -> Vec<f32> {
+ let frames = samples.len() / channels.max(1);
+ extract_rms_range(samples, channels, sample_rate, bucket_count, 0, frames)
+}
diff --git a/rust/crates/audio/src/clips.rs b/rust/crates/audio/src/clips.rs
new file mode 100644
index 00000000..1f4c839c
--- /dev/null
+++ b/rust/crates/audio/src/clips.rs
@@ -0,0 +1,169 @@
+use std::path::PathBuf;
+
+use animation::{has_keyframes_for_path, resolve_number_at_time};
+use scene::{
+ AudioElement, BaseTimelineElement, Element, ElementAnimations, ParamValue, RetimeConfig,
+ SceneTracks, Track,
+};
+use time::MediaTime;
+
+pub const MIN_RETIME_RATE: f64 = 0.01;
+pub const MAX_RETIME_RATE: f64 = 5.0;
+pub const VOLUME_DB_MIN: f64 = -60.0;
+pub const VOLUME_DB_MAX: f64 = 20.0;
+
+pub fn clamp_retime_rate(rate: f64) -> f64 {
+ if !rate.is_finite() || rate <= 0.0 {
+ return 1.0;
+ }
+ rate.clamp(MIN_RETIME_RATE, MAX_RETIME_RATE)
+}
+
+pub fn source_time_at_clip_time(clip_time: f64, retime: Option<&RetimeConfig>) -> f64 {
+ clip_time * clamp_retime_rate(retime.map(|config| config.rate).unwrap_or(1.0))
+}
+
+pub fn clamp_db(value: f64) -> f64 {
+ if !value.is_finite() {
+ return 0.0;
+ }
+ value.clamp(VOLUME_DB_MIN, VOLUME_DB_MAX)
+}
+
+pub fn db_to_linear(db: f64) -> f64 {
+ 10_f64.powf(clamp_db(db) / 20.0)
+}
+
+#[derive(Debug, Clone)]
+pub struct AudibleClip {
+ pub element_id: String,
+ pub path: PathBuf,
+ pub start_time: MediaTime,
+ pub duration: MediaTime,
+ pub trim_start: MediaTime,
+ pub trim_end: MediaTime,
+ pub volume_db: f64,
+ pub retime: Option<RetimeConfig>,
+ pub animations: Option<ElementAnimations>,
+}
+
+impl AudibleClip {
+ pub fn end_time(&self) -> MediaTime {
+ self.start_time + self.duration
+ }
+
+ pub fn effective_rate(&self) -> f64 {
+ clamp_retime_rate(self.retime.map(|config| config.rate).unwrap_or(1.0))
+ }
+
+ pub fn requires_maintain_pitch(&self) -> bool {
+ self.retime.is_some_and(|config| {
+ config.maintain_pitch == Some(true)
+ && config.rate.is_finite()
+ && config.rate > 0.0
+ && clamp_retime_rate(config.rate) != 1.0
+ })
+ }
+
+ pub fn has_animated_volume(&self) -> bool {
+ has_keyframes_for_path(self.animations.as_ref(), "volume")
+ }
+
+ pub fn gain_at(&self, local_time: MediaTime) -> f64 {
+ db_to_linear(resolve_number_at_time(
+ self.volume_db,
+ self.animations.as_ref(),
+ "volume",
+ local_time,
+ ))
+ }
+
+ pub fn static_gain(&self) -> f64 {
+ db_to_linear(self.volume_db)
+ }
+}
+
+fn track_audio_muted(track: &Track) -> bool {
+ match track {
+ Track::Video { muted, .. } | Track::Audio { muted, .. } => *muted,
+ _ => false,
+ }
+}
+
+fn element_source(element: &Element) -> Option<(&BaseTimelineElement, &str, Option<RetimeConfig>)> {
+ match element {
+ Element::Audio(AudioElement::Upload(upload)) => {
+ Some((&upload.base, upload.media_id.as_str(), upload.retime))
+ }
+ Element::Audio(AudioElement::Library(library)) => {
+ Some((&library.base, library.source_url.as_str(), library.retime))
+ }
+ Element::Video(video) => {
+ if video.hidden == Some(true) || video.is_source_audio_enabled == Some(false) {
+ return None;
+ }
+ Some((&video.base, video.media_id.as_str(), video.retime))
+ }
+ _ => None,
+ }
+}
+
+fn collect_from_track(
+ track: &Track,
+ media: &dyn Fn(&str) -> Option<PathBuf>,
+ range: (MediaTime, MediaTime),
+ clips: &mut Vec<AudibleClip>,
+) {
+ if track_audio_muted(track) {
+ return;
+ }
+ for element in track.elements() {
+ let Some((base, source_key, retime)) = element_source(element) else {
+ continue;
+ };
+ if base.duration <= MediaTime::ZERO {
+ continue;
+ }
+ let end_time = base.start_time + base.duration;
+ if end_time <= range.0 || base.start_time >= range.1 {
+ continue;
+ }
+ if base.params.get("muted") == Some(&ParamValue::Bool(true)) {
+ continue;
+ }
+ let volume_db = match base.params.get("volume") {
+ Some(ParamValue::Number(value)) => *value,
+ _ => 0.0,
+ };
+ let Some(path) = media(source_key) else {
+ continue;
+ };
+ clips.push(AudibleClip {
+ element_id: base.id.clone(),
+ path,
+ start_time: base.start_time,
+ duration: base.duration,
+ trim_start: base.trim_start,
+ trim_end: base.trim_end,
+ volume_db,
+ retime,
+ animations: base.animations.clone(),
+ });
+ }
+}
+
+pub fn collect_audible_elements(
+ tracks: &SceneTracks,
+ media: &dyn Fn(&str) -> Option<PathBuf>,
+ range: (MediaTime, MediaTime),
+) -> Vec<AudibleClip> {
+ let mut clips = Vec::new();
+ for track in &tracks.overlay {
+ collect_from_track(track, media, range, &mut clips);
+ }
+ collect_from_track(&tracks.main, media, range, &mut clips);
+ for track in &tracks.audio {
+ collect_from_track(track, media, range, &mut clips);
+ }
+ clips
+}
diff --git a/rust/crates/audio/src/lib.rs b/rust/crates/audio/src/lib.rs
new file mode 100644
index 00000000..0bce5647
--- /dev/null
+++ b/rust/crates/audio/src/lib.rs
@@ -0,0 +1,29 @@
+mod buckets;
+mod clips;
+mod mix;
+
+pub use buckets::{
+ extract_peak_range, extract_rms_buckets, extract_rms_range, extract_rms_windowed,
+ sample_bucket_range,
+};
+pub use clips::{
+ AudibleClip, clamp_db, clamp_retime_rate, collect_audible_elements, db_to_linear,
+ source_time_at_clip_time,
+};
+pub use mix::{apply_mastering, mix_range, render_export_buffer};
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error(
+ "element {element_id} requires pitch-preserving retime at rate {rate}, which is not supported"
+ )]
+ MaintainPitchUnsupported { element_id: String, rate: f64 },
+ #[error("invalid time range: end must be after start")]
+ InvalidTimeRange,
+ #[error("sample rate must be at least 1")]
+ InvalidSampleRate,
+ #[error("channel count must be at least 1")]
+ InvalidChannelCount,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
diff --git a/rust/crates/audio/src/mix.rs b/rust/crates/audio/src/mix.rs
new file mode 100644
index 00000000..887111eb
--- /dev/null
+++ b/rust/crates/audio/src/mix.rs
@@ -0,0 +1,181 @@
+use media::{AudioSamples, Decoder};
+use time::MediaTime;
+
+use crate::Error;
+use crate::clips::AudibleClip;
+
+pub const MASTER_LIMITER_THRESHOLD_DB: f64 = -1.0;
+pub const MASTER_LIMITER_RATIO: f64 = 20.0;
+pub const MASTER_LIMITER_ATTACK_SECONDS: f64 = 0.001;
+pub const MASTER_LIMITER_RELEASE_SECONDS: f64 = 0.12;
+pub const MASTER_OUTPUT_HEADROOM: f64 = 0.98;
+pub const MASTER_OUTPUT_HEADROOM_F32: f32 = 0.98;
+
+fn decode_clip_source(clip: &AudibleClip, source_start: f64, source_end: f64) -> Option<AudioSamples> {
+ let start = MediaTime::from_seconds_f64(source_start.max(0.0))?;
+ let end = MediaTime::from_seconds_f64(source_end.max(0.0))?;
+ if end <= start {
+ return None;
+ }
+ let mut decoder = Decoder::open(&clip.path).ok()?;
+ let stream = decoder.info().audio_streams.first().cloned()?;
+ decoder.decode_audio_range(stream.index, start, end).ok()
+}
+
+fn mix_clip(
+ clip: &AudibleClip,
+ range_start_seconds: f64,
+ out_frames: usize,
+ sample_rate: u32,
+ channel_count: usize,
+ output: &mut [f32],
+) {
+ let clip_start_seconds = clip.start_time.to_seconds_f64();
+ let clip_end_seconds = clip_start_seconds + clip.duration.to_seconds_f64();
+ let rate = clip.effective_rate();
+ let trim_start_seconds = clip.trim_start.to_seconds_f64();
+ let rate_f64 = f64::from(sample_rate);
+
+ let first_frame = ((clip_start_seconds - range_start_seconds) * rate_f64)
+ .ceil()
+ .max(0.0) as usize;
+ let last_frame = (((clip_end_seconds - range_start_seconds) * rate_f64)
+ .ceil()
+ .max(0.0) as usize)
+ .min(out_frames);
+ if last_frame <= first_frame {
+ return;
+ }
+
+ let source_start = trim_start_seconds
+ + (first_frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds) * rate;
+ let source_end = trim_start_seconds
+ + (last_frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds) * rate;
+ let Some(decoded) = decode_clip_source(clip, source_start, source_end) else {
+ return;
+ };
+ let decoded_channels = decoded.channels as usize;
+ let decoded_frames = decoded.samples.len() / decoded_channels;
+ let decoded_rate = f64::from(decoded.sample_rate);
+ let static_gain = clip.static_gain() as f32;
+ let animated_volume = clip.has_animated_volume();
+
+ for frame in first_frame..last_frame {
+ let clip_time = frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds;
+ let source_time = trim_start_seconds + clip_time * rate;
+ let source_position = (source_time - source_start) * decoded_rate;
+ if source_position >= decoded_frames as f64 {
+ break;
+ }
+ let source_position = source_position.max(0.0);
+ let lower = source_position.floor() as usize;
+ let upper = (lower + 1).min(decoded_frames.saturating_sub(1));
+ let fraction = (source_position - lower as f64) as f32;
+ let gain = if animated_volume {
+ clip.gain_at(MediaTime::from_seconds_f64(clip_time).unwrap_or(MediaTime::ZERO)) as f32
+ } else {
+ static_gain
+ };
+ for channel in 0..channel_count {
+ let source_channel = channel.min(decoded_channels - 1);
+ let a = decoded.samples[lower * decoded_channels + source_channel];
+ let b = decoded.samples[upper * decoded_channels + source_channel];
+ output[frame * channel_count + channel] +=
+ (a * (1.0 - fraction) + b * fraction) * gain;
+ }
+ }
+}
+
+pub fn mix_range(
+ clips: &[AudibleClip],
+ range: (MediaTime, MediaTime),
+ sample_rate: u32,
+ channels: u32,
+) -> Result<Vec<f32>, Error> {
+ if range.1 <= range.0 {
+ return Err(Error::InvalidTimeRange);
+ }
+ if sample_rate == 0 {
+ return Err(Error::InvalidSampleRate);
+ }
+ if channels == 0 {
+ return Err(Error::InvalidChannelCount);
+ }
+ for clip in clips {
+ if clip.requires_maintain_pitch()
+ && clip.end_time() > range.0
+ && clip.start_time < range.1
+ {
+ return Err(Error::MaintainPitchUnsupported {
+ element_id: clip.element_id.clone(),
+ rate: clip.effective_rate(),
+ });
+ }
+ }
+
+ let range_seconds = (range.1 - range.0).to_seconds_f64();
+ let out_frames = (range_seconds * f64::from(sample_rate)).ceil() as usize;
+ let channel_count = channels as usize;
+ let mut output = vec![0.0_f32; out_frames * channel_count];
+ let range_start_seconds = range.0.to_seconds_f64();
+ for clip in clips {
+ mix_clip(
+ clip,
+ range_start_seconds,
+ out_frames,
+ sample_rate,
+ channel_count,
+ &mut output,
+ );
+ }
+ Ok(output)
+}
+
+pub fn apply_mastering(samples: &mut [f32], channels: usize, sample_rate: u32) {
+ let peak = samples.iter().fold(0.0_f32, |peak, s| peak.max(s.abs()));
+ if f64::from(peak) <= MASTER_OUTPUT_HEADROOM {
+ return;
+ }
+ let attack_coeff =
+ (-1.0 / (MASTER_LIMITER_ATTACK_SECONDS * f64::from(sample_rate))).exp();
+ let release_coeff =
+ (-1.0 / (MASTER_LIMITER_RELEASE_SECONDS * f64::from(sample_rate))).exp();
+ let mut envelope_db = 0.0_f64;
+ for frame in samples.chunks_mut(channels) {
+ let level = frame.iter().fold(0.0_f32, |level, s| level.max(s.abs()));
+ let level_db = 20.0 * f64::from(level.max(1e-6)).log10();
+ let over = level_db - MASTER_LIMITER_THRESHOLD_DB;
+ let target_db = if over > 0.0 {
+ -over * (1.0 - 1.0 / MASTER_LIMITER_RATIO)
+ } else {
+ 0.0
+ };
+ let coeff = if target_db < envelope_db {
+ attack_coeff
+ } else {
+ release_coeff
+ };
+ envelope_db = target_db + (envelope_db - target_db) * coeff;
+ let gain = 10_f64.powf(envelope_db / 20.0) * MASTER_OUTPUT_HEADROOM;
+ for sample in frame.iter_mut() {
+ *sample = (f64::from(*sample) * gain) as f32;
+ }
+ }
+ for sample in samples.iter_mut() {
+ *sample = sample.clamp(-MASTER_OUTPUT_HEADROOM_F32, MASTER_OUTPUT_HEADROOM_F32);
+ }
+}
+
+pub fn render_export_buffer(
+ clips: &[AudibleClip],
+ total_duration: MediaTime,
+ sample_rate: u32,
+ channels: u32,
+) -> Result<Vec<f32>, Error> {
+ if clips.is_empty() || total_duration <= MediaTime::ZERO {
+ return Ok(Vec::new());
+ }
+ let mut buffer = mix_range(clips, (MediaTime::ZERO, total_duration), sample_rate, channels)?;
+ apply_mastering(&mut buffer, channels as usize, sample_rate);
+ Ok(buffer)
+}
diff --git a/rust/crates/audio/tests/collection.rs b/rust/crates/audio/tests/collection.rs
new file mode 100644
index 00000000..6c1f8a9f
--- /dev/null
+++ b/rust/crates/audio/tests/collection.rs
@@ -0,0 +1,263 @@
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+
+use audio::collect_audible_elements;
+use scene::{
+ AudioElement, BaseTimelineElement, Element, LibraryAudioElement, ParamValue, RetimeConfig,
+ SceneTracks, Track, UploadAudioElement, VideoElement,
+};
+use time::MediaTime;
+
+fn seconds(value: f64) -> MediaTime {
+ MediaTime::from_seconds_f64(value).expect("finite seconds")
+}
+
+fn base(id: &str, start: f64, duration: f64) -> BaseTimelineElement {
+ BaseTimelineElement {
+ id: id.to_string(),
+ name: id.to_string(),
+ duration: seconds(duration),
+ start_time: seconds(start),
+ trim_start: MediaTime::ZERO,
+ trim_end: MediaTime::ZERO,
+ source_duration: None,
+ animations: None,
+ params: BTreeMap::new(),
+ }
+}
+
+fn upload_audio(id: &str, start: f64, duration: f64) -> Element {
+ Element::Audio(AudioElement::Upload(UploadAudioElement {
+ base: base(id, start, duration),
+ retime: None,
+ media_id: format!("media-{id}"),
+ }))
+}
+
+fn library_audio(id: &str, start: f64, duration: f64) -> Element {
+ Element::Audio(AudioElement::Library(LibraryAudioElement {
+ base: base(id, start, duration),
+ retime: None,
+ source_url: format!("https://cdn.test/{id}.mp3"),
+ }))
+}
+
+fn video(id: &str, start: f64, duration: f64) -> Element {
+ Element::Video(VideoElement {
+ base: base(id, start, duration),
+ media_id: format!("media-{id}"),
+ is_source_audio_enabled: None,
+ hidden: None,
+ retime: None,
+ effects: None,
+ masks: None,
+ })
+}
+
+fn main_track(elements: Vec<Element>) -> Track {
+ Track::Video {
+ id: "main".to_string(),
+ name: "main".to_string(),
+ elements,
+ muted: false,
+ hidden: false,
+ }
+}
+
+fn audio_track(id: &str, muted: bool, elements: Vec<Element>) -> Track {
+ Track::Audio {
+ id: id.to_string(),
+ name: id.to_string(),
+ elements,
+ muted,
+ }
+}
+
+fn tracks(main: Vec<Element>, audio: Vec<Track>) -> SceneTracks {
+ SceneTracks {
+ overlay: vec![],
+ main: main_track(main),
+ audio,
+ }
+}
+
+fn resolve_any(key: &str) -> Option<PathBuf> {
+ Some(PathBuf::from(key))
+}
+
+fn full_range() -> (MediaTime, MediaTime) {
+ (MediaTime::ZERO, seconds(60.0))
+}
+
+#[test]
+fn collects_upload_audio_and_video_with_source_audio() {
+ let scene = tracks(
+ vec![video("v1", 0.0, 2.0)],
+ vec![audio_track("a", false, vec![upload_audio("u1", 1.0, 1.0)])],
+ );
+ let clips = collect_audible_elements(&scene, &resolve_any, full_range());
+ assert_eq!(clips.len(), 2);
+ assert_eq!(clips[0].element_id, "v1");
+ assert_eq!(clips[1].element_id, "u1");
+ assert_eq!(clips[0].path, PathBuf::from("media-v1"));
+ assert_eq!(clips[1].start_time, seconds(1.0));
+ assert_eq!(clips[1].duration, seconds(1.0));
+}
+
+#[test]
+fn library_audio_resolves_by_source_url() {
+ let scene = tracks(
+ vec![],
+ vec![audio_track("a", false, vec![library_audio("lib", 0.0, 1.0)])],
+ );
+ let clips = collect_audible_elements(&scene, &resolve_any, full_range());
+ assert_eq!(clips.len(), 1);
+ assert_eq!(clips[0].path, PathBuf::from("https://cdn.test/lib.mp3"));
+}
+
+#[test]
+fn muted_track_excludes_its_elements() {
+ let scene = tracks(
+ vec![],
+ vec![
+ audio_track("muted-track", true, vec![upload_audio("u1", 0.0, 1.0)]),
+ audio_track("live-track", false, vec![upload_audio("u2", 0.0, 1.0)]),
+ ],
+ );
+ let clips = collect_audible_elements(&scene, &resolve_any, full_range());
+ assert_eq!(clips.len(), 1);
+ assert_eq!(clips[0].element_id, "u2");
+}
+
+#[test]
+fn muted_element_is_excluded() {
+ let mut element = match upload_audio("u1", 0.0, 1.0) {
+ Element::Audio(AudioElement::Upload(mut upload)) => {
+ upload
+ .base
+ .params
+ .insert("muted".to_string(), ParamValue::Bool(true));
+ Element::Audio(AudioElement::Upload(upload))
+ }
+ other => other,
+ };
+ let scene = tracks(vec![], vec![audio_track("a", false, vec![element])]);
+ let clips = collect_audible_elements(&scene, &resolve_any, full_range());
+ assert!(clips.is_empty());
+
+ element = match upload_audio("u2", 0.0, 1.0) {
+ Element::Audio(AudioElement::Upload(mut upload)) => {
+ upload
+ .base
+ .params
+ .insert("muted".to_string(), ParamValue::Bool(false));
+ Element::Audio(AudioElement::Upload(upload))
+ }
+ other => other,
+ };
+ let scene = tracks(vec![], vec![audio_track("a", false, vec![element])]);
+ assert_eq!(collect_audible_elements(&scene, &resolve_any, full_range()).len(), 1);
+}
+
+#[test]
+fn hidden_video_is_excluded() {
+ let hidden = Element::Video(VideoElement {
+ hidden: Some(true),
+ ..match video("v1", 0.0, 2.0) {
+ Element::Video(v) => v,
+ _ => unreachable!(),
+ }
+ });
+ let scene = tracks(vec![hidden], vec![]);
+ assert!(collect_audible_elements(&scene, &resolve_any, full_range()).is_empty());
+}
+
+#[test]
+fn video_with_source_audio_disabled_is_excluded() {
+ let disabled = Element::Video(VideoElement {
+ is_source_audio_enabled: Some(false),
+ ..match video("v1", 0.0, 2.0) {
+ Element::Video(v) => v,
+ _ => unreachable!(),
+ }
+ });
+ let enabled = Element::Video(VideoElement {
+ is_source_audio_enabled: Some(true),
+ ..match video("v2", 0.0, 2.0) {
+ Element::Video(v) => v,
+ _ => unreachable!(),
+ }
+ });
+ let scene = tracks(vec![disabled, enabled], vec![]);
+ let clips = collect_audible_elements(&scene, &resolve_any, full_range());
+ assert_eq!(clips.len(), 1);
+ assert_eq!(clips[0].element_id, "v2");
+}
+
+#[test]
+fn range_overlap_trims_collection() {
+ let scene = tracks(
+ vec![video("v1", 2.0, 2.0), video("v2", 10.0, 2.0)],
+ vec![],
+ );
+ let clips = collect_audible_elements(&scene, &resolve_any, (seconds(3.0), seconds(4.5)));
+ assert_eq!(clips.len(), 1);
+ assert_eq!(clips[0].element_id, "v1");
+
+ let clips = collect_audible_elements(&scene, &resolve_any, (seconds(4.0), seconds(10.0)));
+ assert!(clips.is_empty());
+
+ let clips = collect_audible_elements(&scene, &resolve_any, (seconds(4.0), seconds(10.5)));
+ assert_eq!(clips.len(), 1);
+ assert_eq!(clips[0].element_id, "v2");
+}
+
+#[test]
+fn volume_param_is_decibels_with_zero_default() {
+ let quiet = match upload_audio("u1", 0.0, 1.0) {
+ Element::Audio(AudioElement::Upload(mut upload)) => {
+ upload
+ .base
+ .params
+ .insert("volume".to_string(), ParamValue::Number(-6.0));
+ Element::Audio(AudioElement::Upload(upload))
+ }
+ other => other,
+ };
+ let plain = upload_audio("u2", 0.0, 1.0);
+ let scene = tracks(vec![], vec![audio_track("a", false, vec![quiet, plain])]);
+ let clips = collect_audible_elements(&scene, &resolve_any, full_range());
+ assert_eq!(clips[0].volume_db, -6.0);
+ assert_eq!(clips[1].volume_db, 0.0);
+ assert!((clips[0].static_gain() - 0.5012).abs() < 0.001);
+ assert_eq!(clips[1].static_gain(), 1.0);
+}
+
+#[test]
+fn retime_and_trim_are_carried() {
+ let element = match upload_audio("u1", 0.0, 0.5) {
+ Element::Audio(AudioElement::Upload(mut upload)) => {
+ upload.base.trim_start = seconds(0.25);
+ upload.base.trim_end = seconds(0.25);
+ upload.retime = Some(RetimeConfig {
+ rate: 2.0,
+ maintain_pitch: None,
+ });
+ Element::Audio(AudioElement::Upload(upload))
+ }
+ other => other,
+ };
+ let scene = tracks(vec![], vec![audio_track("a", false, vec![element])]);
+ let clips = collect_audible_elements(&scene, &resolve_any, full_range());
+ assert_eq!(clips.len(), 1);
+ assert_eq!(clips[0].trim_start, seconds(0.25));
+ assert_eq!(clips[0].effective_rate(), 2.0);
+ assert!(!clips[0].requires_maintain_pitch());
+}
+
+#[test]
+fn unresolvable_media_is_excluded() {
+ let scene = tracks(vec![video("v1", 0.0, 1.0)], vec![]);
+ let clips = collect_audible_elements(&scene, &|_| None, full_range());
+ assert!(clips.is_empty());
+}
diff --git a/rust/crates/audio/tests/fixtures/sample.mp4 b/rust/crates/audio/tests/fixtures/sample.mp4
new file mode 100644
index 00000000..13c113e5
--- /dev/null
+++ b/rust/crates/audio/tests/fixtures/sample.mp4
Binary files differ
diff --git a/rust/crates/audio/tests/fixtures/tone.wav b/rust/crates/audio/tests/fixtures/tone.wav
new file mode 100644
index 00000000..8d5f7886
--- /dev/null
+++ b/rust/crates/audio/tests/fixtures/tone.wav
Binary files differ
diff --git a/rust/crates/audio/tests/mix.rs b/rust/crates/audio/tests/mix.rs
new file mode 100644
index 00000000..6d0ed0d1
--- /dev/null
+++ b/rust/crates/audio/tests/mix.rs
@@ -0,0 +1,183 @@
+use std::path::PathBuf;
+
+use audio::{
+ AudibleClip, Error, apply_mastering, extract_rms_buckets, mix_range, render_export_buffer,
+};
+use media::Decoder;
+use scene::RetimeConfig;
+use time::MediaTime;
+
+const TONE_WAV: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/tone.wav");
+
+fn seconds(value: f64) -> MediaTime {
+ MediaTime::from_seconds_f64(value).expect("finite seconds")
+}
+
+fn tone_clip(id: &str, start: f64, duration: f64) -> AudibleClip {
+ AudibleClip {
+ element_id: id.to_string(),
+ path: PathBuf::from(TONE_WAV),
+ start_time: seconds(start),
+ duration: seconds(duration),
+ trim_start: MediaTime::ZERO,
+ trim_end: MediaTime::ZERO,
+ volume_db: 0.0,
+ retime: None,
+ animations: None,
+ }
+}
+
+fn decode_tone(start: f64, end: f64) -> (Vec<f32>, u32, usize) {
+ let mut decoder = Decoder::open(TONE_WAV).expect("open tone.wav");
+ let stream = decoder.info().audio_streams.first().cloned().expect("audio stream");
+ let decoded = decoder
+ .decode_audio_range(stream.index, seconds(start), seconds(end))
+ .expect("decode tone");
+ (decoded.samples, decoded.sample_rate, decoded.channels as usize)
+}
+
+#[test]
+fn two_overlapping_tones_sum() {
+ let clip_a = tone_clip("a", 0.0, 1.0);
+ let clip_b = tone_clip("b", 0.0, 1.0);
+ let single = mix_range(&[clip_a.clone()], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap();
+ let doubled = mix_range(&[clip_a, clip_b], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap();
+ assert_eq!(single.len(), doubled.len());
+ for (one, two) in single.iter().zip(doubled.iter()) {
+ assert!((two - 2.0 * one).abs() < 1e-6, "{two} != 2 * {one}");
+ }
+}
+
+#[test]
+fn volume_param_scales_samples() {
+ let full = tone_clip("full", 0.0, 1.0);
+ let mut half = tone_clip("half", 0.0, 1.0);
+ half.volume_db = -6.020599913279624;
+ let full_mix = mix_range(&[full], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap();
+ let half_mix = mix_range(&[half], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap();
+ for (f, h) in full_mix.iter().zip(half_mix.iter()) {
+ assert!((h - f * 0.5).abs() < 1e-3, "{h} != 0.5 * {f}");
+ }
+}
+
+#[test]
+fn retime_2x_consumes_source_twice_as_fast() {
+ let normal = tone_clip("normal", 0.0, 1.0);
+ let mut fast = tone_clip("fast", 0.0, 0.5);
+ fast.retime = Some(RetimeConfig {
+ rate: 2.0,
+ maintain_pitch: None,
+ });
+ let normal_mix = mix_range(&[normal], (seconds(0.0), seconds(1.0)), 44100, 2).unwrap();
+ let fast_mix = mix_range(&[fast], (seconds(0.0), seconds(0.5)), 44100, 2).unwrap();
+ let frames = fast_mix.len() / 2;
+ for i in 0..frames {
+ for channel in 0..2 {
+ let expected = normal_mix[2 * i * 2 + channel];
+ let actual = fast_mix[i * 2 + channel];
+ assert!((actual - expected).abs() < 1e-6, "frame {i}: {actual} != {expected}");
+ }
+ }
+}
+
+#[test]
+fn trim_start_offsets_source() {
+ let mut clip = tone_clip("trimmed", 0.0, 0.5);
+ clip.trim_start = seconds(0.5);
+ let mix = mix_range(&[clip], (seconds(0.0), seconds(0.5)), 44100, 2).unwrap();
+ let (decoded, _rate, channels) = decode_tone(0.5, 1.0);
+ assert_eq!(channels, 1);
+ let frames = mix.len() / 2;
+ for i in 0..frames {
+ for channel in 0..2 {
+ let actual = mix[i * 2 + channel];
+ assert!((actual - decoded[i]).abs() < 1e-6, "frame {i}: {actual} != {}", decoded[i]);
+ }
+ }
+}
+
+#[test]
+fn rms_buckets_of_tone_match_hand_computed() {
+ let (samples, rate, channels) = decode_tone(0.0, 1.0);
+ let peak = samples.iter().fold(0.0_f32, |p, s| p.max(s.abs()));
+ let expected_rms = peak / 2.0_f32.sqrt();
+ let buckets = extract_rms_buckets(&samples, channels, rate, 10);
+ assert_eq!(buckets.len(), 10);
+ for bucket in &buckets {
+ let relative = (bucket - expected_rms).abs() / expected_rms;
+ assert!(relative < 0.05, "bucket {bucket} vs expected {expected_rms}");
+ }
+}
+
+#[test]
+fn mastering_limits_peaks_on_hot_input() {
+ let frames = 4096;
+ let mut samples = vec![1.5_f32; frames * 2];
+ apply_mastering(&mut samples, 2, 44100);
+ let peak = samples.iter().fold(0.0_f32, |p, s| p.max(s.abs()));
+ assert!(peak <= 0.98, "peak was {peak}");
+ assert!(peak > 0.9, "limiter over-attenuated: peak {peak}");
+}
+
+#[test]
+fn mastering_leaves_quiet_input_untouched() {
+ let mut samples: Vec<f32> = (0..1000).map(|i| (i as f32 / 1000.0) * 0.5).collect();
+ let original = samples.clone();
+ apply_mastering(&mut samples, 1, 44100);
+ assert_eq!(samples, original);
+}
+
+#[test]
+fn maintain_pitch_true_returns_typed_error() {
+ let mut clip = tone_clip("pitched", 0.0, 0.5);
+ clip.retime = Some(RetimeConfig {
+ rate: 2.0,
+ maintain_pitch: Some(true),
+ });
+ let result = mix_range(&[clip], (seconds(0.0), seconds(1.0)), 44100, 2);
+ match result {
+ Err(Error::MaintainPitchUnsupported { element_id, rate }) => {
+ assert_eq!(element_id, "pitched");
+ assert_eq!(rate, 2.0);
+ }
+ other => panic!("expected MaintainPitchUnsupported, got {other:?}"),
+ }
+}
+
+#[test]
+fn maintain_pitch_at_unity_rate_is_allowed() {
+ let mut clip = tone_clip("unity", 0.0, 0.5);
+ clip.retime = Some(RetimeConfig {
+ rate: 1.0,
+ maintain_pitch: Some(true),
+ });
+ assert!(mix_range(&[clip], (seconds(0.0), seconds(0.5)), 44100, 2).is_ok());
+}
+
+#[test]
+fn render_export_buffer_mixes_and_masters() {
+ let clip = tone_clip("a", 0.0, 1.0);
+ let buffer = render_export_buffer(&[clip], seconds(1.0), 44100, 2).unwrap();
+ assert_eq!(buffer.len(), 44100 * 2);
+ let peak = buffer.iter().fold(0.0_f32, |p, s| p.max(s.abs()));
+ assert!(peak > 0.05, "peak was {peak}");
+ assert!(render_export_buffer(&[], seconds(1.0), 44100, 2).unwrap().is_empty());
+}
+
+#[test]
+fn range_render_only_covers_requested_window() {
+ let clip = tone_clip("a", 1.0, 1.0);
+ let buffer = mix_range(&[clip], (seconds(0.0), seconds(2.0)), 44100, 2).unwrap();
+ let first_half = &buffer[..44100 * 2];
+ assert!(first_half.iter().all(|s| *s == 0.0));
+ let second_peak = buffer[44100 * 2..].iter().fold(0.0_f32, |p, s| p.max(s.abs()));
+ assert!(second_peak > 0.05);
+}
+
+#[test]
+fn undecodable_clip_contributes_silence() {
+ let mut clip = tone_clip("missing", 0.0, 1.0);
+ clip.path = PathBuf::from("/nonexistent/nope.wav");
+ let buffer = mix_range(&[clip], (seconds(0.0), seconds(0.5)), 44100, 2).unwrap();
+ assert!(buffer.iter().all(|s| *s == 0.0));
+}