summaryrefslogtreecommitdiff
path: root/rust/crates/audio/src
diff options
context:
space:
mode:
Diffstat (limited to 'rust/crates/audio/src')
-rw-r--r--rust/crates/audio/src/buckets.rs105
-rw-r--r--rust/crates/audio/src/clips.rs169
-rw-r--r--rust/crates/audio/src/lib.rs29
-rw-r--r--rust/crates/audio/src/mix.rs181
4 files changed, 484 insertions, 0 deletions
diff --git a/rust/crates/audio/src/buckets.rs b/rust/crates/audio/src/buckets.rs
new file mode 100644
index 00000000..261c2373
--- /dev/null
+++ b/rust/crates/audio/src/buckets.rs
@@ -0,0 +1,105 @@
+pub const RMS_ANALYSIS_WINDOW_SECONDS: f64 = 0.02;
+
+pub fn sample_bucket_range(
+ start_sample: usize,
+ end_sample: usize,
+ bucket_index: usize,
+ bucket_count: usize,
+) -> (usize, usize) {
+ let range_length = end_sample.saturating_sub(start_sample);
+ let bucket_start = start_sample + bucket_index * range_length / bucket_count;
+ let bucket_end = start_sample + (bucket_index + 1) * range_length / bucket_count;
+ (bucket_start, bucket_end.max(bucket_start))
+}
+
+pub fn extract_peak_range(
+ samples: &[f32],
+ channels: usize,
+ count: usize,
+ start_sample: usize,
+ end_sample: usize,
+) -> Vec<f32> {
+ let channels = channels.max(1);
+ let frames = samples.len() / channels;
+ let end = end_sample.min(frames);
+ let start = start_sample.min(end);
+ let mut peaks = vec![0.0_f32; count];
+ for (index, peak) in peaks.iter_mut().enumerate() {
+ let (bucket_start, bucket_end) = sample_bucket_range(start, end, index, count);
+ let mut bucket_peak = 0.0_f32;
+ for frame in bucket_start..bucket_end {
+ for channel in 0..channels {
+ let magnitude = samples[frame * channels + channel].abs();
+ if magnitude > bucket_peak {
+ bucket_peak = magnitude;
+ }
+ }
+ }
+ *peak = bucket_peak;
+ }
+ peaks
+}
+
+pub fn extract_rms_windowed(
+ samples: &[f32],
+ channels: usize,
+ sample_rate: u32,
+ buckets: &[(usize, usize)],
+) -> Vec<f32> {
+ let channels = channels.max(1);
+ let max_window =
+ (f64::from(sample_rate) * RMS_ANALYSIS_WINDOW_SECONDS).floor().max(1.0) as usize;
+ buckets
+ .iter()
+ .map(|&(bucket_start, bucket_end)| {
+ let bucket_length = bucket_end.saturating_sub(bucket_start);
+ if bucket_length == 0 {
+ return 0.0;
+ }
+ let window_length = bucket_length.min(max_window).max(1);
+ let mut max_mean_square = 0.0_f64;
+ let mut window_start = bucket_start;
+ while window_start < bucket_end {
+ let window_end = (window_start + window_length).min(bucket_end);
+ let n = window_end - window_start;
+ let mut sum = 0.0_f64;
+ for frame in window_start..window_end {
+ for channel in 0..channels {
+ let value = f64::from(samples[frame * channels + channel]);
+ sum += value * value;
+ }
+ }
+ let mean_square = sum / (n * channels) as f64;
+ if mean_square > max_mean_square {
+ max_mean_square = mean_square;
+ }
+ window_start = window_end;
+ }
+ max_mean_square.sqrt() as f32
+ })
+ .collect()
+}
+
+pub fn extract_rms_range(
+ samples: &[f32],
+ channels: usize,
+ sample_rate: u32,
+ count: usize,
+ start_sample: usize,
+ end_sample: usize,
+) -> Vec<f32> {
+ let buckets: Vec<(usize, usize)> = (0..count)
+ .map(|index| sample_bucket_range(start_sample, end_sample, index, count))
+ .collect();
+ extract_rms_windowed(samples, channels, sample_rate, &buckets)
+}
+
+pub fn extract_rms_buckets(
+ samples: &[f32],
+ channels: usize,
+ sample_rate: u32,
+ bucket_count: usize,
+) -> Vec<f32> {
+ let frames = samples.len() / channels.max(1);
+ extract_rms_range(samples, channels, sample_rate, bucket_count, 0, frames)
+}
diff --git a/rust/crates/audio/src/clips.rs b/rust/crates/audio/src/clips.rs
new file mode 100644
index 00000000..1f4c839c
--- /dev/null
+++ b/rust/crates/audio/src/clips.rs
@@ -0,0 +1,169 @@
+use std::path::PathBuf;
+
+use animation::{has_keyframes_for_path, resolve_number_at_time};
+use scene::{
+ AudioElement, BaseTimelineElement, Element, ElementAnimations, ParamValue, RetimeConfig,
+ SceneTracks, Track,
+};
+use time::MediaTime;
+
+pub const MIN_RETIME_RATE: f64 = 0.01;
+pub const MAX_RETIME_RATE: f64 = 5.0;
+pub const VOLUME_DB_MIN: f64 = -60.0;
+pub const VOLUME_DB_MAX: f64 = 20.0;
+
+pub fn clamp_retime_rate(rate: f64) -> f64 {
+ if !rate.is_finite() || rate <= 0.0 {
+ return 1.0;
+ }
+ rate.clamp(MIN_RETIME_RATE, MAX_RETIME_RATE)
+}
+
+pub fn source_time_at_clip_time(clip_time: f64, retime: Option<&RetimeConfig>) -> f64 {
+ clip_time * clamp_retime_rate(retime.map(|config| config.rate).unwrap_or(1.0))
+}
+
+pub fn clamp_db(value: f64) -> f64 {
+ if !value.is_finite() {
+ return 0.0;
+ }
+ value.clamp(VOLUME_DB_MIN, VOLUME_DB_MAX)
+}
+
+pub fn db_to_linear(db: f64) -> f64 {
+ 10_f64.powf(clamp_db(db) / 20.0)
+}
+
+#[derive(Debug, Clone)]
+pub struct AudibleClip {
+ pub element_id: String,
+ pub path: PathBuf,
+ pub start_time: MediaTime,
+ pub duration: MediaTime,
+ pub trim_start: MediaTime,
+ pub trim_end: MediaTime,
+ pub volume_db: f64,
+ pub retime: Option<RetimeConfig>,
+ pub animations: Option<ElementAnimations>,
+}
+
+impl AudibleClip {
+ pub fn end_time(&self) -> MediaTime {
+ self.start_time + self.duration
+ }
+
+ pub fn effective_rate(&self) -> f64 {
+ clamp_retime_rate(self.retime.map(|config| config.rate).unwrap_or(1.0))
+ }
+
+ pub fn requires_maintain_pitch(&self) -> bool {
+ self.retime.is_some_and(|config| {
+ config.maintain_pitch == Some(true)
+ && config.rate.is_finite()
+ && config.rate > 0.0
+ && clamp_retime_rate(config.rate) != 1.0
+ })
+ }
+
+ pub fn has_animated_volume(&self) -> bool {
+ has_keyframes_for_path(self.animations.as_ref(), "volume")
+ }
+
+ pub fn gain_at(&self, local_time: MediaTime) -> f64 {
+ db_to_linear(resolve_number_at_time(
+ self.volume_db,
+ self.animations.as_ref(),
+ "volume",
+ local_time,
+ ))
+ }
+
+ pub fn static_gain(&self) -> f64 {
+ db_to_linear(self.volume_db)
+ }
+}
+
+fn track_audio_muted(track: &Track) -> bool {
+ match track {
+ Track::Video { muted, .. } | Track::Audio { muted, .. } => *muted,
+ _ => false,
+ }
+}
+
+fn element_source(element: &Element) -> Option<(&BaseTimelineElement, &str, Option<RetimeConfig>)> {
+ match element {
+ Element::Audio(AudioElement::Upload(upload)) => {
+ Some((&upload.base, upload.media_id.as_str(), upload.retime))
+ }
+ Element::Audio(AudioElement::Library(library)) => {
+ Some((&library.base, library.source_url.as_str(), library.retime))
+ }
+ Element::Video(video) => {
+ if video.hidden == Some(true) || video.is_source_audio_enabled == Some(false) {
+ return None;
+ }
+ Some((&video.base, video.media_id.as_str(), video.retime))
+ }
+ _ => None,
+ }
+}
+
+fn collect_from_track(
+ track: &Track,
+ media: &dyn Fn(&str) -> Option<PathBuf>,
+ range: (MediaTime, MediaTime),
+ clips: &mut Vec<AudibleClip>,
+) {
+ if track_audio_muted(track) {
+ return;
+ }
+ for element in track.elements() {
+ let Some((base, source_key, retime)) = element_source(element) else {
+ continue;
+ };
+ if base.duration <= MediaTime::ZERO {
+ continue;
+ }
+ let end_time = base.start_time + base.duration;
+ if end_time <= range.0 || base.start_time >= range.1 {
+ continue;
+ }
+ if base.params.get("muted") == Some(&ParamValue::Bool(true)) {
+ continue;
+ }
+ let volume_db = match base.params.get("volume") {
+ Some(ParamValue::Number(value)) => *value,
+ _ => 0.0,
+ };
+ let Some(path) = media(source_key) else {
+ continue;
+ };
+ clips.push(AudibleClip {
+ element_id: base.id.clone(),
+ path,
+ start_time: base.start_time,
+ duration: base.duration,
+ trim_start: base.trim_start,
+ trim_end: base.trim_end,
+ volume_db,
+ retime,
+ animations: base.animations.clone(),
+ });
+ }
+}
+
+pub fn collect_audible_elements(
+ tracks: &SceneTracks,
+ media: &dyn Fn(&str) -> Option<PathBuf>,
+ range: (MediaTime, MediaTime),
+) -> Vec<AudibleClip> {
+ let mut clips = Vec::new();
+ for track in &tracks.overlay {
+ collect_from_track(track, media, range, &mut clips);
+ }
+ collect_from_track(&tracks.main, media, range, &mut clips);
+ for track in &tracks.audio {
+ collect_from_track(track, media, range, &mut clips);
+ }
+ clips
+}
diff --git a/rust/crates/audio/src/lib.rs b/rust/crates/audio/src/lib.rs
new file mode 100644
index 00000000..0bce5647
--- /dev/null
+++ b/rust/crates/audio/src/lib.rs
@@ -0,0 +1,29 @@
+mod buckets;
+mod clips;
+mod mix;
+
+pub use buckets::{
+ extract_peak_range, extract_rms_buckets, extract_rms_range, extract_rms_windowed,
+ sample_bucket_range,
+};
+pub use clips::{
+ AudibleClip, clamp_db, clamp_retime_rate, collect_audible_elements, db_to_linear,
+ source_time_at_clip_time,
+};
+pub use mix::{apply_mastering, mix_range, render_export_buffer};
+
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+ #[error(
+ "element {element_id} requires pitch-preserving retime at rate {rate}, which is not supported"
+ )]
+ MaintainPitchUnsupported { element_id: String, rate: f64 },
+ #[error("invalid time range: end must be after start")]
+ InvalidTimeRange,
+ #[error("sample rate must be at least 1")]
+ InvalidSampleRate,
+ #[error("channel count must be at least 1")]
+ InvalidChannelCount,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
diff --git a/rust/crates/audio/src/mix.rs b/rust/crates/audio/src/mix.rs
new file mode 100644
index 00000000..887111eb
--- /dev/null
+++ b/rust/crates/audio/src/mix.rs
@@ -0,0 +1,181 @@
+use media::{AudioSamples, Decoder};
+use time::MediaTime;
+
+use crate::Error;
+use crate::clips::AudibleClip;
+
+pub const MASTER_LIMITER_THRESHOLD_DB: f64 = -1.0;
+pub const MASTER_LIMITER_RATIO: f64 = 20.0;
+pub const MASTER_LIMITER_ATTACK_SECONDS: f64 = 0.001;
+pub const MASTER_LIMITER_RELEASE_SECONDS: f64 = 0.12;
+pub const MASTER_OUTPUT_HEADROOM: f64 = 0.98;
+pub const MASTER_OUTPUT_HEADROOM_F32: f32 = 0.98;
+
+fn decode_clip_source(clip: &AudibleClip, source_start: f64, source_end: f64) -> Option<AudioSamples> {
+ let start = MediaTime::from_seconds_f64(source_start.max(0.0))?;
+ let end = MediaTime::from_seconds_f64(source_end.max(0.0))?;
+ if end <= start {
+ return None;
+ }
+ let mut decoder = Decoder::open(&clip.path).ok()?;
+ let stream = decoder.info().audio_streams.first().cloned()?;
+ decoder.decode_audio_range(stream.index, start, end).ok()
+}
+
+fn mix_clip(
+ clip: &AudibleClip,
+ range_start_seconds: f64,
+ out_frames: usize,
+ sample_rate: u32,
+ channel_count: usize,
+ output: &mut [f32],
+) {
+ let clip_start_seconds = clip.start_time.to_seconds_f64();
+ let clip_end_seconds = clip_start_seconds + clip.duration.to_seconds_f64();
+ let rate = clip.effective_rate();
+ let trim_start_seconds = clip.trim_start.to_seconds_f64();
+ let rate_f64 = f64::from(sample_rate);
+
+ let first_frame = ((clip_start_seconds - range_start_seconds) * rate_f64)
+ .ceil()
+ .max(0.0) as usize;
+ let last_frame = (((clip_end_seconds - range_start_seconds) * rate_f64)
+ .ceil()
+ .max(0.0) as usize)
+ .min(out_frames);
+ if last_frame <= first_frame {
+ return;
+ }
+
+ let source_start = trim_start_seconds
+ + (first_frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds) * rate;
+ let source_end = trim_start_seconds
+ + (last_frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds) * rate;
+ let Some(decoded) = decode_clip_source(clip, source_start, source_end) else {
+ return;
+ };
+ let decoded_channels = decoded.channels as usize;
+ let decoded_frames = decoded.samples.len() / decoded_channels;
+ let decoded_rate = f64::from(decoded.sample_rate);
+ let static_gain = clip.static_gain() as f32;
+ let animated_volume = clip.has_animated_volume();
+
+ for frame in first_frame..last_frame {
+ let clip_time = frame as f64 / rate_f64 + range_start_seconds - clip_start_seconds;
+ let source_time = trim_start_seconds + clip_time * rate;
+ let source_position = (source_time - source_start) * decoded_rate;
+ if source_position >= decoded_frames as f64 {
+ break;
+ }
+ let source_position = source_position.max(0.0);
+ let lower = source_position.floor() as usize;
+ let upper = (lower + 1).min(decoded_frames.saturating_sub(1));
+ let fraction = (source_position - lower as f64) as f32;
+ let gain = if animated_volume {
+ clip.gain_at(MediaTime::from_seconds_f64(clip_time).unwrap_or(MediaTime::ZERO)) as f32
+ } else {
+ static_gain
+ };
+ for channel in 0..channel_count {
+ let source_channel = channel.min(decoded_channels - 1);
+ let a = decoded.samples[lower * decoded_channels + source_channel];
+ let b = decoded.samples[upper * decoded_channels + source_channel];
+ output[frame * channel_count + channel] +=
+ (a * (1.0 - fraction) + b * fraction) * gain;
+ }
+ }
+}
+
+pub fn mix_range(
+ clips: &[AudibleClip],
+ range: (MediaTime, MediaTime),
+ sample_rate: u32,
+ channels: u32,
+) -> Result<Vec<f32>, Error> {
+ if range.1 <= range.0 {
+ return Err(Error::InvalidTimeRange);
+ }
+ if sample_rate == 0 {
+ return Err(Error::InvalidSampleRate);
+ }
+ if channels == 0 {
+ return Err(Error::InvalidChannelCount);
+ }
+ for clip in clips {
+ if clip.requires_maintain_pitch()
+ && clip.end_time() > range.0
+ && clip.start_time < range.1
+ {
+ return Err(Error::MaintainPitchUnsupported {
+ element_id: clip.element_id.clone(),
+ rate: clip.effective_rate(),
+ });
+ }
+ }
+
+ let range_seconds = (range.1 - range.0).to_seconds_f64();
+ let out_frames = (range_seconds * f64::from(sample_rate)).ceil() as usize;
+ let channel_count = channels as usize;
+ let mut output = vec![0.0_f32; out_frames * channel_count];
+ let range_start_seconds = range.0.to_seconds_f64();
+ for clip in clips {
+ mix_clip(
+ clip,
+ range_start_seconds,
+ out_frames,
+ sample_rate,
+ channel_count,
+ &mut output,
+ );
+ }
+ Ok(output)
+}
+
+pub fn apply_mastering(samples: &mut [f32], channels: usize, sample_rate: u32) {
+ let peak = samples.iter().fold(0.0_f32, |peak, s| peak.max(s.abs()));
+ if f64::from(peak) <= MASTER_OUTPUT_HEADROOM {
+ return;
+ }
+ let attack_coeff =
+ (-1.0 / (MASTER_LIMITER_ATTACK_SECONDS * f64::from(sample_rate))).exp();
+ let release_coeff =
+ (-1.0 / (MASTER_LIMITER_RELEASE_SECONDS * f64::from(sample_rate))).exp();
+ let mut envelope_db = 0.0_f64;
+ for frame in samples.chunks_mut(channels) {
+ let level = frame.iter().fold(0.0_f32, |level, s| level.max(s.abs()));
+ let level_db = 20.0 * f64::from(level.max(1e-6)).log10();
+ let over = level_db - MASTER_LIMITER_THRESHOLD_DB;
+ let target_db = if over > 0.0 {
+ -over * (1.0 - 1.0 / MASTER_LIMITER_RATIO)
+ } else {
+ 0.0
+ };
+ let coeff = if target_db < envelope_db {
+ attack_coeff
+ } else {
+ release_coeff
+ };
+ envelope_db = target_db + (envelope_db - target_db) * coeff;
+ let gain = 10_f64.powf(envelope_db / 20.0) * MASTER_OUTPUT_HEADROOM;
+ for sample in frame.iter_mut() {
+ *sample = (f64::from(*sample) * gain) as f32;
+ }
+ }
+ for sample in samples.iter_mut() {
+ *sample = sample.clamp(-MASTER_OUTPUT_HEADROOM_F32, MASTER_OUTPUT_HEADROOM_F32);
+ }
+}
+
+pub fn render_export_buffer(
+ clips: &[AudibleClip],
+ total_duration: MediaTime,
+ sample_rate: u32,
+ channels: u32,
+) -> Result<Vec<f32>, Error> {
+ if clips.is_empty() || total_duration <= MediaTime::ZERO {
+ return Ok(Vec::new());
+ }
+ let mut buffer = mix_range(clips, (MediaTime::ZERO, total_duration), sample_rate, channels)?;
+ apply_mastering(&mut buffer, channels as usize, sample_rate);
+ Ok(buffer)
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-17 08:31:17 +0000