feat: clip volume line and timeline waveform improvements

2 files changed, 353 insertions, 42 deletions

diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts
index 04ea580f..675cc3a5 100644
--- a/apps/web/src/lib/media/audio.ts
+++ b/apps/web/src/lib/media/audio.ts
@@ -20,10 +20,13 @@ import { mediaSupportsAudio } from "@/lib/media/media-utils";
 import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime";
 import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
 import { TICKS_PER_SECOND } from "@/lib/wasm";
+import {
+	computeRmsBuckets,
+	type SampleBucket,
+} from "@/lib/media/waveform-summary";
 
 const MAX_AUDIO_CHANNELS = 2;
 const EXPORT_SAMPLE_RATE = 44100;
-const COARSE_SAMPLE_COUNT = 2048;
 
 export interface CollectedAudioElement {
 	timelineElement: AudioCapableElement;
@@ -177,8 +180,8 @@ export async function collectAudioElements({
 			if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
 
 			pendingElements.push(
-				resolveAudioBufferForVideoElement({
-					mediaAsset,
+				resolveAudioBufferForAsset({
+					asset: mediaAsset,
 					audioContext,
 				}).then((audioBuffer) => {
 					if (!audioBuffer) return null;
@@ -222,10 +225,8 @@ async function resolveAudioBufferForElement({
 	try {
 		if (element.sourceType === "upload") {
 			const asset = mediaMap.get(element.mediaId);
-			if (!asset || asset.type !== "audio") return null;
-
-			const arrayBuffer = await asset.file.arrayBuffer();
-			return await audioContext.decodeAudioData(arrayBuffer.slice(0));
+			if (!asset) return null;
+			return await resolveAudioBufferForAsset({ asset, audioContext });
 		}
 
 		if (element.buffer) return element.buffer;
@@ -243,15 +244,25 @@ async function resolveAudioBufferForElement({
 	}
 }
 
-async function resolveAudioBufferForVideoElement({
-	mediaAsset,
+async function resolveAudioBufferForAsset({
+	asset,
 	audioContext,
 }: {
-	mediaAsset: MediaAsset;
+	asset: MediaAsset;
 	audioContext: AudioContext;
 }): Promise<AudioBuffer | null> {
+	if (asset.type === "audio") {
+		try {
+			const arrayBuffer = await asset.file.arrayBuffer();
+			return await audioContext.decodeAudioData(arrayBuffer.slice(0));
+		} catch (error) {
+			console.warn("Failed to decode audio asset:", error);
+			return null;
+		}
+	}
+
 	const input = new Input({
-		source: new BlobSource(mediaAsset.file),
+		source: new BlobSource(asset.file),
 		formats: ALL_FORMATS,
 	});
 
@@ -319,7 +330,7 @@ async function resolveAudioBufferForVideoElement({
 
 		return await offlineContext.startRendering();
 	} catch (error) {
-		console.warn("Failed to decode video audio:", error);
+		console.warn("Failed to decode asset audio:", error);
 		return null;
 	} finally {
 		input.dispose();
@@ -675,63 +686,130 @@ export async function createTimelineAudioBuffer({
 	return await applyAudioMasteringToBuffer({ audioBuffer: outputBuffer });
 }
 
-export function computeGlobalMaxRms({
+function collectPeakRange({
 	buffer,
+	count,
+	startSample,
+	endSample,
 }: {
 	buffer: AudioBuffer;
-}): number {
+	count: number;
+	startSample: number;
+	endSample: number;
+}): Float32Array {
 	const channels = buffer.numberOfChannels;
-	const step = Math.max(1, Math.floor(buffer.length / COARSE_SAMPLE_COUNT));
-	let globalMax = 0;
+	const peaks = new Float32Array(count);
 
 	for (let c = 0; c < channels; c++) {
 		const data = buffer.getChannelData(c);
-		for (let i = 0; i + step <= buffer.length; i += step) {
-			for (let j = i; j < i + step; j++) {
+		for (let i = 0; i < count; i++) {
+			const { bucketStart: start, bucketEnd: end } = getSampleBucketRange({
+				startSample,
+				endSample,
+				bucketIndex: i,
+				bucketCount: count,
+			});
+			for (let j = start; j < end; j++) {
 				const abs = Math.abs(data[j]);
-				if (abs > globalMax) globalMax = abs;
+				if (abs > peaks[i]) peaks[i] = abs;
 			}
 		}
 	}
 
-	return globalMax || 1;
+	return peaks;
 }
 
-export function extractRmsRange({
+export function extractPeakRange({
 	buffer,
 	count,
 	startSample,
 	endSample,
-	globalMax,
 }: {
 	buffer: AudioBuffer;
 	count: number;
 	startSample: number;
 	endSample: number;
-	globalMax: number;
 }): number[] {
-	const channels = buffer.numberOfChannels;
-	const rangeLength = endSample - startSample;
-	const step = Math.max(1, Math.floor(rangeLength / count));
-	const peaks = new Float32Array(count);
+	return Array.from(
+		collectPeakRange({
+			buffer,
+			count,
+			startSample,
+			endSample,
+		}),
+	);
+}
 
-	for (let c = 0; c < channels; c++) {
-		const data = buffer.getChannelData(c);
-		for (let i = 0; i < count; i++) {
-			const start = startSample + i * step;
-			const end = Math.min(start + step, endSample);
-			for (let j = start; j < end; j++) {
-				const abs = Math.abs(data[j]);
-				if (abs > peaks[i]) peaks[i] = abs;
-			}
-		}
-	}
+export function getSampleBucketRange({
+	startSample,
+	endSample,
+	bucketIndex,
+	bucketCount,
+}: {
+	startSample: number;
+	endSample: number;
+	bucketIndex: number;
+	bucketCount: number;
+}): {
+	bucketStart: number;
+	bucketEnd: number;
+} {
+	const rangeLength = Math.max(0, endSample - startSample);
+	const bucketStart =
+		startSample + Math.floor((bucketIndex * rangeLength) / bucketCount);
+	const bucketEnd =
+		startSample + Math.floor(((bucketIndex + 1) * rangeLength) / bucketCount);
+	return {
+		bucketStart,
+		bucketEnd: Math.max(bucketStart, bucketEnd),
+	};
+}
 
-	const norm = 1 / globalMax;
-	const result = new Array<number>(count);
-	for (let i = 0; i < count; i++) result[i] = Math.min(1, peaks[i] * norm);
+export function extractRmsBuckets({
+	buffer,
+	buckets,
+}: {
+	buffer: AudioBuffer;
+	buckets: SampleBucket[];
+}): number[] {
+	return computeRmsBuckets({ buffer, buckets });
+}
 
-	return result;
+/**
+ * Computes per-bucket waveform amplitude using the maximum RMS over a short
+ * analysis window inside each bucket.
+ *
+ * A naive mean-RMS over a whole bucket averages silence together with nearby
+ * sound, which smears transitions (e.g. the onset of speech) across the
+ * bucket and makes the waveform respond late. Taking the max over fixed
+ * short windows (~20 ms) preserves the smooth, non-jittery RMS character
+ * while making transitions land where they actually happen in the audio.
+ *
+ * Channels are combined per-window before taking the max, so the measure
+ * reflects total energy regardless of stereo layout.
+ */
+export function extractRmsRange({
+	buffer,
+	count,
+	startSample,
+	endSample,
+}: {
+	buffer: AudioBuffer;
+	count: number;
+	startSample: number;
+	endSample: number;
+}): number[] {
+	return extractRmsBuckets({
+		buffer,
+		buckets: Array.from({ length: count }, (_, bucketIndex) =>
+			getSampleBucketRange({
+				startSample,
+				endSample,
+				bucketIndex,
+				bucketCount: count,
+			}),
+		),
+	});
 }
 
 function mixAudioChannels({
diff --git a/apps/web/src/lib/media/waveform-summary.ts b/apps/web/src/lib/media/waveform-summary.ts
new file mode 100644
index 00000000..522d6e04
--- /dev/null
+++ b/apps/web/src/lib/media/waveform-summary.ts
@@ -0,0 +1,233 @@
+"use client";
+
+import { getSourceTimeAtClipTime } from "@/lib/retime";
+import type { RetimeConfig } from "@/lib/timeline";
+
+const RMS_ANALYSIS_WINDOW_SECONDS = 0.02;
+const DEFAULT_SOURCE_WAVEFORM_BUCKET_SIZE = 128;
+
+function computePeakBuckets({
+	buffer,
+	buckets,
+}: {
+	buffer: AudioBuffer;
+	buckets: SampleBucket[];
+}): number[] {
+	const channels = buffer.numberOfChannels;
+	const channelData: Float32Array[] = Array.from({ length: channels }, (_, c) =>
+		buffer.getChannelData(c),
+	);
+
+	return buckets.map(({ bucketStart, bucketEnd }) => {
+		let peak = 0;
+		for (let c = 0; c < channels; c++) {
+			const data = channelData[c];
+			for (let j = bucketStart; j < bucketEnd; j++) {
+				const abs = Math.abs(data[j] ?? 0);
+				if (abs > peak) {
+					peak = abs;
+				}
+			}
+		}
+		return peak;
+	});
+}
+
+export interface SampleBucket {
+	bucketStart: number;
+	bucketEnd: number;
+}
+
+export interface SourceWaveformSummary {
+	sourceKey: string;
+	sampleRate: number;
+	totalSamples: number;
+	bucketSize: number;
+	amplitudes: Float32Array;
+}
+
+export function buildWaveformSourceKey({
+	kind,
+	id,
+}: {
+	kind: "media" | "library";
+	id: string;
+}): string {
+	return `${kind}:${id}`;
+}
+
+export function buildSourceWaveformSummary({
+	sourceKey,
+	buffer,
+	bucketSize = DEFAULT_SOURCE_WAVEFORM_BUCKET_SIZE,
+}: {
+	sourceKey: string;
+	buffer: AudioBuffer;
+	bucketSize?: number;
+}): SourceWaveformSummary {
+	const safeBucketSize = Math.max(1, Math.floor(bucketSize));
+	const bucketCount = Math.max(1, Math.ceil(buffer.length / safeBucketSize));
+	const amplitudes = computePeakBuckets({
+		buffer,
+		buckets: Array.from({ length: bucketCount }, (_, bucketIndex) => {
+			const bucketStart = bucketIndex * safeBucketSize;
+			const bucketEnd = Math.min(buffer.length, bucketStart + safeBucketSize);
+			return { bucketStart, bucketEnd };
+		}),
+	});
+
+	return {
+		sourceKey,
+		sampleRate: buffer.sampleRate,
+		totalSamples: buffer.length,
+		bucketSize: safeBucketSize,
+		amplitudes: Float32Array.from(amplitudes),
+	};
+}
+
+export function buildWaveformSampleBuckets({
+	clipLeftPx,
+	clipRightPx,
+	barCount,
+	pixelsPerSecond,
+	clipDurationSec,
+	sourceStartSec,
+	retime,
+	sampleRate,
+	maxSampleExclusive,
+	barStepPx,
+}: {
+	clipLeftPx: number;
+	clipRightPx: number;
+	barCount: number;
+	pixelsPerSecond: number;
+	clipDurationSec: number;
+	sourceStartSec: number;
+	retime?: RetimeConfig;
+	sampleRate: number;
+	maxSampleExclusive: number;
+	barStepPx: number;
+}): SampleBucket[] {
+	return Array.from({ length: barCount }, (_, index) => {
+		const bucketLeftPx = clipLeftPx + index * barStepPx;
+		const bucketRightPx = Math.min(clipRightPx, bucketLeftPx + barStepPx);
+		const clipStartSec = Math.max(
+			0,
+			Math.min(clipDurationSec, bucketLeftPx / pixelsPerSecond),
+		);
+		const clipEndSec = Math.max(
+			clipStartSec,
+			Math.min(clipDurationSec, bucketRightPx / pixelsPerSecond),
+		);
+		const sourceBucketStartSec =
+			sourceStartSec +
+			getSourceTimeAtClipTime({
+				clipTime: clipStartSec,
+				retime,
+			});
+		const sourceBucketEndSec =
+			sourceStartSec +
+			getSourceTimeAtClipTime({
+				clipTime: clipEndSec,
+				retime,
+			});
+
+		return {
+			bucketStart: Math.max(0, Math.floor(sourceBucketStartSec * sampleRate)),
+			bucketEnd: Math.min(
+				maxSampleExclusive,
+				Math.max(0, Math.ceil(sourceBucketEndSec * sampleRate)),
+			),
+		};
+	});
+}
+
+export function sampleSourceWaveformSummary({
+	summary,
+	buckets,
+}: {
+	summary: SourceWaveformSummary;
+	buckets: SampleBucket[];
+}): number[] {
+	return buckets.map(({ bucketStart, bucketEnd }) => {
+		if (bucketEnd <= bucketStart) {
+			return 0;
+		}
+
+		const startIndex = Math.max(
+			0,
+			Math.floor(bucketStart / summary.bucketSize),
+		);
+		const endIndex = Math.min(
+			summary.amplitudes.length,
+			Math.max(startIndex + 1, Math.ceil(bucketEnd / summary.bucketSize)),
+		);
+
+		let maxAmplitude = 0;
+		for (let i = startIndex; i < endIndex; i++) {
+			const amplitude = summary.amplitudes[i] ?? 0;
+			if (amplitude > maxAmplitude) {
+				maxAmplitude = amplitude;
+			}
+		}
+
+		return maxAmplitude;
+	});
+}
+
+export function computeRmsBuckets({
+	buffer,
+	buckets,
+}: {
+	buffer: AudioBuffer;
+	buckets: SampleBucket[];
+}): number[] {
+	const channels = buffer.numberOfChannels;
+	const maxWindowLength = Math.max(
+		1,
+		Math.floor(buffer.sampleRate * RMS_ANALYSIS_WINDOW_SECONDS),
+	);
+
+	const channelData: Float32Array[] = new Array(channels);
+	for (let c = 0; c < channels; c++) {
+		channelData[c] = buffer.getChannelData(c);
+	}
+
+	const result = new Array<number>(buckets.length);
+
+	for (let i = 0; i < buckets.length; i++) {
+		const { bucketStart, bucketEnd } = buckets[i];
+		const bucketLength = bucketEnd - bucketStart;
+		if (bucketLength <= 0) {
+			result[i] = 0;
+			continue;
+		}
+
+		const windowLength = Math.max(1, Math.min(bucketLength, maxWindowLength));
+		let maxMeanSquare = 0;
+
+		for (let winStart = bucketStart; winStart < bucketEnd; ) {
+			const winEnd = Math.min(winStart + windowLength, bucketEnd);
+			const n = winEnd - winStart;
+			if (n > 0) {
+				let sum = 0;
+				for (let c = 0; c < channels; c++) {
+					const data = channelData[c];
+					for (let j = winStart; j < winEnd; j++) {
+						const v = data[j];
+						sum += v * v;
+					}
+				}
+				const meanSquare = sum / (n * channels);
+				if (meanSquare > maxMeanSquare) {
+					maxMeanSquare = meanSquare;
+				}
+			}
+			winStart = winEnd;
+		}
+
+		result[i] = Math.sqrt(maxMeanSquare);
+	}
+
+	return result;
+}