feat: masks, properties refactor, shaders, storage migrations, and more

1 files changed, 31 insertions, 99 deletions

diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts
index 11fe9837..c2d1d7f3 100644
--- a/apps/web/src/lib/media/mediabunny.ts
+++ b/apps/web/src/lib/media/mediabunny.ts
@@ -1,7 +1,7 @@
-import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
-import { collectAudioMixSources } from "@/lib/media/audio";
-import type { TimelineTrack } from "@/types/timeline";
-import type { MediaAsset } from "@/types/assets";
+import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
+import { createTimelineAudioBuffer } from "@/lib/media/audio";
+import type { TimelineTrack } from "@/lib/timeline";
+import type { MediaAsset } from "@/lib/media/types";
 
 export async function getVideoInfo({
 	videoFile,
@@ -12,6 +12,7 @@ export async function getVideoInfo({
 	width: number;
 	height: number;
 	fps: number;
+	hasAudio: boolean;
 }> {
 	const input = new Input({
 		source: new BlobSource(videoFile),
@@ -27,12 +28,14 @@ export async function getVideoInfo({
 
 	const packetStats = await videoTrack.computePacketStats(100);
 	const fps = packetStats.averagePacketRate;
+	const audioTrack = await input.getPrimaryAudioTrack();
 
 	return {
 		duration,
 		width: videoTrack.displayWidth,
 		height: videoTrack.displayHeight,
 		fps,
+		hasAudio: audioTrack !== null,
 	};
 }
 
@@ -54,12 +57,16 @@ export const extractTimelineAudio = async ({
 		return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });
 	}
 
-	const audioMixSources = await collectAudioMixSources({
+	onProgress?.(10);
+
+	const audioBuffer = await createTimelineAudioBuffer({
 		tracks,
 		mediaAssets,
+		duration: totalDuration,
+		sampleRate: SAMPLE_RATE,
 	});
 
-	if (audioMixSources.length === 0) {
+	if (!audioBuffer) {
 		const silentDuration = Math.max(1, totalDuration);
 		const silentSamples = new Float32Array(
 			Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,
@@ -67,108 +74,33 @@ export const extractTimelineAudio = async ({
 		return createWavBlob({ samples: silentSamples });
 	}
 
-	const totalSamples = Math.ceil(totalDuration * SAMPLE_RATE);
-	const mixBuffers = [
-		new Float32Array(totalSamples),
-		new Float32Array(totalSamples),
-	];
-
-	for (let i = 0; i < audioMixSources.length; i++) {
-		const source = audioMixSources[i];
-
-		if (onProgress) {
-			onProgress((i / audioMixSources.length) * 90);
-		}
-
-		try {
-			await decodeAndMixAudioSource({
-				source,
-				mixBuffers,
-				totalSamples,
-			});
-		} catch (error) {
-			console.warn(
-				`Failed to process audio source ${source.file.name}:`,
-				error,
-			);
-		}
-	}
-
-	// clamp to prevent clipping
-	for (const channel of mixBuffers) {
-		for (let i = 0; i < channel.length; i++) {
-			channel[i] = Math.max(-1, Math.min(1, channel[i]));
-		}
-	}
+	onProgress?.(90);
 
-	// interleave channels for wav output
-	const interleavedSamples = new Float32Array(totalSamples * NUM_CHANNELS);
-	for (let i = 0; i < totalSamples; i++) {
-		interleavedSamples[i * 2] = mixBuffers[0][i];
-		interleavedSamples[i * 2 + 1] = mixBuffers[1][i];
-	}
-
-	if (onProgress) {
-		onProgress(100);
-	}
+	const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
+	onProgress?.(100);
 
 	return createWavBlob({ samples: interleavedSamples });
 };
 
-async function decodeAndMixAudioSource({
-	source,
-	mixBuffers,
-	totalSamples,
+function interleaveAudioBuffer({
+	audioBuffer,
 }: {
-	source: {
-		file: File;
-		startTime: number;
-		duration: number;
-		trimStart: number;
-	};
-	mixBuffers: Float32Array[];
-	totalSamples: number;
-}): Promise<void> {
-	const input = new Input({
-		source: new BlobSource(source.file),
-		formats: ALL_FORMATS,
-	});
-
-	const audioTrack = await input.getPrimaryAudioTrack();
-	if (!audioTrack) return;
-
-	const sink = new AudioBufferSink(audioTrack);
-	const trimEnd = source.trimStart + source.duration;
+	audioBuffer: AudioBuffer;
+}): Float32Array {
+	const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
+	const interleavedSamples = new Float32Array(
+		audioBuffer.length * NUM_CHANNELS,
+	);
 
-	for await (const { buffer, timestamp } of sink.buffers(
-		source.trimStart,
-		trimEnd,
-	)) {
-		const relativeTime = timestamp - source.trimStart;
-		const outputStartSample = Math.floor(
-			(source.startTime + relativeTime) * SAMPLE_RATE,
-		);
-
-		// resample if needed
-		const resampleRatio = SAMPLE_RATE / buffer.sampleRate;
-
-		for (let ch = 0; ch < NUM_CHANNELS; ch++) {
-			const sourceChannel = Math.min(ch, buffer.numberOfChannels - 1);
-			const channelData = buffer.getChannelData(sourceChannel);
-			const outputChannel = mixBuffers[ch];
-
-			const resampledLength = Math.floor(channelData.length * resampleRatio);
-			for (let i = 0; i < resampledLength; i++) {
-				const outputIdx = outputStartSample + i;
-				if (outputIdx < 0 || outputIdx >= totalSamples) continue;
-
-				const sourceIdx = Math.floor(i / resampleRatio);
-				if (sourceIdx < channelData.length) {
-					outputChannel[outputIdx] += channelData[sourceIdx];
-				}
-			}
+	for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
+		for (let channel = 0; channel < NUM_CHANNELS; channel++) {
+			const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
+			interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
+				audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
 		}
 	}
+
+	return interleavedSamples;
 }
 
 function createWavBlob({ samples }: { samples: Float32Array }): Blob {