feat: masks, properties refactor, shaders, storage migrations, and more

1 files changed, 340 insertions, 12 deletions

diff --git a/apps/web/src/core/managers/audio-manager.ts b/apps/web/src/core/managers/audio-manager.ts
index 20b124b6..54fda6c2 100644
--- a/apps/web/src/core/managers/audio-manager.ts
+++ b/apps/web/src/core/managers/audio-manager.ts
@@ -1,7 +1,21 @@
 import type { EditorCore } from "@/core";
+import {
+	clampRetimeRate,
+	shouldMaintainPitch,
+} from "@/constants/retime-constants";
 import type { AudioClipSource } from "@/lib/media/audio";
 import { createAudioContext, collectAudioClips } from "@/lib/media/audio";
 import {
+	buildAudioGainAutomation,
+	hasAnimatedVolume,
+} from "@/lib/timeline/audio-state";
+import { createAudioMasteringChain } from "@/lib/media/audio-mastering";
+import {
+	getClipTimeAtSourceTime,
+	getSourceTimeAtClipTime,
+	renderRetimedBuffer,
+} from "@/lib/retime";
+import {
 	ALL_FORMATS,
 	AudioBufferSink,
 	BlobSource,
@@ -26,6 +40,8 @@ export class AudioManager {
 		AsyncGenerator<WrappedAudioBuffer, void, unknown>
 	>();
 	private queuedSources = new Set<AudioBufferSourceNode>();
+	private preparedClipBuffers = new Map<string, Promise<AudioBuffer | null>>();
+	private decodedBuffers = new Map<string, Promise<AudioBuffer | null>>();
 	private playbackSessionId = 0;
 	private lastIsPlaying = false;
 	private lastVolume = 1;
@@ -55,6 +71,8 @@ export class AudioManager {
 			window.removeEventListener("playback-seek", this.handleSeek);
 		}
 		this.disposeSinks();
+		this.preparedClipBuffers.clear();
+		this.decodedBuffers.clear();
 		if (this.audioContext) {
 			void this.audioContext.close();
 			this.audioContext = null;
@@ -102,6 +120,8 @@ export class AudioManager {
 
 	private handleTimelineChange = (): void => {
 		this.disposeSinks();
+		this.preparedClipBuffers.clear();
+		this.decodedBuffers.clear();
 
 		if (!this.editor.playback.getIsPlaying()) return;
 
@@ -113,9 +133,12 @@ export class AudioManager {
 		if (typeof window === "undefined") return null;
 
 		this.audioContext = createAudioContext();
-		this.masterGain = this.audioContext.createGain();
+		const { input } = createAudioMasteringChain({
+			audioContext: this.audioContext,
+			destination: this.audioContext.destination,
+		});
+		this.masterGain = input;
 		this.masterGain.gain.value = this.lastVolume;
-		this.masterGain.connect(this.audioContext.destination);
 		return this.audioContext;
 	}
 
@@ -179,11 +202,19 @@ export class AudioManager {
 			if (clip.startTime > windowEnd) continue;
 
 			this.activeClipIds.add(clip.id);
-			void this.runClipIterator({
-				clip,
-				startTime: currentTime,
-				sessionId: this.playbackSessionId,
-			});
+			if (this.shouldUsePreparedClipBuffer({ clip })) {
+				void this.schedulePreparedClip({
+					clip,
+					startTime: currentTime,
+					sessionId: this.playbackSessionId,
+				});
+			} else {
+				void this.runClipIterator({
+					clip,
+					startTime: currentTime,
+					sessionId: this.playbackSessionId,
+				});
+			}
 		}
 	}
 
@@ -236,7 +267,11 @@ export class AudioManager {
 			return;
 		}
 		const sourceStartTime =
-			clip.trimStart + (iteratorStartTime - clip.startTime);
+			clip.trimStart +
+			getSourceTimeAtClipTime({
+				clipTime: iteratorStartTime - clip.startTime,
+				retime: clip.retime,
+			});
 
 		const iterator = sink.buffers(sourceStartTime);
 		this.clipIterators.set(clip.id, iterator);
@@ -246,12 +281,23 @@ export class AudioManager {
 			if (!this.editor.playback.getIsPlaying()) return;
 			if (sessionId !== this.playbackSessionId) return;
 
-			const timelineTime = clip.startTime + (timestamp - clip.trimStart);
+			const timelineTime =
+				clip.startTime +
+				getClipTimeAtSourceTime({
+					sourceTime: timestamp - clip.trimStart,
+					retime: clip.retime,
+				});
 			if (timelineTime >= clipEnd) break;
 
 			const node = audioContext.createBufferSource();
 			node.buffer = buffer;
-			node.connect(this.masterGain ?? audioContext.destination);
+			if (clip.retime) {
+				node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate });
+			}
+			const clipGain = audioContext.createGain();
+			clipGain.gain.value = clip.volume;
+			node.connect(clipGain);
+			clipGain.connect(this.masterGain ?? audioContext.destination);
 
 			const startTimestamp =
 				this.playbackStartContextTime +
@@ -277,8 +323,7 @@ export class AudioManager {
 							nextCompensationSeconds >
 							this.playbackLatencyCompensationSeconds + 0.001
 						) {
-							this.playbackLatencyCompensationSeconds =
-								nextCompensationSeconds;
+							this.playbackLatencyCompensationSeconds = nextCompensationSeconds;
 						}
 						const resyncStartTime = this.getPlaybackTime();
 						this.clipIterators.delete(clip.id);
@@ -296,6 +341,7 @@ export class AudioManager {
 			this.queuedSources.add(node);
 			node.addEventListener("ended", () => {
 				node.disconnect();
+				clipGain.disconnect();
 				this.queuedSources.delete(node);
 			});
 
@@ -311,6 +357,73 @@ export class AudioManager {
 		// the set is cleared on stopPlayback anyway
 	}
 
+	private async schedulePreparedClip({
+		clip,
+		startTime,
+		sessionId,
+	}: {
+		clip: AudioClipSource;
+		startTime: number;
+		sessionId: number;
+	}): Promise<void> {
+		const audioContext = this.ensureAudioContext();
+		if (!audioContext) return;
+
+		const buffer = await this.getPreparedClipBuffer({ clip });
+		if (!buffer || !this.editor.playback.getIsPlaying()) return;
+		if (sessionId !== this.playbackSessionId) return;
+
+		const clipStart = clip.startTime;
+		const clipEnd = clip.startTime + clip.duration;
+		const playbackTimeAfterReady = this.getPlaybackTime();
+		const effectiveStartTime = Math.max(
+			startTime,
+			clipStart,
+			playbackTimeAfterReady,
+		);
+		if (effectiveStartTime >= clipEnd) {
+			return;
+		}
+
+		const node = audioContext.createBufferSource();
+		node.buffer = buffer;
+		const clipGain = audioContext.createGain();
+		node.connect(clipGain);
+		clipGain.connect(this.masterGain ?? audioContext.destination);
+
+		const startTimestamp =
+			this.playbackStartContextTime +
+			this.playbackLatencyCompensationSeconds +
+			(effectiveStartTime - this.playbackStartTime);
+		const clipOffset = effectiveStartTime - clipStart;
+		let actualStartTimestamp = startTimestamp;
+		let actualClipOffset = clipOffset;
+
+		if (startTimestamp >= audioContext.currentTime) {
+			node.start(startTimestamp, clipOffset);
+		} else {
+			const lateOffset = audioContext.currentTime - startTimestamp;
+			actualStartTimestamp = audioContext.currentTime;
+			actualClipOffset = clipOffset + lateOffset;
+			node.start(actualStartTimestamp, actualClipOffset);
+		}
+
+		this.scheduleClipGainAutomation({
+			audioContext,
+			clip,
+			clipGain,
+			startTimestamp: actualStartTimestamp,
+			startLocalTime: actualClipOffset,
+		});
+
+		this.queuedSources.add(node);
+		node.addEventListener("ended", () => {
+			node.disconnect();
+			clipGain.disconnect();
+			this.queuedSources.delete(node);
+		});
+	}
+
 	private waitUntilCaughtUp({
 		timelineTime,
 		targetAhead,
@@ -349,6 +462,221 @@ export class AudioManager {
 		this.sinks.clear();
 	}
 
+	private shouldUsePreparedClipBuffer({
+		clip,
+	}: {
+		clip: AudioClipSource;
+	}): boolean {
+		return (
+			this.hasCurveRetime({ clip }) ||
+			hasAnimatedVolume({ element: clip.timelineElement }) ||
+			shouldMaintainPitch({
+				rate: clip.retime?.rate ?? 1,
+				maintainPitch: clip.retime?.maintainPitch,
+			})
+		);
+	}
+
+	private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean {
+		const mode = (clip.retime as { mode?: unknown } | undefined)?.mode;
+		return mode === "curve";
+	}
+
+	private scheduleClipGainAutomation({
+		audioContext,
+		clip,
+		clipGain,
+		startTimestamp,
+		startLocalTime,
+	}: {
+		audioContext: AudioContext;
+		clip: AudioClipSource;
+		clipGain: GainNode;
+		startTimestamp: number;
+		startLocalTime: number;
+	}): void {
+		clipGain.gain.cancelScheduledValues(startTimestamp);
+		clipGain.gain.setValueAtTime(clip.volume, startTimestamp);
+
+		if (!hasAnimatedVolume({ element: clip.timelineElement })) {
+			return;
+		}
+
+		const points = buildAudioGainAutomation({
+			element: clip.timelineElement,
+			fromLocalTime: startLocalTime,
+			toLocalTime: clip.duration,
+		});
+
+		if (points.length === 0) {
+			return;
+		}
+
+		clipGain.gain.setValueAtTime(points[0].gain, startTimestamp);
+		for (let index = 1; index < points.length; index++) {
+			const point = points[index];
+			const pointTimestamp =
+				startTimestamp + (point.localTime - startLocalTime);
+			if (pointTimestamp < audioContext.currentTime) {
+				continue;
+			}
+
+			clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp);
+		}
+	}
+
+	private buildPreparedClipCacheKey({
+		clip,
+	}: {
+		clip: AudioClipSource;
+	}): string {
+		return JSON.stringify({
+			id: clip.id,
+			sourceKey: clip.sourceKey,
+			startTime: clip.startTime,
+			duration: clip.duration,
+			trimStart: clip.trimStart,
+			trimEnd: clip.trimEnd,
+			retime: clip.retime ?? null,
+		});
+	}
+
+	private async getPreparedClipBuffer({
+		clip,
+	}: {
+		clip: AudioClipSource;
+	}): Promise<AudioBuffer | null> {
+		const cacheKey = this.buildPreparedClipCacheKey({ clip });
+		const existing = this.preparedClipBuffers.get(cacheKey);
+		if (existing) {
+			return existing;
+		}
+
+		const promise = (async () => {
+			const audioContext = this.ensureAudioContext();
+			if (!audioContext) {
+				return null;
+			}
+
+			const decodedBuffer = await this.getDecodedBuffer({ clip });
+			if (!decodedBuffer) {
+				return null;
+			}
+
+			return await renderRetimedBuffer({
+				audioContext,
+				sourceBuffer: decodedBuffer,
+				trimStart: clip.trimStart,
+				clipDuration: clip.duration,
+				retime: clip.retime,
+				maintainPitch: clip.retime?.maintainPitch === true,
+			});
+		})();
+
+		this.preparedClipBuffers.set(cacheKey, promise);
+		return promise;
+	}
+
+	private async getDecodedBuffer({
+		clip,
+	}: {
+		clip: AudioClipSource;
+	}): Promise<AudioBuffer | null> {
+		const existing = this.decodedBuffers.get(clip.sourceKey);
+		if (existing) {
+			return existing;
+		}
+
+		const promise = this.decodeClipBuffer({ clip });
+		this.decodedBuffers.set(clip.sourceKey, promise);
+		return promise;
+	}
+
+	private async decodeClipBuffer({
+		clip,
+	}: {
+		clip: AudioClipSource;
+	}): Promise<AudioBuffer | null> {
+		const audioContext = this.ensureAudioContext();
+		if (!audioContext) {
+			return null;
+		}
+
+		const input = new Input({
+			source: new BlobSource(clip.file),
+			formats: ALL_FORMATS,
+		});
+
+		try {
+			const audioTrack = await input.getPrimaryAudioTrack();
+			if (!audioTrack) {
+				return null;
+			}
+
+			const sink = new AudioBufferSink(audioTrack);
+			const chunks: AudioBuffer[] = [];
+			let totalSamples = 0;
+
+			for await (const { buffer } of sink.buffers(0)) {
+				chunks.push(buffer);
+				totalSamples += buffer.length;
+			}
+
+			if (chunks.length === 0) {
+				return null;
+			}
+
+			const targetSampleRate = audioContext.sampleRate;
+			const nativeSampleRate = chunks[0].sampleRate;
+			const numChannels = Math.min(2, chunks[0].numberOfChannels);
+			const nativeChannels = Array.from(
+				{ length: numChannels },
+				() => new Float32Array(totalSamples),
+			);
+
+			let offset = 0;
+			for (const chunk of chunks) {
+				for (let channel = 0; channel < numChannels; channel++) {
+					nativeChannels[channel].set(
+						chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)),
+						offset,
+					);
+				}
+				offset += chunk.length;
+			}
+
+			const outputSamples = Math.ceil(
+				totalSamples * (targetSampleRate / nativeSampleRate),
+			);
+			const offlineContext = new OfflineAudioContext(
+				numChannels,
+				outputSamples,
+				targetSampleRate,
+			);
+			const nativeBuffer = audioContext.createBuffer(
+				numChannels,
+				totalSamples,
+				nativeSampleRate,
+			);
+
+			for (let channel = 0; channel < numChannels; channel++) {
+				nativeBuffer.copyToChannel(nativeChannels[channel], channel);
+			}
+
+			const sourceNode = offlineContext.createBufferSource();
+			sourceNode.buffer = nativeBuffer;
+			sourceNode.connect(offlineContext.destination);
+			sourceNode.start(0);
+
+			return await offlineContext.startRendering();
+		} catch (error) {
+			console.warn("Failed to decode clip audio:", error);
+			return null;
+		} finally {
+			input.dispose();
+		}
+	}
+
 	private async getAudioSink({
 		clip,
 	}: {