From e4b67094e757917588fb624a30aa046667403add Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Tue, 7 Apr 2026 01:09:13 +0200 Subject: refactor: split effects and masks into dedicated rust crates, introduce MediaTime and FrameRate --- apps/web/src/core/managers/audio-manager.ts | 1417 ++++++++++++------------ apps/web/src/core/managers/playback-manager.ts | 29 +- apps/web/src/core/managers/project-manager.ts | 2 +- apps/web/src/core/managers/renderer-manager.ts | 12 +- apps/web/src/core/managers/timeline-manager.ts | 6 +- 5 files changed, 739 insertions(+), 727 deletions(-) (limited to 'apps/web/src/core') diff --git a/apps/web/src/core/managers/audio-manager.ts b/apps/web/src/core/managers/audio-manager.ts index 0f642b1f..aaeb7ade 100644 --- a/apps/web/src/core/managers/audio-manager.ts +++ b/apps/web/src/core/managers/audio-manager.ts @@ -1,708 +1,709 @@ -import type { EditorCore } from "@/core"; -import { - clampRetimeRate, - shouldMaintainPitch, -} from "@/lib/retime/rate"; -import type { AudioClipSource } from "@/lib/media/audio"; -import { createAudioContext, collectAudioClips } from "@/lib/media/audio"; -import { - buildAudioGainAutomation, - hasAnimatedVolume, -} from "@/lib/timeline/audio-state"; -import { createAudioMasteringChain } from "@/lib/media/audio-mastering"; -import { - getClipTimeAtSourceTime, - getSourceTimeAtClipTime, - renderRetimedBuffer, -} from "@/lib/retime"; -import { - ALL_FORMATS, - AudioBufferSink, - BlobSource, - Input, - type WrappedAudioBuffer, -} from "mediabunny"; - -export class AudioManager { - private audioContext: AudioContext | null = null; - private masterGain: GainNode | null = null; - private playbackStartTime = 0; - private playbackStartContextTime = 0; - private scheduleTimer: number | null = null; - private lookaheadSeconds = 2; - private scheduleIntervalMs = 500; - private clips: AudioClipSource[] = []; - private sinks = new Map(); - private inputs = new Map(); - private activeClipIds = new Set(); - private clipIterators = new Map< - string, - AsyncGenerator - >(); - private queuedSources = new Set(); - private preparedClipBuffers = new Map>(); - private decodedBuffers = new Map>(); - private playbackSessionId = 0; - private lastIsPlaying = false; - private lastVolume = 1; - private playbackLatencyCompensationSeconds = 0; - private unsubscribers: Array<() => void> = []; - - constructor(private editor: EditorCore) { - this.lastVolume = this.editor.playback.getVolume(); - - this.unsubscribers.push( - this.editor.playback.subscribe(this.handlePlaybackChange), - this.editor.timeline.subscribe(this.handleTimelineChange), - this.editor.media.subscribe(this.handleTimelineChange), - ); - if (typeof window !== "undefined") { - window.addEventListener("playback-seek", this.handleSeek); - } - } - - dispose(): void { - this.stopPlayback(); - for (const unsub of this.unsubscribers) { - unsub(); - } - this.unsubscribers = []; - if (typeof window !== "undefined") { - window.removeEventListener("playback-seek", this.handleSeek); - } - this.disposeSinks(); - this.preparedClipBuffers.clear(); - this.decodedBuffers.clear(); - if (this.audioContext) { - void this.audioContext.close(); - this.audioContext = null; - this.masterGain = null; - } - } - - private handlePlaybackChange = (): void => { - const isPlaying = this.editor.playback.getIsPlaying(); - const volume = this.editor.playback.getVolume(); - - if (volume !== this.lastVolume) { - this.lastVolume = volume; - this.updateGain(); - } - - if (isPlaying !== this.lastIsPlaying) { - this.lastIsPlaying = isPlaying; - if (isPlaying) { - void this.startPlayback({ - time: this.editor.playback.getCurrentTime(), - }); - } else { - this.stopPlayback(); - } - } - }; - - private handleSeek = (event: Event): void => { - const detail = (event as CustomEvent<{ time: number }>).detail; - if (!detail) return; - - if (this.editor.playback.getIsScrubbing()) { - this.stopPlayback(); - return; - } - - if (this.editor.playback.getIsPlaying()) { - void this.startPlayback({ time: detail.time }); - return; - } - - this.stopPlayback(); - }; - - private handleTimelineChange = (): void => { - this.disposeSinks(); - this.preparedClipBuffers.clear(); - this.decodedBuffers.clear(); - - if (!this.editor.playback.getIsPlaying()) return; - - void this.startPlayback({ time: this.editor.playback.getCurrentTime() }); - }; - - private ensureAudioContext(): AudioContext | null { - if (this.audioContext) return this.audioContext; - if (typeof window === "undefined") return null; - - this.audioContext = createAudioContext(); - const { input } = createAudioMasteringChain({ - audioContext: this.audioContext, - destination: this.audioContext.destination, - }); - this.masterGain = input; - this.masterGain.gain.value = this.lastVolume; - return this.audioContext; - } - - private updateGain(): void { - if (!this.masterGain) return; - this.masterGain.gain.value = this.lastVolume; - } - - private getPlaybackTime(): number { - if (!this.audioContext) return this.playbackStartTime; - const elapsed = - this.audioContext.currentTime - this.playbackStartContextTime; - return this.playbackStartTime + elapsed; - } - - private async startPlayback({ time }: { time: number }): Promise { - const audioContext = this.ensureAudioContext(); - if (!audioContext) return; - - this.stopPlayback(); - this.playbackSessionId++; - this.playbackLatencyCompensationSeconds = 0; - - const tracks = this.editor.timeline.getTracks(); - const mediaAssets = this.editor.media.getAssets(); - const duration = this.editor.timeline.getTotalDuration(); - - if (duration <= 0) return; - - if (audioContext.state === "suspended") { - await audioContext.resume(); - } - - this.clips = await collectAudioClips({ tracks, mediaAssets }); - if (!this.editor.playback.getIsPlaying()) return; - - this.playbackStartTime = time; - this.playbackStartContextTime = audioContext.currentTime; - - this.scheduleUpcomingClips(); - - if (typeof window !== "undefined") { - this.scheduleTimer = window.setInterval(() => { - this.scheduleUpcomingClips(); - }, this.scheduleIntervalMs); - } - } - - private scheduleUpcomingClips(): void { - if (!this.editor.playback.getIsPlaying()) return; - - const currentTime = this.getPlaybackTime(); - const windowEnd = currentTime + this.lookaheadSeconds; - - for (const clip of this.clips) { - if (clip.muted) continue; - if (this.activeClipIds.has(clip.id)) continue; - - const clipEnd = clip.startTime + clip.duration; - if (clipEnd <= currentTime) continue; - if (clip.startTime > windowEnd) continue; - - this.activeClipIds.add(clip.id); - if (this.shouldUsePreparedClipBuffer({ clip })) { - void this.schedulePreparedClip({ - clip, - startTime: currentTime, - sessionId: this.playbackSessionId, - }); - } else { - void this.runClipIterator({ - clip, - startTime: currentTime, - sessionId: this.playbackSessionId, - }); - } - } - } - - private stopPlayback(): void { - if (this.scheduleTimer && typeof window !== "undefined") { - window.clearInterval(this.scheduleTimer); - } - this.scheduleTimer = null; - - for (const iterator of this.clipIterators.values()) { - void iterator.return(); - } - this.clipIterators.clear(); - this.activeClipIds.clear(); - - for (const source of this.queuedSources) { - try { - source.stop(); - } catch {} - source.disconnect(); - } - this.queuedSources.clear(); - } - - private async runClipIterator({ - clip, - startTime, - sessionId, - }: { - clip: AudioClipSource; - startTime: number; - sessionId: number; - }): Promise { - const audioContext = this.ensureAudioContext(); - if (!audioContext) return; - - const sink = await this.getAudioSink({ clip }); - if (!sink || !this.editor.playback.getIsPlaying()) return; - if (sessionId !== this.playbackSessionId) return; - - const clipStart = clip.startTime; - const clipEnd = clip.startTime + clip.duration; - const playbackTimeAfterSinkReady = this.getPlaybackTime(); - const iteratorStartTime = Math.max( - startTime, - clipStart, - playbackTimeAfterSinkReady, - ); - if (iteratorStartTime >= clipEnd) { - return; - } - const sourceStartTime = - clip.trimStart + - getSourceTimeAtClipTime({ - clipTime: iteratorStartTime - clip.startTime, - retime: clip.retime, - }); - - const iterator = sink.buffers(sourceStartTime); - this.clipIterators.set(clip.id, iterator); - let consecutiveDroppedBufferCount = 0; - - for await (const { buffer, timestamp } of iterator) { - if (!this.editor.playback.getIsPlaying()) return; - if (sessionId !== this.playbackSessionId) return; - - const timelineTime = - clip.startTime + - getClipTimeAtSourceTime({ - sourceTime: timestamp - clip.trimStart, - retime: clip.retime, - }); - if (timelineTime >= clipEnd) break; - - const node = audioContext.createBufferSource(); - node.buffer = buffer; - if (clip.retime) { - node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate }); - } - const clipGain = audioContext.createGain(); - clipGain.gain.value = clip.volume; - node.connect(clipGain); - clipGain.connect(this.masterGain ?? audioContext.destination); - - const startTimestamp = - this.playbackStartContextTime + - this.playbackLatencyCompensationSeconds + - (timelineTime - this.playbackStartTime); - - if (startTimestamp >= audioContext.currentTime) { - node.start(startTimestamp); - consecutiveDroppedBufferCount = 0; - } else { - const offset = audioContext.currentTime - startTimestamp; - if (offset < buffer.duration) { - node.start(audioContext.currentTime, offset); - consecutiveDroppedBufferCount = 0; - } else { - consecutiveDroppedBufferCount += 1; - if (consecutiveDroppedBufferCount >= 5) { - const nextCompensationSeconds = Math.max( - this.playbackLatencyCompensationSeconds, - Math.min(0.25, offset + 0.01), - ); - if ( - nextCompensationSeconds > - this.playbackLatencyCompensationSeconds + 0.001 - ) { - this.playbackLatencyCompensationSeconds = nextCompensationSeconds; - } - const resyncStartTime = this.getPlaybackTime(); - this.clipIterators.delete(clip.id); - void this.runClipIterator({ - clip, - startTime: resyncStartTime, - sessionId, - }); - return; - } - continue; - } - } - - this.queuedSources.add(node); - node.addEventListener("ended", () => { - node.disconnect(); - clipGain.disconnect(); - this.queuedSources.delete(node); - }); - - const aheadTime = timelineTime - this.getPlaybackTime(); - if (aheadTime >= 1) { - await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 }); - if (sessionId !== this.playbackSessionId) return; - } - } - - this.clipIterators.delete(clip.id); - // don't remove from activeClipIds - prevents scheduler from restarting this clip - // the set is cleared on stopPlayback anyway - } - - private async schedulePreparedClip({ - clip, - startTime, - sessionId, - }: { - clip: AudioClipSource; - startTime: number; - sessionId: number; - }): Promise { - const audioContext = this.ensureAudioContext(); - if (!audioContext) return; - - const buffer = await this.getPreparedClipBuffer({ clip }); - if (!buffer || !this.editor.playback.getIsPlaying()) return; - if (sessionId !== this.playbackSessionId) return; - - const clipStart = clip.startTime; - const clipEnd = clip.startTime + clip.duration; - const playbackTimeAfterReady = this.getPlaybackTime(); - const effectiveStartTime = Math.max( - startTime, - clipStart, - playbackTimeAfterReady, - ); - if (effectiveStartTime >= clipEnd) { - return; - } - - const node = audioContext.createBufferSource(); - node.buffer = buffer; - const clipGain = audioContext.createGain(); - node.connect(clipGain); - clipGain.connect(this.masterGain ?? audioContext.destination); - - const startTimestamp = - this.playbackStartContextTime + - this.playbackLatencyCompensationSeconds + - (effectiveStartTime - this.playbackStartTime); - const clipOffset = effectiveStartTime - clipStart; - let actualStartTimestamp = startTimestamp; - let actualClipOffset = clipOffset; - - if (startTimestamp >= audioContext.currentTime) { - node.start(startTimestamp, clipOffset); - } else { - const lateOffset = audioContext.currentTime - startTimestamp; - actualStartTimestamp = audioContext.currentTime; - actualClipOffset = clipOffset + lateOffset; - node.start(actualStartTimestamp, actualClipOffset); - } - - this.scheduleClipGainAutomation({ - audioContext, - clip, - clipGain, - startTimestamp: actualStartTimestamp, - startLocalTime: actualClipOffset, - }); - - this.queuedSources.add(node); - node.addEventListener("ended", () => { - node.disconnect(); - clipGain.disconnect(); - this.queuedSources.delete(node); - }); - } - - private waitUntilCaughtUp({ - timelineTime, - targetAhead, - }: { - timelineTime: number; - targetAhead: number; - }): Promise { - return new Promise((resolve) => { - const checkInterval = setInterval(() => { - if (!this.editor.playback.getIsPlaying()) { - clearInterval(checkInterval); - resolve(); - return; - } - - const playbackTime = this.getPlaybackTime(); - if (timelineTime - playbackTime < targetAhead) { - clearInterval(checkInterval); - resolve(); - } - }, 100); - }); - } - - private disposeSinks(): void { - for (const iterator of this.clipIterators.values()) { - void iterator.return(); - } - this.clipIterators.clear(); - this.activeClipIds.clear(); - - for (const input of this.inputs.values()) { - input.dispose(); - } - this.inputs.clear(); - this.sinks.clear(); - } - - private shouldUsePreparedClipBuffer({ - clip, - }: { - clip: AudioClipSource; - }): boolean { - return ( - this.hasCurveRetime({ clip }) || - hasAnimatedVolume({ element: clip.timelineElement }) || - shouldMaintainPitch({ - rate: clip.retime?.rate ?? 1, - maintainPitch: clip.retime?.maintainPitch, - }) - ); - } - - private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean { - const mode = (clip.retime as { mode?: unknown } | undefined)?.mode; - return mode === "curve"; - } - - private scheduleClipGainAutomation({ - audioContext, - clip, - clipGain, - startTimestamp, - startLocalTime, - }: { - audioContext: AudioContext; - clip: AudioClipSource; - clipGain: GainNode; - startTimestamp: number; - startLocalTime: number; - }): void { - clipGain.gain.cancelScheduledValues(startTimestamp); - clipGain.gain.setValueAtTime(clip.volume, startTimestamp); - - if (!hasAnimatedVolume({ element: clip.timelineElement })) { - return; - } - - const points = buildAudioGainAutomation({ - element: clip.timelineElement, - fromLocalTime: startLocalTime, - toLocalTime: clip.duration, - }); - - if (points.length === 0) { - return; - } - - clipGain.gain.setValueAtTime(points[0].gain, startTimestamp); - for (let index = 1; index < points.length; index++) { - const point = points[index]; - const pointTimestamp = - startTimestamp + (point.localTime - startLocalTime); - if (pointTimestamp < audioContext.currentTime) { - continue; - } - - clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp); - } - } - - private buildPreparedClipCacheKey({ - clip, - }: { - clip: AudioClipSource; - }): string { - return JSON.stringify({ - id: clip.id, - sourceKey: clip.sourceKey, - startTime: clip.startTime, - duration: clip.duration, - trimStart: clip.trimStart, - trimEnd: clip.trimEnd, - retime: clip.retime ?? null, - }); - } - - private async getPreparedClipBuffer({ - clip, - }: { - clip: AudioClipSource; - }): Promise { - const cacheKey = this.buildPreparedClipCacheKey({ clip }); - const existing = this.preparedClipBuffers.get(cacheKey); - if (existing) { - return existing; - } - - const promise = (async () => { - const audioContext = this.ensureAudioContext(); - if (!audioContext) { - return null; - } - - const decodedBuffer = await this.getDecodedBuffer({ clip }); - if (!decodedBuffer) { - return null; - } - - return await renderRetimedBuffer({ - audioContext, - sourceBuffer: decodedBuffer, - trimStart: clip.trimStart, - clipDuration: clip.duration, - retime: clip.retime, - maintainPitch: clip.retime?.maintainPitch === true, - }); - })(); - - this.preparedClipBuffers.set(cacheKey, promise); - return promise; - } - - private async getDecodedBuffer({ - clip, - }: { - clip: AudioClipSource; - }): Promise { - const existing = this.decodedBuffers.get(clip.sourceKey); - if (existing) { - return existing; - } - - const promise = this.decodeClipBuffer({ clip }); - this.decodedBuffers.set(clip.sourceKey, promise); - return promise; - } - - private async decodeClipBuffer({ - clip, - }: { - clip: AudioClipSource; - }): Promise { - const audioContext = this.ensureAudioContext(); - if (!audioContext) { - return null; - } - - const input = new Input({ - source: new BlobSource(clip.file), - formats: ALL_FORMATS, - }); - - try { - const audioTrack = await input.getPrimaryAudioTrack(); - if (!audioTrack) { - return null; - } - - const sink = new AudioBufferSink(audioTrack); - const chunks: AudioBuffer[] = []; - let totalSamples = 0; - - for await (const { buffer } of sink.buffers(0)) { - chunks.push(buffer); - totalSamples += buffer.length; - } - - if (chunks.length === 0) { - return null; - } - - const targetSampleRate = audioContext.sampleRate; - const nativeSampleRate = chunks[0].sampleRate; - const numChannels = Math.min(2, chunks[0].numberOfChannels); - const nativeChannels = Array.from( - { length: numChannels }, - () => new Float32Array(totalSamples), - ); - - let offset = 0; - for (const chunk of chunks) { - for (let channel = 0; channel < numChannels; channel++) { - nativeChannels[channel].set( - chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)), - offset, - ); - } - offset += chunk.length; - } - - const outputSamples = Math.ceil( - totalSamples * (targetSampleRate / nativeSampleRate), - ); - const offlineContext = new OfflineAudioContext( - numChannels, - outputSamples, - targetSampleRate, - ); - const nativeBuffer = audioContext.createBuffer( - numChannels, - totalSamples, - nativeSampleRate, - ); - - for (let channel = 0; channel < numChannels; channel++) { - nativeBuffer.copyToChannel(nativeChannels[channel], channel); - } - - const sourceNode = offlineContext.createBufferSource(); - sourceNode.buffer = nativeBuffer; - sourceNode.connect(offlineContext.destination); - sourceNode.start(0); - - return await offlineContext.startRendering(); - } catch (error) { - console.warn("Failed to decode clip audio:", error); - return null; - } finally { - input.dispose(); - } - } - - private async getAudioSink({ - clip, - }: { - clip: AudioClipSource; - }): Promise { - const existingSink = this.sinks.get(clip.sourceKey); - if (existingSink) return existingSink; - - try { - const input = new Input({ - source: new BlobSource(clip.file), - formats: ALL_FORMATS, - }); - const audioTrack = await input.getPrimaryAudioTrack(); - if (!audioTrack) { - input.dispose(); - return null; - } - - const sink = new AudioBufferSink(audioTrack); - this.inputs.set(clip.sourceKey, input); - this.sinks.set(clip.sourceKey, sink); - return sink; - } catch (error) { - console.warn("Failed to initialize audio sink:", error); - return null; - } - } -} +import type { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { + clampRetimeRate, + shouldMaintainPitch, +} from "@/lib/retime/rate"; +import type { AudioClipSource } from "@/lib/media/audio"; +import { createAudioContext, collectAudioClips } from "@/lib/media/audio"; +import { + buildAudioGainAutomation, + hasAnimatedVolume, +} from "@/lib/timeline/audio-state"; +import { createAudioMasteringChain } from "@/lib/media/audio-mastering"; +import { + getClipTimeAtSourceTime, + getSourceTimeAtClipTime, + renderRetimedBuffer, +} from "@/lib/retime"; +import { + ALL_FORMATS, + AudioBufferSink, + BlobSource, + Input, + type WrappedAudioBuffer, +} from "mediabunny"; + +export class AudioManager { + private audioContext: AudioContext | null = null; + private masterGain: GainNode | null = null; + private playbackStartTime = 0; + private playbackStartContextTime = 0; + private scheduleTimer: number | null = null; + private lookaheadSeconds = 2; + private scheduleIntervalMs = 500; + private clips: AudioClipSource[] = []; + private sinks = new Map(); + private inputs = new Map(); + private activeClipIds = new Set(); + private clipIterators = new Map< + string, + AsyncGenerator + >(); + private queuedSources = new Set(); + private preparedClipBuffers = new Map>(); + private decodedBuffers = new Map>(); + private playbackSessionId = 0; + private lastIsPlaying = false; + private lastVolume = 1; + private playbackLatencyCompensationSeconds = 0; + private unsubscribers: Array<() => void> = []; + + constructor(private editor: EditorCore) { + this.lastVolume = this.editor.playback.getVolume(); + + this.unsubscribers.push( + this.editor.playback.subscribe(this.handlePlaybackChange), + this.editor.timeline.subscribe(this.handleTimelineChange), + this.editor.media.subscribe(this.handleTimelineChange), + ); + if (typeof window !== "undefined") { + window.addEventListener("playback-seek", this.handleSeek); + } + } + + dispose(): void { + this.stopPlayback(); + for (const unsub of this.unsubscribers) { + unsub(); + } + this.unsubscribers = []; + if (typeof window !== "undefined") { + window.removeEventListener("playback-seek", this.handleSeek); + } + this.disposeSinks(); + this.preparedClipBuffers.clear(); + this.decodedBuffers.clear(); + if (this.audioContext) { + void this.audioContext.close(); + this.audioContext = null; + this.masterGain = null; + } + } + + private handlePlaybackChange = (): void => { + const isPlaying = this.editor.playback.getIsPlaying(); + const volume = this.editor.playback.getVolume(); + + if (volume !== this.lastVolume) { + this.lastVolume = volume; + this.updateGain(); + } + + if (isPlaying !== this.lastIsPlaying) { + this.lastIsPlaying = isPlaying; + if (isPlaying) { + void this.startPlayback({ + time: this.editor.playback.getCurrentTime() / TICKS_PER_SECOND, + }); + } else { + this.stopPlayback(); + } + } + }; + + private handleSeek = (event: Event): void => { + const detail = (event as CustomEvent<{ time: number }>).detail; + if (!detail) return; + + if (this.editor.playback.getIsScrubbing()) { + this.stopPlayback(); + return; + } + + if (this.editor.playback.getIsPlaying()) { + void this.startPlayback({ time: detail.time / TICKS_PER_SECOND }); + return; + } + + this.stopPlayback(); + }; + + private handleTimelineChange = (): void => { + this.disposeSinks(); + this.preparedClipBuffers.clear(); + this.decodedBuffers.clear(); + + if (!this.editor.playback.getIsPlaying()) return; + + void this.startPlayback({ time: this.editor.playback.getCurrentTime() / TICKS_PER_SECOND }); + }; + + private ensureAudioContext(): AudioContext | null { + if (this.audioContext) return this.audioContext; + if (typeof window === "undefined") return null; + + this.audioContext = createAudioContext(); + const { input } = createAudioMasteringChain({ + audioContext: this.audioContext, + destination: this.audioContext.destination, + }); + this.masterGain = input; + this.masterGain.gain.value = this.lastVolume; + return this.audioContext; + } + + private updateGain(): void { + if (!this.masterGain) return; + this.masterGain.gain.value = this.lastVolume; + } + + private getPlaybackTime(): number { + if (!this.audioContext) return this.playbackStartTime; + const elapsed = + this.audioContext.currentTime - this.playbackStartContextTime; + return this.playbackStartTime + elapsed; + } + + private async startPlayback({ time }: { time: number }): Promise { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + this.stopPlayback(); + this.playbackSessionId++; + this.playbackLatencyCompensationSeconds = 0; + + const tracks = this.editor.timeline.getTracks(); + const mediaAssets = this.editor.media.getAssets(); + const duration = this.editor.timeline.getTotalDuration(); + + if (duration <= 0) return; + + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + this.clips = await collectAudioClips({ tracks, mediaAssets }); + if (!this.editor.playback.getIsPlaying()) return; + + this.playbackStartTime = time; + this.playbackStartContextTime = audioContext.currentTime; + + this.scheduleUpcomingClips(); + + if (typeof window !== "undefined") { + this.scheduleTimer = window.setInterval(() => { + this.scheduleUpcomingClips(); + }, this.scheduleIntervalMs); + } + } + + private scheduleUpcomingClips(): void { + if (!this.editor.playback.getIsPlaying()) return; + + const currentTime = this.getPlaybackTime(); + const windowEnd = currentTime + this.lookaheadSeconds; + + for (const clip of this.clips) { + if (clip.muted) continue; + if (this.activeClipIds.has(clip.id)) continue; + + const clipEnd = clip.startTime + clip.duration; + if (clipEnd <= currentTime) continue; + if (clip.startTime > windowEnd) continue; + + this.activeClipIds.add(clip.id); + if (this.shouldUsePreparedClipBuffer({ clip })) { + void this.schedulePreparedClip({ + clip, + startTime: currentTime, + sessionId: this.playbackSessionId, + }); + } else { + void this.runClipIterator({ + clip, + startTime: currentTime, + sessionId: this.playbackSessionId, + }); + } + } + } + + private stopPlayback(): void { + if (this.scheduleTimer && typeof window !== "undefined") { + window.clearInterval(this.scheduleTimer); + } + this.scheduleTimer = null; + + for (const iterator of this.clipIterators.values()) { + void iterator.return(); + } + this.clipIterators.clear(); + this.activeClipIds.clear(); + + for (const source of this.queuedSources) { + try { + source.stop(); + } catch {} + source.disconnect(); + } + this.queuedSources.clear(); + } + + private async runClipIterator({ + clip, + startTime, + sessionId, + }: { + clip: AudioClipSource; + startTime: number; + sessionId: number; + }): Promise { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + const sink = await this.getAudioSink({ clip }); + if (!sink || !this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const clipStart = clip.startTime; + const clipEnd = clip.startTime + clip.duration; + const playbackTimeAfterSinkReady = this.getPlaybackTime(); + const iteratorStartTime = Math.max( + startTime, + clipStart, + playbackTimeAfterSinkReady, + ); + if (iteratorStartTime >= clipEnd) { + return; + } + const sourceStartTime = + clip.trimStart + + getSourceTimeAtClipTime({ + clipTime: iteratorStartTime - clip.startTime, + retime: clip.retime, + }); + + const iterator = sink.buffers(sourceStartTime); + this.clipIterators.set(clip.id, iterator); + let consecutiveDroppedBufferCount = 0; + + for await (const { buffer, timestamp } of iterator) { + if (!this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const timelineTime = + clip.startTime + + getClipTimeAtSourceTime({ + sourceTime: timestamp - clip.trimStart, + retime: clip.retime, + }); + if (timelineTime >= clipEnd) break; + + const node = audioContext.createBufferSource(); + node.buffer = buffer; + if (clip.retime) { + node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate }); + } + const clipGain = audioContext.createGain(); + clipGain.gain.value = clip.volume; + node.connect(clipGain); + clipGain.connect(this.masterGain ?? audioContext.destination); + + const startTimestamp = + this.playbackStartContextTime + + this.playbackLatencyCompensationSeconds + + (timelineTime - this.playbackStartTime); + + if (startTimestamp >= audioContext.currentTime) { + node.start(startTimestamp); + consecutiveDroppedBufferCount = 0; + } else { + const offset = audioContext.currentTime - startTimestamp; + if (offset < buffer.duration) { + node.start(audioContext.currentTime, offset); + consecutiveDroppedBufferCount = 0; + } else { + consecutiveDroppedBufferCount += 1; + if (consecutiveDroppedBufferCount >= 5) { + const nextCompensationSeconds = Math.max( + this.playbackLatencyCompensationSeconds, + Math.min(0.25, offset + 0.01), + ); + if ( + nextCompensationSeconds > + this.playbackLatencyCompensationSeconds + 0.001 + ) { + this.playbackLatencyCompensationSeconds = nextCompensationSeconds; + } + const resyncStartTime = this.getPlaybackTime(); + this.clipIterators.delete(clip.id); + void this.runClipIterator({ + clip, + startTime: resyncStartTime, + sessionId, + }); + return; + } + continue; + } + } + + this.queuedSources.add(node); + node.addEventListener("ended", () => { + node.disconnect(); + clipGain.disconnect(); + this.queuedSources.delete(node); + }); + + const aheadTime = timelineTime - this.getPlaybackTime(); + if (aheadTime >= 1) { + await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 }); + if (sessionId !== this.playbackSessionId) return; + } + } + + this.clipIterators.delete(clip.id); + // don't remove from activeClipIds - prevents scheduler from restarting this clip + // the set is cleared on stopPlayback anyway + } + + private async schedulePreparedClip({ + clip, + startTime, + sessionId, + }: { + clip: AudioClipSource; + startTime: number; + sessionId: number; + }): Promise { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + const buffer = await this.getPreparedClipBuffer({ clip }); + if (!buffer || !this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const clipStart = clip.startTime; + const clipEnd = clip.startTime + clip.duration; + const playbackTimeAfterReady = this.getPlaybackTime(); + const effectiveStartTime = Math.max( + startTime, + clipStart, + playbackTimeAfterReady, + ); + if (effectiveStartTime >= clipEnd) { + return; + } + + const node = audioContext.createBufferSource(); + node.buffer = buffer; + const clipGain = audioContext.createGain(); + node.connect(clipGain); + clipGain.connect(this.masterGain ?? audioContext.destination); + + const startTimestamp = + this.playbackStartContextTime + + this.playbackLatencyCompensationSeconds + + (effectiveStartTime - this.playbackStartTime); + const clipOffset = effectiveStartTime - clipStart; + let actualStartTimestamp = startTimestamp; + let actualClipOffset = clipOffset; + + if (startTimestamp >= audioContext.currentTime) { + node.start(startTimestamp, clipOffset); + } else { + const lateOffset = audioContext.currentTime - startTimestamp; + actualStartTimestamp = audioContext.currentTime; + actualClipOffset = clipOffset + lateOffset; + node.start(actualStartTimestamp, actualClipOffset); + } + + this.scheduleClipGainAutomation({ + audioContext, + clip, + clipGain, + startTimestamp: actualStartTimestamp, + startLocalTime: actualClipOffset, + }); + + this.queuedSources.add(node); + node.addEventListener("ended", () => { + node.disconnect(); + clipGain.disconnect(); + this.queuedSources.delete(node); + }); + } + + private waitUntilCaughtUp({ + timelineTime, + targetAhead, + }: { + timelineTime: number; + targetAhead: number; + }): Promise { + return new Promise((resolve) => { + const checkInterval = setInterval(() => { + if (!this.editor.playback.getIsPlaying()) { + clearInterval(checkInterval); + resolve(); + return; + } + + const playbackTime = this.getPlaybackTime(); + if (timelineTime - playbackTime < targetAhead) { + clearInterval(checkInterval); + resolve(); + } + }, 100); + }); + } + + private disposeSinks(): void { + for (const iterator of this.clipIterators.values()) { + void iterator.return(); + } + this.clipIterators.clear(); + this.activeClipIds.clear(); + + for (const input of this.inputs.values()) { + input.dispose(); + } + this.inputs.clear(); + this.sinks.clear(); + } + + private shouldUsePreparedClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): boolean { + return ( + this.hasCurveRetime({ clip }) || + hasAnimatedVolume({ element: clip.timelineElement }) || + shouldMaintainPitch({ + rate: clip.retime?.rate ?? 1, + maintainPitch: clip.retime?.maintainPitch, + }) + ); + } + + private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean { + const mode = (clip.retime as { mode?: unknown } | undefined)?.mode; + return mode === "curve"; + } + + private scheduleClipGainAutomation({ + audioContext, + clip, + clipGain, + startTimestamp, + startLocalTime, + }: { + audioContext: AudioContext; + clip: AudioClipSource; + clipGain: GainNode; + startTimestamp: number; + startLocalTime: number; + }): void { + clipGain.gain.cancelScheduledValues(startTimestamp); + clipGain.gain.setValueAtTime(clip.volume, startTimestamp); + + if (!hasAnimatedVolume({ element: clip.timelineElement })) { + return; + } + + const points = buildAudioGainAutomation({ + element: clip.timelineElement, + fromLocalTime: startLocalTime, + toLocalTime: clip.duration, + }); + + if (points.length === 0) { + return; + } + + clipGain.gain.setValueAtTime(points[0].gain, startTimestamp); + for (let index = 1; index < points.length; index++) { + const point = points[index]; + const pointTimestamp = + startTimestamp + (point.localTime - startLocalTime); + if (pointTimestamp < audioContext.currentTime) { + continue; + } + + clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp); + } + } + + private buildPreparedClipCacheKey({ + clip, + }: { + clip: AudioClipSource; + }): string { + return JSON.stringify({ + id: clip.id, + sourceKey: clip.sourceKey, + startTime: clip.startTime, + duration: clip.duration, + trimStart: clip.trimStart, + trimEnd: clip.trimEnd, + retime: clip.retime ?? null, + }); + } + + private async getPreparedClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise { + const cacheKey = this.buildPreparedClipCacheKey({ clip }); + const existing = this.preparedClipBuffers.get(cacheKey); + if (existing) { + return existing; + } + + const promise = (async () => { + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + + const decodedBuffer = await this.getDecodedBuffer({ clip }); + if (!decodedBuffer) { + return null; + } + + return await renderRetimedBuffer({ + audioContext, + sourceBuffer: decodedBuffer, + trimStart: clip.trimStart, + clipDuration: clip.duration, + retime: clip.retime, + maintainPitch: clip.retime?.maintainPitch === true, + }); + })(); + + this.preparedClipBuffers.set(cacheKey, promise); + return promise; + } + + private async getDecodedBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise { + const existing = this.decodedBuffers.get(clip.sourceKey); + if (existing) { + return existing; + } + + const promise = this.decodeClipBuffer({ clip }); + this.decodedBuffers.set(clip.sourceKey, promise); + return promise; + } + + private async decodeClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise { + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + + const input = new Input({ + source: new BlobSource(clip.file), + formats: ALL_FORMATS, + }); + + try { + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + return null; + } + + const sink = new AudioBufferSink(audioTrack); + const chunks: AudioBuffer[] = []; + let totalSamples = 0; + + for await (const { buffer } of sink.buffers(0)) { + chunks.push(buffer); + totalSamples += buffer.length; + } + + if (chunks.length === 0) { + return null; + } + + const targetSampleRate = audioContext.sampleRate; + const nativeSampleRate = chunks[0].sampleRate; + const numChannels = Math.min(2, chunks[0].numberOfChannels); + const nativeChannels = Array.from( + { length: numChannels }, + () => new Float32Array(totalSamples), + ); + + let offset = 0; + for (const chunk of chunks) { + for (let channel = 0; channel < numChannels; channel++) { + nativeChannels[channel].set( + chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)), + offset, + ); + } + offset += chunk.length; + } + + const outputSamples = Math.ceil( + totalSamples * (targetSampleRate / nativeSampleRate), + ); + const offlineContext = new OfflineAudioContext( + numChannels, + outputSamples, + targetSampleRate, + ); + const nativeBuffer = audioContext.createBuffer( + numChannels, + totalSamples, + nativeSampleRate, + ); + + for (let channel = 0; channel < numChannels; channel++) { + nativeBuffer.copyToChannel(nativeChannels[channel], channel); + } + + const sourceNode = offlineContext.createBufferSource(); + sourceNode.buffer = nativeBuffer; + sourceNode.connect(offlineContext.destination); + sourceNode.start(0); + + return await offlineContext.startRendering(); + } catch (error) { + console.warn("Failed to decode clip audio:", error); + return null; + } finally { + input.dispose(); + } + } + + private async getAudioSink({ + clip, + }: { + clip: AudioClipSource; + }): Promise { + const existingSink = this.sinks.get(clip.sourceKey); + if (existingSink) return existingSink; + + try { + const input = new Input({ + source: new BlobSource(clip.file), + formats: ALL_FORMATS, + }); + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + input.dispose(); + return null; + } + + const sink = new AudioBufferSink(audioTrack); + this.inputs.set(clip.sourceKey, input); + this.sinks.set(clip.sourceKey, sink); + return sink; + } catch (error) { + console.warn("Failed to initialize audio sink:", error); + return null; + } + } +} diff --git a/apps/web/src/core/managers/playback-manager.ts b/apps/web/src/core/managers/playback-manager.ts index 6943f187..47909bfe 100644 --- a/apps/web/src/core/managers/playback-manager.ts +++ b/apps/web/src/core/managers/playback-manager.ts @@ -1,4 +1,6 @@ import type { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { roundToFrame } from "opencut-wasm"; export class PlaybackManager { private isPlaying = false; @@ -9,11 +11,12 @@ export class PlaybackManager { private isScrubbing = false; private listeners = new Set<() => void>(); private playbackTimer: number | null = null; - private lastUpdate = 0; + private playbackStartWallTime = 0; + private playbackStartTime = 0; constructor(private editor: EditorCore) { this.editor.timeline.subscribe(() => { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); if (this.currentTime > maxTime && maxTime > 0) { this.currentTime = maxTime; this.notify(); @@ -22,7 +25,7 @@ export class PlaybackManager { } play(): void { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); if (maxTime > 0) { if (this.currentTime >= maxTime) { @@ -50,8 +53,12 @@ export class PlaybackManager { } seek({ time }: { time: number }): void { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); this.currentTime = Math.max(0, Math.min(maxTime, time)); + if (this.isPlaying) { + this.playbackStartWallTime = performance.now(); + this.playbackStartTime = this.currentTime; + } this.notify(); window.dispatchEvent( @@ -135,7 +142,8 @@ export class PlaybackManager { cancelAnimationFrame(this.playbackTimer); } - this.lastUpdate = performance.now(); + this.playbackStartWallTime = performance.now(); + this.playbackStartTime = this.currentTime; this.updateTime(); } @@ -149,12 +157,11 @@ export class PlaybackManager { private updateTime = (): void => { if (!this.isPlaying) return; - const now = performance.now(); - const delta = (now - this.lastUpdate) / 1000; - this.lastUpdate = now; - - const newTime = this.currentTime + delta; - const maxTime = this.editor.timeline.getLastSeekableTime(); + const fps = this.editor.project.getActive()?.settings.fps; + const elapsedSeconds = (performance.now() - this.playbackStartWallTime) / 1000; + const rawTime = this.playbackStartTime + Math.round(elapsedSeconds * TICKS_PER_SECOND); + const newTime = fps ? (roundToFrame({ time: rawTime, rate: fps }) ?? rawTime) : rawTime; + const maxTime = this.editor.timeline.getTotalDuration(); if (maxTime > 0 && newTime >= maxTime) { this.pause(); diff --git a/apps/web/src/core/managers/project-manager.ts b/apps/web/src/core/managers/project-manager.ts index 10a9229d..513eab94 100644 --- a/apps/web/src/core/managers/project-manager.ts +++ b/apps/web/src/core/managers/project-manager.ts @@ -496,7 +496,7 @@ export class ProjectManager { importedAssets, }: { importedAssets: Array>; - }): number | null { + }): import("opencut-wasm").FrameRate | null { if (!this.active) return null; const nextFps = getRaisedProjectFpsForImportedMedia({ diff --git a/apps/web/src/core/managers/renderer-manager.ts b/apps/web/src/core/managers/renderer-manager.ts index aa34f496..e0d5b895 100644 --- a/apps/web/src/core/managers/renderer-manager.ts +++ b/apps/web/src/core/managers/renderer-manager.ts @@ -5,7 +5,8 @@ import { CanvasRenderer } from "@/services/renderer/canvas-renderer"; import { SceneExporter } from "@/services/renderer/scene-exporter"; import { buildScene } from "@/services/renderer/scene-builder"; import { createTimelineAudioBuffer } from "@/lib/media/audio"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode } from "opencut-wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; import { downloadBlob } from "@/utils/browser"; type SnapshotResult = @@ -81,7 +82,10 @@ export class RendererManager { } const { canvasSize, fps } = activeProject.settings; - const renderTime = this.editor.playback.getCurrentTime(); + const renderTime = Math.min( + this.editor.playback.getCurrentTime(), + this.editor.timeline.getLastFrameTime(), + ); const renderer = new CanvasRenderer({ width: canvasSize.width, @@ -107,7 +111,7 @@ export class RendererManager { return { success: false, error: "Failed to create image" }; } - const timecode = formatTimeCode({ timeInSeconds: renderTime, fps })!.replace(/:/g, "-"); + const timecode = formatTimecode({ time: renderTime, rate: fps })!.replace(/:/g, "-"); const safeName = activeProject.metadata.name.replace(/[<>:"/\\|?*]/g, "-").trim() || "snapshot"; @@ -148,7 +152,7 @@ export class RendererManager { return { success: false, error: "Project is empty" }; } - const exportFps = fps || activeProject.settings.fps; + const exportFps = fps ?? activeProject.settings.fps; const canvasSize = activeProject.settings.canvasSize; let audioBuffer: AudioBuffer | null = null; diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 3abde47b..a5cb428e 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -18,7 +18,7 @@ import type { AnimationValue, ScalarCurveKeyframePatch, } from "@/lib/animation/types"; -import { getLastFrameTime } from "opencut-wasm"; +import { lastFrameTime } from "opencut-wasm"; import { BatchCommand } from "@/lib/commands"; import { AddTrackCommand, @@ -196,11 +196,11 @@ export class TimelineManager { return calculateTotalDuration({ tracks: this.getTracks() }); } - getLastSeekableTime(): number { + getLastFrameTime(): number { const duration = this.getTotalDuration(); const fps = this.editor.project.getActive()?.settings.fps; if (!fps || duration <= 0) return duration; - return getLastFrameTime({ duration, fps }); + return lastFrameTime({ duration, rate: fps }) ?? duration; } getTrackById({ trackId }: { trackId: string }): TimelineTrack | null { -- cgit v1.3.1