diff options
| author | Maze Winther <mazewinther@gmail.com> | 2026-04-07 01:09:13 +0200 |
|---|---|---|
| committer | Maze Winther <mazewinther@gmail.com> | 2026-04-07 01:09:13 +0200 |
| commit | e4b67094e757917588fb624a30aa046667403add (patch) | |
| tree | 55452692eac69a5a63b58f2a5cdb701dd7023d68 /apps/web/src/core/managers | |
| parent | 79df73643185c02493364a077402726d1fa7b9ec (diff) | |
| download | opencut-e4b67094e757917588fb624a30aa046667403add.tar.gz | |
refactor: split effects and masks into dedicated rust crates, introduce MediaTime and FrameRate
Diffstat (limited to 'apps/web/src/core/managers')
| -rw-r--r-- | apps/web/src/core/managers/audio-manager.ts | 1417 | ||||
| -rw-r--r-- | apps/web/src/core/managers/playback-manager.ts | 29 | ||||
| -rw-r--r-- | apps/web/src/core/managers/project-manager.ts | 2 | ||||
| -rw-r--r-- | apps/web/src/core/managers/renderer-manager.ts | 12 | ||||
| -rw-r--r-- | apps/web/src/core/managers/timeline-manager.ts | 6 |
5 files changed, 739 insertions, 727 deletions
diff --git a/apps/web/src/core/managers/audio-manager.ts b/apps/web/src/core/managers/audio-manager.ts index 0f642b1f..aaeb7ade 100644 --- a/apps/web/src/core/managers/audio-manager.ts +++ b/apps/web/src/core/managers/audio-manager.ts @@ -1,708 +1,709 @@ -import type { EditorCore } from "@/core";
-import {
- clampRetimeRate,
- shouldMaintainPitch,
-} from "@/lib/retime/rate";
-import type { AudioClipSource } from "@/lib/media/audio";
-import { createAudioContext, collectAudioClips } from "@/lib/media/audio";
-import {
- buildAudioGainAutomation,
- hasAnimatedVolume,
-} from "@/lib/timeline/audio-state";
-import { createAudioMasteringChain } from "@/lib/media/audio-mastering";
-import {
- getClipTimeAtSourceTime,
- getSourceTimeAtClipTime,
- renderRetimedBuffer,
-} from "@/lib/retime";
-import {
- ALL_FORMATS,
- AudioBufferSink,
- BlobSource,
- Input,
- type WrappedAudioBuffer,
-} from "mediabunny";
-
-export class AudioManager {
- private audioContext: AudioContext | null = null;
- private masterGain: GainNode | null = null;
- private playbackStartTime = 0;
- private playbackStartContextTime = 0;
- private scheduleTimer: number | null = null;
- private lookaheadSeconds = 2;
- private scheduleIntervalMs = 500;
- private clips: AudioClipSource[] = [];
- private sinks = new Map<string, AudioBufferSink>();
- private inputs = new Map<string, Input>();
- private activeClipIds = new Set<string>();
- private clipIterators = new Map<
- string,
- AsyncGenerator<WrappedAudioBuffer, void, unknown>
- >();
- private queuedSources = new Set<AudioBufferSourceNode>();
- private preparedClipBuffers = new Map<string, Promise<AudioBuffer | null>>();
- private decodedBuffers = new Map<string, Promise<AudioBuffer | null>>();
- private playbackSessionId = 0;
- private lastIsPlaying = false;
- private lastVolume = 1;
- private playbackLatencyCompensationSeconds = 0;
- private unsubscribers: Array<() => void> = [];
-
- constructor(private editor: EditorCore) {
- this.lastVolume = this.editor.playback.getVolume();
-
- this.unsubscribers.push(
- this.editor.playback.subscribe(this.handlePlaybackChange),
- this.editor.timeline.subscribe(this.handleTimelineChange),
- this.editor.media.subscribe(this.handleTimelineChange),
- );
- if (typeof window !== "undefined") {
- window.addEventListener("playback-seek", this.handleSeek);
- }
- }
-
- dispose(): void {
- this.stopPlayback();
- for (const unsub of this.unsubscribers) {
- unsub();
- }
- this.unsubscribers = [];
- if (typeof window !== "undefined") {
- window.removeEventListener("playback-seek", this.handleSeek);
- }
- this.disposeSinks();
- this.preparedClipBuffers.clear();
- this.decodedBuffers.clear();
- if (this.audioContext) {
- void this.audioContext.close();
- this.audioContext = null;
- this.masterGain = null;
- }
- }
-
- private handlePlaybackChange = (): void => {
- const isPlaying = this.editor.playback.getIsPlaying();
- const volume = this.editor.playback.getVolume();
-
- if (volume !== this.lastVolume) {
- this.lastVolume = volume;
- this.updateGain();
- }
-
- if (isPlaying !== this.lastIsPlaying) {
- this.lastIsPlaying = isPlaying;
- if (isPlaying) {
- void this.startPlayback({
- time: this.editor.playback.getCurrentTime(),
- });
- } else {
- this.stopPlayback();
- }
- }
- };
-
- private handleSeek = (event: Event): void => {
- const detail = (event as CustomEvent<{ time: number }>).detail;
- if (!detail) return;
-
- if (this.editor.playback.getIsScrubbing()) {
- this.stopPlayback();
- return;
- }
-
- if (this.editor.playback.getIsPlaying()) {
- void this.startPlayback({ time: detail.time });
- return;
- }
-
- this.stopPlayback();
- };
-
- private handleTimelineChange = (): void => {
- this.disposeSinks();
- this.preparedClipBuffers.clear();
- this.decodedBuffers.clear();
-
- if (!this.editor.playback.getIsPlaying()) return;
-
- void this.startPlayback({ time: this.editor.playback.getCurrentTime() });
- };
-
- private ensureAudioContext(): AudioContext | null {
- if (this.audioContext) return this.audioContext;
- if (typeof window === "undefined") return null;
-
- this.audioContext = createAudioContext();
- const { input } = createAudioMasteringChain({
- audioContext: this.audioContext,
- destination: this.audioContext.destination,
- });
- this.masterGain = input;
- this.masterGain.gain.value = this.lastVolume;
- return this.audioContext;
- }
-
- private updateGain(): void {
- if (!this.masterGain) return;
- this.masterGain.gain.value = this.lastVolume;
- }
-
- private getPlaybackTime(): number {
- if (!this.audioContext) return this.playbackStartTime;
- const elapsed =
- this.audioContext.currentTime - this.playbackStartContextTime;
- return this.playbackStartTime + elapsed;
- }
-
- private async startPlayback({ time }: { time: number }): Promise<void> {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) return;
-
- this.stopPlayback();
- this.playbackSessionId++;
- this.playbackLatencyCompensationSeconds = 0;
-
- const tracks = this.editor.timeline.getTracks();
- const mediaAssets = this.editor.media.getAssets();
- const duration = this.editor.timeline.getTotalDuration();
-
- if (duration <= 0) return;
-
- if (audioContext.state === "suspended") {
- await audioContext.resume();
- }
-
- this.clips = await collectAudioClips({ tracks, mediaAssets });
- if (!this.editor.playback.getIsPlaying()) return;
-
- this.playbackStartTime = time;
- this.playbackStartContextTime = audioContext.currentTime;
-
- this.scheduleUpcomingClips();
-
- if (typeof window !== "undefined") {
- this.scheduleTimer = window.setInterval(() => {
- this.scheduleUpcomingClips();
- }, this.scheduleIntervalMs);
- }
- }
-
- private scheduleUpcomingClips(): void {
- if (!this.editor.playback.getIsPlaying()) return;
-
- const currentTime = this.getPlaybackTime();
- const windowEnd = currentTime + this.lookaheadSeconds;
-
- for (const clip of this.clips) {
- if (clip.muted) continue;
- if (this.activeClipIds.has(clip.id)) continue;
-
- const clipEnd = clip.startTime + clip.duration;
- if (clipEnd <= currentTime) continue;
- if (clip.startTime > windowEnd) continue;
-
- this.activeClipIds.add(clip.id);
- if (this.shouldUsePreparedClipBuffer({ clip })) {
- void this.schedulePreparedClip({
- clip,
- startTime: currentTime,
- sessionId: this.playbackSessionId,
- });
- } else {
- void this.runClipIterator({
- clip,
- startTime: currentTime,
- sessionId: this.playbackSessionId,
- });
- }
- }
- }
-
- private stopPlayback(): void {
- if (this.scheduleTimer && typeof window !== "undefined") {
- window.clearInterval(this.scheduleTimer);
- }
- this.scheduleTimer = null;
-
- for (const iterator of this.clipIterators.values()) {
- void iterator.return();
- }
- this.clipIterators.clear();
- this.activeClipIds.clear();
-
- for (const source of this.queuedSources) {
- try {
- source.stop();
- } catch {}
- source.disconnect();
- }
- this.queuedSources.clear();
- }
-
- private async runClipIterator({
- clip,
- startTime,
- sessionId,
- }: {
- clip: AudioClipSource;
- startTime: number;
- sessionId: number;
- }): Promise<void> {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) return;
-
- const sink = await this.getAudioSink({ clip });
- if (!sink || !this.editor.playback.getIsPlaying()) return;
- if (sessionId !== this.playbackSessionId) return;
-
- const clipStart = clip.startTime;
- const clipEnd = clip.startTime + clip.duration;
- const playbackTimeAfterSinkReady = this.getPlaybackTime();
- const iteratorStartTime = Math.max(
- startTime,
- clipStart,
- playbackTimeAfterSinkReady,
- );
- if (iteratorStartTime >= clipEnd) {
- return;
- }
- const sourceStartTime =
- clip.trimStart +
- getSourceTimeAtClipTime({
- clipTime: iteratorStartTime - clip.startTime,
- retime: clip.retime,
- });
-
- const iterator = sink.buffers(sourceStartTime);
- this.clipIterators.set(clip.id, iterator);
- let consecutiveDroppedBufferCount = 0;
-
- for await (const { buffer, timestamp } of iterator) {
- if (!this.editor.playback.getIsPlaying()) return;
- if (sessionId !== this.playbackSessionId) return;
-
- const timelineTime =
- clip.startTime +
- getClipTimeAtSourceTime({
- sourceTime: timestamp - clip.trimStart,
- retime: clip.retime,
- });
- if (timelineTime >= clipEnd) break;
-
- const node = audioContext.createBufferSource();
- node.buffer = buffer;
- if (clip.retime) {
- node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate });
- }
- const clipGain = audioContext.createGain();
- clipGain.gain.value = clip.volume;
- node.connect(clipGain);
- clipGain.connect(this.masterGain ?? audioContext.destination);
-
- const startTimestamp =
- this.playbackStartContextTime +
- this.playbackLatencyCompensationSeconds +
- (timelineTime - this.playbackStartTime);
-
- if (startTimestamp >= audioContext.currentTime) {
- node.start(startTimestamp);
- consecutiveDroppedBufferCount = 0;
- } else {
- const offset = audioContext.currentTime - startTimestamp;
- if (offset < buffer.duration) {
- node.start(audioContext.currentTime, offset);
- consecutiveDroppedBufferCount = 0;
- } else {
- consecutiveDroppedBufferCount += 1;
- if (consecutiveDroppedBufferCount >= 5) {
- const nextCompensationSeconds = Math.max(
- this.playbackLatencyCompensationSeconds,
- Math.min(0.25, offset + 0.01),
- );
- if (
- nextCompensationSeconds >
- this.playbackLatencyCompensationSeconds + 0.001
- ) {
- this.playbackLatencyCompensationSeconds = nextCompensationSeconds;
- }
- const resyncStartTime = this.getPlaybackTime();
- this.clipIterators.delete(clip.id);
- void this.runClipIterator({
- clip,
- startTime: resyncStartTime,
- sessionId,
- });
- return;
- }
- continue;
- }
- }
-
- this.queuedSources.add(node);
- node.addEventListener("ended", () => {
- node.disconnect();
- clipGain.disconnect();
- this.queuedSources.delete(node);
- });
-
- const aheadTime = timelineTime - this.getPlaybackTime();
- if (aheadTime >= 1) {
- await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 });
- if (sessionId !== this.playbackSessionId) return;
- }
- }
-
- this.clipIterators.delete(clip.id);
- // don't remove from activeClipIds - prevents scheduler from restarting this clip
- // the set is cleared on stopPlayback anyway
- }
-
- private async schedulePreparedClip({
- clip,
- startTime,
- sessionId,
- }: {
- clip: AudioClipSource;
- startTime: number;
- sessionId: number;
- }): Promise<void> {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) return;
-
- const buffer = await this.getPreparedClipBuffer({ clip });
- if (!buffer || !this.editor.playback.getIsPlaying()) return;
- if (sessionId !== this.playbackSessionId) return;
-
- const clipStart = clip.startTime;
- const clipEnd = clip.startTime + clip.duration;
- const playbackTimeAfterReady = this.getPlaybackTime();
- const effectiveStartTime = Math.max(
- startTime,
- clipStart,
- playbackTimeAfterReady,
- );
- if (effectiveStartTime >= clipEnd) {
- return;
- }
-
- const node = audioContext.createBufferSource();
- node.buffer = buffer;
- const clipGain = audioContext.createGain();
- node.connect(clipGain);
- clipGain.connect(this.masterGain ?? audioContext.destination);
-
- const startTimestamp =
- this.playbackStartContextTime +
- this.playbackLatencyCompensationSeconds +
- (effectiveStartTime - this.playbackStartTime);
- const clipOffset = effectiveStartTime - clipStart;
- let actualStartTimestamp = startTimestamp;
- let actualClipOffset = clipOffset;
-
- if (startTimestamp >= audioContext.currentTime) {
- node.start(startTimestamp, clipOffset);
- } else {
- const lateOffset = audioContext.currentTime - startTimestamp;
- actualStartTimestamp = audioContext.currentTime;
- actualClipOffset = clipOffset + lateOffset;
- node.start(actualStartTimestamp, actualClipOffset);
- }
-
- this.scheduleClipGainAutomation({
- audioContext,
- clip,
- clipGain,
- startTimestamp: actualStartTimestamp,
- startLocalTime: actualClipOffset,
- });
-
- this.queuedSources.add(node);
- node.addEventListener("ended", () => {
- node.disconnect();
- clipGain.disconnect();
- this.queuedSources.delete(node);
- });
- }
-
- private waitUntilCaughtUp({
- timelineTime,
- targetAhead,
- }: {
- timelineTime: number;
- targetAhead: number;
- }): Promise<void> {
- return new Promise((resolve) => {
- const checkInterval = setInterval(() => {
- if (!this.editor.playback.getIsPlaying()) {
- clearInterval(checkInterval);
- resolve();
- return;
- }
-
- const playbackTime = this.getPlaybackTime();
- if (timelineTime - playbackTime < targetAhead) {
- clearInterval(checkInterval);
- resolve();
- }
- }, 100);
- });
- }
-
- private disposeSinks(): void {
- for (const iterator of this.clipIterators.values()) {
- void iterator.return();
- }
- this.clipIterators.clear();
- this.activeClipIds.clear();
-
- for (const input of this.inputs.values()) {
- input.dispose();
- }
- this.inputs.clear();
- this.sinks.clear();
- }
-
- private shouldUsePreparedClipBuffer({
- clip,
- }: {
- clip: AudioClipSource;
- }): boolean {
- return (
- this.hasCurveRetime({ clip }) ||
- hasAnimatedVolume({ element: clip.timelineElement }) ||
- shouldMaintainPitch({
- rate: clip.retime?.rate ?? 1,
- maintainPitch: clip.retime?.maintainPitch,
- })
- );
- }
-
- private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean {
- const mode = (clip.retime as { mode?: unknown } | undefined)?.mode;
- return mode === "curve";
- }
-
- private scheduleClipGainAutomation({
- audioContext,
- clip,
- clipGain,
- startTimestamp,
- startLocalTime,
- }: {
- audioContext: AudioContext;
- clip: AudioClipSource;
- clipGain: GainNode;
- startTimestamp: number;
- startLocalTime: number;
- }): void {
- clipGain.gain.cancelScheduledValues(startTimestamp);
- clipGain.gain.setValueAtTime(clip.volume, startTimestamp);
-
- if (!hasAnimatedVolume({ element: clip.timelineElement })) {
- return;
- }
-
- const points = buildAudioGainAutomation({
- element: clip.timelineElement,
- fromLocalTime: startLocalTime,
- toLocalTime: clip.duration,
- });
-
- if (points.length === 0) {
- return;
- }
-
- clipGain.gain.setValueAtTime(points[0].gain, startTimestamp);
- for (let index = 1; index < points.length; index++) {
- const point = points[index];
- const pointTimestamp =
- startTimestamp + (point.localTime - startLocalTime);
- if (pointTimestamp < audioContext.currentTime) {
- continue;
- }
-
- clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp);
- }
- }
-
- private buildPreparedClipCacheKey({
- clip,
- }: {
- clip: AudioClipSource;
- }): string {
- return JSON.stringify({
- id: clip.id,
- sourceKey: clip.sourceKey,
- startTime: clip.startTime,
- duration: clip.duration,
- trimStart: clip.trimStart,
- trimEnd: clip.trimEnd,
- retime: clip.retime ?? null,
- });
- }
-
- private async getPreparedClipBuffer({
- clip,
- }: {
- clip: AudioClipSource;
- }): Promise<AudioBuffer | null> {
- const cacheKey = this.buildPreparedClipCacheKey({ clip });
- const existing = this.preparedClipBuffers.get(cacheKey);
- if (existing) {
- return existing;
- }
-
- const promise = (async () => {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) {
- return null;
- }
-
- const decodedBuffer = await this.getDecodedBuffer({ clip });
- if (!decodedBuffer) {
- return null;
- }
-
- return await renderRetimedBuffer({
- audioContext,
- sourceBuffer: decodedBuffer,
- trimStart: clip.trimStart,
- clipDuration: clip.duration,
- retime: clip.retime,
- maintainPitch: clip.retime?.maintainPitch === true,
- });
- })();
-
- this.preparedClipBuffers.set(cacheKey, promise);
- return promise;
- }
-
- private async getDecodedBuffer({
- clip,
- }: {
- clip: AudioClipSource;
- }): Promise<AudioBuffer | null> {
- const existing = this.decodedBuffers.get(clip.sourceKey);
- if (existing) {
- return existing;
- }
-
- const promise = this.decodeClipBuffer({ clip });
- this.decodedBuffers.set(clip.sourceKey, promise);
- return promise;
- }
-
- private async decodeClipBuffer({
- clip,
- }: {
- clip: AudioClipSource;
- }): Promise<AudioBuffer | null> {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) {
- return null;
- }
-
- const input = new Input({
- source: new BlobSource(clip.file),
- formats: ALL_FORMATS,
- });
-
- try {
- const audioTrack = await input.getPrimaryAudioTrack();
- if (!audioTrack) {
- return null;
- }
-
- const sink = new AudioBufferSink(audioTrack);
- const chunks: AudioBuffer[] = [];
- let totalSamples = 0;
-
- for await (const { buffer } of sink.buffers(0)) {
- chunks.push(buffer);
- totalSamples += buffer.length;
- }
-
- if (chunks.length === 0) {
- return null;
- }
-
- const targetSampleRate = audioContext.sampleRate;
- const nativeSampleRate = chunks[0].sampleRate;
- const numChannels = Math.min(2, chunks[0].numberOfChannels);
- const nativeChannels = Array.from(
- { length: numChannels },
- () => new Float32Array(totalSamples),
- );
-
- let offset = 0;
- for (const chunk of chunks) {
- for (let channel = 0; channel < numChannels; channel++) {
- nativeChannels[channel].set(
- chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)),
- offset,
- );
- }
- offset += chunk.length;
- }
-
- const outputSamples = Math.ceil(
- totalSamples * (targetSampleRate / nativeSampleRate),
- );
- const offlineContext = new OfflineAudioContext(
- numChannels,
- outputSamples,
- targetSampleRate,
- );
- const nativeBuffer = audioContext.createBuffer(
- numChannels,
- totalSamples,
- nativeSampleRate,
- );
-
- for (let channel = 0; channel < numChannels; channel++) {
- nativeBuffer.copyToChannel(nativeChannels[channel], channel);
- }
-
- const sourceNode = offlineContext.createBufferSource();
- sourceNode.buffer = nativeBuffer;
- sourceNode.connect(offlineContext.destination);
- sourceNode.start(0);
-
- return await offlineContext.startRendering();
- } catch (error) {
- console.warn("Failed to decode clip audio:", error);
- return null;
- } finally {
- input.dispose();
- }
- }
-
- private async getAudioSink({
- clip,
- }: {
- clip: AudioClipSource;
- }): Promise<AudioBufferSink | null> {
- const existingSink = this.sinks.get(clip.sourceKey);
- if (existingSink) return existingSink;
-
- try {
- const input = new Input({
- source: new BlobSource(clip.file),
- formats: ALL_FORMATS,
- });
- const audioTrack = await input.getPrimaryAudioTrack();
- if (!audioTrack) {
- input.dispose();
- return null;
- }
-
- const sink = new AudioBufferSink(audioTrack);
- this.inputs.set(clip.sourceKey, input);
- this.sinks.set(clip.sourceKey, sink);
- return sink;
- } catch (error) {
- console.warn("Failed to initialize audio sink:", error);
- return null;
- }
- }
-}
+import type { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { + clampRetimeRate, + shouldMaintainPitch, +} from "@/lib/retime/rate"; +import type { AudioClipSource } from "@/lib/media/audio"; +import { createAudioContext, collectAudioClips } from "@/lib/media/audio"; +import { + buildAudioGainAutomation, + hasAnimatedVolume, +} from "@/lib/timeline/audio-state"; +import { createAudioMasteringChain } from "@/lib/media/audio-mastering"; +import { + getClipTimeAtSourceTime, + getSourceTimeAtClipTime, + renderRetimedBuffer, +} from "@/lib/retime"; +import { + ALL_FORMATS, + AudioBufferSink, + BlobSource, + Input, + type WrappedAudioBuffer, +} from "mediabunny"; + +export class AudioManager { + private audioContext: AudioContext | null = null; + private masterGain: GainNode | null = null; + private playbackStartTime = 0; + private playbackStartContextTime = 0; + private scheduleTimer: number | null = null; + private lookaheadSeconds = 2; + private scheduleIntervalMs = 500; + private clips: AudioClipSource[] = []; + private sinks = new Map<string, AudioBufferSink>(); + private inputs = new Map<string, Input>(); + private activeClipIds = new Set<string>(); + private clipIterators = new Map< + string, + AsyncGenerator<WrappedAudioBuffer, void, unknown> + >(); + private queuedSources = new Set<AudioBufferSourceNode>(); + private preparedClipBuffers = new Map<string, Promise<AudioBuffer | null>>(); + private decodedBuffers = new Map<string, Promise<AudioBuffer | null>>(); + private playbackSessionId = 0; + private lastIsPlaying = false; + private lastVolume = 1; + private playbackLatencyCompensationSeconds = 0; + private unsubscribers: Array<() => void> = []; + + constructor(private editor: EditorCore) { + this.lastVolume = this.editor.playback.getVolume(); + + this.unsubscribers.push( + this.editor.playback.subscribe(this.handlePlaybackChange), + this.editor.timeline.subscribe(this.handleTimelineChange), + this.editor.media.subscribe(this.handleTimelineChange), + ); + if (typeof window !== "undefined") { + window.addEventListener("playback-seek", this.handleSeek); + } + } + + dispose(): void { + this.stopPlayback(); + for (const unsub of this.unsubscribers) { + unsub(); + } + this.unsubscribers = []; + if (typeof window !== "undefined") { + window.removeEventListener("playback-seek", this.handleSeek); + } + this.disposeSinks(); + this.preparedClipBuffers.clear(); + this.decodedBuffers.clear(); + if (this.audioContext) { + void this.audioContext.close(); + this.audioContext = null; + this.masterGain = null; + } + } + + private handlePlaybackChange = (): void => { + const isPlaying = this.editor.playback.getIsPlaying(); + const volume = this.editor.playback.getVolume(); + + if (volume !== this.lastVolume) { + this.lastVolume = volume; + this.updateGain(); + } + + if (isPlaying !== this.lastIsPlaying) { + this.lastIsPlaying = isPlaying; + if (isPlaying) { + void this.startPlayback({ + time: this.editor.playback.getCurrentTime() / TICKS_PER_SECOND, + }); + } else { + this.stopPlayback(); + } + } + }; + + private handleSeek = (event: Event): void => { + const detail = (event as CustomEvent<{ time: number }>).detail; + if (!detail) return; + + if (this.editor.playback.getIsScrubbing()) { + this.stopPlayback(); + return; + } + + if (this.editor.playback.getIsPlaying()) { + void this.startPlayback({ time: detail.time / TICKS_PER_SECOND }); + return; + } + + this.stopPlayback(); + }; + + private handleTimelineChange = (): void => { + this.disposeSinks(); + this.preparedClipBuffers.clear(); + this.decodedBuffers.clear(); + + if (!this.editor.playback.getIsPlaying()) return; + + void this.startPlayback({ time: this.editor.playback.getCurrentTime() / TICKS_PER_SECOND }); + }; + + private ensureAudioContext(): AudioContext | null { + if (this.audioContext) return this.audioContext; + if (typeof window === "undefined") return null; + + this.audioContext = createAudioContext(); + const { input } = createAudioMasteringChain({ + audioContext: this.audioContext, + destination: this.audioContext.destination, + }); + this.masterGain = input; + this.masterGain.gain.value = this.lastVolume; + return this.audioContext; + } + + private updateGain(): void { + if (!this.masterGain) return; + this.masterGain.gain.value = this.lastVolume; + } + + private getPlaybackTime(): number { + if (!this.audioContext) return this.playbackStartTime; + const elapsed = + this.audioContext.currentTime - this.playbackStartContextTime; + return this.playbackStartTime + elapsed; + } + + private async startPlayback({ time }: { time: number }): Promise<void> { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + this.stopPlayback(); + this.playbackSessionId++; + this.playbackLatencyCompensationSeconds = 0; + + const tracks = this.editor.timeline.getTracks(); + const mediaAssets = this.editor.media.getAssets(); + const duration = this.editor.timeline.getTotalDuration(); + + if (duration <= 0) return; + + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + this.clips = await collectAudioClips({ tracks, mediaAssets }); + if (!this.editor.playback.getIsPlaying()) return; + + this.playbackStartTime = time; + this.playbackStartContextTime = audioContext.currentTime; + + this.scheduleUpcomingClips(); + + if (typeof window !== "undefined") { + this.scheduleTimer = window.setInterval(() => { + this.scheduleUpcomingClips(); + }, this.scheduleIntervalMs); + } + } + + private scheduleUpcomingClips(): void { + if (!this.editor.playback.getIsPlaying()) return; + + const currentTime = this.getPlaybackTime(); + const windowEnd = currentTime + this.lookaheadSeconds; + + for (const clip of this.clips) { + if (clip.muted) continue; + if (this.activeClipIds.has(clip.id)) continue; + + const clipEnd = clip.startTime + clip.duration; + if (clipEnd <= currentTime) continue; + if (clip.startTime > windowEnd) continue; + + this.activeClipIds.add(clip.id); + if (this.shouldUsePreparedClipBuffer({ clip })) { + void this.schedulePreparedClip({ + clip, + startTime: currentTime, + sessionId: this.playbackSessionId, + }); + } else { + void this.runClipIterator({ + clip, + startTime: currentTime, + sessionId: this.playbackSessionId, + }); + } + } + } + + private stopPlayback(): void { + if (this.scheduleTimer && typeof window !== "undefined") { + window.clearInterval(this.scheduleTimer); + } + this.scheduleTimer = null; + + for (const iterator of this.clipIterators.values()) { + void iterator.return(); + } + this.clipIterators.clear(); + this.activeClipIds.clear(); + + for (const source of this.queuedSources) { + try { + source.stop(); + } catch {} + source.disconnect(); + } + this.queuedSources.clear(); + } + + private async runClipIterator({ + clip, + startTime, + sessionId, + }: { + clip: AudioClipSource; + startTime: number; + sessionId: number; + }): Promise<void> { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + const sink = await this.getAudioSink({ clip }); + if (!sink || !this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const clipStart = clip.startTime; + const clipEnd = clip.startTime + clip.duration; + const playbackTimeAfterSinkReady = this.getPlaybackTime(); + const iteratorStartTime = Math.max( + startTime, + clipStart, + playbackTimeAfterSinkReady, + ); + if (iteratorStartTime >= clipEnd) { + return; + } + const sourceStartTime = + clip.trimStart + + getSourceTimeAtClipTime({ + clipTime: iteratorStartTime - clip.startTime, + retime: clip.retime, + }); + + const iterator = sink.buffers(sourceStartTime); + this.clipIterators.set(clip.id, iterator); + let consecutiveDroppedBufferCount = 0; + + for await (const { buffer, timestamp } of iterator) { + if (!this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const timelineTime = + clip.startTime + + getClipTimeAtSourceTime({ + sourceTime: timestamp - clip.trimStart, + retime: clip.retime, + }); + if (timelineTime >= clipEnd) break; + + const node = audioContext.createBufferSource(); + node.buffer = buffer; + if (clip.retime) { + node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate }); + } + const clipGain = audioContext.createGain(); + clipGain.gain.value = clip.volume; + node.connect(clipGain); + clipGain.connect(this.masterGain ?? audioContext.destination); + + const startTimestamp = + this.playbackStartContextTime + + this.playbackLatencyCompensationSeconds + + (timelineTime - this.playbackStartTime); + + if (startTimestamp >= audioContext.currentTime) { + node.start(startTimestamp); + consecutiveDroppedBufferCount = 0; + } else { + const offset = audioContext.currentTime - startTimestamp; + if (offset < buffer.duration) { + node.start(audioContext.currentTime, offset); + consecutiveDroppedBufferCount = 0; + } else { + consecutiveDroppedBufferCount += 1; + if (consecutiveDroppedBufferCount >= 5) { + const nextCompensationSeconds = Math.max( + this.playbackLatencyCompensationSeconds, + Math.min(0.25, offset + 0.01), + ); + if ( + nextCompensationSeconds > + this.playbackLatencyCompensationSeconds + 0.001 + ) { + this.playbackLatencyCompensationSeconds = nextCompensationSeconds; + } + const resyncStartTime = this.getPlaybackTime(); + this.clipIterators.delete(clip.id); + void this.runClipIterator({ + clip, + startTime: resyncStartTime, + sessionId, + }); + return; + } + continue; + } + } + + this.queuedSources.add(node); + node.addEventListener("ended", () => { + node.disconnect(); + clipGain.disconnect(); + this.queuedSources.delete(node); + }); + + const aheadTime = timelineTime - this.getPlaybackTime(); + if (aheadTime >= 1) { + await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 }); + if (sessionId !== this.playbackSessionId) return; + } + } + + this.clipIterators.delete(clip.id); + // don't remove from activeClipIds - prevents scheduler from restarting this clip + // the set is cleared on stopPlayback anyway + } + + private async schedulePreparedClip({ + clip, + startTime, + sessionId, + }: { + clip: AudioClipSource; + startTime: number; + sessionId: number; + }): Promise<void> { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + const buffer = await this.getPreparedClipBuffer({ clip }); + if (!buffer || !this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const clipStart = clip.startTime; + const clipEnd = clip.startTime + clip.duration; + const playbackTimeAfterReady = this.getPlaybackTime(); + const effectiveStartTime = Math.max( + startTime, + clipStart, + playbackTimeAfterReady, + ); + if (effectiveStartTime >= clipEnd) { + return; + } + + const node = audioContext.createBufferSource(); + node.buffer = buffer; + const clipGain = audioContext.createGain(); + node.connect(clipGain); + clipGain.connect(this.masterGain ?? audioContext.destination); + + const startTimestamp = + this.playbackStartContextTime + + this.playbackLatencyCompensationSeconds + + (effectiveStartTime - this.playbackStartTime); + const clipOffset = effectiveStartTime - clipStart; + let actualStartTimestamp = startTimestamp; + let actualClipOffset = clipOffset; + + if (startTimestamp >= audioContext.currentTime) { + node.start(startTimestamp, clipOffset); + } else { + const lateOffset = audioContext.currentTime - startTimestamp; + actualStartTimestamp = audioContext.currentTime; + actualClipOffset = clipOffset + lateOffset; + node.start(actualStartTimestamp, actualClipOffset); + } + + this.scheduleClipGainAutomation({ + audioContext, + clip, + clipGain, + startTimestamp: actualStartTimestamp, + startLocalTime: actualClipOffset, + }); + + this.queuedSources.add(node); + node.addEventListener("ended", () => { + node.disconnect(); + clipGain.disconnect(); + this.queuedSources.delete(node); + }); + } + + private waitUntilCaughtUp({ + timelineTime, + targetAhead, + }: { + timelineTime: number; + targetAhead: number; + }): Promise<void> { + return new Promise((resolve) => { + const checkInterval = setInterval(() => { + if (!this.editor.playback.getIsPlaying()) { + clearInterval(checkInterval); + resolve(); + return; + } + + const playbackTime = this.getPlaybackTime(); + if (timelineTime - playbackTime < targetAhead) { + clearInterval(checkInterval); + resolve(); + } + }, 100); + }); + } + + private disposeSinks(): void { + for (const iterator of this.clipIterators.values()) { + void iterator.return(); + } + this.clipIterators.clear(); + this.activeClipIds.clear(); + + for (const input of this.inputs.values()) { + input.dispose(); + } + this.inputs.clear(); + this.sinks.clear(); + } + + private shouldUsePreparedClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): boolean { + return ( + this.hasCurveRetime({ clip }) || + hasAnimatedVolume({ element: clip.timelineElement }) || + shouldMaintainPitch({ + rate: clip.retime?.rate ?? 1, + maintainPitch: clip.retime?.maintainPitch, + }) + ); + } + + private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean { + const mode = (clip.retime as { mode?: unknown } | undefined)?.mode; + return mode === "curve"; + } + + private scheduleClipGainAutomation({ + audioContext, + clip, + clipGain, + startTimestamp, + startLocalTime, + }: { + audioContext: AudioContext; + clip: AudioClipSource; + clipGain: GainNode; + startTimestamp: number; + startLocalTime: number; + }): void { + clipGain.gain.cancelScheduledValues(startTimestamp); + clipGain.gain.setValueAtTime(clip.volume, startTimestamp); + + if (!hasAnimatedVolume({ element: clip.timelineElement })) { + return; + } + + const points = buildAudioGainAutomation({ + element: clip.timelineElement, + fromLocalTime: startLocalTime, + toLocalTime: clip.duration, + }); + + if (points.length === 0) { + return; + } + + clipGain.gain.setValueAtTime(points[0].gain, startTimestamp); + for (let index = 1; index < points.length; index++) { + const point = points[index]; + const pointTimestamp = + startTimestamp + (point.localTime - startLocalTime); + if (pointTimestamp < audioContext.currentTime) { + continue; + } + + clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp); + } + } + + private buildPreparedClipCacheKey({ + clip, + }: { + clip: AudioClipSource; + }): string { + return JSON.stringify({ + id: clip.id, + sourceKey: clip.sourceKey, + startTime: clip.startTime, + duration: clip.duration, + trimStart: clip.trimStart, + trimEnd: clip.trimEnd, + retime: clip.retime ?? null, + }); + } + + private async getPreparedClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise<AudioBuffer | null> { + const cacheKey = this.buildPreparedClipCacheKey({ clip }); + const existing = this.preparedClipBuffers.get(cacheKey); + if (existing) { + return existing; + } + + const promise = (async () => { + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + + const decodedBuffer = await this.getDecodedBuffer({ clip }); + if (!decodedBuffer) { + return null; + } + + return await renderRetimedBuffer({ + audioContext, + sourceBuffer: decodedBuffer, + trimStart: clip.trimStart, + clipDuration: clip.duration, + retime: clip.retime, + maintainPitch: clip.retime?.maintainPitch === true, + }); + })(); + + this.preparedClipBuffers.set(cacheKey, promise); + return promise; + } + + private async getDecodedBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise<AudioBuffer | null> { + const existing = this.decodedBuffers.get(clip.sourceKey); + if (existing) { + return existing; + } + + const promise = this.decodeClipBuffer({ clip }); + this.decodedBuffers.set(clip.sourceKey, promise); + return promise; + } + + private async decodeClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise<AudioBuffer | null> { + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + + const input = new Input({ + source: new BlobSource(clip.file), + formats: ALL_FORMATS, + }); + + try { + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + return null; + } + + const sink = new AudioBufferSink(audioTrack); + const chunks: AudioBuffer[] = []; + let totalSamples = 0; + + for await (const { buffer } of sink.buffers(0)) { + chunks.push(buffer); + totalSamples += buffer.length; + } + + if (chunks.length === 0) { + return null; + } + + const targetSampleRate = audioContext.sampleRate; + const nativeSampleRate = chunks[0].sampleRate; + const numChannels = Math.min(2, chunks[0].numberOfChannels); + const nativeChannels = Array.from( + { length: numChannels }, + () => new Float32Array(totalSamples), + ); + + let offset = 0; + for (const chunk of chunks) { + for (let channel = 0; channel < numChannels; channel++) { + nativeChannels[channel].set( + chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)), + offset, + ); + } + offset += chunk.length; + } + + const outputSamples = Math.ceil( + totalSamples * (targetSampleRate / nativeSampleRate), + ); + const offlineContext = new OfflineAudioContext( + numChannels, + outputSamples, + targetSampleRate, + ); + const nativeBuffer = audioContext.createBuffer( + numChannels, + totalSamples, + nativeSampleRate, + ); + + for (let channel = 0; channel < numChannels; channel++) { + nativeBuffer.copyToChannel(nativeChannels[channel], channel); + } + + const sourceNode = offlineContext.createBufferSource(); + sourceNode.buffer = nativeBuffer; + sourceNode.connect(offlineContext.destination); + sourceNode.start(0); + + return await offlineContext.startRendering(); + } catch (error) { + console.warn("Failed to decode clip audio:", error); + return null; + } finally { + input.dispose(); + } + } + + private async getAudioSink({ + clip, + }: { + clip: AudioClipSource; + }): Promise<AudioBufferSink | null> { + const existingSink = this.sinks.get(clip.sourceKey); + if (existingSink) return existingSink; + + try { + const input = new Input({ + source: new BlobSource(clip.file), + formats: ALL_FORMATS, + }); + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + input.dispose(); + return null; + } + + const sink = new AudioBufferSink(audioTrack); + this.inputs.set(clip.sourceKey, input); + this.sinks.set(clip.sourceKey, sink); + return sink; + } catch (error) { + console.warn("Failed to initialize audio sink:", error); + return null; + } + } +} diff --git a/apps/web/src/core/managers/playback-manager.ts b/apps/web/src/core/managers/playback-manager.ts index 6943f187..47909bfe 100644 --- a/apps/web/src/core/managers/playback-manager.ts +++ b/apps/web/src/core/managers/playback-manager.ts @@ -1,4 +1,6 @@ import type { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { roundToFrame } from "opencut-wasm"; export class PlaybackManager { private isPlaying = false; @@ -9,11 +11,12 @@ export class PlaybackManager { private isScrubbing = false; private listeners = new Set<() => void>(); private playbackTimer: number | null = null; - private lastUpdate = 0; + private playbackStartWallTime = 0; + private playbackStartTime = 0; constructor(private editor: EditorCore) { this.editor.timeline.subscribe(() => { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); if (this.currentTime > maxTime && maxTime > 0) { this.currentTime = maxTime; this.notify(); @@ -22,7 +25,7 @@ export class PlaybackManager { } play(): void { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); if (maxTime > 0) { if (this.currentTime >= maxTime) { @@ -50,8 +53,12 @@ export class PlaybackManager { } seek({ time }: { time: number }): void { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); this.currentTime = Math.max(0, Math.min(maxTime, time)); + if (this.isPlaying) { + this.playbackStartWallTime = performance.now(); + this.playbackStartTime = this.currentTime; + } this.notify(); window.dispatchEvent( @@ -135,7 +142,8 @@ export class PlaybackManager { cancelAnimationFrame(this.playbackTimer); } - this.lastUpdate = performance.now(); + this.playbackStartWallTime = performance.now(); + this.playbackStartTime = this.currentTime; this.updateTime(); } @@ -149,12 +157,11 @@ export class PlaybackManager { private updateTime = (): void => { if (!this.isPlaying) return; - const now = performance.now(); - const delta = (now - this.lastUpdate) / 1000; - this.lastUpdate = now; - - const newTime = this.currentTime + delta; - const maxTime = this.editor.timeline.getLastSeekableTime(); + const fps = this.editor.project.getActive()?.settings.fps; + const elapsedSeconds = (performance.now() - this.playbackStartWallTime) / 1000; + const rawTime = this.playbackStartTime + Math.round(elapsedSeconds * TICKS_PER_SECOND); + const newTime = fps ? (roundToFrame({ time: rawTime, rate: fps }) ?? rawTime) : rawTime; + const maxTime = this.editor.timeline.getTotalDuration(); if (maxTime > 0 && newTime >= maxTime) { this.pause(); diff --git a/apps/web/src/core/managers/project-manager.ts b/apps/web/src/core/managers/project-manager.ts index 10a9229d..513eab94 100644 --- a/apps/web/src/core/managers/project-manager.ts +++ b/apps/web/src/core/managers/project-manager.ts @@ -496,7 +496,7 @@ export class ProjectManager { importedAssets, }: { importedAssets: Array<Pick<MediaAsset, "type" | "fps">>; - }): number | null { + }): import("opencut-wasm").FrameRate | null { if (!this.active) return null; const nextFps = getRaisedProjectFpsForImportedMedia({ diff --git a/apps/web/src/core/managers/renderer-manager.ts b/apps/web/src/core/managers/renderer-manager.ts index aa34f496..e0d5b895 100644 --- a/apps/web/src/core/managers/renderer-manager.ts +++ b/apps/web/src/core/managers/renderer-manager.ts @@ -5,7 +5,8 @@ import { CanvasRenderer } from "@/services/renderer/canvas-renderer"; import { SceneExporter } from "@/services/renderer/scene-exporter"; import { buildScene } from "@/services/renderer/scene-builder"; import { createTimelineAudioBuffer } from "@/lib/media/audio"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode } from "opencut-wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; import { downloadBlob } from "@/utils/browser"; type SnapshotResult = @@ -81,7 +82,10 @@ export class RendererManager { } const { canvasSize, fps } = activeProject.settings; - const renderTime = this.editor.playback.getCurrentTime(); + const renderTime = Math.min( + this.editor.playback.getCurrentTime(), + this.editor.timeline.getLastFrameTime(), + ); const renderer = new CanvasRenderer({ width: canvasSize.width, @@ -107,7 +111,7 @@ export class RendererManager { return { success: false, error: "Failed to create image" }; } - const timecode = formatTimeCode({ timeInSeconds: renderTime, fps })!.replace(/:/g, "-"); + const timecode = formatTimecode({ time: renderTime, rate: fps })!.replace(/:/g, "-"); const safeName = activeProject.metadata.name.replace(/[<>:"/\\|?*]/g, "-").trim() || "snapshot"; @@ -148,7 +152,7 @@ export class RendererManager { return { success: false, error: "Project is empty" }; } - const exportFps = fps || activeProject.settings.fps; + const exportFps = fps ?? activeProject.settings.fps; const canvasSize = activeProject.settings.canvasSize; let audioBuffer: AudioBuffer | null = null; diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 3abde47b..a5cb428e 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -18,7 +18,7 @@ import type { AnimationValue, ScalarCurveKeyframePatch, } from "@/lib/animation/types"; -import { getLastFrameTime } from "opencut-wasm"; +import { lastFrameTime } from "opencut-wasm"; import { BatchCommand } from "@/lib/commands"; import { AddTrackCommand, @@ -196,11 +196,11 @@ export class TimelineManager { return calculateTotalDuration({ tracks: this.getTracks() }); } - getLastSeekableTime(): number { + getLastFrameTime(): number { const duration = this.getTotalDuration(); const fps = this.editor.project.getActive()?.settings.fps; if (!fps || duration <= 0) return duration; - return getLastFrameTime({ duration, fps }); + return lastFrameTime({ duration, rate: fps }) ?? duration; } getTrackById({ trackId }: { trackId: string }): TimelineTrack | null { |
