refactor: split effects and masks into dedicated rust crates, introduce MediaTime and FrameRate

5 files changed, 218 insertions, 209 deletions

diff --git a/apps/web/src/services/renderer/canvas-renderer.ts b/apps/web/src/services/renderer/canvas-renderer.ts
index 840d8b52..b142d97a 100644
--- a/apps/web/src/services/renderer/canvas-renderer.ts
+++ b/apps/web/src/services/renderer/canvas-renderer.ts
@@ -1,9 +1,11 @@
+import type { FrameRate } from "opencut-wasm";
+import { frameRateToFloat } from "@/lib/fps/utils";
 import type { BaseNode } from "./nodes/base-node";
 
 export type CanvasRendererParams = {
 	width: number;
 	height: number;
-	fps: number;
+	fps: FrameRate;
 };
 
 export class CanvasRenderer {
@@ -11,7 +13,7 @@ export class CanvasRenderer {
 	context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D;
 	width: number;
 	height: number;
-	fps: number;
+	fps: FrameRate;
 
 	constructor({ width, height, fps }: CanvasRendererParams) {
 		this.width = width;
diff --git a/apps/web/src/services/renderer/nodes/blur-background-node.ts b/apps/web/src/services/renderer/nodes/blur-background-node.ts
index c67a5113..4cb45b33 100644
--- a/apps/web/src/services/renderer/nodes/blur-background-node.ts
+++ b/apps/web/src/services/renderer/nodes/blur-background-node.ts
@@ -1,5 +1,5 @@
-import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants";
 import { buildGaussianBlurPasses, intensityToSigma } from "@/lib/effects/definitions/blur";
+import { mediaTimeToSeconds } from "opencut-wasm";
 import { getSourceTimeAtClipTime } from "@/lib/retime";
 import { videoCache } from "@/services/video-cache/service";
 import type { RetimeConfig } from "@/lib/timeline";
@@ -39,9 +39,7 @@ export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> {
 
 	private isInRange({ time }: { time: number }): boolean {
 		const localTime = time - this.params.timeOffset;
-		return (
-			localTime >= -TIME_EPSILON_SECONDS && localTime < this.params.duration
-		);
+		return localTime >= 0 && localTime < this.params.duration;
 	}
 
 	private getSourceLocalTime({ time }: { time: number }): number {
@@ -61,10 +59,11 @@ export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> {
 		time: number;
 	}): Promise<BackdropSource | null> {
 		if (this.params.mediaType === "video") {
+			const sourceTimeTicks = this.getSourceLocalTime({ time });
 			const frame = await videoCache.getFrameAt({
 				mediaId: this.params.mediaId,
 				file: this.params.file,
-				time: this.getSourceLocalTime({ time }),
+				time: mediaTimeToSeconds({ time: sourceTimeTicks }),
 			});
 
 			if (!frame) {
diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts
index 3bb836ec..4f225567 100644
--- a/apps/web/src/services/renderer/nodes/video-node.ts
+++ b/apps/web/src/services/renderer/nodes/video-node.ts
@@ -1,37 +1,39 @@
-import type { CanvasRenderer } from "../canvas-renderer";

-import { VisualNode, type VisualNodeParams } from "./visual-node";

-import { videoCache } from "@/services/video-cache/service";

-

-export interface VideoNodeParams extends VisualNodeParams {

-	url: string;

-	file: File;

-	mediaId: string;

-}

-

-export class VideoNode extends VisualNode<VideoNodeParams> {

-	async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {

-		await super.render({ renderer, time });

-

-		if (!this.isInRange({ time })) {

-			return;

-		}

-

-		const videoTime = this.getSourceLocalTime({ time });

-

-		const frame = await videoCache.getFrameAt({

-			mediaId: this.params.mediaId,

-			file: this.params.file,

-			time: videoTime,

-		});

-

-		if (frame) {

-			this.renderVisual({

-				renderer,

-				source: frame.canvas,

-				sourceWidth: frame.canvas.width,

-				sourceHeight: frame.canvas.height,

-				timelineTime: time,

-			});

-		}

-	}

-}

+import type { CanvasRenderer } from "../canvas-renderer";
+import { mediaTimeToSeconds } from "opencut-wasm";
+import { VisualNode, type VisualNodeParams } from "./visual-node";
+import { videoCache } from "@/services/video-cache/service";
+
+export interface VideoNodeParams extends VisualNodeParams {
+	url: string;
+	file: File;
+	mediaId: string;
+}
+
+export class VideoNode extends VisualNode<VideoNodeParams> {
+	async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
+		await super.render({ renderer, time });
+
+		if (!this.isInRange({ time })) {
+			return;
+		}
+
+		const videoTimeTicks = this.getSourceLocalTime({ time });
+		const videoTimeSeconds = mediaTimeToSeconds({ time: videoTimeTicks });
+
+		const frame = await videoCache.getFrameAt({
+			mediaId: this.params.mediaId,
+			file: this.params.file,
+			time: videoTimeSeconds,
+		});
+
+		if (frame) {
+			this.renderVisual({
+				renderer,
+				source: frame.canvas,
+				sourceWidth: frame.canvas.width,
+				sourceHeight: frame.canvas.height,
+				timelineTime: time,
+			});
+		}
+	}
+}
diff --git a/apps/web/src/services/renderer/nodes/visual-node.ts b/apps/web/src/services/renderer/nodes/visual-node.ts
index 82093da4..4a5fcdc6 100644
--- a/apps/web/src/services/renderer/nodes/visual-node.ts
+++ b/apps/web/src/services/renderer/nodes/visual-node.ts
@@ -12,7 +12,6 @@ import {
 	resolveTransformAtTime,
 } from "@/lib/animation";
 import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel";
-import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants";
 import { effectsRegistry, resolveEffectPasses } from "@/lib/effects";
 import { masksRegistry } from "@/lib/masks";
 import { getSourceTimeAtClipTime } from "@/lib/retime";
@@ -58,7 +57,7 @@ export abstract class VisualNode<
 	protected isInRange({ time }: { time: number }): boolean {
 		const localTime = time - this.params.timeOffset;
 		return (
-			localTime >= -TIME_EPSILON_SECONDS &&
+			localTime >= 0 &&
 			localTime < this.params.duration
 		);
 	}
diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts
index 2dbb7cdd..0c2ee17f 100644
--- a/apps/web/src/services/renderer/scene-exporter.ts
+++ b/apps/web/src/services/renderer/scene-exporter.ts
@@ -1,163 +1,170 @@
-import EventEmitter from "eventemitter3";

-

-import {

-	Output,

-	Mp4OutputFormat,

-	WebMOutputFormat,

-	BufferTarget,

-	CanvasSource,

-	AudioBufferSource,

-	QUALITY_LOW,

-	QUALITY_MEDIUM,

-	QUALITY_HIGH,

-	QUALITY_VERY_HIGH,

-} from "mediabunny";

-import type { RootNode } from "./nodes/root-node";

-import type { ExportFormat, ExportQuality } from "@/lib/export";

-import { CanvasRenderer } from "./canvas-renderer";

-

-type ExportParams = {

-	width: number;

-	height: number;

-	fps: number;

-	format: ExportFormat;

-	quality: ExportQuality;

-	shouldIncludeAudio?: boolean;

-	audioBuffer?: AudioBuffer;

-};

-

-const qualityMap = {

-	low: QUALITY_LOW,

-	medium: QUALITY_MEDIUM,

-	high: QUALITY_HIGH,

-	very_high: QUALITY_VERY_HIGH,

-};

-

-export type SceneExporterEvents = {

-	progress: [progress: number];

-	complete: [buffer: ArrayBuffer];

-	error: [error: Error];

-	cancelled: [];

-};

-

-export class SceneExporter extends EventEmitter<SceneExporterEvents> {

-	private renderer: CanvasRenderer;

-	private format: ExportFormat;

-	private quality: ExportQuality;

-	private shouldIncludeAudio: boolean;

-	private audioBuffer?: AudioBuffer;

-

-	private isCancelled = false;

-

-	constructor({

-		width,

-		height,

-		fps,

-		format,

-		quality,

-		shouldIncludeAudio,

-		audioBuffer,

-	}: ExportParams) {

-		super();

-		this.renderer = new CanvasRenderer({

-			width,

-			height,

-			fps,

-		});

-

-		this.format = format;

-		this.quality = quality;

-		this.shouldIncludeAudio = shouldIncludeAudio ?? false;

-		this.audioBuffer = audioBuffer;

-	}

-

-	cancel(): void {

-		this.isCancelled = true;

-	}

-

-	async export({

-		rootNode,

-	}: {

-		rootNode: RootNode;

-	}): Promise<ArrayBuffer | null> {

-		const { fps } = this.renderer;

-		const frameCount = Math.ceil(rootNode.duration * fps);

-

-		const outputFormat =

-			this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();

-

-		const output = new Output({

-			format: outputFormat,

-			target: new BufferTarget(),

-		});

-

-		const videoSource = new CanvasSource(this.renderer.canvas, {

-			codec: this.format === "webm" ? "vp9" : "avc",

-			bitrate: qualityMap[this.quality],

-		});

-

-		output.addVideoTrack(videoSource, { frameRate: fps });

-

-		let audioSource: AudioBufferSource | null = null;

-		if (this.shouldIncludeAudio && this.audioBuffer) {

-			let audioCodec: "aac" | "opus" =

-				this.format === "webm" ? "opus" : "aac";

-

-			if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") {

-				const { supported } = await AudioEncoder.isConfigSupported({

-					codec: "mp4a.40.2",

-					sampleRate: this.audioBuffer.sampleRate,

-					numberOfChannels: this.audioBuffer.numberOfChannels,

-					bitrate: 192000,

-				});

-				if (!supported) audioCodec = "opus";

-			}

-

-			audioSource = new AudioBufferSource({

-				codec: audioCodec,

-				bitrate: qualityMap[this.quality],

-			});

-			output.addAudioTrack(audioSource);

-		}

-

-		await output.start();

-

-		if (audioSource && this.audioBuffer) {

-			await audioSource.add(this.audioBuffer);

-			audioSource.close();

-		}

-

-		for (let i = 0; i < frameCount; i++) {

-			if (this.isCancelled) {

-				await output.cancel();

-				this.emit("cancelled");

-				return null;

-			}

-

-			const time = i / fps;

-			await this.renderer.render({ node: rootNode, time });

-			await videoSource.add(time, 1 / fps);

-

-			this.emit("progress", i / frameCount);

-		}

-

-		if (this.isCancelled) {

-			await output.cancel();

-			this.emit("cancelled");

-			return null;

-		}

-

-		videoSource.close();

-		await output.finalize();

-		this.emit("progress", 1);

-

-		const buffer = output.target.buffer;

-		if (!buffer) {

-			this.emit("error", new Error("Failed to export video"));

-			return null;

-		}

-

-		this.emit("complete", buffer);

-		return buffer;

-	}

-}

+import EventEmitter from "eventemitter3";
+
+import {
+	Output,
+	Mp4OutputFormat,
+	WebMOutputFormat,
+	BufferTarget,
+	CanvasSource,
+	AudioBufferSource,
+	QUALITY_LOW,
+	QUALITY_MEDIUM,
+	QUALITY_HIGH,
+	QUALITY_VERY_HIGH,
+} from "mediabunny";
+import type { FrameRate } from "opencut-wasm";
+import { mediaTimeToSeconds } from "opencut-wasm";
+import { TICKS_PER_SECOND } from "@/lib/wasm";
+import { frameRateToFloat } from "@/lib/fps/utils";
+import type { RootNode } from "./nodes/root-node";
+import type { ExportFormat, ExportQuality } from "@/lib/export";
+import { CanvasRenderer } from "./canvas-renderer";
+
+type ExportParams = {
+	width: number;
+	height: number;
+	fps: FrameRate;
+	format: ExportFormat;
+	quality: ExportQuality;
+	shouldIncludeAudio?: boolean;
+	audioBuffer?: AudioBuffer;
+};
+
+const qualityMap = {
+	low: QUALITY_LOW,
+	medium: QUALITY_MEDIUM,
+	high: QUALITY_HIGH,
+	very_high: QUALITY_VERY_HIGH,
+};
+
+export type SceneExporterEvents = {
+	progress: [progress: number];
+	complete: [buffer: ArrayBuffer];
+	error: [error: Error];
+	cancelled: [];
+};
+
+export class SceneExporter extends EventEmitter<SceneExporterEvents> {
+	private renderer: CanvasRenderer;
+	private format: ExportFormat;
+	private quality: ExportQuality;
+	private shouldIncludeAudio: boolean;
+	private audioBuffer?: AudioBuffer;
+
+	private isCancelled = false;
+
+	constructor({
+		width,
+		height,
+		fps,
+		format,
+		quality,
+		shouldIncludeAudio,
+		audioBuffer,
+	}: ExportParams) {
+		super();
+		this.renderer = new CanvasRenderer({
+			width,
+			height,
+			fps,
+		});
+
+		this.format = format;
+		this.quality = quality;
+		this.shouldIncludeAudio = shouldIncludeAudio ?? false;
+		this.audioBuffer = audioBuffer;
+	}
+
+	cancel(): void {
+		this.isCancelled = true;
+	}
+
+	async export({
+		rootNode,
+	}: {
+		rootNode: RootNode;
+	}): Promise<ArrayBuffer | null> {
+		const fps = this.renderer.fps;
+		const fpsFloat = frameRateToFloat(fps);
+		const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator);
+		const frameCount = Math.floor(rootNode.duration / ticksPerFrame);
+
+		const outputFormat =
+			this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();
+
+		const output = new Output({
+			format: outputFormat,
+			target: new BufferTarget(),
+		});
+
+		const videoSource = new CanvasSource(this.renderer.canvas, {
+			codec: this.format === "webm" ? "vp9" : "avc",
+			bitrate: qualityMap[this.quality],
+		});
+
+		output.addVideoTrack(videoSource, { frameRate: fpsFloat });
+
+		let audioSource: AudioBufferSource | null = null;
+		if (this.shouldIncludeAudio && this.audioBuffer) {
+			let audioCodec: "aac" | "opus" =
+				this.format === "webm" ? "opus" : "aac";
+
+			if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") {
+				const { supported } = await AudioEncoder.isConfigSupported({
+					codec: "mp4a.40.2",
+					sampleRate: this.audioBuffer.sampleRate,
+					numberOfChannels: this.audioBuffer.numberOfChannels,
+					bitrate: 192000,
+				});
+				if (!supported) audioCodec = "opus";
+			}
+
+			audioSource = new AudioBufferSource({
+				codec: audioCodec,
+				bitrate: qualityMap[this.quality],
+			});
+			output.addAudioTrack(audioSource);
+		}
+
+		await output.start();
+
+		if (audioSource && this.audioBuffer) {
+			await audioSource.add(this.audioBuffer);
+			audioSource.close();
+		}
+
+		for (let i = 0; i < frameCount; i++) {
+			if (this.isCancelled) {
+				await output.cancel();
+				this.emit("cancelled");
+				return null;
+			}
+
+			const timeTicks = i * ticksPerFrame;
+			const timeSeconds = mediaTimeToSeconds({ time: timeTicks });
+			await this.renderer.render({ node: rootNode, time: timeTicks });
+			await videoSource.add(timeSeconds, 1 / fpsFloat);
+
+			this.emit("progress", i / frameCount);
+		}
+
+		if (this.isCancelled) {
+			await output.cancel();
+			this.emit("cancelled");
+			return null;
+		}
+
+		videoSource.close();
+		await output.finalize();
+		this.emit("progress", 1);
+
+		const buffer = output.target.buffer;
+		if (!buffer) {
+			this.emit("error", new Error("Failed to export video"));
+			return null;
+		}
+
+		this.emit("complete", buffer);
+		return buffer;
+	}
+}