From e4b67094e757917588fb624a30aa046667403add Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Tue, 7 Apr 2026 01:09:13 +0200 Subject: refactor: split effects and masks into dedicated rust crates, introduce MediaTime and FrameRate --- apps/web/src/services/renderer/canvas-renderer.ts | 6 +- .../renderer/nodes/blur-background-node.ts | 9 +- apps/web/src/services/renderer/nodes/video-node.ts | 76 ++--- .../web/src/services/renderer/nodes/visual-node.ts | 3 +- apps/web/src/services/renderer/scene-exporter.ts | 333 +++++++++++---------- 5 files changed, 218 insertions(+), 209 deletions(-) (limited to 'apps/web/src/services/renderer') diff --git a/apps/web/src/services/renderer/canvas-renderer.ts b/apps/web/src/services/renderer/canvas-renderer.ts index 840d8b52..b142d97a 100644 --- a/apps/web/src/services/renderer/canvas-renderer.ts +++ b/apps/web/src/services/renderer/canvas-renderer.ts @@ -1,9 +1,11 @@ +import type { FrameRate } from "opencut-wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; import type { BaseNode } from "./nodes/base-node"; export type CanvasRendererParams = { width: number; height: number; - fps: number; + fps: FrameRate; }; export class CanvasRenderer { @@ -11,7 +13,7 @@ export class CanvasRenderer { context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; width: number; height: number; - fps: number; + fps: FrameRate; constructor({ width, height, fps }: CanvasRendererParams) { this.width = width; diff --git a/apps/web/src/services/renderer/nodes/blur-background-node.ts b/apps/web/src/services/renderer/nodes/blur-background-node.ts index c67a5113..4cb45b33 100644 --- a/apps/web/src/services/renderer/nodes/blur-background-node.ts +++ b/apps/web/src/services/renderer/nodes/blur-background-node.ts @@ -1,5 +1,5 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { buildGaussianBlurPasses, intensityToSigma } from "@/lib/effects/definitions/blur"; +import { mediaTimeToSeconds } from "opencut-wasm"; import { getSourceTimeAtClipTime } from "@/lib/retime"; import { videoCache } from "@/services/video-cache/service"; import type { RetimeConfig } from "@/lib/timeline"; @@ -39,9 +39,7 @@ export class BlurBackgroundNode extends BaseNode { private isInRange({ time }: { time: number }): boolean { const localTime = time - this.params.timeOffset; - return ( - localTime >= -TIME_EPSILON_SECONDS && localTime < this.params.duration - ); + return localTime >= 0 && localTime < this.params.duration; } private getSourceLocalTime({ time }: { time: number }): number { @@ -61,10 +59,11 @@ export class BlurBackgroundNode extends BaseNode { time: number; }): Promise { if (this.params.mediaType === "video") { + const sourceTimeTicks = this.getSourceLocalTime({ time }); const frame = await videoCache.getFrameAt({ mediaId: this.params.mediaId, file: this.params.file, - time: this.getSourceLocalTime({ time }), + time: mediaTimeToSeconds({ time: sourceTimeTicks }), }); if (!frame) { diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts index 3bb836ec..4f225567 100644 --- a/apps/web/src/services/renderer/nodes/video-node.ts +++ b/apps/web/src/services/renderer/nodes/video-node.ts @@ -1,37 +1,39 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { VisualNode, type VisualNodeParams } from "./visual-node"; -import { videoCache } from "@/services/video-cache/service"; - -export interface VideoNodeParams extends VisualNodeParams { - url: string; - file: File; - mediaId: string; -} - -export class VideoNode extends VisualNode { - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const videoTime = this.getSourceLocalTime({ time }); - - const frame = await videoCache.getFrameAt({ - mediaId: this.params.mediaId, - file: this.params.file, - time: videoTime, - }); - - if (frame) { - this.renderVisual({ - renderer, - source: frame.canvas, - sourceWidth: frame.canvas.width, - sourceHeight: frame.canvas.height, - timelineTime: time, - }); - } - } -} +import type { CanvasRenderer } from "../canvas-renderer"; +import { mediaTimeToSeconds } from "opencut-wasm"; +import { VisualNode, type VisualNodeParams } from "./visual-node"; +import { videoCache } from "@/services/video-cache/service"; + +export interface VideoNodeParams extends VisualNodeParams { + url: string; + file: File; + mediaId: string; +} + +export class VideoNode extends VisualNode { + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + await super.render({ renderer, time }); + + if (!this.isInRange({ time })) { + return; + } + + const videoTimeTicks = this.getSourceLocalTime({ time }); + const videoTimeSeconds = mediaTimeToSeconds({ time: videoTimeTicks }); + + const frame = await videoCache.getFrameAt({ + mediaId: this.params.mediaId, + file: this.params.file, + time: videoTimeSeconds, + }); + + if (frame) { + this.renderVisual({ + renderer, + source: frame.canvas, + sourceWidth: frame.canvas.width, + sourceHeight: frame.canvas.height, + timelineTime: time, + }); + } + } +} diff --git a/apps/web/src/services/renderer/nodes/visual-node.ts b/apps/web/src/services/renderer/nodes/visual-node.ts index 82093da4..4a5fcdc6 100644 --- a/apps/web/src/services/renderer/nodes/visual-node.ts +++ b/apps/web/src/services/renderer/nodes/visual-node.ts @@ -12,7 +12,6 @@ import { resolveTransformAtTime, } from "@/lib/animation"; import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; import { masksRegistry } from "@/lib/masks"; import { getSourceTimeAtClipTime } from "@/lib/retime"; @@ -58,7 +57,7 @@ export abstract class VisualNode< protected isInRange({ time }: { time: number }): boolean { const localTime = time - this.params.timeOffset; return ( - localTime >= -TIME_EPSILON_SECONDS && + localTime >= 0 && localTime < this.params.duration ); } diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts index 2dbb7cdd..0c2ee17f 100644 --- a/apps/web/src/services/renderer/scene-exporter.ts +++ b/apps/web/src/services/renderer/scene-exporter.ts @@ -1,163 +1,170 @@ -import EventEmitter from "eventemitter3"; - -import { - Output, - Mp4OutputFormat, - WebMOutputFormat, - BufferTarget, - CanvasSource, - AudioBufferSource, - QUALITY_LOW, - QUALITY_MEDIUM, - QUALITY_HIGH, - QUALITY_VERY_HIGH, -} from "mediabunny"; -import type { RootNode } from "./nodes/root-node"; -import type { ExportFormat, ExportQuality } from "@/lib/export"; -import { CanvasRenderer } from "./canvas-renderer"; - -type ExportParams = { - width: number; - height: number; - fps: number; - format: ExportFormat; - quality: ExportQuality; - shouldIncludeAudio?: boolean; - audioBuffer?: AudioBuffer; -}; - -const qualityMap = { - low: QUALITY_LOW, - medium: QUALITY_MEDIUM, - high: QUALITY_HIGH, - very_high: QUALITY_VERY_HIGH, -}; - -export type SceneExporterEvents = { - progress: [progress: number]; - complete: [buffer: ArrayBuffer]; - error: [error: Error]; - cancelled: []; -}; - -export class SceneExporter extends EventEmitter { - private renderer: CanvasRenderer; - private format: ExportFormat; - private quality: ExportQuality; - private shouldIncludeAudio: boolean; - private audioBuffer?: AudioBuffer; - - private isCancelled = false; - - constructor({ - width, - height, - fps, - format, - quality, - shouldIncludeAudio, - audioBuffer, - }: ExportParams) { - super(); - this.renderer = new CanvasRenderer({ - width, - height, - fps, - }); - - this.format = format; - this.quality = quality; - this.shouldIncludeAudio = shouldIncludeAudio ?? false; - this.audioBuffer = audioBuffer; - } - - cancel(): void { - this.isCancelled = true; - } - - async export({ - rootNode, - }: { - rootNode: RootNode; - }): Promise { - const { fps } = this.renderer; - const frameCount = Math.ceil(rootNode.duration * fps); - - const outputFormat = - this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); - - const output = new Output({ - format: outputFormat, - target: new BufferTarget(), - }); - - const videoSource = new CanvasSource(this.renderer.canvas, { - codec: this.format === "webm" ? "vp9" : "avc", - bitrate: qualityMap[this.quality], - }); - - output.addVideoTrack(videoSource, { frameRate: fps }); - - let audioSource: AudioBufferSource | null = null; - if (this.shouldIncludeAudio && this.audioBuffer) { - let audioCodec: "aac" | "opus" = - this.format === "webm" ? "opus" : "aac"; - - if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") { - const { supported } = await AudioEncoder.isConfigSupported({ - codec: "mp4a.40.2", - sampleRate: this.audioBuffer.sampleRate, - numberOfChannels: this.audioBuffer.numberOfChannels, - bitrate: 192000, - }); - if (!supported) audioCodec = "opus"; - } - - audioSource = new AudioBufferSource({ - codec: audioCodec, - bitrate: qualityMap[this.quality], - }); - output.addAudioTrack(audioSource); - } - - await output.start(); - - if (audioSource && this.audioBuffer) { - await audioSource.add(this.audioBuffer); - audioSource.close(); - } - - for (let i = 0; i < frameCount; i++) { - if (this.isCancelled) { - await output.cancel(); - this.emit("cancelled"); - return null; - } - - const time = i / fps; - await this.renderer.render({ node: rootNode, time }); - await videoSource.add(time, 1 / fps); - - this.emit("progress", i / frameCount); - } - - if (this.isCancelled) { - await output.cancel(); - this.emit("cancelled"); - return null; - } - - videoSource.close(); - await output.finalize(); - this.emit("progress", 1); - - const buffer = output.target.buffer; - if (!buffer) { - this.emit("error", new Error("Failed to export video")); - return null; - } - - this.emit("complete", buffer); - return buffer; - } -} +import EventEmitter from "eventemitter3"; + +import { + Output, + Mp4OutputFormat, + WebMOutputFormat, + BufferTarget, + CanvasSource, + AudioBufferSource, + QUALITY_LOW, + QUALITY_MEDIUM, + QUALITY_HIGH, + QUALITY_VERY_HIGH, +} from "mediabunny"; +import type { FrameRate } from "opencut-wasm"; +import { mediaTimeToSeconds } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; +import type { RootNode } from "./nodes/root-node"; +import type { ExportFormat, ExportQuality } from "@/lib/export"; +import { CanvasRenderer } from "./canvas-renderer"; + +type ExportParams = { + width: number; + height: number; + fps: FrameRate; + format: ExportFormat; + quality: ExportQuality; + shouldIncludeAudio?: boolean; + audioBuffer?: AudioBuffer; +}; + +const qualityMap = { + low: QUALITY_LOW, + medium: QUALITY_MEDIUM, + high: QUALITY_HIGH, + very_high: QUALITY_VERY_HIGH, +}; + +export type SceneExporterEvents = { + progress: [progress: number]; + complete: [buffer: ArrayBuffer]; + error: [error: Error]; + cancelled: []; +}; + +export class SceneExporter extends EventEmitter { + private renderer: CanvasRenderer; + private format: ExportFormat; + private quality: ExportQuality; + private shouldIncludeAudio: boolean; + private audioBuffer?: AudioBuffer; + + private isCancelled = false; + + constructor({ + width, + height, + fps, + format, + quality, + shouldIncludeAudio, + audioBuffer, + }: ExportParams) { + super(); + this.renderer = new CanvasRenderer({ + width, + height, + fps, + }); + + this.format = format; + this.quality = quality; + this.shouldIncludeAudio = shouldIncludeAudio ?? false; + this.audioBuffer = audioBuffer; + } + + cancel(): void { + this.isCancelled = true; + } + + async export({ + rootNode, + }: { + rootNode: RootNode; + }): Promise { + const fps = this.renderer.fps; + const fpsFloat = frameRateToFloat(fps); + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); + const frameCount = Math.floor(rootNode.duration / ticksPerFrame); + + const outputFormat = + this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); + + const output = new Output({ + format: outputFormat, + target: new BufferTarget(), + }); + + const videoSource = new CanvasSource(this.renderer.canvas, { + codec: this.format === "webm" ? "vp9" : "avc", + bitrate: qualityMap[this.quality], + }); + + output.addVideoTrack(videoSource, { frameRate: fpsFloat }); + + let audioSource: AudioBufferSource | null = null; + if (this.shouldIncludeAudio && this.audioBuffer) { + let audioCodec: "aac" | "opus" = + this.format === "webm" ? "opus" : "aac"; + + if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") { + const { supported } = await AudioEncoder.isConfigSupported({ + codec: "mp4a.40.2", + sampleRate: this.audioBuffer.sampleRate, + numberOfChannels: this.audioBuffer.numberOfChannels, + bitrate: 192000, + }); + if (!supported) audioCodec = "opus"; + } + + audioSource = new AudioBufferSource({ + codec: audioCodec, + bitrate: qualityMap[this.quality], + }); + output.addAudioTrack(audioSource); + } + + await output.start(); + + if (audioSource && this.audioBuffer) { + await audioSource.add(this.audioBuffer); + audioSource.close(); + } + + for (let i = 0; i < frameCount; i++) { + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + const timeTicks = i * ticksPerFrame; + const timeSeconds = mediaTimeToSeconds({ time: timeTicks }); + await this.renderer.render({ node: rootNode, time: timeTicks }); + await videoSource.add(timeSeconds, 1 / fpsFloat); + + this.emit("progress", i / frameCount); + } + + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + videoSource.close(); + await output.finalize(); + this.emit("progress", 1); + + const buffer = output.target.buffer; + if (!buffer) { + this.emit("error", new Error("Failed to export video")); + return null; + } + + this.emit("complete", buffer); + return buffer; + } +} -- cgit v1.3.1