From 4d127a038bf768eded68aae3cfa5c1b12241bdac Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Fri, 27 Mar 2026 00:55:15 +0100 Subject: chore: normalize line endings --- apps/web/src/services/renderer/canvas-utils.ts | 32 +- apps/web/src/services/renderer/mask-feather.ts | 108 ++-- .../services/renderer/nodes/effect-layer-node.ts | 162 +++--- .../src/services/renderer/nodes/graphic-node.ts | 184 +++---- apps/web/src/services/renderer/nodes/video-node.ts | 74 +-- apps/web/src/services/renderer/scene-builder.ts | 544 +++++++++---------- apps/web/src/services/renderer/scene-exporter.ts | 326 +++++------ apps/web/src/services/renderer/webgl/jfa.ts | 388 +++++++------- .../src/services/renderer/webgl/webgl-context.ts | 98 ++-- .../renderer/webgl/webgl-effect-renderer.ts | 70 +-- .../web/src/services/renderer/webgl/webgl-utils.ts | 596 ++++++++++----------- 11 files changed, 1291 insertions(+), 1291 deletions(-) (limited to 'apps/web/src/services/renderer') diff --git a/apps/web/src/services/renderer/canvas-utils.ts b/apps/web/src/services/renderer/canvas-utils.ts index 093168e6..ea04037b 100644 --- a/apps/web/src/services/renderer/canvas-utils.ts +++ b/apps/web/src/services/renderer/canvas-utils.ts @@ -1,16 +1,16 @@ -export function createOffscreenCanvas({ - width, - height, -}: { - width: number; - height: number; -}): OffscreenCanvas | HTMLCanvasElement { - try { - return new OffscreenCanvas(width, height); - } catch { - const canvas = document.createElement("canvas"); - canvas.width = width; - canvas.height = height; - return canvas; - } -} +export function createOffscreenCanvas({ + width, + height, +}: { + width: number; + height: number; +}): OffscreenCanvas | HTMLCanvasElement { + try { + return new OffscreenCanvas(width, height); + } catch { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + return canvas; + } +} diff --git a/apps/web/src/services/renderer/mask-feather.ts b/apps/web/src/services/renderer/mask-feather.ts index 39ad80a8..f1515117 100644 --- a/apps/web/src/services/renderer/mask-feather.ts +++ b/apps/web/src/services/renderer/mask-feather.ts @@ -1,54 +1,54 @@ -import jfaDistanceShader from "@/lib/masks/shaders/jfa-distance.frag.glsl"; -import { getWebGLContext, readResult } from "./webgl/webgl-context"; -import { computeSignedDistanceField, runPass } from "./webgl/jfa"; -import { compileProgram, createTexture } from "./webgl/webgl-utils"; - -export function applyMaskFeather({ - maskCanvas, - width, - height, - feather, -}: { - maskCanvas: CanvasImageSource; - width: number; - height: number; - feather: number; -}): OffscreenCanvas | HTMLCanvasElement { - const { context, programCache } = getWebGLContext({ width, height }); - const sourceTexture = createTexture({ context, source: maskCanvas }); - - const sdf = computeSignedDistanceField({ - context, - programCache, - sourceTexture, - width, - height, - }); - - const distanceProgram = compileProgram({ - context, - fragmentShaderSource: jfaDistanceShader, - programCache, - }); - - runPass({ - context, - program: distanceProgram, - inputTexture: sdf.insideTexture, - target: null, - width, - height, - uniforms: { u_feather_half: feather / 2.0 }, - extraBindings: [ - { unit: 1, texture: sdf.outsideTexture, name: "u_jfa_outside" }, - ], - }); - - context.deleteTexture(sourceTexture); - sdf.cleanup(); - - context.bindTexture(context.TEXTURE_2D, null); - context.bindFramebuffer(context.FRAMEBUFFER, null); - - return readResult({ width, height }); -} +import jfaDistanceShader from "@/lib/masks/shaders/jfa-distance.frag.glsl"; +import { getWebGLContext, readResult } from "./webgl/webgl-context"; +import { computeSignedDistanceField, runPass } from "./webgl/jfa"; +import { compileProgram, createTexture } from "./webgl/webgl-utils"; + +export function applyMaskFeather({ + maskCanvas, + width, + height, + feather, +}: { + maskCanvas: CanvasImageSource; + width: number; + height: number; + feather: number; +}): OffscreenCanvas | HTMLCanvasElement { + const { context, programCache } = getWebGLContext({ width, height }); + const sourceTexture = createTexture({ context, source: maskCanvas }); + + const sdf = computeSignedDistanceField({ + context, + programCache, + sourceTexture, + width, + height, + }); + + const distanceProgram = compileProgram({ + context, + fragmentShaderSource: jfaDistanceShader, + programCache, + }); + + runPass({ + context, + program: distanceProgram, + inputTexture: sdf.insideTexture, + target: null, + width, + height, + uniforms: { u_feather_half: feather / 2.0 }, + extraBindings: [ + { unit: 1, texture: sdf.outsideTexture, name: "u_jfa_outside" }, + ], + }); + + context.deleteTexture(sourceTexture); + sdf.cleanup(); + + context.bindTexture(context.TEXTURE_2D, null); + context.bindFramebuffer(context.FRAMEBUFFER, null); + + return readResult({ width, height }); +} diff --git a/apps/web/src/services/renderer/nodes/effect-layer-node.ts b/apps/web/src/services/renderer/nodes/effect-layer-node.ts index 82aee1b3..51de2398 100644 --- a/apps/web/src/services/renderer/nodes/effect-layer-node.ts +++ b/apps/web/src/services/renderer/nodes/effect-layer-node.ts @@ -1,81 +1,81 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; -import type { ParamValues } from "@/lib/params"; -import { BaseNode } from "./base-node"; -import { webglEffectRenderer } from "../webgl/webgl-effect-renderer"; - -const TIME_EPSILON = 1e-6; - -export type EffectLayerNodeParams = { - effectType: string; - effectParams: ParamValues; - timeOffset: number; - duration: number; -}; - -function isInRange({ - time, - timeOffset, - duration, -}: { - time: number; - timeOffset: number; - duration: number; -}): boolean { - return ( - time >= timeOffset - TIME_EPSILON && - time < timeOffset + duration + TIME_EPSILON - ); -} - -// snapshots whatever is currently on the canvas, applies the effect, draws it back -export class EffectLayerNode extends BaseNode { - async render({ - renderer, - time, - }: { - renderer: CanvasRenderer; - time: number; - }): Promise { - if ( - !isInRange({ - time, - timeOffset: this.params.timeOffset, - duration: this.params.duration, - }) - ) { - return; - } - - const source = renderer.context.canvas as CanvasImageSource; - - const effectDefinition = effectsRegistry.get(this.params.effectType); - - const passes = resolveEffectPasses({ - definition: effectDefinition, - effectParams: this.params.effectParams, - width: renderer.width, - height: renderer.height, - }); - if (passes.length === 0) { - return; - } - const effectResult = webglEffectRenderer.applyEffect({ - source, - width: renderer.width, - height: renderer.height, - passes, - }); - - renderer.context.save(); - renderer.context.clearRect(0, 0, renderer.width, renderer.height); - renderer.context.drawImage( - effectResult, - 0, - 0, - renderer.width, - renderer.height, - ); - renderer.context.restore(); - } -} +import type { CanvasRenderer } from "../canvas-renderer"; +import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; +import type { ParamValues } from "@/lib/params"; +import { BaseNode } from "./base-node"; +import { webglEffectRenderer } from "../webgl/webgl-effect-renderer"; + +const TIME_EPSILON = 1e-6; + +export type EffectLayerNodeParams = { + effectType: string; + effectParams: ParamValues; + timeOffset: number; + duration: number; +}; + +function isInRange({ + time, + timeOffset, + duration, +}: { + time: number; + timeOffset: number; + duration: number; +}): boolean { + return ( + time >= timeOffset - TIME_EPSILON && + time < timeOffset + duration + TIME_EPSILON + ); +} + +// snapshots whatever is currently on the canvas, applies the effect, draws it back +export class EffectLayerNode extends BaseNode { + async render({ + renderer, + time, + }: { + renderer: CanvasRenderer; + time: number; + }): Promise { + if ( + !isInRange({ + time, + timeOffset: this.params.timeOffset, + duration: this.params.duration, + }) + ) { + return; + } + + const source = renderer.context.canvas as CanvasImageSource; + + const effectDefinition = effectsRegistry.get(this.params.effectType); + + const passes = resolveEffectPasses({ + definition: effectDefinition, + effectParams: this.params.effectParams, + width: renderer.width, + height: renderer.height, + }); + if (passes.length === 0) { + return; + } + const effectResult = webglEffectRenderer.applyEffect({ + source, + width: renderer.width, + height: renderer.height, + passes, + }); + + renderer.context.save(); + renderer.context.clearRect(0, 0, renderer.width, renderer.height); + renderer.context.drawImage( + effectResult, + 0, + 0, + renderer.width, + renderer.height, + ); + renderer.context.restore(); + } +} diff --git a/apps/web/src/services/renderer/nodes/graphic-node.ts b/apps/web/src/services/renderer/nodes/graphic-node.ts index b258a353..0a3dc408 100644 --- a/apps/web/src/services/renderer/nodes/graphic-node.ts +++ b/apps/web/src/services/renderer/nodes/graphic-node.ts @@ -1,92 +1,92 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { createOffscreenCanvas } from "../canvas-utils"; -import { - DEFAULT_GRAPHIC_SOURCE_SIZE, - getGraphicDefinition, - registerDefaultGraphics, -} from "@/lib/graphics"; -import { resolveGraphicParamsAtTime } from "@/lib/animation"; -import type { ParamValues } from "@/lib/params"; -import { VisualNode, type VisualNodeParams } from "./visual-node"; - -export interface GraphicNodeParams extends VisualNodeParams { - definitionId: string; - params: ParamValues; -} - -export class GraphicNode extends VisualNode { - private cachedKey: string | null = null; - private cachedSource: OffscreenCanvas | HTMLCanvasElement | null = null; - - constructor(params: GraphicNodeParams) { - super(params); - registerDefaultGraphics(); - } - - private getSource({ - localTime, - }: { - localTime: number; - }): OffscreenCanvas | HTMLCanvasElement | null { - const definition = getGraphicDefinition({ - definitionId: this.params.definitionId, - }); - const resolvedParams = resolveGraphicParamsAtTime({ - element: this.params, - localTime, - }); - const cacheKey = JSON.stringify({ - definitionId: this.params.definitionId, - params: resolvedParams, - }); - if (this.cachedSource && this.cachedKey === cacheKey) { - return this.cachedSource; - } - - const canvas = createOffscreenCanvas({ - width: DEFAULT_GRAPHIC_SOURCE_SIZE, - height: DEFAULT_GRAPHIC_SOURCE_SIZE, - }); - const ctx = canvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!ctx) { - return null; - } - - definition.render({ - ctx, - params: resolvedParams, - width: DEFAULT_GRAPHIC_SOURCE_SIZE, - height: DEFAULT_GRAPHIC_SOURCE_SIZE, - }); - - this.cachedKey = cacheKey; - this.cachedSource = canvas; - return canvas; - } - - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const source = this.getSource({ - localTime: this.getAnimationLocalTime({ time }), - }); - if (!source) { - return; - } - - this.renderVisual({ - renderer, - source, - sourceWidth: DEFAULT_GRAPHIC_SOURCE_SIZE, - sourceHeight: DEFAULT_GRAPHIC_SOURCE_SIZE, - timelineTime: time, - }); - } -} +import type { CanvasRenderer } from "../canvas-renderer"; +import { createOffscreenCanvas } from "../canvas-utils"; +import { + DEFAULT_GRAPHIC_SOURCE_SIZE, + getGraphicDefinition, + registerDefaultGraphics, +} from "@/lib/graphics"; +import { resolveGraphicParamsAtTime } from "@/lib/animation"; +import type { ParamValues } from "@/lib/params"; +import { VisualNode, type VisualNodeParams } from "./visual-node"; + +export interface GraphicNodeParams extends VisualNodeParams { + definitionId: string; + params: ParamValues; +} + +export class GraphicNode extends VisualNode { + private cachedKey: string | null = null; + private cachedSource: OffscreenCanvas | HTMLCanvasElement | null = null; + + constructor(params: GraphicNodeParams) { + super(params); + registerDefaultGraphics(); + } + + private getSource({ + localTime, + }: { + localTime: number; + }): OffscreenCanvas | HTMLCanvasElement | null { + const definition = getGraphicDefinition({ + definitionId: this.params.definitionId, + }); + const resolvedParams = resolveGraphicParamsAtTime({ + element: this.params, + localTime, + }); + const cacheKey = JSON.stringify({ + definitionId: this.params.definitionId, + params: resolvedParams, + }); + if (this.cachedSource && this.cachedKey === cacheKey) { + return this.cachedSource; + } + + const canvas = createOffscreenCanvas({ + width: DEFAULT_GRAPHIC_SOURCE_SIZE, + height: DEFAULT_GRAPHIC_SOURCE_SIZE, + }); + const ctx = canvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!ctx) { + return null; + } + + definition.render({ + ctx, + params: resolvedParams, + width: DEFAULT_GRAPHIC_SOURCE_SIZE, + height: DEFAULT_GRAPHIC_SOURCE_SIZE, + }); + + this.cachedKey = cacheKey; + this.cachedSource = canvas; + return canvas; + } + + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + await super.render({ renderer, time }); + + if (!this.isInRange({ time })) { + return; + } + + const source = this.getSource({ + localTime: this.getAnimationLocalTime({ time }), + }); + if (!source) { + return; + } + + this.renderVisual({ + renderer, + source, + sourceWidth: DEFAULT_GRAPHIC_SOURCE_SIZE, + sourceHeight: DEFAULT_GRAPHIC_SOURCE_SIZE, + timelineTime: time, + }); + } +} diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts index af39d26f..3bb836ec 100644 --- a/apps/web/src/services/renderer/nodes/video-node.ts +++ b/apps/web/src/services/renderer/nodes/video-node.ts @@ -1,37 +1,37 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { VisualNode, type VisualNodeParams } from "./visual-node"; -import { videoCache } from "@/services/video-cache/service"; - -export interface VideoNodeParams extends VisualNodeParams { - url: string; - file: File; - mediaId: string; -} - -export class VideoNode extends VisualNode { - async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { - await super.render({ renderer, time }); - - if (!this.isInRange({ time })) { - return; - } - - const videoTime = this.getSourceLocalTime({ time }); - - const frame = await videoCache.getFrameAt({ - mediaId: this.params.mediaId, - file: this.params.file, - time: videoTime, - }); - - if (frame) { - this.renderVisual({ - renderer, - source: frame.canvas, - sourceWidth: frame.canvas.width, - sourceHeight: frame.canvas.height, - timelineTime: time, - }); - } - } -} +import type { CanvasRenderer } from "../canvas-renderer"; +import { VisualNode, type VisualNodeParams } from "./visual-node"; +import { videoCache } from "@/services/video-cache/service"; + +export interface VideoNodeParams extends VisualNodeParams { + url: string; + file: File; + mediaId: string; +} + +export class VideoNode extends VisualNode { + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + await super.render({ renderer, time }); + + if (!this.isInRange({ time })) { + return; + } + + const videoTime = this.getSourceLocalTime({ time }); + + const frame = await videoCache.getFrameAt({ + mediaId: this.params.mediaId, + file: this.params.file, + time: videoTime, + }); + + if (frame) { + this.renderVisual({ + renderer, + source: frame.canvas, + sourceWidth: frame.canvas.width, + sourceHeight: frame.canvas.height, + timelineTime: time, + }); + } + } +} diff --git a/apps/web/src/services/renderer/scene-builder.ts b/apps/web/src/services/renderer/scene-builder.ts index 7d5f9fe9..1c3e6ad2 100644 --- a/apps/web/src/services/renderer/scene-builder.ts +++ b/apps/web/src/services/renderer/scene-builder.ts @@ -1,272 +1,272 @@ -import type { TimelineTrack } from "@/lib/timeline"; -import type { MediaAsset } from "@/lib/media/types"; -import { RootNode } from "./nodes/root-node"; -import { VideoNode } from "./nodes/video-node"; -import { ImageNode } from "./nodes/image-node"; -import { TextNode } from "./nodes/text-node"; -import { StickerNode } from "./nodes/sticker-node"; -import { GraphicNode } from "./nodes/graphic-node"; -import { ColorNode } from "./nodes/color-node"; -import { BlurBackgroundNode } from "./nodes/blur-background-node"; -import { EffectLayerNode } from "./nodes/effect-layer-node"; -import type { BaseNode } from "./nodes/base-node"; -import type { TBackground, TCanvasSize } from "@/lib/project/types"; -import { DEFAULT_BLUR_INTENSITY } from "@/constants/project-constants"; -import { isMainTrack } from "@/lib/timeline"; - -const PREVIEW_MAX_IMAGE_SIZE = 2048; - -function getVisibleSortedElements({ track }: { track: TimelineTrack }) { - return track.elements - .filter((element) => !("hidden" in element && element.hidden)) - .slice() - .sort((a, b) => { - if (a.startTime !== b.startTime) return a.startTime - b.startTime; - return a.id.localeCompare(b.id); - }); -} - -function buildTrackNodes({ - tracks, - mediaMap, - canvasSize, - isPreview, -}: { - tracks: TimelineTrack[]; - mediaMap: Map; - canvasSize: TCanvasSize; - isPreview?: boolean; -}): BaseNode[] { - const nodes: BaseNode[] = []; - - for (const track of tracks) { - const elements = getVisibleSortedElements({ track }); - - for (const element of elements) { - if (element.type === "effect") { - nodes.push( - new EffectLayerNode({ - effectType: element.effectType, - effectParams: element.params, - timeOffset: element.startTime, - duration: element.duration, - }), - ); - continue; - } - - if (element.type === "video" || element.type === "image") { - const mediaAsset = mediaMap.get(element.mediaId); - if (!mediaAsset?.file || !mediaAsset?.url) { - continue; - } - - if (element.type === "video" && mediaAsset.type === "video") { - nodes.push( - new VideoNode({ - mediaId: mediaAsset.id, - url: mediaAsset.url, - file: mediaAsset.file, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - retime: element.retime, - transform: element.transform, - animations: element.animations, - opacity: element.opacity, - blendMode: element.blendMode, - effects: element.effects, - masks: element.masks, - }), - ); - } - if (element.type === "image" && mediaAsset.type === "image") { - nodes.push( - new ImageNode({ - url: mediaAsset.url, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - transform: element.transform, - animations: element.animations, - opacity: element.opacity, - blendMode: element.blendMode, - effects: element.effects, - masks: element.masks, - ...(isPreview && { - maxSourceSize: PREVIEW_MAX_IMAGE_SIZE, - }), - }), - ); - } - } - - if (element.type === "text") { - nodes.push( - new TextNode({ - ...element, - canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 }, - canvasHeight: canvasSize.height, - textBaseline: "middle", - effects: element.effects, - }), - ); - } - - if (element.type === "sticker") { - nodes.push( - new StickerNode({ - stickerId: element.stickerId, - intrinsicWidth: element.intrinsicWidth, - intrinsicHeight: element.intrinsicHeight, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - transform: element.transform, - animations: element.animations, - opacity: element.opacity, - blendMode: element.blendMode, - effects: element.effects, - }), - ); - } - - if (element.type === "graphic") { - nodes.push( - new GraphicNode({ - definitionId: element.definitionId, - params: element.params, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - transform: element.transform, - animations: element.animations, - opacity: element.opacity, - blendMode: element.blendMode, - effects: element.effects, - masks: element.masks, - }), - ); - } - } - } - - return nodes; -} - -function buildBlurBackgroundNodes({ - track, - mediaMap, - blurIntensity, -}: { - track: TimelineTrack | undefined; - mediaMap: Map; - blurIntensity: number; -}): BaseNode[] { - if (!track) { - return []; - } - - const nodes: BaseNode[] = []; - const elements = getVisibleSortedElements({ track }); - - for (const element of elements) { - if (element.type !== "video" && element.type !== "image") { - continue; - } - - const mediaAsset = mediaMap.get(element.mediaId); - if ( - !mediaAsset?.file || - !mediaAsset?.url || - (mediaAsset.type !== "video" && mediaAsset.type !== "image") - ) { - continue; - } - - nodes.push( - new BlurBackgroundNode({ - mediaId: mediaAsset.id, - url: mediaAsset.url, - file: mediaAsset.file, - mediaType: mediaAsset.type, - duration: element.duration, - timeOffset: element.startTime, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - retime: element.type === "video" ? element.retime : undefined, - blurIntensity, - }), - ); - } - - return nodes; -} - -export type BuildSceneParams = { - canvasSize: TCanvasSize; - tracks: TimelineTrack[]; - mediaAssets: MediaAsset[]; - duration: number; - background: TBackground; - isPreview?: boolean; -}; - -export function buildScene({ - canvasSize, - tracks, - mediaAssets, - duration, - background, - isPreview, -}: BuildSceneParams) { - const rootNode = new RootNode({ duration }); - const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); - - const visibleTracks = tracks.filter( - (track) => !("hidden" in track && track.hidden), - ); - - const orderedTracksTopToBottom = [ - ...visibleTracks.filter((track) => !isMainTrack(track)), - ...visibleTracks.filter((track) => isMainTrack(track)), - ]; - - const orderedTracksBottomToTop = orderedTracksTopToBottom.slice().reverse(); - const mainTrack = orderedTracksBottomToTop.find((track) => - isMainTrack(track), - ); - - const allNodes = buildTrackNodes({ - tracks: orderedTracksBottomToTop, - mediaMap, - canvasSize, - isPreview, - }); - - if (background.type === "blur") { - const blurNodes = buildBlurBackgroundNodes({ - track: mainTrack, - mediaMap, - blurIntensity: background.blurIntensity ?? DEFAULT_BLUR_INTENSITY, - }); - for (const node of blurNodes) { - rootNode.add(node); - } - } else if ( - background.type === "color" && - background.color !== "transparent" - ) { - rootNode.add(new ColorNode({ color: background.color })); - } - - for (const node of allNodes) { - rootNode.add(node); - } - - return rootNode; -} +import type { TimelineTrack } from "@/lib/timeline"; +import type { MediaAsset } from "@/lib/media/types"; +import { RootNode } from "./nodes/root-node"; +import { VideoNode } from "./nodes/video-node"; +import { ImageNode } from "./nodes/image-node"; +import { TextNode } from "./nodes/text-node"; +import { StickerNode } from "./nodes/sticker-node"; +import { GraphicNode } from "./nodes/graphic-node"; +import { ColorNode } from "./nodes/color-node"; +import { BlurBackgroundNode } from "./nodes/blur-background-node"; +import { EffectLayerNode } from "./nodes/effect-layer-node"; +import type { BaseNode } from "./nodes/base-node"; +import type { TBackground, TCanvasSize } from "@/lib/project/types"; +import { DEFAULT_BLUR_INTENSITY } from "@/constants/project-constants"; +import { isMainTrack } from "@/lib/timeline"; + +const PREVIEW_MAX_IMAGE_SIZE = 2048; + +function getVisibleSortedElements({ track }: { track: TimelineTrack }) { + return track.elements + .filter((element) => !("hidden" in element && element.hidden)) + .slice() + .sort((a, b) => { + if (a.startTime !== b.startTime) return a.startTime - b.startTime; + return a.id.localeCompare(b.id); + }); +} + +function buildTrackNodes({ + tracks, + mediaMap, + canvasSize, + isPreview, +}: { + tracks: TimelineTrack[]; + mediaMap: Map; + canvasSize: TCanvasSize; + isPreview?: boolean; +}): BaseNode[] { + const nodes: BaseNode[] = []; + + for (const track of tracks) { + const elements = getVisibleSortedElements({ track }); + + for (const element of elements) { + if (element.type === "effect") { + nodes.push( + new EffectLayerNode({ + effectType: element.effectType, + effectParams: element.params, + timeOffset: element.startTime, + duration: element.duration, + }), + ); + continue; + } + + if (element.type === "video" || element.type === "image") { + const mediaAsset = mediaMap.get(element.mediaId); + if (!mediaAsset?.file || !mediaAsset?.url) { + continue; + } + + if (element.type === "video" && mediaAsset.type === "video") { + nodes.push( + new VideoNode({ + mediaId: mediaAsset.id, + url: mediaAsset.url, + file: mediaAsset.file, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + retime: element.retime, + transform: element.transform, + animations: element.animations, + opacity: element.opacity, + blendMode: element.blendMode, + effects: element.effects, + masks: element.masks, + }), + ); + } + if (element.type === "image" && mediaAsset.type === "image") { + nodes.push( + new ImageNode({ + url: mediaAsset.url, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + transform: element.transform, + animations: element.animations, + opacity: element.opacity, + blendMode: element.blendMode, + effects: element.effects, + masks: element.masks, + ...(isPreview && { + maxSourceSize: PREVIEW_MAX_IMAGE_SIZE, + }), + }), + ); + } + } + + if (element.type === "text") { + nodes.push( + new TextNode({ + ...element, + canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 }, + canvasHeight: canvasSize.height, + textBaseline: "middle", + effects: element.effects, + }), + ); + } + + if (element.type === "sticker") { + nodes.push( + new StickerNode({ + stickerId: element.stickerId, + intrinsicWidth: element.intrinsicWidth, + intrinsicHeight: element.intrinsicHeight, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + transform: element.transform, + animations: element.animations, + opacity: element.opacity, + blendMode: element.blendMode, + effects: element.effects, + }), + ); + } + + if (element.type === "graphic") { + nodes.push( + new GraphicNode({ + definitionId: element.definitionId, + params: element.params, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + transform: element.transform, + animations: element.animations, + opacity: element.opacity, + blendMode: element.blendMode, + effects: element.effects, + masks: element.masks, + }), + ); + } + } + } + + return nodes; +} + +function buildBlurBackgroundNodes({ + track, + mediaMap, + blurIntensity, +}: { + track: TimelineTrack | undefined; + mediaMap: Map; + blurIntensity: number; +}): BaseNode[] { + if (!track) { + return []; + } + + const nodes: BaseNode[] = []; + const elements = getVisibleSortedElements({ track }); + + for (const element of elements) { + if (element.type !== "video" && element.type !== "image") { + continue; + } + + const mediaAsset = mediaMap.get(element.mediaId); + if ( + !mediaAsset?.file || + !mediaAsset?.url || + (mediaAsset.type !== "video" && mediaAsset.type !== "image") + ) { + continue; + } + + nodes.push( + new BlurBackgroundNode({ + mediaId: mediaAsset.id, + url: mediaAsset.url, + file: mediaAsset.file, + mediaType: mediaAsset.type, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + retime: element.type === "video" ? element.retime : undefined, + blurIntensity, + }), + ); + } + + return nodes; +} + +export type BuildSceneParams = { + canvasSize: TCanvasSize; + tracks: TimelineTrack[]; + mediaAssets: MediaAsset[]; + duration: number; + background: TBackground; + isPreview?: boolean; +}; + +export function buildScene({ + canvasSize, + tracks, + mediaAssets, + duration, + background, + isPreview, +}: BuildSceneParams) { + const rootNode = new RootNode({ duration }); + const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); + + const visibleTracks = tracks.filter( + (track) => !("hidden" in track && track.hidden), + ); + + const orderedTracksTopToBottom = [ + ...visibleTracks.filter((track) => !isMainTrack(track)), + ...visibleTracks.filter((track) => isMainTrack(track)), + ]; + + const orderedTracksBottomToTop = orderedTracksTopToBottom.slice().reverse(); + const mainTrack = orderedTracksBottomToTop.find((track) => + isMainTrack(track), + ); + + const allNodes = buildTrackNodes({ + tracks: orderedTracksBottomToTop, + mediaMap, + canvasSize, + isPreview, + }); + + if (background.type === "blur") { + const blurNodes = buildBlurBackgroundNodes({ + track: mainTrack, + mediaMap, + blurIntensity: background.blurIntensity ?? DEFAULT_BLUR_INTENSITY, + }); + for (const node of blurNodes) { + rootNode.add(node); + } + } else if ( + background.type === "color" && + background.color !== "transparent" + ) { + rootNode.add(new ColorNode({ color: background.color })); + } + + for (const node of allNodes) { + rootNode.add(node); + } + + return rootNode; +} diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts index 6ab8c6f3..2dbb7cdd 100644 --- a/apps/web/src/services/renderer/scene-exporter.ts +++ b/apps/web/src/services/renderer/scene-exporter.ts @@ -1,163 +1,163 @@ -import EventEmitter from "eventemitter3"; - -import { - Output, - Mp4OutputFormat, - WebMOutputFormat, - BufferTarget, - CanvasSource, - AudioBufferSource, - QUALITY_LOW, - QUALITY_MEDIUM, - QUALITY_HIGH, - QUALITY_VERY_HIGH, -} from "mediabunny"; -import type { RootNode } from "./nodes/root-node"; -import type { ExportFormat, ExportQuality } from "@/lib/export"; -import { CanvasRenderer } from "./canvas-renderer"; - -type ExportParams = { - width: number; - height: number; - fps: number; - format: ExportFormat; - quality: ExportQuality; - shouldIncludeAudio?: boolean; - audioBuffer?: AudioBuffer; -}; - -const qualityMap = { - low: QUALITY_LOW, - medium: QUALITY_MEDIUM, - high: QUALITY_HIGH, - very_high: QUALITY_VERY_HIGH, -}; - -export type SceneExporterEvents = { - progress: [progress: number]; - complete: [buffer: ArrayBuffer]; - error: [error: Error]; - cancelled: []; -}; - -export class SceneExporter extends EventEmitter { - private renderer: CanvasRenderer; - private format: ExportFormat; - private quality: ExportQuality; - private shouldIncludeAudio: boolean; - private audioBuffer?: AudioBuffer; - - private isCancelled = false; - - constructor({ - width, - height, - fps, - format, - quality, - shouldIncludeAudio, - audioBuffer, - }: ExportParams) { - super(); - this.renderer = new CanvasRenderer({ - width, - height, - fps, - }); - - this.format = format; - this.quality = quality; - this.shouldIncludeAudio = shouldIncludeAudio ?? false; - this.audioBuffer = audioBuffer; - } - - cancel(): void { - this.isCancelled = true; - } - - async export({ - rootNode, - }: { - rootNode: RootNode; - }): Promise { - const { fps } = this.renderer; - const frameCount = Math.ceil(rootNode.duration * fps); - - const outputFormat = - this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); - - const output = new Output({ - format: outputFormat, - target: new BufferTarget(), - }); - - const videoSource = new CanvasSource(this.renderer.canvas, { - codec: this.format === "webm" ? "vp9" : "avc", - bitrate: qualityMap[this.quality], - }); - - output.addVideoTrack(videoSource, { frameRate: fps }); - - let audioSource: AudioBufferSource | null = null; - if (this.shouldIncludeAudio && this.audioBuffer) { - let audioCodec: "aac" | "opus" = - this.format === "webm" ? "opus" : "aac"; - - if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") { - const { supported } = await AudioEncoder.isConfigSupported({ - codec: "mp4a.40.2", - sampleRate: this.audioBuffer.sampleRate, - numberOfChannels: this.audioBuffer.numberOfChannels, - bitrate: 192000, - }); - if (!supported) audioCodec = "opus"; - } - - audioSource = new AudioBufferSource({ - codec: audioCodec, - bitrate: qualityMap[this.quality], - }); - output.addAudioTrack(audioSource); - } - - await output.start(); - - if (audioSource && this.audioBuffer) { - await audioSource.add(this.audioBuffer); - audioSource.close(); - } - - for (let i = 0; i < frameCount; i++) { - if (this.isCancelled) { - await output.cancel(); - this.emit("cancelled"); - return null; - } - - const time = i / fps; - await this.renderer.render({ node: rootNode, time }); - await videoSource.add(time, 1 / fps); - - this.emit("progress", i / frameCount); - } - - if (this.isCancelled) { - await output.cancel(); - this.emit("cancelled"); - return null; - } - - videoSource.close(); - await output.finalize(); - this.emit("progress", 1); - - const buffer = output.target.buffer; - if (!buffer) { - this.emit("error", new Error("Failed to export video")); - return null; - } - - this.emit("complete", buffer); - return buffer; - } -} +import EventEmitter from "eventemitter3"; + +import { + Output, + Mp4OutputFormat, + WebMOutputFormat, + BufferTarget, + CanvasSource, + AudioBufferSource, + QUALITY_LOW, + QUALITY_MEDIUM, + QUALITY_HIGH, + QUALITY_VERY_HIGH, +} from "mediabunny"; +import type { RootNode } from "./nodes/root-node"; +import type { ExportFormat, ExportQuality } from "@/lib/export"; +import { CanvasRenderer } from "./canvas-renderer"; + +type ExportParams = { + width: number; + height: number; + fps: number; + format: ExportFormat; + quality: ExportQuality; + shouldIncludeAudio?: boolean; + audioBuffer?: AudioBuffer; +}; + +const qualityMap = { + low: QUALITY_LOW, + medium: QUALITY_MEDIUM, + high: QUALITY_HIGH, + very_high: QUALITY_VERY_HIGH, +}; + +export type SceneExporterEvents = { + progress: [progress: number]; + complete: [buffer: ArrayBuffer]; + error: [error: Error]; + cancelled: []; +}; + +export class SceneExporter extends EventEmitter { + private renderer: CanvasRenderer; + private format: ExportFormat; + private quality: ExportQuality; + private shouldIncludeAudio: boolean; + private audioBuffer?: AudioBuffer; + + private isCancelled = false; + + constructor({ + width, + height, + fps, + format, + quality, + shouldIncludeAudio, + audioBuffer, + }: ExportParams) { + super(); + this.renderer = new CanvasRenderer({ + width, + height, + fps, + }); + + this.format = format; + this.quality = quality; + this.shouldIncludeAudio = shouldIncludeAudio ?? false; + this.audioBuffer = audioBuffer; + } + + cancel(): void { + this.isCancelled = true; + } + + async export({ + rootNode, + }: { + rootNode: RootNode; + }): Promise { + const { fps } = this.renderer; + const frameCount = Math.ceil(rootNode.duration * fps); + + const outputFormat = + this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); + + const output = new Output({ + format: outputFormat, + target: new BufferTarget(), + }); + + const videoSource = new CanvasSource(this.renderer.canvas, { + codec: this.format === "webm" ? "vp9" : "avc", + bitrate: qualityMap[this.quality], + }); + + output.addVideoTrack(videoSource, { frameRate: fps }); + + let audioSource: AudioBufferSource | null = null; + if (this.shouldIncludeAudio && this.audioBuffer) { + let audioCodec: "aac" | "opus" = + this.format === "webm" ? "opus" : "aac"; + + if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") { + const { supported } = await AudioEncoder.isConfigSupported({ + codec: "mp4a.40.2", + sampleRate: this.audioBuffer.sampleRate, + numberOfChannels: this.audioBuffer.numberOfChannels, + bitrate: 192000, + }); + if (!supported) audioCodec = "opus"; + } + + audioSource = new AudioBufferSource({ + codec: audioCodec, + bitrate: qualityMap[this.quality], + }); + output.addAudioTrack(audioSource); + } + + await output.start(); + + if (audioSource && this.audioBuffer) { + await audioSource.add(this.audioBuffer); + audioSource.close(); + } + + for (let i = 0; i < frameCount; i++) { + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + const time = i / fps; + await this.renderer.render({ node: rootNode, time }); + await videoSource.add(time, 1 / fps); + + this.emit("progress", i / frameCount); + } + + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + videoSource.close(); + await output.finalize(); + this.emit("progress", 1); + + const buffer = output.target.buffer; + if (!buffer) { + this.emit("error", new Error("Failed to export video")); + return null; + } + + this.emit("complete", buffer); + return buffer; + } +} diff --git a/apps/web/src/services/renderer/webgl/jfa.ts b/apps/web/src/services/renderer/webgl/jfa.ts index b69a9d06..9c87e7fa 100644 --- a/apps/web/src/services/renderer/webgl/jfa.ts +++ b/apps/web/src/services/renderer/webgl/jfa.ts @@ -1,194 +1,194 @@ -import jfaInitShader from "@/lib/shaders/jfa-init.frag.glsl"; -import jfaStepShader from "@/lib/shaders/jfa-step.frag.glsl"; -import { - compileProgram, - createFramebufferTexture, - setUniforms, - drawFullscreenQuad, -} from "./webgl-utils"; - -interface FBPair { - texture: WebGLTexture; - framebuffer: WebGLFramebuffer; -} - -function runPass({ - context, - program, - inputTexture, - target, - width, - height, - uniforms, - extraBindings, -}: { - context: WebGLRenderingContext; - program: WebGLProgram; - inputTexture: WebGLTexture; - target: WebGLFramebuffer | null; - width: number; - height: number; - uniforms: Record; - extraBindings?: Array<{ unit: number; texture: WebGLTexture; name: string }>; -}): void { - context.bindFramebuffer(context.FRAMEBUFFER, target); - // biome-ignore lint/correctness/useHookAtTopLevel: WebGL API method, not a React hook - context.useProgram(program); - - context.activeTexture(context.TEXTURE0); - context.bindTexture(context.TEXTURE_2D, inputTexture); - const uTexLoc = context.getUniformLocation(program, "u_texture"); - if (uTexLoc) context.uniform1i(uTexLoc, 0); - - if (extraBindings) { - for (const binding of extraBindings) { - context.activeTexture(context.TEXTURE0 + binding.unit); - context.bindTexture(context.TEXTURE_2D, binding.texture); - const loc = context.getUniformLocation(program, binding.name); - if (loc) context.uniform1i(loc, binding.unit); - } - } - - setUniforms({ - context, - program, - uniforms: { ...uniforms, u_resolution: [width, height] }, - }); - drawFullscreenQuad({ context, program, width, height }); -} - -function runJFA({ - context, - programCache, - sourceTexture, - width, - height, - isInverted, -}: { - context: WebGLRenderingContext; - programCache: Map; - sourceTexture: WebGLTexture; - width: number; - height: number; - isInverted: boolean; -}): { - resultTexture: WebGLTexture; - resultFB: WebGLFramebuffer; - tempFBs: FBPair[]; -} { - const numSteps = Math.ceil(Math.log2(Math.max(width, height))); - - const fbA = createFramebufferTexture({ context, width, height }); - const fbB = createFramebufferTexture({ context, width, height }); - - const initProgram = compileProgram({ - context, - fragmentShaderSource: jfaInitShader, - programCache, - }); - runPass({ - context, - program: initProgram, - inputTexture: sourceTexture, - target: fbA.framebuffer, - width, - height, - uniforms: { u_invert: isInverted ? 1.0 : 0.0 }, - }); - - const stepProgram = compileProgram({ - context, - fragmentShaderSource: jfaStepShader, - programCache, - }); - - let readFB = fbA; - let writeFB = fbB; - - for (let i = numSteps - 1; i >= 0; i--) { - const stepSize = 2 ** i; - runPass({ - context, - program: stepProgram, - inputTexture: readFB.texture, - target: writeFB.framebuffer, - width, - height, - uniforms: { u_step_size: stepSize }, - }); - const tmp = readFB; - readFB = writeFB; - writeFB = tmp; - } - - return { - resultTexture: readFB.texture, - resultFB: readFB.framebuffer, - tempFBs: [writeFB], - }; -} - -function cleanupJFAResult({ - context, - result, -}: { - context: WebGLRenderingContext; - result: { - resultTexture: WebGLTexture; - resultFB: WebGLFramebuffer; - tempFBs: FBPair[]; - }; -}): void { - context.deleteTexture(result.resultTexture); - context.deleteFramebuffer(result.resultFB); - for (const fb of result.tempFBs) { - context.deleteTexture(fb.texture); - context.deleteFramebuffer(fb.framebuffer); - } -} - -export { runPass }; - -export function computeSignedDistanceField({ - context, - programCache, - sourceTexture, - width, - height, -}: { - context: WebGLRenderingContext; - programCache: Map; - sourceTexture: WebGLTexture; - width: number; - height: number; -}): { - insideTexture: WebGLTexture; - outsideTexture: WebGLTexture; - cleanup: () => void; -} { - const inside = runJFA({ - context, - programCache, - sourceTexture, - width, - height, - isInverted: false, - }); - const outside = runJFA({ - context, - programCache, - sourceTexture, - width, - height, - isInverted: true, - }); - - return { - insideTexture: inside.resultTexture, - outsideTexture: outside.resultTexture, - cleanup: () => { - cleanupJFAResult({ context, result: inside }); - cleanupJFAResult({ context, result: outside }); - }, - }; -} +import jfaInitShader from "@/lib/shaders/jfa-init.frag.glsl"; +import jfaStepShader from "@/lib/shaders/jfa-step.frag.glsl"; +import { + compileProgram, + createFramebufferTexture, + setUniforms, + drawFullscreenQuad, +} from "./webgl-utils"; + +interface FBPair { + texture: WebGLTexture; + framebuffer: WebGLFramebuffer; +} + +function runPass({ + context, + program, + inputTexture, + target, + width, + height, + uniforms, + extraBindings, +}: { + context: WebGLRenderingContext; + program: WebGLProgram; + inputTexture: WebGLTexture; + target: WebGLFramebuffer | null; + width: number; + height: number; + uniforms: Record; + extraBindings?: Array<{ unit: number; texture: WebGLTexture; name: string }>; +}): void { + context.bindFramebuffer(context.FRAMEBUFFER, target); + // biome-ignore lint/correctness/useHookAtTopLevel: WebGL API method, not a React hook + context.useProgram(program); + + context.activeTexture(context.TEXTURE0); + context.bindTexture(context.TEXTURE_2D, inputTexture); + const uTexLoc = context.getUniformLocation(program, "u_texture"); + if (uTexLoc) context.uniform1i(uTexLoc, 0); + + if (extraBindings) { + for (const binding of extraBindings) { + context.activeTexture(context.TEXTURE0 + binding.unit); + context.bindTexture(context.TEXTURE_2D, binding.texture); + const loc = context.getUniformLocation(program, binding.name); + if (loc) context.uniform1i(loc, binding.unit); + } + } + + setUniforms({ + context, + program, + uniforms: { ...uniforms, u_resolution: [width, height] }, + }); + drawFullscreenQuad({ context, program, width, height }); +} + +function runJFA({ + context, + programCache, + sourceTexture, + width, + height, + isInverted, +}: { + context: WebGLRenderingContext; + programCache: Map; + sourceTexture: WebGLTexture; + width: number; + height: number; + isInverted: boolean; +}): { + resultTexture: WebGLTexture; + resultFB: WebGLFramebuffer; + tempFBs: FBPair[]; +} { + const numSteps = Math.ceil(Math.log2(Math.max(width, height))); + + const fbA = createFramebufferTexture({ context, width, height }); + const fbB = createFramebufferTexture({ context, width, height }); + + const initProgram = compileProgram({ + context, + fragmentShaderSource: jfaInitShader, + programCache, + }); + runPass({ + context, + program: initProgram, + inputTexture: sourceTexture, + target: fbA.framebuffer, + width, + height, + uniforms: { u_invert: isInverted ? 1.0 : 0.0 }, + }); + + const stepProgram = compileProgram({ + context, + fragmentShaderSource: jfaStepShader, + programCache, + }); + + let readFB = fbA; + let writeFB = fbB; + + for (let i = numSteps - 1; i >= 0; i--) { + const stepSize = 2 ** i; + runPass({ + context, + program: stepProgram, + inputTexture: readFB.texture, + target: writeFB.framebuffer, + width, + height, + uniforms: { u_step_size: stepSize }, + }); + const tmp = readFB; + readFB = writeFB; + writeFB = tmp; + } + + return { + resultTexture: readFB.texture, + resultFB: readFB.framebuffer, + tempFBs: [writeFB], + }; +} + +function cleanupJFAResult({ + context, + result, +}: { + context: WebGLRenderingContext; + result: { + resultTexture: WebGLTexture; + resultFB: WebGLFramebuffer; + tempFBs: FBPair[]; + }; +}): void { + context.deleteTexture(result.resultTexture); + context.deleteFramebuffer(result.resultFB); + for (const fb of result.tempFBs) { + context.deleteTexture(fb.texture); + context.deleteFramebuffer(fb.framebuffer); + } +} + +export { runPass }; + +export function computeSignedDistanceField({ + context, + programCache, + sourceTexture, + width, + height, +}: { + context: WebGLRenderingContext; + programCache: Map; + sourceTexture: WebGLTexture; + width: number; + height: number; +}): { + insideTexture: WebGLTexture; + outsideTexture: WebGLTexture; + cleanup: () => void; +} { + const inside = runJFA({ + context, + programCache, + sourceTexture, + width, + height, + isInverted: false, + }); + const outside = runJFA({ + context, + programCache, + sourceTexture, + width, + height, + isInverted: true, + }); + + return { + insideTexture: inside.resultTexture, + outsideTexture: outside.resultTexture, + cleanup: () => { + cleanupJFAResult({ context, result: inside }); + cleanupJFAResult({ context, result: outside }); + }, + }; +} diff --git a/apps/web/src/services/renderer/webgl/webgl-context.ts b/apps/web/src/services/renderer/webgl/webgl-context.ts index aeefc503..39869a02 100644 --- a/apps/web/src/services/renderer/webgl/webgl-context.ts +++ b/apps/web/src/services/renderer/webgl/webgl-context.ts @@ -1,49 +1,49 @@ -import { createOffscreenCanvas } from "../canvas-utils"; - -let gl: WebGLRenderingContext | null = null; -let webglCanvas: OffscreenCanvas | HTMLCanvasElement | null = null; -const programCache = new Map(); - -export function getWebGLContext({ - width, - height, -}: { - width: number; - height: number; -}): { - context: WebGLRenderingContext; - programCache: Map; -} { - if (!webglCanvas) { - webglCanvas = createOffscreenCanvas({ width, height }); - gl = webglCanvas.getContext("webgl", { - premultipliedAlpha: false, - }) as WebGLRenderingContext | null; - if (!gl) throw new Error("WebGL not supported"); - } - if (webglCanvas.width !== width || webglCanvas.height !== height) { - webglCanvas.width = width; - webglCanvas.height = height; - } - if (!gl) throw new Error("WebGL context lost"); - return { context: gl, programCache }; -} - -export function readResult({ - width, - height, -}: { - width: number; - height: number; -}): OffscreenCanvas | HTMLCanvasElement { - if (!webglCanvas) throw new Error("WebGL canvas not initialized"); - const outputCanvas = createOffscreenCanvas({ width, height }); - const outputCtx = outputCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (outputCtx) { - outputCtx.drawImage(webglCanvas, 0, 0, width, height); - } - return outputCanvas; -} +import { createOffscreenCanvas } from "../canvas-utils"; + +let gl: WebGLRenderingContext | null = null; +let webglCanvas: OffscreenCanvas | HTMLCanvasElement | null = null; +const programCache = new Map(); + +export function getWebGLContext({ + width, + height, +}: { + width: number; + height: number; +}): { + context: WebGLRenderingContext; + programCache: Map; +} { + if (!webglCanvas) { + webglCanvas = createOffscreenCanvas({ width, height }); + gl = webglCanvas.getContext("webgl", { + premultipliedAlpha: false, + }) as WebGLRenderingContext | null; + if (!gl) throw new Error("WebGL not supported"); + } + if (webglCanvas.width !== width || webglCanvas.height !== height) { + webglCanvas.width = width; + webglCanvas.height = height; + } + if (!gl) throw new Error("WebGL context lost"); + return { context: gl, programCache }; +} + +export function readResult({ + width, + height, +}: { + width: number; + height: number; +}): OffscreenCanvas | HTMLCanvasElement { + if (!webglCanvas) throw new Error("WebGL canvas not initialized"); + const outputCanvas = createOffscreenCanvas({ width, height }); + const outputCtx = outputCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (outputCtx) { + outputCtx.drawImage(webglCanvas, 0, 0, width, height); + } + return outputCanvas; +} diff --git a/apps/web/src/services/renderer/webgl/webgl-effect-renderer.ts b/apps/web/src/services/renderer/webgl/webgl-effect-renderer.ts index 89dcb2ae..95ef990b 100644 --- a/apps/web/src/services/renderer/webgl/webgl-effect-renderer.ts +++ b/apps/web/src/services/renderer/webgl/webgl-effect-renderer.ts @@ -1,35 +1,35 @@ -import { getWebGLContext, readResult } from "./webgl-context"; -import { applyMultiPassEffect } from "./webgl-utils"; -import type { EffectPassData } from "./webgl-utils"; - -export interface ApplyEffectParams { - source: CanvasImageSource; - width: number; - height: number; - passes: EffectPassData[]; -} - -function applyEffect({ - source, - width, - height, - passes, -}: ApplyEffectParams): CanvasImageSource { - if (passes.length === 0) { - return source; - } - const { context, programCache } = getWebGLContext({ width, height }); - applyMultiPassEffect({ - context, - source, - width, - height, - passes, - programCache, - }); - return readResult({ width, height }); -} - -export const webglEffectRenderer = { - applyEffect, -}; +import { getWebGLContext, readResult } from "./webgl-context"; +import { applyMultiPassEffect } from "./webgl-utils"; +import type { EffectPassData } from "./webgl-utils"; + +export interface ApplyEffectParams { + source: CanvasImageSource; + width: number; + height: number; + passes: EffectPassData[]; +} + +function applyEffect({ + source, + width, + height, + passes, +}: ApplyEffectParams): CanvasImageSource { + if (passes.length === 0) { + return source; + } + const { context, programCache } = getWebGLContext({ width, height }); + applyMultiPassEffect({ + context, + source, + width, + height, + passes, + programCache, + }); + return readResult({ width, height }); +} + +export const webglEffectRenderer = { + applyEffect, +}; diff --git a/apps/web/src/services/renderer/webgl/webgl-utils.ts b/apps/web/src/services/renderer/webgl/webgl-utils.ts index 834c652e..ec207546 100644 --- a/apps/web/src/services/renderer/webgl/webgl-utils.ts +++ b/apps/web/src/services/renderer/webgl/webgl-utils.ts @@ -1,298 +1,298 @@ -import VERTEX_SHADER_SOURCE from "@/lib/effects/effect.vert.glsl"; - -export interface EffectPassData { - fragmentShader: string; - uniforms: Record; -} - -export const QUAD_POSITIONS = new Float32Array([ - -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1, -]); - -export function compileProgram({ - context, - fragmentShaderSource, - programCache, -}: { - context: WebGLRenderingContext; - fragmentShaderSource: string; - programCache: Map; -}): WebGLProgram { - const cached = programCache.get(fragmentShaderSource); - if (cached) { - return cached; - } - const vertexShader = compileShader({ - context, - source: VERTEX_SHADER_SOURCE, - type: context.VERTEX_SHADER, - }); - const fragmentShader = compileShader({ - context, - source: fragmentShaderSource, - type: context.FRAGMENT_SHADER, - }); - const program = context.createProgram(); - if (!program) { - throw new Error("Failed to create WebGL program"); - } - context.attachShader(program, vertexShader); - context.attachShader(program, fragmentShader); - context.linkProgram(program); - if (!context.getProgramParameter(program, context.LINK_STATUS)) { - const info = context.getProgramInfoLog(program); - context.deleteProgram(program); - throw new Error(`WebGL program link failed: ${info}`); - } - context.deleteShader(vertexShader); - context.deleteShader(fragmentShader); - programCache.set(fragmentShaderSource, program); - return program; -} - -export function compileShader({ - context, - source, - type, -}: { - context: WebGLRenderingContext; - source: string; - type: number; -}): WebGLShader { - const shader = context.createShader(type); - if (!shader) { - throw new Error("Failed to create WebGL shader"); - } - context.shaderSource(shader, source); - context.compileShader(shader); - if (!context.getShaderParameter(shader, context.COMPILE_STATUS)) { - const info = context.getShaderInfoLog(shader); - context.deleteShader(shader); - throw new Error(`WebGL shader compile failed: ${info}`); - } - return shader; -} - -export function createTexture({ - context, - source, -}: { - context: WebGLRenderingContext; - source: CanvasImageSource; -}): WebGLTexture { - const texture = context.createTexture(); - if (!texture) { - throw new Error("Failed to create WebGL texture"); - } - context.activeTexture(context.TEXTURE0); - context.bindTexture(context.TEXTURE_2D, texture); - context.pixelStorei(context.UNPACK_FLIP_Y_WEBGL, 1); - context.texParameteri( - context.TEXTURE_2D, - context.TEXTURE_WRAP_S, - context.CLAMP_TO_EDGE, - ); - context.texParameteri( - context.TEXTURE_2D, - context.TEXTURE_WRAP_T, - context.CLAMP_TO_EDGE, - ); - context.texParameteri( - context.TEXTURE_2D, - context.TEXTURE_MIN_FILTER, - context.LINEAR, - ); - context.texParameteri( - context.TEXTURE_2D, - context.TEXTURE_MAG_FILTER, - context.LINEAR, - ); - context.texImage2D( - context.TEXTURE_2D, - 0, - context.RGBA, - context.RGBA, - context.UNSIGNED_BYTE, - source as TexImageSource, - ); - return texture; -} - -export function setUniforms({ - context, - program, - uniforms, -}: { - context: WebGLRenderingContext; - program: WebGLProgram; - uniforms: Record; -}): void { - for (const [name, value] of Object.entries(uniforms)) { - const location = context.getUniformLocation(program, name); - if (location === null) continue; - - if (typeof value === "number") { - context.uniform1f(location, value); - } else if (Array.isArray(value)) { - if (value.length === 2) { - context.uniform2fv(location, new Float32Array(value)); - } else if (value.length === 3) { - context.uniform3fv(location, new Float32Array(value)); - } else if (value.length === 4) { - context.uniform4fv(location, new Float32Array(value)); - } - } - } -} - -export function drawFullscreenQuad({ - context, - program, - width, - height, -}: { - context: WebGLRenderingContext; - program: WebGLProgram; - width: number; - height: number; -}): void { - const positionLocation = context.getAttribLocation(program, "a_position"); - const buffer = context.createBuffer(); - context.bindBuffer(context.ARRAY_BUFFER, buffer); - context.bufferData(context.ARRAY_BUFFER, QUAD_POSITIONS, context.STATIC_DRAW); - context.enableVertexAttribArray(positionLocation); - context.vertexAttribPointer(positionLocation, 2, context.FLOAT, false, 0, 0); - - context.viewport(0, 0, width, height); - context.clearColor(0, 0, 0, 0); - context.clear(context.COLOR_BUFFER_BIT); - context.drawArrays(context.TRIANGLES, 0, 6); -} - -export function createFramebufferTexture({ - context, - width, - height, -}: { - context: WebGLRenderingContext; - width: number; - height: number; -}): { texture: WebGLTexture; framebuffer: WebGLFramebuffer } { - const texture = context.createTexture(); - if (!texture) throw new Error("Failed to create framebuffer texture"); - context.bindTexture(context.TEXTURE_2D, texture); - context.texImage2D( - context.TEXTURE_2D, - 0, - context.RGBA, - width, - height, - 0, - context.RGBA, - context.UNSIGNED_BYTE, - null, - ); - context.texParameteri( - context.TEXTURE_2D, - context.TEXTURE_WRAP_S, - context.CLAMP_TO_EDGE, - ); - context.texParameteri( - context.TEXTURE_2D, - context.TEXTURE_WRAP_T, - context.CLAMP_TO_EDGE, - ); - context.texParameteri( - context.TEXTURE_2D, - context.TEXTURE_MIN_FILTER, - context.LINEAR, - ); - context.texParameteri( - context.TEXTURE_2D, - context.TEXTURE_MAG_FILTER, - context.LINEAR, - ); - context.bindTexture(context.TEXTURE_2D, null); - - const framebuffer = context.createFramebuffer(); - if (!framebuffer) throw new Error("Failed to create framebuffer"); - context.bindFramebuffer(context.FRAMEBUFFER, framebuffer); - context.framebufferTexture2D( - context.FRAMEBUFFER, - context.COLOR_ATTACHMENT0, - context.TEXTURE_2D, - texture, - 0, - ); - context.bindFramebuffer(context.FRAMEBUFFER, null); - - return { texture, framebuffer }; -} - -export function applyMultiPassEffect({ - context, - source, - width, - height, - passes, - programCache, -}: { - context: WebGLRenderingContext; - source: CanvasImageSource; - width: number; - height: number; - passes: EffectPassData[]; - programCache: Map; -}): void { - const sourceTexture = createTexture({ context, source }); - let currentTexture: WebGLTexture = sourceTexture; - - const intermediates: Array<{ - texture: WebGLTexture; - framebuffer: WebGLFramebuffer; - }> = []; - for (let i = 0; i < passes.length - 1; i++) { - intermediates.push(createFramebufferTexture({ context, width, height })); - } - - for (let i = 0; i < passes.length; i++) { - const pass = passes[i]; - const program = compileProgram({ - context, - fragmentShaderSource: pass.fragmentShader, - programCache, - }); - const isLastPass = i === passes.length - 1; - const targetFramebuffer = isLastPass ? null : intermediates[i].framebuffer; - - context.bindFramebuffer(context.FRAMEBUFFER, targetFramebuffer); - // biome-ignore lint/correctness/useHookAtTopLevel: WebGL API method, not a React hook - context.useProgram(program); - context.activeTexture(context.TEXTURE0); - context.bindTexture(context.TEXTURE_2D, currentTexture); - - const uTextureLocation = context.getUniformLocation(program, "u_texture"); - if (uTextureLocation) { - context.uniform1i(uTextureLocation, 0); - } - - setUniforms({ - context, - program, - uniforms: { ...pass.uniforms, u_resolution: [width, height] }, - }); - drawFullscreenQuad({ context, program, width, height }); - - if (!isLastPass) { - currentTexture = intermediates[i].texture; - } - } - - context.deleteTexture(sourceTexture); - for (const intermediate of intermediates) { - context.deleteTexture(intermediate.texture); - context.deleteFramebuffer(intermediate.framebuffer); - } - context.bindTexture(context.TEXTURE_2D, null); - context.bindFramebuffer(context.FRAMEBUFFER, null); -} +import VERTEX_SHADER_SOURCE from "@/lib/effects/effect.vert.glsl"; + +export interface EffectPassData { + fragmentShader: string; + uniforms: Record; +} + +export const QUAD_POSITIONS = new Float32Array([ + -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1, +]); + +export function compileProgram({ + context, + fragmentShaderSource, + programCache, +}: { + context: WebGLRenderingContext; + fragmentShaderSource: string; + programCache: Map; +}): WebGLProgram { + const cached = programCache.get(fragmentShaderSource); + if (cached) { + return cached; + } + const vertexShader = compileShader({ + context, + source: VERTEX_SHADER_SOURCE, + type: context.VERTEX_SHADER, + }); + const fragmentShader = compileShader({ + context, + source: fragmentShaderSource, + type: context.FRAGMENT_SHADER, + }); + const program = context.createProgram(); + if (!program) { + throw new Error("Failed to create WebGL program"); + } + context.attachShader(program, vertexShader); + context.attachShader(program, fragmentShader); + context.linkProgram(program); + if (!context.getProgramParameter(program, context.LINK_STATUS)) { + const info = context.getProgramInfoLog(program); + context.deleteProgram(program); + throw new Error(`WebGL program link failed: ${info}`); + } + context.deleteShader(vertexShader); + context.deleteShader(fragmentShader); + programCache.set(fragmentShaderSource, program); + return program; +} + +export function compileShader({ + context, + source, + type, +}: { + context: WebGLRenderingContext; + source: string; + type: number; +}): WebGLShader { + const shader = context.createShader(type); + if (!shader) { + throw new Error("Failed to create WebGL shader"); + } + context.shaderSource(shader, source); + context.compileShader(shader); + if (!context.getShaderParameter(shader, context.COMPILE_STATUS)) { + const info = context.getShaderInfoLog(shader); + context.deleteShader(shader); + throw new Error(`WebGL shader compile failed: ${info}`); + } + return shader; +} + +export function createTexture({ + context, + source, +}: { + context: WebGLRenderingContext; + source: CanvasImageSource; +}): WebGLTexture { + const texture = context.createTexture(); + if (!texture) { + throw new Error("Failed to create WebGL texture"); + } + context.activeTexture(context.TEXTURE0); + context.bindTexture(context.TEXTURE_2D, texture); + context.pixelStorei(context.UNPACK_FLIP_Y_WEBGL, 1); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_WRAP_S, + context.CLAMP_TO_EDGE, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_WRAP_T, + context.CLAMP_TO_EDGE, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_MIN_FILTER, + context.LINEAR, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_MAG_FILTER, + context.LINEAR, + ); + context.texImage2D( + context.TEXTURE_2D, + 0, + context.RGBA, + context.RGBA, + context.UNSIGNED_BYTE, + source as TexImageSource, + ); + return texture; +} + +export function setUniforms({ + context, + program, + uniforms, +}: { + context: WebGLRenderingContext; + program: WebGLProgram; + uniforms: Record; +}): void { + for (const [name, value] of Object.entries(uniforms)) { + const location = context.getUniformLocation(program, name); + if (location === null) continue; + + if (typeof value === "number") { + context.uniform1f(location, value); + } else if (Array.isArray(value)) { + if (value.length === 2) { + context.uniform2fv(location, new Float32Array(value)); + } else if (value.length === 3) { + context.uniform3fv(location, new Float32Array(value)); + } else if (value.length === 4) { + context.uniform4fv(location, new Float32Array(value)); + } + } + } +} + +export function drawFullscreenQuad({ + context, + program, + width, + height, +}: { + context: WebGLRenderingContext; + program: WebGLProgram; + width: number; + height: number; +}): void { + const positionLocation = context.getAttribLocation(program, "a_position"); + const buffer = context.createBuffer(); + context.bindBuffer(context.ARRAY_BUFFER, buffer); + context.bufferData(context.ARRAY_BUFFER, QUAD_POSITIONS, context.STATIC_DRAW); + context.enableVertexAttribArray(positionLocation); + context.vertexAttribPointer(positionLocation, 2, context.FLOAT, false, 0, 0); + + context.viewport(0, 0, width, height); + context.clearColor(0, 0, 0, 0); + context.clear(context.COLOR_BUFFER_BIT); + context.drawArrays(context.TRIANGLES, 0, 6); +} + +export function createFramebufferTexture({ + context, + width, + height, +}: { + context: WebGLRenderingContext; + width: number; + height: number; +}): { texture: WebGLTexture; framebuffer: WebGLFramebuffer } { + const texture = context.createTexture(); + if (!texture) throw new Error("Failed to create framebuffer texture"); + context.bindTexture(context.TEXTURE_2D, texture); + context.texImage2D( + context.TEXTURE_2D, + 0, + context.RGBA, + width, + height, + 0, + context.RGBA, + context.UNSIGNED_BYTE, + null, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_WRAP_S, + context.CLAMP_TO_EDGE, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_WRAP_T, + context.CLAMP_TO_EDGE, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_MIN_FILTER, + context.LINEAR, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_MAG_FILTER, + context.LINEAR, + ); + context.bindTexture(context.TEXTURE_2D, null); + + const framebuffer = context.createFramebuffer(); + if (!framebuffer) throw new Error("Failed to create framebuffer"); + context.bindFramebuffer(context.FRAMEBUFFER, framebuffer); + context.framebufferTexture2D( + context.FRAMEBUFFER, + context.COLOR_ATTACHMENT0, + context.TEXTURE_2D, + texture, + 0, + ); + context.bindFramebuffer(context.FRAMEBUFFER, null); + + return { texture, framebuffer }; +} + +export function applyMultiPassEffect({ + context, + source, + width, + height, + passes, + programCache, +}: { + context: WebGLRenderingContext; + source: CanvasImageSource; + width: number; + height: number; + passes: EffectPassData[]; + programCache: Map; +}): void { + const sourceTexture = createTexture({ context, source }); + let currentTexture: WebGLTexture = sourceTexture; + + const intermediates: Array<{ + texture: WebGLTexture; + framebuffer: WebGLFramebuffer; + }> = []; + for (let i = 0; i < passes.length - 1; i++) { + intermediates.push(createFramebufferTexture({ context, width, height })); + } + + for (let i = 0; i < passes.length; i++) { + const pass = passes[i]; + const program = compileProgram({ + context, + fragmentShaderSource: pass.fragmentShader, + programCache, + }); + const isLastPass = i === passes.length - 1; + const targetFramebuffer = isLastPass ? null : intermediates[i].framebuffer; + + context.bindFramebuffer(context.FRAMEBUFFER, targetFramebuffer); + // biome-ignore lint/correctness/useHookAtTopLevel: WebGL API method, not a React hook + context.useProgram(program); + context.activeTexture(context.TEXTURE0); + context.bindTexture(context.TEXTURE_2D, currentTexture); + + const uTextureLocation = context.getUniformLocation(program, "u_texture"); + if (uTextureLocation) { + context.uniform1i(uTextureLocation, 0); + } + + setUniforms({ + context, + program, + uniforms: { ...pass.uniforms, u_resolution: [width, height] }, + }); + drawFullscreenQuad({ context, program, width, height }); + + if (!isLastPass) { + currentTexture = intermediates[i].texture; + } + } + + context.deleteTexture(sourceTexture); + for (const intermediate of intermediates) { + context.deleteTexture(intermediate.texture); + context.deleteFramebuffer(intermediate.framebuffer); + } + context.bindTexture(context.TEXTURE_2D, null); + context.bindFramebuffer(context.FRAMEBUFFER, null); +} -- cgit v1.3.1