diff options
Diffstat (limited to 'apps/web/src/services')
20 files changed, 1580 insertions, 218 deletions
diff --git a/apps/web/src/services/renderer/canvas-utils.ts b/apps/web/src/services/renderer/canvas-utils.ts new file mode 100644 index 00000000..093168e6 --- /dev/null +++ b/apps/web/src/services/renderer/canvas-utils.ts @@ -0,0 +1,16 @@ +export function createOffscreenCanvas({ + width, + height, +}: { + width: number; + height: number; +}): OffscreenCanvas | HTMLCanvasElement { + try { + return new OffscreenCanvas(width, height); + } catch { + const canvas = document.createElement("canvas"); + canvas.width = width; + canvas.height = height; + return canvas; + } +} diff --git a/apps/web/src/services/renderer/effect-preview.ts b/apps/web/src/services/renderer/effect-preview.ts new file mode 100644 index 00000000..2cebec5d --- /dev/null +++ b/apps/web/src/services/renderer/effect-preview.ts @@ -0,0 +1,194 @@ +import { createOffscreenCanvas } from "./canvas-utils"; +import { getEffect } from "@/lib/effects"; +import type { EffectParamValues } from "@/types/effects"; +import { applyMultiPassEffect } from "./webgl-utils"; +import type { EffectPassData } from "./webgl-utils"; + +const PREVIEW_SIZE = 160; +const PREVIEW_IMAGE_PATH = "/effects/preview.jpg"; + +let previewGl: WebGLRenderingContext | null = null; +let previewCanvas: OffscreenCanvas | HTMLCanvasElement | null = null; +let testSourceCanvas: OffscreenCanvas | HTMLCanvasElement | null = null; +let previewImageElement: HTMLImageElement | null = null; +const programCache = new Map<string, WebGLProgram>(); +const onReadyCallbacks = new Set<() => void>(); + +export function onPreviewImageReady({ + callback, +}: { + callback: () => void; +}): () => void { + onReadyCallbacks.add(callback); + return () => onReadyCallbacks.delete(callback); +} + +function loadPreviewImage(): void { + if (typeof window === "undefined") return; + const image = new Image(); + image.onload = () => { + testSourceCanvas = null; + for (const callback of onReadyCallbacks) { + callback(); + } + }; + image.src = PREVIEW_IMAGE_PATH; + previewImageElement = image; +} + +loadPreviewImage(); + +function buildDefaultParams({ + effectType, +}: { + effectType: string; +}): EffectParamValues { + const definition = getEffect({ effectType }); + const params: EffectParamValues = {}; + for (const paramDef of definition.params) { + params[paramDef.key] = paramDef.default; + } + return params; +} + +function createTestSource({ + width, + height, +}: { + width: number; + height: number; +}): OffscreenCanvas | HTMLCanvasElement | null { + const isImageReady = + previewImageElement?.complete && + (previewImageElement.naturalWidth ?? 0) > 0; + if (!isImageReady || !previewImageElement) { + return null; + } + + const canvas = createOffscreenCanvas({ width, height }); + const ctx = canvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!ctx) { + throw new Error("failed to get 2d context for test source"); + } + ctx.drawImage(previewImageElement, 0, 0, width, height); + return canvas; +} + +function getOrCreatePreviewContext({ + width, + height, +}: { + width: number; + height: number; +}): { canvas: OffscreenCanvas | HTMLCanvasElement; gl: WebGLRenderingContext } { + if (!previewCanvas || !previewGl) { + previewCanvas = createOffscreenCanvas({ width, height }); + previewGl = previewCanvas.getContext("webgl", { + premultipliedAlpha: false, + }) as WebGLRenderingContext | null; + if (!previewGl) { + throw new Error("WebGL not supported"); + } + } + if (previewCanvas.width !== width || previewCanvas.height !== height) { + previewCanvas.width = width; + previewCanvas.height = height; + } + return { canvas: previewCanvas, gl: previewGl }; +} + +function getTestSource({ + width, + height, +}: { + width: number; + height: number; +}): CanvasImageSource | null { + if ( + !testSourceCanvas || + testSourceCanvas.width !== width || + testSourceCanvas.height !== height + ) { + testSourceCanvas = createTestSource({ width, height }); + } + return testSourceCanvas; +} + +function applyWebGlEffect({ + source, + width, + height, + passes, +}: { + source: CanvasImageSource; + width: number; + height: number; + passes: EffectPassData[]; +}): OffscreenCanvas | HTMLCanvasElement { + const { canvas: glCanvas, gl } = getOrCreatePreviewContext({ width, height }); + + applyMultiPassEffect({ context: gl, source, width, height, passes, programCache }); + + const outputCanvas = createOffscreenCanvas({ width, height }); + const outputCtx = outputCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (outputCtx) { + outputCtx.drawImage(glCanvas, 0, 0, width, height); + } + return outputCanvas; +} + +export function renderPreview({ + effectType, + params, + targetCanvas, +}: { + effectType: string; + params: EffectParamValues; + targetCanvas: HTMLCanvasElement; +}): void { + const size = PREVIEW_SIZE; + const source = getTestSource({ width: size, height: size }); + if (!source) return; + + const definition = getEffect({ effectType }); + const resolvedParams = + Object.keys(params).length > 0 + ? params + : buildDefaultParams({ effectType }); + + const passes = definition.renderer.passes.map((pass) => ({ + fragmentShader: pass.fragmentShader, + uniforms: pass.uniforms({ + effectParams: resolvedParams, + width: size, + height: size, + }), + })); + const result = applyWebGlEffect({ + source, + width: size, + height: size, + passes, + }); + + const targetCtx = targetCanvas.getContext( + "2d", + ) as CanvasRenderingContext2D | null; + if (targetCtx) { + targetCanvas.width = size; + targetCanvas.height = size; + targetCtx.drawImage(result, 0, 0, size, size); + } +} + +export const effectPreviewService = { + renderPreview, + onPreviewImageReady, + PREVIEW_SIZE, +}; diff --git a/apps/web/src/services/renderer/nodes/blur-background-node.ts b/apps/web/src/services/renderer/nodes/blur-background-node.ts deleted file mode 100644 index 504161d9..00000000 --- a/apps/web/src/services/renderer/nodes/blur-background-node.ts +++ /dev/null @@ -1,75 +0,0 @@ -import type { CanvasRenderer } from "../canvas-renderer"; -import { BaseNode } from "./base-node"; - -export type BlurBackgroundNodeParams = { - blurIntensity: number; - contentNodes: BaseNode[]; -}; - -export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> { - private blurIntensity: number; - private contentNodes: BaseNode[]; - - constructor(params: BlurBackgroundNodeParams) { - super(params); - this.blurIntensity = params.blurIntensity; - this.contentNodes = params.contentNodes; - } - - async render({ - renderer, - time, - }: { - renderer: CanvasRenderer; - time: number; - }): Promise<void> { - let offscreen: OffscreenCanvas | HTMLCanvasElement; - let offscreenCtx: - | OffscreenCanvasRenderingContext2D - | CanvasRenderingContext2D; - - try { - offscreen = new OffscreenCanvas(renderer.width, renderer.height); - const ctx = offscreen.getContext("2d"); - if (!ctx) { - throw new Error("failed to get offscreen canvas context"); - } - offscreenCtx = ctx; - } catch { - offscreen = document.createElement("canvas"); - offscreen.width = renderer.width; - offscreen.height = renderer.height; - const ctx = offscreen.getContext("2d"); - if (!ctx) { - throw new Error("failed to get canvas context"); - } - offscreenCtx = ctx; - } - - const originalContext = renderer.context; - renderer.context = offscreenCtx; - - for (const node of this.contentNodes) { - await node.render({ renderer, time }); - } - - renderer.context = originalContext; - - const zoomScale = 1.4; - const scaledWidth = renderer.width * zoomScale; - const scaledHeight = renderer.height * zoomScale; - const offsetX = (renderer.width - scaledWidth) / 2; - const offsetY = (renderer.height - scaledHeight) / 2; - - renderer.context.save(); - renderer.context.filter = `blur(${this.blurIntensity}px)`; - renderer.context.drawImage( - offscreen as CanvasImageSource, - offsetX, - offsetY, - scaledWidth, - scaledHeight, - ); - renderer.context.restore(); - } -} diff --git a/apps/web/src/services/renderer/nodes/composite-effect-node.ts b/apps/web/src/services/renderer/nodes/composite-effect-node.ts new file mode 100644 index 00000000..3fbfffc1 --- /dev/null +++ b/apps/web/src/services/renderer/nodes/composite-effect-node.ts @@ -0,0 +1,77 @@ +import type { CanvasRenderer } from "../canvas-renderer"; +import { createOffscreenCanvas } from "../canvas-utils"; +import { getEffect } from "@/lib/effects"; +import type { EffectParamValues } from "@/types/effects"; +import { BaseNode } from "./base-node"; +import { webglEffectRenderer } from "../webgl-effect-renderer"; + +export type CompositeEffectNodeParams = { + contentNodes: BaseNode[]; + effectType: string; + effectParams: EffectParamValues; + scale: number; +}; + +export class CompositeEffectNode extends BaseNode<CompositeEffectNodeParams> { + async render({ + renderer, + time, + }: { + renderer: CanvasRenderer; + time: number; + }): Promise<void> { + const offscreen = createOffscreenCanvas({ + width: renderer.width, + height: renderer.height, + }); + const offscreenCtx = offscreen.getContext("2d") as OffscreenCanvasRenderingContext2D | null; + if (!offscreenCtx) { + throw new Error("failed to get offscreen canvas context"); + } + + const originalContext = renderer.context; + renderer.context = offscreenCtx; + + for (const node of this.params.contentNodes) { + await node.render({ renderer, time }); + } + + renderer.context = originalContext; + + const effectDefinition = getEffect({ effectType: this.params.effectType }); + const scale = this.params.scale; + const scaledWidth = renderer.width * scale; + const scaledHeight = renderer.height * scale; + const offsetX = (renderer.width - scaledWidth) / 2; + const offsetY = (renderer.height - scaledHeight) / 2; + + const passes = effectDefinition.renderer.passes.map((pass) => ({ + fragmentShader: pass.fragmentShader, + uniforms: pass.uniforms({ + effectParams: this.params.effectParams, + width: renderer.width, + height: renderer.height, + }), + })); + const effectResult = webglEffectRenderer.applyEffect({ + source: offscreen as CanvasImageSource, + width: renderer.width, + height: renderer.height, + passes, + }); + + renderer.context.save(); + renderer.context.drawImage( + effectResult, + 0, + 0, + renderer.width, + renderer.height, + offsetX, + offsetY, + scaledWidth, + scaledHeight, + ); + renderer.context.restore(); + } +} diff --git a/apps/web/src/services/renderer/nodes/effect-layer-node.ts b/apps/web/src/services/renderer/nodes/effect-layer-node.ts new file mode 100644 index 00000000..3d84be2b --- /dev/null +++ b/apps/web/src/services/renderer/nodes/effect-layer-node.ts @@ -0,0 +1,82 @@ +import type { CanvasRenderer } from "../canvas-renderer"; +import { getEffect } from "@/lib/effects"; +import type { EffectParamValues } from "@/types/effects"; +import { BaseNode } from "./base-node"; +import { webglEffectRenderer } from "../webgl-effect-renderer"; + +const TIME_EPSILON = 1e-6; + +export type EffectLayerNodeParams = { + effectType: string; + effectParams: EffectParamValues; + timeOffset: number; + duration: number; +}; + +function isInRange({ + time, + timeOffset, + duration, +}: { + time: number; + timeOffset: number; + duration: number; +}): boolean { + return ( + time >= timeOffset - TIME_EPSILON && + time < timeOffset + duration + TIME_EPSILON + ); +} + +// snapshots whatever is currently on the canvas, applies the effect, draws it back +export class EffectLayerNode extends BaseNode<EffectLayerNodeParams> { + async render({ + renderer, + time, + }: { + renderer: CanvasRenderer; + time: number; + }): Promise<void> { + if ( + !isInRange({ + time, + timeOffset: this.params.timeOffset, + duration: this.params.duration, + }) + ) { + return; + } + + const source = renderer.context.canvas as CanvasImageSource; + + const effectDefinition = getEffect({ + effectType: this.params.effectType, + }); + + const passes = effectDefinition.renderer.passes.map((pass) => ({ + fragmentShader: pass.fragmentShader, + uniforms: pass.uniforms({ + effectParams: this.params.effectParams, + width: renderer.width, + height: renderer.height, + }), + })); + const effectResult = webglEffectRenderer.applyEffect({ + source, + width: renderer.width, + height: renderer.height, + passes, + }); + + renderer.context.save(); + renderer.context.clearRect(0, 0, renderer.width, renderer.height); + renderer.context.drawImage( + effectResult, + 0, + 0, + renderer.width, + renderer.height, + ); + renderer.context.restore(); + } +} diff --git a/apps/web/src/services/renderer/nodes/image-node.ts b/apps/web/src/services/renderer/nodes/image-node.ts index b3a0be74..6720a5e5 100644 --- a/apps/web/src/services/renderer/nodes/image-node.ts +++ b/apps/web/src/services/renderer/nodes/image-node.ts @@ -73,7 +73,7 @@ export class ImageNode extends VisualNode<ImageNodeParams> { async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { await super.render({ renderer, time }); - if (!this.isInRange(time)) { + if (!this.isInRange({ time })) { return; } @@ -84,6 +84,7 @@ export class ImageNode extends VisualNode<ImageNodeParams> { source, sourceWidth: width || renderer.width, sourceHeight: height || renderer.height, + timelineTime: time, }); } } diff --git a/apps/web/src/services/renderer/nodes/sticker-node.ts b/apps/web/src/services/renderer/nodes/sticker-node.ts index acf638eb..da271685 100644 --- a/apps/web/src/services/renderer/nodes/sticker-node.ts +++ b/apps/web/src/services/renderer/nodes/sticker-node.ts @@ -51,7 +51,7 @@ export class StickerNode extends VisualNode<StickerNodeParams> { async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { await super.render({ renderer, time }); - if (!this.isInRange(time)) { + if (!this.isInRange({ time })) { return; } @@ -62,6 +62,7 @@ export class StickerNode extends VisualNode<StickerNodeParams> { source, sourceWidth: width, sourceHeight: height, + timelineTime: time, }); } } diff --git a/apps/web/src/services/renderer/nodes/text-node.ts b/apps/web/src/services/renderer/nodes/text-node.ts index 76d78b31..86e24fdf 100644 --- a/apps/web/src/services/renderer/nodes/text-node.ts +++ b/apps/web/src/services/renderer/nodes/text-node.ts @@ -1,10 +1,14 @@ import type { CanvasRenderer } from "../canvas-renderer"; +import { createOffscreenCanvas } from "../canvas-utils"; import { BaseNode } from "./base-node"; import type { TextElement } from "@/types/timeline"; import { + DEFAULT_TEXT_BACKGROUND, DEFAULT_TEXT_ELEMENT, DEFAULT_LINE_HEIGHT, FONT_SIZE_SCALE_REFERENCE, + CORNER_RADIUS_MAX, + CORNER_RADIUS_MIN, } from "@/constants/text-constants"; import { getMetricAscent, @@ -12,6 +16,17 @@ import { getTextBackgroundRect, measureTextBlock, } from "@/lib/text/layout"; +import { + getElementLocalTime, + resolveColorAtTime, + resolveNumberAtTime, + resolveOpacityAtTime, + resolveTransformAtTime, +} from "@/lib/animation"; +import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel"; +import { getEffect } from "@/lib/effects"; +import { webglEffectRenderer } from "../webgl-effect-renderer"; +import { clamp } from "@/utils/math"; function scaleFontSize({ fontSize, @@ -27,6 +42,9 @@ function quoteFontFamily({ fontFamily }: { fontFamily: string }): string { return `"${fontFamily.replace(/"/g, '\\"')}"`; } +const TEXT_DECORATION_THICKNESS_RATIO = 0.07; +const STRIKETHROUGH_VERTICAL_RATIO = 0.35; + function drawTextDecoration({ ctx, textDecoration, @@ -46,7 +64,7 @@ function drawTextDecoration({ }): void { if (textDecoration === "none" || !textDecoration) return; - const thickness = Math.max(1, scaledFontSize * 0.07); + const thickness = Math.max(1, scaledFontSize * TEXT_DECORATION_THICKNESS_RATIO); const ascent = getMetricAscent({ metrics, fallbackFontSize: scaledFontSize }); const descent = getMetricDescent({ metrics, fallbackFontSize: scaledFontSize }); @@ -60,7 +78,7 @@ function drawTextDecoration({ } if (textDecoration === "line-through") { - const strikeY = lineY - (ascent - descent) * 0.35; + const strikeY = lineY - (ascent - descent) * STRIKETHROUGH_VERTICAL_RATIO; ctx.fillRect(xStart, strikeY, lineWidth, thickness); } } @@ -84,19 +102,24 @@ export class TextNode extends BaseNode<TextNodeParams> { return; } - renderer.context.save(); - - const x = this.params.transform.position.x + this.params.canvasCenter.x; - const y = this.params.transform.position.y + this.params.canvasCenter.y; + const localTime = getElementLocalTime({ + timelineTime: time, + elementStartTime: this.params.startTime, + elementDuration: this.params.duration, + }); + const transform = resolveTransformAtTime({ + baseTransform: this.params.transform, + animations: this.params.animations, + localTime, + }); + const opacity = resolveOpacityAtTime({ + baseOpacity: this.params.opacity, + animations: this.params.animations, + localTime, + }); - renderer.context.translate(x, y); - renderer.context.scale( - this.params.transform.scale, - this.params.transform.scale, - ); - if (this.params.transform.rotate) { - renderer.context.rotate((this.params.transform.rotate * Math.PI) / 180); - } + const x = transform.position.x + this.params.canvasCenter.x; + const y = transform.position.y + this.params.canvasCenter.y; const fontWeight = this.params.fontWeight === "bold" ? "bold" : "normal"; const fontStyle = this.params.fontStyle === "italic" ? "italic" : "normal"; @@ -105,83 +128,193 @@ export class TextNode extends BaseNode<TextNodeParams> { canvasHeight: this.params.canvasHeight, }); const fontFamily = quoteFontFamily({ fontFamily: this.params.fontFamily }); - renderer.context.font = `${fontStyle} ${fontWeight} ${scaledFontSize}px ${fontFamily}, sans-serif`; - renderer.context.textAlign = this.params.textAlign; - renderer.context.fillStyle = this.params.color; - + const fontString = `${fontStyle} ${fontWeight} ${scaledFontSize}px ${fontFamily}, sans-serif`; const letterSpacing = this.params.letterSpacing ?? 0; const lineHeight = this.params.lineHeight ?? DEFAULT_LINE_HEIGHT; - if ("letterSpacing" in renderer.context) { - ( - renderer.context as CanvasRenderingContext2D & { letterSpacing: string } - ).letterSpacing = `${letterSpacing}px`; - } - const lines = this.params.content.split("\n"); const lineHeightPx = scaledFontSize * lineHeight; const fontSizeRatio = this.params.fontSize / DEFAULT_TEXT_ELEMENT.fontSize; const baseline = this.params.textBaseline ?? "middle"; + const blendMode = ( + this.params.blendMode && this.params.blendMode !== "normal" + ? this.params.blendMode + : "source-over" + ) as GlobalCompositeOperation; + renderer.context.save(); + renderer.context.font = fontString; renderer.context.textBaseline = baseline; + if ("letterSpacing" in renderer.context) { + (renderer.context as CanvasRenderingContext2D & { letterSpacing: string }).letterSpacing = `${letterSpacing}px`; + } const lineMetrics = lines.map((line) => renderer.context.measureText(line)); + renderer.context.restore(); + const lineCount = lines.length; + const block = measureTextBlock({ lineMetrics, lineHeightPx, fallbackFontSize: scaledFontSize }); - const block = measureTextBlock({ - lineMetrics, - lineHeightPx, - fallbackFontSize: scaledFontSize, + const textColor = resolveColorAtTime({ + baseColor: this.params.color, + animations: this.params.animations, + propertyPath: "color", + localTime, }); + const bg = this.params.background; + const resolvedBackground = { + ...bg, + color: resolveColorAtTime({ + baseColor: bg.color, + animations: this.params.animations, + propertyPath: "background.color", + localTime, + }), + paddingX: resolveNumberAtTime({ + baseValue: bg.paddingX ?? DEFAULT_TEXT_BACKGROUND.paddingX, + animations: this.params.animations, + propertyPath: "background.paddingX", + localTime, + }), + paddingY: resolveNumberAtTime({ + baseValue: bg.paddingY ?? DEFAULT_TEXT_BACKGROUND.paddingY, + animations: this.params.animations, + propertyPath: "background.paddingY", + localTime, + }), + offsetX: resolveNumberAtTime({ + baseValue: bg.offsetX ?? DEFAULT_TEXT_BACKGROUND.offsetX, + animations: this.params.animations, + propertyPath: "background.offsetX", + localTime, + }), + offsetY: resolveNumberAtTime({ + baseValue: bg.offsetY ?? DEFAULT_TEXT_BACKGROUND.offsetY, + animations: this.params.animations, + propertyPath: "background.offsetY", + localTime, + }), + cornerRadius: resolveNumberAtTime({ + baseValue: bg.cornerRadius ?? CORNER_RADIUS_MIN, + animations: this.params.animations, + propertyPath: "background.cornerRadius", + localTime, + }), + }; - const prevAlpha = renderer.context.globalAlpha; - renderer.context.globalCompositeOperation = ( - this.params.blendMode && this.params.blendMode !== "normal" - ? this.params.blendMode - : "source-over" - ) as GlobalCompositeOperation; - renderer.context.globalAlpha = this.params.opacity; + const drawContent = (ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) => { + ctx.font = fontString; + ctx.textAlign = this.params.textAlign; + ctx.textBaseline = baseline; + ctx.fillStyle = textColor; + if ("letterSpacing" in ctx) { + (ctx as CanvasRenderingContext2D & { letterSpacing: string }).letterSpacing = `${letterSpacing}px`; + } - if ( - this.params.background.color && - this.params.background.color !== "transparent" && - lineCount > 0 - ) { - const { color, cornerRadius = 0 } = this.params.background; - const backgroundRect = getTextBackgroundRect({ - textAlign: this.params.textAlign, - block, - background: this.params.background, - fontSizeRatio, - }); - if (backgroundRect) { - renderer.context.fillStyle = color; - renderer.context.beginPath(); - renderer.context.roundRect( - backgroundRect.left, - backgroundRect.top, - backgroundRect.width, - backgroundRect.height, - cornerRadius, - ); - renderer.context.fill(); - renderer.context.fillStyle = this.params.color; + if ( + this.params.background.enabled && + this.params.background.color && + this.params.background.color !== "transparent" && + lineCount > 0 + ) { + const backgroundRect = getTextBackgroundRect({ + textAlign: this.params.textAlign, + block, + background: resolvedBackground, + fontSizeRatio, + }); + if (backgroundRect) { + const p = clamp({ value: resolvedBackground.cornerRadius, min: CORNER_RADIUS_MIN, max: CORNER_RADIUS_MAX }) / 100; + const radius = Math.min(backgroundRect.width, backgroundRect.height) / 2 * p; + ctx.fillStyle = resolvedBackground.color; + ctx.beginPath(); + ctx.roundRect(backgroundRect.left, backgroundRect.top, backgroundRect.width, backgroundRect.height, radius); + ctx.fill(); + ctx.fillStyle = textColor; + } } + + for (let i = 0; i < lineCount; i++) { + const lineY = i * lineHeightPx - block.visualCenterOffset; + ctx.fillText(lines[i], 0, lineY); + drawTextDecoration({ + ctx, + textDecoration: this.params.textDecoration ?? "none", + lineWidth: lineMetrics[i].width, + lineY, + metrics: lineMetrics[i], + scaledFontSize, + textAlign: this.params.textAlign, + }); + } + }; + + const applyTransform = (ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D) => { + ctx.translate(x, y); + ctx.scale(transform.scale, transform.scale); + if (transform.rotate) { + ctx.rotate((transform.rotate * Math.PI) / 180); + } + }; + + const enabledEffects = this.params.effects?.filter((effect) => effect.enabled) ?? []; + + if (enabledEffects.length === 0) { + renderer.context.save(); + applyTransform(renderer.context); + renderer.context.globalCompositeOperation = blendMode; + renderer.context.globalAlpha = opacity; + drawContent(renderer.context); + renderer.context.restore(); + return; + } + + // Effects path: render text to a same-size offscreen canvas so the blur + // can spread into the surrounding transparent area without hard clipping. + const offscreen = createOffscreenCanvas({ width: renderer.width, height: renderer.height }); + const offscreenCtx = offscreen.getContext("2d") as OffscreenCanvasRenderingContext2D | null; + + if (!offscreenCtx) { + renderer.context.save(); + applyTransform(renderer.context); + renderer.context.globalCompositeOperation = blendMode; + renderer.context.globalAlpha = opacity; + drawContent(renderer.context); + renderer.context.restore(); + return; } - for (let i = 0; i < lineCount; i++) { - const y = i * lineHeightPx - block.visualCenterOffset; - renderer.context.fillText(lines[i], 0, y); - drawTextDecoration({ - ctx: renderer.context, - textDecoration: this.params.textDecoration ?? "none", - lineWidth: lineMetrics[i].width, - lineY: y, - metrics: lineMetrics[i], - scaledFontSize, - textAlign: this.params.textAlign, + offscreenCtx.save(); + applyTransform(offscreenCtx); + drawContent(offscreenCtx); + offscreenCtx.restore(); + + let currentSource: CanvasImageSource = offscreen; + for (const effect of enabledEffects) { + const resolvedParams = resolveEffectParamsAtTime({ + effect, + animations: this.params.animations, + localTime, + }); + const definition = getEffect({ effectType: effect.type }); + const passes = definition.renderer.passes.map((pass) => ({ + fragmentShader: pass.fragmentShader, + uniforms: pass.uniforms({ + effectParams: resolvedParams, + width: renderer.width, + height: renderer.height, + }), + })); + currentSource = webglEffectRenderer.applyEffect({ + source: currentSource, + width: renderer.width, + height: renderer.height, + passes, }); } - renderer.context.globalAlpha = prevAlpha; + renderer.context.save(); + renderer.context.globalCompositeOperation = blendMode; + renderer.context.globalAlpha = opacity; + renderer.context.drawImage(currentSource, 0, 0); renderer.context.restore(); } } diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts index 8e2ddccd..503c9c03 100644 --- a/apps/web/src/services/renderer/nodes/video-node.ts +++ b/apps/web/src/services/renderer/nodes/video-node.ts @@ -12,11 +12,11 @@ export class VideoNode extends VisualNode<VideoNodeParams> { async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { await super.render({ renderer, time }); - if (!this.isInRange(time)) { + if (!this.isInRange({ time })) { return; } - const videoTime = this.getLocalTime(time); + const videoTime = this.getSourceLocalTime({ time }); const frame = await videoCache.getFrameAt({ mediaId: this.params.mediaId, file: this.params.file, @@ -29,6 +29,7 @@ export class VideoNode extends VisualNode<VideoNodeParams> { source: frame.canvas, sourceWidth: frame.canvas.width, sourceHeight: frame.canvas.height, + timelineTime: time, }); } } diff --git a/apps/web/src/services/renderer/nodes/visual-node.ts b/apps/web/src/services/renderer/nodes/visual-node.ts index 7179369e..6c50aa12 100644 --- a/apps/web/src/services/renderer/nodes/visual-node.ts +++ b/apps/web/src/services/renderer/nodes/visual-node.ts @@ -1,9 +1,19 @@ import type { CanvasRenderer } from "../canvas-renderer"; +import { createOffscreenCanvas } from "../canvas-utils"; import { BaseNode } from "./base-node"; +import type { Effect } from "@/types/effects"; import type { BlendMode } from "@/types/rendering"; import type { Transform } from "@/types/timeline"; - -const VISUAL_EPSILON = 1 / 1000; +import type { ElementAnimations } from "@/types/animation"; +import { + getElementLocalTime, + resolveOpacityAtTime, + resolveTransformAtTime, +} from "@/lib/animation"; +import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel"; +import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; +import { getEffect } from "@/lib/effects"; +import { webglEffectRenderer } from "../webgl-effect-renderer"; export interface VisualNodeParams { duration: number; @@ -11,21 +21,31 @@ export interface VisualNodeParams { trimStart: number; trimEnd: number; transform: Transform; + animations?: ElementAnimations; opacity: number; blendMode?: BlendMode; + effects?: Effect[]; } export abstract class VisualNode< Params extends VisualNodeParams = VisualNodeParams, > extends BaseNode<Params> { - protected getLocalTime(time: number): number { + protected getSourceLocalTime({ time }: { time: number }): number { return time - this.params.timeOffset + this.params.trimStart; } - protected isInRange(time: number): boolean { - const localTime = this.getLocalTime(time); + protected getAnimationLocalTime({ time }: { time: number }): number { + return getElementLocalTime({ + timelineTime: time, + elementStartTime: this.params.timeOffset, + elementDuration: this.params.duration, + }); + } + + protected isInRange({ time }: { time: number }): boolean { + const localTime = this.getSourceLocalTime({ time }); return ( - localTime >= this.params.trimStart - VISUAL_EPSILON && + localTime >= this.params.trimStart - TIME_EPSILON_SECONDS && localTime < this.params.trimStart + this.params.duration ); } @@ -35,15 +55,27 @@ export abstract class VisualNode< source, sourceWidth, sourceHeight, + timelineTime, }: { renderer: CanvasRenderer; source: CanvasImageSource; sourceWidth: number; sourceHeight: number; + timelineTime: number; }): void { renderer.context.save(); - const { transform, opacity } = this.params; + const animationLocalTime = this.getAnimationLocalTime({ time: timelineTime }); + const transform = resolveTransformAtTime({ + baseTransform: this.params.transform, + animations: this.params.animations, + localTime: animationLocalTime, + }); + const opacity = resolveOpacityAtTime({ + baseOpacity: this.params.opacity, + animations: this.params.animations, + localTime: animationLocalTime, + }); const containScale = Math.min( renderer.width / sourceWidth, renderer.height / sourceHeight, @@ -68,7 +100,63 @@ export abstract class VisualNode< renderer.context.translate(-centerX, -centerY); } - renderer.context.drawImage(source, x, y, scaledWidth, scaledHeight); + const enabledEffects = + this.params.effects?.filter((effect) => effect.enabled) ?? []; + + if (enabledEffects.length === 0) { + renderer.context.drawImage(source, x, y, scaledWidth, scaledHeight); + renderer.context.restore(); + return; + } + + const elementCanvas = createOffscreenCanvas({ + width: Math.round(scaledWidth), + height: Math.round(scaledHeight), + }); + const elementCtx = elementCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!elementCtx) { + renderer.context.drawImage(source, x, y, scaledWidth, scaledHeight); + renderer.context.restore(); + return; + } + + elementCtx.drawImage(source, 0, 0, scaledWidth, scaledHeight); + + let currentResult: CanvasImageSource = elementCanvas; + + for (const effect of enabledEffects) { + const resolvedParams = resolveEffectParamsAtTime({ + effect, + animations: this.params.animations, + localTime: animationLocalTime, + }); + const definition = getEffect({ effectType: effect.type }); + const passes = definition.renderer.passes.map((pass) => ({ + fragmentShader: pass.fragmentShader, + uniforms: pass.uniforms({ + effectParams: resolvedParams, + width: scaledWidth, + height: scaledHeight, + }), + })); + currentResult = webglEffectRenderer.applyEffect({ + source: currentResult, + width: Math.round(scaledWidth), + height: Math.round(scaledHeight), + passes, + }); + } + + renderer.context.drawImage( + currentResult, + x, + y, + scaledWidth, + scaledHeight, + ); renderer.context.restore(); } } diff --git a/apps/web/src/services/renderer/scene-builder.ts b/apps/web/src/services/renderer/scene-builder.ts index 530b2eb3..695e077f 100644 --- a/apps/web/src/services/renderer/scene-builder.ts +++ b/apps/web/src/services/renderer/scene-builder.ts @@ -6,51 +6,59 @@ import { ImageNode } from "./nodes/image-node"; import { TextNode } from "./nodes/text-node"; import { StickerNode } from "./nodes/sticker-node"; import { ColorNode } from "./nodes/color-node"; -import { BlurBackgroundNode } from "./nodes/blur-background-node"; +import { CompositeEffectNode } from "./nodes/composite-effect-node"; +import { EffectLayerNode } from "./nodes/effect-layer-node"; +import type { BaseNode } from "./nodes/base-node"; import type { TBackground, TCanvasSize } from "@/types/project"; import { DEFAULT_BLUR_INTENSITY } from "@/constants/project-constants"; import { isMainTrack } from "@/lib/timeline"; const PREVIEW_MAX_IMAGE_SIZE = 2048; +const BLUR_BACKGROUND_ZOOM_SCALE = 1.4; -export type BuildSceneParams = { - canvasSize: TCanvasSize; +function getVisibleSortedElements({ + track, +}: { + track: TimelineTrack; +}) { + return track.elements + .filter((element) => !("hidden" in element && element.hidden)) + .slice() + .sort((a, b) => { + if (a.startTime !== b.startTime) return a.startTime - b.startTime; + return a.id.localeCompare(b.id); + }); +} + +function buildTrackNodes({ + tracks, + mediaMap, + canvasSize, + isPreview, +}: { tracks: TimelineTrack[]; - mediaAssets: MediaAsset[]; - duration: number; - background: TBackground; + mediaMap: Map<string, MediaAsset>; + canvasSize: TCanvasSize; isPreview?: boolean; -}; - -export function buildScene(params: BuildSceneParams) { - const { tracks, mediaAssets, duration, canvasSize, background } = params; - - const rootNode = new RootNode({ duration }); - const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); +}): BaseNode[] { + const nodes: BaseNode[] = []; - const visibleTracks = tracks.filter( - (track) => !("hidden" in track && track.hidden), - ); - - const orderedTracksTopToBottom = [ - ...visibleTracks.filter((track) => !isMainTrack(track)), - ...visibleTracks.filter((track) => isMainTrack(track)), - ]; - - const orderedTracksBottomToTop = orderedTracksTopToBottom.slice().reverse(); - - const contentNodes = []; - - for (const track of orderedTracksBottomToTop) { - const elements = track.elements - .filter((element) => !("hidden" in element && element.hidden)) - .slice() - .sort((a, b) => { - if (a.startTime !== b.startTime) return a.startTime - b.startTime; - return a.id.localeCompare(b.id); - }); + for (const track of tracks) { + const elements = getVisibleSortedElements({ track }); for (const element of elements) { + if (element.type === "effect") { + nodes.push( + new EffectLayerNode({ + effectType: element.effectType, + effectParams: element.params, + timeOffset: element.startTime, + duration: element.duration, + }), + ); + continue; + } + if (element.type === "video" || element.type === "image") { const mediaAsset = mediaMap.get(element.mediaId); if (!mediaAsset?.file || !mediaAsset?.url) { @@ -58,7 +66,7 @@ export function buildScene(params: BuildSceneParams) { } if (mediaAsset.type === "video") { - contentNodes.push( + nodes.push( new VideoNode({ mediaId: mediaAsset.id, url: mediaAsset.url, @@ -68,13 +76,15 @@ export function buildScene(params: BuildSceneParams) { trimStart: element.trimStart, trimEnd: element.trimEnd, transform: element.transform, + animations: element.animations, opacity: element.opacity, blendMode: element.blendMode, + effects: element.effects, }), ); } if (mediaAsset.type === "image") { - contentNodes.push( + nodes.push( new ImageNode({ url: mediaAsset.url, duration: element.duration, @@ -82,9 +92,11 @@ export function buildScene(params: BuildSceneParams) { trimStart: element.trimStart, trimEnd: element.trimEnd, transform: element.transform, + animations: element.animations, opacity: element.opacity, blendMode: element.blendMode, - ...(params.isPreview && { + effects: element.effects, + ...(isPreview && { maxSourceSize: PREVIEW_MAX_IMAGE_SIZE, }), }), @@ -93,18 +105,19 @@ export function buildScene(params: BuildSceneParams) { } if (element.type === "text") { - contentNodes.push( + nodes.push( new TextNode({ ...element, canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 }, canvasHeight: canvasSize.height, textBaseline: "middle", + effects: element.effects, }), ); } if (element.type === "sticker") { - contentNodes.push( + nodes.push( new StickerNode({ stickerId: element.stickerId, duration: element.duration, @@ -112,31 +125,77 @@ export function buildScene(params: BuildSceneParams) { trimStart: element.trimStart, trimEnd: element.trimEnd, transform: element.transform, + animations: element.animations, opacity: element.opacity, blendMode: element.blendMode, + effects: element.effects, }), ); } } } + return nodes; +} + +export type BuildSceneParams = { + canvasSize: TCanvasSize; + tracks: TimelineTrack[]; + mediaAssets: MediaAsset[]; + duration: number; + background: TBackground; + isPreview?: boolean; +}; + +export function buildScene({ + canvasSize, + tracks, + mediaAssets, + duration, + background, + isPreview, +}: BuildSceneParams) { + const rootNode = new RootNode({ duration }); + const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); + + const visibleTracks = tracks.filter( + (track) => !("hidden" in track && track.hidden), + ); + + const orderedTracksTopToBottom = [ + ...visibleTracks.filter((track) => !isMainTrack(track)), + ...visibleTracks.filter((track) => isMainTrack(track)), + ]; + + const orderedTracksBottomToTop = orderedTracksTopToBottom.slice().reverse(); + + const allNodes = buildTrackNodes({ + tracks: orderedTracksBottomToTop, + mediaMap, + canvasSize, + isPreview, + }); + if (background.type === "blur") { rootNode.add( - new BlurBackgroundNode({ - blurIntensity: background.blurIntensity ?? DEFAULT_BLUR_INTENSITY, - contentNodes, + new CompositeEffectNode({ + contentNodes: allNodes.filter( + (node) => !(node instanceof EffectLayerNode), + ), + effectType: "blur", + effectParams: { + intensity: + background.blurIntensity ?? DEFAULT_BLUR_INTENSITY, + }, + scale: BLUR_BACKGROUND_ZOOM_SCALE, }), ); - for (const node of contentNodes) { - rootNode.add(node); - } - } else { - if (background.type === "color" && background.color !== "transparent") { - rootNode.add(new ColorNode({ color: background.color })); - } - for (const node of contentNodes) { - rootNode.add(node); - } + } else if (background.type === "color" && background.color !== "transparent") { + rootNode.add(new ColorNode({ color: background.color })); + } + + for (const node of allNodes) { + rootNode.add(node); } return rootNode; diff --git a/apps/web/src/services/renderer/webgl-effect-renderer.ts b/apps/web/src/services/renderer/webgl-effect-renderer.ts new file mode 100644 index 00000000..ad5cd40e --- /dev/null +++ b/apps/web/src/services/renderer/webgl-effect-renderer.ts @@ -0,0 +1,73 @@ +import { createOffscreenCanvas } from "./canvas-utils"; +import { applyMultiPassEffect } from "./webgl-utils"; +import type { EffectPassData } from "./webgl-utils"; + +export interface ApplyEffectParams { + source: CanvasImageSource; + width: number; + height: number; + passes: EffectPassData[]; +} + +let gl: WebGLRenderingContext | null = null; +let canvas: OffscreenCanvas | HTMLCanvasElement | null = null; +const programCache = new Map<string, WebGLProgram>(); + +function getOrCreateCanvas({ + width, + height, +}: { + width: number; + height: number; +}): OffscreenCanvas | HTMLCanvasElement { + if (!canvas) { + canvas = createOffscreenCanvas({ width, height }); + gl = canvas.getContext("webgl", { + premultipliedAlpha: false, + }) as WebGLRenderingContext | null; + if (!gl) { + throw new Error("WebGL not supported"); + } + } + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + return canvas; +} + +function applyEffect({ + source, + width, + height, + passes, +}: ApplyEffectParams): OffscreenCanvas | HTMLCanvasElement { + const targetCanvas = getOrCreateCanvas({ width, height }); + const context = gl; + if (!context) { + throw new Error("WebGL context not initialized"); + } + + applyMultiPassEffect({ + context, + source, + width, + height, + passes, + programCache, + }); + + const outputCanvas = createOffscreenCanvas({ width, height }); + const outputCtx = outputCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (outputCtx) { + outputCtx.drawImage(targetCanvas, 0, 0, width, height); + } + return outputCanvas; +} + +export const webglEffectRenderer = { + applyEffect, +}; diff --git a/apps/web/src/services/renderer/webgl-utils.ts b/apps/web/src/services/renderer/webgl-utils.ts new file mode 100644 index 00000000..834c652e --- /dev/null +++ b/apps/web/src/services/renderer/webgl-utils.ts @@ -0,0 +1,298 @@ +import VERTEX_SHADER_SOURCE from "@/lib/effects/effect.vert.glsl"; + +export interface EffectPassData { + fragmentShader: string; + uniforms: Record<string, number | number[]>; +} + +export const QUAD_POSITIONS = new Float32Array([ + -1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1, +]); + +export function compileProgram({ + context, + fragmentShaderSource, + programCache, +}: { + context: WebGLRenderingContext; + fragmentShaderSource: string; + programCache: Map<string, WebGLProgram>; +}): WebGLProgram { + const cached = programCache.get(fragmentShaderSource); + if (cached) { + return cached; + } + const vertexShader = compileShader({ + context, + source: VERTEX_SHADER_SOURCE, + type: context.VERTEX_SHADER, + }); + const fragmentShader = compileShader({ + context, + source: fragmentShaderSource, + type: context.FRAGMENT_SHADER, + }); + const program = context.createProgram(); + if (!program) { + throw new Error("Failed to create WebGL program"); + } + context.attachShader(program, vertexShader); + context.attachShader(program, fragmentShader); + context.linkProgram(program); + if (!context.getProgramParameter(program, context.LINK_STATUS)) { + const info = context.getProgramInfoLog(program); + context.deleteProgram(program); + throw new Error(`WebGL program link failed: ${info}`); + } + context.deleteShader(vertexShader); + context.deleteShader(fragmentShader); + programCache.set(fragmentShaderSource, program); + return program; +} + +export function compileShader({ + context, + source, + type, +}: { + context: WebGLRenderingContext; + source: string; + type: number; +}): WebGLShader { + const shader = context.createShader(type); + if (!shader) { + throw new Error("Failed to create WebGL shader"); + } + context.shaderSource(shader, source); + context.compileShader(shader); + if (!context.getShaderParameter(shader, context.COMPILE_STATUS)) { + const info = context.getShaderInfoLog(shader); + context.deleteShader(shader); + throw new Error(`WebGL shader compile failed: ${info}`); + } + return shader; +} + +export function createTexture({ + context, + source, +}: { + context: WebGLRenderingContext; + source: CanvasImageSource; +}): WebGLTexture { + const texture = context.createTexture(); + if (!texture) { + throw new Error("Failed to create WebGL texture"); + } + context.activeTexture(context.TEXTURE0); + context.bindTexture(context.TEXTURE_2D, texture); + context.pixelStorei(context.UNPACK_FLIP_Y_WEBGL, 1); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_WRAP_S, + context.CLAMP_TO_EDGE, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_WRAP_T, + context.CLAMP_TO_EDGE, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_MIN_FILTER, + context.LINEAR, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_MAG_FILTER, + context.LINEAR, + ); + context.texImage2D( + context.TEXTURE_2D, + 0, + context.RGBA, + context.RGBA, + context.UNSIGNED_BYTE, + source as TexImageSource, + ); + return texture; +} + +export function setUniforms({ + context, + program, + uniforms, +}: { + context: WebGLRenderingContext; + program: WebGLProgram; + uniforms: Record<string, number | number[]>; +}): void { + for (const [name, value] of Object.entries(uniforms)) { + const location = context.getUniformLocation(program, name); + if (location === null) continue; + + if (typeof value === "number") { + context.uniform1f(location, value); + } else if (Array.isArray(value)) { + if (value.length === 2) { + context.uniform2fv(location, new Float32Array(value)); + } else if (value.length === 3) { + context.uniform3fv(location, new Float32Array(value)); + } else if (value.length === 4) { + context.uniform4fv(location, new Float32Array(value)); + } + } + } +} + +export function drawFullscreenQuad({ + context, + program, + width, + height, +}: { + context: WebGLRenderingContext; + program: WebGLProgram; + width: number; + height: number; +}): void { + const positionLocation = context.getAttribLocation(program, "a_position"); + const buffer = context.createBuffer(); + context.bindBuffer(context.ARRAY_BUFFER, buffer); + context.bufferData(context.ARRAY_BUFFER, QUAD_POSITIONS, context.STATIC_DRAW); + context.enableVertexAttribArray(positionLocation); + context.vertexAttribPointer(positionLocation, 2, context.FLOAT, false, 0, 0); + + context.viewport(0, 0, width, height); + context.clearColor(0, 0, 0, 0); + context.clear(context.COLOR_BUFFER_BIT); + context.drawArrays(context.TRIANGLES, 0, 6); +} + +export function createFramebufferTexture({ + context, + width, + height, +}: { + context: WebGLRenderingContext; + width: number; + height: number; +}): { texture: WebGLTexture; framebuffer: WebGLFramebuffer } { + const texture = context.createTexture(); + if (!texture) throw new Error("Failed to create framebuffer texture"); + context.bindTexture(context.TEXTURE_2D, texture); + context.texImage2D( + context.TEXTURE_2D, + 0, + context.RGBA, + width, + height, + 0, + context.RGBA, + context.UNSIGNED_BYTE, + null, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_WRAP_S, + context.CLAMP_TO_EDGE, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_WRAP_T, + context.CLAMP_TO_EDGE, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_MIN_FILTER, + context.LINEAR, + ); + context.texParameteri( + context.TEXTURE_2D, + context.TEXTURE_MAG_FILTER, + context.LINEAR, + ); + context.bindTexture(context.TEXTURE_2D, null); + + const framebuffer = context.createFramebuffer(); + if (!framebuffer) throw new Error("Failed to create framebuffer"); + context.bindFramebuffer(context.FRAMEBUFFER, framebuffer); + context.framebufferTexture2D( + context.FRAMEBUFFER, + context.COLOR_ATTACHMENT0, + context.TEXTURE_2D, + texture, + 0, + ); + context.bindFramebuffer(context.FRAMEBUFFER, null); + + return { texture, framebuffer }; +} + +export function applyMultiPassEffect({ + context, + source, + width, + height, + passes, + programCache, +}: { + context: WebGLRenderingContext; + source: CanvasImageSource; + width: number; + height: number; + passes: EffectPassData[]; + programCache: Map<string, WebGLProgram>; +}): void { + const sourceTexture = createTexture({ context, source }); + let currentTexture: WebGLTexture = sourceTexture; + + const intermediates: Array<{ + texture: WebGLTexture; + framebuffer: WebGLFramebuffer; + }> = []; + for (let i = 0; i < passes.length - 1; i++) { + intermediates.push(createFramebufferTexture({ context, width, height })); + } + + for (let i = 0; i < passes.length; i++) { + const pass = passes[i]; + const program = compileProgram({ + context, + fragmentShaderSource: pass.fragmentShader, + programCache, + }); + const isLastPass = i === passes.length - 1; + const targetFramebuffer = isLastPass ? null : intermediates[i].framebuffer; + + context.bindFramebuffer(context.FRAMEBUFFER, targetFramebuffer); + // biome-ignore lint/correctness/useHookAtTopLevel: WebGL API method, not a React hook + context.useProgram(program); + context.activeTexture(context.TEXTURE0); + context.bindTexture(context.TEXTURE_2D, currentTexture); + + const uTextureLocation = context.getUniformLocation(program, "u_texture"); + if (uTextureLocation) { + context.uniform1i(uTextureLocation, 0); + } + + setUniforms({ + context, + program, + uniforms: { ...pass.uniforms, u_resolution: [width, height] }, + }); + drawFullscreenQuad({ context, program, width, height }); + + if (!isLastPass) { + currentTexture = intermediates[i].texture; + } + } + + context.deleteTexture(sourceTexture); + for (const intermediate of intermediates) { + context.deleteTexture(intermediate.texture); + context.deleteFramebuffer(intermediate.framebuffer); + } + context.bindTexture(context.TEXTURE_2D, null); + context.bindFramebuffer(context.FRAMEBUFFER, null); +} diff --git a/apps/web/src/services/storage/migrations/__tests__/v8-to-v9.test.ts b/apps/web/src/services/storage/migrations/__tests__/v8-to-v9.test.ts new file mode 100644 index 00000000..51e91f78 --- /dev/null +++ b/apps/web/src/services/storage/migrations/__tests__/v8-to-v9.test.ts @@ -0,0 +1,175 @@ +import { describe, expect, test } from "bun:test"; +import { transformProjectV8ToV9 } from "../transformers/v8-to-v9"; + +const v8ProjectWithText = { + id: "project-v8-text", + version: 8, + metadata: { + id: "project-v8-text", + name: "V8 Project with Text", + createdAt: "2024-01-01T00:00:00.000Z", + updatedAt: "2024-01-01T00:00:00.000Z", + }, + settings: { + fps: 30, + canvasSize: { width: 1920, height: 1080 }, + background: { type: "color", color: "#000000" }, + }, + currentSceneId: "scene-main", + scenes: [ + { + id: "scene-main", + name: "Main scene", + isMain: true, + tracks: [ + { + id: "track-text", + type: "text", + name: "Text Track", + hidden: false, + elements: [ + { + id: "el-1", + type: "text", + content: "With color", + startTime: 0, + duration: 5, + background: { + color: "#ff0000", + cornerRadius: 0, + paddingX: 8, + paddingY: 4, + }, + }, + { + id: "el-2", + type: "text", + content: "Transparent", + startTime: 5, + duration: 5, + background: { + color: "transparent", + paddingX: 30, + paddingY: 42, + }, + }, + ], + }, + ], + bookmarks: [], + createdAt: "2024-01-01T00:00:00.000Z", + updatedAt: "2024-01-01T00:00:00.000Z", + }, + ], +} as Parameters<typeof transformProjectV8ToV9>[0]["project"]; + +describe("V8 to V9 Migration", () => { + test("adds background.enabled from color (transparent => false, otherwise true)", () => { + const result = transformProjectV8ToV9({ project: v8ProjectWithText }); + + expect(result.skipped).toBe(false); + expect(result.project.version).toBe(9); + + const track = ( + result.project.scenes as Array<{ tracks: Array<{ elements: unknown[] }> }> + )[0].tracks[0]; + const elements = track.elements as Array<{ background: { enabled: boolean; color: string } }>; + + expect(elements[0].background.enabled).toBe(true); + expect(elements[0].background.color).toBe("#ff0000"); + + expect(elements[1].background.enabled).toBe(false); + expect(elements[1].background.color).toBe("transparent"); + }); + + test("preserves existing background.enabled if already present", () => { + const projectWithEnabled = { + ...v8ProjectWithText, + scenes: [ + { + ...(v8ProjectWithText.scenes as Record<string, unknown>[])[0], + tracks: [ + { + id: "track-text", + type: "text", + name: "Text Track", + hidden: false, + elements: [ + { + id: "el-1", + type: "text", + content: "Already has enabled", + startTime: 0, + duration: 5, + background: { + enabled: false, + color: "#00ff00", + }, + }, + ], + }, + ], + }, + ], + } as Parameters<typeof transformProjectV8ToV9>[0]["project"]; + + const result = transformProjectV8ToV9({ project: projectWithEnabled }); + + expect(result.skipped).toBe(false); + const elements = ( + result.project.scenes as Array<{ tracks: Array<{ elements: unknown[] }> }> + )[0].tracks[0].elements as Array<{ background: { enabled: boolean } }>; + expect(elements[0].background.enabled).toBe(false); + }); + + test("skips non-text elements and tracks", () => { + const projectWithVideoOnly = { + ...v8ProjectWithText, + scenes: [ + { + id: "scene-main", + name: "Main scene", + isMain: true, + tracks: [ + { + id: "track-video", + type: "video", + name: "Video Track", + isMain: true, + elements: [], + }, + ], + bookmarks: [], + createdAt: "2024-01-01T00:00:00.000Z", + updatedAt: "2024-01-01T00:00:00.000Z", + }, + ], + } as Parameters<typeof transformProjectV8ToV9>[0]["project"]; + + const result = transformProjectV8ToV9({ project: projectWithVideoOnly }); + + expect(result.skipped).toBe(false); + expect(result.project.version).toBe(9); + }); + + test("skips projects that are already v9", () => { + const result = transformProjectV8ToV9({ + project: { ...v8ProjectWithText, version: 9 }, + }); + + expect(result.skipped).toBe(true); + expect(result.reason).toBe("already v9"); + }); + + test("skips projects with no id", () => { + const result = transformProjectV8ToV9({ + project: { + version: 8, + scenes: [], + } as Parameters<typeof transformProjectV8ToV9>[0]["project"], + }); + + expect(result.skipped).toBe(true); + expect(result.reason).toBe("no project id"); + }); +}); diff --git a/apps/web/src/services/storage/migrations/index.ts b/apps/web/src/services/storage/migrations/index.ts index fd40f14c..bffacb66 100644 --- a/apps/web/src/services/storage/migrations/index.ts +++ b/apps/web/src/services/storage/migrations/index.ts @@ -6,10 +6,12 @@ import { V3toV4Migration } from "./v3-to-v4"; import { V4toV5Migration } from "./v4-to-v5"; import { V5toV6Migration } from "./v5-to-v6"; import { V6toV7Migration } from "./v6-to-v7"; +import { V7toV8Migration } from "./v7-to-v8"; +import { V8toV9Migration } from "./v8-to-v9"; export { runStorageMigrations } from "./runner"; export type { MigrationProgress } from "./runner"; -export const CURRENT_PROJECT_VERSION = 7; +export const CURRENT_PROJECT_VERSION = 9; export const migrations = [ new V0toV1Migration(), @@ -19,4 +21,6 @@ export const migrations = [ new V4toV5Migration(), new V5toV6Migration(), new V6toV7Migration(), + new V7toV8Migration(), + new V8toV9Migration(), ]; diff --git a/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts b/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts index 098101d6..297d1c58 100644 --- a/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts +++ b/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts @@ -388,7 +388,7 @@ async function transformMediaTrack({ ); const validElements = transformedElements.filter( - (el): el is VideoElement | ImageElement => el !== null, + (element): element is VideoElement | ImageElement => element !== null, ); return { @@ -450,10 +450,18 @@ function transformTextTrack({ value: textElement.color, fallback: "#000000", }), - backgroundColor: getStringValue({ - value: textElement.backgroundColor, - fallback: "#FFFFFF", - }), + background: { + enabled: false, + color: getStringValue({ + value: textElement.backgroundColor, + fallback: "transparent", + }), + cornerRadius: 0, + paddingX: 8, + paddingY: 4, + offsetX: 0, + offsetY: 0, + }, textAlign: (getStringValue({ value: textElement.textAlign, fallback: "left", @@ -479,7 +487,7 @@ function transformTextTrack({ trimEnd: getNumberValue({ value: element.trimEnd, fallback: 0 }), }; }) - .filter((el): el is TextElement => el !== null); + .filter((element): element is TextElement => element !== null); return { id: getStringValue({ value: track.id, fallback: "" }), @@ -522,7 +530,7 @@ function transformAudioTrack({ trimEnd: getNumberValue({ value: element.trimEnd, fallback: 0 }), }; }) - .filter((el): el is AudioElement => el !== null); + .filter((element): element is AudioElement => element !== null); return { id: getStringValue({ value: track.id, fallback: "" }), diff --git a/apps/web/src/services/storage/migrations/transformers/v7-to-v8.ts b/apps/web/src/services/storage/migrations/transformers/v7-to-v8.ts new file mode 100644 index 00000000..7393f776 --- /dev/null +++ b/apps/web/src/services/storage/migrations/transformers/v7-to-v8.ts @@ -0,0 +1,96 @@ +import type { MigrationResult, ProjectRecord } from "./types"; +import { getProjectId, isRecord } from "./utils"; + +export function transformProjectV7ToV8({ + project, +}: { + project: ProjectRecord; +}): MigrationResult<ProjectRecord> { + const projectId = getProjectId({ project }); + if (!projectId) { + return { project, skipped: true, reason: "no project id" }; + } + + if (isV8Project({ project })) { + return { project, skipped: true, reason: "already v8" }; + } + + const migratedProject = migrateProjectElements({ project }); + + return { + project: { ...migratedProject, version: 8 }, + skipped: false, + }; +} + +function migrateProjectElements({ + project, +}: { + project: ProjectRecord; +}): ProjectRecord { + const scenesValue = project.scenes; + if (!Array.isArray(scenesValue)) return project; + + let hasChanges = false; + const migratedScenes = scenesValue.map((scene) => { + const migrated = migrateSceneElements({ scene }); + if (migrated !== scene) hasChanges = true; + return migrated; + }); + + if (!hasChanges) return project; + return { ...project, scenes: migratedScenes }; +} + +function migrateSceneElements({ scene }: { scene: unknown }): unknown { + if (!isRecord(scene)) return scene; + + const tracksValue = scene.tracks; + if (!Array.isArray(tracksValue)) return scene; + + let hasChanges = false; + const migratedTracks = tracksValue.map((track) => { + const migrated = migrateTrackElements({ track }); + if (migrated !== track) hasChanges = true; + return migrated; + }); + + if (!hasChanges) return scene; + return { ...scene, tracks: migratedTracks }; +} + +function migrateTrackElements({ track }: { track: unknown }): unknown { + if (!isRecord(track)) return track; + + const elementsValue = track.elements; + if (!Array.isArray(elementsValue)) return track; + + let hasChanges = false; + const migratedElements = elementsValue.map((element) => { + const migrated = migrateElement({ element }); + if (migrated !== element) hasChanges = true; + return migrated; + }); + + if (!hasChanges) return track; + return { ...track, elements: migratedElements }; +} + +function migrateElement({ element }: { element: unknown }): unknown { + if (!isRecord(element)) return element; + if (element.type !== "video" && element.type !== "audio") return element; + if (typeof element.sourceDuration === "number") return element; + + const trimStart = typeof element.trimStart === "number" ? element.trimStart : 0; + const duration = typeof element.duration === "number" ? element.duration : 0; + const trimEnd = typeof element.trimEnd === "number" ? element.trimEnd : 0; + + return { + ...element, + sourceDuration: trimStart + duration + trimEnd, + }; +} + +function isV8Project({ project }: { project: ProjectRecord }): boolean { + return typeof project.version === "number" && project.version >= 8; +} diff --git a/apps/web/src/services/storage/migrations/transformers/v8-to-v9.ts b/apps/web/src/services/storage/migrations/transformers/v8-to-v9.ts new file mode 100644 index 00000000..1dc895bc --- /dev/null +++ b/apps/web/src/services/storage/migrations/transformers/v8-to-v9.ts @@ -0,0 +1,99 @@ +import type { MigrationResult, ProjectRecord } from "./types"; +import { getProjectId, isRecord } from "./utils"; + +export function transformProjectV8ToV9({ + project, +}: { + project: ProjectRecord; +}): MigrationResult<ProjectRecord> { + const projectId = getProjectId({ project }); + if (!projectId) { + return { project, skipped: true, reason: "no project id" }; + } + + if (isV9Project({ project })) { + return { project, skipped: true, reason: "already v9" }; + } + + const migratedProject = migrateProjectTextElements({ project }); + + return { + project: { ...migratedProject, version: 9 }, + skipped: false, + }; +} + +function migrateProjectTextElements({ + project, +}: { + project: ProjectRecord; +}): ProjectRecord { + const scenesValue = project.scenes; + if (!Array.isArray(scenesValue)) return project; + + let hasChanges = false; + const migratedScenes = scenesValue.map((scene) => { + const migrated = migrateSceneTextElements({ scene }); + if (migrated !== scene) hasChanges = true; + return migrated; + }); + + if (!hasChanges) return project; + return { ...project, scenes: migratedScenes }; +} + +function migrateSceneTextElements({ scene }: { scene: unknown }): unknown { + if (!isRecord(scene)) return scene; + + const tracksValue = scene.tracks; + if (!Array.isArray(tracksValue)) return scene; + + let hasChanges = false; + const migratedTracks = tracksValue.map((track) => { + const migrated = migrateTrackTextElements({ track }); + if (migrated !== track) hasChanges = true; + return migrated; + }); + + if (!hasChanges) return scene; + return { ...scene, tracks: migratedTracks }; +} + +function migrateTrackTextElements({ track }: { track: unknown }): unknown { + if (!isRecord(track)) return track; + if (track.type !== "text") return track; + + const elementsValue = track.elements; + if (!Array.isArray(elementsValue)) return track; + + let hasChanges = false; + const migratedElements = elementsValue.map((element) => { + const migrated = migrateTextElement({ element }); + if (migrated !== element) hasChanges = true; + return migrated; + }); + + if (!hasChanges) return track; + return { ...track, elements: migratedElements }; +} + +function migrateTextElement({ element }: { element: unknown }): unknown { + if (!isRecord(element)) return element; + if (element.type !== "text") return element; + + const bg = element.background; + if (!isRecord(bg)) return element; + if (typeof bg.enabled === "boolean") return element; + + const color = typeof bg.color === "string" ? bg.color : "transparent"; + const enabled = color !== "transparent"; + + return { + ...element, + background: { ...bg, enabled }, + }; +} + +function isV9Project({ project }: { project: ProjectRecord }): boolean { + return typeof project.version === "number" && project.version >= 9; +} diff --git a/apps/web/src/services/storage/migrations/v7-to-v8.ts b/apps/web/src/services/storage/migrations/v7-to-v8.ts new file mode 100644 index 00000000..33372c0a --- /dev/null +++ b/apps/web/src/services/storage/migrations/v7-to-v8.ts @@ -0,0 +1,16 @@ +import { StorageMigration } from "./base"; +import type { ProjectRecord } from "./transformers/types"; +import { transformProjectV7ToV8 } from "./transformers/v7-to-v8"; + +export class V7toV8Migration extends StorageMigration { + from = 7; + to = 8; + + async transform(project: ProjectRecord): Promise<{ + project: ProjectRecord; + skipped: boolean; + reason?: string; + }> { + return transformProjectV7ToV8({ project }); + } +} diff --git a/apps/web/src/services/storage/migrations/v8-to-v9.ts b/apps/web/src/services/storage/migrations/v8-to-v9.ts new file mode 100644 index 00000000..6222eeac --- /dev/null +++ b/apps/web/src/services/storage/migrations/v8-to-v9.ts @@ -0,0 +1,16 @@ +import { StorageMigration } from "./base"; +import type { ProjectRecord } from "./transformers/types"; +import { transformProjectV8ToV9 } from "./transformers/v8-to-v9"; + +export class V8toV9Migration extends StorageMigration { + from = 8; + to = 9; + + async transform(project: ProjectRecord): Promise<{ + project: ProjectRecord; + skipped: boolean; + reason?: string; + }> { + return transformProjectV8ToV9({ project }); + } +} |