diff options
Diffstat (limited to 'apps/web/src/services/renderer')
4 files changed, 402 insertions, 259 deletions
diff --git a/apps/web/src/services/renderer/canvas-renderer.ts b/apps/web/src/services/renderer/canvas-renderer.ts index 50f7fd12..4a002675 100644 --- a/apps/web/src/services/renderer/canvas-renderer.ts +++ b/apps/web/src/services/renderer/canvas-renderer.ts @@ -3,6 +3,11 @@ import type { AnyBaseNode } from "./nodes/base-node"; import { buildFrameDescriptor } from "./compositor/frame-descriptor"; import { wasmCompositor } from "./compositor/wasm-compositor"; import { resolveRenderTree } from "./resolve"; +import { + measureSpanAsync, + measureSpanSync, + onRenderPerfFrameComplete, +} from "@/diagnostics/render-perf"; export type CanvasRendererParams = { width: number; @@ -69,17 +74,26 @@ export class CanvasRenderer { } async render({ node, time }: { node: AnyBaseNode; time: number }) { - await resolveRenderTree({ node, renderer: this, time }); - const { frame, textures } = await buildFrameDescriptor({ - node, - renderer: this, + await measureSpanAsync({ + name: "resolve", + fn: () => resolveRenderTree({ node, renderer: this, time }), + }); + const { frame, textures } = await measureSpanAsync({ + name: "buildFrame", + fn: () => buildFrameDescriptor({ node, renderer: this }), }); wasmCompositor.ensureInitialized({ width: this.width, height: this.height, }); - wasmCompositor.syncTextures(textures); - wasmCompositor.render(frame); + measureSpanSync({ + name: "syncTextures", + fn: () => wasmCompositor.syncTextures(textures), + }); + measureSpanSync({ + name: "renderFrame", + fn: () => wasmCompositor.render(frame), + }); } async renderToCanvas({ @@ -98,12 +112,17 @@ export class CanvasRenderer { throw new Error("Failed to get target canvas context"); } - ctx.drawImage( - wasmCompositor.getCanvas(), - 0, - 0, - targetCanvas.width, - targetCanvas.height, - ); + measureSpanSync({ + name: "drawImage", + fn: () => + ctx.drawImage( + wasmCompositor.getCanvas(), + 0, + 0, + targetCanvas.width, + targetCanvas.height, + ), + }); + onRenderPerfFrameComplete(); } } diff --git a/apps/web/src/services/renderer/compositor/frame-descriptor.ts b/apps/web/src/services/renderer/compositor/frame-descriptor.ts index 666bea03..190a0237 100644 --- a/apps/web/src/services/renderer/compositor/frame-descriptor.ts +++ b/apps/web/src/services/renderer/compositor/frame-descriptor.ts @@ -1,5 +1,6 @@ import { drawCssBackground } from "@/gradients"; import { masksRegistry } from "@/masks"; +import { incrementCounter } from "@/diagnostics/render-perf"; import type { AnyBaseNode } from "../nodes/base-node"; import type { CanvasRenderer } from "../canvas-renderer"; import { createOffscreenCanvas } from "../canvas-utils"; @@ -21,16 +22,11 @@ import type { FrameItemDescriptor, LayerMaskDescriptor, QuadTransformDescriptor, + TextureCanvasDrawFn, + TextureUploadDescriptor, } from "./types"; import { DEFAULT_GRAPHIC_SOURCE_SIZE } from "@/graphics"; -export type TextureUploadDescriptor = { - id: string; - source: CanvasImageSource; - width: number; - height: number; -}; - export async function buildFrameDescriptor({ node, renderer, @@ -52,6 +48,9 @@ export async function buildFrameDescriptor({ textures, }); + incrementCounter({ name: "frameItems", by: items.length }); + incrementCounter({ name: "frameTextures", by: textures.size }); + return { frame: { width: renderer.width, @@ -93,31 +92,21 @@ async function collectNode({ if (node instanceof ColorNode) { const textureId = `${path}:color`; - const canvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const ctx = canvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!ctx) return; - if (/gradient\(/i.test(node.params.color)) { - drawCssBackground({ - ctx, - width: renderer.width, - height: renderer.height, - css: node.params.color, - }); - } else { - ctx.fillStyle = node.params.color; - ctx.fillRect(0, 0, renderer.width, renderer.height); - } + const { width, height } = renderer; textures.set(textureId, { + kind: "rendered", id: textureId, - source: canvas, - width: renderer.width, - height: renderer.height, + contentHash: `color:${node.params.color}:${width}x${height}`, + width, + height, + draw: (ctx) => { + if (/gradient\(/i.test(node.params.color)) { + drawCssBackground({ ctx, width, height, css: node.params.color }); + } else { + ctx.fillStyle = node.params.color; + ctx.fillRect(0, 0, width, height); + } + }, }); items.push({ type: "layer", @@ -147,36 +136,35 @@ async function collectNode({ return; } const textureId = `${path}:blur-background`; - const backdropCanvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const backdropCtx = backdropCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!backdropCtx) return; + const { width, height } = renderer; const { backdropSource, passes } = node.resolved; - const coverScale = Math.max( - renderer.width / backdropSource.width, - renderer.height / backdropSource.height, - ); - const scaledWidth = backdropSource.width * coverScale; - const scaledHeight = backdropSource.height * coverScale; - const offsetX = (renderer.width - scaledWidth) / 2; - const offsetY = (renderer.height - scaledHeight) / 2; - backdropCtx.drawImage( - backdropSource.source, - offsetX, - offsetY, - scaledWidth, - scaledHeight, - ); + // Backdrop pixels come from a decoded video/image frame whose identity + // already changes when it changes. Hashing the source reference is + // enough to let us skip redraws on frozen frames. + const contentHash = `blur:${identityKey(backdropSource.source)}:${backdropSource.width}x${backdropSource.height}:${width}x${height}`; textures.set(textureId, { + kind: "rendered", id: textureId, - source: backdropCanvas, - width: renderer.width, - height: renderer.height, + contentHash, + width, + height, + draw: (ctx) => { + const coverScale = Math.max( + width / backdropSource.width, + height / backdropSource.height, + ); + const scaledWidth = backdropSource.width * coverScale; + const scaledHeight = backdropSource.height * coverScale; + const offsetX = (width - scaledWidth) / 2; + const offsetY = (height - scaledHeight) / 2; + ctx.drawImage( + backdropSource.source, + offsetX, + offsetY, + scaledWidth, + scaledHeight, + ); + }, }); items.push({ type: "layer", @@ -253,6 +241,7 @@ async function collectVisualSourceNode({ const textureId = `${path}:source`; textures.set(textureId, { + kind: "external", id: textureId, source, width: sourceWidth, @@ -305,28 +294,24 @@ function collectTextNode({ } const textureId = `${path}:text`; - const canvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const ctx = canvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!ctx) { - return; - } - - renderTextToContext({ - node, - ctx, - }); - + const { width, height } = renderer; + // Text output is fully determined by node.params + node.resolved. Both are + // plain data we can stringify cheaply; the resolved measured layout is the + // expensive part of text setup, so stringifying it here is orders of + // magnitude cheaper than re-rasterizing when nothing changed. + const contentHash = `text:${width}x${height}:${JSON.stringify({ + params: node.params, + resolved: node.resolved, + })}`; textures.set(textureId, { + kind: "rendered", id: textureId, - source: canvas, - width: renderer.width, - height: renderer.height, + contentHash, + width, + height, + draw: (ctx) => { + renderTextToContext({ node, ctx }); + }, }); items.push({ type: "layer", @@ -411,20 +396,6 @@ function buildMaskArtifacts({ return { mask: null, strokeLayer: null }; } - const elementMaskCanvas = createOffscreenCanvas({ - width: Math.round(transform.width), - height: Math.round(transform.height), - }); - const elementMaskCtx = elementMaskCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!elementMaskCtx) { - return { mask: null, strokeLayer: null }; - } - elementMaskCtx.clearRect(0, 0, transform.width, transform.height); - - let strokePath: Path2D | null = null; let feather = mask.params.feather; const canRenderMaskDirectly = Boolean(definition.renderer.renderMask); const shouldRenderMaskDirectly = @@ -432,81 +403,78 @@ function buildMaskArtifacts({ (!definition.renderer.buildPath || (mask.params.feather > 0 && definition.renderer.renderMaskHandlesFeather)); - if (shouldRenderMaskDirectly && definition.renderer.renderMask) { - definition.renderer.renderMask({ - resolvedParams: mask.params, - ctx: elementMaskCtx, + if ( + shouldRenderMaskDirectly && + definition.renderer.renderMaskHandlesFeather + ) { + feather = 0; + } + + const maskTextureId = `${path}:mask`; + const { width: canvasWidth, height: canvasHeight } = renderer; + const maskContentHash = `mask:${mask.type}:${JSON.stringify(mask.params)}:${transformHash(transform)}:${canvasWidth}x${canvasHeight}:direct=${shouldRenderMaskDirectly}`; + const drawMask: TextureCanvasDrawFn = (ctx) => { + const elementMaskCanvas = createOffscreenCanvas({ width: Math.round(transform.width), height: Math.round(transform.height), - feather: mask.params.feather, }); - if (definition.renderer.renderMaskHandlesFeather) { - feather = 0; - } - strokePath = - definition.renderer.buildStrokePath?.({ + const elementMaskCtx = elementMaskCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!elementMaskCtx) return; + + if (shouldRenderMaskDirectly && definition.renderer.renderMask) { + definition.renderer.renderMask({ resolvedParams: mask.params, - width: transform.width, - height: transform.height, - }) ?? null; - } else { - if (!definition.renderer.buildPath) { - return { mask: null, strokeLayer: null }; - } - const path2d = definition.renderer.buildPath({ - resolvedParams: mask.params, - width: transform.width, - height: transform.height, - }); - elementMaskCtx.fillStyle = "white"; - elementMaskCtx.fill(path2d); - strokePath = - definition.renderer.buildStrokePath?.({ + ctx: elementMaskCtx, + width: Math.round(transform.width), + height: Math.round(transform.height), + feather: mask.params.feather, + }); + } else if (definition.renderer.buildPath) { + const path2d = definition.renderer.buildPath({ resolvedParams: mask.params, width: transform.width, height: transform.height, - }) ?? path2d; - } - - const fullMaskCanvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const fullMaskCtx = fullMaskCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (!fullMaskCtx) { - return { mask: null, strokeLayer: null }; - } - drawTransformedCanvas({ - ctx: fullMaskCtx, - source: elementMaskCanvas, - transform, - }); + }); + elementMaskCtx.fillStyle = "white"; + elementMaskCtx.fill(path2d); + } else { + return; + } - const maskTextureId = `${path}:mask`; + drawTransformedCanvas({ ctx, source: elementMaskCanvas, transform }); + }; textures.set(maskTextureId, { + kind: "rendered", id: maskTextureId, - source: fullMaskCanvas, - width: renderer.width, - height: renderer.height, + contentHash: maskContentHash, + width: canvasWidth, + height: canvasHeight, + draw: drawMask, }); - let strokeLayer: FrameItemDescriptor | null = null; - if ( + const hasStroke = mask.params.strokeWidth > 0 && - (strokePath || definition.renderer.renderStroke) - ) { - const strokeCanvas = createOffscreenCanvas({ - width: Math.round(transform.width), - height: Math.round(transform.height), - }); - const strokeCtx = strokeCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (strokeCtx) { + (definition.renderer.renderStroke || + definition.renderer.buildStrokePath || + definition.renderer.buildPath); + let strokeLayer: FrameItemDescriptor | null = null; + if (hasStroke) { + const strokeTextureId = `${path}:mask-stroke`; + const strokeContentHash = `stroke:${mask.type}:${JSON.stringify(mask.params)}:${transformHash(transform)}:${canvasWidth}x${canvasHeight}`; + const drawStroke: TextureCanvasDrawFn = (ctx) => { + const strokeCanvas = createOffscreenCanvas({ + width: Math.round(transform.width), + height: Math.round(transform.height), + }); + const strokeCtx = strokeCanvas.getContext("2d") as + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | null; + if (!strokeCtx) return; + if (definition.renderer.renderStroke) { definition.renderer.renderStroke({ resolvedParams: mask.params, @@ -514,44 +482,44 @@ function buildMaskArtifacts({ width: transform.width, height: transform.height, }); - } else if (strokePath) { + } else { + const strokePath = + definition.renderer.buildStrokePath?.({ + resolvedParams: mask.params, + width: transform.width, + height: transform.height, + }) ?? + definition.renderer.buildPath?.({ + resolvedParams: mask.params, + width: transform.width, + height: transform.height, + }) ?? + null; + if (!strokePath) return; strokeCtx.strokeStyle = mask.params.strokeColor; strokeCtx.lineWidth = mask.params.strokeWidth; strokeCtx.stroke(strokePath); } - const fullStrokeCanvas = createOffscreenCanvas({ - width: renderer.width, - height: renderer.height, - }); - const fullStrokeCtx = fullStrokeCanvas.getContext("2d") as - | CanvasRenderingContext2D - | OffscreenCanvasRenderingContext2D - | null; - if (fullStrokeCtx) { - drawTransformedCanvas({ - ctx: fullStrokeCtx, - source: strokeCanvas, - transform, - }); - const strokeTextureId = `${path}:mask-stroke`; - textures.set(strokeTextureId, { - id: strokeTextureId, - source: fullStrokeCanvas, - width: renderer.width, - height: renderer.height, - }); - strokeLayer = { - type: "layer", - textureId: strokeTextureId, - transform: fullCanvasTransform(renderer), - opacity: 1, - blendMode: "normal", - effectPassGroups: [], - mask: null, - }; - } - } + drawTransformedCanvas({ ctx, source: strokeCanvas, transform }); + }; + textures.set(strokeTextureId, { + kind: "rendered", + id: strokeTextureId, + contentHash: strokeContentHash, + width: canvasWidth, + height: canvasHeight, + draw: drawStroke, + }); + strokeLayer = { + type: "layer", + textureId: strokeTextureId, + transform: fullCanvasTransform(renderer), + opacity: 1, + blendMode: "normal", + effectPassGroups: [], + mask: null, + }; } return { @@ -590,3 +558,23 @@ function drawTransformedCanvas({ ctx.drawImage(source, x, y, transform.width, transform.height); ctx.restore(); } + +function transformHash(transform: QuadTransformDescriptor): string { + return `${transform.centerX}:${transform.centerY}:${transform.width}:${transform.height}:${transform.rotationDegrees}:${transform.flipX ? 1 : 0}:${transform.flipY ? 1 : 0}`; +} + +// Stable identity key for CanvasImageSource. Using a WeakMap → counter keeps +// hash string length bounded and avoids holding sources alive. +const identityKeys = new WeakMap<object, number>(); +let nextIdentity = 1; +function identityKey(source: CanvasImageSource): string { + if (typeof source === "object" && source !== null) { + let key = identityKeys.get(source); + if (key === undefined) { + key = nextIdentity++; + identityKeys.set(source, key); + } + return `@${key}`; + } + return "@?"; +} diff --git a/apps/web/src/services/renderer/compositor/types.ts b/apps/web/src/services/renderer/compositor/types.ts index af5a58c2..1a509909 100644 --- a/apps/web/src/services/renderer/compositor/types.ts +++ b/apps/web/src/services/renderer/compositor/types.ts @@ -40,3 +40,39 @@ export type LayerMaskDescriptor = { feather: number; inverted: boolean; }; + +export type TextureCanvasDrawFn = ( + ctx: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, +) => void; + +/** + * A layer texture whose pixels come from somewhere outside the renderer — + * typically a decoded video/image frame or a sticker. Cached by reference + * identity of the source object. + */ +export type ExternalTextureDescriptor = { + kind: "external"; + id: string; + source: CanvasImageSource; + width: number; + height: number; +}; + +/** + * A layer texture that the renderer rasterizes from scene state (color fill, + * text layout, mask shape, blur backdrop). Cached by `contentHash`: when it + * matches the previous frame's hash for this id, the upload is skipped + * entirely and the persistent canvas is not even cleared. + */ +export type RenderedTextureDescriptor = { + kind: "rendered"; + id: string; + contentHash: string; + width: number; + height: number; + draw: TextureCanvasDrawFn; +}; + +export type TextureUploadDescriptor = + | ExternalTextureDescriptor + | RenderedTextureDescriptor; diff --git a/apps/web/src/services/renderer/compositor/wasm-compositor.ts b/apps/web/src/services/renderer/compositor/wasm-compositor.ts index 16b03853..10684f11 100644 --- a/apps/web/src/services/renderer/compositor/wasm-compositor.ts +++ b/apps/web/src/services/renderer/compositor/wasm-compositor.ts @@ -1,44 +1,39 @@ import { getCompositorCanvas, + getLastFrameProfile, initCompositor, releaseTexture, renderFrame, resizeCompositor, uploadTexture, } from "opencut-wasm"; -import type { FrameDescriptor } from "./types"; +import { + incrementCounter, + isRenderPerfEnabled, + recordWasmFrameProfile, +} from "@/diagnostics/render-perf"; +import type { + ExternalTextureDescriptor, + FrameDescriptor, + RenderedTextureDescriptor, + TextureUploadDescriptor, +} from "./types"; -function ensureOffscreenCanvas({ - source, - width, - height, - label, -}: { - source: CanvasImageSource; +/** + * One slot in the derived-texture cache. The OffscreenCanvas is persistent — + * we reuse and redraw into it across frames, which is the change that takes + * the WebGL upload path off the per-frame critical path for static content. + */ +type RenderedCacheEntry = { + kind: "rendered"; + canvas: OffscreenCanvas; + contentHash: string; width: number; height: number; - label: string; -}): OffscreenCanvas { - if (source instanceof OffscreenCanvas) { - return source; - } - - if (typeof OffscreenCanvas === "undefined") { - throw new Error(`OffscreenCanvas is required for ${label}`); - } - - const canvas = new OffscreenCanvas(width, height); - const context = canvas.getContext("2d"); - if (!context) { - throw new Error(`Failed to get 2d context for ${label}`); - } - context.clearRect(0, 0, width, height); - context.drawImage(source, 0, 0, width, height); - return canvas; -} +}; -export type TextureUploadDescriptor = { - id: string; +type ExternalCacheEntry = { + kind: "external"; source: CanvasImageSource; width: number; height: number; @@ -47,11 +42,7 @@ export type TextureUploadDescriptor = { class WasmCompositor { private canvas: HTMLCanvasElement | null = null; private initializedSize: { width: number; height: number } | null = null; - private retainedTextureIds = new Set<string>(); - private uploadedTextures = new Map< - string, - { source: CanvasImageSource; width: number; height: number } - >(); + private cache = new Map<string, RenderedCacheEntry | ExternalCacheEntry>(); ensureInitialized({ width, height }: { width: number; height: number }) { if (!this.canvas) { @@ -80,48 +71,157 @@ class WasmCompositor { syncTextures(textures: TextureUploadDescriptor[]) { const nextIds = new Set(textures.map((texture) => texture.id)); - for (const previousId of this.retainedTextureIds) { + for (const previousId of this.cache.keys()) { if (!nextIds.has(previousId)) { releaseTexture(previousId); - this.uploadedTextures.delete(previousId); + this.cache.delete(previousId); } } for (const texture of textures) { - const previousTexture = this.uploadedTextures.get(texture.id); - if ( - previousTexture?.source === texture.source && - previousTexture.width === texture.width && - previousTexture.height === texture.height - ) { - continue; + if (texture.kind === "external") { + this.syncExternalTexture(texture); + } else { + this.syncRenderedTexture(texture); } + } + } - const sourceCanvas = ensureOffscreenCanvas({ + render(frame: FrameDescriptor) { + renderFrame(frame); + if (isRenderPerfEnabled()) { + recordWasmFrameProfile( + getLastFrameProfile() as Array<{ name: string; durationMs: number }>, + ); + } + } + + private syncExternalTexture(texture: ExternalTextureDescriptor) { + const previous = this.cache.get(texture.id); + if ( + previous?.kind === "external" && + previous.source === texture.source && + previous.width === texture.width && + previous.height === texture.height + ) { + incrementCounter({ name: "textureCacheHit" }); + return; + } + + incrementCounter({ name: "textureUpload" }); + incrementCounter({ + name: "textureUploadPixels", + by: texture.width * texture.height, + }); + uploadTexture({ + id: texture.id, + source: ensureOffscreenCanvas({ source: texture.source, width: texture.width, height: texture.height, label: `texture upload ${texture.id}`, - }); - uploadTexture({ - id: texture.id, - source: sourceCanvas, - width: texture.width, - height: texture.height, - }); - this.uploadedTextures.set(texture.id, { - source: texture.source, - width: texture.width, - height: texture.height, - }); + }), + width: texture.width, + height: texture.height, + }); + this.cache.set(texture.id, { + kind: "external", + source: texture.source, + width: texture.width, + height: texture.height, + }); + } + + private syncRenderedTexture(texture: RenderedTextureDescriptor) { + const previous = this.cache.get(texture.id); + if ( + previous?.kind === "rendered" && + previous.contentHash === texture.contentHash && + previous.width === texture.width && + previous.height === texture.height + ) { + incrementCounter({ name: "textureCacheHit" }); + return; } - this.retainedTextureIds = nextIds; - } + const canvas = + previous?.kind === "rendered" && + previous.width === texture.width && + previous.height === texture.height + ? previous.canvas + : createBackingCanvas({ + width: texture.width, + height: texture.height, + }); - render(frame: FrameDescriptor) { - renderFrame(frame); + const ctx = canvas.getContext("2d"); + if (!ctx) { + throw new Error(`Failed to get 2d context for texture ${texture.id}`); + } + ctx.clearRect(0, 0, texture.width, texture.height); + texture.draw(ctx); + + incrementCounter({ name: "textureUpload" }); + incrementCounter({ + name: "textureUploadPixels", + by: texture.width * texture.height, + }); + uploadTexture({ + id: texture.id, + source: canvas, + width: texture.width, + height: texture.height, + }); + this.cache.set(texture.id, { + kind: "rendered", + canvas, + contentHash: texture.contentHash, + width: texture.width, + height: texture.height, + }); } } export const wasmCompositor = new WasmCompositor(); + +function createBackingCanvas({ + width, + height, +}: { + width: number; + height: number; +}): OffscreenCanvas { + if (typeof OffscreenCanvas === "undefined") { + throw new Error("OffscreenCanvas is not supported in this environment"); + } + return new OffscreenCanvas(width, height); +} + +function ensureOffscreenCanvas({ + source, + width, + height, + label, +}: { + source: CanvasImageSource; + width: number; + height: number; + label: string; +}): OffscreenCanvas { + if (source instanceof OffscreenCanvas) { + return source; + } + + if (typeof OffscreenCanvas === "undefined") { + throw new Error(`OffscreenCanvas is required for ${label}`); + } + + const canvas = new OffscreenCanvas(width, height); + const context = canvas.getContext("2d"); + if (!context) { + throw new Error(`Failed to get 2d context for ${label}`); + } + context.clearRect(0, 0, width, height); + context.drawImage(source, 0, 0, width, height); + return canvas; +} |