diff options
| author | cashmere <cashmere@autistici.org> | 2026-07-28 19:41:14 +0200 |
|---|---|---|
| committer | cashmere <cashmere@autistici.org> | 2026-07-28 19:41:14 +0200 |
| commit | e837e7504379e100ff3b56282577ec67bb88bec9 (patch) | |
| tree | eec7e6f964725a783a9e907e931862591bb46b55 /apps/web/src/services | |
| parent | da99dec8a2de1f3849aebbf5695c1e812b2dbdf9 (diff) | |
| download | opencut-e837e7504379e100ff3b56282577ec67bb88bec9.tar.gz | |
feat: ripple-extend resize, timeline filmstrip, export hardening
Yesterday's verified web session (gates green: 316 tests, tsc, build):
- ripple-extend pushes right-hand chain on static-element resize
- per-position video filmstrip thumbnails with zoom-adaptive density
- fps-aware export bitrate, 2s keyframes, even-dim export canvas
- freeze frame uses fixed 3s duration, prompt removed
Diffstat (limited to 'apps/web/src/services')
| -rw-r--r-- | apps/web/src/services/renderer/__tests__/export-encoding.test.ts | 60 | ||||
| -rw-r--r-- | apps/web/src/services/renderer/export-encoding.ts | 22 | ||||
| -rw-r--r-- | apps/web/src/services/renderer/scene-exporter.ts | 17 | ||||
| -rw-r--r-- | apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts | 258 | ||||
| -rw-r--r-- | apps/web/src/services/thumbnail-cache/service.ts | 272 | ||||
| -rw-r--r-- | apps/web/src/services/thumbnail-cache/tiling.ts | 121 |
6 files changed, 747 insertions, 3 deletions
diff --git a/apps/web/src/services/renderer/__tests__/export-encoding.test.ts b/apps/web/src/services/renderer/__tests__/export-encoding.test.ts new file mode 100644 index 00000000..fbf354eb --- /dev/null +++ b/apps/web/src/services/renderer/__tests__/export-encoding.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from "bun:test"; +import { computeVideoBitrate } from "../export-encoding"; + +describe("computeVideoBitrate", () => { + test("scales linearly with frame rate at the same resolution and quality", () => { + const at30 = computeVideoBitrate({ + width: 1920, + height: 1080, + fps: 30, + quality: "high", + }); + const at60 = computeVideoBitrate({ + width: 1920, + height: 1080, + fps: 60, + quality: "high", + }); + + expect(at60).toBe(at30 * 2); + }); + + test("applies the bits-per-pixel ladder per quality tier", () => { + const params = { width: 1920, height: 1080, fps: 30 }; + + expect(computeVideoBitrate({ ...params, quality: "low" })).toBe(3_110_400); + expect(computeVideoBitrate({ ...params, quality: "medium" })).toBe( + 6_220_800, + ); + expect(computeVideoBitrate({ ...params, quality: "high" })).toBe( + 12_441_600, + ); + expect(computeVideoBitrate({ ...params, quality: "very_high" })).toBe( + 24_883_200, + ); + }); + + test("increases monotonically across quality tiers", () => { + const params = { width: 1280, height: 720, fps: 30 }; + const bitrates = (["low", "medium", "high", "very_high"] as const).map( + (quality) => computeVideoBitrate({ ...params, quality }), + ); + + for (let i = 1; i < bitrates.length; i++) { + expect(bitrates[i]).toBeGreaterThan(bitrates[i - 1]); + } + }); + + test("returns a positive integer for fractional frame rates", () => { + const bitrate = computeVideoBitrate({ + width: 1920, + height: 1080, + fps: 29.97, + quality: "high", + }); + + expect(bitrate).toBe(12_429_158); + expect(Number.isInteger(bitrate)).toBe(true); + expect(bitrate).toBeGreaterThan(0); + }); +}); diff --git a/apps/web/src/services/renderer/export-encoding.ts b/apps/web/src/services/renderer/export-encoding.ts new file mode 100644 index 00000000..cc333ea4 --- /dev/null +++ b/apps/web/src/services/renderer/export-encoding.ts @@ -0,0 +1,22 @@ +import type { ExportQuality } from "@/export"; + +const videoBitsPerPixelFrame: Record<ExportQuality, number> = { + low: 0.05, + medium: 0.1, + high: 0.2, + very_high: 0.4, +}; + +export function computeVideoBitrate({ + width, + height, + fps, + quality, +}: { + width: number; + height: number; + fps: number; + quality: ExportQuality; +}): number { + return Math.round(width * height * fps * videoBitsPerPixelFrame[quality]); +} diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts index 66df9485..04c14277 100644 --- a/apps/web/src/services/renderer/scene-exporter.ts +++ b/apps/web/src/services/renderer/scene-exporter.ts @@ -19,6 +19,7 @@ import { frameRateToFloat } from "@/fps/utils"; import type { RootNode } from "./nodes/root-node"; import type { ExportFormat, ExportQuality } from "@/export"; import { CanvasRenderer } from "./canvas-renderer"; +import { computeVideoBitrate } from "./export-encoding"; type ExportParams = { width: number; @@ -30,13 +31,15 @@ type ExportParams = { audioBuffer?: AudioBuffer; }; -const qualityMap = { +const audioQualityMap = { low: QUALITY_LOW, medium: QUALITY_MEDIUM, high: QUALITY_HIGH, very_high: QUALITY_VERY_HIGH, }; +const KEY_FRAME_INTERVAL_SECONDS = 2; + export type SceneExporterEvents = { progress: [progress: number]; complete: [buffer: ArrayBuffer]; @@ -99,9 +102,17 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> { target: new BufferTarget(), }); + const videoBitrate = computeVideoBitrate({ + width: this.renderer.width, + height: this.renderer.height, + fps: fpsFloat, + quality: this.quality, + }); + const videoSource = new CanvasSource(this.renderer.getOutputCanvas(), { codec: this.format === "webm" ? "vp9" : "avc", - bitrate: qualityMap[this.quality], + bitrate: videoBitrate, + keyFrameInterval: KEY_FRAME_INTERVAL_SECONDS, }); output.addVideoTrack(videoSource, { frameRate: fpsFloat }); @@ -122,7 +133,7 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> { audioSource = new AudioBufferSource({ codec: audioCodec, - bitrate: qualityMap[this.quality], + bitrate: audioQualityMap[this.quality], }); output.addAudioTrack(audioSource); } diff --git a/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts b/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts new file mode 100644 index 00000000..1fe52bb1 --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts @@ -0,0 +1,258 @@ +import { describe, expect, test } from "bun:test"; +import { + buildThumbnailTileKey, + FILMSTRIP_MAX_INTERVAL_SECONDS, + FILMSTRIP_MIN_INTERVAL_SECONDS, + getFilmstripInterval, + getFilmstripSlotCount, + getFilmstripSlotSourceTime, + getVisibleSlotRange, + MEDIA_TILE_ASPECT_RATIO, +} from "../tiling"; + +const TRACK_HEIGHT_PX = 65; +const MIN_TILE_WIDTH_PX = TRACK_HEIGHT_PX * MEDIA_TILE_ASPECT_RATIO; + +describe("getFilmstripInterval", () => { + test("snaps to the nearest power of two seconds", () => { + expect( + getFilmstripInterval({ + pixelsPerSecond: 100, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(1); + expect( + getFilmstripInterval({ + pixelsPerSecond: 50, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(2); + expect( + getFilmstripInterval({ + pixelsPerSecond: 200, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(0.5); + }); + + test("keeps tile width near the minimum across zoom levels", () => { + for (const pixelsPerSecond of [5, 12, 25, 40, 80, 160, 320, 640]) { + const interval = getFilmstripInterval({ + pixelsPerSecond, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + const tileWidthPx = interval * pixelsPerSecond; + expect(tileWidthPx).toBeGreaterThanOrEqual(MIN_TILE_WIDTH_PX * 0.7); + expect(tileWidthPx).toBeLessThanOrEqual(MIN_TILE_WIDTH_PX * 1.45); + } + }); + + test("is stable under tiny zoom changes", () => { + const coarse = getFilmstripInterval({ + pixelsPerSecond: 100, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + const nudged = getFilmstripInterval({ + pixelsPerSecond: 110, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(nudged).toBe(coarse); + }); + + test("clamps the interval at extreme zoom levels", () => { + const zoomedIn = getFilmstripInterval({ + pixelsPerSecond: 5000, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(zoomedIn).toBeGreaterThanOrEqual(FILMSTRIP_MIN_INTERVAL_SECONDS); + + const zoomedOut = getFilmstripInterval({ + pixelsPerSecond: 5, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(zoomedOut).toBeLessThanOrEqual(FILMSTRIP_MAX_INTERVAL_SECONDS); + }); + + test("falls back to the max interval for invalid zoom", () => { + expect( + getFilmstripInterval({ + pixelsPerSecond: 0, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(FILMSTRIP_MAX_INTERVAL_SECONDS); + expect( + getFilmstripInterval({ + pixelsPerSecond: Number.NaN, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(FILMSTRIP_MAX_INTERVAL_SECONDS); + }); +}); + +describe("getFilmstripSlotCount", () => { + test("covers the clip duration with exact tiles", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 10, interval: 2 })).toBe(5); + expect(getFilmstripSlotCount({ clipDurationSec: 10.5, interval: 2 })).toBe( + 6, + ); + }); + + test("does not add a trailing tile on exact multiples", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 4, interval: 2 })).toBe(2); + }); + + test("returns zero for empty clips", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 0, interval: 2 })).toBe(0); + expect(getFilmstripSlotCount({ clipDurationSec: -3, interval: 2 })).toBe(0); + }); +}); + +describe("getVisibleSlotRange", () => { + test("returns the slots intersecting the visible window plus overscan", () => { + const range = getVisibleSlotRange({ + slotCount: 100, + tileWidthPx: 100, + visibleLeftPx: 500, + visibleRightPx: 1000, + overscanSlots: 2, + }); + expect(range).toEqual({ firstSlot: 3, lastSlot: 11 }); + }); + + test("clamps to the slot bounds", () => { + const range = getVisibleSlotRange({ + slotCount: 8, + tileWidthPx: 100, + visibleLeftPx: 0, + visibleRightPx: 1000, + overscanSlots: 2, + }); + expect(range).toEqual({ firstSlot: 0, lastSlot: 7 }); + }); + + test("returns an empty range when nothing is visible", () => { + const range = getVisibleSlotRange({ + slotCount: 10, + tileWidthPx: 100, + visibleLeftPx: 500, + visibleRightPx: 500, + overscanSlots: 2, + }); + expect(range.lastSlot).toBeLessThan(range.firstSlot); + }); +}); + +describe("getFilmstripSlotSourceTime", () => { + test("offsets by the trim start", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 2, + interval: 1, + trimStartSec: 5, + rate: 1, + }), + ).toBe(7); + }); + + test("scales clip time by the retime rate", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 2, + interval: 1, + trimStartSec: 5, + rate: 2, + }), + ).toBe(9); + }); + + test("clamps to the end of the asset", () => { + const clamped = getFilmstripSlotSourceTime({ + slot: 100, + interval: 1, + trimStartSec: 0, + rate: 1, + assetDurationSec: 10, + }); + expect(clamped).toBeLessThan(10); + expect(clamped).toBeGreaterThan(9); + }); + + test("never returns negative times", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 0, + interval: 1, + trimStartSec: -4, + rate: 1, + }), + ).toBe(0); + }); +}); + +describe("buildThumbnailTileKey", () => { + test("changes per media, size, and quantized time", () => { + const base = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-2", + time: 1, + width: 232, + height: 130, + }), + ); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-1", + time: 2, + width: 232, + height: 130, + }), + ); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 464, + height: 260, + }), + ); + }); + + test("shares keys for times within one quantum", () => { + const quantum = 1 / 4096; + const first = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + const second = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1 + quantum / 4, + width: 232, + height: 130, + }); + expect(second).toBe(first); + }); + + test("splits keys across quantum boundaries", () => { + const first = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + const second = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1 + 1 / 4096, + width: 232, + height: 130, + }); + expect(second).not.toBe(first); + }); +}); diff --git a/apps/web/src/services/thumbnail-cache/service.ts b/apps/web/src/services/thumbnail-cache/service.ts new file mode 100644 index 00000000..7002cd9c --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/service.ts @@ -0,0 +1,272 @@ +"use client"; + +import { + ALL_FORMATS, + BlobSource, + CanvasSink, + Input, + type WrappedCanvas, +} from "mediabunny"; +import { buildThumbnailTileKey } from "./tiling"; + +const MAX_TILE_ENTRIES = 400; +const MAX_SOURCE_ENTRIES = 8; +const MAX_CONCURRENT_FLUSHES = 2; +const TILE_JPEG_QUALITY = 0.7; +const CANVAS_POOL_SIZE = 2; + +interface PendingTile { + time: number; + resolve: (url: string | null) => void; +} + +interface TileSource { + width: number; + height: number; + init: Promise<CanvasSink | null>; + pending: PendingTile[]; + flushing: boolean; + flushScheduled: boolean; + dispose: () => void; +} + +function renderTileDataUrl({ + wrapped, +}: { + wrapped: WrappedCanvas; +}): string | null { + const canvas = wrapped.canvas; + if (canvas instanceof HTMLCanvasElement) { + return canvas.toDataURL("image/jpeg", TILE_JPEG_QUALITY); + } + return null; +} + +export class ThumbnailCache { + private tiles = new Map<string, Promise<string | null>>(); + private sources = new Map<string, TileSource>(); + private activeFlushes = 0; + private flushQueue: Array<() => void> = []; + + getTile({ + mediaId, + file, + time, + width, + height, + }: { + mediaId: string; + file: File; + time: number; + width: number; + height: number; + }): Promise<string | null> { + const key = buildThumbnailTileKey({ mediaId, time, width, height }); + const existing = this.tiles.get(key); + if (existing) { + this.tiles.delete(key); + this.tiles.set(key, existing); + return existing; + } + + const source = this.ensureSource({ mediaId, file, width, height }); + const requested = new Promise<string | null>((resolve) => { + source.pending.push({ time, resolve }); + }); + this.scheduleFlush({ mediaId }); + + const tracked = requested.then((url) => { + if (url === null && this.tiles.get(key) === tracked) { + this.tiles.delete(key); + } + return url; + }); + + this.tiles.set(key, tracked); + this.evictTiles(); + return tracked; + } + + clearMedia({ mediaId }: { mediaId: string }): void { + for (const key of [...this.tiles.keys()]) { + if (key.startsWith(`${mediaId}:`)) { + this.tiles.delete(key); + } + } + const source = this.sources.get(mediaId); + if (source) { + this.sources.delete(mediaId); + this.releaseSource({ source }); + } + } + + clearAll(): void { + this.tiles.clear(); + for (const source of this.sources.values()) { + this.releaseSource({ source }); + } + this.sources.clear(); + } + + private releaseSource({ source }: { source: TileSource }): void { + const pending = source.pending.splice(0); + for (const tile of pending) { + tile.resolve(null); + } + source.dispose(); + } + + private ensureSource({ + mediaId, + file, + width, + height, + }: { + mediaId: string; + file: File; + width: number; + height: number; + }): TileSource { + const existing = this.sources.get(mediaId); + if (existing && existing.width === width && existing.height === height) { + this.sources.delete(mediaId); + this.sources.set(mediaId, existing); + return existing; + } + if (existing) { + this.sources.delete(mediaId); + this.releaseSource({ source: existing }); + } + + const input = new Input({ + source: new BlobSource(file), + formats: ALL_FORMATS, + }); + const init = input + .getPrimaryVideoTrack() + .then((track) => { + if (!track) { + return null; + } + return new CanvasSink(track, { + width, + height, + fit: "cover", + poolSize: CANVAS_POOL_SIZE, + }); + }) + .catch(() => null); + + const source: TileSource = { + width, + height, + init, + pending: [], + flushing: false, + flushScheduled: false, + dispose: () => { + void init.finally(() => input.dispose()); + }, + }; + this.sources.set(mediaId, source); + this.evictSources(); + return source; + } + + private evictTiles(): void { + while (this.tiles.size > MAX_TILE_ENTRIES) { + const oldestKey = this.tiles.keys().next().value; + if (oldestKey === undefined) { + return; + } + this.tiles.delete(oldestKey); + } + } + + private evictSources(): void { + while (this.sources.size > MAX_SOURCE_ENTRIES) { + const oldestKey = this.sources.keys().next().value; + if (oldestKey === undefined) { + return; + } + const oldest = this.sources.get(oldestKey); + this.sources.delete(oldestKey); + if (oldest) { + this.releaseSource({ source: oldest }); + } + } + } + + private scheduleFlush({ mediaId }: { mediaId: string }): void { + const source = this.sources.get(mediaId); + if (!source || source.flushScheduled) { + return; + } + source.flushScheduled = true; + queueMicrotask(() => { + source.flushScheduled = false; + void this.flush({ mediaId }); + }); + } + + private async flush({ mediaId }: { mediaId: string }): Promise<void> { + const source = this.sources.get(mediaId); + if (!source || source.flushing || source.pending.length === 0) { + return; + } + source.flushing = true; + const batch = source.pending.splice(0).sort((a, b) => a.time - b.time); + + await this.acquireFlushSlot(); + try { + const sink = await source.init; + if (!sink) { + for (const tile of batch) { + tile.resolve(null); + } + return; + } + + const iterator = sink.canvasesAtTimestamps( + batch.map((tile) => tile.time), + ); + for (const tile of batch) { + const { value, done } = await iterator.next(); + tile.resolve( + !done && value ? renderTileDataUrl({ wrapped: value }) : null, + ); + } + } catch { + for (const tile of batch) { + tile.resolve(null); + } + } finally { + this.releaseFlushSlot(); + source.flushing = false; + if (source.pending.length > 0) { + this.scheduleFlush({ mediaId }); + } + } + } + + private async acquireFlushSlot(): Promise<void> { + if (this.activeFlushes < MAX_CONCURRENT_FLUSHES) { + this.activeFlushes++; + return; + } + await new Promise<void>((resolve) => { + this.flushQueue.push(resolve); + }); + this.activeFlushes++; + } + + private releaseFlushSlot(): void { + this.activeFlushes--; + const next = this.flushQueue.shift(); + if (next) { + next(); + } + } +} + +export const thumbnailCache = new ThumbnailCache(); diff --git a/apps/web/src/services/thumbnail-cache/tiling.ts b/apps/web/src/services/thumbnail-cache/tiling.ts new file mode 100644 index 00000000..bc8d31e5 --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/tiling.ts @@ -0,0 +1,121 @@ +export const MEDIA_TILE_ASPECT_RATIO = 16 / 9; +export const FILMSTRIP_MIN_INTERVAL_SECONDS = 0.1; +export const FILMSTRIP_MAX_INTERVAL_SECONDS = 64; +export const FILMSTRIP_OVERSCAN_SLOTS = 2; + +const SOURCE_TIME_END_PADDING_SECONDS = 0.05; +const TILE_TIME_QUANTUM_PER_SECOND = 4096; + +export function getFilmstripInterval({ + pixelsPerSecond, + minTileWidthPx, +}: { + pixelsPerSecond: number; + minTileWidthPx: number; +}): number { + if ( + !Number.isFinite(pixelsPerSecond) || + pixelsPerSecond <= 0 || + !Number.isFinite(minTileWidthPx) || + minTileWidthPx <= 0 + ) { + return FILMSTRIP_MAX_INTERVAL_SECONDS; + } + + const ideal = minTileWidthPx / pixelsPerSecond; + const clamped = Math.min( + FILMSTRIP_MAX_INTERVAL_SECONDS, + Math.max(FILMSTRIP_MIN_INTERVAL_SECONDS, ideal), + ); + return 2 ** Math.round(Math.log2(clamped)); +} + +export function getFilmstripSlotCount({ + clipDurationSec, + interval, +}: { + clipDurationSec: number; + interval: number; +}): number { + if ( + !Number.isFinite(clipDurationSec) || + clipDurationSec <= 0 || + interval <= 0 + ) { + return 0; + } + return Math.max(1, Math.ceil(clipDurationSec / interval - 1e-6)); +} + +export function getVisibleSlotRange({ + slotCount, + tileWidthPx, + visibleLeftPx, + visibleRightPx, + overscanSlots = FILMSTRIP_OVERSCAN_SLOTS, +}: { + slotCount: number; + tileWidthPx: number; + visibleLeftPx: number; + visibleRightPx: number; + overscanSlots?: number; +}): { firstSlot: number; lastSlot: number } { + if (slotCount <= 0 || tileWidthPx <= 0 || visibleRightPx <= visibleLeftPx) { + return { firstSlot: 0, lastSlot: -1 }; + } + + const firstSlot = Math.max( + 0, + Math.floor(visibleLeftPx / tileWidthPx) - overscanSlots, + ); + const lastSlot = Math.min( + slotCount - 1, + Math.ceil(visibleRightPx / tileWidthPx) - 1 + overscanSlots, + ); + return { firstSlot, lastSlot }; +} + +export function getFilmstripSlotSourceTime({ + slot, + interval, + trimStartSec, + rate, + assetDurationSec, +}: { + slot: number; + interval: number; + trimStartSec: number; + rate: number; + assetDurationSec?: number; +}): number { + const safeRate = Number.isFinite(rate) && rate > 0 ? rate : 1; + const raw = Math.max(0, trimStartSec + slot * interval * safeRate); + if ( + assetDurationSec === undefined || + !Number.isFinite(assetDurationSec) || + assetDurationSec <= 0 + ) { + return raw; + } + return Math.min( + raw, + Math.max(0, assetDurationSec - SOURCE_TIME_END_PADDING_SECONDS), + ); +} + +export function buildThumbnailTileKey({ + mediaId, + time, + width, + height, +}: { + mediaId: string; + time: number; + width: number; + height: number; +}): string { + const quantizedTime = Math.round( + Math.max(0, time) * TILE_TIME_QUANTUM_PER_SECOND, + ); + return `${mediaId}:${width}x${height}:${quantizedTime}`; +} |
