From e837e7504379e100ff3b56282577ec67bb88bec9 Mon Sep 17 00:00:00 2001 From: cashmere Date: Tue, 28 Jul 2026 19:41:14 +0200 Subject: feat: ripple-extend resize, timeline filmstrip, export hardening Yesterday's verified web session (gates green: 316 tests, tsc, build): - ripple-extend pushes right-hand chain on static-element resize - per-position video filmstrip thumbnails with zoom-adaptive density - fps-aware export bitrate, 2s keyframes, even-dim export canvas - freeze frame uses fixed 3s duration, prompt removed --- .../thumbnail-cache/__tests__/tiling.test.ts | 258 +++++++++++++++++++ apps/web/src/services/thumbnail-cache/service.ts | 272 +++++++++++++++++++++ apps/web/src/services/thumbnail-cache/tiling.ts | 121 +++++++++ 3 files changed, 651 insertions(+) create mode 100644 apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts create mode 100644 apps/web/src/services/thumbnail-cache/service.ts create mode 100644 apps/web/src/services/thumbnail-cache/tiling.ts (limited to 'apps/web/src/services/thumbnail-cache') diff --git a/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts b/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts new file mode 100644 index 00000000..1fe52bb1 --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts @@ -0,0 +1,258 @@ +import { describe, expect, test } from "bun:test"; +import { + buildThumbnailTileKey, + FILMSTRIP_MAX_INTERVAL_SECONDS, + FILMSTRIP_MIN_INTERVAL_SECONDS, + getFilmstripInterval, + getFilmstripSlotCount, + getFilmstripSlotSourceTime, + getVisibleSlotRange, + MEDIA_TILE_ASPECT_RATIO, +} from "../tiling"; + +const TRACK_HEIGHT_PX = 65; +const MIN_TILE_WIDTH_PX = TRACK_HEIGHT_PX * MEDIA_TILE_ASPECT_RATIO; + +describe("getFilmstripInterval", () => { + test("snaps to the nearest power of two seconds", () => { + expect( + getFilmstripInterval({ + pixelsPerSecond: 100, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(1); + expect( + getFilmstripInterval({ + pixelsPerSecond: 50, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(2); + expect( + getFilmstripInterval({ + pixelsPerSecond: 200, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(0.5); + }); + + test("keeps tile width near the minimum across zoom levels", () => { + for (const pixelsPerSecond of [5, 12, 25, 40, 80, 160, 320, 640]) { + const interval = getFilmstripInterval({ + pixelsPerSecond, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + const tileWidthPx = interval * pixelsPerSecond; + expect(tileWidthPx).toBeGreaterThanOrEqual(MIN_TILE_WIDTH_PX * 0.7); + expect(tileWidthPx).toBeLessThanOrEqual(MIN_TILE_WIDTH_PX * 1.45); + } + }); + + test("is stable under tiny zoom changes", () => { + const coarse = getFilmstripInterval({ + pixelsPerSecond: 100, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + const nudged = getFilmstripInterval({ + pixelsPerSecond: 110, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(nudged).toBe(coarse); + }); + + test("clamps the interval at extreme zoom levels", () => { + const zoomedIn = getFilmstripInterval({ + pixelsPerSecond: 5000, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(zoomedIn).toBeGreaterThanOrEqual(FILMSTRIP_MIN_INTERVAL_SECONDS); + + const zoomedOut = getFilmstripInterval({ + pixelsPerSecond: 5, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(zoomedOut).toBeLessThanOrEqual(FILMSTRIP_MAX_INTERVAL_SECONDS); + }); + + test("falls back to the max interval for invalid zoom", () => { + expect( + getFilmstripInterval({ + pixelsPerSecond: 0, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(FILMSTRIP_MAX_INTERVAL_SECONDS); + expect( + getFilmstripInterval({ + pixelsPerSecond: Number.NaN, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(FILMSTRIP_MAX_INTERVAL_SECONDS); + }); +}); + +describe("getFilmstripSlotCount", () => { + test("covers the clip duration with exact tiles", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 10, interval: 2 })).toBe(5); + expect(getFilmstripSlotCount({ clipDurationSec: 10.5, interval: 2 })).toBe( + 6, + ); + }); + + test("does not add a trailing tile on exact multiples", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 4, interval: 2 })).toBe(2); + }); + + test("returns zero for empty clips", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 0, interval: 2 })).toBe(0); + expect(getFilmstripSlotCount({ clipDurationSec: -3, interval: 2 })).toBe(0); + }); +}); + +describe("getVisibleSlotRange", () => { + test("returns the slots intersecting the visible window plus overscan", () => { + const range = getVisibleSlotRange({ + slotCount: 100, + tileWidthPx: 100, + visibleLeftPx: 500, + visibleRightPx: 1000, + overscanSlots: 2, + }); + expect(range).toEqual({ firstSlot: 3, lastSlot: 11 }); + }); + + test("clamps to the slot bounds", () => { + const range = getVisibleSlotRange({ + slotCount: 8, + tileWidthPx: 100, + visibleLeftPx: 0, + visibleRightPx: 1000, + overscanSlots: 2, + }); + expect(range).toEqual({ firstSlot: 0, lastSlot: 7 }); + }); + + test("returns an empty range when nothing is visible", () => { + const range = getVisibleSlotRange({ + slotCount: 10, + tileWidthPx: 100, + visibleLeftPx: 500, + visibleRightPx: 500, + overscanSlots: 2, + }); + expect(range.lastSlot).toBeLessThan(range.firstSlot); + }); +}); + +describe("getFilmstripSlotSourceTime", () => { + test("offsets by the trim start", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 2, + interval: 1, + trimStartSec: 5, + rate: 1, + }), + ).toBe(7); + }); + + test("scales clip time by the retime rate", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 2, + interval: 1, + trimStartSec: 5, + rate: 2, + }), + ).toBe(9); + }); + + test("clamps to the end of the asset", () => { + const clamped = getFilmstripSlotSourceTime({ + slot: 100, + interval: 1, + trimStartSec: 0, + rate: 1, + assetDurationSec: 10, + }); + expect(clamped).toBeLessThan(10); + expect(clamped).toBeGreaterThan(9); + }); + + test("never returns negative times", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 0, + interval: 1, + trimStartSec: -4, + rate: 1, + }), + ).toBe(0); + }); +}); + +describe("buildThumbnailTileKey", () => { + test("changes per media, size, and quantized time", () => { + const base = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-2", + time: 1, + width: 232, + height: 130, + }), + ); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-1", + time: 2, + width: 232, + height: 130, + }), + ); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 464, + height: 260, + }), + ); + }); + + test("shares keys for times within one quantum", () => { + const quantum = 1 / 4096; + const first = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + const second = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1 + quantum / 4, + width: 232, + height: 130, + }); + expect(second).toBe(first); + }); + + test("splits keys across quantum boundaries", () => { + const first = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + const second = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1 + 1 / 4096, + width: 232, + height: 130, + }); + expect(second).not.toBe(first); + }); +}); diff --git a/apps/web/src/services/thumbnail-cache/service.ts b/apps/web/src/services/thumbnail-cache/service.ts new file mode 100644 index 00000000..7002cd9c --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/service.ts @@ -0,0 +1,272 @@ +"use client"; + +import { + ALL_FORMATS, + BlobSource, + CanvasSink, + Input, + type WrappedCanvas, +} from "mediabunny"; +import { buildThumbnailTileKey } from "./tiling"; + +const MAX_TILE_ENTRIES = 400; +const MAX_SOURCE_ENTRIES = 8; +const MAX_CONCURRENT_FLUSHES = 2; +const TILE_JPEG_QUALITY = 0.7; +const CANVAS_POOL_SIZE = 2; + +interface PendingTile { + time: number; + resolve: (url: string | null) => void; +} + +interface TileSource { + width: number; + height: number; + init: Promise; + pending: PendingTile[]; + flushing: boolean; + flushScheduled: boolean; + dispose: () => void; +} + +function renderTileDataUrl({ + wrapped, +}: { + wrapped: WrappedCanvas; +}): string | null { + const canvas = wrapped.canvas; + if (canvas instanceof HTMLCanvasElement) { + return canvas.toDataURL("image/jpeg", TILE_JPEG_QUALITY); + } + return null; +} + +export class ThumbnailCache { + private tiles = new Map>(); + private sources = new Map(); + private activeFlushes = 0; + private flushQueue: Array<() => void> = []; + + getTile({ + mediaId, + file, + time, + width, + height, + }: { + mediaId: string; + file: File; + time: number; + width: number; + height: number; + }): Promise { + const key = buildThumbnailTileKey({ mediaId, time, width, height }); + const existing = this.tiles.get(key); + if (existing) { + this.tiles.delete(key); + this.tiles.set(key, existing); + return existing; + } + + const source = this.ensureSource({ mediaId, file, width, height }); + const requested = new Promise((resolve) => { + source.pending.push({ time, resolve }); + }); + this.scheduleFlush({ mediaId }); + + const tracked = requested.then((url) => { + if (url === null && this.tiles.get(key) === tracked) { + this.tiles.delete(key); + } + return url; + }); + + this.tiles.set(key, tracked); + this.evictTiles(); + return tracked; + } + + clearMedia({ mediaId }: { mediaId: string }): void { + for (const key of [...this.tiles.keys()]) { + if (key.startsWith(`${mediaId}:`)) { + this.tiles.delete(key); + } + } + const source = this.sources.get(mediaId); + if (source) { + this.sources.delete(mediaId); + this.releaseSource({ source }); + } + } + + clearAll(): void { + this.tiles.clear(); + for (const source of this.sources.values()) { + this.releaseSource({ source }); + } + this.sources.clear(); + } + + private releaseSource({ source }: { source: TileSource }): void { + const pending = source.pending.splice(0); + for (const tile of pending) { + tile.resolve(null); + } + source.dispose(); + } + + private ensureSource({ + mediaId, + file, + width, + height, + }: { + mediaId: string; + file: File; + width: number; + height: number; + }): TileSource { + const existing = this.sources.get(mediaId); + if (existing && existing.width === width && existing.height === height) { + this.sources.delete(mediaId); + this.sources.set(mediaId, existing); + return existing; + } + if (existing) { + this.sources.delete(mediaId); + this.releaseSource({ source: existing }); + } + + const input = new Input({ + source: new BlobSource(file), + formats: ALL_FORMATS, + }); + const init = input + .getPrimaryVideoTrack() + .then((track) => { + if (!track) { + return null; + } + return new CanvasSink(track, { + width, + height, + fit: "cover", + poolSize: CANVAS_POOL_SIZE, + }); + }) + .catch(() => null); + + const source: TileSource = { + width, + height, + init, + pending: [], + flushing: false, + flushScheduled: false, + dispose: () => { + void init.finally(() => input.dispose()); + }, + }; + this.sources.set(mediaId, source); + this.evictSources(); + return source; + } + + private evictTiles(): void { + while (this.tiles.size > MAX_TILE_ENTRIES) { + const oldestKey = this.tiles.keys().next().value; + if (oldestKey === undefined) { + return; + } + this.tiles.delete(oldestKey); + } + } + + private evictSources(): void { + while (this.sources.size > MAX_SOURCE_ENTRIES) { + const oldestKey = this.sources.keys().next().value; + if (oldestKey === undefined) { + return; + } + const oldest = this.sources.get(oldestKey); + this.sources.delete(oldestKey); + if (oldest) { + this.releaseSource({ source: oldest }); + } + } + } + + private scheduleFlush({ mediaId }: { mediaId: string }): void { + const source = this.sources.get(mediaId); + if (!source || source.flushScheduled) { + return; + } + source.flushScheduled = true; + queueMicrotask(() => { + source.flushScheduled = false; + void this.flush({ mediaId }); + }); + } + + private async flush({ mediaId }: { mediaId: string }): Promise { + const source = this.sources.get(mediaId); + if (!source || source.flushing || source.pending.length === 0) { + return; + } + source.flushing = true; + const batch = source.pending.splice(0).sort((a, b) => a.time - b.time); + + await this.acquireFlushSlot(); + try { + const sink = await source.init; + if (!sink) { + for (const tile of batch) { + tile.resolve(null); + } + return; + } + + const iterator = sink.canvasesAtTimestamps( + batch.map((tile) => tile.time), + ); + for (const tile of batch) { + const { value, done } = await iterator.next(); + tile.resolve( + !done && value ? renderTileDataUrl({ wrapped: value }) : null, + ); + } + } catch { + for (const tile of batch) { + tile.resolve(null); + } + } finally { + this.releaseFlushSlot(); + source.flushing = false; + if (source.pending.length > 0) { + this.scheduleFlush({ mediaId }); + } + } + } + + private async acquireFlushSlot(): Promise { + if (this.activeFlushes < MAX_CONCURRENT_FLUSHES) { + this.activeFlushes++; + return; + } + await new Promise((resolve) => { + this.flushQueue.push(resolve); + }); + this.activeFlushes++; + } + + private releaseFlushSlot(): void { + this.activeFlushes--; + const next = this.flushQueue.shift(); + if (next) { + next(); + } + } +} + +export const thumbnailCache = new ThumbnailCache(); diff --git a/apps/web/src/services/thumbnail-cache/tiling.ts b/apps/web/src/services/thumbnail-cache/tiling.ts new file mode 100644 index 00000000..bc8d31e5 --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/tiling.ts @@ -0,0 +1,121 @@ +export const MEDIA_TILE_ASPECT_RATIO = 16 / 9; +export const FILMSTRIP_MIN_INTERVAL_SECONDS = 0.1; +export const FILMSTRIP_MAX_INTERVAL_SECONDS = 64; +export const FILMSTRIP_OVERSCAN_SLOTS = 2; + +const SOURCE_TIME_END_PADDING_SECONDS = 0.05; +const TILE_TIME_QUANTUM_PER_SECOND = 4096; + +export function getFilmstripInterval({ + pixelsPerSecond, + minTileWidthPx, +}: { + pixelsPerSecond: number; + minTileWidthPx: number; +}): number { + if ( + !Number.isFinite(pixelsPerSecond) || + pixelsPerSecond <= 0 || + !Number.isFinite(minTileWidthPx) || + minTileWidthPx <= 0 + ) { + return FILMSTRIP_MAX_INTERVAL_SECONDS; + } + + const ideal = minTileWidthPx / pixelsPerSecond; + const clamped = Math.min( + FILMSTRIP_MAX_INTERVAL_SECONDS, + Math.max(FILMSTRIP_MIN_INTERVAL_SECONDS, ideal), + ); + return 2 ** Math.round(Math.log2(clamped)); +} + +export function getFilmstripSlotCount({ + clipDurationSec, + interval, +}: { + clipDurationSec: number; + interval: number; +}): number { + if ( + !Number.isFinite(clipDurationSec) || + clipDurationSec <= 0 || + interval <= 0 + ) { + return 0; + } + return Math.max(1, Math.ceil(clipDurationSec / interval - 1e-6)); +} + +export function getVisibleSlotRange({ + slotCount, + tileWidthPx, + visibleLeftPx, + visibleRightPx, + overscanSlots = FILMSTRIP_OVERSCAN_SLOTS, +}: { + slotCount: number; + tileWidthPx: number; + visibleLeftPx: number; + visibleRightPx: number; + overscanSlots?: number; +}): { firstSlot: number; lastSlot: number } { + if (slotCount <= 0 || tileWidthPx <= 0 || visibleRightPx <= visibleLeftPx) { + return { firstSlot: 0, lastSlot: -1 }; + } + + const firstSlot = Math.max( + 0, + Math.floor(visibleLeftPx / tileWidthPx) - overscanSlots, + ); + const lastSlot = Math.min( + slotCount - 1, + Math.ceil(visibleRightPx / tileWidthPx) - 1 + overscanSlots, + ); + return { firstSlot, lastSlot }; +} + +export function getFilmstripSlotSourceTime({ + slot, + interval, + trimStartSec, + rate, + assetDurationSec, +}: { + slot: number; + interval: number; + trimStartSec: number; + rate: number; + assetDurationSec?: number; +}): number { + const safeRate = Number.isFinite(rate) && rate > 0 ? rate : 1; + const raw = Math.max(0, trimStartSec + slot * interval * safeRate); + if ( + assetDurationSec === undefined || + !Number.isFinite(assetDurationSec) || + assetDurationSec <= 0 + ) { + return raw; + } + return Math.min( + raw, + Math.max(0, assetDurationSec - SOURCE_TIME_END_PADDING_SECONDS), + ); +} + +export function buildThumbnailTileKey({ + mediaId, + time, + width, + height, +}: { + mediaId: string; + time: number; + width: number; + height: number; +}): string { + const quantizedTime = Math.round( + Math.max(0, time) * TILE_TIME_QUANTUM_PER_SECOND, + ); + return `${mediaId}:${width}x${height}:${quantizedTime}`; +} -- cgit v1.3.1