diff options
Diffstat (limited to 'apps/web')
23 files changed, 1755 insertions, 383 deletions
diff --git a/apps/web/.gitignore b/apps/web/.gitignore index 2c4edca4..ddb836b6 100644 --- a/apps/web/.gitignore +++ b/apps/web/.gitignore @@ -11,4 +11,4 @@ next-env.d.ts
-*.tsbuildinfo
\ No newline at end of file +*.tsbuildinfo
diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index 71d96c23..66babe11 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -1,54 +1,55 @@ -import type { NextConfig } from "next";
-import { withBotId } from "botid/next/config";
-import { withContentCollections } from "@content-collections/next";
-
-const nextConfig: NextConfig = {
- compiler: {
- removeConsole: process.env.NODE_ENV === "production",
- },
- reactStrictMode: true,
- productionBrowserSourceMaps: true,
- output: "standalone",
- images: {
- remotePatterns: [
- {
- protocol: "https",
- hostname: "plus.unsplash.com",
- },
- {
- protocol: "https",
- hostname: "images.unsplash.com",
- },
- {
- protocol: "https",
- hostname: "images.marblecms.com",
- },
- {
- protocol: "https",
- hostname: "lh3.googleusercontent.com",
- },
- {
- protocol: "https",
- hostname: "avatars.githubusercontent.com",
- },
- {
- protocol: "https",
- hostname: "api.iconify.design",
- },
- {
- protocol: "https",
- hostname: "api.simplesvg.com",
- },
- {
- protocol: "https",
- hostname: "api.unisvg.com",
- },
- {
- protocol: "https",
- hostname: "cdn.brandfetch.io",
- },
- ],
- },
-};
-
-export default withContentCollections(withBotId(nextConfig));
+import type { NextConfig } from "next"; +import { withBotId } from "botid/next/config"; +import { withContentCollections } from "@content-collections/next"; + +const nextConfig: NextConfig = { + compiler: { + removeConsole: process.env.NODE_ENV === "production", + }, + devIndicators: false, + reactStrictMode: true, + productionBrowserSourceMaps: true, + output: "standalone", + images: { + remotePatterns: [ + { + protocol: "https", + hostname: "plus.unsplash.com", + }, + { + protocol: "https", + hostname: "images.unsplash.com", + }, + { + protocol: "https", + hostname: "images.marblecms.com", + }, + { + protocol: "https", + hostname: "lh3.googleusercontent.com", + }, + { + protocol: "https", + hostname: "avatars.githubusercontent.com", + }, + { + protocol: "https", + hostname: "api.iconify.design", + }, + { + protocol: "https", + hostname: "api.simplesvg.com", + }, + { + protocol: "https", + hostname: "api.unisvg.com", + }, + { + protocol: "https", + hostname: "cdn.brandfetch.io", + }, + ], + }, +}; + +export default withContentCollections(withBotId(nextConfig)); diff --git a/apps/web/src/actions/__tests__/freeze-frame-store.test.ts b/apps/web/src/actions/__tests__/freeze-frame-store.test.ts deleted file mode 100644 index 94e1566e..00000000 --- a/apps/web/src/actions/__tests__/freeze-frame-store.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { beforeEach, describe, expect, test } from "bun:test"; -import { - DEFAULT_FREEZE_DURATION_SECONDS, - isValidFreezeDuration, - MAX_FREEZE_DURATION_SECONDS, - useFreezeFrameStore, -} from "../freeze-frame-store"; - -function resetStore() { - useFreezeFrameStore.setState({ - isOpen: false, - lastDurationSeconds: DEFAULT_FREEZE_DURATION_SECONDS, - resolver: null, - }); -} - -describe("freeze-frame store", () => { - beforeEach(() => { - resetStore(); - }); - - describe("isValidFreezeDuration", () => { - test("accepts positive durations up to the max, including decimals", () => { - expect(isValidFreezeDuration({ seconds: 3 })).toBe(true); - expect(isValidFreezeDuration({ seconds: 2.5 })).toBe(true); - expect(isValidFreezeDuration({ seconds: 0.1 })).toBe(true); - expect( - isValidFreezeDuration({ seconds: MAX_FREEZE_DURATION_SECONDS }), - ).toBe(true); - }); - - test("rejects non-positive, over-max, and non-finite durations", () => { - expect(isValidFreezeDuration({ seconds: 0 })).toBe(false); - expect(isValidFreezeDuration({ seconds: -1 })).toBe(false); - expect(isValidFreezeDuration({ seconds: 600.1 })).toBe(false); - expect(isValidFreezeDuration({ seconds: Number.NaN })).toBe(false); - expect(isValidFreezeDuration({ seconds: Number.POSITIVE_INFINITY })).toBe( - false, - ); - }); - }); - - describe("requestDuration", () => { - test("opens the dialog and resolves with the confirmed seconds", async () => { - const promise = useFreezeFrameStore.getState().requestDuration(); - expect(useFreezeFrameStore.getState().isOpen).toBe(true); - - useFreezeFrameStore.getState().confirmDuration({ seconds: 5 }); - - await expect(promise).resolves.toBe(5); - expect(useFreezeFrameStore.getState().isOpen).toBe(false); - }); - - test("remembers the confirmed duration as the last-used value", async () => { - const promise = useFreezeFrameStore.getState().requestDuration(); - useFreezeFrameStore.getState().confirmDuration({ seconds: 5 }); - await promise; - - expect(useFreezeFrameStore.getState().lastDurationSeconds).toBe(5); - }); - - test("resolves null on cancel and keeps the last-used duration", async () => { - useFreezeFrameStore.setState({ lastDurationSeconds: 7 }); - - const promise = useFreezeFrameStore.getState().requestDuration(); - expect(useFreezeFrameStore.getState().isOpen).toBe(true); - - useFreezeFrameStore.getState().cancelDuration(); - - await expect(promise).resolves.toBeNull(); - expect(useFreezeFrameStore.getState().isOpen).toBe(false); - expect(useFreezeFrameStore.getState().lastDurationSeconds).toBe(7); - }); - - test("ignores an invalid confirm and leaves the dialog open", () => { - useFreezeFrameStore.getState().requestDuration(); - - useFreezeFrameStore.getState().confirmDuration({ seconds: 0 }); - useFreezeFrameStore.getState().confirmDuration({ seconds: 601 }); - - expect(useFreezeFrameStore.getState().isOpen).toBe(true); - expect(useFreezeFrameStore.getState().lastDurationSeconds).toBe( - DEFAULT_FREEZE_DURATION_SECONDS, - ); - }); - - test("a new request supersedes a pending one by cancelling it", async () => { - const first = useFreezeFrameStore.getState().requestDuration(); - const second = useFreezeFrameStore.getState().requestDuration(); - - await expect(first).resolves.toBeNull(); - expect(useFreezeFrameStore.getState().isOpen).toBe(true); - - useFreezeFrameStore.getState().confirmDuration({ seconds: 2 }); - await expect(second).resolves.toBe(2); - }); - }); -}); diff --git a/apps/web/src/actions/components/freeze-frame-dialog.tsx b/apps/web/src/actions/components/freeze-frame-dialog.tsx deleted file mode 100644 index a6a7e0ae..00000000 --- a/apps/web/src/actions/components/freeze-frame-dialog.tsx +++ /dev/null @@ -1,109 +0,0 @@ -"use client"; - -import { useState } from "react"; -import { Button } from "@/components/ui/button"; -import { - Dialog, - DialogBody, - DialogContent, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { - isValidFreezeDuration, - MAX_FREEZE_DURATION_SECONDS, - useFreezeFrameStore, -} from "@/actions/freeze-frame-store"; - -export function FreezeFrameDialog() { - const isOpen = useFreezeFrameStore((state) => state.isOpen); - const cancelDuration = useFreezeFrameStore((state) => state.cancelDuration); - - return ( - <Dialog - open={isOpen} - onOpenChange={(open) => { - if (!open) { - cancelDuration(); - } - }} - > - <DialogContent className="sm:max-w-md"> - <DialogHeader> - <DialogTitle>Freeze frame</DialogTitle> - <DialogDescription> - How long should the frozen frame stay on screen? - </DialogDescription> - </DialogHeader> - {isOpen && <FreezeFrameForm />} - </DialogContent> - </Dialog> - ); -} - -function FreezeFrameForm() { - const lastDurationSeconds = useFreezeFrameStore( - (state) => state.lastDurationSeconds, - ); - const confirmDuration = useFreezeFrameStore((state) => state.confirmDuration); - const cancelDuration = useFreezeFrameStore((state) => state.cancelDuration); - const [value, setValue] = useState(String(lastDurationSeconds)); - - const seconds = Number(value); - const showError = value.trim() !== "" && !isValidFreezeDuration({ seconds }); - const canConfirm = isValidFreezeDuration({ seconds }); - - const handleConfirm = () => { - if (canConfirm) { - confirmDuration({ seconds }); - } - }; - - return ( - <> - <DialogBody className="gap-3"> - <Label htmlFor="freeze-frame-duration">Duration (seconds)</Label> - <Input - id="freeze-frame-duration" - value={value} - inputMode="decimal" - aria-invalid={showError} - onChange={(e) => setValue(e.target.value)} - onKeyDown={(e) => { - if (e.key === "Enter") { - e.preventDefault(); - handleConfirm(); - } - }} - placeholder="e.g. 2.5" - /> - {showError && ( - <p className="text-destructive text-xs"> - Enter a duration greater than 0 and at most{" "} - {MAX_FREEZE_DURATION_SECONDS} seconds. - </p> - )} - </DialogBody> - - <DialogFooter> - <Button - variant="outline" - onClick={(e) => { - e.preventDefault(); - e.stopPropagation(); - cancelDuration(); - }} - > - Cancel - </Button> - <Button onClick={handleConfirm} disabled={!canConfirm}> - Freeze - </Button> - </DialogFooter> - </> - ); -} diff --git a/apps/web/src/actions/freeze-frame-store.ts b/apps/web/src/actions/freeze-frame-store.ts deleted file mode 100644 index 17102780..00000000 --- a/apps/web/src/actions/freeze-frame-store.ts +++ /dev/null @@ -1,72 +0,0 @@ -"use client"; - -import { create } from "zustand"; -import { persist } from "zustand/middleware"; - -export const DEFAULT_FREEZE_DURATION_SECONDS = 3; -export const MAX_FREEZE_DURATION_SECONDS = 600; - -export function isValidFreezeDuration({ - seconds, -}: { - seconds: number; -}): boolean { - return ( - Number.isFinite(seconds) && - seconds > 0 && - seconds <= MAX_FREEZE_DURATION_SECONDS - ); -} - -type DurationResolver = ({ seconds }: { seconds: number | null }) => void; - -interface FreezeFrameState { - isOpen: boolean; - lastDurationSeconds: number; - resolver: DurationResolver | null; - requestDuration: () => Promise<number | null>; - confirmDuration: ({ seconds }: { seconds: number }) => void; - cancelDuration: () => void; -} - -export const useFreezeFrameStore = create<FreezeFrameState>()( - persist( - (set, get) => ({ - isOpen: false, - lastDurationSeconds: DEFAULT_FREEZE_DURATION_SECONDS, - resolver: null, - - requestDuration: () => - new Promise<number | null>((resolve) => { - // A new invocation supersedes any pending one: resolve it as - // cancelled so its handler aborts instead of hanging. - get().resolver?.({ seconds: null }); - set({ - isOpen: true, - resolver: ({ seconds }) => resolve(seconds), - }); - }), - confirmDuration: ({ seconds }) => { - if (!isValidFreezeDuration({ seconds })) { - return; - } - get().resolver?.({ seconds }); - set({ - isOpen: false, - resolver: null, - lastDurationSeconds: seconds, - }); - }, - cancelDuration: () => { - get().resolver?.({ seconds: null }); - set({ isOpen: false, resolver: null }); - }, - }), - { - name: "freeze-frame-store", - partialize: (state) => ({ - lastDurationSeconds: state.lastDurationSeconds, - }), - }, - ), -); diff --git a/apps/web/src/actions/use-editor-actions.ts b/apps/web/src/actions/use-editor-actions.ts index c99f65e2..ca16e989 100644 --- a/apps/web/src/actions/use-editor-actions.ts +++ b/apps/web/src/actions/use-editor-actions.ts @@ -28,7 +28,6 @@ import { import { useCommittedRef } from "@/hooks/use-committed-ref"; import { toast } from "sonner"; import { generateImageThumbnail } from "@/media/processing"; -import { useFreezeFrameStore } from "@/actions/freeze-frame-store"; import { AddMediaAssetCommand, BatchCommand, @@ -47,6 +46,8 @@ import { import type { VoiceOverRecordingResult } from "@/voiceover/recorder"; import { useVoiceOverStore } from "@/voiceover/voiceover-store"; +const FREEZE_DURATION_SECONDS = 3; + export function useEditorActions() { const editor = useEditor(); const { selectedElements, setElementSelection } = useElementSelection(); @@ -421,13 +422,6 @@ export function useEditorActions() { return; } - const freezeDurationSeconds = await useFreezeFrameStore - .getState() - .requestDuration(); - if (freezeDurationSeconds === null) { - return; - } - const blob = await editor.renderer.captureFrame(); if (!blob) { toast.error("Failed to capture frame"); @@ -442,7 +436,7 @@ export function useEditorActions() { }); const freezeDuration = mediaTimeFromSeconds({ - seconds: freezeDurationSeconds, + seconds: FREEZE_DURATION_SECONDS, }); const addMediaCommand = new AddMediaAssetCommand({ projectId: activeProject.metadata.id, diff --git a/apps/web/src/app/editor/[project_id]/page.tsx b/apps/web/src/app/editor/[project_id]/page.tsx index b3f1a933..b759b9c2 100644 --- a/apps/web/src/app/editor/[project_id]/page.tsx +++ b/apps/web/src/app/editor/[project_id]/page.tsx @@ -14,7 +14,6 @@ import { EditorHeader } from "@/components/editor/editor-header"; import { EditorProvider } from "@/components/providers/editor-provider"; import { Onboarding } from "@/components/editor/onboarding"; import { MigrationDialog } from "@/project/components/migration-dialog"; -import { FreezeFrameDialog } from "@/actions/components/freeze-frame-dialog"; import { usePanelStore } from "@/editor/panel-store"; import { usePasteMedia } from "@/media/use-paste-media"; import { MobileGate } from "@/components/editor/mobile-gate"; @@ -37,8 +36,8 @@ import { } from "@/timeline/bookmarks/index"; export default function Editor() { - const params = useParams(); - const projectId = params.project_id as string; + const params = useParams<{ project_id: string }>(); + const projectId = params.project_id; return ( <MobileGate> @@ -51,7 +50,6 @@ export default function Editor() { </div> <Onboarding /> <MigrationDialog /> - <FreezeFrameDialog /> <ChangelogNotification /> </div> </EditorProvider> diff --git a/apps/web/src/commands/media/remove-media-asset.ts b/apps/web/src/commands/media/remove-media-asset.ts index ab267f36..2180c302 100644 --- a/apps/web/src/commands/media/remove-media-asset.ts +++ b/apps/web/src/commands/media/remove-media-asset.ts @@ -4,6 +4,7 @@ import { toast } from "sonner"; import type { MediaAsset } from "@/media/types"; import { buildWaveformSourceKey } from "@/media/waveform-summary"; import { storageService } from "@/services/storage/service"; +import { thumbnailCache } from "@/services/thumbnail-cache/service"; import { videoCache } from "@/services/video-cache/service"; import { waveformCache } from "@/services/waveform-cache/service"; import { hasMediaId } from "@/timeline/element-utils"; @@ -67,6 +68,7 @@ export class RemoveMediaAssetCommand extends Command { } videoCache.clearVideo({ mediaId: this.assetId }); + thumbnailCache.clearMedia({ mediaId: this.assetId }); waveformCache.clearSource({ sourceKey: buildWaveformSourceKey({ kind: "media", diff --git a/apps/web/src/core/managers/media-manager.ts b/apps/web/src/core/managers/media-manager.ts index 69da6e80..739fd3fa 100644 --- a/apps/web/src/core/managers/media-manager.ts +++ b/apps/web/src/core/managers/media-manager.ts @@ -4,6 +4,7 @@ import type { MediaAsset } from "@/media/types"; import { storageService } from "@/services/storage/service"; import { generateUUID } from "@/utils/id"; import { videoCache } from "@/services/video-cache/service"; +import { thumbnailCache } from "@/services/thumbnail-cache/service"; import { waveformCache } from "@/services/waveform-cache/service"; import { BatchCommand, RemoveMediaAssetCommand } from "@/commands"; @@ -103,6 +104,7 @@ export class MediaManager { } async clearProjectMedia({ projectId }: { projectId: string }): Promise<void> { + thumbnailCache.clearAll(); waveformCache.clearAll(); this.assets.forEach((asset) => { @@ -131,6 +133,7 @@ export class MediaManager { clearAllAssets(): void { videoCache.clearAll(); + thumbnailCache.clearAll(); waveformCache.clearAll(); this.assets.forEach((asset) => { diff --git a/apps/web/src/core/managers/renderer-manager.ts b/apps/web/src/core/managers/renderer-manager.ts index 38d1e57f..d73e3e58 100644 --- a/apps/web/src/core/managers/renderer-manager.ts +++ b/apps/web/src/core/managers/renderer-manager.ts @@ -6,7 +6,6 @@ import { SceneExporter } from "@/services/renderer/scene-exporter"; import { buildScene } from "@/services/renderer/scene-builder"; import { createTimelineAudioBuffer } from "@/media/audio"; import { formatTimecode } from "opencut-wasm"; -import { frameRateToFloat } from "@/fps/utils"; import { downloadBlob } from "@/utils/browser"; type SnapshotResult = @@ -192,6 +191,10 @@ export class RendererManager { const exportFps = fps ?? activeProject.settings.fps; const canvasSize = activeProject.settings.canvasSize; + const exportSize = { + width: 2 * Math.round(canvasSize.width / 2), + height: 2 * Math.round(canvasSize.height / 2), + }; let audioBuffer: AudioBuffer | null = null; if (includeAudio) { @@ -207,13 +210,13 @@ export class RendererManager { tracks, mediaAssets, duration, - canvasSize, + canvasSize: exportSize, background: activeProject.settings.background, }); const exporter = new SceneExporter({ - width: canvasSize.width, - height: canvasSize.height, + width: exportSize.width, + height: exportSize.height, fps: exportFps, format, quality, diff --git a/apps/web/src/services/renderer/__tests__/export-encoding.test.ts b/apps/web/src/services/renderer/__tests__/export-encoding.test.ts new file mode 100644 index 00000000..fbf354eb --- /dev/null +++ b/apps/web/src/services/renderer/__tests__/export-encoding.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from "bun:test"; +import { computeVideoBitrate } from "../export-encoding"; + +describe("computeVideoBitrate", () => { + test("scales linearly with frame rate at the same resolution and quality", () => { + const at30 = computeVideoBitrate({ + width: 1920, + height: 1080, + fps: 30, + quality: "high", + }); + const at60 = computeVideoBitrate({ + width: 1920, + height: 1080, + fps: 60, + quality: "high", + }); + + expect(at60).toBe(at30 * 2); + }); + + test("applies the bits-per-pixel ladder per quality tier", () => { + const params = { width: 1920, height: 1080, fps: 30 }; + + expect(computeVideoBitrate({ ...params, quality: "low" })).toBe(3_110_400); + expect(computeVideoBitrate({ ...params, quality: "medium" })).toBe( + 6_220_800, + ); + expect(computeVideoBitrate({ ...params, quality: "high" })).toBe( + 12_441_600, + ); + expect(computeVideoBitrate({ ...params, quality: "very_high" })).toBe( + 24_883_200, + ); + }); + + test("increases monotonically across quality tiers", () => { + const params = { width: 1280, height: 720, fps: 30 }; + const bitrates = (["low", "medium", "high", "very_high"] as const).map( + (quality) => computeVideoBitrate({ ...params, quality }), + ); + + for (let i = 1; i < bitrates.length; i++) { + expect(bitrates[i]).toBeGreaterThan(bitrates[i - 1]); + } + }); + + test("returns a positive integer for fractional frame rates", () => { + const bitrate = computeVideoBitrate({ + width: 1920, + height: 1080, + fps: 29.97, + quality: "high", + }); + + expect(bitrate).toBe(12_429_158); + expect(Number.isInteger(bitrate)).toBe(true); + expect(bitrate).toBeGreaterThan(0); + }); +}); diff --git a/apps/web/src/services/renderer/export-encoding.ts b/apps/web/src/services/renderer/export-encoding.ts new file mode 100644 index 00000000..cc333ea4 --- /dev/null +++ b/apps/web/src/services/renderer/export-encoding.ts @@ -0,0 +1,22 @@ +import type { ExportQuality } from "@/export"; + +const videoBitsPerPixelFrame: Record<ExportQuality, number> = { + low: 0.05, + medium: 0.1, + high: 0.2, + very_high: 0.4, +}; + +export function computeVideoBitrate({ + width, + height, + fps, + quality, +}: { + width: number; + height: number; + fps: number; + quality: ExportQuality; +}): number { + return Math.round(width * height * fps * videoBitsPerPixelFrame[quality]); +} diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts index 66df9485..04c14277 100644 --- a/apps/web/src/services/renderer/scene-exporter.ts +++ b/apps/web/src/services/renderer/scene-exporter.ts @@ -19,6 +19,7 @@ import { frameRateToFloat } from "@/fps/utils"; import type { RootNode } from "./nodes/root-node"; import type { ExportFormat, ExportQuality } from "@/export"; import { CanvasRenderer } from "./canvas-renderer"; +import { computeVideoBitrate } from "./export-encoding"; type ExportParams = { width: number; @@ -30,13 +31,15 @@ type ExportParams = { audioBuffer?: AudioBuffer; }; -const qualityMap = { +const audioQualityMap = { low: QUALITY_LOW, medium: QUALITY_MEDIUM, high: QUALITY_HIGH, very_high: QUALITY_VERY_HIGH, }; +const KEY_FRAME_INTERVAL_SECONDS = 2; + export type SceneExporterEvents = { progress: [progress: number]; complete: [buffer: ArrayBuffer]; @@ -99,9 +102,17 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> { target: new BufferTarget(), }); + const videoBitrate = computeVideoBitrate({ + width: this.renderer.width, + height: this.renderer.height, + fps: fpsFloat, + quality: this.quality, + }); + const videoSource = new CanvasSource(this.renderer.getOutputCanvas(), { codec: this.format === "webm" ? "vp9" : "avc", - bitrate: qualityMap[this.quality], + bitrate: videoBitrate, + keyFrameInterval: KEY_FRAME_INTERVAL_SECONDS, }); output.addVideoTrack(videoSource, { frameRate: fpsFloat }); @@ -122,7 +133,7 @@ export class SceneExporter extends EventEmitter<SceneExporterEvents> { audioSource = new AudioBufferSource({ codec: audioCodec, - bitrate: qualityMap[this.quality], + bitrate: audioQualityMap[this.quality], }); output.addAudioTrack(audioSource); } diff --git a/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts b/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts new file mode 100644 index 00000000..1fe52bb1 --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/__tests__/tiling.test.ts @@ -0,0 +1,258 @@ +import { describe, expect, test } from "bun:test"; +import { + buildThumbnailTileKey, + FILMSTRIP_MAX_INTERVAL_SECONDS, + FILMSTRIP_MIN_INTERVAL_SECONDS, + getFilmstripInterval, + getFilmstripSlotCount, + getFilmstripSlotSourceTime, + getVisibleSlotRange, + MEDIA_TILE_ASPECT_RATIO, +} from "../tiling"; + +const TRACK_HEIGHT_PX = 65; +const MIN_TILE_WIDTH_PX = TRACK_HEIGHT_PX * MEDIA_TILE_ASPECT_RATIO; + +describe("getFilmstripInterval", () => { + test("snaps to the nearest power of two seconds", () => { + expect( + getFilmstripInterval({ + pixelsPerSecond: 100, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(1); + expect( + getFilmstripInterval({ + pixelsPerSecond: 50, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(2); + expect( + getFilmstripInterval({ + pixelsPerSecond: 200, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(0.5); + }); + + test("keeps tile width near the minimum across zoom levels", () => { + for (const pixelsPerSecond of [5, 12, 25, 40, 80, 160, 320, 640]) { + const interval = getFilmstripInterval({ + pixelsPerSecond, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + const tileWidthPx = interval * pixelsPerSecond; + expect(tileWidthPx).toBeGreaterThanOrEqual(MIN_TILE_WIDTH_PX * 0.7); + expect(tileWidthPx).toBeLessThanOrEqual(MIN_TILE_WIDTH_PX * 1.45); + } + }); + + test("is stable under tiny zoom changes", () => { + const coarse = getFilmstripInterval({ + pixelsPerSecond: 100, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + const nudged = getFilmstripInterval({ + pixelsPerSecond: 110, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(nudged).toBe(coarse); + }); + + test("clamps the interval at extreme zoom levels", () => { + const zoomedIn = getFilmstripInterval({ + pixelsPerSecond: 5000, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(zoomedIn).toBeGreaterThanOrEqual(FILMSTRIP_MIN_INTERVAL_SECONDS); + + const zoomedOut = getFilmstripInterval({ + pixelsPerSecond: 5, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }); + expect(zoomedOut).toBeLessThanOrEqual(FILMSTRIP_MAX_INTERVAL_SECONDS); + }); + + test("falls back to the max interval for invalid zoom", () => { + expect( + getFilmstripInterval({ + pixelsPerSecond: 0, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(FILMSTRIP_MAX_INTERVAL_SECONDS); + expect( + getFilmstripInterval({ + pixelsPerSecond: Number.NaN, + minTileWidthPx: MIN_TILE_WIDTH_PX, + }), + ).toBe(FILMSTRIP_MAX_INTERVAL_SECONDS); + }); +}); + +describe("getFilmstripSlotCount", () => { + test("covers the clip duration with exact tiles", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 10, interval: 2 })).toBe(5); + expect(getFilmstripSlotCount({ clipDurationSec: 10.5, interval: 2 })).toBe( + 6, + ); + }); + + test("does not add a trailing tile on exact multiples", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 4, interval: 2 })).toBe(2); + }); + + test("returns zero for empty clips", () => { + expect(getFilmstripSlotCount({ clipDurationSec: 0, interval: 2 })).toBe(0); + expect(getFilmstripSlotCount({ clipDurationSec: -3, interval: 2 })).toBe(0); + }); +}); + +describe("getVisibleSlotRange", () => { + test("returns the slots intersecting the visible window plus overscan", () => { + const range = getVisibleSlotRange({ + slotCount: 100, + tileWidthPx: 100, + visibleLeftPx: 500, + visibleRightPx: 1000, + overscanSlots: 2, + }); + expect(range).toEqual({ firstSlot: 3, lastSlot: 11 }); + }); + + test("clamps to the slot bounds", () => { + const range = getVisibleSlotRange({ + slotCount: 8, + tileWidthPx: 100, + visibleLeftPx: 0, + visibleRightPx: 1000, + overscanSlots: 2, + }); + expect(range).toEqual({ firstSlot: 0, lastSlot: 7 }); + }); + + test("returns an empty range when nothing is visible", () => { + const range = getVisibleSlotRange({ + slotCount: 10, + tileWidthPx: 100, + visibleLeftPx: 500, + visibleRightPx: 500, + overscanSlots: 2, + }); + expect(range.lastSlot).toBeLessThan(range.firstSlot); + }); +}); + +describe("getFilmstripSlotSourceTime", () => { + test("offsets by the trim start", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 2, + interval: 1, + trimStartSec: 5, + rate: 1, + }), + ).toBe(7); + }); + + test("scales clip time by the retime rate", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 2, + interval: 1, + trimStartSec: 5, + rate: 2, + }), + ).toBe(9); + }); + + test("clamps to the end of the asset", () => { + const clamped = getFilmstripSlotSourceTime({ + slot: 100, + interval: 1, + trimStartSec: 0, + rate: 1, + assetDurationSec: 10, + }); + expect(clamped).toBeLessThan(10); + expect(clamped).toBeGreaterThan(9); + }); + + test("never returns negative times", () => { + expect( + getFilmstripSlotSourceTime({ + slot: 0, + interval: 1, + trimStartSec: -4, + rate: 1, + }), + ).toBe(0); + }); +}); + +describe("buildThumbnailTileKey", () => { + test("changes per media, size, and quantized time", () => { + const base = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-2", + time: 1, + width: 232, + height: 130, + }), + ); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-1", + time: 2, + width: 232, + height: 130, + }), + ); + expect(base).not.toBe( + buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 464, + height: 260, + }), + ); + }); + + test("shares keys for times within one quantum", () => { + const quantum = 1 / 4096; + const first = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + const second = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1 + quantum / 4, + width: 232, + height: 130, + }); + expect(second).toBe(first); + }); + + test("splits keys across quantum boundaries", () => { + const first = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1, + width: 232, + height: 130, + }); + const second = buildThumbnailTileKey({ + mediaId: "media-1", + time: 1 + 1 / 4096, + width: 232, + height: 130, + }); + expect(second).not.toBe(first); + }); +}); diff --git a/apps/web/src/services/thumbnail-cache/service.ts b/apps/web/src/services/thumbnail-cache/service.ts new file mode 100644 index 00000000..7002cd9c --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/service.ts @@ -0,0 +1,272 @@ +"use client"; + +import { + ALL_FORMATS, + BlobSource, + CanvasSink, + Input, + type WrappedCanvas, +} from "mediabunny"; +import { buildThumbnailTileKey } from "./tiling"; + +const MAX_TILE_ENTRIES = 400; +const MAX_SOURCE_ENTRIES = 8; +const MAX_CONCURRENT_FLUSHES = 2; +const TILE_JPEG_QUALITY = 0.7; +const CANVAS_POOL_SIZE = 2; + +interface PendingTile { + time: number; + resolve: (url: string | null) => void; +} + +interface TileSource { + width: number; + height: number; + init: Promise<CanvasSink | null>; + pending: PendingTile[]; + flushing: boolean; + flushScheduled: boolean; + dispose: () => void; +} + +function renderTileDataUrl({ + wrapped, +}: { + wrapped: WrappedCanvas; +}): string | null { + const canvas = wrapped.canvas; + if (canvas instanceof HTMLCanvasElement) { + return canvas.toDataURL("image/jpeg", TILE_JPEG_QUALITY); + } + return null; +} + +export class ThumbnailCache { + private tiles = new Map<string, Promise<string | null>>(); + private sources = new Map<string, TileSource>(); + private activeFlushes = 0; + private flushQueue: Array<() => void> = []; + + getTile({ + mediaId, + file, + time, + width, + height, + }: { + mediaId: string; + file: File; + time: number; + width: number; + height: number; + }): Promise<string | null> { + const key = buildThumbnailTileKey({ mediaId, time, width, height }); + const existing = this.tiles.get(key); + if (existing) { + this.tiles.delete(key); + this.tiles.set(key, existing); + return existing; + } + + const source = this.ensureSource({ mediaId, file, width, height }); + const requested = new Promise<string | null>((resolve) => { + source.pending.push({ time, resolve }); + }); + this.scheduleFlush({ mediaId }); + + const tracked = requested.then((url) => { + if (url === null && this.tiles.get(key) === tracked) { + this.tiles.delete(key); + } + return url; + }); + + this.tiles.set(key, tracked); + this.evictTiles(); + return tracked; + } + + clearMedia({ mediaId }: { mediaId: string }): void { + for (const key of [...this.tiles.keys()]) { + if (key.startsWith(`${mediaId}:`)) { + this.tiles.delete(key); + } + } + const source = this.sources.get(mediaId); + if (source) { + this.sources.delete(mediaId); + this.releaseSource({ source }); + } + } + + clearAll(): void { + this.tiles.clear(); + for (const source of this.sources.values()) { + this.releaseSource({ source }); + } + this.sources.clear(); + } + + private releaseSource({ source }: { source: TileSource }): void { + const pending = source.pending.splice(0); + for (const tile of pending) { + tile.resolve(null); + } + source.dispose(); + } + + private ensureSource({ + mediaId, + file, + width, + height, + }: { + mediaId: string; + file: File; + width: number; + height: number; + }): TileSource { + const existing = this.sources.get(mediaId); + if (existing && existing.width === width && existing.height === height) { + this.sources.delete(mediaId); + this.sources.set(mediaId, existing); + return existing; + } + if (existing) { + this.sources.delete(mediaId); + this.releaseSource({ source: existing }); + } + + const input = new Input({ + source: new BlobSource(file), + formats: ALL_FORMATS, + }); + const init = input + .getPrimaryVideoTrack() + .then((track) => { + if (!track) { + return null; + } + return new CanvasSink(track, { + width, + height, + fit: "cover", + poolSize: CANVAS_POOL_SIZE, + }); + }) + .catch(() => null); + + const source: TileSource = { + width, + height, + init, + pending: [], + flushing: false, + flushScheduled: false, + dispose: () => { + void init.finally(() => input.dispose()); + }, + }; + this.sources.set(mediaId, source); + this.evictSources(); + return source; + } + + private evictTiles(): void { + while (this.tiles.size > MAX_TILE_ENTRIES) { + const oldestKey = this.tiles.keys().next().value; + if (oldestKey === undefined) { + return; + } + this.tiles.delete(oldestKey); + } + } + + private evictSources(): void { + while (this.sources.size > MAX_SOURCE_ENTRIES) { + const oldestKey = this.sources.keys().next().value; + if (oldestKey === undefined) { + return; + } + const oldest = this.sources.get(oldestKey); + this.sources.delete(oldestKey); + if (oldest) { + this.releaseSource({ source: oldest }); + } + } + } + + private scheduleFlush({ mediaId }: { mediaId: string }): void { + const source = this.sources.get(mediaId); + if (!source || source.flushScheduled) { + return; + } + source.flushScheduled = true; + queueMicrotask(() => { + source.flushScheduled = false; + void this.flush({ mediaId }); + }); + } + + private async flush({ mediaId }: { mediaId: string }): Promise<void> { + const source = this.sources.get(mediaId); + if (!source || source.flushing || source.pending.length === 0) { + return; + } + source.flushing = true; + const batch = source.pending.splice(0).sort((a, b) => a.time - b.time); + + await this.acquireFlushSlot(); + try { + const sink = await source.init; + if (!sink) { + for (const tile of batch) { + tile.resolve(null); + } + return; + } + + const iterator = sink.canvasesAtTimestamps( + batch.map((tile) => tile.time), + ); + for (const tile of batch) { + const { value, done } = await iterator.next(); + tile.resolve( + !done && value ? renderTileDataUrl({ wrapped: value }) : null, + ); + } + } catch { + for (const tile of batch) { + tile.resolve(null); + } + } finally { + this.releaseFlushSlot(); + source.flushing = false; + if (source.pending.length > 0) { + this.scheduleFlush({ mediaId }); + } + } + } + + private async acquireFlushSlot(): Promise<void> { + if (this.activeFlushes < MAX_CONCURRENT_FLUSHES) { + this.activeFlushes++; + return; + } + await new Promise<void>((resolve) => { + this.flushQueue.push(resolve); + }); + this.activeFlushes++; + } + + private releaseFlushSlot(): void { + this.activeFlushes--; + const next = this.flushQueue.shift(); + if (next) { + next(); + } + } +} + +export const thumbnailCache = new ThumbnailCache(); diff --git a/apps/web/src/services/thumbnail-cache/tiling.ts b/apps/web/src/services/thumbnail-cache/tiling.ts new file mode 100644 index 00000000..bc8d31e5 --- /dev/null +++ b/apps/web/src/services/thumbnail-cache/tiling.ts @@ -0,0 +1,121 @@ +export const MEDIA_TILE_ASPECT_RATIO = 16 / 9; +export const FILMSTRIP_MIN_INTERVAL_SECONDS = 0.1; +export const FILMSTRIP_MAX_INTERVAL_SECONDS = 64; +export const FILMSTRIP_OVERSCAN_SLOTS = 2; + +const SOURCE_TIME_END_PADDING_SECONDS = 0.05; +const TILE_TIME_QUANTUM_PER_SECOND = 4096; + +export function getFilmstripInterval({ + pixelsPerSecond, + minTileWidthPx, +}: { + pixelsPerSecond: number; + minTileWidthPx: number; +}): number { + if ( + !Number.isFinite(pixelsPerSecond) || + pixelsPerSecond <= 0 || + !Number.isFinite(minTileWidthPx) || + minTileWidthPx <= 0 + ) { + return FILMSTRIP_MAX_INTERVAL_SECONDS; + } + + const ideal = minTileWidthPx / pixelsPerSecond; + const clamped = Math.min( + FILMSTRIP_MAX_INTERVAL_SECONDS, + Math.max(FILMSTRIP_MIN_INTERVAL_SECONDS, ideal), + ); + return 2 ** Math.round(Math.log2(clamped)); +} + +export function getFilmstripSlotCount({ + clipDurationSec, + interval, +}: { + clipDurationSec: number; + interval: number; +}): number { + if ( + !Number.isFinite(clipDurationSec) || + clipDurationSec <= 0 || + interval <= 0 + ) { + return 0; + } + return Math.max(1, Math.ceil(clipDurationSec / interval - 1e-6)); +} + +export function getVisibleSlotRange({ + slotCount, + tileWidthPx, + visibleLeftPx, + visibleRightPx, + overscanSlots = FILMSTRIP_OVERSCAN_SLOTS, +}: { + slotCount: number; + tileWidthPx: number; + visibleLeftPx: number; + visibleRightPx: number; + overscanSlots?: number; +}): { firstSlot: number; lastSlot: number } { + if (slotCount <= 0 || tileWidthPx <= 0 || visibleRightPx <= visibleLeftPx) { + return { firstSlot: 0, lastSlot: -1 }; + } + + const firstSlot = Math.max( + 0, + Math.floor(visibleLeftPx / tileWidthPx) - overscanSlots, + ); + const lastSlot = Math.min( + slotCount - 1, + Math.ceil(visibleRightPx / tileWidthPx) - 1 + overscanSlots, + ); + return { firstSlot, lastSlot }; +} + +export function getFilmstripSlotSourceTime({ + slot, + interval, + trimStartSec, + rate, + assetDurationSec, +}: { + slot: number; + interval: number; + trimStartSec: number; + rate: number; + assetDurationSec?: number; +}): number { + const safeRate = Number.isFinite(rate) && rate > 0 ? rate : 1; + const raw = Math.max(0, trimStartSec + slot * interval * safeRate); + if ( + assetDurationSec === undefined || + !Number.isFinite(assetDurationSec) || + assetDurationSec <= 0 + ) { + return raw; + } + return Math.min( + raw, + Math.max(0, assetDurationSec - SOURCE_TIME_END_PADDING_SECONDS), + ); +} + +export function buildThumbnailTileKey({ + mediaId, + time, + width, + height, +}: { + mediaId: string; + time: number; + width: number; + height: number; +}): string { + const quantizedTime = Math.round( + Math.max(0, time) * TILE_TIME_QUANTUM_PER_SECOND, + ); + return `${mediaId}:${width}x${height}:${quantizedTime}`; +} diff --git a/apps/web/src/timeline/components/timeline-element.tsx b/apps/web/src/timeline/components/timeline-element.tsx index 9dfeab8b..b25a7a28 100644 --- a/apps/web/src/timeline/components/timeline-element.tsx +++ b/apps/web/src/timeline/components/timeline-element.tsx @@ -5,6 +5,8 @@ import { useEditor } from "@/editor/use-editor"; import { FEATURES } from "@/features"; import { useAssetsPanelStore } from "@/components/editor/panels/assets/assets-panel-store"; import { AudioWaveform, WAVEFORM_GAIN_SAMPLE_COUNT } from "./audio-waveform"; +import { VideoFilmstrip } from "./video-filmstrip"; +import { MEDIA_TILE_ASPECT_RATIO } from "@/services/thumbnail-cache/tiling"; import { AudioVolumeLine } from "./audio-volume-line"; import { useElementPreview } from "@/timeline/hooks/use-element-preview"; import { @@ -95,7 +97,6 @@ const KEYFRAME_INDICATOR_MIN_WIDTH_PX = 40; const ELEMENT_RING_WIDTH_PX = 1.5; const PixelsPerSecondContext = createContext<number | null>(null); -const THUMBNAIL_ASPECT_RATIO = 16 / 9; interface KeyframeIndicator { time: MediaTime; @@ -1082,20 +1083,78 @@ function EffectsButton({ ); } -function TiledMediaContent({ +function VideoElementContent({ element, track, }: { - element: VideoElement | ImageElement; + element: VideoElement; + track: TimelineTrack; +}) { + const pixelsPerSecond = useContext(PixelsPerSecondContext); + if (pixelsPerSecond === null) { + throw new Error( + "VideoElementContent must be rendered inside PixelsPerSecondContext.Provider", + ); + } + const mediaAssets = useEditor((e) => e.media.getAssets()); + const mediaAsset = mediaAssets.find((asset) => asset.id === element.mediaId); + + if (!mediaAsset) { + return ( + <span className="text-foreground/80 truncate text-xs"> + {element.name} + </span> + ); + } + + const trackHeight = getTrackHeight({ type: track.type }); + const fallbackTileWidth = trackHeight * MEDIA_TILE_ASPECT_RATIO; + + return ( + <> + {mediaAsset.thumbnailUrl && ( + <div + className="absolute inset-0" + style={{ + backgroundColor: "var(--muted)", + backgroundImage: `url(${mediaAsset.thumbnailUrl})`, + backgroundRepeat: "repeat-x", + backgroundSize: `${fallbackTileWidth}px ${trackHeight}px`, + backgroundPosition: "left center", + pointerEvents: "none", + }} + /> + )} + <VideoFilmstrip + element={element} + mediaAsset={mediaAsset} + pixelsPerSecond={pixelsPerSecond} + trackHeight={trackHeight} + /> + <MediaElementHeader + name={mediaAsset.name} + leading={ + FEATURES.effects && hasElementEffects({ element }) ? ( + <EffectsButton element={element} track={track} /> + ) : null + } + hasFade={true} + /> + </> + ); +} + +function ImageElementContent({ + element, + track, +}: { + element: ImageElement; track: TimelineTrack; }) { const mediaAssets = useEditor((e) => e.media.getAssets()); const mediaAsset = mediaAssets.find((asset) => asset.id === element.mediaId); - const imageUrl = - element.type === "video" - ? mediaAsset?.thumbnailUrl - : (mediaAsset?.thumbnailUrl ?? mediaAsset?.url); + const imageUrl = mediaAsset?.thumbnailUrl ?? mediaAsset?.url; if (!imageUrl) { return ( @@ -1106,7 +1165,7 @@ function TiledMediaContent({ } const trackHeight = getTrackHeight({ type: track.type }); - const tileWidth = trackHeight * THUMBNAIL_ASPECT_RATIO; + const tileWidth = trackHeight * MEDIA_TILE_ASPECT_RATIO; return ( <> @@ -1177,8 +1236,9 @@ function ElementContent({ element, track }: ElementContentProps) { case "audio": return <AudioElementContent element={element} trackId={track.id} />; case "video": + return <VideoElementContent element={element} track={track} />; case "image": - return <TiledMediaContent element={element} track={track} />; + return <ImageElementContent element={element} track={track} />; } } diff --git a/apps/web/src/timeline/components/video-filmstrip.tsx b/apps/web/src/timeline/components/video-filmstrip.tsx new file mode 100644 index 00000000..e05f6bf4 --- /dev/null +++ b/apps/web/src/timeline/components/video-filmstrip.tsx @@ -0,0 +1,208 @@ +"use client"; + +import { + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from "react"; +import { useResizeObserver } from "@/hooks/use-resize-observer"; +import type { MediaAsset } from "@/media/types"; +import { getEffectiveRateAt } from "@/retime/resolve"; +import { thumbnailCache } from "@/services/thumbnail-cache/service"; +import { + getFilmstripInterval, + getFilmstripSlotCount, + getFilmstripSlotSourceTime, + getVisibleSlotRange, + MEDIA_TILE_ASPECT_RATIO, +} from "@/services/thumbnail-cache/tiling"; +import type { VideoElement } from "@/timeline"; +import { findScrollParent } from "@/utils/browser"; +import { mediaTimeToSeconds } from "@/wasm"; + +interface VideoFilmstripProps { + element: VideoElement; + mediaAsset: MediaAsset; + pixelsPerSecond: number; + trackHeight: number; +} + +export function VideoFilmstrip({ + element, + mediaAsset, + pixelsPerSecond, + trackHeight, +}: VideoFilmstripProps) { + const containerRef = useRef<HTMLDivElement>(null); + const [visibleRange, setVisibleRange] = useState({ left: 0, right: 0 }); + + const clipDurationSec = mediaTimeToSeconds({ time: element.duration }); + const trimStartSec = mediaTimeToSeconds({ time: element.trimStart }); + const rate = getEffectiveRateAt({ retime: element.retime }); + const minTileWidthPx = trackHeight * MEDIA_TILE_ASPECT_RATIO; + const interval = getFilmstripInterval({ pixelsPerSecond, minTileWidthPx }); + const tileWidthPx = interval * pixelsPerSecond; + const slotCount = getFilmstripSlotCount({ clipDurationSec, interval }); + + const updateVisibleRange = useCallback(() => { + const container = containerRef.current; + if (!container) { + return; + } + const rect = container.getBoundingClientRect(); + if (rect.width <= 0) { + return; + } + const scrollParent = findScrollParent({ element: container }); + let left = 0; + let right = rect.width; + if (scrollParent) { + const parentRect = scrollParent.getBoundingClientRect(); + left = Math.max(0, parentRect.left - rect.left); + right = Math.min(rect.width, parentRect.right - rect.left); + } else { + left = Math.max(0, -rect.left); + right = Math.min(rect.width, window.innerWidth - rect.left); + } + setVisibleRange((previous) => + previous.left === left && previous.right === right + ? previous + : { left, right }, + ); + }, []); + + useLayoutEffect(() => { + updateVisibleRange(); + }, [updateVisibleRange, pixelsPerSecond, clipDurationSec, interval]); + + useEffect(() => { + const container = containerRef.current; + if (!container) { + return; + } + const scrollParent = findScrollParent({ element: container }); + if (!scrollParent) { + return; + } + scrollParent.addEventListener("scroll", updateVisibleRange, { + passive: true, + }); + return () => scrollParent.removeEventListener("scroll", updateVisibleRange); + }, [updateVisibleRange]); + + useResizeObserver({ ref: containerRef, onResize: updateVisibleRange }); + + const { firstSlot, lastSlot } = getVisibleSlotRange({ + slotCount, + tileWidthPx, + visibleLeftPx: visibleRange.left, + visibleRightPx: visibleRange.right, + }); + + const bitmapSize = useMemo(() => { + const dpr = + typeof window === "undefined" ? 1 : window.devicePixelRatio || 1; + return { + width: Math.max(1, Math.round(minTileWidthPx * dpr)), + height: Math.max(1, Math.round(trackHeight * dpr)), + }; + }, [minTileWidthPx, trackHeight]); + + const slots: number[] = []; + for (let slot = firstSlot; slot <= lastSlot; slot++) { + slots.push(slot); + } + + return ( + <div + ref={containerRef} + className="pointer-events-none absolute inset-0 overflow-hidden" + > + {slots.map((slot) => ( + <FilmstripTile + key={slot} + mediaId={mediaAsset.id} + file={mediaAsset.file} + sourceTimeSec={getFilmstripSlotSourceTime({ + slot, + interval, + trimStartSec, + rate, + assetDurationSec: mediaAsset.duration, + })} + leftPx={slot * tileWidthPx} + widthPx={tileWidthPx} + heightPx={trackHeight} + bitmapWidth={bitmapSize.width} + bitmapHeight={bitmapSize.height} + /> + ))} + </div> + ); +} + +function FilmstripTile({ + mediaId, + file, + sourceTimeSec, + leftPx, + widthPx, + heightPx, + bitmapWidth, + bitmapHeight, +}: { + mediaId: string; + file: File; + sourceTimeSec: number; + leftPx: number; + widthPx: number; + heightPx: number; + bitmapWidth: number; + bitmapHeight: number; +}) { + const requestKey = `${mediaId}:${sourceTimeSec}:${bitmapWidth}x${bitmapHeight}`; + const [loaded, setLoaded] = useState<{ + requestKey: string; + url: string; + } | null>(null); + + useEffect(() => { + let isCancelled = false; + void thumbnailCache + .getTile({ + mediaId, + file, + time: sourceTimeSec, + width: bitmapWidth, + height: bitmapHeight, + }) + .then((url) => { + if (!isCancelled && url) { + setLoaded({ requestKey, url }); + } + }); + return () => { + isCancelled = true; + }; + }, [mediaId, file, sourceTimeSec, bitmapWidth, bitmapHeight, requestKey]); + + if (!loaded || loaded.requestKey !== requestKey) { + return null; + } + + return ( + <div + className="absolute top-0" + style={{ + left: `${leftPx}px`, + width: `${widthPx}px`, + height: `${heightPx}px`, + backgroundImage: `url(${loaded.url})`, + backgroundSize: "100% 100%", + }} + /> + ); +} diff --git a/apps/web/src/timeline/controllers/resize-controller.ts b/apps/web/src/timeline/controllers/resize-controller.ts index a1098915..a657c695 100644 --- a/apps/web/src/timeline/controllers/resize-controller.ts +++ b/apps/web/src/timeline/controllers/resize-controller.ts @@ -109,18 +109,30 @@ export function buildResizeMembers({ ? elementEnd : maxMediaTime({ a: bound, b: elementEnd }); }, null); - const rightNeighborBound = otherElements - .filter( - (el) => - el.startTime >= addMediaTime({ a: element.startTime, b: element.duration }), - ) - .reduce<MediaTime | null>( - (bound, el) => - bound === null - ? el.startTime - : minMediaTime({ a: bound, b: el.startTime }), - null, - ); + const rightElements = otherElements.filter( + (el) => + el.startTime >= addMediaTime({ a: element.startTime, b: element.duration }), + ); + const rightNeighborBound = rightElements.reduce<MediaTime | null>( + (bound, el) => + bound === null + ? el.startTime + : minMediaTime({ a: bound, b: el.startTime }), + null, + ); + const rightPushChain = + element.sourceDuration == null + ? rightElements + .sort((a, b) => (a.startTime > b.startTime ? 1 : -1)) + .map((el) => ({ + trackId, + elementId: el.id, + startTime: el.startTime, + duration: el.duration, + trimStart: el.trimStart, + trimEnd: el.trimEnd, + })) + : undefined; return [ { @@ -134,6 +146,7 @@ export function buildResizeMembers({ retime: isRetimableElement(element) ? element.retime : undefined, leftNeighborBound, rightNeighborBound, + rightPushChain, }, ]; }); diff --git a/apps/web/src/timeline/group-resize/__tests__/compute-resize.test.ts b/apps/web/src/timeline/group-resize/__tests__/compute-resize.test.ts new file mode 100644 index 00000000..a678b18d --- /dev/null +++ b/apps/web/src/timeline/group-resize/__tests__/compute-resize.test.ts @@ -0,0 +1,559 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import type { FrameRate } from "opencut-wasm"; +import { EditorCore } from "@/core"; +import { UpdateElementsCommand } from "@/commands"; +import { buildDefaultScene } from "@/timeline/scenes"; +import type { ImageElement, TimelineElement, VideoElement } from "@/timeline"; +import { buildResizeMembers } from "@/timeline/controllers/resize-controller"; +import { computeGroupResize } from "@/timeline/group-resize"; +import type { + GroupResizeMember, + GroupResizePushTarget, + GroupResizeUpdate, +} from "@/timeline/group-resize"; +import { + addMediaTime, + type MediaTime, + mediaTime, + mediaTimeFromSeconds, + TICKS_PER_SECOND, + ZERO_MEDIA_TIME, +} from "@/wasm"; + +const FPS: FrameRate = { numerator: 30, denominator: 1 }; + +function seconds({ value }: { value: number }): MediaTime { + return mediaTimeFromSeconds({ seconds: value }); +} + +function buildPushTarget({ + elementId, + startTime, + duration, +}: { + elementId: string; + startTime: MediaTime; + duration: MediaTime; +}): GroupResizePushTarget { + return { + trackId: "track-1", + elementId, + startTime, + duration, + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + }; +} + +function buildStaticMember({ + startTime, + duration, + leftNeighborBound = null, + rightNeighborBound, + rightPushChain, +}: { + startTime: MediaTime; + duration: MediaTime; + leftNeighborBound?: MediaTime | null; + rightNeighborBound: MediaTime | null; + rightPushChain?: GroupResizePushTarget[]; +}): GroupResizeMember { + return { + trackId: "track-1", + elementId: "element-1", + startTime, + duration, + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + leftNeighborBound, + rightNeighborBound, + rightPushChain, + }; +} + +function buildVideoMember({ + startTime, + duration, + sourceDuration, + rightNeighborBound, +}: { + startTime: MediaTime; + duration: MediaTime; + sourceDuration: MediaTime; + rightNeighborBound: MediaTime | null; +}): GroupResizeMember { + return { + trackId: "track-1", + elementId: "element-1", + startTime, + duration, + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + sourceDuration, + leftNeighborBound: null, + rightNeighborBound, + }; +} + +function findUpdate({ + updates, + elementId, +}: { + updates: GroupResizeUpdate[]; + elementId: string; +}): GroupResizeUpdate | undefined { + return updates.find((update) => update.elementId === elementId); +} + +describe("computeGroupResize ripple push", () => { + test("extends into the gap before pushing anything", () => { + const member = buildStaticMember({ + startTime: ZERO_MEDIA_TIME, + duration: seconds({ value: 5 }), + rightNeighborBound: seconds({ value: 10 }), + rightPushChain: [ + buildPushTarget({ + elementId: "neighbor-1", + startTime: seconds({ value: 10 }), + duration: seconds({ value: 10 }), + }), + ], + }); + + const result = computeGroupResize({ + members: [member], + side: "right", + deltaTime: seconds({ value: 3 }), + fps: FPS, + }); + + expect(result.deltaTime).toBe(seconds({ value: 3 })); + expect(result.updates).toHaveLength(1); + expect( + findUpdate({ updates: result.updates, elementId: "element-1" })?.patch + .duration, + ).toBe(seconds({ value: 8 })); + }); + + test("pushes the flush right neighbor by the overflow", () => { + const member = buildStaticMember({ + startTime: ZERO_MEDIA_TIME, + duration: seconds({ value: 5 }), + rightNeighborBound: seconds({ value: 5 }), + rightPushChain: [ + buildPushTarget({ + elementId: "neighbor-1", + startTime: seconds({ value: 5 }), + duration: seconds({ value: 10 }), + }), + ], + }); + + const result = computeGroupResize({ + members: [member], + side: "right", + deltaTime: seconds({ value: 3 }), + fps: FPS, + }); + + expect(result.deltaTime).toBe(seconds({ value: 3 })); + expect(result.updates).toHaveLength(2); + expect( + findUpdate({ updates: result.updates, elementId: "element-1" })?.patch + .duration, + ).toBe(seconds({ value: 8 })); + const push = findUpdate({ updates: result.updates, elementId: "neighbor-1" }); + expect(push?.patch.startTime).toBe(seconds({ value: 8 })); + expect(push?.patch.duration).toBe(seconds({ value: 10 })); + expect(push?.patch.trimStart).toBe(ZERO_MEDIA_TIME); + expect(push?.patch.trimEnd).toBe(ZERO_MEDIA_TIME); + }); + + test("pushes a chain of several neighbors by the same overflow", () => { + const member = buildStaticMember({ + startTime: ZERO_MEDIA_TIME, + duration: seconds({ value: 5 }), + rightNeighborBound: seconds({ value: 5 }), + rightPushChain: [ + buildPushTarget({ + elementId: "neighbor-1", + startTime: seconds({ value: 5 }), + duration: seconds({ value: 10 }), + }), + buildPushTarget({ + elementId: "neighbor-2", + startTime: seconds({ value: 15 }), + duration: seconds({ value: 5 }), + }), + buildPushTarget({ + elementId: "neighbor-3", + startTime: seconds({ value: 25 }), + duration: seconds({ value: 2 }), + }), + ], + }); + + const result = computeGroupResize({ + members: [member], + side: "right", + deltaTime: seconds({ value: 4 }), + fps: FPS, + }); + + expect(result.deltaTime).toBe(seconds({ value: 4 })); + expect(result.updates).toHaveLength(4); + expect( + findUpdate({ updates: result.updates, elementId: "neighbor-1" })?.patch + .startTime, + ).toBe(seconds({ value: 9 })); + expect( + findUpdate({ updates: result.updates, elementId: "neighbor-2" })?.patch + .startTime, + ).toBe(seconds({ value: 19 })); + expect( + findUpdate({ updates: result.updates, elementId: "neighbor-3" })?.patch + .startTime, + ).toBe(seconds({ value: 29 })); + }); + + test("push stops at the end of the track", () => { + const member = buildStaticMember({ + startTime: ZERO_MEDIA_TIME, + duration: seconds({ value: 5 }), + rightNeighborBound: null, + rightPushChain: [], + }); + + const result = computeGroupResize({ + members: [member], + side: "right", + deltaTime: seconds({ value: 3 }), + fps: FPS, + }); + + expect(result.deltaTime).toBe(seconds({ value: 3 })); + expect(result.updates).toHaveLength(1); + expect( + findUpdate({ updates: result.updates, elementId: "element-1" })?.patch + .duration, + ).toBe(seconds({ value: 8 })); + }); + + test("video element clamps at its right neighbor instead of pushing", () => { + const member = buildVideoMember({ + startTime: ZERO_MEDIA_TIME, + duration: seconds({ value: 5 }), + sourceDuration: seconds({ value: 6 }), + rightNeighborBound: seconds({ value: 5 }), + }); + + const result = computeGroupResize({ + members: [member], + side: "right", + deltaTime: seconds({ value: 10 }), + fps: FPS, + }); + + expect(result.deltaTime).toBe(ZERO_MEDIA_TIME); + expect(result.updates).toHaveLength(1); + expect( + findUpdate({ updates: result.updates, elementId: "element-1" })?.patch + .duration, + ).toBe(seconds({ value: 5 })); + }); + + test("video element clamps at its source extent when there is no neighbor", () => { + const member = buildVideoMember({ + startTime: ZERO_MEDIA_TIME, + duration: seconds({ value: 5 }), + sourceDuration: seconds({ value: 6 }), + rightNeighborBound: null, + }); + + const result = computeGroupResize({ + members: [member], + side: "right", + deltaTime: seconds({ value: 10 }), + fps: FPS, + }); + + expect(result.deltaTime).toBe(seconds({ value: 1 })); + expect(result.updates).toHaveLength(1); + expect( + findUpdate({ updates: result.updates, elementId: "element-1" })?.patch + .duration, + ).toBe(seconds({ value: 6 })); + }); + + test("static element without a push chain keeps the old neighbor clamp", () => { + const member = buildStaticMember({ + startTime: ZERO_MEDIA_TIME, + duration: seconds({ value: 5 }), + rightNeighborBound: seconds({ value: 5 }), + }); + + const result = computeGroupResize({ + members: [member], + side: "right", + deltaTime: seconds({ value: 3 }), + fps: FPS, + }); + + expect(result.deltaTime).toBe(ZERO_MEDIA_TIME); + expect(result.updates).toHaveLength(1); + }); + + test("shortening still respects the one-frame minimum duration", () => { + const oneFrame = mediaTime({ + ticks: Math.round(TICKS_PER_SECOND / 30), + }); + const member = buildStaticMember({ + startTime: ZERO_MEDIA_TIME, + duration: seconds({ value: 5 }), + rightNeighborBound: null, + rightPushChain: [], + }); + + const result = computeGroupResize({ + members: [member], + side: "right", + deltaTime: seconds({ value: -10 }), + fps: FPS, + }); + + expect(result.updates).toHaveLength(1); + expect( + findUpdate({ updates: result.updates, elementId: "element-1" })?.patch + .duration, + ).toBe(oneFrame); + }); + + test("left-edge resize still clamps at the left neighbor and pushes nothing", () => { + const member = buildStaticMember({ + startTime: seconds({ value: 5 }), + duration: seconds({ value: 5 }), + leftNeighborBound: seconds({ value: 2 }), + rightNeighborBound: null, + rightPushChain: [ + buildPushTarget({ + elementId: "neighbor-1", + startTime: seconds({ value: 10 }), + duration: seconds({ value: 10 }), + }), + ], + }); + + const result = computeGroupResize({ + members: [member], + side: "left", + deltaTime: seconds({ value: -10 }), + fps: FPS, + }); + + expect(result.deltaTime).toBe(seconds({ value: -3 })); + expect(result.updates).toHaveLength(1); + const update = findUpdate({ updates: result.updates, elementId: "element-1" }); + expect(update?.patch.startTime).toBe(seconds({ value: 2 })); + expect(update?.patch.duration).toBe(seconds({ value: 8 })); + }); +}); + +const IMAGE_DURATION = seconds({ value: 5 }); +const VIDEO_DURATION = seconds({ value: 10 }); +const TAIL_DURATION = seconds({ value: 5 }); +const PUSH_DELTA = seconds({ value: 3 }); + +function buildImageElement(): ImageElement { + return { + id: "image-1", + type: "image", + name: "image-1", + startTime: ZERO_MEDIA_TIME, + duration: IMAGE_DURATION, + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + mediaId: "media-image-1", + params: { + "transform.positionX": 0, + "transform.positionY": 0, + "transform.scaleX": 1, + "transform.scaleY": 1, + "transform.rotate": 0, + opacity: 1, + }, + }; +} + +function buildVideoElement({ + id, + startTime, + duration, +}: { + id: string; + startTime: MediaTime; + duration: MediaTime; +}): VideoElement { + return { + id, + type: "video", + name: id, + startTime, + duration, + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + sourceDuration: duration, + mediaId: `media-${id}`, + params: { + "transform.positionX": 0, + "transform.positionY": 0, + "transform.scaleX": 1, + "transform.scaleY": 1, + "transform.rotate": 0, + opacity: 1, + }, + }; +} + +function seedScene(): { editor: EditorCore; trackId: string } { + const editor = EditorCore.getInstance(); + const scene = buildDefaultScene({ name: "Main scene", isMain: true }); + scene.tracks.main.elements = [ + buildImageElement(), + buildVideoElement({ + id: "video-1", + startTime: IMAGE_DURATION, + duration: VIDEO_DURATION, + }), + buildVideoElement({ + id: "video-2", + startTime: addMediaTime({ a: IMAGE_DURATION, b: VIDEO_DURATION }), + duration: TAIL_DURATION, + }), + ]; + editor.project.setActiveProject({ + project: { + metadata: { + id: "project-1", + name: "Test project", + duration: addMediaTime({ + a: IMAGE_DURATION, + b: addMediaTime({ a: VIDEO_DURATION, b: TAIL_DURATION }), + }), + createdAt: new Date("2026-01-01T00:00:00Z"), + updatedAt: new Date("2026-01-01T00:00:00Z"), + }, + scenes: [scene], + currentSceneId: scene.id, + settings: { + fps: FPS, + canvasSize: { width: 1920, height: 1080 }, + background: { type: "color", color: "#000000" }, + }, + version: 1, + }, + }); + editor.scenes.setScenes({ scenes: [scene], activeSceneId: scene.id }); + return { editor, trackId: scene.tracks.main.id }; +} + +function getElementById({ + editor, + elementId, +}: { + editor: EditorCore; + elementId: string; +}): TimelineElement | undefined { + return editor.scenes + .getActiveScene() + .tracks.main.elements.find((element) => element.id === elementId); +} + +function commitResizeResult({ + editor, + trackId, +}: { + editor: EditorCore; + trackId: string; +}): void { + const members = buildResizeMembers({ + tracks: editor.scenes.getActiveScene().tracks, + selectedElements: [{ trackId, elementId: "image-1" }], + }); + const result = computeGroupResize({ + members, + side: "right", + deltaTime: PUSH_DELTA, + fps: FPS, + }); + editor.command.execute({ + command: new UpdateElementsCommand({ + updates: result.updates.map( + ({ trackId: updateTrackId, elementId, patch }) => ({ + trackId: updateTrackId, + elementId, + patch: patch as Partial<TimelineElement>, + }), + ), + }), + }); +} + +function expectPushedLayout({ editor }: { editor: EditorCore }): void { + expect(getElementById({ editor, elementId: "image-1" })?.duration).toBe( + addMediaTime({ a: IMAGE_DURATION, b: PUSH_DELTA }), + ); + expect(getElementById({ editor, elementId: "video-1" })?.startTime).toBe( + addMediaTime({ a: IMAGE_DURATION, b: PUSH_DELTA }), + ); + expect(getElementById({ editor, elementId: "video-2" })?.startTime).toBe( + addMediaTime({ + a: addMediaTime({ a: IMAGE_DURATION, b: VIDEO_DURATION }), + b: PUSH_DELTA, + }), + ); +} + +function expectOriginalLayout({ editor }: { editor: EditorCore }): void { + expect(getElementById({ editor, elementId: "image-1" })?.duration).toBe( + IMAGE_DURATION, + ); + expect(getElementById({ editor, elementId: "video-1" })?.startTime).toBe( + IMAGE_DURATION, + ); + expect(getElementById({ editor, elementId: "video-2" })?.startTime).toBe( + addMediaTime({ a: IMAGE_DURATION, b: VIDEO_DURATION }), + ); +} + +describe("ripple push commit", () => { + beforeEach(() => { + EditorCore.reset(); + }); + + test("resize and push land in one undoable step", () => { + const { editor, trackId } = seedScene(); + + commitResizeResult({ editor, trackId }); + + expectPushedLayout({ editor }); + }); + + test("undo restores the original positions", () => { + const { editor, trackId } = seedScene(); + + commitResizeResult({ editor, trackId }); + editor.command.undo(); + + expectOriginalLayout({ editor }); + }); + + test("redo after undo re-applies the resize and push", () => { + const { editor, trackId } = seedScene(); + + commitResizeResult({ editor, trackId }); + editor.command.undo(); + editor.command.redo(); + + expectPushedLayout({ editor }); + }); +}); diff --git a/apps/web/src/timeline/group-resize/compute-resize.ts b/apps/web/src/timeline/group-resize/compute-resize.ts index ed335997..2674aa8d 100644 --- a/apps/web/src/timeline/group-resize/compute-resize.ts +++ b/apps/web/src/timeline/group-resize/compute-resize.ts @@ -18,6 +18,7 @@ import { import type { ComputeGroupResizeArgs, GroupResizeMember, + GroupResizePushTarget, GroupResizeResult, GroupResizeUpdate, ResizeSide, @@ -103,16 +104,72 @@ export function computeGroupResize({ return { deltaTime: Object.is(finalDeltaTime, -0) ? ZERO_MEDIA_TIME : finalDeltaTime, - updates: members.map((member) => - buildResizeUpdate({ - member, - side, - deltaTime: finalDeltaTime, - }), - ), + updates: [ + ...members.map((member) => + buildResizeUpdate({ + member, + side, + deltaTime: finalDeltaTime, + }), + ), + ...buildRipplePushUpdates({ members, side, deltaTime: finalDeltaTime }), + ], }; } +function buildRipplePushUpdates({ + members, + side, + deltaTime, +}: { + members: GroupResizeMember[]; + side: ResizeSide; + deltaTime: MediaTime; +}): GroupResizeUpdate[] { + if (side !== "right" || deltaTime <= 0) { + return []; + } + + const overflowByTarget = new Map< + string, + { target: GroupResizePushTarget; overflow: MediaTime } + >(); + for (const member of members) { + if ( + member.sourceDuration != null || + member.rightPushChain === undefined || + member.rightNeighborBound === null + ) { + continue; + } + const neighborCeiling = subMediaTime({ + a: member.rightNeighborBound, + b: addMediaTime({ a: member.startTime, b: member.duration }), + }); + const overflow = subMediaTime({ a: deltaTime, b: neighborCeiling }); + if (overflow <= 0) { + continue; + } + for (const target of member.rightPushChain) { + const existing = overflowByTarget.get(target.elementId); + if (existing === undefined || overflow > existing.overflow) { + overflowByTarget.set(target.elementId, { target, overflow }); + } + } + } + + return [...overflowByTarget.values()].map(({ target, overflow }) => ({ + trackId: target.trackId, + elementId: target.elementId, + patch: { + trimStart: target.trimStart, + trimEnd: target.trimEnd, + startTime: addMediaTime({ a: target.startTime, b: overflow }), + duration: target.duration, + }, + })); +} + function buildResizeUpdate({ member, side, @@ -219,7 +276,7 @@ function getMaximumAllowedDeltaTime({ b: addMediaTime({ a: member.startTime, b: member.duration }), }); if (member.sourceDuration == null) { - return rightNeighborCeiling; + return member.rightPushChain === undefined ? rightNeighborCeiling : null; } const maximumVisibleSourceSpan = subMediaTime({ diff --git a/apps/web/src/timeline/group-resize/index.ts b/apps/web/src/timeline/group-resize/index.ts index bc0a7b35..662ae24e 100644 --- a/apps/web/src/timeline/group-resize/index.ts +++ b/apps/web/src/timeline/group-resize/index.ts @@ -3,6 +3,7 @@ export { computeGroupResize } from "./compute-resize"; export type { ComputeGroupResizeArgs, GroupResizeMember, + GroupResizePushTarget, GroupResizeResult, GroupResizeUpdate, ResizeSide, diff --git a/apps/web/src/timeline/group-resize/types.ts b/apps/web/src/timeline/group-resize/types.ts index 5c0f77f6..f483d156 100644 --- a/apps/web/src/timeline/group-resize/types.ts +++ b/apps/web/src/timeline/group-resize/types.ts @@ -4,6 +4,13 @@ import type { MediaTime } from "@/wasm"; export type ResizeSide = "left" | "right"; +export interface GroupResizePushTarget extends ElementRef { + startTime: MediaTime; + duration: MediaTime; + trimStart: MediaTime; + trimEnd: MediaTime; +} + export interface GroupResizeMember extends ElementRef { startTime: MediaTime; duration: MediaTime; @@ -13,6 +20,7 @@ export interface GroupResizeMember extends ElementRef { retime?: RetimeConfig; leftNeighborBound: MediaTime | null; rightNeighborBound: MediaTime | null; + rightPushChain?: GroupResizePushTarget[]; } export interface GroupResizeUpdate extends ElementRef { |