diff options
102 files changed, 4977 insertions, 3707 deletions
@@ -1428,6 +1428,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] +name = "effects" +version = "0.1.0" +dependencies = [ + "bytemuck", + "gpu", + "thiserror 2.0.18", + "wgpu", +] + +[[package]] name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3218,6 +3228,15 @@ dependencies = [ ] [[package]] +name = "masks" +version = "0.1.0" +dependencies = [ + "bytemuck", + "gpu", + "wgpu", +] + +[[package]] name = "maybe-rayon" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3777,10 +3796,14 @@ dependencies = [ [[package]] name = "opencut-wasm" -version = "0.1.3" +version = "0.2.3" dependencies = [ + "bridge", + "effects", "gpu", "js-sys", + "masks", + "num-traits", "serde", "serde-wasm-bindgen", "time", @@ -5581,6 +5604,7 @@ name = "time" version = "0.1.0" dependencies = [ "bridge", + "num-traits", "serde", "tsify-next", "wasm-bindgen", @@ -4,5 +4,8 @@ members = [ "apps/desktop", "rust/crates/time", "rust/crates/bridge", - "rust/wasm", "rust/crates/gpu", + "rust/crates/effects", + "rust/crates/gpu", + "rust/crates/masks", + "rust/wasm", ] diff --git a/apps/web/content/changelog/0.3.0.md b/apps/web/content/changelog/0.3.0.md index 42c7c9a7..92675321 100644 --- a/apps/web/content/changelog/0.3.0.md +++ b/apps/web/content/changelog/0.3.0.md @@ -79,5 +79,7 @@ changes: text: "Auto-generated captions were sometimes inaccurate. The underlying issue has been fixed."
- type: new
text: "You can now import transcript files to generate captions instead of running auto-transcription."
+ - type: new
+ text: "Graph editor for keyframe curves. Shape the easing between keyframes by dragging bezier handles."
---
diff --git a/apps/web/package.json b/apps/web/package.json index 0317d3b9..7ee5fe1a 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -52,7 +52,7 @@ "nanoid": "^5.1.5", "next": "16.1.3", "next-themes": "^0.4.4", - "opencut-wasm": "^0.1.3", + "opencut-wasm": "^0.2.3", "pg": "^8.16.2", "postgres": "^3.4.5", "radix-ui": "^1.4.3", diff --git a/apps/web/src/app/projects/page.tsx b/apps/web/src/app/projects/page.tsx index 8ee9d504..93d6e189 100644 --- a/apps/web/src/app/projects/page.tsx +++ b/apps/web/src/app/projects/page.tsx @@ -21,7 +21,7 @@ import type { TProjectSortKey, TProjectSortOption, } from "@/lib/project/types"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode, mediaTimeToSeconds } from "opencut-wasm"; import { formatDate } from "@/utils/date"; import { HugeiconsIcon } from "@hugeicons/react"; import { @@ -76,8 +76,9 @@ const formatProjectDuration = ({ return null; } - const format = duration >= 3600 ? "HH:MM:SS" : "MM:SS"; - return formatTimeCode({ timeInSeconds: duration, format }) ?? ""; + const durationSeconds = mediaTimeToSeconds({ time: duration }); + const format = durationSeconds >= 3600 ? "HH:MM:SS" : "MM:SS"; + return formatTimecode({ time: duration, format }) ?? ""; }; const VIEW_MODE_OPTIONS = [ diff --git a/apps/web/src/components/editable-timecode.tsx b/apps/web/src/components/editable-timecode.tsx index 6a323a1c..276cb490 100644 --- a/apps/web/src/components/editable-timecode.tsx +++ b/apps/web/src/components/editable-timecode.tsx @@ -1,14 +1,14 @@ "use client"; import { useEffect, useRef, useState } from "react"; -import { formatTimeCode, parseTimeCode, type TimeCodeFormat } from "opencut-wasm"; +import { formatTimecode, parseTimecode, snappedSeekTime, type FrameRate, type TimeCodeFormat } from "opencut-wasm"; import { cn } from "@/utils/ui"; interface EditableTimecodeProps { time: number; duration: number; format?: TimeCodeFormat; - fps: number; + fps: FrameRate; onTimeChange?: ({ time }: { time: number }) => void; className?: string; disabled?: boolean; @@ -28,7 +28,7 @@ export function EditableTimecode({ const [hasError, setHasError] = useState(false); const inputRef = useRef<HTMLInputElement>(null); const enterPressedRef = useRef(false); - const formattedTime = formatTimeCode({ timeInSeconds: time, format, fps }) ?? ""; + const formattedTime = formatTimecode({ time, format, rate: fps }) ?? ""; const startEditing = () => { if (disabled) return; @@ -46,17 +46,16 @@ export function EditableTimecode({ }; const applyEdit = () => { - const parsedTime = parseTimeCode({ timeCode: inputValue, format, fps }); + const parsedTime = parseTimecode({ timeCode: inputValue, format, rate: fps }); if (parsedTime == null) { setHasError(true); return; } - const clampedTime = Math.max( - 0, - duration ? Math.min(duration, parsedTime) : parsedTime, - ); + const clampedTime = duration + ? (snappedSeekTime({ time: parsedTime, duration, rate: fps }) ?? parsedTime) + : parsedTime; onTimeChange?.({ time: clampedTime }); setIsEditing(false); diff --git a/apps/web/src/components/editor/dialogs/project-info-dialog.tsx b/apps/web/src/components/editor/dialogs/project-info-dialog.tsx index c31c8e96..ed4eb990 100644 --- a/apps/web/src/components/editor/dialogs/project-info-dialog.tsx +++ b/apps/web/src/components/editor/dialogs/project-info-dialog.tsx @@ -8,7 +8,7 @@ import { } from "@/components/ui/dialog"; import type { TProjectMetadata } from "@/lib/project/types"; import { formatDate } from "@/utils/date"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode, mediaTimeToSeconds } from "opencut-wasm"; import { Button } from "@/components/ui/button"; function InfoRow({ @@ -35,9 +35,10 @@ export function ProjectInfoDialog({ onOpenChange: (open: boolean) => void; project: TProjectMetadata; }) { + const durationSeconds = mediaTimeToSeconds({ time: project.duration }); const durationFormatted = project.duration > 0 - ? (formatTimeCode({ timeInSeconds: project.duration, format: project.duration >= 3600 ? "HH:MM:SS" : "MM:SS" }) ?? "") + ? (formatTimecode({ time: project.duration, format: durationSeconds >= 3600 ? "HH:MM:SS" : "MM:SS" }) ?? "") : "0:00"; return ( diff --git a/apps/web/src/components/editor/panels/assets/views/assets.tsx b/apps/web/src/components/editor/panels/assets/views/assets.tsx index 031d2e6a..85888dce 100644 --- a/apps/web/src/components/editor/panels/assets/views/assets.tsx +++ b/apps/web/src/components/editor/panels/assets/views/assets.tsx @@ -25,7 +25,7 @@ import { TooltipProvider, TooltipTrigger, } from "@/components/ui/tooltip"; -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; import { useEditor } from "@/hooks/use-editor"; import { useFileUpload } from "@/hooks/use-file-upload"; import { invokeAction } from "@/lib/actions"; @@ -257,7 +257,7 @@ function MediaAssetDraggable({ startTime: number; }) => { const duration = - asset.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + asset.duration ?? DEFAULT_NEW_ELEMENT_DURATION; const element = buildElementFromMedia({ mediaId: asset.id, mediaType: asset.type, diff --git a/apps/web/src/components/editor/panels/assets/views/settings/index.tsx b/apps/web/src/components/editor/panels/assets/views/settings/index.tsx index 6c819a36..16faad3d 100644 --- a/apps/web/src/components/editor/panels/assets/views/settings/index.tsx +++ b/apps/web/src/components/editor/panels/assets/views/settings/index.tsx @@ -10,6 +10,7 @@ import { SelectValue, } from "@/components/ui/select"; import { FPS_PRESETS } from "@/lib/fps/constants"; +import { floatToFrameRate, frameRateToFloat } from "@/lib/fps/utils"; import { useEditor } from "@/hooks/use-editor"; import { Section, @@ -232,12 +233,12 @@ export function SettingsView() { <Section showTopBorder={false}> <SectionHeader className="justify-between"> <SectionTitle className="flex-1">Frame rate</SectionTitle> - <Select - value={activeProject.settings.fps.toString()} - onValueChange={(value) => { - const fps = parseFloat(value); - editor.project.updateSettings({ settings: { fps } }); - }} + <Select + value={String(Math.round(frameRateToFloat(activeProject.settings.fps)))} + onValueChange={(value) => { + const fps = floatToFrameRate(parseFloat(value)); + editor.project.updateSettings({ settings: { fps } }); + }} > <SelectTrigger className="bg-transparent border-none p-1 h-auto"> <SelectValue placeholder="Select a frame rate" /> diff --git a/apps/web/src/components/editor/panels/preview/index.tsx b/apps/web/src/components/editor/panels/preview/index.tsx index 8cfe7f49..786d9a46 100644 --- a/apps/web/src/components/editor/panels/preview/index.tsx +++ b/apps/web/src/components/editor/panels/preview/index.tsx @@ -7,6 +7,7 @@ import { useRafLoop } from "@/hooks/use-raf-loop"; import { useContainerSize } from "@/hooks/use-container-size"; import { useFullscreen } from "@/hooks/use-fullscreen"; import { CanvasRenderer } from "@/services/renderer/canvas-renderer"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import type { RootNode } from "@/services/renderer/nodes/root-node"; import { buildScene } from "@/services/renderer/scene-builder"; import { PreviewInteractionOverlay } from "./preview-interaction-overlay"; @@ -129,12 +130,16 @@ function PreviewCanvas({ height: nativeHeight, fps: activeProject.settings.fps, }); - }, [nativeWidth, nativeHeight, activeProject.settings.fps]); + }, [nativeWidth, nativeHeight, activeProject.settings.fps.numerator, activeProject.settings.fps.denominator]); const render = useCallback(() => { if (canvasRef.current && renderTree && !renderingRef.current) { - const renderTime = editor.playback.getCurrentTime(); - const frame = Math.floor(renderTime * renderer.fps); + const renderTime = Math.min( + editor.playback.getCurrentTime(), + editor.timeline.getLastFrameTime(), + ); + const ticksPerFrame = Math.round(TICKS_PER_SECOND * renderer.fps.denominator / renderer.fps.numerator); + const frame = Math.floor(renderTime / ticksPerFrame); if ( frame !== lastFrameRef.current || diff --git a/apps/web/src/components/editor/panels/preview/toolbar.tsx b/apps/web/src/components/editor/panels/preview/toolbar.tsx index 0be1ec46..555cde17 100644 --- a/apps/web/src/components/editor/panels/preview/toolbar.tsx +++ b/apps/web/src/components/editor/panels/preview/toolbar.tsx @@ -2,7 +2,7 @@ import { useState, useEffect } from "react"; import { useEditor } from "@/hooks/use-editor"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode } from "opencut-wasm"; import { invokeAction } from "@/lib/actions"; import { EditableTimecode } from "@/components/editable-timecode"; import { Button } from "@/components/ui/button"; @@ -93,7 +93,7 @@ function TimecodeDisplay() { /> <span className="text-muted-foreground px-2 font-mono text-xs">/</span> <span className="text-muted-foreground font-mono text-xs"> - {formatTimeCode({ timeInSeconds: totalDuration, format: "HH:MM:SS:FF", fps })} + {formatTimecode({ time: totalDuration, format: "HH:MM:SS:FF", rate: fps })} </span> </div> ); diff --git a/apps/web/src/components/editor/panels/properties/hooks/use-element-playhead.ts b/apps/web/src/components/editor/panels/properties/hooks/use-element-playhead.ts index a900cb13..27300784 100644 --- a/apps/web/src/components/editor/panels/properties/hooks/use-element-playhead.ts +++ b/apps/web/src/components/editor/panels/properties/hooks/use-element-playhead.ts @@ -1,23 +1,22 @@ -import { useEditor } from "@/hooks/use-editor";
-import { getElementLocalTime } from "@/lib/animation";
-import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants";
-
-export function useElementPlayhead({
- startTime,
- duration,
-}: {
- startTime: number;
- duration: number;
-}) {
- const playheadTime = useEditor((editor) => editor.playback.getCurrentTime());
- const localTime = getElementLocalTime({
- timelineTime: playheadTime,
- elementStartTime: startTime,
- elementDuration: duration,
- });
- const isPlayheadWithinElementRange =
- playheadTime >= startTime - TIME_EPSILON_SECONDS &&
- playheadTime <= startTime + duration + TIME_EPSILON_SECONDS;
-
- return { localTime, isPlayheadWithinElementRange };
-}
+import { useEditor } from "@/hooks/use-editor"; +import { getElementLocalTime } from "@/lib/animation"; + +export function useElementPlayhead({ + startTime, + duration, +}: { + startTime: number; + duration: number; +}) { + const playheadTime = useEditor((editor) => editor.playback.getCurrentTime()); + const localTime = getElementLocalTime({ + timelineTime: playheadTime, + elementStartTime: startTime, + elementDuration: duration, + }); + const isPlayheadWithinElementRange = + playheadTime >= startTime && + playheadTime <= startTime + duration; + + return { localTime, isPlayheadWithinElementRange }; +} diff --git a/apps/web/src/components/editor/panels/properties/tabs/masks-tab.tsx b/apps/web/src/components/editor/panels/properties/tabs/masks-tab.tsx index f48a50cb..9b0ffc88 100644 --- a/apps/web/src/components/editor/panels/properties/tabs/masks-tab.tsx +++ b/apps/web/src/components/editor/panels/properties/tabs/masks-tab.tsx @@ -15,7 +15,6 @@ import { PlusSignIcon, RotateClockwiseIcon, } from "@hugeicons/core-free-icons"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { useMemo, useState } from "react"; import { Button } from "@/components/ui/button"; import { ColorPicker } from "@/components/ui/color-picker"; @@ -102,7 +101,7 @@ export function MasksTab({ element, trackId }: MasksTabProps) { const elementBounds = useMemo(() => { const clampedTime = Math.min( Math.max(currentTime, element.startTime), - element.startTime + element.duration - TIME_EPSILON_SECONDS, + element.startTime + element.duration - 1, ); return ( diff --git a/apps/web/src/components/editor/panels/timeline/bookmarks.tsx b/apps/web/src/components/editor/panels/timeline/bookmarks.tsx index d4b1657e..9342c30e 100644 --- a/apps/web/src/components/editor/panels/timeline/bookmarks.tsx +++ b/apps/web/src/components/editor/panels/timeline/bookmarks.tsx @@ -4,13 +4,12 @@ import { useEffect, useState } from "react"; import type { EditorCore } from "@/core"; import { useEditor } from "@/hooks/use-editor"; import type { BookmarkDragState } from "@/hooks/timeline/use-bookmark-drag"; -import { BOOKMARK_TIME_EPSILON } from "@/lib/timeline/bookmarks"; import { DEFAULT_TIMELINE_BOOKMARK_COLOR, } from "./theme"; import { TIMELINE_BOOKMARK_ROW_HEIGHT_PX } from "./layout"; import { DEFAULT_FPS } from "@/lib/fps/constants"; -import { getSnappedSeekTime } from "opencut-wasm"; +import { snappedSeekTime } from "opencut-wasm"; import { ArrowTurnBackwardIcon, Delete02Icon, @@ -49,8 +48,8 @@ function seekToBookmarkTime({ }) { const activeProject = editor.project.getActive(); const duration = editor.timeline.getTotalDuration(); - const fps = activeProject?.settings.fps ?? DEFAULT_FPS; - const snappedTime = getSnappedSeekTime({ rawTime: time, duration, fps }); + const rate = activeProject?.settings.fps ?? DEFAULT_FPS; + const snappedTime = snappedSeekTime({ time, duration, rate }) ?? time; editor.playback.seek({ time: snappedTime }); } @@ -139,7 +138,7 @@ function TimelineBookmark({ const isDragging = dragState.isDragging && dragState.bookmarkTime !== null && - Math.abs(dragState.bookmarkTime - bookmark.time) < BOOKMARK_TIME_EPSILON; + dragState.bookmarkTime === bookmark.time; const displayTime = isDragging ? dragState.currentTime : bookmark.time; const time = bookmark.time; diff --git a/apps/web/src/components/editor/panels/timeline/selection-hit-testing.ts b/apps/web/src/components/editor/panels/timeline/selection-hit-testing.ts index 6488e29a..551393e9 100644 --- a/apps/web/src/components/editor/panels/timeline/selection-hit-testing.ts +++ b/apps/web/src/components/editor/panels/timeline/selection-hit-testing.ts @@ -1,5 +1,5 @@ import type { TimelineTrack } from "@/lib/timeline"; -import { getTimelinePixelsPerSecond } from "@/lib/timeline/pixel-utils"; +import { timelineTimeToPixels } from "@/lib/timeline/pixel-utils"; import { TIMELINE_CONTENT_TOP_PADDING_PX, } from "./layout"; @@ -96,7 +96,6 @@ export function resolveTimelineElementIntersections({ startPos, endPos: currentPos, }); - const pixelsPerSecond = getTimelinePixelsPerSecond({ zoomLevel }); const selectedElements: TimelineElementRef[] = []; for (const [trackIndex, track] of tracks.entries()) { @@ -109,8 +108,8 @@ export function resolveTimelineElementIntersections({ const elementBottom = elementTop + trackHeight; for (const element of track.elements) { - const elementLeft = element.startTime * pixelsPerSecond; - const elementRight = elementLeft + element.duration * pixelsPerSecond; + const elementLeft = timelineTimeToPixels({ time: element.startTime, zoomLevel }); + const elementRight = timelineTimeToPixels({ time: element.startTime + element.duration, zoomLevel }); const elementRectangle = { left: elementLeft, top: elementTop, diff --git a/apps/web/src/components/editor/panels/timeline/timeline-playhead.tsx b/apps/web/src/components/editor/panels/timeline/timeline-playhead.tsx index 361f1398..d9218371 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-playhead.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-playhead.tsx @@ -7,6 +7,7 @@ import { timelineTimeToSnappedPixels, } from "@/lib/timeline"; import { useTimelinePlayhead } from "@/hooks/timeline/use-timeline-playhead"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { useEditor } from "@/hooks/use-editor"; import { TIMELINE_SCROLLBAR_SIZE_PX, @@ -72,10 +73,11 @@ export function TimelinePlayhead({ if (event.key !== "ArrowLeft" && event.key !== "ArrowRight") return; event.preventDefault(); - const step = 1 / Math.max(1, editor.project.getActive().settings.fps); + const fps = editor.project.getActive().settings.fps; + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); const direction = event.key === "ArrowRight" ? 1 : -1; const now = editor.playback.getCurrentTime(); - const nextTime = Math.max(0, Math.min(duration, now + direction * step)); + const nextTime = Math.max(0, Math.min(duration, now + direction * ticksPerFrame)); editor.playback.seek({ time: nextTime }); }; diff --git a/apps/web/src/components/editor/panels/timeline/timeline-ruler.tsx b/apps/web/src/components/editor/panels/timeline/timeline-ruler.tsx index 9e49bf6c..8b8dab32 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-ruler.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-ruler.tsx @@ -2,6 +2,8 @@ import { type JSX, useLayoutEffect, useRef } from "react"; import { BASE_TIMELINE_PIXELS_PER_SECOND, } from "@/lib/timeline/scale"; +import { mediaTimeToSeconds } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { TIMELINE_RULER_HEIGHT_PX } from "./layout"; import { DEFAULT_FPS } from "@/lib/fps/constants"; import { useEditor } from "@/hooks/use-editor"; @@ -30,17 +32,18 @@ export function TimelineRuler({ handleRulerTrackingMouseDown, handleRulerMouseDown, }: TimelineRulerProps) { - const duration = useEditor((e) => e.timeline.getTotalDuration()); + const durationTicks = useEditor((e) => e.timeline.getTotalDuration()); + const durationSeconds = mediaTimeToSeconds({ time: durationTicks }); const pixelsPerSecond = BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; - const visibleDuration = dynamicTimelineWidth / pixelsPerSecond; - const effectiveDuration = Math.max(duration, visibleDuration); + const visibleDurationSeconds = dynamicTimelineWidth / pixelsPerSecond; + const effectiveDurationSeconds = Math.max(durationSeconds, visibleDurationSeconds); const fps = useEditor((e) => e.project.getActiveOrNull()?.settings.fps) ?? DEFAULT_FPS; const { labelIntervalSeconds, tickIntervalSeconds } = getRulerConfig({ zoomLevel, fps, }); - const tickCount = Math.ceil(effectiveDuration / tickIntervalSeconds) + 1; + const tickCount = Math.ceil(effectiveDurationSeconds / tickIntervalSeconds) + 1; const { scrollLeft, viewportWidth } = useScrollPosition({ scrollRef: tracksScrollRef, @@ -61,20 +64,20 @@ export function TimelineRuler({ prevZoomRef.current = zoomLevel; }, [zoomLevel]); - const visibleStartTime = Math.max( + const visibleStartTimeSeconds = Math.max( 0, (scrollLeft - bufferPx) / pixelsPerSecond, ); - const visibleEndTime = + const visibleEndTimeSeconds = (scrollLeft + viewportWidth + bufferPx) / pixelsPerSecond; const startTickIndex = Math.max( 0, - Math.floor(visibleStartTime / tickIntervalSeconds), + Math.floor(visibleStartTimeSeconds / tickIntervalSeconds), ); const endTickIndex = Math.min( tickCount - 1, - Math.ceil(visibleEndTime / tickIntervalSeconds), + Math.ceil(visibleEndTimeSeconds / tickIntervalSeconds), ); const timelineTicks: Array<JSX.Element> = []; @@ -83,14 +86,16 @@ export function TimelineRuler({ tickIndex <= endTickIndex; tickIndex += 1 ) { - const time = tickIndex * tickIntervalSeconds; - if (time > effectiveDuration) break; + const timeSeconds = tickIndex * tickIntervalSeconds; + if (timeSeconds > effectiveDurationSeconds) break; - const showLabel = shouldShowLabel({ time, labelIntervalSeconds }); + const timeTicks = Math.round(timeSeconds * TICKS_PER_SECOND); + const showLabel = shouldShowLabel({ time: timeSeconds, labelIntervalSeconds }); timelineTicks.push( <TimelineTick key={tickIndex} - time={time} + time={timeTicks} + timeInSeconds={timeSeconds} zoomLevel={zoomLevel} fps={fps} showLabel={showLabel} @@ -104,7 +109,7 @@ export function TimelineRuler({ tabIndex={0} aria-label="Timeline ruler" aria-valuemin={0} - aria-valuemax={effectiveDuration} + aria-valuemax={effectiveDurationSeconds} aria-valuenow={0} className="relative flex-1 overflow-x-visible" style={{ height: TIMELINE_RULER_HEIGHT_PX }} diff --git a/apps/web/src/components/editor/panels/timeline/timeline-tick.tsx b/apps/web/src/components/editor/panels/timeline/timeline-tick.tsx index 7d9a4bc2..fca55752 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-tick.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-tick.tsx @@ -1,39 +1,42 @@ -"use client";
-
-import { timelineTimeToSnappedPixels } from "@/lib/timeline";
-import { formatRulerLabel } from "@/lib/timeline/ruler-utils";
-
-interface TimelineTickProps {
- time: number;
- zoomLevel: number;
- fps: number;
- showLabel: boolean;
-}
-
-export function TimelineTick({
- time,
- zoomLevel,
- fps,
- showLabel,
-}: TimelineTickProps) {
- const leftPosition = timelineTimeToSnappedPixels({ time, zoomLevel });
-
- if (showLabel) {
- const label = formatRulerLabel({ timeInSeconds: time, fps });
- return (
- <span
- className="text-muted-foreground/85 absolute top-1 select-none text-[10px] leading-none"
- style={{ left: `${leftPosition}px` }}
- >
- {label}
- </span>
- );
- }
-
- return (
- <div
- className="border-muted-foreground/25 absolute top-1.5 h-1.5 border-l"
- style={{ left: `${leftPosition}px` }}
- />
- );
-}
+"use client"; + +import type { FrameRate } from "opencut-wasm"; +import { timelineTimeToSnappedPixels } from "@/lib/timeline"; +import { formatRulerLabel } from "@/lib/timeline/ruler-utils"; + +interface TimelineTickProps { + time: number; + timeInSeconds: number; + zoomLevel: number; + fps: FrameRate; + showLabel: boolean; +} + +export function TimelineTick({ + time, + timeInSeconds, + zoomLevel, + fps, + showLabel, +}: TimelineTickProps) { + const leftPosition = timelineTimeToSnappedPixels({ time, zoomLevel }); + + if (showLabel) { + const label = formatRulerLabel({ timeInSeconds, fps }); + return ( + <span + className="text-muted-foreground/85 absolute top-1 select-none text-[10px] leading-none" + style={{ left: `${leftPosition}px` }} + > + {label} + </span> + ); + } + + return ( + <div + className="border-muted-foreground/25 absolute top-1.5 h-1.5 border-l" + style={{ left: `${leftPosition}px` }} + /> + ); +} diff --git a/apps/web/src/constants/animation-constants.ts b/apps/web/src/constants/animation-constants.ts index 17056114..ab932ea2 100644 --- a/apps/web/src/constants/animation-constants.ts +++ b/apps/web/src/constants/animation-constants.ts @@ -1,2 +1 @@ -export const TIME_EPSILON_SECONDS = 1 / 1000;
-export const MIN_TRANSFORM_SCALE = 0.01;
+export const MIN_TRANSFORM_SCALE = 0.01; diff --git a/apps/web/src/core/managers/audio-manager.ts b/apps/web/src/core/managers/audio-manager.ts index 0f642b1f..aaeb7ade 100644 --- a/apps/web/src/core/managers/audio-manager.ts +++ b/apps/web/src/core/managers/audio-manager.ts @@ -1,708 +1,709 @@ -import type { EditorCore } from "@/core";
-import {
- clampRetimeRate,
- shouldMaintainPitch,
-} from "@/lib/retime/rate";
-import type { AudioClipSource } from "@/lib/media/audio";
-import { createAudioContext, collectAudioClips } from "@/lib/media/audio";
-import {
- buildAudioGainAutomation,
- hasAnimatedVolume,
-} from "@/lib/timeline/audio-state";
-import { createAudioMasteringChain } from "@/lib/media/audio-mastering";
-import {
- getClipTimeAtSourceTime,
- getSourceTimeAtClipTime,
- renderRetimedBuffer,
-} from "@/lib/retime";
-import {
- ALL_FORMATS,
- AudioBufferSink,
- BlobSource,
- Input,
- type WrappedAudioBuffer,
-} from "mediabunny";
-
-export class AudioManager {
- private audioContext: AudioContext | null = null;
- private masterGain: GainNode | null = null;
- private playbackStartTime = 0;
- private playbackStartContextTime = 0;
- private scheduleTimer: number | null = null;
- private lookaheadSeconds = 2;
- private scheduleIntervalMs = 500;
- private clips: AudioClipSource[] = [];
- private sinks = new Map<string, AudioBufferSink>();
- private inputs = new Map<string, Input>();
- private activeClipIds = new Set<string>();
- private clipIterators = new Map<
- string,
- AsyncGenerator<WrappedAudioBuffer, void, unknown>
- >();
- private queuedSources = new Set<AudioBufferSourceNode>();
- private preparedClipBuffers = new Map<string, Promise<AudioBuffer | null>>();
- private decodedBuffers = new Map<string, Promise<AudioBuffer | null>>();
- private playbackSessionId = 0;
- private lastIsPlaying = false;
- private lastVolume = 1;
- private playbackLatencyCompensationSeconds = 0;
- private unsubscribers: Array<() => void> = [];
-
- constructor(private editor: EditorCore) {
- this.lastVolume = this.editor.playback.getVolume();
-
- this.unsubscribers.push(
- this.editor.playback.subscribe(this.handlePlaybackChange),
- this.editor.timeline.subscribe(this.handleTimelineChange),
- this.editor.media.subscribe(this.handleTimelineChange),
- );
- if (typeof window !== "undefined") {
- window.addEventListener("playback-seek", this.handleSeek);
- }
- }
-
- dispose(): void {
- this.stopPlayback();
- for (const unsub of this.unsubscribers) {
- unsub();
- }
- this.unsubscribers = [];
- if (typeof window !== "undefined") {
- window.removeEventListener("playback-seek", this.handleSeek);
- }
- this.disposeSinks();
- this.preparedClipBuffers.clear();
- this.decodedBuffers.clear();
- if (this.audioContext) {
- void this.audioContext.close();
- this.audioContext = null;
- this.masterGain = null;
- }
- }
-
- private handlePlaybackChange = (): void => {
- const isPlaying = this.editor.playback.getIsPlaying();
- const volume = this.editor.playback.getVolume();
-
- if (volume !== this.lastVolume) {
- this.lastVolume = volume;
- this.updateGain();
- }
-
- if (isPlaying !== this.lastIsPlaying) {
- this.lastIsPlaying = isPlaying;
- if (isPlaying) {
- void this.startPlayback({
- time: this.editor.playback.getCurrentTime(),
- });
- } else {
- this.stopPlayback();
- }
- }
- };
-
- private handleSeek = (event: Event): void => {
- const detail = (event as CustomEvent<{ time: number }>).detail;
- if (!detail) return;
-
- if (this.editor.playback.getIsScrubbing()) {
- this.stopPlayback();
- return;
- }
-
- if (this.editor.playback.getIsPlaying()) {
- void this.startPlayback({ time: detail.time });
- return;
- }
-
- this.stopPlayback();
- };
-
- private handleTimelineChange = (): void => {
- this.disposeSinks();
- this.preparedClipBuffers.clear();
- this.decodedBuffers.clear();
-
- if (!this.editor.playback.getIsPlaying()) return;
-
- void this.startPlayback({ time: this.editor.playback.getCurrentTime() });
- };
-
- private ensureAudioContext(): AudioContext | null {
- if (this.audioContext) return this.audioContext;
- if (typeof window === "undefined") return null;
-
- this.audioContext = createAudioContext();
- const { input } = createAudioMasteringChain({
- audioContext: this.audioContext,
- destination: this.audioContext.destination,
- });
- this.masterGain = input;
- this.masterGain.gain.value = this.lastVolume;
- return this.audioContext;
- }
-
- private updateGain(): void {
- if (!this.masterGain) return;
- this.masterGain.gain.value = this.lastVolume;
- }
-
- private getPlaybackTime(): number {
- if (!this.audioContext) return this.playbackStartTime;
- const elapsed =
- this.audioContext.currentTime - this.playbackStartContextTime;
- return this.playbackStartTime + elapsed;
- }
-
- private async startPlayback({ time }: { time: number }): Promise<void> {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) return;
-
- this.stopPlayback();
- this.playbackSessionId++;
- this.playbackLatencyCompensationSeconds = 0;
-
- const tracks = this.editor.timeline.getTracks();
- const mediaAssets = this.editor.media.getAssets();
- const duration = this.editor.timeline.getTotalDuration();
-
- if (duration <= 0) return;
-
- if (audioContext.state === "suspended") {
- await audioContext.resume();
- }
-
- this.clips = await collectAudioClips({ tracks, mediaAssets });
- if (!this.editor.playback.getIsPlaying()) return;
-
- this.playbackStartTime = time;
- this.playbackStartContextTime = audioContext.currentTime;
-
- this.scheduleUpcomingClips();
-
- if (typeof window !== "undefined") {
- this.scheduleTimer = window.setInterval(() => {
- this.scheduleUpcomingClips();
- }, this.scheduleIntervalMs);
- }
- }
-
- private scheduleUpcomingClips(): void {
- if (!this.editor.playback.getIsPlaying()) return;
-
- const currentTime = this.getPlaybackTime();
- const windowEnd = currentTime + this.lookaheadSeconds;
-
- for (const clip of this.clips) {
- if (clip.muted) continue;
- if (this.activeClipIds.has(clip.id)) continue;
-
- const clipEnd = clip.startTime + clip.duration;
- if (clipEnd <= currentTime) continue;
- if (clip.startTime > windowEnd) continue;
-
- this.activeClipIds.add(clip.id);
- if (this.shouldUsePreparedClipBuffer({ clip })) {
- void this.schedulePreparedClip({
- clip,
- startTime: currentTime,
- sessionId: this.playbackSessionId,
- });
- } else {
- void this.runClipIterator({
- clip,
- startTime: currentTime,
- sessionId: this.playbackSessionId,
- });
- }
- }
- }
-
- private stopPlayback(): void {
- if (this.scheduleTimer && typeof window !== "undefined") {
- window.clearInterval(this.scheduleTimer);
- }
- this.scheduleTimer = null;
-
- for (const iterator of this.clipIterators.values()) {
- void iterator.return();
- }
- this.clipIterators.clear();
- this.activeClipIds.clear();
-
- for (const source of this.queuedSources) {
- try {
- source.stop();
- } catch {}
- source.disconnect();
- }
- this.queuedSources.clear();
- }
-
- private async runClipIterator({
- clip,
- startTime,
- sessionId,
- }: {
- clip: AudioClipSource;
- startTime: number;
- sessionId: number;
- }): Promise<void> {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) return;
-
- const sink = await this.getAudioSink({ clip });
- if (!sink || !this.editor.playback.getIsPlaying()) return;
- if (sessionId !== this.playbackSessionId) return;
-
- const clipStart = clip.startTime;
- const clipEnd = clip.startTime + clip.duration;
- const playbackTimeAfterSinkReady = this.getPlaybackTime();
- const iteratorStartTime = Math.max(
- startTime,
- clipStart,
- playbackTimeAfterSinkReady,
- );
- if (iteratorStartTime >= clipEnd) {
- return;
- }
- const sourceStartTime =
- clip.trimStart +
- getSourceTimeAtClipTime({
- clipTime: iteratorStartTime - clip.startTime,
- retime: clip.retime,
- });
-
- const iterator = sink.buffers(sourceStartTime);
- this.clipIterators.set(clip.id, iterator);
- let consecutiveDroppedBufferCount = 0;
-
- for await (const { buffer, timestamp } of iterator) {
- if (!this.editor.playback.getIsPlaying()) return;
- if (sessionId !== this.playbackSessionId) return;
-
- const timelineTime =
- clip.startTime +
- getClipTimeAtSourceTime({
- sourceTime: timestamp - clip.trimStart,
- retime: clip.retime,
- });
- if (timelineTime >= clipEnd) break;
-
- const node = audioContext.createBufferSource();
- node.buffer = buffer;
- if (clip.retime) {
- node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate });
- }
- const clipGain = audioContext.createGain();
- clipGain.gain.value = clip.volume;
- node.connect(clipGain);
- clipGain.connect(this.masterGain ?? audioContext.destination);
-
- const startTimestamp =
- this.playbackStartContextTime +
- this.playbackLatencyCompensationSeconds +
- (timelineTime - this.playbackStartTime);
-
- if (startTimestamp >= audioContext.currentTime) {
- node.start(startTimestamp);
- consecutiveDroppedBufferCount = 0;
- } else {
- const offset = audioContext.currentTime - startTimestamp;
- if (offset < buffer.duration) {
- node.start(audioContext.currentTime, offset);
- consecutiveDroppedBufferCount = 0;
- } else {
- consecutiveDroppedBufferCount += 1;
- if (consecutiveDroppedBufferCount >= 5) {
- const nextCompensationSeconds = Math.max(
- this.playbackLatencyCompensationSeconds,
- Math.min(0.25, offset + 0.01),
- );
- if (
- nextCompensationSeconds >
- this.playbackLatencyCompensationSeconds + 0.001
- ) {
- this.playbackLatencyCompensationSeconds = nextCompensationSeconds;
- }
- const resyncStartTime = this.getPlaybackTime();
- this.clipIterators.delete(clip.id);
- void this.runClipIterator({
- clip,
- startTime: resyncStartTime,
- sessionId,
- });
- return;
- }
- continue;
- }
- }
-
- this.queuedSources.add(node);
- node.addEventListener("ended", () => {
- node.disconnect();
- clipGain.disconnect();
- this.queuedSources.delete(node);
- });
-
- const aheadTime = timelineTime - this.getPlaybackTime();
- if (aheadTime >= 1) {
- await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 });
- if (sessionId !== this.playbackSessionId) return;
- }
- }
-
- this.clipIterators.delete(clip.id);
- // don't remove from activeClipIds - prevents scheduler from restarting this clip
- // the set is cleared on stopPlayback anyway
- }
-
- private async schedulePreparedClip({
- clip,
- startTime,
- sessionId,
- }: {
- clip: AudioClipSource;
- startTime: number;
- sessionId: number;
- }): Promise<void> {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) return;
-
- const buffer = await this.getPreparedClipBuffer({ clip });
- if (!buffer || !this.editor.playback.getIsPlaying()) return;
- if (sessionId !== this.playbackSessionId) return;
-
- const clipStart = clip.startTime;
- const clipEnd = clip.startTime + clip.duration;
- const playbackTimeAfterReady = this.getPlaybackTime();
- const effectiveStartTime = Math.max(
- startTime,
- clipStart,
- playbackTimeAfterReady,
- );
- if (effectiveStartTime >= clipEnd) {
- return;
- }
-
- const node = audioContext.createBufferSource();
- node.buffer = buffer;
- const clipGain = audioContext.createGain();
- node.connect(clipGain);
- clipGain.connect(this.masterGain ?? audioContext.destination);
-
- const startTimestamp =
- this.playbackStartContextTime +
- this.playbackLatencyCompensationSeconds +
- (effectiveStartTime - this.playbackStartTime);
- const clipOffset = effectiveStartTime - clipStart;
- let actualStartTimestamp = startTimestamp;
- let actualClipOffset = clipOffset;
-
- if (startTimestamp >= audioContext.currentTime) {
- node.start(startTimestamp, clipOffset);
- } else {
- const lateOffset = audioContext.currentTime - startTimestamp;
- actualStartTimestamp = audioContext.currentTime;
- actualClipOffset = clipOffset + lateOffset;
- node.start(actualStartTimestamp, actualClipOffset);
- }
-
- this.scheduleClipGainAutomation({
- audioContext,
- clip,
- clipGain,
- startTimestamp: actualStartTimestamp,
- startLocalTime: actualClipOffset,
- });
-
- this.queuedSources.add(node);
- node.addEventListener("ended", () => {
- node.disconnect();
- clipGain.disconnect();
- this.queuedSources.delete(node);
- });
- }
-
- private waitUntilCaughtUp({
- timelineTime,
- targetAhead,
- }: {
- timelineTime: number;
- targetAhead: number;
- }): Promise<void> {
- return new Promise((resolve) => {
- const checkInterval = setInterval(() => {
- if (!this.editor.playback.getIsPlaying()) {
- clearInterval(checkInterval);
- resolve();
- return;
- }
-
- const playbackTime = this.getPlaybackTime();
- if (timelineTime - playbackTime < targetAhead) {
- clearInterval(checkInterval);
- resolve();
- }
- }, 100);
- });
- }
-
- private disposeSinks(): void {
- for (const iterator of this.clipIterators.values()) {
- void iterator.return();
- }
- this.clipIterators.clear();
- this.activeClipIds.clear();
-
- for (const input of this.inputs.values()) {
- input.dispose();
- }
- this.inputs.clear();
- this.sinks.clear();
- }
-
- private shouldUsePreparedClipBuffer({
- clip,
- }: {
- clip: AudioClipSource;
- }): boolean {
- return (
- this.hasCurveRetime({ clip }) ||
- hasAnimatedVolume({ element: clip.timelineElement }) ||
- shouldMaintainPitch({
- rate: clip.retime?.rate ?? 1,
- maintainPitch: clip.retime?.maintainPitch,
- })
- );
- }
-
- private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean {
- const mode = (clip.retime as { mode?: unknown } | undefined)?.mode;
- return mode === "curve";
- }
-
- private scheduleClipGainAutomation({
- audioContext,
- clip,
- clipGain,
- startTimestamp,
- startLocalTime,
- }: {
- audioContext: AudioContext;
- clip: AudioClipSource;
- clipGain: GainNode;
- startTimestamp: number;
- startLocalTime: number;
- }): void {
- clipGain.gain.cancelScheduledValues(startTimestamp);
- clipGain.gain.setValueAtTime(clip.volume, startTimestamp);
-
- if (!hasAnimatedVolume({ element: clip.timelineElement })) {
- return;
- }
-
- const points = buildAudioGainAutomation({
- element: clip.timelineElement,
- fromLocalTime: startLocalTime,
- toLocalTime: clip.duration,
- });
-
- if (points.length === 0) {
- return;
- }
-
- clipGain.gain.setValueAtTime(points[0].gain, startTimestamp);
- for (let index = 1; index < points.length; index++) {
- const point = points[index];
- const pointTimestamp =
- startTimestamp + (point.localTime - startLocalTime);
- if (pointTimestamp < audioContext.currentTime) {
- continue;
- }
-
- clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp);
- }
- }
-
- private buildPreparedClipCacheKey({
- clip,
- }: {
- clip: AudioClipSource;
- }): string {
- return JSON.stringify({
- id: clip.id,
- sourceKey: clip.sourceKey,
- startTime: clip.startTime,
- duration: clip.duration,
- trimStart: clip.trimStart,
- trimEnd: clip.trimEnd,
- retime: clip.retime ?? null,
- });
- }
-
- private async getPreparedClipBuffer({
- clip,
- }: {
- clip: AudioClipSource;
- }): Promise<AudioBuffer | null> {
- const cacheKey = this.buildPreparedClipCacheKey({ clip });
- const existing = this.preparedClipBuffers.get(cacheKey);
- if (existing) {
- return existing;
- }
-
- const promise = (async () => {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) {
- return null;
- }
-
- const decodedBuffer = await this.getDecodedBuffer({ clip });
- if (!decodedBuffer) {
- return null;
- }
-
- return await renderRetimedBuffer({
- audioContext,
- sourceBuffer: decodedBuffer,
- trimStart: clip.trimStart,
- clipDuration: clip.duration,
- retime: clip.retime,
- maintainPitch: clip.retime?.maintainPitch === true,
- });
- })();
-
- this.preparedClipBuffers.set(cacheKey, promise);
- return promise;
- }
-
- private async getDecodedBuffer({
- clip,
- }: {
- clip: AudioClipSource;
- }): Promise<AudioBuffer | null> {
- const existing = this.decodedBuffers.get(clip.sourceKey);
- if (existing) {
- return existing;
- }
-
- const promise = this.decodeClipBuffer({ clip });
- this.decodedBuffers.set(clip.sourceKey, promise);
- return promise;
- }
-
- private async decodeClipBuffer({
- clip,
- }: {
- clip: AudioClipSource;
- }): Promise<AudioBuffer | null> {
- const audioContext = this.ensureAudioContext();
- if (!audioContext) {
- return null;
- }
-
- const input = new Input({
- source: new BlobSource(clip.file),
- formats: ALL_FORMATS,
- });
-
- try {
- const audioTrack = await input.getPrimaryAudioTrack();
- if (!audioTrack) {
- return null;
- }
-
- const sink = new AudioBufferSink(audioTrack);
- const chunks: AudioBuffer[] = [];
- let totalSamples = 0;
-
- for await (const { buffer } of sink.buffers(0)) {
- chunks.push(buffer);
- totalSamples += buffer.length;
- }
-
- if (chunks.length === 0) {
- return null;
- }
-
- const targetSampleRate = audioContext.sampleRate;
- const nativeSampleRate = chunks[0].sampleRate;
- const numChannels = Math.min(2, chunks[0].numberOfChannels);
- const nativeChannels = Array.from(
- { length: numChannels },
- () => new Float32Array(totalSamples),
- );
-
- let offset = 0;
- for (const chunk of chunks) {
- for (let channel = 0; channel < numChannels; channel++) {
- nativeChannels[channel].set(
- chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)),
- offset,
- );
- }
- offset += chunk.length;
- }
-
- const outputSamples = Math.ceil(
- totalSamples * (targetSampleRate / nativeSampleRate),
- );
- const offlineContext = new OfflineAudioContext(
- numChannels,
- outputSamples,
- targetSampleRate,
- );
- const nativeBuffer = audioContext.createBuffer(
- numChannels,
- totalSamples,
- nativeSampleRate,
- );
-
- for (let channel = 0; channel < numChannels; channel++) {
- nativeBuffer.copyToChannel(nativeChannels[channel], channel);
- }
-
- const sourceNode = offlineContext.createBufferSource();
- sourceNode.buffer = nativeBuffer;
- sourceNode.connect(offlineContext.destination);
- sourceNode.start(0);
-
- return await offlineContext.startRendering();
- } catch (error) {
- console.warn("Failed to decode clip audio:", error);
- return null;
- } finally {
- input.dispose();
- }
- }
-
- private async getAudioSink({
- clip,
- }: {
- clip: AudioClipSource;
- }): Promise<AudioBufferSink | null> {
- const existingSink = this.sinks.get(clip.sourceKey);
- if (existingSink) return existingSink;
-
- try {
- const input = new Input({
- source: new BlobSource(clip.file),
- formats: ALL_FORMATS,
- });
- const audioTrack = await input.getPrimaryAudioTrack();
- if (!audioTrack) {
- input.dispose();
- return null;
- }
-
- const sink = new AudioBufferSink(audioTrack);
- this.inputs.set(clip.sourceKey, input);
- this.sinks.set(clip.sourceKey, sink);
- return sink;
- } catch (error) {
- console.warn("Failed to initialize audio sink:", error);
- return null;
- }
- }
-}
+import type { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { + clampRetimeRate, + shouldMaintainPitch, +} from "@/lib/retime/rate"; +import type { AudioClipSource } from "@/lib/media/audio"; +import { createAudioContext, collectAudioClips } from "@/lib/media/audio"; +import { + buildAudioGainAutomation, + hasAnimatedVolume, +} from "@/lib/timeline/audio-state"; +import { createAudioMasteringChain } from "@/lib/media/audio-mastering"; +import { + getClipTimeAtSourceTime, + getSourceTimeAtClipTime, + renderRetimedBuffer, +} from "@/lib/retime"; +import { + ALL_FORMATS, + AudioBufferSink, + BlobSource, + Input, + type WrappedAudioBuffer, +} from "mediabunny"; + +export class AudioManager { + private audioContext: AudioContext | null = null; + private masterGain: GainNode | null = null; + private playbackStartTime = 0; + private playbackStartContextTime = 0; + private scheduleTimer: number | null = null; + private lookaheadSeconds = 2; + private scheduleIntervalMs = 500; + private clips: AudioClipSource[] = []; + private sinks = new Map<string, AudioBufferSink>(); + private inputs = new Map<string, Input>(); + private activeClipIds = new Set<string>(); + private clipIterators = new Map< + string, + AsyncGenerator<WrappedAudioBuffer, void, unknown> + >(); + private queuedSources = new Set<AudioBufferSourceNode>(); + private preparedClipBuffers = new Map<string, Promise<AudioBuffer | null>>(); + private decodedBuffers = new Map<string, Promise<AudioBuffer | null>>(); + private playbackSessionId = 0; + private lastIsPlaying = false; + private lastVolume = 1; + private playbackLatencyCompensationSeconds = 0; + private unsubscribers: Array<() => void> = []; + + constructor(private editor: EditorCore) { + this.lastVolume = this.editor.playback.getVolume(); + + this.unsubscribers.push( + this.editor.playback.subscribe(this.handlePlaybackChange), + this.editor.timeline.subscribe(this.handleTimelineChange), + this.editor.media.subscribe(this.handleTimelineChange), + ); + if (typeof window !== "undefined") { + window.addEventListener("playback-seek", this.handleSeek); + } + } + + dispose(): void { + this.stopPlayback(); + for (const unsub of this.unsubscribers) { + unsub(); + } + this.unsubscribers = []; + if (typeof window !== "undefined") { + window.removeEventListener("playback-seek", this.handleSeek); + } + this.disposeSinks(); + this.preparedClipBuffers.clear(); + this.decodedBuffers.clear(); + if (this.audioContext) { + void this.audioContext.close(); + this.audioContext = null; + this.masterGain = null; + } + } + + private handlePlaybackChange = (): void => { + const isPlaying = this.editor.playback.getIsPlaying(); + const volume = this.editor.playback.getVolume(); + + if (volume !== this.lastVolume) { + this.lastVolume = volume; + this.updateGain(); + } + + if (isPlaying !== this.lastIsPlaying) { + this.lastIsPlaying = isPlaying; + if (isPlaying) { + void this.startPlayback({ + time: this.editor.playback.getCurrentTime() / TICKS_PER_SECOND, + }); + } else { + this.stopPlayback(); + } + } + }; + + private handleSeek = (event: Event): void => { + const detail = (event as CustomEvent<{ time: number }>).detail; + if (!detail) return; + + if (this.editor.playback.getIsScrubbing()) { + this.stopPlayback(); + return; + } + + if (this.editor.playback.getIsPlaying()) { + void this.startPlayback({ time: detail.time / TICKS_PER_SECOND }); + return; + } + + this.stopPlayback(); + }; + + private handleTimelineChange = (): void => { + this.disposeSinks(); + this.preparedClipBuffers.clear(); + this.decodedBuffers.clear(); + + if (!this.editor.playback.getIsPlaying()) return; + + void this.startPlayback({ time: this.editor.playback.getCurrentTime() / TICKS_PER_SECOND }); + }; + + private ensureAudioContext(): AudioContext | null { + if (this.audioContext) return this.audioContext; + if (typeof window === "undefined") return null; + + this.audioContext = createAudioContext(); + const { input } = createAudioMasteringChain({ + audioContext: this.audioContext, + destination: this.audioContext.destination, + }); + this.masterGain = input; + this.masterGain.gain.value = this.lastVolume; + return this.audioContext; + } + + private updateGain(): void { + if (!this.masterGain) return; + this.masterGain.gain.value = this.lastVolume; + } + + private getPlaybackTime(): number { + if (!this.audioContext) return this.playbackStartTime; + const elapsed = + this.audioContext.currentTime - this.playbackStartContextTime; + return this.playbackStartTime + elapsed; + } + + private async startPlayback({ time }: { time: number }): Promise<void> { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + this.stopPlayback(); + this.playbackSessionId++; + this.playbackLatencyCompensationSeconds = 0; + + const tracks = this.editor.timeline.getTracks(); + const mediaAssets = this.editor.media.getAssets(); + const duration = this.editor.timeline.getTotalDuration(); + + if (duration <= 0) return; + + if (audioContext.state === "suspended") { + await audioContext.resume(); + } + + this.clips = await collectAudioClips({ tracks, mediaAssets }); + if (!this.editor.playback.getIsPlaying()) return; + + this.playbackStartTime = time; + this.playbackStartContextTime = audioContext.currentTime; + + this.scheduleUpcomingClips(); + + if (typeof window !== "undefined") { + this.scheduleTimer = window.setInterval(() => { + this.scheduleUpcomingClips(); + }, this.scheduleIntervalMs); + } + } + + private scheduleUpcomingClips(): void { + if (!this.editor.playback.getIsPlaying()) return; + + const currentTime = this.getPlaybackTime(); + const windowEnd = currentTime + this.lookaheadSeconds; + + for (const clip of this.clips) { + if (clip.muted) continue; + if (this.activeClipIds.has(clip.id)) continue; + + const clipEnd = clip.startTime + clip.duration; + if (clipEnd <= currentTime) continue; + if (clip.startTime > windowEnd) continue; + + this.activeClipIds.add(clip.id); + if (this.shouldUsePreparedClipBuffer({ clip })) { + void this.schedulePreparedClip({ + clip, + startTime: currentTime, + sessionId: this.playbackSessionId, + }); + } else { + void this.runClipIterator({ + clip, + startTime: currentTime, + sessionId: this.playbackSessionId, + }); + } + } + } + + private stopPlayback(): void { + if (this.scheduleTimer && typeof window !== "undefined") { + window.clearInterval(this.scheduleTimer); + } + this.scheduleTimer = null; + + for (const iterator of this.clipIterators.values()) { + void iterator.return(); + } + this.clipIterators.clear(); + this.activeClipIds.clear(); + + for (const source of this.queuedSources) { + try { + source.stop(); + } catch {} + source.disconnect(); + } + this.queuedSources.clear(); + } + + private async runClipIterator({ + clip, + startTime, + sessionId, + }: { + clip: AudioClipSource; + startTime: number; + sessionId: number; + }): Promise<void> { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + const sink = await this.getAudioSink({ clip }); + if (!sink || !this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const clipStart = clip.startTime; + const clipEnd = clip.startTime + clip.duration; + const playbackTimeAfterSinkReady = this.getPlaybackTime(); + const iteratorStartTime = Math.max( + startTime, + clipStart, + playbackTimeAfterSinkReady, + ); + if (iteratorStartTime >= clipEnd) { + return; + } + const sourceStartTime = + clip.trimStart + + getSourceTimeAtClipTime({ + clipTime: iteratorStartTime - clip.startTime, + retime: clip.retime, + }); + + const iterator = sink.buffers(sourceStartTime); + this.clipIterators.set(clip.id, iterator); + let consecutiveDroppedBufferCount = 0; + + for await (const { buffer, timestamp } of iterator) { + if (!this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const timelineTime = + clip.startTime + + getClipTimeAtSourceTime({ + sourceTime: timestamp - clip.trimStart, + retime: clip.retime, + }); + if (timelineTime >= clipEnd) break; + + const node = audioContext.createBufferSource(); + node.buffer = buffer; + if (clip.retime) { + node.playbackRate.value = clampRetimeRate({ rate: clip.retime.rate }); + } + const clipGain = audioContext.createGain(); + clipGain.gain.value = clip.volume; + node.connect(clipGain); + clipGain.connect(this.masterGain ?? audioContext.destination); + + const startTimestamp = + this.playbackStartContextTime + + this.playbackLatencyCompensationSeconds + + (timelineTime - this.playbackStartTime); + + if (startTimestamp >= audioContext.currentTime) { + node.start(startTimestamp); + consecutiveDroppedBufferCount = 0; + } else { + const offset = audioContext.currentTime - startTimestamp; + if (offset < buffer.duration) { + node.start(audioContext.currentTime, offset); + consecutiveDroppedBufferCount = 0; + } else { + consecutiveDroppedBufferCount += 1; + if (consecutiveDroppedBufferCount >= 5) { + const nextCompensationSeconds = Math.max( + this.playbackLatencyCompensationSeconds, + Math.min(0.25, offset + 0.01), + ); + if ( + nextCompensationSeconds > + this.playbackLatencyCompensationSeconds + 0.001 + ) { + this.playbackLatencyCompensationSeconds = nextCompensationSeconds; + } + const resyncStartTime = this.getPlaybackTime(); + this.clipIterators.delete(clip.id); + void this.runClipIterator({ + clip, + startTime: resyncStartTime, + sessionId, + }); + return; + } + continue; + } + } + + this.queuedSources.add(node); + node.addEventListener("ended", () => { + node.disconnect(); + clipGain.disconnect(); + this.queuedSources.delete(node); + }); + + const aheadTime = timelineTime - this.getPlaybackTime(); + if (aheadTime >= 1) { + await this.waitUntilCaughtUp({ timelineTime, targetAhead: 1 }); + if (sessionId !== this.playbackSessionId) return; + } + } + + this.clipIterators.delete(clip.id); + // don't remove from activeClipIds - prevents scheduler from restarting this clip + // the set is cleared on stopPlayback anyway + } + + private async schedulePreparedClip({ + clip, + startTime, + sessionId, + }: { + clip: AudioClipSource; + startTime: number; + sessionId: number; + }): Promise<void> { + const audioContext = this.ensureAudioContext(); + if (!audioContext) return; + + const buffer = await this.getPreparedClipBuffer({ clip }); + if (!buffer || !this.editor.playback.getIsPlaying()) return; + if (sessionId !== this.playbackSessionId) return; + + const clipStart = clip.startTime; + const clipEnd = clip.startTime + clip.duration; + const playbackTimeAfterReady = this.getPlaybackTime(); + const effectiveStartTime = Math.max( + startTime, + clipStart, + playbackTimeAfterReady, + ); + if (effectiveStartTime >= clipEnd) { + return; + } + + const node = audioContext.createBufferSource(); + node.buffer = buffer; + const clipGain = audioContext.createGain(); + node.connect(clipGain); + clipGain.connect(this.masterGain ?? audioContext.destination); + + const startTimestamp = + this.playbackStartContextTime + + this.playbackLatencyCompensationSeconds + + (effectiveStartTime - this.playbackStartTime); + const clipOffset = effectiveStartTime - clipStart; + let actualStartTimestamp = startTimestamp; + let actualClipOffset = clipOffset; + + if (startTimestamp >= audioContext.currentTime) { + node.start(startTimestamp, clipOffset); + } else { + const lateOffset = audioContext.currentTime - startTimestamp; + actualStartTimestamp = audioContext.currentTime; + actualClipOffset = clipOffset + lateOffset; + node.start(actualStartTimestamp, actualClipOffset); + } + + this.scheduleClipGainAutomation({ + audioContext, + clip, + clipGain, + startTimestamp: actualStartTimestamp, + startLocalTime: actualClipOffset, + }); + + this.queuedSources.add(node); + node.addEventListener("ended", () => { + node.disconnect(); + clipGain.disconnect(); + this.queuedSources.delete(node); + }); + } + + private waitUntilCaughtUp({ + timelineTime, + targetAhead, + }: { + timelineTime: number; + targetAhead: number; + }): Promise<void> { + return new Promise((resolve) => { + const checkInterval = setInterval(() => { + if (!this.editor.playback.getIsPlaying()) { + clearInterval(checkInterval); + resolve(); + return; + } + + const playbackTime = this.getPlaybackTime(); + if (timelineTime - playbackTime < targetAhead) { + clearInterval(checkInterval); + resolve(); + } + }, 100); + }); + } + + private disposeSinks(): void { + for (const iterator of this.clipIterators.values()) { + void iterator.return(); + } + this.clipIterators.clear(); + this.activeClipIds.clear(); + + for (const input of this.inputs.values()) { + input.dispose(); + } + this.inputs.clear(); + this.sinks.clear(); + } + + private shouldUsePreparedClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): boolean { + return ( + this.hasCurveRetime({ clip }) || + hasAnimatedVolume({ element: clip.timelineElement }) || + shouldMaintainPitch({ + rate: clip.retime?.rate ?? 1, + maintainPitch: clip.retime?.maintainPitch, + }) + ); + } + + private hasCurveRetime({ clip }: { clip: AudioClipSource }): boolean { + const mode = (clip.retime as { mode?: unknown } | undefined)?.mode; + return mode === "curve"; + } + + private scheduleClipGainAutomation({ + audioContext, + clip, + clipGain, + startTimestamp, + startLocalTime, + }: { + audioContext: AudioContext; + clip: AudioClipSource; + clipGain: GainNode; + startTimestamp: number; + startLocalTime: number; + }): void { + clipGain.gain.cancelScheduledValues(startTimestamp); + clipGain.gain.setValueAtTime(clip.volume, startTimestamp); + + if (!hasAnimatedVolume({ element: clip.timelineElement })) { + return; + } + + const points = buildAudioGainAutomation({ + element: clip.timelineElement, + fromLocalTime: startLocalTime, + toLocalTime: clip.duration, + }); + + if (points.length === 0) { + return; + } + + clipGain.gain.setValueAtTime(points[0].gain, startTimestamp); + for (let index = 1; index < points.length; index++) { + const point = points[index]; + const pointTimestamp = + startTimestamp + (point.localTime - startLocalTime); + if (pointTimestamp < audioContext.currentTime) { + continue; + } + + clipGain.gain.linearRampToValueAtTime(point.gain, pointTimestamp); + } + } + + private buildPreparedClipCacheKey({ + clip, + }: { + clip: AudioClipSource; + }): string { + return JSON.stringify({ + id: clip.id, + sourceKey: clip.sourceKey, + startTime: clip.startTime, + duration: clip.duration, + trimStart: clip.trimStart, + trimEnd: clip.trimEnd, + retime: clip.retime ?? null, + }); + } + + private async getPreparedClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise<AudioBuffer | null> { + const cacheKey = this.buildPreparedClipCacheKey({ clip }); + const existing = this.preparedClipBuffers.get(cacheKey); + if (existing) { + return existing; + } + + const promise = (async () => { + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + + const decodedBuffer = await this.getDecodedBuffer({ clip }); + if (!decodedBuffer) { + return null; + } + + return await renderRetimedBuffer({ + audioContext, + sourceBuffer: decodedBuffer, + trimStart: clip.trimStart, + clipDuration: clip.duration, + retime: clip.retime, + maintainPitch: clip.retime?.maintainPitch === true, + }); + })(); + + this.preparedClipBuffers.set(cacheKey, promise); + return promise; + } + + private async getDecodedBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise<AudioBuffer | null> { + const existing = this.decodedBuffers.get(clip.sourceKey); + if (existing) { + return existing; + } + + const promise = this.decodeClipBuffer({ clip }); + this.decodedBuffers.set(clip.sourceKey, promise); + return promise; + } + + private async decodeClipBuffer({ + clip, + }: { + clip: AudioClipSource; + }): Promise<AudioBuffer | null> { + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + + const input = new Input({ + source: new BlobSource(clip.file), + formats: ALL_FORMATS, + }); + + try { + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + return null; + } + + const sink = new AudioBufferSink(audioTrack); + const chunks: AudioBuffer[] = []; + let totalSamples = 0; + + for await (const { buffer } of sink.buffers(0)) { + chunks.push(buffer); + totalSamples += buffer.length; + } + + if (chunks.length === 0) { + return null; + } + + const targetSampleRate = audioContext.sampleRate; + const nativeSampleRate = chunks[0].sampleRate; + const numChannels = Math.min(2, chunks[0].numberOfChannels); + const nativeChannels = Array.from( + { length: numChannels }, + () => new Float32Array(totalSamples), + ); + + let offset = 0; + for (const chunk of chunks) { + for (let channel = 0; channel < numChannels; channel++) { + nativeChannels[channel].set( + chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1)), + offset, + ); + } + offset += chunk.length; + } + + const outputSamples = Math.ceil( + totalSamples * (targetSampleRate / nativeSampleRate), + ); + const offlineContext = new OfflineAudioContext( + numChannels, + outputSamples, + targetSampleRate, + ); + const nativeBuffer = audioContext.createBuffer( + numChannels, + totalSamples, + nativeSampleRate, + ); + + for (let channel = 0; channel < numChannels; channel++) { + nativeBuffer.copyToChannel(nativeChannels[channel], channel); + } + + const sourceNode = offlineContext.createBufferSource(); + sourceNode.buffer = nativeBuffer; + sourceNode.connect(offlineContext.destination); + sourceNode.start(0); + + return await offlineContext.startRendering(); + } catch (error) { + console.warn("Failed to decode clip audio:", error); + return null; + } finally { + input.dispose(); + } + } + + private async getAudioSink({ + clip, + }: { + clip: AudioClipSource; + }): Promise<AudioBufferSink | null> { + const existingSink = this.sinks.get(clip.sourceKey); + if (existingSink) return existingSink; + + try { + const input = new Input({ + source: new BlobSource(clip.file), + formats: ALL_FORMATS, + }); + const audioTrack = await input.getPrimaryAudioTrack(); + if (!audioTrack) { + input.dispose(); + return null; + } + + const sink = new AudioBufferSink(audioTrack); + this.inputs.set(clip.sourceKey, input); + this.sinks.set(clip.sourceKey, sink); + return sink; + } catch (error) { + console.warn("Failed to initialize audio sink:", error); + return null; + } + } +} diff --git a/apps/web/src/core/managers/playback-manager.ts b/apps/web/src/core/managers/playback-manager.ts index 6943f187..47909bfe 100644 --- a/apps/web/src/core/managers/playback-manager.ts +++ b/apps/web/src/core/managers/playback-manager.ts @@ -1,4 +1,6 @@ import type { EditorCore } from "@/core"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { roundToFrame } from "opencut-wasm"; export class PlaybackManager { private isPlaying = false; @@ -9,11 +11,12 @@ export class PlaybackManager { private isScrubbing = false; private listeners = new Set<() => void>(); private playbackTimer: number | null = null; - private lastUpdate = 0; + private playbackStartWallTime = 0; + private playbackStartTime = 0; constructor(private editor: EditorCore) { this.editor.timeline.subscribe(() => { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); if (this.currentTime > maxTime && maxTime > 0) { this.currentTime = maxTime; this.notify(); @@ -22,7 +25,7 @@ export class PlaybackManager { } play(): void { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); if (maxTime > 0) { if (this.currentTime >= maxTime) { @@ -50,8 +53,12 @@ export class PlaybackManager { } seek({ time }: { time: number }): void { - const maxTime = this.editor.timeline.getLastSeekableTime(); + const maxTime = this.editor.timeline.getTotalDuration(); this.currentTime = Math.max(0, Math.min(maxTime, time)); + if (this.isPlaying) { + this.playbackStartWallTime = performance.now(); + this.playbackStartTime = this.currentTime; + } this.notify(); window.dispatchEvent( @@ -135,7 +142,8 @@ export class PlaybackManager { cancelAnimationFrame(this.playbackTimer); } - this.lastUpdate = performance.now(); + this.playbackStartWallTime = performance.now(); + this.playbackStartTime = this.currentTime; this.updateTime(); } @@ -149,12 +157,11 @@ export class PlaybackManager { private updateTime = (): void => { if (!this.isPlaying) return; - const now = performance.now(); - const delta = (now - this.lastUpdate) / 1000; - this.lastUpdate = now; - - const newTime = this.currentTime + delta; - const maxTime = this.editor.timeline.getLastSeekableTime(); + const fps = this.editor.project.getActive()?.settings.fps; + const elapsedSeconds = (performance.now() - this.playbackStartWallTime) / 1000; + const rawTime = this.playbackStartTime + Math.round(elapsedSeconds * TICKS_PER_SECOND); + const newTime = fps ? (roundToFrame({ time: rawTime, rate: fps }) ?? rawTime) : rawTime; + const maxTime = this.editor.timeline.getTotalDuration(); if (maxTime > 0 && newTime >= maxTime) { this.pause(); diff --git a/apps/web/src/core/managers/project-manager.ts b/apps/web/src/core/managers/project-manager.ts index 10a9229d..513eab94 100644 --- a/apps/web/src/core/managers/project-manager.ts +++ b/apps/web/src/core/managers/project-manager.ts @@ -496,7 +496,7 @@ export class ProjectManager { importedAssets, }: { importedAssets: Array<Pick<MediaAsset, "type" | "fps">>; - }): number | null { + }): import("opencut-wasm").FrameRate | null { if (!this.active) return null; const nextFps = getRaisedProjectFpsForImportedMedia({ diff --git a/apps/web/src/core/managers/renderer-manager.ts b/apps/web/src/core/managers/renderer-manager.ts index aa34f496..e0d5b895 100644 --- a/apps/web/src/core/managers/renderer-manager.ts +++ b/apps/web/src/core/managers/renderer-manager.ts @@ -5,7 +5,8 @@ import { CanvasRenderer } from "@/services/renderer/canvas-renderer"; import { SceneExporter } from "@/services/renderer/scene-exporter"; import { buildScene } from "@/services/renderer/scene-builder"; import { createTimelineAudioBuffer } from "@/lib/media/audio"; -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode } from "opencut-wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; import { downloadBlob } from "@/utils/browser"; type SnapshotResult = @@ -81,7 +82,10 @@ export class RendererManager { } const { canvasSize, fps } = activeProject.settings; - const renderTime = this.editor.playback.getCurrentTime(); + const renderTime = Math.min( + this.editor.playback.getCurrentTime(), + this.editor.timeline.getLastFrameTime(), + ); const renderer = new CanvasRenderer({ width: canvasSize.width, @@ -107,7 +111,7 @@ export class RendererManager { return { success: false, error: "Failed to create image" }; } - const timecode = formatTimeCode({ timeInSeconds: renderTime, fps })!.replace(/:/g, "-"); + const timecode = formatTimecode({ time: renderTime, rate: fps })!.replace(/:/g, "-"); const safeName = activeProject.metadata.name.replace(/[<>:"/\\|?*]/g, "-").trim() || "snapshot"; @@ -148,7 +152,7 @@ export class RendererManager { return { success: false, error: "Project is empty" }; } - const exportFps = fps || activeProject.settings.fps; + const exportFps = fps ?? activeProject.settings.fps; const canvasSize = activeProject.settings.canvasSize; let audioBuffer: AudioBuffer | null = null; diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 3abde47b..a5cb428e 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -18,7 +18,7 @@ import type { AnimationValue, ScalarCurveKeyframePatch, } from "@/lib/animation/types"; -import { getLastFrameTime } from "opencut-wasm"; +import { lastFrameTime } from "opencut-wasm"; import { BatchCommand } from "@/lib/commands"; import { AddTrackCommand, @@ -196,11 +196,11 @@ export class TimelineManager { return calculateTotalDuration({ tracks: this.getTracks() }); } - getLastSeekableTime(): number { + getLastFrameTime(): number { const duration = this.getTotalDuration(); const fps = this.editor.project.getActive()?.settings.fps; if (!fps || duration <= 0) return duration; - return getLastFrameTime({ duration, fps }); + return lastFrameTime({ duration, rate: fps }) ?? duration; } getTrackById({ trackId }: { trackId: string }): TimelineTrack | null { diff --git a/apps/web/src/hooks/actions/use-editor-actions.ts b/apps/web/src/hooks/actions/use-editor-actions.ts index 76f705e9..599b4260 100644 --- a/apps/web/src/hooks/actions/use-editor-actions.ts +++ b/apps/web/src/hooks/actions/use-editor-actions.ts @@ -5,6 +5,7 @@ import { useTimelineStore } from "@/stores/timeline-store"; import { useActionHandler } from "@/hooks/actions/use-action-handler"; import { useEditor } from "../use-editor"; import { useElementSelection } from "../timeline/element/use-element-selection"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { useKeyframeSelection } from "../timeline/element/use-keyframe-selection"; import { getElementsAtTime, @@ -110,10 +111,11 @@ export function useEditorActions() { "frame-step-forward", () => { const fps = editor.project.getActive().settings.fps; + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); editor.playback.seek({ time: Math.min( editor.timeline.getTotalDuration(), - editor.playback.getCurrentTime() + 1 / fps, + editor.playback.getCurrentTime() + ticksPerFrame, ), }); }, @@ -124,8 +126,9 @@ export function useEditorActions() { "frame-step-backward", () => { const fps = editor.project.getActive().settings.fps; + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); editor.playback.seek({ - time: Math.max(0, editor.playback.getCurrentTime() - 1 / fps), + time: Math.max(0, editor.playback.getCurrentTime() - ticksPerFrame), }); }, undefined, diff --git a/apps/web/src/hooks/timeline/element/use-element-interaction.ts b/apps/web/src/hooks/timeline/element/use-element-interaction.ts index 71ae4704..6fcd4168 100644 --- a/apps/web/src/hooks/timeline/element/use-element-interaction.ts +++ b/apps/web/src/hooks/timeline/element/use-element-interaction.ts @@ -10,8 +10,9 @@ import { useEditor } from "@/hooks/use-editor"; import { useShiftKey } from "@/hooks/use-shift-key"; import { useElementSelection } from "@/hooks/timeline/element/use-element-selection"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { TIMELINE_DRAG_THRESHOLD_PX } from "@/components/editor/panels/timeline/interaction"; -import { snapTimeToFrame } from "opencut-wasm"; +import { roundToFrame } from "opencut-wasm"; import { computeDropTarget } from "@/components/editor/panels/timeline/drop-target"; import { getMouseTimeFromClientX } from "@/lib/timeline/drag-utils"; import { generateUUID } from "@/utils/id"; @@ -67,7 +68,8 @@ function getClickOffsetTime({ zoomLevel: number; }): number { const clickOffsetX = clientX - elementRect.left; - return clickOffsetX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel); + const seconds = clickOffsetX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel); + return Math.round(seconds * TICKS_PER_SECOND); } function getVerticalDragDirection({ @@ -297,15 +299,17 @@ export function useElementInteraction({ zoomLevel, scrollLeft, }); - const adjustedTime = - mouseTime - pendingDragRef.current.clickOffsetTime; - const snappedTime = snapTimeToFrame({ - time: adjustedTime, - fps: activeProject.settings.fps, - }); - startDrag({ - ...pendingDragRef.current, - initialCurrentTime: snappedTime, + const adjustedTime = Math.max( + 0, + mouseTime - pendingDragRef.current.clickOffsetTime, + ); + const snappedTime = roundToFrame({ + time: adjustedTime, + rate: activeProject.settings.fps, + }) ?? adjustedTime; + startDrag({ + ...pendingDragRef.current, + initialCurrentTime: snappedTime, initialCurrentMouseY: clientY, }); startedDragThisEvent = true; @@ -343,9 +347,9 @@ export function useElementInteraction({ zoomLevel, scrollLeft, }); - const adjustedTime = mouseTime - dragState.clickOffsetTime; + const adjustedTime = Math.max(0, mouseTime - dragState.clickOffsetTime); const fps = activeProject.settings.fps; - const frameSnappedTime = snapTimeToFrame({ time: adjustedTime, fps }); + const frameSnappedTime = roundToFrame({ time: adjustedTime, rate: fps }) ?? adjustedTime; const sourceTrack = tracks.find(({ id }) => id === dragState.trackId); const movingElement = sourceTrack?.elements.find( diff --git a/apps/web/src/hooks/timeline/element/use-element-resize.ts b/apps/web/src/hooks/timeline/element/use-element-resize.ts index 3dd98bea..dec87eef 100644 --- a/apps/web/src/hooks/timeline/element/use-element-resize.ts +++ b/apps/web/src/hooks/timeline/element/use-element-resize.ts @@ -1,6 +1,7 @@ import { useState, useEffect, useRef, useCallback } from "react"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; -import { snapTimeToFrame } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { roundToFrame } from "opencut-wasm"; import type { TimelineElement, TimelineTrack } from "@/lib/timeline"; import { useEditor } from "@/hooks/use-editor"; import { useShiftKey } from "@/hooks/use-shift-key"; @@ -185,13 +186,14 @@ export function useTimelineElementResize({ ({ clientX }: { clientX: number }) => { if (!resizing) return; - const deltaX = clientX - resizing.startX; - let deltaTime = - deltaX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel); - let resizeSnapPoint: SnapPoint | null = null; + const deltaX = clientX - resizing.startX; + let deltaTime = Math.round( + (deltaX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel)) * TICKS_PER_SECOND, + ); + let resizeSnapPoint: SnapPoint | null = null; - const projectFps = editor.project.getActive().settings.fps; - const minDurationSeconds = 1 / projectFps; + const projectFps = editor.project.getActive().settings.fps; + const minDuration = Math.round(TICKS_PER_SECOND * projectFps.denominator / projectFps.numerator); const shouldSnap = snappingEnabled && !isShiftHeldRef.current; if (shouldSnap) { const tracks = editor.timeline.getTracks(); @@ -277,19 +279,19 @@ export function useTimelineElementResize({ const maxAllowed = sourceDuration - resizing.initialTrimEnd - - getVisibleSourceSpanForDuration(minDurationSeconds); + getVisibleSourceSpanForDuration(minDuration); const calculated = resizing.initialTrimStart + getSourceDeltaForClipDelta(deltaTime); if (calculated >= 0 && calculated <= maxAllowed) { - const newTrimStart = snapTimeToFrame({ time: Math.min(maxAllowed, Math.max(minTrimStartForNeighbor, calculated)), fps: projectFps }); + const newTrimStart = roundToFrame({ time: Math.min(maxAllowed, Math.max(minTrimStartForNeighbor, calculated)), rate: projectFps }) ?? Math.min(maxAllowed, Math.max(minTrimStartForNeighbor, calculated)); const visibleSourceSpan = Math.max( 0, sourceDuration - newTrimStart - resizing.initialTrimEnd, ); - const newDuration = snapTimeToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), fps: projectFps }); + const newDuration = roundToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), rate: projectFps }) ?? getDurationForVisibleSourceSpan(visibleSourceSpan); const trimDelta = resizing.initialDuration - newDuration; - const newStartTime = snapTimeToFrame({ time: resizing.initialStartTime + trimDelta, fps: projectFps }); + const newStartTime = roundToFrame({ time: resizing.initialStartTime + trimDelta, rate: projectFps }) ?? resizing.initialStartTime + trimDelta; setCurrentTrimStart(newTrimStart); setCurrentStartTime(newStartTime); @@ -311,8 +313,8 @@ export function useTimelineElementResize({ ) : Math.min(extensionAmount, maxExtension), ); - const newStartTime = snapTimeToFrame({ time: resizing.initialStartTime - actualExtension, fps: projectFps }); - const newDuration = snapTimeToFrame({ time: resizing.initialDuration + actualExtension, fps: projectFps }); + const newStartTime = roundToFrame({ time: resizing.initialStartTime - actualExtension, rate: projectFps }) ?? resizing.initialStartTime - actualExtension; + const newDuration = roundToFrame({ time: resizing.initialDuration + actualExtension, rate: projectFps }) ?? resizing.initialDuration + actualExtension; setCurrentTrimStart(0); setCurrentStartTime(newStartTime); @@ -338,8 +340,8 @@ export function useTimelineElementResize({ 0, sourceDuration - newTrimStart - resizing.initialTrimEnd, ); - const newDuration = snapTimeToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), fps: projectFps }); - const newStartTime = snapTimeToFrame({ time: resizing.initialStartTime + (resizing.initialDuration - newDuration), fps: projectFps }); + const newDuration = roundToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), rate: projectFps }) ?? getDurationForVisibleSourceSpan(visibleSourceSpan); + const newStartTime = roundToFrame({ time: resizing.initialStartTime + (resizing.initialDuration - newDuration), rate: projectFps }) ?? resizing.initialStartTime + (resizing.initialDuration - newDuration); setCurrentTrimStart(newTrimStart); setCurrentStartTime(newStartTime); @@ -366,7 +368,7 @@ export function useTimelineElementResize({ const extensionNeeded = Math.abs(newTrimEnd); const baseDuration = resizing.initialDuration + resizing.initialTrimEnd; - const newDuration = snapTimeToFrame({ time: Math.min(baseDuration + extensionNeeded, maxAllowedDuration), fps: projectFps }); + const newDuration = roundToFrame({ time: Math.min(baseDuration + extensionNeeded, maxAllowedDuration), rate: projectFps }) ?? Math.min(baseDuration + extensionNeeded, maxAllowedDuration); setCurrentDuration(newDuration); setCurrentTrimEnd(0); @@ -376,7 +378,7 @@ export function useTimelineElementResize({ const unclampedDuration = getDurationForVisibleSourceSpan( Math.max(0, sourceDuration - resizing.initialTrimStart), ); - const newDuration = snapTimeToFrame({ time: Math.min(unclampedDuration, maxAllowedDuration), fps: projectFps }); + const newDuration = roundToFrame({ time: Math.min(unclampedDuration, maxAllowedDuration), rate: projectFps }) ?? Math.min(unclampedDuration, maxAllowedDuration); setCurrentDuration(newDuration); setCurrentTrimEnd(0); @@ -395,17 +397,17 @@ export function useTimelineElementResize({ const maxTrimEnd = sourceDuration - resizing.initialTrimStart - - getVisibleSourceSpanForDuration(minDurationSeconds); + getVisibleSourceSpanForDuration(minDuration); const clampedTrimEnd = Math.min( maxTrimEnd, Math.max(minTrimEndForNeighbor, newTrimEnd), ); - const finalTrimEnd = snapTimeToFrame({ time: clampedTrimEnd, fps: projectFps }); + const finalTrimEnd = roundToFrame({ time: clampedTrimEnd, rate: projectFps }) ?? clampedTrimEnd; const visibleSourceSpan = Math.max( 0, sourceDuration - resizing.initialTrimStart - finalTrimEnd, ); - const newDuration = snapTimeToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), fps: projectFps }); + const newDuration = roundToFrame({ time: getDurationForVisibleSourceSpan(visibleSourceSpan), rate: projectFps }) ?? getDurationForVisibleSourceSpan(visibleSourceSpan); setCurrentTrimEnd(finalTrimEnd); setCurrentDuration(newDuration); diff --git a/apps/web/src/hooks/timeline/element/use-keyframe-drag.ts b/apps/web/src/hooks/timeline/element/use-keyframe-drag.ts index b66bbea8..24bff26a 100644 --- a/apps/web/src/hooks/timeline/element/use-keyframe-drag.ts +++ b/apps/web/src/hooks/timeline/element/use-keyframe-drag.ts @@ -8,9 +8,10 @@ import { import { useEditor } from "@/hooks/use-editor"; import { getKeyframeById } from "@/lib/animation"; import { useKeyframeSelection } from "./use-keyframe-selection"; -import { snapTimeToFrame, getSnappedSeekTime } from "opencut-wasm"; +import { roundToFrame, snappedSeekTime } from "opencut-wasm"; import { timelineTimeToSnappedPixels } from "@/lib/timeline"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { TIMELINE_DRAG_THRESHOLD_PX } from "@/components/editor/panels/timeline/interaction"; import { RetimeKeyframeCommand } from "@/lib/commands/timeline/element/keyframes/retime-keyframe"; import { BatchCommand } from "@/lib/commands"; @@ -144,9 +145,9 @@ export function useKeyframeDrag({ if (!dragState.isDragging) return; - const startX = mouseDownXRef.current ?? clientX; - const rawDelta = (clientX - startX) / pixelsPerSecond; - const snappedDelta = snapTimeToFrame({ time: rawDelta, fps }); + const startX = mouseDownXRef.current ?? clientX; + const rawDelta = Math.round(((clientX - startX) / pixelsPerSecond) * TICKS_PER_SECOND); + const snappedDelta = roundToFrame({ time: rawDelta, rate: fps }) ?? rawDelta; setDragState((previous) => ({ ...previous, deltaTime: snappedDelta })); }; @@ -255,7 +256,7 @@ export function useKeyframeDrag({ if (wasDrag) return; const duration = editor.timeline.getTotalDuration(); - const seekTime = getSnappedSeekTime({ rawTime: displayedStartTime + indicatorTime, duration, fps }); + const seekTime = snappedSeekTime({ time: displayedStartTime + indicatorTime, duration, rate: fps }) ?? displayedStartTime + indicatorTime; editor.playback.seek({ time: seekTime }); if (event.shiftKey) { diff --git a/apps/web/src/hooks/timeline/use-bookmark-drag.ts b/apps/web/src/hooks/timeline/use-bookmark-drag.ts index 14d58d9a..90edbede 100644 --- a/apps/web/src/hooks/timeline/use-bookmark-drag.ts +++ b/apps/web/src/hooks/timeline/use-bookmark-drag.ts @@ -8,7 +8,7 @@ import { import { useEditor } from "@/hooks/use-editor"; import { useShiftKey } from "@/hooks/use-shift-key"; import { TIMELINE_DRAG_THRESHOLD_PX } from "@/components/editor/panels/timeline/interaction"; -import { snapTimeToFrame } from "opencut-wasm"; +import { roundToFrame } from "opencut-wasm"; import { getMouseTimeFromClientX } from "@/lib/timeline/drag-utils"; import { findSnapPoints, @@ -148,7 +148,7 @@ export function useBookmarkDrag({ zoomLevel, scrollLeft, }); - const frameSnappedTime = snapTimeToFrame({ time: Math.max(0, Math.min(mouseTime, duration)), fps: activeProject.settings.fps }); + const frameSnappedTime = roundToFrame({ time: Math.max(0, Math.min(mouseTime, duration)), rate: activeProject.settings.fps }) ?? Math.max(0, Math.min(mouseTime, duration)); const { snappedTime: initialTime } = getSnapResult({ rawTime: frameSnappedTime, excludeBookmarkTime: bookmarkTime, @@ -176,9 +176,9 @@ export function useBookmarkDrag({ scrollLeft, }); const clampedTime = Math.max(0, Math.min(mouseTime, duration)); - const frameSnappedTime = snapTimeToFrame({ time: clampedTime, fps: activeProject.settings.fps }); - const snapResult = getSnapResult({ - rawTime: frameSnappedTime, + const frameSnappedTime = roundToFrame({ time: clampedTime, rate: activeProject.settings.fps }) ?? clampedTime; + const snapResult = getSnapResult({ + rawTime: frameSnappedTime, excludeBookmarkTime: dragState.bookmarkTime, }); diff --git a/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts b/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts index 8e87510c..7b9ee2ee 100644 --- a/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts +++ b/apps/web/src/hooks/timeline/use-timeline-drag-drop.ts @@ -3,9 +3,9 @@ import { useEditor } from "@/hooks/use-editor"; import { processMediaAssets } from "@/lib/media/processing"; import { toast } from "sonner"; import { showMediaUploadToast } from "@/lib/media/upload-toast"; -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; -import { snapTimeToFrame } from "opencut-wasm"; +import { roundToFrame } from "opencut-wasm"; import { buildTextElement, buildGraphicElement, @@ -47,7 +47,7 @@ export function useTimelineDragDrop({ const getSnappedTime = useCallback( ({ time }: { time: number }) => { const projectFps = editor.project.getActive().settings.fps; - return snapTimeToFrame({ time, fps: projectFps }); + return roundToFrame({ time, rate: projectFps }) ?? time; }, [editor], ); @@ -83,14 +83,14 @@ export function useTimelineDragDrop({ elementType === "sticker" || elementType === "effect" ) { - return DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + return DEFAULT_NEW_ELEMENT_DURATION; } if (mediaId) { const mediaAssets = editor.media.getAssets(); const media = mediaAssets.find((m) => m.id === mediaId); - return media?.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + return media?.duration ?? DEFAULT_NEW_ELEMENT_DURATION; } - return DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + return DEFAULT_NEW_ELEMENT_DURATION; }, [editor], ); @@ -331,7 +331,7 @@ export function useTimelineDragDrop({ dragData.mediaType === "audio" ? "audio" : "video"; const duration = - mediaAsset.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + mediaAsset.duration ?? DEFAULT_NEW_ELEMENT_DURATION; const element = buildElementFromMedia({ mediaId: mediaAsset.id, mediaType: mediaAsset.type, @@ -446,7 +446,7 @@ export function useTimelineDragDrop({ const duration = createdAsset.duration ?? - DEFAULT_NEW_ELEMENT_DURATION_SECONDS; + DEFAULT_NEW_ELEMENT_DURATION; const currentTracks = editor.timeline.getTracks(); const currentTime = editor.playback.getCurrentTime(); const onlyTrack = currentTracks[0]; diff --git a/apps/web/src/hooks/timeline/use-timeline-playhead.ts b/apps/web/src/hooks/timeline/use-timeline-playhead.ts index fce84135..70364e90 100644 --- a/apps/web/src/hooks/timeline/use-timeline-playhead.ts +++ b/apps/web/src/hooks/timeline/use-timeline-playhead.ts @@ -1,4 +1,5 @@ -import { getSnappedSeekTime } from "opencut-wasm"; +import { snappedSeekTime } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { useEffect, useCallback, useRef } from "react"; import { useEdgeAutoScroll } from "@/hooks/timeline/use-edge-auto-scroll"; import { useEditor } from "../use-editor"; @@ -6,6 +7,7 @@ import { useShiftKey } from "@/hooks/use-shift-key"; import { findSnapPoints, snapToNearestPoint } from "@/lib/timeline/snap-utils"; import { getCenteredLineLeft, + timelineTimeToPixels, timelineTimeToSnappedPixels, } from "@/lib/timeline"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; @@ -66,24 +68,24 @@ export function useTimelinePlayhead({ const rulerRect = ruler.getBoundingClientRect(); const relativeMouseX = event.clientX - rulerRect.left; - const timelineContentWidth = - duration * BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; + const timelineContentWidth = timelineTimeToPixels({ time: duration, zoomLevel }); - const clampedMouseX = Math.max( - 0, - Math.min(timelineContentWidth, relativeMouseX), - ); + const clampedMouseX = Math.max( + 0, + Math.min(timelineContentWidth, relativeMouseX), + ); - const rawTime = Math.max( - 0, - Math.min( - duration, - clampedMouseX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), - ), - ); + const rawTimeSeconds = Math.max( + 0, + Math.min( + duration / TICKS_PER_SECOND, + clampedMouseX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), + ), + ); + const rawTime = Math.round(rawTimeSeconds * TICKS_PER_SECOND); - const framesPerSecond = activeProject.settings.fps; - const frameTime = getSnappedSeekTime({ rawTime, duration, fps: framesPerSecond }); + const rate = activeProject.settings.fps; + const frameTime = snappedSeekTime({ time: rawTime, duration, rate }) ?? rawTime; const shouldSnap = snappingEnabled && !isShiftHeldRef.current; const time = (() => { @@ -161,7 +163,7 @@ export function useTimelinePlayhead({ getMouseClientX: () => lastMouseXRef.current, rulerScrollRef, tracksScrollRef, - contentWidth: duration * BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel, + contentWidth: timelineTimeToPixels({ time: duration, zoomLevel }), }); useEffect(() => { @@ -247,8 +249,10 @@ export function useTimelinePlayhead({ const tracksViewport = tracksScrollRef.current; if (!rulerViewport || !tracksViewport) return; - const playheadPixels = - time * BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevelRef.current; + const playheadPixels = timelineTimeToPixels({ + time, + zoomLevel: zoomLevelRef.current, + }); const viewportWidth = rulerViewport.clientWidth; const scrollMinimum = 0; const scrollMaximum = rulerViewport.scrollWidth - viewportWidth; diff --git a/apps/web/src/hooks/timeline/use-timeline-seek.ts b/apps/web/src/hooks/timeline/use-timeline-seek.ts index 1690153e..756ec7ee 100644 --- a/apps/web/src/hooks/timeline/use-timeline-seek.ts +++ b/apps/web/src/hooks/timeline/use-timeline-seek.ts @@ -1,7 +1,8 @@ import { useCallback, useRef } from "react"; import type { MutableRefObject, RefObject } from "react"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; -import { getSnappedSeekTime } from "opencut-wasm"; +import { snappedSeekTime } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; import { useEditor } from "../use-editor"; interface UseTimelineSeekProps { @@ -127,17 +128,18 @@ export function useTimelineSeek({ const mouseX = event.clientX - rect.left; const scrollLeft = scrollContainer.scrollLeft; - const rawTime = Math.max( - 0, - Math.min( - duration, - (mouseX + scrollLeft) / - (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), - ), - ); + const rawTimeSeconds = Math.max( + 0, + Math.min( + duration / TICKS_PER_SECOND, + (mouseX + scrollLeft) / + (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), + ), + ); + const rawTime = Math.round(rawTimeSeconds * TICKS_PER_SECOND); - const projectFps = activeProject?.settings.fps || 30; - const time = getSnappedSeekTime({ rawTime, duration, fps: projectFps }); + const rate = activeProject?.settings.fps; + const time = rate ? (snappedSeekTime({ time: rawTime, duration, rate }) ?? rawTime) : rawTime; seek(time); editor.project.setTimelineViewState({ viewState: { @@ -154,7 +156,8 @@ export function useTimelineSeek({ tracksScrollRef, seek, editor, - activeProject?.settings.fps, + activeProject?.settings.fps.numerator, + activeProject?.settings.fps.denominator, ], ); diff --git a/apps/web/src/hooks/timeline/use-timeline-zoom.ts b/apps/web/src/hooks/timeline/use-timeline-zoom.ts index a1740384..1f79a5b2 100644 --- a/apps/web/src/hooks/timeline/use-timeline-zoom.ts +++ b/apps/web/src/hooks/timeline/use-timeline-zoom.ts @@ -13,8 +13,8 @@ import { import { TIMELINE_ZOOM_MAX, TIMELINE_ZOOM_MIN, - BASE_TIMELINE_PIXELS_PER_SECOND, } from "@/lib/timeline/scale"; +import { timelineTimeToPixels } from "@/lib/timeline/pixel-utils"; import { useEditor } from "@/hooks/use-editor"; import { zoomToSlider } from "@/lib/timeline/zoom-utils"; @@ -186,10 +186,8 @@ export function useTimelineZoom({ } if (sliderPercent >= TIMELINE_ZOOM_ANCHOR_PLAYHEAD_THRESHOLD) { - const playheadPixelsBefore = - playheadTime * BASE_TIMELINE_PIXELS_PER_SECOND * previousZoom; - const playheadPixelsAfter = - playheadTime * BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; + const playheadPixelsBefore = timelineTimeToPixels({ time: playheadTime, zoomLevel: previousZoom }); + const playheadPixelsAfter = timelineTimeToPixels({ time: playheadTime, zoomLevel }); const viewportOffset = playheadPixelsBefore - currentScrollLeft; const newScrollLeft = playheadPixelsAfter - viewportOffset; diff --git a/apps/web/src/hooks/use-paste-media.ts b/apps/web/src/hooks/use-paste-media.ts index 1cba34c0..36d70ebe 100644 --- a/apps/web/src/hooks/use-paste-media.ts +++ b/apps/web/src/hooks/use-paste-media.ts @@ -1,114 +1,114 @@ -import { useEffect } from "react";
-import { useEditor } from "@/hooks/use-editor";
-import { processMediaAssets } from "@/lib/media/processing";
-import { showMediaUploadToast } from "@/lib/media/upload-toast";
-import { invokeAction } from "@/lib/actions";
-import { buildElementFromMedia } from "@/lib/timeline/element-utils";
-import { AddMediaAssetCommand } from "@/lib/commands/media";
-import { InsertElementCommand } from "@/lib/commands/timeline";
-import { BatchCommand } from "@/lib/commands";
-import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation";
-import { isTypableDOMElement } from "@/utils/browser";
-import type { MediaType } from "@/lib/media/types";
-
-const MEDIA_MIME_PREFIXES: MediaType[] = ["image", "video", "audio"];
-
-function isMediaMimeType({ type }: { type: string }): boolean {
- return MEDIA_MIME_PREFIXES.some((prefix) => type.startsWith(`${prefix}/`));
-}
-
-function extractMediaFilesFromClipboard({
- clipboardData,
-}: {
- clipboardData: DataTransfer | null;
-}): File[] {
- if (!clipboardData?.items) return [];
-
- const files: File[] = [];
- for (const item of clipboardData.items) {
- if (item.kind !== "file") continue;
- if (!isMediaMimeType({ type: item.type })) continue;
-
- const file = item.getAsFile();
- if (file) files.push(file);
- }
- return files;
-}
-
-export function usePasteMedia() {
- const editor = useEditor();
-
- useEffect(() => {
- const handlePaste = async (event: ClipboardEvent) => {
- const activeElement = document.activeElement as HTMLElement;
-
- if (activeElement && isTypableDOMElement({ element: activeElement })) {
- return;
- }
-
- const files = extractMediaFilesFromClipboard({
- clipboardData: event.clipboardData,
- });
- if (files.length === 0) {
- event.preventDefault();
- invokeAction("paste-copied");
- return;
- }
-
- event.preventDefault();
-
- const activeProject = editor.project.getActive();
- if (!activeProject) return;
-
- try {
- await showMediaUploadToast({
- filesCount: files.length,
- promise: async () => {
- const processedAssets = await processMediaAssets({ files });
- const startTime = editor.playback.getCurrentTime();
-
- for (const asset of processedAssets) {
- const addMediaCmd = new AddMediaAssetCommand(
- activeProject.metadata.id,
- asset,
- );
- const assetId = addMediaCmd.getAssetId();
- const duration =
- asset.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS;
- const trackType = asset.type === "audio" ? "audio" : "video";
-
- const element = buildElementFromMedia({
- mediaId: assetId,
- mediaType: asset.type,
- name: asset.name,
- duration,
- startTime,
- buffer:
- asset.type === "audio"
- ? new AudioBuffer({ length: 1, sampleRate: 44100 })
- : undefined,
- });
-
- const insertCmd = new InsertElementCommand({
- element,
- placement: { mode: "auto", trackType },
- });
- const batchCmd = new BatchCommand([addMediaCmd, insertCmd]);
- editor.command.execute({ command: batchCmd });
- }
-
- return {
- uploadedCount: processedAssets.length,
- assetNames: processedAssets.map((asset) => asset.name),
- };
- },
- });
- } catch (error) {
- console.error("Failed to paste media:", error);
- }
- };
-
- window.addEventListener("paste", handlePaste);
- return () => window.removeEventListener("paste", handlePaste);
- }, [editor]);
-}
+import { useEffect } from "react"; +import { useEditor } from "@/hooks/use-editor"; +import { processMediaAssets } from "@/lib/media/processing"; +import { showMediaUploadToast } from "@/lib/media/upload-toast"; +import { invokeAction } from "@/lib/actions"; +import { buildElementFromMedia } from "@/lib/timeline/element-utils"; +import { AddMediaAssetCommand } from "@/lib/commands/media"; +import { InsertElementCommand } from "@/lib/commands/timeline"; +import { BatchCommand } from "@/lib/commands"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import { isTypableDOMElement } from "@/utils/browser"; +import type { MediaType } from "@/lib/media/types"; + +const MEDIA_MIME_PREFIXES: MediaType[] = ["image", "video", "audio"]; + +function isMediaMimeType({ type }: { type: string }): boolean { + return MEDIA_MIME_PREFIXES.some((prefix) => type.startsWith(`${prefix}/`)); +} + +function extractMediaFilesFromClipboard({ + clipboardData, +}: { + clipboardData: DataTransfer | null; +}): File[] { + if (!clipboardData?.items) return []; + + const files: File[] = []; + for (const item of clipboardData.items) { + if (item.kind !== "file") continue; + if (!isMediaMimeType({ type: item.type })) continue; + + const file = item.getAsFile(); + if (file) files.push(file); + } + return files; +} + +export function usePasteMedia() { + const editor = useEditor(); + + useEffect(() => { + const handlePaste = async (event: ClipboardEvent) => { + const activeElement = document.activeElement as HTMLElement; + + if (activeElement && isTypableDOMElement({ element: activeElement })) { + return; + } + + const files = extractMediaFilesFromClipboard({ + clipboardData: event.clipboardData, + }); + if (files.length === 0) { + event.preventDefault(); + invokeAction("paste-copied"); + return; + } + + event.preventDefault(); + + const activeProject = editor.project.getActive(); + if (!activeProject) return; + + try { + await showMediaUploadToast({ + filesCount: files.length, + promise: async () => { + const processedAssets = await processMediaAssets({ files }); + const startTime = editor.playback.getCurrentTime(); + + for (const asset of processedAssets) { + const addMediaCmd = new AddMediaAssetCommand( + activeProject.metadata.id, + asset, + ); + const assetId = addMediaCmd.getAssetId(); + const duration = + asset.duration ?? DEFAULT_NEW_ELEMENT_DURATION; + const trackType = asset.type === "audio" ? "audio" : "video"; + + const element = buildElementFromMedia({ + mediaId: assetId, + mediaType: asset.type, + name: asset.name, + duration, + startTime, + buffer: + asset.type === "audio" + ? new AudioBuffer({ length: 1, sampleRate: 44100 }) + : undefined, + }); + + const insertCmd = new InsertElementCommand({ + element, + placement: { mode: "auto", trackType }, + }); + const batchCmd = new BatchCommand([addMediaCmd, insertCmd]); + editor.command.execute({ command: batchCmd }); + } + + return { + uploadedCount: processedAssets.length, + assetNames: processedAssets.map((asset) => asset.name), + }; + }, + }); + } catch (error) { + console.error("Failed to paste media:", error); + } + }; + + window.addEventListener("paste", handlePaste); + return () => window.removeEventListener("paste", handlePaste); + }, [editor]); +} diff --git a/apps/web/src/lib/animation/curve-bridge.ts b/apps/web/src/lib/animation/curve-bridge.ts index 55f96933..12103c3f 100644 --- a/apps/web/src/lib/animation/curve-bridge.ts +++ b/apps/web/src/lib/animation/curve-bridge.ts @@ -1,4 +1,3 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { getDefaultLeftHandle, getDefaultRightHandle, @@ -25,7 +24,7 @@ export function getNormalizedCubicBezierForScalarSegment({ const spanTime = rightKey.time - leftKey.time; const spanValue = rightKey.value - leftKey.value; if ( - Math.abs(spanTime) <= TIME_EPSILON_SECONDS || + spanTime === 0 || Math.abs(spanValue) <= VALUE_EPSILON ) { return null; @@ -59,7 +58,7 @@ export function getCurveHandlesForNormalizedCubicBezier({ const spanTime = rightKey.time - leftKey.time; const spanValue = rightKey.value - leftKey.value; if ( - Math.abs(spanTime) <= TIME_EPSILON_SECONDS || + spanTime === 0 || Math.abs(spanValue) <= VALUE_EPSILON ) { return null; diff --git a/apps/web/src/lib/animation/interpolation.ts b/apps/web/src/lib/animation/interpolation.ts index 67456aa0..1a3c0fed 100644 --- a/apps/web/src/lib/animation/interpolation.ts +++ b/apps/web/src/lib/animation/interpolation.ts @@ -8,7 +8,6 @@ import type { ScalarAnimationKey, ScalarSegmentType, } from "@/lib/animation/types"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { clamp } from "@/utils/math"; import { getBezierPoint, @@ -36,10 +35,7 @@ function isWithinTimePair({ leftTime: number; rightTime: number; }): boolean { - return ( - time >= leftTime - TIME_EPSILON_SECONDS && - time <= rightTime + TIME_EPSILON_SECONDS - ); + return time >= leftTime && time <= rightTime; } function lerpNumber({ @@ -67,7 +63,7 @@ function normalizeRightHandle({ return undefined; } - const span = Math.max(TIME_EPSILON_SECONDS, rightKey.time - leftKey.time); + const span = Math.max(1, rightKey.time - leftKey.time); return { dt: Math.min(span, Math.max(0, handle.dt)), dv: handle.dv, @@ -87,7 +83,7 @@ function normalizeLeftHandle({ return undefined; } - const span = Math.max(TIME_EPSILON_SECONDS, rightKey.time - leftKey.time); + const span = Math.max(1, rightKey.time - leftKey.time); return { dt: Math.max(-span, Math.min(0, handle.dt)), dv: handle.dv, @@ -187,7 +183,7 @@ function extrapolateScalarEdge({ } const span = neighborKey.time - edgeKey.time; - if (Math.abs(span) <= TIME_EPSILON_SECONDS) { + if (span === 0) { return edgeKey.value; } @@ -228,8 +224,8 @@ export function getScalarChannelValueAtTime({ return fallbackValue; } - if (time <= firstKey.time + TIME_EPSILON_SECONDS) { - if (time < firstKey.time - TIME_EPSILON_SECONDS) { + if (time <= firstKey.time) { + if (time < firstKey.time) { return extrapolateScalarEdge({ mode: normalizedChannel.extrapolation?.before ?? "hold", edgeKey: firstKey, @@ -241,8 +237,8 @@ export function getScalarChannelValueAtTime({ return firstKey.value; } - if (time >= lastKey.time - TIME_EPSILON_SECONDS) { - if (time > lastKey.time + TIME_EPSILON_SECONDS) { + if (time >= lastKey.time) { + if (time > lastKey.time) { return extrapolateScalarEdge({ mode: normalizedChannel.extrapolation?.after ?? "hold", edgeKey: lastKey, @@ -261,7 +257,7 @@ export function getScalarChannelValueAtTime({ ) { const leftKey = normalizedChannel.keys[keyIndex]; const rightKey = normalizedChannel.keys[keyIndex + 1]; - if (Math.abs(time - rightKey.time) <= TIME_EPSILON_SECONDS) { + if (time === rightKey.time) { return rightKey.value; } @@ -280,7 +276,7 @@ export function getScalarChannelValueAtTime({ } const span = rightKey.time - leftKey.time; - if (Math.abs(span) <= TIME_EPSILON_SECONDS) { + if (span === 0) { return rightKey.value; } @@ -336,7 +332,7 @@ export function getDiscreteChannelValueAtTime({ }); let currentValue = fallbackValue; for (const key of normalizedChannel.keys) { - if (time + TIME_EPSILON_SECONDS < key.time) { + if (time < key.time) { break; } currentValue = key.value; diff --git a/apps/web/src/lib/animation/keyframe-query.ts b/apps/web/src/lib/animation/keyframe-query.ts index 304a6698..dbcd041c 100644 --- a/apps/web/src/lib/animation/keyframe-query.ts +++ b/apps/web/src/lib/animation/keyframe-query.ts @@ -1,4 +1,3 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import type { AnimationBindingInstance, AnimationChannel, @@ -76,8 +75,7 @@ function getUniqueBindingKeyframeMatches({ const previousMatch = uniqueMatches[uniqueMatches.length - 1]; if ( !previousMatch || - Math.abs(previousMatch.keyframe.time - match.keyframe.time) > - TIME_EPSILON_SECONDS + previousMatch.keyframe.time !== match.keyframe.time ) { uniqueMatches.push(match); continue; @@ -249,9 +247,7 @@ export function getKeyframeAtTime({ matches: getBindingKeyframeMatches({ animations, binding, - }).filter(({ keyframe }) => - Math.abs(keyframe.time - time) <= TIME_EPSILON_SECONDS, - ), + }).filter(({ keyframe }) => keyframe.time === time), }); if (!keyframeMatch) { return null; diff --git a/apps/web/src/lib/animation/keyframes.ts b/apps/web/src/lib/animation/keyframes.ts index d72f9e16..05290c1d 100644 --- a/apps/web/src/lib/animation/keyframes.ts +++ b/apps/web/src/lib/animation/keyframes.ts @@ -14,7 +14,6 @@ import type { ScalarCurveKeyframePatch, ScalarSegmentType, } from "@/lib/animation/types"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { generateUUID } from "@/utils/id"; import { cloneAnimationBinding, @@ -44,7 +43,7 @@ function isNearlySameTime({ leftTime: number; rightTime: number; }): boolean { - return Math.abs(leftTime - rightTime) <= TIME_EPSILON_SECONDS; + return leftTime === rightTime; } function hasChannelKeys({ @@ -1032,8 +1031,8 @@ function splitScalarChannelAtTime({ const rightKey = normalizedChannel.keys[keyIndex + 1]; if ( !( - splitTime > leftKey.time + TIME_EPSILON_SECONDS && - splitTime < rightKey.time - TIME_EPSILON_SECONDS + splitTime > leftKey.time && + splitTime < rightKey.time ) ) { continue; diff --git a/apps/web/src/lib/commands/media/add-media-asset.ts b/apps/web/src/lib/commands/media/add-media-asset.ts index c12d7ea1..8b3067a1 100644 --- a/apps/web/src/lib/commands/media/add-media-asset.ts +++ b/apps/web/src/lib/commands/media/add-media-asset.ts @@ -4,16 +4,17 @@ import { toast } from "sonner"; import type { MediaAsset } from "@/lib/media/types"; import { generateUUID } from "@/utils/id"; import { storageService } from "@/services/storage/service"; +import type { FrameRate } from "opencut-wasm"; import { hasMediaId } from "@/lib/timeline/element-utils"; -import { getHighestImportedVideoFps } from "@/lib/fps/utils"; +import { frameRatesEqual, getHighestImportedVideoFps } from "@/lib/fps/utils"; import { UpdateProjectSettingsCommand } from "@/lib/commands/project"; export class AddMediaAssetCommand extends Command { private assetId: string; private savedAssets: MediaAsset[] | null = null; private createdAsset: MediaAsset | null = null; - private previousProjectFps: number | null = null; - private appliedProjectFps: number | null = null; + private previousProjectFps: FrameRate | null = null; + private appliedProjectFps: FrameRate | null = null; constructor( private projectId: string, @@ -112,14 +113,15 @@ export class AddMediaAssetCommand extends Command { const activeProject = editor.project.getActiveOrNull(); if (!activeProject) return; - if (activeProject.settings.fps !== this.appliedProjectFps) return; + if (!this.appliedProjectFps || !frameRatesEqual(activeProject.settings.fps, this.appliedProjectFps)) return; const highestRemainingVideoFps = getHighestImportedVideoFps({ mediaAssets: editor.media.getAssets(), }); + const appliedFpsFloat = this.appliedProjectFps.numerator / this.appliedProjectFps.denominator; if ( highestRemainingVideoFps !== null && - highestRemainingVideoFps >= this.appliedProjectFps + highestRemainingVideoFps >= appliedFpsFloat ) { return; } diff --git a/apps/web/src/lib/commands/timeline/element/insert-element.ts b/apps/web/src/lib/commands/timeline/element/insert-element.ts index fbef1613..9f0bf941 100644 --- a/apps/web/src/lib/commands/timeline/element/insert-element.ts +++ b/apps/web/src/lib/commands/timeline/element/insert-element.ts @@ -1,264 +1,265 @@ -import { Command, type CommandResult } from "@/lib/commands/base-command";
-import { EditorCore } from "@/core";
-import type {
- CreateTimelineElement,
- TimelineTrack,
- TimelineElement,
- TrackType,
-} from "@/lib/timeline";
-import { generateUUID } from "@/utils/id";
-import { requiresMediaId } from "@/lib/timeline/element-utils";
-import type { MediaAsset } from "@/lib/media/types";
-import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation";
-import { graphicsRegistry, registerDefaultGraphics } from "@/lib/graphics";
-import {
- applyPlacement,
- canElementGoOnTrack,
- resolveTrackPlacement,
- validateElementTrackCompatibility,
-} from "@/lib/timeline/placement";
-
-type InsertElementPlacement =
- | { mode: "explicit"; trackId: string }
- | { mode: "auto"; trackType?: TrackType; insertIndex?: number };
-
-export interface InsertElementParams {
- element: CreateTimelineElement;
- placement: InsertElementPlacement;
-}
-
-export class InsertElementCommand extends Command {
- private elementId: string;
- private savedState: TimelineTrack[] | null = null;
- private targetTrackId: string | null = null;
-
- constructor({ element, placement }: InsertElementParams) {
- super();
- this.elementId = generateUUID();
- this.element = element;
- this.placement = placement;
- }
-
- private element: CreateTimelineElement;
- private placement: InsertElementPlacement;
-
- execute(): CommandResult | undefined {
- const editor = EditorCore.getInstance();
- this.savedState = editor.timeline.getTracks();
-
- if (!this.savedState) {
- console.error("Tracks not available");
- return;
- }
-
- if (!this.validateElementBasics({ element: this.element })) {
- return;
- }
-
- const totalElementsInTimeline = this.savedState.reduce(
- (total, t) => total + t.elements.length,
- 0,
- );
- const isFirstElement = totalElementsInTimeline === 0;
-
- const newElement = this.buildElement({ element: this.element });
- const updateResult = this.applyPlacementResult({
- tracks: this.savedState,
- element: newElement,
- });
-
- if (!updateResult) {
- return;
- }
-
- const { updatedTracks, targetTrackId } = updateResult;
- this.targetTrackId = targetTrackId;
-
- const isVisualMedia =
- newElement.type === "video" || newElement.type === "image";
-
- if (isFirstElement && isVisualMedia) {
- const mediaAssets = editor.media.getAssets();
- const activeProject = editor.project.getActive();
- const asset = mediaAssets.find(
- (item: MediaAsset) => item.id === newElement.mediaId,
- );
-
- if (asset?.width && asset?.height) {
- const nextCanvasSize = { width: asset.width, height: asset.height };
- const shouldSetOriginalCanvasSize =
- !activeProject?.settings.originalCanvasSize;
- editor.project.updateSettings({
- settings: {
- canvasSize: nextCanvasSize,
- ...(shouldSetOriginalCanvasSize
- ? { originalCanvasSize: nextCanvasSize }
- : {}),
- },
- pushHistory: false,
- });
- }
-
- if (asset?.type === "video" && asset?.fps) {
- editor.project.updateSettings({
- settings: { fps: asset.fps },
- pushHistory: false,
- });
- }
- }
-
- editor.timeline.updateTracks(updatedTracks);
- }
-
- undo(): void {
- if (this.savedState) {
- const editor = EditorCore.getInstance();
- editor.timeline.updateTracks(this.savedState);
- }
- }
-
- getElementId(): string {
- return this.elementId;
- }
-
- getTrackId(): string | null {
- return this.targetTrackId;
- }
-
- private buildElement({
- element,
- }: {
- element: CreateTimelineElement;
- }): TimelineElement {
- return {
- ...element,
- id: this.elementId,
- startTime: element.startTime,
- trimStart: element.trimStart ?? 0,
- trimEnd: element.trimEnd ?? 0,
- duration: element.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS,
- } as TimelineElement;
- }
-
- private validateElementBasics({
- element,
- }: {
- element: CreateTimelineElement;
- }): boolean {
- if (requiresMediaId({ element }) && !("mediaId" in element)) {
- console.error("Element requires mediaId");
- return false;
- }
-
- if (
- element.type === "audio" &&
- element.sourceType === "library" &&
- !element.sourceUrl
- ) {
- console.error("Library audio element must have sourceUrl");
- return false;
- }
-
- if (element.type === "sticker" && !element.stickerId) {
- console.error("Sticker element must have stickerId");
- return false;
- }
-
- if (element.type === "graphic") {
- registerDefaultGraphics();
- if (!element.definitionId || !graphicsRegistry.has(element.definitionId)) {
- console.error("Graphic element must have a valid definitionId");
- return false;
- }
- }
-
- if (element.type === "text" && !element.content) {
- console.error("Text element must have content");
- return false;
- }
-
- if (element.type === "effect" && !element.effectType) {
- console.error("Effect element must have effectType");
- return false;
- }
-
- return true;
- }
-
- private applyPlacementResult({
- tracks,
- element,
- }: {
- tracks: TimelineTrack[];
- element: TimelineElement;
- }): { updatedTracks: TimelineTrack[]; targetTrackId: string } | null {
- const placement = this.placement;
-
- if (
- placement.mode === "auto" &&
- placement.trackType &&
- !canElementGoOnTrack({
- elementType: element.type,
- trackType: placement.trackType,
- })
- ) {
- console.error(
- `${element.type} elements cannot be placed on ${placement.trackType} tracks`,
- );
- return null;
- }
-
- const placementResult = resolveTrackPlacement({
- tracks,
- ...(placement.mode === "auto" && placement.trackType
- ? { trackType: placement.trackType }
- : { elementType: element.type }),
- timeSpans: [
- {
- startTime: element.startTime,
- duration: element.duration,
- },
- ],
- strategy:
- placement.mode === "explicit"
- ? { type: "explicit", trackId: placement.trackId }
- : { type: "firstAvailable" },
- });
- if (!placementResult) {
- if (placement.mode === "explicit") {
- const targetTrack = tracks.find((track) => track.id === placement.trackId);
- if (!targetTrack) {
- console.error("Track not found:", placement.trackId);
- return null;
- }
-
- const validation = validateElementTrackCompatibility({
- element,
- track: targetTrack,
- });
- console.error(validation.errorMessage);
- }
-
- return null;
- }
-
- const elementToPlace =
- placementResult.kind === "existingTrack"
- ? {
- ...element,
- startTime:
- placementResult.adjustedStartTime ?? element.startTime,
- }
- : element;
-
- return applyPlacement({
- tracks,
- placementResult,
- elements: [elementToPlace],
- newTrackInsertIndexOverride:
- placement.mode === "auto" && typeof placement.insertIndex === "number"
- ? placement.insertIndex
- : undefined,
- });
- }
-}
+import { Command, type CommandResult } from "@/lib/commands/base-command"; +import { EditorCore } from "@/core"; +import type { + CreateTimelineElement, + TimelineTrack, + TimelineElement, + TrackType, +} from "@/lib/timeline"; +import { generateUUID } from "@/utils/id"; +import { requiresMediaId } from "@/lib/timeline/element-utils"; +import type { MediaAsset } from "@/lib/media/types"; +import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import { floatToFrameRate } from "@/lib/fps/utils"; +import { graphicsRegistry, registerDefaultGraphics } from "@/lib/graphics"; +import { + applyPlacement, + canElementGoOnTrack, + resolveTrackPlacement, + validateElementTrackCompatibility, +} from "@/lib/timeline/placement"; + +type InsertElementPlacement = + | { mode: "explicit"; trackId: string } + | { mode: "auto"; trackType?: TrackType; insertIndex?: number }; + +export interface InsertElementParams { + element: CreateTimelineElement; + placement: InsertElementPlacement; +} + +export class InsertElementCommand extends Command { + private elementId: string; + private savedState: TimelineTrack[] | null = null; + private targetTrackId: string | null = null; + + constructor({ element, placement }: InsertElementParams) { + super(); + this.elementId = generateUUID(); + this.element = element; + this.placement = placement; + } + + private element: CreateTimelineElement; + private placement: InsertElementPlacement; + + execute(): CommandResult | undefined { + const editor = EditorCore.getInstance(); + this.savedState = editor.timeline.getTracks(); + + if (!this.savedState) { + console.error("Tracks not available"); + return; + } + + if (!this.validateElementBasics({ element: this.element })) { + return; + } + + const totalElementsInTimeline = this.savedState.reduce( + (total, t) => total + t.elements.length, + 0, + ); + const isFirstElement = totalElementsInTimeline === 0; + + const newElement = this.buildElement({ element: this.element }); + const updateResult = this.applyPlacementResult({ + tracks: this.savedState, + element: newElement, + }); + + if (!updateResult) { + return; + } + + const { updatedTracks, targetTrackId } = updateResult; + this.targetTrackId = targetTrackId; + + const isVisualMedia = + newElement.type === "video" || newElement.type === "image"; + + if (isFirstElement && isVisualMedia) { + const mediaAssets = editor.media.getAssets(); + const activeProject = editor.project.getActive(); + const asset = mediaAssets.find( + (item: MediaAsset) => item.id === newElement.mediaId, + ); + + if (asset?.width && asset?.height) { + const nextCanvasSize = { width: asset.width, height: asset.height }; + const shouldSetOriginalCanvasSize = + !activeProject?.settings.originalCanvasSize; + editor.project.updateSettings({ + settings: { + canvasSize: nextCanvasSize, + ...(shouldSetOriginalCanvasSize + ? { originalCanvasSize: nextCanvasSize } + : {}), + }, + pushHistory: false, + }); + } + + if (asset?.type === "video" && asset?.fps) { + editor.project.updateSettings({ + settings: { fps: floatToFrameRate(asset.fps) }, + pushHistory: false, + }); + } + } + + editor.timeline.updateTracks(updatedTracks); + } + + undo(): void { + if (this.savedState) { + const editor = EditorCore.getInstance(); + editor.timeline.updateTracks(this.savedState); + } + } + + getElementId(): string { + return this.elementId; + } + + getTrackId(): string | null { + return this.targetTrackId; + } + + private buildElement({ + element, + }: { + element: CreateTimelineElement; + }): TimelineElement { + return { + ...element, + id: this.elementId, + startTime: element.startTime, + trimStart: element.trimStart ?? 0, + trimEnd: element.trimEnd ?? 0, + duration: element.duration ?? DEFAULT_NEW_ELEMENT_DURATION, + } as TimelineElement; + } + + private validateElementBasics({ + element, + }: { + element: CreateTimelineElement; + }): boolean { + if (requiresMediaId({ element }) && !("mediaId" in element)) { + console.error("Element requires mediaId"); + return false; + } + + if ( + element.type === "audio" && + element.sourceType === "library" && + !element.sourceUrl + ) { + console.error("Library audio element must have sourceUrl"); + return false; + } + + if (element.type === "sticker" && !element.stickerId) { + console.error("Sticker element must have stickerId"); + return false; + } + + if (element.type === "graphic") { + registerDefaultGraphics(); + if (!element.definitionId || !graphicsRegistry.has(element.definitionId)) { + console.error("Graphic element must have a valid definitionId"); + return false; + } + } + + if (element.type === "text" && !element.content) { + console.error("Text element must have content"); + return false; + } + + if (element.type === "effect" && !element.effectType) { + console.error("Effect element must have effectType"); + return false; + } + + return true; + } + + private applyPlacementResult({ + tracks, + element, + }: { + tracks: TimelineTrack[]; + element: TimelineElement; + }): { updatedTracks: TimelineTrack[]; targetTrackId: string } | null { + const placement = this.placement; + + if ( + placement.mode === "auto" && + placement.trackType && + !canElementGoOnTrack({ + elementType: element.type, + trackType: placement.trackType, + }) + ) { + console.error( + `${element.type} elements cannot be placed on ${placement.trackType} tracks`, + ); + return null; + } + + const placementResult = resolveTrackPlacement({ + tracks, + ...(placement.mode === "auto" && placement.trackType + ? { trackType: placement.trackType } + : { elementType: element.type }), + timeSpans: [ + { + startTime: element.startTime, + duration: element.duration, + }, + ], + strategy: + placement.mode === "explicit" + ? { type: "explicit", trackId: placement.trackId } + : { type: "firstAvailable" }, + }); + if (!placementResult) { + if (placement.mode === "explicit") { + const targetTrack = tracks.find((track) => track.id === placement.trackId); + if (!targetTrack) { + console.error("Track not found:", placement.trackId); + return null; + } + + const validation = validateElementTrackCompatibility({ + element, + track: targetTrack, + }); + console.error(validation.errorMessage); + } + + return null; + } + + const elementToPlace = + placementResult.kind === "existingTrack" + ? { + ...element, + startTime: + placementResult.adjustedStartTime ?? element.startTime, + } + : element; + + return applyPlacement({ + tracks, + placementResult, + elements: [elementToPlace], + newTrackInsertIndexOverride: + placement.mode === "auto" && typeof placement.insertIndex === "number" + ? placement.insertIndex + : undefined, + }); + } +} diff --git a/apps/web/src/lib/export.ts b/apps/web/src/lib/export.ts index 68d2bc7f..f397ac4b 100644 --- a/apps/web/src/lib/export.ts +++ b/apps/web/src/lib/export.ts @@ -1,3 +1,4 @@ +import type { FrameRate } from "opencut-wasm"; import { EXPORT_MIME_TYPES } from "@/constants/export-constants"; export const EXPORT_QUALITY_VALUES = [ @@ -15,7 +16,7 @@ export type ExportQuality = (typeof EXPORT_QUALITY_VALUES)[number]; export interface ExportOptions { format: ExportFormat; quality: ExportQuality; - fps?: number; + fps?: FrameRate; includeAudio?: boolean; } diff --git a/apps/web/src/lib/fps/__tests__/fps.test.ts b/apps/web/src/lib/fps/__tests__/fps.test.ts index fd082bb2..10eed8d9 100644 --- a/apps/web/src/lib/fps/__tests__/fps.test.ts +++ b/apps/web/src/lib/fps/__tests__/fps.test.ts @@ -36,16 +36,16 @@ describe("getRaisedProjectFpsForImportedMedia", () => { test("raises the project fps to match a higher-fps import", () => { expect( getRaisedProjectFpsForImportedMedia({ - currentFps: 30, + currentFps: { numerator: 30, denominator: 1 }, importedAssets: [{ type: "video", fps: 60 }], }), - ).toBe(60); + ).toEqual({ numerator: 60, denominator: 1 }); }); test("does not lower the project fps for lower-fps imports", () => { expect( getRaisedProjectFpsForImportedMedia({ - currentFps: 60, + currentFps: { numerator: 60, denominator: 1 }, importedAssets: [{ type: "video", fps: 10 }], }), ).toBeNull(); @@ -54,7 +54,7 @@ describe("getRaisedProjectFpsForImportedMedia", () => { test("ignores non-video imports", () => { expect( getRaisedProjectFpsForImportedMedia({ - currentFps: 30, + currentFps: { numerator: 30, denominator: 1 }, importedAssets: [ { type: "image", fps: 60 }, { type: "audio", fps: 120 }, diff --git a/apps/web/src/lib/fps/constants.ts b/apps/web/src/lib/fps/constants.ts index 100c043a..30c3d2bc 100644 --- a/apps/web/src/lib/fps/constants.ts +++ b/apps/web/src/lib/fps/constants.ts @@ -1,3 +1,5 @@ +import type { FrameRate } from "opencut-wasm"; + export const FPS_PRESETS = [ { value: "24", label: "24 fps" }, { value: "25", label: "25 fps" }, @@ -6,4 +8,4 @@ export const FPS_PRESETS = [ { value: "120", label: "120 fps" }, ] as const; -export const DEFAULT_FPS = 30; +export const DEFAULT_FPS: FrameRate = { numerator: 30, denominator: 1 }; diff --git a/apps/web/src/lib/fps/utils.ts b/apps/web/src/lib/fps/utils.ts index 8e3221a1..932788c4 100644 --- a/apps/web/src/lib/fps/utils.ts +++ b/apps/web/src/lib/fps/utils.ts @@ -1,7 +1,61 @@ +import type { FrameRate } from "opencut-wasm"; import type { MediaAsset } from "@/lib/media/types"; type MediaAssetFpsInput = Pick<MediaAsset, "type" | "fps">; +const STANDARD_FRAME_RATES: Array<{ value: number; rate: FrameRate }> = [ + { value: 24_000 / 1_001, rate: { numerator: 24_000, denominator: 1_001 } }, + { value: 24, rate: { numerator: 24, denominator: 1 } }, + { value: 25, rate: { numerator: 25, denominator: 1 } }, + { value: 30_000 / 1_001, rate: { numerator: 30_000, denominator: 1_001 } }, + { value: 30, rate: { numerator: 30, denominator: 1 } }, + { value: 48, rate: { numerator: 48, denominator: 1 } }, + { value: 50, rate: { numerator: 50, denominator: 1 } }, + { value: 60_000 / 1_001, rate: { numerator: 60_000, denominator: 1_001 } }, + { value: 60, rate: { numerator: 60, denominator: 1 } }, + { value: 120, rate: { numerator: 120, denominator: 1 } }, +]; + +const STANDARD_FRAME_RATE_TOLERANCE = 0.01; + +export function frameRateToFloat(rate: FrameRate): number { + return rate.numerator / rate.denominator; +} + +export function frameRatesEqual(a: FrameRate, b: FrameRate): boolean { + return a.numerator === b.numerator && a.denominator === b.denominator; +} + +export function floatToFrameRate(fps: number): FrameRate { + const standard = STANDARD_FRAME_RATES.find( + (candidate) => Math.abs(fps - candidate.value) <= STANDARD_FRAME_RATE_TOLERANCE, + ); + if (standard) return standard.rate; + + if (Number.isInteger(fps)) { + return { numerator: fps, denominator: 1 }; + } + + const ARBITRARY_DENOMINATOR = 1_000_000; + const scaledNumerator = Math.round(fps * ARBITRARY_DENOMINATOR); + const divisor = gcd(scaledNumerator, ARBITRARY_DENOMINATOR); + return { + numerator: scaledNumerator / divisor, + denominator: ARBITRARY_DENOMINATOR / divisor, + }; +} + +function gcd(left: number, right: number): number { + let a = Math.abs(left); + let b = Math.abs(right); + while (b !== 0) { + const remainder = a % b; + a = b; + b = remainder; + } + return a || 1; +} + export function getHighestImportedVideoFps({ mediaAssets, }: { @@ -24,16 +78,18 @@ export function getRaisedProjectFpsForImportedMedia({ currentFps, importedAssets, }: { - currentFps: number; + currentFps: FrameRate; importedAssets: MediaAssetFpsInput[]; -}): number | null { +}): FrameRate | null { const highestImportedVideoFps = getHighestImportedVideoFps({ mediaAssets: importedAssets, }); - if (highestImportedVideoFps === null || highestImportedVideoFps <= currentFps) { + const currentFpsFloat = frameRateToFloat(currentFps); + + if (highestImportedVideoFps === null || highestImportedVideoFps <= currentFpsFloat) { return null; } - return highestImportedVideoFps; + return floatToFrameRate(highestImportedVideoFps); } diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts index d1025aa9..2fbea32e 100644 --- a/apps/web/src/lib/media/audio.ts +++ b/apps/web/src/lib/media/audio.ts @@ -21,6 +21,8 @@ import { mediaSupportsAudio } from "@/lib/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime"; import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + const MAX_AUDIO_CHANNELS = 2; const EXPORT_SAMPLE_RATE = 44100; const COARSE_SAMPLE_COUNT = 2048; @@ -122,10 +124,10 @@ export async function collectAudioElements({ return { timelineElement: element, buffer: audioBuffer, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, volume: resolveEffectiveAudioGain({ element, trackMuted: isTrackMuted, @@ -152,10 +154,10 @@ export async function collectAudioElements({ return { timelineElement: element, buffer: audioBuffer, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, volume: resolveEffectiveAudioGain({ element, trackMuted: isTrackMuted, @@ -337,16 +339,16 @@ async function fetchLibraryAudioSource({ type: "audio/mpeg", }); - return { - timelineElement: element, - file, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, - volume, - retime: element.retime, - }; + return { + timelineElement: element, + file, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, + volume, + retime: element.retime, + }; } catch (error) { console.warn("Failed to fetch library audio:", error); return null; @@ -404,10 +406,10 @@ function collectMediaAudioSource({ return { timelineElement: element, file: mediaAsset.file, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, volume, retime: element.retime, }; @@ -429,10 +431,10 @@ function collectMediaAudioClip({ id: element.id, sourceKey: mediaAsset.id, file: mediaAsset.file, - startTime: element.startTime, - duration: element.duration, - trimStart: element.trimStart, - trimEnd: element.trimEnd, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, volume, muted, retime: element.retime, @@ -602,7 +604,8 @@ export async function createTimelineAudioBuffer({ if (audioElements.length === 0) return null; const outputChannels = 2; - const outputLength = Math.ceil(duration * sampleRate); + const durationSeconds = duration / TICKS_PER_SECOND; + const outputLength = Math.ceil(durationSeconds * sampleRate); const outputBuffer = context.createBuffer( outputChannels, outputLength, diff --git a/apps/web/src/lib/project/types.ts b/apps/web/src/lib/project/types.ts index 8f45db9e..0a454e07 100644 --- a/apps/web/src/lib/project/types.ts +++ b/apps/web/src/lib/project/types.ts @@ -1,53 +1,54 @@ -import type { TScene } from "@/lib/timeline/types";
-
-export type TBackground =
- | {
- type: "color";
- color: string;
- }
- | {
- type: "blur";
- blurIntensity: number;
- };
-
-export interface TCanvasSize {
- width: number;
- height: number;
-}
-
-export interface TProjectMetadata {
- id: string;
- name: string;
- thumbnail?: string;
- duration: number;
- createdAt: Date;
- updatedAt: Date;
-}
-
-export interface TProjectSettings {
- fps: number;
- canvasSize: TCanvasSize;
- canvasSizeMode?: "preset" | "custom";
- lastCustomCanvasSize?: TCanvasSize | null;
- originalCanvasSize?: TCanvasSize | null;
- background: TBackground;
-}
-
-export interface TTimelineViewState {
- zoomLevel: number;
- scrollLeft: number;
- playheadTime: number;
-}
-
-export interface TProject {
- metadata: TProjectMetadata;
- scenes: TScene[];
- currentSceneId: string;
- settings: TProjectSettings;
- version: number;
- timelineViewState?: TTimelineViewState;
-}
-
-export type TProjectSortKey = "createdAt" | "updatedAt" | "name" | "duration";
-export type TSortOrder = "asc" | "desc";
-export type TProjectSortOption = `${TProjectSortKey}-${TSortOrder}`;
+import type { FrameRate } from "opencut-wasm"; +import type { TScene } from "@/lib/timeline/types"; + +export type TBackground = + | { + type: "color"; + color: string; + } + | { + type: "blur"; + blurIntensity: number; + }; + +export interface TCanvasSize { + width: number; + height: number; +} + +export interface TProjectMetadata { + id: string; + name: string; + thumbnail?: string; + duration: number; + createdAt: Date; + updatedAt: Date; +} + +export interface TProjectSettings { + fps: FrameRate; + canvasSize: TCanvasSize; + canvasSizeMode?: "preset" | "custom"; + lastCustomCanvasSize?: TCanvasSize | null; + originalCanvasSize?: TCanvasSize | null; + background: TBackground; +} + +export interface TTimelineViewState { + zoomLevel: number; + scrollLeft: number; + playheadTime: number; +} + +export interface TProject { + metadata: TProjectMetadata; + scenes: TScene[]; + currentSceneId: string; + settings: TProjectSettings; + version: number; + timelineViewState?: TTimelineViewState; +} + +export type TProjectSortKey = "createdAt" | "updatedAt" | "name" | "duration"; +export type TSortOrder = "asc" | "desc"; +export type TProjectSortOption = `${TProjectSortKey}-${TSortOrder}`; diff --git a/apps/web/src/lib/ripple/diff.ts b/apps/web/src/lib/ripple/diff.ts index 8a9730e1..22b8d996 100644 --- a/apps/web/src/lib/ripple/diff.ts +++ b/apps/web/src/lib/ripple/diff.ts @@ -1,4 +1,4 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; + import type { TimelineElement, TimelineTrack } from "@/lib/timeline/types"; import type { RippleAdjustment } from "./apply"; @@ -105,7 +105,7 @@ function collectTrackIntervals({ continue; } - if (beforeElement.endTime > afterElement.endTime + TIME_EPSILON_SECONDS) { + if (beforeElement.endTime > afterElement.endTime) { pushInterval({ intervals: vacatedIntervals, startTime: afterElement.endTime, @@ -141,7 +141,7 @@ function buildAdjustments({ }): RippleAdjustment[] { return intervals.flatMap((interval): RippleAdjustment[] => { const shiftAmount = interval.endTime - interval.startTime; - if (shiftAmount <= TIME_EPSILON_SECONDS) { + if (shiftAmount <= 0) { return []; } @@ -203,7 +203,7 @@ function normalizeIntervals({ const mergedIntervals: Interval[] = [{ ...sortedIntervals[0] }]; for (const interval of sortedIntervals.slice(1)) { const previousInterval = mergedIntervals[mergedIntervals.length - 1]; - if (interval.startTime <= previousInterval.endTime + TIME_EPSILON_SECONDS) { + if (interval.startTime <= previousInterval.endTime) { previousInterval.endTime = Math.max( previousInterval.endTime, interval.endTime, @@ -228,12 +228,10 @@ function subtractSingleInterval({ for (const overlappingInterval of overlappingIntervals) { remainingIntervals = remainingIntervals.flatMap((remainingInterval) => { - if ( - overlappingInterval.endTime <= - remainingInterval.startTime + TIME_EPSILON_SECONDS || - overlappingInterval.startTime >= - remainingInterval.endTime - TIME_EPSILON_SECONDS - ) { + if ( + overlappingInterval.endTime <= remainingInterval.startTime || + overlappingInterval.startTime >= remainingInterval.endTime + ) { return [remainingInterval]; } @@ -264,7 +262,7 @@ function pushInterval({ startTime, endTime, }: { intervals: Interval[]; startTime: number; endTime: number }): void { - if (endTime - startTime <= TIME_EPSILON_SECONDS) { + if (endTime <= startTime) { return; } diff --git a/apps/web/src/lib/timeline/audio-state.ts b/apps/web/src/lib/timeline/audio-state.ts index 28e381c2..9625c482 100644 --- a/apps/web/src/lib/timeline/audio-state.ts +++ b/apps/web/src/lib/timeline/audio-state.ts @@ -1,100 +1,102 @@ -import { hasKeyframesForPath } from "@/lib/animation/keyframe-query";
-import { resolveNumberAtTime } from "@/lib/animation/resolve";
-import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "./audio-constants";
-import type { TimelineElement } from "./types";
-const DEFAULT_STEP_SECONDS = 1 / 60;
-
-export type AudioCapableElement = Extract<
- TimelineElement,
- { type: "audio" | "video" }
->;
-
-export function clampDb(value: number): number {
- if (!Number.isFinite(value)) {
- return 0;
- }
-
- return Math.min(VOLUME_DB_MAX, Math.max(VOLUME_DB_MIN, value));
-}
-
-export function dBToLinear(db: number): number {
- return 10 ** (clampDb(db) / 20);
-}
-
-export function hasAnimatedVolume({
- element,
-}: {
- element: AudioCapableElement;
-}): boolean {
- return hasKeyframesForPath({
- animations: element.animations,
- propertyPath: "volume",
- });
-}
-
-export function resolveEffectiveAudioGain({
- element,
- trackMuted = false,
- localTime,
-}: {
- element: AudioCapableElement;
- trackMuted?: boolean;
- localTime: number;
-}): number {
- if (trackMuted || element.muted === true) {
- return 0;
- }
-
- const resolvedDb = resolveNumberAtTime({
- baseValue: element.volume ?? 0,
- animations: element.animations,
- propertyPath: "volume",
- localTime,
- });
-
- return dBToLinear(resolvedDb);
-}
-
-export function buildAudioGainAutomation({
- element,
- trackMuted = false,
- fromLocalTime,
- toLocalTime,
- stepSeconds = DEFAULT_STEP_SECONDS,
-}: {
- element: AudioCapableElement;
- trackMuted?: boolean;
- fromLocalTime: number;
- toLocalTime: number;
- stepSeconds?: number;
-}): Array<{ localTime: number; gain: number }> {
- const startTime = Math.max(0, fromLocalTime);
- const endTime = Math.max(startTime, toLocalTime);
- const safeStep =
- Number.isFinite(stepSeconds) && stepSeconds > 0
- ? stepSeconds
- : DEFAULT_STEP_SECONDS;
- const points: Array<{ localTime: number; gain: number }> = [];
-
- for (let localTime = startTime; localTime < endTime; localTime += safeStep) {
- points.push({
- localTime,
- gain: resolveEffectiveAudioGain({
- element,
- trackMuted,
- localTime,
- }),
- });
- }
-
- points.push({
- localTime: endTime,
- gain: resolveEffectiveAudioGain({
- element,
- trackMuted,
- localTime: endTime,
- }),
- });
-
- return points;
-}
+import { hasKeyframesForPath } from "@/lib/animation/keyframe-query"; +import { resolveNumberAtTime } from "@/lib/animation/resolve"; +import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "./audio-constants"; +import type { TimelineElement } from "./types"; +const DEFAULT_STEP_SECONDS = 1 / 60; + +export type AudioCapableElement = Extract< + TimelineElement, + { type: "audio" | "video" } +>; + +export function clampDb(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + + return Math.min(VOLUME_DB_MAX, Math.max(VOLUME_DB_MIN, value)); +} + +export function dBToLinear(db: number): number { + return 10 ** (clampDb(db) / 20); +} + +export function hasAnimatedVolume({ + element, +}: { + element: AudioCapableElement; +}): boolean { + return hasKeyframesForPath({ + animations: element.animations, + propertyPath: "volume", + }); +} + +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export function resolveEffectiveAudioGain({ + element, + trackMuted = false, + localTime, +}: { + element: AudioCapableElement; + trackMuted?: boolean; + localTime: number; +}): number { + if (trackMuted || element.muted === true) { + return 0; + } + + const resolvedDb = resolveNumberAtTime({ + baseValue: element.volume ?? 0, + animations: element.animations, + propertyPath: "volume", + localTime: Math.round(localTime * TICKS_PER_SECOND), + }); + + return dBToLinear(resolvedDb); +} + +export function buildAudioGainAutomation({ + element, + trackMuted = false, + fromLocalTime, + toLocalTime, + stepSeconds = DEFAULT_STEP_SECONDS, +}: { + element: AudioCapableElement; + trackMuted?: boolean; + fromLocalTime: number; + toLocalTime: number; + stepSeconds?: number; +}): Array<{ localTime: number; gain: number }> { + const startTime = Math.max(0, fromLocalTime); + const endTime = Math.max(startTime, toLocalTime); + const safeStep = + Number.isFinite(stepSeconds) && stepSeconds > 0 + ? stepSeconds + : DEFAULT_STEP_SECONDS; + const points: Array<{ localTime: number; gain: number }> = []; + + for (let localTime = startTime; localTime < endTime; localTime += safeStep) { + points.push({ + localTime, + gain: resolveEffectiveAudioGain({ + element, + trackMuted, + localTime, + }), + }); + } + + points.push({ + localTime: endTime, + gain: resolveEffectiveAudioGain({ + element, + trackMuted, + localTime: endTime, + }), + }); + + return points; +} diff --git a/apps/web/src/lib/timeline/bookmarks.ts b/apps/web/src/lib/timeline/bookmarks.ts index 70d1b5ee..1fbf17bf 100644 --- a/apps/web/src/lib/timeline/bookmarks.ts +++ b/apps/web/src/lib/timeline/bookmarks.ts @@ -1,8 +1,7 @@ import type { Bookmark } from "@/lib/timeline"; +import type { FrameRate } from "opencut-wasm"; import { roundToFrame } from "opencut-wasm"; -export const BOOKMARK_TIME_EPSILON = 0.001; - function bookmarkTimeEqual({ bookmarkTime, frameTime, @@ -10,7 +9,7 @@ function bookmarkTimeEqual({ bookmarkTime: number; frameTime: number; }): boolean { - return Math.abs(bookmarkTime - frameTime) < BOOKMARK_TIME_EPSILON; + return bookmarkTime === frameTime; } export function findBookmarkIndex({ @@ -112,9 +111,9 @@ export function getFrameTime({ fps, }: { time: number; - fps: number; + fps: FrameRate; }): number { - return roundToFrame({ time, fps }); + return roundToFrame({ time, rate: fps }) ?? time; } export function getBookmarkAtTime({ @@ -141,9 +140,6 @@ export function getBookmarksActiveAtTime({ bookmark.duration != null && bookmark.duration > 0 ? start + bookmark.duration : start; - return ( - time >= start - BOOKMARK_TIME_EPSILON && - time <= end + BOOKMARK_TIME_EPSILON - ); + return time >= start && time <= end; }); } diff --git a/apps/web/src/lib/timeline/creation.ts b/apps/web/src/lib/timeline/creation.ts index 335ab912..a339d806 100644 --- a/apps/web/src/lib/timeline/creation.ts +++ b/apps/web/src/lib/timeline/creation.ts @@ -1 +1,3 @@ -export const DEFAULT_NEW_ELEMENT_DURATION_SECONDS = 5; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export const DEFAULT_NEW_ELEMENT_DURATION = 5 * TICKS_PER_SECOND; diff --git a/apps/web/src/lib/timeline/defaults.ts b/apps/web/src/lib/timeline/defaults.ts index bc968a3c..cf452800 100644 --- a/apps/web/src/lib/timeline/defaults.ts +++ b/apps/web/src/lib/timeline/defaults.ts @@ -1,77 +1,77 @@ -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation";
-import type { TTimelineViewState } from "@/lib/project/types";
-import type { BlendMode, Transform } from "@/lib/rendering";
-import type { TextElement } from "./types";
-
-const defaultTransform: Transform = {
- scaleX: 1,
- scaleY: 1,
- position: { x: 0, y: 0 },
- rotate: 0,
-};
-
-const defaultOpacity = 1;
-const defaultBlendMode: BlendMode = "normal";
-const defaultVolume = 0;
-
-const defaultTextLetterSpacing = 0;
-const defaultTextLineHeight = 1.2;
-
-const defaultTextBackground = {
- enabled: false,
- color: "#000000",
- cornerRadius: 0,
- paddingX: 30,
- paddingY: 42,
- offsetX: 0,
- offsetY: 0,
-};
-
-const defaultTextElement: Omit<TextElement, "id"> = {
- type: "text",
- name: "Text",
- content: "Default text",
- fontSize: 15,
- fontFamily: "Arial",
- color: "#ffffff",
- background: { ...defaultTextBackground },
- textAlign: "center",
- fontWeight: "normal",
- fontStyle: "normal",
- textDecoration: "none",
- letterSpacing: defaultTextLetterSpacing,
- lineHeight: defaultTextLineHeight,
- duration: DEFAULT_NEW_ELEMENT_DURATION_SECONDS,
- startTime: 0,
- trimStart: 0,
- trimEnd: 0,
- transform: {
- ...defaultTransform,
- position: { ...defaultTransform.position },
- },
- opacity: defaultOpacity,
-};
-
-const defaultTimelineViewState: TTimelineViewState = {
- zoomLevel: 1,
- scrollLeft: 0,
- playheadTime: 0,
-};
-
-export const DEFAULTS = {
- element: {
- transform: defaultTransform,
- opacity: defaultOpacity,
- blendMode: defaultBlendMode,
- volume: defaultVolume,
- },
- text: {
- letterSpacing: defaultTextLetterSpacing,
- lineHeight: defaultTextLineHeight,
- background: defaultTextBackground,
- element: defaultTextElement,
- },
- timeline: {
- viewState: defaultTimelineViewState,
- },
-};
+import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import type { TTimelineViewState } from "@/lib/project/types"; +import type { BlendMode, Transform } from "@/lib/rendering"; +import type { TextElement } from "./types"; + +const defaultTransform: Transform = { + scaleX: 1, + scaleY: 1, + position: { x: 0, y: 0 }, + rotate: 0, +}; + +const defaultOpacity = 1; +const defaultBlendMode: BlendMode = "normal"; +const defaultVolume = 0; + +const defaultTextLetterSpacing = 0; +const defaultTextLineHeight = 1.2; + +const defaultTextBackground = { + enabled: false, + color: "#000000", + cornerRadius: 0, + paddingX: 30, + paddingY: 42, + offsetX: 0, + offsetY: 0, +}; + +const defaultTextElement: Omit<TextElement, "id"> = { + type: "text", + name: "Text", + content: "Default text", + fontSize: 15, + fontFamily: "Arial", + color: "#ffffff", + background: { ...defaultTextBackground }, + textAlign: "center", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + letterSpacing: defaultTextLetterSpacing, + lineHeight: defaultTextLineHeight, + duration: DEFAULT_NEW_ELEMENT_DURATION, + startTime: 0, + trimStart: 0, + trimEnd: 0, + transform: { + ...defaultTransform, + position: { ...defaultTransform.position }, + }, + opacity: defaultOpacity, +}; + +const defaultTimelineViewState: TTimelineViewState = { + zoomLevel: 1, + scrollLeft: 0, + playheadTime: 0, +}; + +export const DEFAULTS = { + element: { + transform: defaultTransform, + opacity: defaultOpacity, + blendMode: defaultBlendMode, + volume: defaultVolume, + }, + text: { + letterSpacing: defaultTextLetterSpacing, + lineHeight: defaultTextLineHeight, + background: defaultTextBackground, + element: defaultTextElement, + }, + timeline: { + viewState: defaultTimelineViewState, + }, +}; diff --git a/apps/web/src/lib/timeline/drag-utils.ts b/apps/web/src/lib/timeline/drag-utils.ts index 8efeab79..35d00613 100644 --- a/apps/web/src/lib/timeline/drag-utils.ts +++ b/apps/web/src/lib/timeline/drag-utils.ts @@ -1,19 +1,21 @@ -import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale";
-
-export function getMouseTimeFromClientX({
- clientX,
- containerRect,
- zoomLevel,
- scrollLeft,
-}: {
- clientX: number;
- containerRect: DOMRect;
- zoomLevel: number;
- scrollLeft: number;
-}): number {
- const mouseX = clientX - containerRect.left + scrollLeft;
- return Math.max(
- 0,
- mouseX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel),
- );
-}
+import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export function getMouseTimeFromClientX({ + clientX, + containerRect, + zoomLevel, + scrollLeft, +}: { + clientX: number; + containerRect: DOMRect; + zoomLevel: number; + scrollLeft: number; +}): number { + const mouseX = clientX - containerRect.left + scrollLeft; + const seconds = Math.max( + 0, + mouseX / (BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel), + ); + return Math.round(seconds * TICKS_PER_SECOND); +} diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.ts index 96588903..6f9e9dbc 100644 --- a/apps/web/src/lib/timeline/element-utils.ts +++ b/apps/web/src/lib/timeline/element-utils.ts @@ -1,418 +1,418 @@ -import { DEFAULT_NEW_ELEMENT_DURATION_SECONDS } from "@/lib/timeline/creation";
-import {
- MASKABLE_ELEMENT_TYPES,
- RETIMABLE_ELEMENT_TYPES,
- VISUAL_ELEMENT_TYPES,
- type CreateEffectElement,
- type CreateGraphicElement,
- type CreateTimelineElement,
- type CreateVideoElement,
- type CreateImageElement,
- type CreateStickerElement,
- type CreateUploadAudioElement,
- type CreateLibraryAudioElement,
- type TextBackground,
- type TextElement,
- type TimelineElement,
- type TimelineTrack,
- type AudioElement,
- type VideoElement,
- type ImageElement,
- type MaskableElement,
- type RetimableElement,
- type VisualElement,
- type UploadAudioElement,
-} from "@/lib/timeline";
-import { DEFAULTS } from "@/lib/timeline/defaults";
-import type { MediaType } from "@/lib/media/types";
-import { buildDefaultEffectInstance } from "@/lib/effects";
-import { buildDefaultGraphicInstance } from "@/lib/graphics";
-import type { ParamValues } from "@/lib/params";
-import { capitalizeFirstLetter } from "@/utils/string";
-
-export function canElementHaveAudio(
- element: TimelineElement,
-): element is AudioElement | VideoElement {
- return element.type === "audio" || element.type === "video";
-}
-
-export function isVisualElement(
- element: TimelineElement,
-): element is VisualElement {
- return (VISUAL_ELEMENT_TYPES as readonly string[]).includes(element.type);
-}
-
-export function isMaskableElement(
- element: TimelineElement,
-): element is MaskableElement {
- return (MASKABLE_ELEMENT_TYPES as readonly string[]).includes(element.type);
-}
-
-export function isRetimableElement(
- element: TimelineElement,
-): element is RetimableElement {
- return (RETIMABLE_ELEMENT_TYPES as readonly string[]).includes(element.type);
-}
-
-export function canElementBeHidden(
- element: TimelineElement,
-): element is VisualElement {
- return isVisualElement(element);
-}
-
-export function hasElementEffects({
- element,
-}: {
- element: TimelineElement;
-}): boolean {
- return isVisualElement(element) && (element.effects?.length ?? 0) > 0;
-}
-
-export function hasMediaId(
- element: TimelineElement,
-): element is UploadAudioElement | VideoElement | ImageElement {
- return "mediaId" in element;
-}
-
-export function requiresMediaId({
- element,
-}: {
- element: CreateTimelineElement;
-}): boolean {
- return (
- element.type === "video" ||
- element.type === "image" ||
- (element.type === "audio" && element.sourceType === "upload")
- );
-}
-
-function buildTextBackground(
- raw: Partial<TextBackground> | undefined,
-): TextBackground {
- const color = raw?.color ?? DEFAULTS.text.element.background.color;
- const enabled = raw?.enabled ?? color !== "transparent";
- return {
- enabled,
- color,
- cornerRadius: raw?.cornerRadius,
- paddingX: raw?.paddingX,
- paddingY: raw?.paddingY,
- offsetX: raw?.offsetX,
- offsetY: raw?.offsetY,
- };
-}
-
-export function buildTextElement({
- raw,
- startTime,
-}: {
- raw: Partial<Omit<TextElement, "type" | "id">>;
- startTime: number;
-}): CreateTimelineElement {
- const t = raw as Partial<TextElement>;
-
- return {
- type: "text",
- name: t.name ?? DEFAULTS.text.element.name,
- content: t.content ?? DEFAULTS.text.element.content,
- duration: t.duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS,
- startTime,
- trimStart: 0,
- trimEnd: 0,
- fontSize: t.fontSize ?? DEFAULTS.text.element.fontSize,
- fontFamily: t.fontFamily ?? DEFAULTS.text.element.fontFamily,
- color: t.color ?? DEFAULTS.text.element.color,
- background: buildTextBackground(t.background),
- textAlign: t.textAlign ?? DEFAULTS.text.element.textAlign,
- fontWeight: t.fontWeight ?? DEFAULTS.text.element.fontWeight,
- fontStyle: t.fontStyle ?? DEFAULTS.text.element.fontStyle,
- textDecoration: t.textDecoration ?? DEFAULTS.text.element.textDecoration,
- letterSpacing: t.letterSpacing ?? DEFAULTS.text.element.letterSpacing,
- lineHeight: t.lineHeight ?? DEFAULTS.text.element.lineHeight,
- transform: t.transform ?? DEFAULTS.text.element.transform,
- opacity: t.opacity ?? DEFAULTS.text.element.opacity,
- blendMode: t.blendMode ?? DEFAULTS.element.blendMode,
- };
-}
-
-export function buildEffectElement({
- effectType,
- startTime,
- duration,
-}: {
- effectType: string;
- startTime: number;
- duration?: number;
-}): CreateEffectElement {
- const instance = buildDefaultEffectInstance({ effectType });
- return {
- type: "effect",
- name: capitalizeFirstLetter({ string: instance.type }),
- effectType,
- params: instance.params,
- duration: duration ?? DEFAULT_NEW_ELEMENT_DURATION_SECONDS,
- startTime,
- trimStart: 0,
- trimEnd: 0,
- };
-}
-
-export function buildStickerElement({
- stickerId,
- name,
- startTime,
- intrinsicWidth,
- intrinsicHeight,
-}: {
- stickerId: string;
- name?: string;
- startTime: number;
- intrinsicWidth?: number;
- intrinsicHeight?: number;
-}): CreateStickerElement {
- const stickerNameFromId =
- stickerId.split(":").slice(1).pop()?.replaceAll("-", " ") ?? stickerId;
- return {
- type: "sticker",
- name: name ?? stickerNameFromId,
- stickerId,
- intrinsicWidth,
- intrinsicHeight,
- duration: DEFAULT_NEW_ELEMENT_DURATION_SECONDS,
- startTime,
- trimStart: 0,
- trimEnd: 0,
- transform: {
- ...DEFAULTS.element.transform,
- position: { ...DEFAULTS.element.transform.position },
- },
- opacity: DEFAULTS.element.opacity,
- blendMode: DEFAULTS.element.blendMode,
- };
-}
-
-export function buildGraphicElement({
- definitionId,
- name,
- startTime,
- params,
-}: {
- definitionId: string;
- name?: string;
- startTime: number;
- params?: Partial<ParamValues>;
-}): CreateGraphicElement {
- const instance = buildDefaultGraphicInstance({ definitionId });
- return {
- type: "graphic",
- name: name ?? capitalizeFirstLetter({ string: instance.definitionId }),
- definitionId: instance.definitionId,
- params: { ...instance.params, ...(params ?? {}) } as ParamValues,
- duration: DEFAULT_NEW_ELEMENT_DURATION_SECONDS,
- startTime,
- trimStart: 0,
- trimEnd: 0,
- transform: {
- ...DEFAULTS.element.transform,
- position: { ...DEFAULTS.element.transform.position },
- },
- opacity: DEFAULTS.element.opacity,
- blendMode: DEFAULTS.element.blendMode,
- };
-}
-
-function buildVideoElement({
- mediaId,
- name,
- duration,
- startTime,
-}: {
- mediaId: string;
- name: string;
- duration: number;
- startTime: number;
-}): CreateVideoElement {
- return {
- type: "video",
- mediaId,
- name,
- duration,
- startTime,
- trimStart: 0,
- trimEnd: 0,
- sourceDuration: duration,
- muted: false,
- isSourceAudioEnabled: true,
- hidden: false,
- transform: {
- ...DEFAULTS.element.transform,
- position: { ...DEFAULTS.element.transform.position },
- },
- opacity: DEFAULTS.element.opacity,
- blendMode: DEFAULTS.element.blendMode,
- volume: DEFAULTS.element.volume,
- };
-}
-
-function buildImageElement({
- mediaId,
- name,
- duration,
- startTime,
-}: {
- mediaId: string;
- name: string;
- duration: number;
- startTime: number;
-}): CreateImageElement {
- return {
- type: "image",
- mediaId,
- name,
- duration,
- startTime,
- trimStart: 0,
- trimEnd: 0,
- hidden: false,
- transform: {
- ...DEFAULTS.element.transform,
- position: { ...DEFAULTS.element.transform.position },
- },
- opacity: DEFAULTS.element.opacity,
- blendMode: DEFAULTS.element.blendMode,
- };
-}
-
-function buildUploadAudioElement({
- mediaId,
- name,
- duration,
- startTime,
- buffer,
-}: {
- mediaId: string;
- name: string;
- duration: number;
- startTime: number;
- buffer?: AudioBuffer;
-}): CreateUploadAudioElement {
- const element: CreateUploadAudioElement = {
- type: "audio",
- sourceType: "upload",
- mediaId,
- name,
- duration,
- startTime,
- trimStart: 0,
- trimEnd: 0,
- sourceDuration: duration,
- volume: DEFAULTS.element.volume,
- muted: false,
- };
- if (buffer) {
- element.buffer = buffer;
- }
- return element;
-}
-
-export function buildElementFromMedia({
- mediaId,
- mediaType,
- name,
- duration,
- startTime,
- buffer,
-}: {
- mediaId: string;
- mediaType: MediaType;
- name: string;
- duration: number;
- startTime: number;
- buffer?: AudioBuffer;
-}): CreateTimelineElement {
- switch (mediaType) {
- case "audio":
- return buildUploadAudioElement({
- mediaId,
- name,
- duration,
- startTime,
- buffer,
- });
- case "video":
- return buildVideoElement({ mediaId, name, duration, startTime });
- case "image":
- return buildImageElement({ mediaId, name, duration, startTime });
- }
-}
-
-export function buildLibraryAudioElement({
- sourceUrl,
- name,
- duration,
- startTime,
- buffer,
-}: {
- sourceUrl: string;
- name: string;
- duration: number;
- startTime: number;
- buffer?: AudioBuffer;
-}): CreateLibraryAudioElement {
- const element: CreateLibraryAudioElement = {
- type: "audio",
- sourceType: "library",
- sourceUrl,
- name,
- duration,
- startTime,
- trimStart: 0,
- trimEnd: 0,
- sourceDuration: duration,
- volume: DEFAULTS.element.volume,
- muted: false,
- };
- if (buffer) {
- element.buffer = buffer;
- }
- return element;
-}
-
-export function getElementsAtTime({
- tracks,
- time,
-}: {
- tracks: TimelineTrack[];
- time: number;
-}): { trackId: string; elementId: string }[] {
- const result: { trackId: string; elementId: string }[] = [];
-
- for (const track of tracks) {
- for (const element of track.elements) {
- const elementStart = element.startTime;
- const elementEnd = element.startTime + element.duration;
-
- if (time > elementStart && time < elementEnd) {
- result.push({ trackId: track.id, elementId: element.id });
- }
- }
- }
-
- return result;
-}
-
-export function getElementFontFamilies({
- tracks,
-}: {
- tracks: TimelineTrack[];
-}): string[] {
- const families = new Set<string>();
- for (const track of tracks) {
- for (const element of track.elements) {
- if (element.type === "text" && element.fontFamily) {
- families.add(element.fontFamily);
- }
- }
- }
- return [...families];
-}
+import { DEFAULT_NEW_ELEMENT_DURATION } from "@/lib/timeline/creation"; +import { + MASKABLE_ELEMENT_TYPES, + RETIMABLE_ELEMENT_TYPES, + VISUAL_ELEMENT_TYPES, + type CreateEffectElement, + type CreateGraphicElement, + type CreateTimelineElement, + type CreateVideoElement, + type CreateImageElement, + type CreateStickerElement, + type CreateUploadAudioElement, + type CreateLibraryAudioElement, + type TextBackground, + type TextElement, + type TimelineElement, + type TimelineTrack, + type AudioElement, + type VideoElement, + type ImageElement, + type MaskableElement, + type RetimableElement, + type VisualElement, + type UploadAudioElement, +} from "@/lib/timeline"; +import { DEFAULTS } from "@/lib/timeline/defaults"; +import type { MediaType } from "@/lib/media/types"; +import { buildDefaultEffectInstance } from "@/lib/effects"; +import { buildDefaultGraphicInstance } from "@/lib/graphics"; +import type { ParamValues } from "@/lib/params"; +import { capitalizeFirstLetter } from "@/utils/string"; + +export function canElementHaveAudio( + element: TimelineElement, +): element is AudioElement | VideoElement { + return element.type === "audio" || element.type === "video"; +} + +export function isVisualElement( + element: TimelineElement, +): element is VisualElement { + return (VISUAL_ELEMENT_TYPES as readonly string[]).includes(element.type); +} + +export function isMaskableElement( + element: TimelineElement, +): element is MaskableElement { + return (MASKABLE_ELEMENT_TYPES as readonly string[]).includes(element.type); +} + +export function isRetimableElement( + element: TimelineElement, +): element is RetimableElement { + return (RETIMABLE_ELEMENT_TYPES as readonly string[]).includes(element.type); +} + +export function canElementBeHidden( + element: TimelineElement, +): element is VisualElement { + return isVisualElement(element); +} + +export function hasElementEffects({ + element, +}: { + element: TimelineElement; +}): boolean { + return isVisualElement(element) && (element.effects?.length ?? 0) > 0; +} + +export function hasMediaId( + element: TimelineElement, +): element is UploadAudioElement | VideoElement | ImageElement { + return "mediaId" in element; +} + +export function requiresMediaId({ + element, +}: { + element: CreateTimelineElement; +}): boolean { + return ( + element.type === "video" || + element.type === "image" || + (element.type === "audio" && element.sourceType === "upload") + ); +} + +function buildTextBackground( + raw: Partial<TextBackground> | undefined, +): TextBackground { + const color = raw?.color ?? DEFAULTS.text.element.background.color; + const enabled = raw?.enabled ?? color !== "transparent"; + return { + enabled, + color, + cornerRadius: raw?.cornerRadius, + paddingX: raw?.paddingX, + paddingY: raw?.paddingY, + offsetX: raw?.offsetX, + offsetY: raw?.offsetY, + }; +} + +export function buildTextElement({ + raw, + startTime, +}: { + raw: Partial<Omit<TextElement, "type" | "id">>; + startTime: number; +}): CreateTimelineElement { + const t = raw as Partial<TextElement>; + + return { + type: "text", + name: t.name ?? DEFAULTS.text.element.name, + content: t.content ?? DEFAULTS.text.element.content, + duration: t.duration ?? DEFAULT_NEW_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + fontSize: t.fontSize ?? DEFAULTS.text.element.fontSize, + fontFamily: t.fontFamily ?? DEFAULTS.text.element.fontFamily, + color: t.color ?? DEFAULTS.text.element.color, + background: buildTextBackground(t.background), + textAlign: t.textAlign ?? DEFAULTS.text.element.textAlign, + fontWeight: t.fontWeight ?? DEFAULTS.text.element.fontWeight, + fontStyle: t.fontStyle ?? DEFAULTS.text.element.fontStyle, + textDecoration: t.textDecoration ?? DEFAULTS.text.element.textDecoration, + letterSpacing: t.letterSpacing ?? DEFAULTS.text.element.letterSpacing, + lineHeight: t.lineHeight ?? DEFAULTS.text.element.lineHeight, + transform: t.transform ?? DEFAULTS.text.element.transform, + opacity: t.opacity ?? DEFAULTS.text.element.opacity, + blendMode: t.blendMode ?? DEFAULTS.element.blendMode, + }; +} + +export function buildEffectElement({ + effectType, + startTime, + duration, +}: { + effectType: string; + startTime: number; + duration?: number; +}): CreateEffectElement { + const instance = buildDefaultEffectInstance({ effectType }); + return { + type: "effect", + name: capitalizeFirstLetter({ string: instance.type }), + effectType, + params: instance.params, + duration: duration ?? DEFAULT_NEW_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + }; +} + +export function buildStickerElement({ + stickerId, + name, + startTime, + intrinsicWidth, + intrinsicHeight, +}: { + stickerId: string; + name?: string; + startTime: number; + intrinsicWidth?: number; + intrinsicHeight?: number; +}): CreateStickerElement { + const stickerNameFromId = + stickerId.split(":").slice(1).pop()?.replaceAll("-", " ") ?? stickerId; + return { + type: "sticker", + name: name ?? stickerNameFromId, + stickerId, + intrinsicWidth, + intrinsicHeight, + duration: DEFAULT_NEW_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + transform: { + ...DEFAULTS.element.transform, + position: { ...DEFAULTS.element.transform.position }, + }, + opacity: DEFAULTS.element.opacity, + blendMode: DEFAULTS.element.blendMode, + }; +} + +export function buildGraphicElement({ + definitionId, + name, + startTime, + params, +}: { + definitionId: string; + name?: string; + startTime: number; + params?: Partial<ParamValues>; +}): CreateGraphicElement { + const instance = buildDefaultGraphicInstance({ definitionId }); + return { + type: "graphic", + name: name ?? capitalizeFirstLetter({ string: instance.definitionId }), + definitionId: instance.definitionId, + params: { ...instance.params, ...(params ?? {}) } as ParamValues, + duration: DEFAULT_NEW_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + transform: { + ...DEFAULTS.element.transform, + position: { ...DEFAULTS.element.transform.position }, + }, + opacity: DEFAULTS.element.opacity, + blendMode: DEFAULTS.element.blendMode, + }; +} + +function buildVideoElement({ + mediaId, + name, + duration, + startTime, +}: { + mediaId: string; + name: string; + duration: number; + startTime: number; +}): CreateVideoElement { + return { + type: "video", + mediaId, + name, + duration, + startTime, + trimStart: 0, + trimEnd: 0, + sourceDuration: duration, + muted: false, + isSourceAudioEnabled: true, + hidden: false, + transform: { + ...DEFAULTS.element.transform, + position: { ...DEFAULTS.element.transform.position }, + }, + opacity: DEFAULTS.element.opacity, + blendMode: DEFAULTS.element.blendMode, + volume: DEFAULTS.element.volume, + }; +} + +function buildImageElement({ + mediaId, + name, + duration, + startTime, +}: { + mediaId: string; + name: string; + duration: number; + startTime: number; +}): CreateImageElement { + return { + type: "image", + mediaId, + name, + duration, + startTime, + trimStart: 0, + trimEnd: 0, + hidden: false, + transform: { + ...DEFAULTS.element.transform, + position: { ...DEFAULTS.element.transform.position }, + }, + opacity: DEFAULTS.element.opacity, + blendMode: DEFAULTS.element.blendMode, + }; +} + +function buildUploadAudioElement({ + mediaId, + name, + duration, + startTime, + buffer, +}: { + mediaId: string; + name: string; + duration: number; + startTime: number; + buffer?: AudioBuffer; +}): CreateUploadAudioElement { + const element: CreateUploadAudioElement = { + type: "audio", + sourceType: "upload", + mediaId, + name, + duration, + startTime, + trimStart: 0, + trimEnd: 0, + sourceDuration: duration, + volume: DEFAULTS.element.volume, + muted: false, + }; + if (buffer) { + element.buffer = buffer; + } + return element; +} + +export function buildElementFromMedia({ + mediaId, + mediaType, + name, + duration, + startTime, + buffer, +}: { + mediaId: string; + mediaType: MediaType; + name: string; + duration: number; + startTime: number; + buffer?: AudioBuffer; +}): CreateTimelineElement { + switch (mediaType) { + case "audio": + return buildUploadAudioElement({ + mediaId, + name, + duration, + startTime, + buffer, + }); + case "video": + return buildVideoElement({ mediaId, name, duration, startTime }); + case "image": + return buildImageElement({ mediaId, name, duration, startTime }); + } +} + +export function buildLibraryAudioElement({ + sourceUrl, + name, + duration, + startTime, + buffer, +}: { + sourceUrl: string; + name: string; + duration: number; + startTime: number; + buffer?: AudioBuffer; +}): CreateLibraryAudioElement { + const element: CreateLibraryAudioElement = { + type: "audio", + sourceType: "library", + sourceUrl, + name, + duration, + startTime, + trimStart: 0, + trimEnd: 0, + sourceDuration: duration, + volume: DEFAULTS.element.volume, + muted: false, + }; + if (buffer) { + element.buffer = buffer; + } + return element; +} + +export function getElementsAtTime({ + tracks, + time, +}: { + tracks: TimelineTrack[]; + time: number; +}): { trackId: string; elementId: string }[] { + const result: { trackId: string; elementId: string }[] = []; + + for (const track of tracks) { + for (const element of track.elements) { + const elementStart = element.startTime; + const elementEnd = element.startTime + element.duration; + + if (time > elementStart && time < elementEnd) { + result.push({ trackId: track.id, elementId: element.id }); + } + } + } + + return result; +} + +export function getElementFontFamilies({ + tracks, +}: { + tracks: TimelineTrack[]; +}): string[] { + const families = new Set<string>(); + for (const track of tracks) { + for (const element of track.elements) { + if (element.type === "text" && element.fontFamily) { + families.add(element.fontFamily); + } + } + } + return [...families]; +} diff --git a/apps/web/src/lib/timeline/pixel-utils.ts b/apps/web/src/lib/timeline/pixel-utils.ts index e93b5ab5..10f258aa 100644 --- a/apps/web/src/lib/timeline/pixel-utils.ts +++ b/apps/web/src/lib/timeline/pixel-utils.ts @@ -1,79 +1,80 @@ -import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale";
-
-export const TIMELINE_INDICATOR_LINE_WIDTH_PX = 2;
-
-function getDevicePixelRatio({
- devicePixelRatio,
-}: {
- devicePixelRatio?: number;
-}): number {
- if (
- typeof devicePixelRatio === "number" &&
- Number.isFinite(devicePixelRatio) &&
- devicePixelRatio > 0
- ) {
- return devicePixelRatio;
- }
-
- if (typeof window === "undefined") {
- return 1;
- }
-
- if (Number.isFinite(window.devicePixelRatio) && window.devicePixelRatio > 0) {
- return window.devicePixelRatio;
- }
-
- return 1;
-}
-
-export function getTimelinePixelsPerSecond({
- zoomLevel,
-}: {
- zoomLevel: number;
-}): number {
- return BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel;
-}
-
-export function timelineTimeToPixels({
- time,
- zoomLevel,
-}: {
- time: number;
- zoomLevel: number;
-}): number {
- return time * getTimelinePixelsPerSecond({ zoomLevel });
-}
-
-export function snapPixelToDeviceGrid({
- pixel,
- devicePixelRatio,
-}: {
- pixel: number;
- devicePixelRatio?: number;
-}): number {
- const safeDevicePixelRatio = getDevicePixelRatio({ devicePixelRatio });
- return Math.round(pixel * safeDevicePixelRatio) / safeDevicePixelRatio;
-}
-
-export function timelineTimeToSnappedPixels({
- time,
- zoomLevel,
- devicePixelRatio,
-}: {
- time: number;
- zoomLevel: number;
- devicePixelRatio?: number;
-}): number {
- const rawPixel = timelineTimeToPixels({ time, zoomLevel });
- return snapPixelToDeviceGrid({ pixel: rawPixel, devicePixelRatio });
-}
-
-export function getCenteredLineLeft({
- centerPixel,
- lineWidthPx = TIMELINE_INDICATOR_LINE_WIDTH_PX,
-}: {
- centerPixel: number;
- lineWidthPx?: number;
-}): number {
- return centerPixel - lineWidthPx / 2;
-}
+import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export const TIMELINE_INDICATOR_LINE_WIDTH_PX = 2; + +function getDevicePixelRatio({ + devicePixelRatio, +}: { + devicePixelRatio?: number; +}): number { + if ( + typeof devicePixelRatio === "number" && + Number.isFinite(devicePixelRatio) && + devicePixelRatio > 0 + ) { + return devicePixelRatio; + } + + if (typeof window === "undefined") { + return 1; + } + + if (Number.isFinite(window.devicePixelRatio) && window.devicePixelRatio > 0) { + return window.devicePixelRatio; + } + + return 1; +} + +export function getTimelinePixelsPerSecond({ + zoomLevel, +}: { + zoomLevel: number; +}): number { + return BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; +} + +export function timelineTimeToPixels({ + time, + zoomLevel, +}: { + time: number; + zoomLevel: number; +}): number { + return (time / TICKS_PER_SECOND) * getTimelinePixelsPerSecond({ zoomLevel }); +} + +export function snapPixelToDeviceGrid({ + pixel, + devicePixelRatio, +}: { + pixel: number; + devicePixelRatio?: number; +}): number { + const safeDevicePixelRatio = getDevicePixelRatio({ devicePixelRatio }); + return Math.round(pixel * safeDevicePixelRatio) / safeDevicePixelRatio; +} + +export function timelineTimeToSnappedPixels({ + time, + zoomLevel, + devicePixelRatio, +}: { + time: number; + zoomLevel: number; + devicePixelRatio?: number; +}): number { + const rawPixel = timelineTimeToPixels({ time, zoomLevel }); + return snapPixelToDeviceGrid({ pixel: rawPixel, devicePixelRatio }); +} + +export function getCenteredLineLeft({ + centerPixel, + lineWidthPx = TIMELINE_INDICATOR_LINE_WIDTH_PX, +}: { + centerPixel: number; + lineWidthPx?: number; +}): number { + return centerPixel - lineWidthPx / 2; +} diff --git a/apps/web/src/lib/timeline/ruler-utils.ts b/apps/web/src/lib/timeline/ruler-utils.ts index 732d9ea4..ea2982aa 100644 --- a/apps/web/src/lib/timeline/ruler-utils.ts +++ b/apps/web/src/lib/timeline/ruler-utils.ts @@ -1,4 +1,6 @@ +import type { FrameRate } from "opencut-wasm"; import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { frameRateToFloat } from "@/lib/fps/utils"; /** * frame intervals for labels - starts at 2 so there's always at least @@ -54,15 +56,16 @@ export function getRulerConfig({ fps, }: { zoomLevel: number; - fps: number; + fps: FrameRate; }): RulerConfig { + const fpsFloat = frameRateToFloat(fps); const pixelsPerSecond = BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; - const pixelsPerFrame = pixelsPerSecond / fps; + const pixelsPerFrame = pixelsPerSecond / fpsFloat; const labelIntervalSeconds = findOptimalInterval({ pixelsPerFrame, pixelsPerSecond, - fps, + fps: fpsFloat, minSpacingPx: MIN_LABEL_SPACING_PX, frameIntervals: LABEL_FRAME_INTERVALS, }); @@ -70,7 +73,7 @@ export function getRulerConfig({ const rawTickIntervalSeconds = findOptimalInterval({ pixelsPerFrame, pixelsPerSecond, - fps, + fps: fpsFloat, minSpacingPx: MIN_TICK_SPACING_PX, frameIntervals: TICK_FRAME_INTERVALS, }); @@ -81,7 +84,7 @@ export function getRulerConfig({ labelIntervalSeconds, pixelsPerFrame, pixelsPerSecond, - fps, + fps: fpsFloat, }); return { labelIntervalSeconds, tickIntervalSeconds }; @@ -197,13 +200,13 @@ export function formatRulerLabel({ fps, }: { timeInSeconds: number; - fps: number; + fps: FrameRate; }): string { if (isSecondBoundary({ timeInSeconds })) { return formatTimestamp({ timeInSeconds }); } - const frameWithinSecond = getFrameWithinSecond({ timeInSeconds, fps }); + const frameWithinSecond = getFrameWithinSecond({ timeInSeconds, fps: frameRateToFloat(fps) }); return `${frameWithinSecond}f`; } diff --git a/apps/web/src/lib/timeline/snap-utils.ts b/apps/web/src/lib/timeline/snap-utils.ts index ef5d6e11..2e7bdcec 100644 --- a/apps/web/src/lib/timeline/snap-utils.ts +++ b/apps/web/src/lib/timeline/snap-utils.ts @@ -1,172 +1,172 @@ -import type { Bookmark, TimelineTrack } from "@/lib/timeline";
-import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale";
-import { BOOKMARK_TIME_EPSILON } from "@/lib/timeline/bookmarks";
-import { getElementKeyframes } from "@/lib/animation";
-
-export interface SnapPoint {
- time: number;
- type: "element-start" | "element-end" | "playhead" | "bookmark" | "keyframe";
- elementId?: string;
- trackId?: string;
-}
-
-export interface SnapResult {
- snappedTime: number;
- snapPoint: SnapPoint | null;
- snapDistance: number;
-}
-
-const DEFAULT_SNAP_THRESHOLD_PX = 10;
-
-export function findSnapPoints({
- tracks,
- playheadTime,
- excludeElementId,
- bookmarks = [],
- excludeBookmarkTime,
- enableElementSnapping = true,
- enablePlayheadSnapping = true,
- enableBookmarkSnapping = true,
- enableKeyframeSnapping = true,
-}: {
- tracks: Array<TimelineTrack>;
- playheadTime: number;
- excludeElementId?: string;
- bookmarks?: Array<Bookmark>;
- excludeBookmarkTime?: number;
- enableElementSnapping?: boolean;
- enablePlayheadSnapping?: boolean;
- enableBookmarkSnapping?: boolean;
- enableKeyframeSnapping?: boolean;
-}): SnapPoint[] {
- const snapPoints: SnapPoint[] = [];
-
- for (const track of tracks) {
- for (const element of track.elements) {
- if (element.id === excludeElementId) continue;
-
- if (enableElementSnapping) {
- snapPoints.push(
- {
- time: element.startTime,
- type: "element-start",
- elementId: element.id,
- trackId: track.id,
- },
- {
- time: element.startTime + element.duration,
- type: "element-end",
- elementId: element.id,
- trackId: track.id,
- },
- );
- }
-
- if (enableKeyframeSnapping) {
- for (const keyframe of getElementKeyframes({
- animations: element.animations,
- })) {
- snapPoints.push({
- time: element.startTime + keyframe.time,
- type: "keyframe",
- elementId: element.id,
- trackId: track.id,
- });
- }
- }
- }
- }
-
- if (enablePlayheadSnapping) {
- snapPoints.push({ time: playheadTime, type: "playhead" });
- }
-
- if (enableBookmarkSnapping) {
- for (const bookmark of bookmarks) {
- if (
- excludeBookmarkTime != null &&
- Math.abs(bookmark.time - excludeBookmarkTime) < BOOKMARK_TIME_EPSILON
- ) {
- continue;
- }
- snapPoints.push({ time: bookmark.time, type: "bookmark" });
- }
- }
-
- return snapPoints;
-}
-
-export function snapToNearestPoint({
- targetTime,
- snapPoints,
- zoomLevel,
- snapThreshold = DEFAULT_SNAP_THRESHOLD_PX,
-}: {
- targetTime: number;
- snapPoints: Array<SnapPoint>;
- zoomLevel: number;
- snapThreshold?: number;
-}): SnapResult {
- const pixelsPerSecond = BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel;
- const thresholdInSeconds = snapThreshold / pixelsPerSecond;
-
- let closestSnapPoint: SnapPoint | null = null;
- let closestDistance = Infinity;
-
- for (const snapPoint of snapPoints) {
- const distance = Math.abs(targetTime - snapPoint.time);
- if (distance < thresholdInSeconds && distance < closestDistance) {
- closestDistance = distance;
- closestSnapPoint = snapPoint;
- }
- }
-
- return {
- snappedTime: closestSnapPoint ? closestSnapPoint.time : targetTime,
- snapPoint: closestSnapPoint,
- snapDistance: closestDistance,
- };
-}
-
-export function snapElementEdge({
- targetTime,
- elementDuration,
- tracks,
- playheadTime,
- zoomLevel,
- excludeElementId,
- snapToStart = true,
- bookmarks = [],
-}: {
- targetTime: number;
- elementDuration: number;
- tracks: Array<TimelineTrack>;
- playheadTime: number;
- zoomLevel: number;
- excludeElementId?: string;
- snapToStart?: boolean;
- bookmarks?: Array<Bookmark>;
-}): SnapResult {
- const snapPoints = findSnapPoints({
- tracks,
- playheadTime,
- excludeElementId,
- bookmarks,
- });
-
- const effectiveTargetTime = snapToStart
- ? targetTime
- : targetTime + elementDuration;
-
- const snapResult = snapToNearestPoint({
- targetTime: effectiveTargetTime,
- snapPoints,
- zoomLevel,
- });
-
- if (!snapToStart && snapResult.snapPoint) {
- snapResult.snappedTime = snapResult.snappedTime - elementDuration;
- }
-
- return snapResult;
-}
+import type { Bookmark, TimelineTrack } from "@/lib/timeline"; +import { BASE_TIMELINE_PIXELS_PER_SECOND } from "@/lib/timeline/scale"; +import { getElementKeyframes } from "@/lib/animation"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export interface SnapPoint { + time: number; + type: "element-start" | "element-end" | "playhead" | "bookmark" | "keyframe"; + elementId?: string; + trackId?: string; +} + +export interface SnapResult { + snappedTime: number; + snapPoint: SnapPoint | null; + snapDistance: number; +} + +const DEFAULT_SNAP_THRESHOLD_PX = 10; + +export function findSnapPoints({ + tracks, + playheadTime, + excludeElementId, + bookmarks = [], + excludeBookmarkTime, + enableElementSnapping = true, + enablePlayheadSnapping = true, + enableBookmarkSnapping = true, + enableKeyframeSnapping = true, +}: { + tracks: Array<TimelineTrack>; + playheadTime: number; + excludeElementId?: string; + bookmarks?: Array<Bookmark>; + excludeBookmarkTime?: number; + enableElementSnapping?: boolean; + enablePlayheadSnapping?: boolean; + enableBookmarkSnapping?: boolean; + enableKeyframeSnapping?: boolean; +}): SnapPoint[] { + const snapPoints: SnapPoint[] = []; + + for (const track of tracks) { + for (const element of track.elements) { + if (element.id === excludeElementId) continue; + + if (enableElementSnapping) { + snapPoints.push( + { + time: element.startTime, + type: "element-start", + elementId: element.id, + trackId: track.id, + }, + { + time: element.startTime + element.duration, + type: "element-end", + elementId: element.id, + trackId: track.id, + }, + ); + } + + if (enableKeyframeSnapping) { + for (const keyframe of getElementKeyframes({ + animations: element.animations, + })) { + snapPoints.push({ + time: element.startTime + keyframe.time, + type: "keyframe", + elementId: element.id, + trackId: track.id, + }); + } + } + } + } + + if (enablePlayheadSnapping) { + snapPoints.push({ time: playheadTime, type: "playhead" }); + } + + if (enableBookmarkSnapping) { + for (const bookmark of bookmarks) { + if ( + excludeBookmarkTime != null && + bookmark.time === excludeBookmarkTime + ) { + continue; + } + snapPoints.push({ time: bookmark.time, type: "bookmark" }); + } + } + + return snapPoints; +} + +export function snapToNearestPoint({ + targetTime, + snapPoints, + zoomLevel, + snapThreshold = DEFAULT_SNAP_THRESHOLD_PX, +}: { + targetTime: number; + snapPoints: Array<SnapPoint>; + zoomLevel: number; + snapThreshold?: number; +}): SnapResult { + const pixelsPerSecond = BASE_TIMELINE_PIXELS_PER_SECOND * zoomLevel; + const thresholdInTicks = (snapThreshold / pixelsPerSecond) * TICKS_PER_SECOND; + + let closestSnapPoint: SnapPoint | null = null; + let closestDistance = Infinity; + + for (const snapPoint of snapPoints) { + const distance = Math.abs(targetTime - snapPoint.time); + if (distance < thresholdInTicks && distance < closestDistance) { + closestDistance = distance; + closestSnapPoint = snapPoint; + } + } + + return { + snappedTime: closestSnapPoint ? closestSnapPoint.time : targetTime, + snapPoint: closestSnapPoint, + snapDistance: closestDistance, + }; +} + +export function snapElementEdge({ + targetTime, + elementDuration, + tracks, + playheadTime, + zoomLevel, + excludeElementId, + snapToStart = true, + bookmarks = [], +}: { + targetTime: number; + elementDuration: number; + tracks: Array<TimelineTrack>; + playheadTime: number; + zoomLevel: number; + excludeElementId?: string; + snapToStart?: boolean; + bookmarks?: Array<Bookmark>; +}): SnapResult { + const snapPoints = findSnapPoints({ + tracks, + playheadTime, + excludeElementId, + bookmarks, + }); + + const effectiveTargetTime = snapToStart + ? targetTime + : targetTime + elementDuration; + + const snapResult = snapToNearestPoint({ + targetTime: effectiveTargetTime, + snapPoints, + zoomLevel, + }); + + if (!snapToStart && snapResult.snapPoint) { + snapResult.snappedTime = snapResult.snappedTime - elementDuration; + } + + return snapResult; +} diff --git a/apps/web/src/lib/timeline/update-pipeline.ts b/apps/web/src/lib/timeline/update-pipeline.ts index cef99abc..61d09a9d 100644 --- a/apps/web/src/lib/timeline/update-pipeline.ts +++ b/apps/web/src/lib/timeline/update-pipeline.ts @@ -8,7 +8,7 @@ import { enforceMainTrackStart } from "@/lib/timeline/placement"; import type { RetimeConfig, TimelineElement, TimelineTrack } from "@/lib/timeline"; import { isRetimableElement } from "@/lib/timeline"; -type ElementUpdateField = keyof TimelineElement; +type ElementUpdateField = keyof TimelineElement | string; export interface ElementUpdateContext { tracks: TimelineTrack[]; diff --git a/apps/web/src/lib/timeline/zoom-utils.ts b/apps/web/src/lib/timeline/zoom-utils.ts index c06957d0..680f8d0d 100644 --- a/apps/web/src/lib/timeline/zoom-utils.ts +++ b/apps/web/src/lib/timeline/zoom-utils.ts @@ -2,6 +2,7 @@ import { BASE_TIMELINE_PIXELS_PER_SECOND, TIMELINE_ZOOM_MAX, } from "@/lib/timeline/scale"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; const PADDING_MAX_RATIO = 0.75; const PADDING_MIN_RATIO = 0.15; @@ -14,12 +15,12 @@ export function getTimelineZoomMin({ duration: number; containerWidth: number | null | undefined; }): number { - const safeDuration = Math.max(duration, 1); + const safeDurationSeconds = Math.max(duration / TICKS_PER_SECOND, 1); const safeContainerWidth = containerWidth ?? 1000; const contentRatioAtMinZoom = 1 - PADDING_MAX_RATIO; const availableWidth = safeContainerWidth * contentRatioAtMinZoom; const zoomToFit = - availableWidth / (safeDuration * BASE_TIMELINE_PIXELS_PER_SECOND); + availableWidth / (safeDurationSeconds * BASE_TIMELINE_PIXELS_PER_SECOND); return Math.min(TIMELINE_ZOOM_MAX, zoomToFit); } diff --git a/apps/web/src/lib/wasm/constants.ts b/apps/web/src/lib/wasm/constants.ts new file mode 100644 index 00000000..ababe6b3 --- /dev/null +++ b/apps/web/src/lib/wasm/constants.ts @@ -0,0 +1,3 @@ +import { TICKS_PER_SECOND as _TICKS_PER_SECOND } from "opencut-wasm"; + +export const TICKS_PER_SECOND = _TICKS_PER_SECOND(); diff --git a/apps/web/src/lib/wasm/index.ts b/apps/web/src/lib/wasm/index.ts new file mode 100644 index 00000000..b04bfcf7 --- /dev/null +++ b/apps/web/src/lib/wasm/index.ts @@ -0,0 +1 @@ +export * from "./constants"; diff --git a/apps/web/src/services/renderer/canvas-renderer.ts b/apps/web/src/services/renderer/canvas-renderer.ts index 840d8b52..b142d97a 100644 --- a/apps/web/src/services/renderer/canvas-renderer.ts +++ b/apps/web/src/services/renderer/canvas-renderer.ts @@ -1,9 +1,11 @@ +import type { FrameRate } from "opencut-wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; import type { BaseNode } from "./nodes/base-node"; export type CanvasRendererParams = { width: number; height: number; - fps: number; + fps: FrameRate; }; export class CanvasRenderer { @@ -11,7 +13,7 @@ export class CanvasRenderer { context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; width: number; height: number; - fps: number; + fps: FrameRate; constructor({ width, height, fps }: CanvasRendererParams) { this.width = width; diff --git a/apps/web/src/services/renderer/nodes/blur-background-node.ts b/apps/web/src/services/renderer/nodes/blur-background-node.ts index c67a5113..4cb45b33 100644 --- a/apps/web/src/services/renderer/nodes/blur-background-node.ts +++ b/apps/web/src/services/renderer/nodes/blur-background-node.ts @@ -1,5 +1,5 @@ -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { buildGaussianBlurPasses, intensityToSigma } from "@/lib/effects/definitions/blur"; +import { mediaTimeToSeconds } from "opencut-wasm"; import { getSourceTimeAtClipTime } from "@/lib/retime"; import { videoCache } from "@/services/video-cache/service"; import type { RetimeConfig } from "@/lib/timeline"; @@ -39,9 +39,7 @@ export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> { private isInRange({ time }: { time: number }): boolean { const localTime = time - this.params.timeOffset; - return ( - localTime >= -TIME_EPSILON_SECONDS && localTime < this.params.duration - ); + return localTime >= 0 && localTime < this.params.duration; } private getSourceLocalTime({ time }: { time: number }): number { @@ -61,10 +59,11 @@ export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> { time: number; }): Promise<BackdropSource | null> { if (this.params.mediaType === "video") { + const sourceTimeTicks = this.getSourceLocalTime({ time }); const frame = await videoCache.getFrameAt({ mediaId: this.params.mediaId, file: this.params.file, - time: this.getSourceLocalTime({ time }), + time: mediaTimeToSeconds({ time: sourceTimeTicks }), }); if (!frame) { diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts index 3bb836ec..4f225567 100644 --- a/apps/web/src/services/renderer/nodes/video-node.ts +++ b/apps/web/src/services/renderer/nodes/video-node.ts @@ -1,37 +1,39 @@ -import type { CanvasRenderer } from "../canvas-renderer";
-import { VisualNode, type VisualNodeParams } from "./visual-node";
-import { videoCache } from "@/services/video-cache/service";
-
-export interface VideoNodeParams extends VisualNodeParams {
- url: string;
- file: File;
- mediaId: string;
-}
-
-export class VideoNode extends VisualNode<VideoNodeParams> {
- async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) {
- await super.render({ renderer, time });
-
- if (!this.isInRange({ time })) {
- return;
- }
-
- const videoTime = this.getSourceLocalTime({ time });
-
- const frame = await videoCache.getFrameAt({
- mediaId: this.params.mediaId,
- file: this.params.file,
- time: videoTime,
- });
-
- if (frame) {
- this.renderVisual({
- renderer,
- source: frame.canvas,
- sourceWidth: frame.canvas.width,
- sourceHeight: frame.canvas.height,
- timelineTime: time,
- });
- }
- }
-}
+import type { CanvasRenderer } from "../canvas-renderer"; +import { mediaTimeToSeconds } from "opencut-wasm"; +import { VisualNode, type VisualNodeParams } from "./visual-node"; +import { videoCache } from "@/services/video-cache/service"; + +export interface VideoNodeParams extends VisualNodeParams { + url: string; + file: File; + mediaId: string; +} + +export class VideoNode extends VisualNode<VideoNodeParams> { + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + await super.render({ renderer, time }); + + if (!this.isInRange({ time })) { + return; + } + + const videoTimeTicks = this.getSourceLocalTime({ time }); + const videoTimeSeconds = mediaTimeToSeconds({ time: videoTimeTicks }); + + const frame = await videoCache.getFrameAt({ + mediaId: this.params.mediaId, + file: this.params.file, + time: videoTimeSeconds, + }); + + if (frame) { + this.renderVisual({ + renderer, + source: frame.canvas, + sourceWidth: frame.canvas.width, + sourceHeight: frame.canvas.height, + timelineTime: time, + }); + } + } +} diff --git a/apps/web/src/services/renderer/nodes/visual-node.ts b/apps/web/src/services/renderer/nodes/visual-node.ts index 82093da4..4a5fcdc6 100644 --- a/apps/web/src/services/renderer/nodes/visual-node.ts +++ b/apps/web/src/services/renderer/nodes/visual-node.ts @@ -12,7 +12,6 @@ import { resolveTransformAtTime, } from "@/lib/animation"; import { resolveEffectParamsAtTime } from "@/lib/animation/effect-param-channel"; -import { TIME_EPSILON_SECONDS } from "@/constants/animation-constants"; import { effectsRegistry, resolveEffectPasses } from "@/lib/effects"; import { masksRegistry } from "@/lib/masks"; import { getSourceTimeAtClipTime } from "@/lib/retime"; @@ -58,7 +57,7 @@ export abstract class VisualNode< protected isInRange({ time }: { time: number }): boolean { const localTime = time - this.params.timeOffset; return ( - localTime >= -TIME_EPSILON_SECONDS && + localTime >= 0 && localTime < this.params.duration ); } diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts index 2dbb7cdd..0c2ee17f 100644 --- a/apps/web/src/services/renderer/scene-exporter.ts +++ b/apps/web/src/services/renderer/scene-exporter.ts @@ -1,163 +1,170 @@ -import EventEmitter from "eventemitter3";
-
-import {
- Output,
- Mp4OutputFormat,
- WebMOutputFormat,
- BufferTarget,
- CanvasSource,
- AudioBufferSource,
- QUALITY_LOW,
- QUALITY_MEDIUM,
- QUALITY_HIGH,
- QUALITY_VERY_HIGH,
-} from "mediabunny";
-import type { RootNode } from "./nodes/root-node";
-import type { ExportFormat, ExportQuality } from "@/lib/export";
-import { CanvasRenderer } from "./canvas-renderer";
-
-type ExportParams = {
- width: number;
- height: number;
- fps: number;
- format: ExportFormat;
- quality: ExportQuality;
- shouldIncludeAudio?: boolean;
- audioBuffer?: AudioBuffer;
-};
-
-const qualityMap = {
- low: QUALITY_LOW,
- medium: QUALITY_MEDIUM,
- high: QUALITY_HIGH,
- very_high: QUALITY_VERY_HIGH,
-};
-
-export type SceneExporterEvents = {
- progress: [progress: number];
- complete: [buffer: ArrayBuffer];
- error: [error: Error];
- cancelled: [];
-};
-
-export class SceneExporter extends EventEmitter<SceneExporterEvents> {
- private renderer: CanvasRenderer;
- private format: ExportFormat;
- private quality: ExportQuality;
- private shouldIncludeAudio: boolean;
- private audioBuffer?: AudioBuffer;
-
- private isCancelled = false;
-
- constructor({
- width,
- height,
- fps,
- format,
- quality,
- shouldIncludeAudio,
- audioBuffer,
- }: ExportParams) {
- super();
- this.renderer = new CanvasRenderer({
- width,
- height,
- fps,
- });
-
- this.format = format;
- this.quality = quality;
- this.shouldIncludeAudio = shouldIncludeAudio ?? false;
- this.audioBuffer = audioBuffer;
- }
-
- cancel(): void {
- this.isCancelled = true;
- }
-
- async export({
- rootNode,
- }: {
- rootNode: RootNode;
- }): Promise<ArrayBuffer | null> {
- const { fps } = this.renderer;
- const frameCount = Math.ceil(rootNode.duration * fps);
-
- const outputFormat =
- this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat();
-
- const output = new Output({
- format: outputFormat,
- target: new BufferTarget(),
- });
-
- const videoSource = new CanvasSource(this.renderer.canvas, {
- codec: this.format === "webm" ? "vp9" : "avc",
- bitrate: qualityMap[this.quality],
- });
-
- output.addVideoTrack(videoSource, { frameRate: fps });
-
- let audioSource: AudioBufferSource | null = null;
- if (this.shouldIncludeAudio && this.audioBuffer) {
- let audioCodec: "aac" | "opus" =
- this.format === "webm" ? "opus" : "aac";
-
- if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") {
- const { supported } = await AudioEncoder.isConfigSupported({
- codec: "mp4a.40.2",
- sampleRate: this.audioBuffer.sampleRate,
- numberOfChannels: this.audioBuffer.numberOfChannels,
- bitrate: 192000,
- });
- if (!supported) audioCodec = "opus";
- }
-
- audioSource = new AudioBufferSource({
- codec: audioCodec,
- bitrate: qualityMap[this.quality],
- });
- output.addAudioTrack(audioSource);
- }
-
- await output.start();
-
- if (audioSource && this.audioBuffer) {
- await audioSource.add(this.audioBuffer);
- audioSource.close();
- }
-
- for (let i = 0; i < frameCount; i++) {
- if (this.isCancelled) {
- await output.cancel();
- this.emit("cancelled");
- return null;
- }
-
- const time = i / fps;
- await this.renderer.render({ node: rootNode, time });
- await videoSource.add(time, 1 / fps);
-
- this.emit("progress", i / frameCount);
- }
-
- if (this.isCancelled) {
- await output.cancel();
- this.emit("cancelled");
- return null;
- }
-
- videoSource.close();
- await output.finalize();
- this.emit("progress", 1);
-
- const buffer = output.target.buffer;
- if (!buffer) {
- this.emit("error", new Error("Failed to export video"));
- return null;
- }
-
- this.emit("complete", buffer);
- return buffer;
- }
-}
+import EventEmitter from "eventemitter3"; + +import { + Output, + Mp4OutputFormat, + WebMOutputFormat, + BufferTarget, + CanvasSource, + AudioBufferSource, + QUALITY_LOW, + QUALITY_MEDIUM, + QUALITY_HIGH, + QUALITY_VERY_HIGH, +} from "mediabunny"; +import type { FrameRate } from "opencut-wasm"; +import { mediaTimeToSeconds } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; +import { frameRateToFloat } from "@/lib/fps/utils"; +import type { RootNode } from "./nodes/root-node"; +import type { ExportFormat, ExportQuality } from "@/lib/export"; +import { CanvasRenderer } from "./canvas-renderer"; + +type ExportParams = { + width: number; + height: number; + fps: FrameRate; + format: ExportFormat; + quality: ExportQuality; + shouldIncludeAudio?: boolean; + audioBuffer?: AudioBuffer; +}; + +const qualityMap = { + low: QUALITY_LOW, + medium: QUALITY_MEDIUM, + high: QUALITY_HIGH, + very_high: QUALITY_VERY_HIGH, +}; + +export type SceneExporterEvents = { + progress: [progress: number]; + complete: [buffer: ArrayBuffer]; + error: [error: Error]; + cancelled: []; +}; + +export class SceneExporter extends EventEmitter<SceneExporterEvents> { + private renderer: CanvasRenderer; + private format: ExportFormat; + private quality: ExportQuality; + private shouldIncludeAudio: boolean; + private audioBuffer?: AudioBuffer; + + private isCancelled = false; + + constructor({ + width, + height, + fps, + format, + quality, + shouldIncludeAudio, + audioBuffer, + }: ExportParams) { + super(); + this.renderer = new CanvasRenderer({ + width, + height, + fps, + }); + + this.format = format; + this.quality = quality; + this.shouldIncludeAudio = shouldIncludeAudio ?? false; + this.audioBuffer = audioBuffer; + } + + cancel(): void { + this.isCancelled = true; + } + + async export({ + rootNode, + }: { + rootNode: RootNode; + }): Promise<ArrayBuffer | null> { + const fps = this.renderer.fps; + const fpsFloat = frameRateToFloat(fps); + const ticksPerFrame = Math.round(TICKS_PER_SECOND * fps.denominator / fps.numerator); + const frameCount = Math.floor(rootNode.duration / ticksPerFrame); + + const outputFormat = + this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); + + const output = new Output({ + format: outputFormat, + target: new BufferTarget(), + }); + + const videoSource = new CanvasSource(this.renderer.canvas, { + codec: this.format === "webm" ? "vp9" : "avc", + bitrate: qualityMap[this.quality], + }); + + output.addVideoTrack(videoSource, { frameRate: fpsFloat }); + + let audioSource: AudioBufferSource | null = null; + if (this.shouldIncludeAudio && this.audioBuffer) { + let audioCodec: "aac" | "opus" = + this.format === "webm" ? "opus" : "aac"; + + if (audioCodec === "aac" && typeof AudioEncoder !== "undefined") { + const { supported } = await AudioEncoder.isConfigSupported({ + codec: "mp4a.40.2", + sampleRate: this.audioBuffer.sampleRate, + numberOfChannels: this.audioBuffer.numberOfChannels, + bitrate: 192000, + }); + if (!supported) audioCodec = "opus"; + } + + audioSource = new AudioBufferSource({ + codec: audioCodec, + bitrate: qualityMap[this.quality], + }); + output.addAudioTrack(audioSource); + } + + await output.start(); + + if (audioSource && this.audioBuffer) { + await audioSource.add(this.audioBuffer); + audioSource.close(); + } + + for (let i = 0; i < frameCount; i++) { + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + const timeTicks = i * ticksPerFrame; + const timeSeconds = mediaTimeToSeconds({ time: timeTicks }); + await this.renderer.render({ node: rootNode, time: timeTicks }); + await videoSource.add(timeSeconds, 1 / fpsFloat); + + this.emit("progress", i / frameCount); + } + + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + videoSource.close(); + await output.finalize(); + this.emit("progress", 1); + + const buffer = output.target.buffer; + if (!buffer) { + this.emit("error", new Error("Failed to export video")); + return null; + } + + this.emit("complete", buffer); + return buffer; + } +} diff --git a/apps/web/src/services/storage/migrations/__tests__/v1-to-v2.test.ts b/apps/web/src/services/storage/migrations/__tests__/v1-to-v2.test.ts index 7a4dd641..7a0617a7 100644 --- a/apps/web/src/services/storage/migrations/__tests__/v1-to-v2.test.ts +++ b/apps/web/src/services/storage/migrations/__tests__/v1-to-v2.test.ts @@ -4,7 +4,7 @@ import { DEFAULT_BACKGROUND_COLOR, } from "@/lib/background/constants"; import { DEFAULT_CANVAS_SIZE } from "@/lib/canvas/constants"; -import { DEFAULT_FPS } from "@/lib/fps/constants"; +const DEFAULT_FPS = 30; import type { MediaAssetData } from "@/services/storage/types"; import { getProjectId, transformProjectV1ToV2 } from "../transformers/v1-to-v2"; import { diff --git a/apps/web/src/services/storage/migrations/__tests__/v22-to-v23.test.ts b/apps/web/src/services/storage/migrations/__tests__/v22-to-v23.test.ts new file mode 100644 index 00000000..5b764b48 --- /dev/null +++ b/apps/web/src/services/storage/migrations/__tests__/v22-to-v23.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, test } from "bun:test"; +import { transformProjectV22ToV23 } from "../transformers/v22-to-v23"; + +describe("V22 to V23 Migration", () => { + test("converts project time values from seconds to ticks and fps to a frame-rate object", () => { + const result = transformProjectV22ToV23({ + project: { + id: "project-v22-time", + version: 22, + metadata: { + id: "project-v22-time", + name: "Project", + duration: 15.5, + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + settings: { + fps: 29.97, + canvasSize: { width: 1920, height: 1080 }, + background: { type: "color", color: "#000000" }, + }, + timelineViewState: { + zoomLevel: 1, + scrollLeft: 120, + playheadTime: 1.25, + }, + scenes: [ + { + id: "scene-1", + bookmarks: [ + { time: 2.5, duration: 0.75, note: "Marker", color: "#ff0000" }, + { time: 4.5 }, + ], + tracks: [ + { + id: "track-1", + type: "video", + elements: [ + { + id: "element-1", + type: "video", + startTime: 1.25, + duration: 5.5, + trimStart: 0.25, + trimEnd: 0.5, + sourceDuration: 6.25, + animations: { + bindings: { + opacity: { + path: "opacity", + kind: "number", + components: [ + { + key: "value", + channelId: "opacity:value", + }, + ], + }, + }, + channels: { + "opacity:value": { + kind: "scalar", + keys: [ + { + id: "key-1", + time: 0.5, + value: 1, + segmentToNext: "bezier", + tangentMode: "flat", + rightHandle: { + dt: 0.25, + dv: 0.2, + }, + }, + { + id: "key-2", + time: 1.0, + value: 0.4, + segmentToNext: "linear", + tangentMode: "flat", + leftHandle: { + dt: -0.125, + dv: -0.1, + }, + }, + ], + }, + }, + }, + }, + ], + }, + ], + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }, + ], + }, + }); + + expect(result.skipped).toBe(false); + expect(result.project.version).toBe(23); + + const metadata = result.project.metadata as Record<string, unknown>; + expect(metadata.duration).toBe(1_860_000); + + const settings = result.project.settings as Record<string, unknown>; + expect(settings.fps).toEqual({ numerator: 30_000, denominator: 1_001 }); + + const timelineViewState = result.project.timelineViewState as Record< + string, + unknown + >; + expect(timelineViewState.playheadTime).toBe(150_000); + expect(timelineViewState.scrollLeft).toBe(120); + + const scenes = result.project.scenes as Array<Record<string, unknown>>; + const scene = scenes[0]; + expect(scene.bookmarks).toEqual([ + { + time: 300_000, + duration: 90_000, + note: "Marker", + color: "#ff0000", + }, + { time: 540_000 }, + ]); + + const tracks = scene.tracks as Array<Record<string, unknown>>; + const elements = tracks[0].elements as Array<Record<string, unknown>>; + const element = elements[0]; + expect(element.startTime).toBe(150_000); + expect(element.duration).toBe(660_000); + expect(element.trimStart).toBe(30_000); + expect(element.trimEnd).toBe(60_000); + expect(element.sourceDuration).toBe(750_000); + + const animations = element.animations as Record<string, unknown>; + const channels = animations.channels as Record<string, Record<string, unknown>>; + expect(channels["opacity:value"]).toEqual({ + kind: "scalar", + keys: [ + { + id: "key-1", + time: 60_000, + value: 1, + segmentToNext: "bezier", + tangentMode: "flat", + rightHandle: { + dt: 30_000, + dv: 0.2, + }, + }, + { + id: "key-2", + time: 120_000, + value: 0.4, + segmentToNext: "linear", + tangentMode: "flat", + leftHandle: { + dt: -15_000, + dv: -0.1, + }, + }, + ], + }); + }); + + test("skips projects already on v23", () => { + const result = transformProjectV22ToV23({ + project: { + id: "project-v23", + version: 23, + }, + }); + + expect(result.skipped).toBe(true); + expect(result.reason).toBe("already v23"); + }); + + test("skips projects not on v22", () => { + const result = transformProjectV22ToV23({ + project: { + id: "project-v21", + version: 21, + }, + }); + + expect(result.skipped).toBe(true); + expect(result.reason).toBe("not v22"); + }); +}); diff --git a/apps/web/src/services/storage/migrations/index.ts b/apps/web/src/services/storage/migrations/index.ts index ef41673e..9f0f3e0d 100644 --- a/apps/web/src/services/storage/migrations/index.ts +++ b/apps/web/src/services/storage/migrations/index.ts @@ -21,10 +21,11 @@ import { V18toV19Migration } from "./v18-to-v19"; import { V19toV20Migration } from "./v19-to-v20";
import { V20toV21Migration } from "./v20-to-v21";
import { V21toV22Migration } from "./v21-to-v22";
+import { V22toV23Migration } from "./v22-to-v23";
export { runStorageMigrations } from "./runner";
export type { MigrationProgress } from "./runner";
-export const CURRENT_PROJECT_VERSION = 22;
+export const CURRENT_PROJECT_VERSION = 23;
export const migrations = [
new V0toV1Migration(),
@@ -49,4 +50,5 @@ export const migrations = [ new V19toV20Migration(),
new V20toV21Migration(),
new V21toV22Migration(),
+ new V22toV23Migration(),
];
diff --git a/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts b/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts index a2c72478..6313c311 100644 --- a/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts +++ b/apps/web/src/services/storage/migrations/transformers/v1-to-v2.ts @@ -3,7 +3,7 @@ import { DEFAULT_BACKGROUND_COLOR, } from "@/lib/background/constants"; import { DEFAULT_CANVAS_SIZE } from "@/lib/canvas/constants"; -import { DEFAULT_FPS } from "@/lib/fps/constants"; +const DEFAULT_FPS = 30; import { IndexedDBAdapter } from "@/services/storage/indexeddb-adapter"; import type { MediaAssetData } from "@/services/storage/types"; import type { MigrationResult, ProjectRecord } from "./types"; diff --git a/apps/web/src/services/storage/migrations/transformers/v22-to-v23.ts b/apps/web/src/services/storage/migrations/transformers/v22-to-v23.ts new file mode 100644 index 00000000..cde1294e --- /dev/null +++ b/apps/web/src/services/storage/migrations/transformers/v22-to-v23.ts @@ -0,0 +1,335 @@ +import type { MigrationResult, ProjectRecord } from "./types"; +import { getProjectId, isRecord } from "./utils"; + +import { TICKS_PER_SECOND } from "@/lib/wasm"; +const ARBITRARY_FPS_DENOMINATOR = 1_000_000; +const STANDARD_FRAME_RATES = [ + { value: 24_000 / 1_001, numerator: 24_000, denominator: 1_001 }, + { value: 24, numerator: 24, denominator: 1 }, + { value: 25, numerator: 25, denominator: 1 }, + { value: 30_000 / 1_001, numerator: 30_000, denominator: 1_001 }, + { value: 30, numerator: 30, denominator: 1 }, + { value: 48, numerator: 48, denominator: 1 }, + { value: 50, numerator: 50, denominator: 1 }, + { value: 60_000 / 1_001, numerator: 60_000, denominator: 1_001 }, + { value: 60, numerator: 60, denominator: 1 }, + { value: 120, numerator: 120, denominator: 1 }, +] as const; +const STANDARD_FRAME_RATE_TOLERANCE = 0.01; + +export function transformProjectV22ToV23({ + project, +}: { + project: ProjectRecord; +}): MigrationResult<ProjectRecord> { + if (!getProjectId({ project })) { + return { project, skipped: true, reason: "no project id" }; + } + + const version = project.version; + if (typeof version !== "number") { + return { project, skipped: true, reason: "invalid version" }; + } + if (version >= 23) { + return { project, skipped: true, reason: "already v23" }; + } + if (version !== 22) { + return { project, skipped: true, reason: "not v22" }; + } + + return { + project: { + ...migrateProject({ project }), + version: 23, + }, + skipped: false, + }; +} + +function migrateProject({ + project, +}: { + project: ProjectRecord; +}): ProjectRecord { + const nextProject = { ...project }; + + if (isRecord(project.metadata)) { + nextProject.metadata = migrateMetadata({ metadata: project.metadata }); + } + + if (isRecord(project.settings)) { + nextProject.settings = migrateSettings({ settings: project.settings }); + } + + if (isRecord(project.timelineViewState)) { + nextProject.timelineViewState = migrateTimelineViewState({ + timelineViewState: project.timelineViewState, + }); + } + + if (Array.isArray(project.scenes)) { + nextProject.scenes = project.scenes.map((scene) => migrateScene({ scene })); + } + + return nextProject; +} + +function migrateMetadata({ + metadata, +}: { + metadata: ProjectRecord; +}): ProjectRecord { + return migrateTimeFields({ + record: metadata, + keys: ["duration"], + }); +} + +function migrateSettings({ + settings, +}: { + settings: ProjectRecord; +}): ProjectRecord { + const nextSettings = { ...settings }; + if ("fps" in settings) { + nextSettings.fps = migrateFrameRate({ fps: settings.fps }); + } + return nextSettings; +} + +function migrateTimelineViewState({ + timelineViewState, +}: { + timelineViewState: ProjectRecord; +}): ProjectRecord { + return migrateTimeFields({ + record: timelineViewState, + keys: ["playheadTime"], + }); +} + +function migrateScene({ scene }: { scene: unknown }): unknown { + if (!isRecord(scene)) { + return scene; + } + + const nextScene = { ...scene }; + + if (Array.isArray(scene.bookmarks)) { + nextScene.bookmarks = scene.bookmarks.map((bookmark) => + migrateBookmark({ bookmark }), + ); + } + + if (Array.isArray(scene.tracks)) { + nextScene.tracks = scene.tracks.map((track) => migrateTrack({ track })); + } + + return nextScene; +} + +function migrateTrack({ track }: { track: unknown }): unknown { + if (!isRecord(track)) { + return track; + } + + if (!Array.isArray(track.elements)) { + return track; + } + + return { + ...track, + elements: track.elements.map((element) => migrateElement({ element })), + }; +} + +function migrateElement({ element }: { element: unknown }): unknown { + if (!isRecord(element)) { + return element; + } + + const nextElement = migrateTimeFields({ + record: element, + keys: ["duration", "startTime", "trimStart", "trimEnd", "sourceDuration"], + }); + + if (isRecord(element.animations)) { + nextElement.animations = migrateAnimations({ + animations: element.animations, + }); + } + + return nextElement; +} + +function migrateAnimations({ + animations, +}: { + animations: ProjectRecord; +}): ProjectRecord { + if (!isRecord(animations.channels)) { + return animations; + } + + return { + ...animations, + channels: Object.fromEntries( + Object.entries(animations.channels).map(([channelId, channel]) => [ + channelId, + migrateAnimationChannel({ channel }), + ]), + ), + }; +} + +function migrateAnimationChannel({ channel }: { channel: unknown }): unknown { + if (!isRecord(channel)) { + return channel; + } + + if (!Array.isArray(channel.keys)) { + return channel; + } + + return { + ...channel, + keys: channel.keys.map((keyframe) => migrateAnimationKeyframe({ keyframe })), + }; +} + +function migrateAnimationKeyframe({ + keyframe, +}: { + keyframe: unknown; +}): unknown { + if (!isRecord(keyframe)) { + return keyframe; + } + + const nextKeyframe = migrateTimeFields({ + record: keyframe, + keys: ["time"], + }); + + if (isRecord(keyframe.leftHandle)) { + nextKeyframe.leftHandle = migrateCurveHandle({ + handle: keyframe.leftHandle, + }); + } + + if (isRecord(keyframe.rightHandle)) { + nextKeyframe.rightHandle = migrateCurveHandle({ + handle: keyframe.rightHandle, + }); + } + + return nextKeyframe; +} + +function migrateCurveHandle({ + handle, +}: { + handle: ProjectRecord; +}): ProjectRecord { + return migrateTimeFields({ + record: handle, + keys: ["dt"], + }); +} + +function migrateBookmark({ bookmark }: { bookmark: unknown }): unknown { + if (!isRecord(bookmark)) { + return bookmark; + } + + return migrateTimeFields({ + record: bookmark, + keys: ["time", "duration"], + }); +} + +function migrateTimeFields({ + record, + keys, +}: { + record: ProjectRecord; + keys: string[]; +}): ProjectRecord { + const nextRecord = { ...record }; + + for (const key of keys) { + if (!(key in record)) { + continue; + } + + nextRecord[key] = migrateTimeValue({ value: record[key] }); + } + + return nextRecord; +} + +function migrateTimeValue({ value }: { value: unknown }): unknown { + if (typeof value !== "number" || !Number.isFinite(value)) { + return value; + } + + return secondsToTicks({ value }); +} + +function secondsToTicks({ value }: { value: number }): number { + return Math.round(value * TICKS_PER_SECOND); +} + +function migrateFrameRate({ fps }: { fps: unknown }): unknown { + if (isRecord(fps)) { + return fps; + } + + if (typeof fps !== "number" || !Number.isFinite(fps) || fps <= 0) { + return fps; + } + + const standardFrameRate = STANDARD_FRAME_RATES.find( + (candidate) => Math.abs(fps - candidate.value) <= STANDARD_FRAME_RATE_TOLERANCE, + ); + if (standardFrameRate) { + return { + numerator: standardFrameRate.numerator, + denominator: standardFrameRate.denominator, + }; + } + + if (Number.isInteger(fps)) { + return { numerator: fps, denominator: 1 }; + } + + const scaledNumerator = Math.round(fps * ARBITRARY_FPS_DENOMINATOR); + const divisor = greatestCommonDivisor({ + left: scaledNumerator, + right: ARBITRARY_FPS_DENOMINATOR, + }); + + return { + numerator: scaledNumerator / divisor, + denominator: ARBITRARY_FPS_DENOMINATOR / divisor, + }; +} + +function greatestCommonDivisor({ + left, + right, +}: { + left: number; + right: number; +}): number { + let nextLeft = Math.abs(left); + let nextRight = Math.abs(right); + + while (nextRight !== 0) { + const remainder = nextLeft % nextRight; + nextLeft = nextRight; + nextRight = remainder; + } + + return nextLeft || 1; +} diff --git a/apps/web/src/services/storage/migrations/v22-to-v23.ts b/apps/web/src/services/storage/migrations/v22-to-v23.ts new file mode 100644 index 00000000..cd352e45 --- /dev/null +++ b/apps/web/src/services/storage/migrations/v22-to-v23.ts @@ -0,0 +1,16 @@ +import { StorageMigration } from "./base"; +import type { ProjectRecord } from "./transformers/types"; +import { transformProjectV22ToV23 } from "./transformers/v22-to-v23"; + +export class V22toV23Migration extends StorageMigration { + from = 22; + to = 23; + + async transform(project: ProjectRecord): Promise<{ + project: ProjectRecord; + skipped: boolean; + reason?: string; + }> { + return transformProjectV22ToV23({ project }); + } +} @@ -54,7 +54,7 @@ "nanoid": "^5.1.5", "next": "16.1.3", "next-themes": "^0.4.4", - "opencut-wasm": "^0.1.3", + "opencut-wasm": "^0.2.3", "pg": "^8.16.2", "postgres": "^3.4.5", "radix-ui": "^1.4.3", @@ -1359,7 +1359,7 @@ "onnxruntime-web": ["onnxruntime-web@1.22.0-dev.20250409-89f8206ba4", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ=="], - "opencut-wasm": ["opencut-wasm@0.1.3", "", {}, "sha512-3MlWL8J8NCRBfm/6LrdvW08rgIPqyhk5dCG3j/zokKDOf5mwULox9Vpqi2jgZ+dSKIWReHuUCZq2b0cGCFuVHQ=="], + "opencut-wasm": ["opencut-wasm@0.2.3", "", {}, "sha512-SaLe2fgvLK+EcW7qqggmcfXaQwo6SOh83/h3dtEEbdqEka7gn1PGJECKnTZq4OWhkNEgf8SKJy/HiJOQvyZC+Q=="], "p-limit": ["p-limit@6.2.0", "", { "dependencies": { "yocto-queue": "^1.1.1" } }, "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA=="], diff --git a/rust/crates/bridge/src/bridge.rs b/rust/crates/bridge/src/bridge.rs index 52dee731..99b00615 100644 --- a/rust/crates/bridge/src/bridge.rs +++ b/rust/crates/bridge/src/bridge.rs @@ -1,10 +1,36 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{ItemFn, parse_macro_input}; +use syn::{FnArg, Item, ItemConst, ItemFn, parse_macro_input}; #[proc_macro_attribute] pub fn export(_attr: TokenStream, item: TokenStream) -> TokenStream { - let function = parse_macro_input!(item as ItemFn); + match parse_macro_input!(item as Item) { + Item::Fn(function) => export_fn(function), + Item::Const(constant) => export_const(constant), + other => syn::Error::new_spanned(other, "#[export] only supports fn and const items") + .to_compile_error() + .into(), + } +} + +fn export_fn(function: ItemFn) -> TokenStream { + let param_count = function + .sig + .inputs + .iter() + .filter(|arg| matches!(arg, FnArg::Typed(_))) + .count(); + + if param_count > 1 { + return syn::Error::new_spanned( + &function.sig.inputs, + "#[export] functions must accept a single options struct, not positional arguments. \ + Wrap parameters in a struct: `fn foo(FooOptions { a, b }: FooOptions)`", + ) + .to_compile_error() + .into(); + } + let js_name = snake_to_camel(&function.sig.ident.to_string()); quote! { @@ -14,6 +40,26 @@ pub fn export(_attr: TokenStream, item: TokenStream) -> TokenStream { .into() } +fn export_const(constant: ItemConst) -> TokenStream { + let js_name = constant.ident.to_string(); + let const_ident = &constant.ident; + let getter_ident = syn::Ident::new( + &format!("__const_{}", constant.ident.to_string().to_lowercase()), + constant.ident.span(), + ); + + quote! { + #constant + + #[cfg(feature = "wasm")] + #[::wasm_bindgen::prelude::wasm_bindgen(js_name = #js_name)] + pub fn #getter_ident() -> f64 { + #const_ident as f64 + } + } + .into() +} + fn snake_to_camel(name: &str) -> String { let mut camel = String::with_capacity(name.len()); let mut should_uppercase_next = false; diff --git a/rust/crates/effects/Cargo.toml b/rust/crates/effects/Cargo.toml new file mode 100644 index 00000000..5b6e2a1e --- /dev/null +++ b/rust/crates/effects/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "effects" +version = "0.1.0" +edition = "2024" + +[lib] +path = "src/effects.rs" +crate-type = ["rlib"] + +[dependencies] +bytemuck = { version = "1.25.0", features = ["derive"] } +gpu = { version = "0.1.0", path = "../gpu" } +thiserror = "2.0.18" +wgpu = "29.0.1" diff --git a/rust/crates/effects/src/effects.rs b/rust/crates/effects/src/effects.rs new file mode 100644 index 00000000..c6d750bb --- /dev/null +++ b/rust/crates/effects/src/effects.rs @@ -0,0 +1,5 @@ +mod pipeline; +mod types; + +pub use pipeline::{ApplyEffectsOptions, EffectPipeline, EffectsError}; +pub use types::{EffectPass, UniformValue}; diff --git a/rust/crates/effects/src/pipeline.rs b/rust/crates/effects/src/pipeline.rs new file mode 100644 index 00000000..58692bde --- /dev/null +++ b/rust/crates/effects/src/pipeline.rs @@ -0,0 +1,306 @@ +use std::collections::HashMap; + +use bytemuck::{Pod, Zeroable}; +use gpu::{FULLSCREEN_SHADER_SOURCE, GPU_TEXTURE_FORMAT, GpuContext}; +use thiserror::Error; +use wgpu::util::DeviceExt; + +use crate::{EffectPass, UniformValue}; + +const GAUSSIAN_BLUR_SHADER_ID: &str = "gaussian-blur"; +const GAUSSIAN_BLUR_SHADER_SOURCE: &str = include_str!("shaders/gaussian_blur.wgsl"); + +pub struct ApplyEffectsOptions<'a> { + pub source: &'a wgpu::Texture, + pub width: u32, + pub height: u32, + pub passes: &'a [EffectPass], +} + +pub struct EffectPipeline { + uniform_bind_group_layout: wgpu::BindGroupLayout, + pipelines: HashMap<String, wgpu::RenderPipeline>, +} + +#[derive(Debug, Error)] +pub enum EffectsError { + #[error("At least one effect pass is required")] + MissingEffectPasses, + #[error("Unknown effect shader '{shader}'")] + UnknownEffectShader { shader: String }, + #[error("Missing uniform '{uniform}' for shader '{shader}'")] + MissingUniform { shader: String, uniform: String }, + #[error("Uniform '{uniform}' for shader '{shader}' must be a number")] + InvalidNumberUniform { shader: String, uniform: String }, + #[error( + "Uniform '{uniform}' for shader '{shader}' must be a vector of length {expected_length}" + )] + InvalidVectorUniform { + shader: String, + uniform: String, + expected_length: usize, + }, + #[error("Shader '{shader}' does not support uniform '{uniform}'")] + UnsupportedUniform { shader: String, uniform: String }, +} + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct EffectUniformBuffer { + resolution: [f32; 2], + direction: [f32; 2], + scalars: [f32; 4], +} + +impl EffectPipeline { + pub fn new(context: &GpuContext) -> Self { + let uniform_bind_group_layout = + context + .device() + .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("effects-uniform-bind-group-layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }], + }); + let vertex_shader_module = + context + .device() + .create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("effects-fullscreen-shader"), + source: wgpu::ShaderSource::Wgsl(FULLSCREEN_SHADER_SOURCE.into()), + }); + let gaussian_blur_shader_module = + context + .device() + .create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("effects-gaussian-blur-shader"), + source: wgpu::ShaderSource::Wgsl(GAUSSIAN_BLUR_SHADER_SOURCE.into()), + }); + let pipeline_layout = + context + .device() + .create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("effects-pipeline-layout"), + bind_group_layouts: &[ + Some(context.texture_sampler_bind_group_layout()), + Some(&uniform_bind_group_layout), + ], + immediate_size: 0, + }); + let gaussian_blur_pipeline = + context + .device() + .create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("effects-gaussian-blur-pipeline"), + layout: Some(&pipeline_layout), + vertex: wgpu::VertexState { + module: &vertex_shader_module, + entry_point: Some("vertex_main"), + buffers: &[wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::<[f32; 2]>() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &[wgpu::VertexAttribute { + format: wgpu::VertexFormat::Float32x2, + offset: 0, + shader_location: 0, + }], + }], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &gaussian_blur_shader_module, + entry_point: Some("fragment_main"), + targets: &[Some(wgpu::ColorTargetState { + format: GPU_TEXTURE_FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview_mask: None, + cache: None, + }); + let pipelines = + HashMap::from([(GAUSSIAN_BLUR_SHADER_ID.to_string(), gaussian_blur_pipeline)]); + + Self { + uniform_bind_group_layout, + pipelines, + } + } + + pub fn apply( + &self, + context: &GpuContext, + ApplyEffectsOptions { + source, + width, + height, + passes, + }: ApplyEffectsOptions<'_>, + ) -> Result<wgpu::Texture, EffectsError> { + let mut current_texture: Option<wgpu::Texture> = None; + + for pass in passes { + let input_texture = current_texture.as_ref().unwrap_or(source); + let output_texture = + context.create_render_texture(width, height, "effects-pass-output"); + let input_view = input_texture.create_view(&wgpu::TextureViewDescriptor::default()); + let output_view = output_texture.create_view(&wgpu::TextureViewDescriptor::default()); + let texture_bind_group = + context + .device() + .create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("effects-texture-bind-group"), + layout: context.texture_sampler_bind_group_layout(), + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView(&input_view), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(context.linear_sampler()), + }, + ], + }); + let uniform_buffer = + context + .device() + .create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("effects-uniform-buffer"), + contents: bytemuck::bytes_of(&pack_effect_uniforms(pass, width, height)?), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + let uniform_bind_group = + context + .device() + .create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("effects-uniform-bind-group"), + layout: &self.uniform_bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: uniform_buffer.as_entire_binding(), + }], + }); + let pipeline = self.pipelines.get(&pass.shader).ok_or_else(|| { + EffectsError::UnknownEffectShader { + shader: pass.shader.clone(), + } + })?; + let mut encoder = + context + .device() + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("effects-command-encoder"), + }); + + { + let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("effects-render-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: &output_view, + resolve_target: None, + depth_slice: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), + store: wgpu::StoreOp::Store, + }, + })], + depth_stencil_attachment: None, + occlusion_query_set: None, + timestamp_writes: None, + multiview_mask: None, + }); + render_pass.set_pipeline(pipeline); + render_pass.set_vertex_buffer(0, context.fullscreen_quad().slice(..)); + render_pass.set_bind_group(0, &texture_bind_group, &[]); + render_pass.set_bind_group(1, &uniform_bind_group, &[]); + render_pass.draw(0..6, 0..1); + } + + context.queue().submit([encoder.finish()]); + current_texture = Some(output_texture); + } + + current_texture.ok_or(EffectsError::MissingEffectPasses) + } +} + +fn pack_effect_uniforms( + pass: &EffectPass, + width: u32, + height: u32, +) -> Result<EffectUniformBuffer, EffectsError> { + let shader = pass.shader.as_str(); + let sigma = read_number_uniform(pass, "u_sigma")?; + let step = read_number_uniform(pass, "u_step")?; + let direction = read_vec2_uniform(pass, "u_direction")?; + + for uniform in pass.uniforms.keys() { + if uniform == "u_sigma" || uniform == "u_step" || uniform == "u_direction" { + continue; + } + return Err(EffectsError::UnsupportedUniform { + shader: shader.to_string(), + uniform: uniform.clone(), + }); + } + + Ok(EffectUniformBuffer { + resolution: [width as f32, height as f32], + direction, + scalars: [sigma, step, 0.0, 0.0], + }) +} + +fn read_number_uniform(pass: &EffectPass, uniform: &str) -> Result<f32, EffectsError> { + let Some(value) = pass.uniforms.get(uniform) else { + return Err(EffectsError::MissingUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + }); + }; + match value { + UniformValue::Number(value) => Ok(*value), + UniformValue::Vector(_) => Err(EffectsError::InvalidNumberUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + }), + } +} + +fn read_vec2_uniform(pass: &EffectPass, uniform: &str) -> Result<[f32; 2], EffectsError> { + let Some(value) = pass.uniforms.get(uniform) else { + return Err(EffectsError::MissingUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + }); + }; + let UniformValue::Vector(values) = value else { + return Err(EffectsError::InvalidVectorUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + expected_length: 2, + }); + }; + if values.len() != 2 { + return Err(EffectsError::InvalidVectorUniform { + shader: pass.shader.clone(), + uniform: uniform.to_string(), + expected_length: 2, + }); + } + Ok([values[0], values[1]]) +} diff --git a/rust/crates/gpu/src/shaders/gaussian_blur.wgsl b/rust/crates/effects/src/shaders/gaussian_blur.wgsl index 6c8d311d..6c8d311d 100644 --- a/rust/crates/gpu/src/shaders/gaussian_blur.wgsl +++ b/rust/crates/effects/src/shaders/gaussian_blur.wgsl diff --git a/rust/crates/effects/src/types.rs b/rust/crates/effects/src/types.rs new file mode 100644 index 00000000..bd013195 --- /dev/null +++ b/rust/crates/effects/src/types.rs @@ -0,0 +1,13 @@ +use std::collections::HashMap; + +#[derive(Clone, Debug)] +pub struct EffectPass { + pub shader: String, + pub uniforms: HashMap<String, UniformValue>, +} + +#[derive(Clone, Debug)] +pub enum UniformValue { + Number(f32), + Vector(Vec<f32>), +} diff --git a/rust/crates/gpu/Cargo.toml b/rust/crates/gpu/Cargo.toml index d68e8a65..4be1e8d6 100644 --- a/rust/crates/gpu/Cargo.toml +++ b/rust/crates/gpu/Cargo.toml @@ -7,3 +7,7 @@ edition = "2024" bytemuck = { version = "1.25.0", features = ["derive"] } thiserror = "2.0.18" wgpu = "29.0.1" + +[features] +default = [] +wasm = [] diff --git a/rust/crates/gpu/src/context.rs b/rust/crates/gpu/src/context.rs index 702f65ba..1526a67d 100644 --- a/rust/crates/gpu/src/context.rs +++ b/rust/crates/gpu/src/context.rs @@ -1,12 +1,8 @@ use wgpu::util::DeviceExt; -use crate::{ - GPU_TEXTURE_FORMAT, GpuError, - effect_pipeline::{ApplyEffectsOptions, apply_effects}, - mask_feather::{ApplyMaskFeatherOptions, MaskFeatherPipeline}, - sdf_pipeline::SdfPipeline, - shader_registry::ShaderRegistry, -}; +use crate::{FULLSCREEN_SHADER_SOURCE, GPU_TEXTURE_FORMAT, GpuError}; + +const BLIT_SHADER_SOURCE: &str = include_str!("shaders/blit.wgsl"); const FULLSCREEN_QUAD_POSITIONS: [[f32; 2]; 6] = [ [-1.0, -1.0], @@ -25,9 +21,8 @@ pub struct GpuContext { fullscreen_quad: wgpu::Buffer, linear_sampler: wgpu::Sampler, nearest_sampler: wgpu::Sampler, - shader_registry: ShaderRegistry, - sdf_pipeline: SdfPipeline, - mask_feather_pipeline: MaskFeatherPipeline, + texture_sampler_bind_group_layout: wgpu::BindGroupLayout, + blit_pipeline: wgpu::RenderPipeline, } impl GpuContext { @@ -77,9 +72,74 @@ impl GpuContext { mipmap_filter: wgpu::MipmapFilterMode::Nearest, ..Default::default() }); - let shader_registry = ShaderRegistry::new(&device); - let sdf_pipeline = SdfPipeline::new(&device); - let mask_feather_pipeline = MaskFeatherPipeline::new(&device); + let texture_sampler_bind_group_layout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("gpu-texture-sampler-bind-group-layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + view_dimension: wgpu::TextureViewDimension::D2, + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let vertex_shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("gpu-fullscreen-shader"), + source: wgpu::ShaderSource::Wgsl(FULLSCREEN_SHADER_SOURCE.into()), + }); + let blit_shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { + label: Some("gpu-blit-shader"), + source: wgpu::ShaderSource::Wgsl(BLIT_SHADER_SOURCE.into()), + }); + let blit_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("gpu-blit-pipeline-layout"), + bind_group_layouts: &[Some(&texture_sampler_bind_group_layout)], + immediate_size: 0, + }); + let blit_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("gpu-blit-pipeline"), + layout: Some(&blit_pipeline_layout), + vertex: wgpu::VertexState { + module: &vertex_shader_module, + entry_point: Some("vertex_main"), + buffers: &[wgpu::VertexBufferLayout { + array_stride: std::mem::size_of::<[f32; 2]>() as u64, + step_mode: wgpu::VertexStepMode::Vertex, + attributes: &[wgpu::VertexAttribute { + format: wgpu::VertexFormat::Float32x2, + offset: 0, + shader_location: 0, + }], + }], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &blit_shader_module, + entry_point: Some("fragment_main"), + targets: &[Some(wgpu::ColorTargetState { + format: GPU_TEXTURE_FORMAT, + blend: None, + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview_mask: None, + cache: None, + }); Ok(Self { instance, @@ -89,26 +149,11 @@ impl GpuContext { fullscreen_quad, linear_sampler, nearest_sampler, - shader_registry, - sdf_pipeline, - mask_feather_pipeline, + texture_sampler_bind_group_layout, + blit_pipeline, }) } - pub fn apply_effects( - &self, - options: ApplyEffectsOptions<'_>, - ) -> Result<wgpu::Texture, GpuError> { - apply_effects(self, options) - } - - pub fn apply_mask_feather( - &self, - options: ApplyMaskFeatherOptions<'_>, - ) -> wgpu::Texture { - self.mask_feather_pipeline.apply_mask_feather(self, options) - } - pub fn create_render_texture( &self, width: u32, @@ -162,12 +207,8 @@ impl GpuContext { &self.nearest_sampler } - pub fn shader_registry(&self) -> &ShaderRegistry { - &self.shader_registry - } - - pub fn sdf_pipeline(&self) -> &SdfPipeline { - &self.sdf_pipeline + pub fn texture_sampler_bind_group_layout(&self) -> &wgpu::BindGroupLayout { + &self.texture_sampler_bind_group_layout } pub fn render_texture_to_surface( @@ -202,7 +243,7 @@ impl GpuContext { .create_view(&wgpu::TextureViewDescriptor::default()); let bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("gpu-blit-bind-group"), - layout: self.shader_registry.effect_texture_bind_group_layout(), + layout: &self.texture_sampler_bind_group_layout, entries: &[ wgpu::BindGroupEntry { binding: 0, @@ -237,7 +278,7 @@ impl GpuContext { timestamp_writes: None, multiview_mask: None, }); - render_pass.set_pipeline(self.shader_registry.blit_pipeline()); + render_pass.set_pipeline(&self.blit_pipeline); render_pass.set_vertex_buffer(0, self.fullscreen_quad.slice(..)); render_pass.set_bind_group(0, &bind_group, &[]); render_pass.draw(0..6, 0..1); @@ -247,4 +288,50 @@ impl GpuContext { surface_texture.present(); Ok(()) } + + #[cfg(all(feature = "wasm", target_arch = "wasm32"))] + pub fn import_offscreen_canvas_texture( + &self, + canvas: &wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + label: &'static str, + ) -> wgpu::Texture { + let texture = self.create_render_texture(width, height, label); + self.queue.copy_external_image_to_texture( + &wgpu::CopyExternalImageSourceInfo { + source: wgpu::ExternalImageSource::OffscreenCanvas(canvas.clone()), + origin: wgpu::Origin2d::ZERO, + flip_y: true, + }, + wgpu::CopyExternalImageDestInfo { + texture: &texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + color_space: wgpu::PredefinedColorSpace::Srgb, + premultiplied_alpha: false, + }, + wgpu::Extent3d { + width, + height, + depth_or_array_layers: 1, + }, + ); + texture + } + + #[cfg(all(feature = "wasm", target_arch = "wasm32"))] + pub fn render_texture_to_offscreen_canvas( + &self, + texture: &wgpu::Texture, + canvas: &wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + ) -> Result<(), GpuError> { + let surface = self + .instance + .create_surface(wgpu::SurfaceTarget::OffscreenCanvas(canvas.clone()))?; + self.render_texture_to_surface(texture, &surface, width, height) + } } diff --git a/rust/crates/gpu/src/effect_pipeline.rs b/rust/crates/gpu/src/effect_pipeline.rs deleted file mode 100644 index c600551a..00000000 --- a/rust/crates/gpu/src/effect_pipeline.rs +++ /dev/null @@ -1,194 +0,0 @@ -use std::collections::HashMap; - -use bytemuck::{Pod, Zeroable}; -use wgpu::util::DeviceExt; - -use crate::{GpuError, context::GpuContext}; - -pub struct ApplyEffectsOptions<'a> { - pub source: &'a wgpu::Texture, - pub width: u32, - pub height: u32, - pub passes: &'a [EffectPass], -} - -#[derive(Clone, Debug)] -pub struct EffectPass { - pub shader: String, - pub uniforms: HashMap<String, UniformValue>, -} - -#[derive(Clone, Debug)] -pub enum UniformValue { - Number(f32), - Vector(Vec<f32>), -} - -#[repr(C)] -#[derive(Clone, Copy, Pod, Zeroable)] -struct EffectUniformBuffer { - resolution: [f32; 2], - direction: [f32; 2], - scalars: [f32; 4], -} - -pub fn apply_effects( - context: &GpuContext, - ApplyEffectsOptions { - source, - width, - height, - passes, - }: ApplyEffectsOptions<'_>, -) -> Result<wgpu::Texture, GpuError> { - let mut current_texture: Option<wgpu::Texture> = None; - - for pass in passes { - let input_texture = current_texture.as_ref().unwrap_or(source); - let output_texture = - context.create_render_texture(width, height, "gpu-effect-pass-output"); - let input_view = input_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let output_view = output_texture.create_view(&wgpu::TextureViewDescriptor::default()); - let texture_bind_group = context - .device() - .create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("gpu-effect-texture-bind-group"), - layout: context - .shader_registry() - .effect_texture_bind_group_layout(), - entries: &[ - wgpu::BindGroupEntry { - binding: 0, - resource: wgpu::BindingResource::TextureView(&input_view), - }, - wgpu::BindGroupEntry { - binding: 1, - resource: wgpu::BindingResource::Sampler(context.linear_sampler()), - }, - ], - }); - let uniform_buffer = context - .device() - .create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("gpu-effect-uniform-buffer"), - contents: bytemuck::bytes_of(&pack_effect_uniforms(pass, width, height)?), - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - }); - let uniform_bind_group = context - .device() - .create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("gpu-effect-uniform-bind-group"), - layout: context - .shader_registry() - .effect_uniform_bind_group_layout(), - entries: &[wgpu::BindGroupEntry { - binding: 0, - resource: uniform_buffer.as_entire_binding(), - }], - }); - let pipeline = context.shader_registry().get_effect_pipeline(&pass.shader)?; - let mut encoder = context - .device() - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("gpu-effect-command-encoder"), - }); - - { - let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { - label: Some("gpu-effect-render-pass"), - color_attachments: &[Some(wgpu::RenderPassColorAttachment { - view: &output_view, - resolve_target: None, - depth_slice: None, - ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), - store: wgpu::StoreOp::Store, - }, - })], - depth_stencil_attachment: None, - occlusion_query_set: None, - timestamp_writes: None, - multiview_mask: None, - }); - render_pass.set_pipeline(pipeline); - render_pass.set_vertex_buffer(0, context.fullscreen_quad().slice(..)); - render_pass.set_bind_group(0, &texture_bind_group, &[]); - render_pass.set_bind_group(1, &uniform_bind_group, &[]); - render_pass.draw(0..6, 0..1); - } - - context.queue().submit([encoder.finish()]); - current_texture = Some(output_texture); - } - - current_texture.ok_or_else(|| GpuError::UnknownEffectShader { - shader: "missing-effect-pass".to_string(), - }) -} - -fn pack_effect_uniforms( - pass: &EffectPass, - width: u32, - height: u32, -) -> Result<EffectUniformBuffer, GpuError> { - let shader = pass.shader.as_str(); - let sigma = read_number_uniform(pass, "u_sigma")?; - let step = read_number_uniform(pass, "u_step")?; - let direction = read_vec2_uniform(pass, "u_direction")?; - - for uniform in pass.uniforms.keys() { - if uniform == "u_sigma" || uniform == "u_step" || uniform == "u_direction" { - continue; - } - return Err(GpuError::UnsupportedUniform { - shader: shader.to_string(), - uniform: uniform.clone(), - }); - } - - Ok(EffectUniformBuffer { - resolution: [width as f32, height as f32], - direction, - scalars: [sigma, step, 0.0, 0.0], - }) -} - -fn read_number_uniform(pass: &EffectPass, uniform: &str) -> Result<f32, GpuError> { - let Some(value) = pass.uniforms.get(uniform) else { - return Err(GpuError::MissingUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - }); - }; - match value { - UniformValue::Number(value) => Ok(*value), - UniformValue::Vector(_) => Err(GpuError::InvalidNumberUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - }), - } -} - -fn read_vec2_uniform(pass: &EffectPass, uniform: &str) -> Result<[f32; 2], GpuError> { - let Some(value) = pass.uniforms.get(uniform) else { - return Err(GpuError::MissingUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - }); - }; - let UniformValue::Vector(values) = value else { - return Err(GpuError::InvalidVectorUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - expected_length: 2, - }); - }; - if values.len() != 2 { - return Err(GpuError::InvalidVectorUniform { - shader: pass.shader.clone(), - uniform: uniform.to_string(), - expected_length: 2, - }); - } - Ok([values[0], values[1]]) -} diff --git a/rust/crates/gpu/src/lib.rs b/rust/crates/gpu/src/lib.rs index c53e63ab..b6ca6c1c 100644 --- a/rust/crates/gpu/src/lib.rs +++ b/rust/crates/gpu/src/lib.rs @@ -1,17 +1,12 @@ mod context; -mod effect_pipeline; -mod mask_feather; -mod sdf_pipeline; -mod shader_registry; use thiserror::Error; -pub use wgpu; pub use context::GpuContext; -pub use effect_pipeline::{ApplyEffectsOptions, EffectPass, UniformValue}; -pub use mask_feather::ApplyMaskFeatherOptions; +pub use wgpu; pub const GPU_TEXTURE_FORMAT: wgpu::TextureFormat = wgpu::TextureFormat::Bgra8Unorm; +pub const FULLSCREEN_SHADER_SOURCE: &str = include_str!("shaders/fullscreen.wgsl"); #[derive(Debug, Error)] pub enum GpuError { @@ -23,18 +18,4 @@ pub enum GpuError { CreateSurface(#[from] wgpu::CreateSurfaceError), #[error("The output surface does not support the required texture format")] UnsupportedSurfaceFormat, - #[error("Unknown effect shader '{shader}'")] - UnknownEffectShader { shader: String }, - #[error("Missing uniform '{uniform}' for shader '{shader}'")] - MissingUniform { shader: String, uniform: String }, - #[error("Uniform '{uniform}' for shader '{shader}' must be a number")] - InvalidNumberUniform { shader: String, uniform: String }, - #[error("Uniform '{uniform}' for shader '{shader}' must be a vector of length {expected_length}")] - InvalidVectorUniform { - shader: String, - uniform: String, - expected_length: usize, - }, - #[error("Shader '{shader}' does not support uniform '{uniform}'")] - UnsupportedUniform { shader: String, uniform: String }, } diff --git a/rust/crates/gpu/src/shader_registry.rs b/rust/crates/gpu/src/shader_registry.rs deleted file mode 100644 index b6af173e..00000000 --- a/rust/crates/gpu/src/shader_registry.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::collections::HashMap; - -use crate::{GPU_TEXTURE_FORMAT, GpuError}; - -const FULLSCREEN_SHADER_SOURCE: &str = include_str!("shaders/fullscreen.wgsl"); -const GAUSSIAN_BLUR_SHADER_SOURCE: &str = include_str!("shaders/gaussian_blur.wgsl"); -const BLIT_SHADER_SOURCE: &str = include_str!("shaders/blit.wgsl"); -pub const GAUSSIAN_BLUR_SHADER_ID: &str = "gaussian-blur"; - -pub struct ShaderRegistry { - effect_texture_bind_group_layout: wgpu::BindGroupLayout, - effect_uniform_bind_group_layout: wgpu::BindGroupLayout, - effect_pipelines: HashMap<String, wgpu::RenderPipeline>, - blit_pipeline: wgpu::RenderPipeline, -} - -impl ShaderRegistry { - pub fn new(device: &wgpu::Device) -> Self { - let effect_texture_bind_group_layout = - device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("gpu-effect-texture-bind-group-layout"), - entries: &[ - wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Texture { - multisampled: false, - view_dimension: wgpu::TextureViewDimension::D2, - sample_type: wgpu::TextureSampleType::Float { filterable: true }, - }, - count: None, - }, - wgpu::BindGroupLayoutEntry { - binding: 1, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), - count: None, - }, - ], - }); - let effect_uniform_bind_group_layout = - device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { - label: Some("gpu-effect-uniform-bind-group-layout"), - entries: &[wgpu::BindGroupLayoutEntry { - binding: 0, - visibility: wgpu::ShaderStages::FRAGMENT, - ty: wgpu::BindingType::Buffer { - ty: wgpu::BufferBindingType::Uniform, - has_dynamic_offset: false, - min_binding_size: None, - }, - count: None, - }], - }); - - let vertex_shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("gpu-fullscreen-shader"), - source: wgpu::ShaderSource::Wgsl(FULLSCREEN_SHADER_SOURCE.into()), - }); - let gaussian_blur_shader_module = - device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("gpu-gaussian-blur-shader"), - source: wgpu::ShaderSource::Wgsl(GAUSSIAN_BLUR_SHADER_SOURCE.into()), - }); - let blit_shader_module = device.create_shader_module(wgpu::ShaderModuleDescriptor { - label: Some("gpu-blit-shader"), - source: wgpu::ShaderSource::Wgsl(BLIT_SHADER_SOURCE.into()), - }); - let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("gpu-effect-pipeline-layout"), - bind_group_layouts: &[ - Some(&effect_texture_bind_group_layout), - Some(&effect_uniform_bind_group_layout), - ], - immediate_size: 0, - }); - let blit_pipeline_layout = - device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { - label: Some("gpu-blit-pipeline-layout"), - bind_group_layouts: &[Some(&effect_texture_bind_group_layout)], - immediate_size: 0, - }); - let gaussian_blur_pipeline = - device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("gpu-gaussian-blur-pipeline"), - layout: Some(&pipeline_layout), - vertex: wgpu::VertexState { - module: &vertex_shader_module, - entry_point: Some("vertex_main"), - buffers: &[wgpu::VertexBufferLayout { - array_stride: std::mem::size_of::<[f32; 2]>() as u64, - step_mode: wgpu::VertexStepMode::Vertex, - attributes: &[wgpu::VertexAttribute { - format: wgpu::VertexFormat::Float32x2, - offset: 0, - shader_location: 0, - }], - }], - compilation_options: wgpu::PipelineCompilationOptions::default(), - }, - fragment: Some(wgpu::FragmentState { - module: &gaussian_blur_shader_module, - entry_point: Some("fragment_main"), - targets: &[Some(wgpu::ColorTargetState { - format: GPU_TEXTURE_FORMAT, - blend: None, - write_mask: wgpu::ColorWrites::ALL, - })], - compilation_options: wgpu::PipelineCompilationOptions::default(), - }), - primitive: wgpu::PrimitiveState::default(), - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview_mask: None, - cache: None, - }); - let blit_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { - label: Some("gpu-blit-pipeline"), - layout: Some(&blit_pipeline_layout), - vertex: wgpu::VertexState { - module: &vertex_shader_module, - entry_point: Some("vertex_main"), - buffers: &[wgpu::VertexBufferLayout { - array_stride: std::mem::size_of::<[f32; 2]>() as u64, - step_mode: wgpu::VertexStepMode::Vertex, - attributes: &[wgpu::VertexAttribute { - format: wgpu::VertexFormat::Float32x2, - offset: 0, - shader_location: 0, - }], - }], - compilation_options: wgpu::PipelineCompilationOptions::default(), - }, - fragment: Some(wgpu::FragmentState { - module: &blit_shader_module, - entry_point: Some("fragment_main"), - targets: &[Some(wgpu::ColorTargetState { - format: GPU_TEXTURE_FORMAT, - blend: None, - write_mask: wgpu::ColorWrites::ALL, - })], - compilation_options: wgpu::PipelineCompilationOptions::default(), - }), - primitive: wgpu::PrimitiveState::default(), - depth_stencil: None, - multisample: wgpu::MultisampleState::default(), - multiview_mask: None, - cache: None, - }); - - let effect_pipelines = HashMap::from([( - GAUSSIAN_BLUR_SHADER_ID.to_string(), - gaussian_blur_pipeline, - )]); - - Self { - effect_texture_bind_group_layout, - effect_uniform_bind_group_layout, - effect_pipelines, - blit_pipeline, - } - } - - pub fn get_effect_pipeline(&self, shader: &str) -> Result<&wgpu::RenderPipeline, GpuError> { - self.effect_pipelines - .get(shader) - .ok_or_else(|| GpuError::UnknownEffectShader { - shader: shader.to_string(), - }) - } - - pub fn effect_texture_bind_group_layout(&self) -> &wgpu::BindGroupLayout { - &self.effect_texture_bind_group_layout - } - - pub fn effect_uniform_bind_group_layout(&self) -> &wgpu::BindGroupLayout { - &self.effect_uniform_bind_group_layout - } - - pub fn blit_pipeline(&self) -> &wgpu::RenderPipeline { - &self.blit_pipeline - } -} diff --git a/rust/crates/masks/Cargo.toml b/rust/crates/masks/Cargo.toml new file mode 100644 index 00000000..58e10712 --- /dev/null +++ b/rust/crates/masks/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "masks" +version = "0.1.0" +edition = "2024" + +[lib] +path = "src/masks.rs" +crate-type = ["rlib"] + +[dependencies] +bytemuck = { version = "1.25.0", features = ["derive"] } +gpu = { version = "0.1.0", path = "../gpu" } +wgpu = "29.0.1" diff --git a/rust/crates/gpu/src/mask_feather.rs b/rust/crates/masks/src/feather.rs index 029b59ed..e58572f4 100644 --- a/rust/crates/gpu/src/mask_feather.rs +++ b/rust/crates/masks/src/feather.rs @@ -1,9 +1,9 @@ use bytemuck::{Pod, Zeroable}; +use gpu::{FULLSCREEN_SHADER_SOURCE, GPU_TEXTURE_FORMAT, GpuContext}; use wgpu::util::DeviceExt; -use crate::{GPU_TEXTURE_FORMAT, context::GpuContext}; +use crate::SdfPipeline; -const FULLSCREEN_SHADER_SOURCE: &str = include_str!("shaders/fullscreen.wgsl"); const JFA_DISTANCE_SHADER_SOURCE: &str = include_str!("shaders/jfa_distance.wgsl"); pub struct ApplyMaskFeatherOptions<'a> { @@ -14,6 +14,7 @@ pub struct ApplyMaskFeatherOptions<'a> { } pub struct MaskFeatherPipeline { + sdf_pipeline: SdfPipeline, inside_texture_bind_group_layout: wgpu::BindGroupLayout, outside_texture_bind_group_layout: wgpu::BindGroupLayout, uniform_bind_group_layout: wgpu::BindGroupLayout, @@ -29,7 +30,8 @@ struct DistanceUniformBuffer { } impl MaskFeatherPipeline { - pub fn new(device: &wgpu::Device) -> Self { + pub fn new(context: &GpuContext) -> Self { + let device = context.device(); let inside_texture_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("gpu-mask-distance-inside-layout"), @@ -140,6 +142,7 @@ impl MaskFeatherPipeline { }); Self { + sdf_pipeline: SdfPipeline::new(context), inside_texture_bind_group_layout, outside_texture_bind_group_layout, uniform_bind_group_layout, @@ -157,11 +160,10 @@ impl MaskFeatherPipeline { feather, }: ApplyMaskFeatherOptions<'_>, ) -> wgpu::Texture { - let sdf = context - .sdf_pipeline() + let sdf = self + .sdf_pipeline .compute_signed_distance_field(context, mask, width, height); - let output_texture = - context.create_render_texture(width, height, "gpu-mask-feather-output"); + let output_texture = context.create_render_texture(width, height, "masks-feather-output"); let inside_view = sdf .inside_texture .create_view(&wgpu::TextureViewDescriptor::default()); @@ -201,17 +203,18 @@ impl MaskFeatherPipeline { }, ], }); - let uniform_buffer = context - .device() - .create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("gpu-mask-distance-uniform-buffer"), - contents: bytemuck::bytes_of(&DistanceUniformBuffer { - resolution: [width as f32, height as f32], - feather_half: feather / 2.0, - _padding: 0.0, - }), - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - }); + let uniform_buffer = + context + .device() + .create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("gpu-mask-distance-uniform-buffer"), + contents: bytemuck::bytes_of(&DistanceUniformBuffer { + resolution: [width as f32, height as f32], + feather_half: feather / 2.0, + _padding: 0.0, + }), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); let uniform_bind_group = context .device() .create_bind_group(&wgpu::BindGroupDescriptor { @@ -222,11 +225,12 @@ impl MaskFeatherPipeline { resource: uniform_buffer.as_entire_binding(), }], }); - let mut encoder = context - .device() - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("gpu-mask-distance-command-encoder"), - }); + let mut encoder = + context + .device() + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("gpu-mask-distance-command-encoder"), + }); { let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { diff --git a/rust/crates/masks/src/masks.rs b/rust/crates/masks/src/masks.rs new file mode 100644 index 00000000..2bf33342 --- /dev/null +++ b/rust/crates/masks/src/masks.rs @@ -0,0 +1,5 @@ +mod feather; +mod sdf; + +pub use feather::{ApplyMaskFeatherOptions, MaskFeatherPipeline}; +pub use sdf::{SdfPipeline, SignedDistanceFieldTextures}; diff --git a/rust/crates/gpu/src/sdf_pipeline.rs b/rust/crates/masks/src/sdf.rs index d2612fe1..0e7ec01a 100644 --- a/rust/crates/gpu/src/sdf_pipeline.rs +++ b/rust/crates/masks/src/sdf.rs @@ -1,9 +1,7 @@ use bytemuck::{Pod, Zeroable}; +use gpu::{FULLSCREEN_SHADER_SOURCE, GPU_TEXTURE_FORMAT, GpuContext}; use wgpu::util::DeviceExt; -use crate::{GPU_TEXTURE_FORMAT, context::GpuContext}; - -const FULLSCREEN_SHADER_SOURCE: &str = include_str!("shaders/fullscreen.wgsl"); const JFA_INIT_SHADER_SOURCE: &str = include_str!("shaders/jfa_init.wgsl"); const JFA_STEP_SHADER_SOURCE: &str = include_str!("shaders/jfa_step.wgsl"); @@ -36,7 +34,8 @@ struct JfaStepUniformBuffer { } impl SdfPipeline { - pub fn new(device: &wgpu::Device) -> Self { + pub fn new(context: &GpuContext) -> Self { + let device = context.device(); let texture_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("gpu-sdf-texture-bind-group-layout"), @@ -213,13 +212,14 @@ impl SdfPipeline { }, ], }); - let uniform_buffer = context - .device() - .create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: Some("gpu-sdf-uniform-buffer"), - contents: uniform_buffer_bytes, - usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, - }); + let uniform_buffer = + context + .device() + .create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("gpu-sdf-uniform-buffer"), + contents: uniform_buffer_bytes, + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); let uniform_bind_group = context .device() .create_bind_group(&wgpu::BindGroupDescriptor { @@ -230,11 +230,12 @@ impl SdfPipeline { resource: uniform_buffer.as_entire_binding(), }], }); - let mut encoder = context - .device() - .create_command_encoder(&wgpu::CommandEncoderDescriptor { - label: Some("gpu-sdf-command-encoder"), - }); + let mut encoder = + context + .device() + .create_command_encoder(&wgpu::CommandEncoderDescriptor { + label: Some("gpu-sdf-command-encoder"), + }); { let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { diff --git a/rust/crates/gpu/src/shaders/jfa_distance.wgsl b/rust/crates/masks/src/shaders/jfa_distance.wgsl index e65a7023..e65a7023 100644 --- a/rust/crates/gpu/src/shaders/jfa_distance.wgsl +++ b/rust/crates/masks/src/shaders/jfa_distance.wgsl diff --git a/rust/crates/gpu/src/shaders/jfa_init.wgsl b/rust/crates/masks/src/shaders/jfa_init.wgsl index 29aa38c7..29aa38c7 100644 --- a/rust/crates/gpu/src/shaders/jfa_init.wgsl +++ b/rust/crates/masks/src/shaders/jfa_init.wgsl diff --git a/rust/crates/gpu/src/shaders/jfa_step.wgsl b/rust/crates/masks/src/shaders/jfa_step.wgsl index e20d7fb0..e20d7fb0 100644 --- a/rust/crates/gpu/src/shaders/jfa_step.wgsl +++ b/rust/crates/masks/src/shaders/jfa_step.wgsl diff --git a/rust/crates/time/Cargo.toml b/rust/crates/time/Cargo.toml index bf66e4e2..070695dc 100644 --- a/rust/crates/time/Cargo.toml +++ b/rust/crates/time/Cargo.toml @@ -5,13 +5,14 @@ edition = "2024" [lib] path = "src/time.rs" -crate-type = ["rlib", "cdylib"] +crate-type = ["rlib"] [dependencies] bridge = { version = "0.1.0", path = "../bridge" } +num-traits = "0.2.19" serde = { version = "1", features = ["derive"] } tsify-next = { version = "0.5", optional = true } wasm-bindgen = { version = "0.2.115", optional = true } [features] -wasm = ["dep:wasm-bindgen", "dep:tsify-next", "tsify-next/js"]
\ No newline at end of file +wasm = ["dep:wasm-bindgen", "dep:tsify-next", "tsify-next/js"] diff --git a/rust/crates/time/src/frame_rate.rs b/rust/crates/time/src/frame_rate.rs new file mode 100644 index 00000000..7af10649 --- /dev/null +++ b/rust/crates/time/src/frame_rate.rs @@ -0,0 +1,121 @@ +use serde::{Deserialize, Serialize}; + +use crate::media_time::TICKS_PER_SECOND; + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi, into_wasm_abi))] +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, PartialEq)] +pub struct FrameRate { + pub numerator: u32, + pub denominator: u32, +} + +impl FrameRate { + pub const FPS_23_976: Self = Self { + numerator: 24_000, + denominator: 1_001, + }; + pub const FPS_24: Self = Self { + numerator: 24, + denominator: 1, + }; + pub const FPS_25: Self = Self { + numerator: 25, + denominator: 1, + }; + pub const FPS_29_97: Self = Self { + numerator: 30_000, + denominator: 1_001, + }; + pub const FPS_30: Self = Self { + numerator: 30, + denominator: 1, + }; + pub const FPS_48: Self = Self { + numerator: 48, + denominator: 1, + }; + pub const FPS_50: Self = Self { + numerator: 50, + denominator: 1, + }; + pub const FPS_59_94: Self = Self { + numerator: 60_000, + denominator: 1_001, + }; + pub const FPS_60: Self = Self { + numerator: 60, + denominator: 1, + }; + pub const FPS_120: Self = Self { + numerator: 120, + denominator: 1, + }; + + pub const fn new(numerator: u32, denominator: u32) -> Self { + Self { + numerator, + denominator, + } + } + + pub const fn is_valid(self) -> bool { + self.numerator > 0 && self.denominator > 0 + } + + pub fn as_f64(self) -> Option<f64> { + if !self.is_valid() { + return None; + } + + Some(f64::from(self.numerator) / f64::from(self.denominator)) + } + + pub fn frame_number_upper_bound(self) -> Option<u32> { + if !self.is_valid() { + return None; + } + + Some(self.numerator.div_ceil(self.denominator)) + } + + pub fn ticks_per_frame(self) -> Option<i64> { + if !self.is_valid() { + return None; + } + + let tick_numerator = TICKS_PER_SECOND.checked_mul(i64::from(self.denominator))?; + let tick_denominator = i64::from(self.numerator); + if tick_numerator % tick_denominator != 0 { + return None; + } + + Some(tick_numerator / tick_denominator) + } +} + +#[cfg(test)] +mod tests { + use super::FrameRate; + + #[test] + fn resolves_ticks_per_standard_frame_rate() { + assert_eq!(FrameRate::FPS_23_976.ticks_per_frame(), Some(5_005)); + assert_eq!(FrameRate::FPS_24.ticks_per_frame(), Some(5_000)); + assert_eq!(FrameRate::FPS_25.ticks_per_frame(), Some(4_800)); + assert_eq!(FrameRate::FPS_29_97.ticks_per_frame(), Some(4_004)); + assert_eq!(FrameRate::FPS_30.ticks_per_frame(), Some(4_000)); + assert_eq!(FrameRate::FPS_48.ticks_per_frame(), Some(2_500)); + assert_eq!(FrameRate::FPS_50.ticks_per_frame(), Some(2_400)); + assert_eq!(FrameRate::FPS_59_94.ticks_per_frame(), Some(2_002)); + assert_eq!(FrameRate::FPS_60.ticks_per_frame(), Some(2_000)); + assert_eq!(FrameRate::FPS_120.ticks_per_frame(), Some(1_000)); + } + + #[test] + fn rejects_invalid_or_unsupported_rates() { + assert_eq!(FrameRate::new(0, 1).ticks_per_frame(), None); + assert_eq!(FrameRate::new(1, 0).ticks_per_frame(), None); + assert_eq!(FrameRate::new(7, 3).ticks_per_frame(), None); + } +} diff --git a/rust/crates/time/src/media_time.rs b/rust/crates/time/src/media_time.rs new file mode 100644 index 00000000..11d268e5 --- /dev/null +++ b/rust/crates/time/src/media_time.rs @@ -0,0 +1,428 @@ +use std::ops::{Add, Div, Mul, Neg, Sub}; + +use bridge::export; +use num_traits::ToPrimitive; +use serde::{Deserialize, Serialize}; + +use crate::frame_rate::FrameRate; + +#[export] +pub const TICKS_PER_SECOND: i64 = 120_000; +const TICKS_PER_SECOND_F64: f64 = TICKS_PER_SECOND as f64; + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi, into_wasm_abi))] +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct MediaTime(i64); + +impl MediaTime { + pub const ZERO: Self = Self(0); + pub const ONE_TICK: Self = Self(1); + + pub const fn from_ticks(ticks: i64) -> Self { + Self(ticks) + } + + pub const fn as_ticks(self) -> i64 { + self.0 + } + + pub fn from_seconds_f64(seconds: f64) -> Option<Self> { + if !seconds.is_finite() { + return None; + } + + let ticks = (seconds * TICKS_PER_SECOND_F64).round().to_i64()?; + Some(Self(ticks)) + } + + pub fn to_seconds_f64(self) -> f64 { + self.0.to_f64().unwrap_or(0.0) / TICKS_PER_SECOND_F64 + } + + pub fn from_frame(frame: i64, rate: FrameRate) -> Option<Self> { + let ticks_per_frame = rate.ticks_per_frame()?; + Some(Self(frame.checked_mul(ticks_per_frame)?)) + } + + pub fn to_frame_round(self, rate: FrameRate) -> Option<i64> { + let ticks_per_frame = rate.ticks_per_frame()?; + let remainder = self.0.rem_euclid(ticks_per_frame); + let floor = self.0.div_euclid(ticks_per_frame); + if remainder * 2 >= ticks_per_frame { + Some(floor + 1) + } else { + Some(floor) + } + } + + pub fn to_frame_floor(self, rate: FrameRate) -> Option<i64> { + let ticks_per_frame = rate.ticks_per_frame()?; + Some(self.0.div_euclid(ticks_per_frame)) + } + + pub fn round_to_frame(self, rate: FrameRate) -> Option<Self> { + Self::from_frame(self.to_frame_round(rate)?, rate) + } + + pub fn floor_to_frame(self, rate: FrameRate) -> Option<Self> { + let ticks_per_frame = rate.ticks_per_frame()?; + Some(Self(self.0.div_euclid(ticks_per_frame) * ticks_per_frame)) + } + + pub fn is_frame_aligned(self, rate: FrameRate) -> Option<bool> { + let ticks_per_frame = rate.ticks_per_frame()?; + Some(self.0.rem_euclid(ticks_per_frame) == 0) + } + + pub fn last_frame_time(self, rate: FrameRate) -> Option<Self> { + if self <= Self::ZERO { + return Some(Self::ZERO); + } + + let last_inclusive_tick = self.0.checked_sub(1).unwrap_or(0); + Self::from_ticks(last_inclusive_tick).floor_to_frame(rate) + } + + pub fn snapped_seek_time(self, duration: Self, rate: FrameRate) -> Option<Self> { + let snapped = self.round_to_frame(rate)?; + Some(snapped.clamp(Self::ZERO, duration)) + } + + pub fn clamp(self, min: Self, max: Self) -> Self { + Self(self.0.clamp(min.0, max.0)) + } + + pub fn min(self, other: Self) -> Self { + Self(self.0.min(other.0)) + } + + pub fn max(self, other: Self) -> Self { + Self(self.0.max(other.0)) + } +} + +impl Add for MediaTime { + type Output = Self; + + fn add(self, rhs: Self) -> Self::Output { + Self(self.0 + rhs.0) + } +} + +impl Sub for MediaTime { + type Output = Self; + + fn sub(self, rhs: Self) -> Self::Output { + Self(self.0 - rhs.0) + } +} + +impl Neg for MediaTime { + type Output = Self; + + fn neg(self) -> Self::Output { + Self(-self.0) + } +} + +impl Mul<i64> for MediaTime { + type Output = Self; + + fn mul(self, rhs: i64) -> Self::Output { + Self(self.0 * rhs) + } +} + +impl Div<i64> for MediaTime { + type Output = Self; + + fn div(self, rhs: i64) -> Self::Output { + Self(self.0 / rhs) + } +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeFromSecondsOptions { + pub seconds: f64, +} + +#[export] +pub fn media_time_from_seconds( + MediaTimeFromSecondsOptions { seconds }: MediaTimeFromSecondsOptions, +) -> Option<MediaTime> { + MediaTime::from_seconds_f64(seconds) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeToSecondsOptions { + pub time: MediaTime, +} + +#[export] +pub fn media_time_to_seconds(MediaTimeToSecondsOptions { time }: MediaTimeToSecondsOptions) -> f64 { + time.to_seconds_f64() +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeFromFrameOptions { + pub frame: i64, + pub rate: FrameRate, +} + +#[export] +pub fn media_time_from_frame( + MediaTimeFromFrameOptions { frame, rate }: MediaTimeFromFrameOptions, +) -> Option<MediaTime> { + MediaTime::from_frame(frame, rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeToFrameOptions { + pub time: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn media_time_to_frame( + MediaTimeToFrameOptions { time, rate }: MediaTimeToFrameOptions, +) -> Option<i64> { + time.to_frame_round(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RoundToFrameOptions { + pub time: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn round_to_frame( + RoundToFrameOptions { time, rate }: RoundToFrameOptions, +) -> Option<MediaTime> { + time.round_to_frame(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FloorToFrameOptions { + pub time: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn floor_to_frame( + FloorToFrameOptions { time, rate }: FloorToFrameOptions, +) -> Option<MediaTime> { + time.floor_to_frame(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct IsFrameAlignedOptions { + pub time: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn is_frame_aligned( + IsFrameAlignedOptions { time, rate }: IsFrameAlignedOptions, +) -> Option<bool> { + time.is_frame_aligned(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LastFrameTimeOptions { + pub duration: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn last_frame_time( + LastFrameTimeOptions { duration, rate }: LastFrameTimeOptions, +) -> Option<MediaTime> { + duration.last_frame_time(rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SnappedSeekTimeOptions { + pub time: MediaTime, + pub duration: MediaTime, + pub rate: FrameRate, +} + +#[export] +pub fn snapped_seek_time( + SnappedSeekTimeOptions { + time, + duration, + rate, + }: SnappedSeekTimeOptions, +) -> Option<MediaTime> { + time.snapped_seek_time(duration, rate) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeAddOptions { + pub lhs: MediaTime, + pub rhs: MediaTime, +} + +#[export] +pub fn media_time_add(MediaTimeAddOptions { lhs, rhs }: MediaTimeAddOptions) -> MediaTime { + lhs + rhs +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeSubOptions { + pub lhs: MediaTime, + pub rhs: MediaTime, +} + +#[export] +pub fn media_time_sub(MediaTimeSubOptions { lhs, rhs }: MediaTimeSubOptions) -> MediaTime { + lhs - rhs +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeMinOptions { + pub lhs: MediaTime, + pub rhs: MediaTime, +} + +#[export] +pub fn media_time_min(MediaTimeMinOptions { lhs, rhs }: MediaTimeMinOptions) -> MediaTime { + lhs.min(rhs) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeMaxOptions { + pub lhs: MediaTime, + pub rhs: MediaTime, +} + +#[export] +pub fn media_time_max(MediaTimeMaxOptions { lhs, rhs }: MediaTimeMaxOptions) -> MediaTime { + lhs.max(rhs) +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaTimeClampOptions { + pub time: MediaTime, + pub min: MediaTime, + pub max: MediaTime, +} + +#[export] +pub fn media_time_clamp( + MediaTimeClampOptions { time, min, max }: MediaTimeClampOptions, +) -> MediaTime { + time.clamp(min, max) +} + +#[cfg(test)] +mod tests { + use crate::frame_rate::FrameRate; + + use super::{MediaTime, TICKS_PER_SECOND}; + + #[test] + fn converts_between_seconds_and_ticks() { + assert_eq!( + MediaTime::from_seconds_f64(1.5), + Some(MediaTime::from_ticks(180_000)) + ); + assert_eq!(MediaTime::from_ticks(180_000).to_seconds_f64(), 1.5); + assert_eq!(TICKS_PER_SECOND, 120_000); + } + + #[test] + fn rejects_non_finite_seconds() { + assert_eq!(MediaTime::from_seconds_f64(f64::NAN), None); + assert_eq!(MediaTime::from_seconds_f64(f64::INFINITY), None); + assert_eq!(MediaTime::from_seconds_f64(f64::NEG_INFINITY), None); + } + + #[test] + fn snaps_to_the_nearest_frame() { + let rate = FrameRate::FPS_30; + let time = MediaTime::from_seconds_f64(1.26).unwrap(); + + assert_eq!(time.to_frame_round(rate), Some(38)); + assert_eq!( + time.round_to_frame(rate), + Some(MediaTime::from_ticks(152_000)) + ); + } + + #[test] + fn floors_to_frame() { + let rate = FrameRate::FPS_30; + let ticks_per_frame = 4_000; + let time = MediaTime::from_ticks(ticks_per_frame * 5 + 1); + + assert_eq!(time.to_frame_floor(rate), Some(5)); + assert_eq!(time.to_frame_round(rate), Some(5)); + + let almost_next = MediaTime::from_ticks(ticks_per_frame * 5 + ticks_per_frame / 2); + assert_eq!(almost_next.to_frame_floor(rate), Some(5)); + assert_eq!(almost_next.to_frame_round(rate), Some(6)); + } + + #[test] + fn computes_last_frame_time_and_snapped_seek_time() { + let rate = FrameRate::new(5, 1); + let duration = MediaTime::from_seconds_f64(10.0).unwrap(); + + assert_eq!( + duration.last_frame_time(rate), + Some(MediaTime::from_seconds_f64(9.8).unwrap()), + ); + assert_eq!( + MediaTime::from_seconds_f64(10.0) + .unwrap() + .snapped_seek_time(duration, rate), + Some(MediaTime::from_seconds_f64(10.0).unwrap()), + ); + } +} diff --git a/rust/crates/time/src/time.rs b/rust/crates/time/src/time.rs index 84b69327..3d45c47f 100644 --- a/rust/crates/time/src/time.rs +++ b/rust/crates/time/src/time.rs @@ -1,422 +1,19 @@ -use bridge::export; -use serde::{Deserialize, Serialize}; +mod frame_rate; +mod media_time; +mod timecode; -const SECONDS_PER_HOUR: f64 = 3600.0; -const SECONDS_PER_MINUTE: f64 = 60.0; -const CENTISECONDS_PER_SECOND: f64 = 100.0; - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi, into_wasm_abi))] -#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, PartialEq)] -pub enum TimeCodeFormat { - #[serde(rename = "MM:SS")] - MmSs, - #[serde(rename = "HH:MM:SS")] - HhMmSs, - #[serde(rename = "HH:MM:SS:CS")] - HhMmSsCs, - #[serde(rename = "HH:MM:SS:FF")] - HhMmSsFf, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct RoundToFrameOptions { - pub time: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct FormatTimeCodeOptions { - pub time_in_seconds: f64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub format: Option<TimeCodeFormat>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fps: Option<f64>, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct ParseTimeCodeOptions { - pub time_code: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub format: Option<TimeCodeFormat>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub fps: Option<f64>, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GuessTimeCodeFormatOptions { - pub time_code: String, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct TimeToFrameOptions { - pub time: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct FrameToTimeOptions { - pub frame: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct SnapTimeToFrameOptions { - pub time: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetSnappedSeekTimeOptions { - pub raw_time: f64, - pub duration: f64, - pub fps: f64, -} - -#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] -#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetLastFrameTimeOptions { - pub duration: f64, - pub fps: f64, -} - -#[export] -pub fn round_to_frame(RoundToFrameOptions { time, fps }: RoundToFrameOptions) -> f64 { - (time * fps).round() / fps -} - -#[export] -pub fn format_time_code( - FormatTimeCodeOptions { - time_in_seconds, - format, - fps, - }: FormatTimeCodeOptions, -) -> Option<String> { - let format = format.unwrap_or(TimeCodeFormat::HhMmSsCs); - let hours = (time_in_seconds / SECONDS_PER_HOUR).floor() as u64; - let minutes = ((time_in_seconds % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE).floor() as u64; - let seconds = (time_in_seconds % SECONDS_PER_MINUTE).floor() as u64; - let centiseconds = ((time_in_seconds % 1.0) * CENTISECONDS_PER_SECOND).floor() as u64; - - match format { - TimeCodeFormat::MmSs => Some(format!("{minutes:02}:{seconds:02}")), - TimeCodeFormat::HhMmSs => Some(format!("{hours:02}:{minutes:02}:{seconds:02}")), - TimeCodeFormat::HhMmSsCs => Some(format!( - "{hours:02}:{minutes:02}:{seconds:02}:{centiseconds:02}", - )), - TimeCodeFormat::HhMmSsFf => { - let fps = fps?; - if fps <= 0.0 { - return None; - } - - let frames = ((time_in_seconds % 1.0) * fps).floor() as u64; - Some(format!( - "{hours:02}:{minutes:02}:{seconds:02}:{frames:02}", - )) - } - } -} - -#[export] -pub fn parse_time_code( - ParseTimeCodeOptions { - time_code, - format, - fps, - }: ParseTimeCodeOptions, -) -> Option<f64> { - if time_code.trim().is_empty() { - return None; - } - - let format = format.unwrap_or(TimeCodeFormat::HhMmSsCs); - let parts = time_code - .trim() - .split(':') - .map(|part| part.parse::<u32>().ok()) - .collect::<Option<Vec<_>>>()?; - - match format { - TimeCodeFormat::MmSs => { - let [minutes, seconds] = parts.as_slice() else { - return None; - }; - if *seconds >= SECONDS_PER_MINUTE as u32 { - return None; - } - - Some((*minutes as f64 * SECONDS_PER_MINUTE) + *seconds as f64) - } - TimeCodeFormat::HhMmSs => { - let [hours, minutes, seconds] = parts.as_slice() else { - return None; - }; - if *minutes >= SECONDS_PER_MINUTE as u32 || *seconds >= SECONDS_PER_MINUTE as u32 { - return None; - } - - Some( - (*hours as f64 * SECONDS_PER_HOUR) - + (*minutes as f64 * SECONDS_PER_MINUTE) - + *seconds as f64, - ) - } - TimeCodeFormat::HhMmSsCs => { - let [hours, minutes, seconds, centiseconds] = parts.as_slice() else { - return None; - }; - if *minutes >= SECONDS_PER_MINUTE as u32 - || *seconds >= SECONDS_PER_MINUTE as u32 - || *centiseconds >= CENTISECONDS_PER_SECOND as u32 - { - return None; - } - - Some( - (*hours as f64 * SECONDS_PER_HOUR) - + (*minutes as f64 * SECONDS_PER_MINUTE) - + *seconds as f64 - + (*centiseconds as f64 / CENTISECONDS_PER_SECOND), - ) - } - TimeCodeFormat::HhMmSsFf => { - let fps = fps?; - if fps <= 0.0 { - return None; - } - - let [hours, minutes, seconds, frames] = parts.as_slice() else { - return None; - }; - if *minutes >= SECONDS_PER_MINUTE as u32 - || *seconds >= SECONDS_PER_MINUTE as u32 - || *frames as f64 >= fps - { - return None; - } - - Some( - (*hours as f64 * SECONDS_PER_HOUR) - + (*minutes as f64 * SECONDS_PER_MINUTE) - + *seconds as f64 - + (*frames as f64 / fps), - ) - } - } -} - -#[export] -pub fn guess_time_code_format( - GuessTimeCodeFormatOptions { time_code }: GuessTimeCodeFormatOptions, -) -> Option<TimeCodeFormat> { - if time_code.trim().is_empty() { - return None; - } - - let part_count = time_code - .split(':') - .try_fold(0usize, |count, part| { - part.parse::<u32>().ok().map(|_| count + 1) - })?; - - match part_count { - 2 => Some(TimeCodeFormat::MmSs), - 3 => Some(TimeCodeFormat::HhMmSs), - 4 => Some(TimeCodeFormat::HhMmSsFf), - _ => None, - } -} - -#[export] -pub fn time_to_frame(TimeToFrameOptions { time, fps }: TimeToFrameOptions) -> f64 { - (time * fps).round() -} - -#[export] -pub fn frame_to_time(FrameToTimeOptions { frame, fps }: FrameToTimeOptions) -> f64 { - frame / fps -} - -#[export] -pub fn snap_time_to_frame(SnapTimeToFrameOptions { time, fps }: SnapTimeToFrameOptions) -> f64 { - if fps <= 0.0 { - return time; - } - - frame_to_time(FrameToTimeOptions { - frame: time_to_frame(TimeToFrameOptions { time, fps }), - fps, - }) -} - -#[export] -pub fn get_snapped_seek_time( - GetSnappedSeekTimeOptions { - raw_time, - duration, - fps, - }: GetSnappedSeekTimeOptions, -) -> f64 { - let snapped_time = snap_time_to_frame(SnapTimeToFrameOptions { time: raw_time, fps }); - let last_frame = get_last_frame_time(GetLastFrameTimeOptions { duration, fps }); - snapped_time.clamp(0.0, last_frame) -} - -#[export] -pub fn get_last_frame_time( - GetLastFrameTimeOptions { duration, fps }: GetLastFrameTimeOptions, -) -> f64 { - if duration <= 0.0 { - return 0.0; - } - - if fps <= 0.0 { - return duration; - } - - let frame_offset = 1.0 / fps; - (duration - frame_offset).max(0.0) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn rounds_to_the_nearest_frame() { - assert_eq!(round_to_frame(RoundToFrameOptions { time: 1.24, fps: 10.0 }), 1.2); - assert_eq!(round_to_frame(RoundToFrameOptions { time: 1.26, fps: 10.0 }), 1.3); - } - - #[test] - fn formats_default_time_codes() { - assert_eq!( - format_time_code(FormatTimeCodeOptions { - time_in_seconds: 3723.45, - format: None, - fps: None, - }), - Some("01:02:03:44".to_string()), - ); - assert_eq!( - format_time_code(FormatTimeCodeOptions { - time_in_seconds: 65.0, - format: Some(TimeCodeFormat::MmSs), - fps: None, - }), - Some("01:05".to_string()), - ); - } - - #[test] - fn formats_frame_based_time_codes() { - assert_eq!( - format_time_code(FormatTimeCodeOptions { - time_in_seconds: 1.5, - format: Some(TimeCodeFormat::HhMmSsFf), - fps: Some(30.0), - }), - Some("00:00:01:15".to_string()), - ); - assert_eq!( - format_time_code(FormatTimeCodeOptions { - time_in_seconds: 1.5, - format: Some(TimeCodeFormat::HhMmSsFf), - fps: None, - }), - None, - ); - } - - #[test] - fn parses_time_codes() { - assert_eq!( - parse_time_code(ParseTimeCodeOptions { - time_code: "01:05".to_string(), - format: Some(TimeCodeFormat::MmSs), - fps: None, - }), - Some(65.0), - ); - assert_eq!( - parse_time_code(ParseTimeCodeOptions { - time_code: "00:00:01:15".to_string(), - format: Some(TimeCodeFormat::HhMmSsFf), - fps: Some(30.0), - }), - Some(1.5), - ); - assert_eq!( - parse_time_code(ParseTimeCodeOptions { - time_code: "00:00:01:30".to_string(), - format: Some(TimeCodeFormat::HhMmSsFf), - fps: Some(30.0), - }), - None, - ); - } - - #[test] - fn guesses_time_code_formats() { - assert_eq!( - guess_time_code_format(GuessTimeCodeFormatOptions { time_code: "01:05".to_string() }), - Some(TimeCodeFormat::MmSs), - ); - assert_eq!( - guess_time_code_format(GuessTimeCodeFormatOptions { - time_code: "00:00:01".to_string(), - }), - Some(TimeCodeFormat::HhMmSs), - ); - assert_eq!( - guess_time_code_format(GuessTimeCodeFormatOptions { - time_code: "00:00:01:15".to_string(), - }), - Some(TimeCodeFormat::HhMmSsFf), - ); - } - - #[test] - fn snaps_and_clamps_seek_time() { - assert_eq!(time_to_frame(TimeToFrameOptions { time: 1.26, fps: 10.0 }), 13.0); - assert_eq!(frame_to_time(FrameToTimeOptions { frame: 13.0, fps: 10.0 }), 1.3); - assert_eq!(snap_time_to_frame(SnapTimeToFrameOptions { time: 1.26, fps: 10.0 }), 1.3); - assert_eq!(get_last_frame_time(GetLastFrameTimeOptions { duration: 10.0, fps: 5.0 }), 9.8); - assert_eq!( - get_snapped_seek_time(GetSnappedSeekTimeOptions { - raw_time: 10.0, - duration: 10.0, - fps: 5.0, - }), - 9.8, - ); - } -} +pub use frame_rate::FrameRate; +pub use media_time::{ + FloorToFrameOptions, IsFrameAlignedOptions, LastFrameTimeOptions, MediaTime, + MediaTimeAddOptions, MediaTimeClampOptions, MediaTimeFromFrameOptions, + MediaTimeFromSecondsOptions, MediaTimeMaxOptions, MediaTimeMinOptions, MediaTimeSubOptions, + MediaTimeToFrameOptions, MediaTimeToSecondsOptions, RoundToFrameOptions, + SnappedSeekTimeOptions, TICKS_PER_SECOND, floor_to_frame, is_frame_aligned, last_frame_time, + media_time_add, media_time_clamp, media_time_from_frame, media_time_from_seconds, + media_time_max, media_time_min, media_time_sub, media_time_to_frame, media_time_to_seconds, + round_to_frame, snapped_seek_time, +}; +pub use timecode::{ + FormatTimecodeOptions, GuessTimecodeFormatOptions, ParseTimecodeOptions, TimeCodeFormat, + format_timecode, guess_timecode_format, parse_timecode, +}; diff --git a/rust/crates/time/src/timecode.rs b/rust/crates/time/src/timecode.rs new file mode 100644 index 00000000..0eb484d3 --- /dev/null +++ b/rust/crates/time/src/timecode.rs @@ -0,0 +1,287 @@ +use bridge::export; +use serde::{Deserialize, Serialize}; + +use crate::{ + frame_rate::FrameRate, + media_time::{MediaTime, TICKS_PER_SECOND}, +}; + +const SECONDS_PER_HOUR: i64 = 3_600; +const SECONDS_PER_MINUTE: i64 = 60; +const CENTISECONDS_PER_SECOND: i64 = 100; +const TICKS_PER_CENTISECOND: i64 = TICKS_PER_SECOND / CENTISECONDS_PER_SECOND; + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi, into_wasm_abi))] +#[derive(Serialize, Deserialize, Clone, Copy, Debug, Eq, PartialEq)] +pub enum TimeCodeFormat { + #[serde(rename = "MM:SS")] + MmSs, + #[serde(rename = "HH:MM:SS")] + HhMmSs, + #[serde(rename = "HH:MM:SS:CS")] + HhMmSsCs, + #[serde(rename = "HH:MM:SS:FF")] + HhMmSsFf, +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FormatTimecodeOptions { + pub time: MediaTime, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option<TimeCodeFormat>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate: Option<FrameRate>, +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ParseTimecodeOptions { + pub time_code: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub format: Option<TimeCodeFormat>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rate: Option<FrameRate>, +} + +#[cfg_attr(feature = "wasm", derive(tsify_next::Tsify))] +#[cfg_attr(feature = "wasm", tsify(from_wasm_abi))] +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GuessTimecodeFormatOptions { + pub time_code: String, +} + +#[export] +pub fn guess_timecode_format( + GuessTimecodeFormatOptions { time_code }: GuessTimecodeFormatOptions, +) -> Option<TimeCodeFormat> { + if time_code.trim().is_empty() { + return None; + } + + let part_count = time_code + .trim() + .split(':') + .try_fold(0usize, |count, part| { + part.parse::<u32>().ok().map(|_| count + 1) + })?; + + match part_count { + 2 => Some(TimeCodeFormat::MmSs), + 3 => Some(TimeCodeFormat::HhMmSs), + 4 => Some(TimeCodeFormat::HhMmSsFf), + _ => None, + } +} + +#[export] +pub fn format_timecode( + FormatTimecodeOptions { time, format, rate }: FormatTimecodeOptions, +) -> Option<String> { + let format = format.unwrap_or(TimeCodeFormat::HhMmSsCs); + let total_ticks = u64::try_from(time.as_ticks().max(0)).ok()?; + let ticks_per_second = u64::try_from(TICKS_PER_SECOND).ok()?; + let total_seconds = total_ticks / ticks_per_second; + let hour_ticks = u64::try_from(SECONDS_PER_HOUR).ok()? * ticks_per_second; + let minute_ticks = u64::try_from(SECONDS_PER_MINUTE).ok()? * ticks_per_second; + let seconds_per_minute = u64::try_from(SECONDS_PER_MINUTE).ok()?; + let ticks_per_centisecond = u64::try_from(TICKS_PER_CENTISECOND).ok()?; + + let hours = total_ticks / hour_ticks; + let minutes = (total_ticks % hour_ticks) / minute_ticks; + let seconds = total_seconds % seconds_per_minute; + let second_ticks = total_ticks % ticks_per_second; + let centiseconds = second_ticks / ticks_per_centisecond; + + match format { + TimeCodeFormat::MmSs => Some(format!("{minutes:02}:{seconds:02}")), + TimeCodeFormat::HhMmSs => Some(format!("{hours:02}:{minutes:02}:{seconds:02}")), + TimeCodeFormat::HhMmSsCs => Some(format!( + "{hours:02}:{minutes:02}:{seconds:02}:{centiseconds:02}" + )), + TimeCodeFormat::HhMmSsFf => { + let rate = rate?; + let ticks_per_frame = rate.ticks_per_frame()?; + let frames = second_ticks / u64::try_from(ticks_per_frame).ok()?; + Some(format!("{hours:02}:{minutes:02}:{seconds:02}:{frames:02}")) + } + } +} + +#[export] +pub fn parse_timecode( + ParseTimecodeOptions { + time_code, + format, + rate, + }: ParseTimecodeOptions, +) -> Option<MediaTime> { + if time_code.trim().is_empty() { + return None; + } + + let format = format.unwrap_or(TimeCodeFormat::HhMmSsCs); + let parts = time_code + .trim() + .split(':') + .map(|part| part.parse::<u32>().ok()) + .collect::<Option<Vec<_>>>()?; + + match format { + TimeCodeFormat::MmSs => { + let [minutes, seconds] = parts.as_slice() else { + return None; + }; + if i64::from(*seconds) >= SECONDS_PER_MINUTE { + return None; + } + + Some(MediaTime::from_ticks( + (i64::from(*minutes) * SECONDS_PER_MINUTE + i64::from(*seconds)) * TICKS_PER_SECOND, + )) + } + TimeCodeFormat::HhMmSs => { + let [hours, minutes, seconds] = parts.as_slice() else { + return None; + }; + if i64::from(*minutes) >= SECONDS_PER_MINUTE + || i64::from(*seconds) >= SECONDS_PER_MINUTE + { + return None; + } + + Some(MediaTime::from_ticks( + (i64::from(*hours) * SECONDS_PER_HOUR + + i64::from(*minutes) * SECONDS_PER_MINUTE + + i64::from(*seconds)) + * TICKS_PER_SECOND, + )) + } + TimeCodeFormat::HhMmSsCs => { + let [hours, minutes, seconds, centiseconds] = parts.as_slice() else { + return None; + }; + if i64::from(*minutes) >= SECONDS_PER_MINUTE + || i64::from(*seconds) >= SECONDS_PER_MINUTE + || i64::from(*centiseconds) >= CENTISECONDS_PER_SECOND + { + return None; + } + + Some(MediaTime::from_ticks( + (i64::from(*hours) * SECONDS_PER_HOUR + + i64::from(*minutes) * SECONDS_PER_MINUTE + + i64::from(*seconds)) + * TICKS_PER_SECOND + + i64::from(*centiseconds) * TICKS_PER_CENTISECOND, + )) + } + TimeCodeFormat::HhMmSsFf => { + let rate = rate?; + let frame_upper_bound = rate.frame_number_upper_bound()?; + let [hours, minutes, seconds, frames] = parts.as_slice() else { + return None; + }; + if i64::from(*minutes) >= SECONDS_PER_MINUTE + || i64::from(*seconds) >= SECONDS_PER_MINUTE + || *frames >= frame_upper_bound + { + return None; + } + + Some( + MediaTime::from_ticks( + (i64::from(*hours) * SECONDS_PER_HOUR + + i64::from(*minutes) * SECONDS_PER_MINUTE + + i64::from(*seconds)) + * TICKS_PER_SECOND, + ) + MediaTime::from_frame(i64::from(*frames), rate)?, + ) + } + } +} + +#[cfg(test)] +mod tests { + use crate::frame_rate::FrameRate; + use crate::media_time::MediaTime; + + use super::{FormatTimecodeOptions, GuessTimecodeFormatOptions, ParseTimecodeOptions}; + use super::{TimeCodeFormat, format_timecode, guess_timecode_format, parse_timecode}; + + #[test] + fn formats_default_and_frame_timecodes() { + assert_eq!( + format_timecode(FormatTimecodeOptions { + time: MediaTime::from_seconds_f64(3723.45).unwrap(), + format: None, + rate: None, + }), + Some("01:02:03:45".to_string()), + ); + assert_eq!( + format_timecode(FormatTimecodeOptions { + time: MediaTime::from_seconds_f64(1.5).unwrap(), + format: Some(TimeCodeFormat::HhMmSsFf), + rate: Some(FrameRate::FPS_30), + }), + Some("00:00:01:15".to_string()), + ); + } + + #[test] + fn parses_timecodes() { + assert_eq!( + parse_timecode(ParseTimecodeOptions { + time_code: "01:05".to_string(), + format: Some(TimeCodeFormat::MmSs), + rate: None, + }), + Some(MediaTime::from_seconds_f64(65.0).unwrap()), + ); + assert_eq!( + parse_timecode(ParseTimecodeOptions { + time_code: "00:00:01:15".to_string(), + format: Some(TimeCodeFormat::HhMmSsFf), + rate: Some(FrameRate::FPS_30), + }), + Some(MediaTime::from_seconds_f64(1.5).unwrap()), + ); + assert_eq!( + parse_timecode(ParseTimecodeOptions { + time_code: "00:00:01:30".to_string(), + format: Some(TimeCodeFormat::HhMmSsFf), + rate: Some(FrameRate::FPS_30), + }), + None, + ); + } + + #[test] + fn guesses_timecode_formats() { + assert_eq!( + guess_timecode_format(GuessTimecodeFormatOptions { + time_code: "01:05".to_string(), + }), + Some(TimeCodeFormat::MmSs), + ); + assert_eq!( + guess_timecode_format(GuessTimecodeFormatOptions { + time_code: "00:00:01".to_string(), + }), + Some(TimeCodeFormat::HhMmSs), + ); + assert_eq!( + guess_timecode_format(GuessTimecodeFormatOptions { + time_code: "00:00:01:15".to_string(), + }), + Some(TimeCodeFormat::HhMmSsFf), + ); + } +} diff --git a/rust/wasm/Cargo.toml b/rust/wasm/Cargo.toml index a5b0d9e0..88f92a5c 100644 --- a/rust/wasm/Cargo.toml +++ b/rust/wasm/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "opencut-wasm" -version = "0.1.3" +version = "0.2.3" edition = "2024" description = "Shared video editor logic compiled to WebAssembly" repository = "https://github.com/opencut/opencut" @@ -11,11 +11,19 @@ path = "src/wasm.rs" crate-type = ["cdylib", "rlib"] [dependencies] -gpu = { version = "0.1.0", path = "../crates/gpu" } +bridge = { version = "0.1.0", path = "../crates/bridge" } +effects = { version = "0.1.0", path = "../crates/effects" } +gpu = { version = "0.1.0", path = "../crates/gpu", features = ["wasm"] } js-sys = "0.3.93" +masks = { version = "0.1.0", path = "../crates/masks" } +num-traits = "0.2.19" serde = { version = "1.0.228", features = ["derive"] } serde-wasm-bindgen = "0.6.5" time = { version = "0.1.0", path = "../crates/time", features = ["wasm"] } wasm-bindgen = "0.2.116" wasm-bindgen-futures = "0.4.66" web-sys = { version = "0.3.93", features = ["OffscreenCanvas", "HtmlCanvasElement", "CanvasRenderingContext2d", "Document", "Window"] } + +[features] +default = ["wasm"] +wasm = [] diff --git a/rust/wasm/README.md b/rust/wasm/README.md index e90c58c2..8147bb08 100644 --- a/rust/wasm/README.md +++ b/rust/wasm/README.md @@ -11,7 +11,10 @@ npm install opencut-wasm ## Usage ```ts -import { formatTimeCode } from "opencut-wasm"; +import { formatTimecode, mediaTimeFromSeconds } from "opencut-wasm"; + +const ticks = mediaTimeFromSeconds(1.5); +const label = formatTimecode({ ticks }); ``` All exports are documented in the [TypeScript definitions](./opencut_wasm.d.ts). diff --git a/rust/wasm/src/effects.rs b/rust/wasm/src/effects.rs new file mode 100644 index 00000000..1f5c2947 --- /dev/null +++ b/rust/wasm/src/effects.rs @@ -0,0 +1,101 @@ +#![cfg(target_arch = "wasm32")] + +use effects::{ApplyEffectsOptions, EffectPass, UniformValue}; +use gpu::wgpu; +use js_sys::Object; +use serde::Deserialize; +use wasm_bindgen::{JsCast, JsValue, prelude::wasm_bindgen}; + +use crate::gpu::{ + import_canvas_texture, read_offscreen_canvas_property, read_serde_property, read_u32_property, + render_texture_to_canvas, with_gpu_runtime, +}; + +struct ApplyEffectPassesOptions { + source: wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + passes: Vec<EffectPassInput>, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EffectPassInput { + shader: String, + uniforms: Vec<EffectUniformInput>, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EffectUniformInput { + name: String, + value: Vec<f32>, +} + +#[wasm_bindgen(js_name = applyEffectPasses)] +pub fn apply_effect_passes(options: JsValue) -> Result<wgpu::web_sys::OffscreenCanvas, JsValue> { + let ApplyEffectPassesOptions { + source, + width, + height, + passes, + } = parse_apply_effect_passes_options(options)?; + + with_gpu_runtime(|runtime| { + let source_texture = import_canvas_texture( + &runtime.context, + &source, + width, + height, + "effects-input-texture", + ); + let effect_passes = map_effect_passes(passes); + let result_texture = runtime + .effects + .apply( + &runtime.context, + ApplyEffectsOptions { + source: &source_texture, + width, + height, + passes: &effect_passes, + }, + ) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + render_texture_to_canvas(&runtime.context, &result_texture, width, height) + }) +} + +fn map_effect_passes(effect_passes: Vec<EffectPassInput>) -> Vec<EffectPass> { + effect_passes + .into_iter() + .map(|pass| EffectPass { + shader: pass.shader, + uniforms: pass + .uniforms + .into_iter() + .map(|uniform| { + let value = if uniform.value.len() == 1 { + UniformValue::Number(uniform.value[0]) + } else { + UniformValue::Vector(uniform.value) + }; + (uniform.name, value) + }) + .collect(), + }) + .collect() +} + +fn parse_apply_effect_passes_options(value: JsValue) -> Result<ApplyEffectPassesOptions, JsValue> { + let object: Object = value + .dyn_into() + .map_err(|_| JsValue::from_str("applyEffectPasses expects an options object"))?; + + Ok(ApplyEffectPassesOptions { + source: read_offscreen_canvas_property(&object, "source")?, + width: read_u32_property(&object, "width")?, + height: read_u32_property(&object, "height")?, + passes: read_serde_property(&object, "passes")?, + }) +} diff --git a/rust/wasm/src/gpu.rs b/rust/wasm/src/gpu.rs new file mode 100644 index 00000000..70f3a241 --- /dev/null +++ b/rust/wasm/src/gpu.rs @@ -0,0 +1,123 @@ +#![cfg(target_arch = "wasm32")] + +use std::cell::RefCell; + +use effects::EffectPipeline; +use gpu::{GpuContext, wgpu}; +use js_sys::{Object, Reflect}; +use masks::MaskFeatherPipeline; +use serde::Deserialize; +use wasm_bindgen::{JsCast, JsValue, prelude::wasm_bindgen}; + +pub(crate) struct GpuRuntime { + pub(crate) context: GpuContext, + pub(crate) effects: EffectPipeline, + pub(crate) masks: MaskFeatherPipeline, +} + +thread_local! { + static GPU_RUNTIME: RefCell<Option<GpuRuntime>> = const { RefCell::new(None) }; +} + +#[wasm_bindgen(js_name = initializeGpu)] +pub async fn initialize_gpu() -> Result<(), JsValue> { + if GPU_RUNTIME.with(|runtime| runtime.borrow().is_some()) { + return Ok(()); + } + + let context = GpuContext::new() + .await + .map_err(|error| JsValue::from_str(&error.to_string()))?; + let effects = EffectPipeline::new(&context); + let masks = MaskFeatherPipeline::new(&context); + + GPU_RUNTIME.with(|runtime| { + runtime.replace(Some(GpuRuntime { + context, + effects, + masks, + })); + }); + + Ok(()) +} + +pub(crate) fn with_gpu_runtime<T>( + action: impl FnOnce(&GpuRuntime) -> Result<T, JsValue>, +) -> Result<T, JsValue> { + GPU_RUNTIME.with(|runtime| { + let borrow = runtime.borrow(); + let Some(gpu_runtime) = borrow.as_ref() else { + return Err(JsValue::from_str( + "GPU context not initialized. Call initializeGpu() first.", + )); + }; + action(gpu_runtime) + }) +} + +pub(crate) fn import_canvas_texture( + context: &GpuContext, + canvas: &wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + label: &'static str, +) -> wgpu::Texture { + context.import_offscreen_canvas_texture(canvas, width, height, label) +} + +pub(crate) fn render_texture_to_canvas( + context: &GpuContext, + texture: &wgpu::Texture, + width: u32, + height: u32, +) -> Result<wgpu::web_sys::OffscreenCanvas, JsValue> { + let canvas = wgpu::web_sys::OffscreenCanvas::new(width, height)?; + context + .render_texture_to_offscreen_canvas(texture, &canvas, width, height) + .map_err(|error| JsValue::from_str(&error.to_string()))?; + Ok(canvas) +} + +pub(crate) fn read_property(object: &Object, name: &str) -> Result<JsValue, JsValue> { + Reflect::get(object, &JsValue::from_str(name)) + .map_err(|_| JsValue::from_str(&format!("Missing property '{name}'"))) +} + +pub(crate) fn read_offscreen_canvas_property( + object: &Object, + name: &str, +) -> Result<wgpu::web_sys::OffscreenCanvas, JsValue> { + read_property(object, name)? + .dyn_into::<wgpu::web_sys::OffscreenCanvas>() + .map_err(|_| JsValue::from_str(&format!("Property '{name}' must be an OffscreenCanvas"))) +} + +pub(crate) fn read_u32_property(object: &Object, name: &str) -> Result<u32, JsValue> { + let value = read_property(object, name)?; + let Some(number) = value.as_f64() else { + return Err(JsValue::from_str(&format!( + "Property '{name}' must be a number" + ))); + }; + Ok(number as u32) +} + +pub(crate) fn read_f32_property(object: &Object, name: &str) -> Result<f32, JsValue> { + let value = read_property(object, name)?; + let Some(number) = value.as_f64() else { + return Err(JsValue::from_str(&format!( + "Property '{name}' must be a number" + ))); + }; + Ok(number as f32) +} + +pub(crate) fn read_serde_property<T>(object: &Object, name: &str) -> Result<T, JsValue> +where + T: for<'de> Deserialize<'de>, +{ + let value = read_property(object, name)?; + serde_wasm_bindgen::from_value(value) + .map_err(|error| JsValue::from_str(&format!("Invalid property '{name}': {error}"))) +} diff --git a/rust/wasm/src/gpu_bridge.rs b/rust/wasm/src/gpu_bridge.rs deleted file mode 100644 index 85db0c8b..00000000 --- a/rust/wasm/src/gpu_bridge.rs +++ /dev/null @@ -1,256 +0,0 @@ -#![cfg(target_arch = "wasm32")] - -use std::cell::RefCell; - -use gpu::{EffectPass, GpuContext, UniformValue, wgpu}; -use js_sys::{Object, Reflect}; -use serde::Deserialize; -use wasm_bindgen::{JsCast, JsValue, prelude::wasm_bindgen}; - -thread_local! { - static GPU_CONTEXT: RefCell<Option<GpuContext>> = const { RefCell::new(None) }; -} - -struct ApplyEffectPassesOptions { - source: wgpu::web_sys::OffscreenCanvas, - width: u32, - height: u32, - passes: Vec<EffectPassInput>, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct EffectPassInput { - shader: String, - uniforms: Vec<EffectUniformInput>, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct EffectUniformInput { - name: String, - value: Vec<f32>, -} - -struct ApplyMaskFeatherOptions { - mask: wgpu::web_sys::OffscreenCanvas, - width: u32, - height: u32, - feather: f32, -} - -#[wasm_bindgen(js_name = initializeGpu)] -pub async fn initialize_gpu() -> Result<(), JsValue> { - if GPU_CONTEXT.with(|context| context.borrow().is_some()) { - return Ok(()); - } - - let context = GpuContext::new() - .await - .map_err(|error| JsValue::from_str(&error.to_string()))?; - GPU_CONTEXT.with(|gpu_context| { - gpu_context.replace(Some(context)); - }); - Ok(()) -} - -#[wasm_bindgen(js_name = applyEffectPasses)] -pub fn apply_effect_passes( - options: JsValue, -) -> Result<wgpu::web_sys::OffscreenCanvas, JsValue> { - let ApplyEffectPassesOptions { - source, - width, - height, - passes, - } = parse_apply_effect_passes_options(options)?; - - with_gpu_context(|context| { - let source_texture = import_canvas_texture(context, &source, width, height)?; - let effect_passes = map_effect_passes(passes); - let result_texture = context - .apply_effects(gpu::ApplyEffectsOptions { - source: &source_texture, - width, - height, - passes: &effect_passes, - }) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - let output_canvas = wgpu::web_sys::OffscreenCanvas::new(width, height)?; - let surface = context - .instance() - .create_surface(wgpu::SurfaceTarget::OffscreenCanvas(output_canvas.clone())) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - - context - .render_texture_to_surface(&result_texture, &surface, width, height) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - Ok(output_canvas) - }) -} - -#[wasm_bindgen(js_name = applyMaskFeather)] -pub fn apply_mask_feather( - options: JsValue, -) -> Result<wgpu::web_sys::OffscreenCanvas, JsValue> { - let ApplyMaskFeatherOptions { - mask, - width, - height, - feather, - } = parse_apply_mask_feather_options(options)?; - - with_gpu_context(|context| { - let mask_texture = import_canvas_texture(context, &mask, width, height)?; - let result_texture = context.apply_mask_feather(gpu::ApplyMaskFeatherOptions { - mask: &mask_texture, - width, - height, - feather, - }); - let output_canvas = wgpu::web_sys::OffscreenCanvas::new(width, height)?; - let surface = context - .instance() - .create_surface(wgpu::SurfaceTarget::OffscreenCanvas(output_canvas.clone())) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - - context - .render_texture_to_surface(&result_texture, &surface, width, height) - .map_err(|error| JsValue::from_str(&error.to_string()))?; - Ok(output_canvas) - }) -} - -fn with_gpu_context<T>( - action: impl FnOnce(&GpuContext) -> Result<T, JsValue>, -) -> Result<T, JsValue> { - GPU_CONTEXT.with(|context| { - let borrow = context.borrow(); - let Some(gpu_context) = borrow.as_ref() else { - return Err(JsValue::from_str( - "GPU context not initialized. Call initializeGpu() first.", - )); - }; - action(gpu_context) - }) -} - -fn import_canvas_texture( - context: &GpuContext, - canvas: &wgpu::web_sys::OffscreenCanvas, - width: u32, - height: u32, -) -> Result<wgpu::Texture, JsValue> { - let texture = context.create_render_texture(width, height, "gpu-bridge-input-texture"); - context.queue().copy_external_image_to_texture( - &wgpu::CopyExternalImageSourceInfo { - source: wgpu::ExternalImageSource::OffscreenCanvas(canvas.clone()), - origin: wgpu::Origin2d::ZERO, - flip_y: true, - }, - wgpu::CopyExternalImageDestInfo { - texture: &texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - color_space: wgpu::PredefinedColorSpace::Srgb, - premultiplied_alpha: false, - }, - wgpu::Extent3d { - width, - height, - depth_or_array_layers: 1, - }, - ); - Ok(texture) -} - -fn map_effect_passes(effect_passes: Vec<EffectPassInput>) -> Vec<EffectPass> { - effect_passes - .into_iter() - .map(|pass| EffectPass { - shader: pass.shader, - uniforms: pass - .uniforms - .into_iter() - .map(|uniform| { - let value = if uniform.value.len() == 1 { - UniformValue::Number(uniform.value[0]) - } else { - UniformValue::Vector(uniform.value) - }; - (uniform.name, value) - }) - .collect(), - }) - .collect() -} - -fn parse_apply_effect_passes_options(value: JsValue) -> Result<ApplyEffectPassesOptions, JsValue> { - let object: Object = value - .dyn_into() - .map_err(|_| JsValue::from_str("applyEffectPasses expects an options object"))?; - - Ok(ApplyEffectPassesOptions { - source: read_offscreen_canvas_property(&object, "source")?, - width: read_u32_property(&object, "width")?, - height: read_u32_property(&object, "height")?, - passes: read_serde_property(&object, "passes")?, - }) -} - -fn parse_apply_mask_feather_options(value: JsValue) -> Result<ApplyMaskFeatherOptions, JsValue> { - let object: Object = value - .dyn_into() - .map_err(|_| JsValue::from_str("applyMaskFeather expects an options object"))?; - - Ok(ApplyMaskFeatherOptions { - mask: read_offscreen_canvas_property(&object, "mask")?, - width: read_u32_property(&object, "width")?, - height: read_u32_property(&object, "height")?, - feather: read_f32_property(&object, "feather")?, - }) -} - -fn read_property(object: &Object, name: &str) -> Result<JsValue, JsValue> { - Reflect::get(object, &JsValue::from_str(name)) - .map_err(|_| JsValue::from_str(&format!("Missing property '{name}'"))) -} - -fn read_offscreen_canvas_property( - object: &Object, - name: &str, -) -> Result<wgpu::web_sys::OffscreenCanvas, JsValue> { - read_property(object, name)? - .dyn_into::<wgpu::web_sys::OffscreenCanvas>() - .map_err(|_| JsValue::from_str(&format!("Property '{name}' must be an OffscreenCanvas"))) -} - -fn read_u32_property(object: &Object, name: &str) -> Result<u32, JsValue> { - let value = read_property(object, name)?; - let Some(number) = value.as_f64() else { - return Err(JsValue::from_str(&format!( - "Property '{name}' must be a number" - ))); - }; - Ok(number as u32) -} - -fn read_f32_property(object: &Object, name: &str) -> Result<f32, JsValue> { - let value = read_property(object, name)?; - let Some(number) = value.as_f64() else { - return Err(JsValue::from_str(&format!( - "Property '{name}' must be a number" - ))); - }; - Ok(number as f32) -} - -fn read_serde_property<T>(object: &Object, name: &str) -> Result<T, JsValue> -where - T: for<'de> Deserialize<'de>, -{ - let value = read_property(object, name)?; - serde_wasm_bindgen::from_value(value) - .map_err(|error| JsValue::from_str(&format!("Invalid property '{name}': {error}"))) -} diff --git a/rust/wasm/src/masks.rs b/rust/wasm/src/masks.rs new file mode 100644 index 00000000..e4f30e21 --- /dev/null +++ b/rust/wasm/src/masks.rs @@ -0,0 +1,60 @@ +#![cfg(target_arch = "wasm32")] + +use gpu::wgpu; +use js_sys::Object; +use wasm_bindgen::{JsCast, JsValue, prelude::wasm_bindgen}; + +use crate::gpu::{ + import_canvas_texture, read_f32_property, read_offscreen_canvas_property, read_u32_property, + render_texture_to_canvas, with_gpu_runtime, +}; + +struct ApplyMaskFeatherOptions { + mask: wgpu::web_sys::OffscreenCanvas, + width: u32, + height: u32, + feather: f32, +} + +#[wasm_bindgen(js_name = applyMaskFeather)] +pub fn apply_mask_feather(options: JsValue) -> Result<wgpu::web_sys::OffscreenCanvas, JsValue> { + let ApplyMaskFeatherOptions { + mask, + width, + height, + feather, + } = parse_apply_mask_feather_options(options)?; + + with_gpu_runtime(|runtime| { + let mask_texture = import_canvas_texture( + &runtime.context, + &mask, + width, + height, + "masks-input-texture", + ); + let result_texture = runtime.masks.apply_mask_feather( + &runtime.context, + masks::ApplyMaskFeatherOptions { + mask: &mask_texture, + width, + height, + feather, + }, + ); + render_texture_to_canvas(&runtime.context, &result_texture, width, height) + }) +} + +fn parse_apply_mask_feather_options(value: JsValue) -> Result<ApplyMaskFeatherOptions, JsValue> { + let object: Object = value + .dyn_into() + .map_err(|_| JsValue::from_str("applyMaskFeather expects an options object"))?; + + Ok(ApplyMaskFeatherOptions { + mask: read_offscreen_canvas_property(&object, "mask")?, + width: read_u32_property(&object, "width")?, + height: read_u32_property(&object, "height")?, + feather: read_f32_property(&object, "feather")?, + }) +} diff --git a/rust/wasm/src/wasm.rs b/rust/wasm/src/wasm.rs index 0a78a314..27ece766 100644 --- a/rust/wasm/src/wasm.rs +++ b/rust/wasm/src/wasm.rs @@ -1,6 +1,14 @@ #[cfg(target_arch = "wasm32")] -mod gpu_bridge; +mod effects; +#[cfg(target_arch = "wasm32")] +mod gpu; +#[cfg(target_arch = "wasm32")] +mod masks; #[cfg(target_arch = "wasm32")] -pub use gpu_bridge::*; +pub use effects::*; +#[cfg(target_arch = "wasm32")] +pub use gpu::*; +#[cfg(target_arch = "wasm32")] +pub use masks::*; pub use time::*; |