From f1043734438c8518a302f698d9ae6583a75f0c14 Mon Sep 17 00:00:00 2001 From: cashmere Date: Sun, 16 Aug 2026 06:16:56 +0200 Subject: feat: add transitions, track mixing, and voiceover controls --- apps/web/src/actions/use-editor-actions.ts | 41 ++- apps/web/src/commands/timeline/track/index.ts | 1 + .../commands/timeline/track/update-track-volume.ts | 38 +++ .../src/components/editor/panels/assets/index.tsx | 12 +- .../editor/panels/assets/views/transitions.tsx | 258 +++++++++++++++++ .../components/track-audio-properties.tsx | 158 ++++++++++ .../components/editor/panels/properties/index.tsx | 21 +- .../editor/panels/properties/registry.tsx | 38 ++- .../panels/properties/stores/properties-store.ts | 44 +-- apps/web/src/components/ui/slider.tsx | 63 ++-- apps/web/src/core/managers/audio-manager.ts | 1 + apps/web/src/core/managers/timeline-manager.ts | 15 +- apps/web/src/media/audio.ts | 42 ++- apps/web/src/params/registry.ts | 77 +++-- .../web/src/rendering/__tests__/transition.test.ts | 166 +++++++++++ apps/web/src/rendering/transition.ts | 194 +++++++++++++ .../__tests__/scene-builder-transition.test.ts | 99 +++++++ apps/web/src/services/renderer/nodes/text-node.ts | 2 + .../web/src/services/renderer/nodes/visual-node.ts | 3 + apps/web/src/services/renderer/resolve.ts | 67 +++-- apps/web/src/services/renderer/scene-builder.ts | 46 ++- .../web/src/timeline/__tests__/audio-state.test.ts | 65 +++++ apps/web/src/timeline/audio-state.ts | 20 +- .../__tests__/drop-target-transition.test.ts | 221 ++++++++++++++ apps/web/src/timeline/components/drop-target.ts | 90 +++++- apps/web/src/timeline/components/index.tsx | 140 ++++++--- .../src/timeline/components/timeline-toolbar.tsx | 27 +- .../drag-drop-controller-transition.test.ts | 190 ++++++++++++ .../timeline/controllers/drag-drop-controller.ts | 71 ++++- apps/web/src/timeline/defaults.ts | 9 + apps/web/src/timeline/drag.ts | 92 +++--- .../src/timeline/hooks/use-timeline-drag-drop.ts | 1 + apps/web/src/timeline/placement/track-factory.ts | 2 + apps/web/src/timeline/scenes.ts | 229 +++++++-------- apps/web/src/timeline/types.ts | 10 + apps/web/src/voiceover/__tests__/devices.test.ts | 72 +++++ .../src/voiceover/__tests__/enhancements.test.ts | 133 +++++++++ apps/web/src/voiceover/devices.ts | 93 ++++++ apps/web/src/voiceover/enhancements.ts | 321 +++++++++++++++++++++ apps/web/src/voiceover/live-waveform.tsx | 171 +++++++++++ apps/web/src/voiceover/recorder.ts | 40 ++- apps/web/src/voiceover/voiceover-popover.tsx | 276 ++++++++++++++++++ apps/web/src/voiceover/voiceover-store.ts | 57 +++- 43 files changed, 3355 insertions(+), 361 deletions(-) create mode 100644 apps/web/src/commands/timeline/track/update-track-volume.ts create mode 100644 apps/web/src/components/editor/panels/assets/views/transitions.tsx create mode 100644 apps/web/src/components/editor/panels/properties/components/track-audio-properties.tsx create mode 100644 apps/web/src/rendering/__tests__/transition.test.ts create mode 100644 apps/web/src/rendering/transition.ts create mode 100644 apps/web/src/services/renderer/__tests__/scene-builder-transition.test.ts create mode 100644 apps/web/src/timeline/__tests__/audio-state.test.ts create mode 100644 apps/web/src/timeline/components/__tests__/drop-target-transition.test.ts create mode 100644 apps/web/src/timeline/controllers/__tests__/drag-drop-controller-transition.test.ts create mode 100644 apps/web/src/voiceover/__tests__/devices.test.ts create mode 100644 apps/web/src/voiceover/__tests__/enhancements.test.ts create mode 100644 apps/web/src/voiceover/devices.ts create mode 100644 apps/web/src/voiceover/enhancements.ts create mode 100644 apps/web/src/voiceover/live-waveform.tsx create mode 100644 apps/web/src/voiceover/voiceover-popover.tsx (limited to 'apps/web') diff --git a/apps/web/src/actions/use-editor-actions.ts b/apps/web/src/actions/use-editor-actions.ts index ca16e989..b165d5ef 100644 --- a/apps/web/src/actions/use-editor-actions.ts +++ b/apps/web/src/actions/use-editor-actions.ts @@ -44,7 +44,11 @@ import { stopVoiceOverRecording, } from "@/voiceover/recorder"; import type { VoiceOverRecordingResult } from "@/voiceover/recorder"; -import { useVoiceOverStore } from "@/voiceover/voiceover-store"; +import { + useVoiceOverPreferencesStore, + useVoiceOverStore, +} from "@/voiceover/voiceover-store"; +import { applyVoiceOverEnhancementsToBlob } from "@/voiceover/enhancements"; const FREEZE_DURATION_SECONDS = 3; @@ -651,7 +655,9 @@ export function useEditorActions() { if (voiceOverState.status === "idle") { try { - await startVoiceOverRecording(); + await startVoiceOverRecording({ + deviceId: useVoiceOverPreferencesStore.getState().selectedDeviceId, + }); } catch (error) { toast.error(getVoiceOverErrorMessage({ error })); return; @@ -662,11 +668,16 @@ export function useEditorActions() { return; } - useVoiceOverStore.getState().stopRecording(); + if (voiceOverState.status === "processing") { + return; + } + + useVoiceOverStore.getState().processRecording(); const activeProject = editor.project.getActive(); if (!activeProject) { cancelVoiceOverRecording(); + useVoiceOverStore.getState().stopRecording(); return; } @@ -675,18 +686,35 @@ export function useEditorActions() { recording = await stopVoiceOverRecording(); } catch (error) { toast.error(getVoiceOverErrorMessage({ error })); + useVoiceOverStore.getState().stopRecording(); return; } if (recording.blob.size === 0) { toast.error("Voice-over recording is empty"); + useVoiceOverStore.getState().stopRecording(); + return; + } + + const enhancements = useVoiceOverPreferencesStore.getState().enhancements; + let assetBlob = recording.blob; + let assetMimeType = recording.mimeType; + try { + assetBlob = await applyVoiceOverEnhancementsToBlob({ + blob: recording.blob, + settings: enhancements, + }); + assetMimeType = assetBlob.type || recording.mimeType; + } catch (error) { + toast.error(getVoiceOverErrorMessage({ error })); + useVoiceOverStore.getState().stopRecording(); return; } const file = new File( - [recording.blob], - `voice-over.${getFileExtensionForMimeType({ mimeType: recording.mimeType })}`, - { type: recording.mimeType }, + [assetBlob], + `voice-over.${getFileExtensionForMimeType({ mimeType: assetMimeType })}`, + { type: assetMimeType }, ); const addMediaCommand = new AddMediaAssetCommand({ projectId: activeProject.metadata.id, @@ -715,6 +743,7 @@ export function useEditorActions() { editor.command.execute({ command: new BatchCommand([addMediaCommand, insertCommand]), }); + useVoiceOverStore.getState().stopRecording(); }, undefined, ); diff --git a/apps/web/src/commands/timeline/track/index.ts b/apps/web/src/commands/timeline/track/index.ts index 9d4bd9c3..4aa4ceaf 100644 --- a/apps/web/src/commands/timeline/track/index.ts +++ b/apps/web/src/commands/timeline/track/index.ts @@ -2,3 +2,4 @@ export { AddTrackCommand } from "./add-track"; export { RemoveTrackCommand } from "./remove-track"; export { ToggleTrackMuteCommand } from "./toggle-track-mute"; export { ToggleTrackVisibilityCommand } from "./toggle-track-visibility"; +export { UpdateTrackVolumeCommand } from "./update-track-volume"; diff --git a/apps/web/src/commands/timeline/track/update-track-volume.ts b/apps/web/src/commands/timeline/track/update-track-volume.ts new file mode 100644 index 00000000..ecd7f6ac --- /dev/null +++ b/apps/web/src/commands/timeline/track/update-track-volume.ts @@ -0,0 +1,38 @@ +import { Command, type CommandResult } from "@/commands/base-command"; +import { EditorCore } from "@/core"; +import type { SceneTracks } from "@/timeline"; +import { canTrackHaveAudio, updateTrackInSceneTracks } from "@/timeline"; +import { clampDb } from "@/timeline/audio-state"; + +export class UpdateTrackVolumeCommand extends Command { + private savedState: SceneTracks | null = null; + + constructor({ trackId, volume }: { trackId: string; volume: number }) { + super(); + this.trackId = trackId; + this.volume = clampDb(volume); + } + + private trackId: string; + private volume: number; + + execute(): CommandResult | undefined { + const editor = EditorCore.getInstance(); + this.savedState = editor.scenes.getActiveScene().tracks; + const updatedTracks = updateTrackInSceneTracks({ + tracks: this.savedState, + trackId: this.trackId, + update: (track) => + canTrackHaveAudio(track) ? { ...track, volume: this.volume } : track, + }); + editor.timeline.updateTracks(updatedTracks); + return undefined; + } + + undo(): void { + if (this.savedState) { + const editor = EditorCore.getInstance(); + editor.timeline.updateTracks(this.savedState); + } + } +} diff --git a/apps/web/src/components/editor/panels/assets/index.tsx b/apps/web/src/components/editor/panels/assets/index.tsx index c0204b12..6e134eda 100644 --- a/apps/web/src/components/editor/panels/assets/index.tsx +++ b/apps/web/src/components/editor/panels/assets/index.tsx @@ -1,10 +1,14 @@ "use client"; import { Separator } from "@/components/ui/separator"; -import { type Tab, useAssetsPanelStore } from "@/components/editor/panels/assets/assets-panel-store"; +import { + type Tab, + useAssetsPanelStore, +} from "@/components/editor/panels/assets/assets-panel-store"; import { TabBar } from "./tabbar"; import { MediaView } from "./views/assets"; import { SettingsView } from "./views/settings"; +import { TransitionsView } from "./views/transitions"; import { SoundsView } from "@/sounds/components/assets-view"; import { StickersView } from "@/stickers/components/assets-view"; import { TextView } from "@/text/components/assets-view"; @@ -21,11 +25,7 @@ export function AssetsPanel() { stickers: FEATURES.stickers ? : null, effects: FEATURES.effects ? : null, lens: FEATURES.effects ? : null, - transitions: ( -
- Transitions view coming soon... -
- ), + transitions: , settings: , }; diff --git a/apps/web/src/components/editor/panels/assets/views/transitions.tsx b/apps/web/src/components/editor/panels/assets/views/transitions.tsx new file mode 100644 index 00000000..e0236a2f --- /dev/null +++ b/apps/web/src/components/editor/panels/assets/views/transitions.tsx @@ -0,0 +1,258 @@ +"use client"; + +import { + ArrowRightDoubleIcon, + MagicWand05Icon, +} from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { PanelView } from "./base-panel"; +import { Button } from "@/components/ui/button"; +import { useEditor } from "@/editor/use-editor"; +import { useElementSelection } from "@/timeline/hooks/element/use-element-selection"; +import { isVisualElement } from "@/timeline/element-utils"; +import type { TimelineElement, TimelineTrack } from "@/timeline"; +import { + type TransitionKind, + withVisualTransitionParams, +} from "@/rendering/transition"; + +const transitions: Array<{ + kind: TransitionKind; + name: string; + description: string; + inDuration: number; + outDuration: number; +}> = [ + { + kind: "fade", + name: "Soft fade", + description: "Clean dissolve for slides and stills.", + inDuration: 0.35, + outDuration: 0, + }, + { + kind: "push-left", + name: "Slide left", + description: "Deck-like horizontal slide motion.", + inDuration: 0.45, + outDuration: 0, + }, + { + kind: "push-right", + name: "Slide right", + description: "Reverse horizontal slide motion.", + inDuration: 0.45, + outDuration: 0, + }, + { + kind: "zoom", + name: "Zoom in", + description: "Subtle punch-in with a fade edge.", + inDuration: 0.4, + outDuration: 0, + }, + { + kind: "zoom-out", + name: "Zoom out", + description: "Settle inward from a wider opening.", + inDuration: 0.4, + outDuration: 0, + }, + { + kind: "spin", + name: "Spin dissolve", + description: "A restrained turn with a soft fade.", + inDuration: 0.45, + outDuration: 0, + }, +]; + +export function TransitionsView() { + const editor = useEditor(); + const { selectedElements } = useElementSelection(); + const scene = useEditor((currentEditor) => + currentEditor.scenes.getActiveSceneOrNull(), + ); + const currentTime = useEditor((currentEditor) => + currentEditor.playback.getCurrentTime(), + ); + const selectedVisualElements = editor.timeline + .getElementsWithTracks({ elements: selectedElements }) + .filter(({ element }) => isVisualElement(element)); + const nearestTarget = scene + ? findNearestTransitionTarget({ + tracks: [scene.tracks.main, ...scene.tracks.overlay], + currentTime, + }) + : null; + const targets = + selectedVisualElements.length > 0 + ? selectedVisualElements + : nearestTarget + ? [nearestTarget] + : []; + const selectedCount = selectedVisualElements.length; + + const applyTransition = ({ + kind, + inDuration, + outDuration, + }: { + kind: TransitionKind; + inDuration: number; + outDuration: number; + }) => { + if (targets.length === 0) { + return; + } + + editor.timeline.updateElements({ + updates: targets.map(({ track, element }) => ({ + trackId: track.id, + elementId: element.id, + patch: { + params: withVisualTransitionParams({ + params: element.params, + transitionKind: kind, + inDuration, + outDuration, + }), + }, + })), + }); + }; + + return ( + +
+
+ + Apply at the cut +
+

+ {selectedCount > 0 + ? `${selectedCount} selected clip${selectedCount === 1 ? "" : "s"}.` + : nearestTarget + ? `Ready for the cut into ${nearestTarget.element.name}.` + : "Add at least two image or video clips to one track."} +

+
+ +
+ {transitions.map((transition) => ( + + ))} +
+
+ ); +} + +function findNearestTransitionTarget({ + tracks, + currentTime, +}: { + tracks: TimelineTrack[]; + currentTime: number; +}): { track: TimelineTrack; element: TimelineElement } | null { + let nearest: { track: TimelineTrack; element: TimelineElement } | null = null; + let nearestDistance = Number.POSITIVE_INFINITY; + for (const track of tracks) { + const elements = track.elements + .filter((element) => element.type === "video" || element.type === "image") + .slice() + .sort((left, right) => left.startTime - right.startTime); + for (let index = 1; index < elements.length; index += 1) { + const element = elements[index]; + const distance = Math.abs(element.startTime - currentTime); + if (distance < nearestDistance) { + nearest = { track, element }; + nearestDistance = distance; + } + } + } + return nearest; +} + +function TransitionCard({ + transition, + disabled, + onApply, +}: { + transition: (typeof transitions)[number]; + disabled: boolean; + onApply: (transition: (typeof transitions)[number]) => void; +}) { + const editor = useEditor(); + return ( + + ); +} + +function TransitionPreview({ kind }: { kind: TransitionKind }) { + return ( +
+
+
+
+ {kind === "fade" && ( +
+ )} + {kind === "push-left" && ( + + )} + {kind === "push-right" && ( + + )} + {kind === "zoom" && ( +
+ )} + {kind === "zoom-out" && ( +
+ )} + {kind === "spin" && ( +
+ )} +
+ ); +} diff --git a/apps/web/src/components/editor/panels/properties/components/track-audio-properties.tsx b/apps/web/src/components/editor/panels/properties/components/track-audio-properties.tsx new file mode 100644 index 00000000..4738d902 --- /dev/null +++ b/apps/web/src/components/editor/panels/properties/components/track-audio-properties.tsx @@ -0,0 +1,158 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { MusicNote03Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { NumberField } from "@/components/ui/number-field"; +import { Slider } from "@/components/ui/slider"; +import { Switch } from "@/components/ui/switch"; +import { + Section, + SectionContent, + SectionField, + SectionFields, +} from "@/components/section"; +import { useEditor } from "@/editor/use-editor"; +import type { AudioTrack, VideoTrack } from "@/timeline"; +import { clampDb, getTrackVolume } from "@/timeline/audio-state"; +import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/timeline/audio-constants"; +import { + formatNumberForDisplay, + getFractionDigitsForStep, + snapToStep, +} from "@/utils/math"; + +const VOLUME_STEP = 0.1; +const VOLUME_FRACTION_DIGITS = getFractionDigitsForStep({ step: VOLUME_STEP }); + +function normalizeVolume(value: number): number { + return clampDb(snapToStep({ value, step: VOLUME_STEP })); +} + +function formatVolume(value: number): string { + return formatNumberForDisplay({ + value, + maxFractionDigits: VOLUME_FRACTION_DIGITS, + }); +} + +export function TrackAudioProperties({ + track, +}: { + track: AudioTrack | VideoTrack; +}) { + const editor = useEditor(); + const sourceVolume = getTrackVolume({ track }); + const [volume, setVolume] = useState(sourceVolume); + const [volumeDraft, setVolumeDraft] = useState(formatVolume(sourceVolume)); + const textEditingRef = useRef(false); + const sliderEditingRef = useRef(false); + const previousTrackIdRef = useRef(track.id); + + useEffect(() => { + const trackChanged = previousTrackIdRef.current !== track.id; + if (trackChanged) { + previousTrackIdRef.current = track.id; + textEditingRef.current = false; + sliderEditingRef.current = false; + } + if (!trackChanged && (textEditingRef.current || sliderEditingRef.current)) { + return; + } + setVolume(sourceVolume); + setVolumeDraft(formatVolume(sourceVolume)); + }, [sourceVolume, track.id]); + + const commitVolume = (nextVolume: number) => { + const normalized = normalizeVolume(nextVolume); + setVolume(normalized); + setVolumeDraft(formatVolume(normalized)); + if (normalized !== sourceVolume) { + editor.timeline.updateTrackVolume({ + trackId: track.id, + volume: normalized, + }); + } + }; + + const commitVolumeDraft = () => { + textEditingRef.current = false; + const parsed = parseFloat(volumeDraft); + if (Number.isNaN(parsed)) { + setVolume(sourceVolume); + setVolumeDraft(formatVolume(sourceVolume)); + return; + } + commitVolume(parsed); + }; + + return ( +
+
+
+ +
+
+
{track.name}
+
Track audio
+
+
+
+ + + +
+ { + sliderEditingRef.current = true; + const nextVolume = normalizeVolume(values[0]); + setVolume(nextVolume); + setVolumeDraft(formatVolume(nextVolume)); + }} + onValueCommit={(values) => { + sliderEditingRef.current = false; + commitVolume(values[0]); + }} + /> + { + textEditingRef.current = true; + }} + onChange={(event) => { + const nextDraft = event.currentTarget.value; + setVolumeDraft(nextDraft); + const parsed = parseFloat(nextDraft); + if (!Number.isNaN(parsed)) { + setVolume(normalizeVolume(parsed)); + } + }} + onBlur={commitVolumeDraft} + /> +
+
+ + { + editor.timeline.toggleTrackMute({ trackId: track.id }); + }} + /> + +
+
+
+
+ ); +} diff --git a/apps/web/src/components/editor/panels/properties/index.tsx b/apps/web/src/components/editor/panels/properties/index.tsx index d946027d..f4ce9651 100644 --- a/apps/web/src/components/editor/panels/properties/index.tsx +++ b/apps/web/src/components/editor/panels/properties/index.tsx @@ -14,13 +14,32 @@ import { usePropertiesStore } from "./stores/properties-store"; import { getPropertiesConfig } from "./registry"; import { cn } from "@/utils/ui"; import { EmptyView } from "./empty-view"; +import { TrackAudioProperties } from "./components/track-audio-properties"; export function PropertiesPanel() { const editor = useEditor(); useEditor((e) => e.scenes.getActiveSceneOrNull()); useEditor((e) => e.media.getAssets()); const { selectedElements } = useElementSelection(); - const { activeTabPerType, setActiveTab } = usePropertiesStore(); + const { activeTabPerType, setActiveTab, selectedTrackId } = + usePropertiesStore(); + const selectedTrack = selectedTrackId + ? editor.timeline.getTrackById({ trackId: selectedTrackId }) + : null; + + if ( + selectedElements.length === 0 && + selectedTrack && + (selectedTrack.type === "audio" || selectedTrack.type === "video") + ) { + return ( +
+ + + +
+ ); + } if (selectedElements.length === 0) { return ( diff --git a/apps/web/src/components/editor/panels/properties/registry.tsx b/apps/web/src/components/editor/panels/properties/registry.tsx index 15f010f7..805098f8 100644 --- a/apps/web/src/components/editor/panels/properties/registry.tsx +++ b/apps/web/src/components/editor/panels/properties/registry.tsx @@ -23,7 +23,10 @@ import { DashboardSpeed02Icon, } from "@hugeicons/core-free-icons"; import { ElementParamsTab } from "./components/element-params-tab"; -import { ClipEffectsTab, StandaloneEffectTab } from "@/effects/components/effects-tab"; +import { + ClipEffectsTab, + StandaloneEffectTab, +} from "@/effects/components/effects-tab"; import { MasksTab } from "@/masks/components/masks-tab"; import { SpeedTab } from "@/speed/components/speed-tab"; import { GraphicTab } from "@/graphics/components/graphic-tab"; @@ -40,6 +43,11 @@ const TRANSFORM_PARAM_KEYS = [ const BLENDING_PARAM_KEYS = ["opacity", "blendMode"] as const; const AUDIO_PARAM_KEYS = ["volume", "muted"] as const; +const TRANSITION_PARAM_KEYS = [ + "transition.kind", + "transition.inDuration", + "transition.outDuration", +] as const; const TEXT_PARAM_KEYS = [ "content", "fontFamily", @@ -135,6 +143,25 @@ function buildAudioTab({ ), }; } +function buildTransitionTab({ + element, +}: { + element: VisualElement; +}): PropertiesTabDef { + return { + id: "transition", + label: "Transition", + icon: , + content: ({ trackId }) => ( + + ), + }; +} function buildSpeedTab({ element, @@ -202,7 +229,9 @@ function buildGraphicTab({ id: "graphic", label: "Graphic", icon: , - content: ({ trackId }) => , + content: ({ trackId }) => ( + + ), }; } @@ -232,6 +261,7 @@ function getTextConfig({ buildTextTab({ element }), buildTransformTab({ element }), buildBlendingTab({ element }), + buildTransitionTab({ element }), ], }; } @@ -251,6 +281,7 @@ function getVideoConfig({ ...(showAudioTab ? [buildAudioTab({ element })] : []), buildSpeedTab({ element }), buildBlendingTab({ element }), + buildTransitionTab({ element }), buildMasksTab({ element }), ...(FEATURES.effects ? [buildClipEffectsTab({ element })] : []), ], @@ -267,6 +298,7 @@ function getImageConfig({ tabs: [ buildTransformTab({ element }), buildBlendingTab({ element }), + buildTransitionTab({ element }), buildMasksTab({ element }), ...(FEATURES.effects ? [buildClipEffectsTab({ element })] : []), ], @@ -283,6 +315,7 @@ function getStickerConfig({ tabs: [ buildTransformTab({ element }), buildBlendingTab({ element }), + buildTransitionTab({ element }), ...(FEATURES.effects ? [buildClipEffectsTab({ element })] : []), ], }; @@ -299,6 +332,7 @@ function getGraphicConfig({ buildGraphicTab({ element }), buildTransformTab({ element }), buildBlendingTab({ element }), + buildTransitionTab({ element }), buildMasksTab({ element }), ...(FEATURES.effects ? [buildClipEffectsTab({ element })] : []), ], diff --git a/apps/web/src/components/editor/panels/properties/stores/properties-store.ts b/apps/web/src/components/editor/panels/properties/stores/properties-store.ts index f6304349..3303dae2 100644 --- a/apps/web/src/components/editor/panels/properties/stores/properties-store.ts +++ b/apps/web/src/components/editor/panels/properties/stores/properties-store.ts @@ -1,19 +1,25 @@ -import { create } from "zustand"; - -interface PropertiesState { - activeTabPerType: Record; - setActiveTab: (args: { elementType: string; tabId: string }) => void; - isTransformScaleLocked: boolean; - setTransformScaleLocked: (args: { locked: boolean }) => void; -} - -export const usePropertiesStore = create()((set) => ({ - activeTabPerType: {}, - setActiveTab: ({ elementType, tabId }) => - set((state) => ({ - activeTabPerType: { ...state.activeTabPerType, [elementType]: tabId }, - })), - isTransformScaleLocked: false, - setTransformScaleLocked: ({ locked }) => - set({ isTransformScaleLocked: locked }), -})); +import { create } from "zustand"; + +interface PropertiesState { + activeTabPerType: Record; + setActiveTab: (args: { elementType: string; tabId: string }) => void; + isTransformScaleLocked: boolean; + setTransformScaleLocked: (args: { locked: boolean }) => void; + selectedTrackId: string | null; + selectTrack: ({ trackId }: { trackId: string }) => void; + clearTrackSelection: () => void; +} + +export const usePropertiesStore = create()((set) => ({ + activeTabPerType: {}, + setActiveTab: ({ elementType, tabId }) => + set((state) => ({ + activeTabPerType: { ...state.activeTabPerType, [elementType]: tabId }, + })), + isTransformScaleLocked: false, + setTransformScaleLocked: ({ locked }) => + set({ isTransformScaleLocked: locked }), + selectedTrackId: null, + selectTrack: ({ trackId }) => set({ selectedTrackId: trackId }), + clearTrackSelection: () => set({ selectedTrackId: null }), +})); diff --git a/apps/web/src/components/ui/slider.tsx b/apps/web/src/components/ui/slider.tsx index aa14dfbd..35faf7e8 100644 --- a/apps/web/src/components/ui/slider.tsx +++ b/apps/web/src/components/ui/slider.tsx @@ -1,30 +1,33 @@ -"use client"; - -import * as React from "react"; -import { Slider as SliderPrimitive } from "radix-ui"; - -import { cn } from "@/utils/ui"; - -const Slider = React.forwardRef< - React.ElementRef, - React.ComponentPropsWithoutRef & { - className?: string; - } ->(({ className, ...props }, ref) => ( - - - - - - -)); -Slider.displayName = SliderPrimitive.Root.displayName; - -export { Slider }; +"use client"; + +import * as React from "react"; +import { Slider as SliderPrimitive } from "radix-ui"; + +import { cn } from "@/utils/ui"; + +const Slider = React.forwardRef< + React.ElementRef, + React.ComponentPropsWithoutRef & { + className?: string; + } +>(({ className, ...props }, ref) => ( + + + + + + +)); +Slider.displayName = SliderPrimitive.Root.displayName; + +export { Slider }; diff --git a/apps/web/src/core/managers/audio-manager.ts b/apps/web/src/core/managers/audio-manager.ts index d0716aad..59e3ba80 100644 --- a/apps/web/src/core/managers/audio-manager.ts +++ b/apps/web/src/core/managers/audio-manager.ts @@ -497,6 +497,7 @@ export class AudioManager { const points = buildAudioGainAutomation({ element: clip.timelineElement, fromLocalTime: startLocalTime, + trackVolume: clip.trackVolume, toLocalTime: clip.duration, }); diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 6fb28625..82987200 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -34,6 +34,7 @@ import { RemoveTrackCommand, ToggleTrackMuteCommand, ToggleTrackVisibilityCommand, + UpdateTrackVolumeCommand, InsertElementCommand, DeleteElementsCommand, DuplicateElementsCommand, @@ -178,6 +179,16 @@ export class TimelineManager { const command = new ToggleTrackMuteCommand(trackId); this.editor.command.execute({ command }); } + updateTrackVolume({ + trackId, + volume, + }: { + trackId: string; + volume: number; + }): void { + const command = new UpdateTrackVolumeCommand({ trackId, volume }); + this.editor.command.execute({ command }); + } toggleTrackVisibility({ trackId }: { trackId: string }): void { const command = new ToggleTrackVisibilityCommand(trackId); @@ -841,7 +852,9 @@ export class TimelineManager { }): void { const shouldMute = elements.some(({ trackId, elementId }) => { const element = this.getElementByRef({ trackId, elementId }); - return element && canElementHaveAudio(element) && !isElementMuted({ element }); + return ( + element && canElementHaveAudio(element) && !isElementMuted({ element }) + ); }); const nextUpdates = elements.flatMap(({ trackId, elementId }) => { diff --git a/apps/web/src/media/audio.ts b/apps/web/src/media/audio.ts index 7abaf90e..0e8ee826 100644 --- a/apps/web/src/media/audio.ts +++ b/apps/web/src/media/audio.ts @@ -10,6 +10,7 @@ import type { MediaAsset } from "@/media/types"; import { applyAudioMasteringToBuffer } from "@/media/audio-mastering"; import type { AudioCapableElement } from "@/timeline/audio-state"; import { + getTrackVolume, hasAnimatedVolume, isElementMuted, resolveEffectiveAudioGain, @@ -21,10 +22,7 @@ import { mediaSupportsAudio } from "@/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/retime"; import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny"; import { TICKS_PER_SECOND } from "@/wasm"; -import { - computeRmsBuckets, - type SampleBucket, -} from "@/media/waveform-summary"; +import { computeRmsBuckets, type SampleBucket } from "@/media/waveform-summary"; const MAX_AUDIO_CHANNELS = 2; const EXPORT_SAMPLE_RATE = 44100; @@ -37,6 +35,7 @@ export interface CollectedAudioElement { trimStart: number; trimEnd: number; volume: number; + trackVolume: number; muted: boolean; retime?: RetimeConfig; } @@ -89,6 +88,7 @@ export async function decodeAudioToFloat32({ export interface AudibleElementCandidate { element: AudioElement | VideoElement; mediaAsset: MediaAsset | null; + trackVolume: number; } export function collectAudibleCandidates({ @@ -114,7 +114,11 @@ export function collectAudibleCandidates({ : null; if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; - candidates.push({ element, mediaAsset }); + candidates.push({ + element, + mediaAsset, + trackVolume: canTrackHaveAudio(track) ? getTrackVolume({ track }) : 0, + }); } } @@ -148,7 +152,7 @@ export async function collectAudioElements({ ); const pendingElements: Array> = []; - for (const { element, mediaAsset } of candidates) { + for (const { element, mediaAsset, trackVolume } of candidates) { if (element.type === "audio") { pendingElements.push( resolveAudioBufferForElement({ @@ -166,10 +170,11 @@ export async function collectAudioElements({ trimEnd: element.trimEnd / TICKS_PER_SECOND, volume: resolveEffectiveAudioGain({ element, - trackMuted: false, + trackVolume, localTime: 0, }), muted: isElementMuted({ element }), + trackVolume, retime: element.retime, }; }), @@ -195,10 +200,11 @@ export async function collectAudioElements({ trimEnd: element.trimEnd / TICKS_PER_SECOND, volume: resolveEffectiveAudioGain({ element, - trackMuted: false, + trackVolume, localTime: 0, }), muted: isElementMuted({ element }), + trackVolume, retime: element.retime, }; }), @@ -359,6 +365,7 @@ export interface AudioClipSource { trimStart: number; trimEnd: number; volume: number; + trackVolume: number; muted: boolean; retime?: RetimeConfig; } @@ -401,10 +408,12 @@ async function fetchLibraryAudioClip({ element, muted, volume, + trackVolume, }: { element: LibraryAudioElement; muted: boolean; volume: number; + trackVolume: number; }): Promise { try { const response = await fetch(element.sourceUrl); @@ -427,6 +436,7 @@ async function fetchLibraryAudioClip({ trimStart: element.trimStart, trimEnd: element.trimEnd, volume, + trackVolume, muted, retime: element.retime, }; @@ -462,11 +472,13 @@ function collectMediaAudioClip({ mediaAsset, muted, volume, + trackVolume, }: { element: AudioCapableElement; mediaAsset: MediaAsset; muted: boolean; volume: number; + trackVolume: number; }): AudioClipSource { return { timelineElement: element, @@ -478,6 +490,7 @@ function collectMediaAudioClip({ trimStart: element.trimStart / TICKS_PER_SECOND, trimEnd: element.trimEnd / TICKS_PER_SECOND, volume, + trackVolume, muted, retime: element.retime, }; @@ -507,8 +520,12 @@ export async function collectAudioMixSources({ ? (mediaMap.get(element.mediaId) ?? null) : null; if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; + const trackVolume = canTrackHaveAudio(track) + ? getTrackVolume({ track }) + : 0; const volume = resolveEffectiveAudioGain({ element, + trackVolume, localTime: 0, }); @@ -562,6 +579,9 @@ export async function collectAudioClips({ for (const track of orderedTracks) { const isTrackMuted = canTrackHaveAudio(track) && track.muted; + const trackVolume = canTrackHaveAudio(track) + ? getTrackVolume({ track }) + : 0; for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; @@ -575,6 +595,7 @@ export async function collectAudioClips({ const volume = resolveEffectiveAudioGain({ element, trackMuted: isTrackMuted, + trackVolume, localTime: 0, }); @@ -589,11 +610,12 @@ export async function collectAudioClips({ mediaAsset, muted, volume, + trackVolume, }), ); } else { pendingLibraryClips.push( - fetchLibraryAudioClip({ element, muted, volume }), + fetchLibraryAudioClip({ element, muted, volume, trackVolume }), ); } continue; @@ -607,6 +629,7 @@ export async function collectAudioClips({ mediaAsset, muted, volume, + trackVolume, }), ); } @@ -855,6 +878,7 @@ function mixAudioChannels({ const gain = hasAnimatedVolume({ element: element.timelineElement }) ? resolveEffectiveAudioGain({ element: element.timelineElement, + trackVolume: element.trackVolume, localTime: clipTime, }) : element.volume; diff --git a/apps/web/src/params/registry.ts b/apps/web/src/params/registry.ts index 9e363c88..6456a360 100644 --- a/apps/web/src/params/registry.ts +++ b/apps/web/src/params/registry.ts @@ -1,20 +1,11 @@ -import type { - ParamDefinition, - ParamValue, - ParamValues, -} from "@/params"; +import type { ParamDefinition, ParamValue, ParamValues } from "@/params"; import { MIN_TRANSFORM_SCALE } from "@/animation/transform"; import type { BlendMode } from "@/rendering"; -import type { - ElementType, - TimelineElement, -} from "@/timeline"; +import { TRANSITION_KIND_OPTIONS } from "@/rendering/transition"; +import type { ElementType, TimelineElement } from "@/timeline"; import { DEFAULTS } from "@/timeline/defaults"; import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/timeline/audio-constants"; -import { - CORNER_RADIUS_MAX, - CORNER_RADIUS_MIN, -} from "@/text/background"; +import { CORNER_RADIUS_MAX, CORNER_RADIUS_MIN } from "@/text/background"; export type ElementParamDefinition = ParamDefinition & { @@ -46,13 +37,7 @@ export class DefinitionRegistry { this.entityName = entityName; } - register({ - key, - definition, - }: { - key: TKey; - definition: TDefinition; - }): void { + register({ key, definition }: { key: TKey; definition: TDefinition }): void { this.definitions.set(key, definition); } @@ -153,6 +138,36 @@ const visualElementParams: ElementParamDefinition[] = [ options: BLEND_MODE_OPTIONS, }, ]; +const transitionElementParams: ElementParamDefinition[] = [ + { + key: "transition.kind", + label: "Style", + type: "select", + default: DEFAULTS.element.transitionKind, + keyframable: false, + options: TRANSITION_KIND_OPTIONS, + }, + { + key: "transition.inDuration", + label: "In", + type: "number", + default: DEFAULTS.element.transitionDuration, + min: 0, + max: 5, + step: 0.1, + keyframable: false, + }, + { + key: "transition.outDuration", + label: "Out", + type: "number", + default: DEFAULTS.element.transitionDuration, + min: 0, + max: 5, + step: 0.1, + keyframable: false, + }, +]; const audioElementParams: ElementParamDefinition[] = [ { @@ -333,20 +348,31 @@ export const elementParamRegistry = new DefinitionRegistry< elementParamRegistry.register({ key: "video", - definition: [...visualElementParams, ...audioElementParams], + definition: [ + ...visualElementParams, + ...transitionElementParams, + ...audioElementParams, + ], +}); +elementParamRegistry.register({ + key: "image", + definition: [...visualElementParams, ...transitionElementParams], }); -elementParamRegistry.register({ key: "image", definition: visualElementParams }); elementParamRegistry.register({ key: "text", - definition: [...textElementParams, ...visualElementParams], + definition: [ + ...textElementParams, + ...visualElementParams, + ...transitionElementParams, + ], }); elementParamRegistry.register({ key: "sticker", - definition: visualElementParams, + definition: [...visualElementParams, ...transitionElementParams], }); elementParamRegistry.register({ key: "graphic", - definition: visualElementParams, + definition: [...visualElementParams, ...transitionElementParams], }); elementParamRegistry.register({ key: "audio", definition: audioElementParams }); elementParamRegistry.register({ key: "effect", definition: [] }); @@ -435,4 +461,3 @@ export function buildElementParamValues({ } return values; } - diff --git a/apps/web/src/rendering/__tests__/transition.test.ts b/apps/web/src/rendering/__tests__/transition.test.ts new file mode 100644 index 00000000..4421fe58 --- /dev/null +++ b/apps/web/src/rendering/__tests__/transition.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, test } from "bun:test"; +import { + applyVisualTransition, + buildTransitionFromParams, +} from "@/rendering/transition"; +import type { Transform } from "@/rendering"; +import { TICKS_PER_SECOND } from "@/wasm"; + +const baseTransform: Transform = { + scaleX: 1, + scaleY: 1, + position: { x: 0, y: 0 }, + rotate: 0, +}; + +describe("visual transitions", () => { + test("fade ramps opacity at clip edges", () => { + const transition = buildTransitionFromParams({ + params: { + "transition.kind": "fade", + "transition.inDuration": 1, + "transition.outDuration": 1, + }, + }); + + expect( + applyVisualTransition({ + baseTransform, + transition, + localTime: 0, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }).opacityMultiplier, + ).toBe(0); + expect( + applyVisualTransition({ + baseTransform, + transition, + localTime: 2 * TICKS_PER_SECOND, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }).opacityMultiplier, + ).toBe(1); + expect( + applyVisualTransition({ + baseTransform, + transition, + localTime: 4 * TICKS_PER_SECOND, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }).opacityMultiplier, + ).toBe(0); + }); + + test("slide left moves in and out across canvas", () => { + const transition = buildTransitionFromParams({ + params: { + "transition.kind": "push-left", + "transition.inDuration": 1, + "transition.outDuration": 1, + }, + }); + + const enter = applyVisualTransition({ + baseTransform, + transition, + localTime: 0, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }); + const middle = applyVisualTransition({ + baseTransform, + transition, + localTime: 2 * TICKS_PER_SECOND, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }); + const leave = applyVisualTransition({ + baseTransform, + transition, + localTime: 4 * TICKS_PER_SECOND, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }); + + expect(enter.transform.position.x).toBe(100); + expect(middle.transform.position.x).toBe(0); + expect(leave.transform.position.x).toBe(-100); + expect(leave.opacityMultiplier).toBe(1); + }); + + test("slide right mirrors horizontal entry and exit", () => { + const transition = buildTransitionFromParams({ + params: { + "transition.kind": "push-right", + "transition.inDuration": 1, + "transition.outDuration": 1, + }, + }); + + const enter = applyVisualTransition({ + baseTransform, + transition, + localTime: 0, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }); + const middle = applyVisualTransition({ + baseTransform, + transition, + localTime: 2 * TICKS_PER_SECOND, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }); + const leave = applyVisualTransition({ + baseTransform, + transition, + localTime: 4 * TICKS_PER_SECOND, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }); + + expect(enter.transform.position.x).toBe(-100); + expect(middle.transform.position.x).toBe(0); + expect(leave.transform.position.x).toBe(100); + expect(enter.opacityMultiplier).toBe(1); + }); + + test("zoom and spin variants converge on the base transform", () => { + const applyKind = ({ + kind, + localTime, + }: { + kind: "zoom" | "zoom-out" | "spin"; + localTime: number; + }) => + applyVisualTransition({ + baseTransform, + transition: buildTransitionFromParams({ + params: { + "transition.kind": kind, + "transition.inDuration": 1, + "transition.outDuration": 0, + }, + }), + localTime, + duration: 4 * TICKS_PER_SECOND, + canvasWidth: 100, + }); + + expect(applyKind({ kind: "zoom", localTime: 0 }).transform.scaleX).toBe( + 0.96, + ); + expect(applyKind({ kind: "zoom-out", localTime: 0 }).transform.scaleX).toBe( + 1.04, + ); + expect(applyKind({ kind: "spin", localTime: 0 }).transform.rotate).toBe(-6); + + const settledSpin = applyKind({ + kind: "spin", + localTime: 2 * TICKS_PER_SECOND, + }); + expect(settledSpin.transform.scaleX).toBe(1); + expect(settledSpin.transform.rotate).toBe(0); + }); +}); diff --git a/apps/web/src/rendering/transition.ts b/apps/web/src/rendering/transition.ts new file mode 100644 index 00000000..874975a1 --- /dev/null +++ b/apps/web/src/rendering/transition.ts @@ -0,0 +1,194 @@ +import type { ParamValues } from "@/params"; +import { clamp } from "@/utils/math"; +import { TICKS_PER_SECOND } from "@/wasm"; +import type { Transform } from "./index"; + +export type TransitionKind = + | "fade" + | "push-left" + | "push-right" + | "zoom" + | "zoom-out" + | "spin"; + +export interface VisualTransition { + kind: TransitionKind; + inDuration: number; + outDuration: number; +} + +export interface TransitionState { + opacityMultiplier: number; + transform: Transform; +} + +export const TRANSITION_KIND_OPTIONS: Array<{ + value: TransitionKind; + label: string; +}> = [ + { value: "fade", label: "Soft fade" }, + { value: "push-left", label: "Slide left" }, + { value: "push-right", label: "Slide right" }, + { value: "zoom", label: "Zoom in" }, + { value: "zoom-out", label: "Zoom out" }, + { value: "spin", label: "Spin dissolve" }, +]; + +export function isTransitionKind(value: string): value is TransitionKind { + return TRANSITION_KIND_OPTIONS.some((option) => option.value === value); +} +export function withVisualTransitionParams({ + params, + transitionKind, + inDuration, + outDuration, +}: { + params: ParamValues; + transitionKind: TransitionKind; + inDuration: number; + outDuration: number; +}): ParamValues { + return { + ...params, + "transition.kind": transitionKind, + "transition.inDuration": inDuration, + "transition.outDuration": outDuration, + }; +} + +export function buildTransitionFromParams({ + params, +}: { + params: ParamValues; +}): VisualTransition { + const kindValue = params["transition.kind"]; + const inDurationValue = params["transition.inDuration"]; + const outDurationValue = params["transition.outDuration"]; + return { + kind: + typeof kindValue === "string" && isTransitionKind(kindValue) + ? kindValue + : "fade", + inDuration: readDuration({ value: inDurationValue }), + outDuration: readDuration({ value: outDurationValue }), + }; +} + +export function applyVisualTransition({ + baseTransform, + transition, + localTime, + duration, + canvasWidth, +}: { + baseTransform: Transform; + transition?: VisualTransition; + localTime: number; + duration: number; + canvasWidth: number; +}): TransitionState { + if (!transition || duration <= 0) { + return { opacityMultiplier: 1, transform: baseTransform }; + } + + const inProgress = progressWithin({ + elapsed: localTime, + duration: transition.inDuration, + }); + const outProgress = progressWithin({ + elapsed: duration - localTime, + duration: transition.outDuration, + }); + const progress = Math.min(inProgress, outProgress); + const isSlide = + transition.kind === "push-left" || transition.kind === "push-right"; + const opacityMultiplier = isSlide ? 1 : ease({ progress }); + const transform = applyTransitionTransform({ + transform: baseTransform, + transition, + inProgress, + outProgress, + canvasWidth, + }); + return { opacityMultiplier, transform }; +} + +function readDuration({ value }: { value: unknown }): number { + const seconds = + typeof value === "number" && Number.isFinite(value) + ? clamp({ value, min: 0, max: 5 }) + : 0; + return seconds * TICKS_PER_SECOND; +} + +function progressWithin({ + elapsed, + duration, +}: { + elapsed: number; + duration: number; +}): number { + if (duration <= 0) { + return 1; + } + return clamp({ value: elapsed / duration, min: 0, max: 1 }); +} + +function ease({ progress }: { progress: number }): number { + return progress * progress * (3 - 2 * progress); +} + +function applyTransitionTransform({ + transform, + transition, + inProgress, + outProgress, + canvasWidth, +}: { + transform: Transform; + transition: VisualTransition; + inProgress: number; + outProgress: number; + canvasWidth: number; +}): Transform { + if (transition.kind === "fade") { + return transform; + } + + if (transition.kind === "zoom" || transition.kind === "zoom-out") { + const progress = Math.min(inProgress, outProgress); + const eased = ease({ progress }); + const scale = + transition.kind === "zoom" ? 0.96 + eased * 0.04 : 1.04 - eased * 0.04; + return { + ...transform, + scaleX: transform.scaleX * scale, + scaleY: transform.scaleY * scale, + }; + } + + if (transition.kind === "spin") { + const progress = Math.min(inProgress, outProgress); + const eased = ease({ progress }); + const scale = 0.96 + eased * 0.04; + return { + ...transform, + scaleX: transform.scaleX * scale, + scaleY: transform.scaleY * scale, + rotate: transform.rotate - (1 - eased) * 6, + }; + } + + const direction = transition.kind === "push-right" ? -1 : 1; + const enteringOffset = + (1 - ease({ progress: inProgress })) * canvasWidth * direction; + const leavingOffset = + (ease({ progress: outProgress }) - 1) * canvasWidth * direction; + return { + ...transform, + position: { + x: transform.position.x + enteringOffset + leavingOffset, + y: transform.position.y, + }, + }; +} diff --git a/apps/web/src/services/renderer/__tests__/scene-builder-transition.test.ts b/apps/web/src/services/renderer/__tests__/scene-builder-transition.test.ts new file mode 100644 index 00000000..dd644f2a --- /dev/null +++ b/apps/web/src/services/renderer/__tests__/scene-builder-transition.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, test } from "bun:test"; +import { buildScene } from "@/services/renderer/scene-builder"; +import { ImageNode } from "@/services/renderer/nodes/image-node"; +import type { ImageElement, SceneTracks } from "@/timeline"; +import type { MediaAsset } from "@/media/types"; +import { + mediaTime, + type MediaTime, + TICKS_PER_SECOND, + ZERO_MEDIA_TIME, +} from "@/wasm"; + +function imageElement({ + id, + mediaId, + startTime, + transitionDuration = 0, +}: { + id: string; + mediaId: string; + startTime: MediaTime; + transitionDuration?: number; +}): ImageElement { + return { + id, + name: id, + type: "image", + mediaId, + startTime, + duration: mediaTime({ ticks: 2 * TICKS_PER_SECOND }), + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + params: { + "transform.positionX": 0, + "transform.positionY": 0, + "transform.scaleX": 1, + "transform.scaleY": 1, + "transform.rotate": 0, + opacity: 1, + blendMode: "normal", + "transition.kind": "fade", + "transition.inDuration": transitionDuration, + "transition.outDuration": 0, + }, + }; +} + +function imageAsset(id: string): MediaAsset { + return { + id, + name: id, + type: "image", + file: new File([], `${id}.png`, { type: "image/png" }), + url: `blob:${id}`, + }; +} + +describe("scene transition boundaries", () => { + test("holds the outgoing slide through the incoming transition", () => { + const first = imageElement({ + id: "first", + mediaId: "media-first", + startTime: ZERO_MEDIA_TIME, + }); + const second = imageElement({ + id: "second", + mediaId: "media-second", + startTime: mediaTime({ ticks: 3 * TICKS_PER_SECOND }), + transitionDuration: 1, + }); + const tracks: SceneTracks = { + overlay: [], + main: { + id: "main", + name: "Main Track", + type: "video", + elements: [first, second], + muted: false, + hidden: false, + }, + audio: [], + }; + const scene = buildScene({ + canvasSize: { width: 1920, height: 1080 }, + tracks, + mediaAssets: [imageAsset("media-first"), imageAsset("media-second")], + duration: 5 * TICKS_PER_SECOND, + background: { type: "color", color: "transparent" }, + }); + const imageNodes = scene.children.filter( + (node): node is ImageNode => node instanceof ImageNode, + ); + + expect(imageNodes).toHaveLength(2); + expect(imageNodes[0].params.contentDuration).toBe(2 * TICKS_PER_SECOND); + expect(imageNodes[0].params.duration).toBe(4 * TICKS_PER_SECOND); + expect(imageNodes[1].params.transition?.inDuration).toBe(TICKS_PER_SECOND); + }); +}); diff --git a/apps/web/src/services/renderer/nodes/text-node.ts b/apps/web/src/services/renderer/nodes/text-node.ts index 6635a031..79ed0272 100644 --- a/apps/web/src/services/renderer/nodes/text-node.ts +++ b/apps/web/src/services/renderer/nodes/text-node.ts @@ -2,6 +2,7 @@ import { BaseNode } from "./base-node"; import type { TextElement } from "@/timeline"; import type { EffectPass } from "@/effects/types"; import type { BlendMode, Transform } from "@/rendering"; +import type { VisualTransition } from "@/rendering/transition"; import { drawMeasuredTextLayout } from "@/text/primitives"; import type { MeasuredTextElement } from "@/text/measure-element"; @@ -9,6 +10,7 @@ export type TextNodeParams = TextElement & { transform: Transform; opacity: number; blendMode?: BlendMode; + transition?: VisualTransition; canvasCenter: { x: number; y: number }; canvasHeight: number; textBaseline?: CanvasTextBaseline; diff --git a/apps/web/src/services/renderer/nodes/visual-node.ts b/apps/web/src/services/renderer/nodes/visual-node.ts index b9bf1709..e3f9b42f 100644 --- a/apps/web/src/services/renderer/nodes/visual-node.ts +++ b/apps/web/src/services/renderer/nodes/visual-node.ts @@ -2,10 +2,12 @@ import { BaseNode } from "./base-node"; import type { Effect, EffectPass } from "@/effects/types"; import type { Mask } from "@/masks/types"; import type { BlendMode, Transform } from "@/rendering"; +import type { VisualTransition } from "@/rendering/transition"; import type { RetimeConfig, VisualElement } from "@/timeline"; export interface VisualNodeParams { duration: number; + contentDuration?: number; timeOffset: number; trimStart: number; trimEnd: number; @@ -14,6 +16,7 @@ export interface VisualNodeParams { animations?: VisualElement["animations"]; opacity: number; blendMode?: BlendMode; + transition?: VisualTransition; effects?: Effect[]; masks?: Mask[]; } diff --git a/apps/web/src/services/renderer/resolve.ts b/apps/web/src/services/renderer/resolve.ts index 12f57e30..54a6eca4 100644 --- a/apps/web/src/services/renderer/resolve.ts +++ b/apps/web/src/services/renderer/resolve.ts @@ -22,6 +22,7 @@ import { } from "@/text/measure-element"; import { resolveColorAtTime, resolveOpacityAtTime } from "@/animation/values"; import { resolveTransformAtTime } from "@/rendering/animation-values"; +import { applyVisualTransition } from "@/rendering/transition"; import { videoCache } from "@/services/video-cache/service"; import type { CanvasRenderer } from "./canvas-renderer"; import type { AnyBaseNode } from "./nodes/base-node"; @@ -148,12 +149,13 @@ function resolveVisualState({ return null; } + const contentDuration = params.contentDuration ?? params.duration; const localTime = getElementLocalTime({ timelineTime: context.time, elementStartTime: params.timeOffset, - elementDuration: params.duration, + elementDuration: contentDuration, }); - const transform = applyMotionEffects({ + const baseTransform = applyMotionEffects({ baseTransform: resolveTransformAtTime({ baseTransform: params.transform, animations: params.animations, @@ -162,15 +164,24 @@ function resolveVisualState({ effects: params.effects, animations: params.animations, localTime, - duration: params.duration, + duration: contentDuration, canvasWidth: context.renderer.width, canvasHeight: context.renderer.height, }); - const opacity = resolveOpacityAtTime({ + const baseOpacity = resolveOpacityAtTime({ baseOpacity: params.opacity, animations: params.animations, localTime, }); + const transitionState = applyVisualTransition({ + baseTransform, + transition: params.transition, + localTime, + duration: contentDuration, + canvasWidth: context.renderer.width, + }); + const transform = transitionState.transform; + const opacity = baseOpacity * transitionState.opacityMultiplier; const containScale = Math.min( context.renderer.width / sourceWidth, context.renderer.height / sourceHeight, @@ -207,17 +218,21 @@ async function resolveVideoNode({ if (clipTime < 0 || clipTime >= node.params.duration) { return null; } + const contentDuration = node.params.contentDuration ?? node.params.duration; + const sourceClipTime = Math.min(clipTime, Math.max(0, contentDuration - 1)); const sourceTimeTicks = node.params.trimStart + getSourceTimeAtClipTime({ - clipTime, + clipTime: sourceClipTime, retime: node.params.retime, }); const frame = await videoCache.getFrameAt({ mediaId: node.params.mediaId, file: node.params.file, - time: mediaTimeToSeconds({ time: roundMediaTime({ time: sourceTimeTicks }) }), + time: mediaTimeToSeconds({ + time: roundMediaTime({ time: sourceTimeTicks }), + }), }); if (!frame) { return null; @@ -344,18 +359,27 @@ function resolveTextNode({ elementDuration: node.params.duration, }); const background = buildTextBackgroundFromElement({ element: node.params }); + const baseTransform = resolveTransformAtTime({ + baseTransform: node.params.transform, + animations: node.params.animations, + localTime, + }); + const baseOpacity = resolveOpacityAtTime({ + baseOpacity: node.params.opacity, + animations: node.params.animations, + localTime, + }); + const transitionState = applyVisualTransition({ + baseTransform, + transition: node.params.transition, + localTime, + duration: node.params.duration, + canvasWidth: context.renderer.width, + }); return { - transform: resolveTransformAtTime({ - baseTransform: node.params.transform, - animations: node.params.animations, - localTime, - }), - opacity: resolveOpacityAtTime({ - baseOpacity: node.params.opacity, - animations: node.params.animations, - localTime, - }), + transform: transitionState.transform, + opacity: baseOpacity * transitionState.opacityMultiplier, textColor: resolveColorAtTime({ baseColor: typeof node.params.params.color === "string" @@ -438,7 +462,9 @@ async function resolveBackdropSource({ const frame = await videoCache.getFrameAt({ mediaId: node.params.mediaId, file: node.params.file, - time: mediaTimeToSeconds({ time: roundMediaTime({ time: sourceTimeTicks }) }), + time: mediaTimeToSeconds({ + time: roundMediaTime({ time: sourceTimeTicks }), + }), }); if (!frame) { return null; @@ -485,10 +511,9 @@ function resolveEffectLayerNode({ const motion = definition.motion?.evaluate({ effectParams: node.params.effectParams, progress: sampleEasingBezier({ - bezier: - BUILTIN_PRESETS.find( - (entry) => entry.id === node.params.effectParams.easing, - )?.value ?? [0.25, 0.1, 0.25, 1], + bezier: BUILTIN_PRESETS.find( + (entry) => entry.id === node.params.effectParams.easing, + )?.value ?? [0.25, 0.1, 0.25, 1], x: clamp({ value: (time - node.params.timeOffset) / node.params.duration, min: 0, diff --git a/apps/web/src/services/renderer/scene-builder.ts b/apps/web/src/services/renderer/scene-builder.ts index 431b8269..9df154ef 100644 --- a/apps/web/src/services/renderer/scene-builder.ts +++ b/apps/web/src/services/renderer/scene-builder.ts @@ -1,4 +1,4 @@ -import type { SceneTracks, TimelineTrack } from "@/timeline"; +import type { SceneTracks, TimelineElement, TimelineTrack } from "@/timeline"; import type { MediaAsset } from "@/media/types"; import { RootNode } from "./nodes/root-node"; import { VideoNode } from "./nodes/video-node"; @@ -17,6 +17,7 @@ import { readBlendModeFromParams, readOpacityFromParams, } from "@/rendering"; +import { buildTransitionFromParams } from "@/rendering/transition"; const PREVIEW_MAX_IMAGE_SIZE = 2048; @@ -29,6 +30,32 @@ function getVisibleSortedElements({ track }: { track: TimelineTrack }) { return a.id.localeCompare(b.id); }); } +function getTailHoldDuration({ + elements, + elementIndex, +}: { + elements: TimelineElement[]; + elementIndex: number; +}): number { + const element = elements[elementIndex]; + const nextElement = elements[elementIndex + 1]; + if ( + !element || + !nextElement || + (element.type !== "video" && element.type !== "image") || + (nextElement.type !== "video" && nextElement.type !== "image") + ) { + return 0; + } + const gap = nextElement.startTime - (element.startTime + element.duration); + if (gap < 0) { + return 0; + } + const transitionDuration = buildTransitionFromParams({ + params: nextElement.params, + }).inDuration; + return transitionDuration > 0 ? gap + transitionDuration : 0; +} function buildTrackNodes({ tracks, @@ -46,7 +73,11 @@ function buildTrackNodes({ for (const track of tracks) { const elements = getVisibleSortedElements({ track }); - for (const element of elements) { + for (const [elementIndex, element] of elements.entries()) { + const tailHoldDuration = getTailHoldDuration({ + elements, + elementIndex, + }); if (element.type === "effect") { nodes.push( new EffectLayerNode({ @@ -71,7 +102,8 @@ function buildTrackNodes({ mediaId: mediaAsset.id, url: mediaAsset.url, file: mediaAsset.file, - duration: element.duration, + duration: element.duration + tailHoldDuration, + contentDuration: element.duration, timeOffset: element.startTime, trimStart: element.trimStart, trimEnd: element.trimEnd, @@ -80,6 +112,7 @@ function buildTrackNodes({ animations: element.animations, opacity: readOpacityFromParams({ params: element.params }), blendMode: readBlendModeFromParams({ params: element.params }), + transition: buildTransitionFromParams({ params: element.params }), effects: element.effects ?? [], masks: element.masks ?? [], }), @@ -89,7 +122,8 @@ function buildTrackNodes({ nodes.push( new ImageNode({ url: mediaAsset.url, - duration: element.duration, + duration: element.duration + tailHoldDuration, + contentDuration: element.duration, timeOffset: element.startTime, trimStart: element.trimStart, trimEnd: element.trimEnd, @@ -97,6 +131,7 @@ function buildTrackNodes({ animations: element.animations, opacity: readOpacityFromParams({ params: element.params }), blendMode: readBlendModeFromParams({ params: element.params }), + transition: buildTransitionFromParams({ params: element.params }), effects: element.effects ?? [], masks: element.masks ?? [], ...(isPreview && { @@ -114,6 +149,7 @@ function buildTrackNodes({ transform: buildTransformFromParams({ params: element.params }), opacity: readOpacityFromParams({ params: element.params }), blendMode: readBlendModeFromParams({ params: element.params }), + transition: buildTransitionFromParams({ params: element.params }), canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 }, canvasHeight: canvasSize.height, textBaseline: "middle", @@ -136,6 +172,7 @@ function buildTrackNodes({ animations: element.animations, opacity: readOpacityFromParams({ params: element.params }), blendMode: readBlendModeFromParams({ params: element.params }), + transition: buildTransitionFromParams({ params: element.params }), effects: element.effects ?? [], }), ); @@ -154,6 +191,7 @@ function buildTrackNodes({ animations: element.animations, opacity: readOpacityFromParams({ params: element.params }), blendMode: readBlendModeFromParams({ params: element.params }), + transition: buildTransitionFromParams({ params: element.params }), effects: element.effects ?? [], masks: element.masks ?? [], }), diff --git a/apps/web/src/timeline/__tests__/audio-state.test.ts b/apps/web/src/timeline/__tests__/audio-state.test.ts new file mode 100644 index 00000000..91c422cb --- /dev/null +++ b/apps/web/src/timeline/__tests__/audio-state.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test } from "bun:test"; +import { + dBToLinear, + getTrackVolume, + resolveEffectiveAudioGain, +} from "@/timeline/audio-state"; +import type { AudioElement, AudioTrack } from "@/timeline"; +import { ZERO_MEDIA_TIME } from "@/wasm"; + +function audioElement(params?: AudioElement["params"]): AudioElement { + return { + id: "audio-1", + name: "Audio", + type: "audio", + sourceType: "upload", + mediaId: "media-1", + duration: ZERO_MEDIA_TIME, + startTime: ZERO_MEDIA_TIME, + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + params: params ?? { volume: 0, muted: false }, + }; +} + +describe("audio state", () => { + test("track volume offsets element gain in decibels", () => { + const gain = resolveEffectiveAudioGain({ + element: audioElement({ volume: -6, muted: false }), + trackVolume: -6, + localTime: 0, + }); + + expect(gain).toBeCloseTo(dBToLinear(-12), 6); + }); + test("track volume and mute resolve to playback gain", () => { + const element = audioElement({ volume: 0, muted: false }); + + expect( + resolveEffectiveAudioGain({ element, trackVolume: 0, localTime: 0 }), + ).toBe(1); + expect( + resolveEffectiveAudioGain({ element, trackVolume: -6, localTime: 0 }), + ).toBeCloseTo(dBToLinear(-6), 6); + expect( + resolveEffectiveAudioGain({ + element, + trackMuted: true, + trackVolume: 0, + localTime: 0, + }), + ).toBe(0); + }); + + test("missing track volume defaults to unity", () => { + const track: AudioTrack = { + id: "track-1", + name: "Audio track", + type: "audio", + elements: [], + muted: false, + }; + + expect(getTrackVolume({ track })).toBe(0); + }); +}); diff --git a/apps/web/src/timeline/audio-state.ts b/apps/web/src/timeline/audio-state.ts index 6c8f3044..34220cc9 100644 --- a/apps/web/src/timeline/audio-state.ts +++ b/apps/web/src/timeline/audio-state.ts @@ -1,7 +1,8 @@ import { hasKeyframesForPath } from "@/animation/keyframe-query"; import { resolveNumberAtTime } from "@/animation/values"; import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "./audio-constants"; -import type { TimelineElement } from "./types"; +import type { TimelineElement, TimelineTrack } from "./types"; +import { TICKS_PER_SECOND } from "@/wasm"; const DEFAULT_STEP_SECONDS = 1 / 60; export type AudioCapableElement = Extract< @@ -29,6 +30,13 @@ export function getElementVolume({ const value = element.params.volume; return typeof value === "number" ? value : 0; } +export function getTrackVolume({ + track, +}: { + track: Extract; +}): number { + return typeof track.volume === "number" ? track.volume : 0; +} export function isElementMuted({ element, @@ -49,15 +57,15 @@ export function hasAnimatedVolume({ }); } -import { TICKS_PER_SECOND } from "@/wasm"; - export function resolveEffectiveAudioGain({ element, trackMuted = false, + trackVolume = 0, localTime, }: { element: AudioCapableElement; trackMuted?: boolean; + trackVolume?: number; localTime: number; }): number { if (trackMuted || isElementMuted({ element })) { @@ -71,7 +79,7 @@ export function resolveEffectiveAudioGain({ localTime: Math.round(localTime * TICKS_PER_SECOND), }); - return dBToLinear(resolvedDb); + return dBToLinear(resolvedDb + trackVolume); } export function buildWaveformGainSamples({ @@ -91,12 +99,14 @@ export function buildWaveformGainSamples({ export function buildAudioGainAutomation({ element, trackMuted = false, + trackVolume = 0, fromLocalTime, toLocalTime, stepSeconds = DEFAULT_STEP_SECONDS, }: { element: AudioCapableElement; trackMuted?: boolean; + trackVolume?: number; fromLocalTime: number; toLocalTime: number; stepSeconds?: number; @@ -115,6 +125,7 @@ export function buildAudioGainAutomation({ gain: resolveEffectiveAudioGain({ element, trackMuted, + trackVolume, localTime, }), }); @@ -126,6 +137,7 @@ export function buildAudioGainAutomation({ element, trackMuted, localTime: endTime, + trackVolume, }), }); diff --git a/apps/web/src/timeline/components/__tests__/drop-target-transition.test.ts b/apps/web/src/timeline/components/__tests__/drop-target-transition.test.ts new file mode 100644 index 00000000..d3246a19 --- /dev/null +++ b/apps/web/src/timeline/components/__tests__/drop-target-transition.test.ts @@ -0,0 +1,221 @@ +import { describe, expect, test } from "bun:test"; +import type { + AudioTrack, + EffectTrack, + ImageElement, + SceneTracks, + TextTrack, + VideoElement, + VideoTrack, +} from "@/timeline"; +import { computeTransitionCutDropTarget } from "@/timeline/components/drop-target"; +import { mediaTime, TICKS_PER_SECOND, ZERO_MEDIA_TIME } from "@/wasm"; + +const PIXELS_PER_SECOND = 50; + +function buildImage({ + id, + start, + duration, +}: { + id: string; + start: number; + duration: number; +}): ImageElement { + return { + id, + type: "image", + name: id, + mediaId: `media-${id}`, + startTime: mediaTime({ ticks: start }), + duration: mediaTime({ ticks: duration }), + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + params: { opacity: 0.75 }, + }; +} + +function buildVideo({ + id, + start, + duration, +}: { + id: string; + start: number; + duration: number; +}): VideoElement { + return { + id, + type: "video", + name: id, + mediaId: `media-${id}`, + startTime: mediaTime({ ticks: start }), + duration: mediaTime({ ticks: duration }), + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + params: { opacity: 1 }, + }; +} + +function buildVideoTrack({ + elements, +}: { + elements: VideoTrack["elements"]; +}): VideoTrack { + return { + id: "visual", + type: "video", + name: "Visual", + elements, + muted: false, + hidden: false, + }; +} + +function buildTracks({ + main, + overlay = [], + audio = [], +}: { + main: VideoTrack; + overlay?: SceneTracks["overlay"]; + audio?: AudioTrack[]; +}): SceneTracks { + return { main, overlay, audio }; +} + +function resolve({ + tracks, + mouseX, + mouseY = 10, +}: { + tracks: SceneTracks; + mouseX: number; + mouseY?: number; +}) { + return computeTransitionCutDropTarget({ + mouseX, + mouseY, + tracks, + pixelsPerSecond: PIXELS_PER_SECOND, + zoomLevel: 1, + }); +} + +const oneSecond = TICKS_PER_SECOND; + +function adjacentTrack(): VideoTrack { + return buildVideoTrack({ + elements: [ + buildImage({ id: "outgoing", start: 0, duration: oneSecond }), + buildVideo({ id: "incoming", start: oneSecond, duration: oneSecond }), + ], + }); +} + +describe("computeTransitionCutDropTarget", () => { + test("resolves an adjacent image and video cut within 12 screen pixels", () => { + const target = resolve({ + tracks: buildTracks({ main: adjacentTrack() }), + mouseX: 55, + }); + + expect(target).toEqual({ + trackIndex: 0, + isNewTrack: false, + insertPosition: null, + xPosition: mediaTime({ ticks: oneSecond }), + targetElement: null, + targetCut: { + trackId: "visual", + outgoingElementId: "outgoing", + incomingElementId: "incoming", + time: mediaTime({ ticks: oneSecond }), + }, + }); + }); + + test("rejects a pointer beyond 12 screen pixels", () => { + expect( + resolve({ tracks: buildTracks({ main: adjacentTrack() }), mouseX: 63 }), + ).toBeNull(); + }); + + test("rejects gaps and overlaps", () => { + const incoming = buildVideo({ + id: "incoming", + start: oneSecond, + duration: oneSecond, + }); + const gap = buildVideoTrack({ + elements: [ + buildImage({ id: "outgoing", start: 0, duration: oneSecond - 1 }), + incoming, + ], + }); + const overlap = buildVideoTrack({ + elements: [ + buildImage({ id: "outgoing", start: 0, duration: oneSecond + 1 }), + incoming, + ], + }); + + expect( + resolve({ tracks: buildTracks({ main: gap }), mouseX: 50 }), + ).toBeNull(); + expect( + resolve({ tracks: buildTracks({ main: overlap }), mouseX: 50 }), + ).toBeNull(); + }); + + test("rejects the first visual clip boundary", () => { + expect( + resolve({ tracks: buildTracks({ main: adjacentTrack() }), mouseX: 0 }), + ).toBeNull(); + }); + + test("rejects audio, text, and effect tracks", () => { + const textTrack: TextTrack = { + id: "text", + type: "text", + name: "Text", + elements: [], + hidden: false, + }; + const effectTrack: EffectTrack = { + id: "effect", + type: "effect", + name: "Effect", + elements: [], + hidden: false, + }; + const audioTrack: AudioTrack = { + id: "audio", + type: "audio", + name: "Audio", + elements: [], + muted: false, + }; + const main = adjacentTrack(); + + expect( + resolve({ + tracks: buildTracks({ main, overlay: [textTrack] }), + mouseX: 50, + }), + ).toBeNull(); + expect( + resolve({ + tracks: buildTracks({ main, overlay: [effectTrack] }), + mouseX: 50, + }), + ).toBeNull(); + expect( + resolve({ + tracks: buildTracks({ main, audio: [audioTrack] }), + mouseX: 50, + mouseY: 72, + }), + ).toBeNull(); + }); +}); diff --git a/apps/web/src/timeline/components/drop-target.ts b/apps/web/src/timeline/components/drop-target.ts index f68a4b72..e22c8af9 100644 --- a/apps/web/src/timeline/components/drop-target.ts +++ b/apps/web/src/timeline/components/drop-target.ts @@ -1,5 +1,9 @@ -import type { TimelineTrack, TimelineElement } from "@/timeline"; -import type { ComputeDropTargetParams, DropTarget } from "@/timeline"; +import type { + ComputeDropTargetParams, + DropTarget, + TimelineElement, + TimelineTrack, +} from "@/timeline"; import { resolveTrackPlacement } from "@/timeline/placement"; import { TIMELINE_TRACK_GAP_PX } from "./layout"; import { getTrackHeight } from "./track-layout"; @@ -97,6 +101,71 @@ function fallbackNewTrackDropTarget({ insertPosition: null, xPosition, targetElement: EMPTY_TARGET_ELEMENT, + targetCut: null, + }; +} + +export function computeTransitionCutDropTarget({ + mouseX, + mouseY, + tracks, + pixelsPerSecond, + zoomLevel, +}: { + mouseX: number; + mouseY: number; + tracks: ComputeDropTargetParams["tracks"]; + pixelsPerSecond: number; + zoomLevel: number; +}): DropTarget | null { + const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; + const trackAtMouse = getTrackAtY({ mouseY, tracks: orderedTracks }); + if (!trackAtMouse) return null; + + const track = orderedTracks[trackAtMouse.trackIndex]; + if (!track || track.type !== "video") return null; + + const elements = track.elements + .filter((element) => element.type === "video" || element.type === "image") + .slice() + .sort((left, right) => left.startTime - right.startTime); + let nearest: { + outgoingElementId: string; + incomingElementId: string; + time: MediaTime; + distance: number; + } | null = null; + + for (let index = 1; index < elements.length; index += 1) { + const outgoing = elements[index - 1]; + const incoming = elements[index]; + const cutTime = incoming.startTime; + if (outgoing.startTime + outgoing.duration !== cutTime) continue; + + const cutX = (cutTime / TICKS_PER_SECOND) * pixelsPerSecond * zoomLevel; + const distance = Math.abs(mouseX - cutX); + if (distance > 12 || (nearest && nearest.distance <= distance)) continue; + nearest = { + outgoingElementId: outgoing.id, + incomingElementId: incoming.id, + time: cutTime, + distance, + }; + } + + if (!nearest) return null; + return { + trackIndex: trackAtMouse.trackIndex, + isNewTrack: false, + insertPosition: null, + xPosition: nearest.time, + targetElement: null, + targetCut: { + trackId: track.id, + outgoingElementId: nearest.outgoingElementId, + incomingElementId: nearest.incomingElementId, + time: nearest.time, + }, }; } @@ -133,7 +202,9 @@ export function computeDropTarget({ const placementResult = resolveTrackPlacement({ tracks, elementType, - timeSpans: [{ startTime: xPosition, duration: elementDuration, excludeElementId }], + timeSpans: [ + { startTime: xPosition, duration: elementDuration, excludeElementId }, + ], strategy: { type: "preferIndex", trackIndex: 0, @@ -153,6 +224,7 @@ export function computeDropTarget({ insertPosition: emptyTimelineResult.insertPosition, xPosition, targetElement: EMPTY_TARGET_ELEMENT, + targetCut: null, }; } @@ -168,7 +240,9 @@ export function computeDropTarget({ const placementResult = resolveTrackPlacement({ tracks, elementType, - timeSpans: [{ startTime: xPosition, duration: elementDuration, excludeElementId }], + timeSpans: [ + { startTime: xPosition, duration: elementDuration, excludeElementId }, + ], strategy: { type: "preferIndex", trackIndex: isAboveAllTracks ? 0 : orderedTracks.length - 1, @@ -188,6 +262,7 @@ export function computeDropTarget({ insertPosition: outOfBoundsResult.insertPosition, xPosition, targetElement: EMPTY_TARGET_ELEMENT, + targetCut: null, }; } @@ -210,6 +285,7 @@ export function computeDropTarget({ insertPosition: null, xPosition, targetElement, + targetCut: null, }; } } @@ -218,7 +294,9 @@ export function computeDropTarget({ const placementResult = resolveTrackPlacement({ tracks, elementType, - timeSpans: [{ startTime: xPosition, duration: elementDuration, excludeElementId }], + timeSpans: [ + { startTime: xPosition, duration: elementDuration, excludeElementId }, + ], strategy: { type: "preferIndex", trackIndex, @@ -242,6 +320,7 @@ export function computeDropTarget({ insertPosition: null, xPosition: adjustedXPosition, targetElement: EMPTY_TARGET_ELEMENT, + targetCut: null, }; } @@ -251,6 +330,7 @@ export function computeDropTarget({ insertPosition: placementResult.insertPosition, xPosition, targetElement: EMPTY_TARGET_ELEMENT, + targetCut: null, }; } diff --git a/apps/web/src/timeline/components/index.tsx b/apps/web/src/timeline/components/index.tsx index 8edbb6fc..527f6d8d 100644 --- a/apps/web/src/timeline/components/index.tsx +++ b/apps/web/src/timeline/components/index.tsx @@ -53,6 +53,8 @@ import { getTimelineZoomMin, getTimelinePaddingPx, } from "@/timeline"; +import { doesElementHaveEnabledAudio } from "@/timeline/audio-separation"; +import type { MediaAsset } from "@/media/types"; import { timelineTimeToPixels } from "@/timeline/pixel-utils"; import { getTrackHeight, @@ -84,9 +86,11 @@ import { useEditor } from "@/editor/use-editor"; import { useScrollPosition } from "@/timeline/hooks/use-scroll-position"; import { useTimelinePlayhead } from "@/timeline/hooks/use-timeline-playhead"; import { DragLine } from "./drag-line"; +import { TIMELINE_LAYERS } from "./layers"; import { invokeAction } from "@/actions"; import { resolveTimelineElementIntersections } from "./selection-hit-testing"; import { cn } from "@/utils/ui"; +import { usePropertiesStore } from "@/components/editor/panels/properties/stores/properties-store"; const TRACKS_CONTAINER_MAX_HEIGHT = 800; const FALLBACK_CONTAINER_WIDTH = 1000; @@ -302,12 +306,12 @@ export function Timeline() { const { dragView, handleElementMouseDown, handleElementClick } = useElementInteraction({ - zoomLevel, - tracksContainerRef, - tracksScrollRef, - snappingEnabled, - onSnapPointChange: handleSnapPointChange, - }); + zoomLevel, + tracksContainerRef, + tracksScrollRef, + snappingEnabled, + onSnapPointChange: handleSnapPointChange, + }); const isElementDragging = dragView.kind === "dragging"; const { @@ -455,13 +459,13 @@ export function Timeline() { className="relative isolate flex flex-1 flex-col overflow-hidden" ref={tracksContainerRef} > - + e.scenes.getActiveSceneOrNull()); + const mediaAssets = useEditor((e) => e.media.getAssets()); + const selectTrack = usePropertiesStore((state) => state.selectTrack); const tracks = useMemo( () => scene @@ -619,7 +625,7 @@ function TrackLabelsPanel({ : [], [scene], ); - const { selectedElements } = useElementSelection(); + const { selectedElements, clearElementSelection } = useElementSelection(); const tracksWithSelection = useMemo( () => new Set(selectedElements.map((el) => el.trackId)), [selectedElements], @@ -653,6 +659,7 @@ function TrackLabelsPanel({ {tracks.map((track, index) => { const expandedRows = trackExpandedRowsMap[index]; const baseHeight = getTrackHeight({ type: track.type }); + const hasAudio = trackContainsAudio({ track, mediaAssets }); return (
- {canTrackHaveAudio(track) && ( - { + clearElementSelection(); + selectTrack({ trackId: track.id }); }} - onClick={() => - editor.timeline.toggleTrackMute({ - trackId: track.id, - }) - } /> )} {canTrackBeHidden(track) && ( @@ -780,8 +782,8 @@ function TimelineTrackRows({ const draggingElementIds = useMemo( () => dragView.kind === "dragging" - ? dragView.memberTimeOffsets - : (null as ReadonlyMap | null), + ? dragView.memberTimeOffsets + : (null as ReadonlyMap | null), [dragView], ); const sortedTracks = useMemo(() => { @@ -833,6 +835,18 @@ function TimelineTrackRows({ : null } /> + {isDragOver && dropTarget?.targetCut?.trackId === track.id && ( +
+ )}
@@ -915,24 +929,72 @@ function TrackToggleIcon({ }; onClick: () => void; }) { + const icon = isOff ? icons.off : icons.on; return ( - <> - {isOff ? ( - - ) : ( - + ); } +function TrackAudioButton({ + track, + onClick, +}: { + track: Extract; + onClick: () => void; +}) { + return ( + + ); +} + +function trackContainsAudio({ + track, + mediaAssets, +}: { + track: TimelineTrack; + mediaAssets: MediaAsset[]; +}): boolean { + if (!canTrackHaveAudio(track)) { + return false; + } + + for (const element of track.elements) { + if (element.type === "audio") { + return true; + } + if (element.type !== "video") { + continue; + } + const mediaAsset = mediaAssets.find( + (asset) => asset.id === element.mediaId, + ); + if (doesElementHaveEnabledAudio({ element, mediaAsset })) { + return true; + } + } + return false; +} function PropertyTree({ rows }: { rows: ExpandedRow[] }) { return ( diff --git a/apps/web/src/timeline/components/timeline-toolbar.tsx b/apps/web/src/timeline/components/timeline-toolbar.tsx index 47906931..ba9a99fa 100644 --- a/apps/web/src/timeline/components/timeline-toolbar.tsx +++ b/apps/web/src/timeline/components/timeline-toolbar.tsx @@ -43,17 +43,13 @@ import { Layers01Icon, Chart03Icon, Unlink02Icon, - MicIcon, - StopIcon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { OcRippleIcon } from "@/components/icons"; import { GraphEditorPopover } from "./graph-editor/popover"; import { PopoverTrigger } from "@/components/ui/popover"; import { useGraphEditorController } from "./graph-editor/use-controller"; -import { formatElapsedSeconds } from "@/voiceover/recorder"; -import { useRecordingElapsedSeconds } from "@/voiceover/use-recording-elapsed"; -import { useVoiceOverStore } from "@/voiceover/voiceover-store"; +import { VoiceOverRecorderPopover } from "@/voiceover/voiceover-popover"; export function TimelineToolbar({ zoomLevel, @@ -100,10 +96,6 @@ function ToolbarLeftSection() { const isCurrentlyBookmarked = useEditor((e) => e.scenes.isBookmarked({ time: e.playback.getCurrentTime() }), ); - const isRecordingVoiceOver = useVoiceOverStore( - (state) => state.status === "recording", - ); - const voiceOverElapsedSeconds = useRecordingElapsedSeconds(); const canFreezeFrame = useEditor((e) => { const time = e.playback.getCurrentTime(); const { overlay, main } = e.scenes.getActiveScene().tracks; @@ -211,22 +203,7 @@ function ToolbarLeftSection() { onClick={({ event }) => handleAction({ action: "freeze-frame", event })} /> - - } - tooltip={ - isRecordingVoiceOver ? "Stop voice-over" : "Record voice-over" - } - isActive={isRecordingVoiceOver} - onClick={({ event }) => handleAction({ action: "voice-over", event })} - /> - - {isRecordingVoiceOver && ( - - {formatElapsedSeconds({ seconds: voiceOverElapsedSeconds })} - - )} + } diff --git a/apps/web/src/timeline/controllers/__tests__/drag-drop-controller-transition.test.ts b/apps/web/src/timeline/controllers/__tests__/drag-drop-controller-transition.test.ts new file mode 100644 index 00000000..b24f9ad9 --- /dev/null +++ b/apps/web/src/timeline/controllers/__tests__/drag-drop-controller-transition.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, mock, test } from "bun:test"; +import type { DragEvent } from "react"; +import type { SceneTracks, VideoElement, VideoTrack } from "@/timeline"; +import { + DragDropController, + type DragDropConfig, + type DragDropConfigRef, +} from "@/timeline/controllers/drag-drop-controller"; +import { TimelineDragSource } from "@/timeline/drag-source"; +import { mediaTime, TICKS_PER_SECOND, ZERO_MEDIA_TIME } from "@/wasm"; + +function buildVideo({ + id, + start, +}: { + id: string; + start: number; +}): VideoElement { + return { + id, + type: "video", + name: id, + mediaId: `media-${id}`, + startTime: mediaTime({ ticks: start }), + duration: mediaTime({ ticks: TICKS_PER_SECOND }), + trimStart: ZERO_MEDIA_TIME, + trimEnd: ZERO_MEDIA_TIME, + params: { opacity: id === "incoming" ? 0.8 : 1 }, + }; +} + +function buildTracks(): SceneTracks { + const main: VideoTrack = { + id: "visual", + type: "video", + name: "Visual", + elements: [ + buildVideo({ id: "outgoing", start: 0 }), + buildVideo({ id: "incoming", start: TICKS_PER_SECOND }), + ], + muted: false, + hidden: false, + }; + return { overlay: [], main, audio: [] }; +} + +function buildDataTransfer(): DataTransfer { + return { + types: [], + files: [], + dropEffect: "none", + effectAllowed: "none", + setData: mock(() => {}), + } as unknown as DataTransfer; +} + +function buildDragEvent({ + clientX, + dataTransfer, +}: { + clientX: number; + dataTransfer: DataTransfer; +}): DragEvent { + return { + clientX, + clientY: 10, + dataTransfer, + preventDefault: mock(() => {}), + } as unknown as DragEvent; +} + +function setup() { + const dragSource = new TimelineDragSource(); + const updateElements = mock(() => {}); + const insertElement = mock(() => {}); + const executeCommand = mock(() => {}); + const tracksElement = { + scrollLeft: 0, + scrollTop: 0, + getBoundingClientRect: () => ({ left: 0, top: 0 }), + } as unknown as HTMLDivElement; + const config: DragDropConfig = { + zoomLevel: 1, + getContainerEl: () => tracksElement, + getHeaderEl: () => null, + getTracksScrollEl: () => tracksElement, + getActiveProjectFps: () => null, + getActiveProjectId: () => null, + getSceneTracks: buildTracks, + getCurrentPlayheadTime: () => ZERO_MEDIA_TIME, + getMediaAssets: () => [], + dragSource, + addMediaAsset: () => Promise.resolve(null), + executeCommand, + insertElement, + updateElements, + addClipEffect: mock(() => {}), + }; + const configRef: DragDropConfigRef = { current: config }; + return { + controller: new DragDropController({ configRef }), + dragSource, + updateElements, + insertElement, + executeCommand, + }; +} + +function beginTransitionDrag( + dragSource: TimelineDragSource, + dataTransfer: DataTransfer, +) { + dragSource.begin({ + dataTransfer, + dragData: { + type: "transition", + id: "push-left", + name: "Slide left", + transitionKind: "push-left", + inDuration: 0.45, + outDuration: 0, + }, + }); +} + +describe("DragDropController transition drops", () => { + test("hovers and applies a transition only to the incoming clip", () => { + const { + controller, + dragSource, + updateElements, + insertElement, + executeCommand, + } = setup(); + const dataTransfer = buildDataTransfer(); + beginTransitionDrag(dragSource, dataTransfer); + + controller.onDragOver(buildDragEvent({ clientX: 50, dataTransfer })); + + expect(dataTransfer.dropEffect).toBe("copy"); + expect(controller.dropTarget?.targetCut).toEqual({ + trackId: "visual", + outgoingElementId: "outgoing", + incomingElementId: "incoming", + time: mediaTime({ ticks: TICKS_PER_SECOND }), + }); + + controller.onDrop(buildDragEvent({ clientX: 50, dataTransfer })); + + expect(updateElements).toHaveBeenCalledTimes(1); + expect(updateElements).toHaveBeenCalledWith({ + updates: [ + { + trackId: "visual", + elementId: "incoming", + patch: { + params: { + opacity: 0.8, + "transition.kind": "push-left", + "transition.inDuration": 0.45, + "transition.outDuration": 0, + }, + }, + }, + ], + }); + expect(insertElement).not.toHaveBeenCalled(); + expect(executeCommand).not.toHaveBeenCalled(); + }); + + test("does nothing when the transition has no cut target", () => { + const { + controller, + dragSource, + updateElements, + insertElement, + executeCommand, + } = setup(); + const dataTransfer = buildDataTransfer(); + beginTransitionDrag(dragSource, dataTransfer); + + controller.onDragOver(buildDragEvent({ clientX: 80, dataTransfer })); + expect(controller.dropTarget).toBeNull(); + controller.onDrop(buildDragEvent({ clientX: 80, dataTransfer })); + + expect(updateElements).not.toHaveBeenCalled(); + expect(insertElement).not.toHaveBeenCalled(); + expect(executeCommand).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/src/timeline/controllers/drag-drop-controller.ts b/apps/web/src/timeline/controllers/drag-drop-controller.ts index a419101f..6b4c8bbf 100644 --- a/apps/web/src/timeline/controllers/drag-drop-controller.ts +++ b/apps/web/src/timeline/controllers/drag-drop-controller.ts @@ -17,7 +17,10 @@ import { import { AddTrackCommand, InsertElementCommand } from "@/commands/timeline"; import { BatchCommand } from "@/commands"; import type { Command } from "@/commands/base-command"; -import { computeDropTarget } from "@/timeline/components/drop-target"; +import { + computeDropTarget, + computeTransitionCutDropTarget, +} from "@/timeline/components/drop-target"; import type { TimelineDragSource } from "@/timeline/drag-source"; import type { TrackType, @@ -26,11 +29,13 @@ import type { SceneTracks, TimelineTrack, CreateTimelineElement, + TimelineElement, } from "@/timeline"; import type { TimelineDragData } from "@/timeline/drag"; import type { MediaAsset } from "@/media/types"; import type { ProcessedMediaAsset } from "@/media/processing"; import { roundFrameTime, type MediaTime } from "@/wasm"; +import { withVisualTransitionParams } from "@/rendering/transition"; // --- Config --- @@ -54,6 +59,14 @@ export interface DragDropConfig { placement: { mode: "explicit"; trackId: string }; element: CreateTimelineElement; }) => void; + updateElements: (args: { + updates: Array<{ + trackId: string; + elementId: string; + patch: Partial; + }>; + pushHistory?: boolean; + }) => void; addClipEffect: (args: { trackId: string; elementId: string; @@ -85,7 +98,7 @@ interface TimelineCoords { function elementTypeFromDrag({ dragData, }: { - dragData: TimelineDragData; + dragData: Exclude; }): ElementType { switch (dragData.type) { case "text": @@ -104,7 +117,7 @@ function elementTypeFromDrag({ function getTargetElementTypesForDrag({ dragData, }: { - dragData: TimelineDragData; + dragData: Exclude; }): string[] | undefined { if (dragData.type === "effect") return dragData.targetElementTypes; if (dragData.type === "media") return dragData.targetElementTypes; @@ -115,7 +128,7 @@ function getDurationForDrag({ dragData, mediaAssets, }: { - dragData: TimelineDragData; + dragData: Exclude; mediaAssets: MediaAsset[]; }): MediaTime { if (dragData.type !== "media") return DEFAULT_NEW_ELEMENT_DURATION; @@ -202,6 +215,18 @@ export class DragDropController { } return; } + if (dragData.type === "transition") { + const target = computeTransitionCutDropTarget({ + mouseX: coords.mouseX, + mouseY: coords.mouseY, + tracks: this.config.getSceneTracks(), + pixelsPerSecond: BASE_TIMELINE_PIXELS_PER_SECOND, + zoomLevel: this.config.zoomLevel, + }); + this.setOver({ dropTarget: target, elementType: null }); + if (target) event.dataTransfer.dropEffect = "copy"; + return; + } const elementType = elementTypeFromDrag({ dragData }); const duration = getDurationForDrag({ @@ -374,9 +399,47 @@ export class DragDropController { case "media": this.executeMediaDrop({ target, dragData }); return; + case "transition": + this.executeTransitionDrop({ target, dragData }); + return; } } + private executeTransitionDrop({ + target, + dragData, + }: { + target: DropTarget; + dragData: Extract; + }): void { + const cut = target.targetCut; + if (!cut) return; + + const tracks = orderedTracks({ sceneTracks: this.config.getSceneTracks() }); + const track = tracks.find(({ id }) => id === cut.trackId); + const incoming = track?.elements.find( + ({ id }) => id === cut.incomingElementId, + ); + if (!incoming) return; + + this.config.updateElements({ + updates: [ + { + trackId: cut.trackId, + elementId: cut.incomingElementId, + patch: { + params: withVisualTransitionParams({ + params: incoming.params, + transitionKind: dragData.transitionKind, + inDuration: dragData.inDuration, + outDuration: dragData.outDuration, + }), + }, + }, + ], + }); + } + private executeTextDrop({ target, dragData, diff --git a/apps/web/src/timeline/defaults.ts b/apps/web/src/timeline/defaults.ts index d32fb649..bc679ab9 100644 --- a/apps/web/src/timeline/defaults.ts +++ b/apps/web/src/timeline/defaults.ts @@ -14,6 +14,9 @@ const defaultTransform: Transform = { const defaultOpacity = 1; const defaultBlendMode: BlendMode = "normal"; const defaultVolume = 0; +const defaultTransitionKind = "fade"; +const defaultTransitionDuration = 0; +const defaultTrackVolume = 0; const defaultTextLetterSpacing = 0; const defaultTextLineHeight = 1.2; @@ -60,6 +63,9 @@ const defaultTextElement: Omit = { "transform.rotate": defaultTransform.rotate, opacity: defaultOpacity, blendMode: defaultBlendMode, + "transition.kind": defaultTransitionKind, + "transition.inDuration": defaultTransitionDuration, + "transition.outDuration": defaultTransitionDuration, }, }; @@ -75,6 +81,9 @@ export const DEFAULTS = { opacity: defaultOpacity, blendMode: defaultBlendMode, volume: defaultVolume, + transitionKind: defaultTransitionKind, + transitionDuration: defaultTransitionDuration, + trackVolume: defaultTrackVolume, }, text: { letterSpacing: defaultTextLetterSpacing, diff --git a/apps/web/src/timeline/drag.ts b/apps/web/src/timeline/drag.ts index 7bf99a6d..06f721da 100644 --- a/apps/web/src/timeline/drag.ts +++ b/apps/web/src/timeline/drag.ts @@ -1,42 +1,50 @@ -import type { MaskableElement, VisualElement } from "./types"; -import type { ParamValues } from "@/params"; - -interface BaseDragData { - id: string; - name: string; -} - -export interface MediaDragData extends BaseDragData { - type: "media"; - mediaType: "image" | "video" | "audio"; - targetElementTypes?: MaskableElement["type"][]; -} - -export interface TextDragData extends BaseDragData { - type: "text"; - content: string; -} - -export interface StickerDragData extends BaseDragData { - type: "sticker"; - stickerId: string; -} - -export interface GraphicDragData extends BaseDragData { - type: "graphic"; - definitionId: string; - params: Partial; -} - -export interface EffectDragData extends BaseDragData { - type: "effect"; - effectType: string; - targetElementTypes: VisualElement["type"][]; -} - -export type TimelineDragData = - | MediaDragData - | TextDragData - | StickerDragData - | GraphicDragData - | EffectDragData; +import type { MaskableElement, VisualElement } from "./types"; +import type { ParamValues } from "@/params"; +import type { TransitionKind } from "@/rendering/transition"; + +interface BaseDragData { + id: string; + name: string; +} + +export interface MediaDragData extends BaseDragData { + type: "media"; + mediaType: "image" | "video" | "audio"; + targetElementTypes?: MaskableElement["type"][]; +} + +export interface TextDragData extends BaseDragData { + type: "text"; + content: string; +} + +export interface StickerDragData extends BaseDragData { + type: "sticker"; + stickerId: string; +} + +export interface GraphicDragData extends BaseDragData { + type: "graphic"; + definitionId: string; + params: Partial; +} + +export interface EffectDragData extends BaseDragData { + type: "effect"; + effectType: string; + targetElementTypes: VisualElement["type"][]; +} +export interface TransitionDragData extends BaseDragData { + type: "transition"; + transitionKind: TransitionKind; + inDuration: number; + outDuration: number; +} + +export type TimelineDragData = + | MediaDragData + | TextDragData + | StickerDragData + | GraphicDragData + | EffectDragData + | TransitionDragData; diff --git a/apps/web/src/timeline/hooks/use-timeline-drag-drop.ts b/apps/web/src/timeline/hooks/use-timeline-drag-drop.ts index c405819e..cde3874f 100644 --- a/apps/web/src/timeline/hooks/use-timeline-drag-drop.ts +++ b/apps/web/src/timeline/hooks/use-timeline-drag-drop.ts @@ -36,6 +36,7 @@ export function useTimelineDragDrop({ addMediaAsset: (args) => editor.media.addMediaAsset(args), executeCommand: (command) => editor.command.execute({ command }), insertElement: (args) => editor.timeline.insertElement(args), + updateElements: (args) => editor.timeline.updateElements(args), addClipEffect: (args) => editor.timeline.addClipEffect(args), }; const configRef = useCommittedRef(config); diff --git a/apps/web/src/timeline/placement/track-factory.ts b/apps/web/src/timeline/placement/track-factory.ts index 9c69ea8c..290141fc 100644 --- a/apps/web/src/timeline/placement/track-factory.ts +++ b/apps/web/src/timeline/placement/track-factory.ts @@ -84,6 +84,7 @@ export function buildEmptyTrack({ elements: [], hidden: false, muted: false, + volume: 0, }; case "text": return { @@ -108,6 +109,7 @@ export function buildEmptyTrack({ type: "audio", elements: [], muted: false, + volume: 0, }; case "effect": return { diff --git a/apps/web/src/timeline/scenes.ts b/apps/web/src/timeline/scenes.ts index 8aa1b6b5..155bda96 100644 --- a/apps/web/src/timeline/scenes.ts +++ b/apps/web/src/timeline/scenes.ts @@ -1,114 +1,115 @@ -import type { TScene } from "@/timeline"; -import { generateUUID } from "@/utils/id"; -import { calculateTotalDuration } from "@/timeline"; -import { MAIN_TRACK_NAME } from "@/timeline/placement/main-track"; -import { type MediaTime, ZERO_MEDIA_TIME } from "@/wasm"; - -export function getMainScene({ scenes }: { scenes: TScene[] }): TScene | null { - return scenes.find((scene) => scene.isMain) || null; -} - -export function ensureMainScene({ scenes }: { scenes: TScene[] }): TScene[] { - const hasMain = scenes.some((scene) => scene.isMain); - if (!hasMain) { - const mainScene = buildDefaultScene({ name: "Main scene", isMain: true }); - return [mainScene, ...scenes]; - } - return scenes; -} - -export function buildDefaultScene({ - name, - isMain, -}: { - name: string; - isMain: boolean; -}): TScene { - return { - id: generateUUID(), - name, - isMain, - tracks: { - overlay: [], - main: { - id: generateUUID(), - name: MAIN_TRACK_NAME, - type: "video", - elements: [], - muted: false, - hidden: false, - }, - audio: [], - }, - bookmarks: [], - createdAt: new Date(), - updatedAt: new Date(), - }; -} - -export function canDeleteScene({ scene }: { scene: TScene }): { - canDelete: boolean; - reason?: string; -} { - if (scene.isMain) { - return { canDelete: false, reason: "Cannot delete main scene" }; - } - return { canDelete: true }; -} - -export function getFallbackSceneAfterDelete({ - scenes, - deletedSceneId, - currentSceneId, -}: { - scenes: TScene[]; - deletedSceneId: string; - currentSceneId: string | null; -}): TScene | null { - if (currentSceneId !== deletedSceneId) { - return scenes.find((s) => s.id === currentSceneId) || null; - } - return getMainScene({ scenes }); -} - -export function findCurrentScene({ - scenes, - currentSceneId, -}: { - scenes: TScene[]; - currentSceneId: string; -}): TScene | null { - return ( - scenes.find((s) => s.id === currentSceneId) || - getMainScene({ scenes }) || - scenes[0] || - null - ); -} - -export function getProjectDurationFromScenes({ - scenes, -}: { - scenes: TScene[]; -}): MediaTime { - const mainScene = getMainScene({ scenes }) ?? scenes[0] ?? null; - if (!mainScene?.tracks) { - return ZERO_MEDIA_TIME; - } - - return calculateTotalDuration({ tracks: mainScene.tracks }); -} - -export function updateSceneInArray({ - scenes, - sceneId, - updates, -}: { - scenes: TScene[]; - sceneId: string; - updates: Partial; -}): TScene[] { - return scenes.map((scene) => - scene.id === sceneId ? { ...scene, ...updates } : scene, - ); -} +import type { TScene } from "@/timeline"; +import { generateUUID } from "@/utils/id"; +import { calculateTotalDuration } from "@/timeline"; +import { MAIN_TRACK_NAME } from "@/timeline/placement/main-track"; +import { type MediaTime, ZERO_MEDIA_TIME } from "@/wasm"; + +export function getMainScene({ scenes }: { scenes: TScene[] }): TScene | null { + return scenes.find((scene) => scene.isMain) || null; +} + +export function ensureMainScene({ scenes }: { scenes: TScene[] }): TScene[] { + const hasMain = scenes.some((scene) => scene.isMain); + if (!hasMain) { + const mainScene = buildDefaultScene({ name: "Main scene", isMain: true }); + return [mainScene, ...scenes]; + } + return scenes; +} + +export function buildDefaultScene({ + name, + isMain, +}: { + name: string; + isMain: boolean; +}): TScene { + return { + id: generateUUID(), + name, + isMain, + tracks: { + overlay: [], + main: { + id: generateUUID(), + name: MAIN_TRACK_NAME, + type: "video", + elements: [], + muted: false, + volume: 0, + hidden: false, + }, + audio: [], + }, + bookmarks: [], + createdAt: new Date(), + updatedAt: new Date(), + }; +} + +export function canDeleteScene({ scene }: { scene: TScene }): { + canDelete: boolean; + reason?: string; +} { + if (scene.isMain) { + return { canDelete: false, reason: "Cannot delete main scene" }; + } + return { canDelete: true }; +} + +export function getFallbackSceneAfterDelete({ + scenes, + deletedSceneId, + currentSceneId, +}: { + scenes: TScene[]; + deletedSceneId: string; + currentSceneId: string | null; +}): TScene | null { + if (currentSceneId !== deletedSceneId) { + return scenes.find((s) => s.id === currentSceneId) || null; + } + return getMainScene({ scenes }); +} + +export function findCurrentScene({ + scenes, + currentSceneId, +}: { + scenes: TScene[]; + currentSceneId: string; +}): TScene | null { + return ( + scenes.find((s) => s.id === currentSceneId) || + getMainScene({ scenes }) || + scenes[0] || + null + ); +} + +export function getProjectDurationFromScenes({ + scenes, +}: { + scenes: TScene[]; +}): MediaTime { + const mainScene = getMainScene({ scenes }) ?? scenes[0] ?? null; + if (!mainScene?.tracks) { + return ZERO_MEDIA_TIME; + } + + return calculateTotalDuration({ tracks: mainScene.tracks }); +} + +export function updateSceneInArray({ + scenes, + sceneId, + updates, +}: { + scenes: TScene[]; + sceneId: string; + updates: Partial; +}): TScene[] { + return scenes.map((scene) => + scene.id === sceneId ? { ...scene, ...updates } : scene, + ); +} diff --git a/apps/web/src/timeline/types.ts b/apps/web/src/timeline/types.ts index 5b3e4d29..48f3f592 100644 --- a/apps/web/src/timeline/types.ts +++ b/apps/web/src/timeline/types.ts @@ -37,6 +37,7 @@ export interface VideoTrack extends BaseTrack { type: "video"; elements: (VideoElement | ImageElement)[]; muted: boolean; + volume?: number; hidden: boolean; } @@ -50,6 +51,7 @@ export interface AudioTrack extends BaseTrack { type: "audio"; elements: AudioElement[]; muted: boolean; + volume?: number; } export interface GraphicTrack extends BaseTrack { @@ -256,12 +258,20 @@ export type ElementDragView = readonly dropTarget: DropTarget | null; }; +export interface TransitionCutTarget { + trackId: string; + outgoingElementId: string; + incomingElementId: string; + time: MediaTime; +} + export interface DropTarget { trackIndex: number; isNewTrack: boolean; insertPosition: "above" | "below" | null; xPosition: MediaTime; targetElement: { elementId: string; trackId: string } | null; + targetCut: TransitionCutTarget | null; } export interface ComputeDropTargetParams { diff --git a/apps/web/src/voiceover/__tests__/devices.test.ts b/apps/web/src/voiceover/__tests__/devices.test.ts new file mode 100644 index 00000000..524e8c3f --- /dev/null +++ b/apps/web/src/voiceover/__tests__/devices.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, test } from "bun:test"; +import { + DEFAULT_AUDIO_INPUT_DEVICE_ID, + buildVoiceOverAudioConstraints, + normalizeSelectedDeviceId, + type VoiceOverInputDevice, +} from "@/voiceover/devices"; + +const devices: VoiceOverInputDevice[] = [ + { + deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID, + label: "System default microphone", + isDefault: true, + }, + { + deviceId: "blue-yeti-snowball", + label: "Blue Snowball", + isDefault: false, + }, +]; + +describe("buildVoiceOverAudioConstraints", () => { + test("uses default processing constraints without an exact device", () => { + const constraints = buildVoiceOverAudioConstraints({ deviceId: null }); + expect(constraints.audio).toEqual({ + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); + }); + + test("adds an exact device constraint for a selected microphone", () => { + const constraints = buildVoiceOverAudioConstraints({ + deviceId: "blue-yeti-snowball", + }); + expect(constraints.audio).toEqual({ + deviceId: { exact: "blue-yeti-snowball" }, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); + }); +}); + +describe("normalizeSelectedDeviceId", () => { + test("keeps the selected device when it is still present", () => { + expect( + normalizeSelectedDeviceId({ + deviceId: "blue-yeti-snowball", + devices, + }), + ).toBe("blue-yeti-snowball"); + }); + + test("clears a selected device when it is no longer present", () => { + expect( + normalizeSelectedDeviceId({ + deviceId: "missing", + devices, + }), + ).toBeNull(); + }); + + test("keeps the browser default device id", () => { + expect( + normalizeSelectedDeviceId({ + deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID, + devices: [], + }), + ).toBe(DEFAULT_AUDIO_INPUT_DEVICE_ID); + }); +}); diff --git a/apps/web/src/voiceover/__tests__/enhancements.test.ts b/apps/web/src/voiceover/__tests__/enhancements.test.ts new file mode 100644 index 00000000..bc5b4b7c --- /dev/null +++ b/apps/web/src/voiceover/__tests__/enhancements.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, test } from "bun:test"; +import { + DEFAULT_VOICE_OVER_ENHANCEMENTS, + encodeAudioBufferToWav, + getAudioBufferPeak, + hasEnabledVoiceOverEnhancement, + type VoiceOverEnhancementSettings, +} from "@/voiceover/enhancements"; + +/* eslint-disable opencut/prefer-object-params -- Browser AudioBuffer methods are positional. */ +class FakeAudioBuffer implements AudioBuffer { + duration: number; + length: number; + numberOfChannels: number; + sampleRate: number; + + constructor( + private readonly channels: Array>, + sampleRate: number, + ) { + this.length = channels[0]?.length ?? 0; + this.numberOfChannels = channels.length; + this.sampleRate = sampleRate; + this.duration = this.length / sampleRate; + } + + copyFromChannel( + destination: Float32Array, + channelNumber: number, + bufferOffset = 0, + ): void { + destination.set( + this.getChannelData(channelNumber).slice( + bufferOffset, + bufferOffset + destination.length, + ), + ); + } + + copyToChannel( + source: Float32Array, + channelNumber: number, + bufferOffset = 0, + ): void { + this.getChannelData(channelNumber).set(source, bufferOffset); + } + + getChannelData(channel: number): Float32Array { + const data = this.channels[channel]; + if (!data) { + throw new Error(`Missing channel ${channel}`); + } + return data; + } +} +/* eslint-enable opencut/prefer-object-params */ + +describe("hasEnabledVoiceOverEnhancement", () => { + test("detects enabled defaults", () => { + expect( + hasEnabledVoiceOverEnhancement({ + settings: DEFAULT_VOICE_OVER_ENHANCEMENTS, + }), + ).toBe(true); + }); + + test("detects an all-disabled chain", () => { + const settings: VoiceOverEnhancementSettings = { + voiceBoost: false, + noiseGate: false, + highPass: false, + deEss: false, + limiter: false, + normalize: false, + }; + expect(hasEnabledVoiceOverEnhancement({ settings })).toBe(false); + }); +}); + +describe("getAudioBufferPeak", () => { + test("returns the maximum absolute sample", () => { + const buffer = new FakeAudioBuffer( + [ + createSamples([0, -0.25, 0.5]), + createSamples([0.1, -0.8, 0.2]), + ], + 48000, + ); + + expect(getAudioBufferPeak({ buffer })).toBeCloseTo(0.8); + }); +}); + +describe("encodeAudioBufferToWav", () => { + test("writes a 16-bit PCM WAV header", async () => { + const buffer = new FakeAudioBuffer( + [createSamples([0, 0.5, -0.5])], + 44100, + ); + + const blob = encodeAudioBufferToWav({ buffer }); + const view = new DataView(await blob.arrayBuffer()); + + expect(blob.type).toBe("audio/wav"); + expect(blob.size).toBe(50); + expect(readAscii({ view, offset: 0, length: 4 })).toBe("RIFF"); + expect(readAscii({ view, offset: 8, length: 4 })).toBe("WAVE"); + expect(view.getUint16(22, true)).toBe(1); + expect(view.getUint32(24, true)).toBe(44100); + expect(view.getUint16(34, true)).toBe(16); + expect(readAscii({ view, offset: 36, length: 4 })).toBe("data"); + }); +}); + +function readAscii({ + view, + offset, + length, +}: { + view: DataView; + offset: number; + length: number; +}): string { + return Array.from({ length }, (_, index) => + String.fromCharCode(view.getUint8(offset + index)), + ).join(""); +} + +function createSamples(values: number[]): Float32Array { + const samples = new Float32Array(new ArrayBuffer(values.length * 4)); + samples.set(values); + return samples; +} diff --git a/apps/web/src/voiceover/devices.ts b/apps/web/src/voiceover/devices.ts new file mode 100644 index 00000000..dd5b680c --- /dev/null +++ b/apps/web/src/voiceover/devices.ts @@ -0,0 +1,93 @@ +"use client"; + +export const DEFAULT_AUDIO_INPUT_DEVICE_ID = "default"; + +export interface VoiceOverInputDevice { + deviceId: string; + label: string; + isDefault: boolean; +} + +export function buildVoiceOverAudioConstraints({ + deviceId, +}: { + deviceId?: string | null; +}): MediaStreamConstraints { + const audio: MediaTrackConstraints = { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }; + + if (deviceId && deviceId !== DEFAULT_AUDIO_INPUT_DEVICE_ID) { + audio.deviceId = { exact: deviceId }; + } + + return { audio }; +} + +export function normalizeSelectedDeviceId({ + deviceId, + devices, +}: { + deviceId: string | null; + devices: VoiceOverInputDevice[]; +}): string | null { + if (!deviceId) { + return null; + } + + if (deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID) { + return deviceId; + } + + return devices.some((device) => device.deviceId === deviceId) + ? deviceId + : null; +} + +export async function listVoiceOverInputDevices({ + requestPermission = false, +}: { + requestPermission?: boolean; +} = {}): Promise { + const mediaDevices = navigator.mediaDevices; + if (!mediaDevices?.enumerateDevices) { + throw new Error("Microphone selection is not supported in this browser"); + } + + let permissionStream: MediaStream | null = null; + if (requestPermission) { + permissionStream = await mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + } + + try { + const devices = await mediaDevices.enumerateDevices(); + let inputIndex = 1; + return devices + .filter((device) => device.kind === "audioinput") + .map((device) => { + const isDefault = device.deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID; + const fallbackLabel = isDefault + ? "System default microphone" + : `Microphone ${inputIndex++}`; + return { + deviceId: device.deviceId, + label: device.label || fallbackLabel, + isDefault, + }; + }); + } finally { + if (permissionStream) { + for (const track of permissionStream.getTracks()) { + track.stop(); + } + } + } +} diff --git a/apps/web/src/voiceover/enhancements.ts b/apps/web/src/voiceover/enhancements.ts new file mode 100644 index 00000000..8718406f --- /dev/null +++ b/apps/web/src/voiceover/enhancements.ts @@ -0,0 +1,321 @@ +"use client"; + +export interface VoiceOverEnhancementSettings { + voiceBoost: boolean; + noiseGate: boolean; + highPass: boolean; + deEss: boolean; + limiter: boolean; + normalize: boolean; +} + +export const DEFAULT_VOICE_OVER_ENHANCEMENTS: VoiceOverEnhancementSettings = { + voiceBoost: true, + noiseGate: true, + highPass: true, + deEss: false, + limiter: true, + normalize: true, +}; + +export const VOICE_OVER_ENHANCEMENT_LABELS: Record< + keyof VoiceOverEnhancementSettings, + { label: string; description: string } +> = { + voiceBoost: { + label: "Voice boost", + description: "Adds compression and presence for a stronger voice.", + }, + noiseGate: { + label: "Noise gate", + description: "Turns down quiet room noise between phrases.", + }, + highPass: { + label: "High-pass", + description: "Removes low rumble and desk vibration.", + }, + deEss: { + label: "De-ess", + description: "Softens harsh S sounds and upper fizz.", + }, + limiter: { + label: "Limiter", + description: "Catches peaks after boosts.", + }, + normalize: { + label: "Normalize", + description: "Raises the finished take to a consistent level.", + }, +}; + +const WAV_BITS_PER_SAMPLE = 16; +const WAV_HEADER_BYTES = 44; +const TARGET_PEAK = 0.92; +const NOISE_GATE_THRESHOLD = 0.018; +const NOISE_GATE_FLOOR = 0.18; + +export function hasEnabledVoiceOverEnhancement({ + settings, +}: { + settings: VoiceOverEnhancementSettings; +}): boolean { + return Object.values(settings).some(Boolean); +} + +export async function applyVoiceOverEnhancementsToBlob({ + blob, + settings, +}: { + blob: Blob; + settings: VoiceOverEnhancementSettings; +}): Promise { + if (!hasEnabledVoiceOverEnhancement({ settings })) { + return blob; + } + + const audioContext = createAudioContext(); + try { + const buffer = await audioContext.decodeAudioData(await blob.arrayBuffer()); + const renderedBuffer = await renderEnhancedVoiceOverBuffer({ + buffer, + settings, + }); + return encodeAudioBufferToWav({ buffer: renderedBuffer }); + } finally { + void audioContext.close(); + } +} + +export async function renderEnhancedVoiceOverBuffer({ + buffer, + settings, +}: { + buffer: AudioBuffer; + settings: VoiceOverEnhancementSettings; +}): Promise { + const offlineContext = new OfflineAudioContext( + buffer.numberOfChannels, + Math.max(1, buffer.length), + buffer.sampleRate, + ); + const source = offlineContext.createBufferSource(); + source.buffer = settings.noiseGate + ? applyNoiseGateToBuffer({ + buffer, + context: offlineContext, + threshold: NOISE_GATE_THRESHOLD, + floor: NOISE_GATE_FLOOR, + }) + : buffer; + + let current: AudioNode = source; + + if (settings.highPass) { + const highPass = offlineContext.createBiquadFilter(); + highPass.type = "highpass"; + highPass.frequency.value = 85; + highPass.Q.value = 0.7; + current.connect(highPass); + current = highPass; + } + + if (settings.voiceBoost) { + const presence = offlineContext.createBiquadFilter(); + presence.type = "peaking"; + presence.frequency.value = 3200; + presence.Q.value = 0.9; + presence.gain.value = 2.5; + + const compressor = offlineContext.createDynamicsCompressor(); + compressor.threshold.value = -24; + compressor.knee.value = 12; + compressor.ratio.value = 3.5; + compressor.attack.value = 0.004; + compressor.release.value = 0.16; + + const makeupGain = offlineContext.createGain(); + makeupGain.gain.value = 1.45; + + current.connect(presence); + presence.connect(compressor); + compressor.connect(makeupGain); + current = makeupGain; + } + + if (settings.deEss) { + const deEss = offlineContext.createBiquadFilter(); + deEss.type = "peaking"; + deEss.frequency.value = 6400; + deEss.Q.value = 2.2; + deEss.gain.value = -3; + current.connect(deEss); + current = deEss; + } + + if (settings.limiter) { + const limiter = offlineContext.createDynamicsCompressor(); + limiter.threshold.value = -1; + limiter.knee.value = 0; + limiter.ratio.value = 20; + limiter.attack.value = 0.001; + limiter.release.value = 0.1; + current.connect(limiter); + current = limiter; + } + + current.connect(offlineContext.destination); + source.start(0); + + const rendered = await offlineContext.startRendering(); + if (settings.normalize) { + normalizeAudioBufferPeak({ buffer: rendered, targetPeak: TARGET_PEAK }); + } + clampAudioBuffer({ buffer: rendered, maxPeak: 0.98 }); + return rendered; +} + +export function encodeAudioBufferToWav({ + buffer, +}: { + buffer: AudioBuffer; +}): Blob { + const numChannels = buffer.numberOfChannels; + const bytesPerSample = WAV_BITS_PER_SAMPLE / 8; + const dataSize = buffer.length * numChannels * bytesPerSample; + const arrayBuffer = new ArrayBuffer(WAV_HEADER_BYTES + dataSize); + const view = new DataView(arrayBuffer); + + writeAsciiString({ view, offset: 0, value: "RIFF" }); + view.setUint32(4, 36 + dataSize, true); + writeAsciiString({ view, offset: 8, value: "WAVE" }); + writeAsciiString({ view, offset: 12, value: "fmt " }); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, buffer.sampleRate, true); + view.setUint32(28, buffer.sampleRate * numChannels * bytesPerSample, true); + view.setUint16(32, numChannels * bytesPerSample, true); + view.setUint16(34, WAV_BITS_PER_SAMPLE, true); + writeAsciiString({ view, offset: 36, value: "data" }); + view.setUint32(40, dataSize, true); + + let offset = WAV_HEADER_BYTES; + for (let sampleIndex = 0; sampleIndex < buffer.length; sampleIndex++) { + for (let channel = 0; channel < numChannels; channel++) { + const sample = Math.max( + -1, + Math.min(1, buffer.getChannelData(channel)[sampleIndex] ?? 0), + ); + const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff; + view.setInt16(offset, int16, true); + offset += bytesPerSample; + } + } + + return new Blob([arrayBuffer], { type: "audio/wav" }); +} + +export function getAudioBufferPeak({ buffer }: { buffer: AudioBuffer }): number { + let peak = 0; + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channelData = buffer.getChannelData(channel); + for (let index = 0; index < channelData.length; index++) { + peak = Math.max(peak, Math.abs(channelData[index] ?? 0)); + } + } + return peak; +} + +function normalizeAudioBufferPeak({ + buffer, + targetPeak, +}: { + buffer: AudioBuffer; + targetPeak: number; +}): void { + const peak = getAudioBufferPeak({ buffer }); + if (peak <= 0 || peak >= targetPeak) { + return; + } + + const gain = targetPeak / peak; + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channelData = buffer.getChannelData(channel); + for (let index = 0; index < channelData.length; index++) { + channelData[index] *= gain; + } + } +} + +function clampAudioBuffer({ + buffer, + maxPeak, +}: { + buffer: AudioBuffer; + maxPeak: number; +}): void { + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channelData = buffer.getChannelData(channel); + for (let index = 0; index < channelData.length; index++) { + channelData[index] = Math.max( + -maxPeak, + Math.min(maxPeak, channelData[index] ?? 0), + ); + } + } +} + +function applyNoiseGateToBuffer({ + buffer, + context, + threshold, + floor, +}: { + buffer: AudioBuffer; + context: OfflineAudioContext; + threshold: number; + floor: number; +}): AudioBuffer { + const gated = context.createBuffer( + buffer.numberOfChannels, + buffer.length, + buffer.sampleRate, + ); + + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const input = buffer.getChannelData(channel); + const output = gated.getChannelData(channel); + for (let index = 0; index < input.length; index++) { + const sample = input[index] ?? 0; + output[index] = + Math.abs(sample) < threshold + ? sample * floor + : sample; + } + } + + return gated; +} + +function createAudioContext(): AudioContext { + const AudioContextConstructor = + window.AudioContext || + (window as typeof window & { webkitAudioContext?: typeof AudioContext }) + .webkitAudioContext; + + return new AudioContextConstructor(); +} + +function writeAsciiString({ + view, + offset, + value, +}: { + view: DataView; + offset: number; + value: string; +}): void { + for (let index = 0; index < value.length; index++) { + view.setUint8(offset + index, value.charCodeAt(index)); + } +} diff --git a/apps/web/src/voiceover/live-waveform.tsx b/apps/web/src/voiceover/live-waveform.tsx new file mode 100644 index 00000000..cd25c833 --- /dev/null +++ b/apps/web/src/voiceover/live-waveform.tsx @@ -0,0 +1,171 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { cn } from "@/utils/ui"; +import { getActiveVoiceOverRecordingStream } from "@/voiceover/recorder"; + +const FFT_SIZE = 1024; +const HISTORY_SIZE = 96; + +export function VoiceOverLiveWaveform({ + recording, + className, +}: { + recording: boolean; + className?: string; +}) { + const canvasRef = useRef(null); + const levelRef = useRef(0); + const historyRef = useRef( + Array.from({ length: HISTORY_SIZE }, () => 0), + ); + const [level, setLevel] = useState(0); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || !recording) { + drawWaveform({ + canvas, + history: historyRef.current, + level: 0, + }); + setLevel(0); + return; + } + + const stream = getActiveVoiceOverRecordingStream(); + if (!stream) { + return; + } + + const audioContext = createAudioContext(); + const source = audioContext.createMediaStreamSource(stream); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = FFT_SIZE; + analyser.smoothingTimeConstant = 0.72; + source.connect(analyser); + + const samples = new Uint8Array(analyser.fftSize); + let animationFrame = 0; + let lastLevelUpdate = 0; + + const render = (timestamp: number) => { + analyser.getByteTimeDomainData(samples); + const rms = getRmsLevel({ samples }); + levelRef.current = rms; + historyRef.current = [...historyRef.current.slice(1), rms]; + drawWaveform({ + canvas, + history: historyRef.current, + level: rms, + }); + + if (timestamp - lastLevelUpdate > 80) { + lastLevelUpdate = timestamp; + setLevel(rms); + } + animationFrame = requestAnimationFrame(render); + }; + + animationFrame = requestAnimationFrame(render); + + return () => { + cancelAnimationFrame(animationFrame); + source.disconnect(); + analyser.disconnect(); + void audioContext.close(); + }; + }, [recording]); + + return ( +
+
+ +
+
+
+
+
+ ); +} + +function getRmsLevel({ samples }: { samples: Uint8Array }): number { + let sum = 0; + for (let index = 0; index < samples.length; index++) { + const centered = ((samples[index] ?? 128) - 128) / 128; + sum += centered * centered; + } + return Math.sqrt(sum / samples.length); +} + +function drawWaveform({ + canvas, + history, + level, +}: { + canvas: HTMLCanvasElement | null; + history: number[]; + level: number; +}): void { + if (!canvas) { + return; + } + + const rect = canvas.getBoundingClientRect(); + const dpr = window.devicePixelRatio || 1; + const width = Math.max(1, Math.round(rect.width * dpr)); + const height = Math.max(1, Math.round(rect.height * dpr)); + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + + const ctx = canvas.getContext("2d"); + if (!ctx) { + return; + } + + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, width, height); + + const centerY = height / 2; + const step = width / Math.max(1, history.length); + + ctx.strokeStyle = "rgba(148, 163, 184, 0.35)"; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(0, centerY); + ctx.lineTo(width, centerY); + ctx.stroke(); + + for (let index = 0; index < history.length; index++) { + const value = Math.max(0, history[index] ?? 0); + const barHeight = Math.max(1, Math.min(height, value * height * 2.8)); + const x = Math.round(index * step); + const y = Math.round(centerY - barHeight / 2); + ctx.fillStyle = + value > 0.62 + ? "rgba(239, 68, 68, 0.9)" + : value > 0.32 + ? "rgba(245, 158, 11, 0.9)" + : "rgba(34, 197, 94, 0.9)"; + ctx.fillRect(x, y, Math.max(1, Math.floor(step - 1)), barHeight); + } + + if (level > 0.78) { + ctx.fillStyle = "rgba(239, 68, 68, 0.18)"; + ctx.fillRect(0, 0, width, height); + } +} + +function createAudioContext(): AudioContext { + const AudioContextConstructor = + window.AudioContext || + (window as typeof window & { webkitAudioContext?: typeof AudioContext }) + .webkitAudioContext; + + return new AudioContextConstructor(); +} diff --git a/apps/web/src/voiceover/recorder.ts b/apps/web/src/voiceover/recorder.ts index b71ca74f..48322898 100644 --- a/apps/web/src/voiceover/recorder.ts +++ b/apps/web/src/voiceover/recorder.ts @@ -5,6 +5,8 @@ * actual microphone stream. */ +import { buildVoiceOverAudioConstraints } from "@/voiceover/devices"; + export const PREFERRED_MIME_TYPES = [ "audio/webm;codecs=opus", "audio/webm", @@ -95,6 +97,11 @@ interface ActiveRecording { startedAtMs: number; } +export interface VoiceOverRecordingSession { + stream: MediaStream; + mimeType: string; +} + let activeRecording: ActiveRecording | null = null; function stopStreamTracks({ stream }: { stream: MediaStream }): void { @@ -103,7 +110,15 @@ function stopStreamTracks({ stream }: { stream: MediaStream }): void { } } -export async function startVoiceOverRecording(): Promise { +export function getActiveVoiceOverRecordingStream(): MediaStream | null { + return activeRecording?.stream ?? null; +} + +export async function startVoiceOverRecording({ + deviceId, +}: { + deviceId?: string | null; +} = {}): Promise { if (activeRecording) { throw new Error("A voice-over recording is already in progress"); } @@ -113,7 +128,24 @@ export async function startVoiceOverRecording(): Promise { throw new Error("Voice-over recording is not supported in this browser"); } - const stream = await mediaDevices.getUserMedia({ audio: true }); + let stream: MediaStream; + try { + stream = await mediaDevices.getUserMedia( + buildVoiceOverAudioConstraints({ deviceId }), + ); + } catch (error) { + if ( + deviceId && + error instanceof DOMException && + (error.name === "NotFoundError" || error.name === "OverconstrainedError") + ) { + stream = await mediaDevices.getUserMedia( + buildVoiceOverAudioConstraints({ deviceId: null }), + ); + } else { + throw error; + } + } try { const mimeType = pickSupportedMimeType({ isTypeSupported: (candidate) => MediaRecorder.isTypeSupported(candidate), @@ -135,6 +167,10 @@ export async function startVoiceOverRecording(): Promise { mimeType: recorder.mimeType || mimeType || "audio/webm", startedAtMs: performance.now(), }; + return { + stream, + mimeType: activeRecording.mimeType, + }; } catch (error) { stopStreamTracks({ stream }); throw error; diff --git a/apps/web/src/voiceover/voiceover-popover.tsx b/apps/web/src/voiceover/voiceover-popover.tsx new file mode 100644 index 00000000..a1f74646 --- /dev/null +++ b/apps/web/src/voiceover/voiceover-popover.tsx @@ -0,0 +1,276 @@ +"use client"; + +import { useCallback, useMemo, useState } from "react"; +import { MicIcon, StopIcon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { RefreshCw } from "lucide-react"; +import { toast } from "sonner"; +import { invokeAction } from "@/actions"; +import { Button } from "@/components/ui/button"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { cn } from "@/utils/ui"; +import { + DEFAULT_AUDIO_INPUT_DEVICE_ID, + listVoiceOverInputDevices, + normalizeSelectedDeviceId, + type VoiceOverInputDevice, +} from "@/voiceover/devices"; +import { + VOICE_OVER_ENHANCEMENT_LABELS, + type VoiceOverEnhancementSettings, +} from "@/voiceover/enhancements"; +import { VoiceOverLiveWaveform } from "@/voiceover/live-waveform"; +import { + formatElapsedSeconds, + getVoiceOverErrorMessage, +} from "@/voiceover/recorder"; +import { useRecordingElapsedSeconds } from "@/voiceover/use-recording-elapsed"; +import { + useVoiceOverPreferencesStore, + useVoiceOverStore, +} from "@/voiceover/voiceover-store"; + +const ENHANCEMENT_ORDER: Array = [ + "voiceBoost", + "noiseGate", + "highPass", + "deEss", + "limiter", + "normalize", +]; + +export function VoiceOverRecorderPopover() { + const [open, setOpen] = useState(false); + const [devices, setDevices] = useState([]); + const [isLoadingDevices, setIsLoadingDevices] = useState(false); + const status = useVoiceOverStore((state) => state.status); + const isRecording = status === "recording"; + const isProcessing = status === "processing"; + const elapsedSeconds = useRecordingElapsedSeconds(); + const selectedDeviceId = useVoiceOverPreferencesStore( + (state) => state.selectedDeviceId, + ); + const enhancements = useVoiceOverPreferencesStore( + (state) => state.enhancements, + ); + const setSelectedDeviceId = useVoiceOverPreferencesStore( + (state) => state.setSelectedDeviceId, + ); + const setEnhancementEnabled = useVoiceOverPreferencesStore( + (state) => state.setEnhancementEnabled, + ); + + const devicesWithDefault = useMemo(() => { + if ( + devices.some( + (device) => device.deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID, + ) + ) { + return devices; + } + return [ + { + deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID, + label: "System default microphone", + isDefault: true, + }, + ...devices, + ]; + }, [devices]); + + const selectedValue = selectedDeviceId ?? DEFAULT_AUDIO_INPUT_DEVICE_ID; + + const refreshDevices = useCallback( + async ({ requestPermission }: { requestPermission: boolean }) => { + setIsLoadingDevices(true); + try { + const nextDevices = await listVoiceOverInputDevices({ + requestPermission, + }); + setDevices(nextDevices); + const normalizedDeviceId = normalizeSelectedDeviceId({ + deviceId: selectedDeviceId, + devices: nextDevices, + }); + if (normalizedDeviceId !== selectedDeviceId) { + setSelectedDeviceId({ deviceId: normalizedDeviceId }); + } + } catch (error) { + toast.error(getVoiceOverErrorMessage({ error })); + } finally { + setIsLoadingDevices(false); + } + }, + [selectedDeviceId, setSelectedDeviceId], + ); + + const handleOpenChange = useCallback( + (nextOpen: boolean) => { + setOpen(nextOpen); + if (nextOpen) { + void refreshDevices({ requestPermission: true }); + } + }, + [refreshDevices], + ); + + const handleRecordClick = (event: React.MouseEvent) => { + event.stopPropagation(); + void invokeAction("voice-over"); + }; + + return ( + + + + + + + + + + {isProcessing + ? "Saving voice-over" + : isRecording + ? "Stop voice-over" + : "Record voice-over"} + + + + + event.stopPropagation()} + > +
+
+
+
Voice-over
+
+ {isRecording + ? formatElapsedSeconds({ seconds: elapsedSeconds }) + : isProcessing + ? "Saving" + : "Ready"} +
+
+ +
+ + + +
+
+
+ Input +
+ +
+ +
+ +
+
+ Enhancements +
+
+ {ENHANCEMENT_ORDER.map((id) => { + const option = VOICE_OVER_ENHANCEMENT_LABELS[id]; + return ( +
+ + {option.label} + + {option.description} + + + + setEnhancementEnabled({ id, enabled }) + } + /> +
+ ); + })} +
+
+
+
+
+ ); +} diff --git a/apps/web/src/voiceover/voiceover-store.ts b/apps/web/src/voiceover/voiceover-store.ts index 44ccae6c..e4024c1e 100644 --- a/apps/web/src/voiceover/voiceover-store.ts +++ b/apps/web/src/voiceover/voiceover-store.ts @@ -6,18 +6,37 @@ */ import { create } from "zustand"; +import { persist } from "zustand/middleware"; import type { MediaTime } from "@/wasm"; +import { + DEFAULT_VOICE_OVER_ENHANCEMENTS, + type VoiceOverEnhancementSettings, +} from "@/voiceover/enhancements"; -export type VoiceOverStatus = "idle" | "recording"; +export type VoiceOverStatus = "idle" | "recording" | "processing"; interface VoiceOverStore { status: VoiceOverStatus; startedAtMs: number | null; playheadStartTime: MediaTime | null; startRecording: ({ playheadTime }: { playheadTime: MediaTime }) => void; + processRecording: () => void; stopRecording: () => void; } +interface VoiceOverPreferencesStore { + selectedDeviceId: string | null; + enhancements: VoiceOverEnhancementSettings; + setSelectedDeviceId: ({ deviceId }: { deviceId: string | null }) => void; + setEnhancementEnabled: ({ + id, + enabled, + }: { + id: keyof VoiceOverEnhancementSettings; + enabled: boolean; + }) => void; +} + export const useVoiceOverStore = create()((set) => ({ status: "idle", startedAtMs: null, @@ -31,6 +50,13 @@ export const useVoiceOverStore = create()((set) => ({ }); }, + processRecording: () => { + set({ + status: "processing", + startedAtMs: null, + }); + }, + stopRecording: () => { set({ status: "idle", @@ -39,3 +65,32 @@ export const useVoiceOverStore = create()((set) => ({ }); }, })); + +export const useVoiceOverPreferencesStore = create()( + persist( + (set) => ({ + selectedDeviceId: null, + enhancements: DEFAULT_VOICE_OVER_ENHANCEMENTS, + + setSelectedDeviceId: ({ deviceId }) => { + set({ selectedDeviceId: deviceId }); + }, + + setEnhancementEnabled: ({ id, enabled }) => { + set((state) => ({ + enhancements: { + ...state.enhancements, + [id]: enabled, + }, + })); + }, + }), + { + name: "voiceover-preferences-store", + partialize: (state) => ({ + selectedDeviceId: state.selectedDeviceId, + enhancements: state.enhancements, + }), + }, + ), +); -- cgit v1.3.1