diff options
18 files changed, 915 insertions, 53 deletions
diff --git a/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx b/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx index 4dbc1a17..5ac0fbb7 100644 --- a/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx +++ b/apps/web/src/components/editor/panels/properties/tabs/audio-tab.tsx @@ -1,5 +1,7 @@ +import { Button } from "@/components/ui/button";
import { NumberField } from "@/components/ui/number-field";
import { VOLUME_DB_MAX, VOLUME_DB_MIN } from "@/lib/timeline/audio-constants";
+import { isSourceAudioSeparated } from "@/lib/timeline/audio-separation";
import { DEFAULTS } from "@/lib/timeline/defaults";
import {
clamp,
@@ -10,6 +12,7 @@ import { } from "@/utils/math";
import type { AudioElement, VideoElement } from "@/lib/timeline";
import { resolveNumberAtTime } from "@/lib/animation";
+import { useEditor } from "@/hooks/use-editor";
import { useElementPlayhead } from "../hooks/use-element-playhead";
import { useKeyframedNumberProperty } from "../hooks/use-keyframed-number-property";
import { KeyframeToggle } from "../components/keyframe-toggle";
@@ -34,6 +37,7 @@ export function AudioTab({ element: AudioElement | VideoElement;
trackId: string;
}) {
+ const editor = useEditor();
const { localTime, isPlayheadWithinElementRange } = useElementPlayhead({
startTime: element.startTime,
duration: element.duration,
@@ -80,46 +84,68 @@ export function AudioTab({ rightValue: DEFAULTS.element.volume,
})
: (element.volume ?? DEFAULTS.element.volume) === DEFAULTS.element.volume;
+ const isSeparated =
+ element.type === "video" && isSourceAudioSeparated({ element });
return (
- <Section collapsible sectionKey={`${element.id}:audio`}>
- <SectionHeader>
- <SectionTitle>Audio</SectionTitle>
- </SectionHeader>
- <SectionContent>
- <SectionFields>
- <SectionField
- label="Volume"
- beforeLabel={
- <KeyframeToggle
- isActive={volume.isKeyframedAtTime}
- isDisabled={!isPlayheadWithinElementRange}
- title="Toggle volume keyframe"
- onToggle={volume.toggleKeyframe}
- />
+ <>
+ {isSeparated && (
+ <div className="mx-4 mt-4 rounded-md border bg-muted/30 p-3">
+ <p className="text-sm">Audio has been separated.</p>
+ <Button
+ className="mt-3"
+ size="sm"
+ variant="secondary"
+ onClick={() =>
+ editor.timeline.toggleSourceAudioSeparation({
+ trackId,
+ elementId: element.id,
+ })
}
>
- <NumberField
- icon={<HugeiconsIcon icon={VolumeHighIcon} />}
- value={volume.displayValue}
- onFocus={volume.onFocus}
- onChange={volume.onChange}
- onBlur={volume.onBlur}
- dragSensitivity="slow"
- scrubClamp={{ min: VOLUME_DB_MIN, max: VOLUME_DB_MAX }}
- onScrub={volume.scrubTo}
- onScrubEnd={volume.commitScrub}
- onReset={() =>
- volume.commitValue({
- value: DEFAULTS.element.volume,
- })
+ Recover audio
+ </Button>
+ </div>
+ )}
+ <Section collapsible sectionKey={`${element.id}:audio`}>
+ <SectionHeader>
+ <SectionTitle>Audio</SectionTitle>
+ </SectionHeader>
+ <SectionContent>
+ <SectionFields>
+ <SectionField
+ label="Volume"
+ beforeLabel={
+ <KeyframeToggle
+ isActive={volume.isKeyframedAtTime}
+ isDisabled={!isPlayheadWithinElementRange}
+ title="Toggle volume keyframe"
+ onToggle={volume.toggleKeyframe}
+ />
}
- isDefault={isDefault}
- suffix="dB"
- />
- </SectionField>
- </SectionFields>
- </SectionContent>
- </Section>
+ >
+ <NumberField
+ icon={<HugeiconsIcon icon={VolumeHighIcon} />}
+ value={volume.displayValue}
+ onFocus={volume.onFocus}
+ onChange={volume.onChange}
+ onBlur={volume.onBlur}
+ dragSensitivity="slow"
+ scrubClamp={{ min: VOLUME_DB_MIN, max: VOLUME_DB_MAX }}
+ onScrub={volume.scrubTo}
+ onScrubEnd={volume.commitScrub}
+ onReset={() =>
+ volume.commitValue({
+ value: DEFAULTS.element.volume,
+ })
+ }
+ isDefault={isDefault}
+ suffix="dB"
+ />
+ </SectionField>
+ </SectionFields>
+ </SectionContent>
+ </Section>
+ </>
);
}
diff --git a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx index f196da44..bb30b0b0 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx @@ -40,6 +40,11 @@ import type { import type { MediaAsset } from "@/lib/media/types";
import { mediaSupportsAudio } from "@/lib/media/media-utils";
import {
+ canToggleSourceAudio,
+ getSourceAudioActionLabel,
+ isSourceAudioSeparated,
+} from "@/lib/timeline/audio-separation";
+import {
getActionDefinition,
type TAction,
type TActionWithOptionalArgs,
@@ -61,7 +66,9 @@ import { Search01Icon,
Exchange01Icon,
KeyframeIcon,
+ Link02Icon,
MagicWand05Icon,
+ Unlink02Icon,
} from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import { uppercase } from "@/utils/string";
@@ -280,6 +287,19 @@ export function TimelineElement({ };
const isMuted = canElementHaveAudio(element) && element.muted === true;
+ const canToggleCurrentSourceAudio =
+ selectedElements.length === 1 &&
+ isCurrentElementSelected &&
+ canToggleSourceAudio({
+ element,
+ mediaAsset,
+ });
+ const sourceAudioLabel =
+ element.type === "video"
+ ? getSourceAudioActionLabel({ element })
+ : "Extract audio";
+ const isElementSourceAudioSeparated =
+ element.type === "video" && isSourceAudioSeparated({ element });
return (
<ContextMenu>
@@ -334,6 +354,21 @@ export function TimelineElement({ isMuted={isMuted}
/>
)}
+ {canToggleCurrentSourceAudio && (
+ <ContextMenuItem
+ icon={
+ <HugeiconsIcon
+ icon={isElementSourceAudioSeparated ? Unlink02Icon : Link02Icon}
+ />
+ }
+ onClick={(event: React.MouseEvent) => {
+ event.stopPropagation();
+ invokeAction("toggle-source-audio");
+ }}
+ >
+ {sourceAudioLabel}
+ </ContextMenuItem>
+ )}
{canElementBeHidden(element) && (
<VisibilityMenuItem
element={element}
diff --git a/apps/web/src/components/editor/panels/timeline/timeline-toolbar.tsx b/apps/web/src/components/editor/panels/timeline/timeline-toolbar.tsx index f398264c..abe36f88 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-toolbar.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-toolbar.tsx @@ -1,4 +1,5 @@ import { useEditor } from "@/hooks/use-editor"; +import { useElementSelection } from "@/hooks/timeline/element/use-element-selection"; import { TooltipProvider, Tooltip, @@ -6,7 +7,6 @@ import { TooltipContent, } from "@/components/ui/tooltip"; import { Button } from "@/components/ui/button"; -import { SplitSquareHorizontal } from "lucide-react"; import { SplitButton, SplitButtonLeft, @@ -18,6 +18,12 @@ import { TIMELINE_CONSTANTS } from "@/constants/timeline-constants"; import { sliderToZoom, zoomToSlider } from "@/lib/timeline/zoom-utils"; import { ScenesView } from "@/components/editor/scenes-view"; import { type TActionWithOptionalArgs, invokeAction } from "@/lib/actions"; +import { + canToggleSourceAudio, + getSourceAudioActionLabel, + isSourceAudioSeparated, +} from "@/lib/timeline/audio-separation"; +import { hasMediaId } from "@/lib/timeline"; import { cn } from "@/utils/ui"; import { useTimelineStore } from "@/stores/timeline-store"; import { ScrollArea } from "@/components/ui/scroll-area"; @@ -33,8 +39,10 @@ import { Copy01Icon, AlignLeftIcon, AlignRightIcon, + Link02Icon, Layers01Icon, Chart03Icon, + Unlink02Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { OcRippleIcon } from "@/components/icons"; @@ -78,9 +86,47 @@ export function TimelineToolbar({ } function ToolbarLeftSection() { + const editor = useEditor(); + const mediaAssets = useEditor((currentEditor) => currentEditor.media.getAssets()); + const { selectedElements } = useElementSelection(); const isCurrentlyBookmarked = useEditor((e) => e.scenes.isBookmarked({ time: e.playback.getCurrentTime() }), ); + const selectedElement = + selectedElements.length === 1 + ? (editor.timeline.getElementsWithTracks({ + elements: selectedElements, + })[0] ?? null) + : null; + const selectedMediaAsset = (() => { + if (!selectedElement) { + return null; + } + + const { element } = selectedElement; + if (!hasMediaId(element)) { + return null; + } + + return mediaAssets.find((asset) => asset.id === element.mediaId) ?? null; + })(); + const canToggleSelectedSourceAudio = + !!selectedElement && + canToggleSourceAudio({ + element: selectedElement.element, + mediaAsset: selectedMediaAsset, + }); + const sourceAudioLabel = + selectedElement?.element.type === "video" + ? getSourceAudioActionLabel({ + element: selectedElement.element, + }) + : "Extract audio"; + const isSelectedSourceAudioSeparated = + selectedElement?.element.type === "video" && + isSourceAudioSeparated({ + element: selectedElement.element, + }); const handleAction = ({ action, @@ -117,10 +163,16 @@ function ToolbarLeftSection() { /> <ToolbarButton - icon={<SplitSquareHorizontal />} - tooltip="Separate audio (coming soon)" - disabled={true} - onClick={({ event: _event }) => {}} + icon={ + <HugeiconsIcon + icon={isSelectedSourceAudioSeparated ? Unlink02Icon : Link02Icon} + /> + } + tooltip={sourceAudioLabel} + disabled={!canToggleSelectedSourceAudio} + onClick={({ event }) => + handleAction({ action: "toggle-source-audio", event }) + } /> <ToolbarButton @@ -275,6 +327,7 @@ function ToolbarButton({ <Button variant={isActive ? "secondary" : "text"} size="icon" + disabled={disabled} onClick={(event) => onClick({ event })} className={cn( "rounded-sm", diff --git a/apps/web/src/core/managers/timeline-manager.ts b/apps/web/src/core/managers/timeline-manager.ts index 7189761c..6f262e57 100644 --- a/apps/web/src/core/managers/timeline-manager.ts +++ b/apps/web/src/core/managers/timeline-manager.ts @@ -45,6 +45,7 @@ import { ToggleMaskInvertedCommand, UpsertEffectParamKeyframeCommand, RemoveEffectParamKeyframeCommand, + ToggleSourceAudioSeparationCommand, } from "@/lib/commands/timeline"; import { BatchCommand } from "@/lib/commands"; import type { InsertElementParams } from "@/lib/commands/timeline/element/insert-element"; @@ -281,6 +282,20 @@ export class TimelineManager { this.editor.command.execute({ command }); } + toggleSourceAudioSeparation({ + trackId, + elementId, + }: { + trackId: string; + elementId: string; + }): void { + const command = new ToggleSourceAudioSeparationCommand({ + trackId, + elementId, + }); + this.editor.command.execute({ command }); + } + updateElements({ updates, pushHistory = true, diff --git a/apps/web/src/hooks/actions/use-editor-actions.ts b/apps/web/src/hooks/actions/use-editor-actions.ts index ba480a30..87aa6b48 100644 --- a/apps/web/src/hooks/actions/use-editor-actions.ts +++ b/apps/web/src/hooks/actions/use-editor-actions.ts @@ -6,9 +6,13 @@ import { useActionHandler } from "@/hooks/actions/use-action-handler"; import { useEditor } from "../use-editor"; import { useElementSelection } from "../timeline/element/use-element-selection"; import { useKeyframeSelection } from "../timeline/element/use-keyframe-selection"; -import { getElementsAtTime } from "@/lib/timeline"; +import { + getElementsAtTime, + hasMediaId, +} from "@/lib/timeline"; import { cancelInteraction } from "@/lib/cancel-interaction"; import { invokeAction } from "@/lib/actions"; +import { canToggleSourceAudio } from "@/lib/timeline/audio-separation"; import { activateScope, clearActiveScope, @@ -266,6 +270,48 @@ export function useEditorActions() { ); useActionHandler( + "toggle-source-audio", + () => { + if (selectedElements.length !== 1) { + return; + } + + const selectedElement = editor.timeline.getElementsWithTracks({ + elements: selectedElements, + })[0]; + if (!selectedElement) { + return; + } + + const mediaAsset = (() => { + const { element } = selectedElement; + if (!hasMediaId(element)) { + return null; + } + + return ( + editor.media.getAssets().find((asset) => asset.id === element.mediaId) ?? + null + ); + })(); + if ( + !canToggleSourceAudio({ + element: selectedElement.element, + mediaAsset, + }) + ) { + return; + } + + editor.timeline.toggleSourceAudioSeparation({ + trackId: selectedElement.track.id, + elementId: selectedElement.element.id, + }); + }, + undefined, + ); + + useActionHandler( "select-all", () => { const allElements = editor.timeline.getTracks().flatMap((track) => diff --git a/apps/web/src/lib/actions/definitions.ts b/apps/web/src/lib/actions/definitions.ts index e48126d0..3059a2e3 100644 --- a/apps/web/src/lib/actions/definitions.ts +++ b/apps/web/src/lib/actions/definitions.ts @@ -101,6 +101,10 @@ export const ACTIONS = { description: "Toggle ripple editing",
category: "editing",
},
+ "toggle-source-audio": {
+ description: "Extract or recover source audio",
+ category: "editing",
+ },
"select-all": {
description: "Select all elements",
category: "selection",
diff --git a/apps/web/src/lib/commands/timeline/element/index.ts b/apps/web/src/lib/commands/timeline/element/index.ts index a7e513fe..5ce48d55 100644 --- a/apps/web/src/lib/commands/timeline/element/index.ts +++ b/apps/web/src/lib/commands/timeline/element/index.ts @@ -8,6 +8,7 @@ export { SplitElementsCommand } from "./split-elements"; export { UpdateElementCommand } from "./update-element";
export { ToggleElementsVisibilityCommand } from "./toggle-elements-visibility";
export { ToggleElementsMutedCommand } from "./toggle-elements-muted";
+export { ToggleSourceAudioSeparationCommand } from "./toggle-source-audio-separation";
export { MoveElementCommand } from "./move-elements";
export * from "./keyframes";
export * from "./effects";
diff --git a/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts new file mode 100644 index 00000000..d0907c5b --- /dev/null +++ b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts @@ -0,0 +1,143 @@ +import { EditorCore } from "@/core"; +import { Command } from "@/lib/commands/base-command"; +import { + buildSeparatedAudioElement, + canExtractSourceAudio, + canRecoverSourceAudio, +} from "@/lib/timeline/audio-separation"; +import { + applyPlacement, + resolveTrackPlacement, +} from "@/lib/timeline/placement"; +import { updateElementInTracks } from "@/lib/timeline/track-element-update"; +import type { TimelineTrack, VideoElement } from "@/lib/timeline/types"; +import { generateUUID } from "@/utils/id"; + +export class ToggleSourceAudioSeparationCommand extends Command { + private savedState: TimelineTrack[] | null = null; + + constructor( + private readonly params: { + trackId: string; + elementId: string; + }, + ) { + super(); + } + + execute(): void { + const editor = EditorCore.getInstance(); + this.savedState = editor.timeline.getTracks(); + + const trackIndex = this.savedState.findIndex( + (track) => track.id === this.params.trackId, + ); + if (trackIndex < 0) { + return; + } + + const sourceTrack = this.savedState[trackIndex]; + const sourceElement = sourceTrack.elements.find( + (element) => element.id === this.params.elementId, + ); + if (!sourceElement) { + return; + } + + if (canRecoverSourceAudio({ element: sourceElement })) { + editor.timeline.updateTracks( + updateSourceAudioEnabled({ + tracks: this.savedState, + trackId: this.params.trackId, + elementId: this.params.elementId, + isSourceAudioEnabled: true, + }), + ); + return; + } + + const mediaAsset = editor + .media + .getAssets() + .find((asset) => + sourceElement.type === "video" ? asset.id === sourceElement.mediaId : false, + ); + if (!canExtractSourceAudio({ element: sourceElement, mediaAsset })) { + return; + } + if (sourceElement.duration <= 0) { + return; + } + + const separatedAudioElement = { + ...buildSeparatedAudioElement({ + sourceElement, + }), + id: generateUUID(), + }; + const placementResult = resolveTrackPlacement({ + tracks: this.savedState, + trackType: "audio", + timeSpans: [ + { + startTime: separatedAudioElement.startTime, + duration: separatedAudioElement.duration, + }, + ], + strategy: { type: "aboveSource", sourceTrackIndex: trackIndex }, + }); + if (!placementResult) { + return; + } + + const appliedPlacement = applyPlacement({ + tracks: this.savedState, + placementResult, + elements: [separatedAudioElement], + }); + if (!appliedPlacement) { + return; + } + + editor.timeline.updateTracks( + updateSourceAudioEnabled({ + tracks: appliedPlacement.updatedTracks, + trackId: this.params.trackId, + elementId: this.params.elementId, + isSourceAudioEnabled: false, + }), + ); + } + + undo(): void { + if (!this.savedState) { + return; + } + + const editor = EditorCore.getInstance(); + editor.timeline.updateTracks(this.savedState); + } +} + +function updateSourceAudioEnabled({ + tracks, + trackId, + elementId, + isSourceAudioEnabled, +}: { + tracks: TimelineTrack[]; + trackId: string; + elementId: string; + isSourceAudioEnabled: boolean; +}): TimelineTrack[] { + return updateElementInTracks({ + tracks, + trackId, + elementId, + elementPredicate: (element): element is VideoElement => element.type === "video", + update: (element) => ({ + ...element, + isSourceAudioEnabled, + }), + }); +} diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts index 41d3b38e..0afc42fb 100644 --- a/apps/web/src/lib/media/audio.ts +++ b/apps/web/src/lib/media/audio.ts @@ -12,7 +12,10 @@ import { hasAnimatedVolume, resolveEffectiveAudioGain, } from "@/lib/timeline/audio-state"; -import { canElementHaveAudio } from "@/lib/timeline/element-utils"; +import { + doesElementHaveEnabledAudio, +} from "@/lib/timeline/audio-separation"; +import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils"; import { canTracktHaveAudio } from "@/lib/timeline"; import { mediaSupportsAudio } from "@/lib/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime"; @@ -100,6 +103,11 @@ export async function collectAudioElements({ if (!canElementHaveAudio(element)) continue; if (element.duration <= 0) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; + const isTrackMuted = canTracktHaveAudio(track) && track.muted; if (element.type === "audio") { @@ -132,7 +140,6 @@ export async function collectAudioElements({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; pendingElements.push( @@ -451,6 +458,10 @@ export async function collectAudioMixSources({ for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; if (element.muted === true) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; const volume = resolveEffectiveAudioGain({ element, localTime: 0, @@ -473,10 +484,7 @@ export async function collectAudioMixSources({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); - if (!mediaAsset) continue; - - if (mediaSupportsAudio({ media: mediaAsset })) { + if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) { audioMixSources.push( collectMediaAudioSource({ element, mediaAsset, volume }), ); @@ -512,6 +520,11 @@ export async function collectAudioClips({ for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; + const isElementMuted = "muted" in element ? (element.muted ?? false) : false; const muted = isTrackMuted || isElementMuted; @@ -543,10 +556,7 @@ export async function collectAudioClips({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); - if (!mediaAsset) continue; - - if (mediaSupportsAudio({ media: mediaAsset })) { + if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) { clips.push( collectMediaAudioClip({ element, diff --git a/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts new file mode 100644 index 00000000..173b6402 --- /dev/null +++ b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, test } from "bun:test"; +import type { AudioElement, VideoElement } from "@/lib/timeline"; +import { + buildSeparatedAudioElement, + doesElementHaveEnabledAudio, + isSourceAudioEnabled, + isSourceAudioSeparated, +} from ".."; + +describe("audio separation", () => { + test("treats missing source audio state as enabled", () => { + const element = buildVideoElement({}); + + expect(isSourceAudioEnabled({ element })).toBe(true); + expect(isSourceAudioSeparated({ element })).toBe(false); + }); + + test("builds a detached audio element from the source clip audio state", () => { + const element = buildVideoElement({ + duration: 8, + startTime: 3, + trimStart: 1.5, + trimEnd: 0.5, + sourceDuration: 12, + volume: -6, + muted: true, + retime: { rate: 1.25, maintainPitch: true }, + animations: { + channels: { + volume: { + valueKind: "number", + keyframes: [ + { + id: "volume-keyframe", + time: 2, + value: -12, + interpolation: "linear", + }, + ], + }, + opacity: { + valueKind: "number", + keyframes: [ + { + id: "opacity-keyframe", + time: 1, + value: 0.5, + interpolation: "linear", + }, + ], + }, + }, + }, + }); + + const separatedAudioElement = buildSeparatedAudioElement({ + sourceElement: element, + }); + + expect(separatedAudioElement).toMatchObject({ + type: "audio", + sourceType: "upload", + mediaId: element.mediaId, + name: element.name, + duration: 8, + startTime: 3, + trimStart: 1.5, + trimEnd: 0.5, + sourceDuration: 12, + volume: -6, + muted: true, + retime: { rate: 1.25, maintainPitch: true }, + }); + expect(Object.keys(separatedAudioElement.animations?.channels ?? {})).toEqual([ + "volume", + ]); + expect( + separatedAudioElement.animations?.channels.volume?.keyframes[0]?.id, + ).not.toBe("volume-keyframe"); + }); + + test("skips source audio collection when the source clip is separated", () => { + const mediaAsset = { + id: "media-1", + type: "video", + name: "Clip", + size: 1, + lastModified: 1, + file: new File(["video"], "clip.mp4", { type: "video/mp4" }), + url: "blob:clip", + hasAudio: true, + }; + const videoElement = buildVideoElement({ + isSourceAudioEnabled: false, + }); + const audioElement = { + id: "audio-1", + type: "audio", + sourceType: "upload", + mediaId: "audio-media-1", + name: "Detached audio", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + } as AudioElement; + + expect( + doesElementHaveEnabledAudio({ + element: videoElement, + mediaAsset, + }), + ).toBe(false); + expect( + doesElementHaveEnabledAudio({ + element: audioElement, + }), + ).toBe(true); + }); +}); + +function buildVideoElement( + overrides: Partial<VideoElement>, +): VideoElement { + return { + id: "video-1", + type: "video", + mediaId: "media-1", + name: "Clip", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + muted: false, + isSourceAudioEnabled: true, + transform: { + scaleX: 1, + scaleY: 1, + position: { x: 0, y: 0 }, + rotate: 0, + }, + opacity: 1, + ...overrides, + }; +} diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts new file mode 100644 index 00000000..b92cb79b --- /dev/null +++ b/apps/web/src/lib/timeline/audio-separation/index.ts @@ -0,0 +1,133 @@ +import { cloneAnimations, getChannel } from "@/lib/animation"; +import type { ElementAnimations } from "@/lib/animation/types"; +import type { MediaAsset } from "@/lib/media/types"; +import { DEFAULTS } from "@/lib/timeline/defaults"; +import type { + CreateUploadAudioElement, + TimelineElement, + AudioElement, + VideoElement, +} from "../types"; + +export function isSourceAudioEnabled({ + element, +}: { + element: VideoElement; +}): boolean { + return element.isSourceAudioEnabled !== false; +} + +export function isSourceAudioSeparated({ + element, +}: { + element: VideoElement; +}): boolean { + return !isSourceAudioEnabled({ element }); +} + +export function canExtractSourceAudio({ + element, + mediaAsset, +}: { + element: TimelineElement; + mediaAsset: MediaAsset | null | undefined; +}): element is VideoElement { + return ( + element.type === "video" && + isSourceAudioEnabled({ element }) && + !!mediaAsset && + mediaAsset.hasAudio !== false + ); +} + +export function canRecoverSourceAudio({ + element, +}: { + element: TimelineElement; +}): element is VideoElement { + return element.type === "video" && isSourceAudioSeparated({ element }); +} + +export function canToggleSourceAudio({ + element, + mediaAsset, +}: { + element: TimelineElement; + mediaAsset: MediaAsset | null | undefined; +}): element is VideoElement { + return ( + canRecoverSourceAudio({ element }) || + canExtractSourceAudio({ element, mediaAsset }) + ); +} + +export function doesElementHaveEnabledAudio({ + element, + mediaAsset, +}: { + element: AudioElement | VideoElement; + mediaAsset?: MediaAsset | null; +}): boolean { + if (element.type === "audio") { + return true; + } + + return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element }); +} + +export function buildSeparatedAudioElement({ + sourceElement, +}: { + sourceElement: VideoElement; +}): CreateUploadAudioElement { + return { + type: "audio", + sourceType: "upload", + mediaId: sourceElement.mediaId, + name: sourceElement.name, + duration: sourceElement.duration, + startTime: sourceElement.startTime, + trimStart: sourceElement.trimStart, + trimEnd: sourceElement.trimEnd, + sourceDuration: sourceElement.sourceDuration, + volume: sourceElement.volume ?? DEFAULTS.element.volume, + muted: sourceElement.muted ?? false, + retime: sourceElement.retime + ? { + rate: sourceElement.retime.rate, + maintainPitch: sourceElement.retime.maintainPitch, + } + : undefined, + animations: cloneVolumeAnimations({ + animations: sourceElement.animations, + }), + }; +} + +export function getSourceAudioActionLabel({ + element, +}: { + element: VideoElement; +}): "Extract audio" | "Recover audio" { + return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio"; +} + +function cloneVolumeAnimations({ + animations, +}: { + animations: ElementAnimations | undefined; +}): ElementAnimations | undefined { + const volumeChannel = getChannel({ animations, propertyPath: "volume" }); + if (!volumeChannel) { + return undefined; + } + + return cloneAnimations({ + animations: { + channels: { + volume: volumeChannel, + }, + }, + shouldRegenerateKeyframeIds: true, + }); +} diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.ts index ed7bb174..5820b96d 100644 --- a/apps/web/src/lib/timeline/element-utils.ts +++ b/apps/web/src/lib/timeline/element-utils.ts @@ -242,6 +242,7 @@ function buildVideoElement({ trimEnd: 0,
sourceDuration: duration,
muted: false,
+ isSourceAudioEnabled: true,
hidden: false,
transform: {
...DEFAULTS.element.transform,
diff --git a/apps/web/src/lib/timeline/index.ts b/apps/web/src/lib/timeline/index.ts index d4337833..83a6a0f0 100644 --- a/apps/web/src/lib/timeline/index.ts +++ b/apps/web/src/lib/timeline/index.ts @@ -5,6 +5,7 @@ export * from "./drag"; export * from "./track-utils";
export * from "./track-element-update";
export * from "./element-utils";
+export * from "./audio-separation";
export * from "./zoom-utils";
export * from "./ruler-utils";
export * from "./ripple-utils";
diff --git a/apps/web/src/lib/timeline/types.ts b/apps/web/src/lib/timeline/types.ts index 69d363c7..ebdc4c23 100644 --- a/apps/web/src/lib/timeline/types.ts +++ b/apps/web/src/lib/timeline/types.ts @@ -113,6 +113,7 @@ export interface VideoElement extends BaseTimelineElement { mediaId: string;
volume?: number;
muted?: boolean;
+ isSourceAudioEnabled?: boolean;
hidden?: boolean;
retime?: RetimeConfig;
transform: Transform;
diff --git a/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts b/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts new file mode 100644 index 00000000..0e0cfbbe --- /dev/null +++ b/apps/web/src/services/storage/migrations/__tests__/v19-to-v20.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, test } from "bun:test"; +import { transformProjectV19ToV20 } from "../transformers/v19-to-v20"; + +describe("V19 to V20 Migration", () => { + test("backfills source audio enabled state on video elements", () => { + const result = transformProjectV19ToV20({ + project: { + id: "project-v19-source-audio", + version: 19, + metadata: { + id: "project-v19-source-audio", + name: "Project", + createdAt: "2024-01-01T00:00:00.000Z", + updatedAt: "2024-01-01T00:00:00.000Z", + }, + settings: { + fps: 30, + canvasSize: { width: 1920, height: 1080 }, + background: { type: "color", color: "#000000" }, + }, + currentSceneId: "scene-main", + scenes: [ + { + id: "scene-main", + name: "Main", + isMain: true, + bookmarks: [], + createdAt: "2024-01-01T00:00:00.000Z", + updatedAt: "2024-01-01T00:00:00.000Z", + tracks: [ + { + id: "track-video", + type: "video", + name: "Video", + isMain: true, + muted: false, + hidden: false, + elements: [ + { + id: "video-1", + type: "video", + name: "Clip", + mediaId: "media-1", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + transform: { + position: { x: 0, y: 0 }, + scale: { x: 1, y: 1 }, + rotation: 0, + }, + opacity: 1, + }, + ], + }, + { + id: "track-audio", + type: "audio", + name: "Audio", + muted: false, + elements: [ + { + id: "audio-1", + type: "audio", + sourceType: "upload", + mediaId: "media-audio-1", + name: "Audio", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + }, + ], + }, + ], + }, + ], + }, + }); + + expect(result.skipped).toBe(false); + expect(result.project.version).toBe(20); + expect( + ((result.project.scenes as Array<Record<string, unknown>>)[0] + .tracks as Array<Record<string, unknown>>)[0].elements, + ).toEqual([ + expect.objectContaining({ + id: "video-1", + isSourceAudioEnabled: true, + }), + ]); + expect( + (((result.project.scenes as Array<Record<string, unknown>>)[0] + .tracks as Array<Record<string, unknown>>)[1] + .elements as Array<Record<string, unknown>>)[0].isSourceAudioEnabled, + ).toBeUndefined(); + }); + + test("preserves existing explicit source audio state", () => { + const result = transformProjectV19ToV20({ + project: { + id: "project-v19-existing-state", + version: 19, + scenes: [ + { + tracks: [ + { + elements: [ + { + type: "video", + isSourceAudioEnabled: false, + }, + ], + }, + ], + }, + ], + }, + }); + + expect( + (((result.project.scenes as Array<Record<string, unknown>>)[0] + .tracks as Array<Record<string, unknown>>)[0] + .elements as Array<Record<string, unknown>>)[0].isSourceAudioEnabled, + ).toBe(false); + }); + + test("skips projects already on v20", () => { + const result = transformProjectV19ToV20({ + project: { + id: "project-v20", + version: 20, + }, + }); + + expect(result.skipped).toBe(true); + expect(result.reason).toBe("already v20"); + }); +}); diff --git a/apps/web/src/services/storage/migrations/index.ts b/apps/web/src/services/storage/migrations/index.ts index e87d1cb9..d278263b 100644 --- a/apps/web/src/services/storage/migrations/index.ts +++ b/apps/web/src/services/storage/migrations/index.ts @@ -18,10 +18,11 @@ import { V15toV16Migration } from "./v15-to-v16"; import { V16toV17Migration } from "./v16-to-v17";
import { V17toV18Migration } from "./v17-to-v18";
import { V18toV19Migration } from "./v18-to-v19";
+import { V19toV20Migration } from "./v19-to-v20";
export { runStorageMigrations } from "./runner";
export type { MigrationProgress } from "./runner";
-export const CURRENT_PROJECT_VERSION = 19;
+export const CURRENT_PROJECT_VERSION = 20;
export const migrations = [
new V0toV1Migration(),
@@ -43,4 +44,5 @@ export const migrations = [ new V16toV17Migration(),
new V17toV18Migration(),
new V18toV19Migration(),
+ new V19toV20Migration(),
];
diff --git a/apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts b/apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts new file mode 100644 index 00000000..f72812a4 --- /dev/null +++ b/apps/web/src/services/storage/migrations/transformers/v19-to-v20.ts @@ -0,0 +1,87 @@ +import type { MigrationResult, ProjectRecord } from "./types"; +import { getProjectId, isRecord } from "./utils"; + +export function transformProjectV19ToV20({ + project, +}: { + project: ProjectRecord; +}): MigrationResult<ProjectRecord> { + if (!getProjectId({ project })) { + return { project, skipped: true, reason: "no project id" }; + } + + if (typeof project.version === "number" && project.version >= 20) { + return { project, skipped: true, reason: "already v20" }; + } + + return { + project: { + ...migrateVideoSourceAudioState({ project }), + version: 20, + }, + skipped: false, + }; +} + +function migrateVideoSourceAudioState({ + project, +}: { + project: ProjectRecord; +}): ProjectRecord { + if (!Array.isArray(project.scenes)) { + return project; + } + + return { + ...project, + scenes: project.scenes.map((scene) => migrateScene({ scene })), + }; +} + +function migrateScene({ + scene, +}: { + scene: unknown; +}): unknown { + if (!isRecord(scene) || !Array.isArray(scene.tracks)) { + return scene; + } + + return { + ...scene, + tracks: scene.tracks.map((track) => migrateTrack({ track })), + }; +} + +function migrateTrack({ + track, +}: { + track: unknown; +}): unknown { + if (!isRecord(track) || !Array.isArray(track.elements)) { + return track; + } + + return { + ...track, + elements: track.elements.map((element) => migrateElement({ element })), + }; +} + +function migrateElement({ + element, +}: { + element: unknown; +}): unknown { + if (!isRecord(element) || element.type !== "video") { + return element; + } + + return { + ...element, + isSourceAudioEnabled: + typeof element.isSourceAudioEnabled === "boolean" + ? element.isSourceAudioEnabled + : true, + }; +} diff --git a/apps/web/src/services/storage/migrations/v19-to-v20.ts b/apps/web/src/services/storage/migrations/v19-to-v20.ts new file mode 100644 index 00000000..c6809067 --- /dev/null +++ b/apps/web/src/services/storage/migrations/v19-to-v20.ts @@ -0,0 +1,16 @@ +import { StorageMigration } from "./base"; +import type { ProjectRecord } from "./transformers/types"; +import { transformProjectV19ToV20 } from "./transformers/v19-to-v20"; + +export class V19toV20Migration extends StorageMigration { + from = 19; + to = 20; + + async transform(project: ProjectRecord): Promise<{ + project: ProjectRecord; + skipped: boolean; + reason?: string; + }> { + return transformProjectV19ToV20({ project }); + } +} |