From f1d5932c54178c4f8eac61d3d3e0e4a30101780f Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Tue, 31 Mar 2026 23:23:36 +0200 Subject: feat: separate audio from elements --- apps/web/src/lib/actions/definitions.ts | 4 + .../web/src/lib/commands/timeline/element/index.ts | 1 + .../element/toggle-source-audio-separation.ts | 143 ++++++++++++++++++++ apps/web/src/lib/media/audio.ts | 30 +++-- .../audio-separation/__tests__/index.test.ts | 147 +++++++++++++++++++++ .../web/src/lib/timeline/audio-separation/index.ts | 133 +++++++++++++++++++ apps/web/src/lib/timeline/element-utils.ts | 1 + apps/web/src/lib/timeline/index.ts | 1 + apps/web/src/lib/timeline/types.ts | 1 + 9 files changed, 451 insertions(+), 10 deletions(-) create mode 100644 apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts create mode 100644 apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts create mode 100644 apps/web/src/lib/timeline/audio-separation/index.ts (limited to 'apps/web/src/lib') diff --git a/apps/web/src/lib/actions/definitions.ts b/apps/web/src/lib/actions/definitions.ts index e48126d0..3059a2e3 100644 --- a/apps/web/src/lib/actions/definitions.ts +++ b/apps/web/src/lib/actions/definitions.ts @@ -101,6 +101,10 @@ export const ACTIONS = { description: "Toggle ripple editing", category: "editing", }, + "toggle-source-audio": { + description: "Extract or recover source audio", + category: "editing", + }, "select-all": { description: "Select all elements", category: "selection", diff --git a/apps/web/src/lib/commands/timeline/element/index.ts b/apps/web/src/lib/commands/timeline/element/index.ts index a7e513fe..5ce48d55 100644 --- a/apps/web/src/lib/commands/timeline/element/index.ts +++ b/apps/web/src/lib/commands/timeline/element/index.ts @@ -8,6 +8,7 @@ export { SplitElementsCommand } from "./split-elements"; export { UpdateElementCommand } from "./update-element"; export { ToggleElementsVisibilityCommand } from "./toggle-elements-visibility"; export { ToggleElementsMutedCommand } from "./toggle-elements-muted"; +export { ToggleSourceAudioSeparationCommand } from "./toggle-source-audio-separation"; export { MoveElementCommand } from "./move-elements"; export * from "./keyframes"; export * from "./effects"; diff --git a/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts new file mode 100644 index 00000000..d0907c5b --- /dev/null +++ b/apps/web/src/lib/commands/timeline/element/toggle-source-audio-separation.ts @@ -0,0 +1,143 @@ +import { EditorCore } from "@/core"; +import { Command } from "@/lib/commands/base-command"; +import { + buildSeparatedAudioElement, + canExtractSourceAudio, + canRecoverSourceAudio, +} from "@/lib/timeline/audio-separation"; +import { + applyPlacement, + resolveTrackPlacement, +} from "@/lib/timeline/placement"; +import { updateElementInTracks } from "@/lib/timeline/track-element-update"; +import type { TimelineTrack, VideoElement } from "@/lib/timeline/types"; +import { generateUUID } from "@/utils/id"; + +export class ToggleSourceAudioSeparationCommand extends Command { + private savedState: TimelineTrack[] | null = null; + + constructor( + private readonly params: { + trackId: string; + elementId: string; + }, + ) { + super(); + } + + execute(): void { + const editor = EditorCore.getInstance(); + this.savedState = editor.timeline.getTracks(); + + const trackIndex = this.savedState.findIndex( + (track) => track.id === this.params.trackId, + ); + if (trackIndex < 0) { + return; + } + + const sourceTrack = this.savedState[trackIndex]; + const sourceElement = sourceTrack.elements.find( + (element) => element.id === this.params.elementId, + ); + if (!sourceElement) { + return; + } + + if (canRecoverSourceAudio({ element: sourceElement })) { + editor.timeline.updateTracks( + updateSourceAudioEnabled({ + tracks: this.savedState, + trackId: this.params.trackId, + elementId: this.params.elementId, + isSourceAudioEnabled: true, + }), + ); + return; + } + + const mediaAsset = editor + .media + .getAssets() + .find((asset) => + sourceElement.type === "video" ? asset.id === sourceElement.mediaId : false, + ); + if (!canExtractSourceAudio({ element: sourceElement, mediaAsset })) { + return; + } + if (sourceElement.duration <= 0) { + return; + } + + const separatedAudioElement = { + ...buildSeparatedAudioElement({ + sourceElement, + }), + id: generateUUID(), + }; + const placementResult = resolveTrackPlacement({ + tracks: this.savedState, + trackType: "audio", + timeSpans: [ + { + startTime: separatedAudioElement.startTime, + duration: separatedAudioElement.duration, + }, + ], + strategy: { type: "aboveSource", sourceTrackIndex: trackIndex }, + }); + if (!placementResult) { + return; + } + + const appliedPlacement = applyPlacement({ + tracks: this.savedState, + placementResult, + elements: [separatedAudioElement], + }); + if (!appliedPlacement) { + return; + } + + editor.timeline.updateTracks( + updateSourceAudioEnabled({ + tracks: appliedPlacement.updatedTracks, + trackId: this.params.trackId, + elementId: this.params.elementId, + isSourceAudioEnabled: false, + }), + ); + } + + undo(): void { + if (!this.savedState) { + return; + } + + const editor = EditorCore.getInstance(); + editor.timeline.updateTracks(this.savedState); + } +} + +function updateSourceAudioEnabled({ + tracks, + trackId, + elementId, + isSourceAudioEnabled, +}: { + tracks: TimelineTrack[]; + trackId: string; + elementId: string; + isSourceAudioEnabled: boolean; +}): TimelineTrack[] { + return updateElementInTracks({ + tracks, + trackId, + elementId, + elementPredicate: (element): element is VideoElement => element.type === "video", + update: (element) => ({ + ...element, + isSourceAudioEnabled, + }), + }); +} diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts index 41d3b38e..0afc42fb 100644 --- a/apps/web/src/lib/media/audio.ts +++ b/apps/web/src/lib/media/audio.ts @@ -12,7 +12,10 @@ import { hasAnimatedVolume, resolveEffectiveAudioGain, } from "@/lib/timeline/audio-state"; -import { canElementHaveAudio } from "@/lib/timeline/element-utils"; +import { + doesElementHaveEnabledAudio, +} from "@/lib/timeline/audio-separation"; +import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils"; import { canTracktHaveAudio } from "@/lib/timeline"; import { mediaSupportsAudio } from "@/lib/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime"; @@ -100,6 +103,11 @@ export async function collectAudioElements({ if (!canElementHaveAudio(element)) continue; if (element.duration <= 0) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; + const isTrackMuted = canTracktHaveAudio(track) && track.muted; if (element.type === "audio") { @@ -132,7 +140,6 @@ export async function collectAudioElements({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; pendingElements.push( @@ -451,6 +458,10 @@ export async function collectAudioMixSources({ for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; if (element.muted === true) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; const volume = resolveEffectiveAudioGain({ element, localTime: 0, @@ -473,10 +484,7 @@ export async function collectAudioMixSources({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); - if (!mediaAsset) continue; - - if (mediaSupportsAudio({ media: mediaAsset })) { + if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) { audioMixSources.push( collectMediaAudioSource({ element, mediaAsset, volume }), ); @@ -512,6 +520,11 @@ export async function collectAudioClips({ for (const element of track.elements) { if (!canElementHaveAudio(element)) continue; + const mediaAsset = hasMediaId(element) + ? (mediaMap.get(element.mediaId) ?? null) + : null; + if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; + const isElementMuted = "muted" in element ? (element.muted ?? false) : false; const muted = isTrackMuted || isElementMuted; @@ -543,10 +556,7 @@ export async function collectAudioClips({ } if (element.type === "video") { - const mediaAsset = mediaMap.get(element.mediaId); - if (!mediaAsset) continue; - - if (mediaSupportsAudio({ media: mediaAsset })) { + if (mediaAsset && mediaSupportsAudio({ media: mediaAsset })) { clips.push( collectMediaAudioClip({ element, diff --git a/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts new file mode 100644 index 00000000..173b6402 --- /dev/null +++ b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, test } from "bun:test"; +import type { AudioElement, VideoElement } from "@/lib/timeline"; +import { + buildSeparatedAudioElement, + doesElementHaveEnabledAudio, + isSourceAudioEnabled, + isSourceAudioSeparated, +} from ".."; + +describe("audio separation", () => { + test("treats missing source audio state as enabled", () => { + const element = buildVideoElement({}); + + expect(isSourceAudioEnabled({ element })).toBe(true); + expect(isSourceAudioSeparated({ element })).toBe(false); + }); + + test("builds a detached audio element from the source clip audio state", () => { + const element = buildVideoElement({ + duration: 8, + startTime: 3, + trimStart: 1.5, + trimEnd: 0.5, + sourceDuration: 12, + volume: -6, + muted: true, + retime: { rate: 1.25, maintainPitch: true }, + animations: { + channels: { + volume: { + valueKind: "number", + keyframes: [ + { + id: "volume-keyframe", + time: 2, + value: -12, + interpolation: "linear", + }, + ], + }, + opacity: { + valueKind: "number", + keyframes: [ + { + id: "opacity-keyframe", + time: 1, + value: 0.5, + interpolation: "linear", + }, + ], + }, + }, + }, + }); + + const separatedAudioElement = buildSeparatedAudioElement({ + sourceElement: element, + }); + + expect(separatedAudioElement).toMatchObject({ + type: "audio", + sourceType: "upload", + mediaId: element.mediaId, + name: element.name, + duration: 8, + startTime: 3, + trimStart: 1.5, + trimEnd: 0.5, + sourceDuration: 12, + volume: -6, + muted: true, + retime: { rate: 1.25, maintainPitch: true }, + }); + expect(Object.keys(separatedAudioElement.animations?.channels ?? {})).toEqual([ + "volume", + ]); + expect( + separatedAudioElement.animations?.channels.volume?.keyframes[0]?.id, + ).not.toBe("volume-keyframe"); + }); + + test("skips source audio collection when the source clip is separated", () => { + const mediaAsset = { + id: "media-1", + type: "video", + name: "Clip", + size: 1, + lastModified: 1, + file: new File(["video"], "clip.mp4", { type: "video/mp4" }), + url: "blob:clip", + hasAudio: true, + }; + const videoElement = buildVideoElement({ + isSourceAudioEnabled: false, + }); + const audioElement = { + id: "audio-1", + type: "audio", + sourceType: "upload", + mediaId: "audio-media-1", + name: "Detached audio", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + } as AudioElement; + + expect( + doesElementHaveEnabledAudio({ + element: videoElement, + mediaAsset, + }), + ).toBe(false); + expect( + doesElementHaveEnabledAudio({ + element: audioElement, + }), + ).toBe(true); + }); +}); + +function buildVideoElement( + overrides: Partial, +): VideoElement { + return { + id: "video-1", + type: "video", + mediaId: "media-1", + name: "Clip", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + muted: false, + isSourceAudioEnabled: true, + transform: { + scaleX: 1, + scaleY: 1, + position: { x: 0, y: 0 }, + rotate: 0, + }, + opacity: 1, + ...overrides, + }; +} diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts new file mode 100644 index 00000000..b92cb79b --- /dev/null +++ b/apps/web/src/lib/timeline/audio-separation/index.ts @@ -0,0 +1,133 @@ +import { cloneAnimations, getChannel } from "@/lib/animation"; +import type { ElementAnimations } from "@/lib/animation/types"; +import type { MediaAsset } from "@/lib/media/types"; +import { DEFAULTS } from "@/lib/timeline/defaults"; +import type { + CreateUploadAudioElement, + TimelineElement, + AudioElement, + VideoElement, +} from "../types"; + +export function isSourceAudioEnabled({ + element, +}: { + element: VideoElement; +}): boolean { + return element.isSourceAudioEnabled !== false; +} + +export function isSourceAudioSeparated({ + element, +}: { + element: VideoElement; +}): boolean { + return !isSourceAudioEnabled({ element }); +} + +export function canExtractSourceAudio({ + element, + mediaAsset, +}: { + element: TimelineElement; + mediaAsset: MediaAsset | null | undefined; +}): element is VideoElement { + return ( + element.type === "video" && + isSourceAudioEnabled({ element }) && + !!mediaAsset && + mediaAsset.hasAudio !== false + ); +} + +export function canRecoverSourceAudio({ + element, +}: { + element: TimelineElement; +}): element is VideoElement { + return element.type === "video" && isSourceAudioSeparated({ element }); +} + +export function canToggleSourceAudio({ + element, + mediaAsset, +}: { + element: TimelineElement; + mediaAsset: MediaAsset | null | undefined; +}): element is VideoElement { + return ( + canRecoverSourceAudio({ element }) || + canExtractSourceAudio({ element, mediaAsset }) + ); +} + +export function doesElementHaveEnabledAudio({ + element, + mediaAsset, +}: { + element: AudioElement | VideoElement; + mediaAsset?: MediaAsset | null; +}): boolean { + if (element.type === "audio") { + return true; + } + + return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element }); +} + +export function buildSeparatedAudioElement({ + sourceElement, +}: { + sourceElement: VideoElement; +}): CreateUploadAudioElement { + return { + type: "audio", + sourceType: "upload", + mediaId: sourceElement.mediaId, + name: sourceElement.name, + duration: sourceElement.duration, + startTime: sourceElement.startTime, + trimStart: sourceElement.trimStart, + trimEnd: sourceElement.trimEnd, + sourceDuration: sourceElement.sourceDuration, + volume: sourceElement.volume ?? DEFAULTS.element.volume, + muted: sourceElement.muted ?? false, + retime: sourceElement.retime + ? { + rate: sourceElement.retime.rate, + maintainPitch: sourceElement.retime.maintainPitch, + } + : undefined, + animations: cloneVolumeAnimations({ + animations: sourceElement.animations, + }), + }; +} + +export function getSourceAudioActionLabel({ + element, +}: { + element: VideoElement; +}): "Extract audio" | "Recover audio" { + return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio"; +} + +function cloneVolumeAnimations({ + animations, +}: { + animations: ElementAnimations | undefined; +}): ElementAnimations | undefined { + const volumeChannel = getChannel({ animations, propertyPath: "volume" }); + if (!volumeChannel) { + return undefined; + } + + return cloneAnimations({ + animations: { + channels: { + volume: volumeChannel, + }, + }, + shouldRegenerateKeyframeIds: true, + }); +} diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.ts index ed7bb174..5820b96d 100644 --- a/apps/web/src/lib/timeline/element-utils.ts +++ b/apps/web/src/lib/timeline/element-utils.ts @@ -242,6 +242,7 @@ function buildVideoElement({ trimEnd: 0, sourceDuration: duration, muted: false, + isSourceAudioEnabled: true, hidden: false, transform: { ...DEFAULTS.element.transform, diff --git a/apps/web/src/lib/timeline/index.ts b/apps/web/src/lib/timeline/index.ts index d4337833..83a6a0f0 100644 --- a/apps/web/src/lib/timeline/index.ts +++ b/apps/web/src/lib/timeline/index.ts @@ -5,6 +5,7 @@ export * from "./drag"; export * from "./track-utils"; export * from "./track-element-update"; export * from "./element-utils"; +export * from "./audio-separation"; export * from "./zoom-utils"; export * from "./ruler-utils"; export * from "./ripple-utils"; diff --git a/apps/web/src/lib/timeline/types.ts b/apps/web/src/lib/timeline/types.ts index 69d363c7..ebdc4c23 100644 --- a/apps/web/src/lib/timeline/types.ts +++ b/apps/web/src/lib/timeline/types.ts @@ -113,6 +113,7 @@ export interface VideoElement extends BaseTimelineElement { mediaId: string; volume?: number; muted?: boolean; + isSourceAudioEnabled?: boolean; hidden?: boolean; retime?: RetimeConfig; transform: Transform; -- cgit v1.3.1