From f1d5932c54178c4f8eac61d3d3e0e4a30101780f Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Tue, 31 Mar 2026 23:23:36 +0200 Subject: feat: separate audio from elements --- .../audio-separation/__tests__/index.test.ts | 147 +++++++++++++++++++++ .../web/src/lib/timeline/audio-separation/index.ts | 133 +++++++++++++++++++ apps/web/src/lib/timeline/element-utils.ts | 1 + apps/web/src/lib/timeline/index.ts | 1 + apps/web/src/lib/timeline/types.ts | 1 + 5 files changed, 283 insertions(+) create mode 100644 apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts create mode 100644 apps/web/src/lib/timeline/audio-separation/index.ts (limited to 'apps/web/src/lib/timeline') diff --git a/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts new file mode 100644 index 00000000..173b6402 --- /dev/null +++ b/apps/web/src/lib/timeline/audio-separation/__tests__/index.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, test } from "bun:test"; +import type { AudioElement, VideoElement } from "@/lib/timeline"; +import { + buildSeparatedAudioElement, + doesElementHaveEnabledAudio, + isSourceAudioEnabled, + isSourceAudioSeparated, +} from ".."; + +describe("audio separation", () => { + test("treats missing source audio state as enabled", () => { + const element = buildVideoElement({}); + + expect(isSourceAudioEnabled({ element })).toBe(true); + expect(isSourceAudioSeparated({ element })).toBe(false); + }); + + test("builds a detached audio element from the source clip audio state", () => { + const element = buildVideoElement({ + duration: 8, + startTime: 3, + trimStart: 1.5, + trimEnd: 0.5, + sourceDuration: 12, + volume: -6, + muted: true, + retime: { rate: 1.25, maintainPitch: true }, + animations: { + channels: { + volume: { + valueKind: "number", + keyframes: [ + { + id: "volume-keyframe", + time: 2, + value: -12, + interpolation: "linear", + }, + ], + }, + opacity: { + valueKind: "number", + keyframes: [ + { + id: "opacity-keyframe", + time: 1, + value: 0.5, + interpolation: "linear", + }, + ], + }, + }, + }, + }); + + const separatedAudioElement = buildSeparatedAudioElement({ + sourceElement: element, + }); + + expect(separatedAudioElement).toMatchObject({ + type: "audio", + sourceType: "upload", + mediaId: element.mediaId, + name: element.name, + duration: 8, + startTime: 3, + trimStart: 1.5, + trimEnd: 0.5, + sourceDuration: 12, + volume: -6, + muted: true, + retime: { rate: 1.25, maintainPitch: true }, + }); + expect(Object.keys(separatedAudioElement.animations?.channels ?? {})).toEqual([ + "volume", + ]); + expect( + separatedAudioElement.animations?.channels.volume?.keyframes[0]?.id, + ).not.toBe("volume-keyframe"); + }); + + test("skips source audio collection when the source clip is separated", () => { + const mediaAsset = { + id: "media-1", + type: "video", + name: "Clip", + size: 1, + lastModified: 1, + file: new File(["video"], "clip.mp4", { type: "video/mp4" }), + url: "blob:clip", + hasAudio: true, + }; + const videoElement = buildVideoElement({ + isSourceAudioEnabled: false, + }); + const audioElement = { + id: "audio-1", + type: "audio", + sourceType: "upload", + mediaId: "audio-media-1", + name: "Detached audio", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + } as AudioElement; + + expect( + doesElementHaveEnabledAudio({ + element: videoElement, + mediaAsset, + }), + ).toBe(false); + expect( + doesElementHaveEnabledAudio({ + element: audioElement, + }), + ).toBe(true); + }); +}); + +function buildVideoElement( + overrides: Partial, +): VideoElement { + return { + id: "video-1", + type: "video", + mediaId: "media-1", + name: "Clip", + duration: 5, + startTime: 0, + trimStart: 0, + trimEnd: 0, + volume: 0, + muted: false, + isSourceAudioEnabled: true, + transform: { + scaleX: 1, + scaleY: 1, + position: { x: 0, y: 0 }, + rotate: 0, + }, + opacity: 1, + ...overrides, + }; +} diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts new file mode 100644 index 00000000..b92cb79b --- /dev/null +++ b/apps/web/src/lib/timeline/audio-separation/index.ts @@ -0,0 +1,133 @@ +import { cloneAnimations, getChannel } from "@/lib/animation"; +import type { ElementAnimations } from "@/lib/animation/types"; +import type { MediaAsset } from "@/lib/media/types"; +import { DEFAULTS } from "@/lib/timeline/defaults"; +import type { + CreateUploadAudioElement, + TimelineElement, + AudioElement, + VideoElement, +} from "../types"; + +export function isSourceAudioEnabled({ + element, +}: { + element: VideoElement; +}): boolean { + return element.isSourceAudioEnabled !== false; +} + +export function isSourceAudioSeparated({ + element, +}: { + element: VideoElement; +}): boolean { + return !isSourceAudioEnabled({ element }); +} + +export function canExtractSourceAudio({ + element, + mediaAsset, +}: { + element: TimelineElement; + mediaAsset: MediaAsset | null | undefined; +}): element is VideoElement { + return ( + element.type === "video" && + isSourceAudioEnabled({ element }) && + !!mediaAsset && + mediaAsset.hasAudio !== false + ); +} + +export function canRecoverSourceAudio({ + element, +}: { + element: TimelineElement; +}): element is VideoElement { + return element.type === "video" && isSourceAudioSeparated({ element }); +} + +export function canToggleSourceAudio({ + element, + mediaAsset, +}: { + element: TimelineElement; + mediaAsset: MediaAsset | null | undefined; +}): element is VideoElement { + return ( + canRecoverSourceAudio({ element }) || + canExtractSourceAudio({ element, mediaAsset }) + ); +} + +export function doesElementHaveEnabledAudio({ + element, + mediaAsset, +}: { + element: AudioElement | VideoElement; + mediaAsset?: MediaAsset | null; +}): boolean { + if (element.type === "audio") { + return true; + } + + return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element }); +} + +export function buildSeparatedAudioElement({ + sourceElement, +}: { + sourceElement: VideoElement; +}): CreateUploadAudioElement { + return { + type: "audio", + sourceType: "upload", + mediaId: sourceElement.mediaId, + name: sourceElement.name, + duration: sourceElement.duration, + startTime: sourceElement.startTime, + trimStart: sourceElement.trimStart, + trimEnd: sourceElement.trimEnd, + sourceDuration: sourceElement.sourceDuration, + volume: sourceElement.volume ?? DEFAULTS.element.volume, + muted: sourceElement.muted ?? false, + retime: sourceElement.retime + ? { + rate: sourceElement.retime.rate, + maintainPitch: sourceElement.retime.maintainPitch, + } + : undefined, + animations: cloneVolumeAnimations({ + animations: sourceElement.animations, + }), + }; +} + +export function getSourceAudioActionLabel({ + element, +}: { + element: VideoElement; +}): "Extract audio" | "Recover audio" { + return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio"; +} + +function cloneVolumeAnimations({ + animations, +}: { + animations: ElementAnimations | undefined; +}): ElementAnimations | undefined { + const volumeChannel = getChannel({ animations, propertyPath: "volume" }); + if (!volumeChannel) { + return undefined; + } + + return cloneAnimations({ + animations: { + channels: { + volume: volumeChannel, + }, + }, + shouldRegenerateKeyframeIds: true, + }); +} diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.ts index ed7bb174..5820b96d 100644 --- a/apps/web/src/lib/timeline/element-utils.ts +++ b/apps/web/src/lib/timeline/element-utils.ts @@ -242,6 +242,7 @@ function buildVideoElement({ trimEnd: 0, sourceDuration: duration, muted: false, + isSourceAudioEnabled: true, hidden: false, transform: { ...DEFAULTS.element.transform, diff --git a/apps/web/src/lib/timeline/index.ts b/apps/web/src/lib/timeline/index.ts index d4337833..83a6a0f0 100644 --- a/apps/web/src/lib/timeline/index.ts +++ b/apps/web/src/lib/timeline/index.ts @@ -5,6 +5,7 @@ export * from "./drag"; export * from "./track-utils"; export * from "./track-element-update"; export * from "./element-utils"; +export * from "./audio-separation"; export * from "./zoom-utils"; export * from "./ruler-utils"; export * from "./ripple-utils"; diff --git a/apps/web/src/lib/timeline/types.ts b/apps/web/src/lib/timeline/types.ts index 69d363c7..ebdc4c23 100644 --- a/apps/web/src/lib/timeline/types.ts +++ b/apps/web/src/lib/timeline/types.ts @@ -113,6 +113,7 @@ export interface VideoElement extends BaseTimelineElement { mediaId: string; volume?: number; muted?: boolean; + isSourceAudioEnabled?: boolean; hidden?: boolean; retime?: RetimeConfig; transform: Transform; -- cgit v1.3.1