summaryrefslogtreecommitdiff
path: root/apps/web/src/lib/media/audio.ts
diff options
context:
space:
mode:
Diffstat (limited to 'apps/web/src/lib/media/audio.ts')
-rw-r--r--apps/web/src/lib/media/audio.ts459
1 files changed, 459 insertions, 0 deletions
diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts
new file mode 100644
index 00000000..b45e348e
--- /dev/null
+++ b/apps/web/src/lib/media/audio.ts
@@ -0,0 +1,459 @@
+import type {
+ AudioElement,
+ LibraryAudioElement,
+ TimelineElement,
+ TimelineTrack,
+} from "@/types/timeline";
+import type { MediaAsset } from "@/types/assets";
+import { canElementHaveAudio } from "@/lib/timeline/element-utils";
+import { canTracktHaveAudio } from "@/lib/timeline";
+import { mediaSupportsAudio } from "@/lib/media/media-utils";
+
+export type CollectedAudioElement = Omit<
+ AudioElement,
+ "type" | "mediaId" | "volume" | "id" | "name" | "sourceType" | "sourceUrl"
+> & { buffer: AudioBuffer };
+
+export function createAudioContext(): AudioContext {
+ const AudioContextConstructor =
+ window.AudioContext ||
+ (window as typeof window & { webkitAudioContext?: typeof AudioContext })
+ .webkitAudioContext;
+
+ return new AudioContextConstructor();
+}
+
+export interface DecodedAudio {
+ samples: Float32Array;
+ sampleRate: number;
+}
+
+export async function decodeAudioToFloat32({
+ audioBlob,
+}: {
+ audioBlob: Blob;
+}): Promise<DecodedAudio> {
+ const audioContext = createAudioContext();
+ const arrayBuffer = await audioBlob.arrayBuffer();
+ const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+
+ // mix down to mono
+ const numChannels = audioBuffer.numberOfChannels;
+ const length = audioBuffer.length;
+ const samples = new Float32Array(length);
+
+ for (let i = 0; i < length; i++) {
+ let sum = 0;
+ for (let channel = 0; channel < numChannels; channel++) {
+ sum += audioBuffer.getChannelData(channel)[i];
+ }
+ samples[i] = sum / numChannels;
+ }
+
+ return { samples, sampleRate: audioBuffer.sampleRate };
+}
+
+export async function collectAudioElements({
+ tracks,
+ mediaAssets,
+ audioContext,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+ audioContext: AudioContext;
+}): Promise<CollectedAudioElement[]> {
+ const mediaMap = new Map<string, MediaAsset>(
+ mediaAssets.map((media) => [media.id, media]),
+ );
+ const pendingElements: Array<Promise<CollectedAudioElement | null>> = [];
+
+ for (const track of tracks) {
+ if (canTracktHaveAudio(track) && track.muted) continue;
+
+ for (const element of track.elements) {
+ if (element.type !== "audio") continue;
+ if (element.duration <= 0) continue;
+
+ const isTrackMuted = canTracktHaveAudio(track) && track.muted;
+ pendingElements.push(
+ resolveAudioBufferForElement({
+ element,
+ mediaMap,
+ audioContext,
+ }).then((audioBuffer) => {
+ if (!audioBuffer) return null;
+ return {
+ buffer: audioBuffer,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ muted: element.muted || isTrackMuted,
+ };
+ }),
+ );
+ }
+ }
+
+ const resolvedElements = await Promise.all(pendingElements);
+ const audioElements: CollectedAudioElement[] = [];
+ for (const element of resolvedElements) {
+ if (element) audioElements.push(element);
+ }
+ return audioElements;
+}
+
+async function resolveAudioBufferForElement({
+ element,
+ mediaMap,
+ audioContext,
+}: {
+ element: AudioElement;
+ mediaMap: Map<string, MediaAsset>;
+ audioContext: AudioContext;
+}): Promise<AudioBuffer | null> {
+ try {
+ if (element.sourceType === "upload") {
+ const asset = mediaMap.get(element.mediaId);
+ if (!asset || asset.type !== "audio") return null;
+
+ const arrayBuffer = await asset.file.arrayBuffer();
+ return await audioContext.decodeAudioData(arrayBuffer.slice(0));
+ }
+
+ if (element.buffer) return element.buffer;
+
+ const response = await fetch(element.sourceUrl);
+ if (!response.ok) {
+ throw new Error(`Library audio fetch failed: ${response.status}`);
+ }
+
+ const arrayBuffer = await response.arrayBuffer();
+ return await audioContext.decodeAudioData(arrayBuffer.slice(0));
+ } catch (error) {
+ console.warn("Failed to decode audio:", error);
+ return null;
+ }
+}
+
+interface AudioMixSource {
+ file: File;
+ startTime: number;
+ duration: number;
+ trimStart: number;
+ trimEnd: number;
+}
+
+export interface AudioClipSource {
+ id: string;
+ sourceKey: string;
+ file: File;
+ startTime: number;
+ duration: number;
+ trimStart: number;
+ trimEnd: number;
+ muted: boolean;
+}
+
+async function fetchLibraryAudioSource({
+ element,
+}: {
+ element: LibraryAudioElement;
+}): Promise<AudioMixSource | null> {
+ try {
+ const response = await fetch(element.sourceUrl);
+ if (!response.ok) {
+ throw new Error(`Library audio fetch failed: ${response.status}`);
+ }
+
+ const blob = await response.blob();
+ const file = new File([blob], `${element.name}.mp3`, {
+ type: "audio/mpeg",
+ });
+
+ return {
+ file,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ };
+ } catch (error) {
+ console.warn("Failed to fetch library audio:", error);
+ return null;
+ }
+}
+
+async function fetchLibraryAudioClip({
+ element,
+ muted,
+}: {
+ element: LibraryAudioElement;
+ muted: boolean;
+}): Promise<AudioClipSource | null> {
+ try {
+ const response = await fetch(element.sourceUrl);
+ if (!response.ok) {
+ throw new Error(`Library audio fetch failed: ${response.status}`);
+ }
+
+ const blob = await response.blob();
+ const file = new File([blob], `${element.name}.mp3`, {
+ type: "audio/mpeg",
+ });
+
+ return {
+ id: element.id,
+ sourceKey: element.id,
+ file,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ muted,
+ };
+ } catch (error) {
+ console.warn("Failed to fetch library audio:", error);
+ return null;
+ }
+}
+
+function collectMediaAudioSource({
+ element,
+ mediaAsset,
+}: {
+ element: TimelineElement;
+ mediaAsset: MediaAsset;
+}): AudioMixSource {
+ return {
+ file: mediaAsset.file,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ };
+}
+
+function collectMediaAudioClip({
+ element,
+ mediaAsset,
+ muted,
+}: {
+ element: TimelineElement;
+ mediaAsset: MediaAsset;
+ muted: boolean;
+}): AudioClipSource {
+ return {
+ id: element.id,
+ sourceKey: mediaAsset.id,
+ file: mediaAsset.file,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ muted,
+ };
+}
+
+export async function collectAudioMixSources({
+ tracks,
+ mediaAssets,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+}): Promise<AudioMixSource[]> {
+ const audioMixSources: AudioMixSource[] = [];
+ const mediaMap = new Map<string, MediaAsset>(
+ mediaAssets.map((asset) => [asset.id, asset]),
+ );
+ const pendingLibrarySources: Array<Promise<AudioMixSource | null>> = [];
+
+ for (const track of tracks) {
+ if (canTracktHaveAudio(track) && track.muted) continue;
+
+ for (const element of track.elements) {
+ if (!canElementHaveAudio(element)) continue;
+
+ if (element.type === "audio") {
+ if (element.sourceType === "upload") {
+ const mediaAsset = mediaMap.get(element.mediaId);
+ if (!mediaAsset) continue;
+
+ audioMixSources.push(
+ collectMediaAudioSource({ element, mediaAsset }),
+ );
+ } else {
+ pendingLibrarySources.push(fetchLibraryAudioSource({ element }));
+ }
+ continue;
+ }
+
+ if (element.type === "video") {
+ const mediaAsset = mediaMap.get(element.mediaId);
+ if (!mediaAsset) continue;
+
+ if (mediaSupportsAudio({ media: mediaAsset })) {
+ audioMixSources.push(
+ collectMediaAudioSource({ element, mediaAsset }),
+ );
+ }
+ }
+ }
+ }
+
+ const resolvedLibrarySources = await Promise.all(pendingLibrarySources);
+ for (const source of resolvedLibrarySources) {
+ if (source) audioMixSources.push(source);
+ }
+
+ return audioMixSources;
+}
+
+export async function collectAudioClips({
+ tracks,
+ mediaAssets,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+}): Promise<AudioClipSource[]> {
+ const clips: AudioClipSource[] = [];
+ const mediaMap = new Map<string, MediaAsset>(
+ mediaAssets.map((asset) => [asset.id, asset]),
+ );
+ const pendingLibraryClips: Array<Promise<AudioClipSource | null>> = [];
+
+ for (const track of tracks) {
+ const isTrackMuted = canTracktHaveAudio(track) && track.muted;
+
+ for (const element of track.elements) {
+ if (!canElementHaveAudio(element)) continue;
+
+ const isElementMuted =
+ "muted" in element ? (element.muted ?? false) : false;
+ const muted = isTrackMuted || isElementMuted;
+
+ if (element.type === "audio") {
+ if (element.sourceType === "upload") {
+ const mediaAsset = mediaMap.get(element.mediaId);
+ if (!mediaAsset) continue;
+
+ clips.push(
+ collectMediaAudioClip({
+ element,
+ mediaAsset,
+ muted,
+ }),
+ );
+ } else {
+ pendingLibraryClips.push(fetchLibraryAudioClip({ element, muted }));
+ }
+ continue;
+ }
+
+ if (element.type === "video") {
+ const mediaAsset = mediaMap.get(element.mediaId);
+ if (!mediaAsset) continue;
+
+ if (mediaSupportsAudio({ media: mediaAsset })) {
+ clips.push(
+ collectMediaAudioClip({
+ element,
+ mediaAsset,
+ muted,
+ }),
+ );
+ }
+ }
+ }
+ }
+
+ const resolvedLibraryClips = await Promise.all(pendingLibraryClips);
+ for (const clip of resolvedLibraryClips) {
+ if (clip) clips.push(clip);
+ }
+
+ return clips;
+}
+
+export async function createTimelineAudioBuffer({
+ tracks,
+ mediaAssets,
+ duration,
+ sampleRate = 44100,
+ audioContext,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+ duration: number;
+ sampleRate?: number;
+ audioContext?: AudioContext;
+}): Promise<AudioBuffer | null> {
+ const context = audioContext ?? createAudioContext();
+
+ const audioElements = await collectAudioElements({
+ tracks,
+ mediaAssets,
+ audioContext: context,
+ });
+
+ if (audioElements.length === 0) return null;
+
+ const outputChannels = 2;
+ const outputLength = Math.ceil(duration * sampleRate);
+ const outputBuffer = context.createBuffer(
+ outputChannels,
+ outputLength,
+ sampleRate,
+ );
+
+ for (const element of audioElements) {
+ if (element.muted) continue;
+
+ mixAudioChannels({
+ element,
+ outputBuffer,
+ outputLength,
+ sampleRate,
+ });
+ }
+
+ return outputBuffer;
+}
+
+function mixAudioChannels({
+ element,
+ outputBuffer,
+ outputLength,
+ sampleRate,
+}: {
+ element: CollectedAudioElement;
+ outputBuffer: AudioBuffer;
+ outputLength: number;
+ sampleRate: number;
+}): void {
+ const { buffer, startTime, trimStart, duration: elementDuration } = element;
+
+ const sourceStartSample = Math.floor(trimStart * buffer.sampleRate);
+ const sourceLengthSamples = Math.floor(elementDuration * buffer.sampleRate);
+ const outputStartSample = Math.floor(startTime * sampleRate);
+
+ const resampleRatio = sampleRate / buffer.sampleRate;
+ const resampledLength = Math.floor(sourceLengthSamples * resampleRatio);
+
+ const outputChannels = 2;
+ for (let channel = 0; channel < outputChannels; channel++) {
+ const outputData = outputBuffer.getChannelData(channel);
+ const sourceChannel = Math.min(channel, buffer.numberOfChannels - 1);
+ const sourceData = buffer.getChannelData(sourceChannel);
+
+ for (let i = 0; i < resampledLength; i++) {
+ const outputIndex = outputStartSample + i;
+ if (outputIndex >= outputLength) break;
+
+ const sourceIndex = sourceStartSample + Math.floor(i / resampleRatio);
+ if (sourceIndex >= sourceData.length) break;
+
+ outputData[outputIndex] += sourceData[sourceIndex];
+ }
+ }
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-17 08:28:59 +0000