From 70a9bfebb196d5c0bde35ce0a165b696bb9ae1d0 Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Thu, 28 Aug 2025 21:52:03 +0200 Subject: feat: add audio support for export --- apps/web/src/lib/export.ts | 443 ++++++++++++++++++++++++++++++--------------- 1 file changed, 296 insertions(+), 147 deletions(-) (limited to 'apps/web/src/lib/export.ts') diff --git a/apps/web/src/lib/export.ts b/apps/web/src/lib/export.ts index 69bc8d1d..f25f5f6b 100644 --- a/apps/web/src/lib/export.ts +++ b/apps/web/src/lib/export.ts @@ -1,147 +1,296 @@ -import { - Output, - Mp4OutputFormat, - WebMOutputFormat, - BufferTarget, - CanvasSource, - QUALITY_LOW, - QUALITY_MEDIUM, - QUALITY_HIGH, - QUALITY_VERY_HIGH, -} from "mediabunny"; -import { renderTimelineFrame } from "./timeline-renderer"; -import { useTimelineStore } from "@/stores/timeline-store"; -import { useMediaStore } from "@/stores/media-store"; -import { useProjectStore } from "@/stores/project-store"; -import { DEFAULT_FPS } from "@/stores/project-store"; -import { ExportOptions, ExportResult } from "@/types/export"; - -export const DEFAULT_EXPORT_OPTIONS: ExportOptions = { - format: "mp4", - quality: "high", -}; - -const qualityMap = { - low: QUALITY_LOW, - medium: QUALITY_MEDIUM, - high: QUALITY_HIGH, - very_high: QUALITY_VERY_HIGH, -}; - -export async function exportProject( - options: ExportOptions -): Promise { - const { format, quality, fps, onProgress, onCancel } = options; - - try { - const timelineStore = useTimelineStore.getState(); - const mediaStore = useMediaStore.getState(); - const projectStore = useProjectStore.getState(); - - const { tracks, getTotalDuration } = timelineStore; - const { mediaFiles } = mediaStore; - const { activeProject } = projectStore; - - if (!activeProject) { - return { success: false, error: "No active project" }; - } - - const duration = getTotalDuration(); - if (duration === 0) { - return { success: false, error: "Project is empty" }; - } - - const exportFps = fps || activeProject.fps || DEFAULT_FPS; - const canvasSize = activeProject.canvasSize; - - const outputFormat = - format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); - - // BufferTarget for smaller files, StreamTarget for larger ones - // TODO: Implement StreamTarget - const output = new Output({ - format: outputFormat, - target: new BufferTarget(), - }); - - // Canvas for rendering - const canvas = document.createElement("canvas"); - canvas.width = canvasSize.width; - canvas.height = canvasSize.height; - const ctx = canvas.getContext("2d"); - - if (!ctx) { - return { success: false, error: "Failed to create canvas context" }; - } - - const videoSource = new CanvasSource(canvas, { - codec: format === "webm" ? "vp9" : "avc", // VP9 for WebM, H.264 for MP4 - bitrate: qualityMap[quality], - }); - - output.addVideoTrack(videoSource, { frameRate: exportFps }); - - // Start the output - await output.start(); - - const totalFrames = Math.ceil(duration * exportFps); - let cancelled = false; - - // Render each frame - for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) { - // Check for cancellation - if (onCancel?.()) { - cancelled = true; - break; - } - - const time = frameIndex / exportFps; - - await renderTimelineFrame({ - ctx, - time, - canvasWidth: canvas.width, - canvasHeight: canvas.height, - tracks, - mediaFiles, - backgroundColor: - activeProject.backgroundType === "blur" - ? "transparent" - : activeProject.backgroundColor || "#000000", - projectCanvasSize: canvasSize, - }); - - const frameDuration = 1 / exportFps; - await videoSource.add(time, frameDuration); - - onProgress?.(frameIndex / totalFrames); - } - - if (cancelled) { - await output.cancel(); - return { success: false, cancelled: true }; - } - videoSource.close(); - await output.finalize(); - onProgress?.(1); - - return { - success: true, - buffer: output.target.buffer || undefined, - }; - } catch (error) { - console.error("Export failed:", error); - return { - success: false, - error: error instanceof Error ? error.message : "Unknown export error", - }; - } -} - -export function getExportMimeType(format: "mp4" | "webm"): string { - return format === "webm" ? "video/webm" : "video/mp4"; -} - -export function getExportFileExtension(format: "mp4" | "webm"): string { - return `.${format}`; -} +import { + Output, + Mp4OutputFormat, + WebMOutputFormat, + BufferTarget, + CanvasSource, + AudioBufferSource, + QUALITY_LOW, + QUALITY_MEDIUM, + QUALITY_HIGH, + QUALITY_VERY_HIGH, +} from "mediabunny"; +import { renderTimelineFrame } from "./timeline-renderer"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { useMediaStore } from "@/stores/media-store"; +import { useProjectStore } from "@/stores/project-store"; +import { DEFAULT_FPS } from "@/stores/project-store"; +import { ExportOptions, ExportResult } from "@/types/export"; + +export const DEFAULT_EXPORT_OPTIONS: ExportOptions = { + format: "mp4", + quality: "high", + includeAudio: true, +}; + +const qualityMap = { + low: QUALITY_LOW, + medium: QUALITY_MEDIUM, + high: QUALITY_HIGH, + very_high: QUALITY_VERY_HIGH, +}; + +interface AudioElement { + buffer: AudioBuffer; + startTime: number; + duration: number; + trimStart: number; + trimEnd: number; + muted: boolean; +} + +async function createTimelineAudioBuffer( + tracks: any[], + mediaFiles: any[], + duration: number, + sampleRate: number = 44100 +): Promise { + // Get Web Audio context + const audioContext = new (window.AudioContext || + (window as any).webkitAudioContext)(); + + // Collect all audio elements from timeline + const audioElements: AudioElement[] = []; + const mediaMap = new Map(mediaFiles.map((m) => [m.id, m])); + + for (const track of tracks) { + if (track.muted) continue; + + for (const element of track.elements) { + if (element.type !== "media") continue; + + const mediaItem = mediaMap.get(element.mediaId); + if (!mediaItem || mediaItem.type !== "audio") continue; + + const visibleDuration = + element.duration - element.trimStart - element.trimEnd; + if (visibleDuration <= 0) continue; + + try { + // Decode audio file + const arrayBuffer = await mediaItem.file.arrayBuffer(); + const audioBuffer = await audioContext.decodeAudioData( + arrayBuffer.slice(0) + ); + + audioElements.push({ + buffer: audioBuffer, + startTime: element.startTime, + duration: element.duration, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + muted: element.muted || track.muted || false, + }); + } catch (error) { + console.warn(`Failed to decode audio file ${mediaItem.name}:`, error); + } + } + } + + if (audioElements.length === 0) { + return null; // No audio to mix + } + + // Create output buffer + const outputChannels = 2; // Stereo + const outputLength = Math.ceil(duration * sampleRate); + const outputBuffer = audioContext.createBuffer( + outputChannels, + outputLength, + sampleRate + ); + + // Mix all audio elements + for (const element of audioElements) { + if (element.muted) continue; + + const { + buffer, + startTime, + trimStart, + trimEnd, + duration: elementDuration, + } = element; + + // Calculate timing + const sourceStartSample = Math.floor(trimStart * buffer.sampleRate); + const sourceDuration = elementDuration - trimStart - trimEnd; + const sourceLengthSamples = Math.floor(sourceDuration * buffer.sampleRate); + const outputStartSample = Math.floor(startTime * sampleRate); + + // Resample if needed (simple approach) + const resampleRatio = sampleRate / buffer.sampleRate; + const resampledLength = Math.floor(sourceLengthSamples * resampleRatio); + + // Mix each channel + for (let channel = 0; channel < outputChannels; channel++) { + const outputData = outputBuffer.getChannelData(channel); + const sourceChannel = Math.min(channel, buffer.numberOfChannels - 1); + const sourceData = buffer.getChannelData(sourceChannel); + + for (let i = 0; i < resampledLength; i++) { + const outputIndex = outputStartSample + i; + if (outputIndex >= outputLength) break; + + // Simple resampling (could be improved with proper interpolation) + const sourceIndex = sourceStartSample + Math.floor(i / resampleRatio); + if (sourceIndex >= sourceData.length) break; + + outputData[outputIndex] += sourceData[sourceIndex]; + } + } + } + + return outputBuffer; +} + +export async function exportProject( + options: ExportOptions +): Promise { + const { format, quality, fps, includeAudio, onProgress, onCancel } = options; + + try { + const timelineStore = useTimelineStore.getState(); + const mediaStore = useMediaStore.getState(); + const projectStore = useProjectStore.getState(); + + const { tracks, getTotalDuration } = timelineStore; + const { mediaFiles } = mediaStore; + const { activeProject } = projectStore; + + if (!activeProject) { + return { success: false, error: "No active project" }; + } + + const duration = getTotalDuration(); + if (duration === 0) { + return { success: false, error: "Project is empty" }; + } + + const exportFps = fps || activeProject.fps || DEFAULT_FPS; + const canvasSize = activeProject.canvasSize; + + const outputFormat = + format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); + + // BufferTarget for smaller files, StreamTarget for larger ones + // TODO: Implement StreamTarget + const output = new Output({ + format: outputFormat, + target: new BufferTarget(), + }); + + // Canvas for rendering + const canvas = document.createElement("canvas"); + canvas.width = canvasSize.width; + canvas.height = canvasSize.height; + const ctx = canvas.getContext("2d"); + + if (!ctx) { + return { success: false, error: "Failed to create canvas context" }; + } + + const videoSource = new CanvasSource(canvas, { + codec: format === "webm" ? "vp9" : "avc", // VP9 for WebM, H.264 for MP4 + bitrate: qualityMap[quality], + }); + + output.addVideoTrack(videoSource, { frameRate: exportFps }); + + // Add audio track if requested (but don't add data yet) + let audioSource: AudioBufferSource | null = null; + let audioBuffer: AudioBuffer | null = null; + + if (includeAudio) { + onProgress?.(0.05); // 5% for audio processing + + audioBuffer = await createTimelineAudioBuffer( + tracks, + mediaFiles, + duration + ); + + if (audioBuffer) { + audioSource = new AudioBufferSource({ + codec: format === "webm" ? "opus" : "aac", // Opus for WebM, AAC for MP4 + bitrate: qualityMap[quality], // Use same quality for audio + }); + + output.addAudioTrack(audioSource); + } + } + + // Start the output (after all tracks are added) + await output.start(); + + // Now add audio data after starting + if (audioSource && audioBuffer) { + await audioSource.add(audioBuffer); + audioSource.close(); + } + + const totalFrames = Math.ceil(duration * exportFps); + let cancelled = false; + + // Render each frame + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex++) { + // Check for cancellation + if (onCancel?.()) { + cancelled = true; + break; + } + + const time = frameIndex / exportFps; + + await renderTimelineFrame({ + ctx, + time, + canvasWidth: canvas.width, + canvasHeight: canvas.height, + tracks, + mediaFiles, + backgroundColor: + activeProject.backgroundType === "blur" + ? "transparent" + : activeProject.backgroundColor || "#000000", + projectCanvasSize: canvasSize, + }); + + const frameDuration = 1 / exportFps; + await videoSource.add(time, frameDuration); + + // Adjust progress to account for audio processing (5% at start) + const videoProgress = includeAudio + ? 0.05 + (frameIndex / totalFrames) * 0.95 + : frameIndex / totalFrames; + onProgress?.(videoProgress); + } + + if (cancelled) { + await output.cancel(); + return { success: false, cancelled: true }; + } + videoSource.close(); + await output.finalize(); + onProgress?.(1); + + return { + success: true, + buffer: output.target.buffer || undefined, + }; + } catch (error) { + console.error("Export failed:", error); + return { + success: false, + error: error instanceof Error ? error.message : "Unknown export error", + }; + } +} + +export function getExportMimeType(format: "mp4" | "webm"): string { + return format === "webm" ? "video/webm" : "video/mp4"; +} + +export function getExportFileExtension(format: "mp4" | "webm"): string { + return `.${format}`; +} -- cgit v1.3.1