From 5b059ce6deb2b4132b445a0db3816388367acccf Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Mon, 11 Aug 2025 22:24:54 +0200 Subject: feat: auto-captions --- apps/web/src/lib/ffmpeg-utils.ts | 214 ++++++++++++++++++++++++++++++-- apps/web/src/lib/transcription-utils.ts | 13 ++ apps/web/src/lib/zk-encryption.ts | 71 +++++++++++ 3 files changed, 290 insertions(+), 8 deletions(-) create mode 100644 apps/web/src/lib/transcription-utils.ts create mode 100644 apps/web/src/lib/zk-encryption.ts (limited to 'apps/web/src/lib') diff --git a/apps/web/src/lib/ffmpeg-utils.ts b/apps/web/src/lib/ffmpeg-utils.ts index c34200f6..0643ecb9 100644 --- a/apps/web/src/lib/ffmpeg-utils.ts +++ b/apps/web/src/lib/ffmpeg-utils.ts @@ -1,5 +1,7 @@ import { FFmpeg } from "@ffmpeg/ffmpeg"; import { toBlobURL } from "@ffmpeg/util"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { useMediaStore } from "@/stores/media-store"; let ffmpeg: FFmpeg | null = null; @@ -7,14 +9,7 @@ export const initFFmpeg = async (): Promise => { if (ffmpeg) return ffmpeg; ffmpeg = new FFmpeg(); - - // Use locally hosted files instead of CDN - const baseURL = "/ffmpeg"; - - await ffmpeg.load({ - coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, "text/javascript"), - wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, "application/wasm"), - }); + await ffmpeg.load(); // Use default config return ffmpeg; }; @@ -268,3 +263,206 @@ export const extractAudio = async ( return blob; }; + +export const extractTimelineAudio = async ( + onProgress?: (progress: number) => void +): Promise => { + // Create fresh FFmpeg instance for this operation + const ffmpeg = new FFmpeg(); + + try { + await ffmpeg.load(); + } catch (error) { + console.error("Failed to load fresh FFmpeg instance:", error); + throw new Error("Unable to initialize audio processing. Please try again."); + } + + const timeline = useTimelineStore.getState(); + const mediaStore = useMediaStore.getState(); + + const tracks = timeline.tracks; + const totalDuration = timeline.getTotalDuration(); + + if (totalDuration === 0) { + const emptyAudioData = new ArrayBuffer(44); + return new Blob([emptyAudioData], { type: "audio/wav" }); + } + + if (onProgress) { + ffmpeg.on("progress", ({ progress }) => { + onProgress(progress * 100); + }); + } + + const audioElements: Array<{ + file: File; + startTime: number; + duration: number; + trimStart: number; + trimEnd: number; + trackMuted: boolean; + }> = []; + + for (const track of tracks) { + if (track.muted) continue; + + for (const element of track.elements) { + if (element.type === "media") { + const mediaItem = mediaStore.mediaItems.find( + (m) => m.id === element.mediaId + ); + if (!mediaItem) continue; + + if (mediaItem.type === "video" || mediaItem.type === "audio") { + audioElements.push({ + file: mediaItem.file, + startTime: element.startTime, + duration: element.duration, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + trackMuted: track.muted || false, + }); + } + } + } + } + + if (audioElements.length === 0) { + // Return silent audio if no audio elements + const silentDuration = Math.max(1, totalDuration); // At least 1 second + try { + const silentAudio = await generateSilentAudio(silentDuration); + return silentAudio; + } catch (error) { + console.error("Failed to generate silent audio:", error); + throw new Error("Unable to generate audio for empty timeline."); + } + } + + // Create a complex filter to mix all audio sources + const inputFiles: string[] = []; + const filterInputs: string[] = []; + + try { + for (let i = 0; i < audioElements.length; i++) { + const element = audioElements[i]; + const inputName = `input_${i}.${element.file.name.split(".").pop()}`; + inputFiles.push(inputName); + + try { + await ffmpeg.writeFile( + inputName, + new Uint8Array(await element.file.arrayBuffer()) + ); + } catch (error) { + console.error(`Failed to write file ${element.file.name}:`, error); + throw new Error( + `Unable to process file: ${element.file.name}. The file may be corrupted or in an unsupported format.` + ); + } + + const actualStart = element.trimStart; + const actualDuration = + element.duration - element.trimStart - element.trimEnd; + + const filterName = `audio_${i}`; + filterInputs.push( + `[${i}:a]atrim=start=${actualStart}:duration=${actualDuration},asetpts=PTS-STARTPTS,adelay=${element.startTime * 1000}|${element.startTime * 1000}[${filterName}]` + ); + } + + const mixFilter = + audioElements.length === 1 + ? `[audio_0]aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]` + : `${filterInputs.map((_, i) => `[audio_${i}]`).join("")}amix=inputs=${audioElements.length}:duration=longest:dropout_transition=2,aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]`; + + const complexFilter = [...filterInputs, mixFilter].join(";"); + const outputName = "timeline_audio.wav"; + + const ffmpegArgs = [ + ...inputFiles.flatMap((name) => ["-i", name]), + "-filter_complex", + complexFilter, + "-map", + "[out]", + "-t", + totalDuration.toString(), + "-c:a", + "pcm_s16le", + "-ar", + "44100", + outputName, + ]; + + try { + await ffmpeg.exec(ffmpegArgs); + } catch (error) { + console.error("FFmpeg execution failed:", error); + throw new Error( + "Audio processing failed. Some audio files may be corrupted or incompatible." + ); + } + + const data = await ffmpeg.readFile(outputName); + const blob = new Blob([data], { type: "audio/wav" }); + + return blob; + } catch (error) { + for (const inputFile of inputFiles) { + try { + await ffmpeg.deleteFile(inputFile); + } catch (cleanupError) { + console.warn(`Failed to cleanup file ${inputFile}:`, cleanupError); + } + } + try { + await ffmpeg.deleteFile("timeline_audio.wav"); + } catch (cleanupError) { + console.warn("Failed to cleanup output file:", cleanupError); + } + + throw error; + } finally { + for (const inputFile of inputFiles) { + try { + await ffmpeg.deleteFile(inputFile); + } catch (cleanupError) {} + } + try { + await ffmpeg.deleteFile("timeline_audio.wav"); + } catch (cleanupError) {} + } +}; + +const generateSilentAudio = async (durationSeconds: number): Promise => { + const ffmpeg = await initFFmpeg(); + const outputName = "silent.wav"; + + try { + await ffmpeg.exec([ + "-f", + "lavfi", + "-i", + `anullsrc=channel_layout=stereo:sample_rate=44100`, + "-t", + durationSeconds.toString(), + "-c:a", + "pcm_s16le", + outputName, + ]); + + const data = await ffmpeg.readFile(outputName); + const blob = new Blob([data], { type: "audio/wav" }); + + return blob; + } catch (error) { + console.error("Failed to generate silent audio:", error); + throw error; + } finally { + try { + await ffmpeg.deleteFile(outputName); + } catch (cleanupError) { + // Silent cleanup + } + } +}; diff --git a/apps/web/src/lib/transcription-utils.ts b/apps/web/src/lib/transcription-utils.ts new file mode 100644 index 00000000..6388a3b6 --- /dev/null +++ b/apps/web/src/lib/transcription-utils.ts @@ -0,0 +1,13 @@ +import { env } from "@/env"; + +export function isTranscriptionConfigured() { + const missingVars = []; + + if (!env.CLOUDFLARE_ACCOUNT_ID) missingVars.push("CLOUDFLARE_ACCOUNT_ID"); + if (!env.R2_ACCESS_KEY_ID) missingVars.push("R2_ACCESS_KEY_ID"); + if (!env.R2_SECRET_ACCESS_KEY) missingVars.push("R2_SECRET_ACCESS_KEY"); + if (!env.R2_BUCKET_NAME) missingVars.push("R2_BUCKET_NAME"); + if (!env.MODAL_TRANSCRIPTION_URL) missingVars.push("MODAL_TRANSCRIPTION_URL"); + + return { configured: missingVars.length === 0, missingVars }; +} diff --git a/apps/web/src/lib/zk-encryption.ts b/apps/web/src/lib/zk-encryption.ts new file mode 100644 index 00000000..09e5e861 --- /dev/null +++ b/apps/web/src/lib/zk-encryption.ts @@ -0,0 +1,71 @@ +/** + * True zero-knowledge encryption utilities + * Keys are generated randomly in the browser and never derived from server secrets + */ + +export interface ZeroKnowledgeEncryptionResult { + encryptedData: ArrayBuffer; + key: ArrayBuffer; + iv: ArrayBuffer; +} + +/** + * Encrypt data with a randomly generated key (true zero-knowledge) + */ +export async function encryptWithRandomKey( + data: ArrayBuffer +): Promise { + // Generate a truly random 256-bit key + const key = crypto.getRandomValues(new Uint8Array(32)); + + // Generate random IV + const iv = crypto.getRandomValues(new Uint8Array(12)); + + // Import the key for encryption + const cryptoKey = await crypto.subtle.importKey( + "raw", + key, + { name: " " }, + false, + ["encrypt"] + ); + + // Encrypt the data + const encryptedResult = await crypto.subtle.encrypt( + { name: "AES-GCM", iv }, + cryptoKey, + data + ); + + // For AES-GCM, we need to append the authentication tag + // The encrypted result contains both ciphertext and tag + return { + encryptedData: encryptedResult, + key: key.buffer, + iv: iv.buffer, + }; +} + +/** + * Convert ArrayBuffer to base64 string for transmission + */ +export function arrayBufferToBase64(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer); + let binary = ""; + for (let i = 0; i < bytes.byteLength; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +/** + * Convert base64 string back to ArrayBuffer + */ +export function base64ToArrayBuffer(base64: string): ArrayBuffer { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes.buffer; +} -- cgit v1.3.1