From 5b059ce6deb2b4132b445a0db3816388367acccf Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Mon, 11 Aug 2025 22:24:54 +0200 Subject: feat: auto-captions --- apps/web/src/app/api/get-upload-url/route.ts | 128 ++++++++ apps/web/src/app/api/transcribe/route.ts | 189 ++++++++++++ apps/web/src/components/editor-header.tsx | 2 +- .../editor/media-panel/views/base-view.tsx | 16 +- .../editor/media-panel/views/captions.tsx | 322 ++++++++++++++++++++- apps/web/src/components/editor/preview-panel.tsx | 12 +- apps/web/src/components/language-select.tsx | 204 +++++++++++++ apps/web/src/components/onboarding.tsx | 2 +- apps/web/src/components/ui/button.tsx | 2 + apps/web/src/env.ts | 14 + apps/web/src/lib/ffmpeg-utils.ts | 214 +++++++++++++- apps/web/src/lib/transcription-utils.ts | 13 + apps/web/src/lib/zk-encryption.ts | 71 +++++ 13 files changed, 1160 insertions(+), 29 deletions(-) create mode 100644 apps/web/src/app/api/get-upload-url/route.ts create mode 100644 apps/web/src/app/api/transcribe/route.ts create mode 100644 apps/web/src/components/language-select.tsx create mode 100644 apps/web/src/lib/transcription-utils.ts create mode 100644 apps/web/src/lib/zk-encryption.ts (limited to 'apps/web/src') diff --git a/apps/web/src/app/api/get-upload-url/route.ts b/apps/web/src/app/api/get-upload-url/route.ts new file mode 100644 index 00000000..dc5b7328 --- /dev/null +++ b/apps/web/src/app/api/get-upload-url/route.ts @@ -0,0 +1,128 @@ +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { AwsClient } from "aws4fetch"; +import { nanoid } from "nanoid"; +import { env } from "@/env"; +import { baseRateLimit } from "@/lib/rate-limit"; +import { isTranscriptionConfigured } from "@/lib/transcription-utils"; + +const uploadRequestSchema = z.object({ + fileExtension: z.enum(["wav", "mp3", "m4a", "flac"], { + errorMap: () => ({ + message: "File extension must be wav, mp3, m4a, or flac", + }), + }), +}); + +const apiResponseSchema = z.object({ + uploadUrl: z.string().url(), + fileName: z.string().min(1), +}); + +export async function POST(request: NextRequest) { + try { + // Rate limiting + const ip = request.headers.get("x-forwarded-for") ?? "anonymous"; + const { success } = await baseRateLimit.limit(ip); + + if (!success) { + return NextResponse.json({ error: "Too many requests" }, { status: 429 }); + } + + // Check transcription configuration + const transcriptionCheck = isTranscriptionConfigured(); + if (!transcriptionCheck.configured) { + console.error( + "Missing environment variables:", + JSON.stringify(transcriptionCheck.missingVars) + ); + + return NextResponse.json( + { + error: "Transcription not configured", + message: `Auto-captions require environment variables: ${transcriptionCheck.missingVars.join(", ")}. Check README for setup instructions.`, + }, + { status: 503 } + ); + } + + // Parse and validate request body + const rawBody = await request.json().catch(() => null); + if (!rawBody) { + return NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 } + ); + } + + const validationResult = uploadRequestSchema.safeParse(rawBody); + if (!validationResult.success) { + return NextResponse.json( + { + error: "Invalid request parameters", + details: validationResult.error.flatten().fieldErrors, + }, + { status: 400 } + ); + } + + const { fileExtension } = validationResult.data; + + // Initialize R2 client + const client = new AwsClient({ + accessKeyId: env.R2_ACCESS_KEY_ID, + secretAccessKey: env.R2_SECRET_ACCESS_KEY, + }); + + // Generate unique filename with timestamp + const timestamp = Date.now(); + const fileName = `audio/${timestamp}-${nanoid()}.${fileExtension}`; + + // Create presigned URL + const url = new URL( + `https://${env.R2_BUCKET_NAME}.${env.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com/${fileName}` + ); + + url.searchParams.set("X-Amz-Expires", "3600"); // 1 hour expiry + + const signed = await client.sign(new Request(url, { method: "PUT" }), { + aws: { signQuery: true }, + }); + + if (!signed.url) { + throw new Error("Failed to generate presigned URL"); + } + + // Prepare and validate response + const responseData = { + uploadUrl: signed.url, + fileName, + }; + + const responseValidation = apiResponseSchema.safeParse(responseData); + if (!responseValidation.success) { + console.error( + "Invalid API response structure:", + responseValidation.error + ); + return NextResponse.json( + { error: "Internal response formatting error" }, + { status: 500 } + ); + } + + return NextResponse.json(responseValidation.data); + } catch (error) { + console.error("Error generating upload URL:", error); + return NextResponse.json( + { + error: "Failed to generate upload URL", + message: + error instanceof Error + ? error.message + : "An unexpected error occurred", + }, + { status: 500 } + ); + } +} diff --git a/apps/web/src/app/api/transcribe/route.ts b/apps/web/src/app/api/transcribe/route.ts new file mode 100644 index 00000000..9a497f65 --- /dev/null +++ b/apps/web/src/app/api/transcribe/route.ts @@ -0,0 +1,189 @@ +import { NextRequest, NextResponse } from "next/server"; +import { z } from "zod"; +import { env } from "@/env"; +import { baseRateLimit } from "@/lib/rate-limit"; +import { isTranscriptionConfigured } from "@/lib/transcription-utils"; + +const transcribeRequestSchema = z.object({ + filename: z.string().min(1, "Filename is required"), + language: z.string().optional().default("auto"), + decryptionKey: z.string().min(1, "Decryption key is required").optional(), + iv: z.string().min(1, "IV is required").optional(), +}); + +const modalResponseSchema = z.object({ + text: z.string(), + segments: z.array( + z.object({ + id: z.number(), + seek: z.number(), + start: z.number(), + end: z.number(), + text: z.string(), + tokens: z.array(z.number()), + temperature: z.number(), + avg_logprob: z.number(), + compression_ratio: z.number(), + no_speech_prob: z.number(), + }) + ), + language: z.string(), +}); + +const apiResponseSchema = z.object({ + text: z.string(), + segments: z.array( + z.object({ + id: z.number(), + seek: z.number(), + start: z.number(), + end: z.number(), + text: z.string(), + tokens: z.array(z.number()), + temperature: z.number(), + avg_logprob: z.number(), + compression_ratio: z.number(), + no_speech_prob: z.number(), + }) + ), + language: z.string(), +}); + +export async function POST(request: NextRequest) { + try { + // Rate limiting + const ip = request.headers.get("x-forwarded-for") ?? "anonymous"; + const { success } = await baseRateLimit.limit(ip); + const origin = request.headers.get("origin"); + + if (!success) { + return NextResponse.json({ error: "Too many requests" }, { status: 429 }); + } + + // Check transcription configuration + const transcriptionCheck = isTranscriptionConfigured(); + if (!transcriptionCheck.configured) { + console.error( + "Missing environment variables:", + JSON.stringify(transcriptionCheck.missingVars) + ); + + return NextResponse.json( + { + error: "Transcription not configured", + message: `Auto-captions require environment variables: ${transcriptionCheck.missingVars.join(", ")}. Check README for setup instructions.`, + }, + { status: 503 } + ); + } + + // Parse and validate request body + const rawBody = await request.json().catch(() => null); + if (!rawBody) { + return NextResponse.json( + { error: "Invalid JSON in request body" }, + { status: 400 } + ); + } + + const validationResult = transcribeRequestSchema.safeParse(rawBody); + if (!validationResult.success) { + return NextResponse.json( + { + error: "Invalid request parameters", + details: validationResult.error.flatten().fieldErrors, + }, + { status: 400 } + ); + } + + const { filename, language, decryptionKey, iv } = validationResult.data; + + // Prepare request body for Modal + const modalRequestBody: any = { + filename, + language, + }; + + // Add encryption parameters if provided (zero-knowledge) + if (decryptionKey && iv) { + modalRequestBody.decryptionKey = decryptionKey; + modalRequestBody.iv = iv; + } + + // Call Modal transcription service + const response = await fetch(env.MODAL_TRANSCRIPTION_URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(modalRequestBody), + }); + + if (!response.ok) { + const errorText = await response.text(); + console.error("Modal API error:", response.status, errorText); + + let errorMessage = "Transcription service unavailable"; + try { + const errorData = JSON.parse(errorText); + errorMessage = errorData.error || errorMessage; + } catch { + // Use default message if parsing fails + } + + return NextResponse.json( + { + error: errorMessage, + message: "Failed to process transcription request", + }, + { status: response.status >= 500 ? 502 : response.status } + ); + } + + const rawResult = await response.json(); + console.log("Raw Modal response:", JSON.stringify(rawResult, null, 2)); + + // Validate Modal response + const modalValidation = modalResponseSchema.safeParse(rawResult); + if (!modalValidation.success) { + console.error("Invalid Modal API response:", modalValidation.error); + return NextResponse.json( + { error: "Invalid response from transcription service" }, + { status: 502 } + ); + } + + const result = modalValidation.data; + + // Prepare and validate API response + const responseData = { + text: result.text, + segments: result.segments, + language: result.language, + }; + + const responseValidation = apiResponseSchema.safeParse(responseData); + if (!responseValidation.success) { + console.error( + "Invalid API response structure:", + responseValidation.error + ); + return NextResponse.json( + { error: "Internal response formatting error" }, + { status: 500 } + ); + } + + return NextResponse.json(responseValidation.data); + } catch (error) { + console.error("Transcription API error:", error); + return NextResponse.json( + { + error: "Internal server error", + message: "An unexpected error occurred during transcription", + }, + { status: 500 } + ); + } +} diff --git a/apps/web/src/components/editor-header.tsx b/apps/web/src/components/editor-header.tsx index 6db2543b..98aec2b9 100644 --- a/apps/web/src/components/editor-header.tsx +++ b/apps/web/src/components/editor-header.tsx @@ -180,7 +180,7 @@ function ExportButton() { className="flex items-center gap-1.5 bg-[#38BDF8] text-white rounded-md px-[0.1rem] py-[0.1rem] cursor-pointer hover:brightness-95 transition-all duration-200" onClick={handleExport} > -
+
Export
diff --git a/apps/web/src/components/editor/media-panel/views/base-view.tsx b/apps/web/src/components/editor/media-panel/views/base-view.tsx index 296295fc..2cd33535 100644 --- a/apps/web/src/components/editor/media-panel/views/base-view.tsx +++ b/apps/web/src/components/editor/media-panel/views/base-view.tsx @@ -11,12 +11,19 @@ interface BaseViewProps { content: React.ReactNode; }[]; className?: string; + ref?: React.RefObject; } -function ViewContent({ children }: { children: React.ReactNode }) { +function ViewContent({ + children, + className, +}: { + children: React.ReactNode; + className?: string; +}) { return ( -
{children}
+
{children}
 ); } @@ -26,11 +33,12 @@ export function BaseView({ defaultTab, tabs, className = "", + ref, }: BaseViewProps) { return ( -
+
{!tabs || tabs.length === 0 ? ( - {children} + {children} ) : (
diff --git a/apps/web/src/components/editor/media-panel/views/captions.tsx b/apps/web/src/components/editor/media-panel/views/captions.tsx index ef919fec..4fe2cbd7 100644 --- a/apps/web/src/components/editor/media-panel/views/captions.tsx +++ b/apps/web/src/components/editor/media-panel/views/captions.tsx @@ -1,9 +1,313 @@ -import { BaseView } from "./base-view"; - -export function Captions() { - return ( - -
Captions
- - ); -} +import { Button } from "@/components/ui/button"; +import { PropertyGroup } from "../../properties-panel/property-item"; +import { BaseView } from "./base-view"; +import { Language, LanguageSelect } from "@/components/language-select"; +import { useState, useRef, useEffect } from "react"; +import { extractTimelineAudio } from "@/lib/ffmpeg-utils"; +import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { Loader2, Shield, Trash2, Upload } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { TextElement } from "@/types/timeline"; + +export const languages: Language[] = [ + { code: "US", name: "English" }, + { code: "ES", name: "Spanish" }, + { code: "IT", name: "Italian" }, + { code: "FR", name: "French" }, + { code: "DE", name: "German" }, + { code: "PT", name: "Portuguese" }, + { code: "RU", name: "Russian" }, + { code: "JP", name: "Japanese" }, + { code: "CN", name: "Chinese" }, +]; + +const PRIVACY_DIALOG_KEY = "opencut-transcription-privacy-accepted"; + +export function Captions() { + const [selectedCountry, setSelectedCountry] = useState("auto"); + const [isProcessing, setIsProcessing] = useState(false); + const [processingStep, setProcessingStep] = useState(""); + const [error, setError] = useState(null); + const [showPrivacyDialog, setShowPrivacyDialog] = useState(false); + const [hasAcceptedPrivacy, setHasAcceptedPrivacy] = useState(false); + const containerRef = useRef(null); + const { insertTrackAt, addElementToTrack } = useTimelineStore(); + + // Check if user has already accepted privacy on mount + useEffect(() => { + const hasAccepted = localStorage.getItem(PRIVACY_DIALOG_KEY) === "true"; + setHasAcceptedPrivacy(hasAccepted); + }, []); + + const handleGenerateTranscript = async () => { + try { + setIsProcessing(true); + setError(null); + setProcessingStep("Extracting audio..."); + + const audioBlob = await extractTimelineAudio(); + + setProcessingStep("Encrypting audio..."); + + // Encrypt the audio with a random key (zero-knowledge) + const audioBuffer = await audioBlob.arrayBuffer(); + const encryptionResult = await encryptWithRandomKey(audioBuffer); + + // Convert encrypted data to blob for upload + const encryptedBlob = new Blob([encryptionResult.encryptedData]); + + setProcessingStep("Uploading..."); + const uploadResponse = await fetch("/api/get-upload-url", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fileExtension: "wav" }), + }); + + if (!uploadResponse.ok) { + const error = await uploadResponse.json(); + throw new Error(error.message || "Failed to get upload URL"); + } + + const { uploadUrl, fileName } = await uploadResponse.json(); + + // Upload to R2 + await fetch(uploadUrl, { + method: "PUT", + body: encryptedBlob, + }); + + setProcessingStep("Transcribing..."); + + // Call Modal transcription API with encryption parameters + const transcriptionResponse = await fetch("/api/transcribe", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + filename: fileName, + language: + selectedCountry === "auto" ? "auto" : selectedCountry.toLowerCase(), + // Send the raw encryption key and IV (zero-knowledge) + decryptionKey: arrayBufferToBase64(encryptionResult.key), + iv: arrayBufferToBase64(encryptionResult.iv), + }), + }); + + if (!transcriptionResponse.ok) { + const error = await transcriptionResponse.json(); + throw new Error(error.message || "Transcription failed"); + } + + const { text, segments } = await transcriptionResponse.json(); + + console.log("Transcription completed:", { text, segments }); + + const shortCaptions: Array<{ + text: string; + startTime: number; + duration: number; + }> = []; + + let globalEndTime = 0; // Track the end time of the last caption globally + + segments.forEach((segment: any) => { + const words = segment.text.trim().split(/\s+/); + const segmentDuration = segment.end - segment.start; + const wordsPerSecond = words.length / segmentDuration; + + // Split into chunks of 2-4 words + const chunks: string[] = []; + for (let i = 0; i < words.length; i += 3) { + chunks.push(words.slice(i, i + 3).join(" ")); + } + + // Calculate timing for each chunk to place them sequentially + let chunkStartTime = segment.start; + chunks.forEach((chunk) => { + const chunkWords = chunk.split(/\s+/).length; + const chunkDuration = Math.max(0.8, chunkWords / wordsPerSecond); // Minimum 0.8s per chunk + + let adjustedStartTime = chunkStartTime; + + // Prevent overlapping: if this caption would start before the last one ends, + // start it right after the last one ends + if (adjustedStartTime < globalEndTime) { + adjustedStartTime = globalEndTime; + } + + shortCaptions.push({ + text: chunk, + startTime: adjustedStartTime, + duration: chunkDuration, + }); + + // Update global end time + globalEndTime = adjustedStartTime + chunkDuration; + + // Next chunk starts when this one ends (for within-segment timing) + chunkStartTime += chunkDuration; + }); + }); + + // Create a single track for all captions + const captionTrackId = insertTrackAt("text", 0); + + // Add all caption elements to the same track + shortCaptions.forEach((caption, index) => { + addElementToTrack(captionTrackId, { + type: "text", + name: `Caption ${index + 1}`, + content: caption.text, + duration: caption.duration, + startTime: caption.startTime, + trimStart: 0, + trimEnd: 0, + fontSize: 65, + fontFamily: "Arial", + color: "#ffffff", + textAlign: "center", + fontWeight: "bold", + fontStyle: "normal", + textDecoration: "none", + x: 0, + y: 0, + rotation: 0, + opacity: 1, + } as TextElement); + }); + + console.log( + `✅ ${shortCaptions.length} short-form caption chunks added to timeline!` + ); + } catch (error) { + console.error("Transcription failed:", error); + setError( + error instanceof Error ? error.message : "An unexpected error occurred" + ); + } finally { + setIsProcessing(false); + setProcessingStep(""); + } + }; + + return ( + + + + + +
+ {error && ( +
+

{error}

+
+ )} + + + + + + + + + Audio Processing Notice + + +

+ To generate captions, we need to process your timeline audio + using speech-to-text technology. +

+ +
+
+ + + Zero-knowledge encryption - we cannot decrypt your files + even if we wanted to + +
+ +
+ + + Encryption keys generated randomly in your browser, never + stored anywhere + +
+ +
+ + + Audio encrypted before upload - raw audio never leaves + your device + +
+ +
+ + + Everything permanently deleted within seconds after + transcription + +
+
+ +

+ True zero-knowledge privacy: Encryption keys + are generated randomly in your browser and never stored + anywhere. It's cryptographically impossible for us, our cloud + providers, or anyone else to decrypt your audio files. +

+ + + + + + + + +
+ + ); +} diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsx index a47e446a..785162a6 100644 --- a/apps/web/src/components/editor/preview-panel.tsx +++ b/apps/web/src/components/editor/preview-panel.tsx @@ -356,7 +356,7 @@ export function PreviewPanel() { return (
handleTextMouseDown(e, element, elementData.track.id) } @@ -377,7 +377,7 @@ export function PreviewPanel() { canvasSize.height) * 100 }%`, - transform: `translate(-50%, -50%) rotate(${element.rotation}deg) scale(${scaleRatio})`, + transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`, opacity: element.opacity, zIndex: 100 + index, // Text elements on top }} @@ -385,16 +385,16 @@ export function PreviewPanel() {
void; + containerRef: React.RefObject; + languages: Language[]; +} + +function FlagPreloader({ languages }: { languages: Language[] }) { + return ( +
+ {languages.map((language) => ( + + ))} +
+ ); +} + +export function LanguageSelect({ + selectedCountry, + onSelect, + containerRef, + languages, +}: LanguageSelectProps) { + const [expanded, setExpanded] = useState(false); + const [isTapping, setIsTapping] = useState(false); + const [isClosing, setIsClosing] = useState(false); + const collapsedHeight = "2.5rem"; + const expandHeight = "12rem"; + const buttonRef = useRef(null); + + const expand = () => { + setIsTapping(true); + setTimeout(() => setIsTapping(false), 600); + setExpanded(true); + buttonRef.current?.focus(); + }; + + useEffect(() => { + if (!expanded) return; + + const handleClickOutside = (event: MouseEvent) => { + if ( + buttonRef.current && + !buttonRef.current.contains(event.target as Node) + ) { + setIsClosing(true); + setTimeout(() => setIsClosing(false), 600); + setExpanded(false); + buttonRef.current?.blur(); + } + }; + + document.addEventListener("mousedown", handleClickOutside); + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, [expanded]); + + const selectedLanguage = languages.find( + (lang) => lang.code === selectedCountry + ); + + const handleSelect = ({ + code, + e, + }: { + code: string; + e: React.MouseEvent; + }) => { + e.stopPropagation(); + e.preventDefault(); + onSelect(code); + setExpanded(false); + }; + + return ( +
+ + + {!expanded ? ( +
+
+ {selectedCountry === "auto" ? ( + + ) : ( + + )} + + {selectedCountry === "auto" ? "Auto" : selectedLanguage?.name} + +
+
+ ) : ( +
+ + {languages.map((language) => ( + + ))} +
+ )} + + + + + +
+ ); +} + +function LanguageButton({ + language, + onSelect, + selectedCountry, +}: { + language: Language; + onSelect: ({ + code, + e, + }: { + code: string; + e: React.MouseEvent; + }) => void; + selectedCountry: string; +}) { + return ( + + ); +} diff --git a/apps/web/src/components/onboarding.tsx b/apps/web/src/components/onboarding.tsx index 36ae68d9..d72ac0f6 100644 --- a/apps/web/src/components/onboarding.tsx +++ b/apps/web/src/components/onboarding.tsx @@ -79,7 +79,7 @@ export function Onboarding() { return ( - + {getStepTitle()} diff --git a/apps/web/src/components/ui/button.tsx b/apps/web/src/components/ui/button.tsx index d3656790..5558eee0 100644 --- a/apps/web/src/components/ui/button.tsx +++ b/apps/web/src/components/ui/button.tsx @@ -13,6 +13,8 @@ const buttonVariants = cva( "bg-foreground text-background shadow-sm hover:bg-foreground/90", primary: "bg-primary text-primary-foreground shadow-sm hover:bg-primary/90", + "primary-gradient": + "bg-gradient-to-r from-cyan-400 to-blue-500 text-white hover:opacity-85 transition-opacity", destructive: "bg-destructive text-destructive-foreground shadow-xs hover:bg-destructive/90", outline: diff --git a/apps/web/src/env.ts b/apps/web/src/env.ts index fd4ad523..71614300 100644 --- a/apps/web/src/env.ts +++ b/apps/web/src/env.ts @@ -17,6 +17,13 @@ export const env = createEnv({ UPSTASH_REDIS_REST_TOKEN: z.string(), FREESOUND_CLIENT_ID: z.string(), FREESOUND_API_KEY: z.string(), + // R2 / Cloudflare + CLOUDFLARE_ACCOUNT_ID: z.string(), + R2_ACCESS_KEY_ID: z.string(), + R2_SECRET_ACCESS_KEY: z.string(), + R2_BUCKET_NAME: z.string(), + // Modal transcription + MODAL_TRANSCRIPTION_URL: z.string(), }, client: {}, runtimeEnv: { @@ -27,5 +34,12 @@ export const env = createEnv({ UPSTASH_REDIS_REST_TOKEN: process.env.UPSTASH_REDIS_REST_TOKEN, FREESOUND_CLIENT_ID: process.env.FREESOUND_CLIENT_ID, FREESOUND_API_KEY: process.env.FREESOUND_API_KEY, + // R2 / Cloudflare + CLOUDFLARE_ACCOUNT_ID: process.env.CLOUDFLARE_ACCOUNT_ID, + R2_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID, + R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY, + R2_BUCKET_NAME: process.env.R2_BUCKET_NAME, + // Modal transcription + MODAL_TRANSCRIPTION_URL: process.env.MODAL_TRANSCRIPTION_URL, }, }); diff --git a/apps/web/src/lib/ffmpeg-utils.ts b/apps/web/src/lib/ffmpeg-utils.ts index c34200f6..0643ecb9 100644 --- a/apps/web/src/lib/ffmpeg-utils.ts +++ b/apps/web/src/lib/ffmpeg-utils.ts @@ -1,5 +1,7 @@ import { FFmpeg } from "@ffmpeg/ffmpeg"; import { toBlobURL } from "@ffmpeg/util"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { useMediaStore } from "@/stores/media-store"; let ffmpeg: FFmpeg | null = null; @@ -7,14 +9,7 @@ export const initFFmpeg = async (): Promise => { if (ffmpeg) return ffmpeg; ffmpeg = new FFmpeg(); - - // Use locally hosted files instead of CDN - const baseURL = "/ffmpeg"; - - await ffmpeg.load({ - coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, "text/javascript"), - wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, "application/wasm"), - }); + await ffmpeg.load(); // Use default config return ffmpeg; }; @@ -268,3 +263,206 @@ export const extractAudio = async ( return blob; }; + +export const extractTimelineAudio = async ( + onProgress?: (progress: number) => void +): Promise => { + // Create fresh FFmpeg instance for this operation + const ffmpeg = new FFmpeg(); + + try { + await ffmpeg.load(); + } catch (error) { + console.error("Failed to load fresh FFmpeg instance:", error); + throw new Error("Unable to initialize audio processing. Please try again."); + } + + const timeline = useTimelineStore.getState(); + const mediaStore = useMediaStore.getState(); + + const tracks = timeline.tracks; + const totalDuration = timeline.getTotalDuration(); + + if (totalDuration === 0) { + const emptyAudioData = new ArrayBuffer(44); + return new Blob([emptyAudioData], { type: "audio/wav" }); + } + + if (onProgress) { + ffmpeg.on("progress", ({ progress }) => { + onProgress(progress * 100); + }); + } + + const audioElements: Array<{ + file: File; + startTime: number; + duration: number; + trimStart: number; + trimEnd: number; + trackMuted: boolean; + }> = []; + + for (const track of tracks) { + if (track.muted) continue; + + for (const element of track.elements) { + if (element.type === "media") { + const mediaItem = mediaStore.mediaItems.find( + (m) => m.id === element.mediaId + ); + if (!mediaItem) continue; + + if (mediaItem.type === "video" || mediaItem.type === "audio") { + audioElements.push({ + file: mediaItem.file, + startTime: element.startTime, + duration: element.duration, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + trackMuted: track.muted || false, + }); + } + } + } + } + + if (audioElements.length === 0) { + // Return silent audio if no audio elements + const silentDuration = Math.max(1, totalDuration); // At least 1 second + try { + const silentAudio = await generateSilentAudio(silentDuration); + return silentAudio; + } catch (error) { + console.error("Failed to generate silent audio:", error); + throw new Error("Unable to generate audio for empty timeline."); + } + } + + // Create a complex filter to mix all audio sources + const inputFiles: string[] = []; + const filterInputs: string[] = []; + + try { + for (let i = 0; i < audioElements.length; i++) { + const element = audioElements[i]; + const inputName = `input_${i}.${element.file.name.split(".").pop()}`; + inputFiles.push(inputName); + + try { + await ffmpeg.writeFile( + inputName, + new Uint8Array(await element.file.arrayBuffer()) + ); + } catch (error) { + console.error(`Failed to write file ${element.file.name}:`, error); + throw new Error( + `Unable to process file: ${element.file.name}. The file may be corrupted or in an unsupported format.` + ); + } + + const actualStart = element.trimStart; + const actualDuration = + element.duration - element.trimStart - element.trimEnd; + + const filterName = `audio_${i}`; + filterInputs.push( + `[${i}:a]atrim=start=${actualStart}:duration=${actualDuration},asetpts=PTS-STARTPTS,adelay=${element.startTime * 1000}|${element.startTime * 1000}[${filterName}]` + ); + } + + const mixFilter = + audioElements.length === 1 + ? `[audio_0]aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]` + : `${filterInputs.map((_, i) => `[audio_${i}]`).join("")}amix=inputs=${audioElements.length}:duration=longest:dropout_transition=2,aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]`; + + const complexFilter = [...filterInputs, mixFilter].join(";"); + const outputName = "timeline_audio.wav"; + + const ffmpegArgs = [ + ...inputFiles.flatMap((name) => ["-i", name]), + "-filter_complex", + complexFilter, + "-map", + "[out]", + "-t", + totalDuration.toString(), + "-c:a", + "pcm_s16le", + "-ar", + "44100", + outputName, + ]; + + try { + await ffmpeg.exec(ffmpegArgs); + } catch (error) { + console.error("FFmpeg execution failed:", error); + throw new Error( + "Audio processing failed. Some audio files may be corrupted or incompatible." + ); + } + + const data = await ffmpeg.readFile(outputName); + const blob = new Blob([data], { type: "audio/wav" }); + + return blob; + } catch (error) { + for (const inputFile of inputFiles) { + try { + await ffmpeg.deleteFile(inputFile); + } catch (cleanupError) { + console.warn(`Failed to cleanup file ${inputFile}:`, cleanupError); + } + } + try { + await ffmpeg.deleteFile("timeline_audio.wav"); + } catch (cleanupError) { + console.warn("Failed to cleanup output file:", cleanupError); + } + + throw error; + } finally { + for (const inputFile of inputFiles) { + try { + await ffmpeg.deleteFile(inputFile); + } catch (cleanupError) {} + } + try { + await ffmpeg.deleteFile("timeline_audio.wav"); + } catch (cleanupError) {} + } +}; + +const generateSilentAudio = async (durationSeconds: number): Promise => { + const ffmpeg = await initFFmpeg(); + const outputName = "silent.wav"; + + try { + await ffmpeg.exec([ + "-f", + "lavfi", + "-i", + `anullsrc=channel_layout=stereo:sample_rate=44100`, + "-t", + durationSeconds.toString(), + "-c:a", + "pcm_s16le", + outputName, + ]); + + const data = await ffmpeg.readFile(outputName); + const blob = new Blob([data], { type: "audio/wav" }); + + return blob; + } catch (error) { + console.error("Failed to generate silent audio:", error); + throw error; + } finally { + try { + await ffmpeg.deleteFile(outputName); + } catch (cleanupError) { + // Silent cleanup + } + } +}; diff --git a/apps/web/src/lib/transcription-utils.ts b/apps/web/src/lib/transcription-utils.ts new file mode 100644 index 00000000..6388a3b6 --- /dev/null +++ b/apps/web/src/lib/transcription-utils.ts @@ -0,0 +1,13 @@ +import { env } from "@/env"; + +export function isTranscriptionConfigured() { + const missingVars = []; + + if (!env.CLOUDFLARE_ACCOUNT_ID) missingVars.push("CLOUDFLARE_ACCOUNT_ID"); + if (!env.R2_ACCESS_KEY_ID) missingVars.push("R2_ACCESS_KEY_ID"); + if (!env.R2_SECRET_ACCESS_KEY) missingVars.push("R2_SECRET_ACCESS_KEY"); + if (!env.R2_BUCKET_NAME) missingVars.push("R2_BUCKET_NAME"); + if (!env.MODAL_TRANSCRIPTION_URL) missingVars.push("MODAL_TRANSCRIPTION_URL"); + + return { configured: missingVars.length === 0, missingVars }; +} diff --git a/apps/web/src/lib/zk-encryption.ts b/apps/web/src/lib/zk-encryption.ts new file mode 100644 index 00000000..09e5e861 --- /dev/null +++ b/apps/web/src/lib/zk-encryption.ts @@ -0,0 +1,71 @@ +/** + * True zero-knowledge encryption utilities + * Keys are generated randomly in the browser and never derived from server secrets + */ + +export interface ZeroKnowledgeEncryptionResult { + encryptedData: ArrayBuffer; + key: ArrayBuffer; + iv: ArrayBuffer; +} + +/** + * Encrypt data with a randomly generated key (true zero-knowledge) + */ +export async function encryptWithRandomKey( + data: ArrayBuffer +): Promise { + // Generate a truly random 256-bit key + const key = crypto.getRandomValues(new Uint8Array(32)); + + // Generate random IV + const iv = crypto.getRandomValues(new Uint8Array(12)); + + // Import the key for encryption + const cryptoKey = await crypto.subtle.importKey( + "raw", + key, + { name: " " }, + false, + ["encrypt"] + ); + + // Encrypt the data + const encryptedResult = await crypto.subtle.encrypt( + { name: "AES-GCM", iv }, + cryptoKey, + data + ); + + // For AES-GCM, we need to append the authentication tag + // The encrypted result contains both ciphertext and tag + return { + encryptedData: encryptedResult, + key: key.buffer, + iv: iv.buffer, + }; +} + +/** + * Convert ArrayBuffer to base64 string for transmission + */ +export function arrayBufferToBase64(buffer: ArrayBuffer): string { + const bytes = new Uint8Array(buffer); + let binary = ""; + for (let i = 0; i < bytes.byteLength; i++) { + binary += String.fromCharCode(bytes[i]); + } + return btoa(binary); +} + +/** + * Convert base64 string back to ArrayBuffer + */ +export function base64ToArrayBuffer(base64: string): ArrayBuffer { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes.buffer; +} -- cgit v1.3.1