From 5b059ce6deb2b4132b445a0db3816388367acccf Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Mon, 11 Aug 2025 22:24:54 +0200 Subject: feat: auto-captions --- apps/web/src/components/editor-header.tsx | 2 +- .../editor/media-panel/views/base-view.tsx | 16 +- .../editor/media-panel/views/captions.tsx | 322 ++++++++++++++++++++- apps/web/src/components/editor/preview-panel.tsx | 12 +- apps/web/src/components/language-select.tsx | 204 +++++++++++++ apps/web/src/components/onboarding.tsx | 2 +- apps/web/src/components/ui/button.tsx | 2 + 7 files changed, 539 insertions(+), 21 deletions(-) create mode 100644 apps/web/src/components/language-select.tsx (limited to 'apps/web/src/components') diff --git a/apps/web/src/components/editor-header.tsx b/apps/web/src/components/editor-header.tsx index 6db2543b..98aec2b9 100644 --- a/apps/web/src/components/editor-header.tsx +++ b/apps/web/src/components/editor-header.tsx @@ -180,7 +180,7 @@ function ExportButton() { className="flex items-center gap-1.5 bg-[#38BDF8] text-white rounded-md px-[0.1rem] py-[0.1rem] cursor-pointer hover:brightness-95 transition-all duration-200" onClick={handleExport} > -
+
Export
diff --git a/apps/web/src/components/editor/media-panel/views/base-view.tsx b/apps/web/src/components/editor/media-panel/views/base-view.tsx index 296295fc..2cd33535 100644 --- a/apps/web/src/components/editor/media-panel/views/base-view.tsx +++ b/apps/web/src/components/editor/media-panel/views/base-view.tsx @@ -11,12 +11,19 @@ interface BaseViewProps { content: React.ReactNode; }[]; className?: string; + ref?: React.RefObject; } -function ViewContent({ children }: { children: React.ReactNode }) { +function ViewContent({ + children, + className, +}: { + children: React.ReactNode; + className?: string; +}) { return ( -
{children}
+
{children}
 ); } @@ -26,11 +33,12 @@ export function BaseView({ defaultTab, tabs, className = "", + ref, }: BaseViewProps) { return ( -
+
{!tabs || tabs.length === 0 ? ( - {children} + {children} ) : (
diff --git a/apps/web/src/components/editor/media-panel/views/captions.tsx b/apps/web/src/components/editor/media-panel/views/captions.tsx index ef919fec..4fe2cbd7 100644 --- a/apps/web/src/components/editor/media-panel/views/captions.tsx +++ b/apps/web/src/components/editor/media-panel/views/captions.tsx @@ -1,9 +1,313 @@ -import { BaseView } from "./base-view"; - -export function Captions() { - return ( - -
Captions
- - ); -} +import { Button } from "@/components/ui/button"; +import { PropertyGroup } from "../../properties-panel/property-item"; +import { BaseView } from "./base-view"; +import { Language, LanguageSelect } from "@/components/language-select"; +import { useState, useRef, useEffect } from "react"; +import { extractTimelineAudio } from "@/lib/ffmpeg-utils"; +import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { Loader2, Shield, Trash2, Upload } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { TextElement } from "@/types/timeline"; + +export const languages: Language[] = [ + { code: "US", name: "English" }, + { code: "ES", name: "Spanish" }, + { code: "IT", name: "Italian" }, + { code: "FR", name: "French" }, + { code: "DE", name: "German" }, + { code: "PT", name: "Portuguese" }, + { code: "RU", name: "Russian" }, + { code: "JP", name: "Japanese" }, + { code: "CN", name: "Chinese" }, +]; + +const PRIVACY_DIALOG_KEY = "opencut-transcription-privacy-accepted"; + +export function Captions() { + const [selectedCountry, setSelectedCountry] = useState("auto"); + const [isProcessing, setIsProcessing] = useState(false); + const [processingStep, setProcessingStep] = useState(""); + const [error, setError] = useState(null); + const [showPrivacyDialog, setShowPrivacyDialog] = useState(false); + const [hasAcceptedPrivacy, setHasAcceptedPrivacy] = useState(false); + const containerRef = useRef(null); + const { insertTrackAt, addElementToTrack } = useTimelineStore(); + + // Check if user has already accepted privacy on mount + useEffect(() => { + const hasAccepted = localStorage.getItem(PRIVACY_DIALOG_KEY) === "true"; + setHasAcceptedPrivacy(hasAccepted); + }, []); + + const handleGenerateTranscript = async () => { + try { + setIsProcessing(true); + setError(null); + setProcessingStep("Extracting audio..."); + + const audioBlob = await extractTimelineAudio(); + + setProcessingStep("Encrypting audio..."); + + // Encrypt the audio with a random key (zero-knowledge) + const audioBuffer = await audioBlob.arrayBuffer(); + const encryptionResult = await encryptWithRandomKey(audioBuffer); + + // Convert encrypted data to blob for upload + const encryptedBlob = new Blob([encryptionResult.encryptedData]); + + setProcessingStep("Uploading..."); + const uploadResponse = await fetch("/api/get-upload-url", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fileExtension: "wav" }), + }); + + if (!uploadResponse.ok) { + const error = await uploadResponse.json(); + throw new Error(error.message || "Failed to get upload URL"); + } + + const { uploadUrl, fileName } = await uploadResponse.json(); + + // Upload to R2 + await fetch(uploadUrl, { + method: "PUT", + body: encryptedBlob, + }); + + setProcessingStep("Transcribing..."); + + // Call Modal transcription API with encryption parameters + const transcriptionResponse = await fetch("/api/transcribe", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + filename: fileName, + language: + selectedCountry === "auto" ? "auto" : selectedCountry.toLowerCase(), + // Send the raw encryption key and IV (zero-knowledge) + decryptionKey: arrayBufferToBase64(encryptionResult.key), + iv: arrayBufferToBase64(encryptionResult.iv), + }), + }); + + if (!transcriptionResponse.ok) { + const error = await transcriptionResponse.json(); + throw new Error(error.message || "Transcription failed"); + } + + const { text, segments } = await transcriptionResponse.json(); + + console.log("Transcription completed:", { text, segments }); + + const shortCaptions: Array<{ + text: string; + startTime: number; + duration: number; + }> = []; + + let globalEndTime = 0; // Track the end time of the last caption globally + + segments.forEach((segment: any) => { + const words = segment.text.trim().split(/\s+/); + const segmentDuration = segment.end - segment.start; + const wordsPerSecond = words.length / segmentDuration; + + // Split into chunks of 2-4 words + const chunks: string[] = []; + for (let i = 0; i < words.length; i += 3) { + chunks.push(words.slice(i, i + 3).join(" ")); + } + + // Calculate timing for each chunk to place them sequentially + let chunkStartTime = segment.start; + chunks.forEach((chunk) => { + const chunkWords = chunk.split(/\s+/).length; + const chunkDuration = Math.max(0.8, chunkWords / wordsPerSecond); // Minimum 0.8s per chunk + + let adjustedStartTime = chunkStartTime; + + // Prevent overlapping: if this caption would start before the last one ends, + // start it right after the last one ends + if (adjustedStartTime < globalEndTime) { + adjustedStartTime = globalEndTime; + } + + shortCaptions.push({ + text: chunk, + startTime: adjustedStartTime, + duration: chunkDuration, + }); + + // Update global end time + globalEndTime = adjustedStartTime + chunkDuration; + + // Next chunk starts when this one ends (for within-segment timing) + chunkStartTime += chunkDuration; + }); + }); + + // Create a single track for all captions + const captionTrackId = insertTrackAt("text", 0); + + // Add all caption elements to the same track + shortCaptions.forEach((caption, index) => { + addElementToTrack(captionTrackId, { + type: "text", + name: `Caption ${index + 1}`, + content: caption.text, + duration: caption.duration, + startTime: caption.startTime, + trimStart: 0, + trimEnd: 0, + fontSize: 65, + fontFamily: "Arial", + color: "#ffffff", + textAlign: "center", + fontWeight: "bold", + fontStyle: "normal", + textDecoration: "none", + x: 0, + y: 0, + rotation: 0, + opacity: 1, + } as TextElement); + }); + + console.log( + `✅ ${shortCaptions.length} short-form caption chunks added to timeline!` + ); + } catch (error) { + console.error("Transcription failed:", error); + setError( + error instanceof Error ? error.message : "An unexpected error occurred" + ); + } finally { + setIsProcessing(false); + setProcessingStep(""); + } + }; + + return ( + + + + + +
+ {error && ( +
+

{error}

+
+ )} + + + + + + + + + Audio Processing Notice + + +

+ To generate captions, we need to process your timeline audio + using speech-to-text technology. +

+ +
+
+ + + Zero-knowledge encryption - we cannot decrypt your files + even if we wanted to + +
+ +
+ + + Encryption keys generated randomly in your browser, never + stored anywhere + +
+ +
+ + + Audio encrypted before upload - raw audio never leaves + your device + +
+ +
+ + + Everything permanently deleted within seconds after + transcription + +
+
+ +

+ True zero-knowledge privacy: Encryption keys + are generated randomly in your browser and never stored + anywhere. It's cryptographically impossible for us, our cloud + providers, or anyone else to decrypt your audio files. +

+ + + + + + + + +
+ + ); +} diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsx index a47e446a..785162a6 100644 --- a/apps/web/src/components/editor/preview-panel.tsx +++ b/apps/web/src/components/editor/preview-panel.tsx @@ -356,7 +356,7 @@ export function PreviewPanel() { return (
handleTextMouseDown(e, element, elementData.track.id) } @@ -377,7 +377,7 @@ export function PreviewPanel() { canvasSize.height) * 100 }%`, - transform: `translate(-50%, -50%) rotate(${element.rotation}deg) scale(${scaleRatio})`, + transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`, opacity: element.opacity, zIndex: 100 + index, // Text elements on top }} @@ -385,16 +385,16 @@ export function PreviewPanel() {
void; + containerRef: React.RefObject; + languages: Language[]; +} + +function FlagPreloader({ languages }: { languages: Language[] }) { + return ( +
+ {languages.map((language) => ( + + ))} +
+ ); +} + +export function LanguageSelect({ + selectedCountry, + onSelect, + containerRef, + languages, +}: LanguageSelectProps) { + const [expanded, setExpanded] = useState(false); + const [isTapping, setIsTapping] = useState(false); + const [isClosing, setIsClosing] = useState(false); + const collapsedHeight = "2.5rem"; + const expandHeight = "12rem"; + const buttonRef = useRef(null); + + const expand = () => { + setIsTapping(true); + setTimeout(() => setIsTapping(false), 600); + setExpanded(true); + buttonRef.current?.focus(); + }; + + useEffect(() => { + if (!expanded) return; + + const handleClickOutside = (event: MouseEvent) => { + if ( + buttonRef.current && + !buttonRef.current.contains(event.target as Node) + ) { + setIsClosing(true); + setTimeout(() => setIsClosing(false), 600); + setExpanded(false); + buttonRef.current?.blur(); + } + }; + + document.addEventListener("mousedown", handleClickOutside); + return () => { + document.removeEventListener("mousedown", handleClickOutside); + }; + }, [expanded]); + + const selectedLanguage = languages.find( + (lang) => lang.code === selectedCountry + ); + + const handleSelect = ({ + code, + e, + }: { + code: string; + e: React.MouseEvent; + }) => { + e.stopPropagation(); + e.preventDefault(); + onSelect(code); + setExpanded(false); + }; + + return ( +
+ + + {!expanded ? ( +
+
+ {selectedCountry === "auto" ? ( + + ) : ( + + )} + + {selectedCountry === "auto" ? "Auto" : selectedLanguage?.name} + +
+
+ ) : ( +
+ + {languages.map((language) => ( + + ))} +
+ )} + + + + + +
+ ); +} + +function LanguageButton({ + language, + onSelect, + selectedCountry, +}: { + language: Language; + onSelect: ({ + code, + e, + }: { + code: string; + e: React.MouseEvent; + }) => void; + selectedCountry: string; +}) { + return ( + + ); +} diff --git a/apps/web/src/components/onboarding.tsx b/apps/web/src/components/onboarding.tsx index 36ae68d9..d72ac0f6 100644 --- a/apps/web/src/components/onboarding.tsx +++ b/apps/web/src/components/onboarding.tsx @@ -79,7 +79,7 @@ export function Onboarding() { return ( - + {getStepTitle()} diff --git a/apps/web/src/components/ui/button.tsx b/apps/web/src/components/ui/button.tsx index d3656790..5558eee0 100644 --- a/apps/web/src/components/ui/button.tsx +++ b/apps/web/src/components/ui/button.tsx @@ -13,6 +13,8 @@ const buttonVariants = cva( "bg-foreground text-background shadow-sm hover:bg-foreground/90", primary: "bg-primary text-primary-foreground shadow-sm hover:bg-primary/90", + "primary-gradient": + "bg-gradient-to-r from-cyan-400 to-blue-500 text-white hover:opacity-85 transition-opacity", destructive: "bg-destructive text-destructive-foreground shadow-xs hover:bg-destructive/90", outline: -- cgit v1.3.1