diff options
| author | Maze Winther <mazewinther@gmail.com> | 2025-08-11 22:24:54 +0200 |
|---|---|---|
| committer | Maze Winther <mazewinther@gmail.com> | 2025-08-11 22:24:54 +0200 |
| commit | 5b059ce6deb2b4132b445a0db3816388367acccf (patch) | |
| tree | 9650dcb1efe10b15650aa3b4e152854bbaae0b2d /apps/web | |
| parent | 38f82767884250fe7c9e99672d06cb59ac957cd0 (diff) | |
| download | opencut-5b059ce6deb2b4132b445a0db3816388367acccf.tar.gz | |
feat: auto-captions
Diffstat (limited to 'apps/web')
| -rw-r--r-- | apps/web/.env.example | 12 | ||||
| -rw-r--r-- | apps/web/package.json | 4 | ||||
| -rw-r--r-- | apps/web/src/app/api/get-upload-url/route.ts | 128 | ||||
| -rw-r--r-- | apps/web/src/app/api/transcribe/route.ts | 189 | ||||
| -rw-r--r-- | apps/web/src/components/editor-header.tsx | 2 | ||||
| -rw-r--r-- | apps/web/src/components/editor/media-panel/views/base-view.tsx | 16 | ||||
| -rw-r--r-- | apps/web/src/components/editor/media-panel/views/captions.tsx | 322 | ||||
| -rw-r--r-- | apps/web/src/components/editor/preview-panel.tsx | 12 | ||||
| -rw-r--r-- | apps/web/src/components/language-select.tsx | 204 | ||||
| -rw-r--r-- | apps/web/src/components/onboarding.tsx | 2 | ||||
| -rw-r--r-- | apps/web/src/components/ui/button.tsx | 2 | ||||
| -rw-r--r-- | apps/web/src/env.ts | 14 | ||||
| -rw-r--r-- | apps/web/src/lib/ffmpeg-utils.ts | 214 | ||||
| -rw-r--r-- | apps/web/src/lib/transcription-utils.ts | 13 | ||||
| -rw-r--r-- | apps/web/src/lib/zk-encryption.ts | 71 |
15 files changed, 1174 insertions, 31 deletions
diff --git a/apps/web/.env.example b/apps/web/.env.example index af84fd77..e77d3c5a 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -20,4 +20,14 @@ NEXT_PUBLIC_MARBLE_API_URL=https://api.marblecms.com # Freesound (generate at https://freesound.org/apiv2/apply/) FREESOUND_CLIENT_ID=... -FREESOUND_API_KEY=...
\ No newline at end of file +FREESOUND_API_KEY=... + +# Cloudflare R2 (for auto-captions/transcription) +# Get these from Cloudflare Dashboard > R2 > Manage R2 API tokens +CLOUDFLARE_ACCOUNT_ID=your-account-id +R2_ACCESS_KEY_ID=your-access-key-id +R2_SECRET_ACCESS_KEY=your-secret-access-key +R2_BUCKET_NAME=opencut-transcription + +# Modal transcription endpoint (from modal deploy transcription.py) +MODAL_TRANSCRIPTION_URL=https://your-username--opencut-transcription-transcribe-audio.modal.run
\ No newline at end of file diff --git a/apps/web/package.json b/apps/web/package.json index d9947643..48a211db 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -4,7 +4,7 @@ "private": true, "packageManager": "bun@1.2.18", "scripts": { - "dev": "next dev", + "dev": "next dev --turbopack", "build": "next build", "start": "next start", "lint": "biome check src/", @@ -29,6 +29,7 @@ "@upstash/ratelimit": "^2.0.5", "@upstash/redis": "^1.35.0", "@vercel/analytics": "^1.4.1", + "aws4fetch": "^1.0.20", "better-auth": "^1.2.7", "botid": "^1.4.2", "class-variance-authority": "^0.7.1", @@ -43,6 +44,7 @@ "input-otp": "^1.4.1", "lucide-react": "^0.468.0", "motion": "^12.18.1", + "nanoid": "^5.1.5", "next": "^15.4.5", "next-themes": "^0.4.4", "pg": "^8.16.2", diff --git a/apps/web/src/app/api/get-upload-url/route.ts b/apps/web/src/app/api/get-upload-url/route.ts new file mode 100644 index 00000000..dc5b7328 --- /dev/null +++ b/apps/web/src/app/api/get-upload-url/route.ts @@ -0,0 +1,128 @@ +import { NextRequest, NextResponse } from "next/server";
+import { z } from "zod";
+import { AwsClient } from "aws4fetch";
+import { nanoid } from "nanoid";
+import { env } from "@/env";
+import { baseRateLimit } from "@/lib/rate-limit";
+import { isTranscriptionConfigured } from "@/lib/transcription-utils";
+
+const uploadRequestSchema = z.object({
+ fileExtension: z.enum(["wav", "mp3", "m4a", "flac"], {
+ errorMap: () => ({
+ message: "File extension must be wav, mp3, m4a, or flac",
+ }),
+ }),
+});
+
+const apiResponseSchema = z.object({
+ uploadUrl: z.string().url(),
+ fileName: z.string().min(1),
+});
+
+export async function POST(request: NextRequest) {
+ try {
+ // Rate limiting
+ const ip = request.headers.get("x-forwarded-for") ?? "anonymous";
+ const { success } = await baseRateLimit.limit(ip);
+
+ if (!success) {
+ return NextResponse.json({ error: "Too many requests" }, { status: 429 });
+ }
+
+ // Check transcription configuration
+ const transcriptionCheck = isTranscriptionConfigured();
+ if (!transcriptionCheck.configured) {
+ console.error(
+ "Missing environment variables:",
+ JSON.stringify(transcriptionCheck.missingVars)
+ );
+
+ return NextResponse.json(
+ {
+ error: "Transcription not configured",
+ message: `Auto-captions require environment variables: ${transcriptionCheck.missingVars.join(", ")}. Check README for setup instructions.`,
+ },
+ { status: 503 }
+ );
+ }
+
+ // Parse and validate request body
+ const rawBody = await request.json().catch(() => null);
+ if (!rawBody) {
+ return NextResponse.json(
+ { error: "Invalid JSON in request body" },
+ { status: 400 }
+ );
+ }
+
+ const validationResult = uploadRequestSchema.safeParse(rawBody);
+ if (!validationResult.success) {
+ return NextResponse.json(
+ {
+ error: "Invalid request parameters",
+ details: validationResult.error.flatten().fieldErrors,
+ },
+ { status: 400 }
+ );
+ }
+
+ const { fileExtension } = validationResult.data;
+
+ // Initialize R2 client
+ const client = new AwsClient({
+ accessKeyId: env.R2_ACCESS_KEY_ID,
+ secretAccessKey: env.R2_SECRET_ACCESS_KEY,
+ });
+
+ // Generate unique filename with timestamp
+ const timestamp = Date.now();
+ const fileName = `audio/${timestamp}-${nanoid()}.${fileExtension}`;
+
+ // Create presigned URL
+ const url = new URL(
+ `https://${env.R2_BUCKET_NAME}.${env.CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com/${fileName}`
+ );
+
+ url.searchParams.set("X-Amz-Expires", "3600"); // 1 hour expiry
+
+ const signed = await client.sign(new Request(url, { method: "PUT" }), {
+ aws: { signQuery: true },
+ });
+
+ if (!signed.url) {
+ throw new Error("Failed to generate presigned URL");
+ }
+
+ // Prepare and validate response
+ const responseData = {
+ uploadUrl: signed.url,
+ fileName,
+ };
+
+ const responseValidation = apiResponseSchema.safeParse(responseData);
+ if (!responseValidation.success) {
+ console.error(
+ "Invalid API response structure:",
+ responseValidation.error
+ );
+ return NextResponse.json(
+ { error: "Internal response formatting error" },
+ { status: 500 }
+ );
+ }
+
+ return NextResponse.json(responseValidation.data);
+ } catch (error) {
+ console.error("Error generating upload URL:", error);
+ return NextResponse.json(
+ {
+ error: "Failed to generate upload URL",
+ message:
+ error instanceof Error
+ ? error.message
+ : "An unexpected error occurred",
+ },
+ { status: 500 }
+ );
+ }
+}
diff --git a/apps/web/src/app/api/transcribe/route.ts b/apps/web/src/app/api/transcribe/route.ts new file mode 100644 index 00000000..9a497f65 --- /dev/null +++ b/apps/web/src/app/api/transcribe/route.ts @@ -0,0 +1,189 @@ +import { NextRequest, NextResponse } from "next/server";
+import { z } from "zod";
+import { env } from "@/env";
+import { baseRateLimit } from "@/lib/rate-limit";
+import { isTranscriptionConfigured } from "@/lib/transcription-utils";
+
+const transcribeRequestSchema = z.object({
+ filename: z.string().min(1, "Filename is required"),
+ language: z.string().optional().default("auto"),
+ decryptionKey: z.string().min(1, "Decryption key is required").optional(),
+ iv: z.string().min(1, "IV is required").optional(),
+});
+
+const modalResponseSchema = z.object({
+ text: z.string(),
+ segments: z.array(
+ z.object({
+ id: z.number(),
+ seek: z.number(),
+ start: z.number(),
+ end: z.number(),
+ text: z.string(),
+ tokens: z.array(z.number()),
+ temperature: z.number(),
+ avg_logprob: z.number(),
+ compression_ratio: z.number(),
+ no_speech_prob: z.number(),
+ })
+ ),
+ language: z.string(),
+});
+
+const apiResponseSchema = z.object({
+ text: z.string(),
+ segments: z.array(
+ z.object({
+ id: z.number(),
+ seek: z.number(),
+ start: z.number(),
+ end: z.number(),
+ text: z.string(),
+ tokens: z.array(z.number()),
+ temperature: z.number(),
+ avg_logprob: z.number(),
+ compression_ratio: z.number(),
+ no_speech_prob: z.number(),
+ })
+ ),
+ language: z.string(),
+});
+
+export async function POST(request: NextRequest) {
+ try {
+ // Rate limiting
+ const ip = request.headers.get("x-forwarded-for") ?? "anonymous";
+ const { success } = await baseRateLimit.limit(ip);
+ const origin = request.headers.get("origin");
+
+ if (!success) {
+ return NextResponse.json({ error: "Too many requests" }, { status: 429 });
+ }
+
+ // Check transcription configuration
+ const transcriptionCheck = isTranscriptionConfigured();
+ if (!transcriptionCheck.configured) {
+ console.error(
+ "Missing environment variables:",
+ JSON.stringify(transcriptionCheck.missingVars)
+ );
+
+ return NextResponse.json(
+ {
+ error: "Transcription not configured",
+ message: `Auto-captions require environment variables: ${transcriptionCheck.missingVars.join(", ")}. Check README for setup instructions.`,
+ },
+ { status: 503 }
+ );
+ }
+
+ // Parse and validate request body
+ const rawBody = await request.json().catch(() => null);
+ if (!rawBody) {
+ return NextResponse.json(
+ { error: "Invalid JSON in request body" },
+ { status: 400 }
+ );
+ }
+
+ const validationResult = transcribeRequestSchema.safeParse(rawBody);
+ if (!validationResult.success) {
+ return NextResponse.json(
+ {
+ error: "Invalid request parameters",
+ details: validationResult.error.flatten().fieldErrors,
+ },
+ { status: 400 }
+ );
+ }
+
+ const { filename, language, decryptionKey, iv } = validationResult.data;
+
+ // Prepare request body for Modal
+ const modalRequestBody: any = {
+ filename,
+ language,
+ };
+
+ // Add encryption parameters if provided (zero-knowledge)
+ if (decryptionKey && iv) {
+ modalRequestBody.decryptionKey = decryptionKey;
+ modalRequestBody.iv = iv;
+ }
+
+ // Call Modal transcription service
+ const response = await fetch(env.MODAL_TRANSCRIPTION_URL, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify(modalRequestBody),
+ });
+
+ if (!response.ok) {
+ const errorText = await response.text();
+ console.error("Modal API error:", response.status, errorText);
+
+ let errorMessage = "Transcription service unavailable";
+ try {
+ const errorData = JSON.parse(errorText);
+ errorMessage = errorData.error || errorMessage;
+ } catch {
+ // Use default message if parsing fails
+ }
+
+ return NextResponse.json(
+ {
+ error: errorMessage,
+ message: "Failed to process transcription request",
+ },
+ { status: response.status >= 500 ? 502 : response.status }
+ );
+ }
+
+ const rawResult = await response.json();
+ console.log("Raw Modal response:", JSON.stringify(rawResult, null, 2));
+
+ // Validate Modal response
+ const modalValidation = modalResponseSchema.safeParse(rawResult);
+ if (!modalValidation.success) {
+ console.error("Invalid Modal API response:", modalValidation.error);
+ return NextResponse.json(
+ { error: "Invalid response from transcription service" },
+ { status: 502 }
+ );
+ }
+
+ const result = modalValidation.data;
+
+ // Prepare and validate API response
+ const responseData = {
+ text: result.text,
+ segments: result.segments,
+ language: result.language,
+ };
+
+ const responseValidation = apiResponseSchema.safeParse(responseData);
+ if (!responseValidation.success) {
+ console.error(
+ "Invalid API response structure:",
+ responseValidation.error
+ );
+ return NextResponse.json(
+ { error: "Internal response formatting error" },
+ { status: 500 }
+ );
+ }
+
+ return NextResponse.json(responseValidation.data);
+ } catch (error) {
+ console.error("Transcription API error:", error);
+ return NextResponse.json(
+ {
+ error: "Internal server error",
+ message: "An unexpected error occurred during transcription",
+ },
+ { status: 500 }
+ );
+ }
+}
diff --git a/apps/web/src/components/editor-header.tsx b/apps/web/src/components/editor-header.tsx index 6db2543b..98aec2b9 100644 --- a/apps/web/src/components/editor-header.tsx +++ b/apps/web/src/components/editor-header.tsx @@ -180,7 +180,7 @@ function ExportButton() { className="flex items-center gap-1.5 bg-[#38BDF8] text-white rounded-md px-[0.1rem] py-[0.1rem] cursor-pointer hover:brightness-95 transition-all duration-200" onClick={handleExport} > - <div className="flex items-center gap-1.5 bg-linear-270 from-[#2567EC] to-[#37B6F7] rounded-[0.8rem] px-4 py-1 relative shadow-[0_1px_3px_0px_rgba(0,0,0,0.45)]"> + <div className="flex items-center gap-1.5 bg-linear-270 from-[#2567EC] to-[#37B6F7] rounded-[0.8rem] px-4 py-1 relative shadow-[0_1px_3px_0px_rgba(0,0,0,0.45)]"> <TransitionUpIcon className="z-50" /> <span className="text-[0.875rem] z-50">Export</span> <div className="absolute w-full h-full left-0 top-0 bg-linear-to-t from-white/0 to-white/50 z-10 rounded-[0.8rem] flex items-center justify-center"> diff --git a/apps/web/src/components/editor/media-panel/views/base-view.tsx b/apps/web/src/components/editor/media-panel/views/base-view.tsx index 296295fc..2cd33535 100644 --- a/apps/web/src/components/editor/media-panel/views/base-view.tsx +++ b/apps/web/src/components/editor/media-panel/views/base-view.tsx @@ -11,12 +11,19 @@ interface BaseViewProps { content: React.ReactNode;
}[];
className?: string;
+ ref?: React.RefObject<HTMLDivElement>;
}
-function ViewContent({ children }: { children: React.ReactNode }) {
+function ViewContent({
+ children,
+ className,
+}: {
+ children: React.ReactNode;
+ className?: string;
+}) {
return (
<ScrollArea className="flex-1">
- <div className="p-5">{children}</div>
+ <div className={`p-5 h-full ${className}`}>{children}</div>
</ScrollArea>
);
}
@@ -26,11 +33,12 @@ export function BaseView({ defaultTab,
tabs,
className = "",
+ ref,
}: BaseViewProps) {
return (
- <div className={`h-full flex flex-col ${className}`}>
+ <div className={`h-full flex flex-col ${className}`} ref={ref}>
{!tabs || tabs.length === 0 ? (
- <ViewContent>{children}</ViewContent>
+ <ViewContent className={className}>{children}</ViewContent>
) : (
<Tabs defaultValue={defaultTab} className="flex flex-col h-full">
<div className="px-3 pt-4 pb-0">
diff --git a/apps/web/src/components/editor/media-panel/views/captions.tsx b/apps/web/src/components/editor/media-panel/views/captions.tsx index ef919fec..4fe2cbd7 100644 --- a/apps/web/src/components/editor/media-panel/views/captions.tsx +++ b/apps/web/src/components/editor/media-panel/views/captions.tsx @@ -1,9 +1,313 @@ -import { BaseView } from "./base-view";
-
-export function Captions() {
- return (
- <BaseView>
- <div>Captions</div>
- </BaseView>
- );
-}
+import { Button } from "@/components/ui/button"; +import { PropertyGroup } from "../../properties-panel/property-item"; +import { BaseView } from "./base-view"; +import { Language, LanguageSelect } from "@/components/language-select"; +import { useState, useRef, useEffect } from "react"; +import { extractTimelineAudio } from "@/lib/ffmpeg-utils"; +import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { Loader2, Shield, Trash2, Upload } from "lucide-react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { TextElement } from "@/types/timeline"; + +export const languages: Language[] = [ + { code: "US", name: "English" }, + { code: "ES", name: "Spanish" }, + { code: "IT", name: "Italian" }, + { code: "FR", name: "French" }, + { code: "DE", name: "German" }, + { code: "PT", name: "Portuguese" }, + { code: "RU", name: "Russian" }, + { code: "JP", name: "Japanese" }, + { code: "CN", name: "Chinese" }, +]; + +const PRIVACY_DIALOG_KEY = "opencut-transcription-privacy-accepted"; + +export function Captions() { + const [selectedCountry, setSelectedCountry] = useState("auto"); + const [isProcessing, setIsProcessing] = useState(false); + const [processingStep, setProcessingStep] = useState<string>(""); + const [error, setError] = useState<string | null>(null); + const [showPrivacyDialog, setShowPrivacyDialog] = useState(false); + const [hasAcceptedPrivacy, setHasAcceptedPrivacy] = useState(false); + const containerRef = useRef<HTMLDivElement>(null); + const { insertTrackAt, addElementToTrack } = useTimelineStore(); + + // Check if user has already accepted privacy on mount + useEffect(() => { + const hasAccepted = localStorage.getItem(PRIVACY_DIALOG_KEY) === "true"; + setHasAcceptedPrivacy(hasAccepted); + }, []); + + const handleGenerateTranscript = async () => { + try { + setIsProcessing(true); + setError(null); + setProcessingStep("Extracting audio..."); + + const audioBlob = await extractTimelineAudio(); + + setProcessingStep("Encrypting audio..."); + + // Encrypt the audio with a random key (zero-knowledge) + const audioBuffer = await audioBlob.arrayBuffer(); + const encryptionResult = await encryptWithRandomKey(audioBuffer); + + // Convert encrypted data to blob for upload + const encryptedBlob = new Blob([encryptionResult.encryptedData]); + + setProcessingStep("Uploading..."); + const uploadResponse = await fetch("/api/get-upload-url", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ fileExtension: "wav" }), + }); + + if (!uploadResponse.ok) { + const error = await uploadResponse.json(); + throw new Error(error.message || "Failed to get upload URL"); + } + + const { uploadUrl, fileName } = await uploadResponse.json(); + + // Upload to R2 + await fetch(uploadUrl, { + method: "PUT", + body: encryptedBlob, + }); + + setProcessingStep("Transcribing..."); + + // Call Modal transcription API with encryption parameters + const transcriptionResponse = await fetch("/api/transcribe", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + filename: fileName, + language: + selectedCountry === "auto" ? "auto" : selectedCountry.toLowerCase(), + // Send the raw encryption key and IV (zero-knowledge) + decryptionKey: arrayBufferToBase64(encryptionResult.key), + iv: arrayBufferToBase64(encryptionResult.iv), + }), + }); + + if (!transcriptionResponse.ok) { + const error = await transcriptionResponse.json(); + throw new Error(error.message || "Transcription failed"); + } + + const { text, segments } = await transcriptionResponse.json(); + + console.log("Transcription completed:", { text, segments }); + + const shortCaptions: Array<{ + text: string; + startTime: number; + duration: number; + }> = []; + + let globalEndTime = 0; // Track the end time of the last caption globally + + segments.forEach((segment: any) => { + const words = segment.text.trim().split(/\s+/); + const segmentDuration = segment.end - segment.start; + const wordsPerSecond = words.length / segmentDuration; + + // Split into chunks of 2-4 words + const chunks: string[] = []; + for (let i = 0; i < words.length; i += 3) { + chunks.push(words.slice(i, i + 3).join(" ")); + } + + // Calculate timing for each chunk to place them sequentially + let chunkStartTime = segment.start; + chunks.forEach((chunk) => { + const chunkWords = chunk.split(/\s+/).length; + const chunkDuration = Math.max(0.8, chunkWords / wordsPerSecond); // Minimum 0.8s per chunk + + let adjustedStartTime = chunkStartTime; + + // Prevent overlapping: if this caption would start before the last one ends, + // start it right after the last one ends + if (adjustedStartTime < globalEndTime) { + adjustedStartTime = globalEndTime; + } + + shortCaptions.push({ + text: chunk, + startTime: adjustedStartTime, + duration: chunkDuration, + }); + + // Update global end time + globalEndTime = adjustedStartTime + chunkDuration; + + // Next chunk starts when this one ends (for within-segment timing) + chunkStartTime += chunkDuration; + }); + }); + + // Create a single track for all captions + const captionTrackId = insertTrackAt("text", 0); + + // Add all caption elements to the same track + shortCaptions.forEach((caption, index) => { + addElementToTrack(captionTrackId, { + type: "text", + name: `Caption ${index + 1}`, + content: caption.text, + duration: caption.duration, + startTime: caption.startTime, + trimStart: 0, + trimEnd: 0, + fontSize: 65, + fontFamily: "Arial", + color: "#ffffff", + textAlign: "center", + fontWeight: "bold", + fontStyle: "normal", + textDecoration: "none", + x: 0, + y: 0, + rotation: 0, + opacity: 1, + } as TextElement); + }); + + console.log( + `✅ ${shortCaptions.length} short-form caption chunks added to timeline!` + ); + } catch (error) { + console.error("Transcription failed:", error); + setError( + error instanceof Error ? error.message : "An unexpected error occurred" + ); + } finally { + setIsProcessing(false); + setProcessingStep(""); + } + }; + + return ( + <BaseView ref={containerRef} className="flex flex-col justify-between"> + <PropertyGroup title="Language"> + <LanguageSelect + selectedCountry={selectedCountry} + onSelect={setSelectedCountry} + containerRef={containerRef} + languages={languages} + /> + </PropertyGroup> + + <div className="flex flex-col gap-4"> + {error && ( + <div className="p-3 bg-destructive/10 border border-destructive/20 rounded-md"> + <p className="text-sm text-destructive">{error}</p> + </div> + )} + + <Button + className="w-full" + onClick={() => { + if (hasAcceptedPrivacy) { + handleGenerateTranscript(); + } else { + setShowPrivacyDialog(true); + } + }} + disabled={isProcessing} + > + {isProcessing && <Loader2 className="mr-1 h-4 w-4 animate-spin" />} + {isProcessing ? processingStep : "Generate transcript"} + </Button> + + <Dialog open={showPrivacyDialog} onOpenChange={setShowPrivacyDialog}> + <DialogContent className="sm:max-w-md"> + <DialogHeader> + <DialogTitle className="flex items-center gap-2"> + <Shield className="h-5 w-5" /> + Audio Processing Notice + </DialogTitle> + <DialogDescription className="space-y-3"> + <p> + To generate captions, we need to process your timeline audio + using speech-to-text technology. + </p> + + <div className="space-y-2 pt-2"> + <div className="flex items-start gap-2"> + <Shield className="h-4 w-4 flex-shrink-0" /> + <span className="text-sm"> + Zero-knowledge encryption - we cannot decrypt your files + even if we wanted to + </span> + </div> + + <div className="flex items-start gap-2"> + <Shield className="h-4 w-4 flex-shrink-0" /> + <span className="text-sm"> + Encryption keys generated randomly in your browser, never + stored anywhere + </span> + </div> + + <div className="flex items-start gap-2"> + <Upload className="h-4 w-4 flex-shrink-0" /> + <span className="text-sm"> + Audio encrypted before upload - raw audio never leaves + your device + </span> + </div> + + <div className="flex items-start gap-2"> + <Trash2 className="h-4 w-4 flex-shrink-0" /> + <span className="text-sm"> + Everything permanently deleted within seconds after + transcription + </span> + </div> + </div> + + <p className="text-xs text-muted-foreground"> + <strong>True zero-knowledge privacy:</strong> Encryption keys + are generated randomly in your browser and never stored + anywhere. It's cryptographically impossible for us, our cloud + providers, or anyone else to decrypt your audio files. + </p> + </DialogDescription> + </DialogHeader> + <DialogFooter className="gap-2"> + <Button + variant="outline" + onClick={() => setShowPrivacyDialog(false)} + disabled={isProcessing} + > + Cancel + </Button> + <Button + onClick={() => { + localStorage.setItem(PRIVACY_DIALOG_KEY, "true"); + setHasAcceptedPrivacy(true); + setShowPrivacyDialog(false); + handleGenerateTranscript(); + }} + disabled={isProcessing} + > + Continue & Generate Captions + </Button> + </DialogFooter> + </DialogContent> + </Dialog> + </div> + </BaseView> + ); +} diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsx index a47e446a..785162a6 100644 --- a/apps/web/src/components/editor/preview-panel.tsx +++ b/apps/web/src/components/editor/preview-panel.tsx @@ -356,7 +356,7 @@ export function PreviewPanel() { return ( <div key={element.id} - className="absolute flex items-center justify-center cursor-grab" + className="absolute cursor-grab" onMouseDown={(e) => handleTextMouseDown(e, element, elementData.track.id) } @@ -377,7 +377,7 @@ export function PreviewPanel() { canvasSize.height) * 100 }%`, - transform: `translate(-50%, -50%) rotate(${element.rotation}deg) scale(${scaleRatio})`, + transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`, opacity: element.opacity, zIndex: 100 + index, // Text elements on top }} @@ -385,16 +385,16 @@ export function PreviewPanel() { <div className={fontClassName} style={{ - fontSize: `${element.fontSize}px`, + fontSize: `${element.fontSize * scaleRatio}px`, color: element.color, backgroundColor: element.backgroundColor, textAlign: element.textAlign, fontWeight: element.fontWeight, fontStyle: element.fontStyle, textDecoration: element.textDecoration, - padding: "4px 8px", - borderRadius: "2px", - whiteSpace: "pre-wrap", + padding: `${4 * scaleRatio}px ${8 * scaleRatio}px`, + borderRadius: `${2 * scaleRatio}px`, + whiteSpace: "nowrap", // Fallback for system fonts that don't have classes ...(fontClassName === "" && { fontFamily: element.fontFamily }), }} diff --git a/apps/web/src/components/language-select.tsx b/apps/web/src/components/language-select.tsx new file mode 100644 index 00000000..43ed632d --- /dev/null +++ b/apps/web/src/components/language-select.tsx @@ -0,0 +1,204 @@ +import { useState, useRef, useEffect } from "react";
+import { ChevronDown, Globe } from "lucide-react";
+import { cn } from "@/lib/utils";
+import { motion } from "framer-motion";
+import ReactCountryFlag from "react-country-flag";
+
+export interface Language {
+ code: string;
+ name: string;
+ flag?: string;
+}
+
+interface LanguageSelectProps {
+ selectedCountry: string;
+ onSelect: (country: string) => void;
+ containerRef: React.RefObject<HTMLDivElement>;
+ languages: Language[];
+}
+
+function FlagPreloader({ languages }: { languages: Language[] }) {
+ return (
+ <div className="absolute -top-[9999px] left-0 pointer-events-none">
+ {languages.map((language) => (
+ <ReactCountryFlag
+ key={language.code}
+ countryCode={language.code}
+ svg
+ style={{ width: "1.05rem", height: "1.05rem" }}
+ />
+ ))}
+ </div>
+ );
+}
+
+export function LanguageSelect({
+ selectedCountry,
+ onSelect,
+ containerRef,
+ languages,
+}: LanguageSelectProps) {
+ const [expanded, setExpanded] = useState(false);
+ const [isTapping, setIsTapping] = useState(false);
+ const [isClosing, setIsClosing] = useState(false);
+ const collapsedHeight = "2.5rem";
+ const expandHeight = "12rem";
+ const buttonRef = useRef<HTMLButtonElement>(null);
+
+ const expand = () => {
+ setIsTapping(true);
+ setTimeout(() => setIsTapping(false), 600);
+ setExpanded(true);
+ buttonRef.current?.focus();
+ };
+
+ useEffect(() => {
+ if (!expanded) return;
+
+ const handleClickOutside = (event: MouseEvent) => {
+ if (
+ buttonRef.current &&
+ !buttonRef.current.contains(event.target as Node)
+ ) {
+ setIsClosing(true);
+ setTimeout(() => setIsClosing(false), 600);
+ setExpanded(false);
+ buttonRef.current?.blur();
+ }
+ };
+
+ document.addEventListener("mousedown", handleClickOutside);
+ return () => {
+ document.removeEventListener("mousedown", handleClickOutside);
+ };
+ }, [expanded]);
+
+ const selectedLanguage = languages.find(
+ (lang) => lang.code === selectedCountry
+ );
+
+ const handleSelect = ({
+ code,
+ e,
+ }: {
+ code: string;
+ e: React.MouseEvent<HTMLButtonElement>;
+ }) => {
+ e.stopPropagation();
+ e.preventDefault();
+ onSelect(code);
+ setExpanded(false);
+ };
+
+ return (
+ <div className="relative w-full h-9">
+ <FlagPreloader languages={languages} />
+ <motion.button
+ type="button"
+ className={cn(
+ "absolute w-full h-full flex flex-col overflow-hidden items-start justify-between z-10 rounded-lg px-3 cursor-pointer",
+ "!bg-foreground/10 backdrop-blur-md text-foreground py-0",
+ "transition-[color,box-shadow] focus:border-ring focus:ring-ring/50 focus:ring-[1px]"
+ )}
+ initial={{
+ height: collapsedHeight,
+ scale: 1,
+ }}
+ animate={{
+ height: expanded ? expandHeight : collapsedHeight,
+ scale: isTapping ? [1, 0.985, 1] : 1,
+ }}
+ transition={{
+ height: { duration: 0.25, ease: [0.4, 0, 0.2, 1] },
+ scale: { duration: 0.6, ease: "easeOut" },
+ }}
+ onClick={expand}
+ ref={buttonRef}
+ >
+ {!expanded ? (
+ <div
+ className="flex items-center justify-between w-full"
+ style={{
+ height: collapsedHeight,
+ }}
+ >
+ <div className="flex items-center gap-2">
+ {selectedCountry === "auto" ? (
+ <Globe className="!size-[1.05rem]" />
+ ) : (
+ <ReactCountryFlag
+ countryCode={selectedCountry}
+ svg
+ style={{ width: "1.05rem", height: "1.05rem" }}
+ />
+ )}
+ <span className="pt-[0.05rem]">
+ {selectedCountry === "auto" ? "Auto" : selectedLanguage?.name}
+ </span>
+ </div>
+ </div>
+ ) : (
+ <div className="flex flex-col gap-2 my-2.5 w-full overflow-y-auto scrollbar-hidden">
+ <LanguageButton
+ language={{ code: "auto", name: "Auto" }}
+ onSelect={handleSelect}
+ selectedCountry={selectedCountry}
+ />
+ {languages.map((language) => (
+ <LanguageButton
+ key={language.code}
+ language={language}
+ onSelect={handleSelect}
+ selectedCountry={selectedCountry}
+ />
+ ))}
+ </div>
+ )}
+ </motion.button>
+
+ <motion.div
+ className="absolute top-1/2 right-3 -translate-y-1/2 pointer-events-none z-20 mt-0.5"
+ initial={{ opacity: 1 }}
+ animate={{ opacity: expanded ? 0 : 1 }}
+ transition={{ duration: 0.2, ease: "easeInOut" }}
+ >
+ <ChevronDown className="text-muted-foreground size-4" />
+ </motion.div>
+ </div>
+ );
+}
+
+function LanguageButton({
+ language,
+ onSelect,
+ selectedCountry,
+}: {
+ language: Language;
+ onSelect: ({
+ code,
+ e,
+ }: {
+ code: string;
+ e: React.MouseEvent<HTMLButtonElement>;
+ }) => void;
+ selectedCountry: string;
+}) {
+ return (
+ <button
+ type="button"
+ className="flex items-center gap-2 cursor-pointer text-foreground hover:text-foreground/75"
+ onClick={(e) => onSelect({ code: language.code, e })}
+ >
+ {language.code === "auto" ? (
+ <Globe className="!size-[1.0rem]" />
+ ) : (
+ <ReactCountryFlag
+ countryCode={language.code}
+ svg
+ style={{ width: "1.05rem", height: "1.05rem" }}
+ />
+ )}
+ <span className="pt-[0.05rem]">{language.name}</span>
+ </button>
+ );
+}
diff --git a/apps/web/src/components/onboarding.tsx b/apps/web/src/components/onboarding.tsx index 36ae68d9..d72ac0f6 100644 --- a/apps/web/src/components/onboarding.tsx +++ b/apps/web/src/components/onboarding.tsx @@ -79,7 +79,7 @@ export function Onboarding() { return ( <Dialog open={isOpen} onOpenChange={handleClose}> - <DialogContent className="sm:max-w-[425px] outline-hidden!"> + <DialogContent className="sm:max-w-[425px] !outline-none"> <DialogTitle> <span className="sr-only">{getStepTitle()}</span> </DialogTitle> diff --git a/apps/web/src/components/ui/button.tsx b/apps/web/src/components/ui/button.tsx index d3656790..5558eee0 100644 --- a/apps/web/src/components/ui/button.tsx +++ b/apps/web/src/components/ui/button.tsx @@ -13,6 +13,8 @@ const buttonVariants = cva( "bg-foreground text-background shadow-sm hover:bg-foreground/90", primary: "bg-primary text-primary-foreground shadow-sm hover:bg-primary/90", + "primary-gradient": + "bg-gradient-to-r from-cyan-400 to-blue-500 text-white hover:opacity-85 transition-opacity", destructive: "bg-destructive text-destructive-foreground shadow-xs hover:bg-destructive/90", outline: diff --git a/apps/web/src/env.ts b/apps/web/src/env.ts index fd4ad523..71614300 100644 --- a/apps/web/src/env.ts +++ b/apps/web/src/env.ts @@ -17,6 +17,13 @@ export const env = createEnv({ UPSTASH_REDIS_REST_TOKEN: z.string(), FREESOUND_CLIENT_ID: z.string(), FREESOUND_API_KEY: z.string(), + // R2 / Cloudflare + CLOUDFLARE_ACCOUNT_ID: z.string(), + R2_ACCESS_KEY_ID: z.string(), + R2_SECRET_ACCESS_KEY: z.string(), + R2_BUCKET_NAME: z.string(), + // Modal transcription + MODAL_TRANSCRIPTION_URL: z.string(), }, client: {}, runtimeEnv: { @@ -27,5 +34,12 @@ export const env = createEnv({ UPSTASH_REDIS_REST_TOKEN: process.env.UPSTASH_REDIS_REST_TOKEN, FREESOUND_CLIENT_ID: process.env.FREESOUND_CLIENT_ID, FREESOUND_API_KEY: process.env.FREESOUND_API_KEY, + // R2 / Cloudflare + CLOUDFLARE_ACCOUNT_ID: process.env.CLOUDFLARE_ACCOUNT_ID, + R2_ACCESS_KEY_ID: process.env.R2_ACCESS_KEY_ID, + R2_SECRET_ACCESS_KEY: process.env.R2_SECRET_ACCESS_KEY, + R2_BUCKET_NAME: process.env.R2_BUCKET_NAME, + // Modal transcription + MODAL_TRANSCRIPTION_URL: process.env.MODAL_TRANSCRIPTION_URL, }, }); diff --git a/apps/web/src/lib/ffmpeg-utils.ts b/apps/web/src/lib/ffmpeg-utils.ts index c34200f6..0643ecb9 100644 --- a/apps/web/src/lib/ffmpeg-utils.ts +++ b/apps/web/src/lib/ffmpeg-utils.ts @@ -1,5 +1,7 @@ import { FFmpeg } from "@ffmpeg/ffmpeg"; import { toBlobURL } from "@ffmpeg/util"; +import { useTimelineStore } from "@/stores/timeline-store"; +import { useMediaStore } from "@/stores/media-store"; let ffmpeg: FFmpeg | null = null; @@ -7,14 +9,7 @@ export const initFFmpeg = async (): Promise<FFmpeg> => { if (ffmpeg) return ffmpeg; ffmpeg = new FFmpeg(); - - // Use locally hosted files instead of CDN - const baseURL = "/ffmpeg"; - - await ffmpeg.load({ - coreURL: await toBlobURL(`${baseURL}/ffmpeg-core.js`, "text/javascript"), - wasmURL: await toBlobURL(`${baseURL}/ffmpeg-core.wasm`, "application/wasm"), - }); + await ffmpeg.load(); // Use default config return ffmpeg; }; @@ -268,3 +263,206 @@ export const extractAudio = async ( return blob; }; + +export const extractTimelineAudio = async ( + onProgress?: (progress: number) => void +): Promise<Blob> => { + // Create fresh FFmpeg instance for this operation + const ffmpeg = new FFmpeg(); + + try { + await ffmpeg.load(); + } catch (error) { + console.error("Failed to load fresh FFmpeg instance:", error); + throw new Error("Unable to initialize audio processing. Please try again."); + } + + const timeline = useTimelineStore.getState(); + const mediaStore = useMediaStore.getState(); + + const tracks = timeline.tracks; + const totalDuration = timeline.getTotalDuration(); + + if (totalDuration === 0) { + const emptyAudioData = new ArrayBuffer(44); + return new Blob([emptyAudioData], { type: "audio/wav" }); + } + + if (onProgress) { + ffmpeg.on("progress", ({ progress }) => { + onProgress(progress * 100); + }); + } + + const audioElements: Array<{ + file: File; + startTime: number; + duration: number; + trimStart: number; + trimEnd: number; + trackMuted: boolean; + }> = []; + + for (const track of tracks) { + if (track.muted) continue; + + for (const element of track.elements) { + if (element.type === "media") { + const mediaItem = mediaStore.mediaItems.find( + (m) => m.id === element.mediaId + ); + if (!mediaItem) continue; + + if (mediaItem.type === "video" || mediaItem.type === "audio") { + audioElements.push({ + file: mediaItem.file, + startTime: element.startTime, + duration: element.duration, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + trackMuted: track.muted || false, + }); + } + } + } + } + + if (audioElements.length === 0) { + // Return silent audio if no audio elements + const silentDuration = Math.max(1, totalDuration); // At least 1 second + try { + const silentAudio = await generateSilentAudio(silentDuration); + return silentAudio; + } catch (error) { + console.error("Failed to generate silent audio:", error); + throw new Error("Unable to generate audio for empty timeline."); + } + } + + // Create a complex filter to mix all audio sources + const inputFiles: string[] = []; + const filterInputs: string[] = []; + + try { + for (let i = 0; i < audioElements.length; i++) { + const element = audioElements[i]; + const inputName = `input_${i}.${element.file.name.split(".").pop()}`; + inputFiles.push(inputName); + + try { + await ffmpeg.writeFile( + inputName, + new Uint8Array(await element.file.arrayBuffer()) + ); + } catch (error) { + console.error(`Failed to write file ${element.file.name}:`, error); + throw new Error( + `Unable to process file: ${element.file.name}. The file may be corrupted or in an unsupported format.` + ); + } + + const actualStart = element.trimStart; + const actualDuration = + element.duration - element.trimStart - element.trimEnd; + + const filterName = `audio_${i}`; + filterInputs.push( + `[${i}:a]atrim=start=${actualStart}:duration=${actualDuration},asetpts=PTS-STARTPTS,adelay=${element.startTime * 1000}|${element.startTime * 1000}[${filterName}]` + ); + } + + const mixFilter = + audioElements.length === 1 + ? `[audio_0]aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]` + : `${filterInputs.map((_, i) => `[audio_${i}]`).join("")}amix=inputs=${audioElements.length}:duration=longest:dropout_transition=2,aresample=44100,aformat=sample_fmts=s16:channel_layouts=stereo[out]`; + + const complexFilter = [...filterInputs, mixFilter].join(";"); + const outputName = "timeline_audio.wav"; + + const ffmpegArgs = [ + ...inputFiles.flatMap((name) => ["-i", name]), + "-filter_complex", + complexFilter, + "-map", + "[out]", + "-t", + totalDuration.toString(), + "-c:a", + "pcm_s16le", + "-ar", + "44100", + outputName, + ]; + + try { + await ffmpeg.exec(ffmpegArgs); + } catch (error) { + console.error("FFmpeg execution failed:", error); + throw new Error( + "Audio processing failed. Some audio files may be corrupted or incompatible." + ); + } + + const data = await ffmpeg.readFile(outputName); + const blob = new Blob([data], { type: "audio/wav" }); + + return blob; + } catch (error) { + for (const inputFile of inputFiles) { + try { + await ffmpeg.deleteFile(inputFile); + } catch (cleanupError) { + console.warn(`Failed to cleanup file ${inputFile}:`, cleanupError); + } + } + try { + await ffmpeg.deleteFile("timeline_audio.wav"); + } catch (cleanupError) { + console.warn("Failed to cleanup output file:", cleanupError); + } + + throw error; + } finally { + for (const inputFile of inputFiles) { + try { + await ffmpeg.deleteFile(inputFile); + } catch (cleanupError) {} + } + try { + await ffmpeg.deleteFile("timeline_audio.wav"); + } catch (cleanupError) {} + } +}; + +const generateSilentAudio = async (durationSeconds: number): Promise<Blob> => { + const ffmpeg = await initFFmpeg(); + const outputName = "silent.wav"; + + try { + await ffmpeg.exec([ + "-f", + "lavfi", + "-i", + `anullsrc=channel_layout=stereo:sample_rate=44100`, + "-t", + durationSeconds.toString(), + "-c:a", + "pcm_s16le", + outputName, + ]); + + const data = await ffmpeg.readFile(outputName); + const blob = new Blob([data], { type: "audio/wav" }); + + return blob; + } catch (error) { + console.error("Failed to generate silent audio:", error); + throw error; + } finally { + try { + await ffmpeg.deleteFile(outputName); + } catch (cleanupError) { + // Silent cleanup + } + } +}; diff --git a/apps/web/src/lib/transcription-utils.ts b/apps/web/src/lib/transcription-utils.ts new file mode 100644 index 00000000..6388a3b6 --- /dev/null +++ b/apps/web/src/lib/transcription-utils.ts @@ -0,0 +1,13 @@ +import { env } from "@/env";
+
+export function isTranscriptionConfigured() {
+ const missingVars = [];
+
+ if (!env.CLOUDFLARE_ACCOUNT_ID) missingVars.push("CLOUDFLARE_ACCOUNT_ID");
+ if (!env.R2_ACCESS_KEY_ID) missingVars.push("R2_ACCESS_KEY_ID");
+ if (!env.R2_SECRET_ACCESS_KEY) missingVars.push("R2_SECRET_ACCESS_KEY");
+ if (!env.R2_BUCKET_NAME) missingVars.push("R2_BUCKET_NAME");
+ if (!env.MODAL_TRANSCRIPTION_URL) missingVars.push("MODAL_TRANSCRIPTION_URL");
+
+ return { configured: missingVars.length === 0, missingVars };
+}
diff --git a/apps/web/src/lib/zk-encryption.ts b/apps/web/src/lib/zk-encryption.ts new file mode 100644 index 00000000..09e5e861 --- /dev/null +++ b/apps/web/src/lib/zk-encryption.ts @@ -0,0 +1,71 @@ +/**
+ * True zero-knowledge encryption utilities
+ * Keys are generated randomly in the browser and never derived from server secrets
+ */
+
+export interface ZeroKnowledgeEncryptionResult {
+ encryptedData: ArrayBuffer;
+ key: ArrayBuffer;
+ iv: ArrayBuffer;
+}
+
+/**
+ * Encrypt data with a randomly generated key (true zero-knowledge)
+ */
+export async function encryptWithRandomKey(
+ data: ArrayBuffer
+): Promise<ZeroKnowledgeEncryptionResult> {
+ // Generate a truly random 256-bit key
+ const key = crypto.getRandomValues(new Uint8Array(32));
+
+ // Generate random IV
+ const iv = crypto.getRandomValues(new Uint8Array(12));
+
+ // Import the key for encryption
+ const cryptoKey = await crypto.subtle.importKey(
+ "raw",
+ key,
+ { name: " " },
+ false,
+ ["encrypt"]
+ );
+
+ // Encrypt the data
+ const encryptedResult = await crypto.subtle.encrypt(
+ { name: "AES-GCM", iv },
+ cryptoKey,
+ data
+ );
+
+ // For AES-GCM, we need to append the authentication tag
+ // The encrypted result contains both ciphertext and tag
+ return {
+ encryptedData: encryptedResult,
+ key: key.buffer,
+ iv: iv.buffer,
+ };
+}
+
+/**
+ * Convert ArrayBuffer to base64 string for transmission
+ */
+export function arrayBufferToBase64(buffer: ArrayBuffer): string {
+ const bytes = new Uint8Array(buffer);
+ let binary = "";
+ for (let i = 0; i < bytes.byteLength; i++) {
+ binary += String.fromCharCode(bytes[i]);
+ }
+ return btoa(binary);
+}
+
+/**
+ * Convert base64 string back to ArrayBuffer
+ */
+export function base64ToArrayBuffer(base64: string): ArrayBuffer {
+ const binary = atob(base64);
+ const bytes = new Uint8Array(binary.length);
+ for (let i = 0; i < binary.length; i++) {
+ bytes[i] = binary.charCodeAt(i);
+ }
+ return bytes.buffer;
+}
|
