import { Button } from "@/components/ui/button"; import { PanelView } from "@/components/editor/panels/assets/views/base-panel"; import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select"; import { useReducer, useRef, useState } from "react"; import { extractTimelineAudio } from "@/media/mediabunny"; import { useEditor } from "@/editor/use-editor"; import { TRANSCRIPTION_DIAGNOSTICS_SCOPE } from "@/transcription/diagnostics"; import { DEFAULT_TRANSCRIPTION_SAMPLE_RATE } from "@/transcription/audio"; import { TRANSCRIPTION_LANGUAGES } from "@/transcription/supported-languages"; import type { CaptionChunk, TranscriptionLanguage, TranscriptionProgress, } from "@/transcription/types"; import { transcriptionService } from "@/services/transcription/service"; import { decodeAudioToFloat32 } from "@/media/audio"; import { buildCaptionChunks } from "@/transcription/caption"; import { insertCaptionChunksAsTextTrack } from "@/subtitles/insert"; import { parseSubtitleFile } from "@/subtitles/parse"; import { Spinner } from "@/components/ui/spinner"; import { Section, SectionContent, SectionField, SectionFields, } from "@/components/section"; import { AlertCircleIcon, CloudUploadIcon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger, } from "@/components/ui/tooltip"; import type { DiagnosticSeverity } from "@/diagnostics/types"; const DIAGNOSTIC_BUTTON_VARIANT: Record< DiagnosticSeverity, "caution" | "destructive-foreground" > = { caution: "caution", error: "destructive-foreground", }; type ProcessingState = | { status: "idle"; error: string | null; warnings: string[] } | { status: "processing"; step: string }; type ProcessingAction = | { type: "start"; step: string } | { type: "update_step"; step: string } | { type: "succeed"; warnings: string[] } | { type: "fail"; error: string }; const IDLE_STATE: ProcessingState = { status: "idle", error: null, warnings: [], }; function processingReducer( state: ProcessingState, action: ProcessingAction, ): ProcessingState { switch (action.type) { case "start": return { status: "processing", step: action.step }; case "update_step": if (state.status !== "processing") return state; return { status: "processing", step: action.step }; case "succeed": return { status: "idle", error: null, warnings: action.warnings }; case "fail": return { status: "idle", error: action.error, warnings: [] }; } } export function Captions() { const [selectedLanguage, setSelectedLanguage] = useState("auto"); const [processing, dispatch] = useReducer(processingReducer, IDLE_STATE); const containerRef = useRef(null); const fileInputRef = useRef(null); const editor = useEditor(); const isProcessing = processing.status === "processing"; const activeDiagnostics = useEditor((e) => e.diagnostics.getActive({ scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE }), ); const handleProgress = (progress: TranscriptionProgress) => { if (progress.status === "loading-model") { dispatch({ type: "update_step", step: `Loading model ${Math.round(progress.progress)}%`, }); } else if (progress.status === "transcribing") { dispatch({ type: "update_step", step: "Transcribing..." }); } }; const insertCaptions = ({ captions, }: { captions: CaptionChunk[]; }): boolean => { const trackId = insertCaptionChunksAsTextTrack({ editor, captions }); return trackId !== null; }; const handleGenerateTranscript = async () => { dispatch({ type: "start", step: "Extracting audio..." }); try { const audioBlob = await extractTimelineAudio({ tracks: editor.scenes.getActiveScene().tracks, mediaAssets: editor.media.getAssets(), totalDuration: editor.timeline.getTotalDuration(), }); dispatch({ type: "update_step", step: "Preparing audio..." }); const { samples } = await decodeAudioToFloat32({ audioBlob, sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE, }); const result = await transcriptionService.transcribe({ audioData: samples, language: selectedLanguage === "auto" ? undefined : selectedLanguage, onProgress: handleProgress, }); dispatch({ type: "update_step", step: "Generating captions..." }); const captionChunks = buildCaptionChunks({ segments: result.segments }); if (!insertCaptions({ captions: captionChunks })) { dispatch({ type: "fail", error: "No captions were generated" }); return; } dispatch({ type: "succeed", warnings: [] }); } catch (error) { console.error("Transcription failed:", error); dispatch({ type: "fail", error: error instanceof Error ? error.message : "An unexpected error occurred", }); } }; const handleImportClick = () => { fileInputRef.current?.click(); }; const handleImportFile = async ({ file }: { file: File }) => { dispatch({ type: "start", step: "Reading subtitle file..." }); try { const input = await file.text(); const result = parseSubtitleFile({ fileName: file.name, input, }); if (result.captions.length === 0) { dispatch({ type: "fail", error: "No valid subtitle cues were found in the subtitle file", }); return; } dispatch({ type: "update_step", step: "Importing subtitles..." }); if (!insertCaptions({ captions: result.captions })) { dispatch({ type: "fail", error: "No captions were generated" }); return; } const nextWarnings = [...result.warnings]; if (result.skippedCueCount > 0) { nextWarnings.unshift( `Imported ${result.captions.length} subtitle cue(s) and skipped ${result.skippedCueCount} malformed cue(s).`, ); } dispatch({ type: "succeed", warnings: nextWarnings }); } catch (error) { console.error("Subtitle import failed:", error); dispatch({ type: "fail", error: error instanceof Error ? error.message : "An unexpected error occurred", }); } }; const handleFileChange = async ({ event, }: { event: React.ChangeEvent; }) => { const file = event.target.files?.[0]; if (event.target) { event.target.value = ""; } if (!file) return; await handleImportFile({ file }); }; const handleLanguageChange = ({ value }: { value: string }) => { if (value === "auto") { setSelectedLanguage("auto"); return; } const matchedLanguage = TRANSCRIPTION_LANGUAGES.find( (language) => language.code === value, ); if (!matchedLanguage) return; setSelectedLanguage(matchedLanguage.code); }; const error = processing.status === "idle" ? processing.error : null; const warnings = processing.status === "idle" ? processing.warnings : []; return (
{!isProcessing && activeDiagnostics.map((diagnostic) => ( {diagnostic.message} ))}
} ref={containerRef} > void handleFileChange({ event })} />
{error && (

{error}

)} {warnings.length > 0 && (
    {warnings.map((warning) => (
  • {warning}
  • ))}
)}
); }