diff options
Diffstat (limited to 'apps/web/src/subtitles/components')
| -rw-r--r-- | apps/web/src/subtitles/components/assets-view.tsx | 337 |
1 files changed, 337 insertions, 0 deletions
diff --git a/apps/web/src/subtitles/components/assets-view.tsx b/apps/web/src/subtitles/components/assets-view.tsx new file mode 100644 index 00000000..7e2ddad9 --- /dev/null +++ b/apps/web/src/subtitles/components/assets-view.tsx @@ -0,0 +1,337 @@ +import { Button } from "@/components/ui/button"; +import { PanelView } from "@/components/editor/panels/assets/views/base-panel"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { useReducer, useRef, useState } from "react"; +import { extractTimelineAudio } from "@/media/mediabunny"; +import { useEditor } from "@/editor/use-editor"; +import { TRANSCRIPTION_DIAGNOSTICS_SCOPE } from "@/transcription/diagnostics"; +import { DEFAULT_TRANSCRIPTION_SAMPLE_RATE } from "@/transcription/audio"; +import { TRANSCRIPTION_LANGUAGES } from "@/transcription/supported-languages"; +import type { + CaptionChunk, + TranscriptionLanguage, + TranscriptionProgress, +} from "@/transcription/types"; +import { transcriptionService } from "@/services/transcription/service"; +import { decodeAudioToFloat32 } from "@/media/audio"; +import { buildCaptionChunks } from "@/transcription/caption"; +import { insertCaptionChunksAsTextTrack } from "@/subtitles/insert"; +import { parseSubtitleFile } from "@/subtitles/parse"; +import { Spinner } from "@/components/ui/spinner"; +import { + Section, + SectionContent, + SectionField, + SectionFields, +} from "@/components/section"; +import { AlertCircleIcon, CloudUploadIcon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import type { DiagnosticSeverity } from "@/diagnostics/types"; + +const DIAGNOSTIC_BUTTON_VARIANT: Record< + DiagnosticSeverity, + "caution" | "destructive-foreground" +> = { + caution: "caution", + error: "destructive-foreground", +}; + +type ProcessingState = + | { status: "idle"; error: string | null; warnings: string[] } + | { status: "processing"; step: string }; + +type ProcessingAction = + | { type: "start"; step: string } + | { type: "update_step"; step: string } + | { type: "succeed"; warnings: string[] } + | { type: "fail"; error: string }; + +const IDLE_STATE: ProcessingState = { + status: "idle", + error: null, + warnings: [], +}; + +function processingReducer( + state: ProcessingState, + action: ProcessingAction, +): ProcessingState { + switch (action.type) { + case "start": + return { status: "processing", step: action.step }; + case "update_step": + if (state.status !== "processing") return state; + return { status: "processing", step: action.step }; + case "succeed": + return { status: "idle", error: null, warnings: action.warnings }; + case "fail": + return { status: "idle", error: action.error, warnings: [] }; + } +} + +export function Captions() { + const [selectedLanguage, setSelectedLanguage] = + useState<TranscriptionLanguage>("auto"); + const [processing, dispatch] = useReducer(processingReducer, IDLE_STATE); + const containerRef = useRef<HTMLDivElement>(null); + const fileInputRef = useRef<HTMLInputElement>(null); + const editor = useEditor(); + + const isProcessing = processing.status === "processing"; + + const activeDiagnostics = useEditor((e) => + e.diagnostics.getActive({ scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE }), + ); + + const handleProgress = (progress: TranscriptionProgress) => { + if (progress.status === "loading-model") { + dispatch({ + type: "update_step", + step: `Loading model ${Math.round(progress.progress)}%`, + }); + } else if (progress.status === "transcribing") { + dispatch({ type: "update_step", step: "Transcribing..." }); + } + }; + + const insertCaptions = ({ + captions, + }: { + captions: CaptionChunk[]; + }): boolean => { + const trackId = insertCaptionChunksAsTextTrack({ editor, captions }); + return trackId !== null; + }; + + const handleGenerateTranscript = async () => { + dispatch({ type: "start", step: "Extracting audio..." }); + try { + const audioBlob = await extractTimelineAudio({ + tracks: editor.scenes.getActiveScene().tracks, + mediaAssets: editor.media.getAssets(), + totalDuration: editor.timeline.getTotalDuration(), + }); + + dispatch({ type: "update_step", step: "Preparing audio..." }); + const { samples } = await decodeAudioToFloat32({ + audioBlob, + sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE, + }); + + const result = await transcriptionService.transcribe({ + audioData: samples, + language: selectedLanguage === "auto" ? undefined : selectedLanguage, + onProgress: handleProgress, + }); + + dispatch({ type: "update_step", step: "Generating captions..." }); + const captionChunks = buildCaptionChunks({ segments: result.segments }); + + if (!insertCaptions({ captions: captionChunks })) { + dispatch({ type: "fail", error: "No captions were generated" }); + return; + } + + dispatch({ type: "succeed", warnings: [] }); + } catch (error) { + console.error("Transcription failed:", error); + dispatch({ + type: "fail", + error: + error instanceof Error + ? error.message + : "An unexpected error occurred", + }); + } + }; + + const handleImportClick = () => { + fileInputRef.current?.click(); + }; + + const handleImportFile = async ({ file }: { file: File }) => { + dispatch({ type: "start", step: "Reading subtitle file..." }); + try { + const input = await file.text(); + const result = parseSubtitleFile({ + fileName: file.name, + input, + }); + + if (result.captions.length === 0) { + dispatch({ + type: "fail", + error: "No valid subtitle cues were found in the subtitle file", + }); + return; + } + + dispatch({ type: "update_step", step: "Importing subtitles..." }); + + if (!insertCaptions({ captions: result.captions })) { + dispatch({ type: "fail", error: "No captions were generated" }); + return; + } + + const nextWarnings = [...result.warnings]; + if (result.skippedCueCount > 0) { + nextWarnings.unshift( + `Imported ${result.captions.length} subtitle cue(s) and skipped ${result.skippedCueCount} malformed cue(s).`, + ); + } + + dispatch({ type: "succeed", warnings: nextWarnings }); + } catch (error) { + console.error("Subtitle import failed:", error); + dispatch({ + type: "fail", + error: + error instanceof Error + ? error.message + : "An unexpected error occurred", + }); + } + }; + + const handleFileChange = async ({ + event, + }: { + event: React.ChangeEvent<HTMLInputElement>; + }) => { + const file = event.target.files?.[0]; + if (event.target) { + event.target.value = ""; + } + if (!file) return; + + await handleImportFile({ file }); + }; + + const handleLanguageChange = ({ value }: { value: string }) => { + if (value === "auto") { + setSelectedLanguage("auto"); + return; + } + + const matchedLanguage = TRANSCRIPTION_LANGUAGES.find( + (language) => language.code === value, + ); + if (!matchedLanguage) return; + setSelectedLanguage(matchedLanguage.code); + }; + + const error = processing.status === "idle" ? processing.error : null; + const warnings = processing.status === "idle" ? processing.warnings : []; + + return ( + <PanelView + title="Captions" + contentClassName="px-0 flex flex-col h-full" + actions={ + <TooltipProvider> + <div className="flex items-center gap-1.5"> + {!isProcessing && + activeDiagnostics.map((diagnostic) => ( + <Tooltip key={diagnostic.id}> + <TooltipTrigger asChild> + <Button + variant={DIAGNOSTIC_BUTTON_VARIANT[diagnostic.severity]} + size="icon" + aria-label={diagnostic.message} + > + <HugeiconsIcon icon={AlertCircleIcon} size={16} /> + </Button> + </TooltipTrigger> + <TooltipContent>{diagnostic.message}</TooltipContent> + </Tooltip> + ))} + <Button + type="button" + variant="outline" + size="sm" + onClick={handleImportClick} + disabled={isProcessing} + className="items-center justify-center gap-1.5" + > + <HugeiconsIcon icon={CloudUploadIcon} /> + Import + </Button> + </div> + </TooltipProvider> + } + ref={containerRef} + > + <input + ref={fileInputRef} + type="file" + accept=".srt,.ass" + className="hidden" + onChange={(event) => void handleFileChange({ event })} + /> + <Section + showTopBorder={false} + showBottomBorder={false} + className="flex-1" + > + <SectionContent className="flex flex-col gap-4 h-full pt-1"> + <SectionFields> + <SectionField label="Language"> + <Select + value={selectedLanguage} + onValueChange={(value) => handleLanguageChange({ value })} + > + <SelectTrigger> + <SelectValue placeholder="Select a language" /> + </SelectTrigger> + <SelectContent> + <SelectItem value="auto">Auto detect</SelectItem> + {TRANSCRIPTION_LANGUAGES.map((language) => ( + <SelectItem key={language.code} value={language.code}> + {language.name} + </SelectItem> + ))} + </SelectContent> + </Select> + </SectionField> + </SectionFields> + + <Button + type="button" + className="mt-auto w-full" + onClick={handleGenerateTranscript} + disabled={isProcessing || activeDiagnostics.length > 0} + > + {isProcessing && <Spinner className="mr-1" />} + {isProcessing ? processing.step : "Generate transcript"} + </Button> + {error && ( + <div className="bg-destructive/10 border-destructive/20 rounded-md border p-3"> + <p className="text-destructive text-sm">{error}</p> + </div> + )} + {warnings.length > 0 && ( + <div className="rounded-md border border-amber-500/20 bg-amber-500/10 p-3"> + <ul className="space-y-1 text-sm text-amber-700"> + {warnings.map((warning) => ( + <li key={warning}>{warning}</li> + ))} + </ul> + </div> + )} + </SectionContent> + </Section> + </PanelView> + ); +} |