Add support for basic SRT subtitle import to Captions panel (#756)

* Add SRT subtitle import to the captions panel

[Modified from 0bd641b4102c827ef8d218ada099229f73396077, 2026-03-28 08:46:23 -0400]

Parse .srt files into chunks and insert them as text elements on a new text track using the existing caption styling defaults. Displays a warning when malformed cues are skipped during import.

* Batch caption text insertion

[Modified from eece3bbc5f0578a24a4f90f0ab87fe5ca37e94e9, 2026-03-28 09:13:18 -0400]

Batch caption insertion in a BatchCommand so that undo will batch undo all subtitle insertion from either srt file or ai transcriptions.

* Move subtitle parsing into a shared interface

[Modified from e84404b7857f8c611eda7a65b5d61ee6c9823d75, 2026-03-28 09:52:31 -0400]

Extracted the srt specific extraction behind a generic shared interface to allow easier adding of other subtitle formats in the future.

* Apply better default styling for subtitles.

[Modified from b5deec4285f99873ba5bf57971f7bcab26e8c743, 2026-03-31 11:49:34 -0400]

Shrink font size, aim to layout 5% above bottom, keep subtitle width 80% of video, wrap text on new lines if too long.

* Update Captions Panel UI

Split captions panel into Generate and Import tabs as they are two different approaches for adding captions.

* Add biome formatting fixes

* Add .ass subtitle import support

Extend subtitle import to accept .ass files alongside .srt, map supported .ass styles onto imported text elements, and display warnings when unsupported .ass features are skipped.

* Change “Import subtitles” button style

* refactor: review fixes

* fix: convert subtitle times from seconds to ticks

* refactor: replace captions tab switcher with import action button

---------

Co-authored-by: Maze Winther <mazewinther@gmail.com>

1 files changed, 282 insertions, 179 deletions

diff --git a/apps/web/src/components/editor/panels/assets/views/captions.tsx b/apps/web/src/components/editor/panels/assets/views/captions.tsx
index c89e587b..488875a8 100644
--- a/apps/web/src/components/editor/panels/assets/views/captions.tsx
+++ b/apps/web/src/components/editor/panels/assets/views/captions.tsx
@@ -1,179 +1,282 @@
-import { Button } from "@/components/ui/button";

-import { PanelView } from "@/components/editor/panels/assets/views/base-panel";

-import {

-	Select,

-	SelectContent,

-	SelectItem,

-	SelectTrigger,

-	SelectValue,

-} from "@/components/ui/select";

-import { useState, useRef } from "react";

-import { extractTimelineAudio } from "@/lib/media/mediabunny";

-import { useEditor } from "@/hooks/use-editor";

-import {

-	DEFAULT_TRANSCRIPTION_SAMPLE_RATE,

-	TRANSCRIPTION_LANGUAGES,

-} from "@/constants/transcription-constants";

-import type {

-	TranscriptionLanguage,

-	TranscriptionProgress,

-} from "@/lib/transcription/types";

-import { transcriptionService } from "@/services/transcription/service";

-import { decodeAudioToFloat32 } from "@/lib/media/audio";

-import { buildCaptionChunks } from "@/lib/transcription/caption";

-import { Spinner } from "@/components/ui/spinner";

-import {

-	Section,

-	SectionContent,

-	SectionField,

-	SectionFields,

-} from "@/components/section";

-import { DEFAULTS } from "@/lib/timeline/defaults";

-import {

-	AddTrackCommand,

-	BatchCommand,

-	InsertElementCommand,

-} from "@/lib/commands";

-

-export function Captions() {

-	const [selectedLanguage, setSelectedLanguage] =

-		useState<TranscriptionLanguage>("auto");

-	const [isProcessing, setIsProcessing] = useState(false);

-	const [processingStep, setProcessingStep] = useState("");

-	const [error, setError] = useState<string | null>(null);

-	const containerRef = useRef<HTMLDivElement>(null);

-	const editor = useEditor();

-

-	const handleProgress = (progress: TranscriptionProgress) => {

-		if (progress.status === "loading-model") {

-			setProcessingStep(`Loading model ${Math.round(progress.progress)}%`);

-		} else if (progress.status === "transcribing") {

-			setProcessingStep("Transcribing...");

-		}

-	};

-

-	const handleGenerateTranscript = async () => {

-		try {

-			setIsProcessing(true);

-			setError(null);

-			setProcessingStep("Extracting audio...");

-

-			const audioBlob = await extractTimelineAudio({

-				tracks: editor.scenes.getActiveScene().tracks,

-				mediaAssets: editor.media.getAssets(),

-				totalDuration: editor.timeline.getTotalDuration(),

-			});

-

-			setProcessingStep("Preparing audio...");

-			const { samples } = await decodeAudioToFloat32({

-				audioBlob,

-				sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE,

-			});

-

-			const result = await transcriptionService.transcribe({

-				audioData: samples,

-				language: selectedLanguage === "auto" ? undefined : selectedLanguage,

-				onProgress: handleProgress,

-			});

-

-			setProcessingStep("Generating captions...");

-			const captionChunks = buildCaptionChunks({ segments: result.segments });

-

-			const addTrackCommand = new AddTrackCommand("text", 0);

-			const insertCommands = captionChunks.map(

-				(caption, i) =>

-					new InsertElementCommand({

-						placement: {

-							mode: "explicit",

-							trackId: addTrackCommand.getTrackId(),

-						},

-						element: {

-							...DEFAULTS.text.element,

-							name: `Caption ${i + 1}`,

-							content: caption.text,

-							duration: caption.duration,

-							startTime: caption.startTime,

-							fontSize: 65,

-							fontWeight: "bold",

-						},

-					}),

-			);

-

-			editor.command.execute({

-				command: new BatchCommand([addTrackCommand, ...insertCommands]),

-			});

-		} catch (error) {

-			console.error("Transcription failed:", error);

-			setError(

-				error instanceof Error ? error.message : "An unexpected error occurred",

-			);

-		} finally {

-			setIsProcessing(false);

-			setProcessingStep("");

-		}

-	};

-

-	const handleLanguageChange = ({ value }: { value: string }) => {

-		if (value === "auto") {

-			setSelectedLanguage("auto");

-			return;

-		}

-

-		const matchedLanguage = TRANSCRIPTION_LANGUAGES.find(

-			(language) => language.code === value,

-		);

-		if (!matchedLanguage) return;

-		setSelectedLanguage(matchedLanguage.code);

-	};

-

-	return (

-		<PanelView

-			title="Captions"

-			contentClassName="px-0 flex flex-col h-full"

-			ref={containerRef}

-		>

-			<Section showTopBorder={false} showBottomBorder={false} className="flex-1">

-				<SectionContent className="flex flex-col gap-4 h-full pt-1">

-					<SectionFields>

-						<SectionField label="Language">

-							<Select

-								value={selectedLanguage}

-								onValueChange={(value) => handleLanguageChange({ value })}

-							>

-								<SelectTrigger>

-									<SelectValue placeholder="Select a language" />

-								</SelectTrigger>

-								<SelectContent>

-									<SelectItem value="auto">Auto detect</SelectItem>

-									{TRANSCRIPTION_LANGUAGES.map((language) => (

-										<SelectItem key={language.code} value={language.code}>

-											{language.name}

-										</SelectItem>

-									))}

-								</SelectContent>

-							</Select>

-						</SectionField>

-					</SectionFields>

-

-					{error && (

-						<div className="bg-destructive/10 border-destructive/20 rounded-md border p-3">

-							<p className="text-destructive text-sm">{error}</p>

-						</div>

-					)}

-				</SectionContent>

-			</Section>

-			<Section showBottomBorder={false} showTopBorder={false}>

-				<SectionContent>

-					<Button

-						className="w-full"

-						onClick={handleGenerateTranscript}

-						disabled={isProcessing}

-					>

-						{isProcessing && <Spinner className="mr-1" />}

-						{isProcessing ? processingStep : "Generate transcript"}

-					</Button>

-				</SectionContent>

-			</Section>

-		</PanelView>

-	);

-}

+import { Button } from "@/components/ui/button";
+import { PanelView } from "@/components/editor/panels/assets/views/base-panel";
+import {
+	Select,
+	SelectContent,
+	SelectItem,
+	SelectTrigger,
+	SelectValue,
+} from "@/components/ui/select";
+import { useReducer, useRef, useState } from "react";
+import { extractTimelineAudio } from "@/lib/media/mediabunny";
+import { useEditor } from "@/hooks/use-editor";
+import {
+	DEFAULT_TRANSCRIPTION_SAMPLE_RATE,
+	TRANSCRIPTION_LANGUAGES,
+} from "@/constants/transcription-constants";
+import type {
+	CaptionChunk,
+	TranscriptionLanguage,
+	TranscriptionProgress,
+} from "@/lib/transcription/types";
+import { transcriptionService } from "@/services/transcription/service";
+import { decodeAudioToFloat32 } from "@/lib/media/audio";
+import { buildCaptionChunks } from "@/lib/transcription/caption";
+import { insertCaptionChunksAsTextTrack } from "@/lib/subtitles/insert";
+import { parseSubtitleFile } from "@/lib/subtitles/parse";
+import { Spinner } from "@/components/ui/spinner";
+import {
+	Section,
+	SectionContent,
+	SectionField,
+	SectionFields,
+} from "@/components/section";
+import { CloudUploadIcon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+
+type ProcessingState =
+	| { status: "idle"; error: string | null; warnings: string[] }
+	| { status: "processing"; step: string };
+
+type ProcessingAction =
+	| { type: "start"; step: string }
+	| { type: "update_step"; step: string }
+	| { type: "succeed"; warnings: string[] }
+	| { type: "fail"; error: string };
+
+const IDLE_STATE: ProcessingState = { status: "idle", error: null, warnings: [] };
+
+function processingReducer(
+	state: ProcessingState,
+	action: ProcessingAction,
+): ProcessingState {
+	switch (action.type) {
+		case "start":
+			return { status: "processing", step: action.step };
+		case "update_step":
+			if (state.status !== "processing") return state;
+			return { status: "processing", step: action.step };
+		case "succeed":
+			return { status: "idle", error: null, warnings: action.warnings };
+		case "fail":
+			return { status: "idle", error: action.error, warnings: [] };
+	}
+}
+
+export function Captions() {
+	const [selectedLanguage, setSelectedLanguage] =
+		useState<TranscriptionLanguage>("auto");
+	const [processing, dispatch] = useReducer(processingReducer, IDLE_STATE);
+	const containerRef = useRef<HTMLDivElement>(null);
+	const fileInputRef = useRef<HTMLInputElement>(null);
+	const editor = useEditor();
+
+	const isProcessing = processing.status === "processing";
+
+	const handleProgress = (progress: TranscriptionProgress) => {
+		if (progress.status === "loading-model") {
+			dispatch({
+				type: "update_step",
+				step: `Loading model ${Math.round(progress.progress)}%`,
+			});
+		} else if (progress.status === "transcribing") {
+			dispatch({ type: "update_step", step: "Transcribing..." });
+		}
+	};
+
+	const insertCaptions = ({ captions }: { captions: CaptionChunk[] }): boolean => {
+		const trackId = insertCaptionChunksAsTextTrack({ editor, captions });
+		return trackId !== null;
+	};
+
+	const handleGenerateTranscript = async () => {
+		dispatch({ type: "start", step: "Extracting audio..." });
+		try {
+			const audioBlob = await extractTimelineAudio({
+				tracks: editor.scenes.getActiveScene().tracks,
+				mediaAssets: editor.media.getAssets(),
+				totalDuration: editor.timeline.getTotalDuration(),
+			});
+
+			dispatch({ type: "update_step", step: "Preparing audio..." });
+			const { samples } = await decodeAudioToFloat32({
+				audioBlob,
+				sampleRate: DEFAULT_TRANSCRIPTION_SAMPLE_RATE,
+			});
+
+			const result = await transcriptionService.transcribe({
+				audioData: samples,
+				language: selectedLanguage === "auto" ? undefined : selectedLanguage,
+				onProgress: handleProgress,
+			});
+
+			dispatch({ type: "update_step", step: "Generating captions..." });
+			const captionChunks = buildCaptionChunks({ segments: result.segments });
+
+			if (!insertCaptions({ captions: captionChunks })) {
+				dispatch({ type: "fail", error: "No captions were generated" });
+				return;
+			}
+
+			dispatch({ type: "succeed", warnings: [] });
+		} catch (error) {
+			console.error("Transcription failed:", error);
+			dispatch({
+				type: "fail",
+				error: error instanceof Error ? error.message : "An unexpected error occurred",
+			});
+		}
+	};
+
+	const handleImportClick = () => {
+		fileInputRef.current?.click();
+	};
+
+	const handleImportFile = async ({ file }: { file: File }) => {
+		dispatch({ type: "start", step: "Reading subtitle file..." });
+		try {
+			const input = await file.text();
+			const result = parseSubtitleFile({
+				fileName: file.name,
+				input,
+			});
+
+			if (result.captions.length === 0) {
+				dispatch({
+					type: "fail",
+					error: "No valid subtitle cues were found in the subtitle file",
+				});
+				return;
+			}
+
+			dispatch({ type: "update_step", step: "Importing subtitles..." });
+
+			if (!insertCaptions({ captions: result.captions })) {
+				dispatch({ type: "fail", error: "No captions were generated" });
+				return;
+			}
+
+			const nextWarnings = [...result.warnings];
+			if (result.skippedCueCount > 0) {
+				nextWarnings.unshift(
+					`Imported ${result.captions.length} subtitle cue(s) and skipped ${result.skippedCueCount} malformed cue(s).`,
+				);
+			}
+
+			dispatch({ type: "succeed", warnings: nextWarnings });
+		} catch (error) {
+			console.error("Subtitle import failed:", error);
+			dispatch({
+				type: "fail",
+				error: error instanceof Error ? error.message : "An unexpected error occurred",
+			});
+		}
+	};
+
+	const handleFileChange = async ({
+		event,
+	}: {
+		event: React.ChangeEvent<HTMLInputElement>;
+	}) => {
+		const file = event.target.files?.[0];
+		if (event.target) {
+			event.target.value = "";
+		}
+		if (!file) return;
+
+		await handleImportFile({ file });
+	};
+
+	const handleLanguageChange = ({ value }: { value: string }) => {
+		if (value === "auto") {
+			setSelectedLanguage("auto");
+			return;
+		}
+
+		const matchedLanguage = TRANSCRIPTION_LANGUAGES.find(
+			(language) => language.code === value,
+		);
+		if (!matchedLanguage) return;
+		setSelectedLanguage(matchedLanguage.code);
+	};
+
+	const error = processing.status === "idle" ? processing.error : null;
+	const warnings = processing.status === "idle" ? processing.warnings : [];
+
+	return (
+		<PanelView
+			title="Captions"
+			contentClassName="px-0 flex flex-col h-full"
+			actions={
+				<Button
+					type="button"
+					variant="outline"
+					size="sm"
+					onClick={handleImportClick}
+					disabled={isProcessing}
+					className="items-center justify-center gap-1.5"
+				>
+					<HugeiconsIcon icon={CloudUploadIcon} />
+					Import
+				</Button>
+			}
+			ref={containerRef}
+		>
+			<input
+				ref={fileInputRef}
+				type="file"
+				accept=".srt,.ass"
+				className="hidden"
+				onChange={(event) => void handleFileChange({ event })}
+			/>
+			<Section showTopBorder={false} showBottomBorder={false} className="flex-1">
+				<SectionContent className="flex flex-col gap-4 h-full pt-1">
+					<SectionFields>
+						<SectionField label="Language">
+							<Select
+								value={selectedLanguage}
+								onValueChange={(value) => handleLanguageChange({ value })}
+							>
+								<SelectTrigger>
+									<SelectValue placeholder="Select a language" />
+								</SelectTrigger>
+								<SelectContent>
+									<SelectItem value="auto">Auto detect</SelectItem>
+									{TRANSCRIPTION_LANGUAGES.map((language) => (
+										<SelectItem key={language.code} value={language.code}>
+											{language.name}
+										</SelectItem>
+									))}
+								</SelectContent>
+							</Select>
+						</SectionField>
+					</SectionFields>
+
+					<Button
+						type="button"
+						className="mt-auto w-full"
+						onClick={handleGenerateTranscript}
+						disabled={isProcessing}
+					>
+						{isProcessing && <Spinner className="mr-1" />}
+						{isProcessing ? processing.step : "Generate transcript"}
+					</Button>
+					{error && (
+						<div className="bg-destructive/10 border-destructive/20 rounded-md border p-3">
+							<p className="text-destructive text-sm">{error}</p>
+						</div>
+					)}
+					{warnings.length > 0 && (
+						<div className="rounded-md border border-amber-500/20 bg-amber-500/10 p-3">
+							<ul className="space-y-1 text-sm text-amber-700">
+								{warnings.map((warning) => (
+									<li key={warning}>{warning}</li>
+								))}
+							</ul>
+						</div>
+					)}
+				</SectionContent>
+			</Section>
+		</PanelView>
+	);
+}