feat: transcription diagnostics

5 files changed, 321 insertions, 240 deletions

diff --git a/apps/web/src/lib/diagnostics/types.ts b/apps/web/src/lib/diagnostics/types.ts
new file mode 100644
index 00000000..486f362c
--- /dev/null
+++ b/apps/web/src/lib/diagnostics/types.ts
@@ -0,0 +1,8 @@
+export type DiagnosticSeverity = "caution" | "error";
+
+export interface DiagnosticDefinition {
+	id: string;
+	scope: string;
+	severity: DiagnosticSeverity;
+	message: string;
+}
diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts
index 5a45b6ea..04ea580f 100644
--- a/apps/web/src/lib/media/audio.ts
+++ b/apps/web/src/lib/media/audio.ts
@@ -1,5 +1,6 @@
 import type {
 	AudioElement,
+	VideoElement,
 	LibraryAudioElement,
 	RetimeConfig,
 	SceneTracks,
@@ -12,9 +13,7 @@ import {
 	hasAnimatedVolume,
 	resolveEffectiveAudioGain,
 } from "@/lib/timeline/audio-state";
-import {
-	doesElementHaveEnabledAudio,
-} from "@/lib/timeline/audio-separation";
+import { doesElementHaveEnabledAudio } from "@/lib/timeline/audio-separation";
 import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils";
 import { canTrackHaveAudio } from "@/lib/timeline";
 import { mediaSupportsAudio } from "@/lib/media/media-utils";
@@ -83,22 +82,23 @@ export async function decodeAudioToFloat32({
 	return { samples, sampleRate: audioBuffer.sampleRate };
 }
 
-export async function collectAudioElements({
+export interface AudibleElementCandidate {
+	element: AudioElement | VideoElement;
+	mediaAsset: MediaAsset | null;
+}
+
+export function collectAudibleCandidates({
 	tracks,
 	mediaAssets,
-	audioContext,
 }: {
 	tracks: SceneTracks;
 	mediaAssets: MediaAsset[];
-	audioContext: AudioContext;
-}): Promise<CollectedAudioElement[]> {
-	const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio];
-	const mediaMap = new Map<string, MediaAsset>(
-		mediaAssets.map((media) => [media.id, media]),
-	);
-	const pendingElements: Array<Promise<CollectedAudioElement | null>> = [];
+}): AudibleElementCandidate[] {
+	const allTracks = [...tracks.overlay, tracks.main, ...tracks.audio];
+	const mediaMap = new Map(mediaAssets.map((a) => [a.id, a]));
+	const candidates: AudibleElementCandidate[] = [];
 
-	for (const track of orderedTracks) {
+	for (const track of allTracks) {
 		if (canTrackHaveAudio(track) && track.muted) continue;
 
 		for (const element of track.elements) {
@@ -110,65 +110,95 @@ export async function collectAudioElements({
 				: null;
 			if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;
 
-			const isTrackMuted = canTrackHaveAudio(track) && track.muted;
+			candidates.push({ element, mediaAsset });
+		}
+	}
 
-			if (element.type === "audio") {
-				pendingElements.push(
-					resolveAudioBufferForElement({
-						element,
-						mediaMap,
-						audioContext,
-					}).then((audioBuffer) => {
-						if (!audioBuffer) return null;
-						const muted = element.muted === true || isTrackMuted;
-						return {
-							timelineElement: element,
-							buffer: audioBuffer,
-							startTime: element.startTime / TICKS_PER_SECOND,
-							duration: element.duration / TICKS_PER_SECOND,
-							trimStart: element.trimStart / TICKS_PER_SECOND,
-							trimEnd: element.trimEnd / TICKS_PER_SECOND,
-							volume: resolveEffectiveAudioGain({
-								element,
-								trackMuted: isTrackMuted,
-								localTime: 0,
-							}),
-							muted,
-							retime: element.retime,
-						};
-					}),
-				);
-				continue;
-			}
+	return candidates;
+}
 
-			if (element.type === "video") {
-				if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
+export function timelineHasAudio({
+	tracks,
+	mediaAssets,
+}: {
+	tracks: SceneTracks;
+	mediaAssets: MediaAsset[];
+}): boolean {
+	return collectAudibleCandidates({ tracks, mediaAssets }).some(
+		({ element }) => element.muted !== true,
+	);
+}
 
-				pendingElements.push(
-					resolveAudioBufferForVideoElement({
-						mediaAsset,
-						audioContext,
-					}).then((audioBuffer) => {
-						if (!audioBuffer) return null;
-						const muted = (element.muted ?? false) || isTrackMuted;
-						return {
-							timelineElement: element,
-							buffer: audioBuffer,
-							startTime: element.startTime / TICKS_PER_SECOND,
-							duration: element.duration / TICKS_PER_SECOND,
-							trimStart: element.trimStart / TICKS_PER_SECOND,
-							trimEnd: element.trimEnd / TICKS_PER_SECOND,
-							volume: resolveEffectiveAudioGain({
-								element,
-								trackMuted: isTrackMuted,
-								localTime: 0,
-							}),
-							muted,
-							retime: element.retime,
-						};
-					}),
-				);
-			}
+export async function collectAudioElements({
+	tracks,
+	mediaAssets,
+	audioContext,
+}: {
+	tracks: SceneTracks;
+	mediaAssets: MediaAsset[];
+	audioContext: AudioContext;
+}): Promise<CollectedAudioElement[]> {
+	const candidates = collectAudibleCandidates({ tracks, mediaAssets });
+	const mediaMap = new Map<string, MediaAsset>(
+		mediaAssets.map((media) => [media.id, media]),
+	);
+	const pendingElements: Array<Promise<CollectedAudioElement | null>> = [];
+
+	for (const { element, mediaAsset } of candidates) {
+		if (element.type === "audio") {
+			pendingElements.push(
+				resolveAudioBufferForElement({
+					element,
+					mediaMap,
+					audioContext,
+				}).then((audioBuffer) => {
+					if (!audioBuffer) return null;
+					return {
+						timelineElement: element,
+						buffer: audioBuffer,
+						startTime: element.startTime / TICKS_PER_SECOND,
+						duration: element.duration / TICKS_PER_SECOND,
+						trimStart: element.trimStart / TICKS_PER_SECOND,
+						trimEnd: element.trimEnd / TICKS_PER_SECOND,
+						volume: resolveEffectiveAudioGain({
+							element,
+							trackMuted: false,
+							localTime: 0,
+						}),
+						muted: element.muted === true,
+						retime: element.retime,
+					};
+				}),
+			);
+			continue;
+		}
+
+		if (element.type === "video") {
+			if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
+
+			pendingElements.push(
+				resolveAudioBufferForVideoElement({
+					mediaAsset,
+					audioContext,
+				}).then((audioBuffer) => {
+					if (!audioBuffer) return null;
+					return {
+						timelineElement: element,
+						buffer: audioBuffer,
+						startTime: element.startTime / TICKS_PER_SECOND,
+						duration: element.duration / TICKS_PER_SECOND,
+						trimStart: element.trimStart / TICKS_PER_SECOND,
+						trimEnd: element.trimEnd / TICKS_PER_SECOND,
+						volume: resolveEffectiveAudioGain({
+							element,
+							trackMuted: false,
+							localTime: 0,
+						}),
+						muted: element.muted ?? false,
+						retime: element.retime,
+					};
+				}),
+			);
 		}
 	}
 
@@ -339,16 +369,16 @@ async function fetchLibraryAudioSource({
 			type: "audio/mpeg",
 		});
 
-	return {
-		timelineElement: element,
-		file,
-		startTime: element.startTime / TICKS_PER_SECOND,
-		duration: element.duration / TICKS_PER_SECOND,
-		trimStart: element.trimStart / TICKS_PER_SECOND,
-		trimEnd: element.trimEnd / TICKS_PER_SECOND,
-		volume,
-		retime: element.retime,
-	};
+		return {
+			timelineElement: element,
+			file,
+			startTime: element.startTime / TICKS_PER_SECOND,
+			duration: element.duration / TICKS_PER_SECOND,
+			trimStart: element.trimStart / TICKS_PER_SECOND,
+			trimEnd: element.trimEnd / TICKS_PER_SECOND,
+			volume,
+			retime: element.retime,
+		};
 	} catch (error) {
 		console.warn("Failed to fetch library audio:", error);
 		return null;
diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts
index a313c6f0..1e97fc42 100644
--- a/apps/web/src/lib/media/mediabunny.ts
+++ b/apps/web/src/lib/media/mediabunny.ts
@@ -1,158 +1,168 @@
-import { Input, ALL_FORMATS, BlobSource } from "mediabunny";

-import { createTimelineAudioBuffer } from "@/lib/media/audio";

-import type { SceneTracks } from "@/lib/timeline";

-import type { MediaAsset } from "@/lib/media/types";

-

-export async function getVideoInfo({

-	videoFile,

-}: {

-	videoFile: File;

-}): Promise<{

-	duration: number;

-	width: number;

-	height: number;

-	fps: number;

-	hasAudio: boolean;

-}> {

-	const input = new Input({

-		source: new BlobSource(videoFile),

-		formats: ALL_FORMATS,

-	});

-

-	const duration = await input.computeDuration();

-	const videoTrack = await input.getPrimaryVideoTrack();

-

-	if (!videoTrack) {

-		throw new Error("No video track found in the file");

-	}

-

-	const packetStats = await videoTrack.computePacketStats(100);

-	const fps = packetStats.averagePacketRate;

-	const audioTrack = await input.getPrimaryAudioTrack();

-

-	return {

-		duration,

-		width: videoTrack.displayWidth,

-		height: videoTrack.displayHeight,

-		fps,

-		hasAudio: audioTrack !== null,

-	};

-}

-

-const SAMPLE_RATE = 44100;

-const NUM_CHANNELS = 2;

-

-export const extractTimelineAudio = async ({

-	tracks,

-	mediaAssets,

-	totalDuration,

-	onProgress,

-}: {

-	tracks: SceneTracks;

-	mediaAssets: MediaAsset[];

-	totalDuration: number;

-	onProgress?: (progress: number) => void;

-}): Promise<Blob> => {

-	if (totalDuration === 0) {

-		return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });

-	}

-

-	onProgress?.(10);

-

-	const audioBuffer = await createTimelineAudioBuffer({

-		tracks,

-		mediaAssets,

-		duration: totalDuration,

-		sampleRate: SAMPLE_RATE,

-	});

-

-	if (!audioBuffer) {

-		const silentDuration = Math.max(1, totalDuration);

-		const silentSamples = new Float32Array(

-			Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,

-		);

-		return createWavBlob({ samples: silentSamples });

-	}

-

-	onProgress?.(90);

-

-	const interleavedSamples = interleaveAudioBuffer({ audioBuffer });

-	onProgress?.(100);

-

-	return createWavBlob({ samples: interleavedSamples });

-};

-

-function interleaveAudioBuffer({

-	audioBuffer,

-}: {

-	audioBuffer: AudioBuffer;

-}): Float32Array {

-	const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);

-	const interleavedSamples = new Float32Array(

-		audioBuffer.length * NUM_CHANNELS,

-	);

-

-	for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {

-		for (let channel = 0; channel < NUM_CHANNELS; channel++) {

-			const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));

-			interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =

-				audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;

-		}

-	}

-

-	return interleavedSamples;

-}

-

-function createWavBlob({ samples }: { samples: Float32Array }): Blob {

-	const numChannels = NUM_CHANNELS;

-	const bitsPerSample = 16;

-	const bytesPerSample = bitsPerSample / 8;

-	const numSamples = samples.length / numChannels;

-	const dataSize = numSamples * numChannels * bytesPerSample;

-	const buffer = new ArrayBuffer(44 + dataSize);

-	const view = new DataView(buffer);

-

-	// riff header

-	writeString({ view, offset: 0, str: "RIFF" });

-	view.setUint32(4, 36 + dataSize, true);

-	writeString({ view, offset: 8, str: "WAVE" });

-

-	// fmt chunk

-	writeString({ view, offset: 12, str: "fmt " });

-	view.setUint32(16, 16, true);

-	view.setUint16(20, 1, true);

-	view.setUint16(22, numChannels, true);

-	view.setUint32(24, SAMPLE_RATE, true);

-	view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);

-	view.setUint16(32, numChannels * bytesPerSample, true);

-	view.setUint16(34, bitsPerSample, true);

-

-	// data chunk

-	writeString({ view, offset: 36, str: "data" });

-	view.setUint32(40, dataSize, true);

-

-	// convert float32 to int16 and write

-	let offset = 44;

-	for (let i = 0; i < samples.length; i++) {

-		const sample = Math.max(-1, Math.min(1, samples[i]));

-		const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;

-		view.setInt16(offset, int16, true);

-		offset += 2;

-	}

-

-	return new Blob([buffer], { type: "audio/wav" });

-}

-

-function writeString({

-	view,

-	offset,

-	str,

-}: {

-	view: DataView;

-	offset: number;

-	str: string;

-}): void {

-	for (let i = 0; i < str.length; i++) {

-		view.setUint8(offset + i, str.charCodeAt(i));

-	}

-}

+import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
+import { createTimelineAudioBuffer } from "@/lib/media/audio";
+import type { SceneTracks } from "@/lib/timeline";
+import type { MediaAsset } from "@/lib/media/types";
+import { TICKS_PER_SECOND } from "@/lib/wasm";
+
+export async function getVideoInfo({
+	videoFile,
+}: {
+	videoFile: File;
+}): Promise<{
+	duration: number;
+	width: number;
+	height: number;
+	fps: number;
+	hasAudio: boolean;
+}> {
+	const input = new Input({
+		source: new BlobSource(videoFile),
+		formats: ALL_FORMATS,
+	});
+
+	const duration = await input.computeDuration();
+	const videoTrack = await input.getPrimaryVideoTrack();
+
+	if (!videoTrack) {
+		throw new Error("No video track found in the file");
+	}
+
+	const packetStats = await videoTrack.computePacketStats(100);
+	const fps = packetStats.averagePacketRate;
+	const audioTrack = await input.getPrimaryAudioTrack();
+
+	return {
+		duration,
+		width: videoTrack.displayWidth,
+		height: videoTrack.displayHeight,
+		fps,
+		hasAudio: audioTrack !== null,
+	};
+}
+
+const SAMPLE_RATE = 44100;
+const NUM_CHANNELS = 2;
+const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1;
+const MIN_SILENT_DURATION_SECONDS = 0.001;
+
+export const extractTimelineAudio = async ({
+	tracks,
+	mediaAssets,
+	totalDuration,
+	onProgress,
+}: {
+	tracks: SceneTracks;
+	mediaAssets: MediaAsset[];
+	totalDuration: number;
+	onProgress?: (progress: number) => void;
+}): Promise<Blob> => {
+	if (totalDuration === 0) {
+		return createWavBlob({
+			samples: new Float32Array(
+				SAMPLE_RATE * EMPTY_TIMELINE_SILENT_DURATION_SECONDS,
+			),
+		});
+	}
+
+	onProgress?.(10);
+
+	const audioBuffer = await createTimelineAudioBuffer({
+		tracks,
+		mediaAssets,
+		duration: totalDuration,
+		sampleRate: SAMPLE_RATE,
+	});
+
+	if (!audioBuffer) {
+		const silentDurationSeconds = Math.max(
+			MIN_SILENT_DURATION_SECONDS,
+			totalDuration / TICKS_PER_SECOND,
+		);
+		const silentSamples = new Float32Array(
+			Math.ceil(silentDurationSeconds * SAMPLE_RATE) * NUM_CHANNELS,
+		);
+		return createWavBlob({ samples: silentSamples });
+	}
+
+	onProgress?.(90);
+
+	const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
+	onProgress?.(100);
+
+	return createWavBlob({ samples: interleavedSamples });
+};
+
+function interleaveAudioBuffer({
+	audioBuffer,
+}: {
+	audioBuffer: AudioBuffer;
+}): Float32Array {
+	const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
+	const interleavedSamples = new Float32Array(
+		audioBuffer.length * NUM_CHANNELS,
+	);
+
+	for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
+		for (let channel = 0; channel < NUM_CHANNELS; channel++) {
+			const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
+			interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
+				audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
+		}
+	}
+
+	return interleavedSamples;
+}
+
+function createWavBlob({ samples }: { samples: Float32Array }): Blob {
+	const numChannels = NUM_CHANNELS;
+	const bitsPerSample = 16;
+	const bytesPerSample = bitsPerSample / 8;
+	const numSamples = samples.length / numChannels;
+	const dataSize = numSamples * numChannels * bytesPerSample;
+	const buffer = new ArrayBuffer(44 + dataSize);
+	const view = new DataView(buffer);
+
+	// riff header
+	writeString({ view, offset: 0, str: "RIFF" });
+	view.setUint32(4, 36 + dataSize, true);
+	writeString({ view, offset: 8, str: "WAVE" });
+
+	// fmt chunk
+	writeString({ view, offset: 12, str: "fmt " });
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, numChannels, true);
+	view.setUint32(24, SAMPLE_RATE, true);
+	view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
+	view.setUint16(32, numChannels * bytesPerSample, true);
+	view.setUint16(34, bitsPerSample, true);
+
+	// data chunk
+	writeString({ view, offset: 36, str: "data" });
+	view.setUint32(40, dataSize, true);
+
+	// convert float32 to int16 and write
+	let offset = 44;
+	for (let i = 0; i < samples.length; i++) {
+		const sample = Math.max(-1, Math.min(1, samples[i]));
+		const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+		view.setInt16(offset, int16, true);
+		offset += 2;
+	}
+
+	return new Blob([buffer], { type: "audio/wav" });
+}
+
+function writeString({
+	view,
+	offset,
+	str,
+}: {
+	view: DataView;
+	offset: number;
+	str: string;
+}): void {
+	for (let i = 0; i < str.length; i++) {
+		view.setUint8(offset + i, str.charCodeAt(i));
+	}
+}
diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts
index fe3c1ca4..849dbda3 100644
--- a/apps/web/src/lib/timeline/audio-separation/index.ts
+++ b/apps/web/src/lib/timeline/audio-separation/index.ts
@@ -49,7 +49,9 @@ export function canToggleSourceAudio(
 	element: TimelineElement,
 	mediaAsset: MediaAudioState | null | undefined,
 ): element is VideoElement {
-	return canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset);
+	return (
+		canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset)
+	);
 }
 
 export function doesElementHaveEnabledAudio({
@@ -63,7 +65,11 @@ export function doesElementHaveEnabledAudio({
 		return true;
 	}
 
-	return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element });
+	return (
+		!!mediaAsset &&
+		mediaAsset.hasAudio !== false &&
+		isSourceAudioEnabled({ element })
+	);
 }
 
 export function buildSeparatedAudioElement({
@@ -100,7 +106,9 @@ export function getSourceAudioActionLabel({
 }: {
 	element: VideoElement;
 }): "Extract audio" | "Recover audio" {
-	return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio";
+	return isSourceAudioSeparated({ element })
+		? "Recover audio"
+		: "Extract audio";
 }
 
 function cloneVolumeAnimations({
diff --git a/apps/web/src/lib/transcription/diagnostics.ts b/apps/web/src/lib/transcription/diagnostics.ts
new file mode 100644
index 00000000..a1921699
--- /dev/null
+++ b/apps/web/src/lib/transcription/diagnostics.ts
@@ -0,0 +1,25 @@
+import type { DiagnosticsManager } from "@/core/managers/diagnostics-manager";
+import { timelineHasAudio } from "@/lib/media/audio";
+
+export const TRANSCRIPTION_DIAGNOSTICS_SCOPE = "transcription";
+
+export function registerTranscriptionDiagnostics({
+	diagnostics,
+}: {
+	diagnostics: DiagnosticsManager;
+}): void {
+	diagnostics.register({
+		id: "transcription.no_audio",
+		scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE,
+		severity: "caution",
+		message: "No audio detected. Add a clip with audio to the timeline first.",
+		check: (editor) => {
+			const scene = editor.scenes.getActiveSceneOrNull();
+			if (!scene) return false;
+			return !timelineHasAudio({
+				tracks: scene.tracks,
+				mediaAssets: editor.media.getAssets(),
+			});
+		},
+	});
+}