feat: transcription diagnostics

1 files changed, 168 insertions, 158 deletions

diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts
index a313c6f0..1e97fc42 100644
--- a/apps/web/src/lib/media/mediabunny.ts
+++ b/apps/web/src/lib/media/mediabunny.ts
@@ -1,158 +1,168 @@
-import { Input, ALL_FORMATS, BlobSource } from "mediabunny";

-import { createTimelineAudioBuffer } from "@/lib/media/audio";

-import type { SceneTracks } from "@/lib/timeline";

-import type { MediaAsset } from "@/lib/media/types";

-

-export async function getVideoInfo({

-	videoFile,

-}: {

-	videoFile: File;

-}): Promise<{

-	duration: number;

-	width: number;

-	height: number;

-	fps: number;

-	hasAudio: boolean;

-}> {

-	const input = new Input({

-		source: new BlobSource(videoFile),

-		formats: ALL_FORMATS,

-	});

-

-	const duration = await input.computeDuration();

-	const videoTrack = await input.getPrimaryVideoTrack();

-

-	if (!videoTrack) {

-		throw new Error("No video track found in the file");

-	}

-

-	const packetStats = await videoTrack.computePacketStats(100);

-	const fps = packetStats.averagePacketRate;

-	const audioTrack = await input.getPrimaryAudioTrack();

-

-	return {

-		duration,

-		width: videoTrack.displayWidth,

-		height: videoTrack.displayHeight,

-		fps,

-		hasAudio: audioTrack !== null,

-	};

-}

-

-const SAMPLE_RATE = 44100;

-const NUM_CHANNELS = 2;

-

-export const extractTimelineAudio = async ({

-	tracks,

-	mediaAssets,

-	totalDuration,

-	onProgress,

-}: {

-	tracks: SceneTracks;

-	mediaAssets: MediaAsset[];

-	totalDuration: number;

-	onProgress?: (progress: number) => void;

-}): Promise<Blob> => {

-	if (totalDuration === 0) {

-		return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });

-	}

-

-	onProgress?.(10);

-

-	const audioBuffer = await createTimelineAudioBuffer({

-		tracks,

-		mediaAssets,

-		duration: totalDuration,

-		sampleRate: SAMPLE_RATE,

-	});

-

-	if (!audioBuffer) {

-		const silentDuration = Math.max(1, totalDuration);

-		const silentSamples = new Float32Array(

-			Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,

-		);

-		return createWavBlob({ samples: silentSamples });

-	}

-

-	onProgress?.(90);

-

-	const interleavedSamples = interleaveAudioBuffer({ audioBuffer });

-	onProgress?.(100);

-

-	return createWavBlob({ samples: interleavedSamples });

-};

-

-function interleaveAudioBuffer({

-	audioBuffer,

-}: {

-	audioBuffer: AudioBuffer;

-}): Float32Array {

-	const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);

-	const interleavedSamples = new Float32Array(

-		audioBuffer.length * NUM_CHANNELS,

-	);

-

-	for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {

-		for (let channel = 0; channel < NUM_CHANNELS; channel++) {

-			const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));

-			interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =

-				audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;

-		}

-	}

-

-	return interleavedSamples;

-}

-

-function createWavBlob({ samples }: { samples: Float32Array }): Blob {

-	const numChannels = NUM_CHANNELS;

-	const bitsPerSample = 16;

-	const bytesPerSample = bitsPerSample / 8;

-	const numSamples = samples.length / numChannels;

-	const dataSize = numSamples * numChannels * bytesPerSample;

-	const buffer = new ArrayBuffer(44 + dataSize);

-	const view = new DataView(buffer);

-

-	// riff header

-	writeString({ view, offset: 0, str: "RIFF" });

-	view.setUint32(4, 36 + dataSize, true);

-	writeString({ view, offset: 8, str: "WAVE" });

-

-	// fmt chunk

-	writeString({ view, offset: 12, str: "fmt " });

-	view.setUint32(16, 16, true);

-	view.setUint16(20, 1, true);

-	view.setUint16(22, numChannels, true);

-	view.setUint32(24, SAMPLE_RATE, true);

-	view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);

-	view.setUint16(32, numChannels * bytesPerSample, true);

-	view.setUint16(34, bitsPerSample, true);

-

-	// data chunk

-	writeString({ view, offset: 36, str: "data" });

-	view.setUint32(40, dataSize, true);

-

-	// convert float32 to int16 and write

-	let offset = 44;

-	for (let i = 0; i < samples.length; i++) {

-		const sample = Math.max(-1, Math.min(1, samples[i]));

-		const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;

-		view.setInt16(offset, int16, true);

-		offset += 2;

-	}

-

-	return new Blob([buffer], { type: "audio/wav" });

-}

-

-function writeString({

-	view,

-	offset,

-	str,

-}: {

-	view: DataView;

-	offset: number;

-	str: string;

-}): void {

-	for (let i = 0; i < str.length; i++) {

-		view.setUint8(offset + i, str.charCodeAt(i));

-	}

-}

+import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
+import { createTimelineAudioBuffer } from "@/lib/media/audio";
+import type { SceneTracks } from "@/lib/timeline";
+import type { MediaAsset } from "@/lib/media/types";
+import { TICKS_PER_SECOND } from "@/lib/wasm";
+
+export async function getVideoInfo({
+	videoFile,
+}: {
+	videoFile: File;
+}): Promise<{
+	duration: number;
+	width: number;
+	height: number;
+	fps: number;
+	hasAudio: boolean;
+}> {
+	const input = new Input({
+		source: new BlobSource(videoFile),
+		formats: ALL_FORMATS,
+	});
+
+	const duration = await input.computeDuration();
+	const videoTrack = await input.getPrimaryVideoTrack();
+
+	if (!videoTrack) {
+		throw new Error("No video track found in the file");
+	}
+
+	const packetStats = await videoTrack.computePacketStats(100);
+	const fps = packetStats.averagePacketRate;
+	const audioTrack = await input.getPrimaryAudioTrack();
+
+	return {
+		duration,
+		width: videoTrack.displayWidth,
+		height: videoTrack.displayHeight,
+		fps,
+		hasAudio: audioTrack !== null,
+	};
+}
+
+const SAMPLE_RATE = 44100;
+const NUM_CHANNELS = 2;
+const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1;
+const MIN_SILENT_DURATION_SECONDS = 0.001;
+
+export const extractTimelineAudio = async ({
+	tracks,
+	mediaAssets,
+	totalDuration,
+	onProgress,
+}: {
+	tracks: SceneTracks;
+	mediaAssets: MediaAsset[];
+	totalDuration: number;
+	onProgress?: (progress: number) => void;
+}): Promise<Blob> => {
+	if (totalDuration === 0) {
+		return createWavBlob({
+			samples: new Float32Array(
+				SAMPLE_RATE * EMPTY_TIMELINE_SILENT_DURATION_SECONDS,
+			),
+		});
+	}
+
+	onProgress?.(10);
+
+	const audioBuffer = await createTimelineAudioBuffer({
+		tracks,
+		mediaAssets,
+		duration: totalDuration,
+		sampleRate: SAMPLE_RATE,
+	});
+
+	if (!audioBuffer) {
+		const silentDurationSeconds = Math.max(
+			MIN_SILENT_DURATION_SECONDS,
+			totalDuration / TICKS_PER_SECOND,
+		);
+		const silentSamples = new Float32Array(
+			Math.ceil(silentDurationSeconds * SAMPLE_RATE) * NUM_CHANNELS,
+		);
+		return createWavBlob({ samples: silentSamples });
+	}
+
+	onProgress?.(90);
+
+	const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
+	onProgress?.(100);
+
+	return createWavBlob({ samples: interleavedSamples });
+};
+
+function interleaveAudioBuffer({
+	audioBuffer,
+}: {
+	audioBuffer: AudioBuffer;
+}): Float32Array {
+	const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
+	const interleavedSamples = new Float32Array(
+		audioBuffer.length * NUM_CHANNELS,
+	);
+
+	for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
+		for (let channel = 0; channel < NUM_CHANNELS; channel++) {
+			const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
+			interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
+				audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
+		}
+	}
+
+	return interleavedSamples;
+}
+
+function createWavBlob({ samples }: { samples: Float32Array }): Blob {
+	const numChannels = NUM_CHANNELS;
+	const bitsPerSample = 16;
+	const bytesPerSample = bitsPerSample / 8;
+	const numSamples = samples.length / numChannels;
+	const dataSize = numSamples * numChannels * bytesPerSample;
+	const buffer = new ArrayBuffer(44 + dataSize);
+	const view = new DataView(buffer);
+
+	// riff header
+	writeString({ view, offset: 0, str: "RIFF" });
+	view.setUint32(4, 36 + dataSize, true);
+	writeString({ view, offset: 8, str: "WAVE" });
+
+	// fmt chunk
+	writeString({ view, offset: 12, str: "fmt " });
+	view.setUint32(16, 16, true);
+	view.setUint16(20, 1, true);
+	view.setUint16(22, numChannels, true);
+	view.setUint32(24, SAMPLE_RATE, true);
+	view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
+	view.setUint16(32, numChannels * bytesPerSample, true);
+	view.setUint16(34, bitsPerSample, true);
+
+	// data chunk
+	writeString({ view, offset: 36, str: "data" });
+	view.setUint32(40, dataSize, true);
+
+	// convert float32 to int16 and write
+	let offset = 44;
+	for (let i = 0; i < samples.length; i++) {
+		const sample = Math.max(-1, Math.min(1, samples[i]));
+		const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+		view.setInt16(offset, int16, true);
+		offset += 2;
+	}
+
+	return new Blob([buffer], { type: "audio/wav" });
+}
+
+function writeString({
+	view,
+	offset,
+	str,
+}: {
+	view: DataView;
+	offset: number;
+	str: string;
+}): void {
+	for (let i = 0; i < str.length; i++) {
+		view.setUint8(offset + i, str.charCodeAt(i));
+	}
+}