From 91e1232112421438ce73b9361896f9b58c84fb3e Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Tue, 14 Apr 2026 23:34:57 +0200 Subject: feat: transcription diagnostics --- apps/web/src/lib/diagnostics/types.ts | 8 + apps/web/src/lib/media/audio.ts | 190 +++++++----- apps/web/src/lib/media/mediabunny.ts | 326 +++++++++++---------- .../web/src/lib/timeline/audio-separation/index.ts | 14 +- apps/web/src/lib/transcription/diagnostics.ts | 25 ++ 5 files changed, 322 insertions(+), 241 deletions(-) create mode 100644 apps/web/src/lib/diagnostics/types.ts create mode 100644 apps/web/src/lib/transcription/diagnostics.ts (limited to 'apps/web/src/lib') diff --git a/apps/web/src/lib/diagnostics/types.ts b/apps/web/src/lib/diagnostics/types.ts new file mode 100644 index 00000000..486f362c --- /dev/null +++ b/apps/web/src/lib/diagnostics/types.ts @@ -0,0 +1,8 @@ +export type DiagnosticSeverity = "caution" | "error"; + +export interface DiagnosticDefinition { + id: string; + scope: string; + severity: DiagnosticSeverity; + message: string; +} diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts index 5a45b6ea..04ea580f 100644 --- a/apps/web/src/lib/media/audio.ts +++ b/apps/web/src/lib/media/audio.ts @@ -1,5 +1,6 @@ import type { AudioElement, + VideoElement, LibraryAudioElement, RetimeConfig, SceneTracks, @@ -12,9 +13,7 @@ import { hasAnimatedVolume, resolveEffectiveAudioGain, } from "@/lib/timeline/audio-state"; -import { - doesElementHaveEnabledAudio, -} from "@/lib/timeline/audio-separation"; +import { doesElementHaveEnabledAudio } from "@/lib/timeline/audio-separation"; import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils"; import { canTrackHaveAudio } from "@/lib/timeline"; import { mediaSupportsAudio } from "@/lib/media/media-utils"; @@ -83,22 +82,23 @@ export async function decodeAudioToFloat32({ return { samples, sampleRate: audioBuffer.sampleRate }; } -export async function collectAudioElements({ +export interface AudibleElementCandidate { + element: AudioElement | VideoElement; + mediaAsset: MediaAsset | null; +} + +export function collectAudibleCandidates({ tracks, mediaAssets, - audioContext, }: { tracks: SceneTracks; mediaAssets: MediaAsset[]; - audioContext: AudioContext; -}): Promise { - const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; - const mediaMap = new Map( - mediaAssets.map((media) => [media.id, media]), - ); - const pendingElements: Array> = []; +}): AudibleElementCandidate[] { + const allTracks = [...tracks.overlay, tracks.main, ...tracks.audio]; + const mediaMap = new Map(mediaAssets.map((a) => [a.id, a])); + const candidates: AudibleElementCandidate[] = []; - for (const track of orderedTracks) { + for (const track of allTracks) { if (canTrackHaveAudio(track) && track.muted) continue; for (const element of track.elements) { @@ -110,65 +110,95 @@ export async function collectAudioElements({ : null; if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue; - const isTrackMuted = canTrackHaveAudio(track) && track.muted; + candidates.push({ element, mediaAsset }); + } + } - if (element.type === "audio") { - pendingElements.push( - resolveAudioBufferForElement({ - element, - mediaMap, - audioContext, - }).then((audioBuffer) => { - if (!audioBuffer) return null; - const muted = element.muted === true || isTrackMuted; - return { - timelineElement: element, - buffer: audioBuffer, - startTime: element.startTime / TICKS_PER_SECOND, - duration: element.duration / TICKS_PER_SECOND, - trimStart: element.trimStart / TICKS_PER_SECOND, - trimEnd: element.trimEnd / TICKS_PER_SECOND, - volume: resolveEffectiveAudioGain({ - element, - trackMuted: isTrackMuted, - localTime: 0, - }), - muted, - retime: element.retime, - }; - }), - ); - continue; - } + return candidates; +} - if (element.type === "video") { - if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; - - pendingElements.push( - resolveAudioBufferForVideoElement({ - mediaAsset, - audioContext, - }).then((audioBuffer) => { - if (!audioBuffer) return null; - const muted = (element.muted ?? false) || isTrackMuted; - return { - timelineElement: element, - buffer: audioBuffer, - startTime: element.startTime / TICKS_PER_SECOND, - duration: element.duration / TICKS_PER_SECOND, - trimStart: element.trimStart / TICKS_PER_SECOND, - trimEnd: element.trimEnd / TICKS_PER_SECOND, - volume: resolveEffectiveAudioGain({ - element, - trackMuted: isTrackMuted, - localTime: 0, - }), - muted, - retime: element.retime, - }; - }), - ); - } +export function timelineHasAudio({ + tracks, + mediaAssets, +}: { + tracks: SceneTracks; + mediaAssets: MediaAsset[]; +}): boolean { + return collectAudibleCandidates({ tracks, mediaAssets }).some( + ({ element }) => element.muted !== true, + ); +} + +export async function collectAudioElements({ + tracks, + mediaAssets, + audioContext, +}: { + tracks: SceneTracks; + mediaAssets: MediaAsset[]; + audioContext: AudioContext; +}): Promise { + const candidates = collectAudibleCandidates({ tracks, mediaAssets }); + const mediaMap = new Map( + mediaAssets.map((media) => [media.id, media]), + ); + const pendingElements: Array> = []; + + for (const { element, mediaAsset } of candidates) { + if (element.type === "audio") { + pendingElements.push( + resolveAudioBufferForElement({ + element, + mediaMap, + audioContext, + }).then((audioBuffer) => { + if (!audioBuffer) return null; + return { + timelineElement: element, + buffer: audioBuffer, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, + volume: resolveEffectiveAudioGain({ + element, + trackMuted: false, + localTime: 0, + }), + muted: element.muted === true, + retime: element.retime, + }; + }), + ); + continue; + } + + if (element.type === "video") { + if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue; + + pendingElements.push( + resolveAudioBufferForVideoElement({ + mediaAsset, + audioContext, + }).then((audioBuffer) => { + if (!audioBuffer) return null; + return { + timelineElement: element, + buffer: audioBuffer, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, + volume: resolveEffectiveAudioGain({ + element, + trackMuted: false, + localTime: 0, + }), + muted: element.muted ?? false, + retime: element.retime, + }; + }), + ); } } @@ -339,16 +369,16 @@ async function fetchLibraryAudioSource({ type: "audio/mpeg", }); - return { - timelineElement: element, - file, - startTime: element.startTime / TICKS_PER_SECOND, - duration: element.duration / TICKS_PER_SECOND, - trimStart: element.trimStart / TICKS_PER_SECOND, - trimEnd: element.trimEnd / TICKS_PER_SECOND, - volume, - retime: element.retime, - }; + return { + timelineElement: element, + file, + startTime: element.startTime / TICKS_PER_SECOND, + duration: element.duration / TICKS_PER_SECOND, + trimStart: element.trimStart / TICKS_PER_SECOND, + trimEnd: element.trimEnd / TICKS_PER_SECOND, + volume, + retime: element.retime, + }; } catch (error) { console.warn("Failed to fetch library audio:", error); return null; diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts index a313c6f0..1e97fc42 100644 --- a/apps/web/src/lib/media/mediabunny.ts +++ b/apps/web/src/lib/media/mediabunny.ts @@ -1,158 +1,168 @@ -import { Input, ALL_FORMATS, BlobSource } from "mediabunny"; -import { createTimelineAudioBuffer } from "@/lib/media/audio"; -import type { SceneTracks } from "@/lib/timeline"; -import type { MediaAsset } from "@/lib/media/types"; - -export async function getVideoInfo({ - videoFile, -}: { - videoFile: File; -}): Promise<{ - duration: number; - width: number; - height: number; - fps: number; - hasAudio: boolean; -}> { - const input = new Input({ - source: new BlobSource(videoFile), - formats: ALL_FORMATS, - }); - - const duration = await input.computeDuration(); - const videoTrack = await input.getPrimaryVideoTrack(); - - if (!videoTrack) { - throw new Error("No video track found in the file"); - } - - const packetStats = await videoTrack.computePacketStats(100); - const fps = packetStats.averagePacketRate; - const audioTrack = await input.getPrimaryAudioTrack(); - - return { - duration, - width: videoTrack.displayWidth, - height: videoTrack.displayHeight, - fps, - hasAudio: audioTrack !== null, - }; -} - -const SAMPLE_RATE = 44100; -const NUM_CHANNELS = 2; - -export const extractTimelineAudio = async ({ - tracks, - mediaAssets, - totalDuration, - onProgress, -}: { - tracks: SceneTracks; - mediaAssets: MediaAsset[]; - totalDuration: number; - onProgress?: (progress: number) => void; -}): Promise => { - if (totalDuration === 0) { - return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) }); - } - - onProgress?.(10); - - const audioBuffer = await createTimelineAudioBuffer({ - tracks, - mediaAssets, - duration: totalDuration, - sampleRate: SAMPLE_RATE, - }); - - if (!audioBuffer) { - const silentDuration = Math.max(1, totalDuration); - const silentSamples = new Float32Array( - Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS, - ); - return createWavBlob({ samples: silentSamples }); - } - - onProgress?.(90); - - const interleavedSamples = interleaveAudioBuffer({ audioBuffer }); - onProgress?.(100); - - return createWavBlob({ samples: interleavedSamples }); -}; - -function interleaveAudioBuffer({ - audioBuffer, -}: { - audioBuffer: AudioBuffer; -}): Float32Array { - const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels); - const interleavedSamples = new Float32Array( - audioBuffer.length * NUM_CHANNELS, - ); - - for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) { - for (let channel = 0; channel < NUM_CHANNELS; channel++) { - const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1)); - interleavedSamples[sampleIndex * NUM_CHANNELS + channel] = - audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0; - } - } - - return interleavedSamples; -} - -function createWavBlob({ samples }: { samples: Float32Array }): Blob { - const numChannels = NUM_CHANNELS; - const bitsPerSample = 16; - const bytesPerSample = bitsPerSample / 8; - const numSamples = samples.length / numChannels; - const dataSize = numSamples * numChannels * bytesPerSample; - const buffer = new ArrayBuffer(44 + dataSize); - const view = new DataView(buffer); - - // riff header - writeString({ view, offset: 0, str: "RIFF" }); - view.setUint32(4, 36 + dataSize, true); - writeString({ view, offset: 8, str: "WAVE" }); - - // fmt chunk - writeString({ view, offset: 12, str: "fmt " }); - view.setUint32(16, 16, true); - view.setUint16(20, 1, true); - view.setUint16(22, numChannels, true); - view.setUint32(24, SAMPLE_RATE, true); - view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true); - view.setUint16(32, numChannels * bytesPerSample, true); - view.setUint16(34, bitsPerSample, true); - - // data chunk - writeString({ view, offset: 36, str: "data" }); - view.setUint32(40, dataSize, true); - - // convert float32 to int16 and write - let offset = 44; - for (let i = 0; i < samples.length; i++) { - const sample = Math.max(-1, Math.min(1, samples[i])); - const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff; - view.setInt16(offset, int16, true); - offset += 2; - } - - return new Blob([buffer], { type: "audio/wav" }); -} - -function writeString({ - view, - offset, - str, -}: { - view: DataView; - offset: number; - str: string; -}): void { - for (let i = 0; i < str.length; i++) { - view.setUint8(offset + i, str.charCodeAt(i)); - } -} +import { Input, ALL_FORMATS, BlobSource } from "mediabunny"; +import { createTimelineAudioBuffer } from "@/lib/media/audio"; +import type { SceneTracks } from "@/lib/timeline"; +import type { MediaAsset } from "@/lib/media/types"; +import { TICKS_PER_SECOND } from "@/lib/wasm"; + +export async function getVideoInfo({ + videoFile, +}: { + videoFile: File; +}): Promise<{ + duration: number; + width: number; + height: number; + fps: number; + hasAudio: boolean; +}> { + const input = new Input({ + source: new BlobSource(videoFile), + formats: ALL_FORMATS, + }); + + const duration = await input.computeDuration(); + const videoTrack = await input.getPrimaryVideoTrack(); + + if (!videoTrack) { + throw new Error("No video track found in the file"); + } + + const packetStats = await videoTrack.computePacketStats(100); + const fps = packetStats.averagePacketRate; + const audioTrack = await input.getPrimaryAudioTrack(); + + return { + duration, + width: videoTrack.displayWidth, + height: videoTrack.displayHeight, + fps, + hasAudio: audioTrack !== null, + }; +} + +const SAMPLE_RATE = 44100; +const NUM_CHANNELS = 2; +const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1; +const MIN_SILENT_DURATION_SECONDS = 0.001; + +export const extractTimelineAudio = async ({ + tracks, + mediaAssets, + totalDuration, + onProgress, +}: { + tracks: SceneTracks; + mediaAssets: MediaAsset[]; + totalDuration: number; + onProgress?: (progress: number) => void; +}): Promise => { + if (totalDuration === 0) { + return createWavBlob({ + samples: new Float32Array( + SAMPLE_RATE * EMPTY_TIMELINE_SILENT_DURATION_SECONDS, + ), + }); + } + + onProgress?.(10); + + const audioBuffer = await createTimelineAudioBuffer({ + tracks, + mediaAssets, + duration: totalDuration, + sampleRate: SAMPLE_RATE, + }); + + if (!audioBuffer) { + const silentDurationSeconds = Math.max( + MIN_SILENT_DURATION_SECONDS, + totalDuration / TICKS_PER_SECOND, + ); + const silentSamples = new Float32Array( + Math.ceil(silentDurationSeconds * SAMPLE_RATE) * NUM_CHANNELS, + ); + return createWavBlob({ samples: silentSamples }); + } + + onProgress?.(90); + + const interleavedSamples = interleaveAudioBuffer({ audioBuffer }); + onProgress?.(100); + + return createWavBlob({ samples: interleavedSamples }); +}; + +function interleaveAudioBuffer({ + audioBuffer, +}: { + audioBuffer: AudioBuffer; +}): Float32Array { + const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels); + const interleavedSamples = new Float32Array( + audioBuffer.length * NUM_CHANNELS, + ); + + for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) { + for (let channel = 0; channel < NUM_CHANNELS; channel++) { + const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1)); + interleavedSamples[sampleIndex * NUM_CHANNELS + channel] = + audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0; + } + } + + return interleavedSamples; +} + +function createWavBlob({ samples }: { samples: Float32Array }): Blob { + const numChannels = NUM_CHANNELS; + const bitsPerSample = 16; + const bytesPerSample = bitsPerSample / 8; + const numSamples = samples.length / numChannels; + const dataSize = numSamples * numChannels * bytesPerSample; + const buffer = new ArrayBuffer(44 + dataSize); + const view = new DataView(buffer); + + // riff header + writeString({ view, offset: 0, str: "RIFF" }); + view.setUint32(4, 36 + dataSize, true); + writeString({ view, offset: 8, str: "WAVE" }); + + // fmt chunk + writeString({ view, offset: 12, str: "fmt " }); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, SAMPLE_RATE, true); + view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true); + view.setUint16(32, numChannels * bytesPerSample, true); + view.setUint16(34, bitsPerSample, true); + + // data chunk + writeString({ view, offset: 36, str: "data" }); + view.setUint32(40, dataSize, true); + + // convert float32 to int16 and write + let offset = 44; + for (let i = 0; i < samples.length; i++) { + const sample = Math.max(-1, Math.min(1, samples[i])); + const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff; + view.setInt16(offset, int16, true); + offset += 2; + } + + return new Blob([buffer], { type: "audio/wav" }); +} + +function writeString({ + view, + offset, + str, +}: { + view: DataView; + offset: number; + str: string; +}): void { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)); + } +} diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts index fe3c1ca4..849dbda3 100644 --- a/apps/web/src/lib/timeline/audio-separation/index.ts +++ b/apps/web/src/lib/timeline/audio-separation/index.ts @@ -49,7 +49,9 @@ export function canToggleSourceAudio( element: TimelineElement, mediaAsset: MediaAudioState | null | undefined, ): element is VideoElement { - return canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset); + return ( + canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset) + ); } export function doesElementHaveEnabledAudio({ @@ -63,7 +65,11 @@ export function doesElementHaveEnabledAudio({ return true; } - return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element }); + return ( + !!mediaAsset && + mediaAsset.hasAudio !== false && + isSourceAudioEnabled({ element }) + ); } export function buildSeparatedAudioElement({ @@ -100,7 +106,9 @@ export function getSourceAudioActionLabel({ }: { element: VideoElement; }): "Extract audio" | "Recover audio" { - return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio"; + return isSourceAudioSeparated({ element }) + ? "Recover audio" + : "Extract audio"; } function cloneVolumeAnimations({ diff --git a/apps/web/src/lib/transcription/diagnostics.ts b/apps/web/src/lib/transcription/diagnostics.ts new file mode 100644 index 00000000..a1921699 --- /dev/null +++ b/apps/web/src/lib/transcription/diagnostics.ts @@ -0,0 +1,25 @@ +import type { DiagnosticsManager } from "@/core/managers/diagnostics-manager"; +import { timelineHasAudio } from "@/lib/media/audio"; + +export const TRANSCRIPTION_DIAGNOSTICS_SCOPE = "transcription"; + +export function registerTranscriptionDiagnostics({ + diagnostics, +}: { + diagnostics: DiagnosticsManager; +}): void { + diagnostics.register({ + id: "transcription.no_audio", + scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE, + severity: "caution", + message: "No audio detected. Add a clip with audio to the timeline first.", + check: (editor) => { + const scene = editor.scenes.getActiveSceneOrNull(); + if (!scene) return false; + return !timelineHasAudio({ + tracks: scene.tracks, + mediaAssets: editor.media.getAssets(), + }); + }, + }); +} -- cgit v1.3.1