summaryrefslogtreecommitdiff
path: root/apps/web/src/lib
diff options
context:
space:
mode:
Diffstat (limited to 'apps/web/src/lib')
-rw-r--r--apps/web/src/lib/diagnostics/types.ts8
-rw-r--r--apps/web/src/lib/media/audio.ts188
-rw-r--r--apps/web/src/lib/media/mediabunny.ts326
-rw-r--r--apps/web/src/lib/timeline/audio-separation/index.ts14
-rw-r--r--apps/web/src/lib/transcription/diagnostics.ts25
5 files changed, 321 insertions, 240 deletions
diff --git a/apps/web/src/lib/diagnostics/types.ts b/apps/web/src/lib/diagnostics/types.ts
new file mode 100644
index 00000000..486f362c
--- /dev/null
+++ b/apps/web/src/lib/diagnostics/types.ts
@@ -0,0 +1,8 @@
+export type DiagnosticSeverity = "caution" | "error";
+
+export interface DiagnosticDefinition {
+ id: string;
+ scope: string;
+ severity: DiagnosticSeverity;
+ message: string;
+}
diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts
index 5a45b6ea..04ea580f 100644
--- a/apps/web/src/lib/media/audio.ts
+++ b/apps/web/src/lib/media/audio.ts
@@ -1,5 +1,6 @@
import type {
AudioElement,
+ VideoElement,
LibraryAudioElement,
RetimeConfig,
SceneTracks,
@@ -12,9 +13,7 @@ import {
hasAnimatedVolume,
resolveEffectiveAudioGain,
} from "@/lib/timeline/audio-state";
-import {
- doesElementHaveEnabledAudio,
-} from "@/lib/timeline/audio-separation";
+import { doesElementHaveEnabledAudio } from "@/lib/timeline/audio-separation";
import { canElementHaveAudio, hasMediaId } from "@/lib/timeline/element-utils";
import { canTrackHaveAudio } from "@/lib/timeline";
import { mediaSupportsAudio } from "@/lib/media/media-utils";
@@ -83,22 +82,23 @@ export async function decodeAudioToFloat32({
return { samples, sampleRate: audioBuffer.sampleRate };
}
-export async function collectAudioElements({
+export interface AudibleElementCandidate {
+ element: AudioElement | VideoElement;
+ mediaAsset: MediaAsset | null;
+}
+
+export function collectAudibleCandidates({
tracks,
mediaAssets,
- audioContext,
}: {
tracks: SceneTracks;
mediaAssets: MediaAsset[];
- audioContext: AudioContext;
-}): Promise<CollectedAudioElement[]> {
- const orderedTracks = [...tracks.overlay, tracks.main, ...tracks.audio];
- const mediaMap = new Map<string, MediaAsset>(
- mediaAssets.map((media) => [media.id, media]),
- );
- const pendingElements: Array<Promise<CollectedAudioElement | null>> = [];
+}): AudibleElementCandidate[] {
+ const allTracks = [...tracks.overlay, tracks.main, ...tracks.audio];
+ const mediaMap = new Map(mediaAssets.map((a) => [a.id, a]));
+ const candidates: AudibleElementCandidate[] = [];
- for (const track of orderedTracks) {
+ for (const track of allTracks) {
if (canTrackHaveAudio(track) && track.muted) continue;
for (const element of track.elements) {
@@ -110,65 +110,95 @@ export async function collectAudioElements({
: null;
if (!doesElementHaveEnabledAudio({ element, mediaAsset })) continue;
- const isTrackMuted = canTrackHaveAudio(track) && track.muted;
+ candidates.push({ element, mediaAsset });
+ }
+ }
- if (element.type === "audio") {
- pendingElements.push(
- resolveAudioBufferForElement({
- element,
- mediaMap,
- audioContext,
- }).then((audioBuffer) => {
- if (!audioBuffer) return null;
- const muted = element.muted === true || isTrackMuted;
- return {
- timelineElement: element,
- buffer: audioBuffer,
- startTime: element.startTime / TICKS_PER_SECOND,
- duration: element.duration / TICKS_PER_SECOND,
- trimStart: element.trimStart / TICKS_PER_SECOND,
- trimEnd: element.trimEnd / TICKS_PER_SECOND,
- volume: resolveEffectiveAudioGain({
- element,
- trackMuted: isTrackMuted,
- localTime: 0,
- }),
- muted,
- retime: element.retime,
- };
- }),
- );
- continue;
- }
+ return candidates;
+}
- if (element.type === "video") {
- if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
+export function timelineHasAudio({
+ tracks,
+ mediaAssets,
+}: {
+ tracks: SceneTracks;
+ mediaAssets: MediaAsset[];
+}): boolean {
+ return collectAudibleCandidates({ tracks, mediaAssets }).some(
+ ({ element }) => element.muted !== true,
+ );
+}
- pendingElements.push(
- resolveAudioBufferForVideoElement({
- mediaAsset,
- audioContext,
- }).then((audioBuffer) => {
- if (!audioBuffer) return null;
- const muted = (element.muted ?? false) || isTrackMuted;
- return {
- timelineElement: element,
- buffer: audioBuffer,
- startTime: element.startTime / TICKS_PER_SECOND,
- duration: element.duration / TICKS_PER_SECOND,
- trimStart: element.trimStart / TICKS_PER_SECOND,
- trimEnd: element.trimEnd / TICKS_PER_SECOND,
- volume: resolveEffectiveAudioGain({
- element,
- trackMuted: isTrackMuted,
- localTime: 0,
- }),
- muted,
- retime: element.retime,
- };
- }),
- );
- }
+export async function collectAudioElements({
+ tracks,
+ mediaAssets,
+ audioContext,
+}: {
+ tracks: SceneTracks;
+ mediaAssets: MediaAsset[];
+ audioContext: AudioContext;
+}): Promise<CollectedAudioElement[]> {
+ const candidates = collectAudibleCandidates({ tracks, mediaAssets });
+ const mediaMap = new Map<string, MediaAsset>(
+ mediaAssets.map((media) => [media.id, media]),
+ );
+ const pendingElements: Array<Promise<CollectedAudioElement | null>> = [];
+
+ for (const { element, mediaAsset } of candidates) {
+ if (element.type === "audio") {
+ pendingElements.push(
+ resolveAudioBufferForElement({
+ element,
+ mediaMap,
+ audioContext,
+ }).then((audioBuffer) => {
+ if (!audioBuffer) return null;
+ return {
+ timelineElement: element,
+ buffer: audioBuffer,
+ startTime: element.startTime / TICKS_PER_SECOND,
+ duration: element.duration / TICKS_PER_SECOND,
+ trimStart: element.trimStart / TICKS_PER_SECOND,
+ trimEnd: element.trimEnd / TICKS_PER_SECOND,
+ volume: resolveEffectiveAudioGain({
+ element,
+ trackMuted: false,
+ localTime: 0,
+ }),
+ muted: element.muted === true,
+ retime: element.retime,
+ };
+ }),
+ );
+ continue;
+ }
+
+ if (element.type === "video") {
+ if (!mediaAsset || !mediaSupportsAudio({ media: mediaAsset })) continue;
+
+ pendingElements.push(
+ resolveAudioBufferForVideoElement({
+ mediaAsset,
+ audioContext,
+ }).then((audioBuffer) => {
+ if (!audioBuffer) return null;
+ return {
+ timelineElement: element,
+ buffer: audioBuffer,
+ startTime: element.startTime / TICKS_PER_SECOND,
+ duration: element.duration / TICKS_PER_SECOND,
+ trimStart: element.trimStart / TICKS_PER_SECOND,
+ trimEnd: element.trimEnd / TICKS_PER_SECOND,
+ volume: resolveEffectiveAudioGain({
+ element,
+ trackMuted: false,
+ localTime: 0,
+ }),
+ muted: element.muted ?? false,
+ retime: element.retime,
+ };
+ }),
+ );
}
}
@@ -339,16 +369,16 @@ async function fetchLibraryAudioSource({
type: "audio/mpeg",
});
- return {
- timelineElement: element,
- file,
- startTime: element.startTime / TICKS_PER_SECOND,
- duration: element.duration / TICKS_PER_SECOND,
- trimStart: element.trimStart / TICKS_PER_SECOND,
- trimEnd: element.trimEnd / TICKS_PER_SECOND,
- volume,
- retime: element.retime,
- };
+ return {
+ timelineElement: element,
+ file,
+ startTime: element.startTime / TICKS_PER_SECOND,
+ duration: element.duration / TICKS_PER_SECOND,
+ trimStart: element.trimStart / TICKS_PER_SECOND,
+ trimEnd: element.trimEnd / TICKS_PER_SECOND,
+ volume,
+ retime: element.retime,
+ };
} catch (error) {
console.warn("Failed to fetch library audio:", error);
return null;
diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts
index a313c6f0..1e97fc42 100644
--- a/apps/web/src/lib/media/mediabunny.ts
+++ b/apps/web/src/lib/media/mediabunny.ts
@@ -1,158 +1,168 @@
-import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
-import { createTimelineAudioBuffer } from "@/lib/media/audio";
-import type { SceneTracks } from "@/lib/timeline";
-import type { MediaAsset } from "@/lib/media/types";
-
-export async function getVideoInfo({
- videoFile,
-}: {
- videoFile: File;
-}): Promise<{
- duration: number;
- width: number;
- height: number;
- fps: number;
- hasAudio: boolean;
-}> {
- const input = new Input({
- source: new BlobSource(videoFile),
- formats: ALL_FORMATS,
- });
-
- const duration = await input.computeDuration();
- const videoTrack = await input.getPrimaryVideoTrack();
-
- if (!videoTrack) {
- throw new Error("No video track found in the file");
- }
-
- const packetStats = await videoTrack.computePacketStats(100);
- const fps = packetStats.averagePacketRate;
- const audioTrack = await input.getPrimaryAudioTrack();
-
- return {
- duration,
- width: videoTrack.displayWidth,
- height: videoTrack.displayHeight,
- fps,
- hasAudio: audioTrack !== null,
- };
-}
-
-const SAMPLE_RATE = 44100;
-const NUM_CHANNELS = 2;
-
-export const extractTimelineAudio = async ({
- tracks,
- mediaAssets,
- totalDuration,
- onProgress,
-}: {
- tracks: SceneTracks;
- mediaAssets: MediaAsset[];
- totalDuration: number;
- onProgress?: (progress: number) => void;
-}): Promise<Blob> => {
- if (totalDuration === 0) {
- return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });
- }
-
- onProgress?.(10);
-
- const audioBuffer = await createTimelineAudioBuffer({
- tracks,
- mediaAssets,
- duration: totalDuration,
- sampleRate: SAMPLE_RATE,
- });
-
- if (!audioBuffer) {
- const silentDuration = Math.max(1, totalDuration);
- const silentSamples = new Float32Array(
- Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,
- );
- return createWavBlob({ samples: silentSamples });
- }
-
- onProgress?.(90);
-
- const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
- onProgress?.(100);
-
- return createWavBlob({ samples: interleavedSamples });
-};
-
-function interleaveAudioBuffer({
- audioBuffer,
-}: {
- audioBuffer: AudioBuffer;
-}): Float32Array {
- const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
- const interleavedSamples = new Float32Array(
- audioBuffer.length * NUM_CHANNELS,
- );
-
- for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
- for (let channel = 0; channel < NUM_CHANNELS; channel++) {
- const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
- interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
- audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
- }
- }
-
- return interleavedSamples;
-}
-
-function createWavBlob({ samples }: { samples: Float32Array }): Blob {
- const numChannels = NUM_CHANNELS;
- const bitsPerSample = 16;
- const bytesPerSample = bitsPerSample / 8;
- const numSamples = samples.length / numChannels;
- const dataSize = numSamples * numChannels * bytesPerSample;
- const buffer = new ArrayBuffer(44 + dataSize);
- const view = new DataView(buffer);
-
- // riff header
- writeString({ view, offset: 0, str: "RIFF" });
- view.setUint32(4, 36 + dataSize, true);
- writeString({ view, offset: 8, str: "WAVE" });
-
- // fmt chunk
- writeString({ view, offset: 12, str: "fmt " });
- view.setUint32(16, 16, true);
- view.setUint16(20, 1, true);
- view.setUint16(22, numChannels, true);
- view.setUint32(24, SAMPLE_RATE, true);
- view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
- view.setUint16(32, numChannels * bytesPerSample, true);
- view.setUint16(34, bitsPerSample, true);
-
- // data chunk
- writeString({ view, offset: 36, str: "data" });
- view.setUint32(40, dataSize, true);
-
- // convert float32 to int16 and write
- let offset = 44;
- for (let i = 0; i < samples.length; i++) {
- const sample = Math.max(-1, Math.min(1, samples[i]));
- const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
- view.setInt16(offset, int16, true);
- offset += 2;
- }
-
- return new Blob([buffer], { type: "audio/wav" });
-}
-
-function writeString({
- view,
- offset,
- str,
-}: {
- view: DataView;
- offset: number;
- str: string;
-}): void {
- for (let i = 0; i < str.length; i++) {
- view.setUint8(offset + i, str.charCodeAt(i));
- }
-}
+import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
+import { createTimelineAudioBuffer } from "@/lib/media/audio";
+import type { SceneTracks } from "@/lib/timeline";
+import type { MediaAsset } from "@/lib/media/types";
+import { TICKS_PER_SECOND } from "@/lib/wasm";
+
+export async function getVideoInfo({
+ videoFile,
+}: {
+ videoFile: File;
+}): Promise<{
+ duration: number;
+ width: number;
+ height: number;
+ fps: number;
+ hasAudio: boolean;
+}> {
+ const input = new Input({
+ source: new BlobSource(videoFile),
+ formats: ALL_FORMATS,
+ });
+
+ const duration = await input.computeDuration();
+ const videoTrack = await input.getPrimaryVideoTrack();
+
+ if (!videoTrack) {
+ throw new Error("No video track found in the file");
+ }
+
+ const packetStats = await videoTrack.computePacketStats(100);
+ const fps = packetStats.averagePacketRate;
+ const audioTrack = await input.getPrimaryAudioTrack();
+
+ return {
+ duration,
+ width: videoTrack.displayWidth,
+ height: videoTrack.displayHeight,
+ fps,
+ hasAudio: audioTrack !== null,
+ };
+}
+
+const SAMPLE_RATE = 44100;
+const NUM_CHANNELS = 2;
+const EMPTY_TIMELINE_SILENT_DURATION_SECONDS = 0.1;
+const MIN_SILENT_DURATION_SECONDS = 0.001;
+
+export const extractTimelineAudio = async ({
+ tracks,
+ mediaAssets,
+ totalDuration,
+ onProgress,
+}: {
+ tracks: SceneTracks;
+ mediaAssets: MediaAsset[];
+ totalDuration: number;
+ onProgress?: (progress: number) => void;
+}): Promise<Blob> => {
+ if (totalDuration === 0) {
+ return createWavBlob({
+ samples: new Float32Array(
+ SAMPLE_RATE * EMPTY_TIMELINE_SILENT_DURATION_SECONDS,
+ ),
+ });
+ }
+
+ onProgress?.(10);
+
+ const audioBuffer = await createTimelineAudioBuffer({
+ tracks,
+ mediaAssets,
+ duration: totalDuration,
+ sampleRate: SAMPLE_RATE,
+ });
+
+ if (!audioBuffer) {
+ const silentDurationSeconds = Math.max(
+ MIN_SILENT_DURATION_SECONDS,
+ totalDuration / TICKS_PER_SECOND,
+ );
+ const silentSamples = new Float32Array(
+ Math.ceil(silentDurationSeconds * SAMPLE_RATE) * NUM_CHANNELS,
+ );
+ return createWavBlob({ samples: silentSamples });
+ }
+
+ onProgress?.(90);
+
+ const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
+ onProgress?.(100);
+
+ return createWavBlob({ samples: interleavedSamples });
+};
+
+function interleaveAudioBuffer({
+ audioBuffer,
+}: {
+ audioBuffer: AudioBuffer;
+}): Float32Array {
+ const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
+ const interleavedSamples = new Float32Array(
+ audioBuffer.length * NUM_CHANNELS,
+ );
+
+ for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
+ for (let channel = 0; channel < NUM_CHANNELS; channel++) {
+ const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
+ interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
+ audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
+ }
+ }
+
+ return interleavedSamples;
+}
+
+function createWavBlob({ samples }: { samples: Float32Array }): Blob {
+ const numChannels = NUM_CHANNELS;
+ const bitsPerSample = 16;
+ const bytesPerSample = bitsPerSample / 8;
+ const numSamples = samples.length / numChannels;
+ const dataSize = numSamples * numChannels * bytesPerSample;
+ const buffer = new ArrayBuffer(44 + dataSize);
+ const view = new DataView(buffer);
+
+ // riff header
+ writeString({ view, offset: 0, str: "RIFF" });
+ view.setUint32(4, 36 + dataSize, true);
+ writeString({ view, offset: 8, str: "WAVE" });
+
+ // fmt chunk
+ writeString({ view, offset: 12, str: "fmt " });
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, numChannels, true);
+ view.setUint32(24, SAMPLE_RATE, true);
+ view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
+ view.setUint16(32, numChannels * bytesPerSample, true);
+ view.setUint16(34, bitsPerSample, true);
+
+ // data chunk
+ writeString({ view, offset: 36, str: "data" });
+ view.setUint32(40, dataSize, true);
+
+ // convert float32 to int16 and write
+ let offset = 44;
+ for (let i = 0; i < samples.length; i++) {
+ const sample = Math.max(-1, Math.min(1, samples[i]));
+ const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+ view.setInt16(offset, int16, true);
+ offset += 2;
+ }
+
+ return new Blob([buffer], { type: "audio/wav" });
+}
+
+function writeString({
+ view,
+ offset,
+ str,
+}: {
+ view: DataView;
+ offset: number;
+ str: string;
+}): void {
+ for (let i = 0; i < str.length; i++) {
+ view.setUint8(offset + i, str.charCodeAt(i));
+ }
+}
diff --git a/apps/web/src/lib/timeline/audio-separation/index.ts b/apps/web/src/lib/timeline/audio-separation/index.ts
index fe3c1ca4..849dbda3 100644
--- a/apps/web/src/lib/timeline/audio-separation/index.ts
+++ b/apps/web/src/lib/timeline/audio-separation/index.ts
@@ -49,7 +49,9 @@ export function canToggleSourceAudio(
element: TimelineElement,
mediaAsset: MediaAudioState | null | undefined,
): element is VideoElement {
- return canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset);
+ return (
+ canRecoverSourceAudio(element) || canExtractSourceAudio(element, mediaAsset)
+ );
}
export function doesElementHaveEnabledAudio({
@@ -63,7 +65,11 @@ export function doesElementHaveEnabledAudio({
return true;
}
- return !!mediaAsset && mediaAsset.hasAudio !== false && isSourceAudioEnabled({ element });
+ return (
+ !!mediaAsset &&
+ mediaAsset.hasAudio !== false &&
+ isSourceAudioEnabled({ element })
+ );
}
export function buildSeparatedAudioElement({
@@ -100,7 +106,9 @@ export function getSourceAudioActionLabel({
}: {
element: VideoElement;
}): "Extract audio" | "Recover audio" {
- return isSourceAudioSeparated({ element }) ? "Recover audio" : "Extract audio";
+ return isSourceAudioSeparated({ element })
+ ? "Recover audio"
+ : "Extract audio";
}
function cloneVolumeAnimations({
diff --git a/apps/web/src/lib/transcription/diagnostics.ts b/apps/web/src/lib/transcription/diagnostics.ts
new file mode 100644
index 00000000..a1921699
--- /dev/null
+++ b/apps/web/src/lib/transcription/diagnostics.ts
@@ -0,0 +1,25 @@
+import type { DiagnosticsManager } from "@/core/managers/diagnostics-manager";
+import { timelineHasAudio } from "@/lib/media/audio";
+
+export const TRANSCRIPTION_DIAGNOSTICS_SCOPE = "transcription";
+
+export function registerTranscriptionDiagnostics({
+ diagnostics,
+}: {
+ diagnostics: DiagnosticsManager;
+}): void {
+ diagnostics.register({
+ id: "transcription.no_audio",
+ scope: TRANSCRIPTION_DIAGNOSTICS_SCOPE,
+ severity: "caution",
+ message: "No audio detected. Add a clip with audio to the timeline first.",
+ check: (editor) => {
+ const scene = editor.scenes.getActiveSceneOrNull();
+ if (!scene) return false;
+ return !timelineHasAudio({
+ tracks: scene.tracks,
+ mediaAssets: editor.media.getAssets(),
+ });
+ },
+ });
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-17 10:27:14 +0000