summaryrefslogtreecommitdiff
path: root/apps/web/src/lib/media
diff options
context:
space:
mode:
Diffstat (limited to 'apps/web/src/lib/media')
-rw-r--r--apps/web/src/lib/media/audio-mastering.ts103
-rw-r--r--apps/web/src/lib/media/audio.ts249
-rw-r--r--apps/web/src/lib/media/media-utils.ts2
-rw-r--r--apps/web/src/lib/media/mediabunny.ts130
-rw-r--r--apps/web/src/lib/media/processing.ts90
-rw-r--r--apps/web/src/lib/media/types.ts9
-rw-r--r--apps/web/src/lib/media/upload-toast.ts51
7 files changed, 464 insertions, 170 deletions
diff --git a/apps/web/src/lib/media/audio-mastering.ts b/apps/web/src/lib/media/audio-mastering.ts
new file mode 100644
index 00000000..29af223b
--- /dev/null
+++ b/apps/web/src/lib/media/audio-mastering.ts
@@ -0,0 +1,103 @@
+const MASTER_LIMITER_THRESHOLD_DB = -1;
+const MASTER_LIMITER_KNEE_DB = 0;
+const MASTER_LIMITER_RATIO = 20;
+const MASTER_LIMITER_ATTACK_SECONDS = 0.001;
+const MASTER_LIMITER_RELEASE_SECONDS = 0.12;
+const MASTER_OUTPUT_HEADROOM = 0.98;
+
+export function getAudioBufferPeak({
+ audioBuffer,
+}: {
+ audioBuffer: AudioBuffer;
+}): number {
+ let peak = 0;
+
+ for (let channel = 0; channel < audioBuffer.numberOfChannels; channel++) {
+ const channelData = audioBuffer.getChannelData(channel);
+ for (let index = 0; index < channelData.length; index++) {
+ const magnitude = Math.abs(channelData[index]);
+ if (magnitude > peak) {
+ peak = magnitude;
+ }
+ }
+ }
+
+ return peak;
+}
+
+export function createAudioMasteringChain({
+ audioContext,
+ destination,
+}: {
+ audioContext: AudioContext | OfflineAudioContext;
+ destination: AudioNode;
+}): {
+ input: GainNode;
+} {
+ const input = audioContext.createGain();
+ const limiter = audioContext.createDynamicsCompressor();
+ const outputGain = audioContext.createGain();
+
+ limiter.threshold.value = MASTER_LIMITER_THRESHOLD_DB;
+ limiter.knee.value = MASTER_LIMITER_KNEE_DB;
+ limiter.ratio.value = MASTER_LIMITER_RATIO;
+ limiter.attack.value = MASTER_LIMITER_ATTACK_SECONDS;
+ limiter.release.value = MASTER_LIMITER_RELEASE_SECONDS;
+ outputGain.gain.value = MASTER_OUTPUT_HEADROOM;
+
+ input.connect(limiter);
+ limiter.connect(outputGain);
+ outputGain.connect(destination);
+
+ return { input };
+}
+
+export async function applyAudioMasteringToBuffer({
+ audioBuffer,
+}: {
+ audioBuffer: AudioBuffer;
+}): Promise<AudioBuffer> {
+ if (getAudioBufferPeak({ audioBuffer }) <= MASTER_OUTPUT_HEADROOM) {
+ return audioBuffer;
+ }
+
+ const offlineContext = new OfflineAudioContext(
+ audioBuffer.numberOfChannels,
+ Math.max(1, audioBuffer.length),
+ audioBuffer.sampleRate,
+ );
+ const source = offlineContext.createBufferSource();
+ source.buffer = audioBuffer;
+
+ const { input } = createAudioMasteringChain({
+ audioContext: offlineContext,
+ destination: offlineContext.destination,
+ });
+ source.connect(input);
+ source.start(0);
+
+ const renderedBuffer = await offlineContext.startRendering();
+ clampAudioBufferPeak({
+ audioBuffer: renderedBuffer,
+ maxPeak: MASTER_OUTPUT_HEADROOM,
+ });
+ return renderedBuffer;
+}
+
+function clampAudioBufferPeak({
+ audioBuffer,
+ maxPeak,
+}: {
+ audioBuffer: AudioBuffer;
+ maxPeak: number;
+}): void {
+ for (let channel = 0; channel < audioBuffer.numberOfChannels; channel++) {
+ const channelData = audioBuffer.getChannelData(channel);
+ for (let index = 0; index < channelData.length; index++) {
+ channelData[index] = Math.max(
+ -maxPeak,
+ Math.min(maxPeak, channelData[index]),
+ );
+ }
+ }
+}
diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts
index e96f158b..41d3b38e 100644
--- a/apps/web/src/lib/media/audio.ts
+++ b/apps/web/src/lib/media/audio.ts
@@ -1,24 +1,44 @@
import type {
AudioElement,
LibraryAudioElement,
- TimelineElement,
+ RetimeConfig,
TimelineTrack,
-} from "@/types/timeline";
-import type { MediaAsset } from "@/types/assets";
+} from "@/lib/timeline";
+import { shouldMaintainPitch } from "@/constants/retime-constants";
+import type { MediaAsset } from "@/lib/media/types";
+import { applyAudioMasteringToBuffer } from "@/lib/media/audio-mastering";
+import type { AudioCapableElement } from "@/lib/timeline/audio-state";
+import {
+ hasAnimatedVolume,
+ resolveEffectiveAudioGain,
+} from "@/lib/timeline/audio-state";
import { canElementHaveAudio } from "@/lib/timeline/element-utils";
import { canTracktHaveAudio } from "@/lib/timeline";
import { mediaSupportsAudio } from "@/lib/media/media-utils";
+import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/lib/retime";
import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
const MAX_AUDIO_CHANNELS = 2;
const EXPORT_SAMPLE_RATE = 44100;
+const COARSE_SAMPLE_COUNT = 2048;
-export type CollectedAudioElement = Omit<
- AudioElement,
- "type" | "mediaId" | "volume" | "id" | "name" | "sourceType" | "sourceUrl"
-> & { buffer: AudioBuffer };
+export interface CollectedAudioElement {
+ timelineElement: AudioCapableElement;
+ buffer: AudioBuffer;
+ startTime: number;
+ duration: number;
+ trimStart: number;
+ trimEnd: number;
+ volume: number;
+ muted: boolean;
+ retime?: RetimeConfig;
+}
-export function createAudioContext({ sampleRate }: { sampleRate?: number } = {}): AudioContext {
+export function createAudioContext({
+ sampleRate,
+}: {
+ sampleRate?: number;
+} = {}): AudioContext {
const AudioContextConstructor =
window.AudioContext ||
(window as typeof window & { webkitAudioContext?: typeof AudioContext })
@@ -90,13 +110,21 @@ export async function collectAudioElements({
audioContext,
}).then((audioBuffer) => {
if (!audioBuffer) return null;
+ const muted = element.muted === true || isTrackMuted;
return {
+ timelineElement: element,
buffer: audioBuffer,
startTime: element.startTime,
duration: element.duration,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
- muted: element.muted || isTrackMuted,
+ volume: resolveEffectiveAudioGain({
+ element,
+ trackMuted: isTrackMuted,
+ localTime: 0,
+ }),
+ muted,
+ retime: element.retime,
};
}),
);
@@ -113,14 +141,21 @@ export async function collectAudioElements({
audioContext,
}).then((audioBuffer) => {
if (!audioBuffer) return null;
- const elementMuted = element.muted ?? false;
+ const muted = (element.muted ?? false) || isTrackMuted;
return {
+ timelineElement: element,
buffer: audioBuffer,
startTime: element.startTime,
duration: element.duration,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
- muted: elementMuted || isTrackMuted,
+ volume: resolveEffectiveAudioGain({
+ element,
+ trackMuted: isTrackMuted,
+ localTime: 0,
+ }),
+ muted,
+ retime: element.retime,
};
}),
);
@@ -199,7 +234,10 @@ async function resolveAudioBufferForVideoElement({
if (chunks.length === 0) return null;
const nativeSampleRate = chunks[0].sampleRate;
- const numChannels = Math.min(MAX_AUDIO_CHANNELS, chunks[0].numberOfChannels);
+ const numChannels = Math.min(
+ MAX_AUDIO_CHANNELS,
+ chunks[0].numberOfChannels,
+ );
const nativeChannels = Array.from(
{ length: numChannels },
@@ -208,17 +246,29 @@ async function resolveAudioBufferForVideoElement({
let offset = 0;
for (const chunk of chunks) {
for (let channel = 0; channel < numChannels; channel++) {
- const sourceData = chunk.getChannelData(Math.min(channel, chunk.numberOfChannels - 1));
+ const sourceData = chunk.getChannelData(
+ Math.min(channel, chunk.numberOfChannels - 1),
+ );
nativeChannels[channel].set(sourceData, offset);
}
offset += chunk.length;
}
// use OfflineAudioContext for high-quality resampling to target rate
- const outputSamples = Math.ceil(totalSamples * (targetSampleRate / nativeSampleRate));
- const offlineContext = new OfflineAudioContext(numChannels, outputSamples, targetSampleRate);
+ const outputSamples = Math.ceil(
+ totalSamples * (targetSampleRate / nativeSampleRate),
+ );
+ const offlineContext = new OfflineAudioContext(
+ numChannels,
+ outputSamples,
+ targetSampleRate,
+ );
- const nativeBuffer = audioContext.createBuffer(numChannels, totalSamples, nativeSampleRate);
+ const nativeBuffer = audioContext.createBuffer(
+ numChannels,
+ totalSamples,
+ nativeSampleRate,
+ );
for (let ch = 0; ch < numChannels; ch++) {
nativeBuffer.copyToChannel(nativeChannels[ch], ch);
}
@@ -238,14 +288,18 @@ async function resolveAudioBufferForVideoElement({
}
interface AudioMixSource {
+ timelineElement: AudioCapableElement;
file: File;
startTime: number;
duration: number;
trimStart: number;
trimEnd: number;
+ volume: number;
+ retime?: RetimeConfig;
}
export interface AudioClipSource {
+ timelineElement: AudioCapableElement;
id: string;
sourceKey: string;
file: File;
@@ -253,13 +307,17 @@ export interface AudioClipSource {
duration: number;
trimStart: number;
trimEnd: number;
+ volume: number;
muted: boolean;
+ retime?: RetimeConfig;
}
async function fetchLibraryAudioSource({
element,
+ volume,
}: {
element: LibraryAudioElement;
+ volume: number;
}): Promise<AudioMixSource | null> {
try {
const response = await fetch(element.sourceUrl);
@@ -273,11 +331,14 @@ async function fetchLibraryAudioSource({
});
return {
+ timelineElement: element,
file,
startTime: element.startTime,
duration: element.duration,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
+ volume,
+ retime: element.retime,
};
} catch (error) {
console.warn("Failed to fetch library audio:", error);
@@ -288,9 +349,11 @@ async function fetchLibraryAudioSource({
async function fetchLibraryAudioClip({
element,
muted,
+ volume,
}: {
element: LibraryAudioElement;
muted: boolean;
+ volume: number;
}): Promise<AudioClipSource | null> {
try {
const response = await fetch(element.sourceUrl);
@@ -304,6 +367,7 @@ async function fetchLibraryAudioClip({
});
return {
+ timelineElement: element,
id: element.id,
sourceKey: element.id,
file,
@@ -311,7 +375,9 @@ async function fetchLibraryAudioClip({
duration: element.duration,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
+ volume,
muted,
+ retime: element.retime,
};
} catch (error) {
console.warn("Failed to fetch library audio:", error);
@@ -322,16 +388,21 @@ async function fetchLibraryAudioClip({
function collectMediaAudioSource({
element,
mediaAsset,
+ volume,
}: {
- element: TimelineElement;
+ element: AudioCapableElement;
mediaAsset: MediaAsset;
+ volume: number;
}): AudioMixSource {
return {
+ timelineElement: element,
file: mediaAsset.file,
startTime: element.startTime,
duration: element.duration,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
+ volume,
+ retime: element.retime,
};
}
@@ -339,12 +410,15 @@ function collectMediaAudioClip({
element,
mediaAsset,
muted,
+ volume,
}: {
- element: TimelineElement;
+ element: AudioCapableElement;
mediaAsset: MediaAsset;
muted: boolean;
+ volume: number;
}): AudioClipSource {
return {
+ timelineElement: element,
id: element.id,
sourceKey: mediaAsset.id,
file: mediaAsset.file,
@@ -352,7 +426,9 @@ function collectMediaAudioClip({
duration: element.duration,
trimStart: element.trimStart,
trimEnd: element.trimEnd,
+ volume,
muted,
+ retime: element.retime,
};
}
@@ -374,6 +450,11 @@ export async function collectAudioMixSources({
for (const element of track.elements) {
if (!canElementHaveAudio(element)) continue;
+ if (element.muted === true) continue;
+ const volume = resolveEffectiveAudioGain({
+ element,
+ localTime: 0,
+ });
if (element.type === "audio") {
if (element.sourceType === "upload") {
@@ -381,10 +462,12 @@ export async function collectAudioMixSources({
if (!mediaAsset) continue;
audioMixSources.push(
- collectMediaAudioSource({ element, mediaAsset }),
+ collectMediaAudioSource({ element, mediaAsset, volume }),
);
} else {
- pendingLibrarySources.push(fetchLibraryAudioSource({ element }));
+ pendingLibrarySources.push(
+ fetchLibraryAudioSource({ element, volume }),
+ );
}
continue;
}
@@ -395,7 +478,7 @@ export async function collectAudioMixSources({
if (mediaSupportsAudio({ media: mediaAsset })) {
audioMixSources.push(
- collectMediaAudioSource({ element, mediaAsset }),
+ collectMediaAudioSource({ element, mediaAsset, volume }),
);
}
}
@@ -432,6 +515,11 @@ export async function collectAudioClips({
const isElementMuted =
"muted" in element ? (element.muted ?? false) : false;
const muted = isTrackMuted || isElementMuted;
+ const volume = resolveEffectiveAudioGain({
+ element,
+ trackMuted: isTrackMuted,
+ localTime: 0,
+ });
if (element.type === "audio") {
if (element.sourceType === "upload") {
@@ -443,10 +531,13 @@ export async function collectAudioClips({
element,
mediaAsset,
muted,
+ volume,
}),
);
} else {
- pendingLibraryClips.push(fetchLibraryAudioClip({ element, muted }));
+ pendingLibraryClips.push(
+ fetchLibraryAudioClip({ element, muted, volume }),
+ );
}
continue;
}
@@ -461,6 +552,7 @@ export async function collectAudioClips({
element,
mediaAsset,
muted,
+ volume,
}),
);
}
@@ -510,36 +602,114 @@ export async function createTimelineAudioBuffer({
for (const element of audioElements) {
if (element.muted) continue;
+ const renderedBuffer = shouldMaintainPitch({
+ rate: element.retime?.rate ?? 1,
+ maintainPitch: element.retime?.maintainPitch,
+ })
+ ? await renderRetimedBuffer({
+ audioContext: context,
+ sourceBuffer: element.buffer,
+ trimStart: element.trimStart,
+ clipDuration: element.duration,
+ retime: element.retime,
+ maintainPitch: true,
+ })
+ : undefined;
+
mixAudioChannels({
element,
+ buffer: renderedBuffer ?? element.buffer,
+ trimStart: renderedBuffer ? 0 : element.trimStart,
+ retime: renderedBuffer ? undefined : element.retime,
outputBuffer,
outputLength,
sampleRate,
});
}
- return outputBuffer;
+ return await applyAudioMasteringToBuffer({ audioBuffer: outputBuffer });
+}
+
+export function computeGlobalMaxRms({
+ buffer,
+}: {
+ buffer: AudioBuffer;
+}): number {
+ const channels = buffer.numberOfChannels;
+ const step = Math.max(1, Math.floor(buffer.length / COARSE_SAMPLE_COUNT));
+ let globalMax = 0;
+
+ for (let c = 0; c < channels; c++) {
+ const data = buffer.getChannelData(c);
+ for (let i = 0; i + step <= buffer.length; i += step) {
+ for (let j = i; j < i + step; j++) {
+ const abs = Math.abs(data[j]);
+ if (abs > globalMax) globalMax = abs;
+ }
+ }
+ }
+
+ return globalMax || 1;
+}
+
+export function extractRmsRange({
+ buffer,
+ count,
+ startSample,
+ endSample,
+ globalMax,
+}: {
+ buffer: AudioBuffer;
+ count: number;
+ startSample: number;
+ endSample: number;
+ globalMax: number;
+}): number[] {
+ const channels = buffer.numberOfChannels;
+ const rangeLength = endSample - startSample;
+ const step = Math.max(1, Math.floor(rangeLength / count));
+ const peaks = new Float32Array(count);
+
+ for (let c = 0; c < channels; c++) {
+ const data = buffer.getChannelData(c);
+ for (let i = 0; i < count; i++) {
+ const start = startSample + i * step;
+ const end = Math.min(start + step, endSample);
+ for (let j = start; j < end; j++) {
+ const abs = Math.abs(data[j]);
+ if (abs > peaks[i]) peaks[i] = abs;
+ }
+ }
+ }
+
+ const norm = 1 / globalMax;
+ const result = new Array<number>(count);
+ for (let i = 0; i < count; i++) result[i] = Math.min(1, peaks[i] * norm);
+
+ return result;
}
function mixAudioChannels({
element,
+ buffer,
+ trimStart,
+ retime,
outputBuffer,
outputLength,
sampleRate,
}: {
element: CollectedAudioElement;
+ buffer: AudioBuffer;
+ trimStart: number;
+ retime?: RetimeConfig;
outputBuffer: AudioBuffer;
outputLength: number;
sampleRate: number;
}): void {
- const { buffer, startTime, trimStart, duration: elementDuration } = element;
+ const { startTime, duration: elementDuration } = element;
- const sourceStartSample = Math.floor(trimStart * buffer.sampleRate);
- const sourceLengthSamples = Math.floor(elementDuration * buffer.sampleRate);
const outputStartSample = Math.floor(startTime * sampleRate);
-
- const resampleRatio = sampleRate / buffer.sampleRate;
- const resampledLength = Math.floor(sourceLengthSamples * resampleRatio);
+ const renderedLength = Math.ceil(elementDuration * sampleRate);
const outputChannels = 2;
for (let channel = 0; channel < outputChannels; channel++) {
@@ -547,14 +717,29 @@ function mixAudioChannels({
const sourceChannel = Math.min(channel, buffer.numberOfChannels - 1);
const sourceData = buffer.getChannelData(sourceChannel);
- for (let i = 0; i < resampledLength; i++) {
+ for (let i = 0; i < renderedLength; i++) {
const outputIndex = outputStartSample + i;
if (outputIndex >= outputLength) break;
- const sourceIndex = sourceStartSample + Math.floor(i / resampleRatio);
+ const clipTime = i / sampleRate;
+ const sourceTime =
+ trimStart + getSourceTimeAtClipTime({ clipTime, retime });
+ const sourceIndex = sourceTime * buffer.sampleRate;
if (sourceIndex >= sourceData.length) break;
- outputData[outputIndex] += sourceData[sourceIndex];
+ const lowerIndex = Math.floor(sourceIndex);
+ const upperIndex = Math.min(sourceData.length - 1, lowerIndex + 1);
+ const fraction = sourceIndex - lowerIndex;
+ const gain = hasAnimatedVolume({ element: element.timelineElement })
+ ? resolveEffectiveAudioGain({
+ element: element.timelineElement,
+ localTime: clipTime,
+ })
+ : element.volume;
+ outputData[outputIndex] +=
+ (sourceData[lowerIndex] * (1 - fraction) +
+ sourceData[upperIndex] * fraction) *
+ gain;
}
}
}
diff --git a/apps/web/src/lib/media/media-utils.ts b/apps/web/src/lib/media/media-utils.ts
index 4b487bf2..52c08f0c 100644
--- a/apps/web/src/lib/media/media-utils.ts
+++ b/apps/web/src/lib/media/media-utils.ts
@@ -1,4 +1,4 @@
-import type { MediaAsset, MediaType } from "@/types/assets";
+import type { MediaAsset, MediaType } from "@/lib/media/types";
export const SUPPORTS_AUDIO: readonly MediaType[] = ["audio", "video"];
diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts
index 11fe9837..c2d1d7f3 100644
--- a/apps/web/src/lib/media/mediabunny.ts
+++ b/apps/web/src/lib/media/mediabunny.ts
@@ -1,7 +1,7 @@
-import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
-import { collectAudioMixSources } from "@/lib/media/audio";
-import type { TimelineTrack } from "@/types/timeline";
-import type { MediaAsset } from "@/types/assets";
+import { Input, ALL_FORMATS, BlobSource } from "mediabunny";
+import { createTimelineAudioBuffer } from "@/lib/media/audio";
+import type { TimelineTrack } from "@/lib/timeline";
+import type { MediaAsset } from "@/lib/media/types";
export async function getVideoInfo({
videoFile,
@@ -12,6 +12,7 @@ export async function getVideoInfo({
width: number;
height: number;
fps: number;
+ hasAudio: boolean;
}> {
const input = new Input({
source: new BlobSource(videoFile),
@@ -27,12 +28,14 @@ export async function getVideoInfo({
const packetStats = await videoTrack.computePacketStats(100);
const fps = packetStats.averagePacketRate;
+ const audioTrack = await input.getPrimaryAudioTrack();
return {
duration,
width: videoTrack.displayWidth,
height: videoTrack.displayHeight,
fps,
+ hasAudio: audioTrack !== null,
};
}
@@ -54,12 +57,16 @@ export const extractTimelineAudio = async ({
return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });
}
- const audioMixSources = await collectAudioMixSources({
+ onProgress?.(10);
+
+ const audioBuffer = await createTimelineAudioBuffer({
tracks,
mediaAssets,
+ duration: totalDuration,
+ sampleRate: SAMPLE_RATE,
});
- if (audioMixSources.length === 0) {
+ if (!audioBuffer) {
const silentDuration = Math.max(1, totalDuration);
const silentSamples = new Float32Array(
Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,
@@ -67,108 +74,33 @@ export const extractTimelineAudio = async ({
return createWavBlob({ samples: silentSamples });
}
- const totalSamples = Math.ceil(totalDuration * SAMPLE_RATE);
- const mixBuffers = [
- new Float32Array(totalSamples),
- new Float32Array(totalSamples),
- ];
-
- for (let i = 0; i < audioMixSources.length; i++) {
- const source = audioMixSources[i];
-
- if (onProgress) {
- onProgress((i / audioMixSources.length) * 90);
- }
-
- try {
- await decodeAndMixAudioSource({
- source,
- mixBuffers,
- totalSamples,
- });
- } catch (error) {
- console.warn(
- `Failed to process audio source ${source.file.name}:`,
- error,
- );
- }
- }
-
- // clamp to prevent clipping
- for (const channel of mixBuffers) {
- for (let i = 0; i < channel.length; i++) {
- channel[i] = Math.max(-1, Math.min(1, channel[i]));
- }
- }
+ onProgress?.(90);
- // interleave channels for wav output
- const interleavedSamples = new Float32Array(totalSamples * NUM_CHANNELS);
- for (let i = 0; i < totalSamples; i++) {
- interleavedSamples[i * 2] = mixBuffers[0][i];
- interleavedSamples[i * 2 + 1] = mixBuffers[1][i];
- }
-
- if (onProgress) {
- onProgress(100);
- }
+ const interleavedSamples = interleaveAudioBuffer({ audioBuffer });
+ onProgress?.(100);
return createWavBlob({ samples: interleavedSamples });
};
-async function decodeAndMixAudioSource({
- source,
- mixBuffers,
- totalSamples,
+function interleaveAudioBuffer({
+ audioBuffer,
}: {
- source: {
- file: File;
- startTime: number;
- duration: number;
- trimStart: number;
- };
- mixBuffers: Float32Array[];
- totalSamples: number;
-}): Promise<void> {
- const input = new Input({
- source: new BlobSource(source.file),
- formats: ALL_FORMATS,
- });
-
- const audioTrack = await input.getPrimaryAudioTrack();
- if (!audioTrack) return;
-
- const sink = new AudioBufferSink(audioTrack);
- const trimEnd = source.trimStart + source.duration;
+ audioBuffer: AudioBuffer;
+}): Float32Array {
+ const numChannels = Math.min(NUM_CHANNELS, audioBuffer.numberOfChannels);
+ const interleavedSamples = new Float32Array(
+ audioBuffer.length * NUM_CHANNELS,
+ );
- for await (const { buffer, timestamp } of sink.buffers(
- source.trimStart,
- trimEnd,
- )) {
- const relativeTime = timestamp - source.trimStart;
- const outputStartSample = Math.floor(
- (source.startTime + relativeTime) * SAMPLE_RATE,
- );
-
- // resample if needed
- const resampleRatio = SAMPLE_RATE / buffer.sampleRate;
-
- for (let ch = 0; ch < NUM_CHANNELS; ch++) {
- const sourceChannel = Math.min(ch, buffer.numberOfChannels - 1);
- const channelData = buffer.getChannelData(sourceChannel);
- const outputChannel = mixBuffers[ch];
-
- const resampledLength = Math.floor(channelData.length * resampleRatio);
- for (let i = 0; i < resampledLength; i++) {
- const outputIdx = outputStartSample + i;
- if (outputIdx < 0 || outputIdx >= totalSamples) continue;
-
- const sourceIdx = Math.floor(i / resampleRatio);
- if (sourceIdx < channelData.length) {
- outputChannel[outputIdx] += channelData[sourceIdx];
- }
- }
+ for (let sampleIndex = 0; sampleIndex < audioBuffer.length; sampleIndex++) {
+ for (let channel = 0; channel < NUM_CHANNELS; channel++) {
+ const sourceChannel = Math.min(channel, Math.max(0, numChannels - 1));
+ interleavedSamples[sampleIndex * NUM_CHANNELS + channel] =
+ audioBuffer.getChannelData(sourceChannel)[sampleIndex] ?? 0;
}
}
+
+ return interleavedSamples;
}
function createWavBlob({ samples }: { samples: Float32Array }): Blob {
diff --git a/apps/web/src/lib/media/processing.ts b/apps/web/src/lib/media/processing.ts
index fbe29acd..547e7ed8 100644
--- a/apps/web/src/lib/media/processing.ts
+++ b/apps/web/src/lib/media/processing.ts
@@ -1,14 +1,34 @@
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
import { toast } from "sonner";
-import type { MediaAsset } from "@/types/assets";
import { getMediaTypeFromFile } from "@/lib/media/media-utils";
+import { formatStorageBytes } from "@/services/storage/quota";
+import { storageService } from "@/services/storage/service";
+import type { MediaAsset } from "@/lib/media/types";
import { getVideoInfo } from "./mediabunny";
-import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
export interface ProcessedMediaAsset extends Omit<MediaAsset, "id"> {}
const THUMBNAIL_MAX_WIDTH = 1280;
const THUMBNAIL_MAX_HEIGHT = 720;
+const getStorageLimitDescription = ({
+ fileSize,
+ availableBytes,
+}: {
+ fileSize: number;
+ availableBytes: number | null;
+}): string => {
+ const fileSizeLabel = formatStorageBytes({ bytes: fileSize });
+
+ if (availableBytes === null) {
+ return `File size is ${fileSizeLabel}.`;
+ }
+
+ return `File size is ${fileSizeLabel}, but only ${formatStorageBytes({
+ bytes: availableBytes,
+ })} is safely available in browser storage.`;
+};
+
const getThumbnailSize = ({
width,
height,
@@ -63,7 +83,7 @@ const renderToThumbnailDataUrl = ({
return canvas.toDataURL("image/jpeg", 0.8);
};
-export async function generateThumbnail({
+async function generateThumbnail({
videoFile,
timeInSeconds,
}: {
@@ -106,25 +126,29 @@ export async function generateThumbnail({
}
}
-export async function generateImageThumbnail({
+async function generateImageThumbnail({
imageFile,
}: {
imageFile: File;
-}): Promise<string> {
+}): Promise<{ thumbnailUrl: string; width: number; height: number }> {
return new Promise((resolve, reject) => {
const image = new window.Image();
const objectUrl = URL.createObjectURL(imageFile);
image.addEventListener("load", () => {
try {
- const dataUrl = renderToThumbnailDataUrl({
+ const thumbnailUrl = renderToThumbnailDataUrl({
width: image.naturalWidth,
height: image.naturalHeight,
draw: ({ context, width, height }) => {
context.drawImage(image, 0, 0, width, height);
},
});
- resolve(dataUrl);
+ resolve({
+ thumbnailUrl,
+ width: image.naturalWidth,
+ height: image.naturalHeight,
+ });
} catch (error) {
reject(
error instanceof Error ? error : new Error("Could not render image"),
@@ -166,19 +190,34 @@ export async function processMediaAssets({
continue;
}
+ const storageCheck = await storageService.canStoreFile({
+ size: file.size,
+ });
+
+ if (!storageCheck.canStore) {
+ toast.error(`Not enough browser storage for ${file.name}`, {
+ description: getStorageLimitDescription({
+ fileSize: file.size,
+ availableBytes: storageCheck.availableBytes,
+ }),
+ });
+ continue;
+ }
+
const url = URL.createObjectURL(file);
let thumbnailUrl: string | undefined;
let duration: number | undefined;
let width: number | undefined;
let height: number | undefined;
let fps: number | undefined;
+ let hasAudio: boolean | undefined;
try {
if (fileType === "image") {
- const dimensions = await getImageDimensions({ file });
- width = dimensions.width;
- height = dimensions.height;
- thumbnailUrl = await generateImageThumbnail({ imageFile: file });
+ const result = await generateImageThumbnail({ imageFile: file });
+ thumbnailUrl = result.thumbnailUrl;
+ width = result.width;
+ height = result.height;
} else if (fileType === "video") {
try {
const videoInfo = await getVideoInfo({ videoFile: file });
@@ -188,6 +227,7 @@ export async function processMediaAssets({
fps = Number.isFinite(videoInfo.fps)
? Math.round(videoInfo.fps)
: undefined;
+ hasAudio = videoInfo.hasAudio;
thumbnailUrl = await generateThumbnail({
videoFile: file,
@@ -211,6 +251,7 @@ export async function processMediaAssets({
width,
height,
fps,
+ hasAudio,
});
await new Promise((resolve) => setTimeout(resolve, 0));
@@ -230,33 +271,6 @@ export async function processMediaAssets({
return processedAssets;
}
-const getImageDimensions = ({
- file,
-}: {
- file: File;
-}): Promise<{ width: number; height: number }> => {
- return new Promise((resolve, reject) => {
- const img = new window.Image();
- const objectUrl = URL.createObjectURL(file);
-
- img.addEventListener("load", () => {
- const width = img.naturalWidth;
- const height = img.naturalHeight;
- resolve({ width, height });
- URL.revokeObjectURL(objectUrl);
- img.remove();
- });
-
- img.addEventListener("error", () => {
- reject(new Error("Could not load image"));
- URL.revokeObjectURL(objectUrl);
- img.remove();
- });
-
- img.src = objectUrl;
- });
-};
-
const getMediaDuration = ({ file }: { file: File }): Promise<number> => {
return new Promise((resolve, reject) => {
const element = document.createElement(
diff --git a/apps/web/src/lib/media/types.ts b/apps/web/src/lib/media/types.ts
new file mode 100644
index 00000000..c8dddfc7
--- /dev/null
+++ b/apps/web/src/lib/media/types.ts
@@ -0,0 +1,9 @@
+import type { MediaAssetData } from "@/services/storage/types";
+
+export type MediaType = "image" | "video" | "audio";
+
+export interface MediaAsset
+ extends Omit<MediaAssetData, "size" | "lastModified"> {
+ file: File;
+ url?: string;
+}
diff --git a/apps/web/src/lib/media/upload-toast.ts b/apps/web/src/lib/media/upload-toast.ts
new file mode 100644
index 00000000..e5c5ee28
--- /dev/null
+++ b/apps/web/src/lib/media/upload-toast.ts
@@ -0,0 +1,51 @@
+import { toast } from "sonner";
+
+export interface MediaUploadToastResult {
+ uploadedCount: number;
+ assetNames?: string[];
+}
+
+function getAssetLabel({ count }: { count: number }): string {
+ return count === 1 ? "media asset" : "media assets";
+}
+
+function waitForNextPaint(): Promise<void> {
+ return new Promise((resolve) => {
+ requestAnimationFrame(() => {
+ requestAnimationFrame(() => resolve());
+ });
+ });
+}
+
+export async function showMediaUploadToast<T extends MediaUploadToastResult>({
+ filesCount,
+ promise,
+}: {
+ filesCount: number;
+ promise: Promise<T> | (() => Promise<T>);
+}) {
+ const run = typeof promise === "function" ? promise : () => promise;
+ const toastPromise = toast.promise(async () => {
+ await waitForNextPaint();
+ return run();
+ }, {
+ loading: `Uploading ${getAssetLabel({ count: filesCount })}...`,
+ success: ({ uploadedCount, assetNames }) => {
+ if (uploadedCount === 1) {
+ const assetName = assetNames?.[0];
+ return assetName
+ ? `${assetName} has been uploaded`
+ : "1 media asset has been uploaded";
+ }
+
+ if (uploadedCount > 1) {
+ return `${uploadedCount} media assets have been uploaded`;
+ }
+
+ return "No media assets were uploaded";
+ },
+ error: `Failed to upload ${getAssetLabel({ count: filesCount })}`,
+ });
+
+ return toastPromise.unwrap();
+}
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-17 15:21:39 +0000