summaryrefslogtreecommitdiff
path: root/apps/web/src/lib/media
diff options
context:
space:
mode:
Diffstat (limited to 'apps/web/src/lib/media')
-rw-r--r--apps/web/src/lib/media/audio.ts459
-rw-r--r--apps/web/src/lib/media/media-utils.ts32
-rw-r--r--apps/web/src/lib/media/mediabunny.ts226
-rw-r--r--apps/web/src/lib/media/processing.ts282
4 files changed, 999 insertions, 0 deletions
diff --git a/apps/web/src/lib/media/audio.ts b/apps/web/src/lib/media/audio.ts
new file mode 100644
index 00000000..b45e348e
--- /dev/null
+++ b/apps/web/src/lib/media/audio.ts
@@ -0,0 +1,459 @@
+import type {
+ AudioElement,
+ LibraryAudioElement,
+ TimelineElement,
+ TimelineTrack,
+} from "@/types/timeline";
+import type { MediaAsset } from "@/types/assets";
+import { canElementHaveAudio } from "@/lib/timeline/element-utils";
+import { canTracktHaveAudio } from "@/lib/timeline";
+import { mediaSupportsAudio } from "@/lib/media/media-utils";
+
+export type CollectedAudioElement = Omit<
+ AudioElement,
+ "type" | "mediaId" | "volume" | "id" | "name" | "sourceType" | "sourceUrl"
+> & { buffer: AudioBuffer };
+
+export function createAudioContext(): AudioContext {
+ const AudioContextConstructor =
+ window.AudioContext ||
+ (window as typeof window & { webkitAudioContext?: typeof AudioContext })
+ .webkitAudioContext;
+
+ return new AudioContextConstructor();
+}
+
+export interface DecodedAudio {
+ samples: Float32Array;
+ sampleRate: number;
+}
+
+export async function decodeAudioToFloat32({
+ audioBlob,
+}: {
+ audioBlob: Blob;
+}): Promise<DecodedAudio> {
+ const audioContext = createAudioContext();
+ const arrayBuffer = await audioBlob.arrayBuffer();
+ const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+
+ // mix down to mono
+ const numChannels = audioBuffer.numberOfChannels;
+ const length = audioBuffer.length;
+ const samples = new Float32Array(length);
+
+ for (let i = 0; i < length; i++) {
+ let sum = 0;
+ for (let channel = 0; channel < numChannels; channel++) {
+ sum += audioBuffer.getChannelData(channel)[i];
+ }
+ samples[i] = sum / numChannels;
+ }
+
+ return { samples, sampleRate: audioBuffer.sampleRate };
+}
+
+export async function collectAudioElements({
+ tracks,
+ mediaAssets,
+ audioContext,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+ audioContext: AudioContext;
+}): Promise<CollectedAudioElement[]> {
+ const mediaMap = new Map<string, MediaAsset>(
+ mediaAssets.map((media) => [media.id, media]),
+ );
+ const pendingElements: Array<Promise<CollectedAudioElement | null>> = [];
+
+ for (const track of tracks) {
+ if (canTracktHaveAudio(track) && track.muted) continue;
+
+ for (const element of track.elements) {
+ if (element.type !== "audio") continue;
+ if (element.duration <= 0) continue;
+
+ const isTrackMuted = canTracktHaveAudio(track) && track.muted;
+ pendingElements.push(
+ resolveAudioBufferForElement({
+ element,
+ mediaMap,
+ audioContext,
+ }).then((audioBuffer) => {
+ if (!audioBuffer) return null;
+ return {
+ buffer: audioBuffer,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ muted: element.muted || isTrackMuted,
+ };
+ }),
+ );
+ }
+ }
+
+ const resolvedElements = await Promise.all(pendingElements);
+ const audioElements: CollectedAudioElement[] = [];
+ for (const element of resolvedElements) {
+ if (element) audioElements.push(element);
+ }
+ return audioElements;
+}
+
+async function resolveAudioBufferForElement({
+ element,
+ mediaMap,
+ audioContext,
+}: {
+ element: AudioElement;
+ mediaMap: Map<string, MediaAsset>;
+ audioContext: AudioContext;
+}): Promise<AudioBuffer | null> {
+ try {
+ if (element.sourceType === "upload") {
+ const asset = mediaMap.get(element.mediaId);
+ if (!asset || asset.type !== "audio") return null;
+
+ const arrayBuffer = await asset.file.arrayBuffer();
+ return await audioContext.decodeAudioData(arrayBuffer.slice(0));
+ }
+
+ if (element.buffer) return element.buffer;
+
+ const response = await fetch(element.sourceUrl);
+ if (!response.ok) {
+ throw new Error(`Library audio fetch failed: ${response.status}`);
+ }
+
+ const arrayBuffer = await response.arrayBuffer();
+ return await audioContext.decodeAudioData(arrayBuffer.slice(0));
+ } catch (error) {
+ console.warn("Failed to decode audio:", error);
+ return null;
+ }
+}
+
+interface AudioMixSource {
+ file: File;
+ startTime: number;
+ duration: number;
+ trimStart: number;
+ trimEnd: number;
+}
+
+export interface AudioClipSource {
+ id: string;
+ sourceKey: string;
+ file: File;
+ startTime: number;
+ duration: number;
+ trimStart: number;
+ trimEnd: number;
+ muted: boolean;
+}
+
+async function fetchLibraryAudioSource({
+ element,
+}: {
+ element: LibraryAudioElement;
+}): Promise<AudioMixSource | null> {
+ try {
+ const response = await fetch(element.sourceUrl);
+ if (!response.ok) {
+ throw new Error(`Library audio fetch failed: ${response.status}`);
+ }
+
+ const blob = await response.blob();
+ const file = new File([blob], `${element.name}.mp3`, {
+ type: "audio/mpeg",
+ });
+
+ return {
+ file,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ };
+ } catch (error) {
+ console.warn("Failed to fetch library audio:", error);
+ return null;
+ }
+}
+
+async function fetchLibraryAudioClip({
+ element,
+ muted,
+}: {
+ element: LibraryAudioElement;
+ muted: boolean;
+}): Promise<AudioClipSource | null> {
+ try {
+ const response = await fetch(element.sourceUrl);
+ if (!response.ok) {
+ throw new Error(`Library audio fetch failed: ${response.status}`);
+ }
+
+ const blob = await response.blob();
+ const file = new File([blob], `${element.name}.mp3`, {
+ type: "audio/mpeg",
+ });
+
+ return {
+ id: element.id,
+ sourceKey: element.id,
+ file,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ muted,
+ };
+ } catch (error) {
+ console.warn("Failed to fetch library audio:", error);
+ return null;
+ }
+}
+
+function collectMediaAudioSource({
+ element,
+ mediaAsset,
+}: {
+ element: TimelineElement;
+ mediaAsset: MediaAsset;
+}): AudioMixSource {
+ return {
+ file: mediaAsset.file,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ };
+}
+
+function collectMediaAudioClip({
+ element,
+ mediaAsset,
+ muted,
+}: {
+ element: TimelineElement;
+ mediaAsset: MediaAsset;
+ muted: boolean;
+}): AudioClipSource {
+ return {
+ id: element.id,
+ sourceKey: mediaAsset.id,
+ file: mediaAsset.file,
+ startTime: element.startTime,
+ duration: element.duration,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ muted,
+ };
+}
+
+export async function collectAudioMixSources({
+ tracks,
+ mediaAssets,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+}): Promise<AudioMixSource[]> {
+ const audioMixSources: AudioMixSource[] = [];
+ const mediaMap = new Map<string, MediaAsset>(
+ mediaAssets.map((asset) => [asset.id, asset]),
+ );
+ const pendingLibrarySources: Array<Promise<AudioMixSource | null>> = [];
+
+ for (const track of tracks) {
+ if (canTracktHaveAudio(track) && track.muted) continue;
+
+ for (const element of track.elements) {
+ if (!canElementHaveAudio(element)) continue;
+
+ if (element.type === "audio") {
+ if (element.sourceType === "upload") {
+ const mediaAsset = mediaMap.get(element.mediaId);
+ if (!mediaAsset) continue;
+
+ audioMixSources.push(
+ collectMediaAudioSource({ element, mediaAsset }),
+ );
+ } else {
+ pendingLibrarySources.push(fetchLibraryAudioSource({ element }));
+ }
+ continue;
+ }
+
+ if (element.type === "video") {
+ const mediaAsset = mediaMap.get(element.mediaId);
+ if (!mediaAsset) continue;
+
+ if (mediaSupportsAudio({ media: mediaAsset })) {
+ audioMixSources.push(
+ collectMediaAudioSource({ element, mediaAsset }),
+ );
+ }
+ }
+ }
+ }
+
+ const resolvedLibrarySources = await Promise.all(pendingLibrarySources);
+ for (const source of resolvedLibrarySources) {
+ if (source) audioMixSources.push(source);
+ }
+
+ return audioMixSources;
+}
+
+export async function collectAudioClips({
+ tracks,
+ mediaAssets,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+}): Promise<AudioClipSource[]> {
+ const clips: AudioClipSource[] = [];
+ const mediaMap = new Map<string, MediaAsset>(
+ mediaAssets.map((asset) => [asset.id, asset]),
+ );
+ const pendingLibraryClips: Array<Promise<AudioClipSource | null>> = [];
+
+ for (const track of tracks) {
+ const isTrackMuted = canTracktHaveAudio(track) && track.muted;
+
+ for (const element of track.elements) {
+ if (!canElementHaveAudio(element)) continue;
+
+ const isElementMuted =
+ "muted" in element ? (element.muted ?? false) : false;
+ const muted = isTrackMuted || isElementMuted;
+
+ if (element.type === "audio") {
+ if (element.sourceType === "upload") {
+ const mediaAsset = mediaMap.get(element.mediaId);
+ if (!mediaAsset) continue;
+
+ clips.push(
+ collectMediaAudioClip({
+ element,
+ mediaAsset,
+ muted,
+ }),
+ );
+ } else {
+ pendingLibraryClips.push(fetchLibraryAudioClip({ element, muted }));
+ }
+ continue;
+ }
+
+ if (element.type === "video") {
+ const mediaAsset = mediaMap.get(element.mediaId);
+ if (!mediaAsset) continue;
+
+ if (mediaSupportsAudio({ media: mediaAsset })) {
+ clips.push(
+ collectMediaAudioClip({
+ element,
+ mediaAsset,
+ muted,
+ }),
+ );
+ }
+ }
+ }
+ }
+
+ const resolvedLibraryClips = await Promise.all(pendingLibraryClips);
+ for (const clip of resolvedLibraryClips) {
+ if (clip) clips.push(clip);
+ }
+
+ return clips;
+}
+
+export async function createTimelineAudioBuffer({
+ tracks,
+ mediaAssets,
+ duration,
+ sampleRate = 44100,
+ audioContext,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+ duration: number;
+ sampleRate?: number;
+ audioContext?: AudioContext;
+}): Promise<AudioBuffer | null> {
+ const context = audioContext ?? createAudioContext();
+
+ const audioElements = await collectAudioElements({
+ tracks,
+ mediaAssets,
+ audioContext: context,
+ });
+
+ if (audioElements.length === 0) return null;
+
+ const outputChannels = 2;
+ const outputLength = Math.ceil(duration * sampleRate);
+ const outputBuffer = context.createBuffer(
+ outputChannels,
+ outputLength,
+ sampleRate,
+ );
+
+ for (const element of audioElements) {
+ if (element.muted) continue;
+
+ mixAudioChannels({
+ element,
+ outputBuffer,
+ outputLength,
+ sampleRate,
+ });
+ }
+
+ return outputBuffer;
+}
+
+function mixAudioChannels({
+ element,
+ outputBuffer,
+ outputLength,
+ sampleRate,
+}: {
+ element: CollectedAudioElement;
+ outputBuffer: AudioBuffer;
+ outputLength: number;
+ sampleRate: number;
+}): void {
+ const { buffer, startTime, trimStart, duration: elementDuration } = element;
+
+ const sourceStartSample = Math.floor(trimStart * buffer.sampleRate);
+ const sourceLengthSamples = Math.floor(elementDuration * buffer.sampleRate);
+ const outputStartSample = Math.floor(startTime * sampleRate);
+
+ const resampleRatio = sampleRate / buffer.sampleRate;
+ const resampledLength = Math.floor(sourceLengthSamples * resampleRatio);
+
+ const outputChannels = 2;
+ for (let channel = 0; channel < outputChannels; channel++) {
+ const outputData = outputBuffer.getChannelData(channel);
+ const sourceChannel = Math.min(channel, buffer.numberOfChannels - 1);
+ const sourceData = buffer.getChannelData(sourceChannel);
+
+ for (let i = 0; i < resampledLength; i++) {
+ const outputIndex = outputStartSample + i;
+ if (outputIndex >= outputLength) break;
+
+ const sourceIndex = sourceStartSample + Math.floor(i / resampleRatio);
+ if (sourceIndex >= sourceData.length) break;
+
+ outputData[outputIndex] += sourceData[sourceIndex];
+ }
+ }
+}
diff --git a/apps/web/src/lib/media/media-utils.ts b/apps/web/src/lib/media/media-utils.ts
new file mode 100644
index 00000000..4b487bf2
--- /dev/null
+++ b/apps/web/src/lib/media/media-utils.ts
@@ -0,0 +1,32 @@
+import type { MediaAsset, MediaType } from "@/types/assets";
+
+export const SUPPORTS_AUDIO: readonly MediaType[] = ["audio", "video"];
+
+export function mediaSupportsAudio({
+ media,
+}: {
+ media: MediaAsset | null | undefined;
+}): boolean {
+ if (!media) return false;
+ return SUPPORTS_AUDIO.includes(media.type);
+}
+
+export const getMediaTypeFromFile = ({
+ file,
+}: {
+ file: File;
+}): MediaType | null => {
+ const { type } = file;
+
+ if (type.startsWith("image/")) {
+ return "image";
+ }
+ if (type.startsWith("video/")) {
+ return "video";
+ }
+ if (type.startsWith("audio/")) {
+ return "audio";
+ }
+
+ return null;
+};
diff --git a/apps/web/src/lib/media/mediabunny.ts b/apps/web/src/lib/media/mediabunny.ts
new file mode 100644
index 00000000..11fe9837
--- /dev/null
+++ b/apps/web/src/lib/media/mediabunny.ts
@@ -0,0 +1,226 @@
+import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny";
+import { collectAudioMixSources } from "@/lib/media/audio";
+import type { TimelineTrack } from "@/types/timeline";
+import type { MediaAsset } from "@/types/assets";
+
+export async function getVideoInfo({
+ videoFile,
+}: {
+ videoFile: File;
+}): Promise<{
+ duration: number;
+ width: number;
+ height: number;
+ fps: number;
+}> {
+ const input = new Input({
+ source: new BlobSource(videoFile),
+ formats: ALL_FORMATS,
+ });
+
+ const duration = await input.computeDuration();
+ const videoTrack = await input.getPrimaryVideoTrack();
+
+ if (!videoTrack) {
+ throw new Error("No video track found in the file");
+ }
+
+ const packetStats = await videoTrack.computePacketStats(100);
+ const fps = packetStats.averagePacketRate;
+
+ return {
+ duration,
+ width: videoTrack.displayWidth,
+ height: videoTrack.displayHeight,
+ fps,
+ };
+}
+
+const SAMPLE_RATE = 44100;
+const NUM_CHANNELS = 2;
+
+export const extractTimelineAudio = async ({
+ tracks,
+ mediaAssets,
+ totalDuration,
+ onProgress,
+}: {
+ tracks: TimelineTrack[];
+ mediaAssets: MediaAsset[];
+ totalDuration: number;
+ onProgress?: (progress: number) => void;
+}): Promise<Blob> => {
+ if (totalDuration === 0) {
+ return createWavBlob({ samples: new Float32Array(SAMPLE_RATE * 0.1) });
+ }
+
+ const audioMixSources = await collectAudioMixSources({
+ tracks,
+ mediaAssets,
+ });
+
+ if (audioMixSources.length === 0) {
+ const silentDuration = Math.max(1, totalDuration);
+ const silentSamples = new Float32Array(
+ Math.ceil(silentDuration * SAMPLE_RATE) * NUM_CHANNELS,
+ );
+ return createWavBlob({ samples: silentSamples });
+ }
+
+ const totalSamples = Math.ceil(totalDuration * SAMPLE_RATE);
+ const mixBuffers = [
+ new Float32Array(totalSamples),
+ new Float32Array(totalSamples),
+ ];
+
+ for (let i = 0; i < audioMixSources.length; i++) {
+ const source = audioMixSources[i];
+
+ if (onProgress) {
+ onProgress((i / audioMixSources.length) * 90);
+ }
+
+ try {
+ await decodeAndMixAudioSource({
+ source,
+ mixBuffers,
+ totalSamples,
+ });
+ } catch (error) {
+ console.warn(
+ `Failed to process audio source ${source.file.name}:`,
+ error,
+ );
+ }
+ }
+
+ // clamp to prevent clipping
+ for (const channel of mixBuffers) {
+ for (let i = 0; i < channel.length; i++) {
+ channel[i] = Math.max(-1, Math.min(1, channel[i]));
+ }
+ }
+
+ // interleave channels for wav output
+ const interleavedSamples = new Float32Array(totalSamples * NUM_CHANNELS);
+ for (let i = 0; i < totalSamples; i++) {
+ interleavedSamples[i * 2] = mixBuffers[0][i];
+ interleavedSamples[i * 2 + 1] = mixBuffers[1][i];
+ }
+
+ if (onProgress) {
+ onProgress(100);
+ }
+
+ return createWavBlob({ samples: interleavedSamples });
+};
+
+async function decodeAndMixAudioSource({
+ source,
+ mixBuffers,
+ totalSamples,
+}: {
+ source: {
+ file: File;
+ startTime: number;
+ duration: number;
+ trimStart: number;
+ };
+ mixBuffers: Float32Array[];
+ totalSamples: number;
+}): Promise<void> {
+ const input = new Input({
+ source: new BlobSource(source.file),
+ formats: ALL_FORMATS,
+ });
+
+ const audioTrack = await input.getPrimaryAudioTrack();
+ if (!audioTrack) return;
+
+ const sink = new AudioBufferSink(audioTrack);
+ const trimEnd = source.trimStart + source.duration;
+
+ for await (const { buffer, timestamp } of sink.buffers(
+ source.trimStart,
+ trimEnd,
+ )) {
+ const relativeTime = timestamp - source.trimStart;
+ const outputStartSample = Math.floor(
+ (source.startTime + relativeTime) * SAMPLE_RATE,
+ );
+
+ // resample if needed
+ const resampleRatio = SAMPLE_RATE / buffer.sampleRate;
+
+ for (let ch = 0; ch < NUM_CHANNELS; ch++) {
+ const sourceChannel = Math.min(ch, buffer.numberOfChannels - 1);
+ const channelData = buffer.getChannelData(sourceChannel);
+ const outputChannel = mixBuffers[ch];
+
+ const resampledLength = Math.floor(channelData.length * resampleRatio);
+ for (let i = 0; i < resampledLength; i++) {
+ const outputIdx = outputStartSample + i;
+ if (outputIdx < 0 || outputIdx >= totalSamples) continue;
+
+ const sourceIdx = Math.floor(i / resampleRatio);
+ if (sourceIdx < channelData.length) {
+ outputChannel[outputIdx] += channelData[sourceIdx];
+ }
+ }
+ }
+ }
+}
+
+function createWavBlob({ samples }: { samples: Float32Array }): Blob {
+ const numChannels = NUM_CHANNELS;
+ const bitsPerSample = 16;
+ const bytesPerSample = bitsPerSample / 8;
+ const numSamples = samples.length / numChannels;
+ const dataSize = numSamples * numChannels * bytesPerSample;
+ const buffer = new ArrayBuffer(44 + dataSize);
+ const view = new DataView(buffer);
+
+ // riff header
+ writeString({ view, offset: 0, str: "RIFF" });
+ view.setUint32(4, 36 + dataSize, true);
+ writeString({ view, offset: 8, str: "WAVE" });
+
+ // fmt chunk
+ writeString({ view, offset: 12, str: "fmt " });
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, numChannels, true);
+ view.setUint32(24, SAMPLE_RATE, true);
+ view.setUint32(28, SAMPLE_RATE * numChannels * bytesPerSample, true);
+ view.setUint16(32, numChannels * bytesPerSample, true);
+ view.setUint16(34, bitsPerSample, true);
+
+ // data chunk
+ writeString({ view, offset: 36, str: "data" });
+ view.setUint32(40, dataSize, true);
+
+ // convert float32 to int16 and write
+ let offset = 44;
+ for (let i = 0; i < samples.length; i++) {
+ const sample = Math.max(-1, Math.min(1, samples[i]));
+ const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+ view.setInt16(offset, int16, true);
+ offset += 2;
+ }
+
+ return new Blob([buffer], { type: "audio/wav" });
+}
+
+function writeString({
+ view,
+ offset,
+ str,
+}: {
+ view: DataView;
+ offset: number;
+ str: string;
+}): void {
+ for (let i = 0; i < str.length; i++) {
+ view.setUint8(offset + i, str.charCodeAt(i));
+ }
+}
diff --git a/apps/web/src/lib/media/processing.ts b/apps/web/src/lib/media/processing.ts
new file mode 100644
index 00000000..8f9c045c
--- /dev/null
+++ b/apps/web/src/lib/media/processing.ts
@@ -0,0 +1,282 @@
+import { toast } from "sonner";
+import type { MediaAsset } from "@/types/assets";
+import { getMediaTypeFromFile } from "@/lib/media/media-utils";
+import { getVideoInfo } from "./mediabunny";
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
+
+export interface ProcessedMediaAsset extends Omit<MediaAsset, "id"> {}
+
+const THUMBNAIL_MAX_WIDTH = 1280;
+const THUMBNAIL_MAX_HEIGHT = 720;
+
+const getThumbnailSize = ({
+ width,
+ height,
+}: {
+ width: number;
+ height: number;
+}): { width: number; height: number } => {
+ const aspectRatio = width / height;
+ let targetWidth = width;
+ let targetHeight = height;
+
+ if (targetWidth > THUMBNAIL_MAX_WIDTH) {
+ targetWidth = THUMBNAIL_MAX_WIDTH;
+ targetHeight = Math.round(targetWidth / aspectRatio);
+ }
+ if (targetHeight > THUMBNAIL_MAX_HEIGHT) {
+ targetHeight = THUMBNAIL_MAX_HEIGHT;
+ targetWidth = Math.round(targetHeight * aspectRatio);
+ }
+
+ return { width: targetWidth, height: targetHeight };
+};
+
+const renderToThumbnailDataUrl = ({
+ width,
+ height,
+ draw,
+}: {
+ width: number;
+ height: number;
+ draw: ({
+ context,
+ width,
+ height,
+ }: {
+ context: CanvasRenderingContext2D;
+ width: number;
+ height: number;
+ }) => void;
+}): string => {
+ const size = getThumbnailSize({ width, height });
+ const canvas = document.createElement("canvas");
+ canvas.width = size.width;
+ canvas.height = size.height;
+ const context = canvas.getContext("2d");
+
+ if (!context) {
+ throw new Error("Could not get canvas context");
+ }
+
+ draw({ context, width: size.width, height: size.height });
+ return canvas.toDataURL("image/jpeg", 0.8);
+};
+
+export async function generateThumbnail({
+ videoFile,
+ timeInSeconds,
+}: {
+ videoFile: File;
+ timeInSeconds: number;
+}): Promise<string> {
+ const input = new Input({
+ source: new BlobSource(videoFile),
+ formats: ALL_FORMATS,
+ });
+
+ const videoTrack = await input.getPrimaryVideoTrack();
+ if (!videoTrack) {
+ throw new Error("No video track found in the file");
+ }
+
+ const canDecode = await videoTrack.canDecode();
+ if (!canDecode) {
+ throw new Error("Video codec not supported for decoding");
+ }
+
+ const sink = new VideoSampleSink(videoTrack);
+
+ const frame = await sink.getSample(timeInSeconds);
+
+ if (!frame) {
+ throw new Error("Could not get frame at specified time");
+ }
+
+ try {
+ return renderToThumbnailDataUrl({
+ width: videoTrack.displayWidth,
+ height: videoTrack.displayHeight,
+ draw: ({ context, width, height }) => {
+ frame.draw(context, 0, 0, width, height);
+ },
+ });
+ } finally {
+ frame.close();
+ }
+}
+
+export async function generateImageThumbnail({
+ imageFile,
+}: {
+ imageFile: File;
+}): Promise<string> {
+ return new Promise((resolve, reject) => {
+ const image = new window.Image();
+ const objectUrl = URL.createObjectURL(imageFile);
+
+ image.addEventListener("load", () => {
+ try {
+ const dataUrl = renderToThumbnailDataUrl({
+ width: image.naturalWidth,
+ height: image.naturalHeight,
+ draw: ({ context, width, height }) => {
+ context.drawImage(image, 0, 0, width, height);
+ },
+ });
+ resolve(dataUrl);
+ } catch (error) {
+ reject(
+ error instanceof Error ? error : new Error("Could not render image"),
+ );
+ } finally {
+ URL.revokeObjectURL(objectUrl);
+ image.remove();
+ }
+ });
+
+ image.addEventListener("error", () => {
+ URL.revokeObjectURL(objectUrl);
+ image.remove();
+ reject(new Error("Could not load image"));
+ });
+
+ image.src = objectUrl;
+ });
+}
+
+export async function processMediaAssets({
+ files,
+ onProgress,
+}: {
+ files: FileList | File[];
+ onProgress?: ({ progress }: { progress: number }) => void;
+}): Promise<ProcessedMediaAsset[]> {
+ const fileArray = Array.from(files);
+ const processedAssets: ProcessedMediaAsset[] = [];
+
+ const total = fileArray.length;
+ let completed = 0;
+
+ for (const file of fileArray) {
+ const fileType = getMediaTypeFromFile({ file });
+
+ if (!fileType) {
+ toast.error(`Unsupported file type: ${file.name}`);
+ continue;
+ }
+
+ const url = URL.createObjectURL(file);
+ let thumbnailUrl: string | undefined;
+ let duration: number | undefined;
+ let width: number | undefined;
+ let height: number | undefined;
+ let fps: number | undefined;
+
+ try {
+ if (fileType === "image") {
+ const dimensions = await getImageDimensions({ file });
+ width = dimensions.width;
+ height = dimensions.height;
+ thumbnailUrl = await generateImageThumbnail({ imageFile: file });
+ } else if (fileType === "video") {
+ try {
+ const videoInfo = await getVideoInfo({ videoFile: file });
+ duration = videoInfo.duration;
+ width = videoInfo.width;
+ height = videoInfo.height;
+ fps = Number.isFinite(videoInfo.fps)
+ ? Math.round(videoInfo.fps * 1000) / 1000
+ : undefined;
+
+ thumbnailUrl = await generateThumbnail({
+ videoFile: file,
+ timeInSeconds: 1,
+ });
+ } catch (error) {
+ console.warn("Video processing failed", error);
+ }
+ } else if (fileType === "audio") {
+ // For audio, we don't set width/height/fps (they'll be undefined)
+ duration = await getMediaDuration({ file });
+ }
+
+ processedAssets.push({
+ name: file.name,
+ type: fileType,
+ file,
+ url,
+ thumbnailUrl,
+ duration,
+ width,
+ height,
+ fps,
+ });
+
+ await new Promise((resolve) => setTimeout(resolve, 0));
+
+ completed += 1;
+ if (onProgress) {
+ const percent = Math.round((completed / total) * 100);
+ onProgress({ progress: percent });
+ }
+ } catch (error) {
+ console.error("Error processing file:", file.name, error);
+ toast.error(`Failed to process ${file.name}`);
+ URL.revokeObjectURL(url); // Clean up on error
+ }
+ }
+
+ return processedAssets;
+}
+
+const getImageDimensions = ({
+ file,
+}: {
+ file: File;
+}): Promise<{ width: number; height: number }> => {
+ return new Promise((resolve, reject) => {
+ const img = new window.Image();
+ const objectUrl = URL.createObjectURL(file);
+
+ img.addEventListener("load", () => {
+ const width = img.naturalWidth;
+ const height = img.naturalHeight;
+ resolve({ width, height });
+ URL.revokeObjectURL(objectUrl);
+ img.remove();
+ });
+
+ img.addEventListener("error", () => {
+ reject(new Error("Could not load image"));
+ URL.revokeObjectURL(objectUrl);
+ img.remove();
+ });
+
+ img.src = objectUrl;
+ });
+};
+
+const getMediaDuration = ({ file }: { file: File }): Promise<number> => {
+ return new Promise((resolve, reject) => {
+ const element = document.createElement(
+ file.type.startsWith("video/") ? "video" : "audio",
+ ) as HTMLVideoElement;
+ const objectUrl = URL.createObjectURL(file);
+
+ element.addEventListener("loadedmetadata", () => {
+ resolve(element.duration);
+ URL.revokeObjectURL(objectUrl);
+ element.remove();
+ });
+
+ element.addEventListener("error", () => {
+ reject(new Error("Could not load media"));
+ URL.revokeObjectURL(objectUrl);
+ element.remove();
+ });
+
+ element.src = objectUrl;
+ element.load();
+ });
+};
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-17 09:37:25 +0000