feat: mediabunny + preview canvas refactor

3 files changed, 255 insertions, 244 deletions

diff --git a/apps/web/src/lib/media-processing.ts b/apps/web/src/lib/media-processing.ts
index 353c3113..15c71626 100644
--- a/apps/web/src/lib/media-processing.ts
+++ b/apps/web/src/lib/media-processing.ts
@@ -6,7 +6,7 @@ import {
   getImageDimensions,
   type MediaItem,
 } from "@/stores/media-store";
-import { generateThumbnail, getVideoInfo } from "./ffmpeg-utils";
+import { generateThumbnail, getVideoInfo } from "./mediabunny-utils";
 
 export interface ProcessedMediaItem extends Omit<MediaItem, "id"> {}
 
@@ -37,33 +37,23 @@ export async function processMediaFiles(
 
     try {
       if (fileType === "image") {
-        // Get image dimensions
         const dimensions = await getImageDimensions(file);
         width = dimensions.width;
         height = dimensions.height;
       } else if (fileType === "video") {
         try {
-          // Use FFmpeg for comprehensive video info extraction
-          const videoInfo = await getVideoInfo(file);
+          const videoInfo = await getVideoInfo({ videoFile: file });
           duration = videoInfo.duration;
           width = videoInfo.width;
           height = videoInfo.height;
           fps = videoInfo.fps;
 
-          // Generate thumbnail using FFmpeg
-          thumbnailUrl = await generateThumbnail(file, 1);
+          thumbnailUrl = await generateThumbnail({
+            videoFile: file,
+            timeInSeconds: 1,
+          });
         } catch (error) {
-          console.warn(
-            "FFmpeg processing failed, falling back to basic processing:",
-            error
-          );
-          // Fallback to basic processing
-          const videoResult = await generateVideoThumbnail(file);
-          thumbnailUrl = videoResult.thumbnailUrl;
-          width = videoResult.width;
-          height = videoResult.height;
-          duration = await getMediaDuration(file);
-          // FPS will remain undefined for fallback
+          console.warn("Video processing failed", error);
         }
       } else if (fileType === "audio") {
         // For audio, we don't set width/height/fps (they'll be undefined)
@@ -82,7 +72,6 @@ export async function processMediaFiles(
         fps,
       });
 
-      // Yield back to the event loop to keep the UI responsive
       await new Promise((resolve) => setTimeout(resolve, 0));
 
       completed += 1;
diff --git a/apps/web/src/lib/ffmpeg-utils.ts b/apps/web/src/lib/mediabunny-utils.ts
index 0643ecb9..28e6b97f 100644
--- a/apps/web/src/lib/ffmpeg-utils.ts
+++ b/apps/web/src/lib/mediabunny-utils.ts
@@ -1,7 +1,7 @@
 import { FFmpeg } from "@ffmpeg/ffmpeg";
-import { toBlobURL } from "@ffmpeg/util";
 import { useTimelineStore } from "@/stores/timeline-store";
 import { useMediaStore } from "@/stores/media-store";
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
 
 let ffmpeg: FFmpeg | null = null;
 
@@ -14,256 +14,99 @@ export const initFFmpeg = async (): Promise<FFmpeg> => {
   return ffmpeg;
 };
 
-export const generateThumbnail = async (
-  videoFile: File,
-  timeInSeconds = 1
-): Promise<string> => {
-  const ffmpeg = await initFFmpeg();
-
-  const inputName = "input.mp4";
-  const outputName = "thumbnail.jpg";
-
-  // Write input file
-  await ffmpeg.writeFile(
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  // Generate thumbnail at specific time
-  await ffmpeg.exec([
-    "-i",
-    inputName,
-    "-ss",
-    timeInSeconds.toString(),
-    "-vframes",
-    "1",
-    "-vf",
-    "scale=320:240",
-    "-q:v",
-    "2",
-    outputName,
-  ]);
+export async function generateThumbnail({
+  videoFile,
+  timeInSeconds,
+}: {
+  videoFile: File;
+  timeInSeconds: number;
+}): Promise<string> {
+  const input = new Input({
+    source: new BlobSource(videoFile),
+    formats: ALL_FORMATS,
+  });
 
-  // Read output file
-  const data = await ffmpeg.readFile(outputName);
-  const blob = new Blob([data], { type: "image/jpeg" });
-
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-  await ffmpeg.deleteFile(outputName);
+  const videoTrack = await input.getPrimaryVideoTrack();
+  if (!videoTrack) {
+    throw new Error("No video track found in the file");
+  }
 
-  return URL.createObjectURL(blob);
-};
+  // Check if we can decode this video
+  const canDecode = await videoTrack.canDecode();
+  if (!canDecode) {
+    throw new Error("Video codec not supported for decoding");
+  }
 
-export const trimVideo = async (
-  videoFile: File,
-  startTime: number,
-  endTime: number,
-  onProgress?: (progress: number) => void
-): Promise<Blob> => {
-  const ffmpeg = await initFFmpeg();
+  const sink = new VideoSampleSink(videoTrack);
 
-  const inputName = "input.mp4";
-  const outputName = "output.mp4";
+  const frame = await sink.getSample(timeInSeconds);
 
-  // Set up progress callback
-  if (onProgress) {
-    ffmpeg.on("progress", ({ progress }) => {
-      onProgress(progress * 100);
-    });
+  if (!frame) {
+    throw new Error("Could not get frame at specified time");
   }
 
-  // Write input file
-  await ffmpeg.writeFile(
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  const duration = endTime - startTime;
+  const canvas = document.createElement("canvas");
+  canvas.width = 320;
+  canvas.height = 240;
+  const ctx = canvas.getContext("2d");
 
-  // Trim video
-  await ffmpeg.exec([
-    "-i",
-    inputName,
-    "-ss",
-    startTime.toString(),
-    "-t",
-    duration.toString(),
-    "-c",
-    "copy", // Use stream copy for faster processing
-    outputName,
-  ]);
-
-  // Read output file
-  const data = await ffmpeg.readFile(outputName);
-  const blob = new Blob([data], { type: "video/mp4" });
+  if (!ctx) {
+    throw new Error("Could not get canvas context");
+  }
 
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-  await ffmpeg.deleteFile(outputName);
+  frame.draw(ctx, 0, 0, 320, 240);
 
-  return blob;
-};
+  return new Promise((resolve, reject) => {
+    canvas.toBlob(
+      (blob) => {
+        if (blob) {
+          resolve(URL.createObjectURL(blob));
+        } else {
+          reject(new Error("Failed to create thumbnail blob"));
+        }
+      },
+      "image/jpeg",
+      0.8
+    );
+  });
+}
 
-export const getVideoInfo = async (
-  videoFile: File
-): Promise<{
+export async function getVideoInfo({
+  videoFile,
+}: {
+  videoFile: File;
+}): Promise<{
   duration: number;
   width: number;
   height: number;
   fps: number;
-}> => {
-  const ffmpeg = await initFFmpeg();
-
-  const inputName = "input.mp4";
-
-  // Write input file
-  await ffmpeg.writeFile(
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  // Capture FFmpeg stderr output with a one-time listener pattern
-  let ffmpegOutput = "";
-  let listening = true;
-  const listener = (data: string) => {
-    if (listening) ffmpegOutput += data;
-  };
-  ffmpeg.on("log", ({ message }) => listener(message));
-
-  // Run ffmpeg to get info (stderr will contain the info)
-  try {
-    await ffmpeg.exec(["-i", inputName, "-f", "null", "-"]);
-  } catch (error) {
-    listening = false;
-    await ffmpeg.deleteFile(inputName);
-    console.error("FFmpeg execution failed:", error);
-    throw new Error(
-      "Failed to extract video info. The file may be corrupted or in an unsupported format."
-    );
-  }
+}> {
+  const input = new Input({
+    source: new BlobSource(videoFile),
+    formats: ALL_FORMATS,
+  });
 
-  // Disable listener after exec completes
-  listening = false;
+  const duration = await input.computeDuration();
+  const videoTrack = await input.getPrimaryVideoTrack();
 
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-
-  // Parse output for duration, resolution, and fps
-  // Example: Duration: 00:00:10.00, start: 0.000000, bitrate: 1234 kb/s
-  // Example: Stream #0:0: Video: h264 (High), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn, 60 tbc
-
-  const durationMatch = ffmpegOutput.match(/Duration: (\d+):(\d+):([\d.]+)/);
-  let duration = 0;
-  if (durationMatch) {
-    const [, h, m, s] = durationMatch;
-    duration = parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s);
+  if (!videoTrack) {
+    throw new Error("No video track found in the file");
   }
 
-  const videoStreamMatch = ffmpegOutput.match(
-    /Video:.* (\d+)x(\d+)[^,]*, ([\d.]+) fps/
-  );
-  let width = 0,
-    height = 0,
-    fps = 0;
-  if (videoStreamMatch) {
-    width = parseInt(videoStreamMatch[1]);
-    height = parseInt(videoStreamMatch[2]);
-    fps = parseFloat(videoStreamMatch[3]);
-  }
+  // Get frame rate from packet statistics
+  const packetStats = await videoTrack.computePacketStats(100);
+  const fps = packetStats.averagePacketRate;
 
   return {
     duration,
-    width,
-    height,
+    width: videoTrack.displayWidth,
+    height: videoTrack.displayHeight,
     fps,
   };
-};
-
-export const convertToWebM = async (
-  videoFile: File,
-  onProgress?: (progress: number) => void
-): Promise<Blob> => {
-  const ffmpeg = await initFFmpeg();
-
-  const inputName = "input.mp4";
-  const outputName = "output.webm";
-
-  // Set up progress callback
-  if (onProgress) {
-    ffmpeg.on("progress", ({ progress }) => {
-      onProgress(progress * 100);
-    });
-  }
-
-  // Write input file
-  await ffmpeg.writeFile(
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  // Convert to WebM
-  await ffmpeg.exec([
-    "-i",
-    inputName,
-    "-c:v",
-    "libvpx-vp9",
-    "-crf",
-    "30",
-    "-b:v",
-    "0",
-    "-c:a",
-    "libopus",
-    outputName,
-  ]);
-
-  // Read output file
-  const data = await ffmpeg.readFile(outputName);
-  const blob = new Blob([data], { type: "video/webm" });
-
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-  await ffmpeg.deleteFile(outputName);
-
-  return blob;
-};
-
-export const extractAudio = async (
-  videoFile: File,
-  format: "mp3" | "wav" = "mp3"
-): Promise<Blob> => {
-  const ffmpeg = await initFFmpeg();
-
-  const inputName = "input.mp4";
-  const outputName = `output.${format}`;
-
-  // Write input file
-  await ffmpeg.writeFile(
-    inputName,
-    new Uint8Array(await videoFile.arrayBuffer())
-  );
-
-  // Extract audio
-  await ffmpeg.exec([
-    "-i",
-    inputName,
-    "-vn", // Disable video
-    "-acodec",
-    format === "mp3" ? "libmp3lame" : "pcm_s16le",
-    outputName,
-  ]);
-
-  // Read output file
-  const data = await ffmpeg.readFile(outputName);
-  const blob = new Blob([data], { type: `audio/${format}` });
-
-  // Cleanup
-  await ffmpeg.deleteFile(inputName);
-  await ffmpeg.deleteFile(outputName);
-
-  return blob;
-};
+}
 
+// Audio mixing for timeline - keeping FFmpeg for now due to complexity
+// TODO: Replace with Mediabunny audio processing when implementing canvas preview
 export const extractTimelineAudio = async (
   onProgress?: (progress: number) => void
 ): Promise<Blob> => {
diff --git a/apps/web/src/lib/timeline-renderer.ts b/apps/web/src/lib/timeline-renderer.ts
new file mode 100644
index 00000000..1c1eafcd
--- /dev/null
+++ b/apps/web/src/lib/timeline-renderer.ts
@@ -0,0 +1,179 @@
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";

+import type { TimelineTrack } from "@/types/timeline";

+import type { MediaItem } from "@/stores/media-store";

+

+export interface RenderContext {

+  ctx: CanvasRenderingContext2D;

+  time: number;

+  canvasWidth: number;

+  canvasHeight: number;

+  tracks: TimelineTrack[];

+  mediaItems: MediaItem[];

+  backgroundColor?: string;

+  useCache?: boolean;

+  cache?: Map<

+    string,

+    Map<

+      number,

+      {

+        draw: (

+          c: CanvasRenderingContext2D,

+          x: number,

+          y: number,

+          w: number,

+          h: number

+        ) => void;

+      }

+    >

+  >;

+  fps: number;

+}

+

+export async function renderTimelineFrame({

+  ctx,

+  time,

+  canvasWidth,

+  canvasHeight,

+  tracks,

+  mediaItems,

+  backgroundColor,

+  useCache = true,

+  cache,

+  fps,

+}: RenderContext): Promise<void> {

+  // Background

+  ctx.clearRect(0, 0, canvasWidth, canvasHeight);

+  if (backgroundColor && backgroundColor !== "transparent") {

+    ctx.fillStyle = backgroundColor;

+    ctx.fillRect(0, 0, canvasWidth, canvasHeight);

+  }

+

+  const scaleX = 1;

+  const scaleY = 1;

+  const idToMedia = new Map(mediaItems.map((m) => [m.id, m] as const));

+  const active: Array<{

+    track: TimelineTrack;

+    element: TimelineTrack["elements"][number];

+    mediaItem: MediaItem | null;

+  }> = [];

+

+  for (let t = tracks.length - 1; t >= 0; t -= 1) {

+    const track = tracks[t];

+    for (const element of track.elements) {

+      if ((element as any).hidden) continue;

+      const elementStart = element.startTime;

+      const elementEnd =

+        element.startTime +

+        (element.duration - element.trimStart - element.trimEnd);

+      if (time >= elementStart && time < elementEnd) {

+        let mediaItem: MediaItem | null = null;

+        if (element.type === "media") {

+          mediaItem =

+            element.mediaId === "test"

+              ? null

+              : idToMedia.get(element.mediaId) || null;

+        }

+        active.push({ track, element, mediaItem });

+      }

+    }

+  }

+

+  for (const { element, mediaItem } of active) {

+    if (element.type === "media" && mediaItem) {

+      if (mediaItem.type === "video") {

+        const input = new Input({

+          source: new BlobSource(mediaItem.file),

+          formats: ALL_FORMATS,

+        });

+        const track = await input.getPrimaryVideoTrack();

+        if (!track) continue;

+        const decodable = await track.canDecode();

+        if (!decodable) continue;

+        const sink = new VideoSampleSink(track);

+

+        const localTime = time - element.startTime + element.trimStart;

+        const sample = await sink.getSample(localTime);

+        if (!sample) continue;

+

+        const mediaW = Math.max(1, mediaItem.width || canvasWidth);

+        const mediaH = Math.max(1, mediaItem.height || canvasHeight);

+        const containScale = Math.min(

+          canvasWidth / mediaW,

+          canvasHeight / mediaH

+        );

+        const drawW = mediaW * containScale;

+        const drawH = mediaH * containScale;

+        const drawX = (canvasWidth - drawW) / 2;

+        const drawY = (canvasHeight - drawH) / 2;

+        sample.draw(ctx, drawX, drawY, drawW, drawH);

+      }

+      if (mediaItem.type === "image") {

+        const img = new Image();

+        await new Promise<void>((resolve, reject) => {

+          img.onload = () => resolve();

+          img.onerror = () => reject(new Error("Image load failed"));

+          img.src = mediaItem.url || URL.createObjectURL(mediaItem.file);

+        });

+        const mediaW = Math.max(

+          1,

+          mediaItem.width || img.naturalWidth || canvasWidth

+        );

+        const mediaH = Math.max(

+          1,

+          mediaItem.height || img.naturalHeight || canvasHeight

+        );

+        const containScale = Math.min(

+          canvasWidth / mediaW,

+          canvasHeight / mediaH

+        );

+        const drawW = mediaW * containScale;

+        const drawH = mediaH * containScale;

+        const drawX = (canvasWidth - drawW) / 2;

+        const drawY = (canvasHeight - drawH) / 2;

+        ctx.drawImage(img, drawX, drawY, drawW, drawH);

+      }

+    }

+    if (element.type === "text") {

+      const posX = canvasWidth / 2 + (element as any).x * scaleX;

+      const posY = canvasHeight / 2 + (element as any).y * scaleY;

+      ctx.save();

+      ctx.translate(posX, posY);

+      ctx.rotate(((element as any).rotation * Math.PI) / 180);

+      ctx.globalAlpha = Math.max(0, Math.min(1, (element as any).opacity));

+      const px = (element as any).fontSize * scaleX;

+      const weight = (element as any).fontWeight === "bold" ? "bold " : "";

+      const style = (element as any).fontStyle === "italic" ? "italic " : "";

+      ctx.font = `${style}${weight}${px}px ${(element as any).fontFamily}`;

+      ctx.fillStyle = (element as any).color;

+      ctx.textAlign = (element as any).textAlign;

+      ctx.textBaseline = "middle";

+      const metrics = ctx.measureText((element as any).content);

+      const ascent =

+        (metrics as unknown as { actualBoundingBoxAscent?: number })

+          .actualBoundingBoxAscent ?? px * 0.8;

+      const descent =

+        (metrics as unknown as { actualBoundingBoxDescent?: number })

+          .actualBoundingBoxDescent ?? px * 0.2;

+      const textW = metrics.width;

+      const textH = ascent + descent;

+      const padX = 8 * scaleX;

+      const padY = 4 * scaleX;

+      if ((element as any).backgroundColor) {

+        ctx.save();

+        ctx.fillStyle = (element as any).backgroundColor;

+        let bgLeft = -textW / 2;

+        if (ctx.textAlign === "left") bgLeft = 0;

+        if (ctx.textAlign === "right") bgLeft = -textW;

+        ctx.fillRect(

+          bgLeft - padX,

+          -textH / 2 - padY,

+          textW + padX * 2,

+          textH + padY * 2

+        );

+        ctx.restore();

+      }

+      ctx.fillText((element as any).content, 0, 0);

+      ctx.restore();

+    }

+  }

+}