summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--apps/web/src/components/editor/media-panel/views/captions.tsx2
-rw-r--r--apps/web/src/components/editor/preview-panel.tsx520
-rw-r--r--apps/web/src/lib/media-processing.ts25
-rw-r--r--apps/web/src/lib/mediabunny-utils.ts (renamed from apps/web/src/lib/ffmpeg-utils.ts)295
-rw-r--r--apps/web/src/lib/timeline-renderer.ts179
-rw-r--r--bun.lock11
-rw-r--r--package.json1
7 files changed, 582 insertions, 451 deletions
diff --git a/apps/web/src/components/editor/media-panel/views/captions.tsx b/apps/web/src/components/editor/media-panel/views/captions.tsx
index 4fe2cbd7..6e18781c 100644
--- a/apps/web/src/components/editor/media-panel/views/captions.tsx
+++ b/apps/web/src/components/editor/media-panel/views/captions.tsx
@@ -3,7 +3,7 @@ import { PropertyGroup } from "../../properties-panel/property-item";
import { BaseView } from "./base-view";
import { Language, LanguageSelect } from "@/components/language-select";
import { useState, useRef, useEffect } from "react";
-import { extractTimelineAudio } from "@/lib/ffmpeg-utils";
+import { extractTimelineAudio } from "@/lib/mediabunny-utils";
import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption";
import { useTimelineStore } from "@/stores/timeline-store";
import { Loader2, Shield, Trash2, Upload } from "lucide-react";
diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsx
index 42c4becc..99c4edef 100644
--- a/apps/web/src/components/editor/preview-panel.tsx
+++ b/apps/web/src/components/editor/preview-panel.tsx
@@ -5,16 +5,19 @@ import { TimelineElement, TimelineTrack } from "@/types/timeline";
import { useMediaStore, type MediaItem } from "@/stores/media-store";
import { usePlaybackStore } from "@/stores/playback-store";
import { useEditorStore } from "@/stores/editor-store";
-import { VideoPlayer } from "@/components/ui/video-player";
-import { AudioPlayer } from "@/components/ui/audio-player";
import { Button } from "@/components/ui/button";
import { Play, Pause, Expand, SkipBack, SkipForward } from "lucide-react";
import { useState, useRef, useEffect, useCallback } from "react";
+import { VideoSampleSink } from "mediabunny";
+import { renderTimelineFrame } from "@/lib/timeline-renderer";
import { cn } from "@/lib/utils";
import { formatTimeCode } from "@/lib/time";
import { EditableTimecode } from "@/components/ui/editable-timecode";
-import { FONT_CLASS_MAP } from "@/lib/font-config";
-import { DEFAULT_CANVAS_SIZE, DEFAULT_FPS, useProjectStore } from "@/stores/project-store";
+import {
+ DEFAULT_CANVAS_SIZE,
+ DEFAULT_FPS,
+ useProjectStore,
+} from "@/stores/project-store";
import { TextElementDragState } from "@/types/editor";
import {
Popover,
@@ -37,8 +40,20 @@ export function PreviewPanel() {
const { tracks, getTotalDuration, updateTextElement } = useTimelineStore();
const { mediaItems } = useMediaStore();
const { currentTime, toggle, setCurrentTime } = usePlaybackStore();
+ const { isPlaying, volume, muted } = usePlaybackStore();
const { activeProject } = useProjectStore();
const previewRef = useRef<HTMLDivElement>(null);
+ const canvasRef = useRef<HTMLCanvasElement>(null);
+ const lastFrameTimeRef = useRef(0);
+ const renderSeqRef = useRef(0);
+ const offscreenCanvasRef = useRef<OffscreenCanvas | HTMLCanvasElement | null>(
+ null
+ );
+
+ const audioContextRef = useRef<AudioContext | null>(null);
+ const audioGainRef = useRef<GainNode | null>(null);
+ const audioBuffersRef = useRef<Map<string, AudioBuffer>>(new Map());
+ const playingSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set());
const containerRef = useRef<HTMLDivElement>(null);
const [previewDimensions, setPreviewDimensions] = useState({
width: 0,
@@ -262,227 +277,315 @@ export function PreviewPanel() {
const activeElements = getActiveElements();
- // Get media elements for blur background (video/image only)
- const getBlurBackgroundElements = (): ActiveElement[] => {
- return activeElements.filter(
- ({ element, mediaItem }) =>
- element.type === "media" &&
- mediaItem &&
- (mediaItem.type === "video" || mediaItem.type === "image") &&
- element.mediaId !== "test" // Exclude test elements
- );
- };
-
- const blurBackgroundElements = getBlurBackgroundElements();
+ // Ensure first frame after mount/seek renders immediately
+ useEffect(() => {
+ const onSeek = () => {
+ lastFrameTimeRef.current = -Infinity;
+ renderSeqRef.current++;
+ };
+ window.addEventListener("playback-seek", onSeek as EventListener);
+ lastFrameTimeRef.current = -Infinity;
+ return () => {
+ window.removeEventListener("playback-seek", onSeek as EventListener);
+ };
+ }, []);
- // Render blur background layer
- const renderBlurBackground = () => {
- if (
- !activeProject?.backgroundType ||
- activeProject.backgroundType !== "blur" ||
- blurBackgroundElements.length === 0
- ) {
- return null;
- }
+ // Web Audio: schedule only on play/pause/seek/volume/mute changes
+ useEffect(() => {
+ const stopAll = () => {
+ for (const src of playingSourcesRef.current) {
+ try {
+ src.stop();
+ } catch {}
+ }
+ playingSourcesRef.current.clear();
+ };
- // Use the first media element for background (could be enhanced to use primary/focused element)
- const backgroundElement = blurBackgroundElements[0];
- const { element, mediaItem } = backgroundElement;
+ type WebAudioWindow = Window & {
+ AudioContext?: typeof AudioContext;
+ webkitAudioContext?: typeof AudioContext;
+ };
+ const ensureAudioGraph = async () => {
+ if (!audioContextRef.current) {
+ const win = window as WebAudioWindow;
+ const Ctx = win.AudioContext ?? win.webkitAudioContext;
+ if (!Ctx) return;
+ audioContextRef.current = new Ctx();
+ }
+ if (!audioGainRef.current) {
+ audioGainRef.current = audioContextRef.current!.createGain();
+ audioGainRef.current.connect(audioContextRef.current!.destination);
+ }
+ if (audioContextRef.current!.state === "suspended") {
+ try {
+ await audioContextRef.current!.resume();
+ } catch {}
+ }
+ const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume));
+ audioGainRef.current!.gain.setValueAtTime(
+ gainValue,
+ audioContextRef.current!.currentTime
+ );
+ };
- if (!mediaItem) return null;
+ const scheduleNow = async () => {
+ await ensureAudioGraph();
+ const audioCtx = audioContextRef.current!;
+ const gain = audioGainRef.current!;
- const blurIntensity = activeProject.blurIntensity || 8;
+ const tracksSnapshot = useTimelineStore.getState().tracks;
+ const mediaList = useMediaStore.getState().mediaItems;
+ const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const));
+ const playbackNow = usePlaybackStore.getState().currentTime;
- if (mediaItem.type === "video") {
- return (
- <div
- key={`blur-${element.id}`}
- className="absolute inset-0 overflow-hidden"
- style={{
- filter: `blur(${blurIntensity}px)`,
- transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts
- transformOrigin: "center",
- }}
- >
- <VideoPlayer
- src={mediaItem.url!}
- poster={mediaItem.thumbnailUrl}
- clipStartTime={element.startTime}
- trimStart={element.trimStart}
- trimEnd={element.trimEnd}
- clipDuration={element.duration}
- className="w-full h-full object-cover"
- trackMuted={true}
- />
- </div>
- );
- }
+ const audible: Array<{
+ id: string;
+ elementStart: number;
+ trimStart: number;
+ trimEnd: number;
+ duration: number;
+ muted: boolean;
+ trackMuted: boolean;
+ }> = [];
+ const uniqueIds = new Set<string>();
+ for (const track of tracksSnapshot) {
+ for (const element of track.elements) {
+ if (element.type !== "media") continue;
+ const media = idToMedia.get(element.mediaId);
+ if (!media || media.type !== "audio") continue;
+ const visibleDuration =
+ element.duration - element.trimStart - element.trimEnd;
+ if (visibleDuration <= 0) continue;
+ const localTime = playbackNow - element.startTime + element.trimStart;
+ if (localTime < 0 || localTime >= visibleDuration) continue;
+ audible.push({
+ id: media.id,
+ elementStart: element.startTime,
+ trimStart: element.trimStart,
+ trimEnd: element.trimEnd,
+ duration: element.duration,
+ muted: !!element.muted,
+ trackMuted: !!track.muted,
+ });
+ uniqueIds.add(media.id);
+ }
+ }
- if (mediaItem.type === "image") {
- return (
- <div
- key={`blur-${element.id}`}
- className="absolute inset-0 overflow-hidden"
- style={{
- filter: `blur(${blurIntensity}px)`,
- transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts
- transformOrigin: "center",
- }}
- >
- <img
- src={mediaItem.url!}
- alt={mediaItem.name}
- className="w-full h-full object-cover"
- draggable={false}
- />
- </div>
- );
- }
+ if (audible.length === 0) return;
- return null;
- };
+ // Decode buffers as needed
+ const decodePromises: Array<Promise<void>> = [];
+ for (const id of uniqueIds) {
+ if (!audioBuffersRef.current.has(id)) {
+ const mediaItem = idToMedia.get(id);
+ if (!mediaItem) continue;
+ const p = (async () => {
+ const arr = await mediaItem.file.arrayBuffer();
+ const buf = await audioCtx.decodeAudioData(arr.slice(0));
+ audioBuffersRef.current.set(id, buf);
+ })();
+ decodePromises.push(p);
+ }
+ }
+ await Promise.all(decodePromises);
- // Render an element
- const renderElement = (elementData: ActiveElement, index: number) => {
- const { element, mediaItem } = elementData;
+ const startAt = audioCtx.currentTime + 0.02;
+ for (const entry of audible) {
+ if (entry.muted || entry.trackMuted) continue;
+ const buffer = audioBuffersRef.current.get(entry.id);
+ if (!buffer) continue;
+ const visibleDuration =
+ entry.duration - entry.trimStart - entry.trimEnd;
+ const localTime = Math.max(
+ 0,
+ playbackNow - entry.elementStart + entry.trimStart
+ );
+ const playDuration = Math.max(0, visibleDuration - localTime);
+ if (playDuration <= 0) continue;
+ const src = audioCtx.createBufferSource();
+ src.buffer = buffer;
+ src.connect(gain);
+ try {
+ src.start(startAt, localTime, playDuration);
+ playingSourcesRef.current.add(src);
+ } catch {}
+ }
+ };
- // Text elements
- if (element.type === "text") {
- const fontClassName =
- FONT_CLASS_MAP[element.fontFamily as keyof typeof FONT_CLASS_MAP] || "";
+ const onSeek = () => {
+ if (!isPlaying) return;
+ for (const src of playingSourcesRef.current) {
+ try {
+ src.stop();
+ } catch {}
+ }
+ playingSourcesRef.current.clear();
+ void scheduleNow();
+ };
- const scaleRatio = previewDimensions.width / canvasSize.width;
+ // Apply volume/mute changes immediately
+ void ensureAudioGraph();
- return (
- <div
- key={element.id}
- className="absolute cursor-grab"
- onMouseDown={(e) =>
- handleTextMouseDown(e, element, elementData.track.id)
- }
- style={{
- left: `${
- 50 +
- ((dragState.isDragging && dragState.elementId === element.id
- ? dragState.currentX
- : element.x) /
- canvasSize.width) *
- 100
- }%`,
- top: `${
- 50 +
- ((dragState.isDragging && dragState.elementId === element.id
- ? dragState.currentY
- : element.y) /
- canvasSize.height) *
- 100
- }%`,
- transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`,
- opacity: element.opacity,
- zIndex: 100 + index, // Text elements on top
- }}
- >
- <div
- className={fontClassName}
- style={{
- fontSize: `${element.fontSize * scaleRatio}px`,
- color: element.color,
- backgroundColor: element.backgroundColor,
- textAlign: element.textAlign,
- fontWeight: element.fontWeight,
- fontStyle: element.fontStyle,
- textDecoration: element.textDecoration,
- padding: `${4 * scaleRatio}px ${8 * scaleRatio}px`,
- borderRadius: `${2 * scaleRatio}px`,
- whiteSpace: "nowrap",
- // Fallback for system fonts that don't have classes
- ...(fontClassName === "" && { fontFamily: element.fontFamily }),
- }}
- >
- {element.content}
- </div>
- </div>
- );
+ // Start/stop on play state changes
+ for (const src of playingSourcesRef.current) {
+ try {
+ src.stop();
+ } catch {}
+ }
+ playingSourcesRef.current.clear();
+ if (isPlaying) {
+ void scheduleNow();
}
- // Media elements
- if (element.type === "media") {
- // Test elements
- if (!mediaItem || element.mediaId === "test") {
- return (
- <div
- key={element.id}
- className="absolute inset-0 bg-linear-to-br from-blue-500/20 to-purple-500/20 flex items-center justify-center"
- >
- <div className="text-center">
- <div className="text-2xl mb-2">🎬</div>
- <p className="text-xs text-foreground">{element.name}</p>
- </div>
- </div>
- );
+ window.addEventListener("playback-seek", onSeek as EventListener);
+ return () => {
+ window.removeEventListener("playback-seek", onSeek as EventListener);
+ for (const src of playingSourcesRef.current) {
+ try {
+ src.stop();
+ } catch {}
}
+ playingSourcesRef.current.clear();
+ };
+ }, [isPlaying, volume, muted, mediaItems]);
- // Video elements
- if (mediaItem.type === "video") {
- return (
- <div
- key={element.id}
- className="absolute inset-0 flex items-center justify-center"
- >
- <VideoPlayer
- src={mediaItem.url!}
- poster={mediaItem.thumbnailUrl}
- clipStartTime={element.startTime}
- trimStart={element.trimStart}
- trimEnd={element.trimEnd}
- clipDuration={element.duration}
- trackMuted={element.muted || elementData.track.muted}
- />
- </div>
- );
+ // Canvas: draw current frame for visible elements using offscreen compositing
+ useEffect(() => {
+ const draw = async () => {
+ const canvas = canvasRef.current;
+ if (!canvas) return;
+ const mainCtx = canvas.getContext("2d");
+ if (!mainCtx) return;
+
+ // Set canvas internal resolution to avoid blurry scaling
+ const displayWidth = Math.max(1, Math.floor(previewDimensions.width));
+ const displayHeight = Math.max(1, Math.floor(previewDimensions.height));
+ if (canvas.width !== displayWidth || canvas.height !== displayHeight) {
+ canvas.width = displayWidth;
+ canvas.height = displayHeight;
}
- // Image elements
- if (mediaItem.type === "image") {
- return (
- <div
- key={element.id}
- className="absolute inset-0 flex items-center justify-center"
- >
- <img
- src={mediaItem.url!}
- alt={mediaItem.name}
- className="max-w-full max-h-full object-contain"
- draggable={false}
- />
- </div>
- );
+ // Throttle rendering to project FPS during playback only
+ const fps = activeProject?.fps || DEFAULT_FPS;
+ const minDelta = 1 / fps;
+ if (isPlaying) {
+ if (currentTime - lastFrameTimeRef.current < minDelta) {
+ return;
+ }
+ lastFrameTimeRef.current = currentTime;
}
- // Audio elements (no visual representation)
- if (mediaItem.type === "audio") {
- return (
- <div
- key={element.id}
- className="absolute inset-0"
- style={{ pointerEvents: "none" }}
- >
- <AudioPlayer
- src={mediaItem.url!}
- clipStartTime={element.startTime}
- trimStart={element.trimStart}
- trimEnd={element.trimEnd}
- clipDuration={element.duration}
- trackMuted={element.muted || elementData.track.muted}
- />
- </div>
+ // Invalidate older async renders when user scrubs rapidly
+ const mySeq = (renderSeqRef.current += 1);
+
+ // Offscreen buffer to avoid flicker (reuse canvas)
+ if (!offscreenCanvasRef.current) {
+ const hasOffscreen =
+ typeof (globalThis as unknown as { OffscreenCanvas?: unknown })
+ .OffscreenCanvas !== "undefined";
+ if (hasOffscreen) {
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
+ offscreenCanvasRef.current = new (globalThis as any).OffscreenCanvas(
+ displayWidth,
+ displayHeight
+ ) as OffscreenCanvas;
+ } else {
+ const c = document.createElement("canvas");
+ c.width = displayWidth;
+ c.height = displayHeight;
+ offscreenCanvasRef.current = c;
+ }
+ }
+ // Ensure size matches
+ if (
+ offscreenCanvasRef.current &&
+ (offscreenCanvasRef.current as HTMLCanvasElement).getContext
+ ) {
+ const c = offscreenCanvasRef.current as HTMLCanvasElement;
+ if (c.width !== displayWidth || c.height !== displayHeight) {
+ c.width = displayWidth;
+ c.height = displayHeight;
+ }
+ } else {
+ const c = offscreenCanvasRef.current as OffscreenCanvas;
+ // @ts-ignore width/height exist on OffscreenCanvas in modern browsers
+ if (
+ (c as unknown as { width: number }).width !== displayWidth ||
+ (c as unknown as { height: number }).height !== displayHeight
+ ) {
+ // @ts-ignore
+ (c as unknown as { width: number }).width = displayWidth;
+ // @ts-ignore
+ (c as unknown as { height: number }).height = displayHeight;
+ }
+ }
+ const offscreenCanvas = offscreenCanvasRef.current as
+ | HTMLCanvasElement
+ | OffscreenCanvas;
+ const offCtx = (offscreenCanvas as HTMLCanvasElement).getContext
+ ? (offscreenCanvas as HTMLCanvasElement).getContext("2d")
+ : (offscreenCanvas as OffscreenCanvas).getContext("2d");
+ if (!offCtx) return;
+
+ await renderTimelineFrame({
+ ctx: offCtx as CanvasRenderingContext2D,
+ time: currentTime,
+ canvasWidth: displayWidth,
+ canvasHeight: displayHeight,
+ tracks,
+ mediaItems,
+ backgroundColor:
+ activeProject?.backgroundType === "blur"
+ ? "transparent"
+ : activeProject?.backgroundColor || "#000000",
+ useCache: false,
+ fps: activeProject?.fps || DEFAULT_FPS,
+ });
+
+ // Blit offscreen to visible canvas
+ mainCtx.clearRect(0, 0, displayWidth, displayHeight);
+ if ((offscreenCanvas as HTMLCanvasElement).getContext) {
+ mainCtx.drawImage(offscreenCanvas as HTMLCanvasElement, 0, 0);
+ } else {
+ mainCtx.drawImage(
+ offscreenCanvas as unknown as CanvasImageSource,
+ 0,
+ 0
);
}
- }
+ };
+
+ void draw();
+ }, [
+ activeElements,
+ currentTime,
+ previewDimensions.width,
+ previewDimensions.height,
+ canvasSize.width,
+ canvasSize.height,
+ activeProject?.backgroundType,
+ activeProject?.backgroundColor,
+ ]);
- return null;
+ // Get media elements for blur background (video/image only)
+ const getBlurBackgroundElements = (): ActiveElement[] => {
+ return activeElements.filter(
+ ({ element, mediaItem }) =>
+ element.type === "media" &&
+ mediaItem &&
+ (mediaItem.type === "video" || mediaItem.type === "image") &&
+ element.mediaId !== "test" // Exclude test elements
+ );
};
+ const blurBackgroundElements = getBlurBackgroundElements();
+
+ // Render blur background layer (handled by canvas now)
+ const renderBlurBackground = () => null;
+
+ // Render an element (canvas handles visuals now). Audio playback to be implemented via Web Audio.
+ const renderElement = (_elementData: ActiveElement) => null;
+
return (
<>
<div className="h-full w-full flex flex-col min-h-0 min-w-0 bg-panel rounded-sm relative">
@@ -505,14 +608,23 @@ export function PreviewPanel() {
}}
>
{renderBlurBackground()}
+ <canvas
+ ref={canvasRef}
+ style={{
+ position: "absolute",
+ left: 0,
+ top: 0,
+ width: previewDimensions.width,
+ height: previewDimensions.height,
+ }}
+ aria-label="Video preview canvas"
+ />
{activeElements.length === 0 ? (
<div className="absolute inset-0 flex items-center justify-center text-muted-foreground">
No elements at current time
</div>
) : (
- activeElements.map((elementData, index) =>
- renderElement(elementData, index)
- )
+ activeElements.map((elementData) => renderElement(elementData))
)}
<LayoutGuideOverlay />
</div>
diff --git a/apps/web/src/lib/media-processing.ts b/apps/web/src/lib/media-processing.ts
index 353c3113..15c71626 100644
--- a/apps/web/src/lib/media-processing.ts
+++ b/apps/web/src/lib/media-processing.ts
@@ -6,7 +6,7 @@ import {
getImageDimensions,
type MediaItem,
} from "@/stores/media-store";
-import { generateThumbnail, getVideoInfo } from "./ffmpeg-utils";
+import { generateThumbnail, getVideoInfo } from "./mediabunny-utils";
export interface ProcessedMediaItem extends Omit<MediaItem, "id"> {}
@@ -37,33 +37,23 @@ export async function processMediaFiles(
try {
if (fileType === "image") {
- // Get image dimensions
const dimensions = await getImageDimensions(file);
width = dimensions.width;
height = dimensions.height;
} else if (fileType === "video") {
try {
- // Use FFmpeg for comprehensive video info extraction
- const videoInfo = await getVideoInfo(file);
+ const videoInfo = await getVideoInfo({ videoFile: file });
duration = videoInfo.duration;
width = videoInfo.width;
height = videoInfo.height;
fps = videoInfo.fps;
- // Generate thumbnail using FFmpeg
- thumbnailUrl = await generateThumbnail(file, 1);
+ thumbnailUrl = await generateThumbnail({
+ videoFile: file,
+ timeInSeconds: 1,
+ });
} catch (error) {
- console.warn(
- "FFmpeg processing failed, falling back to basic processing:",
- error
- );
- // Fallback to basic processing
- const videoResult = await generateVideoThumbnail(file);
- thumbnailUrl = videoResult.thumbnailUrl;
- width = videoResult.width;
- height = videoResult.height;
- duration = await getMediaDuration(file);
- // FPS will remain undefined for fallback
+ console.warn("Video processing failed", error);
}
} else if (fileType === "audio") {
// For audio, we don't set width/height/fps (they'll be undefined)
@@ -82,7 +72,6 @@ export async function processMediaFiles(
fps,
});
- // Yield back to the event loop to keep the UI responsive
await new Promise((resolve) => setTimeout(resolve, 0));
completed += 1;
diff --git a/apps/web/src/lib/ffmpeg-utils.ts b/apps/web/src/lib/mediabunny-utils.ts
index 0643ecb9..28e6b97f 100644
--- a/apps/web/src/lib/ffmpeg-utils.ts
+++ b/apps/web/src/lib/mediabunny-utils.ts
@@ -1,7 +1,7 @@
import { FFmpeg } from "@ffmpeg/ffmpeg";
-import { toBlobURL } from "@ffmpeg/util";
import { useTimelineStore } from "@/stores/timeline-store";
import { useMediaStore } from "@/stores/media-store";
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
let ffmpeg: FFmpeg | null = null;
@@ -14,256 +14,99 @@ export const initFFmpeg = async (): Promise<FFmpeg> => {
return ffmpeg;
};
-export const generateThumbnail = async (
- videoFile: File,
- timeInSeconds = 1
-): Promise<string> => {
- const ffmpeg = await initFFmpeg();
-
- const inputName = "input.mp4";
- const outputName = "thumbnail.jpg";
-
- // Write input file
- await ffmpeg.writeFile(
- inputName,
- new Uint8Array(await videoFile.arrayBuffer())
- );
-
- // Generate thumbnail at specific time
- await ffmpeg.exec([
- "-i",
- inputName,
- "-ss",
- timeInSeconds.toString(),
- "-vframes",
- "1",
- "-vf",
- "scale=320:240",
- "-q:v",
- "2",
- outputName,
- ]);
+export async function generateThumbnail({
+ videoFile,
+ timeInSeconds,
+}: {
+ videoFile: File;
+ timeInSeconds: number;
+}): Promise<string> {
+ const input = new Input({
+ source: new BlobSource(videoFile),
+ formats: ALL_FORMATS,
+ });
- // Read output file
- const data = await ffmpeg.readFile(outputName);
- const blob = new Blob([data], { type: "image/jpeg" });
-
- // Cleanup
- await ffmpeg.deleteFile(inputName);
- await ffmpeg.deleteFile(outputName);
+ const videoTrack = await input.getPrimaryVideoTrack();
+ if (!videoTrack) {
+ throw new Error("No video track found in the file");
+ }
- return URL.createObjectURL(blob);
-};
+ // Check if we can decode this video
+ const canDecode = await videoTrack.canDecode();
+ if (!canDecode) {
+ throw new Error("Video codec not supported for decoding");
+ }
-export const trimVideo = async (
- videoFile: File,
- startTime: number,
- endTime: number,
- onProgress?: (progress: number) => void
-): Promise<Blob> => {
- const ffmpeg = await initFFmpeg();
+ const sink = new VideoSampleSink(videoTrack);
- const inputName = "input.mp4";
- const outputName = "output.mp4";
+ const frame = await sink.getSample(timeInSeconds);
- // Set up progress callback
- if (onProgress) {
- ffmpeg.on("progress", ({ progress }) => {
- onProgress(progress * 100);
- });
+ if (!frame) {
+ throw new Error("Could not get frame at specified time");
}
- // Write input file
- await ffmpeg.writeFile(
- inputName,
- new Uint8Array(await videoFile.arrayBuffer())
- );
-
- const duration = endTime - startTime;
+ const canvas = document.createElement("canvas");
+ canvas.width = 320;
+ canvas.height = 240;
+ const ctx = canvas.getContext("2d");
- // Trim video
- await ffmpeg.exec([
- "-i",
- inputName,
- "-ss",
- startTime.toString(),
- "-t",
- duration.toString(),
- "-c",
- "copy", // Use stream copy for faster processing
- outputName,
- ]);
-
- // Read output file
- const data = await ffmpeg.readFile(outputName);
- const blob = new Blob([data], { type: "video/mp4" });
+ if (!ctx) {
+ throw new Error("Could not get canvas context");
+ }
- // Cleanup
- await ffmpeg.deleteFile(inputName);
- await ffmpeg.deleteFile(outputName);
+ frame.draw(ctx, 0, 0, 320, 240);
- return blob;
-};
+ return new Promise((resolve, reject) => {
+ canvas.toBlob(
+ (blob) => {
+ if (blob) {
+ resolve(URL.createObjectURL(blob));
+ } else {
+ reject(new Error("Failed to create thumbnail blob"));
+ }
+ },
+ "image/jpeg",
+ 0.8
+ );
+ });
+}
-export const getVideoInfo = async (
- videoFile: File
-): Promise<{
+export async function getVideoInfo({
+ videoFile,
+}: {
+ videoFile: File;
+}): Promise<{
duration: number;
width: number;
height: number;
fps: number;
-}> => {
- const ffmpeg = await initFFmpeg();
-
- const inputName = "input.mp4";
-
- // Write input file
- await ffmpeg.writeFile(
- inputName,
- new Uint8Array(await videoFile.arrayBuffer())
- );
-
- // Capture FFmpeg stderr output with a one-time listener pattern
- let ffmpegOutput = "";
- let listening = true;
- const listener = (data: string) => {
- if (listening) ffmpegOutput += data;
- };
- ffmpeg.on("log", ({ message }) => listener(message));
-
- // Run ffmpeg to get info (stderr will contain the info)
- try {
- await ffmpeg.exec(["-i", inputName, "-f", "null", "-"]);
- } catch (error) {
- listening = false;
- await ffmpeg.deleteFile(inputName);
- console.error("FFmpeg execution failed:", error);
- throw new Error(
- "Failed to extract video info. The file may be corrupted or in an unsupported format."
- );
- }
+}> {
+ const input = new Input({
+ source: new BlobSource(videoFile),
+ formats: ALL_FORMATS,
+ });
- // Disable listener after exec completes
- listening = false;
+ const duration = await input.computeDuration();
+ const videoTrack = await input.getPrimaryVideoTrack();
- // Cleanup
- await ffmpeg.deleteFile(inputName);
-
- // Parse output for duration, resolution, and fps
- // Example: Duration: 00:00:10.00, start: 0.000000, bitrate: 1234 kb/s
- // Example: Stream #0:0: Video: h264 (High), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn, 60 tbc
-
- const durationMatch = ffmpegOutput.match(/Duration: (\d+):(\d+):([\d.]+)/);
- let duration = 0;
- if (durationMatch) {
- const [, h, m, s] = durationMatch;
- duration = parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s);
+ if (!videoTrack) {
+ throw new Error("No video track found in the file");
}
- const videoStreamMatch = ffmpegOutput.match(
- /Video:.* (\d+)x(\d+)[^,]*, ([\d.]+) fps/
- );
- let width = 0,
- height = 0,
- fps = 0;
- if (videoStreamMatch) {
- width = parseInt(videoStreamMatch[1]);
- height = parseInt(videoStreamMatch[2]);
- fps = parseFloat(videoStreamMatch[3]);
- }
+ // Get frame rate from packet statistics
+ const packetStats = await videoTrack.computePacketStats(100);
+ const fps = packetStats.averagePacketRate;
return {
duration,
- width,
- height,
+ width: videoTrack.displayWidth,
+ height: videoTrack.displayHeight,
fps,
};
-};
-
-export const convertToWebM = async (
- videoFile: File,
- onProgress?: (progress: number) => void
-): Promise<Blob> => {
- const ffmpeg = await initFFmpeg();
-
- const inputName = "input.mp4";
- const outputName = "output.webm";
-
- // Set up progress callback
- if (onProgress) {
- ffmpeg.on("progress", ({ progress }) => {
- onProgress(progress * 100);
- });
- }
-
- // Write input file
- await ffmpeg.writeFile(
- inputName,
- new Uint8Array(await videoFile.arrayBuffer())
- );
-
- // Convert to WebM
- await ffmpeg.exec([
- "-i",
- inputName,
- "-c:v",
- "libvpx-vp9",
- "-crf",
- "30",
- "-b:v",
- "0",
- "-c:a",
- "libopus",
- outputName,
- ]);
-
- // Read output file
- const data = await ffmpeg.readFile(outputName);
- const blob = new Blob([data], { type: "video/webm" });
-
- // Cleanup
- await ffmpeg.deleteFile(inputName);
- await ffmpeg.deleteFile(outputName);
-
- return blob;
-};
-
-export const extractAudio = async (
- videoFile: File,
- format: "mp3" | "wav" = "mp3"
-): Promise<Blob> => {
- const ffmpeg = await initFFmpeg();
-
- const inputName = "input.mp4";
- const outputName = `output.${format}`;
-
- // Write input file
- await ffmpeg.writeFile(
- inputName,
- new Uint8Array(await videoFile.arrayBuffer())
- );
-
- // Extract audio
- await ffmpeg.exec([
- "-i",
- inputName,
- "-vn", // Disable video
- "-acodec",
- format === "mp3" ? "libmp3lame" : "pcm_s16le",
- outputName,
- ]);
-
- // Read output file
- const data = await ffmpeg.readFile(outputName);
- const blob = new Blob([data], { type: `audio/${format}` });
-
- // Cleanup
- await ffmpeg.deleteFile(inputName);
- await ffmpeg.deleteFile(outputName);
-
- return blob;
-};
+}
+// Audio mixing for timeline - keeping FFmpeg for now due to complexity
+// TODO: Replace with Mediabunny audio processing when implementing canvas preview
export const extractTimelineAudio = async (
onProgress?: (progress: number) => void
): Promise<Blob> => {
diff --git a/apps/web/src/lib/timeline-renderer.ts b/apps/web/src/lib/timeline-renderer.ts
new file mode 100644
index 00000000..1c1eafcd
--- /dev/null
+++ b/apps/web/src/lib/timeline-renderer.ts
@@ -0,0 +1,179 @@
+import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
+import type { TimelineTrack } from "@/types/timeline";
+import type { MediaItem } from "@/stores/media-store";
+
+export interface RenderContext {
+ ctx: CanvasRenderingContext2D;
+ time: number;
+ canvasWidth: number;
+ canvasHeight: number;
+ tracks: TimelineTrack[];
+ mediaItems: MediaItem[];
+ backgroundColor?: string;
+ useCache?: boolean;
+ cache?: Map<
+ string,
+ Map<
+ number,
+ {
+ draw: (
+ c: CanvasRenderingContext2D,
+ x: number,
+ y: number,
+ w: number,
+ h: number
+ ) => void;
+ }
+ >
+ >;
+ fps: number;
+}
+
+export async function renderTimelineFrame({
+ ctx,
+ time,
+ canvasWidth,
+ canvasHeight,
+ tracks,
+ mediaItems,
+ backgroundColor,
+ useCache = true,
+ cache,
+ fps,
+}: RenderContext): Promise<void> {
+ // Background
+ ctx.clearRect(0, 0, canvasWidth, canvasHeight);
+ if (backgroundColor && backgroundColor !== "transparent") {
+ ctx.fillStyle = backgroundColor;
+ ctx.fillRect(0, 0, canvasWidth, canvasHeight);
+ }
+
+ const scaleX = 1;
+ const scaleY = 1;
+ const idToMedia = new Map(mediaItems.map((m) => [m.id, m] as const));
+ const active: Array<{
+ track: TimelineTrack;
+ element: TimelineTrack["elements"][number];
+ mediaItem: MediaItem | null;
+ }> = [];
+
+ for (let t = tracks.length - 1; t >= 0; t -= 1) {
+ const track = tracks[t];
+ for (const element of track.elements) {
+ if ((element as any).hidden) continue;
+ const elementStart = element.startTime;
+ const elementEnd =
+ element.startTime +
+ (element.duration - element.trimStart - element.trimEnd);
+ if (time >= elementStart && time < elementEnd) {
+ let mediaItem: MediaItem | null = null;
+ if (element.type === "media") {
+ mediaItem =
+ element.mediaId === "test"
+ ? null
+ : idToMedia.get(element.mediaId) || null;
+ }
+ active.push({ track, element, mediaItem });
+ }
+ }
+ }
+
+ for (const { element, mediaItem } of active) {
+ if (element.type === "media" && mediaItem) {
+ if (mediaItem.type === "video") {
+ const input = new Input({
+ source: new BlobSource(mediaItem.file),
+ formats: ALL_FORMATS,
+ });
+ const track = await input.getPrimaryVideoTrack();
+ if (!track) continue;
+ const decodable = await track.canDecode();
+ if (!decodable) continue;
+ const sink = new VideoSampleSink(track);
+
+ const localTime = time - element.startTime + element.trimStart;
+ const sample = await sink.getSample(localTime);
+ if (!sample) continue;
+
+ const mediaW = Math.max(1, mediaItem.width || canvasWidth);
+ const mediaH = Math.max(1, mediaItem.height || canvasHeight);
+ const containScale = Math.min(
+ canvasWidth / mediaW,
+ canvasHeight / mediaH
+ );
+ const drawW = mediaW * containScale;
+ const drawH = mediaH * containScale;
+ const drawX = (canvasWidth - drawW) / 2;
+ const drawY = (canvasHeight - drawH) / 2;
+ sample.draw(ctx, drawX, drawY, drawW, drawH);
+ }
+ if (mediaItem.type === "image") {
+ const img = new Image();
+ await new Promise<void>((resolve, reject) => {
+ img.onload = () => resolve();
+ img.onerror = () => reject(new Error("Image load failed"));
+ img.src = mediaItem.url || URL.createObjectURL(mediaItem.file);
+ });
+ const mediaW = Math.max(
+ 1,
+ mediaItem.width || img.naturalWidth || canvasWidth
+ );
+ const mediaH = Math.max(
+ 1,
+ mediaItem.height || img.naturalHeight || canvasHeight
+ );
+ const containScale = Math.min(
+ canvasWidth / mediaW,
+ canvasHeight / mediaH
+ );
+ const drawW = mediaW * containScale;
+ const drawH = mediaH * containScale;
+ const drawX = (canvasWidth - drawW) / 2;
+ const drawY = (canvasHeight - drawH) / 2;
+ ctx.drawImage(img, drawX, drawY, drawW, drawH);
+ }
+ }
+ if (element.type === "text") {
+ const posX = canvasWidth / 2 + (element as any).x * scaleX;
+ const posY = canvasHeight / 2 + (element as any).y * scaleY;
+ ctx.save();
+ ctx.translate(posX, posY);
+ ctx.rotate(((element as any).rotation * Math.PI) / 180);
+ ctx.globalAlpha = Math.max(0, Math.min(1, (element as any).opacity));
+ const px = (element as any).fontSize * scaleX;
+ const weight = (element as any).fontWeight === "bold" ? "bold " : "";
+ const style = (element as any).fontStyle === "italic" ? "italic " : "";
+ ctx.font = `${style}${weight}${px}px ${(element as any).fontFamily}`;
+ ctx.fillStyle = (element as any).color;
+ ctx.textAlign = (element as any).textAlign;
+ ctx.textBaseline = "middle";
+ const metrics = ctx.measureText((element as any).content);
+ const ascent =
+ (metrics as unknown as { actualBoundingBoxAscent?: number })
+ .actualBoundingBoxAscent ?? px * 0.8;
+ const descent =
+ (metrics as unknown as { actualBoundingBoxDescent?: number })
+ .actualBoundingBoxDescent ?? px * 0.2;
+ const textW = metrics.width;
+ const textH = ascent + descent;
+ const padX = 8 * scaleX;
+ const padY = 4 * scaleX;
+ if ((element as any).backgroundColor) {
+ ctx.save();
+ ctx.fillStyle = (element as any).backgroundColor;
+ let bgLeft = -textW / 2;
+ if (ctx.textAlign === "left") bgLeft = 0;
+ if (ctx.textAlign === "right") bgLeft = -textW;
+ ctx.fillRect(
+ bgLeft - padX,
+ -textH / 2 - padY,
+ textW + padX * 2,
+ textH + padY * 2
+ );
+ ctx.restore();
+ }
+ ctx.fillText((element as any).content, 0, 0);
+ ctx.restore();
+ }
+ }
+}
diff --git a/bun.lock b/bun.lock
index ea7c8c9a..95f7c874 100644
--- a/bun.lock
+++ b/bun.lock
@@ -4,6 +4,7 @@
"": {
"name": "opencut",
"dependencies": {
+ "mediabunny": "^1.9.3",
"next": "^15.3.4",
"react-country-flag": "^3.1.0",
"wavesurfer.js": "^7.9.8",
@@ -558,6 +559,10 @@
"@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="],
+ "@types/dom-mediacapture-transform": ["@types/dom-mediacapture-transform@0.1.11", "", { "dependencies": { "@types/dom-webcodecs": "*" } }, "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ=="],
+
+ "@types/dom-webcodecs": ["@types/dom-webcodecs@0.1.13", "", {}, "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ=="],
+
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
"@types/estree-jsx": ["@types/estree-jsx@1.0.5", "", { "dependencies": { "@types/estree": "*" } }, "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg=="],
@@ -896,6 +901,8 @@
"mdast-util-to-string": ["mdast-util-to-string@4.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0" } }, "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg=="],
+ "mediabunny": ["mediabunny@1.9.3", "", { "dependencies": { "@types/dom-mediacapture-transform": "^0.1.11", "@types/dom-webcodecs": "0.1.13" } }, "sha512-uVspdzrpwvW8NxtGspHPOJvdNSY0dNFEmOQngSheX9PYX9oXijZENBjfYWOgm9c9IgSSlN2bdJF9iyPyY3f+fQ=="],
+
"micromark": ["micromark@4.0.2", "", { "dependencies": { "@types/debug": "^4.0.0", "debug": "^4.0.0", "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-core-commonmark": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-combine-extensions": "^2.0.0", "micromark-util-decode-numeric-character-reference": "^2.0.0", "micromark-util-encode": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-sanitize-uri": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA=="],
"micromark-core-commonmark": ["micromark-core-commonmark@2.0.3", "", { "dependencies": { "decode-named-character-reference": "^1.0.0", "devlop": "^1.0.0", "micromark-factory-destination": "^2.0.0", "micromark-factory-label": "^2.0.0", "micromark-factory-space": "^2.0.0", "micromark-factory-title": "^2.0.0", "micromark-factory-whitespace": "^2.0.0", "micromark-util-character": "^2.0.0", "micromark-util-chunked": "^2.0.0", "micromark-util-classify-character": "^2.0.0", "micromark-util-html-tag-name": "^2.0.0", "micromark-util-normalize-identifier": "^2.0.0", "micromark-util-resolve-all": "^2.0.0", "micromark-util-subtokenize": "^2.0.0", "micromark-util-symbol": "^2.0.0", "micromark-util-types": "^2.0.0" } }, "sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg=="],
@@ -1320,10 +1327,10 @@
"motion/framer-motion/motion-utils": ["motion-utils@12.23.6", "", {}, "sha512-eAWoPgr4eFEOFfg2WjIsMoqJTW6Z8MTUCgn/GZ3VRpClWBdnbjryiA3ZSNLyxCTmCQx4RmYX6jX1iWHbenUPNQ=="],
- "opencut/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
-
"next/postcss/nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
+ "opencut/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
+
"opencut/next/@next/env": ["@next/env@15.4.5", "", {}, "sha512-ruM+q2SCOVCepUiERoxOmZY9ZVoecR3gcXNwCYZRvQQWRjhOiPJGmQ2fAiLR6YKWXcSAh7G79KEFxN3rwhs4LQ=="],
"opencut/next/@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@15.4.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-84dAN4fkfdC7nX6udDLz9GzQlMUwEMKD7zsseXrl7FTeIItF8vpk1lhLEnsotiiDt+QFu3O1FVWnqwcRD2U3KA=="],
diff --git a/package.json b/package.json
index 45a98282..616c97f7 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,7 @@
"format": "npx ultracite@latest format"
},
"dependencies": {
+ "mediabunny": "^1.9.3",
"next": "^15.3.4",
"react-country-flag": "^3.1.0",
"wavesurfer.js": "^7.9.8"
generated by cgit v1.3.1 (git 2.54.0) at 2026-10-17 08:47:11 +0000