diff options
Diffstat (limited to 'apps')
| -rw-r--r-- | apps/web/src/components/editor/media-panel/views/captions.tsx | 2 | ||||
| -rw-r--r-- | apps/web/src/components/editor/preview-panel.tsx | 520 | ||||
| -rw-r--r-- | apps/web/src/lib/media-processing.ts | 25 | ||||
| -rw-r--r-- | apps/web/src/lib/mediabunny-utils.ts (renamed from apps/web/src/lib/ffmpeg-utils.ts) | 295 | ||||
| -rw-r--r-- | apps/web/src/lib/timeline-renderer.ts | 179 |
5 files changed, 572 insertions, 449 deletions
diff --git a/apps/web/src/components/editor/media-panel/views/captions.tsx b/apps/web/src/components/editor/media-panel/views/captions.tsx index 4fe2cbd7..6e18781c 100644 --- a/apps/web/src/components/editor/media-panel/views/captions.tsx +++ b/apps/web/src/components/editor/media-panel/views/captions.tsx @@ -3,7 +3,7 @@ import { PropertyGroup } from "../../properties-panel/property-item"; import { BaseView } from "./base-view"; import { Language, LanguageSelect } from "@/components/language-select"; import { useState, useRef, useEffect } from "react"; -import { extractTimelineAudio } from "@/lib/ffmpeg-utils"; +import { extractTimelineAudio } from "@/lib/mediabunny-utils"; import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption"; import { useTimelineStore } from "@/stores/timeline-store"; import { Loader2, Shield, Trash2, Upload } from "lucide-react"; diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsx index 42c4becc..99c4edef 100644 --- a/apps/web/src/components/editor/preview-panel.tsx +++ b/apps/web/src/components/editor/preview-panel.tsx @@ -5,16 +5,19 @@ import { TimelineElement, TimelineTrack } from "@/types/timeline"; import { useMediaStore, type MediaItem } from "@/stores/media-store"; import { usePlaybackStore } from "@/stores/playback-store"; import { useEditorStore } from "@/stores/editor-store"; -import { VideoPlayer } from "@/components/ui/video-player"; -import { AudioPlayer } from "@/components/ui/audio-player"; import { Button } from "@/components/ui/button"; import { Play, Pause, Expand, SkipBack, SkipForward } from "lucide-react"; import { useState, useRef, useEffect, useCallback } from "react"; +import { VideoSampleSink } from "mediabunny"; +import { renderTimelineFrame } from "@/lib/timeline-renderer"; import { cn } from "@/lib/utils"; import { formatTimeCode } from "@/lib/time"; import { EditableTimecode } from "@/components/ui/editable-timecode"; -import { FONT_CLASS_MAP } from "@/lib/font-config"; -import { DEFAULT_CANVAS_SIZE, DEFAULT_FPS, useProjectStore } from "@/stores/project-store"; +import { + DEFAULT_CANVAS_SIZE, + DEFAULT_FPS, + useProjectStore, +} from "@/stores/project-store"; import { TextElementDragState } from "@/types/editor"; import { Popover, @@ -37,8 +40,20 @@ export function PreviewPanel() { const { tracks, getTotalDuration, updateTextElement } = useTimelineStore(); const { mediaItems } = useMediaStore(); const { currentTime, toggle, setCurrentTime } = usePlaybackStore(); + const { isPlaying, volume, muted } = usePlaybackStore(); const { activeProject } = useProjectStore(); const previewRef = useRef<HTMLDivElement>(null); + const canvasRef = useRef<HTMLCanvasElement>(null); + const lastFrameTimeRef = useRef(0); + const renderSeqRef = useRef(0); + const offscreenCanvasRef = useRef<OffscreenCanvas | HTMLCanvasElement | null>( + null + ); + + const audioContextRef = useRef<AudioContext | null>(null); + const audioGainRef = useRef<GainNode | null>(null); + const audioBuffersRef = useRef<Map<string, AudioBuffer>>(new Map()); + const playingSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set()); const containerRef = useRef<HTMLDivElement>(null); const [previewDimensions, setPreviewDimensions] = useState({ width: 0, @@ -262,227 +277,315 @@ export function PreviewPanel() { const activeElements = getActiveElements(); - // Get media elements for blur background (video/image only) - const getBlurBackgroundElements = (): ActiveElement[] => { - return activeElements.filter( - ({ element, mediaItem }) => - element.type === "media" && - mediaItem && - (mediaItem.type === "video" || mediaItem.type === "image") && - element.mediaId !== "test" // Exclude test elements - ); - }; - - const blurBackgroundElements = getBlurBackgroundElements(); + // Ensure first frame after mount/seek renders immediately + useEffect(() => { + const onSeek = () => { + lastFrameTimeRef.current = -Infinity; + renderSeqRef.current++; + }; + window.addEventListener("playback-seek", onSeek as EventListener); + lastFrameTimeRef.current = -Infinity; + return () => { + window.removeEventListener("playback-seek", onSeek as EventListener); + }; + }, []); - // Render blur background layer - const renderBlurBackground = () => { - if ( - !activeProject?.backgroundType || - activeProject.backgroundType !== "blur" || - blurBackgroundElements.length === 0 - ) { - return null; - } + // Web Audio: schedule only on play/pause/seek/volume/mute changes + useEffect(() => { + const stopAll = () => { + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + }; - // Use the first media element for background (could be enhanced to use primary/focused element) - const backgroundElement = blurBackgroundElements[0]; - const { element, mediaItem } = backgroundElement; + type WebAudioWindow = Window & { + AudioContext?: typeof AudioContext; + webkitAudioContext?: typeof AudioContext; + }; + const ensureAudioGraph = async () => { + if (!audioContextRef.current) { + const win = window as WebAudioWindow; + const Ctx = win.AudioContext ?? win.webkitAudioContext; + if (!Ctx) return; + audioContextRef.current = new Ctx(); + } + if (!audioGainRef.current) { + audioGainRef.current = audioContextRef.current!.createGain(); + audioGainRef.current.connect(audioContextRef.current!.destination); + } + if (audioContextRef.current!.state === "suspended") { + try { + await audioContextRef.current!.resume(); + } catch {} + } + const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume)); + audioGainRef.current!.gain.setValueAtTime( + gainValue, + audioContextRef.current!.currentTime + ); + }; - if (!mediaItem) return null; + const scheduleNow = async () => { + await ensureAudioGraph(); + const audioCtx = audioContextRef.current!; + const gain = audioGainRef.current!; - const blurIntensity = activeProject.blurIntensity || 8; + const tracksSnapshot = useTimelineStore.getState().tracks; + const mediaList = useMediaStore.getState().mediaItems; + const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const)); + const playbackNow = usePlaybackStore.getState().currentTime; - if (mediaItem.type === "video") { - return ( - <div - key={`blur-${element.id}`} - className="absolute inset-0 overflow-hidden" - style={{ - filter: `blur(${blurIntensity}px)`, - transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts - transformOrigin: "center", - }} - > - <VideoPlayer - src={mediaItem.url!} - poster={mediaItem.thumbnailUrl} - clipStartTime={element.startTime} - trimStart={element.trimStart} - trimEnd={element.trimEnd} - clipDuration={element.duration} - className="w-full h-full object-cover" - trackMuted={true} - /> - </div> - ); - } + const audible: Array<{ + id: string; + elementStart: number; + trimStart: number; + trimEnd: number; + duration: number; + muted: boolean; + trackMuted: boolean; + }> = []; + const uniqueIds = new Set<string>(); + for (const track of tracksSnapshot) { + for (const element of track.elements) { + if (element.type !== "media") continue; + const media = idToMedia.get(element.mediaId); + if (!media || media.type !== "audio") continue; + const visibleDuration = + element.duration - element.trimStart - element.trimEnd; + if (visibleDuration <= 0) continue; + const localTime = playbackNow - element.startTime + element.trimStart; + if (localTime < 0 || localTime >= visibleDuration) continue; + audible.push({ + id: media.id, + elementStart: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + duration: element.duration, + muted: !!element.muted, + trackMuted: !!track.muted, + }); + uniqueIds.add(media.id); + } + } - if (mediaItem.type === "image") { - return ( - <div - key={`blur-${element.id}`} - className="absolute inset-0 overflow-hidden" - style={{ - filter: `blur(${blurIntensity}px)`, - transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts - transformOrigin: "center", - }} - > - <img - src={mediaItem.url!} - alt={mediaItem.name} - className="w-full h-full object-cover" - draggable={false} - /> - </div> - ); - } + if (audible.length === 0) return; - return null; - }; + // Decode buffers as needed + const decodePromises: Array<Promise<void>> = []; + for (const id of uniqueIds) { + if (!audioBuffersRef.current.has(id)) { + const mediaItem = idToMedia.get(id); + if (!mediaItem) continue; + const p = (async () => { + const arr = await mediaItem.file.arrayBuffer(); + const buf = await audioCtx.decodeAudioData(arr.slice(0)); + audioBuffersRef.current.set(id, buf); + })(); + decodePromises.push(p); + } + } + await Promise.all(decodePromises); - // Render an element - const renderElement = (elementData: ActiveElement, index: number) => { - const { element, mediaItem } = elementData; + const startAt = audioCtx.currentTime + 0.02; + for (const entry of audible) { + if (entry.muted || entry.trackMuted) continue; + const buffer = audioBuffersRef.current.get(entry.id); + if (!buffer) continue; + const visibleDuration = + entry.duration - entry.trimStart - entry.trimEnd; + const localTime = Math.max( + 0, + playbackNow - entry.elementStart + entry.trimStart + ); + const playDuration = Math.max(0, visibleDuration - localTime); + if (playDuration <= 0) continue; + const src = audioCtx.createBufferSource(); + src.buffer = buffer; + src.connect(gain); + try { + src.start(startAt, localTime, playDuration); + playingSourcesRef.current.add(src); + } catch {} + } + }; - // Text elements - if (element.type === "text") { - const fontClassName = - FONT_CLASS_MAP[element.fontFamily as keyof typeof FONT_CLASS_MAP] || ""; + const onSeek = () => { + if (!isPlaying) return; + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + void scheduleNow(); + }; - const scaleRatio = previewDimensions.width / canvasSize.width; + // Apply volume/mute changes immediately + void ensureAudioGraph(); - return ( - <div - key={element.id} - className="absolute cursor-grab" - onMouseDown={(e) => - handleTextMouseDown(e, element, elementData.track.id) - } - style={{ - left: `${ - 50 + - ((dragState.isDragging && dragState.elementId === element.id - ? dragState.currentX - : element.x) / - canvasSize.width) * - 100 - }%`, - top: `${ - 50 + - ((dragState.isDragging && dragState.elementId === element.id - ? dragState.currentY - : element.y) / - canvasSize.height) * - 100 - }%`, - transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`, - opacity: element.opacity, - zIndex: 100 + index, // Text elements on top - }} - > - <div - className={fontClassName} - style={{ - fontSize: `${element.fontSize * scaleRatio}px`, - color: element.color, - backgroundColor: element.backgroundColor, - textAlign: element.textAlign, - fontWeight: element.fontWeight, - fontStyle: element.fontStyle, - textDecoration: element.textDecoration, - padding: `${4 * scaleRatio}px ${8 * scaleRatio}px`, - borderRadius: `${2 * scaleRatio}px`, - whiteSpace: "nowrap", - // Fallback for system fonts that don't have classes - ...(fontClassName === "" && { fontFamily: element.fontFamily }), - }} - > - {element.content} - </div> - </div> - ); + // Start/stop on play state changes + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + if (isPlaying) { + void scheduleNow(); } - // Media elements - if (element.type === "media") { - // Test elements - if (!mediaItem || element.mediaId === "test") { - return ( - <div - key={element.id} - className="absolute inset-0 bg-linear-to-br from-blue-500/20 to-purple-500/20 flex items-center justify-center" - > - <div className="text-center"> - <div className="text-2xl mb-2">🎬</div> - <p className="text-xs text-foreground">{element.name}</p> - </div> - </div> - ); + window.addEventListener("playback-seek", onSeek as EventListener); + return () => { + window.removeEventListener("playback-seek", onSeek as EventListener); + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} } + playingSourcesRef.current.clear(); + }; + }, [isPlaying, volume, muted, mediaItems]); - // Video elements - if (mediaItem.type === "video") { - return ( - <div - key={element.id} - className="absolute inset-0 flex items-center justify-center" - > - <VideoPlayer - src={mediaItem.url!} - poster={mediaItem.thumbnailUrl} - clipStartTime={element.startTime} - trimStart={element.trimStart} - trimEnd={element.trimEnd} - clipDuration={element.duration} - trackMuted={element.muted || elementData.track.muted} - /> - </div> - ); + // Canvas: draw current frame for visible elements using offscreen compositing + useEffect(() => { + const draw = async () => { + const canvas = canvasRef.current; + if (!canvas) return; + const mainCtx = canvas.getContext("2d"); + if (!mainCtx) return; + + // Set canvas internal resolution to avoid blurry scaling + const displayWidth = Math.max(1, Math.floor(previewDimensions.width)); + const displayHeight = Math.max(1, Math.floor(previewDimensions.height)); + if (canvas.width !== displayWidth || canvas.height !== displayHeight) { + canvas.width = displayWidth; + canvas.height = displayHeight; } - // Image elements - if (mediaItem.type === "image") { - return ( - <div - key={element.id} - className="absolute inset-0 flex items-center justify-center" - > - <img - src={mediaItem.url!} - alt={mediaItem.name} - className="max-w-full max-h-full object-contain" - draggable={false} - /> - </div> - ); + // Throttle rendering to project FPS during playback only + const fps = activeProject?.fps || DEFAULT_FPS; + const minDelta = 1 / fps; + if (isPlaying) { + if (currentTime - lastFrameTimeRef.current < minDelta) { + return; + } + lastFrameTimeRef.current = currentTime; } - // Audio elements (no visual representation) - if (mediaItem.type === "audio") { - return ( - <div - key={element.id} - className="absolute inset-0" - style={{ pointerEvents: "none" }} - > - <AudioPlayer - src={mediaItem.url!} - clipStartTime={element.startTime} - trimStart={element.trimStart} - trimEnd={element.trimEnd} - clipDuration={element.duration} - trackMuted={element.muted || elementData.track.muted} - /> - </div> + // Invalidate older async renders when user scrubs rapidly + const mySeq = (renderSeqRef.current += 1); + + // Offscreen buffer to avoid flicker (reuse canvas) + if (!offscreenCanvasRef.current) { + const hasOffscreen = + typeof (globalThis as unknown as { OffscreenCanvas?: unknown }) + .OffscreenCanvas !== "undefined"; + if (hasOffscreen) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + offscreenCanvasRef.current = new (globalThis as any).OffscreenCanvas( + displayWidth, + displayHeight + ) as OffscreenCanvas; + } else { + const c = document.createElement("canvas"); + c.width = displayWidth; + c.height = displayHeight; + offscreenCanvasRef.current = c; + } + } + // Ensure size matches + if ( + offscreenCanvasRef.current && + (offscreenCanvasRef.current as HTMLCanvasElement).getContext + ) { + const c = offscreenCanvasRef.current as HTMLCanvasElement; + if (c.width !== displayWidth || c.height !== displayHeight) { + c.width = displayWidth; + c.height = displayHeight; + } + } else { + const c = offscreenCanvasRef.current as OffscreenCanvas; + // @ts-ignore width/height exist on OffscreenCanvas in modern browsers + if ( + (c as unknown as { width: number }).width !== displayWidth || + (c as unknown as { height: number }).height !== displayHeight + ) { + // @ts-ignore + (c as unknown as { width: number }).width = displayWidth; + // @ts-ignore + (c as unknown as { height: number }).height = displayHeight; + } + } + const offscreenCanvas = offscreenCanvasRef.current as + | HTMLCanvasElement + | OffscreenCanvas; + const offCtx = (offscreenCanvas as HTMLCanvasElement).getContext + ? (offscreenCanvas as HTMLCanvasElement).getContext("2d") + : (offscreenCanvas as OffscreenCanvas).getContext("2d"); + if (!offCtx) return; + + await renderTimelineFrame({ + ctx: offCtx as CanvasRenderingContext2D, + time: currentTime, + canvasWidth: displayWidth, + canvasHeight: displayHeight, + tracks, + mediaItems, + backgroundColor: + activeProject?.backgroundType === "blur" + ? "transparent" + : activeProject?.backgroundColor || "#000000", + useCache: false, + fps: activeProject?.fps || DEFAULT_FPS, + }); + + // Blit offscreen to visible canvas + mainCtx.clearRect(0, 0, displayWidth, displayHeight); + if ((offscreenCanvas as HTMLCanvasElement).getContext) { + mainCtx.drawImage(offscreenCanvas as HTMLCanvasElement, 0, 0); + } else { + mainCtx.drawImage( + offscreenCanvas as unknown as CanvasImageSource, + 0, + 0 ); } - } + }; + + void draw(); + }, [ + activeElements, + currentTime, + previewDimensions.width, + previewDimensions.height, + canvasSize.width, + canvasSize.height, + activeProject?.backgroundType, + activeProject?.backgroundColor, + ]); - return null; + // Get media elements for blur background (video/image only) + const getBlurBackgroundElements = (): ActiveElement[] => { + return activeElements.filter( + ({ element, mediaItem }) => + element.type === "media" && + mediaItem && + (mediaItem.type === "video" || mediaItem.type === "image") && + element.mediaId !== "test" // Exclude test elements + ); }; + const blurBackgroundElements = getBlurBackgroundElements(); + + // Render blur background layer (handled by canvas now) + const renderBlurBackground = () => null; + + // Render an element (canvas handles visuals now). Audio playback to be implemented via Web Audio. + const renderElement = (_elementData: ActiveElement) => null; + return ( <> <div className="h-full w-full flex flex-col min-h-0 min-w-0 bg-panel rounded-sm relative"> @@ -505,14 +608,23 @@ export function PreviewPanel() { }} > {renderBlurBackground()} + <canvas + ref={canvasRef} + style={{ + position: "absolute", + left: 0, + top: 0, + width: previewDimensions.width, + height: previewDimensions.height, + }} + aria-label="Video preview canvas" + /> {activeElements.length === 0 ? ( <div className="absolute inset-0 flex items-center justify-center text-muted-foreground"> No elements at current time </div> ) : ( - activeElements.map((elementData, index) => - renderElement(elementData, index) - ) + activeElements.map((elementData) => renderElement(elementData)) )} <LayoutGuideOverlay /> </div> diff --git a/apps/web/src/lib/media-processing.ts b/apps/web/src/lib/media-processing.ts index 353c3113..15c71626 100644 --- a/apps/web/src/lib/media-processing.ts +++ b/apps/web/src/lib/media-processing.ts @@ -6,7 +6,7 @@ import { getImageDimensions, type MediaItem, } from "@/stores/media-store"; -import { generateThumbnail, getVideoInfo } from "./ffmpeg-utils"; +import { generateThumbnail, getVideoInfo } from "./mediabunny-utils"; export interface ProcessedMediaItem extends Omit<MediaItem, "id"> {} @@ -37,33 +37,23 @@ export async function processMediaFiles( try { if (fileType === "image") { - // Get image dimensions const dimensions = await getImageDimensions(file); width = dimensions.width; height = dimensions.height; } else if (fileType === "video") { try { - // Use FFmpeg for comprehensive video info extraction - const videoInfo = await getVideoInfo(file); + const videoInfo = await getVideoInfo({ videoFile: file }); duration = videoInfo.duration; width = videoInfo.width; height = videoInfo.height; fps = videoInfo.fps; - // Generate thumbnail using FFmpeg - thumbnailUrl = await generateThumbnail(file, 1); + thumbnailUrl = await generateThumbnail({ + videoFile: file, + timeInSeconds: 1, + }); } catch (error) { - console.warn( - "FFmpeg processing failed, falling back to basic processing:", - error - ); - // Fallback to basic processing - const videoResult = await generateVideoThumbnail(file); - thumbnailUrl = videoResult.thumbnailUrl; - width = videoResult.width; - height = videoResult.height; - duration = await getMediaDuration(file); - // FPS will remain undefined for fallback + console.warn("Video processing failed", error); } } else if (fileType === "audio") { // For audio, we don't set width/height/fps (they'll be undefined) @@ -82,7 +72,6 @@ export async function processMediaFiles( fps, }); - // Yield back to the event loop to keep the UI responsive await new Promise((resolve) => setTimeout(resolve, 0)); completed += 1; diff --git a/apps/web/src/lib/ffmpeg-utils.ts b/apps/web/src/lib/mediabunny-utils.ts index 0643ecb9..28e6b97f 100644 --- a/apps/web/src/lib/ffmpeg-utils.ts +++ b/apps/web/src/lib/mediabunny-utils.ts @@ -1,7 +1,7 @@ import { FFmpeg } from "@ffmpeg/ffmpeg"; -import { toBlobURL } from "@ffmpeg/util"; import { useTimelineStore } from "@/stores/timeline-store"; import { useMediaStore } from "@/stores/media-store"; +import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny"; let ffmpeg: FFmpeg | null = null; @@ -14,256 +14,99 @@ export const initFFmpeg = async (): Promise<FFmpeg> => { return ffmpeg; }; -export const generateThumbnail = async ( - videoFile: File, - timeInSeconds = 1 -): Promise<string> => { - const ffmpeg = await initFFmpeg(); - - const inputName = "input.mp4"; - const outputName = "thumbnail.jpg"; - - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - // Generate thumbnail at specific time - await ffmpeg.exec([ - "-i", - inputName, - "-ss", - timeInSeconds.toString(), - "-vframes", - "1", - "-vf", - "scale=320:240", - "-q:v", - "2", - outputName, - ]); +export async function generateThumbnail({ + videoFile, + timeInSeconds, +}: { + videoFile: File; + timeInSeconds: number; +}): Promise<string> { + const input = new Input({ + source: new BlobSource(videoFile), + formats: ALL_FORMATS, + }); - // Read output file - const data = await ffmpeg.readFile(outputName); - const blob = new Blob([data], { type: "image/jpeg" }); - - // Cleanup - await ffmpeg.deleteFile(inputName); - await ffmpeg.deleteFile(outputName); + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) { + throw new Error("No video track found in the file"); + } - return URL.createObjectURL(blob); -}; + // Check if we can decode this video + const canDecode = await videoTrack.canDecode(); + if (!canDecode) { + throw new Error("Video codec not supported for decoding"); + } -export const trimVideo = async ( - videoFile: File, - startTime: number, - endTime: number, - onProgress?: (progress: number) => void -): Promise<Blob> => { - const ffmpeg = await initFFmpeg(); + const sink = new VideoSampleSink(videoTrack); - const inputName = "input.mp4"; - const outputName = "output.mp4"; + const frame = await sink.getSample(timeInSeconds); - // Set up progress callback - if (onProgress) { - ffmpeg.on("progress", ({ progress }) => { - onProgress(progress * 100); - }); + if (!frame) { + throw new Error("Could not get frame at specified time"); } - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - const duration = endTime - startTime; + const canvas = document.createElement("canvas"); + canvas.width = 320; + canvas.height = 240; + const ctx = canvas.getContext("2d"); - // Trim video - await ffmpeg.exec([ - "-i", - inputName, - "-ss", - startTime.toString(), - "-t", - duration.toString(), - "-c", - "copy", // Use stream copy for faster processing - outputName, - ]); - - // Read output file - const data = await ffmpeg.readFile(outputName); - const blob = new Blob([data], { type: "video/mp4" }); + if (!ctx) { + throw new Error("Could not get canvas context"); + } - // Cleanup - await ffmpeg.deleteFile(inputName); - await ffmpeg.deleteFile(outputName); + frame.draw(ctx, 0, 0, 320, 240); - return blob; -}; + return new Promise((resolve, reject) => { + canvas.toBlob( + (blob) => { + if (blob) { + resolve(URL.createObjectURL(blob)); + } else { + reject(new Error("Failed to create thumbnail blob")); + } + }, + "image/jpeg", + 0.8 + ); + }); +} -export const getVideoInfo = async ( - videoFile: File -): Promise<{ +export async function getVideoInfo({ + videoFile, +}: { + videoFile: File; +}): Promise<{ duration: number; width: number; height: number; fps: number; -}> => { - const ffmpeg = await initFFmpeg(); - - const inputName = "input.mp4"; - - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - // Capture FFmpeg stderr output with a one-time listener pattern - let ffmpegOutput = ""; - let listening = true; - const listener = (data: string) => { - if (listening) ffmpegOutput += data; - }; - ffmpeg.on("log", ({ message }) => listener(message)); - - // Run ffmpeg to get info (stderr will contain the info) - try { - await ffmpeg.exec(["-i", inputName, "-f", "null", "-"]); - } catch (error) { - listening = false; - await ffmpeg.deleteFile(inputName); - console.error("FFmpeg execution failed:", error); - throw new Error( - "Failed to extract video info. The file may be corrupted or in an unsupported format." - ); - } +}> { + const input = new Input({ + source: new BlobSource(videoFile), + formats: ALL_FORMATS, + }); - // Disable listener after exec completes - listening = false; + const duration = await input.computeDuration(); + const videoTrack = await input.getPrimaryVideoTrack(); - // Cleanup - await ffmpeg.deleteFile(inputName); - - // Parse output for duration, resolution, and fps - // Example: Duration: 00:00:10.00, start: 0.000000, bitrate: 1234 kb/s - // Example: Stream #0:0: Video: h264 (High), yuv420p(progressive), 1920x1080 [SAR 1:1 DAR 16:9], 30 fps, 30 tbr, 90k tbn, 60 tbc - - const durationMatch = ffmpegOutput.match(/Duration: (\d+):(\d+):([\d.]+)/); - let duration = 0; - if (durationMatch) { - const [, h, m, s] = durationMatch; - duration = parseInt(h) * 3600 + parseInt(m) * 60 + parseFloat(s); + if (!videoTrack) { + throw new Error("No video track found in the file"); } - const videoStreamMatch = ffmpegOutput.match( - /Video:.* (\d+)x(\d+)[^,]*, ([\d.]+) fps/ - ); - let width = 0, - height = 0, - fps = 0; - if (videoStreamMatch) { - width = parseInt(videoStreamMatch[1]); - height = parseInt(videoStreamMatch[2]); - fps = parseFloat(videoStreamMatch[3]); - } + // Get frame rate from packet statistics + const packetStats = await videoTrack.computePacketStats(100); + const fps = packetStats.averagePacketRate; return { duration, - width, - height, + width: videoTrack.displayWidth, + height: videoTrack.displayHeight, fps, }; -}; - -export const convertToWebM = async ( - videoFile: File, - onProgress?: (progress: number) => void -): Promise<Blob> => { - const ffmpeg = await initFFmpeg(); - - const inputName = "input.mp4"; - const outputName = "output.webm"; - - // Set up progress callback - if (onProgress) { - ffmpeg.on("progress", ({ progress }) => { - onProgress(progress * 100); - }); - } - - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - // Convert to WebM - await ffmpeg.exec([ - "-i", - inputName, - "-c:v", - "libvpx-vp9", - "-crf", - "30", - "-b:v", - "0", - "-c:a", - "libopus", - outputName, - ]); - - // Read output file - const data = await ffmpeg.readFile(outputName); - const blob = new Blob([data], { type: "video/webm" }); - - // Cleanup - await ffmpeg.deleteFile(inputName); - await ffmpeg.deleteFile(outputName); - - return blob; -}; - -export const extractAudio = async ( - videoFile: File, - format: "mp3" | "wav" = "mp3" -): Promise<Blob> => { - const ffmpeg = await initFFmpeg(); - - const inputName = "input.mp4"; - const outputName = `output.${format}`; - - // Write input file - await ffmpeg.writeFile( - inputName, - new Uint8Array(await videoFile.arrayBuffer()) - ); - - // Extract audio - await ffmpeg.exec([ - "-i", - inputName, - "-vn", // Disable video - "-acodec", - format === "mp3" ? "libmp3lame" : "pcm_s16le", - outputName, - ]); - - // Read output file - const data = await ffmpeg.readFile(outputName); - const blob = new Blob([data], { type: `audio/${format}` }); - - // Cleanup - await ffmpeg.deleteFile(inputName); - await ffmpeg.deleteFile(outputName); - - return blob; -}; +} +// Audio mixing for timeline - keeping FFmpeg for now due to complexity +// TODO: Replace with Mediabunny audio processing when implementing canvas preview export const extractTimelineAudio = async ( onProgress?: (progress: number) => void ): Promise<Blob> => { diff --git a/apps/web/src/lib/timeline-renderer.ts b/apps/web/src/lib/timeline-renderer.ts new file mode 100644 index 00000000..1c1eafcd --- /dev/null +++ b/apps/web/src/lib/timeline-renderer.ts @@ -0,0 +1,179 @@ +import { Input, ALL_FORMATS, BlobSource, VideoSampleSink } from "mediabunny";
+import type { TimelineTrack } from "@/types/timeline";
+import type { MediaItem } from "@/stores/media-store";
+
+export interface RenderContext {
+ ctx: CanvasRenderingContext2D;
+ time: number;
+ canvasWidth: number;
+ canvasHeight: number;
+ tracks: TimelineTrack[];
+ mediaItems: MediaItem[];
+ backgroundColor?: string;
+ useCache?: boolean;
+ cache?: Map<
+ string,
+ Map<
+ number,
+ {
+ draw: (
+ c: CanvasRenderingContext2D,
+ x: number,
+ y: number,
+ w: number,
+ h: number
+ ) => void;
+ }
+ >
+ >;
+ fps: number;
+}
+
+export async function renderTimelineFrame({
+ ctx,
+ time,
+ canvasWidth,
+ canvasHeight,
+ tracks,
+ mediaItems,
+ backgroundColor,
+ useCache = true,
+ cache,
+ fps,
+}: RenderContext): Promise<void> {
+ // Background
+ ctx.clearRect(0, 0, canvasWidth, canvasHeight);
+ if (backgroundColor && backgroundColor !== "transparent") {
+ ctx.fillStyle = backgroundColor;
+ ctx.fillRect(0, 0, canvasWidth, canvasHeight);
+ }
+
+ const scaleX = 1;
+ const scaleY = 1;
+ const idToMedia = new Map(mediaItems.map((m) => [m.id, m] as const));
+ const active: Array<{
+ track: TimelineTrack;
+ element: TimelineTrack["elements"][number];
+ mediaItem: MediaItem | null;
+ }> = [];
+
+ for (let t = tracks.length - 1; t >= 0; t -= 1) {
+ const track = tracks[t];
+ for (const element of track.elements) {
+ if ((element as any).hidden) continue;
+ const elementStart = element.startTime;
+ const elementEnd =
+ element.startTime +
+ (element.duration - element.trimStart - element.trimEnd);
+ if (time >= elementStart && time < elementEnd) {
+ let mediaItem: MediaItem | null = null;
+ if (element.type === "media") {
+ mediaItem =
+ element.mediaId === "test"
+ ? null
+ : idToMedia.get(element.mediaId) || null;
+ }
+ active.push({ track, element, mediaItem });
+ }
+ }
+ }
+
+ for (const { element, mediaItem } of active) {
+ if (element.type === "media" && mediaItem) {
+ if (mediaItem.type === "video") {
+ const input = new Input({
+ source: new BlobSource(mediaItem.file),
+ formats: ALL_FORMATS,
+ });
+ const track = await input.getPrimaryVideoTrack();
+ if (!track) continue;
+ const decodable = await track.canDecode();
+ if (!decodable) continue;
+ const sink = new VideoSampleSink(track);
+
+ const localTime = time - element.startTime + element.trimStart;
+ const sample = await sink.getSample(localTime);
+ if (!sample) continue;
+
+ const mediaW = Math.max(1, mediaItem.width || canvasWidth);
+ const mediaH = Math.max(1, mediaItem.height || canvasHeight);
+ const containScale = Math.min(
+ canvasWidth / mediaW,
+ canvasHeight / mediaH
+ );
+ const drawW = mediaW * containScale;
+ const drawH = mediaH * containScale;
+ const drawX = (canvasWidth - drawW) / 2;
+ const drawY = (canvasHeight - drawH) / 2;
+ sample.draw(ctx, drawX, drawY, drawW, drawH);
+ }
+ if (mediaItem.type === "image") {
+ const img = new Image();
+ await new Promise<void>((resolve, reject) => {
+ img.onload = () => resolve();
+ img.onerror = () => reject(new Error("Image load failed"));
+ img.src = mediaItem.url || URL.createObjectURL(mediaItem.file);
+ });
+ const mediaW = Math.max(
+ 1,
+ mediaItem.width || img.naturalWidth || canvasWidth
+ );
+ const mediaH = Math.max(
+ 1,
+ mediaItem.height || img.naturalHeight || canvasHeight
+ );
+ const containScale = Math.min(
+ canvasWidth / mediaW,
+ canvasHeight / mediaH
+ );
+ const drawW = mediaW * containScale;
+ const drawH = mediaH * containScale;
+ const drawX = (canvasWidth - drawW) / 2;
+ const drawY = (canvasHeight - drawH) / 2;
+ ctx.drawImage(img, drawX, drawY, drawW, drawH);
+ }
+ }
+ if (element.type === "text") {
+ const posX = canvasWidth / 2 + (element as any).x * scaleX;
+ const posY = canvasHeight / 2 + (element as any).y * scaleY;
+ ctx.save();
+ ctx.translate(posX, posY);
+ ctx.rotate(((element as any).rotation * Math.PI) / 180);
+ ctx.globalAlpha = Math.max(0, Math.min(1, (element as any).opacity));
+ const px = (element as any).fontSize * scaleX;
+ const weight = (element as any).fontWeight === "bold" ? "bold " : "";
+ const style = (element as any).fontStyle === "italic" ? "italic " : "";
+ ctx.font = `${style}${weight}${px}px ${(element as any).fontFamily}`;
+ ctx.fillStyle = (element as any).color;
+ ctx.textAlign = (element as any).textAlign;
+ ctx.textBaseline = "middle";
+ const metrics = ctx.measureText((element as any).content);
+ const ascent =
+ (metrics as unknown as { actualBoundingBoxAscent?: number })
+ .actualBoundingBoxAscent ?? px * 0.8;
+ const descent =
+ (metrics as unknown as { actualBoundingBoxDescent?: number })
+ .actualBoundingBoxDescent ?? px * 0.2;
+ const textW = metrics.width;
+ const textH = ascent + descent;
+ const padX = 8 * scaleX;
+ const padY = 4 * scaleX;
+ if ((element as any).backgroundColor) {
+ ctx.save();
+ ctx.fillStyle = (element as any).backgroundColor;
+ let bgLeft = -textW / 2;
+ if (ctx.textAlign === "left") bgLeft = 0;
+ if (ctx.textAlign === "right") bgLeft = -textW;
+ ctx.fillRect(
+ bgLeft - padX,
+ -textH / 2 - padY,
+ textW + padX * 2,
+ textH + padY * 2
+ );
+ ctx.restore();
+ }
+ ctx.fillText((element as any).content, 0, 0);
+ ctx.restore();
+ }
+ }
+}
|