diff options
Diffstat (limited to 'apps/web/src/components')
| -rw-r--r-- | apps/web/src/components/editor/media-panel/views/captions.tsx | 2 | ||||
| -rw-r--r-- | apps/web/src/components/editor/preview-panel.tsx | 520 |
2 files changed, 317 insertions, 205 deletions
diff --git a/apps/web/src/components/editor/media-panel/views/captions.tsx b/apps/web/src/components/editor/media-panel/views/captions.tsx index 4fe2cbd7..6e18781c 100644 --- a/apps/web/src/components/editor/media-panel/views/captions.tsx +++ b/apps/web/src/components/editor/media-panel/views/captions.tsx @@ -3,7 +3,7 @@ import { PropertyGroup } from "../../properties-panel/property-item"; import { BaseView } from "./base-view"; import { Language, LanguageSelect } from "@/components/language-select"; import { useState, useRef, useEffect } from "react"; -import { extractTimelineAudio } from "@/lib/ffmpeg-utils"; +import { extractTimelineAudio } from "@/lib/mediabunny-utils"; import { encryptWithRandomKey, arrayBufferToBase64 } from "@/lib/zk-encryption"; import { useTimelineStore } from "@/stores/timeline-store"; import { Loader2, Shield, Trash2, Upload } from "lucide-react"; diff --git a/apps/web/src/components/editor/preview-panel.tsx b/apps/web/src/components/editor/preview-panel.tsx index 42c4becc..99c4edef 100644 --- a/apps/web/src/components/editor/preview-panel.tsx +++ b/apps/web/src/components/editor/preview-panel.tsx @@ -5,16 +5,19 @@ import { TimelineElement, TimelineTrack } from "@/types/timeline"; import { useMediaStore, type MediaItem } from "@/stores/media-store"; import { usePlaybackStore } from "@/stores/playback-store"; import { useEditorStore } from "@/stores/editor-store"; -import { VideoPlayer } from "@/components/ui/video-player"; -import { AudioPlayer } from "@/components/ui/audio-player"; import { Button } from "@/components/ui/button"; import { Play, Pause, Expand, SkipBack, SkipForward } from "lucide-react"; import { useState, useRef, useEffect, useCallback } from "react"; +import { VideoSampleSink } from "mediabunny"; +import { renderTimelineFrame } from "@/lib/timeline-renderer"; import { cn } from "@/lib/utils"; import { formatTimeCode } from "@/lib/time"; import { EditableTimecode } from "@/components/ui/editable-timecode"; -import { FONT_CLASS_MAP } from "@/lib/font-config"; -import { DEFAULT_CANVAS_SIZE, DEFAULT_FPS, useProjectStore } from "@/stores/project-store"; +import { + DEFAULT_CANVAS_SIZE, + DEFAULT_FPS, + useProjectStore, +} from "@/stores/project-store"; import { TextElementDragState } from "@/types/editor"; import { Popover, @@ -37,8 +40,20 @@ export function PreviewPanel() { const { tracks, getTotalDuration, updateTextElement } = useTimelineStore(); const { mediaItems } = useMediaStore(); const { currentTime, toggle, setCurrentTime } = usePlaybackStore(); + const { isPlaying, volume, muted } = usePlaybackStore(); const { activeProject } = useProjectStore(); const previewRef = useRef<HTMLDivElement>(null); + const canvasRef = useRef<HTMLCanvasElement>(null); + const lastFrameTimeRef = useRef(0); + const renderSeqRef = useRef(0); + const offscreenCanvasRef = useRef<OffscreenCanvas | HTMLCanvasElement | null>( + null + ); + + const audioContextRef = useRef<AudioContext | null>(null); + const audioGainRef = useRef<GainNode | null>(null); + const audioBuffersRef = useRef<Map<string, AudioBuffer>>(new Map()); + const playingSourcesRef = useRef<Set<AudioBufferSourceNode>>(new Set()); const containerRef = useRef<HTMLDivElement>(null); const [previewDimensions, setPreviewDimensions] = useState({ width: 0, @@ -262,227 +277,315 @@ export function PreviewPanel() { const activeElements = getActiveElements(); - // Get media elements for blur background (video/image only) - const getBlurBackgroundElements = (): ActiveElement[] => { - return activeElements.filter( - ({ element, mediaItem }) => - element.type === "media" && - mediaItem && - (mediaItem.type === "video" || mediaItem.type === "image") && - element.mediaId !== "test" // Exclude test elements - ); - }; - - const blurBackgroundElements = getBlurBackgroundElements(); + // Ensure first frame after mount/seek renders immediately + useEffect(() => { + const onSeek = () => { + lastFrameTimeRef.current = -Infinity; + renderSeqRef.current++; + }; + window.addEventListener("playback-seek", onSeek as EventListener); + lastFrameTimeRef.current = -Infinity; + return () => { + window.removeEventListener("playback-seek", onSeek as EventListener); + }; + }, []); - // Render blur background layer - const renderBlurBackground = () => { - if ( - !activeProject?.backgroundType || - activeProject.backgroundType !== "blur" || - blurBackgroundElements.length === 0 - ) { - return null; - } + // Web Audio: schedule only on play/pause/seek/volume/mute changes + useEffect(() => { + const stopAll = () => { + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + }; - // Use the first media element for background (could be enhanced to use primary/focused element) - const backgroundElement = blurBackgroundElements[0]; - const { element, mediaItem } = backgroundElement; + type WebAudioWindow = Window & { + AudioContext?: typeof AudioContext; + webkitAudioContext?: typeof AudioContext; + }; + const ensureAudioGraph = async () => { + if (!audioContextRef.current) { + const win = window as WebAudioWindow; + const Ctx = win.AudioContext ?? win.webkitAudioContext; + if (!Ctx) return; + audioContextRef.current = new Ctx(); + } + if (!audioGainRef.current) { + audioGainRef.current = audioContextRef.current!.createGain(); + audioGainRef.current.connect(audioContextRef.current!.destination); + } + if (audioContextRef.current!.state === "suspended") { + try { + await audioContextRef.current!.resume(); + } catch {} + } + const gainValue = muted ? 0 : Math.max(0, Math.min(1, volume)); + audioGainRef.current!.gain.setValueAtTime( + gainValue, + audioContextRef.current!.currentTime + ); + }; - if (!mediaItem) return null; + const scheduleNow = async () => { + await ensureAudioGraph(); + const audioCtx = audioContextRef.current!; + const gain = audioGainRef.current!; - const blurIntensity = activeProject.blurIntensity || 8; + const tracksSnapshot = useTimelineStore.getState().tracks; + const mediaList = useMediaStore.getState().mediaItems; + const idToMedia = new Map(mediaList.map((m) => [m.id, m] as const)); + const playbackNow = usePlaybackStore.getState().currentTime; - if (mediaItem.type === "video") { - return ( - <div - key={`blur-${element.id}`} - className="absolute inset-0 overflow-hidden" - style={{ - filter: `blur(${blurIntensity}px)`, - transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts - transformOrigin: "center", - }} - > - <VideoPlayer - src={mediaItem.url!} - poster={mediaItem.thumbnailUrl} - clipStartTime={element.startTime} - trimStart={element.trimStart} - trimEnd={element.trimEnd} - clipDuration={element.duration} - className="w-full h-full object-cover" - trackMuted={true} - /> - </div> - ); - } + const audible: Array<{ + id: string; + elementStart: number; + trimStart: number; + trimEnd: number; + duration: number; + muted: boolean; + trackMuted: boolean; + }> = []; + const uniqueIds = new Set<string>(); + for (const track of tracksSnapshot) { + for (const element of track.elements) { + if (element.type !== "media") continue; + const media = idToMedia.get(element.mediaId); + if (!media || media.type !== "audio") continue; + const visibleDuration = + element.duration - element.trimStart - element.trimEnd; + if (visibleDuration <= 0) continue; + const localTime = playbackNow - element.startTime + element.trimStart; + if (localTime < 0 || localTime >= visibleDuration) continue; + audible.push({ + id: media.id, + elementStart: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + duration: element.duration, + muted: !!element.muted, + trackMuted: !!track.muted, + }); + uniqueIds.add(media.id); + } + } - if (mediaItem.type === "image") { - return ( - <div - key={`blur-${element.id}`} - className="absolute inset-0 overflow-hidden" - style={{ - filter: `blur(${blurIntensity}px)`, - transform: "scale(1.1)", // Slightly zoom to avoid blur edge artifacts - transformOrigin: "center", - }} - > - <img - src={mediaItem.url!} - alt={mediaItem.name} - className="w-full h-full object-cover" - draggable={false} - /> - </div> - ); - } + if (audible.length === 0) return; - return null; - }; + // Decode buffers as needed + const decodePromises: Array<Promise<void>> = []; + for (const id of uniqueIds) { + if (!audioBuffersRef.current.has(id)) { + const mediaItem = idToMedia.get(id); + if (!mediaItem) continue; + const p = (async () => { + const arr = await mediaItem.file.arrayBuffer(); + const buf = await audioCtx.decodeAudioData(arr.slice(0)); + audioBuffersRef.current.set(id, buf); + })(); + decodePromises.push(p); + } + } + await Promise.all(decodePromises); - // Render an element - const renderElement = (elementData: ActiveElement, index: number) => { - const { element, mediaItem } = elementData; + const startAt = audioCtx.currentTime + 0.02; + for (const entry of audible) { + if (entry.muted || entry.trackMuted) continue; + const buffer = audioBuffersRef.current.get(entry.id); + if (!buffer) continue; + const visibleDuration = + entry.duration - entry.trimStart - entry.trimEnd; + const localTime = Math.max( + 0, + playbackNow - entry.elementStart + entry.trimStart + ); + const playDuration = Math.max(0, visibleDuration - localTime); + if (playDuration <= 0) continue; + const src = audioCtx.createBufferSource(); + src.buffer = buffer; + src.connect(gain); + try { + src.start(startAt, localTime, playDuration); + playingSourcesRef.current.add(src); + } catch {} + } + }; - // Text elements - if (element.type === "text") { - const fontClassName = - FONT_CLASS_MAP[element.fontFamily as keyof typeof FONT_CLASS_MAP] || ""; + const onSeek = () => { + if (!isPlaying) return; + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + void scheduleNow(); + }; - const scaleRatio = previewDimensions.width / canvasSize.width; + // Apply volume/mute changes immediately + void ensureAudioGraph(); - return ( - <div - key={element.id} - className="absolute cursor-grab" - onMouseDown={(e) => - handleTextMouseDown(e, element, elementData.track.id) - } - style={{ - left: `${ - 50 + - ((dragState.isDragging && dragState.elementId === element.id - ? dragState.currentX - : element.x) / - canvasSize.width) * - 100 - }%`, - top: `${ - 50 + - ((dragState.isDragging && dragState.elementId === element.id - ? dragState.currentY - : element.y) / - canvasSize.height) * - 100 - }%`, - transform: `translate(-50%, -50%) rotate(${element.rotation}deg)`, - opacity: element.opacity, - zIndex: 100 + index, // Text elements on top - }} - > - <div - className={fontClassName} - style={{ - fontSize: `${element.fontSize * scaleRatio}px`, - color: element.color, - backgroundColor: element.backgroundColor, - textAlign: element.textAlign, - fontWeight: element.fontWeight, - fontStyle: element.fontStyle, - textDecoration: element.textDecoration, - padding: `${4 * scaleRatio}px ${8 * scaleRatio}px`, - borderRadius: `${2 * scaleRatio}px`, - whiteSpace: "nowrap", - // Fallback for system fonts that don't have classes - ...(fontClassName === "" && { fontFamily: element.fontFamily }), - }} - > - {element.content} - </div> - </div> - ); + // Start/stop on play state changes + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} + } + playingSourcesRef.current.clear(); + if (isPlaying) { + void scheduleNow(); } - // Media elements - if (element.type === "media") { - // Test elements - if (!mediaItem || element.mediaId === "test") { - return ( - <div - key={element.id} - className="absolute inset-0 bg-linear-to-br from-blue-500/20 to-purple-500/20 flex items-center justify-center" - > - <div className="text-center"> - <div className="text-2xl mb-2">🎬</div> - <p className="text-xs text-foreground">{element.name}</p> - </div> - </div> - ); + window.addEventListener("playback-seek", onSeek as EventListener); + return () => { + window.removeEventListener("playback-seek", onSeek as EventListener); + for (const src of playingSourcesRef.current) { + try { + src.stop(); + } catch {} } + playingSourcesRef.current.clear(); + }; + }, [isPlaying, volume, muted, mediaItems]); - // Video elements - if (mediaItem.type === "video") { - return ( - <div - key={element.id} - className="absolute inset-0 flex items-center justify-center" - > - <VideoPlayer - src={mediaItem.url!} - poster={mediaItem.thumbnailUrl} - clipStartTime={element.startTime} - trimStart={element.trimStart} - trimEnd={element.trimEnd} - clipDuration={element.duration} - trackMuted={element.muted || elementData.track.muted} - /> - </div> - ); + // Canvas: draw current frame for visible elements using offscreen compositing + useEffect(() => { + const draw = async () => { + const canvas = canvasRef.current; + if (!canvas) return; + const mainCtx = canvas.getContext("2d"); + if (!mainCtx) return; + + // Set canvas internal resolution to avoid blurry scaling + const displayWidth = Math.max(1, Math.floor(previewDimensions.width)); + const displayHeight = Math.max(1, Math.floor(previewDimensions.height)); + if (canvas.width !== displayWidth || canvas.height !== displayHeight) { + canvas.width = displayWidth; + canvas.height = displayHeight; } - // Image elements - if (mediaItem.type === "image") { - return ( - <div - key={element.id} - className="absolute inset-0 flex items-center justify-center" - > - <img - src={mediaItem.url!} - alt={mediaItem.name} - className="max-w-full max-h-full object-contain" - draggable={false} - /> - </div> - ); + // Throttle rendering to project FPS during playback only + const fps = activeProject?.fps || DEFAULT_FPS; + const minDelta = 1 / fps; + if (isPlaying) { + if (currentTime - lastFrameTimeRef.current < minDelta) { + return; + } + lastFrameTimeRef.current = currentTime; } - // Audio elements (no visual representation) - if (mediaItem.type === "audio") { - return ( - <div - key={element.id} - className="absolute inset-0" - style={{ pointerEvents: "none" }} - > - <AudioPlayer - src={mediaItem.url!} - clipStartTime={element.startTime} - trimStart={element.trimStart} - trimEnd={element.trimEnd} - clipDuration={element.duration} - trackMuted={element.muted || elementData.track.muted} - /> - </div> + // Invalidate older async renders when user scrubs rapidly + const mySeq = (renderSeqRef.current += 1); + + // Offscreen buffer to avoid flicker (reuse canvas) + if (!offscreenCanvasRef.current) { + const hasOffscreen = + typeof (globalThis as unknown as { OffscreenCanvas?: unknown }) + .OffscreenCanvas !== "undefined"; + if (hasOffscreen) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + offscreenCanvasRef.current = new (globalThis as any).OffscreenCanvas( + displayWidth, + displayHeight + ) as OffscreenCanvas; + } else { + const c = document.createElement("canvas"); + c.width = displayWidth; + c.height = displayHeight; + offscreenCanvasRef.current = c; + } + } + // Ensure size matches + if ( + offscreenCanvasRef.current && + (offscreenCanvasRef.current as HTMLCanvasElement).getContext + ) { + const c = offscreenCanvasRef.current as HTMLCanvasElement; + if (c.width !== displayWidth || c.height !== displayHeight) { + c.width = displayWidth; + c.height = displayHeight; + } + } else { + const c = offscreenCanvasRef.current as OffscreenCanvas; + // @ts-ignore width/height exist on OffscreenCanvas in modern browsers + if ( + (c as unknown as { width: number }).width !== displayWidth || + (c as unknown as { height: number }).height !== displayHeight + ) { + // @ts-ignore + (c as unknown as { width: number }).width = displayWidth; + // @ts-ignore + (c as unknown as { height: number }).height = displayHeight; + } + } + const offscreenCanvas = offscreenCanvasRef.current as + | HTMLCanvasElement + | OffscreenCanvas; + const offCtx = (offscreenCanvas as HTMLCanvasElement).getContext + ? (offscreenCanvas as HTMLCanvasElement).getContext("2d") + : (offscreenCanvas as OffscreenCanvas).getContext("2d"); + if (!offCtx) return; + + await renderTimelineFrame({ + ctx: offCtx as CanvasRenderingContext2D, + time: currentTime, + canvasWidth: displayWidth, + canvasHeight: displayHeight, + tracks, + mediaItems, + backgroundColor: + activeProject?.backgroundType === "blur" + ? "transparent" + : activeProject?.backgroundColor || "#000000", + useCache: false, + fps: activeProject?.fps || DEFAULT_FPS, + }); + + // Blit offscreen to visible canvas + mainCtx.clearRect(0, 0, displayWidth, displayHeight); + if ((offscreenCanvas as HTMLCanvasElement).getContext) { + mainCtx.drawImage(offscreenCanvas as HTMLCanvasElement, 0, 0); + } else { + mainCtx.drawImage( + offscreenCanvas as unknown as CanvasImageSource, + 0, + 0 ); } - } + }; + + void draw(); + }, [ + activeElements, + currentTime, + previewDimensions.width, + previewDimensions.height, + canvasSize.width, + canvasSize.height, + activeProject?.backgroundType, + activeProject?.backgroundColor, + ]); - return null; + // Get media elements for blur background (video/image only) + const getBlurBackgroundElements = (): ActiveElement[] => { + return activeElements.filter( + ({ element, mediaItem }) => + element.type === "media" && + mediaItem && + (mediaItem.type === "video" || mediaItem.type === "image") && + element.mediaId !== "test" // Exclude test elements + ); }; + const blurBackgroundElements = getBlurBackgroundElements(); + + // Render blur background layer (handled by canvas now) + const renderBlurBackground = () => null; + + // Render an element (canvas handles visuals now). Audio playback to be implemented via Web Audio. + const renderElement = (_elementData: ActiveElement) => null; + return ( <> <div className="h-full w-full flex flex-col min-h-0 min-w-0 bg-panel rounded-sm relative"> @@ -505,14 +608,23 @@ export function PreviewPanel() { }} > {renderBlurBackground()} + <canvas + ref={canvasRef} + style={{ + position: "absolute", + left: 0, + top: 0, + width: previewDimensions.width, + height: previewDimensions.height, + }} + aria-label="Video preview canvas" + /> {activeElements.length === 0 ? ( <div className="absolute inset-0 flex items-center justify-center text-muted-foreground"> No elements at current time </div> ) : ( - activeElements.map((elementData, index) => - renderElement(elementData, index) - ) + activeElements.map((elementData) => renderElement(elementData)) )} <LayoutGuideOverlay /> </div> |