From 216e3e0c397be3719a94e1a22bc1c2ce2de3856c Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Sat, 28 Feb 2026 18:41:22 +0100 Subject: feat: introduce WebGL effects system and Blur effect This implements the foundational architecture for video effects, starting with a multi-pass WebGL rendering pipeline and a customizable Gaussian Blur effect. Key changes: - WebGL Engine: Added `raw-loader` for `.glsl` shaders, multi-pass framebuffer rendering, and live offscreen canvas previews. - Node Architecture: Replaced hardcoded background blur with `CompositeEffectNode` and added `EffectLayerNode` to apply effects to specific visual elements. - Timeline & DND: Added a new `effect` track type. Upgraded drag-and-drop to support dropping effects directly onto the timeline. Consolidated track constants into a cleaner `TRACK_CONFIG`. - UI/UX: Added an Effects tab in the assets panel with live previews. Added an Effect Properties panel with sliders and inputs for fine-tuning parameters. - Data Model: Added `sourceDuration` to video and audio elements, and wrote a v8 storage migration to update existing projects to the new schema. - Docs: Added `CHANGELOG.md` tracking v0.1.0 and v0.2.0, plus `docs/effects-renderer.md` to document the new WebGL pipeline. --- .../src/components/editor/panels/assets/index.tsx | 7 +- .../editor/panels/assets/views/assets.tsx | 6 + .../editor/panels/assets/views/effects.tsx | 95 ++++++++ .../editor/panels/properties/effect-properties.tsx | 103 ++++++++ .../components/editor/panels/properties/index.tsx | 65 +++-- .../components/editor/panels/timeline/index.tsx | 12 +- .../editor/panels/timeline/timeline-element.tsx | 266 +++++++++++++-------- .../editor/panels/timeline/timeline-track.tsx | 3 + 8 files changed, 426 insertions(+), 131 deletions(-) create mode 100644 apps/web/src/components/editor/panels/assets/views/effects.tsx create mode 100644 apps/web/src/components/editor/panels/properties/effect-properties.tsx (limited to 'apps/web/src/components/editor/panels') diff --git a/apps/web/src/components/editor/panels/assets/index.tsx b/apps/web/src/components/editor/panels/assets/index.tsx index 3600d00b..fe7db329 100644 --- a/apps/web/src/components/editor/panels/assets/index.tsx +++ b/apps/web/src/components/editor/panels/assets/index.tsx @@ -9,6 +9,7 @@ import { SettingsView } from "./views/settings"; import { SoundsView } from "./views/sounds"; import { StickersView } from "./views/stickers"; import { TextView } from "./views/text"; +import { EffectsView } from "./views/effects"; export function AssetsPanel() { const { activeTab } = useAssetsPanelStore(); @@ -18,11 +19,7 @@ export function AssetsPanel() { sounds: , text: , stickers: , - effects: ( -
- Effects view coming soon... -
- ), + effects: , transitions: (
Transitions view coming soon... diff --git a/apps/web/src/components/editor/panels/assets/views/assets.tsx b/apps/web/src/components/editor/panels/assets/views/assets.tsx index 808961f1..4c6922ff 100644 --- a/apps/web/src/components/editor/panels/assets/views/assets.tsx +++ b/apps/web/src/components/editor/panels/assets/views/assets.tsx @@ -426,6 +426,9 @@ function GridView({ type: "media", mediaType: item.type, name: item.name, + ...(item.type !== "audio" && { + targetElementTypes: ["video", "image"] as const, + }), }} shouldShowPlusOnDrag={false} onAddToTimeline={({ currentTime }) => @@ -476,6 +479,9 @@ function ListView({ type: "media", mediaType: item.type, name: item.name, + ...(item.type !== "audio" && { + targetElementTypes: ["video", "image"] as const, + }), }} shouldShowPlusOnDrag={false} onAddToTimeline={({ currentTime }) => diff --git a/apps/web/src/components/editor/panels/assets/views/effects.tsx b/apps/web/src/components/editor/panels/assets/views/effects.tsx new file mode 100644 index 00000000..c9b8b2dd --- /dev/null +++ b/apps/web/src/components/editor/panels/assets/views/effects.tsx @@ -0,0 +1,95 @@ +"use client"; + +import { useEffect, useRef, useCallback } from "react"; +import { PanelView } from "@/components/editor/panels/assets/views/base-view"; +import { DraggableItem } from "@/components/editor/panels/assets/draggable-item"; +import { getAllEffects, EFFECT_TARGET_ELEMENT_TYPES } from "@/lib/effects"; +import { + effectPreviewService, + onPreviewImageReady, +} from "@/services/renderer/effect-preview"; +import { useEditor } from "@/hooks/use-editor"; +import { buildEffectElement } from "@/lib/timeline/element-utils"; +import type { EffectDefinition } from "@/types/effects"; + +export function EffectsView() { + const effects = getAllEffects(); + + return ( + + + + ); +} + +function EffectsGrid({ effects }: { effects: EffectDefinition[] }) { + return ( +
+ {effects.map((effect) => ( + + ))} +
+ ); +} + +function EffectPreviewCanvas({ effectType }: { effectType: string }) { + const canvasRef = useRef(null); + + useEffect(() => { + const render = () => { + if (canvasRef.current) { + effectPreviewService.renderPreview({ + effectType, + params: {}, + targetCanvas: canvasRef.current, + }); + } + }; + + render(); + return onPreviewImageReady({ callback: render }); + }, [effectType]); + + return ; +} + +function EffectItem({ effect }: { effect: EffectDefinition }) { + const editor = useEditor(); + + const handleAddToTimeline = useCallback(() => { + const currentTime = editor.playback.getCurrentTime(); + const element = buildEffectElement({ + effectType: effect.type, + startTime: currentTime, + }); + + editor.timeline.insertElement({ + placement: { mode: "auto", trackType: "effect" }, + element, + }); + }, [editor, effect.type]); + + const preview = ; + + return ( + + ); +} diff --git a/apps/web/src/components/editor/panels/properties/effect-properties.tsx b/apps/web/src/components/editor/panels/properties/effect-properties.tsx new file mode 100644 index 00000000..bf137291 --- /dev/null +++ b/apps/web/src/components/editor/panels/properties/effect-properties.tsx @@ -0,0 +1,103 @@ +"use client"; + +import type { EffectElement } from "@/types/timeline"; +import type { EffectParamDefinition } from "@/types/effects"; +import { getEffect } from "@/lib/effects/registry"; +import { useEditor } from "@/hooks/use-editor"; +import { clamp } from "@/utils/math"; +import { Section, SectionContent, SectionHeader, SectionField, SectionFields } from "./section"; +import { Slider } from "@/components/ui/slider"; +import { NumberField } from "@/components/ui/number-field"; +import { usePropertyDraft } from "./hooks/use-property-draft"; + +function EffectParamField({ + param, + element, + trackId, +}: { + param: EffectParamDefinition; + element: EffectElement; + trackId: string; +}) { + const editor = useEditor(); + + const currentValue = Number(element.params[param.key] ?? param.default); + const min = param.min ?? 0; + const max = param.max ?? 100; + const step = param.step ?? 1; + + const updateParam = (value: number) => + editor.timeline.previewElements({ + updates: [ + { + trackId, + elementId: element.id, + updates: { params: { ...element.params, [param.key]: value } }, + }, + ], + }); + + const commitParam = () => editor.timeline.commitPreview(); + + const draft = usePropertyDraft({ + displayValue: String(currentValue), + parse: (input) => { + const parsed = parseFloat(input); + if (Number.isNaN(parsed)) return null; + return clamp({ value: parsed, min, max }); + }, + onPreview: updateParam, + onCommit: commitParam, + }); + + return ( + +
+ updateParam(value)} + onValueCommit={commitParam} + /> + +
+
+ ); +} + +export function EffectProperties({ + element, + trackId, +}: { + element: EffectElement; + trackId: string; +}) { + const definition = getEffect({ effectType: element.effectType }); + + return ( +
+ + + + {definition.params.map((param) => ( + + ))} + + +
+ ); +} diff --git a/apps/web/src/components/editor/panels/properties/index.tsx b/apps/web/src/components/editor/panels/properties/index.tsx index 4f285f84..75dd81a9 100644 --- a/apps/web/src/components/editor/panels/properties/index.tsx +++ b/apps/web/src/components/editor/panels/properties/index.tsx @@ -4,9 +4,37 @@ import { ScrollArea } from "@/components/ui/scroll-area"; import { AudioProperties } from "./audio-properties"; import { VideoProperties } from "./video-properties"; import { TextProperties } from "./text-properties"; +import { EffectProperties } from "./effect-properties"; import { EmptyView } from "./empty-view"; import { useEditor } from "@/hooks/use-editor"; import { useElementSelection } from "@/hooks/timeline/element/use-element-selection"; +import type { TimelineElement, TimelineTrack } from "@/types/timeline"; + +function ElementProperties({ + track, + element, +}: { + track: TimelineTrack; + element: TimelineElement; +}) { + if (element.type === "text") { + return ; + } + if (element.type === "audio") { + return ; + } + if ( + element.type === "video" || + element.type === "image" || + element.type === "sticker" + ) { + return ; + } + if (element.type === "effect") { + return ; + } + return null; +} export function PropertiesPanel() { const editor = useEditor(); @@ -16,34 +44,19 @@ export function PropertiesPanel() { elements: selectedElements, }); + const hasSelection = selectedElements.length > 0; + return ( -
- {selectedElements.length > 0 ? ( +
+ {hasSelection ? ( - {elementsWithTracks.map(({ track, element }) => { - if (element.type === "text") { - return ( -
- -
- ); - } - if (element.type === "audio") { - return ; - } - if ( - element.type === "video" || - element.type === "image" || - element.type === "sticker" - ) { - return ( -
- -
- ); - } - return null; - })} + {elementsWithTracks.map(({ track, element }) => ( + + ))}
) : ( diff --git a/apps/web/src/components/editor/panels/timeline/index.tsx b/apps/web/src/components/editor/panels/timeline/index.tsx index 18f768a9..d89d270d 100644 --- a/apps/web/src/components/editor/panels/timeline/index.tsx +++ b/apps/web/src/components/editor/panels/timeline/index.tsx @@ -27,7 +27,7 @@ import type { SnapPoint } from "@/lib/timeline/snap-utils"; import type { TimelineTrack } from "@/types/timeline"; import { TIMELINE_CONSTANTS, - TRACK_ICONS, + TRACK_CONFIG, } from "@/constants/timeline-constants"; import { useElementInteraction } from "@/hooks/timeline/element/use-element-interaction"; import { @@ -161,6 +161,7 @@ export function Timeline() { shouldIgnoreClick, } = useSelectionBox({ containerRef: tracksContainerRef, + headerRef: timelineHeaderRef, onSelectionComplete: (elements) => { setElementSelection({ elements }); }, @@ -326,7 +327,7 @@ export function Timeline() {
@@ -523,7 +529,7 @@ export function Timeline() { } function TrackIcon({ track }: { track: TimelineTrack }) { - return <>{TRACK_ICONS[track.type]}; + return <>{TRACK_CONFIG[track.type].icon}; } function TrackToggleIcon({ diff --git a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx index 7ad63692..38cf953b 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-element.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-element.tsx @@ -46,6 +46,7 @@ import { Search01Icon, Exchange01Icon, KeyframeIcon, + MagicWand05Icon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { uppercase } from "@/utils/string"; @@ -59,7 +60,6 @@ interface KeyframeIndicator { time: number; offsetPx: number; keyframes: SelectedKeyframeRef[]; - isSelected: boolean; } function buildKeyframeIndicator({ @@ -69,7 +69,6 @@ function buildKeyframeIndicator({ displayedStartTime, zoomLevel, elementLeft, - isKeyframeSelected, }: { keyframe: ElementKeyframe; trackId: string; @@ -77,16 +76,10 @@ function buildKeyframeIndicator({ displayedStartTime: number; zoomLevel: number; elementLeft: number; - isKeyframeSelected: ({ - keyframe, - }: { - keyframe: SelectedKeyframeRef; - }) => boolean; }): { time: number; offsetPx: number; keyframeRef: SelectedKeyframeRef; - isSelected: boolean; } { const keyframeRef = { trackId, @@ -102,7 +95,6 @@ function buildKeyframeIndicator({ time: keyframe.time, offsetPx: keyframeLeft - elementLeft, keyframeRef, - isSelected: isKeyframeSelected({ keyframe: keyframeRef }), }; } @@ -114,7 +106,6 @@ function getKeyframeIndicators({ zoomLevel, elementLeft, elementWidth, - isKeyframeSelected, }: { keyframes: ElementKeyframe[]; trackId: string; @@ -123,11 +114,6 @@ function getKeyframeIndicators({ zoomLevel: number; elementLeft: number; elementWidth: number; - isKeyframeSelected: ({ - keyframe, - }: { - keyframe: SelectedKeyframeRef; - }) => boolean; }): KeyframeIndicator[] { if (elementWidth < KEYFRAME_INDICATOR_MIN_WIDTH_PX) { return []; @@ -142,7 +128,6 @@ function getKeyframeIndicators({ displayedStartTime, zoomLevel, elementLeft, - isKeyframeSelected, }); const existingIndicator = keyframesByTime.get(indicator.time); if (!existingIndicator) { @@ -150,14 +135,11 @@ function getKeyframeIndicators({ time: indicator.time, offsetPx: indicator.offsetPx, keyframes: [indicator.keyframeRef], - isSelected: indicator.isSelected, }); continue; } existingIndicator.keyframes.push(indicator.keyframeRef); - existingIndicator.isSelected = - existingIndicator.isSelected || indicator.isSelected; } return [...keyframesByTime.values()].sort((a, b) => a.time - b.time); @@ -187,6 +169,7 @@ interface TimelineElementProps { ) => void; onElementClick: (e: React.MouseEvent, element: TimelineElementType) => void; dragState: ElementDragState; + isDropTarget?: boolean; } export function TimelineElement({ @@ -199,10 +182,10 @@ export function TimelineElement({ onElementMouseDown, onElementClick, dragState, + isDropTarget = false, }: TimelineElementProps) { const editor = useEditor(); const { selectedElements } = useElementSelection(); - const { isKeyframeSelected } = useKeyframeSelection(); const { requestRevealMedia } = useAssetsPanelStore(); const mediaAssets = editor.media.getAssets(); @@ -248,16 +231,17 @@ export function TimelineElement({ time: displayedStartTime, zoomLevel, }); - const keyframeIndicators = getKeyframeIndicators({ - keyframes: getElementKeyframes({ animations: element.animations }), - trackId: track.id, - elementId: element.id, - displayedStartTime, - zoomLevel, - elementLeft, - elementWidth, - isKeyframeSelected, - }); + const keyframeIndicators = isSelected + ? getKeyframeIndicators({ + keyframes: getElementKeyframes({ animations: element.animations }), + trackId: track.id, + elementId: element.id, + displayedStartTime, + zoomLevel, + elementLeft, + elementWidth, + }) + : []; const handleRevealInMedia = ({ event }: { event: React.MouseEvent }) => { event.stopPropagation(); if (hasMediaId(element)) { @@ -291,8 +275,9 @@ export function TimelineElement({ onElementClick={onElementClick} onElementMouseDown={onElementMouseDown} handleResizeStart={handleResizeStart} + isDropTarget={isDropTarget} /> - + {isSelected && }
@@ -365,6 +350,7 @@ function ElementInner({ onElementClick, onElementMouseDown, handleResizeStart, + isDropTarget = false, }: { element: TimelineElementType; track: TimelineTrack; @@ -382,14 +368,20 @@ function ElementInner({ elementId: string; side: "left" | "right"; }) => void; + isDropTarget?: boolean; }) { + const opacityClass = + (canElementBeHidden(element) && element.hidden) || isDropTarget + ? "opacity-50" + : ""; + return (
- )); + return indicators.map((indicator) => { + const isIndicatorSelected = indicator.keyframes.some((keyframe) => + isKeyframeSelected({ keyframe }), + ); + + return ( + + ); + }); } -function ElementContent({ - element, - track, - isSelected, - mediaAssets, -}: { +interface ElementContentProps { element: TimelineElementType; track: TimelineTrack; isSelected: boolean; mediaAssets: MediaAsset[]; -}) { - if (element.type === "text") { +} + +type ElementContentRenderer = (props: ElementContentProps) => ReactNode; + +const ELEMENT_CONTENT_RENDERERS: Record< + TimelineElementType["type"], + ElementContentRenderer +> = { + text: ({ element }) => { + const textElement = element as Extract< + TimelineElementType, + { type: "text" } + >; return ( -
- {element.content} +
+ + {textElement.content} +
); - } - - if (element.type === "sticker") { + }, + effect: ({ element }) => ( +
+ + {element.name} +
+ ), + sticker: ({ element }) => { + const stickerElement = element as Extract< + TimelineElementType, + { type: "sticker" } + >; return (
{element.name} - {element.name} + + {stickerElement.name} +
); - } - - if (element.type === "audio") { + }, + audio: ({ element, mediaAssets }) => { + const audioElement = element as Extract< + TimelineElementType, + { type: "audio" } + >; const audioBuffer = - element.sourceType === "library" ? element.buffer : undefined; - + audioElement.sourceType === "library" ? audioElement.buffer : undefined; const audioUrl = - element.sourceType === "library" - ? element.sourceUrl - : mediaAssets.find((asset) => asset.id === element.mediaId)?.url; + audioElement.sourceType === "library" + ? audioElement.sourceUrl + : mediaAssets.find((asset) => asset.id === audioElement.mediaId)?.url; if (audioBuffer || audioUrl) { return ( @@ -593,28 +614,78 @@ function ElementContent({ return ( - {element.name} + {audioElement.name} ); - } + }, + video: ({ element, track, isSelected, mediaAssets }) => { + const videoElement = element as Extract< + TimelineElementType, + { type: "video" } + >; + const mediaAsset = mediaAssets.find( + (asset) => asset.id === videoElement.mediaId, + ); + + if (!mediaAsset) { + return ( + + {videoElement.name} + + ); + } + + if (mediaAsset.thumbnailUrl) { + const trackHeight = getTrackHeight({ type: track.type }); + const tileWidth = trackHeight * (16 / 9); + + return ( +
+
+
+
+
+ ); + } - const mediaAsset = mediaAssets.find((asset) => asset.id === element.mediaId); - if (!mediaAsset) { return ( - {element.name} + {videoElement.name} ); - } + }, + image: ({ element, track, isSelected, mediaAssets }) => { + const imageElement = element as Extract< + TimelineElementType, + { type: "image" } + >; + const mediaAsset = mediaAssets.find( + (asset) => asset.id === imageElement.mediaId, + ); + + if (!mediaAsset?.url) { + return ( + + {imageElement.name} + + ); + } - if ( - mediaAsset.type === "image" || - (mediaAsset.type === "video" && mediaAsset.thumbnailUrl) - ) { const trackHeight = getTrackHeight({ type: track.type }); const tileWidth = trackHeight * (16 / 9); - const imageUrl = - mediaAsset.type === "image" ? mediaAsset.url : mediaAsset.thumbnailUrl; return (
@@ -624,7 +695,7 @@ function ElementContent({
); - } + }, +}; - return ( - {element.name} - ); +function ElementContent(props: ElementContentProps) { + const renderer = ELEMENT_CONTENT_RENDERERS[props.element.type]; + return <>{renderer(props)}; } function CopyMenuItem() { diff --git a/apps/web/src/components/editor/panels/timeline/timeline-track.tsx b/apps/web/src/components/editor/panels/timeline/timeline-track.tsx index 42faadaf..fbc7cb3f 100644 --- a/apps/web/src/components/editor/panels/timeline/timeline-track.tsx +++ b/apps/web/src/components/editor/panels/timeline/timeline-track.tsx @@ -32,6 +32,7 @@ interface TimelineTrackContentProps { onTrackMouseDown?: (event: React.MouseEvent) => void; onTrackClick?: (event: React.MouseEvent) => void; shouldIgnoreClick?: () => boolean; + targetElementId?: string | null; } export function TimelineTrackContent({ @@ -48,6 +49,7 @@ export function TimelineTrackContent({ onTrackMouseDown, onTrackClick, shouldIgnoreClick, + targetElementId = null, }: TimelineTrackContentProps) { const editor = useEditor(); const { isElementSelected, clearElementSelection } = useElementSelection(); @@ -102,6 +104,7 @@ export function TimelineTrackContent({ onElementClick({ event, element, track }) } dragState={dragState} + isDropTarget={element.id === targetElementId} /> ); }) -- cgit v1.3.1