From 216e3e0c397be3719a94e1a22bc1c2ce2de3856c Mon Sep 17 00:00:00 2001 From: Maze Winther Date: Sat, 28 Feb 2026 18:41:22 +0100 Subject: feat: introduce WebGL effects system and Blur effect This implements the foundational architecture for video effects, starting with a multi-pass WebGL rendering pipeline and a customizable Gaussian Blur effect. Key changes: - WebGL Engine: Added `raw-loader` for `.glsl` shaders, multi-pass framebuffer rendering, and live offscreen canvas previews. - Node Architecture: Replaced hardcoded background blur with `CompositeEffectNode` and added `EffectLayerNode` to apply effects to specific visual elements. - Timeline & DND: Added a new `effect` track type. Upgraded drag-and-drop to support dropping effects directly onto the timeline. Consolidated track constants into a cleaner `TRACK_CONFIG`. - UI/UX: Added an Effects tab in the assets panel with live previews. Added an Effect Properties panel with sliders and inputs for fine-tuning parameters. - Data Model: Added `sourceDuration` to video and audio elements, and wrote a v8 storage migration to update existing projects to the new schema. - Docs: Added `CHANGELOG.md` tracking v0.1.0 and v0.2.0, plus `docs/effects-renderer.md` to document the new WebGL pipeline. --- apps/web/src/lib/timeline/drop-utils.ts | 85 +++++++++++++++++++++++++++--- apps/web/src/lib/timeline/element-utils.ts | 42 ++++++++++++--- apps/web/src/lib/timeline/track-utils.ts | 38 ++++++------- 3 files changed, 134 insertions(+), 31 deletions(-) (limited to 'apps/web/src/lib/timeline') diff --git a/apps/web/src/lib/timeline/drop-utils.ts b/apps/web/src/lib/timeline/drop-utils.ts index 9182892f..b502974d 100644 --- a/apps/web/src/lib/timeline/drop-utils.ts +++ b/apps/web/src/lib/timeline/drop-utils.ts @@ -1,9 +1,42 @@ -import type { TimelineTrack, ElementType } from "@/types/timeline"; -import { TRACK_HEIGHTS, TRACK_GAP } from "@/constants/timeline-constants"; +import type { + TimelineTrack, + ElementType, + TimelineElement, +} from "@/types/timeline"; +import { TRACK_CONFIG, TRACK_GAP } from "@/constants/timeline-constants"; import { wouldElementOverlap } from "./element-utils"; import type { ComputeDropTargetParams, DropTarget } from "@/types/timeline"; import { isMainTrack, enforceMainTrackStart } from "./track-utils"; +function findElementAtPosition({ + mouseX, + tracks, + trackIndex, + targetElementTypes, + pixelsPerSecond, + zoomLevel, +}: { + mouseX: number; + tracks: TimelineTrack[]; + trackIndex: number; + targetElementTypes: string[]; + pixelsPerSecond: number; + zoomLevel: number; +}): { elementId: string; trackId: string } | null { + const time = mouseX / (pixelsPerSecond * zoomLevel); + const track = tracks[trackIndex]; + if (!track || !("elements" in track)) return null; + + const hit = track.elements.find( + (element: TimelineElement) => + targetElementTypes.includes(element.type) && + element.startTime <= time && + time < element.startTime + element.duration, + ); + if (!hit) return null; + return { elementId: hit.id, trackId: track.id }; +} + function getTrackAtY({ mouseY, tracks, @@ -16,7 +49,7 @@ function getTrackAtY({ let cumulativeHeight = 0; for (let i = 0; i < tracks.length; i++) { - const trackHeight = TRACK_HEIGHTS[tracks[i].type]; + const trackHeight = TRACK_CONFIG[tracks[i].type].height; const trackTop = cumulativeHeight; const trackBottom = trackTop + trackHeight; @@ -55,6 +88,7 @@ function isCompatible({ if (elementType === "text") return trackType === "text"; if (elementType === "audio") return trackType === "audio"; if (elementType === "sticker") return trackType === "sticker"; + if (elementType === "effect") return trackType === "effect"; if (elementType === "video" || elementType === "image") { return trackType === "video"; } @@ -100,6 +134,8 @@ function findInsertIndex({ }; } +const EMPTY_TARGET_ELEMENT = null; + export function computeDropTarget({ elementType, mouseX, @@ -113,6 +149,7 @@ export function computeDropTarget({ verticalDragDirection, startTimeOverride, excludeElementId, + targetElementTypes, }: ComputeDropTargetParams): DropTarget { const xPosition = typeof startTimeOverride === "number" @@ -130,9 +167,16 @@ export function computeDropTarget({ isNewTrack: true, insertPosition: "below", xPosition, + targetElement: EMPTY_TARGET_ELEMENT, }; } - return { trackIndex: 0, isNewTrack: true, insertPosition: null, xPosition }; + return { + trackIndex: 0, + isNewTrack: true, + insertPosition: null, + xPosition, + targetElement: EMPTY_TARGET_ELEMENT, + }; } const trackAtMouse = getTrackAtY({ mouseY, tracks, verticalDragDirection }); @@ -146,6 +190,7 @@ export function computeDropTarget({ isNewTrack: true, insertPosition: "below", xPosition, + targetElement: EMPTY_TARGET_ELEMENT, }; } @@ -155,6 +200,7 @@ export function computeDropTarget({ isNewTrack: true, insertPosition: "above", xPosition, + targetElement: EMPTY_TARGET_ELEMENT, }; } @@ -163,12 +209,37 @@ export function computeDropTarget({ isNewTrack: true, insertPosition: "above", xPosition, + targetElement: EMPTY_TARGET_ELEMENT, }; } const { trackIndex, relativeY } = trackAtMouse; const track = tracks[trackIndex]; - const trackHeight = TRACK_HEIGHTS[track.type]; + + if ( + targetElementTypes && + targetElementTypes.length > 0 + ) { + const targetElement = findElementAtPosition({ + mouseX, + tracks, + trackIndex, + targetElementTypes, + pixelsPerSecond, + zoomLevel, + }); + if (targetElement) { + return { + trackIndex, + isNewTrack: false, + insertPosition: null, + xPosition, + targetElement, + }; + } + } + + const trackHeight = TRACK_CONFIG[track.type].height; const isInUpperHalf = relativeY < trackHeight / 2; const isTrackCompatible = isCompatible({ @@ -200,6 +271,7 @@ export function computeDropTarget({ isNewTrack: false, insertPosition: null, xPosition: adjustedXPosition, + targetElement: EMPTY_TARGET_ELEMENT, }; } @@ -220,6 +292,7 @@ export function computeDropTarget({ isNewTrack: true, insertPosition: position, xPosition, + targetElement: EMPTY_TARGET_ELEMENT, }; } @@ -237,7 +310,7 @@ export function getDropLineY({ let y = 0; for (let i = 0; i < safeTrackIndex; i++) { - y += TRACK_HEIGHTS[tracks[i].type] + TRACK_GAP; + y += TRACK_CONFIG[tracks[i].type].height + TRACK_GAP; } return y; diff --git a/apps/web/src/lib/timeline/element-utils.ts b/apps/web/src/lib/timeline/element-utils.ts index 9e191833..8c02c097 100644 --- a/apps/web/src/lib/timeline/element-utils.ts +++ b/apps/web/src/lib/timeline/element-utils.ts @@ -6,6 +6,7 @@ import { TIMELINE_CONSTANTS, } from "@/constants/timeline-constants"; import type { + CreateEffectElement, CreateTimelineElement, CreateVideoElement, CreateImageElement, @@ -22,6 +23,8 @@ import type { UploadAudioElement, } from "@/types/timeline"; import type { MediaType } from "@/types/assets"; +import { buildDefaultEffectInstance } from "@/lib/effects"; +import { capitalizeFirstLetter } from "@/utils/string"; export function canElementHaveAudio( element: TimelineElement, @@ -155,13 +158,13 @@ export function buildTextElement({ fontFamily: t.fontFamily ?? DEFAULT_TEXT_ELEMENT.fontFamily, color: t.color ?? DEFAULT_TEXT_ELEMENT.color, background: { - color: t.background?.color ?? DEFAULT_TEXT_ELEMENT.background.color, - cornerRadius: t.background?.cornerRadius, - paddingX: t.background?.paddingX, - paddingY: t.background?.paddingY, - offsetX: t.background?.offsetX, - offsetY: t.background?.offsetY, - }, + color: t.background?.color ?? DEFAULT_TEXT_ELEMENT.background.color, + cornerRadius: t.background?.cornerRadius, + paddingX: t.background?.paddingX, + paddingY: t.background?.paddingY, + offsetX: t.background?.offsetX, + offsetY: t.background?.offsetY, + }, textAlign: t.textAlign ?? DEFAULT_TEXT_ELEMENT.textAlign, fontWeight: t.fontWeight ?? DEFAULT_TEXT_ELEMENT.fontWeight, fontStyle: t.fontStyle ?? DEFAULT_TEXT_ELEMENT.fontStyle, @@ -174,6 +177,28 @@ export function buildTextElement({ }; } +export function buildEffectElement({ + effectType, + startTime, + duration, +}: { + effectType: string; + startTime: number; + duration?: number; +}): CreateEffectElement { + const instance = buildDefaultEffectInstance({ effectType }); + return { + type: "effect", + name: capitalizeFirstLetter({ string: instance.type }), + effectType, + params: instance.params, + duration: duration ?? TIMELINE_CONSTANTS.DEFAULT_ELEMENT_DURATION, + startTime, + trimStart: 0, + trimEnd: 0, + }; +} + export function buildStickerElement({ stickerId, name, @@ -218,6 +243,7 @@ export function buildVideoElement({ startTime, trimStart: 0, trimEnd: 0, + sourceDuration: duration, muted: false, hidden: false, transform: { ...DEFAULT_TRANSFORM }, @@ -274,6 +300,7 @@ export function buildUploadAudioElement({ startTime, trimStart: 0, trimEnd: 0, + sourceDuration: duration, volume: 1, muted: false, }; @@ -336,6 +363,7 @@ export function buildLibraryAudioElement({ startTime, trimStart: 0, trimEnd: 0, + sourceDuration: duration, volume: 1, muted: false, }; diff --git a/apps/web/src/lib/timeline/track-utils.ts b/apps/web/src/lib/timeline/track-utils.ts index 415d0ace..94ff21ba 100644 --- a/apps/web/src/lib/timeline/track-utils.ts +++ b/apps/web/src/lib/timeline/track-utils.ts @@ -6,11 +6,11 @@ import type { AudioTrack, StickerTrack, TextTrack, + EffectTrack, TimelineElement, } from "@/types/timeline"; import { - TRACK_COLORS, - TRACK_HEIGHTS, + TRACK_CONFIG, TRACK_GAP, } from "@/constants/timeline-constants"; import { generateUUID } from "@/utils/id"; @@ -23,21 +23,20 @@ export function canTracktHaveAudio( export function canTrackBeHidden( track: TimelineTrack, -): track is VideoTrack | TextTrack | StickerTrack { +): track is VideoTrack | TextTrack | StickerTrack | EffectTrack { return track.type !== "audio"; } export function getTrackColor({ type }: { type: TrackType }) { - return TRACK_COLORS[type]; + return TRACK_CONFIG[type]; } export function getTrackClasses({ type }: { type: TrackType }) { - const colors = TRACK_COLORS[type]; - return `${colors.background}`.trim(); + return TRACK_CONFIG[type].background.trim(); } export function getTrackHeight({ type }: { type: TrackType }): number { - return TRACK_HEIGHTS[type]; + return TRACK_CONFIG[type].height; } export function getCumulativeHeightBefore({ @@ -77,17 +76,7 @@ export function buildEmptyTrack({ type: TrackType; name?: string; }): TimelineTrack { - const trackName = - name ?? - (type === "video" - ? "Video track" - : type === "text" - ? "Text track" - : type === "audio" - ? "Audio track" - : type === "sticker" - ? "Sticker track" - : "Track"); + const trackName = name ?? TRACK_CONFIG[type].defaultName; switch (type) { case "video": @@ -124,6 +113,14 @@ export function buildEmptyTrack({ elements: [], muted: false, }; + case "effect": + return { + id, + name: trackName, + type: "effect", + elements: [], + hidden: false, + }; default: throw new Error(`Unsupported track type: ${type}`); } @@ -140,6 +137,10 @@ export function getDefaultInsertIndexForTrack({ return tracks.length; } + if (trackType === "effect") { + return 0; + } + const mainTrackIndex = tracks.findIndex((track) => isMainTrack(track)); if (mainTrackIndex >= 0) { return mainTrackIndex; @@ -216,6 +217,7 @@ export function canElementGoOnTrack({ if (elementType === "text") return trackType === "text"; if (elementType === "audio") return trackType === "audio"; if (elementType === "sticker") return trackType === "sticker"; + if (elementType === "effect") return trackType === "effect"; if (elementType === "video" || elementType === "image") { return trackType === "video"; } -- cgit v1.3.1