diff options
| author | Maze <167211895+mazeincoding@users.noreply.github.com> | 2026-01-31 00:20:04 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2026-01-31 00:20:04 +0100 |
| commit | 7bf0984698e43b9aa2dd879f65951318ed410390 (patch) | |
| tree | ccf2cd590a47d550152afc07129cb55c71a9ec35 /apps/web/src/services | |
| parent | 0173db99448d1c4c6415dcf15272e17551b48b04 (diff) | |
| download | opencut-7bf0984698e43b9aa2dd879f65951318ed410390.tar.gz | |
codebase overhaul (#697)
Diffstat (limited to 'apps/web/src/services')
25 files changed, 3000 insertions, 0 deletions
diff --git a/apps/web/src/services/media/video-cache.ts b/apps/web/src/services/media/video-cache.ts new file mode 100644 index 00000000..f3ddfcb2 --- /dev/null +++ b/apps/web/src/services/media/video-cache.ts @@ -0,0 +1,314 @@ +import { + Input, + ALL_FORMATS, + BlobSource, + CanvasSink, + type WrappedCanvas, +} from "mediabunny"; + +interface VideoSinkData { + sink: CanvasSink; + iterator: AsyncGenerator<WrappedCanvas, void, unknown> | null; + currentFrame: WrappedCanvas | null; + nextFrame: WrappedCanvas | null; + lastTime: number; + prefetching: boolean; + prefetchPromise: Promise<void> | null; +} +export class VideoCache { + private sinks = new Map<string, VideoSinkData>(); + private initPromises = new Map<string, Promise<void>>(); + + async getFrameAt({ + mediaId, + file, + time, + }: { + mediaId: string; + file: File; + time: number; + }): Promise<WrappedCanvas | null> { + await this.ensureSink({ mediaId, file }); + + const sinkData = this.sinks.get(mediaId); + if (!sinkData) return null; + + if (sinkData.nextFrame && sinkData.nextFrame.timestamp <= time) { + sinkData.currentFrame = sinkData.nextFrame; + sinkData.nextFrame = null; + this.startPrefetch({ sinkData }); + } + + if ( + sinkData.currentFrame && + this.isFrameValid({ frame: sinkData.currentFrame, time }) + ) { + if (!sinkData.nextFrame && !sinkData.prefetching) { + this.startPrefetch({ sinkData }); + } + return sinkData.currentFrame; + } + + if ( + sinkData.iterator && + sinkData.currentFrame && + time >= sinkData.lastTime && + time < sinkData.lastTime + 2.0 + ) { + const frame = await this.iterateToTime({ sinkData, targetTime: time }); + if (frame) { + if (!sinkData.nextFrame && !sinkData.prefetching) { + this.startPrefetch({ sinkData }); + } + return frame; + } + } + + const frame = await this.seekToTime({ sinkData, time }); + if (frame && !sinkData.nextFrame && !sinkData.prefetching) { + this.startPrefetch({ sinkData }); + } + return frame; + } + + private isFrameValid({ + frame, + time, + }: { + frame: WrappedCanvas; + time: number; + }): boolean { + return time >= frame.timestamp && time < frame.timestamp + frame.duration; + } + private async iterateToTime({ + sinkData, + targetTime, + }: { + sinkData: VideoSinkData; + targetTime: number; + }): Promise<WrappedCanvas | null> { + if (!sinkData.iterator) return null; + + try { + while (true) { + // Wait for any pending prefetch to finish before touching iterator + if (sinkData.prefetching && sinkData.prefetchPromise) { + await sinkData.prefetchPromise; + } + + // Check if the nextFrame (which might have just arrived) is what we need + if ( + sinkData.nextFrame && + sinkData.nextFrame.timestamp <= targetTime + 0.05 // Tolerance + ) { + sinkData.currentFrame = sinkData.nextFrame; + sinkData.nextFrame = null; + } else { + const { value: frame, done } = await sinkData.iterator.next(); + + if (done || !frame) break; + + sinkData.currentFrame = frame; + } + + const frame = sinkData.currentFrame; + if (!frame) break; + + sinkData.lastTime = frame.timestamp; + + if (this.isFrameValid({ frame, time: targetTime })) { + return frame; + } + + if (frame.timestamp > targetTime + 1.0) break; + } + } catch (error) { + console.warn("Iterator failed, will restart:", error); + sinkData.iterator = null; + } + + return null; + } + private async seekToTime({ + sinkData, + time, + }: { + sinkData: VideoSinkData; + time: number; + }): Promise<WrappedCanvas | null> { + try { + if (sinkData.prefetching && sinkData.prefetchPromise) { + await sinkData.prefetchPromise; + } + + if (sinkData.iterator) { + await sinkData.iterator.return(); + sinkData.iterator = null; + } + + sinkData.nextFrame = null; + sinkData.iterator = sinkData.sink.canvases(time); + sinkData.lastTime = time; + + // Fetch current frame + const { value: frame } = await sinkData.iterator.next(); + + if (frame) { + sinkData.currentFrame = frame; + + // Aggressively fetch next frame immediately to fill buffer + // This matches the mediaplayer example which fetches 2 frames on start + try { + const { value: next } = await sinkData.iterator.next(); + if (next) { + sinkData.nextFrame = next; + } + } catch (e) { + console.warn("Failed to pre-fetch next frame on seek:", e); + } + + return frame; + } + } catch (error) { + console.warn("Failed to seek video:", error); + } + + return null; + } + + private startPrefetch({ sinkData }: { sinkData: VideoSinkData }): void { + if (sinkData.prefetching || !sinkData.iterator || sinkData.nextFrame) { + return; + } + + sinkData.prefetching = true; + sinkData.prefetchPromise = this.prefetchNextFrame({ sinkData }); + } + + private async prefetchNextFrame({ + sinkData, + }: { + sinkData: VideoSinkData; + }): Promise<void> { + if (!sinkData.iterator) { + sinkData.prefetching = false; + sinkData.prefetchPromise = null; + return; + } + + try { + const { value: frame, done } = await sinkData.iterator.next(); + + if (done || !frame) { + sinkData.prefetching = false; + sinkData.prefetchPromise = null; + return; + } + + sinkData.nextFrame = frame; + sinkData.prefetching = false; + sinkData.prefetchPromise = null; + } catch (error) { + console.warn("Prefetch failed:", error); + sinkData.prefetching = false; + sinkData.prefetchPromise = null; + sinkData.iterator = null; + } + } + private async ensureSink({ + mediaId, + file, + }: { + mediaId: string; + file: File; + }): Promise<void> { + if (this.sinks.has(mediaId)) return; + + if (this.initPromises.has(mediaId)) { + await this.initPromises.get(mediaId); + return; + } + + const initPromise = this.initializeSink({ mediaId, file }); + this.initPromises.set(mediaId, initPromise); + + try { + await initPromise; + } finally { + this.initPromises.delete(mediaId); + } + } + private async initializeSink({ + mediaId, + file, + }: { + mediaId: string; + file: File; + }): Promise<void> { + try { + const input = new Input({ + source: new BlobSource(file), + formats: ALL_FORMATS, + }); + + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) { + throw new Error("No video track found"); + } + + const canDecode = await videoTrack.canDecode(); + if (!canDecode) { + throw new Error("Video codec not supported for decoding"); + } + + const sink = new CanvasSink(videoTrack, { + poolSize: 3, + fit: "contain", + }); + + this.sinks.set(mediaId, { + sink, + iterator: null, + currentFrame: null, + nextFrame: null, + lastTime: -1, + prefetching: false, + prefetchPromise: null, + }); + } catch (error) { + console.error(`Failed to initialize video sink for ${mediaId}:`, error); + throw error; + } + } + + clearVideo({ mediaId }: { mediaId: string }): void { + const sinkData = this.sinks.get(mediaId); + if (sinkData) { + if (sinkData.iterator) { + void sinkData.iterator.return(); + } + + this.sinks.delete(mediaId); + } + + this.initPromises.delete(mediaId); + } + + clearAll(): void { + for (const [mediaId] of this.sinks) { + this.clearVideo({ mediaId }); + } + } + + getStats() { + return { + totalSinks: this.sinks.size, + activeSinks: Array.from(this.sinks.values()).filter((s) => s.iterator) + .length, + cachedFrames: Array.from(this.sinks.values()).filter( + (s) => s.currentFrame, + ).length, + }; + } +} +export const videoCache = new VideoCache(); diff --git a/apps/web/src/services/renderer/canvas-renderer.ts b/apps/web/src/services/renderer/canvas-renderer.ts new file mode 100644 index 00000000..840d8b52 --- /dev/null +++ b/apps/web/src/services/renderer/canvas-renderer.ts @@ -0,0 +1,87 @@ +import type { BaseNode } from "./nodes/base-node"; + +export type CanvasRendererParams = { + width: number; + height: number; + fps: number; +}; + +export class CanvasRenderer { + canvas: OffscreenCanvas | HTMLCanvasElement; + context: OffscreenCanvasRenderingContext2D | CanvasRenderingContext2D; + width: number; + height: number; + fps: number; + + constructor({ width, height, fps }: CanvasRendererParams) { + this.width = width; + this.height = height; + this.fps = fps; + + try { + this.canvas = new OffscreenCanvas(width, height); + } catch { + this.canvas = document.createElement("canvas"); + this.canvas.width = width; + this.canvas.height = height; + } + + const context = this.canvas.getContext("2d"); + if (!context) { + throw new Error("Failed to get canvas context"); + } + + this.context = context as + | OffscreenCanvasRenderingContext2D + | CanvasRenderingContext2D; + } + + setSize({ width, height }: { width: number; height: number }) { + this.width = width; + this.height = height; + + if (this.canvas instanceof OffscreenCanvas) { + this.canvas = new OffscreenCanvas(width, height); + } else { + this.canvas.width = width; + this.canvas.height = height; + } + + const context = this.canvas.getContext("2d"); + if (!context) { + throw new Error("Failed to get canvas context"); + } + this.context = context as + | OffscreenCanvasRenderingContext2D + | CanvasRenderingContext2D; + } + + private clear() { + this.context.fillStyle = "black"; + this.context.fillRect(0, 0, this.canvas.width, this.canvas.height); + } + + async render({ node, time }: { node: BaseNode; time: number }) { + this.clear(); + await node.render({ renderer: this, time }); + } + + async renderToCanvas({ + node, + time, + targetCanvas, + }: { + node: BaseNode; + time: number; + targetCanvas: HTMLCanvasElement; + }) { + await this.render({ node, time }); + + const ctx = targetCanvas.getContext("2d"); + if (!ctx) { + throw new Error("Failed to get target canvas context"); + } + + ctx.drawImage(this.canvas, 0, 0, targetCanvas.width, targetCanvas.height); + } +} diff --git a/apps/web/src/services/renderer/nodes/base-node.ts b/apps/web/src/services/renderer/nodes/base-node.ts new file mode 100644 index 00000000..55bd055a --- /dev/null +++ b/apps/web/src/services/renderer/nodes/base-node.ts @@ -0,0 +1,35 @@ +import type { CanvasRenderer } from "../canvas-renderer"; + +export type BaseNodeParams = object | undefined; + +export class BaseNode<Params extends BaseNodeParams = BaseNodeParams> { + params: Params; + + constructor(params?: Params) { + this.params = params ?? ({} as Params); + } + + children: BaseNode[] = []; + + add(child: BaseNode) { + this.children.push(child); + return this; + } + + remove(child: BaseNode) { + this.children = this.children.filter((c) => c !== child); + return this; + } + + async render({ + renderer, + time, + }: { + renderer: CanvasRenderer; + time: number; + }): Promise<void> { + for (const child of this.children) { + await child.render({ renderer, time }); + } + } +} diff --git a/apps/web/src/services/renderer/nodes/blur-background-node.ts b/apps/web/src/services/renderer/nodes/blur-background-node.ts new file mode 100644 index 00000000..504161d9 --- /dev/null +++ b/apps/web/src/services/renderer/nodes/blur-background-node.ts @@ -0,0 +1,75 @@ +import type { CanvasRenderer } from "../canvas-renderer"; +import { BaseNode } from "./base-node"; + +export type BlurBackgroundNodeParams = { + blurIntensity: number; + contentNodes: BaseNode[]; +}; + +export class BlurBackgroundNode extends BaseNode<BlurBackgroundNodeParams> { + private blurIntensity: number; + private contentNodes: BaseNode[]; + + constructor(params: BlurBackgroundNodeParams) { + super(params); + this.blurIntensity = params.blurIntensity; + this.contentNodes = params.contentNodes; + } + + async render({ + renderer, + time, + }: { + renderer: CanvasRenderer; + time: number; + }): Promise<void> { + let offscreen: OffscreenCanvas | HTMLCanvasElement; + let offscreenCtx: + | OffscreenCanvasRenderingContext2D + | CanvasRenderingContext2D; + + try { + offscreen = new OffscreenCanvas(renderer.width, renderer.height); + const ctx = offscreen.getContext("2d"); + if (!ctx) { + throw new Error("failed to get offscreen canvas context"); + } + offscreenCtx = ctx; + } catch { + offscreen = document.createElement("canvas"); + offscreen.width = renderer.width; + offscreen.height = renderer.height; + const ctx = offscreen.getContext("2d"); + if (!ctx) { + throw new Error("failed to get canvas context"); + } + offscreenCtx = ctx; + } + + const originalContext = renderer.context; + renderer.context = offscreenCtx; + + for (const node of this.contentNodes) { + await node.render({ renderer, time }); + } + + renderer.context = originalContext; + + const zoomScale = 1.4; + const scaledWidth = renderer.width * zoomScale; + const scaledHeight = renderer.height * zoomScale; + const offsetX = (renderer.width - scaledWidth) / 2; + const offsetY = (renderer.height - scaledHeight) / 2; + + renderer.context.save(); + renderer.context.filter = `blur(${this.blurIntensity}px)`; + renderer.context.drawImage( + offscreen as CanvasImageSource, + offsetX, + offsetY, + scaledWidth, + scaledHeight, + ); + renderer.context.restore(); + } +} diff --git a/apps/web/src/services/renderer/nodes/color-node.ts b/apps/web/src/services/renderer/nodes/color-node.ts new file mode 100644 index 00000000..d3afbf9b --- /dev/null +++ b/apps/web/src/services/renderer/nodes/color-node.ts @@ -0,0 +1,31 @@ +import { drawCssBackground } from "@/lib/gradients"; +import type { CanvasRenderer } from "../canvas-renderer"; +import { BaseNode } from "./base-node"; + +export type ColorNodeParams = { + color: string; +}; + +export class ColorNode extends BaseNode<ColorNodeParams> { + private color: string; + + constructor(params: ColorNodeParams) { + super(params); + this.color = params.color; + } + + async render({ renderer }: { renderer: CanvasRenderer }) { + if (/gradient\(/i.test(this.color)) { + drawCssBackground({ + ctx: renderer.context, + width: renderer.width, + height: renderer.height, + css: this.color, + }); + return; + } + + renderer.context.fillStyle = this.color; + renderer.context.fillRect(0, 0, renderer.width, renderer.height); + } +} diff --git a/apps/web/src/services/renderer/nodes/image-node.ts b/apps/web/src/services/renderer/nodes/image-node.ts new file mode 100644 index 00000000..2a684d5f --- /dev/null +++ b/apps/web/src/services/renderer/nodes/image-node.ts @@ -0,0 +1,93 @@ +import type { CanvasRenderer } from "../canvas-renderer"; +import { BaseNode } from "./base-node"; +import type { BaseMediaNodeParams } from "./video-node"; + +const IMAGE_EPSILON = 1 / 1000; + +export type ImageNodeParams = BaseMediaNodeParams; + +export class ImageNode extends BaseNode<ImageNodeParams> { + private image?: HTMLImageElement; + private readyPromise: Promise<void>; + + constructor(params: ImageNodeParams) { + super(params); + this.readyPromise = this.load(); + } + + private async load() { + const image = new Image(); + this.image = image; + const url = URL.createObjectURL(this.params.file); + + await new Promise<void>((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => reject(new Error("Image load failed")); + image.src = url; + }); + + URL.revokeObjectURL(url); + } + + private getImageTime(time: number) { + return time - this.params.timeOffset + this.params.trimStart; + } + + private isInRange(time: number) { + const imageTime = this.getImageTime(time); + return ( + imageTime >= this.params.trimStart - IMAGE_EPSILON && + imageTime < this.params.trimStart + this.params.duration + ); + } + + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + await super.render({ renderer, time }); + + if (!this.isInRange(time)) { + return; + } + + await this.readyPromise; + + if (!this.image) { + return; + } + + renderer.context.save(); + + if (this.params.opacity !== undefined) { + renderer.context.globalAlpha = this.params.opacity; + } + + if ( + this.params.x !== undefined && + this.params.y !== undefined && + this.params.width !== undefined && + this.params.height !== undefined + ) { + renderer.context.drawImage( + this.image, + this.params.x, + this.params.y, + this.params.width, + this.params.height, + ); + } else { + const mediaW = this.image.naturalWidth || renderer.width; + const mediaH = this.image.naturalHeight || renderer.height; + const containScale = Math.min( + renderer.width / mediaW, + renderer.height / mediaH, + ); + const drawW = mediaW * containScale; + const drawH = mediaH * containScale; + const drawX = (renderer.width - drawW) / 2; + const drawY = (renderer.height - drawH) / 2; + + renderer.context.drawImage(this.image, drawX, drawY, drawW, drawH); + } + + renderer.context.restore(); + } +} diff --git a/apps/web/src/services/renderer/nodes/root-node.ts b/apps/web/src/services/renderer/nodes/root-node.ts new file mode 100644 index 00000000..336018e2 --- /dev/null +++ b/apps/web/src/services/renderer/nodes/root-node.ts @@ -0,0 +1,11 @@ +import { BaseNode } from "./base-node"; + +export type RootNodeParams = { + duration: number; +}; + +export class RootNode extends BaseNode<RootNodeParams> { + get duration() { + return this.params.duration ?? 0; + } +} diff --git a/apps/web/src/services/renderer/nodes/sticker-node.ts b/apps/web/src/services/renderer/nodes/sticker-node.ts new file mode 100644 index 00000000..5af13a6f --- /dev/null +++ b/apps/web/src/services/renderer/nodes/sticker-node.ts @@ -0,0 +1,87 @@ +import type { CanvasRenderer } from "../canvas-renderer"; +import { BaseNode, type BaseNodeParams } from "./base-node"; +import type { Transform } from "@/types/timeline"; + +const STICKER_EPSILON = 1 / 1000; + +export type StickerNodeParams = BaseNodeParams & { + iconName: string; + duration: number; + timeOffset: number; + trimStart: number; + trimEnd: number; + transform: Transform; + opacity: number; + color?: string; +}; + +export class StickerNode extends BaseNode<StickerNodeParams> { + private image?: HTMLImageElement; + private readyPromise: Promise<void>; + + constructor(params: StickerNodeParams) { + super(params); + this.readyPromise = this.load(); + } + + private async load() { + const image = new Image(); + this.image = image; + const color = this.params.color + ? `&color=${encodeURIComponent(this.params.color)}` + : ""; + const url = `https://api.iconify.design/${this.params.iconName}.svg?width=200&height=200${color}`; + + await new Promise<void>((resolve, reject) => { + image.onload = () => resolve(); + image.onerror = () => + reject(new Error(`Failed to load sticker: ${this.params.iconName}`)); + image.src = url; + }); + } + + private getStickerTime(time: number) { + return time - this.params.timeOffset + this.params.trimStart; + } + + private isInRange(time: number) { + const stickerTime = this.getStickerTime(time); + return ( + stickerTime >= this.params.trimStart - STICKER_EPSILON && + stickerTime < this.params.trimStart + this.params.duration + ); + } + + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + await super.render({ renderer, time }); + + if (!this.isInRange(time)) { + return; + } + + await this.readyPromise; + + if (!this.image) { + return; + } + + const { transform, opacity } = this.params; + const size = 200 * transform.scale; + const x = renderer.width / 2 + transform.position.x - size / 2; + const y = renderer.height / 2 + transform.position.y - size / 2; + + renderer.context.save(); + renderer.context.globalAlpha = opacity; + + if (transform.rotate !== 0) { + const centerX = x + size / 2; + const centerY = y + size / 2; + renderer.context.translate(centerX, centerY); + renderer.context.rotate((transform.rotate * Math.PI) / 180); + renderer.context.translate(-centerX, -centerY); + } + + renderer.context.drawImage(this.image, x, y, size, size); + renderer.context.restore(); + } +} diff --git a/apps/web/src/services/renderer/nodes/text-node.ts b/apps/web/src/services/renderer/nodes/text-node.ts new file mode 100644 index 00000000..2cbf3d2a --- /dev/null +++ b/apps/web/src/services/renderer/nodes/text-node.ts @@ -0,0 +1,74 @@ +import type { CanvasRenderer } from "../canvas-renderer"; +import { BaseNode } from "./base-node"; +import type { TextElement } from "@/types/timeline"; + +export type TextNodeParams = TextElement & { + canvasCenter: { x: number; y: number }; + textBaseline?: CanvasTextBaseline; +}; + +export class TextNode extends BaseNode<TextNodeParams> { + isInRange({ time }: { time: number }) { + return ( + time >= this.params.startTime && + time < this.params.startTime + this.params.duration + ); + } + + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + if (!this.isInRange({ time })) { + return; + } + + renderer.context.save(); + + const x = this.params.transform.position.x + this.params.canvasCenter.x; + const y = this.params.transform.position.y + this.params.canvasCenter.y; + + renderer.context.translate(x, y); + if (this.params.transform.rotate) { + renderer.context.rotate((this.params.transform.rotate * Math.PI) / 180); + } + + const fontWeight = this.params.fontWeight === "bold" ? "bold" : "normal"; + const fontStyle = this.params.fontStyle === "italic" ? "italic" : "normal"; + renderer.context.font = `${fontStyle} ${fontWeight} ${this.params.fontSize}px ${this.params.fontFamily}`; + renderer.context.textAlign = this.params.textAlign; + renderer.context.textBaseline = this.params.textBaseline || "middle"; + renderer.context.fillStyle = this.params.color; + + const prevAlpha = renderer.context.globalAlpha; + renderer.context.globalAlpha = this.params.opacity; + + if (this.params.backgroundColor) { + const metrics = renderer.context.measureText(this.params.content); + const ascent = + metrics.actualBoundingBoxAscent ?? this.params.fontSize * 0.8; + const descent = + metrics.actualBoundingBoxDescent ?? this.params.fontSize * 0.2; + const textW = metrics.width; + const textH = ascent + descent; + const padX = 8; + const padY = 4; + + renderer.context.fillStyle = this.params.backgroundColor; + let bgLeft = -textW / 2; + if (renderer.context.textAlign === "left") bgLeft = 0; + if (renderer.context.textAlign === "right") bgLeft = -textW; + + renderer.context.fillRect( + bgLeft - padX, + -textH / 2 - padY, + textW + padX * 2, + textH + padY * 2, + ); + + renderer.context.fillStyle = this.params.color; + } + + renderer.context.fillText(this.params.content, 0, 0); + + renderer.context.globalAlpha = prevAlpha; + renderer.context.restore(); + } +} diff --git a/apps/web/src/services/renderer/nodes/video-node.ts b/apps/web/src/services/renderer/nodes/video-node.ts new file mode 100644 index 00000000..01394b3d --- /dev/null +++ b/apps/web/src/services/renderer/nodes/video-node.ts @@ -0,0 +1,84 @@ +import type { CanvasRenderer } from "../canvas-renderer"; +import { BaseNode } from "./base-node"; +import { videoCache } from "@/services/media/video-cache"; + +const VIDEO_EPSILON = 1 / 1000; + +export interface BaseMediaNodeParams { + file: File; + duration: number; + timeOffset: number; + trimStart: number; + trimEnd: number; + x?: number; + y?: number; + width?: number; + height?: number; + opacity?: number; +} + +export interface VideoNodeParams extends BaseMediaNodeParams { + mediaId: string; +} + +export class VideoNode extends BaseNode<VideoNodeParams> { + private getVideoTime(time: number) { + return time - this.params.timeOffset + this.params.trimStart; + } + + private isInRange(time: number) { + const videoTime = this.getVideoTime(time); + return ( + videoTime >= this.params.trimStart - VIDEO_EPSILON && + videoTime < this.params.trimStart + this.params.duration + ); + } + + async render({ renderer, time }: { renderer: CanvasRenderer; time: number }) { + await super.render({ renderer, time }); + + if (!this.isInRange(time)) { + return; + } + + const videoTime = this.getVideoTime(time); + const frame = await videoCache.getFrameAt({ + mediaId: this.params.mediaId, + file: this.params.file, + time: videoTime, + }); + + if (frame) { + renderer.context.save(); + + if (this.params.opacity !== undefined) { + renderer.context.globalAlpha = this.params.opacity; + } + + if ( + this.params.x !== undefined && + this.params.y !== undefined && + this.params.width !== undefined && + this.params.height !== undefined + ) { + renderer.context.drawImage( + frame.canvas, + this.params.x, + this.params.y, + this.params.width, + this.params.height, + ); + } else { + renderer.context.drawImage( + frame.canvas, + 0, + 0, + renderer.width, + renderer.height, + ); + } + + renderer.context.restore(); + } + } +} diff --git a/apps/web/src/services/renderer/scene-builder.ts b/apps/web/src/services/renderer/scene-builder.ts new file mode 100644 index 00000000..b36edda1 --- /dev/null +++ b/apps/web/src/services/renderer/scene-builder.ts @@ -0,0 +1,130 @@ +import type { TimelineTrack } from "@/types/timeline"; +import type { MediaAsset } from "@/types/assets"; +import { RootNode } from "./nodes/root-node"; +import { VideoNode } from "./nodes/video-node"; +import { ImageNode } from "./nodes/image-node"; +import { TextNode } from "./nodes/text-node"; +import { StickerNode } from "./nodes/sticker-node"; +import { ColorNode } from "./nodes/color-node"; +import { BlurBackgroundNode } from "./nodes/blur-background-node"; +import type { TBackground, TCanvasSize } from "@/types/project"; +import { DEFAULT_BLUR_INTENSITY } from "@/constants/project-constants"; +import { isMainTrack } from "@/lib/timeline"; + +export type BuildSceneParams = { + canvasSize: TCanvasSize; + tracks: TimelineTrack[]; + mediaAssets: MediaAsset[]; + duration: number; + background: TBackground; +}; + +export function buildScene(params: BuildSceneParams) { + const { tracks, mediaAssets, duration, canvasSize, background } = params; + + const rootNode = new RootNode({ duration }); + const mediaMap = new Map(mediaAssets.map((m) => [m.id, m])); + + const visibleTracks = tracks.filter( + (track) => !("hidden" in track && track.hidden), + ); + + const mainTrack = visibleTracks.find((track) => isMainTrack(track)) ?? null; + const orderedTracksTopToBottom = [ + ...visibleTracks.filter((track) => !isMainTrack(track)), + ...(mainTrack ? [mainTrack] : []), + ]; + + const orderedTracksBottomToTop = orderedTracksTopToBottom.slice().reverse(); + + const contentNodes = []; + + for (const track of orderedTracksBottomToTop) { + const elements = track.elements + .filter((element) => !("hidden" in element && element.hidden)) + .slice() + .sort((a, b) => { + if (a.startTime !== b.startTime) return a.startTime - b.startTime; + return a.id.localeCompare(b.id); + }); + + for (const element of elements) { + if (element.type === "video" || element.type === "image") { + const mediaAsset = mediaMap.get(element.mediaId); + if (!mediaAsset?.file) { + continue; + } + + if (mediaAsset.type === "video") { + contentNodes.push( + new VideoNode({ + mediaId: mediaAsset.id, + file: mediaAsset.file, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + }), + ); + } + if (mediaAsset.type === "image") { + contentNodes.push( + new ImageNode({ + file: mediaAsset.file, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + }), + ); + } + } + + if (element.type === "text") { + contentNodes.push( + new TextNode({ + ...element, + canvasCenter: { x: canvasSize.width / 2, y: canvasSize.height / 2 }, + textBaseline: "middle", + }), + ); + } + + if (element.type === "sticker") { + contentNodes.push( + new StickerNode({ + iconName: element.iconName, + duration: element.duration, + timeOffset: element.startTime, + trimStart: element.trimStart, + trimEnd: element.trimEnd, + transform: element.transform, + opacity: element.opacity, + color: element.color, + }), + ); + } + } + } + + if (background.type === "blur") { + rootNode.add( + new BlurBackgroundNode({ + blurIntensity: background.blurIntensity ?? DEFAULT_BLUR_INTENSITY, + contentNodes, + }), + ); + for (const node of contentNodes) { + rootNode.add(node); + } + } else { + if (background.type === "color" && background.color !== "transparent") { + rootNode.add(new ColorNode({ color: background.color })); + } + for (const node of contentNodes) { + rootNode.add(node); + } + } + + return rootNode; +} diff --git a/apps/web/src/services/renderer/scene-exporter.ts b/apps/web/src/services/renderer/scene-exporter.ts new file mode 100644 index 00000000..39e94e55 --- /dev/null +++ b/apps/web/src/services/renderer/scene-exporter.ts @@ -0,0 +1,152 @@ +import EventEmitter from "eventemitter3"; + +import { + Output, + Mp4OutputFormat, + WebMOutputFormat, + BufferTarget, + CanvasSource, + AudioBufferSource, + QUALITY_LOW, + QUALITY_MEDIUM, + QUALITY_HIGH, + QUALITY_VERY_HIGH, +} from "mediabunny"; +import type { RootNode } from "./nodes/root-node"; +import { CanvasRenderer } from "./canvas-renderer"; + +export type ExportFormat = "mp4" | "webm"; +export type ExportQuality = "low" | "medium" | "high" | "very_high"; + +type ExportParams = { + width: number; + height: number; + fps: number; + format: ExportFormat; + quality: ExportQuality; + shouldIncludeAudio?: boolean; + audioBuffer?: AudioBuffer; +}; + +const qualityMap = { + low: QUALITY_LOW, + medium: QUALITY_MEDIUM, + high: QUALITY_HIGH, + very_high: QUALITY_VERY_HIGH, +}; + +export type SceneExporterEvents = { + progress: [progress: number]; + complete: [buffer: ArrayBuffer]; + error: [error: Error]; + cancelled: []; +}; + +export class SceneExporter extends EventEmitter<SceneExporterEvents> { + private renderer: CanvasRenderer; + private format: ExportFormat; + private quality: ExportQuality; + private shouldIncludeAudio: boolean; + private audioBuffer?: AudioBuffer; + + private isCancelled = false; + + constructor({ + width, + height, + fps, + format, + quality, + shouldIncludeAudio, + audioBuffer, + }: ExportParams) { + super(); + this.renderer = new CanvasRenderer({ + width, + height, + fps, + }); + + this.format = format; + this.quality = quality; + this.shouldIncludeAudio = shouldIncludeAudio ?? false; + this.audioBuffer = audioBuffer; + } + + cancel(): void { + this.isCancelled = true; + } + + async export({ + rootNode, + }: { + rootNode: RootNode; + }): Promise<ArrayBuffer | null> { + const { fps } = this.renderer; + const frameCount = Math.ceil(rootNode.duration * fps); + + const outputFormat = + this.format === "webm" ? new WebMOutputFormat() : new Mp4OutputFormat(); + + const output = new Output({ + format: outputFormat, + target: new BufferTarget(), + }); + + const videoSource = new CanvasSource(this.renderer.canvas, { + codec: this.format === "webm" ? "vp9" : "avc", + bitrate: qualityMap[this.quality], + }); + + output.addVideoTrack(videoSource, { frameRate: fps }); + + let audioSource: AudioBufferSource | null = null; + if (this.shouldIncludeAudio && this.audioBuffer) { + audioSource = new AudioBufferSource({ + codec: this.format === "webm" ? "opus" : "aac", + bitrate: qualityMap[this.quality], + }); + output.addAudioTrack(audioSource); + } + + await output.start(); + + if (audioSource && this.audioBuffer) { + await audioSource.add(this.audioBuffer); + audioSource.close(); + } + + for (let i = 0; i < frameCount; i++) { + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + const time = i / fps; + await this.renderer.render({ node: rootNode, time }); + await videoSource.add(time, 1 / fps); + + this.emit("progress", i / frameCount); + } + + if (this.isCancelled) { + await output.cancel(); + this.emit("cancelled"); + return null; + } + + videoSource.close(); + await output.finalize(); + this.emit("progress", 1); + + const buffer = output.target.buffer; + if (!buffer) { + this.emit("error", new Error("Failed to export video")); + return null; + } + + this.emit("complete", buffer); + return buffer; + } +} diff --git a/apps/web/src/services/storage/indexeddb-adapter.ts b/apps/web/src/services/storage/indexeddb-adapter.ts new file mode 100644 index 00000000..cc4d03a0 --- /dev/null +++ b/apps/web/src/services/storage/indexeddb-adapter.ts @@ -0,0 +1,101 @@ +import type { StorageAdapter } from "./types"; + +export class IndexedDBAdapter<T> implements StorageAdapter<T> { + private dbName: string; + private storeName: string; + private version: number; + + constructor(dbName: string, storeName: string, version = 1) { + this.dbName = dbName; + this.storeName = storeName; + this.version = version; + } + + private async getDB(): Promise<IDBDatabase> { + return new Promise((resolve, reject) => { + const request = indexedDB.open(this.dbName, this.version); + + request.onerror = () => reject(request.error); + request.onsuccess = () => resolve(request.result); + + request.onupgradeneeded = (event) => { + const db = (event.target as IDBOpenDBRequest).result; + if (!db.objectStoreNames.contains(this.storeName)) { + db.createObjectStore(this.storeName, { keyPath: "id" }); + } + }; + }); + } + + async get(key: string): Promise<T | null> { + const db = await this.getDB(); + const transaction = db.transaction([this.storeName], "readonly"); + const store = transaction.objectStore(this.storeName); + + return new Promise((resolve, reject) => { + const request = store.get(key); + request.onerror = () => reject(request.error); + request.onsuccess = () => resolve(request.result || null); + }); + } + + async set(key: string, value: T): Promise<void> { + const db = await this.getDB(); + const transaction = db.transaction([this.storeName], "readwrite"); + const store = transaction.objectStore(this.storeName); + + return new Promise((resolve, reject) => { + const request = store.put({ id: key, ...value }); + request.onerror = () => reject(request.error); + request.onsuccess = () => resolve(); + }); + } + + async remove(key: string): Promise<void> { + const db = await this.getDB(); + const transaction = db.transaction([this.storeName], "readwrite"); + const store = transaction.objectStore(this.storeName); + + return new Promise((resolve, reject) => { + const request = store.delete(key); + request.onerror = () => reject(request.error); + request.onsuccess = () => resolve(); + }); + } + + async list(): Promise<string[]> { + const db = await this.getDB(); + const transaction = db.transaction([this.storeName], "readonly"); + const store = transaction.objectStore(this.storeName); + + return new Promise((resolve, reject) => { + const request = store.getAllKeys(); + request.onerror = () => reject(request.error); + request.onsuccess = () => resolve(request.result as string[]); + }); + } + + async getAll(): Promise<T[]> { + const db = await this.getDB(); + const transaction = db.transaction([this.storeName], "readonly"); + const store = transaction.objectStore(this.storeName); + + return new Promise((resolve, reject) => { + const request = store.getAll(); + request.onerror = () => reject(request.error); + request.onsuccess = () => resolve(request.result || []); + }); + } + + async clear(): Promise<void> { + const db = await this.getDB(); + const transaction = db.transaction([this.storeName], "readwrite"); + const store = transaction.objectStore(this.storeName); + + return new Promise((resolve, reject) => { + const request = store.clear(); + request.onerror = () => reject(request.error); + request.onsuccess = () => resolve(); + }); + } +} diff --git a/apps/web/src/services/storage/migrations/base.ts b/apps/web/src/services/storage/migrations/base.ts new file mode 100644 index 00000000..c7558264 --- /dev/null +++ b/apps/web/src/services/storage/migrations/base.ts @@ -0,0 +1,5 @@ +export abstract class StorageMigration { + abstract from: number; + abstract to: number; + abstract run(): Promise<void>; +} diff --git a/apps/web/src/services/storage/migrations/index.ts b/apps/web/src/services/storage/migrations/index.ts new file mode 100644 index 00000000..2343362f --- /dev/null +++ b/apps/web/src/services/storage/migrations/index.ts @@ -0,0 +1,14 @@ +export { StorageMigration } from "./base"; +export { StorageVersionManager } from "./version-manager"; +export { runStorageMigrations } from "./runner"; +import { V0toV1Migration } from "./v0-to-v1"; +import { V1toV2Migration } from "./v1-to-v2"; +import { V2toV3Migration } from "./v2-to-v3"; + +export const CURRENT_STORAGE_VERSION = 3; + +export const migrations = [ + new V0toV1Migration(), + new V1toV2Migration(), + new V2toV3Migration(), +]; diff --git a/apps/web/src/services/storage/migrations/runner.ts b/apps/web/src/services/storage/migrations/runner.ts new file mode 100644 index 00000000..d9277cb0 --- /dev/null +++ b/apps/web/src/services/storage/migrations/runner.ts @@ -0,0 +1,134 @@ +import { IndexedDBAdapter } from "@/services/storage/indexeddb-adapter"; +import type { StorageMigration } from "./base"; +import { StorageVersionManager } from "./version-manager"; + +export interface StorageMigrationResult { + fromVersion: number; + toVersion: number; + migrated: boolean; +} + +export type StorageMigrationCallbacks = { + onMigrationStart?: ({ + fromVersion, + toVersion, + }: { + fromVersion: number; + toVersion: number; + }) => void; + onMigrationComplete?: ({ + fromVersion, + toVersion, + }: { + fromVersion: number; + toVersion: number; + }) => void; +}; + +type ProjectRecord = Record<string, unknown>; + +export async function runStorageMigrations({ + migrations, + versionManager = new StorageVersionManager(), + callbacks, +}: { + migrations: StorageMigration[]; + versionManager?: StorageVersionManager; + callbacks?: StorageMigrationCallbacks; +}): Promise<StorageMigrationResult> { + const versionRecord = await versionManager.getVersionRecord(); + const inferredVersion = versionRecord + ? null + : await inferStorageVersionFromProjects(); + const fromVersion = + versionRecord?.inProgress?.from ?? + versionRecord?.version ?? + inferredVersion ?? + 0; + + if (!versionRecord) { + await versionManager.setVersion({ version: fromVersion }); + } + + const orderedMigrations = [...migrations].sort((a, b) => a.from - b.from); + let currentVersion = fromVersion; + + for (const migration of orderedMigrations) { + if (migration.from !== currentVersion) { + continue; + } + + await versionManager.setInProgress({ + from: migration.from, + to: migration.to, + }); + callbacks?.onMigrationStart?.({ + fromVersion: migration.from, + toVersion: migration.to, + }); + await migration.run(); + currentVersion = migration.to; + await versionManager.setVersion({ version: currentVersion }); + await versionManager.clearInProgress(); + callbacks?.onMigrationComplete?.({ + fromVersion: migration.from, + toVersion: migration.to, + }); + } + + return { + fromVersion, + toVersion: currentVersion, + migrated: currentVersion !== fromVersion, + }; +} + +async function inferStorageVersionFromProjects(): Promise<number> { + const projectsAdapter = new IndexedDBAdapter<unknown>( + "video-editor-projects", + "projects", + 1, + ); + const projects = await projectsAdapter.getAll(); + + if (projects.length === 0) { + return 0; + } + + let lowestVersion = Number.POSITIVE_INFINITY; + + for (const project of projects) { + const projectVersion = checkProjectVersion({ project }); + if (projectVersion < lowestVersion) { + lowestVersion = projectVersion; + } + } + + if (lowestVersion === Number.POSITIVE_INFINITY) { + return 0; + } + + return lowestVersion; +} + +function checkProjectVersion({ project }: { project: unknown }): number { + if (!isRecord(project)) { + return 0; + } + + const versionValue = project.version; + if (typeof versionValue === "number") { + return versionValue; + } + + const scenesValue = project.scenes; + if (Array.isArray(scenesValue) && scenesValue.length > 0) { + return 1; + } + + return 0; +} + +function isRecord(value: unknown): value is ProjectRecord { + return typeof value === "object" && value !== null; +} diff --git a/apps/web/src/services/storage/migrations/v0-to-v1.ts b/apps/web/src/services/storage/migrations/v0-to-v1.ts new file mode 100644 index 00000000..7ec388f1 --- /dev/null +++ b/apps/web/src/services/storage/migrations/v0-to-v1.ts @@ -0,0 +1,93 @@ +import { IndexedDBAdapter } from "@/services/storage/indexeddb-adapter"; +import type { SerializedScene } from "@/services/storage/types"; +import { buildDefaultScene } from "@/lib/scenes"; +import type { TScene } from "@/types/timeline"; +import { StorageMigration } from "./base"; + +type ProjectRecord = Record<string, unknown>; + +export class V0toV1Migration extends StorageMigration { + from = 0; + to = 1; + + async run(): Promise<void> { + const projectsAdapter = new IndexedDBAdapter<unknown>( + "video-editor-projects", + "projects", + 1, + ); + const projects = await projectsAdapter.getAll(); + + for (const project of projects) { + if (!isRecord(project)) { + continue; + } + + const scenesValue = project.scenes; + if (Array.isArray(scenesValue) && scenesValue.length > 0) { + continue; + } + + const mainScene = buildDefaultScene({ isMain: true, name: "Main scene" }); + const serializedScene = serializeScene({ scene: mainScene }); + const updatedProject: ProjectRecord = { + ...project, + scenes: [serializedScene], + currentSceneId: mainScene.id, + version: 1, + }; + + const updatedAt = new Date().toISOString(); + if (isRecord(project.metadata)) { + updatedProject.metadata = { + ...project.metadata, + updatedAt, + }; + } else { + updatedProject.updatedAt = updatedAt; + } + + const projectId = getProjectId({ project: updatedProject }); + if (!projectId) { + continue; + } + + await projectsAdapter.set(projectId, updatedProject); + } + } +} + +function serializeScene({ scene }: { scene: TScene }): SerializedScene { + return { + id: scene.id, + name: scene.name, + isMain: scene.isMain, + tracks: scene.tracks, + bookmarks: scene.bookmarks, + createdAt: scene.createdAt.toISOString(), + updatedAt: scene.updatedAt.toISOString(), + }; +} + +function getProjectId({ project }: { project: ProjectRecord }): string | null { + const idValue = project.id; + if (typeof idValue === "string" && idValue.length > 0) { + return idValue; + } + + const metadataValue = project.metadata; + if (!isRecord(metadataValue)) { + return null; + } + + const metadataId = metadataValue.id; + if (typeof metadataId === "string" && metadataId.length > 0) { + return metadataId; + } + + return null; +} + +function isRecord(value: unknown): value is ProjectRecord { + return typeof value === "object" && value !== null; +} diff --git a/apps/web/src/services/storage/migrations/v1-to-v2.ts b/apps/web/src/services/storage/migrations/v1-to-v2.ts new file mode 100644 index 00000000..05d302ea --- /dev/null +++ b/apps/web/src/services/storage/migrations/v1-to-v2.ts @@ -0,0 +1,339 @@ +import { + DEFAULT_BLUR_INTENSITY, + DEFAULT_CANVAS_SIZE, + DEFAULT_COLOR, + DEFAULT_FPS, +} from "@/constants/project-constants"; +import { IndexedDBAdapter } from "@/services/storage/indexeddb-adapter"; +import { StorageMigration } from "./base"; + +type ProjectRecord = Record<string, unknown>; + +export class V1toV2Migration extends StorageMigration { + from = 1; + to = 2; + + async run(): Promise<void> { + const projectsAdapter = new IndexedDBAdapter<unknown>( + "video-editor-projects", + "projects", + 1, + ); + const projects = await projectsAdapter.getAll(); + + for (const project of projects) { + if (!isRecord(project)) { + continue; + } + + const projectId = getProjectId({ project }); + if (!projectId) { + continue; + } + + if (isV2Project({ project })) { + continue; + } + + const migratedProject = migrateProject({ project, projectId }); + await projectsAdapter.set(projectId, migratedProject); + } + } +} + +function migrateProject({ + project, + projectId, +}: { + project: ProjectRecord; + projectId: string; +}): ProjectRecord { + const createdAt = normalizeDateString({ value: project.createdAt }); + const updatedAt = normalizeDateString({ value: project.updatedAt }); + const metadataValue = project.metadata; + + const metadata = isRecord(metadataValue) + ? { + id: getStringValue({ value: metadataValue.id, fallback: projectId }), + name: getStringValue({ value: metadataValue.name, fallback: "" }), + thumbnail: getStringValue({ value: metadataValue.thumbnail }), + createdAt: normalizeDateString({ value: metadataValue.createdAt }), + updatedAt: normalizeDateString({ value: metadataValue.updatedAt }), + } + : { + id: projectId, + name: getStringValue({ value: project.name, fallback: "" }), + thumbnail: getStringValue({ value: project.thumbnail }), + createdAt, + updatedAt, + }; + + const scenesValue = project.scenes; + const scenes = Array.isArray(scenesValue) ? scenesValue : []; + const legacyBookmarks = Array.isArray(project.bookmarks) + ? project.bookmarks + : null; + const normalizedScenes = applyLegacyBookmarks({ + scenes, + legacyBookmarks, + }); + + const settingsValue = project.settings; + const settings = isRecord(settingsValue) + ? { + fps: getNumberValue({ + value: settingsValue.fps, + fallback: DEFAULT_FPS, + }), + canvasSize: getCanvasSizeValue({ + value: settingsValue.canvasSize, + fallback: DEFAULT_CANVAS_SIZE, + }), + background: getBackgroundValue({ + value: settingsValue.background, + }), + } + : { + fps: getNumberValue({ value: project.fps, fallback: DEFAULT_FPS }), + canvasSize: getCanvasSizeValue({ + value: project.canvasSize, + fallback: DEFAULT_CANVAS_SIZE, + }), + background: getBackgroundValue({ + value: project.background, + backgroundType: project.backgroundType, + backgroundColor: project.backgroundColor, + blurIntensity: project.blurIntensity, + }), + }; + + const currentSceneId = getCurrentSceneId({ + value: project.currentSceneId, + scenes: normalizedScenes, + }); + + return { + ...project, + metadata, + scenes: normalizedScenes, + currentSceneId, + settings, + version: 2, + }; +} + +function getProjectId({ project }: { project: ProjectRecord }): string | null { + const idValue = project.id; + if (typeof idValue === "string" && idValue.length > 0) { + return idValue; + } + + const metadataValue = project.metadata; + if (!isRecord(metadataValue)) { + return null; + } + + const metadataId = metadataValue.id; + if (typeof metadataId === "string" && metadataId.length > 0) { + return metadataId; + } + + return null; +} + +function getCurrentSceneId({ + value, + scenes, +}: { + value: unknown; + scenes: unknown[]; +}): string { + if (typeof value === "string" && value.length > 0) { + return value; + } + + const mainSceneId = findMainSceneId({ scenes }); + if (mainSceneId) { + return mainSceneId; + } + + return ""; +} + +function findMainSceneId({ scenes }: { scenes: unknown[] }): string | null { + for (const scene of scenes) { + if (!isRecord(scene)) { + continue; + } + + if (scene.isMain === true && typeof scene.id === "string") { + return scene.id; + } + } + + for (const scene of scenes) { + if (!isRecord(scene)) { + continue; + } + + if (typeof scene.id === "string") { + return scene.id; + } + } + + return null; +} + +function applyLegacyBookmarks({ + scenes, + legacyBookmarks, +}: { + scenes: unknown[]; + legacyBookmarks: unknown[] | null; +}): unknown[] { + if (!legacyBookmarks || legacyBookmarks.length === 0) { + return scenes; + } + + const mainSceneId = findMainSceneId({ scenes }); + + return scenes.map((scene) => { + if (!isRecord(scene)) { + return scene; + } + + if (mainSceneId && scene.id !== mainSceneId) { + return scene; + } + + if (Array.isArray(scene.bookmarks) && scene.bookmarks.length > 0) { + return scene; + } + + return { + ...scene, + bookmarks: legacyBookmarks, + }; + }); +} + +function getBackgroundValue({ + value, + backgroundType, + backgroundColor, + blurIntensity, +}: { + value: unknown; + backgroundType?: unknown; + backgroundColor?: unknown; + blurIntensity?: unknown; +}): { + type: "color" | "blur"; + color?: string; + blurIntensity?: number; +} { + if (isRecord(value)) { + const typeValue = value.type; + if (typeValue === "blur") { + return { + type: "blur", + blurIntensity: getNumberValue({ + value: value.blurIntensity, + fallback: DEFAULT_BLUR_INTENSITY, + }), + }; + } + + return { + type: "color", + color: getStringValue({ value: value.color, fallback: DEFAULT_COLOR }), + }; + } + + if (backgroundType === "blur") { + return { + type: "blur", + blurIntensity: getNumberValue({ + value: blurIntensity, + fallback: DEFAULT_BLUR_INTENSITY, + }), + }; + } + + return { + type: "color", + color: getStringValue({ value: backgroundColor, fallback: DEFAULT_COLOR }), + }; +} + +function getCanvasSizeValue({ + value, + fallback, +}: { + value: unknown; + fallback: { width: number; height: number }; +}): { width: number; height: number } { + if (isRecord(value)) { + const width = getNumberValue({ + value: value.width, + fallback: fallback.width, + }); + const height = getNumberValue({ + value: value.height, + fallback: fallback.height, + }); + + return { width, height }; + } + + return fallback; +} + +function getNumberValue({ + value, + fallback, +}: { + value: unknown; + fallback: number; +}): number { + return typeof value === "number" ? value : fallback; +} + +function getStringValue({ + value, + fallback, +}: { + value: unknown; + fallback?: string; +}): string | undefined { + if (typeof value === "string") { + return value; + } + + return fallback; +} + +function normalizeDateString({ value }: { value: unknown }): string { + if (value instanceof Date) { + return value.toISOString(); + } + + if (typeof value === "string") { + return value; + } + + return new Date().toISOString(); +} + +function isV2Project({ project }: { project: ProjectRecord }): boolean { + const versionValue = project.version; + if (typeof versionValue === "number" && versionValue >= 2) { + return true; + } + + return isRecord(project.metadata) && isRecord(project.settings); +} + +function isRecord(value: unknown): value is ProjectRecord { + return typeof value === "object" && value !== null; +} diff --git a/apps/web/src/services/storage/migrations/v2-to-v3.ts b/apps/web/src/services/storage/migrations/v2-to-v3.ts new file mode 100644 index 00000000..eb6db26d --- /dev/null +++ b/apps/web/src/services/storage/migrations/v2-to-v3.ts @@ -0,0 +1,92 @@ +import { getProjectDurationFromScenes } from "@/lib/scenes"; +import { IndexedDBAdapter } from "@/services/storage/indexeddb-adapter"; +import type { TScene } from "@/types/timeline"; +import { StorageMigration } from "./base"; + +type ProjectRecord = Record<string, unknown>; + +export class V2toV3Migration extends StorageMigration { + from = 2; + to = 3; + + async run(): Promise<void> { + const projectsAdapter = new IndexedDBAdapter<unknown>( + "video-editor-projects", + "projects", + 1, + ); + const projects = await projectsAdapter.getAll(); + + for (const project of projects) { + if (!isRecord(project)) { + continue; + } + + const projectId = getProjectId({ project }); + if (!projectId) { + continue; + } + + if (isV3Project({ project })) { + continue; + } + + const scenes = getScenes({ project }); + const duration = getProjectDurationFromScenes({ scenes }); + + const metadataValue = project.metadata; + const metadata = isRecord(metadataValue) + ? { ...metadataValue, duration } + : { duration }; + + const migratedProject = { + ...project, + metadata, + version: 3, + }; + + await projectsAdapter.set(projectId, migratedProject); + } + } +} + +function getProjectId({ project }: { project: ProjectRecord }): string | null { + const idValue = project.id; + if (typeof idValue === "string" && idValue.length > 0) { + return idValue; + } + + const metadataValue = project.metadata; + if (!isRecord(metadataValue)) { + return null; + } + + const metadataId = metadataValue.id; + if (typeof metadataId === "string" && metadataId.length > 0) { + return metadataId; + } + + return null; +} + +function getScenes({ project }: { project: ProjectRecord }): TScene[] { + const scenesValue = project.scenes; + if (!Array.isArray(scenesValue)) { + return []; + } + + return scenesValue.filter(isRecord) as unknown as TScene[]; +} + +function isV3Project({ project }: { project: ProjectRecord }): boolean { + const versionValue = project.version; + if (typeof versionValue === "number" && versionValue >= 3) { + return true; + } + + return isRecord(project.metadata) && typeof project.metadata.duration === "number"; +} + +function isRecord(value: unknown): value is ProjectRecord { + return typeof value === "object" && value !== null; +} diff --git a/apps/web/src/services/storage/migrations/version-manager.ts b/apps/web/src/services/storage/migrations/version-manager.ts new file mode 100644 index 00000000..879498ce --- /dev/null +++ b/apps/web/src/services/storage/migrations/version-manager.ts @@ -0,0 +1,73 @@ +import { IndexedDBAdapter } from "@/services/storage/indexeddb-adapter"; + +type StorageVersionRecord = { + version: number; + inProgress?: { + from: number; + to: number; + }; +}; + +const DEFAULT_DB_NAME = "video-editor-meta"; +const DEFAULT_STORE_NAME = "storage-version"; +const DEFAULT_DB_VERSION = 1; +const STORAGE_VERSION_KEY = "storage-version"; + +export class StorageVersionManager { + private adapter: IndexedDBAdapter<StorageVersionRecord>; + + constructor({ + dbName = DEFAULT_DB_NAME, + storeName = DEFAULT_STORE_NAME, + version = DEFAULT_DB_VERSION, + }: { + dbName?: string; + storeName?: string; + version?: number; + } = {}) { + this.adapter = new IndexedDBAdapter<StorageVersionRecord>( + dbName, + storeName, + version, + ); + } + + async getVersion(): Promise<number> { + const record = await this.adapter.get(STORAGE_VERSION_KEY); + return record?.version ?? 0; + } + + async getVersionRecord(): Promise<StorageVersionRecord | null> { + return this.adapter.get(STORAGE_VERSION_KEY); + } + + async setVersion({ version }: { version: number }): Promise<void> { + const record = await this.getVersionRecord(); + const inProgress = record?.inProgress; + await this.adapter.set(STORAGE_VERSION_KEY, { + version, + ...(inProgress ? { inProgress } : {}), + }); + } + + async setInProgress({ + from, + to, + }: { + from: number; + to: number; + }): Promise<void> { + const record = await this.getVersionRecord(); + const version = record?.version ?? 0; + await this.adapter.set(STORAGE_VERSION_KEY, { + version, + inProgress: { from, to }, + }); + } + + async clearInProgress(): Promise<void> { + const record = await this.getVersionRecord(); + const version = record?.version ?? 0; + await this.adapter.set(STORAGE_VERSION_KEY, { version }); + } +} diff --git a/apps/web/src/services/storage/opfs-adapter.ts b/apps/web/src/services/storage/opfs-adapter.ts new file mode 100644 index 00000000..879223b5 --- /dev/null +++ b/apps/web/src/services/storage/opfs-adapter.ts @@ -0,0 +1,73 @@ +import type { StorageAdapter } from "./types"; + +export class OPFSAdapter implements StorageAdapter<File> { + private directoryName: string; + + constructor(directoryName = "media") { + this.directoryName = directoryName; + } + + private async getDirectory(): Promise<FileSystemDirectoryHandle> { + const opfsRoot = await navigator.storage.getDirectory(); + return await opfsRoot.getDirectoryHandle(this.directoryName, { + create: true, + }); + } + + async get(key: string): Promise<File | null> { + try { + const directory = await this.getDirectory(); + const fileHandle = await directory.getFileHandle(key); + return await fileHandle.getFile(); + } catch (error) { + if ((error as Error).name === "NotFoundError") { + return null; + } + throw error; + } + } + + async set(key: string, file: File): Promise<void> { + const directory = await this.getDirectory(); + const fileHandle = await directory.getFileHandle(key, { create: true }); + const writable = await fileHandle.createWritable(); + + await writable.write(file); + await writable.close(); + } + + async remove(key: string): Promise<void> { + try { + const directory = await this.getDirectory(); + await directory.removeEntry(key); + } catch (error) { + if ((error as Error).name !== "NotFoundError") { + throw error; + } + } + } + + async list(): Promise<string[]> { + const directory = await this.getDirectory(); + const keys: string[] = []; + + for await (const name of directory.keys()) { + keys.push(name); + } + + return keys; + } + + async clear(): Promise<void> { + const directory = await this.getDirectory(); + + for await (const name of directory.keys()) { + await directory.removeEntry(name); + } + } + + // Helper method to check OPFS support + static isSupported(): boolean { + return "storage" in navigator && "getDirectory" in navigator.storage; + } +} diff --git a/apps/web/src/services/storage/storage-service.ts b/apps/web/src/services/storage/storage-service.ts new file mode 100644 index 00000000..cb0b2ced --- /dev/null +++ b/apps/web/src/services/storage/storage-service.ts @@ -0,0 +1,475 @@ +import type { TProject, TProjectMetadata } from "@/types/project"; +import { getProjectDurationFromScenes } from "@/lib/scenes"; +import type { MediaAsset } from "@/types/assets"; +import { IndexedDBAdapter } from "./indexeddb-adapter"; +import { OPFSAdapter } from "./opfs-adapter"; +import type { + MediaAssetData, + StorageConfig, + SerializedProject, + SerializedScene, +} from "./types"; +import type { SavedSoundsData, SavedSound, SoundEffect } from "@/types/sounds"; +import { + migrations, + runStorageMigrations, +} from "@/services/storage/migrations"; +import type { TimelineTrack, TScene } from "@/types/timeline"; + +class StorageService { + private projectsAdapter: IndexedDBAdapter<SerializedProject>; + private savedSoundsAdapter: IndexedDBAdapter<SavedSoundsData>; + private config: StorageConfig; + private migrationsPromise: Promise<void> | null = null; + + constructor() { + this.config = { + projectsDb: "video-editor-projects", + mediaDb: "video-editor-media", + savedSoundsDb: "video-editor-saved-sounds", + version: 1, + }; + + this.projectsAdapter = new IndexedDBAdapter<SerializedProject>( + this.config.projectsDb, + "projects", + this.config.version, + ); + + this.savedSoundsAdapter = new IndexedDBAdapter<SavedSoundsData>( + this.config.savedSoundsDb, + "saved-sounds", + this.config.version, + ); + } + + private async ensureMigrations(): Promise<void> { + if (this.migrationsPromise) { + await this.migrationsPromise; + return; + } + + this.migrationsPromise = runStorageMigrations({ migrations }).then( + () => undefined, + ); + await this.migrationsPromise; + } + + private getProjectMediaAdapters({ projectId }: { projectId: string }) { + const mediaMetadataAdapter = new IndexedDBAdapter<MediaAssetData>( + `${this.config.mediaDb}-${projectId}`, + "media-metadata", + this.config.version, + ); + + const mediaAssetsAdapter = new OPFSAdapter(`media-files-${projectId}`); + + return { mediaMetadataAdapter, mediaAssetsAdapter }; + } + + private stripAudioBuffers({ + tracks, + }: { + tracks: TimelineTrack[]; + }): TimelineTrack[] { + return tracks.map((track) => { + if (track.type !== "audio") return track; + return { + ...track, + elements: track.elements.map((element) => { + const { buffer: _buffer, ...rest } = element; + return rest; + }), + }; + }); + } + + async saveProject({ project }: { project: TProject }): Promise<void> { + const duration = + project.metadata.duration ?? + getProjectDurationFromScenes({ scenes: project.scenes }); + const serializedScenes: SerializedScene[] = project.scenes.map((scene) => ({ + id: scene.id, + name: scene.name, + isMain: scene.isMain, + tracks: this.stripAudioBuffers({ tracks: scene.tracks }), + bookmarks: scene.bookmarks, + createdAt: scene.createdAt.toISOString(), + updatedAt: scene.updatedAt.toISOString(), + })); + + const serializedProject: SerializedProject = { + metadata: { + id: project.metadata.id, + name: project.metadata.name, + thumbnail: project.metadata.thumbnail, + duration, + createdAt: project.metadata.createdAt.toISOString(), + updatedAt: project.metadata.updatedAt.toISOString(), + }, + scenes: serializedScenes, + currentSceneId: project.currentSceneId, + settings: project.settings, + version: project.version, + timelineViewState: project.timelineViewState, + }; + + await this.projectsAdapter.set(project.metadata.id, serializedProject); + } + + async loadProject({ + id, + }: { + id: string; + }): Promise<{ project: TProject } | null> { + await this.ensureMigrations(); + const serializedProject = await this.projectsAdapter.get(id); + + if (!serializedProject) return null; + + const scenes = + serializedProject.scenes?.map((scene) => ({ + id: scene.id, + name: scene.name, + isMain: scene.isMain, + tracks: (scene.tracks ?? []).map((track) => + track.type === "video" + ? { ...track, isMain: track.isMain ?? false } // legacy: isMain was optional + : track, + ), + bookmarks: scene.bookmarks ?? [], + createdAt: new Date(scene.createdAt), + updatedAt: new Date(scene.updatedAt), + })) ?? []; + + const project: TProject = { + metadata: { + id: serializedProject.metadata.id, + name: serializedProject.metadata.name, + thumbnail: serializedProject.metadata.thumbnail, + duration: + serializedProject.metadata.duration ?? + getProjectDurationFromScenes({ scenes }), + createdAt: new Date(serializedProject.metadata.createdAt), + updatedAt: new Date(serializedProject.metadata.updatedAt), + }, + scenes, + currentSceneId: serializedProject.currentSceneId || "", + settings: serializedProject.settings, + version: serializedProject.version, + timelineViewState: serializedProject.timelineViewState, + }; + + return { project }; + } + + async loadAllProjects(): Promise<TProject[]> { + const projectIds = await this.projectsAdapter.list(); + const projects: TProject[] = []; + + for (const id of projectIds) { + const result = await this.loadProject({ id }); + if (result?.project) { + projects.push(result.project); + } + } + + return projects.sort( + (a, b) => b.metadata.updatedAt.getTime() - a.metadata.updatedAt.getTime(), + ); + } + + async loadAllProjectsMetadata(): Promise<TProjectMetadata[]> { + await this.ensureMigrations(); + const serializedProjects = await this.projectsAdapter.getAll(); + + const metadata = serializedProjects.map((serializedProject) => ({ + id: serializedProject.metadata.id, + name: serializedProject.metadata.name, + thumbnail: serializedProject.metadata.thumbnail, + duration: + serializedProject.metadata.duration ?? + getProjectDurationFromScenes({ + scenes: (serializedProject.scenes ?? []) as unknown as TScene[], + }), + createdAt: new Date(serializedProject.metadata.createdAt), + updatedAt: new Date(serializedProject.metadata.updatedAt), + })); + + return metadata.sort( + (a, b) => b.updatedAt.getTime() - a.updatedAt.getTime(), + ); + } + + async deleteProject({ id }: { id: string }): Promise<void> { + await this.projectsAdapter.remove(id); + } + + async saveMediaAsset({ + projectId, + mediaAsset, + }: { + projectId: string; + mediaAsset: MediaAsset; + }): Promise<void> { + const { mediaMetadataAdapter, mediaAssetsAdapter } = + this.getProjectMediaAdapters({ projectId }); + + await mediaAssetsAdapter.set(mediaAsset.id, mediaAsset.file); + + const metadata: MediaAssetData = { + id: mediaAsset.id, + name: mediaAsset.name, + type: mediaAsset.type, + size: mediaAsset.file.size, + lastModified: mediaAsset.file.lastModified, + width: mediaAsset.width, + height: mediaAsset.height, + duration: mediaAsset.duration, + thumbnailUrl: mediaAsset.thumbnailUrl, + ephemeral: mediaAsset.ephemeral, + }; + + await mediaMetadataAdapter.set(mediaAsset.id, metadata); + } + + async loadMediaAsset({ + projectId, + id, + }: { + projectId: string; + id: string; + }): Promise<MediaAsset | null> { + const { mediaMetadataAdapter, mediaAssetsAdapter } = + this.getProjectMediaAdapters({ projectId }); + + const [file, metadata] = await Promise.all([ + mediaAssetsAdapter.get(id), + mediaMetadataAdapter.get(id), + ]); + + if (!file || !metadata) return null; + + let url: string; + if (metadata.type === "image" && (!file.type || file.type === "")) { + try { + const text = await file.text(); + if (text.trim().startsWith("<svg")) { + const svgBlob = new Blob([text], { type: "image/svg+xml" }); + url = URL.createObjectURL(svgBlob); + } else { + url = URL.createObjectURL(file); + } + } catch { + url = URL.createObjectURL(file); + } + } else { + url = URL.createObjectURL(file); + } + + return { + id: metadata.id, + name: metadata.name, + type: metadata.type, + file, + url, + width: metadata.width, + height: metadata.height, + duration: metadata.duration, + thumbnailUrl: metadata.thumbnailUrl, + ephemeral: metadata.ephemeral, + }; + } + + async loadAllMediaAssets({ + projectId, + }: { + projectId: string; + }): Promise<MediaAsset[]> { + const { mediaMetadataAdapter } = this.getProjectMediaAdapters({ + projectId, + }); + + const mediaIds = await mediaMetadataAdapter.list(); + const mediaItems: MediaAsset[] = []; + + for (const id of mediaIds) { + const item = await this.loadMediaAsset({ projectId, id }); + if (item) { + mediaItems.push(item); + } + } + + return mediaItems; + } + + async deleteMediaAsset({ + projectId, + id, + }: { + projectId: string; + id: string; + }): Promise<void> { + const { mediaMetadataAdapter, mediaAssetsAdapter } = + this.getProjectMediaAdapters({ projectId }); + + await Promise.all([ + mediaAssetsAdapter.remove(id), + mediaMetadataAdapter.remove(id), + ]); + } + + async deleteProjectMedia({ + projectId, + }: { + projectId: string; + }): Promise<void> { + const { mediaMetadataAdapter, mediaAssetsAdapter } = + this.getProjectMediaAdapters({ projectId }); + + await Promise.all([ + mediaMetadataAdapter.clear(), + mediaAssetsAdapter.clear(), + ]); + } + + // Utility methods + async clearAllData(): Promise<void> { + // Clear all projects + await this.projectsAdapter.clear(); + + // Note: Project-specific media and timelines will be cleaned up when projects are deleted + } + + async getStorageInfo(): Promise<{ + projects: number; + isOPFSSupported: boolean; + isIndexedDBSupported: boolean; + }> { + const projectIds = await this.projectsAdapter.list(); + + return { + projects: projectIds.length, + isOPFSSupported: this.isOPFSSupported(), + isIndexedDBSupported: this.isIndexedDBSupported(), + }; + } + + async getProjectStorageInfo({ projectId }: { projectId: string }): Promise<{ + mediaItems: number; + }> { + const { mediaMetadataAdapter } = this.getProjectMediaAdapters({ + projectId, + }); + + const mediaIds = await mediaMetadataAdapter.list(); + + return { + mediaItems: mediaIds.length, + }; + } + + async loadSavedSounds(): Promise<SavedSoundsData> { + try { + const savedSoundsData = await this.savedSoundsAdapter.get("user-sounds"); + return ( + savedSoundsData || { + sounds: [], + lastModified: new Date().toISOString(), + } + ); + } catch (error) { + console.error("Failed to load saved sounds:", error); + return { sounds: [], lastModified: new Date().toISOString() }; + } + } + + async saveSoundEffect({ + soundEffect, + }: { + soundEffect: SoundEffect; + }): Promise<void> { + try { + const currentData = await this.loadSavedSounds(); + + // Check if sound is already saved + if (currentData.sounds.some((sound) => sound.id === soundEffect.id)) { + return; // Already saved + } + + const savedSound: SavedSound = { + id: soundEffect.id, + name: soundEffect.name, + username: soundEffect.username, + previewUrl: soundEffect.previewUrl, + downloadUrl: soundEffect.downloadUrl, + duration: soundEffect.duration, + tags: soundEffect.tags, + license: soundEffect.license, + savedAt: new Date().toISOString(), + }; + + const updatedData: SavedSoundsData = { + sounds: [...currentData.sounds, savedSound], + lastModified: new Date().toISOString(), + }; + + await this.savedSoundsAdapter.set("user-sounds", updatedData); + } catch (error) { + console.error("Failed to save sound effect:", error); + throw error; + } + } + + async removeSavedSound({ soundId }: { soundId: number }): Promise<void> { + try { + const currentData = await this.loadSavedSounds(); + + const updatedData: SavedSoundsData = { + sounds: currentData.sounds.filter((sound) => sound.id !== soundId), + lastModified: new Date().toISOString(), + }; + + await this.savedSoundsAdapter.set("user-sounds", updatedData); + } catch (error) { + console.error("Failed to remove saved sound:", error); + throw error; + } + } + + async isSoundSaved({ soundId }: { soundId: number }): Promise<boolean> { + try { + const currentData = await this.loadSavedSounds(); + return currentData.sounds.some((sound) => sound.id === soundId); + } catch (error) { + console.error("Failed to check if sound is saved:", error); + return false; + } + } + + async clearSavedSounds(): Promise<void> { + try { + await this.savedSoundsAdapter.remove("user-sounds"); + } catch (error) { + console.error("Failed to clear saved sounds:", error); + throw error; + } + } + + // Check browser support + isOPFSSupported(): boolean { + return OPFSAdapter.isSupported(); + } + + isIndexedDBSupported(): boolean { + return "indexedDB" in window; + } + + isFullySupported(): boolean { + return this.isIndexedDBSupported() && this.isOPFSSupported(); + } +} + +// Export singleton instance +export const storageService = new StorageService(); +export { StorageService }; diff --git a/apps/web/src/services/storage/types.ts b/apps/web/src/services/storage/types.ts new file mode 100644 index 00000000..e7e045f5 --- /dev/null +++ b/apps/web/src/services/storage/types.ts @@ -0,0 +1,66 @@ +import type { MediaType } from "@/types/assets"; +import type { + TProject, + TProjectMetadata, + TTimelineViewState, +} from "@/types/project"; +import type { TScene } from "@/types/timeline"; + +export interface StorageAdapter<T> { + get(key: string): Promise<T | null>; + set(key: string, value: T): Promise<void>; + remove(key: string): Promise<void>; + list(): Promise<string[]>; + clear(): Promise<void>; +} + +export interface MediaAssetData { + id: string; + name: string; + type: MediaType; + size: number; + lastModified: number; + width?: number; + height?: number; + duration?: number; + fps?: number; + ephemeral?: boolean; + thumbnailUrl?: string; + sourceStickerIconName?: string; +} + +export type SerializedScene = Omit<TScene, "createdAt" | "updatedAt"> & { + createdAt: string; + updatedAt: string; +}; + +export type SerializedProjectMetadata = Omit< + TProjectMetadata, + "createdAt" | "updatedAt" +> & { + createdAt: string; + updatedAt: string; +}; + +export type SerializedProject = Omit<TProject, "metadata" | "scenes"> & { + metadata: SerializedProjectMetadata; + scenes: SerializedScene[]; + timelineViewState?: TTimelineViewState; +}; + +export interface StorageConfig { + projectsDb: string; + mediaDb: string; + savedSoundsDb: string; + version: number; +} + +// TypeScript type augmentation to add async iterator methods to FileSystemDirectoryHandle +// These methods are part of the File System Access API spec but may not be in all type definitions +declare global { + interface FileSystemDirectoryHandle { + keys(): AsyncIterableIterator<string>; + values(): AsyncIterableIterator<FileSystemHandle>; + entries(): AsyncIterableIterator<[string, FileSystemHandle]>; + } +} diff --git a/apps/web/src/services/transcription/index.ts b/apps/web/src/services/transcription/index.ts new file mode 100644 index 00000000..37c650da --- /dev/null +++ b/apps/web/src/services/transcription/index.ts @@ -0,0 +1,186 @@ +import type { + TranscriptionResult, + TranscriptionProgress, + TranscriptionModelId, +} from "@/types/transcription"; +import { + DEFAULT_TRANSCRIPTION_MODEL, + TRANSCRIPTION_MODELS, +} from "@/constants/transcription-constants"; +import type { WorkerMessage, WorkerResponse } from "./worker"; +import type { LanguageCode } from "@/types/language"; + +type ProgressCallback = (progress: TranscriptionProgress) => void; + +class TranscriptionService { + private worker: Worker | null = null; + private currentModelId: TranscriptionModelId | null = null; + private isInitialized = false; + private isInitializing = false; + + async transcribe({ + audioData, + language, + modelId = DEFAULT_TRANSCRIPTION_MODEL, + onProgress, + }: { + audioData: Float32Array; + language?: LanguageCode; + modelId?: TranscriptionModelId; + onProgress?: ProgressCallback; + }): Promise<TranscriptionResult> { + await this.ensureWorker({ modelId, onProgress }); + + return new Promise((resolve, reject) => { + if (!this.worker) { + reject(new Error("Worker not initialized")); + return; + } + + const handleMessage = (event: MessageEvent<WorkerResponse>) => { + const response = event.data; + + switch (response.type) { + case "transcribe-progress": + onProgress?.({ + status: "transcribing", + progress: response.progress, + message: "Transcribing audio...", + }); + break; + + case "transcribe-complete": + this.worker?.removeEventListener("message", handleMessage); + resolve({ + text: response.text, + segments: response.segments, + language, + }); + break; + + case "transcribe-error": + this.worker?.removeEventListener("message", handleMessage); + reject(new Error(response.error)); + break; + + case "cancelled": + this.worker?.removeEventListener("message", handleMessage); + reject(new Error("Transcription cancelled")); + break; + } + }; + + this.worker.addEventListener("message", handleMessage); + + this.worker.postMessage({ + type: "transcribe", + audio: audioData, + language, + } satisfies WorkerMessage); + }); + } + + cancel() { + this.worker?.postMessage({ type: "cancel" } satisfies WorkerMessage); + } + + private async ensureWorker({ + modelId, + onProgress, + }: { + modelId: TranscriptionModelId; + onProgress?: ProgressCallback; + }): Promise<void> { + const needsNewModel = this.currentModelId !== modelId; + + if (this.worker && this.isInitialized && !needsNewModel) { + return; + } + + if (this.isInitializing && !needsNewModel) { + await this.waitForInit(); + return; + } + + this.terminate(); + this.isInitializing = true; + this.isInitialized = false; + + const model = TRANSCRIPTION_MODELS.find((m) => m.id === modelId); + if (!model) { + throw new Error(`Unknown model: ${modelId}`); + } + + this.worker = new Worker(new URL("./worker.ts", import.meta.url), { + type: "module", + }); + + return new Promise((resolve, reject) => { + if (!this.worker) { + reject(new Error("Failed to create worker")); + return; + } + + const handleMessage = (event: MessageEvent<WorkerResponse>) => { + const response = event.data; + + switch (response.type) { + case "init-progress": + onProgress?.({ + status: "loading-model", + progress: response.progress, + message: `Loading ${model.name} model...`, + }); + break; + + case "init-complete": + this.worker?.removeEventListener("message", handleMessage); + this.isInitialized = true; + this.isInitializing = false; + this.currentModelId = modelId; + resolve(); + break; + + case "init-error": + this.worker?.removeEventListener("message", handleMessage); + this.isInitializing = false; + this.terminate(); + reject(new Error(response.error)); + break; + } + }; + + this.worker.addEventListener("message", handleMessage); + + this.worker.postMessage({ + type: "init", + modelId: model.huggingFaceId, + } satisfies WorkerMessage); + }); + } + + private waitForInit(): Promise<void> { + return new Promise((resolve) => { + const checkInit = () => { + if (this.isInitialized) { + resolve(); + } else if (!this.isInitializing) { + resolve(); + } else { + setTimeout(checkInit, 100); + } + }; + checkInit(); + }); + } + + terminate() { + this.worker?.terminate(); + this.worker = null; + this.isInitialized = false; + this.isInitializing = false; + this.currentModelId = null; + } +} + +export const transcriptionService = new TranscriptionService(); diff --git a/apps/web/src/services/transcription/worker.ts b/apps/web/src/services/transcription/worker.ts new file mode 100644 index 00000000..2aa9673c --- /dev/null +++ b/apps/web/src/services/transcription/worker.ts @@ -0,0 +1,176 @@ +import { + pipeline, + type AutomaticSpeechRecognitionPipeline, + type AutomaticSpeechRecognitionOutput, +} from "@huggingface/transformers"; +import type { TranscriptionSegment } from "@/types/transcription"; +import { + DEFAULT_CHUNK_LENGTH_SECONDS, + DEFAULT_STRIDE_SECONDS, +} from "@/constants/transcription-constants"; + +export type WorkerMessage = + | { type: "init"; modelId: string } + | { type: "transcribe"; audio: Float32Array; language: string } + | { type: "cancel" }; + +export type WorkerResponse = + | { type: "init-progress"; progress: number } + | { type: "init-complete" } + | { type: "init-error"; error: string } + | { type: "transcribe-progress"; progress: number } + | { + type: "transcribe-complete"; + text: string; + segments: TranscriptionSegment[]; + } + | { type: "transcribe-error"; error: string } + | { type: "cancelled" }; + +let transcriber: AutomaticSpeechRecognitionPipeline | null = null; +let cancelled = false; +let lastReportedProgress = -1; +const fileBytes = new Map<string, { loaded: number; total: number }>(); + +self.onmessage = async (event: MessageEvent<WorkerMessage>) => { + const message = event.data; + + switch (message.type) { + case "init": + await handleInit({ modelId: message.modelId }); + break; + case "transcribe": + await handleTranscribe({ + audio: message.audio, + language: message.language, + }); + break; + case "cancel": + cancelled = true; + self.postMessage({ type: "cancelled" } satisfies WorkerResponse); + break; + } +}; + +async function handleInit({ modelId }: { modelId: string }) { + lastReportedProgress = -1; + fileBytes.clear(); + + try { + transcriber = (await pipeline("automatic-speech-recognition", modelId, { + dtype: "q4", + device: "auto", + progress_callback: (progressInfo: { + status?: string; + file?: string; + loaded?: number; + total?: number; + }) => { + const file = progressInfo.file; + if (!file) return; + + const loaded = progressInfo.loaded ?? 0; + const total = progressInfo.total ?? 0; + + if (progressInfo.status === "progress" && total > 0) { + fileBytes.set(file, { loaded, total }); + } else if (progressInfo.status === "done") { + const existing = fileBytes.get(file); + if (existing) { + fileBytes.set(file, { + loaded: existing.total, + total: existing.total, + }); + } + } + + // sum all bytes + let totalLoaded = 0; + let totalSize = 0; + for (const { loaded, total } of fileBytes.values()) { + totalLoaded += loaded; + totalSize += total; + } + + if (totalSize === 0) return; + + const overallProgress = (totalLoaded / totalSize) * 100; + const roundedProgress = Math.floor(overallProgress); + + if (roundedProgress !== lastReportedProgress) { + lastReportedProgress = roundedProgress; + self.postMessage({ + type: "init-progress", + progress: roundedProgress, + } satisfies WorkerResponse); + } + }, + })) as unknown as AutomaticSpeechRecognitionPipeline; + + self.postMessage({ type: "init-complete" } satisfies WorkerResponse); + } catch (error) { + self.postMessage({ + type: "init-error", + error: error instanceof Error ? error.message : "Failed to load model", + } satisfies WorkerResponse); + } +} + +async function handleTranscribe({ + audio, + language, +}: { + audio: Float32Array; + language: string; +}) { + if (!transcriber) { + self.postMessage({ + type: "transcribe-error", + error: "Model not initialized", + } satisfies WorkerResponse); + return; + } + + cancelled = false; + + try { + const rawResult = await transcriber(audio, { + chunk_length_s: DEFAULT_CHUNK_LENGTH_SECONDS, + stride_length_s: DEFAULT_STRIDE_SECONDS, + language: language === "auto" ? undefined : language, + return_timestamps: true, + }); + + if (cancelled) return; + + const result: AutomaticSpeechRecognitionOutput = Array.isArray(rawResult) + ? rawResult[0] + : rawResult; + + const segments: TranscriptionSegment[] = []; + + if (result.chunks) { + for (const chunk of result.chunks) { + if (chunk.timestamp && chunk.timestamp.length >= 2) { + segments.push({ + text: chunk.text, + start: chunk.timestamp[0] ?? 0, + end: chunk.timestamp[1] ?? chunk.timestamp[0] ?? 0, + }); + } + } + } + + self.postMessage({ + type: "transcribe-complete", + text: result.text, + segments, + } satisfies WorkerResponse); + } catch (error) { + if (cancelled) return; + self.postMessage({ + type: "transcribe-error", + error: error instanceof Error ? error.message : "Transcription failed", + } satisfies WorkerResponse); + } +} |
