diff options
| author | cashmere <cashmere@autistici.org> | 2026-07-27 01:21:20 +0200 |
|---|---|---|
| committer | cashmere <cashmere@autistici.org> | 2026-07-27 01:21:20 +0200 |
| commit | 783e9ac8462751adc14d78b070095fd63e45bda0 (patch) | |
| tree | 9e28a0937b9155c6b791af836c973f39ced9d541 /apps/web/src | |
| parent | 30a36e45686db503fded68e4da5d4c465a06f978 (diff) | |
| download | opencut-783e9ac8462751adc14d78b070095fd63e45bda0.tar.gz | |
feat: voice-over recording into the timeline
MediaRecorder mic capture (defensive mimeType picking, permission/no-mic
toasts) with elapsed display in the timeline toolbar. Stop produces an
audio asset and inserts it at the playhead on an audio track as ONE
BatchCommand undo step, freeze-frame precedent. voice-over action
registered, no default keybinding. 20 tests; smoke-tested in Chromium
with a fake mic end to end.
Diffstat (limited to 'apps/web/src')
| -rw-r--r-- | apps/web/src/timeline/components/timeline-toolbar.tsx | 26 | ||||
| -rw-r--r-- | apps/web/src/voiceover/__tests__/recorder.test.ts | 126 | ||||
| -rw-r--r-- | apps/web/src/voiceover/__tests__/voice-over-command.test.ts | 187 | ||||
| -rw-r--r-- | apps/web/src/voiceover/recorder.ts | 186 | ||||
| -rw-r--r-- | apps/web/src/voiceover/use-recording-elapsed.ts | 26 | ||||
| -rw-r--r-- | apps/web/src/voiceover/voiceover-store.ts | 41 |
6 files changed, 592 insertions, 0 deletions
diff --git a/apps/web/src/timeline/components/timeline-toolbar.tsx b/apps/web/src/timeline/components/timeline-toolbar.tsx index 6208ef27..47906931 100644 --- a/apps/web/src/timeline/components/timeline-toolbar.tsx +++ b/apps/web/src/timeline/components/timeline-toolbar.tsx @@ -43,12 +43,17 @@ import { Layers01Icon, Chart03Icon, Unlink02Icon, + MicIcon, + StopIcon, } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { OcRippleIcon } from "@/components/icons"; import { GraphEditorPopover } from "./graph-editor/popover"; import { PopoverTrigger } from "@/components/ui/popover"; import { useGraphEditorController } from "./graph-editor/use-controller"; +import { formatElapsedSeconds } from "@/voiceover/recorder"; +import { useRecordingElapsedSeconds } from "@/voiceover/use-recording-elapsed"; +import { useVoiceOverStore } from "@/voiceover/voiceover-store"; export function TimelineToolbar({ zoomLevel, @@ -95,6 +100,10 @@ function ToolbarLeftSection() { const isCurrentlyBookmarked = useEditor((e) => e.scenes.isBookmarked({ time: e.playback.getCurrentTime() }), ); + const isRecordingVoiceOver = useVoiceOverStore( + (state) => state.status === "recording", + ); + const voiceOverElapsedSeconds = useRecordingElapsedSeconds(); const canFreezeFrame = useEditor((e) => { const time = e.playback.getCurrentTime(); const { overlay, main } = e.scenes.getActiveScene().tracks; @@ -203,6 +212,23 @@ function ToolbarLeftSection() { /> <ToolbarButton + icon={ + <HugeiconsIcon icon={isRecordingVoiceOver ? StopIcon : MicIcon} /> + } + tooltip={ + isRecordingVoiceOver ? "Stop voice-over" : "Record voice-over" + } + isActive={isRecordingVoiceOver} + onClick={({ event }) => handleAction({ action: "voice-over", event })} + /> + + {isRecordingVoiceOver && ( + <span className="text-destructive min-w-9 text-center text-xs tabular-nums"> + {formatElapsedSeconds({ seconds: voiceOverElapsedSeconds })} + </span> + )} + + <ToolbarButton icon={<HugeiconsIcon icon={Delete02Icon} />} tooltip="Delete element" onClick={({ event }) => diff --git a/apps/web/src/voiceover/__tests__/recorder.test.ts b/apps/web/src/voiceover/__tests__/recorder.test.ts new file mode 100644 index 00000000..1cc89f4c --- /dev/null +++ b/apps/web/src/voiceover/__tests__/recorder.test.ts @@ -0,0 +1,126 @@ +import { describe, expect, test } from "bun:test"; +import { + formatElapsedSeconds, + getFileExtensionForMimeType, + getVoiceOverErrorMessage, + pickSupportedMimeType, + PREFERRED_MIME_TYPES, +} from "@/voiceover/recorder"; + +describe("pickSupportedMimeType", () => { + test("prefers webm/opus when everything is supported", () => { + const mimeType = pickSupportedMimeType({ + isTypeSupported: () => true, + }); + expect(mimeType).toBe("audio/webm;codecs=opus"); + }); + + test("falls back to the first supported candidate", () => { + const mimeType = pickSupportedMimeType({ + isTypeSupported: (candidate) => candidate === "audio/mp4", + }); + expect(mimeType).toBe("audio/mp4"); + }); + + test("returns null when nothing is supported", () => { + const mimeType = pickSupportedMimeType({ + isTypeSupported: () => false, + }); + expect(mimeType).toBeNull(); + }); + + test("respects custom candidate order", () => { + const mimeType = pickSupportedMimeType({ + isTypeSupported: () => true, + candidates: ["audio/ogg", "audio/webm"], + }); + expect(mimeType).toBe("audio/ogg"); + }); + + test("default candidates put webm before mp4", () => { + expect(PREFERRED_MIME_TYPES.indexOf("audio/webm;codecs=opus")).toBeLessThan( + PREFERRED_MIME_TYPES.indexOf("audio/mp4"), + ); + }); +}); + +describe("getFileExtensionForMimeType", () => { + test("maps known audio mime types", () => { + expect(getFileExtensionForMimeType({ mimeType: "audio/webm" })).toBe( + "webm", + ); + expect( + getFileExtensionForMimeType({ mimeType: "audio/webm;codecs=opus" }), + ).toBe("webm"); + expect(getFileExtensionForMimeType({ mimeType: "audio/ogg" })).toBe("ogg"); + expect(getFileExtensionForMimeType({ mimeType: "audio/mp4" })).toBe("m4a"); + expect(getFileExtensionForMimeType({ mimeType: "audio/mpeg" })).toBe( + "mp3", + ); + }); + + test("falls back to a generic extension for unknown types", () => { + expect(getFileExtensionForMimeType({ mimeType: "audio/flac" })).toBe( + "audio", + ); + expect(getFileExtensionForMimeType({ mimeType: "" })).toBe("audio"); + }); +}); + +describe("formatElapsedSeconds", () => { + test("formats zero and sub-second values", () => { + expect(formatElapsedSeconds({ seconds: 0 })).toBe("0:00"); + expect(formatElapsedSeconds({ seconds: 0.9 })).toBe("0:00"); + }); + + test("pads seconds", () => { + expect(formatElapsedSeconds({ seconds: 9.6 })).toBe("0:09"); + }); + + test("formats minutes", () => { + expect(formatElapsedSeconds({ seconds: 65 })).toBe("1:05"); + expect(formatElapsedSeconds({ seconds: 600 })).toBe("10:00"); + }); + + test("clamps negative values", () => { + expect(formatElapsedSeconds({ seconds: -3 })).toBe("0:00"); + }); +}); + +describe("getVoiceOverErrorMessage", () => { + test("explains denied permission", () => { + const message = getVoiceOverErrorMessage({ + error: new DOMException("denied", "NotAllowedError"), + }); + expect(message).toContain("Microphone access denied"); + }); + + test("explains a missing microphone", () => { + const message = getVoiceOverErrorMessage({ + error: new DOMException("missing", "NotFoundError"), + }); + expect(message).toContain("No microphone found"); + }); + + test("explains a busy microphone", () => { + const message = getVoiceOverErrorMessage({ + error: new DOMException("busy", "NotReadableError"), + }); + expect(message).toContain("already in use"); + }); + + test("surfaces generic error messages", () => { + const message = getVoiceOverErrorMessage({ + error: new Error("Voice-over recording is not supported in this browser"), + }); + expect(message).toBe( + "Voice-over recording is not supported in this browser", + ); + }); + + test("handles non-error values", () => { + expect(getVoiceOverErrorMessage({ error: null })).toBe( + "Could not start voice-over recording.", + ); + }); +}); diff --git a/apps/web/src/voiceover/__tests__/voice-over-command.test.ts b/apps/web/src/voiceover/__tests__/voice-over-command.test.ts new file mode 100644 index 00000000..4ba2b3fd --- /dev/null +++ b/apps/web/src/voiceover/__tests__/voice-over-command.test.ts @@ -0,0 +1,187 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { EditorCore } from "@/core"; +import { + AddMediaAssetCommand, + BatchCommand, + InsertElementCommand, +} from "@/commands"; +import { buildDefaultScene } from "@/timeline/scenes"; +import { buildElementFromMedia } from "@/timeline"; +import { mediaTimeFromSeconds, ZERO_MEDIA_TIME, type MediaTime } from "@/wasm"; +import { storageService } from "@/services/storage/service"; + +const RECORDING_DURATION_SECONDS = 12.5; +const RECORDING_DURATION = mediaTimeFromSeconds({ + seconds: RECORDING_DURATION_SECONDS, +}); +const PLAYHEAD_TIME = mediaTimeFromSeconds({ seconds: 2 }); + +function seedEditorWithEmptyScene(): EditorCore { + const editor = EditorCore.getInstance(); + const scene = buildDefaultScene({ name: "Main scene", isMain: true }); + editor.project.setActiveProject({ + project: { + metadata: { + id: "project-1", + name: "Test project", + duration: ZERO_MEDIA_TIME, + createdAt: new Date("2026-01-01T00:00:00Z"), + updatedAt: new Date("2026-01-01T00:00:00Z"), + }, + scenes: [scene], + currentSceneId: scene.id, + settings: { + fps: { numerator: 30, denominator: 1 }, + canvasSize: { width: 1920, height: 1080 }, + background: { type: "color", color: "#000000" }, + }, + version: 1, + }, + }); + editor.scenes.setScenes({ scenes: [scene], activeSceneId: scene.id }); + editor.media.setAssets({ assets: [] }); + return editor; +} + +/** + * Mirrors the stop-recording branch of the "voice-over" action handler in + * use-editor-actions.ts: one undoable batch that adds the recorded audio as a + * media asset and inserts it at the playhead with automatic audio placement. + */ +function buildVoiceOverBatch({ + projectId, + startTime, +}: { + projectId: string; + startTime: MediaTime; +}): { batch: BatchCommand; assetId: string } { + const file = new File([new Blob(["fake-audio"])], "voice-over.webm", { + type: "audio/webm", + }); + const addMediaCommand = new AddMediaAssetCommand({ + projectId, + asset: { + file, + name: "Voice over", + type: "audio", + url: "blob:voice-over-test", + duration: RECORDING_DURATION_SECONDS, + }, + }); + const insertCommand = new InsertElementCommand({ + element: buildElementFromMedia({ + mediaId: addMediaCommand.getAssetId(), + mediaType: "audio", + name: "Voice over", + duration: RECORDING_DURATION, + startTime, + }), + placement: { mode: "auto", trackType: "audio" }, + }); + return { + batch: new BatchCommand([addMediaCommand, insertCommand]), + assetId: addMediaCommand.getAssetId(), + }; +} + +describe("voice-over batch command", () => { + beforeEach(() => { + EditorCore.reset(); + // IndexedDB is unavailable under bun:test; the command treats a failed + // save as fatal and rolls back the asset, so stub persistence out. + storageService.saveMediaAsset = () => Promise.resolve(); + storageService.deleteMediaAsset = () => Promise.resolve(); + }); + + test("adds an audio asset and inserts it on a new audio track at the playhead", () => { + const editor = seedEditorWithEmptyScene(); + const { batch, assetId } = buildVoiceOverBatch({ + projectId: "project-1", + startTime: PLAYHEAD_TIME, + }); + + batch.execute(); + + const assets = editor.media.getAssets(); + expect(assets).toHaveLength(1); + expect(assets[0].id).toBe(assetId); + expect(assets[0].type).toBe("audio"); + expect(assets[0].name).toBe("Voice over"); + expect(assets[0].duration).toBe(RECORDING_DURATION_SECONDS); + + const { tracks } = editor.scenes.getActiveScene(); + expect(tracks.audio).toHaveLength(1); + expect(tracks.audio[0].elements).toHaveLength(1); + const element = tracks.audio[0].elements[0]; + expect(element.type).toBe("audio"); + expect(element.startTime).toBe(PLAYHEAD_TIME); + expect(element.duration).toBe(RECORDING_DURATION); + if (element.type === "audio" && element.sourceType === "upload") { + expect(element.mediaId).toBe(assetId); + } else { + throw new Error("expected an upload audio element"); + } + }); + + test("undo removes both the element and the asset as a single step", () => { + const editor = seedEditorWithEmptyScene(); + const { batch } = buildVoiceOverBatch({ + projectId: "project-1", + startTime: PLAYHEAD_TIME, + }); + + batch.execute(); + batch.undo(); + + expect(editor.media.getAssets()).toHaveLength(0); + const { tracks } = editor.scenes.getActiveScene(); + expect( + tracks.audio.flatMap((track) => track.elements), + ).toHaveLength(0); + }); + + test("redo re-inserts the element and restores the asset", () => { + const editor = seedEditorWithEmptyScene(); + const { batch, assetId } = buildVoiceOverBatch({ + projectId: "project-1", + startTime: PLAYHEAD_TIME, + }); + + batch.execute(); + batch.undo(); + batch.redo(); + + expect(editor.media.getAssets().map((asset) => asset.id)).toEqual([ + assetId, + ]); + const { tracks } = editor.scenes.getActiveScene(); + const elements = tracks.audio.flatMap((track) => track.elements); + expect(elements).toHaveLength(1); + expect(elements[0].startTime).toBe(PLAYHEAD_TIME); + }); + + test("reuses an existing audio track when the slot is free", () => { + const editor = seedEditorWithEmptyScene(); + const first = buildVoiceOverBatch({ + projectId: "project-1", + startTime: PLAYHEAD_TIME, + }); + first.batch.execute(); + + const secondStart = mediaTimeFromSeconds({ + seconds: 2 + RECORDING_DURATION_SECONDS + 1, + }); + const second = buildVoiceOverBatch({ + projectId: "project-1", + startTime: secondStart, + }); + second.batch.execute(); + + const { tracks } = editor.scenes.getActiveScene(); + expect(tracks.audio).toHaveLength(1); + expect(tracks.audio[0].elements).toHaveLength(2); + expect( + tracks.audio[0].elements.map((element) => element.startTime), + ).toEqual([PLAYHEAD_TIME, secondStart]); + }); +}); diff --git a/apps/web/src/voiceover/recorder.ts b/apps/web/src/voiceover/recorder.ts new file mode 100644 index 00000000..b71ca74f --- /dev/null +++ b/apps/web/src/voiceover/recorder.ts @@ -0,0 +1,186 @@ +/** + * Voice-over recording via MediaRecorder. + * Pure helpers (mime type picking, file extension, elapsed formatting, error + * messages) are exported for unit tests; the recorder singleton below owns the + * actual microphone stream. + */ + +export const PREFERRED_MIME_TYPES = [ + "audio/webm;codecs=opus", + "audio/webm", + "audio/ogg;codecs=opus", + "audio/ogg", + "audio/mp4", +] as const; + +export function pickSupportedMimeType({ + isTypeSupported, + candidates = PREFERRED_MIME_TYPES, +}: { + isTypeSupported: (mimeType: string) => boolean; + candidates?: readonly string[]; +}): string | null { + for (const mimeType of candidates) { + if (isTypeSupported(mimeType)) { + return mimeType; + } + } + return null; +} + +const EXTENSION_BY_MIME_SUBTYPE: Record<string, string> = { + webm: "webm", + ogg: "ogg", + mp4: "m4a", + mpeg: "mp3", + wav: "wav", +}; + +export function getFileExtensionForMimeType({ + mimeType, +}: { + mimeType: string; +}): string { + const subtype = mimeType.split("/").at(1)?.split(";").at(0)?.trim() ?? ""; + return EXTENSION_BY_MIME_SUBTYPE[subtype] ?? "audio"; +} + +export function formatElapsedSeconds({ + seconds, +}: { + seconds: number; +}): string { + const totalSeconds = Math.max(0, Math.floor(seconds)); + const minutes = Math.floor(totalSeconds / 60); + const remainder = totalSeconds % 60; + return `${minutes}:${String(remainder).padStart(2, "0")}`; +} + +export function getVoiceOverErrorMessage({ + error, +}: { + error: unknown; +}): string { + if (error instanceof DOMException) { + switch (error.name) { + case "NotAllowedError": + case "SecurityError": + return "Microphone access denied. Allow microphone access in your browser settings to record a voice-over."; + case "NotFoundError": + case "OverconstrainedError": + return "No microphone found. Connect a microphone and try again."; + case "NotReadableError": + return "The microphone is already in use by another application."; + default: + return "Could not start voice-over recording."; + } + } + if (error instanceof Error && error.message) { + return error.message; + } + return "Could not start voice-over recording."; +} + +export interface VoiceOverRecordingResult { + blob: Blob; + mimeType: string; + durationSeconds: number; +} + +interface ActiveRecording { + recorder: MediaRecorder; + stream: MediaStream; + chunks: Blob[]; + mimeType: string; + startedAtMs: number; +} + +let activeRecording: ActiveRecording | null = null; + +function stopStreamTracks({ stream }: { stream: MediaStream }): void { + for (const track of stream.getTracks()) { + track.stop(); + } +} + +export async function startVoiceOverRecording(): Promise<void> { + if (activeRecording) { + throw new Error("A voice-over recording is already in progress"); + } + + const mediaDevices = navigator.mediaDevices; + if (!mediaDevices?.getUserMedia) { + throw new Error("Voice-over recording is not supported in this browser"); + } + + const stream = await mediaDevices.getUserMedia({ audio: true }); + try { + const mimeType = pickSupportedMimeType({ + isTypeSupported: (candidate) => MediaRecorder.isTypeSupported(candidate), + }); + const recorder = mimeType + ? new MediaRecorder(stream, { mimeType }) + : new MediaRecorder(stream); + const chunks: Blob[] = []; + recorder.addEventListener("dataavailable", (event) => { + if (event.data.size > 0) { + chunks.push(event.data); + } + }); + recorder.start(); + activeRecording = { + recorder, + stream, + chunks, + mimeType: recorder.mimeType || mimeType || "audio/webm", + startedAtMs: performance.now(), + }; + } catch (error) { + stopStreamTracks({ stream }); + throw error; + } +} + +export function stopVoiceOverRecording(): Promise<VoiceOverRecordingResult> { + const recording = activeRecording; + if (!recording) { + return Promise.reject( + new Error("No voice-over recording is in progress"), + ); + } + activeRecording = null; + + const { promise, resolve, reject } = + Promise.withResolvers<VoiceOverRecordingResult>(); + recording.recorder.addEventListener("stop", () => { + stopStreamTracks({ stream: recording.stream }); + resolve({ + blob: new Blob(recording.chunks, { type: recording.mimeType }), + mimeType: recording.mimeType, + durationSeconds: Math.max( + 0, + (performance.now() - recording.startedAtMs) / 1000, + ), + }); + }); + recording.recorder.addEventListener("error", () => { + stopStreamTracks({ stream: recording.stream }); + reject(new Error("Voice-over recording failed")); + }); + recording.recorder.stop(); + return promise; +} + +export function cancelVoiceOverRecording(): void { + const recording = activeRecording; + if (!recording) { + return; + } + activeRecording = null; + try { + recording.recorder.stop(); + } catch { + // recorder may already be stopped; tracks are released either way + } + stopStreamTracks({ stream: recording.stream }); +} diff --git a/apps/web/src/voiceover/use-recording-elapsed.ts b/apps/web/src/voiceover/use-recording-elapsed.ts new file mode 100644 index 00000000..1eeb7f90 --- /dev/null +++ b/apps/web/src/voiceover/use-recording-elapsed.ts @@ -0,0 +1,26 @@ +import { useEffect, useState } from "react"; +import { useVoiceOverStore } from "@/voiceover/voiceover-store"; + +const ELAPSED_TICK_MS = 250; + +/** + * Elapsed seconds of the active voice-over recording, ticking while recording. + * Returns 0 when idle. + */ +export function useRecordingElapsedSeconds(): number { + const status = useVoiceOverStore((state) => state.status); + const startedAtMs = useVoiceOverStore((state) => state.startedAtMs); + const [elapsedSeconds, setElapsedSeconds] = useState(0); + + useEffect(() => { + if (status !== "recording" || startedAtMs === null) { + return; + } + const interval = setInterval(() => { + setElapsedSeconds((Date.now() - startedAtMs) / 1000); + }, ELAPSED_TICK_MS); + return () => clearInterval(interval); + }, [status, startedAtMs]); + + return status === "recording" ? elapsedSeconds : 0; +} diff --git a/apps/web/src/voiceover/voiceover-store.ts b/apps/web/src/voiceover/voiceover-store.ts new file mode 100644 index 00000000..44ccae6c --- /dev/null +++ b/apps/web/src/voiceover/voiceover-store.ts @@ -0,0 +1,41 @@ +/** + * Voice-over recording UI state. + * Transient by nature — a recording never survives a reload, so unlike most + * stores here this one is intentionally NOT persisted (same pattern as + * editor-store / properties-store). + */ + +import { create } from "zustand"; +import type { MediaTime } from "@/wasm"; + +export type VoiceOverStatus = "idle" | "recording"; + +interface VoiceOverStore { + status: VoiceOverStatus; + startedAtMs: number | null; + playheadStartTime: MediaTime | null; + startRecording: ({ playheadTime }: { playheadTime: MediaTime }) => void; + stopRecording: () => void; +} + +export const useVoiceOverStore = create<VoiceOverStore>()((set) => ({ + status: "idle", + startedAtMs: null, + playheadStartTime: null, + + startRecording: ({ playheadTime }) => { + set({ + status: "recording", + startedAtMs: Date.now(), + playheadStartTime: playheadTime, + }); + }, + + stopRecording: () => { + set({ + status: "idle", + startedAtMs: null, + playheadStartTime: null, + }); + }, +})); |
