summaryrefslogtreecommitdiff
path: root/apps/web/src
diff options
context:
space:
mode:
authorcashmere <cashmere@autistici.org>2026-07-27 01:21:20 +0200
committercashmere <cashmere@autistici.org>2026-07-27 01:21:20 +0200
commit783e9ac8462751adc14d78b070095fd63e45bda0 (patch)
tree9e28a0937b9155c6b791af836c973f39ced9d541 /apps/web/src
parent30a36e45686db503fded68e4da5d4c465a06f978 (diff)
downloadopencut-783e9ac8462751adc14d78b070095fd63e45bda0.tar.gz
feat: voice-over recording into the timeline
MediaRecorder mic capture (defensive mimeType picking, permission/no-mic toasts) with elapsed display in the timeline toolbar. Stop produces an audio asset and inserts it at the playhead on an audio track as ONE BatchCommand undo step, freeze-frame precedent. voice-over action registered, no default keybinding. 20 tests; smoke-tested in Chromium with a fake mic end to end.
Diffstat (limited to 'apps/web/src')
-rw-r--r--apps/web/src/timeline/components/timeline-toolbar.tsx26
-rw-r--r--apps/web/src/voiceover/__tests__/recorder.test.ts126
-rw-r--r--apps/web/src/voiceover/__tests__/voice-over-command.test.ts187
-rw-r--r--apps/web/src/voiceover/recorder.ts186
-rw-r--r--apps/web/src/voiceover/use-recording-elapsed.ts26
-rw-r--r--apps/web/src/voiceover/voiceover-store.ts41
6 files changed, 592 insertions, 0 deletions
diff --git a/apps/web/src/timeline/components/timeline-toolbar.tsx b/apps/web/src/timeline/components/timeline-toolbar.tsx
index 6208ef27..47906931 100644
--- a/apps/web/src/timeline/components/timeline-toolbar.tsx
+++ b/apps/web/src/timeline/components/timeline-toolbar.tsx
@@ -43,12 +43,17 @@ import {
Layers01Icon,
Chart03Icon,
Unlink02Icon,
+ MicIcon,
+ StopIcon,
} from "@hugeicons/core-free-icons";
import { HugeiconsIcon } from "@hugeicons/react";
import { OcRippleIcon } from "@/components/icons";
import { GraphEditorPopover } from "./graph-editor/popover";
import { PopoverTrigger } from "@/components/ui/popover";
import { useGraphEditorController } from "./graph-editor/use-controller";
+import { formatElapsedSeconds } from "@/voiceover/recorder";
+import { useRecordingElapsedSeconds } from "@/voiceover/use-recording-elapsed";
+import { useVoiceOverStore } from "@/voiceover/voiceover-store";
export function TimelineToolbar({
zoomLevel,
@@ -95,6 +100,10 @@ function ToolbarLeftSection() {
const isCurrentlyBookmarked = useEditor((e) =>
e.scenes.isBookmarked({ time: e.playback.getCurrentTime() }),
);
+ const isRecordingVoiceOver = useVoiceOverStore(
+ (state) => state.status === "recording",
+ );
+ const voiceOverElapsedSeconds = useRecordingElapsedSeconds();
const canFreezeFrame = useEditor((e) => {
const time = e.playback.getCurrentTime();
const { overlay, main } = e.scenes.getActiveScene().tracks;
@@ -203,6 +212,23 @@ function ToolbarLeftSection() {
/>
<ToolbarButton
+ icon={
+ <HugeiconsIcon icon={isRecordingVoiceOver ? StopIcon : MicIcon} />
+ }
+ tooltip={
+ isRecordingVoiceOver ? "Stop voice-over" : "Record voice-over"
+ }
+ isActive={isRecordingVoiceOver}
+ onClick={({ event }) => handleAction({ action: "voice-over", event })}
+ />
+
+ {isRecordingVoiceOver && (
+ <span className="text-destructive min-w-9 text-center text-xs tabular-nums">
+ {formatElapsedSeconds({ seconds: voiceOverElapsedSeconds })}
+ </span>
+ )}
+
+ <ToolbarButton
icon={<HugeiconsIcon icon={Delete02Icon} />}
tooltip="Delete element"
onClick={({ event }) =>
diff --git a/apps/web/src/voiceover/__tests__/recorder.test.ts b/apps/web/src/voiceover/__tests__/recorder.test.ts
new file mode 100644
index 00000000..1cc89f4c
--- /dev/null
+++ b/apps/web/src/voiceover/__tests__/recorder.test.ts
@@ -0,0 +1,126 @@
+import { describe, expect, test } from "bun:test";
+import {
+ formatElapsedSeconds,
+ getFileExtensionForMimeType,
+ getVoiceOverErrorMessage,
+ pickSupportedMimeType,
+ PREFERRED_MIME_TYPES,
+} from "@/voiceover/recorder";
+
+describe("pickSupportedMimeType", () => {
+ test("prefers webm/opus when everything is supported", () => {
+ const mimeType = pickSupportedMimeType({
+ isTypeSupported: () => true,
+ });
+ expect(mimeType).toBe("audio/webm;codecs=opus");
+ });
+
+ test("falls back to the first supported candidate", () => {
+ const mimeType = pickSupportedMimeType({
+ isTypeSupported: (candidate) => candidate === "audio/mp4",
+ });
+ expect(mimeType).toBe("audio/mp4");
+ });
+
+ test("returns null when nothing is supported", () => {
+ const mimeType = pickSupportedMimeType({
+ isTypeSupported: () => false,
+ });
+ expect(mimeType).toBeNull();
+ });
+
+ test("respects custom candidate order", () => {
+ const mimeType = pickSupportedMimeType({
+ isTypeSupported: () => true,
+ candidates: ["audio/ogg", "audio/webm"],
+ });
+ expect(mimeType).toBe("audio/ogg");
+ });
+
+ test("default candidates put webm before mp4", () => {
+ expect(PREFERRED_MIME_TYPES.indexOf("audio/webm;codecs=opus")).toBeLessThan(
+ PREFERRED_MIME_TYPES.indexOf("audio/mp4"),
+ );
+ });
+});
+
+describe("getFileExtensionForMimeType", () => {
+ test("maps known audio mime types", () => {
+ expect(getFileExtensionForMimeType({ mimeType: "audio/webm" })).toBe(
+ "webm",
+ );
+ expect(
+ getFileExtensionForMimeType({ mimeType: "audio/webm;codecs=opus" }),
+ ).toBe("webm");
+ expect(getFileExtensionForMimeType({ mimeType: "audio/ogg" })).toBe("ogg");
+ expect(getFileExtensionForMimeType({ mimeType: "audio/mp4" })).toBe("m4a");
+ expect(getFileExtensionForMimeType({ mimeType: "audio/mpeg" })).toBe(
+ "mp3",
+ );
+ });
+
+ test("falls back to a generic extension for unknown types", () => {
+ expect(getFileExtensionForMimeType({ mimeType: "audio/flac" })).toBe(
+ "audio",
+ );
+ expect(getFileExtensionForMimeType({ mimeType: "" })).toBe("audio");
+ });
+});
+
+describe("formatElapsedSeconds", () => {
+ test("formats zero and sub-second values", () => {
+ expect(formatElapsedSeconds({ seconds: 0 })).toBe("0:00");
+ expect(formatElapsedSeconds({ seconds: 0.9 })).toBe("0:00");
+ });
+
+ test("pads seconds", () => {
+ expect(formatElapsedSeconds({ seconds: 9.6 })).toBe("0:09");
+ });
+
+ test("formats minutes", () => {
+ expect(formatElapsedSeconds({ seconds: 65 })).toBe("1:05");
+ expect(formatElapsedSeconds({ seconds: 600 })).toBe("10:00");
+ });
+
+ test("clamps negative values", () => {
+ expect(formatElapsedSeconds({ seconds: -3 })).toBe("0:00");
+ });
+});
+
+describe("getVoiceOverErrorMessage", () => {
+ test("explains denied permission", () => {
+ const message = getVoiceOverErrorMessage({
+ error: new DOMException("denied", "NotAllowedError"),
+ });
+ expect(message).toContain("Microphone access denied");
+ });
+
+ test("explains a missing microphone", () => {
+ const message = getVoiceOverErrorMessage({
+ error: new DOMException("missing", "NotFoundError"),
+ });
+ expect(message).toContain("No microphone found");
+ });
+
+ test("explains a busy microphone", () => {
+ const message = getVoiceOverErrorMessage({
+ error: new DOMException("busy", "NotReadableError"),
+ });
+ expect(message).toContain("already in use");
+ });
+
+ test("surfaces generic error messages", () => {
+ const message = getVoiceOverErrorMessage({
+ error: new Error("Voice-over recording is not supported in this browser"),
+ });
+ expect(message).toBe(
+ "Voice-over recording is not supported in this browser",
+ );
+ });
+
+ test("handles non-error values", () => {
+ expect(getVoiceOverErrorMessage({ error: null })).toBe(
+ "Could not start voice-over recording.",
+ );
+ });
+});
diff --git a/apps/web/src/voiceover/__tests__/voice-over-command.test.ts b/apps/web/src/voiceover/__tests__/voice-over-command.test.ts
new file mode 100644
index 00000000..4ba2b3fd
--- /dev/null
+++ b/apps/web/src/voiceover/__tests__/voice-over-command.test.ts
@@ -0,0 +1,187 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+import { EditorCore } from "@/core";
+import {
+ AddMediaAssetCommand,
+ BatchCommand,
+ InsertElementCommand,
+} from "@/commands";
+import { buildDefaultScene } from "@/timeline/scenes";
+import { buildElementFromMedia } from "@/timeline";
+import { mediaTimeFromSeconds, ZERO_MEDIA_TIME, type MediaTime } from "@/wasm";
+import { storageService } from "@/services/storage/service";
+
+const RECORDING_DURATION_SECONDS = 12.5;
+const RECORDING_DURATION = mediaTimeFromSeconds({
+ seconds: RECORDING_DURATION_SECONDS,
+});
+const PLAYHEAD_TIME = mediaTimeFromSeconds({ seconds: 2 });
+
+function seedEditorWithEmptyScene(): EditorCore {
+ const editor = EditorCore.getInstance();
+ const scene = buildDefaultScene({ name: "Main scene", isMain: true });
+ editor.project.setActiveProject({
+ project: {
+ metadata: {
+ id: "project-1",
+ name: "Test project",
+ duration: ZERO_MEDIA_TIME,
+ createdAt: new Date("2026-01-01T00:00:00Z"),
+ updatedAt: new Date("2026-01-01T00:00:00Z"),
+ },
+ scenes: [scene],
+ currentSceneId: scene.id,
+ settings: {
+ fps: { numerator: 30, denominator: 1 },
+ canvasSize: { width: 1920, height: 1080 },
+ background: { type: "color", color: "#000000" },
+ },
+ version: 1,
+ },
+ });
+ editor.scenes.setScenes({ scenes: [scene], activeSceneId: scene.id });
+ editor.media.setAssets({ assets: [] });
+ return editor;
+}
+
+/**
+ * Mirrors the stop-recording branch of the "voice-over" action handler in
+ * use-editor-actions.ts: one undoable batch that adds the recorded audio as a
+ * media asset and inserts it at the playhead with automatic audio placement.
+ */
+function buildVoiceOverBatch({
+ projectId,
+ startTime,
+}: {
+ projectId: string;
+ startTime: MediaTime;
+}): { batch: BatchCommand; assetId: string } {
+ const file = new File([new Blob(["fake-audio"])], "voice-over.webm", {
+ type: "audio/webm",
+ });
+ const addMediaCommand = new AddMediaAssetCommand({
+ projectId,
+ asset: {
+ file,
+ name: "Voice over",
+ type: "audio",
+ url: "blob:voice-over-test",
+ duration: RECORDING_DURATION_SECONDS,
+ },
+ });
+ const insertCommand = new InsertElementCommand({
+ element: buildElementFromMedia({
+ mediaId: addMediaCommand.getAssetId(),
+ mediaType: "audio",
+ name: "Voice over",
+ duration: RECORDING_DURATION,
+ startTime,
+ }),
+ placement: { mode: "auto", trackType: "audio" },
+ });
+ return {
+ batch: new BatchCommand([addMediaCommand, insertCommand]),
+ assetId: addMediaCommand.getAssetId(),
+ };
+}
+
+describe("voice-over batch command", () => {
+ beforeEach(() => {
+ EditorCore.reset();
+ // IndexedDB is unavailable under bun:test; the command treats a failed
+ // save as fatal and rolls back the asset, so stub persistence out.
+ storageService.saveMediaAsset = () => Promise.resolve();
+ storageService.deleteMediaAsset = () => Promise.resolve();
+ });
+
+ test("adds an audio asset and inserts it on a new audio track at the playhead", () => {
+ const editor = seedEditorWithEmptyScene();
+ const { batch, assetId } = buildVoiceOverBatch({
+ projectId: "project-1",
+ startTime: PLAYHEAD_TIME,
+ });
+
+ batch.execute();
+
+ const assets = editor.media.getAssets();
+ expect(assets).toHaveLength(1);
+ expect(assets[0].id).toBe(assetId);
+ expect(assets[0].type).toBe("audio");
+ expect(assets[0].name).toBe("Voice over");
+ expect(assets[0].duration).toBe(RECORDING_DURATION_SECONDS);
+
+ const { tracks } = editor.scenes.getActiveScene();
+ expect(tracks.audio).toHaveLength(1);
+ expect(tracks.audio[0].elements).toHaveLength(1);
+ const element = tracks.audio[0].elements[0];
+ expect(element.type).toBe("audio");
+ expect(element.startTime).toBe(PLAYHEAD_TIME);
+ expect(element.duration).toBe(RECORDING_DURATION);
+ if (element.type === "audio" && element.sourceType === "upload") {
+ expect(element.mediaId).toBe(assetId);
+ } else {
+ throw new Error("expected an upload audio element");
+ }
+ });
+
+ test("undo removes both the element and the asset as a single step", () => {
+ const editor = seedEditorWithEmptyScene();
+ const { batch } = buildVoiceOverBatch({
+ projectId: "project-1",
+ startTime: PLAYHEAD_TIME,
+ });
+
+ batch.execute();
+ batch.undo();
+
+ expect(editor.media.getAssets()).toHaveLength(0);
+ const { tracks } = editor.scenes.getActiveScene();
+ expect(
+ tracks.audio.flatMap((track) => track.elements),
+ ).toHaveLength(0);
+ });
+
+ test("redo re-inserts the element and restores the asset", () => {
+ const editor = seedEditorWithEmptyScene();
+ const { batch, assetId } = buildVoiceOverBatch({
+ projectId: "project-1",
+ startTime: PLAYHEAD_TIME,
+ });
+
+ batch.execute();
+ batch.undo();
+ batch.redo();
+
+ expect(editor.media.getAssets().map((asset) => asset.id)).toEqual([
+ assetId,
+ ]);
+ const { tracks } = editor.scenes.getActiveScene();
+ const elements = tracks.audio.flatMap((track) => track.elements);
+ expect(elements).toHaveLength(1);
+ expect(elements[0].startTime).toBe(PLAYHEAD_TIME);
+ });
+
+ test("reuses an existing audio track when the slot is free", () => {
+ const editor = seedEditorWithEmptyScene();
+ const first = buildVoiceOverBatch({
+ projectId: "project-1",
+ startTime: PLAYHEAD_TIME,
+ });
+ first.batch.execute();
+
+ const secondStart = mediaTimeFromSeconds({
+ seconds: 2 + RECORDING_DURATION_SECONDS + 1,
+ });
+ const second = buildVoiceOverBatch({
+ projectId: "project-1",
+ startTime: secondStart,
+ });
+ second.batch.execute();
+
+ const { tracks } = editor.scenes.getActiveScene();
+ expect(tracks.audio).toHaveLength(1);
+ expect(tracks.audio[0].elements).toHaveLength(2);
+ expect(
+ tracks.audio[0].elements.map((element) => element.startTime),
+ ).toEqual([PLAYHEAD_TIME, secondStart]);
+ });
+});
diff --git a/apps/web/src/voiceover/recorder.ts b/apps/web/src/voiceover/recorder.ts
new file mode 100644
index 00000000..b71ca74f
--- /dev/null
+++ b/apps/web/src/voiceover/recorder.ts
@@ -0,0 +1,186 @@
+/**
+ * Voice-over recording via MediaRecorder.
+ * Pure helpers (mime type picking, file extension, elapsed formatting, error
+ * messages) are exported for unit tests; the recorder singleton below owns the
+ * actual microphone stream.
+ */
+
+export const PREFERRED_MIME_TYPES = [
+ "audio/webm;codecs=opus",
+ "audio/webm",
+ "audio/ogg;codecs=opus",
+ "audio/ogg",
+ "audio/mp4",
+] as const;
+
+export function pickSupportedMimeType({
+ isTypeSupported,
+ candidates = PREFERRED_MIME_TYPES,
+}: {
+ isTypeSupported: (mimeType: string) => boolean;
+ candidates?: readonly string[];
+}): string | null {
+ for (const mimeType of candidates) {
+ if (isTypeSupported(mimeType)) {
+ return mimeType;
+ }
+ }
+ return null;
+}
+
+const EXTENSION_BY_MIME_SUBTYPE: Record<string, string> = {
+ webm: "webm",
+ ogg: "ogg",
+ mp4: "m4a",
+ mpeg: "mp3",
+ wav: "wav",
+};
+
+export function getFileExtensionForMimeType({
+ mimeType,
+}: {
+ mimeType: string;
+}): string {
+ const subtype = mimeType.split("/").at(1)?.split(";").at(0)?.trim() ?? "";
+ return EXTENSION_BY_MIME_SUBTYPE[subtype] ?? "audio";
+}
+
+export function formatElapsedSeconds({
+ seconds,
+}: {
+ seconds: number;
+}): string {
+ const totalSeconds = Math.max(0, Math.floor(seconds));
+ const minutes = Math.floor(totalSeconds / 60);
+ const remainder = totalSeconds % 60;
+ return `${minutes}:${String(remainder).padStart(2, "0")}`;
+}
+
+export function getVoiceOverErrorMessage({
+ error,
+}: {
+ error: unknown;
+}): string {
+ if (error instanceof DOMException) {
+ switch (error.name) {
+ case "NotAllowedError":
+ case "SecurityError":
+ return "Microphone access denied. Allow microphone access in your browser settings to record a voice-over.";
+ case "NotFoundError":
+ case "OverconstrainedError":
+ return "No microphone found. Connect a microphone and try again.";
+ case "NotReadableError":
+ return "The microphone is already in use by another application.";
+ default:
+ return "Could not start voice-over recording.";
+ }
+ }
+ if (error instanceof Error && error.message) {
+ return error.message;
+ }
+ return "Could not start voice-over recording.";
+}
+
+export interface VoiceOverRecordingResult {
+ blob: Blob;
+ mimeType: string;
+ durationSeconds: number;
+}
+
+interface ActiveRecording {
+ recorder: MediaRecorder;
+ stream: MediaStream;
+ chunks: Blob[];
+ mimeType: string;
+ startedAtMs: number;
+}
+
+let activeRecording: ActiveRecording | null = null;
+
+function stopStreamTracks({ stream }: { stream: MediaStream }): void {
+ for (const track of stream.getTracks()) {
+ track.stop();
+ }
+}
+
+export async function startVoiceOverRecording(): Promise<void> {
+ if (activeRecording) {
+ throw new Error("A voice-over recording is already in progress");
+ }
+
+ const mediaDevices = navigator.mediaDevices;
+ if (!mediaDevices?.getUserMedia) {
+ throw new Error("Voice-over recording is not supported in this browser");
+ }
+
+ const stream = await mediaDevices.getUserMedia({ audio: true });
+ try {
+ const mimeType = pickSupportedMimeType({
+ isTypeSupported: (candidate) => MediaRecorder.isTypeSupported(candidate),
+ });
+ const recorder = mimeType
+ ? new MediaRecorder(stream, { mimeType })
+ : new MediaRecorder(stream);
+ const chunks: Blob[] = [];
+ recorder.addEventListener("dataavailable", (event) => {
+ if (event.data.size > 0) {
+ chunks.push(event.data);
+ }
+ });
+ recorder.start();
+ activeRecording = {
+ recorder,
+ stream,
+ chunks,
+ mimeType: recorder.mimeType || mimeType || "audio/webm",
+ startedAtMs: performance.now(),
+ };
+ } catch (error) {
+ stopStreamTracks({ stream });
+ throw error;
+ }
+}
+
+export function stopVoiceOverRecording(): Promise<VoiceOverRecordingResult> {
+ const recording = activeRecording;
+ if (!recording) {
+ return Promise.reject(
+ new Error("No voice-over recording is in progress"),
+ );
+ }
+ activeRecording = null;
+
+ const { promise, resolve, reject } =
+ Promise.withResolvers<VoiceOverRecordingResult>();
+ recording.recorder.addEventListener("stop", () => {
+ stopStreamTracks({ stream: recording.stream });
+ resolve({
+ blob: new Blob(recording.chunks, { type: recording.mimeType }),
+ mimeType: recording.mimeType,
+ durationSeconds: Math.max(
+ 0,
+ (performance.now() - recording.startedAtMs) / 1000,
+ ),
+ });
+ });
+ recording.recorder.addEventListener("error", () => {
+ stopStreamTracks({ stream: recording.stream });
+ reject(new Error("Voice-over recording failed"));
+ });
+ recording.recorder.stop();
+ return promise;
+}
+
+export function cancelVoiceOverRecording(): void {
+ const recording = activeRecording;
+ if (!recording) {
+ return;
+ }
+ activeRecording = null;
+ try {
+ recording.recorder.stop();
+ } catch {
+ // recorder may already be stopped; tracks are released either way
+ }
+ stopStreamTracks({ stream: recording.stream });
+}
diff --git a/apps/web/src/voiceover/use-recording-elapsed.ts b/apps/web/src/voiceover/use-recording-elapsed.ts
new file mode 100644
index 00000000..1eeb7f90
--- /dev/null
+++ b/apps/web/src/voiceover/use-recording-elapsed.ts
@@ -0,0 +1,26 @@
+import { useEffect, useState } from "react";
+import { useVoiceOverStore } from "@/voiceover/voiceover-store";
+
+const ELAPSED_TICK_MS = 250;
+
+/**
+ * Elapsed seconds of the active voice-over recording, ticking while recording.
+ * Returns 0 when idle.
+ */
+export function useRecordingElapsedSeconds(): number {
+ const status = useVoiceOverStore((state) => state.status);
+ const startedAtMs = useVoiceOverStore((state) => state.startedAtMs);
+ const [elapsedSeconds, setElapsedSeconds] = useState(0);
+
+ useEffect(() => {
+ if (status !== "recording" || startedAtMs === null) {
+ return;
+ }
+ const interval = setInterval(() => {
+ setElapsedSeconds((Date.now() - startedAtMs) / 1000);
+ }, ELAPSED_TICK_MS);
+ return () => clearInterval(interval);
+ }, [status, startedAtMs]);
+
+ return status === "recording" ? elapsedSeconds : 0;
+}
diff --git a/apps/web/src/voiceover/voiceover-store.ts b/apps/web/src/voiceover/voiceover-store.ts
new file mode 100644
index 00000000..44ccae6c
--- /dev/null
+++ b/apps/web/src/voiceover/voiceover-store.ts
@@ -0,0 +1,41 @@
+/**
+ * Voice-over recording UI state.
+ * Transient by nature — a recording never survives a reload, so unlike most
+ * stores here this one is intentionally NOT persisted (same pattern as
+ * editor-store / properties-store).
+ */
+
+import { create } from "zustand";
+import type { MediaTime } from "@/wasm";
+
+export type VoiceOverStatus = "idle" | "recording";
+
+interface VoiceOverStore {
+ status: VoiceOverStatus;
+ startedAtMs: number | null;
+ playheadStartTime: MediaTime | null;
+ startRecording: ({ playheadTime }: { playheadTime: MediaTime }) => void;
+ stopRecording: () => void;
+}
+
+export const useVoiceOverStore = create<VoiceOverStore>()((set) => ({
+ status: "idle",
+ startedAtMs: null,
+ playheadStartTime: null,
+
+ startRecording: ({ playheadTime }) => {
+ set({
+ status: "recording",
+ startedAtMs: Date.now(),
+ playheadStartTime: playheadTime,
+ });
+ },
+
+ stopRecording: () => {
+ set({
+ status: "idle",
+ startedAtMs: null,
+ playheadStartTime: null,
+ });
+ },
+}));