summaryrefslogtreecommitdiff
path: root/apps/web/src/voiceover
diff options
context:
space:
mode:
authorcashmere <cashmere@autistici.org>2026-08-16 06:16:56 +0200
committercashmere <cashmere@autistici.org>2026-08-16 06:16:56 +0200
commitf1043734438c8518a302f698d9ae6583a75f0c14 (patch)
treed3e636a96e6349c1ad17afa7b28c9ffe5c882e37 /apps/web/src/voiceover
parent37d22b1e390bd6cea04bf8e15069281ee125cf10 (diff)
downloadopencut-f1043734438c8518a302f698d9ae6583a75f0c14.tar.gz
feat: add transitions, track mixing, and voiceover controls
Diffstat (limited to 'apps/web/src/voiceover')
-rw-r--r--apps/web/src/voiceover/__tests__/devices.test.ts72
-rw-r--r--apps/web/src/voiceover/__tests__/enhancements.test.ts133
-rw-r--r--apps/web/src/voiceover/devices.ts93
-rw-r--r--apps/web/src/voiceover/enhancements.ts321
-rw-r--r--apps/web/src/voiceover/live-waveform.tsx171
-rw-r--r--apps/web/src/voiceover/recorder.ts40
-rw-r--r--apps/web/src/voiceover/voiceover-popover.tsx276
-rw-r--r--apps/web/src/voiceover/voiceover-store.ts57
8 files changed, 1160 insertions, 3 deletions
diff --git a/apps/web/src/voiceover/__tests__/devices.test.ts b/apps/web/src/voiceover/__tests__/devices.test.ts
new file mode 100644
index 00000000..524e8c3f
--- /dev/null
+++ b/apps/web/src/voiceover/__tests__/devices.test.ts
@@ -0,0 +1,72 @@
+import { describe, expect, test } from "bun:test";
+import {
+ DEFAULT_AUDIO_INPUT_DEVICE_ID,
+ buildVoiceOverAudioConstraints,
+ normalizeSelectedDeviceId,
+ type VoiceOverInputDevice,
+} from "@/voiceover/devices";
+
+const devices: VoiceOverInputDevice[] = [
+ {
+ deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID,
+ label: "System default microphone",
+ isDefault: true,
+ },
+ {
+ deviceId: "blue-yeti-snowball",
+ label: "Blue Snowball",
+ isDefault: false,
+ },
+];
+
+describe("buildVoiceOverAudioConstraints", () => {
+ test("uses default processing constraints without an exact device", () => {
+ const constraints = buildVoiceOverAudioConstraints({ deviceId: null });
+ expect(constraints.audio).toEqual({
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+ });
+ });
+
+ test("adds an exact device constraint for a selected microphone", () => {
+ const constraints = buildVoiceOverAudioConstraints({
+ deviceId: "blue-yeti-snowball",
+ });
+ expect(constraints.audio).toEqual({
+ deviceId: { exact: "blue-yeti-snowball" },
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+ });
+ });
+});
+
+describe("normalizeSelectedDeviceId", () => {
+ test("keeps the selected device when it is still present", () => {
+ expect(
+ normalizeSelectedDeviceId({
+ deviceId: "blue-yeti-snowball",
+ devices,
+ }),
+ ).toBe("blue-yeti-snowball");
+ });
+
+ test("clears a selected device when it is no longer present", () => {
+ expect(
+ normalizeSelectedDeviceId({
+ deviceId: "missing",
+ devices,
+ }),
+ ).toBeNull();
+ });
+
+ test("keeps the browser default device id", () => {
+ expect(
+ normalizeSelectedDeviceId({
+ deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID,
+ devices: [],
+ }),
+ ).toBe(DEFAULT_AUDIO_INPUT_DEVICE_ID);
+ });
+});
diff --git a/apps/web/src/voiceover/__tests__/enhancements.test.ts b/apps/web/src/voiceover/__tests__/enhancements.test.ts
new file mode 100644
index 00000000..bc5b4b7c
--- /dev/null
+++ b/apps/web/src/voiceover/__tests__/enhancements.test.ts
@@ -0,0 +1,133 @@
+import { describe, expect, test } from "bun:test";
+import {
+ DEFAULT_VOICE_OVER_ENHANCEMENTS,
+ encodeAudioBufferToWav,
+ getAudioBufferPeak,
+ hasEnabledVoiceOverEnhancement,
+ type VoiceOverEnhancementSettings,
+} from "@/voiceover/enhancements";
+
+/* eslint-disable opencut/prefer-object-params -- Browser AudioBuffer methods are positional. */
+class FakeAudioBuffer implements AudioBuffer {
+ duration: number;
+ length: number;
+ numberOfChannels: number;
+ sampleRate: number;
+
+ constructor(
+ private readonly channels: Array<Float32Array<ArrayBuffer>>,
+ sampleRate: number,
+ ) {
+ this.length = channels[0]?.length ?? 0;
+ this.numberOfChannels = channels.length;
+ this.sampleRate = sampleRate;
+ this.duration = this.length / sampleRate;
+ }
+
+ copyFromChannel(
+ destination: Float32Array<ArrayBuffer>,
+ channelNumber: number,
+ bufferOffset = 0,
+ ): void {
+ destination.set(
+ this.getChannelData(channelNumber).slice(
+ bufferOffset,
+ bufferOffset + destination.length,
+ ),
+ );
+ }
+
+ copyToChannel(
+ source: Float32Array<ArrayBuffer>,
+ channelNumber: number,
+ bufferOffset = 0,
+ ): void {
+ this.getChannelData(channelNumber).set(source, bufferOffset);
+ }
+
+ getChannelData(channel: number): Float32Array<ArrayBuffer> {
+ const data = this.channels[channel];
+ if (!data) {
+ throw new Error(`Missing channel ${channel}`);
+ }
+ return data;
+ }
+}
+/* eslint-enable opencut/prefer-object-params */
+
+describe("hasEnabledVoiceOverEnhancement", () => {
+ test("detects enabled defaults", () => {
+ expect(
+ hasEnabledVoiceOverEnhancement({
+ settings: DEFAULT_VOICE_OVER_ENHANCEMENTS,
+ }),
+ ).toBe(true);
+ });
+
+ test("detects an all-disabled chain", () => {
+ const settings: VoiceOverEnhancementSettings = {
+ voiceBoost: false,
+ noiseGate: false,
+ highPass: false,
+ deEss: false,
+ limiter: false,
+ normalize: false,
+ };
+ expect(hasEnabledVoiceOverEnhancement({ settings })).toBe(false);
+ });
+});
+
+describe("getAudioBufferPeak", () => {
+ test("returns the maximum absolute sample", () => {
+ const buffer = new FakeAudioBuffer(
+ [
+ createSamples([0, -0.25, 0.5]),
+ createSamples([0.1, -0.8, 0.2]),
+ ],
+ 48000,
+ );
+
+ expect(getAudioBufferPeak({ buffer })).toBeCloseTo(0.8);
+ });
+});
+
+describe("encodeAudioBufferToWav", () => {
+ test("writes a 16-bit PCM WAV header", async () => {
+ const buffer = new FakeAudioBuffer(
+ [createSamples([0, 0.5, -0.5])],
+ 44100,
+ );
+
+ const blob = encodeAudioBufferToWav({ buffer });
+ const view = new DataView(await blob.arrayBuffer());
+
+ expect(blob.type).toBe("audio/wav");
+ expect(blob.size).toBe(50);
+ expect(readAscii({ view, offset: 0, length: 4 })).toBe("RIFF");
+ expect(readAscii({ view, offset: 8, length: 4 })).toBe("WAVE");
+ expect(view.getUint16(22, true)).toBe(1);
+ expect(view.getUint32(24, true)).toBe(44100);
+ expect(view.getUint16(34, true)).toBe(16);
+ expect(readAscii({ view, offset: 36, length: 4 })).toBe("data");
+ });
+});
+
+function readAscii({
+ view,
+ offset,
+ length,
+}: {
+ view: DataView;
+ offset: number;
+ length: number;
+}): string {
+ return Array.from({ length }, (_, index) =>
+ String.fromCharCode(view.getUint8(offset + index)),
+ ).join("");
+}
+
+function createSamples(values: number[]): Float32Array<ArrayBuffer> {
+ const samples = new Float32Array(new ArrayBuffer(values.length * 4));
+ samples.set(values);
+ return samples;
+}
diff --git a/apps/web/src/voiceover/devices.ts b/apps/web/src/voiceover/devices.ts
new file mode 100644
index 00000000..dd5b680c
--- /dev/null
+++ b/apps/web/src/voiceover/devices.ts
@@ -0,0 +1,93 @@
+"use client";
+
+export const DEFAULT_AUDIO_INPUT_DEVICE_ID = "default";
+
+export interface VoiceOverInputDevice {
+ deviceId: string;
+ label: string;
+ isDefault: boolean;
+}
+
+export function buildVoiceOverAudioConstraints({
+ deviceId,
+}: {
+ deviceId?: string | null;
+}): MediaStreamConstraints {
+ const audio: MediaTrackConstraints = {
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+ };
+
+ if (deviceId && deviceId !== DEFAULT_AUDIO_INPUT_DEVICE_ID) {
+ audio.deviceId = { exact: deviceId };
+ }
+
+ return { audio };
+}
+
+export function normalizeSelectedDeviceId({
+ deviceId,
+ devices,
+}: {
+ deviceId: string | null;
+ devices: VoiceOverInputDevice[];
+}): string | null {
+ if (!deviceId) {
+ return null;
+ }
+
+ if (deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID) {
+ return deviceId;
+ }
+
+ return devices.some((device) => device.deviceId === deviceId)
+ ? deviceId
+ : null;
+}
+
+export async function listVoiceOverInputDevices({
+ requestPermission = false,
+}: {
+ requestPermission?: boolean;
+} = {}): Promise<VoiceOverInputDevice[]> {
+ const mediaDevices = navigator.mediaDevices;
+ if (!mediaDevices?.enumerateDevices) {
+ throw new Error("Microphone selection is not supported in this browser");
+ }
+
+ let permissionStream: MediaStream | null = null;
+ if (requestPermission) {
+ permissionStream = await mediaDevices.getUserMedia({
+ audio: {
+ echoCancellation: true,
+ noiseSuppression: true,
+ autoGainControl: true,
+ },
+ });
+ }
+
+ try {
+ const devices = await mediaDevices.enumerateDevices();
+ let inputIndex = 1;
+ return devices
+ .filter((device) => device.kind === "audioinput")
+ .map((device) => {
+ const isDefault = device.deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID;
+ const fallbackLabel = isDefault
+ ? "System default microphone"
+ : `Microphone ${inputIndex++}`;
+ return {
+ deviceId: device.deviceId,
+ label: device.label || fallbackLabel,
+ isDefault,
+ };
+ });
+ } finally {
+ if (permissionStream) {
+ for (const track of permissionStream.getTracks()) {
+ track.stop();
+ }
+ }
+ }
+}
diff --git a/apps/web/src/voiceover/enhancements.ts b/apps/web/src/voiceover/enhancements.ts
new file mode 100644
index 00000000..8718406f
--- /dev/null
+++ b/apps/web/src/voiceover/enhancements.ts
@@ -0,0 +1,321 @@
+"use client";
+
+export interface VoiceOverEnhancementSettings {
+ voiceBoost: boolean;
+ noiseGate: boolean;
+ highPass: boolean;
+ deEss: boolean;
+ limiter: boolean;
+ normalize: boolean;
+}
+
+export const DEFAULT_VOICE_OVER_ENHANCEMENTS: VoiceOverEnhancementSettings = {
+ voiceBoost: true,
+ noiseGate: true,
+ highPass: true,
+ deEss: false,
+ limiter: true,
+ normalize: true,
+};
+
+export const VOICE_OVER_ENHANCEMENT_LABELS: Record<
+ keyof VoiceOverEnhancementSettings,
+ { label: string; description: string }
+> = {
+ voiceBoost: {
+ label: "Voice boost",
+ description: "Adds compression and presence for a stronger voice.",
+ },
+ noiseGate: {
+ label: "Noise gate",
+ description: "Turns down quiet room noise between phrases.",
+ },
+ highPass: {
+ label: "High-pass",
+ description: "Removes low rumble and desk vibration.",
+ },
+ deEss: {
+ label: "De-ess",
+ description: "Softens harsh S sounds and upper fizz.",
+ },
+ limiter: {
+ label: "Limiter",
+ description: "Catches peaks after boosts.",
+ },
+ normalize: {
+ label: "Normalize",
+ description: "Raises the finished take to a consistent level.",
+ },
+};
+
+const WAV_BITS_PER_SAMPLE = 16;
+const WAV_HEADER_BYTES = 44;
+const TARGET_PEAK = 0.92;
+const NOISE_GATE_THRESHOLD = 0.018;
+const NOISE_GATE_FLOOR = 0.18;
+
+export function hasEnabledVoiceOverEnhancement({
+ settings,
+}: {
+ settings: VoiceOverEnhancementSettings;
+}): boolean {
+ return Object.values(settings).some(Boolean);
+}
+
+export async function applyVoiceOverEnhancementsToBlob({
+ blob,
+ settings,
+}: {
+ blob: Blob;
+ settings: VoiceOverEnhancementSettings;
+}): Promise<Blob> {
+ if (!hasEnabledVoiceOverEnhancement({ settings })) {
+ return blob;
+ }
+
+ const audioContext = createAudioContext();
+ try {
+ const buffer = await audioContext.decodeAudioData(await blob.arrayBuffer());
+ const renderedBuffer = await renderEnhancedVoiceOverBuffer({
+ buffer,
+ settings,
+ });
+ return encodeAudioBufferToWav({ buffer: renderedBuffer });
+ } finally {
+ void audioContext.close();
+ }
+}
+
+export async function renderEnhancedVoiceOverBuffer({
+ buffer,
+ settings,
+}: {
+ buffer: AudioBuffer;
+ settings: VoiceOverEnhancementSettings;
+}): Promise<AudioBuffer> {
+ const offlineContext = new OfflineAudioContext(
+ buffer.numberOfChannels,
+ Math.max(1, buffer.length),
+ buffer.sampleRate,
+ );
+ const source = offlineContext.createBufferSource();
+ source.buffer = settings.noiseGate
+ ? applyNoiseGateToBuffer({
+ buffer,
+ context: offlineContext,
+ threshold: NOISE_GATE_THRESHOLD,
+ floor: NOISE_GATE_FLOOR,
+ })
+ : buffer;
+
+ let current: AudioNode = source;
+
+ if (settings.highPass) {
+ const highPass = offlineContext.createBiquadFilter();
+ highPass.type = "highpass";
+ highPass.frequency.value = 85;
+ highPass.Q.value = 0.7;
+ current.connect(highPass);
+ current = highPass;
+ }
+
+ if (settings.voiceBoost) {
+ const presence = offlineContext.createBiquadFilter();
+ presence.type = "peaking";
+ presence.frequency.value = 3200;
+ presence.Q.value = 0.9;
+ presence.gain.value = 2.5;
+
+ const compressor = offlineContext.createDynamicsCompressor();
+ compressor.threshold.value = -24;
+ compressor.knee.value = 12;
+ compressor.ratio.value = 3.5;
+ compressor.attack.value = 0.004;
+ compressor.release.value = 0.16;
+
+ const makeupGain = offlineContext.createGain();
+ makeupGain.gain.value = 1.45;
+
+ current.connect(presence);
+ presence.connect(compressor);
+ compressor.connect(makeupGain);
+ current = makeupGain;
+ }
+
+ if (settings.deEss) {
+ const deEss = offlineContext.createBiquadFilter();
+ deEss.type = "peaking";
+ deEss.frequency.value = 6400;
+ deEss.Q.value = 2.2;
+ deEss.gain.value = -3;
+ current.connect(deEss);
+ current = deEss;
+ }
+
+ if (settings.limiter) {
+ const limiter = offlineContext.createDynamicsCompressor();
+ limiter.threshold.value = -1;
+ limiter.knee.value = 0;
+ limiter.ratio.value = 20;
+ limiter.attack.value = 0.001;
+ limiter.release.value = 0.1;
+ current.connect(limiter);
+ current = limiter;
+ }
+
+ current.connect(offlineContext.destination);
+ source.start(0);
+
+ const rendered = await offlineContext.startRendering();
+ if (settings.normalize) {
+ normalizeAudioBufferPeak({ buffer: rendered, targetPeak: TARGET_PEAK });
+ }
+ clampAudioBuffer({ buffer: rendered, maxPeak: 0.98 });
+ return rendered;
+}
+
+export function encodeAudioBufferToWav({
+ buffer,
+}: {
+ buffer: AudioBuffer;
+}): Blob {
+ const numChannels = buffer.numberOfChannels;
+ const bytesPerSample = WAV_BITS_PER_SAMPLE / 8;
+ const dataSize = buffer.length * numChannels * bytesPerSample;
+ const arrayBuffer = new ArrayBuffer(WAV_HEADER_BYTES + dataSize);
+ const view = new DataView(arrayBuffer);
+
+ writeAsciiString({ view, offset: 0, value: "RIFF" });
+ view.setUint32(4, 36 + dataSize, true);
+ writeAsciiString({ view, offset: 8, value: "WAVE" });
+ writeAsciiString({ view, offset: 12, value: "fmt " });
+ view.setUint32(16, 16, true);
+ view.setUint16(20, 1, true);
+ view.setUint16(22, numChannels, true);
+ view.setUint32(24, buffer.sampleRate, true);
+ view.setUint32(28, buffer.sampleRate * numChannels * bytesPerSample, true);
+ view.setUint16(32, numChannels * bytesPerSample, true);
+ view.setUint16(34, WAV_BITS_PER_SAMPLE, true);
+ writeAsciiString({ view, offset: 36, value: "data" });
+ view.setUint32(40, dataSize, true);
+
+ let offset = WAV_HEADER_BYTES;
+ for (let sampleIndex = 0; sampleIndex < buffer.length; sampleIndex++) {
+ for (let channel = 0; channel < numChannels; channel++) {
+ const sample = Math.max(
+ -1,
+ Math.min(1, buffer.getChannelData(channel)[sampleIndex] ?? 0),
+ );
+ const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff;
+ view.setInt16(offset, int16, true);
+ offset += bytesPerSample;
+ }
+ }
+
+ return new Blob([arrayBuffer], { type: "audio/wav" });
+}
+
+export function getAudioBufferPeak({ buffer }: { buffer: AudioBuffer }): number {
+ let peak = 0;
+ for (let channel = 0; channel < buffer.numberOfChannels; channel++) {
+ const channelData = buffer.getChannelData(channel);
+ for (let index = 0; index < channelData.length; index++) {
+ peak = Math.max(peak, Math.abs(channelData[index] ?? 0));
+ }
+ }
+ return peak;
+}
+
+function normalizeAudioBufferPeak({
+ buffer,
+ targetPeak,
+}: {
+ buffer: AudioBuffer;
+ targetPeak: number;
+}): void {
+ const peak = getAudioBufferPeak({ buffer });
+ if (peak <= 0 || peak >= targetPeak) {
+ return;
+ }
+
+ const gain = targetPeak / peak;
+ for (let channel = 0; channel < buffer.numberOfChannels; channel++) {
+ const channelData = buffer.getChannelData(channel);
+ for (let index = 0; index < channelData.length; index++) {
+ channelData[index] *= gain;
+ }
+ }
+}
+
+function clampAudioBuffer({
+ buffer,
+ maxPeak,
+}: {
+ buffer: AudioBuffer;
+ maxPeak: number;
+}): void {
+ for (let channel = 0; channel < buffer.numberOfChannels; channel++) {
+ const channelData = buffer.getChannelData(channel);
+ for (let index = 0; index < channelData.length; index++) {
+ channelData[index] = Math.max(
+ -maxPeak,
+ Math.min(maxPeak, channelData[index] ?? 0),
+ );
+ }
+ }
+}
+
+function applyNoiseGateToBuffer({
+ buffer,
+ context,
+ threshold,
+ floor,
+}: {
+ buffer: AudioBuffer;
+ context: OfflineAudioContext;
+ threshold: number;
+ floor: number;
+}): AudioBuffer {
+ const gated = context.createBuffer(
+ buffer.numberOfChannels,
+ buffer.length,
+ buffer.sampleRate,
+ );
+
+ for (let channel = 0; channel < buffer.numberOfChannels; channel++) {
+ const input = buffer.getChannelData(channel);
+ const output = gated.getChannelData(channel);
+ for (let index = 0; index < input.length; index++) {
+ const sample = input[index] ?? 0;
+ output[index] =
+ Math.abs(sample) < threshold
+ ? sample * floor
+ : sample;
+ }
+ }
+
+ return gated;
+}
+
+function createAudioContext(): AudioContext {
+ const AudioContextConstructor =
+ window.AudioContext ||
+ (window as typeof window & { webkitAudioContext?: typeof AudioContext })
+ .webkitAudioContext;
+
+ return new AudioContextConstructor();
+}
+
+function writeAsciiString({
+ view,
+ offset,
+ value,
+}: {
+ view: DataView;
+ offset: number;
+ value: string;
+}): void {
+ for (let index = 0; index < value.length; index++) {
+ view.setUint8(offset + index, value.charCodeAt(index));
+ }
+}
diff --git a/apps/web/src/voiceover/live-waveform.tsx b/apps/web/src/voiceover/live-waveform.tsx
new file mode 100644
index 00000000..cd25c833
--- /dev/null
+++ b/apps/web/src/voiceover/live-waveform.tsx
@@ -0,0 +1,171 @@
+"use client";
+
+import { useEffect, useRef, useState } from "react";
+import { cn } from "@/utils/ui";
+import { getActiveVoiceOverRecordingStream } from "@/voiceover/recorder";
+
+const FFT_SIZE = 1024;
+const HISTORY_SIZE = 96;
+
+export function VoiceOverLiveWaveform({
+ recording,
+ className,
+}: {
+ recording: boolean;
+ className?: string;
+}) {
+ const canvasRef = useRef<HTMLCanvasElement>(null);
+ const levelRef = useRef(0);
+ const historyRef = useRef<number[]>(
+ Array.from({ length: HISTORY_SIZE }, () => 0),
+ );
+ const [level, setLevel] = useState(0);
+
+ useEffect(() => {
+ const canvas = canvasRef.current;
+ if (!canvas || !recording) {
+ drawWaveform({
+ canvas,
+ history: historyRef.current,
+ level: 0,
+ });
+ setLevel(0);
+ return;
+ }
+
+ const stream = getActiveVoiceOverRecordingStream();
+ if (!stream) {
+ return;
+ }
+
+ const audioContext = createAudioContext();
+ const source = audioContext.createMediaStreamSource(stream);
+ const analyser = audioContext.createAnalyser();
+ analyser.fftSize = FFT_SIZE;
+ analyser.smoothingTimeConstant = 0.72;
+ source.connect(analyser);
+
+ const samples = new Uint8Array(analyser.fftSize);
+ let animationFrame = 0;
+ let lastLevelUpdate = 0;
+
+ const render = (timestamp: number) => {
+ analyser.getByteTimeDomainData(samples);
+ const rms = getRmsLevel({ samples });
+ levelRef.current = rms;
+ historyRef.current = [...historyRef.current.slice(1), rms];
+ drawWaveform({
+ canvas,
+ history: historyRef.current,
+ level: rms,
+ });
+
+ if (timestamp - lastLevelUpdate > 80) {
+ lastLevelUpdate = timestamp;
+ setLevel(rms);
+ }
+ animationFrame = requestAnimationFrame(render);
+ };
+
+ animationFrame = requestAnimationFrame(render);
+
+ return () => {
+ cancelAnimationFrame(animationFrame);
+ source.disconnect();
+ analyser.disconnect();
+ void audioContext.close();
+ };
+ }, [recording]);
+
+ return (
+ <div className={cn("space-y-2", className)}>
+ <div className="bg-muted relative h-16 overflow-hidden rounded-sm border">
+ <canvas ref={canvasRef} className="absolute inset-0 size-full" />
+ </div>
+ <div className="bg-muted h-1.5 overflow-hidden rounded-full">
+ <div
+ className="bg-primary h-full rounded-full transition-[width]"
+ style={{ width: `${Math.min(100, Math.round(level * 140))}%` }}
+ />
+ </div>
+ </div>
+ );
+}
+
+function getRmsLevel({ samples }: { samples: Uint8Array }): number {
+ let sum = 0;
+ for (let index = 0; index < samples.length; index++) {
+ const centered = ((samples[index] ?? 128) - 128) / 128;
+ sum += centered * centered;
+ }
+ return Math.sqrt(sum / samples.length);
+}
+
+function drawWaveform({
+ canvas,
+ history,
+ level,
+}: {
+ canvas: HTMLCanvasElement | null;
+ history: number[];
+ level: number;
+}): void {
+ if (!canvas) {
+ return;
+ }
+
+ const rect = canvas.getBoundingClientRect();
+ const dpr = window.devicePixelRatio || 1;
+ const width = Math.max(1, Math.round(rect.width * dpr));
+ const height = Math.max(1, Math.round(rect.height * dpr));
+ if (canvas.width !== width || canvas.height !== height) {
+ canvas.width = width;
+ canvas.height = height;
+ }
+
+ const ctx = canvas.getContext("2d");
+ if (!ctx) {
+ return;
+ }
+
+ ctx.setTransform(1, 0, 0, 1, 0, 0);
+ ctx.clearRect(0, 0, width, height);
+
+ const centerY = height / 2;
+ const step = width / Math.max(1, history.length);
+
+ ctx.strokeStyle = "rgba(148, 163, 184, 0.35)";
+ ctx.lineWidth = 1;
+ ctx.beginPath();
+ ctx.moveTo(0, centerY);
+ ctx.lineTo(width, centerY);
+ ctx.stroke();
+
+ for (let index = 0; index < history.length; index++) {
+ const value = Math.max(0, history[index] ?? 0);
+ const barHeight = Math.max(1, Math.min(height, value * height * 2.8));
+ const x = Math.round(index * step);
+ const y = Math.round(centerY - barHeight / 2);
+ ctx.fillStyle =
+ value > 0.62
+ ? "rgba(239, 68, 68, 0.9)"
+ : value > 0.32
+ ? "rgba(245, 158, 11, 0.9)"
+ : "rgba(34, 197, 94, 0.9)";
+ ctx.fillRect(x, y, Math.max(1, Math.floor(step - 1)), barHeight);
+ }
+
+ if (level > 0.78) {
+ ctx.fillStyle = "rgba(239, 68, 68, 0.18)";
+ ctx.fillRect(0, 0, width, height);
+ }
+}
+
+function createAudioContext(): AudioContext {
+ const AudioContextConstructor =
+ window.AudioContext ||
+ (window as typeof window & { webkitAudioContext?: typeof AudioContext })
+ .webkitAudioContext;
+
+ return new AudioContextConstructor();
+}
diff --git a/apps/web/src/voiceover/recorder.ts b/apps/web/src/voiceover/recorder.ts
index b71ca74f..48322898 100644
--- a/apps/web/src/voiceover/recorder.ts
+++ b/apps/web/src/voiceover/recorder.ts
@@ -5,6 +5,8 @@
* actual microphone stream.
*/
+import { buildVoiceOverAudioConstraints } from "@/voiceover/devices";
+
export const PREFERRED_MIME_TYPES = [
"audio/webm;codecs=opus",
"audio/webm",
@@ -95,6 +97,11 @@ interface ActiveRecording {
startedAtMs: number;
}
+export interface VoiceOverRecordingSession {
+ stream: MediaStream;
+ mimeType: string;
+}
+
let activeRecording: ActiveRecording | null = null;
function stopStreamTracks({ stream }: { stream: MediaStream }): void {
@@ -103,7 +110,15 @@ function stopStreamTracks({ stream }: { stream: MediaStream }): void {
}
}
-export async function startVoiceOverRecording(): Promise<void> {
+export function getActiveVoiceOverRecordingStream(): MediaStream | null {
+ return activeRecording?.stream ?? null;
+}
+
+export async function startVoiceOverRecording({
+ deviceId,
+}: {
+ deviceId?: string | null;
+} = {}): Promise<VoiceOverRecordingSession> {
if (activeRecording) {
throw new Error("A voice-over recording is already in progress");
}
@@ -113,7 +128,24 @@ export async function startVoiceOverRecording(): Promise<void> {
throw new Error("Voice-over recording is not supported in this browser");
}
- const stream = await mediaDevices.getUserMedia({ audio: true });
+ let stream: MediaStream;
+ try {
+ stream = await mediaDevices.getUserMedia(
+ buildVoiceOverAudioConstraints({ deviceId }),
+ );
+ } catch (error) {
+ if (
+ deviceId &&
+ error instanceof DOMException &&
+ (error.name === "NotFoundError" || error.name === "OverconstrainedError")
+ ) {
+ stream = await mediaDevices.getUserMedia(
+ buildVoiceOverAudioConstraints({ deviceId: null }),
+ );
+ } else {
+ throw error;
+ }
+ }
try {
const mimeType = pickSupportedMimeType({
isTypeSupported: (candidate) => MediaRecorder.isTypeSupported(candidate),
@@ -135,6 +167,10 @@ export async function startVoiceOverRecording(): Promise<void> {
mimeType: recorder.mimeType || mimeType || "audio/webm",
startedAtMs: performance.now(),
};
+ return {
+ stream,
+ mimeType: activeRecording.mimeType,
+ };
} catch (error) {
stopStreamTracks({ stream });
throw error;
diff --git a/apps/web/src/voiceover/voiceover-popover.tsx b/apps/web/src/voiceover/voiceover-popover.tsx
new file mode 100644
index 00000000..a1f74646
--- /dev/null
+++ b/apps/web/src/voiceover/voiceover-popover.tsx
@@ -0,0 +1,276 @@
+"use client";
+
+import { useCallback, useMemo, useState } from "react";
+import { MicIcon, StopIcon } from "@hugeicons/core-free-icons";
+import { HugeiconsIcon } from "@hugeicons/react";
+import { RefreshCw } from "lucide-react";
+import { toast } from "sonner";
+import { invokeAction } from "@/actions";
+import { Button } from "@/components/ui/button";
+import {
+ Popover,
+ PopoverContent,
+ PopoverTrigger,
+} from "@/components/ui/popover";
+import {
+ Select,
+ SelectContent,
+ SelectItem,
+ SelectTrigger,
+ SelectValue,
+} from "@/components/ui/select";
+import { Switch } from "@/components/ui/switch";
+import {
+ Tooltip,
+ TooltipContent,
+ TooltipProvider,
+ TooltipTrigger,
+} from "@/components/ui/tooltip";
+import { cn } from "@/utils/ui";
+import {
+ DEFAULT_AUDIO_INPUT_DEVICE_ID,
+ listVoiceOverInputDevices,
+ normalizeSelectedDeviceId,
+ type VoiceOverInputDevice,
+} from "@/voiceover/devices";
+import {
+ VOICE_OVER_ENHANCEMENT_LABELS,
+ type VoiceOverEnhancementSettings,
+} from "@/voiceover/enhancements";
+import { VoiceOverLiveWaveform } from "@/voiceover/live-waveform";
+import {
+ formatElapsedSeconds,
+ getVoiceOverErrorMessage,
+} from "@/voiceover/recorder";
+import { useRecordingElapsedSeconds } from "@/voiceover/use-recording-elapsed";
+import {
+ useVoiceOverPreferencesStore,
+ useVoiceOverStore,
+} from "@/voiceover/voiceover-store";
+
+const ENHANCEMENT_ORDER: Array<keyof VoiceOverEnhancementSettings> = [
+ "voiceBoost",
+ "noiseGate",
+ "highPass",
+ "deEss",
+ "limiter",
+ "normalize",
+];
+
+export function VoiceOverRecorderPopover() {
+ const [open, setOpen] = useState(false);
+ const [devices, setDevices] = useState<VoiceOverInputDevice[]>([]);
+ const [isLoadingDevices, setIsLoadingDevices] = useState(false);
+ const status = useVoiceOverStore((state) => state.status);
+ const isRecording = status === "recording";
+ const isProcessing = status === "processing";
+ const elapsedSeconds = useRecordingElapsedSeconds();
+ const selectedDeviceId = useVoiceOverPreferencesStore(
+ (state) => state.selectedDeviceId,
+ );
+ const enhancements = useVoiceOverPreferencesStore(
+ (state) => state.enhancements,
+ );
+ const setSelectedDeviceId = useVoiceOverPreferencesStore(
+ (state) => state.setSelectedDeviceId,
+ );
+ const setEnhancementEnabled = useVoiceOverPreferencesStore(
+ (state) => state.setEnhancementEnabled,
+ );
+
+ const devicesWithDefault = useMemo(() => {
+ if (
+ devices.some(
+ (device) => device.deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID,
+ )
+ ) {
+ return devices;
+ }
+ return [
+ {
+ deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID,
+ label: "System default microphone",
+ isDefault: true,
+ },
+ ...devices,
+ ];
+ }, [devices]);
+
+ const selectedValue = selectedDeviceId ?? DEFAULT_AUDIO_INPUT_DEVICE_ID;
+
+ const refreshDevices = useCallback(
+ async ({ requestPermission }: { requestPermission: boolean }) => {
+ setIsLoadingDevices(true);
+ try {
+ const nextDevices = await listVoiceOverInputDevices({
+ requestPermission,
+ });
+ setDevices(nextDevices);
+ const normalizedDeviceId = normalizeSelectedDeviceId({
+ deviceId: selectedDeviceId,
+ devices: nextDevices,
+ });
+ if (normalizedDeviceId !== selectedDeviceId) {
+ setSelectedDeviceId({ deviceId: normalizedDeviceId });
+ }
+ } catch (error) {
+ toast.error(getVoiceOverErrorMessage({ error }));
+ } finally {
+ setIsLoadingDevices(false);
+ }
+ },
+ [selectedDeviceId, setSelectedDeviceId],
+ );
+
+ const handleOpenChange = useCallback(
+ (nextOpen: boolean) => {
+ setOpen(nextOpen);
+ if (nextOpen) {
+ void refreshDevices({ requestPermission: true });
+ }
+ },
+ [refreshDevices],
+ );
+
+ const handleRecordClick = (event: React.MouseEvent) => {
+ event.stopPropagation();
+ void invokeAction("voice-over");
+ };
+
+ return (
+ <Popover open={open} onOpenChange={handleOpenChange}>
+ <TooltipProvider delayDuration={500}>
+ <Tooltip delayDuration={200}>
+ <TooltipTrigger asChild>
+ <PopoverTrigger asChild>
+ <Button
+ variant={isRecording ? "secondary" : "text"}
+ size="icon"
+ disabled={isProcessing}
+ className="rounded-sm"
+ onClick={(event) => event.stopPropagation()}
+ >
+ <HugeiconsIcon icon={isRecording ? StopIcon : MicIcon} />
+ </Button>
+ </PopoverTrigger>
+ </TooltipTrigger>
+ <TooltipContent>
+ {isProcessing
+ ? "Saving voice-over"
+ : isRecording
+ ? "Stop voice-over"
+ : "Record voice-over"}
+ </TooltipContent>
+ </Tooltip>
+ </TooltipProvider>
+
+ <PopoverContent
+ align="start"
+ className="w-80 p-3"
+ onClick={(event) => event.stopPropagation()}
+ >
+ <div className="space-y-3">
+ <div className="flex items-center justify-between gap-3">
+ <div className="min-w-0">
+ <div className="text-sm font-medium">Voice-over</div>
+ <div className="text-muted-foreground text-xs tabular-nums">
+ {isRecording
+ ? formatElapsedSeconds({ seconds: elapsedSeconds })
+ : isProcessing
+ ? "Saving"
+ : "Ready"}
+ </div>
+ </div>
+ <Button
+ variant={isRecording ? "secondary" : "default"}
+ size="sm"
+ disabled={isProcessing}
+ onClick={handleRecordClick}
+ className={cn(
+ "gap-1.5 rounded-sm",
+ isRecording ? "text-destructive" : "",
+ )}
+ >
+ <HugeiconsIcon
+ icon={isRecording ? StopIcon : MicIcon}
+ className="size-4"
+ />
+ {isProcessing ? "Saving" : isRecording ? "Stop" : "Record"}
+ </Button>
+ </div>
+
+ <VoiceOverLiveWaveform recording={isRecording} />
+
+ <div className="space-y-1.5">
+ <div className="flex items-center justify-between gap-2">
+ <div className="text-muted-foreground text-xs font-medium">
+ Input
+ </div>
+ <Button
+ variant="text"
+ size="icon"
+ disabled={isLoadingDevices || isRecording}
+ onClick={() => refreshDevices({ requestPermission: true })}
+ className="size-6 rounded-sm"
+ >
+ <RefreshCw className="size-3.5" />
+ </Button>
+ </div>
+ <Select
+ value={selectedValue}
+ disabled={isRecording}
+ onValueChange={(value) =>
+ setSelectedDeviceId({
+ deviceId:
+ value === DEFAULT_AUDIO_INPUT_DEVICE_ID ? null : value,
+ })
+ }
+ >
+ <SelectTrigger className="w-full" variant="outline" size="sm">
+ <SelectValue placeholder="System default microphone" />
+ </SelectTrigger>
+ <SelectContent>
+ {devicesWithDefault.map((device) => (
+ <SelectItem key={device.deviceId} value={device.deviceId}>
+ {device.label}
+ </SelectItem>
+ ))}
+ </SelectContent>
+ </Select>
+ </div>
+
+ <div className="border-border border-t pt-3">
+ <div className="text-muted-foreground mb-2 text-xs font-medium">
+ Enhancements
+ </div>
+ <div className="space-y-2">
+ {ENHANCEMENT_ORDER.map((id) => {
+ const option = VOICE_OVER_ENHANCEMENT_LABELS[id];
+ return (
+ <div
+ key={id}
+ className="flex items-start justify-between gap-3"
+ >
+ <span className="min-w-0">
+ <span className="block text-sm">{option.label}</span>
+ <span className="text-muted-foreground block text-xs leading-snug">
+ {option.description}
+ </span>
+ </span>
+ <Switch
+ checked={enhancements[id]}
+ aria-label={option.label}
+ onCheckedChange={(enabled) =>
+ setEnhancementEnabled({ id, enabled })
+ }
+ />
+ </div>
+ );
+ })}
+ </div>
+ </div>
+ </div>
+ </PopoverContent>
+ </Popover>
+ );
+}
diff --git a/apps/web/src/voiceover/voiceover-store.ts b/apps/web/src/voiceover/voiceover-store.ts
index 44ccae6c..e4024c1e 100644
--- a/apps/web/src/voiceover/voiceover-store.ts
+++ b/apps/web/src/voiceover/voiceover-store.ts
@@ -6,18 +6,37 @@
*/
import { create } from "zustand";
+import { persist } from "zustand/middleware";
import type { MediaTime } from "@/wasm";
+import {
+ DEFAULT_VOICE_OVER_ENHANCEMENTS,
+ type VoiceOverEnhancementSettings,
+} from "@/voiceover/enhancements";
-export type VoiceOverStatus = "idle" | "recording";
+export type VoiceOverStatus = "idle" | "recording" | "processing";
interface VoiceOverStore {
status: VoiceOverStatus;
startedAtMs: number | null;
playheadStartTime: MediaTime | null;
startRecording: ({ playheadTime }: { playheadTime: MediaTime }) => void;
+ processRecording: () => void;
stopRecording: () => void;
}
+interface VoiceOverPreferencesStore {
+ selectedDeviceId: string | null;
+ enhancements: VoiceOverEnhancementSettings;
+ setSelectedDeviceId: ({ deviceId }: { deviceId: string | null }) => void;
+ setEnhancementEnabled: ({
+ id,
+ enabled,
+ }: {
+ id: keyof VoiceOverEnhancementSettings;
+ enabled: boolean;
+ }) => void;
+}
+
export const useVoiceOverStore = create<VoiceOverStore>()((set) => ({
status: "idle",
startedAtMs: null,
@@ -31,6 +50,13 @@ export const useVoiceOverStore = create<VoiceOverStore>()((set) => ({
});
},
+ processRecording: () => {
+ set({
+ status: "processing",
+ startedAtMs: null,
+ });
+ },
+
stopRecording: () => {
set({
status: "idle",
@@ -39,3 +65,32 @@ export const useVoiceOverStore = create<VoiceOverStore>()((set) => ({
});
},
}));
+
+export const useVoiceOverPreferencesStore = create<VoiceOverPreferencesStore>()(
+ persist(
+ (set) => ({
+ selectedDeviceId: null,
+ enhancements: DEFAULT_VOICE_OVER_ENHANCEMENTS,
+
+ setSelectedDeviceId: ({ deviceId }) => {
+ set({ selectedDeviceId: deviceId });
+ },
+
+ setEnhancementEnabled: ({ id, enabled }) => {
+ set((state) => ({
+ enhancements: {
+ ...state.enhancements,
+ [id]: enabled,
+ },
+ }));
+ },
+ }),
+ {
+ name: "voiceover-preferences-store",
+ partialize: (state) => ({
+ selectedDeviceId: state.selectedDeviceId,
+ enhancements: state.enhancements,
+ }),
+ },
+ ),
+);