diff options
| author | cashmere <cashmere@autistici.org> | 2026-08-16 06:16:56 +0200 |
|---|---|---|
| committer | cashmere <cashmere@autistici.org> | 2026-08-16 06:16:56 +0200 |
| commit | f1043734438c8518a302f698d9ae6583a75f0c14 (patch) | |
| tree | d3e636a96e6349c1ad17afa7b28c9ffe5c882e37 /apps/web/src/voiceover | |
| parent | 37d22b1e390bd6cea04bf8e15069281ee125cf10 (diff) | |
| download | opencut-f1043734438c8518a302f698d9ae6583a75f0c14.tar.gz | |
feat: add transitions, track mixing, and voiceover controls
Diffstat (limited to 'apps/web/src/voiceover')
| -rw-r--r-- | apps/web/src/voiceover/__tests__/devices.test.ts | 72 | ||||
| -rw-r--r-- | apps/web/src/voiceover/__tests__/enhancements.test.ts | 133 | ||||
| -rw-r--r-- | apps/web/src/voiceover/devices.ts | 93 | ||||
| -rw-r--r-- | apps/web/src/voiceover/enhancements.ts | 321 | ||||
| -rw-r--r-- | apps/web/src/voiceover/live-waveform.tsx | 171 | ||||
| -rw-r--r-- | apps/web/src/voiceover/recorder.ts | 40 | ||||
| -rw-r--r-- | apps/web/src/voiceover/voiceover-popover.tsx | 276 | ||||
| -rw-r--r-- | apps/web/src/voiceover/voiceover-store.ts | 57 |
8 files changed, 1160 insertions, 3 deletions
diff --git a/apps/web/src/voiceover/__tests__/devices.test.ts b/apps/web/src/voiceover/__tests__/devices.test.ts new file mode 100644 index 00000000..524e8c3f --- /dev/null +++ b/apps/web/src/voiceover/__tests__/devices.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, test } from "bun:test"; +import { + DEFAULT_AUDIO_INPUT_DEVICE_ID, + buildVoiceOverAudioConstraints, + normalizeSelectedDeviceId, + type VoiceOverInputDevice, +} from "@/voiceover/devices"; + +const devices: VoiceOverInputDevice[] = [ + { + deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID, + label: "System default microphone", + isDefault: true, + }, + { + deviceId: "blue-yeti-snowball", + label: "Blue Snowball", + isDefault: false, + }, +]; + +describe("buildVoiceOverAudioConstraints", () => { + test("uses default processing constraints without an exact device", () => { + const constraints = buildVoiceOverAudioConstraints({ deviceId: null }); + expect(constraints.audio).toEqual({ + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); + }); + + test("adds an exact device constraint for a selected microphone", () => { + const constraints = buildVoiceOverAudioConstraints({ + deviceId: "blue-yeti-snowball", + }); + expect(constraints.audio).toEqual({ + deviceId: { exact: "blue-yeti-snowball" }, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }); + }); +}); + +describe("normalizeSelectedDeviceId", () => { + test("keeps the selected device when it is still present", () => { + expect( + normalizeSelectedDeviceId({ + deviceId: "blue-yeti-snowball", + devices, + }), + ).toBe("blue-yeti-snowball"); + }); + + test("clears a selected device when it is no longer present", () => { + expect( + normalizeSelectedDeviceId({ + deviceId: "missing", + devices, + }), + ).toBeNull(); + }); + + test("keeps the browser default device id", () => { + expect( + normalizeSelectedDeviceId({ + deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID, + devices: [], + }), + ).toBe(DEFAULT_AUDIO_INPUT_DEVICE_ID); + }); +}); diff --git a/apps/web/src/voiceover/__tests__/enhancements.test.ts b/apps/web/src/voiceover/__tests__/enhancements.test.ts new file mode 100644 index 00000000..bc5b4b7c --- /dev/null +++ b/apps/web/src/voiceover/__tests__/enhancements.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, test } from "bun:test"; +import { + DEFAULT_VOICE_OVER_ENHANCEMENTS, + encodeAudioBufferToWav, + getAudioBufferPeak, + hasEnabledVoiceOverEnhancement, + type VoiceOverEnhancementSettings, +} from "@/voiceover/enhancements"; + +/* eslint-disable opencut/prefer-object-params -- Browser AudioBuffer methods are positional. */ +class FakeAudioBuffer implements AudioBuffer { + duration: number; + length: number; + numberOfChannels: number; + sampleRate: number; + + constructor( + private readonly channels: Array<Float32Array<ArrayBuffer>>, + sampleRate: number, + ) { + this.length = channels[0]?.length ?? 0; + this.numberOfChannels = channels.length; + this.sampleRate = sampleRate; + this.duration = this.length / sampleRate; + } + + copyFromChannel( + destination: Float32Array<ArrayBuffer>, + channelNumber: number, + bufferOffset = 0, + ): void { + destination.set( + this.getChannelData(channelNumber).slice( + bufferOffset, + bufferOffset + destination.length, + ), + ); + } + + copyToChannel( + source: Float32Array<ArrayBuffer>, + channelNumber: number, + bufferOffset = 0, + ): void { + this.getChannelData(channelNumber).set(source, bufferOffset); + } + + getChannelData(channel: number): Float32Array<ArrayBuffer> { + const data = this.channels[channel]; + if (!data) { + throw new Error(`Missing channel ${channel}`); + } + return data; + } +} +/* eslint-enable opencut/prefer-object-params */ + +describe("hasEnabledVoiceOverEnhancement", () => { + test("detects enabled defaults", () => { + expect( + hasEnabledVoiceOverEnhancement({ + settings: DEFAULT_VOICE_OVER_ENHANCEMENTS, + }), + ).toBe(true); + }); + + test("detects an all-disabled chain", () => { + const settings: VoiceOverEnhancementSettings = { + voiceBoost: false, + noiseGate: false, + highPass: false, + deEss: false, + limiter: false, + normalize: false, + }; + expect(hasEnabledVoiceOverEnhancement({ settings })).toBe(false); + }); +}); + +describe("getAudioBufferPeak", () => { + test("returns the maximum absolute sample", () => { + const buffer = new FakeAudioBuffer( + [ + createSamples([0, -0.25, 0.5]), + createSamples([0.1, -0.8, 0.2]), + ], + 48000, + ); + + expect(getAudioBufferPeak({ buffer })).toBeCloseTo(0.8); + }); +}); + +describe("encodeAudioBufferToWav", () => { + test("writes a 16-bit PCM WAV header", async () => { + const buffer = new FakeAudioBuffer( + [createSamples([0, 0.5, -0.5])], + 44100, + ); + + const blob = encodeAudioBufferToWav({ buffer }); + const view = new DataView(await blob.arrayBuffer()); + + expect(blob.type).toBe("audio/wav"); + expect(blob.size).toBe(50); + expect(readAscii({ view, offset: 0, length: 4 })).toBe("RIFF"); + expect(readAscii({ view, offset: 8, length: 4 })).toBe("WAVE"); + expect(view.getUint16(22, true)).toBe(1); + expect(view.getUint32(24, true)).toBe(44100); + expect(view.getUint16(34, true)).toBe(16); + expect(readAscii({ view, offset: 36, length: 4 })).toBe("data"); + }); +}); + +function readAscii({ + view, + offset, + length, +}: { + view: DataView; + offset: number; + length: number; +}): string { + return Array.from({ length }, (_, index) => + String.fromCharCode(view.getUint8(offset + index)), + ).join(""); +} + +function createSamples(values: number[]): Float32Array<ArrayBuffer> { + const samples = new Float32Array(new ArrayBuffer(values.length * 4)); + samples.set(values); + return samples; +} diff --git a/apps/web/src/voiceover/devices.ts b/apps/web/src/voiceover/devices.ts new file mode 100644 index 00000000..dd5b680c --- /dev/null +++ b/apps/web/src/voiceover/devices.ts @@ -0,0 +1,93 @@ +"use client"; + +export const DEFAULT_AUDIO_INPUT_DEVICE_ID = "default"; + +export interface VoiceOverInputDevice { + deviceId: string; + label: string; + isDefault: boolean; +} + +export function buildVoiceOverAudioConstraints({ + deviceId, +}: { + deviceId?: string | null; +}): MediaStreamConstraints { + const audio: MediaTrackConstraints = { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }; + + if (deviceId && deviceId !== DEFAULT_AUDIO_INPUT_DEVICE_ID) { + audio.deviceId = { exact: deviceId }; + } + + return { audio }; +} + +export function normalizeSelectedDeviceId({ + deviceId, + devices, +}: { + deviceId: string | null; + devices: VoiceOverInputDevice[]; +}): string | null { + if (!deviceId) { + return null; + } + + if (deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID) { + return deviceId; + } + + return devices.some((device) => device.deviceId === deviceId) + ? deviceId + : null; +} + +export async function listVoiceOverInputDevices({ + requestPermission = false, +}: { + requestPermission?: boolean; +} = {}): Promise<VoiceOverInputDevice[]> { + const mediaDevices = navigator.mediaDevices; + if (!mediaDevices?.enumerateDevices) { + throw new Error("Microphone selection is not supported in this browser"); + } + + let permissionStream: MediaStream | null = null; + if (requestPermission) { + permissionStream = await mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + } + + try { + const devices = await mediaDevices.enumerateDevices(); + let inputIndex = 1; + return devices + .filter((device) => device.kind === "audioinput") + .map((device) => { + const isDefault = device.deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID; + const fallbackLabel = isDefault + ? "System default microphone" + : `Microphone ${inputIndex++}`; + return { + deviceId: device.deviceId, + label: device.label || fallbackLabel, + isDefault, + }; + }); + } finally { + if (permissionStream) { + for (const track of permissionStream.getTracks()) { + track.stop(); + } + } + } +} diff --git a/apps/web/src/voiceover/enhancements.ts b/apps/web/src/voiceover/enhancements.ts new file mode 100644 index 00000000..8718406f --- /dev/null +++ b/apps/web/src/voiceover/enhancements.ts @@ -0,0 +1,321 @@ +"use client"; + +export interface VoiceOverEnhancementSettings { + voiceBoost: boolean; + noiseGate: boolean; + highPass: boolean; + deEss: boolean; + limiter: boolean; + normalize: boolean; +} + +export const DEFAULT_VOICE_OVER_ENHANCEMENTS: VoiceOverEnhancementSettings = { + voiceBoost: true, + noiseGate: true, + highPass: true, + deEss: false, + limiter: true, + normalize: true, +}; + +export const VOICE_OVER_ENHANCEMENT_LABELS: Record< + keyof VoiceOverEnhancementSettings, + { label: string; description: string } +> = { + voiceBoost: { + label: "Voice boost", + description: "Adds compression and presence for a stronger voice.", + }, + noiseGate: { + label: "Noise gate", + description: "Turns down quiet room noise between phrases.", + }, + highPass: { + label: "High-pass", + description: "Removes low rumble and desk vibration.", + }, + deEss: { + label: "De-ess", + description: "Softens harsh S sounds and upper fizz.", + }, + limiter: { + label: "Limiter", + description: "Catches peaks after boosts.", + }, + normalize: { + label: "Normalize", + description: "Raises the finished take to a consistent level.", + }, +}; + +const WAV_BITS_PER_SAMPLE = 16; +const WAV_HEADER_BYTES = 44; +const TARGET_PEAK = 0.92; +const NOISE_GATE_THRESHOLD = 0.018; +const NOISE_GATE_FLOOR = 0.18; + +export function hasEnabledVoiceOverEnhancement({ + settings, +}: { + settings: VoiceOverEnhancementSettings; +}): boolean { + return Object.values(settings).some(Boolean); +} + +export async function applyVoiceOverEnhancementsToBlob({ + blob, + settings, +}: { + blob: Blob; + settings: VoiceOverEnhancementSettings; +}): Promise<Blob> { + if (!hasEnabledVoiceOverEnhancement({ settings })) { + return blob; + } + + const audioContext = createAudioContext(); + try { + const buffer = await audioContext.decodeAudioData(await blob.arrayBuffer()); + const renderedBuffer = await renderEnhancedVoiceOverBuffer({ + buffer, + settings, + }); + return encodeAudioBufferToWav({ buffer: renderedBuffer }); + } finally { + void audioContext.close(); + } +} + +export async function renderEnhancedVoiceOverBuffer({ + buffer, + settings, +}: { + buffer: AudioBuffer; + settings: VoiceOverEnhancementSettings; +}): Promise<AudioBuffer> { + const offlineContext = new OfflineAudioContext( + buffer.numberOfChannels, + Math.max(1, buffer.length), + buffer.sampleRate, + ); + const source = offlineContext.createBufferSource(); + source.buffer = settings.noiseGate + ? applyNoiseGateToBuffer({ + buffer, + context: offlineContext, + threshold: NOISE_GATE_THRESHOLD, + floor: NOISE_GATE_FLOOR, + }) + : buffer; + + let current: AudioNode = source; + + if (settings.highPass) { + const highPass = offlineContext.createBiquadFilter(); + highPass.type = "highpass"; + highPass.frequency.value = 85; + highPass.Q.value = 0.7; + current.connect(highPass); + current = highPass; + } + + if (settings.voiceBoost) { + const presence = offlineContext.createBiquadFilter(); + presence.type = "peaking"; + presence.frequency.value = 3200; + presence.Q.value = 0.9; + presence.gain.value = 2.5; + + const compressor = offlineContext.createDynamicsCompressor(); + compressor.threshold.value = -24; + compressor.knee.value = 12; + compressor.ratio.value = 3.5; + compressor.attack.value = 0.004; + compressor.release.value = 0.16; + + const makeupGain = offlineContext.createGain(); + makeupGain.gain.value = 1.45; + + current.connect(presence); + presence.connect(compressor); + compressor.connect(makeupGain); + current = makeupGain; + } + + if (settings.deEss) { + const deEss = offlineContext.createBiquadFilter(); + deEss.type = "peaking"; + deEss.frequency.value = 6400; + deEss.Q.value = 2.2; + deEss.gain.value = -3; + current.connect(deEss); + current = deEss; + } + + if (settings.limiter) { + const limiter = offlineContext.createDynamicsCompressor(); + limiter.threshold.value = -1; + limiter.knee.value = 0; + limiter.ratio.value = 20; + limiter.attack.value = 0.001; + limiter.release.value = 0.1; + current.connect(limiter); + current = limiter; + } + + current.connect(offlineContext.destination); + source.start(0); + + const rendered = await offlineContext.startRendering(); + if (settings.normalize) { + normalizeAudioBufferPeak({ buffer: rendered, targetPeak: TARGET_PEAK }); + } + clampAudioBuffer({ buffer: rendered, maxPeak: 0.98 }); + return rendered; +} + +export function encodeAudioBufferToWav({ + buffer, +}: { + buffer: AudioBuffer; +}): Blob { + const numChannels = buffer.numberOfChannels; + const bytesPerSample = WAV_BITS_PER_SAMPLE / 8; + const dataSize = buffer.length * numChannels * bytesPerSample; + const arrayBuffer = new ArrayBuffer(WAV_HEADER_BYTES + dataSize); + const view = new DataView(arrayBuffer); + + writeAsciiString({ view, offset: 0, value: "RIFF" }); + view.setUint32(4, 36 + dataSize, true); + writeAsciiString({ view, offset: 8, value: "WAVE" }); + writeAsciiString({ view, offset: 12, value: "fmt " }); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numChannels, true); + view.setUint32(24, buffer.sampleRate, true); + view.setUint32(28, buffer.sampleRate * numChannels * bytesPerSample, true); + view.setUint16(32, numChannels * bytesPerSample, true); + view.setUint16(34, WAV_BITS_PER_SAMPLE, true); + writeAsciiString({ view, offset: 36, value: "data" }); + view.setUint32(40, dataSize, true); + + let offset = WAV_HEADER_BYTES; + for (let sampleIndex = 0; sampleIndex < buffer.length; sampleIndex++) { + for (let channel = 0; channel < numChannels; channel++) { + const sample = Math.max( + -1, + Math.min(1, buffer.getChannelData(channel)[sampleIndex] ?? 0), + ); + const int16 = sample < 0 ? sample * 0x8000 : sample * 0x7fff; + view.setInt16(offset, int16, true); + offset += bytesPerSample; + } + } + + return new Blob([arrayBuffer], { type: "audio/wav" }); +} + +export function getAudioBufferPeak({ buffer }: { buffer: AudioBuffer }): number { + let peak = 0; + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channelData = buffer.getChannelData(channel); + for (let index = 0; index < channelData.length; index++) { + peak = Math.max(peak, Math.abs(channelData[index] ?? 0)); + } + } + return peak; +} + +function normalizeAudioBufferPeak({ + buffer, + targetPeak, +}: { + buffer: AudioBuffer; + targetPeak: number; +}): void { + const peak = getAudioBufferPeak({ buffer }); + if (peak <= 0 || peak >= targetPeak) { + return; + } + + const gain = targetPeak / peak; + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channelData = buffer.getChannelData(channel); + for (let index = 0; index < channelData.length; index++) { + channelData[index] *= gain; + } + } +} + +function clampAudioBuffer({ + buffer, + maxPeak, +}: { + buffer: AudioBuffer; + maxPeak: number; +}): void { + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channelData = buffer.getChannelData(channel); + for (let index = 0; index < channelData.length; index++) { + channelData[index] = Math.max( + -maxPeak, + Math.min(maxPeak, channelData[index] ?? 0), + ); + } + } +} + +function applyNoiseGateToBuffer({ + buffer, + context, + threshold, + floor, +}: { + buffer: AudioBuffer; + context: OfflineAudioContext; + threshold: number; + floor: number; +}): AudioBuffer { + const gated = context.createBuffer( + buffer.numberOfChannels, + buffer.length, + buffer.sampleRate, + ); + + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const input = buffer.getChannelData(channel); + const output = gated.getChannelData(channel); + for (let index = 0; index < input.length; index++) { + const sample = input[index] ?? 0; + output[index] = + Math.abs(sample) < threshold + ? sample * floor + : sample; + } + } + + return gated; +} + +function createAudioContext(): AudioContext { + const AudioContextConstructor = + window.AudioContext || + (window as typeof window & { webkitAudioContext?: typeof AudioContext }) + .webkitAudioContext; + + return new AudioContextConstructor(); +} + +function writeAsciiString({ + view, + offset, + value, +}: { + view: DataView; + offset: number; + value: string; +}): void { + for (let index = 0; index < value.length; index++) { + view.setUint8(offset + index, value.charCodeAt(index)); + } +} diff --git a/apps/web/src/voiceover/live-waveform.tsx b/apps/web/src/voiceover/live-waveform.tsx new file mode 100644 index 00000000..cd25c833 --- /dev/null +++ b/apps/web/src/voiceover/live-waveform.tsx @@ -0,0 +1,171 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { cn } from "@/utils/ui"; +import { getActiveVoiceOverRecordingStream } from "@/voiceover/recorder"; + +const FFT_SIZE = 1024; +const HISTORY_SIZE = 96; + +export function VoiceOverLiveWaveform({ + recording, + className, +}: { + recording: boolean; + className?: string; +}) { + const canvasRef = useRef<HTMLCanvasElement>(null); + const levelRef = useRef(0); + const historyRef = useRef<number[]>( + Array.from({ length: HISTORY_SIZE }, () => 0), + ); + const [level, setLevel] = useState(0); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas || !recording) { + drawWaveform({ + canvas, + history: historyRef.current, + level: 0, + }); + setLevel(0); + return; + } + + const stream = getActiveVoiceOverRecordingStream(); + if (!stream) { + return; + } + + const audioContext = createAudioContext(); + const source = audioContext.createMediaStreamSource(stream); + const analyser = audioContext.createAnalyser(); + analyser.fftSize = FFT_SIZE; + analyser.smoothingTimeConstant = 0.72; + source.connect(analyser); + + const samples = new Uint8Array(analyser.fftSize); + let animationFrame = 0; + let lastLevelUpdate = 0; + + const render = (timestamp: number) => { + analyser.getByteTimeDomainData(samples); + const rms = getRmsLevel({ samples }); + levelRef.current = rms; + historyRef.current = [...historyRef.current.slice(1), rms]; + drawWaveform({ + canvas, + history: historyRef.current, + level: rms, + }); + + if (timestamp - lastLevelUpdate > 80) { + lastLevelUpdate = timestamp; + setLevel(rms); + } + animationFrame = requestAnimationFrame(render); + }; + + animationFrame = requestAnimationFrame(render); + + return () => { + cancelAnimationFrame(animationFrame); + source.disconnect(); + analyser.disconnect(); + void audioContext.close(); + }; + }, [recording]); + + return ( + <div className={cn("space-y-2", className)}> + <div className="bg-muted relative h-16 overflow-hidden rounded-sm border"> + <canvas ref={canvasRef} className="absolute inset-0 size-full" /> + </div> + <div className="bg-muted h-1.5 overflow-hidden rounded-full"> + <div + className="bg-primary h-full rounded-full transition-[width]" + style={{ width: `${Math.min(100, Math.round(level * 140))}%` }} + /> + </div> + </div> + ); +} + +function getRmsLevel({ samples }: { samples: Uint8Array }): number { + let sum = 0; + for (let index = 0; index < samples.length; index++) { + const centered = ((samples[index] ?? 128) - 128) / 128; + sum += centered * centered; + } + return Math.sqrt(sum / samples.length); +} + +function drawWaveform({ + canvas, + history, + level, +}: { + canvas: HTMLCanvasElement | null; + history: number[]; + level: number; +}): void { + if (!canvas) { + return; + } + + const rect = canvas.getBoundingClientRect(); + const dpr = window.devicePixelRatio || 1; + const width = Math.max(1, Math.round(rect.width * dpr)); + const height = Math.max(1, Math.round(rect.height * dpr)); + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + + const ctx = canvas.getContext("2d"); + if (!ctx) { + return; + } + + ctx.setTransform(1, 0, 0, 1, 0, 0); + ctx.clearRect(0, 0, width, height); + + const centerY = height / 2; + const step = width / Math.max(1, history.length); + + ctx.strokeStyle = "rgba(148, 163, 184, 0.35)"; + ctx.lineWidth = 1; + ctx.beginPath(); + ctx.moveTo(0, centerY); + ctx.lineTo(width, centerY); + ctx.stroke(); + + for (let index = 0; index < history.length; index++) { + const value = Math.max(0, history[index] ?? 0); + const barHeight = Math.max(1, Math.min(height, value * height * 2.8)); + const x = Math.round(index * step); + const y = Math.round(centerY - barHeight / 2); + ctx.fillStyle = + value > 0.62 + ? "rgba(239, 68, 68, 0.9)" + : value > 0.32 + ? "rgba(245, 158, 11, 0.9)" + : "rgba(34, 197, 94, 0.9)"; + ctx.fillRect(x, y, Math.max(1, Math.floor(step - 1)), barHeight); + } + + if (level > 0.78) { + ctx.fillStyle = "rgba(239, 68, 68, 0.18)"; + ctx.fillRect(0, 0, width, height); + } +} + +function createAudioContext(): AudioContext { + const AudioContextConstructor = + window.AudioContext || + (window as typeof window & { webkitAudioContext?: typeof AudioContext }) + .webkitAudioContext; + + return new AudioContextConstructor(); +} diff --git a/apps/web/src/voiceover/recorder.ts b/apps/web/src/voiceover/recorder.ts index b71ca74f..48322898 100644 --- a/apps/web/src/voiceover/recorder.ts +++ b/apps/web/src/voiceover/recorder.ts @@ -5,6 +5,8 @@ * actual microphone stream. */ +import { buildVoiceOverAudioConstraints } from "@/voiceover/devices"; + export const PREFERRED_MIME_TYPES = [ "audio/webm;codecs=opus", "audio/webm", @@ -95,6 +97,11 @@ interface ActiveRecording { startedAtMs: number; } +export interface VoiceOverRecordingSession { + stream: MediaStream; + mimeType: string; +} + let activeRecording: ActiveRecording | null = null; function stopStreamTracks({ stream }: { stream: MediaStream }): void { @@ -103,7 +110,15 @@ function stopStreamTracks({ stream }: { stream: MediaStream }): void { } } -export async function startVoiceOverRecording(): Promise<void> { +export function getActiveVoiceOverRecordingStream(): MediaStream | null { + return activeRecording?.stream ?? null; +} + +export async function startVoiceOverRecording({ + deviceId, +}: { + deviceId?: string | null; +} = {}): Promise<VoiceOverRecordingSession> { if (activeRecording) { throw new Error("A voice-over recording is already in progress"); } @@ -113,7 +128,24 @@ export async function startVoiceOverRecording(): Promise<void> { throw new Error("Voice-over recording is not supported in this browser"); } - const stream = await mediaDevices.getUserMedia({ audio: true }); + let stream: MediaStream; + try { + stream = await mediaDevices.getUserMedia( + buildVoiceOverAudioConstraints({ deviceId }), + ); + } catch (error) { + if ( + deviceId && + error instanceof DOMException && + (error.name === "NotFoundError" || error.name === "OverconstrainedError") + ) { + stream = await mediaDevices.getUserMedia( + buildVoiceOverAudioConstraints({ deviceId: null }), + ); + } else { + throw error; + } + } try { const mimeType = pickSupportedMimeType({ isTypeSupported: (candidate) => MediaRecorder.isTypeSupported(candidate), @@ -135,6 +167,10 @@ export async function startVoiceOverRecording(): Promise<void> { mimeType: recorder.mimeType || mimeType || "audio/webm", startedAtMs: performance.now(), }; + return { + stream, + mimeType: activeRecording.mimeType, + }; } catch (error) { stopStreamTracks({ stream }); throw error; diff --git a/apps/web/src/voiceover/voiceover-popover.tsx b/apps/web/src/voiceover/voiceover-popover.tsx new file mode 100644 index 00000000..a1f74646 --- /dev/null +++ b/apps/web/src/voiceover/voiceover-popover.tsx @@ -0,0 +1,276 @@ +"use client"; + +import { useCallback, useMemo, useState } from "react"; +import { MicIcon, StopIcon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { RefreshCw } from "lucide-react"; +import { toast } from "sonner"; +import { invokeAction } from "@/actions"; +import { Button } from "@/components/ui/button"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { cn } from "@/utils/ui"; +import { + DEFAULT_AUDIO_INPUT_DEVICE_ID, + listVoiceOverInputDevices, + normalizeSelectedDeviceId, + type VoiceOverInputDevice, +} from "@/voiceover/devices"; +import { + VOICE_OVER_ENHANCEMENT_LABELS, + type VoiceOverEnhancementSettings, +} from "@/voiceover/enhancements"; +import { VoiceOverLiveWaveform } from "@/voiceover/live-waveform"; +import { + formatElapsedSeconds, + getVoiceOverErrorMessage, +} from "@/voiceover/recorder"; +import { useRecordingElapsedSeconds } from "@/voiceover/use-recording-elapsed"; +import { + useVoiceOverPreferencesStore, + useVoiceOverStore, +} from "@/voiceover/voiceover-store"; + +const ENHANCEMENT_ORDER: Array<keyof VoiceOverEnhancementSettings> = [ + "voiceBoost", + "noiseGate", + "highPass", + "deEss", + "limiter", + "normalize", +]; + +export function VoiceOverRecorderPopover() { + const [open, setOpen] = useState(false); + const [devices, setDevices] = useState<VoiceOverInputDevice[]>([]); + const [isLoadingDevices, setIsLoadingDevices] = useState(false); + const status = useVoiceOverStore((state) => state.status); + const isRecording = status === "recording"; + const isProcessing = status === "processing"; + const elapsedSeconds = useRecordingElapsedSeconds(); + const selectedDeviceId = useVoiceOverPreferencesStore( + (state) => state.selectedDeviceId, + ); + const enhancements = useVoiceOverPreferencesStore( + (state) => state.enhancements, + ); + const setSelectedDeviceId = useVoiceOverPreferencesStore( + (state) => state.setSelectedDeviceId, + ); + const setEnhancementEnabled = useVoiceOverPreferencesStore( + (state) => state.setEnhancementEnabled, + ); + + const devicesWithDefault = useMemo(() => { + if ( + devices.some( + (device) => device.deviceId === DEFAULT_AUDIO_INPUT_DEVICE_ID, + ) + ) { + return devices; + } + return [ + { + deviceId: DEFAULT_AUDIO_INPUT_DEVICE_ID, + label: "System default microphone", + isDefault: true, + }, + ...devices, + ]; + }, [devices]); + + const selectedValue = selectedDeviceId ?? DEFAULT_AUDIO_INPUT_DEVICE_ID; + + const refreshDevices = useCallback( + async ({ requestPermission }: { requestPermission: boolean }) => { + setIsLoadingDevices(true); + try { + const nextDevices = await listVoiceOverInputDevices({ + requestPermission, + }); + setDevices(nextDevices); + const normalizedDeviceId = normalizeSelectedDeviceId({ + deviceId: selectedDeviceId, + devices: nextDevices, + }); + if (normalizedDeviceId !== selectedDeviceId) { + setSelectedDeviceId({ deviceId: normalizedDeviceId }); + } + } catch (error) { + toast.error(getVoiceOverErrorMessage({ error })); + } finally { + setIsLoadingDevices(false); + } + }, + [selectedDeviceId, setSelectedDeviceId], + ); + + const handleOpenChange = useCallback( + (nextOpen: boolean) => { + setOpen(nextOpen); + if (nextOpen) { + void refreshDevices({ requestPermission: true }); + } + }, + [refreshDevices], + ); + + const handleRecordClick = (event: React.MouseEvent) => { + event.stopPropagation(); + void invokeAction("voice-over"); + }; + + return ( + <Popover open={open} onOpenChange={handleOpenChange}> + <TooltipProvider delayDuration={500}> + <Tooltip delayDuration={200}> + <TooltipTrigger asChild> + <PopoverTrigger asChild> + <Button + variant={isRecording ? "secondary" : "text"} + size="icon" + disabled={isProcessing} + className="rounded-sm" + onClick={(event) => event.stopPropagation()} + > + <HugeiconsIcon icon={isRecording ? StopIcon : MicIcon} /> + </Button> + </PopoverTrigger> + </TooltipTrigger> + <TooltipContent> + {isProcessing + ? "Saving voice-over" + : isRecording + ? "Stop voice-over" + : "Record voice-over"} + </TooltipContent> + </Tooltip> + </TooltipProvider> + + <PopoverContent + align="start" + className="w-80 p-3" + onClick={(event) => event.stopPropagation()} + > + <div className="space-y-3"> + <div className="flex items-center justify-between gap-3"> + <div className="min-w-0"> + <div className="text-sm font-medium">Voice-over</div> + <div className="text-muted-foreground text-xs tabular-nums"> + {isRecording + ? formatElapsedSeconds({ seconds: elapsedSeconds }) + : isProcessing + ? "Saving" + : "Ready"} + </div> + </div> + <Button + variant={isRecording ? "secondary" : "default"} + size="sm" + disabled={isProcessing} + onClick={handleRecordClick} + className={cn( + "gap-1.5 rounded-sm", + isRecording ? "text-destructive" : "", + )} + > + <HugeiconsIcon + icon={isRecording ? StopIcon : MicIcon} + className="size-4" + /> + {isProcessing ? "Saving" : isRecording ? "Stop" : "Record"} + </Button> + </div> + + <VoiceOverLiveWaveform recording={isRecording} /> + + <div className="space-y-1.5"> + <div className="flex items-center justify-between gap-2"> + <div className="text-muted-foreground text-xs font-medium"> + Input + </div> + <Button + variant="text" + size="icon" + disabled={isLoadingDevices || isRecording} + onClick={() => refreshDevices({ requestPermission: true })} + className="size-6 rounded-sm" + > + <RefreshCw className="size-3.5" /> + </Button> + </div> + <Select + value={selectedValue} + disabled={isRecording} + onValueChange={(value) => + setSelectedDeviceId({ + deviceId: + value === DEFAULT_AUDIO_INPUT_DEVICE_ID ? null : value, + }) + } + > + <SelectTrigger className="w-full" variant="outline" size="sm"> + <SelectValue placeholder="System default microphone" /> + </SelectTrigger> + <SelectContent> + {devicesWithDefault.map((device) => ( + <SelectItem key={device.deviceId} value={device.deviceId}> + {device.label} + </SelectItem> + ))} + </SelectContent> + </Select> + </div> + + <div className="border-border border-t pt-3"> + <div className="text-muted-foreground mb-2 text-xs font-medium"> + Enhancements + </div> + <div className="space-y-2"> + {ENHANCEMENT_ORDER.map((id) => { + const option = VOICE_OVER_ENHANCEMENT_LABELS[id]; + return ( + <div + key={id} + className="flex items-start justify-between gap-3" + > + <span className="min-w-0"> + <span className="block text-sm">{option.label}</span> + <span className="text-muted-foreground block text-xs leading-snug"> + {option.description} + </span> + </span> + <Switch + checked={enhancements[id]} + aria-label={option.label} + onCheckedChange={(enabled) => + setEnhancementEnabled({ id, enabled }) + } + /> + </div> + ); + })} + </div> + </div> + </div> + </PopoverContent> + </Popover> + ); +} diff --git a/apps/web/src/voiceover/voiceover-store.ts b/apps/web/src/voiceover/voiceover-store.ts index 44ccae6c..e4024c1e 100644 --- a/apps/web/src/voiceover/voiceover-store.ts +++ b/apps/web/src/voiceover/voiceover-store.ts @@ -6,18 +6,37 @@ */ import { create } from "zustand"; +import { persist } from "zustand/middleware"; import type { MediaTime } from "@/wasm"; +import { + DEFAULT_VOICE_OVER_ENHANCEMENTS, + type VoiceOverEnhancementSettings, +} from "@/voiceover/enhancements"; -export type VoiceOverStatus = "idle" | "recording"; +export type VoiceOverStatus = "idle" | "recording" | "processing"; interface VoiceOverStore { status: VoiceOverStatus; startedAtMs: number | null; playheadStartTime: MediaTime | null; startRecording: ({ playheadTime }: { playheadTime: MediaTime }) => void; + processRecording: () => void; stopRecording: () => void; } +interface VoiceOverPreferencesStore { + selectedDeviceId: string | null; + enhancements: VoiceOverEnhancementSettings; + setSelectedDeviceId: ({ deviceId }: { deviceId: string | null }) => void; + setEnhancementEnabled: ({ + id, + enabled, + }: { + id: keyof VoiceOverEnhancementSettings; + enabled: boolean; + }) => void; +} + export const useVoiceOverStore = create<VoiceOverStore>()((set) => ({ status: "idle", startedAtMs: null, @@ -31,6 +50,13 @@ export const useVoiceOverStore = create<VoiceOverStore>()((set) => ({ }); }, + processRecording: () => { + set({ + status: "processing", + startedAtMs: null, + }); + }, + stopRecording: () => { set({ status: "idle", @@ -39,3 +65,32 @@ export const useVoiceOverStore = create<VoiceOverStore>()((set) => ({ }); }, })); + +export const useVoiceOverPreferencesStore = create<VoiceOverPreferencesStore>()( + persist( + (set) => ({ + selectedDeviceId: null, + enhancements: DEFAULT_VOICE_OVER_ENHANCEMENTS, + + setSelectedDeviceId: ({ deviceId }) => { + set({ selectedDeviceId: deviceId }); + }, + + setEnhancementEnabled: ({ id, enabled }) => { + set((state) => ({ + enhancements: { + ...state.enhancements, + [id]: enabled, + }, + })); + }, + }), + { + name: "voiceover-preferences-store", + partialize: (state) => ({ + selectedDeviceId: state.selectedDeviceId, + enhancements: state.enhancements, + }), + }, + ), +); |
