// ── Her own voice (CG-010 v1.12, HEL-900): give a reference recording, hear her say something with it, apply it. ── // Contract: docs/voice-custom-contract-HEL899.md §2. Everything goes through the same-origin API; the page never talks to // the voice panel or TTS itself and holds no credentials. Registration takes raw PCM WAV bytes, so whatever the browser // recorded or the file it was given is decoded here and written out again as 16 kHz mono 16-bit WAV — a WebM renamed to // .wav is not a WAV. The reference you gave and her synthesized preview are separate things and are never swapped. export interface CustomVoice { id: string; label: string; source: 'builtin' | 'custom'; selected: boolean; previewed: boolean; registered_at: number | null } export interface VoiceSelection { voice: string | null; label: string; updated_at: number | null } export type VoiceList = { configured: false } | { configured: true; voices: CustomVoice[]; selection: VoiceSelection }; export interface Registered { id: string; label: string; prompt_text: string; prompt_text_source: 'user' | 'asr' } export const RATE = 16000; export const MIN_SECONDS = 2, SUGGEST_MIN = 5, SUGGEST_MAX = 20, MAX_BYTES = 15 * 1024 * 1024; // Bigger files are refused before decoding (memory); the server's own line is 15 MB of WAV. export const MAX_INPUT_BYTES = 50 * 1024 * 1024; export const LABEL_MAX = 20, TEXT_MAX = 500, PREVIEW_MAX = 200; export const PREVIEW_DEFAULT = '你好,这是我的声音,以后就用它陪你说话。'; export const SAY = { noRoute: '这台服务还没有自定义声音的通路。文字和现有语音不受影响。', notConfigured: '语音还没有在这台服务上配置,暂时不能换声音。文字仍可使用。', offline: '网络断开了。你给的声音和填写的内容都还在,连上后可以再试。', unreachable: '暂时连不上服务器。你给的声音和填写的内容都还在,可以再试。', odd: '服务器这次没有正确响应,请稍后再试。', failed: '这次没有完成,请稍后再试。', tooShort: '这段太短了(至少 2 秒),请录 5–20 秒、念一两句完整的话。', tooBig: '这段声音太大了,请选 5–20 秒的一段。', fileTooBig: '这个文件太大了,请选 5–20 秒的一段。', undecodable: '这个浏览器解不开这段音频,没法转成 WAV。可以换一个 WAV 文件,或在这里直接录一段。', silent: '这段几乎没有声音,请靠近麦克风再录一次。', badPreview: '这次试听的结果不是可播放的声音,没有拿来播放。可以再试一次。', unplayable: '这段试听在这个浏览器里播不出来,没有当作试过。可以再试一次。', denied: '没有拿到麦克风权限,没有开始录音。可以改用上传文件。', noMic: '没有找到可用的麦克风。可以改用上传文件。', micBusy: '麦克风被别的程序占用了,没有开始录音。', insecure: '这个地址不是安全连接,浏览器不允许使用麦克风。可以改用上传文件,或从 HTTPS 入口打开。', noRecorder: '这个浏览器不支持在网页里录音。可以改用上传文件。', micGone: '麦克风被关掉了,这段没有录完,已经丢弃。', recordFailed: '这次录音没有成功,已经丢弃,可以再录一次。', } as const; export class VoiceError extends Error { constructor(message: string, readonly status = 0) { super(message); } } // ── WAV ── export function encodeWav(samples: Float32Array, rate = RATE): ArrayBuffer { const out = new ArrayBuffer(44 + samples.length * 2), v = new DataView(out); const text = (at: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(at + i, s.charCodeAt(i)); }; text(0, 'RIFF'); v.setUint32(4, 36 + samples.length * 2, true); text(8, 'WAVE'); text(12, 'fmt '); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true); v.setUint32(24, rate, true); v.setUint32(28, rate * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true); text(36, 'data'); v.setUint32(40, samples.length * 2, true); for (let i = 0; i < samples.length; i++) { const s = Math.max(-1, Math.min(1, samples[i])); v.setInt16(44 + i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true); } return out; } export interface WavInfo { rate: number; channels: number; bits: number; seconds: number } // Reads the header the way the server checks it (RIFF/WAVE, fmt and data chunks). Anything else is not a WAV. export function wavInfo(buf: ArrayBuffer): WavInfo | null { if (buf.byteLength < 44) return null; const v = new DataView(buf), tag = (at: number) => String.fromCharCode(v.getUint8(at), v.getUint8(at + 1), v.getUint8(at + 2), v.getUint8(at + 3)); if (tag(0) !== 'RIFF' || tag(8) !== 'WAVE') return null; let at = 12, fmt: Omit | null = null; while (at + 8 <= buf.byteLength) { const id = tag(at), size = v.getUint32(at + 4, true); if (id === 'fmt ' && size >= 16 && at + 24 <= buf.byteLength) fmt = { channels: v.getUint16(at + 10, true), rate: v.getUint32(at + 12, true), bits: v.getUint16(at + 22, true) }; if (id === 'data') { if (!fmt || !fmt.rate || !fmt.channels || !fmt.bits) return null; const bytes = Math.min(size, buf.byteLength - at - 8); return { ...fmt, seconds: bytes / (fmt.rate * fmt.channels * (fmt.bits / 8)) }; } at += 8 + size + (size & 1); } return null; } export interface Decoded { sampleRate: number; length: number; numberOfChannels: number; getChannelData(channel: number): Float32Array } export function mixdown(buf: Decoded): Float32Array { const out = new Float32Array(buf.length), n = buf.numberOfChannels; for (let c = 0; c < n; c++) { const d = buf.getChannelData(c); for (let i = 0; i < out.length; i++) out[i] += d[i] / n; } return out; } // Decodes with the browser's own decoder, resampled to 16 kHz by an offline context (no device, nothing to close). export function browserDecode(data: ArrayBuffer): Promise { const Offline = (globalThis as { OfflineAudioContext?: typeof OfflineAudioContext }).OfflineAudioContext; if (!Offline) return Promise.reject(new VoiceError(SAY.undecodable)); return new Offline(1, 1, RATE).decodeAudioData(data); } export interface Reference { wav: Blob; seconds: number } // Any audio the browser can decode → the WAV the server accepts. Refuses what the server would refuse anyway. export async function toReference(input: Blob, decode: (data: ArrayBuffer) => Promise = browserDecode): Promise { if (input.size > MAX_INPUT_BYTES) throw new VoiceError(SAY.fileTooBig); let decoded: Decoded; try { decoded = await decode(await input.arrayBuffer()); } catch { throw new VoiceError(SAY.undecodable); } let samples = mixdown(decoded); if (decoded.sampleRate !== RATE) samples = resample(samples, decoded.sampleRate, RATE); const seconds = samples.length / RATE; if (seconds < MIN_SECONDS) throw new VoiceError(SAY.tooShort); if (!samples.some(s => Math.abs(s) > 0.003)) throw new VoiceError(SAY.silent); const wav = encodeWav(samples, RATE); if (wav.byteLength > MAX_BYTES) throw new VoiceError(SAY.tooBig); return { wav: new Blob([wav], { type: 'audio/wav' }), seconds }; } // Only used when a decoder ignored the requested rate; linear is enough for a reference prompt. export function resample(input: Float32Array, from: number, to: number): Float32Array { const out = new Float32Array(Math.floor(input.length * to / from)), step = from / to; for (let i = 0; i < out.length; i++) { const x = i * step, j = Math.floor(x), f = x - j; out[i] = (input[j] ?? 0) * (1 - f) + (input[j + 1] ?? input[j] ?? 0) * f; } return out; } // ── API (same origin) ── type Fetch = typeof fetch; const online = () => typeof navigator === 'undefined' || navigator.onLine !== false; async function call(f: Fetch, path: string, init: RequestInit): Promise { if (!online()) throw new VoiceError(SAY.offline); try { return await f(`/api/voice/${path}`, init); } catch (e) { if (init.signal?.aborted) throw e; throw new VoiceError(online() ? SAY.unreachable : SAY.offline); } } async function refusal(r: Response, fallback: string = SAY.failed): Promise { const data = await r.json().catch(() => null) as { detail?: unknown } | null; return new VoiceError(typeof data?.detail === 'string' && data.detail ? data.detail : fallback, r.status); } const voiceItem = (v: unknown): v is CustomVoice => { const x = v as Record; return !!x && typeof x === 'object' && typeof x.id === 'string' && /^[\w-]{1,40}$/u.test(x.id) && typeof x.label === 'string' && (x.source === 'builtin' || x.source === 'custom'); }; function selectionOf(v: unknown): VoiceSelection { const s = v as Record | null; return { voice: typeof s?.voice === 'string' && s.voice ? s.voice : null, label: typeof s?.label === 'string' ? s.label : '', updated_at: Number.isFinite(s?.updated_at) ? s!.updated_at as number : null }; } export async function listVoices(signal: AbortSignal, f: Fetch = fetch): Promise { const r = await call(f, 'voices', { signal }); // A server from before HEL-899 has no such route: "not wired", not a failure. if (r.status === 404 || r.status === 405) throw new VoiceError(SAY.noRoute, r.status); if (!r.ok) throw await refusal(r); const data = await r.json().catch(() => null) as { configured?: unknown; voices?: unknown; selection?: unknown } | null; if (!data || typeof data !== 'object') throw new VoiceError(SAY.odd); if (data.configured !== true) return { configured: false }; const voices = (Array.isArray(data.voices) ? data.voices : []).filter(voiceItem).map(v => ({ id: v.id, label: v.label || v.id, source: v.source, selected: v.selected === true, previewed: v.previewed === true, registered_at: Number.isFinite(v.registered_at) ? v.registered_at : null })); return { configured: true, voices, selection: selectionOf(data.selection) }; } export async function registerVoice(wav: Blob, form: { label: string; text: string; allowAsr: boolean }, signal: AbortSignal, f: Fetch = fetch): Promise { const q = new URLSearchParams({ label: form.label.trim(), prompt_text: form.text.trim(), allow_asr: form.allowAsr && !form.text.trim() ? 'true' : 'false' }); const r = await call(f, `voices?${q}`, { method: 'POST', headers: { 'Content-Type': 'audio/wav' }, body: wav, signal }); if (!r.ok) throw await refusal(r); const data = await r.json().catch(() => null) as Partial | null; if (!data || typeof data.id !== 'string' || !data.id) throw new VoiceError(SAY.odd); return { id: data.id, label: typeof data.label === 'string' && data.label ? data.label : data.id, prompt_text: typeof data.prompt_text === 'string' ? data.prompt_text : '', prompt_text_source: data.prompt_text_source === 'asr' ? 'asr' : 'user' }; } // A fresh synthesis every time. Only a real WAV comes back as playable; anything else is refused, never played. export async function previewVoice(voice: string, text: string, signal: AbortSignal, f: Fetch = fetch): Promise { const r = await call(f, 'preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(text.trim() ? { voice, text: text.trim() } : { voice }), signal }); if (!r.ok) throw await refusal(r); const data = await r.arrayBuffer().catch(() => null); const info = data && wavInfo(data); if (!data || !info || info.seconds <= 0) throw new VoiceError(SAY.badPreview); return new Blob([data], { type: 'audio/wav' }); } export async function applyVoice(voice: string, signal: AbortSignal, f: Fetch = fetch): Promise { const r = await call(f, 'selection', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ voice }), signal }); if (!r.ok) throw await refusal(r); const s = selectionOf(await r.json().catch(() => null)); if (s.voice !== voice) throw new VoiceError(SAY.odd); return s; } // ── Recording. Only ever started by a tap; stop, cancel, leaving or a lost track releases the microphone at once. ── export type RecPhase = 'idle' | 'asking' | 'recording' | 'converting'; export interface RecView { phase: RecPhase; seconds: number } export interface MediaParts { getUserMedia(c: MediaStreamConstraints): Promise; recorder(stream: MediaStream): MediaRecorder; secure(): boolean; } export const browserMedia: MediaParts = { getUserMedia: c => navigator.mediaDevices.getUserMedia(c), recorder: stream => new MediaRecorder(stream), secure: () => typeof window !== 'undefined' && window.isSecureContext && !!navigator.mediaDevices?.getUserMedia, }; function micFailure(e: unknown) { const name = (e as { name?: string } | null)?.name || ''; if (name === 'NotFoundError' || name === 'DevicesNotFoundError' || name === 'OverconstrainedError') return SAY.noMic; if (name === 'NotReadableError' || name === 'AbortError') return SAY.micBusy; return SAY.denied; } export class Recorder { private gen = 0; private stream: MediaStream | null = null; private rec: MediaRecorder | null = null; private timer: ReturnType | undefined; private began = 0; view: RecView = { phase: 'idle', seconds: 0 }; constructor(private on: { view(v: RecView): void; done(take: Blob): void; fail(message: string): void }, private media: MediaParts = browserMedia, private maxSeconds = SUGGEST_MAX, private now = () => Date.now()) {} private show(v: RecView) { this.view = v; this.on.view(v); } get busy() { return this.view.phase !== 'idle'; } async start() { if (this.busy) return; const gen = ++this.gen; if (!this.media.secure()) { this.on.fail(SAY.insecure); return; } if (typeof MediaRecorder === 'undefined' && this.media === browserMedia) { this.on.fail(SAY.noRecorder); return; } this.show({ phase: 'asking', seconds: 0 }); let stream: MediaStream; // Audio only: no camera, and the browser's own echo/noise handling stays on. try { stream = await this.media.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true }, video: false }); } catch (e) { if (gen === this.gen) { this.show({ phase: 'idle', seconds: 0 }); this.on.fail(micFailure(e)); } return; } // Cancelled (or left) while the permission prompt was up: the late microphone is closed, never used. if (gen !== this.gen) { stream.getTracks().forEach(t => t.stop()); return; } this.stream = stream; let rec: MediaRecorder; try { rec = this.media.recorder(stream); } catch { this.release(); this.show({ phase: 'idle', seconds: 0 }); this.on.fail(SAY.noRecorder); return; } this.rec = rec; const chunks: Blob[] = []; rec.ondataavailable = e => { if (e.data?.size) chunks.push(e.data); }; rec.onstop = () => { if (gen !== this.gen) return; const take = new Blob(chunks, { type: rec.mimeType || chunks[0]?.type || '' }); this.release(); this.gen++; this.show({ phase: 'converting', seconds: this.view.seconds }); this.on.done(take); }; rec.onerror = () => { if (gen === this.gen) this.abandon(SAY.recordFailed); }; stream.getAudioTracks().forEach(t => t.addEventListener('ended', () => { if (gen === this.gen) this.abandon(SAY.micGone); })); try { rec.start(250); } catch { this.abandon(SAY.recordFailed); return; } this.began = this.now(); this.show({ phase: 'recording', seconds: 0 }); this.timer = setInterval(() => { if (gen !== this.gen) return; const seconds = (this.now() - this.began) / 1000; if (seconds >= this.maxSeconds) { this.show({ phase: 'recording', seconds: this.maxSeconds }); this.stop(); return; } this.show({ phase: 'recording', seconds }); }, 200); } // Keep what was recorded: the recorder flushes, then `done` hands the take over (and the microphone is already off). stop() { clearInterval(this.timer); const rec = this.rec; if (this.view.phase !== 'recording' || !rec || rec.state === 'inactive') { this.cancel(); return; } this.view = { ...this.view, seconds: Math.min(this.maxSeconds, (this.now() - this.began) / 1000) }; try { rec.stop(); } catch { this.abandon(SAY.recordFailed); return; } // The tracks stop now, not when the recorder gets round to its last chunk. this.stream?.getTracks().forEach(t => t.stop()); } // Throw the take away. Safe to call any time, also twice; a late permission answer is closed when it arrives. cancel() { this.gen++; this.release(); if (this.view.phase !== 'idle') this.show({ phase: 'idle', seconds: 0 }); } // The converted take was accepted or refused: back to idle. settled() { if (this.view.phase === 'converting') this.show({ phase: 'idle', seconds: 0 }); } private abandon(message: string) { this.cancel(); this.on.fail(message); } private release() { clearInterval(this.timer); this.timer = undefined; const rec = this.rec; this.rec = null; if (rec) { rec.ondataavailable = null; rec.onstop = null; rec.onerror = null; try { if (rec.state !== 'inactive') rec.stop(); } catch { /* Already stopped. */ } } this.stream?.getTracks().forEach(t => t.stop()); this.stream = null; } }