Files
CyberGril/web/src/voice/custom.ts
T

250 lines
17 KiB
TypeScript

// ── Her own voice (CG-010 v1.12, HEL-900): give a reference recording, hear her say something with it, apply it. ──
// Contract: docs/voice-custom-contract-HEL899.md §2. Everything goes through the same-origin API; the page never talks to
// the voice panel or TTS itself and holds no credentials. Registration takes raw PCM WAV bytes, so whatever the browser
// recorded or the file it was given is decoded here and written out again as 16 kHz mono 16-bit WAV — a WebM renamed to
// .wav is not a WAV. The reference you gave and her synthesized preview are separate things and are never swapped.
export interface CustomVoice { id: string; label: string; source: 'builtin' | 'custom'; selected: boolean; previewed: boolean; registered_at: number | null }
export interface VoiceSelection { voice: string | null; label: string; updated_at: number | null }
export type VoiceList = { configured: false } | { configured: true; voices: CustomVoice[]; selection: VoiceSelection };
export interface Registered { id: string; label: string; prompt_text: string; prompt_text_source: 'user' | 'asr' }
export const RATE = 16000;
export const MIN_SECONDS = 2, SUGGEST_MIN = 5, SUGGEST_MAX = 20, MAX_BYTES = 15 * 1024 * 1024;
// Bigger files are refused before decoding (memory); the server's own line is 15 MB of WAV.
export const MAX_INPUT_BYTES = 50 * 1024 * 1024;
export const LABEL_MAX = 20, TEXT_MAX = 500, PREVIEW_MAX = 200;
export const PREVIEW_DEFAULT = '你好,这是我的声音,以后就用它陪你说话。';
export const SAY = {
noRoute: '这台服务还没有自定义声音的通路。文字和现有语音不受影响。',
notConfigured: '语音还没有在这台服务上配置,暂时不能换声音。文字仍可使用。',
offline: '网络断开了。你给的声音和填写的内容都还在,连上后可以再试。',
unreachable: '暂时连不上服务器。你给的声音和填写的内容都还在,可以再试。',
odd: '服务器这次没有正确响应,请稍后再试。',
failed: '这次没有完成,请稍后再试。',
tooShort: '这段太短了(至少 2 秒),请录 5–20 秒、念一两句完整的话。',
tooBig: '这段声音太大了,请选 5–20 秒的一段。',
fileTooBig: '这个文件太大了,请选 5–20 秒的一段。',
undecodable: '这个浏览器解不开这段音频,没法转成 WAV。可以换一个 WAV 文件,或在这里直接录一段。',
silent: '这段几乎没有声音,请靠近麦克风再录一次。',
badPreview: '这次试听的结果不是可播放的声音,没有拿来播放。可以再试一次。',
unplayable: '这段试听在这个浏览器里播不出来,没有当作试过。可以再试一次。',
denied: '没有拿到麦克风权限,没有开始录音。可以改用上传文件。',
noMic: '没有找到可用的麦克风。可以改用上传文件。',
micBusy: '麦克风被别的程序占用了,没有开始录音。',
insecure: '这个地址不是安全连接,浏览器不允许使用麦克风。可以改用上传文件,或从 HTTPS 入口打开。',
noRecorder: '这个浏览器不支持在网页里录音。可以改用上传文件。',
micGone: '麦克风被关掉了,这段没有录完,已经丢弃。',
recordFailed: '这次录音没有成功,已经丢弃,可以再录一次。',
} as const;
export class VoiceError extends Error { constructor(message: string, readonly status = 0) { super(message); } }
// ── WAV ──
export function encodeWav(samples: Float32Array, rate = RATE): ArrayBuffer {
const out = new ArrayBuffer(44 + samples.length * 2), v = new DataView(out);
const text = (at: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(at + i, s.charCodeAt(i)); };
text(0, 'RIFF'); v.setUint32(4, 36 + samples.length * 2, true); text(8, 'WAVE');
text(12, 'fmt '); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true);
v.setUint32(24, rate, true); v.setUint32(28, rate * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true);
text(36, 'data'); v.setUint32(40, samples.length * 2, true);
for (let i = 0; i < samples.length; i++) { const s = Math.max(-1, Math.min(1, samples[i])); v.setInt16(44 + i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true); }
return out;
}
export interface WavInfo { rate: number; channels: number; bits: number; seconds: number }
// Reads the header the way the server checks it (RIFF/WAVE, fmt and data chunks). Anything else is not a WAV.
export function wavInfo(buf: ArrayBuffer): WavInfo | null {
if (buf.byteLength < 44) return null;
const v = new DataView(buf), tag = (at: number) => String.fromCharCode(v.getUint8(at), v.getUint8(at + 1), v.getUint8(at + 2), v.getUint8(at + 3));
if (tag(0) !== 'RIFF' || tag(8) !== 'WAVE') return null;
let at = 12, fmt: Omit<WavInfo, 'seconds'> | null = null;
while (at + 8 <= buf.byteLength) {
const id = tag(at), size = v.getUint32(at + 4, true);
if (id === 'fmt ' && size >= 16 && at + 24 <= buf.byteLength) fmt = { channels: v.getUint16(at + 10, true), rate: v.getUint32(at + 12, true), bits: v.getUint16(at + 22, true) };
if (id === 'data') {
if (!fmt || !fmt.rate || !fmt.channels || !fmt.bits) return null;
const bytes = Math.min(size, buf.byteLength - at - 8);
return { ...fmt, seconds: bytes / (fmt.rate * fmt.channels * (fmt.bits / 8)) };
}
at += 8 + size + (size & 1);
}
return null;
}
export interface Decoded { sampleRate: number; length: number; numberOfChannels: number; getChannelData(channel: number): Float32Array }
export function mixdown(buf: Decoded): Float32Array {
const out = new Float32Array(buf.length), n = buf.numberOfChannels;
for (let c = 0; c < n; c++) { const d = buf.getChannelData(c); for (let i = 0; i < out.length; i++) out[i] += d[i] / n; }
return out;
}
// Decodes with the browser's own decoder, resampled to 16 kHz by an offline context (no device, nothing to close).
export function browserDecode(data: ArrayBuffer): Promise<Decoded> {
const Offline = (globalThis as { OfflineAudioContext?: typeof OfflineAudioContext }).OfflineAudioContext;
if (!Offline) return Promise.reject(new VoiceError(SAY.undecodable));
return new Offline(1, 1, RATE).decodeAudioData(data);
}
export interface Reference { wav: Blob; seconds: number }
// Any audio the browser can decode → the WAV the server accepts. Refuses what the server would refuse anyway.
export async function toReference(input: Blob, decode: (data: ArrayBuffer) => Promise<Decoded> = browserDecode): Promise<Reference> {
if (input.size > MAX_INPUT_BYTES) throw new VoiceError(SAY.fileTooBig);
let decoded: Decoded;
try { decoded = await decode(await input.arrayBuffer()); } catch { throw new VoiceError(SAY.undecodable); }
let samples = mixdown(decoded);
if (decoded.sampleRate !== RATE) samples = resample(samples, decoded.sampleRate, RATE);
const seconds = samples.length / RATE;
if (seconds < MIN_SECONDS) throw new VoiceError(SAY.tooShort);
if (!samples.some(s => Math.abs(s) > 0.003)) throw new VoiceError(SAY.silent);
const wav = encodeWav(samples, RATE);
if (wav.byteLength > MAX_BYTES) throw new VoiceError(SAY.tooBig);
return { wav: new Blob([wav], { type: 'audio/wav' }), seconds };
}
// Only used when a decoder ignored the requested rate; linear is enough for a reference prompt.
export function resample(input: Float32Array, from: number, to: number): Float32Array {
const out = new Float32Array(Math.floor(input.length * to / from)), step = from / to;
for (let i = 0; i < out.length; i++) { const x = i * step, j = Math.floor(x), f = x - j; out[i] = (input[j] ?? 0) * (1 - f) + (input[j + 1] ?? input[j] ?? 0) * f; }
return out;
}
// ── API (same origin) ──
type Fetch = typeof fetch;
const online = () => typeof navigator === 'undefined' || navigator.onLine !== false;
async function call(f: Fetch, path: string, init: RequestInit): Promise<Response> {
if (!online()) throw new VoiceError(SAY.offline);
try { return await f(`/api/voice/${path}`, init); }
catch (e) { if (init.signal?.aborted) throw e; throw new VoiceError(online() ? SAY.unreachable : SAY.offline); }
}
async function refusal(r: Response, fallback: string = SAY.failed): Promise<VoiceError> {
const data = await r.json().catch(() => null) as { detail?: unknown } | null;
return new VoiceError(typeof data?.detail === 'string' && data.detail ? data.detail : fallback, r.status);
}
const voiceItem = (v: unknown): v is CustomVoice => { const x = v as Record<string, unknown>; return !!x && typeof x === 'object' && typeof x.id === 'string' && /^[\w-]{1,40}$/u.test(x.id) && typeof x.label === 'string' && (x.source === 'builtin' || x.source === 'custom'); };
function selectionOf(v: unknown): VoiceSelection {
const s = v as Record<string, unknown> | null;
return { voice: typeof s?.voice === 'string' && s.voice ? s.voice : null, label: typeof s?.label === 'string' ? s.label : '', updated_at: Number.isFinite(s?.updated_at) ? s!.updated_at as number : null };
}
export async function listVoices(signal: AbortSignal, f: Fetch = fetch): Promise<VoiceList> {
const r = await call(f, 'voices', { signal });
// A server from before HEL-899 has no such route: "not wired", not a failure.
if (r.status === 404 || r.status === 405) throw new VoiceError(SAY.noRoute, r.status);
if (!r.ok) throw await refusal(r);
const data = await r.json().catch(() => null) as { configured?: unknown; voices?: unknown; selection?: unknown } | null;
if (!data || typeof data !== 'object') throw new VoiceError(SAY.odd);
if (data.configured !== true) return { configured: false };
const voices = (Array.isArray(data.voices) ? data.voices : []).filter(voiceItem).map(v => ({ id: v.id, label: v.label || v.id, source: v.source, selected: v.selected === true, previewed: v.previewed === true, registered_at: Number.isFinite(v.registered_at) ? v.registered_at : null }));
return { configured: true, voices, selection: selectionOf(data.selection) };
}
export async function registerVoice(wav: Blob, form: { label: string; text: string; allowAsr: boolean }, signal: AbortSignal, f: Fetch = fetch): Promise<Registered> {
const q = new URLSearchParams({ label: form.label.trim(), prompt_text: form.text.trim(), allow_asr: form.allowAsr && !form.text.trim() ? 'true' : 'false' });
const r = await call(f, `voices?${q}`, { method: 'POST', headers: { 'Content-Type': 'audio/wav' }, body: wav, signal });
if (!r.ok) throw await refusal(r);
const data = await r.json().catch(() => null) as Partial<Registered> | null;
if (!data || typeof data.id !== 'string' || !data.id) throw new VoiceError(SAY.odd);
return { id: data.id, label: typeof data.label === 'string' && data.label ? data.label : data.id, prompt_text: typeof data.prompt_text === 'string' ? data.prompt_text : '', prompt_text_source: data.prompt_text_source === 'asr' ? 'asr' : 'user' };
}
// A fresh synthesis every time. Only a real WAV comes back as playable; anything else is refused, never played.
export async function previewVoice(voice: string, text: string, signal: AbortSignal, f: Fetch = fetch): Promise<Blob> {
const r = await call(f, 'preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(text.trim() ? { voice, text: text.trim() } : { voice }), signal });
if (!r.ok) throw await refusal(r);
const data = await r.arrayBuffer().catch(() => null);
const info = data && wavInfo(data);
if (!data || !info || info.seconds <= 0) throw new VoiceError(SAY.badPreview);
return new Blob([data], { type: 'audio/wav' });
}
export async function applyVoice(voice: string, signal: AbortSignal, f: Fetch = fetch): Promise<VoiceSelection> {
const r = await call(f, 'selection', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ voice }), signal });
if (!r.ok) throw await refusal(r);
const s = selectionOf(await r.json().catch(() => null));
if (s.voice !== voice) throw new VoiceError(SAY.odd);
return s;
}
// ── Recording. Only ever started by a tap; stop, cancel, leaving or a lost track releases the microphone at once. ──
export type RecPhase = 'idle' | 'asking' | 'recording' | 'converting';
export interface RecView { phase: RecPhase; seconds: number }
export interface MediaParts {
getUserMedia(c: MediaStreamConstraints): Promise<MediaStream>;
recorder(stream: MediaStream): MediaRecorder;
secure(): boolean;
}
export const browserMedia: MediaParts = {
getUserMedia: c => navigator.mediaDevices.getUserMedia(c),
recorder: stream => new MediaRecorder(stream),
secure: () => typeof window !== 'undefined' && window.isSecureContext && !!navigator.mediaDevices?.getUserMedia,
};
function micFailure(e: unknown) {
const name = (e as { name?: string } | null)?.name || '';
if (name === 'NotFoundError' || name === 'DevicesNotFoundError' || name === 'OverconstrainedError') return SAY.noMic;
if (name === 'NotReadableError' || name === 'AbortError') return SAY.micBusy;
return SAY.denied;
}
export class Recorder {
private gen = 0;
private stream: MediaStream | null = null;
private rec: MediaRecorder | null = null;
private timer: ReturnType<typeof setInterval> | undefined;
private began = 0;
view: RecView = { phase: 'idle', seconds: 0 };
constructor(private on: { view(v: RecView): void; done(take: Blob): void; fail(message: string): void }, private media: MediaParts = browserMedia, private maxSeconds = SUGGEST_MAX, private now = () => Date.now()) {}
private show(v: RecView) { this.view = v; this.on.view(v); }
get busy() { return this.view.phase !== 'idle'; }
async start() {
if (this.busy) return;
const gen = ++this.gen;
if (!this.media.secure()) { this.on.fail(SAY.insecure); return; }
if (typeof MediaRecorder === 'undefined' && this.media === browserMedia) { this.on.fail(SAY.noRecorder); return; }
this.show({ phase: 'asking', seconds: 0 });
let stream: MediaStream;
// Audio only: no camera, and the browser's own echo/noise handling stays on.
try { stream = await this.media.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true }, video: false }); }
catch (e) { if (gen === this.gen) { this.show({ phase: 'idle', seconds: 0 }); this.on.fail(micFailure(e)); } return; }
// Cancelled (or left) while the permission prompt was up: the late microphone is closed, never used.
if (gen !== this.gen) { stream.getTracks().forEach(t => t.stop()); return; }
this.stream = stream;
let rec: MediaRecorder;
try { rec = this.media.recorder(stream); } catch { this.release(); this.show({ phase: 'idle', seconds: 0 }); this.on.fail(SAY.noRecorder); return; }
this.rec = rec;
const chunks: Blob[] = [];
rec.ondataavailable = e => { if (e.data?.size) chunks.push(e.data); };
rec.onstop = () => {
if (gen !== this.gen) return;
const take = new Blob(chunks, { type: rec.mimeType || chunks[0]?.type || '' });
this.release(); this.gen++;
this.show({ phase: 'converting', seconds: this.view.seconds });
this.on.done(take);
};
rec.onerror = () => { if (gen === this.gen) this.abandon(SAY.recordFailed); };
stream.getAudioTracks().forEach(t => t.addEventListener('ended', () => { if (gen === this.gen) this.abandon(SAY.micGone); }));
try { rec.start(250); } catch { this.abandon(SAY.recordFailed); return; }
this.began = this.now();
this.show({ phase: 'recording', seconds: 0 });
this.timer = setInterval(() => {
if (gen !== this.gen) return;
const seconds = (this.now() - this.began) / 1000;
if (seconds >= this.maxSeconds) { this.show({ phase: 'recording', seconds: this.maxSeconds }); this.stop(); return; }
this.show({ phase: 'recording', seconds });
}, 200);
}
// Keep what was recorded: the recorder flushes, then `done` hands the take over (and the microphone is already off).
stop() {
clearInterval(this.timer);
const rec = this.rec;
if (this.view.phase !== 'recording' || !rec || rec.state === 'inactive') { this.cancel(); return; }
this.view = { ...this.view, seconds: Math.min(this.maxSeconds, (this.now() - this.began) / 1000) };
try { rec.stop(); } catch { this.abandon(SAY.recordFailed); return; }
// The tracks stop now, not when the recorder gets round to its last chunk.
this.stream?.getTracks().forEach(t => t.stop());
}
// Throw the take away. Safe to call any time, also twice; a late permission answer is closed when it arrives.
cancel() { this.gen++; this.release(); if (this.view.phase !== 'idle') this.show({ phase: 'idle', seconds: 0 }); }
// The converted take was accepted or refused: back to idle.
settled() { if (this.view.phase === 'converting') this.show({ phase: 'idle', seconds: 0 }); }
private abandon(message: string) { this.cancel(); this.on.fail(message); }
private release() {
clearInterval(this.timer); this.timer = undefined;
const rec = this.rec; this.rec = null;
if (rec) { rec.ondataavailable = null; rec.onstop = null; rec.onerror = null; try { if (rec.state !== 'inactive') rec.stop(); } catch { /* Already stopped. */ } }
this.stream?.getTracks().forEach(t => t.stop()); this.stream = null;
}
}