250 lines
17 KiB
TypeScript
250 lines
17 KiB
TypeScript
// ── Her own voice (CG-010 v1.12, HEL-900): give a reference recording, hear her say something with it, apply it. ──
|
|
// Contract: docs/voice-custom-contract-HEL899.md §2. Everything goes through the same-origin API; the page never talks to
|
|
// the voice panel or TTS itself and holds no credentials. Registration takes raw PCM WAV bytes, so whatever the browser
|
|
// recorded or the file it was given is decoded here and written out again as 16 kHz mono 16-bit WAV — a WebM renamed to
|
|
// .wav is not a WAV. The reference you gave and her synthesized preview are separate things and are never swapped.
|
|
|
|
export interface CustomVoice { id: string; label: string; source: 'builtin' | 'custom'; selected: boolean; previewed: boolean; registered_at: number | null }
|
|
export interface VoiceSelection { voice: string | null; label: string; updated_at: number | null }
|
|
export type VoiceList = { configured: false } | { configured: true; voices: CustomVoice[]; selection: VoiceSelection };
|
|
export interface Registered { id: string; label: string; prompt_text: string; prompt_text_source: 'user' | 'asr' }
|
|
|
|
export const RATE = 16000;
|
|
export const MIN_SECONDS = 2, SUGGEST_MIN = 5, SUGGEST_MAX = 20, MAX_BYTES = 15 * 1024 * 1024;
|
|
// Bigger files are refused before decoding (memory); the server's own line is 15 MB of WAV.
|
|
export const MAX_INPUT_BYTES = 50 * 1024 * 1024;
|
|
export const LABEL_MAX = 20, TEXT_MAX = 500, PREVIEW_MAX = 200;
|
|
export const PREVIEW_DEFAULT = '你好,这是我的声音,以后就用它陪你说话。';
|
|
|
|
export const SAY = {
|
|
noRoute: '这台服务还没有自定义声音的通路。文字和现有语音不受影响。',
|
|
notConfigured: '语音还没有在这台服务上配置,暂时不能换声音。文字仍可使用。',
|
|
offline: '网络断开了。你给的声音和填写的内容都还在,连上后可以再试。',
|
|
unreachable: '暂时连不上服务器。你给的声音和填写的内容都还在,可以再试。',
|
|
odd: '服务器这次没有正确响应,请稍后再试。',
|
|
failed: '这次没有完成,请稍后再试。',
|
|
tooShort: '这段太短了(至少 2 秒),请录 5–20 秒、念一两句完整的话。',
|
|
tooBig: '这段声音太大了,请选 5–20 秒的一段。',
|
|
fileTooBig: '这个文件太大了,请选 5–20 秒的一段。',
|
|
undecodable: '这个浏览器解不开这段音频,没法转成 WAV。可以换一个 WAV 文件,或在这里直接录一段。',
|
|
silent: '这段几乎没有声音,请靠近麦克风再录一次。',
|
|
badPreview: '这次试听的结果不是可播放的声音,没有拿来播放。可以再试一次。',
|
|
unplayable: '这段试听在这个浏览器里播不出来,没有当作试过。可以再试一次。',
|
|
denied: '没有拿到麦克风权限,没有开始录音。可以改用上传文件。',
|
|
noMic: '没有找到可用的麦克风。可以改用上传文件。',
|
|
micBusy: '麦克风被别的程序占用了,没有开始录音。',
|
|
insecure: '这个地址不是安全连接,浏览器不允许使用麦克风。可以改用上传文件,或从 HTTPS 入口打开。',
|
|
noRecorder: '这个浏览器不支持在网页里录音。可以改用上传文件。',
|
|
micGone: '麦克风被关掉了,这段没有录完,已经丢弃。',
|
|
recordFailed: '这次录音没有成功,已经丢弃,可以再录一次。',
|
|
} as const;
|
|
|
|
export class VoiceError extends Error { constructor(message: string, readonly status = 0) { super(message); } }
|
|
|
|
// ── WAV ──
|
|
export function encodeWav(samples: Float32Array, rate = RATE): ArrayBuffer {
|
|
const out = new ArrayBuffer(44 + samples.length * 2), v = new DataView(out);
|
|
const text = (at: number, s: string) => { for (let i = 0; i < s.length; i++) v.setUint8(at + i, s.charCodeAt(i)); };
|
|
text(0, 'RIFF'); v.setUint32(4, 36 + samples.length * 2, true); text(8, 'WAVE');
|
|
text(12, 'fmt '); v.setUint32(16, 16, true); v.setUint16(20, 1, true); v.setUint16(22, 1, true);
|
|
v.setUint32(24, rate, true); v.setUint32(28, rate * 2, true); v.setUint16(32, 2, true); v.setUint16(34, 16, true);
|
|
text(36, 'data'); v.setUint32(40, samples.length * 2, true);
|
|
for (let i = 0; i < samples.length; i++) { const s = Math.max(-1, Math.min(1, samples[i])); v.setInt16(44 + i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true); }
|
|
return out;
|
|
}
|
|
export interface WavInfo { rate: number; channels: number; bits: number; seconds: number }
|
|
// Reads the header the way the server checks it (RIFF/WAVE, fmt and data chunks). Anything else is not a WAV.
|
|
export function wavInfo(buf: ArrayBuffer): WavInfo | null {
|
|
if (buf.byteLength < 44) return null;
|
|
const v = new DataView(buf), tag = (at: number) => String.fromCharCode(v.getUint8(at), v.getUint8(at + 1), v.getUint8(at + 2), v.getUint8(at + 3));
|
|
if (tag(0) !== 'RIFF' || tag(8) !== 'WAVE') return null;
|
|
let at = 12, fmt: Omit<WavInfo, 'seconds'> | null = null;
|
|
while (at + 8 <= buf.byteLength) {
|
|
const id = tag(at), size = v.getUint32(at + 4, true);
|
|
if (id === 'fmt ' && size >= 16 && at + 24 <= buf.byteLength) fmt = { channels: v.getUint16(at + 10, true), rate: v.getUint32(at + 12, true), bits: v.getUint16(at + 22, true) };
|
|
if (id === 'data') {
|
|
if (!fmt || !fmt.rate || !fmt.channels || !fmt.bits) return null;
|
|
const bytes = Math.min(size, buf.byteLength - at - 8);
|
|
return { ...fmt, seconds: bytes / (fmt.rate * fmt.channels * (fmt.bits / 8)) };
|
|
}
|
|
at += 8 + size + (size & 1);
|
|
}
|
|
return null;
|
|
}
|
|
export interface Decoded { sampleRate: number; length: number; numberOfChannels: number; getChannelData(channel: number): Float32Array }
|
|
export function mixdown(buf: Decoded): Float32Array {
|
|
const out = new Float32Array(buf.length), n = buf.numberOfChannels;
|
|
for (let c = 0; c < n; c++) { const d = buf.getChannelData(c); for (let i = 0; i < out.length; i++) out[i] += d[i] / n; }
|
|
return out;
|
|
}
|
|
// Decodes with the browser's own decoder, resampled to 16 kHz by an offline context (no device, nothing to close).
|
|
export function browserDecode(data: ArrayBuffer): Promise<Decoded> {
|
|
const Offline = (globalThis as { OfflineAudioContext?: typeof OfflineAudioContext }).OfflineAudioContext;
|
|
if (!Offline) return Promise.reject(new VoiceError(SAY.undecodable));
|
|
return new Offline(1, 1, RATE).decodeAudioData(data);
|
|
}
|
|
export interface Reference { wav: Blob; seconds: number }
|
|
// Any audio the browser can decode → the WAV the server accepts. Refuses what the server would refuse anyway.
|
|
export async function toReference(input: Blob, decode: (data: ArrayBuffer) => Promise<Decoded> = browserDecode): Promise<Reference> {
|
|
if (input.size > MAX_INPUT_BYTES) throw new VoiceError(SAY.fileTooBig);
|
|
let decoded: Decoded;
|
|
try { decoded = await decode(await input.arrayBuffer()); } catch { throw new VoiceError(SAY.undecodable); }
|
|
let samples = mixdown(decoded);
|
|
if (decoded.sampleRate !== RATE) samples = resample(samples, decoded.sampleRate, RATE);
|
|
const seconds = samples.length / RATE;
|
|
if (seconds < MIN_SECONDS) throw new VoiceError(SAY.tooShort);
|
|
if (!samples.some(s => Math.abs(s) > 0.003)) throw new VoiceError(SAY.silent);
|
|
const wav = encodeWav(samples, RATE);
|
|
if (wav.byteLength > MAX_BYTES) throw new VoiceError(SAY.tooBig);
|
|
return { wav: new Blob([wav], { type: 'audio/wav' }), seconds };
|
|
}
|
|
// Only used when a decoder ignored the requested rate; linear is enough for a reference prompt.
|
|
export function resample(input: Float32Array, from: number, to: number): Float32Array {
|
|
const out = new Float32Array(Math.floor(input.length * to / from)), step = from / to;
|
|
for (let i = 0; i < out.length; i++) { const x = i * step, j = Math.floor(x), f = x - j; out[i] = (input[j] ?? 0) * (1 - f) + (input[j + 1] ?? input[j] ?? 0) * f; }
|
|
return out;
|
|
}
|
|
|
|
// ── API (same origin) ──
|
|
type Fetch = typeof fetch;
|
|
const online = () => typeof navigator === 'undefined' || navigator.onLine !== false;
|
|
async function call(f: Fetch, path: string, init: RequestInit): Promise<Response> {
|
|
if (!online()) throw new VoiceError(SAY.offline);
|
|
try { return await f(`/api/voice/${path}`, init); }
|
|
catch (e) { if (init.signal?.aborted) throw e; throw new VoiceError(online() ? SAY.unreachable : SAY.offline); }
|
|
}
|
|
async function refusal(r: Response, fallback: string = SAY.failed): Promise<VoiceError> {
|
|
const data = await r.json().catch(() => null) as { detail?: unknown } | null;
|
|
return new VoiceError(typeof data?.detail === 'string' && data.detail ? data.detail : fallback, r.status);
|
|
}
|
|
const voiceItem = (v: unknown): v is CustomVoice => { const x = v as Record<string, unknown>; return !!x && typeof x === 'object' && typeof x.id === 'string' && /^[\w-]{1,40}$/u.test(x.id) && typeof x.label === 'string' && (x.source === 'builtin' || x.source === 'custom'); };
|
|
function selectionOf(v: unknown): VoiceSelection {
|
|
const s = v as Record<string, unknown> | null;
|
|
return { voice: typeof s?.voice === 'string' && s.voice ? s.voice : null, label: typeof s?.label === 'string' ? s.label : '', updated_at: Number.isFinite(s?.updated_at) ? s!.updated_at as number : null };
|
|
}
|
|
export async function listVoices(signal: AbortSignal, f: Fetch = fetch): Promise<VoiceList> {
|
|
const r = await call(f, 'voices', { signal });
|
|
// A server from before HEL-899 has no such route: "not wired", not a failure.
|
|
if (r.status === 404 || r.status === 405) throw new VoiceError(SAY.noRoute, r.status);
|
|
if (!r.ok) throw await refusal(r);
|
|
const data = await r.json().catch(() => null) as { configured?: unknown; voices?: unknown; selection?: unknown } | null;
|
|
if (!data || typeof data !== 'object') throw new VoiceError(SAY.odd);
|
|
if (data.configured !== true) return { configured: false };
|
|
const voices = (Array.isArray(data.voices) ? data.voices : []).filter(voiceItem).map(v => ({ id: v.id, label: v.label || v.id, source: v.source, selected: v.selected === true, previewed: v.previewed === true, registered_at: Number.isFinite(v.registered_at) ? v.registered_at : null }));
|
|
return { configured: true, voices, selection: selectionOf(data.selection) };
|
|
}
|
|
export async function registerVoice(wav: Blob, form: { label: string; text: string; allowAsr: boolean }, signal: AbortSignal, f: Fetch = fetch): Promise<Registered> {
|
|
const q = new URLSearchParams({ label: form.label.trim(), prompt_text: form.text.trim(), allow_asr: form.allowAsr && !form.text.trim() ? 'true' : 'false' });
|
|
const r = await call(f, `voices?${q}`, { method: 'POST', headers: { 'Content-Type': 'audio/wav' }, body: wav, signal });
|
|
if (!r.ok) throw await refusal(r);
|
|
const data = await r.json().catch(() => null) as Partial<Registered> | null;
|
|
if (!data || typeof data.id !== 'string' || !data.id) throw new VoiceError(SAY.odd);
|
|
return { id: data.id, label: typeof data.label === 'string' && data.label ? data.label : data.id, prompt_text: typeof data.prompt_text === 'string' ? data.prompt_text : '', prompt_text_source: data.prompt_text_source === 'asr' ? 'asr' : 'user' };
|
|
}
|
|
// A fresh synthesis every time. Only a real WAV comes back as playable; anything else is refused, never played.
|
|
export async function previewVoice(voice: string, text: string, signal: AbortSignal, f: Fetch = fetch): Promise<Blob> {
|
|
const r = await call(f, 'preview', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(text.trim() ? { voice, text: text.trim() } : { voice }), signal });
|
|
if (!r.ok) throw await refusal(r);
|
|
const data = await r.arrayBuffer().catch(() => null);
|
|
const info = data && wavInfo(data);
|
|
if (!data || !info || info.seconds <= 0) throw new VoiceError(SAY.badPreview);
|
|
return new Blob([data], { type: 'audio/wav' });
|
|
}
|
|
export async function applyVoice(voice: string, signal: AbortSignal, f: Fetch = fetch): Promise<VoiceSelection> {
|
|
const r = await call(f, 'selection', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ voice }), signal });
|
|
if (!r.ok) throw await refusal(r);
|
|
const s = selectionOf(await r.json().catch(() => null));
|
|
if (s.voice !== voice) throw new VoiceError(SAY.odd);
|
|
return s;
|
|
}
|
|
|
|
// ── Recording. Only ever started by a tap; stop, cancel, leaving or a lost track releases the microphone at once. ──
|
|
export type RecPhase = 'idle' | 'asking' | 'recording' | 'converting';
|
|
export interface RecView { phase: RecPhase; seconds: number }
|
|
export interface MediaParts {
|
|
getUserMedia(c: MediaStreamConstraints): Promise<MediaStream>;
|
|
recorder(stream: MediaStream): MediaRecorder;
|
|
secure(): boolean;
|
|
}
|
|
export const browserMedia: MediaParts = {
|
|
getUserMedia: c => navigator.mediaDevices.getUserMedia(c),
|
|
recorder: stream => new MediaRecorder(stream),
|
|
secure: () => typeof window !== 'undefined' && window.isSecureContext && !!navigator.mediaDevices?.getUserMedia,
|
|
};
|
|
function micFailure(e: unknown) {
|
|
const name = (e as { name?: string } | null)?.name || '';
|
|
if (name === 'NotFoundError' || name === 'DevicesNotFoundError' || name === 'OverconstrainedError') return SAY.noMic;
|
|
if (name === 'NotReadableError' || name === 'AbortError') return SAY.micBusy;
|
|
return SAY.denied;
|
|
}
|
|
export class Recorder {
|
|
private gen = 0;
|
|
private stream: MediaStream | null = null;
|
|
private rec: MediaRecorder | null = null;
|
|
private timer: ReturnType<typeof setInterval> | undefined;
|
|
private began = 0;
|
|
view: RecView = { phase: 'idle', seconds: 0 };
|
|
constructor(private on: { view(v: RecView): void; done(take: Blob): void; fail(message: string): void }, private media: MediaParts = browserMedia, private maxSeconds = SUGGEST_MAX, private now = () => Date.now()) {}
|
|
private show(v: RecView) { this.view = v; this.on.view(v); }
|
|
get busy() { return this.view.phase !== 'idle'; }
|
|
|
|
async start() {
|
|
if (this.busy) return;
|
|
const gen = ++this.gen;
|
|
if (!this.media.secure()) { this.on.fail(SAY.insecure); return; }
|
|
if (typeof MediaRecorder === 'undefined' && this.media === browserMedia) { this.on.fail(SAY.noRecorder); return; }
|
|
this.show({ phase: 'asking', seconds: 0 });
|
|
let stream: MediaStream;
|
|
// Audio only: no camera, and the browser's own echo/noise handling stays on.
|
|
try { stream = await this.media.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true }, video: false }); }
|
|
catch (e) { if (gen === this.gen) { this.show({ phase: 'idle', seconds: 0 }); this.on.fail(micFailure(e)); } return; }
|
|
// Cancelled (or left) while the permission prompt was up: the late microphone is closed, never used.
|
|
if (gen !== this.gen) { stream.getTracks().forEach(t => t.stop()); return; }
|
|
this.stream = stream;
|
|
let rec: MediaRecorder;
|
|
try { rec = this.media.recorder(stream); } catch { this.release(); this.show({ phase: 'idle', seconds: 0 }); this.on.fail(SAY.noRecorder); return; }
|
|
this.rec = rec;
|
|
const chunks: Blob[] = [];
|
|
rec.ondataavailable = e => { if (e.data?.size) chunks.push(e.data); };
|
|
rec.onstop = () => {
|
|
if (gen !== this.gen) return;
|
|
const take = new Blob(chunks, { type: rec.mimeType || chunks[0]?.type || '' });
|
|
this.release(); this.gen++;
|
|
this.show({ phase: 'converting', seconds: this.view.seconds });
|
|
this.on.done(take);
|
|
};
|
|
rec.onerror = () => { if (gen === this.gen) this.abandon(SAY.recordFailed); };
|
|
stream.getAudioTracks().forEach(t => t.addEventListener('ended', () => { if (gen === this.gen) this.abandon(SAY.micGone); }));
|
|
try { rec.start(250); } catch { this.abandon(SAY.recordFailed); return; }
|
|
this.began = this.now();
|
|
this.show({ phase: 'recording', seconds: 0 });
|
|
this.timer = setInterval(() => {
|
|
if (gen !== this.gen) return;
|
|
const seconds = (this.now() - this.began) / 1000;
|
|
if (seconds >= this.maxSeconds) { this.show({ phase: 'recording', seconds: this.maxSeconds }); this.stop(); return; }
|
|
this.show({ phase: 'recording', seconds });
|
|
}, 200);
|
|
}
|
|
// Keep what was recorded: the recorder flushes, then `done` hands the take over (and the microphone is already off).
|
|
stop() {
|
|
clearInterval(this.timer);
|
|
const rec = this.rec;
|
|
if (this.view.phase !== 'recording' || !rec || rec.state === 'inactive') { this.cancel(); return; }
|
|
this.view = { ...this.view, seconds: Math.min(this.maxSeconds, (this.now() - this.began) / 1000) };
|
|
try { rec.stop(); } catch { this.abandon(SAY.recordFailed); return; }
|
|
// The tracks stop now, not when the recorder gets round to its last chunk.
|
|
this.stream?.getTracks().forEach(t => t.stop());
|
|
}
|
|
// Throw the take away. Safe to call any time, also twice; a late permission answer is closed when it arrives.
|
|
cancel() { this.gen++; this.release(); if (this.view.phase !== 'idle') this.show({ phase: 'idle', seconds: 0 }); }
|
|
// The converted take was accepted or refused: back to idle.
|
|
settled() { if (this.view.phase === 'converting') this.show({ phase: 'idle', seconds: 0 }); }
|
|
private abandon(message: string) { this.cancel(); this.on.fail(message); }
|
|
private release() {
|
|
clearInterval(this.timer); this.timer = undefined;
|
|
const rec = this.rec; this.rec = null;
|
|
if (rec) { rec.ondataavailable = null; rec.onstop = null; rec.onerror = null; try { if (rec.state !== 'inactive') rec.stop(); } catch { /* Already stopped. */ } }
|
|
this.stream?.getTracks().forEach(t => t.stop()); this.stream = null;
|
|
}
|
|
}
|