249 lines
17 KiB
TypeScript
249 lines
17 KiB
TypeScript
import { describe, expect, it, vi, afterEach } from 'vitest';
|
|
import { mergeTurns, VoiceController, voiceLine, WORDS, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session';
|
|
import { readFileSync } from 'node:fs';
|
|
|
|
// HEL-879 unified review: the frames here are the ACTUAL worker output, generated by
|
|
// scripts/gen_hel879_unified_frames.py driving the fc905fe worker wiring (make_voice_llm +
|
|
// bind_voice_session + speak_proactive_row) on the .12 real SDK (livekit-agents 1.8.8) — see
|
|
// docs/rework/hel879/unified-review/frames/. adversarial-gates.jsonl is hand-built (page-guard
|
|
// probes), never claimed as worker output.
|
|
type Frame = Record<string, unknown> & { seq: number };
|
|
const load = (name: string): Frame[] => readFileSync(new URL(`./fixtures/hel879/${name}.jsonl`, import.meta.url), 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l));
|
|
const sidecar = JSON.parse(readFileSync(new URL('./fixtures/hel879/sidecar.json', import.meta.url), 'utf8')) as { scenarios: Record<string, { turn_id?: string; turn_ids?: Record<string, string>; acks?: string[]; start_while_playing?: boolean; kinds: string[] }> };
|
|
|
|
const gate = () => { let open!: () => void; const promise = new Promise<void>(r => { open = r; }); return { promise, open }; };
|
|
const flush = async (n = 8) => { for (let i = 0; i < n; i++) await Promise.resolve(); await new Promise(r => setTimeout(r, 0)); };
|
|
class Mic implements MicTrack { stopped = 0; ended?: () => void; stop() { this.stopped++; } onEnded(cb: () => void) { this.ended = cb; } }
|
|
class Room implements VoiceRoom {
|
|
closed = 0; published: MicTrack[] = []; hushed: boolean[] = []; on!: RoomEvents;
|
|
async publish(mic: MicTrack) { this.published.push(mic); }
|
|
hush(on: boolean) { this.hushed.push(on); }
|
|
async resumeAudio() {}
|
|
async close() { this.closed++; }
|
|
}
|
|
class Transport {
|
|
rooms: Room[] = []; mics: Mic[] = [];
|
|
async capture() { const m = new Mic(); this.mics.push(m); return m; }
|
|
async join(_url: string, _token: string, on: RoomEvents) { const r = new Room(); r.on = on; this.rooms.push(r); return r; }
|
|
}
|
|
class Server {
|
|
open = new Set<string>(); made = 0; calls: { method: string; url: string }[] = [];
|
|
fetch = vi.fn(async (url: string, init: RequestInit = {}) => {
|
|
const method = init.method || 'GET';
|
|
this.calls.push({ method, url });
|
|
if (url === '/api/voice/status') return new Response(JSON.stringify({ configured: true, tts_ok: true }));
|
|
if (url === '/api/voice/session' && method === 'POST') { const id = `s${++this.made}`; this.open.add(id); return new Response(JSON.stringify({ url: 'wss://lk.test', token: `t-${id}`, room: `cg-voice-${id}`, session_id: id, epoch: 3, expires_at: 0 }), { status: 201 }); }
|
|
const m = /^\/api\/voice\/session\/(.+)$/u.exec(url);
|
|
if (m && method === 'DELETE') return this.open.delete(decodeURIComponent(m[1])) ? new Response('{"ended":true}') : new Response('{"detail":"gone"}', { status: 404 });
|
|
return new Response('{"detail":"Not Found"}', { status: 404 });
|
|
});
|
|
}
|
|
function rig() {
|
|
const server = new Server(), transport = new Transport(), views: VoiceView[] = [], record = vi.fn(), spoke = vi.fn();
|
|
const c = new VoiceController({ transport: async () => transport, onView: v => views.push(v), onRecord: record, onSpoke: spoke, fetch: server.fetch as unknown as typeof fetch, secure: () => true, agentWaitMs: 1000 });
|
|
const view = () => views.at(-1)!;
|
|
return { server, transport, views, view, record, spoke, c };
|
|
}
|
|
async function connected() {
|
|
const r = rig(); await r.c.start(); r.transport.rooms[0].on.agentJoined(); return r;
|
|
}
|
|
const play = (room: Room, frames: Frame[]) => { for (const f of frames) room.on.subtitle(f); };
|
|
// Rebase a fixture's seq namespace (a second recording replayed into the same call must
|
|
// continue past every seq the page has already applied).
|
|
const rebase = (frames: Frame[], by: number): Frame[] => frames.map(f => ({ ...f, seq: f.seq + by }));
|
|
afterEach(() => { vi.restoreAllMocks(); });
|
|
|
|
describe('actual-worker frames (fc905fe on the real SDK) through the page', () => {
|
|
it('normal turn: draft, growing reply, stored final, ack — and the record takes over the stage line', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0], turn = sidecar.scenarios.normal.turn_id!;
|
|
const frames = load('worker-normal');
|
|
play(room, frames.slice(0, 2));
|
|
expect(r.view().heard).toBe('你在听吗');
|
|
const speech = (frames.find(f => f.kind === 'turn_start') as { speech_id: string }).speech_id;
|
|
play(room, frames.slice(2, 3));
|
|
expect(r.view().her).toEqual({ speech, text: '', state: 'open' });
|
|
play(room, frames.slice(3, 6));
|
|
expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'open' });
|
|
play(room, frames.slice(6, 7));
|
|
expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'final', turnId: turn });
|
|
play(room, frames.slice(7, 8));
|
|
expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'done', turnId: turn });
|
|
// The worker confirmed the receipt of exactly this turn (sidecar acks).
|
|
expect(sidecar.scenarios.normal.acks).toEqual([turn]);
|
|
// Stage takeover: while open, the subtitle line stands in with a voice- key; once the
|
|
// stored row (same turn id) is re-read, the line IS that row — no duplicate, no jump.
|
|
const open = { speech, text: '在听呢,你刚才说到一半的那件事', state: 'open' as const, turnId: undefined };
|
|
expect(voiceLine(open, [], 0)?.id).toBe(`voice-${speech}`);
|
|
const rows = [
|
|
{ id: '501', role: 'user' as const, text: '你在听吗', lifeIds: [], turnId: turn, status: 'sent' as const },
|
|
{ id: '502', role: 'companion' as const, text: '在听呢,你刚才说到一半的那件事我还在想。', lifeIds: [], turnId: turn, status: 'sent' as const },
|
|
];
|
|
expect(voiceLine({ ...open, state: 'final', turnId: turn, text: rows[1].text }, rows, 0)).toBeNull();
|
|
const known = [rows[0]];
|
|
const merged = mergeTurns(known, rows)!;
|
|
expect(merged.filter(m => m.turnId === turn)).toHaveLength(2);
|
|
expect(merged.at(-1)?.id).toBe('502');
|
|
});
|
|
|
|
it('barge-in: the cut turn never gets a final; the next turn lands clean with its own receipt', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
const frames = load('worker-interrupt');
|
|
play(room, frames);
|
|
const cut = (frames.find(f => f.kind === 'turn_cancel') as { speech_id: string }).speech_id;
|
|
const nextTurn = sidecar.scenarios.interrupt.turn_ids!.next;
|
|
expect(r.view().her?.turnId).toBe(nextTurn);
|
|
expect(r.view().her?.state).toBe('done');
|
|
expect(sidecar.scenarios.interrupt.acks).toEqual([nextTurn]); // the cut turn was never acked
|
|
expect(frames.filter(f => f.kind === 'assistant_final')).toHaveLength(1);
|
|
expect(frames.find(f => f.kind === 'turn_cancel')?.speech_id).toBe(cut);
|
|
});
|
|
|
|
it('phi: her failed line is heard (spoken), not stored; the speech closes without a final', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
play(room, load('worker-phi'));
|
|
expect(r.view().spoken).toContain('说不出来话');
|
|
expect(r.view().her).toBeNull();
|
|
expect(sidecar.scenarios.phi.acks).toEqual([]);
|
|
});
|
|
|
|
it('no legal receipt: cancel only — the fc905fe gate keeps assistant_final off the wire', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
const frames = load('worker-no_receipt');
|
|
play(room, frames);
|
|
expect(frames.some(f => f.kind === 'assistant_final')).toBe(false);
|
|
expect(frames.at(-1)?.kind).toBe('turn_cancel');
|
|
expect((frames.at(-1) as { reason?: string }).reason).toBe('no-receipt');
|
|
expect(r.view().her).toBeNull();
|
|
expect(sidecar.scenarios.no_receipt.acks).toEqual([]);
|
|
});
|
|
|
|
it('proactive: start is published while the handle is still playing (sidecar), end closes the row-bound line', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
const frames = load('worker-proactive');
|
|
expect(sidecar.scenarios.proactive.start_while_playing).toBe(true);
|
|
play(room, frames.slice(0, 2));
|
|
expect(r.view().her).toMatchObject({ state: 'done', proactive: true, messageId: 301, text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。' });
|
|
// The reach-out keeps the stored row's identity on the stage while it is being read;
|
|
// once that row is the newest line of the record, the subtitle line gives way to it.
|
|
const row = { id: '301', role: 'companion' as const, text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。', lifeIds: [], turnId: 'pro-1', proactive: true, status: 'sent' as const };
|
|
const line = voiceLine({ speech: 'p1', text: row.text, state: 'open', messageId: 301, proactive: true }, [], 0, [row]);
|
|
expect(line?.id).toBe('301');
|
|
expect(line?.proactive).toBe(true);
|
|
expect(voiceLine({ speech: 'p1', text: row.text, state: 'done', messageId: 301, proactive: true }, [row], 0)).toBeNull();
|
|
// The interrupted variant clears without a trace.
|
|
play(room, frames.slice(2));
|
|
expect(r.view().her).toBeNull();
|
|
expect(r.spoke).not.toHaveBeenCalled();
|
|
});
|
|
});
|
|
|
|
describe('page guards against bad frames (adversarial, hand-built)', () => {
|
|
it('replays, late frames of a closed speech and out-of-order frames are all dropped', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
const frames = load('adversarial-gates');
|
|
// Everything up to and including the later user interim, plus the replayed interim
|
|
// (old seq), the closed speech's late frames (fresh seqs, dead speech) and an
|
|
// out-of-order lower seq — none of the bad ones may move anything.
|
|
play(room, frames.slice(0, 11));
|
|
expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' });
|
|
expect(r.view().heard).toBe('再说点别的');
|
|
// The duplicate turn_start for speech_A (still inside the 64-entry window) is ignored.
|
|
play(room, frames.slice(12, 13));
|
|
expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' });
|
|
expect(r.view().her?.text).not.toContain('迟到');
|
|
// The out-of-order lower-seq frame (a lower seq arriving after a higher one) is
|
|
// already in slice(0, 11): it never overwrote the newer draft.
|
|
const outOfOrder = frames.find(f => f.kind === 'interim' && f.text === '乱序旧帧')!;
|
|
expect(outOfOrder.seq).toBeLessThan(9);
|
|
expect(r.view().heard).toBe('再说点别的');
|
|
// A replay of the whole first turn (start of speech_A again, still inside the
|
|
// window) keeps being ignored.
|
|
const dup = frames.filter(f => f.kind === 'turn_start' && f.speech_id === 'speech_A');
|
|
expect(dup.length).toBeGreaterThan(1);
|
|
play(room, dup.slice(1));
|
|
expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' });
|
|
});
|
|
|
|
it('a retired speech stays retired for the whole call, however many speeches closed after it (was: re-entered after 64)', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
const frames = load('adversarial-gates');
|
|
// Up to the end of the 66 squeeze speeches: speech_A is long closed; nothing of it is on the page.
|
|
const dupStart = frames.filter(f => f.kind === 'turn_start' && f.speech_id === 'speech_A').at(-1)!;
|
|
play(room, frames.filter(f => f.seq < dupStart.seq));
|
|
expect(r.view().her).toBeNull();
|
|
expect(dupStart.seq).toBeGreaterThan(140);
|
|
// The current worker never re-emits a speech id (each SDK speech gets a fresh id and interim.begin runs once
|
|
// per stream), so this needs a misbehaving/second source — a low-cost guard, not a fix of a worker fault.
|
|
// Its fresh seqs pass the sequence gate; the retired speech id does not.
|
|
const views = r.views.length;
|
|
play(room, frames.filter(f => f.seq >= dupStart.seq));
|
|
expect(r.view().her).toBeNull();
|
|
expect(r.views.slice(views).some(v => v.her || v.phase !== 'listening')).toBe(false);
|
|
// The seq gate is still strict on its own: nothing at or below the high-water mark gets in.
|
|
const high = frames.at(-1)!.seq;
|
|
room.on.subtitle({ v: 2, seq: high, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_new' });
|
|
expect(r.view().her).toBeNull();
|
|
room.on.subtitle({ v: 2, seq: high + 1, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_new' });
|
|
expect(r.view().her).toEqual({ speech: 'speech_new', text: '', state: 'open' });
|
|
});
|
|
it('the retired set is per call: a new call starts clean, and an old call\'s retired ids never leak or block', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
play(room, load('adversarial-gates'));
|
|
await r.c.end();
|
|
await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined();
|
|
// A new worker job: its own sequence from 1. A speech id the old call retired is not this call's business.
|
|
next.on.subtitle({ v: 2, seq: 1, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_A' });
|
|
next.on.subtitle({ v: 2, seq: 2, kind: 'assistant_interim', role: 'assistant', text: '新的一通。', speech_id: 'speech_A' });
|
|
expect(r.view().her).toEqual({ speech: 'speech_A', text: '新的一通。', state: 'open' });
|
|
// The old room, however late, moves nothing.
|
|
room.on.subtitle({ v: 2, seq: 9999, kind: 'turn_cancel', role: 'assistant', text: '', speech_id: 'speech_A' });
|
|
expect(r.view().her?.state).toBe('open');
|
|
await r.c.end();
|
|
});
|
|
|
|
it('your interim leaves her open line to the server\'s cancel; your final closes it at once', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
const frames = load('worker-interrupt');
|
|
play(room, frames.slice(0, 4));
|
|
expect(r.view().her?.state).toBe('open');
|
|
// A barge-in is the SDK's decision, not any sound: the worker's cancel clears her line.
|
|
play(room, frames.slice(4, 5));
|
|
expect(r.view().her).toBeNull();
|
|
// A user_final closes the current speech immediately (the server owns the turn gate).
|
|
play(room, frames.slice(5, 6));
|
|
expect(r.view().her).toBeNull();
|
|
expect(r.view().heard).toBe('那后来呢');
|
|
expect(r.spoke).toHaveBeenCalled();
|
|
// Your interim alone (a draft, not a turn) leaves her open line alone.
|
|
room.on.subtitle({ v: 2, seq: 99, kind: 'interim', role: 'user', text: '嗯' });
|
|
expect(r.view().heard).toBe('嗯');
|
|
});
|
|
|
|
it('text takeover while she speaks: her line and its audio go at once; a drop clears everything and a new call ignores the old room', async () => {
|
|
const r = await connected(), room = r.transport.rooms[0];
|
|
room.on.agentState('speaking');
|
|
const frames = load('worker-normal');
|
|
play(room, frames.slice(2, 6));
|
|
expect(r.view().her?.state).toBe('open');
|
|
r.c.hush();
|
|
expect(r.view().her).toBeNull();
|
|
expect(room.hushed).toEqual([true]);
|
|
play(room, frames.slice(6)); // her late final from the cut turn
|
|
expect(r.view().her).toBeNull();
|
|
// The room drops mid-sentence: everything clears with the call.
|
|
room.on.agentState('speaking');
|
|
play(room, rebase(load('worker-phi').slice(1, 3), 20));
|
|
expect(r.view().her?.state).toBe('open');
|
|
room.on.dropped(); await flush();
|
|
expect(r.view()).toMatchObject({ phase: 'off', her: null, notice: WORDS.dropped });
|
|
// A new call is a new room and a new sequence; stale frames of the old room change nothing.
|
|
await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined();
|
|
next.on.subtitle(frames.find(f => f.kind === 'turn_start')!);
|
|
next.on.subtitle(frames.find(f => f.kind === 'assistant_interim')!);
|
|
expect(r.view().her).toEqual({ speech: (frames.find(f => f.kind === 'turn_start') as { speech_id: string }).speech_id, text: '在听呢,', state: 'open' });
|
|
room.on.subtitle({ v: 2, seq: 99, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'stale' });
|
|
expect(r.view().her?.speech).not.toBe('stale');
|
|
await r.c.end();
|
|
});
|
|
});
|