import { describe, expect, it, vi, afterEach } from 'vitest'; import { mergeTurns, VoiceController, voiceLine, WORDS, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session'; import { readFileSync } from 'node:fs'; // HEL-879 unified review: the frames here are the ACTUAL worker output, generated by // scripts/gen_hel879_unified_frames.py driving the fc905fe worker wiring (make_voice_llm + // bind_voice_session + speak_proactive_row) on the .12 real SDK (livekit-agents 1.8.8) — see // docs/rework/hel879/unified-review/frames/. adversarial-gates.jsonl is hand-built (page-guard // probes), never claimed as worker output. type Frame = Record & { seq: number }; const load = (name: string): Frame[] => readFileSync(new URL(`./fixtures/hel879/${name}.jsonl`, import.meta.url), 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l)); const sidecar = JSON.parse(readFileSync(new URL('./fixtures/hel879/sidecar.json', import.meta.url), 'utf8')) as { scenarios: Record; acks?: string[]; start_while_playing?: boolean; kinds: string[] }> }; const gate = () => { let open!: () => void; const promise = new Promise(r => { open = r; }); return { promise, open }; }; const flush = async (n = 8) => { for (let i = 0; i < n; i++) await Promise.resolve(); await new Promise(r => setTimeout(r, 0)); }; class Mic implements MicTrack { stopped = 0; ended?: () => void; stop() { this.stopped++; } onEnded(cb: () => void) { this.ended = cb; } } class Room implements VoiceRoom { closed = 0; published: MicTrack[] = []; hushed: boolean[] = []; on!: RoomEvents; async publish(mic: MicTrack) { this.published.push(mic); } hush(on: boolean) { this.hushed.push(on); } async resumeAudio() {} async close() { this.closed++; } } class Transport { rooms: Room[] = []; mics: Mic[] = []; async capture() { const m = new Mic(); this.mics.push(m); return m; } async join(_url: string, _token: string, on: RoomEvents) { const r = new Room(); r.on = on; this.rooms.push(r); return r; } } class Server { open = new Set(); made = 0; calls: { method: string; url: string }[] = []; fetch = vi.fn(async (url: string, init: RequestInit = {}) => { const method = init.method || 'GET'; this.calls.push({ method, url }); if (url === '/api/voice/status') return new Response(JSON.stringify({ configured: true, tts_ok: true })); if (url === '/api/voice/session' && method === 'POST') { const id = `s${++this.made}`; this.open.add(id); return new Response(JSON.stringify({ url: 'wss://lk.test', token: `t-${id}`, room: `cg-voice-${id}`, session_id: id, epoch: 3, expires_at: 0 }), { status: 201 }); } const m = /^\/api\/voice\/session\/(.+)$/u.exec(url); if (m && method === 'DELETE') return this.open.delete(decodeURIComponent(m[1])) ? new Response('{"ended":true}') : new Response('{"detail":"gone"}', { status: 404 }); return new Response('{"detail":"Not Found"}', { status: 404 }); }); } function rig() { const server = new Server(), transport = new Transport(), views: VoiceView[] = [], record = vi.fn(), spoke = vi.fn(); const c = new VoiceController({ transport: async () => transport, onView: v => views.push(v), onRecord: record, onSpoke: spoke, fetch: server.fetch as unknown as typeof fetch, secure: () => true, agentWaitMs: 1000 }); const view = () => views.at(-1)!; return { server, transport, views, view, record, spoke, c }; } async function connected() { const r = rig(); await r.c.start(); r.transport.rooms[0].on.agentJoined(); return r; } const play = (room: Room, frames: Frame[]) => { for (const f of frames) room.on.subtitle(f); }; // Rebase a fixture's seq namespace (a second recording replayed into the same call must // continue past every seq the page has already applied). const rebase = (frames: Frame[], by: number): Frame[] => frames.map(f => ({ ...f, seq: f.seq + by })); afterEach(() => { vi.restoreAllMocks(); }); describe('actual-worker frames (fc905fe on the real SDK) through the page', () => { it('normal turn: draft, growing reply, stored final, ack — and the record takes over the stage line', async () => { const r = await connected(), room = r.transport.rooms[0], turn = sidecar.scenarios.normal.turn_id!; const frames = load('worker-normal'); play(room, frames.slice(0, 2)); expect(r.view().heard).toBe('你在听吗'); const speech = (frames.find(f => f.kind === 'turn_start') as { speech_id: string }).speech_id; play(room, frames.slice(2, 3)); expect(r.view().her).toEqual({ speech, text: '', state: 'open' }); play(room, frames.slice(3, 6)); expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'open' }); play(room, frames.slice(6, 7)); expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'final', turnId: turn }); play(room, frames.slice(7, 8)); expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'done', turnId: turn }); // The worker confirmed the receipt of exactly this turn (sidecar acks). expect(sidecar.scenarios.normal.acks).toEqual([turn]); // Stage takeover: while open, the subtitle line stands in with a voice- key; once the // stored row (same turn id) is re-read, the line IS that row — no duplicate, no jump. const open = { speech, text: '在听呢,你刚才说到一半的那件事', state: 'open' as const, turnId: undefined }; expect(voiceLine(open, [], 0)?.id).toBe(`voice-${speech}`); const rows = [ { id: '501', role: 'user' as const, text: '你在听吗', lifeIds: [], turnId: turn, status: 'sent' as const }, { id: '502', role: 'companion' as const, text: '在听呢,你刚才说到一半的那件事我还在想。', lifeIds: [], turnId: turn, status: 'sent' as const }, ]; expect(voiceLine({ ...open, state: 'final', turnId: turn, text: rows[1].text }, rows, 0)).toBeNull(); const known = [rows[0]]; const merged = mergeTurns(known, rows)!; expect(merged.filter(m => m.turnId === turn)).toHaveLength(2); expect(merged.at(-1)?.id).toBe('502'); }); it('barge-in: the cut turn never gets a final; the next turn lands clean with its own receipt', async () => { const r = await connected(), room = r.transport.rooms[0]; const frames = load('worker-interrupt'); play(room, frames); const cut = (frames.find(f => f.kind === 'turn_cancel') as { speech_id: string }).speech_id; const nextTurn = sidecar.scenarios.interrupt.turn_ids!.next; expect(r.view().her?.turnId).toBe(nextTurn); expect(r.view().her?.state).toBe('done'); expect(sidecar.scenarios.interrupt.acks).toEqual([nextTurn]); // the cut turn was never acked expect(frames.filter(f => f.kind === 'assistant_final')).toHaveLength(1); expect(frames.find(f => f.kind === 'turn_cancel')?.speech_id).toBe(cut); }); it('phi: her failed line is heard (spoken), not stored; the speech closes without a final', async () => { const r = await connected(), room = r.transport.rooms[0]; play(room, load('worker-phi')); expect(r.view().spoken).toContain('说不出来话'); expect(r.view().her).toBeNull(); expect(sidecar.scenarios.phi.acks).toEqual([]); }); it('no legal receipt: cancel only — the fc905fe gate keeps assistant_final off the wire', async () => { const r = await connected(), room = r.transport.rooms[0]; const frames = load('worker-no_receipt'); play(room, frames); expect(frames.some(f => f.kind === 'assistant_final')).toBe(false); expect(frames.at(-1)?.kind).toBe('turn_cancel'); expect((frames.at(-1) as { reason?: string }).reason).toBe('no-receipt'); expect(r.view().her).toBeNull(); expect(sidecar.scenarios.no_receipt.acks).toEqual([]); }); it('proactive: start is published while the handle is still playing (sidecar), end closes the row-bound line', async () => { const r = await connected(), room = r.transport.rooms[0]; const frames = load('worker-proactive'); expect(sidecar.scenarios.proactive.start_while_playing).toBe(true); play(room, frames.slice(0, 2)); expect(r.view().her).toMatchObject({ state: 'done', proactive: true, messageId: 301, text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。' }); // The reach-out keeps the stored row's identity on the stage while it is being read; // once that row is the newest line of the record, the subtitle line gives way to it. const row = { id: '301', role: 'companion' as const, text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。', lifeIds: [], turnId: 'pro-1', proactive: true, status: 'sent' as const }; const line = voiceLine({ speech: 'p1', text: row.text, state: 'open', messageId: 301, proactive: true }, [], 0, [row]); expect(line?.id).toBe('301'); expect(line?.proactive).toBe(true); expect(voiceLine({ speech: 'p1', text: row.text, state: 'done', messageId: 301, proactive: true }, [row], 0)).toBeNull(); // The interrupted variant clears without a trace. play(room, frames.slice(2)); expect(r.view().her).toBeNull(); expect(r.spoke).not.toHaveBeenCalled(); }); }); describe('page guards against bad frames (adversarial, hand-built)', () => { it('replays, late frames of a closed speech and out-of-order frames are all dropped', async () => { const r = await connected(), room = r.transport.rooms[0]; const frames = load('adversarial-gates'); // Everything up to and including the later user interim, plus the replayed interim // (old seq), the closed speech's late frames (fresh seqs, dead speech) and an // out-of-order lower seq — none of the bad ones may move anything. play(room, frames.slice(0, 11)); expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' }); expect(r.view().heard).toBe('再说点别的'); // The duplicate turn_start for speech_A (still inside the 64-entry window) is ignored. play(room, frames.slice(12, 13)); expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' }); expect(r.view().her?.text).not.toContain('迟到'); // The out-of-order lower-seq frame (a lower seq arriving after a higher one) is // already in slice(0, 11): it never overwrote the newer draft. const outOfOrder = frames.find(f => f.kind === 'interim' && f.text === '乱序旧帧')!; expect(outOfOrder.seq).toBeLessThan(9); expect(r.view().heard).toBe('再说点别的'); // A replay of the whole first turn (start of speech_A again, still inside the // window) keeps being ignored. const dup = frames.filter(f => f.kind === 'turn_start' && f.speech_id === 'speech_A'); expect(dup.length).toBeGreaterThan(1); play(room, dup.slice(1)); expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' }); }); it('a retired speech stays retired for the whole call, however many speeches closed after it (was: re-entered after 64)', async () => { const r = await connected(), room = r.transport.rooms[0]; const frames = load('adversarial-gates'); // Up to the end of the 66 squeeze speeches: speech_A is long closed; nothing of it is on the page. const dupStart = frames.filter(f => f.kind === 'turn_start' && f.speech_id === 'speech_A').at(-1)!; play(room, frames.filter(f => f.seq < dupStart.seq)); expect(r.view().her).toBeNull(); expect(dupStart.seq).toBeGreaterThan(140); // The current worker never re-emits a speech id (each SDK speech gets a fresh id and interim.begin runs once // per stream), so this needs a misbehaving/second source — a low-cost guard, not a fix of a worker fault. // Its fresh seqs pass the sequence gate; the retired speech id does not. const views = r.views.length; play(room, frames.filter(f => f.seq >= dupStart.seq)); expect(r.view().her).toBeNull(); expect(r.views.slice(views).some(v => v.her || v.phase !== 'listening')).toBe(false); // The seq gate is still strict on its own: nothing at or below the high-water mark gets in. const high = frames.at(-1)!.seq; room.on.subtitle({ v: 2, seq: high, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_new' }); expect(r.view().her).toBeNull(); room.on.subtitle({ v: 2, seq: high + 1, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_new' }); expect(r.view().her).toEqual({ speech: 'speech_new', text: '', state: 'open' }); }); it('the retired set is per call: a new call starts clean, and an old call\'s retired ids never leak or block', async () => { const r = await connected(), room = r.transport.rooms[0]; play(room, load('adversarial-gates')); await r.c.end(); await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined(); // A new worker job: its own sequence from 1. A speech id the old call retired is not this call's business. next.on.subtitle({ v: 2, seq: 1, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_A' }); next.on.subtitle({ v: 2, seq: 2, kind: 'assistant_interim', role: 'assistant', text: '新的一通。', speech_id: 'speech_A' }); expect(r.view().her).toEqual({ speech: 'speech_A', text: '新的一通。', state: 'open' }); // The old room, however late, moves nothing. room.on.subtitle({ v: 2, seq: 9999, kind: 'turn_cancel', role: 'assistant', text: '', speech_id: 'speech_A' }); expect(r.view().her?.state).toBe('open'); await r.c.end(); }); it('your interim leaves her open line to the server\'s cancel; your final closes it at once', async () => { const r = await connected(), room = r.transport.rooms[0]; const frames = load('worker-interrupt'); play(room, frames.slice(0, 4)); expect(r.view().her?.state).toBe('open'); // A barge-in is the SDK's decision, not any sound: the worker's cancel clears her line. play(room, frames.slice(4, 5)); expect(r.view().her).toBeNull(); // A user_final closes the current speech immediately (the server owns the turn gate). play(room, frames.slice(5, 6)); expect(r.view().her).toBeNull(); expect(r.view().heard).toBe('那后来呢'); expect(r.spoke).toHaveBeenCalled(); // Your interim alone (a draft, not a turn) leaves her open line alone. room.on.subtitle({ v: 2, seq: 99, kind: 'interim', role: 'user', text: '嗯' }); expect(r.view().heard).toBe('嗯'); }); it('text takeover while she speaks: her line and its audio go at once; a drop clears everything and a new call ignores the old room', async () => { const r = await connected(), room = r.transport.rooms[0]; room.on.agentState('speaking'); const frames = load('worker-normal'); play(room, frames.slice(2, 6)); expect(r.view().her?.state).toBe('open'); r.c.hush(); expect(r.view().her).toBeNull(); expect(room.hushed).toEqual([true]); play(room, frames.slice(6)); // her late final from the cut turn expect(r.view().her).toBeNull(); // The room drops mid-sentence: everything clears with the call. room.on.agentState('speaking'); play(room, rebase(load('worker-phi').slice(1, 3), 20)); expect(r.view().her?.state).toBe('open'); room.on.dropped(); await flush(); expect(r.view()).toMatchObject({ phase: 'off', her: null, notice: WORDS.dropped }); // A new call is a new room and a new sequence; stale frames of the old room change nothing. await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined(); next.on.subtitle(frames.find(f => f.kind === 'turn_start')!); next.on.subtitle(frames.find(f => f.kind === 'assistant_interim')!); expect(r.view().her).toEqual({ speech: (frames.find(f => f.kind === 'turn_start') as { speech_id: string }).speech_id, text: '在听呢,', state: 'open' }); room.on.subtitle({ v: 2, seq: 99, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'stale' }); expect(r.view().her?.speech).not.toBe('stale'); await r.c.end(); }); });