Files
CyberGril/web/tests/hel879_frames.test.ts
T

249 lines
17 KiB
TypeScript

import { describe, expect, it, vi, afterEach } from 'vitest';
import { mergeTurns, VoiceController, voiceLine, WORDS, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session';
import { readFileSync } from 'node:fs';
// HEL-879 unified review: the frames here are the ACTUAL worker output, generated by
// scripts/gen_hel879_unified_frames.py driving the fc905fe worker wiring (make_voice_llm +
// bind_voice_session + speak_proactive_row) on the .12 real SDK (livekit-agents 1.8.8) — see
// docs/rework/hel879/unified-review/frames/. adversarial-gates.jsonl is hand-built (page-guard
// probes), never claimed as worker output.
type Frame = Record<string, unknown> & { seq: number };
const load = (name: string): Frame[] => readFileSync(new URL(`./fixtures/hel879/${name}.jsonl`, import.meta.url), 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l));
const sidecar = JSON.parse(readFileSync(new URL('./fixtures/hel879/sidecar.json', import.meta.url), 'utf8')) as { scenarios: Record<string, { turn_id?: string; turn_ids?: Record<string, string>; acks?: string[]; start_while_playing?: boolean; kinds: string[] }> };
const gate = () => { let open!: () => void; const promise = new Promise<void>(r => { open = r; }); return { promise, open }; };
const flush = async (n = 8) => { for (let i = 0; i < n; i++) await Promise.resolve(); await new Promise(r => setTimeout(r, 0)); };
class Mic implements MicTrack { stopped = 0; ended?: () => void; stop() { this.stopped++; } onEnded(cb: () => void) { this.ended = cb; } }
class Room implements VoiceRoom {
closed = 0; published: MicTrack[] = []; hushed: boolean[] = []; on!: RoomEvents;
async publish(mic: MicTrack) { this.published.push(mic); }
hush(on: boolean) { this.hushed.push(on); }
async resumeAudio() {}
async close() { this.closed++; }
}
class Transport {
rooms: Room[] = []; mics: Mic[] = [];
async capture() { const m = new Mic(); this.mics.push(m); return m; }
async join(_url: string, _token: string, on: RoomEvents) { const r = new Room(); r.on = on; this.rooms.push(r); return r; }
}
class Server {
open = new Set<string>(); made = 0; calls: { method: string; url: string }[] = [];
fetch = vi.fn(async (url: string, init: RequestInit = {}) => {
const method = init.method || 'GET';
this.calls.push({ method, url });
if (url === '/api/voice/status') return new Response(JSON.stringify({ configured: true, tts_ok: true }));
if (url === '/api/voice/session' && method === 'POST') { const id = `s${++this.made}`; this.open.add(id); return new Response(JSON.stringify({ url: 'wss://lk.test', token: `t-${id}`, room: `cg-voice-${id}`, session_id: id, epoch: 3, expires_at: 0 }), { status: 201 }); }
const m = /^\/api\/voice\/session\/(.+)$/u.exec(url);
if (m && method === 'DELETE') return this.open.delete(decodeURIComponent(m[1])) ? new Response('{"ended":true}') : new Response('{"detail":"gone"}', { status: 404 });
return new Response('{"detail":"Not Found"}', { status: 404 });
});
}
function rig() {
const server = new Server(), transport = new Transport(), views: VoiceView[] = [], record = vi.fn(), spoke = vi.fn();
const c = new VoiceController({ transport: async () => transport, onView: v => views.push(v), onRecord: record, onSpoke: spoke, fetch: server.fetch as unknown as typeof fetch, secure: () => true, agentWaitMs: 1000 });
const view = () => views.at(-1)!;
return { server, transport, views, view, record, spoke, c };
}
async function connected() {
const r = rig(); await r.c.start(); r.transport.rooms[0].on.agentJoined(); return r;
}
const play = (room: Room, frames: Frame[]) => { for (const f of frames) room.on.subtitle(f); };
// Rebase a fixture's seq namespace (a second recording replayed into the same call must
// continue past every seq the page has already applied).
const rebase = (frames: Frame[], by: number): Frame[] => frames.map(f => ({ ...f, seq: f.seq + by }));
afterEach(() => { vi.restoreAllMocks(); });
describe('actual-worker frames (fc905fe on the real SDK) through the page', () => {
it('normal turn: draft, growing reply, stored final, ack — and the record takes over the stage line', async () => {
const r = await connected(), room = r.transport.rooms[0], turn = sidecar.scenarios.normal.turn_id!;
const frames = load('worker-normal');
play(room, frames.slice(0, 2));
expect(r.view().heard).toBe('你在听吗');
const speech = (frames.find(f => f.kind === 'turn_start') as { speech_id: string }).speech_id;
play(room, frames.slice(2, 3));
expect(r.view().her).toEqual({ speech, text: '', state: 'open' });
play(room, frames.slice(3, 6));
expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'open' });
play(room, frames.slice(6, 7));
expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'final', turnId: turn });
play(room, frames.slice(7, 8));
expect(r.view().her).toEqual({ speech, text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'done', turnId: turn });
// The worker confirmed the receipt of exactly this turn (sidecar acks).
expect(sidecar.scenarios.normal.acks).toEqual([turn]);
// Stage takeover: while open, the subtitle line stands in with a voice- key; once the
// stored row (same turn id) is re-read, the line IS that row — no duplicate, no jump.
const open = { speech, text: '在听呢,你刚才说到一半的那件事', state: 'open' as const, turnId: undefined };
expect(voiceLine(open, [], 0)?.id).toBe(`voice-${speech}`);
const rows = [
{ id: '501', role: 'user' as const, text: '你在听吗', lifeIds: [], turnId: turn, status: 'sent' as const },
{ id: '502', role: 'companion' as const, text: '在听呢,你刚才说到一半的那件事我还在想。', lifeIds: [], turnId: turn, status: 'sent' as const },
];
expect(voiceLine({ ...open, state: 'final', turnId: turn, text: rows[1].text }, rows, 0)).toBeNull();
const known = [rows[0]];
const merged = mergeTurns(known, rows)!;
expect(merged.filter(m => m.turnId === turn)).toHaveLength(2);
expect(merged.at(-1)?.id).toBe('502');
});
it('barge-in: the cut turn never gets a final; the next turn lands clean with its own receipt', async () => {
const r = await connected(), room = r.transport.rooms[0];
const frames = load('worker-interrupt');
play(room, frames);
const cut = (frames.find(f => f.kind === 'turn_cancel') as { speech_id: string }).speech_id;
const nextTurn = sidecar.scenarios.interrupt.turn_ids!.next;
expect(r.view().her?.turnId).toBe(nextTurn);
expect(r.view().her?.state).toBe('done');
expect(sidecar.scenarios.interrupt.acks).toEqual([nextTurn]); // the cut turn was never acked
expect(frames.filter(f => f.kind === 'assistant_final')).toHaveLength(1);
expect(frames.find(f => f.kind === 'turn_cancel')?.speech_id).toBe(cut);
});
it('phi: her failed line is heard (spoken), not stored; the speech closes without a final', async () => {
const r = await connected(), room = r.transport.rooms[0];
play(room, load('worker-phi'));
expect(r.view().spoken).toContain('说不出来话');
expect(r.view().her).toBeNull();
expect(sidecar.scenarios.phi.acks).toEqual([]);
});
it('no legal receipt: cancel only — the fc905fe gate keeps assistant_final off the wire', async () => {
const r = await connected(), room = r.transport.rooms[0];
const frames = load('worker-no_receipt');
play(room, frames);
expect(frames.some(f => f.kind === 'assistant_final')).toBe(false);
expect(frames.at(-1)?.kind).toBe('turn_cancel');
expect((frames.at(-1) as { reason?: string }).reason).toBe('no-receipt');
expect(r.view().her).toBeNull();
expect(sidecar.scenarios.no_receipt.acks).toEqual([]);
});
it('proactive: start is published while the handle is still playing (sidecar), end closes the row-bound line', async () => {
const r = await connected(), room = r.transport.rooms[0];
const frames = load('worker-proactive');
expect(sidecar.scenarios.proactive.start_while_playing).toBe(true);
play(room, frames.slice(0, 2));
expect(r.view().her).toMatchObject({ state: 'done', proactive: true, messageId: 301, text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。' });
// The reach-out keeps the stored row's identity on the stage while it is being read;
// once that row is the newest line of the record, the subtitle line gives way to it.
const row = { id: '301', role: 'companion' as const, text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。', lifeIds: [], turnId: 'pro-1', proactive: true, status: 'sent' as const };
const line = voiceLine({ speech: 'p1', text: row.text, state: 'open', messageId: 301, proactive: true }, [], 0, [row]);
expect(line?.id).toBe('301');
expect(line?.proactive).toBe(true);
expect(voiceLine({ speech: 'p1', text: row.text, state: 'done', messageId: 301, proactive: true }, [row], 0)).toBeNull();
// The interrupted variant clears without a trace.
play(room, frames.slice(2));
expect(r.view().her).toBeNull();
expect(r.spoke).not.toHaveBeenCalled();
});
});
describe('page guards against bad frames (adversarial, hand-built)', () => {
it('replays, late frames of a closed speech and out-of-order frames are all dropped', async () => {
const r = await connected(), room = r.transport.rooms[0];
const frames = load('adversarial-gates');
// Everything up to and including the later user interim, plus the replayed interim
// (old seq), the closed speech's late frames (fresh seqs, dead speech) and an
// out-of-order lower seq — none of the bad ones may move anything.
play(room, frames.slice(0, 11));
expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' });
expect(r.view().heard).toBe('再说点别的');
// The duplicate turn_start for speech_A (still inside the 64-entry window) is ignored.
play(room, frames.slice(12, 13));
expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' });
expect(r.view().her?.text).not.toContain('迟到');
// The out-of-order lower-seq frame (a lower seq arriving after a higher one) is
// already in slice(0, 11): it never overwrote the newer draft.
const outOfOrder = frames.find(f => f.kind === 'interim' && f.text === '乱序旧帧')!;
expect(outOfOrder.seq).toBeLessThan(9);
expect(r.view().heard).toBe('再说点别的');
// A replay of the whole first turn (start of speech_A again, still inside the
// window) keeps being ignored.
const dup = frames.filter(f => f.kind === 'turn_start' && f.speech_id === 'speech_A');
expect(dup.length).toBeGreaterThan(1);
play(room, dup.slice(1));
expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' });
});
it('a retired speech stays retired for the whole call, however many speeches closed after it (was: re-entered after 64)', async () => {
const r = await connected(), room = r.transport.rooms[0];
const frames = load('adversarial-gates');
// Up to the end of the 66 squeeze speeches: speech_A is long closed; nothing of it is on the page.
const dupStart = frames.filter(f => f.kind === 'turn_start' && f.speech_id === 'speech_A').at(-1)!;
play(room, frames.filter(f => f.seq < dupStart.seq));
expect(r.view().her).toBeNull();
expect(dupStart.seq).toBeGreaterThan(140);
// The current worker never re-emits a speech id (each SDK speech gets a fresh id and interim.begin runs once
// per stream), so this needs a misbehaving/second source — a low-cost guard, not a fix of a worker fault.
// Its fresh seqs pass the sequence gate; the retired speech id does not.
const views = r.views.length;
play(room, frames.filter(f => f.seq >= dupStart.seq));
expect(r.view().her).toBeNull();
expect(r.views.slice(views).some(v => v.her || v.phase !== 'listening')).toBe(false);
// The seq gate is still strict on its own: nothing at or below the high-water mark gets in.
const high = frames.at(-1)!.seq;
room.on.subtitle({ v: 2, seq: high, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_new' });
expect(r.view().her).toBeNull();
room.on.subtitle({ v: 2, seq: high + 1, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_new' });
expect(r.view().her).toEqual({ speech: 'speech_new', text: '', state: 'open' });
});
it('the retired set is per call: a new call starts clean, and an old call\'s retired ids never leak or block', async () => {
const r = await connected(), room = r.transport.rooms[0];
play(room, load('adversarial-gates'));
await r.c.end();
await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined();
// A new worker job: its own sequence from 1. A speech id the old call retired is not this call's business.
next.on.subtitle({ v: 2, seq: 1, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'speech_A' });
next.on.subtitle({ v: 2, seq: 2, kind: 'assistant_interim', role: 'assistant', text: '新的一通。', speech_id: 'speech_A' });
expect(r.view().her).toEqual({ speech: 'speech_A', text: '新的一通。', state: 'open' });
// The old room, however late, moves nothing.
room.on.subtitle({ v: 2, seq: 9999, kind: 'turn_cancel', role: 'assistant', text: '', speech_id: 'speech_A' });
expect(r.view().her?.state).toBe('open');
await r.c.end();
});
it('your interim leaves her open line to the server\'s cancel; your final closes it at once', async () => {
const r = await connected(), room = r.transport.rooms[0];
const frames = load('worker-interrupt');
play(room, frames.slice(0, 4));
expect(r.view().her?.state).toBe('open');
// A barge-in is the SDK's decision, not any sound: the worker's cancel clears her line.
play(room, frames.slice(4, 5));
expect(r.view().her).toBeNull();
// A user_final closes the current speech immediately (the server owns the turn gate).
play(room, frames.slice(5, 6));
expect(r.view().her).toBeNull();
expect(r.view().heard).toBe('那后来呢');
expect(r.spoke).toHaveBeenCalled();
// Your interim alone (a draft, not a turn) leaves her open line alone.
room.on.subtitle({ v: 2, seq: 99, kind: 'interim', role: 'user', text: '嗯' });
expect(r.view().heard).toBe('嗯');
});
it('text takeover while she speaks: her line and its audio go at once; a drop clears everything and a new call ignores the old room', async () => {
const r = await connected(), room = r.transport.rooms[0];
room.on.agentState('speaking');
const frames = load('worker-normal');
play(room, frames.slice(2, 6));
expect(r.view().her?.state).toBe('open');
r.c.hush();
expect(r.view().her).toBeNull();
expect(room.hushed).toEqual([true]);
play(room, frames.slice(6)); // her late final from the cut turn
expect(r.view().her).toBeNull();
// The room drops mid-sentence: everything clears with the call.
room.on.agentState('speaking');
play(room, rebase(load('worker-phi').slice(1, 3), 20));
expect(r.view().her?.state).toBe('open');
room.on.dropped(); await flush();
expect(r.view()).toMatchObject({ phase: 'off', her: null, notice: WORDS.dropped });
// A new call is a new room and a new sequence; stale frames of the old room change nothing.
await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined();
next.on.subtitle(frames.find(f => f.kind === 'turn_start')!);
next.on.subtitle(frames.find(f => f.kind === 'assistant_interim')!);
expect(r.view().her).toEqual({ speech: (frames.find(f => f.kind === 'turn_start') as { speech_id: string }).speech_id, text: '在听呢,', state: 'open' });
room.on.subtitle({ v: 2, seq: 99, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'stale' });
expect(r.view().her?.speech).not.toBe('stale');
await r.c.end();
});
});