Files
CyberGril/docs/rework/hel898/HEL-898.patch
T

864 lines
77 KiB
Diff
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
diff --git a/docs/requirements/handoff-HEL-898.json b/docs/requirements/handoff-HEL-898.json
new file mode 100644
--- /dev/null
+++ b/docs/requirements/handoff-HEL-898.json
@@ -0,0 +1,39 @@
+{
+ "baseline_version": "1.11",
+ "read_commit": "3228b1e0d7f822b7030f945e285eac74126771aa",
+ "task": "HEL-898(HEL-879 接线卡,视觉总监本人编码):在已认可的交谈界面接上她的增量字幕(cg.subtitle 契约 v2)与语音内主动开口呈现,处理打断/文字接管/错误/挂断收尾;不另做聊天页、不改主题与整体动效",
+ "requirement_ids": [
+ "CG-004", "CG-006", "CG-010", "CG-011", "CG-018", "CG-020",
+ "VIS-001", "VIS-002", "VIS-003", "VIS-004", "VIS-005", "VIS-006", "VIS-007", "VIS-008", "VIS-009", "VIS-010", "VIS-011", "VIS-012", "VIS-013",
+ "UX-001", "UX-002", "UX-005", "UX-006",
+ "AM-001",
+ "GOV-001", "GOV-002", "GOV-003", "GOV-004", "GOV-005", "GOV-006",
+ "DEL-001"
+ ],
+ "deliverable": "自总管输入包 3228b1e(INPUT-MANIFEST 逐文件 SHA-256)起的前端补丁,只动 web/src/voice/session.ts、web/src/chat/ChatPanel.tsx 与测试:1) session.ts 严格解析 v2 信封(v===2、正整数 seq、kind 白名单、角色与种类对应、她的事件必带 speech_id、assistant_final 必带 turn_id、proactive_* 必带正整数 message_id;不合格整条丢弃且不推进序号);每次通话(控制器代次)独立 seq 高水位,seq<=已应用即丢;按 speech 生命周期只接受当前打开的一轮,已关闭/被取代的 speech 不再显示,旧 speech 的高 seq 事件不能关闭新轮。VoiceView 新增 her(speech/text/state open|final|done/turnId/messageId/proactive),与用户 ASR 草稿 heard 分槽。2) turn_start 开轮,assistant_interim 整句替换,assistant_final 原子替换为落库文本并请页面回读 /api/messages,turn_end 关轮(无 final 的关轮按未成为正式回复即清),turn_cancel/error(φ)/user_final/文字接管 hush/结束/掉线/卸载/重开即时清;φ 原句仍在语音行以“未记入聊天”显示。3) proactive_start 以落库原文、同源 message_id 显示在她的位置,proactive_end 保留至记录或下一轮接替,proactive_cancel 即清;不发任何请求、不标已读、不开麦、不重 TTS、不进入正式记录。4) ChatPanel:她的语音行进入与文字回复相同的舞台槽位(当前一两句、自动续读、暂停/回看、全文),生长中沿用文字流的“最新一句先压住”规则,逐分片不重启动画;落库行到来时同一 turn 同一位置接替,行键不变不重放;被撤回的语音行以 instant 方式立即离场(不淡出);减少动态设置沿用既有逻辑。你用语音开始新一轮时与发送文字一致恢复自动呈现(原先暂停状态下她的回答不会进来)。5) 测试:voice.test.ts 改为 v2 帧并新增 11 项(录制样例三场景回放、信封负例、乱序/重放/迟到/双 handle、φ、主动开口及打断、文字接管/挂断/重开、舞台行键接替);fake-livekit.js 未带信封的帧自动套 v2;hel878_mock.mjs 增 /__proactive;hel878_shots.mjs 的她的 final 与 φ 改为 v2 生命周期;新增 hel898_shots.mjs(两主题×电脑/手机)",
+ "acceptance": [
+ "假 LiveKit 有序回放 normal-turn.jsonl:用户草稿→用户终稿→turn_start→两次增量(整句替换)→final(带 turn_id)→turn_end;她的字在舞台出现时 /__rows 尚无该回复;落库后记录中只出现一次(页面只回读,不写)",
+ "interrupt.jsonl 与负例:turn_cancel 后她的未完成行立即离场;被取消轮之后的高 seq 增量/final/重复 turn_start 均不显示;整段录制重放不改变界面;双 handle 时旧 speech 的 turn_end/turn_cancel 不关闭新轮;无 final 的 turn_end 视为未成为正式回复并清除",
+ "φ:error 原句在语音行显示并标“未记入聊天”,她的增量行立即清除,随后 turn_cancel(error_phi) 不抹掉 φ 原句;不落正式记录",
+ "proactive.jsonl:原文一字不差显示在她的位置并保留落库行身份(id/turn),可被你的语音打断(proactive_cancel 即清);全程页面无任何非 GET 请求(无回执、无已读、无消息写入),不调用 onSpoke/onRecord",
+ "文字接管、结束语音、掉线、新通话:她的行即时清;旧房间迟到帧被代次挡住;新通话序号从 1 重新开始并正常显示",
+ "两主题(月光/雾光)电脑 1440×900 与手机 390×844 截图与连续帧:沿用已认可位置与组件,无新增组件或样式;减少动态设置下她的字照常出现、无入场动画",
+ "工程:tsc 0 错;eslint 0 错 1 条既有警告(ChatPanel fullButton ref,HEL-897 起即有);vitest 8 文件 99/99;vite build 通过;浏览器 hel898 61/61,HEL-878 回归 49/49;修前对照:新 voice.test.ts 在原 session.ts/ChatPanel 上 12 项失败"
+ ],
+ "deferred": [
+ "真实通话未测:真实 LiveKit/WebRTC、真实 worker 与 SDK 回调时序、真机/Safari/iOS、可信 HTTPS+WSS、R5 连续三轮/1 秒打断/停麦无残音/有限预算,均仍由 HEL-797 主卡承接;本卡全部为假 LiveKit、假音频设备与 Chromium 无头 Edge",
+ "CG-011 语音内主动开口仅是“会话开时念一条已落库旧留言”的最小接缝(后台默认关,本卡不启用);会话中持续主动联系、激活节奏、未读联动策略未完成,不得记为 CG-011 完成",
+ "后台并行小修(建房延迟后的清理唤醒、主动 say 的 start 发得过晚、无完整 receipt 错误发 final)由原总工在 HEL-879 处理;本卡按 v2 正确语义实现,最终后台 commit 出来后需按其统一复核(特别是 proactive_start 晚于音频开始时的呈现)",
+ "已知边界:seq 按“一次 worker job 一个序列”实现;若同一房间内 worker 重连且序号归零,前端会按契约丢弃其帧直到超过旧高水位——需后台确认 job 与房间一一对应",
+ "原版 Python 需求检查(scripts/check_requirements.py --base-ref origin/main,含本交接)未在本运行时执行(无 Python、无 git),须总管对实际目标分支复跑",
+ "D1 音色、cap=0、R6/R7、整体沉浸体验由主卡与白栖知决定;工程自测不是用户体验验收"
+ ],
+ "requirement_changes": [],
+ "verification": [
+ "读取:HEL-898 卡(启动交接与原说明)、输入包 3228b1e:docs/voice-subtitle-contract-v2.md、docs/rework/hel879/voice-seam-interfaces.md、subtitle-samples/*.jsonl、src/cybergril/voice/seam.py 与 worker.py(事件发出点)、AGENTS.md、docs/requirements/README.md/current.md(v1.11)/coverage.md/changes.md、sources/integration-plan-v1.md R5",
+ "工具:沿用本机既有离线工具链(HEL-834 留存 node v22.11.0;HEL-897 留存 web/node_modules,package-lock 哈希与本包一致,目录联接使用);未下载、未安装",
+ "web:tsc --noEmit 0;eslint src 0 错 1 既有警告;vitest run 8 文件 99/99(原 88 + 新增);vite build 通过;日志随交付包",
+ "浏览器:node tests/hel878_mock.mjs + node tests/hel898_shots.mjs(无头 Edge、脚本化 LiveKit 替身、Chromium 假音频设备)61/61;node tests/hel878_shots.mjs 回归 49/49;截图与连续帧随交付包",
+ "无模型、嵌入、ASR、TTS 请求,无真实麦克风采集,无部署、合并、服务或绑定改动;未改后台、总装分支或 PR#11"
+ ]
+}
diff --git a/web/src/chat/ChatPanel.tsx b/web/src/chat/ChatPanel.tsx
--- a/web/src/chat/ChatPanel.tsx
+++ b/web/src/chat/ChatPanel.tsx
@@ -10,7 +10,7 @@
import { motionOff } from '../stage/roomScene';
import { LineStage, type PresentHow } from './lineStage';
import { anchorOf, cursorAt, groupTurns, pagesOf, ReadingClock, type Anchor, type Metrics, type ShownMessage } from './reading';
+import { mergeTurns, OFF, recentRows, VoiceController, voiceLine, voiceStatus, WORDS, type VoiceView } from '../voice/session';
-import { mergeTurns, OFF, recentRows, VoiceController, voiceStatus, WORDS, type VoiceView } from '../voice/session';
import { loadTransport } from '../voice/livekit';
import { newestId, serverId } from '../adapters/inbox';
@@ -19,6 +19,8 @@
const typingTarget = (el: Element | null) => !!el && (el.matches('textarea,input,select,[contenteditable="true"]'));
const NO_IDS: string[] = [], NO_MESSAGES: Message[] = [];
+// Marks "your next turn is one you said by voice" until its row is read back and its turn key is known.
+const SPOKEN = '\u0000spoken';
export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation, onRecord, offline, fail, log, setLog, focusLife, focusIds = NO_IDS, real = false, onReceipt, incoming = NO_MESSAGES, onAdmit }: {
real?: boolean;
// Real only: the server accepted a complete-receipt confirmation (the life it may unlock is decided later).
@@ -68,7 +70,8 @@
};
const spoke = useRef(() => {});
// Speaking cancels a text reply still streaming (the server's turn gate already did); it is kept as a cut fragment.
+ // Like sending, it starts a new turn that comes in by itself, even if you had paused reading.
+ spoke.current = () => { if (busyNow.current) stop(true); sentTurn.current = SPOKEN; if (!flow) setFlash('新的一轮会自动呈现,随时可以暂停'); };
- spoke.current = () => { if (busyNow.current) stop(true); };
useEffect(() => {
if (!real) return;
const sync = recordSync.current;
@@ -99,17 +102,27 @@
// Real messages are trimmed by the server before transmission; their lifeIds point at records this page has not
// necessarily loaded, so the demo's local filter would wrongly hide them.
const view = useMemo(() => real ? state : publicView(state), [state, real]);
+ // What she says by voice takes the place her text reply would take (HEL-898). It is shown, never stored: once the
+ // record holds it (same turn, or the same stored reach-out as the newest line) the row takes over under the same
+ // line keys; a finished line also gives way as soon as anything else comes into the record after it.
+ const her = real ? voiceView.her : null, herBase = useRef({ speech: '', at: 0 });
+ if (her && herBase.current.speech !== her.speech) herBase.current = { speech: her.speech, at: view.messages.length };
+ const herLine = useMemo(() => reply ? null : voiceLine(her, view.messages, herBase.current.at, incoming), [her, reply, view.messages, incoming]);
+ const herOpen = !!herLine && her?.state === 'open', talking = busy || herOpen;
// The cut-off part of a stopped reply stays with its own turn (it is shown, never stored).
const shown = useMemo<ShownMessage[]>(() => {
const list: ShownMessage[] = [...view.messages];
if (reply) list.push({ id: 'live-reply', role: 'companion', text: reply, lifeIds: [], turnId: activeTurn, live: 'generating' });
+ else {
+ if (interrupted) {
+ const at = list.map(m => m.turnId).lastIndexOf(interrupted.turn);
+ list.splice(at < 0 ? list.length : at + 1, 0, { id: 'cut-reply', role: 'companion', text: interrupted.text, lifeIds: [], turnId: interrupted.turn, live: 'cancelled' });
+ }
+ if (herLine) list.push(herLine);
- else if (interrupted) {
- const at = list.map(m => m.turnId).lastIndexOf(interrupted.turn);
- list.splice(at < 0 ? list.length : at + 1, 0, { id: 'cut-reply', role: 'companion', text: interrupted.text, lifeIds: [], turnId: interrupted.turn, live: 'cancelled' });
}
return list;
+ }, [view.messages, reply, activeTurn, interrupted, herLine]);
+ const turn = useMemo(() => groupTurns(shown).at(-1), [shown]), opener = useRef(''); opener.current = turn?.items[0]?.role || '';
- }, [view.messages, reply, activeTurn, interrupted]);
- const turn = useMemo(() => groupTurns(shown).at(-1), [shown]);
const stageRef = useRef<HTMLDivElement>(null), lines = useRef<LineStage | null>(null), meter = useRef<HTMLElement>(null);
const [metrics, setMetrics] = useState<Metrics>(() => metricsFor(Math.min(900, innerWidth * .88)));
const pages = useMemo(() => pagesOf(turn, metrics), [turn, metrics]);
@@ -126,7 +139,7 @@
// Her own message that arrives while you are here waits its turn: never while she is answering you, a sheet is
// open, you paused or went back, or the line on screen has not had its time. Then it comes in as a new turn.
const waiting = useRef({ count: 0, busy: false, sheet: false }), admit = useRef(() => {});
+ waiting.current = { count: incoming.length, busy: talking || voiceView.phase === 'thinking' || voiceView.phase === 'speaking', sheet: log || full };
- waiting.current = { count: incoming.length, busy: busy || voiceView.phase === 'thinking' || voiceView.phase === 'speaking', sheet: log || full };
admit.current = () => { const add = incoming.filter(m => !live.current.some(k => k.id === m.id)); onAdmit?.(incoming.map(m => m.id)); if (add.length) change([...live.current, ...add]); };
function stop(quiet = false) { gate.current.cancel(); if (replyLive.current) setInterrupted({ text: replyLive.current, turn: turnLive.current }); replyLive.current = ''; setBusy(false); setReply(''); if (!quiet) onPresence('停下来了'); change(live.current.map(m => m.status === 'pending' ? { ...m, status: 'cancelled' } : m)); }
@@ -154,8 +167,12 @@
const key = turn?.key || '';
if (key === turnKey.current) return;
turnKey.current = key; how.current = 'jump'; head.current = START; setAnchor(START);
+ // Your own new turn (sent, or said by voice: its server turn id is only known once the row is read back).
+ if (key && (key === sentTurn.current || (sentTurn.current === SPOKEN && opener.current === 'user'))) { sentTurn.current = key; setFlowState(true); }
- if (key && key === sentTurn.current) setFlowState(true);
}, [turn?.key]);
+ // Her voice line withdrawn (cut, φ, text took over, you spoke, the call ended): it leaves at once, no fade.
+ const hadLine = useRef(false);
+ useLayoutEffect(() => { if (hadLine.current && !her) how.current = 'instant'; hadLine.current = !!herLine; }, [her, herLine]);
const signature = pages.map(p => p.key).join('|');
useLayoutEffect(() => {
lines.current?.present(pages, cursor, how.current, name, !motionOff());
@@ -165,8 +182,8 @@
useLayoutEffect(() => {
if (!page) return;
const mine = page.message, cancelled = mine.live === 'cancelled' || (mine.status === 'cancelled' && mine.role === 'companion');
+ const text = cancelled && page.last ? '未完成 · 已取消' : mine.role === 'user' && mine.status === 'failed' ? '没发出去' : mine.role === 'user' && mine.status === 'cancelled' && atEnd ? '本轮已取消' : atEnd && talking && mine.role === 'companion' ? '还在说' : '';
+ lines.current?.mark({ text, waiting: atEnd && talking && mine.role === 'companion' });
- const text = cancelled && page.last ? '未完成 · 已取消' : mine.role === 'user' && mine.status === 'failed' ? '没发出去' : mine.role === 'user' && mine.status === 'cancelled' && atEnd ? '本轮已取消' : atEnd && busy && mine.role === 'companion' ? '还在说' : '';
- lines.current?.mark({ text, waiting: atEnd && busy && mine.role === 'companion' });
});
useEffect(() => { lines.current?.rename(name); }, [name]);
@@ -306,7 +323,7 @@
useLayoutEffect(() => { const t = textarea.current; if (!t) return; t.style.height = 'auto'; t.style.height = Math.min(t.scrollHeight, 88) + 'px'; }, [draft]);
const typing = (on: boolean) => document.documentElement.classList.toggle('typing', on);
useEffect(() => () => { document.documentElement.classList.remove('typing'); }, []);
+ const record = shown.filter(m => m !== herLine && (!reply || !m.live));
- const record = reply ? shown.filter(m => !m.live) : shown;
// While a reading sheet is open the table words are out of reach (the sheet holds them all); layout effect so the
// focus hand-back after closing never lands on an inert button.
const conversation = useRef<HTMLElement>(null);
@@ -327,7 +344,7 @@
<div className="reader-bar">
<button className="flow" aria-pressed={flow} onClick={() => setFlow(!flow)}><Icon name={flow ? 'pause' : 'play'}/><span>{flow ? '暂停' : '继续'}</span></button>
<button className="icon" aria-label="上一句" disabled={cursor <= 0} onClick={() => back()}><Icon name="back"/></button>
+ <span className="count" aria-label={pages.length ? `第 ${cursor + 1} 句,共 ${pages.length} 句${talking ? ',还在说' : ''}` : '还没有对话'}><span>{pages.length ? `${cursor + 1} / ${pages.length}${talking ? '…' : ''}` : '0 / 0'}</span><i ref={meter}/></span>
- <span className="count" aria-label={pages.length ? `第 ${cursor + 1} 句,共 ${pages.length} 句${busy ? ',还在说' : ''}` : '还没有对话'}><span>{pages.length ? `${cursor + 1} / ${pages.length}${busy ? '…' : ''}` : '0 / 0'}</span><i ref={meter}/></span>
<button className="icon" aria-label="下一句" disabled={atEnd} onClick={() => forward()}><Icon name="next"/></button>
<button className="read-all" ref={fullButton} disabled={!page} onClick={() => setFull(true)}>全文</button>
{busy && <button className="stop" onClick={() => stop()}>停止回复</button>}
@@ -341,7 +358,7 @@
</div>
</section>
<aside className="record log-panel glass read" hidden={!log} role="dialog" aria-modal="false" aria-labelledby="record-title"><header><div><p className="eyebrow">{real ? '服务器聊天记录' : '完整记录 · 本机演示'}</p><h2 id="record-title">你们说过的话</h2></div><button className="btn-round sm" ref={logClose} aria-label="关闭记录" onClick={() => setLog(false)}><Icon name="x"/></button></header>
+ <div className="scroll" ref={logRef}><p className="guide">内容完整保留;关闭后回到刚才那一句,自动呈现会接着走。</p>{record.map(m => <article key={m.id} data-life={m.lifeIds.join(' ') || undefined} data-mid={m.id} data-role={m.role} className={`message ${m.status === 'failed' ? 'fail' : ''} ${(focusLife && m.lifeIds.includes(focusLife)) || focusIds.includes(m.id) ? 'hl' : ''}`}><div className="author"><i/>{m.role === 'user' ? '你' : name}{m.proactive && <span className="tagm">她主动说的</span>}</div><p>{m.text}</p>{m.status === 'failed' && <button className="btn-text" onClick={() => void send(m)}>没发出去 · 重试</button>}{m.status === 'cancelled' && <span className="incomplete">已取消</span>}{m.live === 'cancelled' && <span className="incomplete">已停止 · 这是未完成片段,未存入聊天记录</span>}</article>)}{reply && <article className="message" data-role="companion"><div className="author"><i/>{name}</div><p>{reply}</p><span className="incomplete">还在说…</span></article>}{herLine && !herLine.proactive && <article className="message" data-role="companion"><div className="author"><i/>{name}</div><p>{herLine.text}</p><span className="incomplete">{herOpen ? '还在说…' : '语音刚说完 · 正在读回聊天记录'}</span></article>}</div></aside>
- <div className="scroll" ref={logRef}><p className="guide">内容完整保留;关闭后回到刚才那一句,自动呈现会接着走。</p>{record.map(m => <article key={m.id} data-life={m.lifeIds.join(' ') || undefined} data-mid={m.id} data-role={m.role} className={`message ${m.status === 'failed' ? 'fail' : ''} ${(focusLife && m.lifeIds.includes(focusLife)) || focusIds.includes(m.id) ? 'hl' : ''}`}><div className="author"><i/>{m.role === 'user' ? '你' : name}{m.proactive && <span className="tagm">她主动说的</span>}</div><p>{m.text}</p>{m.status === 'failed' && <button className="btn-text" onClick={() => void send(m)}>没发出去 · 重试</button>}{m.status === 'cancelled' && <span className="incomplete">已取消</span>}{m.live === 'cancelled' && <span className="incomplete">已停止 · 这是未完成片段,未存入聊天记录</span>}</article>)}{reply && <article className="message" data-role="companion"><div className="author"><i/>{name}</div><p>{reply}</p><span className="incomplete">还在说…</span></article>}</div></aside>
{full && fullMessage && <aside className="record full glass read" role="dialog" aria-modal="false" aria-labelledby="full-title"><header><div><p className="eyebrow">{fullMessage.role === 'user' ? '你' : name} · 全文</p><h2 id="full-title">这一段,慢慢看</h2></div><button className="btn-round sm" ref={fullClose} aria-label="关闭全文" onClick={() => setFull(false)}><Icon name="x"/></button></header>
<div className="scroll"><p className="guide">整段完整可读。关闭后回到刚才那一句。</p><article className="message" data-role={fullMessage.role}><div className="author"><i/>{fullMessage.role === 'user' ? '你' : name}</div><p>{fullMessage.text}</p>{fullMessage.live === 'generating' && <span className="incomplete">还在说…</span>}{fullMessage.live === 'cancelled' && <span className="incomplete">这段回复已停止,内容未完成,未存入聊天记录。</span>}</article></div></aside>}
{real && permission === 'mic' && <Modal title={`开始和${name}语音?`} onClose={() => { setPermission(''); setNotice('没有开启语音,文字仍可使用。'); }} confirm="开始语音" onConfirm={() => { setPermission(''); setNotice(''); void voiceCtl.current?.start(); }}>浏览器会请求麦克风权限。你的声音只送到这台服务器上{name}的语音服务,她用同一份记忆、同一段聊天回应你;你说的话和她的回答照常存进聊天记录。直接开口就能打断她。结束语音、离开交谈或关上页面时,麦克风立即关闭。<br/><small className="voice-note">语音通路还没有经过真机验收,效果可能不稳定。</small></Modal>}
diff --git a/web/src/voice/session.ts b/web/src/voice/session.ts
--- a/web/src/voice/session.ts
+++ b/web/src/voice/session.ts
@@ -7,6 +7,7 @@
// One rule holds the whole lifecycle together: every call starts a new generation, and anything that answers for an
// older generation (a late token, a late join, a late microphone, a late room event) is cleaned up, never shown.
import type { Message } from '../domain/model';
+import type { ShownMessage } from '../chat/reading';
import { api, toMessage, within, type MessagePage } from '../adapters/real';
import { serverId } from '../adapters/inbox';
@@ -17,17 +18,38 @@
heard: string;
// Her line spoken after a failure (φ): heard, not part of the record.
spoken: string;
+ // What she is saying right now, shown on the stage where her text replies appear; never stored by the page.
+ her: HerLine | null;
// Why voice is off or what went wrong; empty while it simply runs.
notice: string;
// The browser held her audio back until a tap (autoplay rules); `resume` lets it out.
blocked: boolean;
}
+// One "she speaks" (an SDK speech): her reply to your voice turn, or a stored reach-out read aloud. `open` grows by
+// replacement (each interim is the whole text so far); `final` is the stored reply's exact text; `done` is closed and
+// stays only until the record (or the next turn) takes its place.
+export interface HerLine { speech: string; text: string; state: 'open' | 'final' | 'done'; turnId?: string; messageId?: number; proactive?: boolean }
+export const OFF: VoiceView = { phase: 'off', heard: '', spoken: '', her: null, notice: '', blocked: false };
+
+// ── cg.subtitle contract v2 (docs/voice-subtitle-contract-v2.md). Anything that is not exactly v2 is dropped. ──
+const USER_KINDS = ['interim', 'user_final'] as const;
+const HER_KINDS = ['turn_start', 'assistant_interim', 'assistant_final', 'turn_end', 'turn_cancel', 'error', 'proactive_start', 'proactive_end', 'proactive_cancel'] as const;
+export type SubtitleKind = typeof USER_KINDS[number] | typeof HER_KINDS[number];
+export interface Subtitle { seq: number; kind: SubtitleKind; role: 'user' | 'assistant'; text: string; speech: string; turnId?: string; messageId?: number; reason?: string }
+const id = (v: unknown) => typeof v === 'string' && v.length > 0 && v.length <= 200 ? v : '';
-export const OFF: VoiceView = { phase: 'off', heard: '', spoken: '', notice: '', blocked: false };
-export interface Subtitle { kind: 'interim' | 'user_final' | 'assistant_final' | 'error'; role: 'user' | 'assistant'; text: string }
export function subtitle(v: unknown): Subtitle | null {
const s = v as Record<string, unknown> | null;
+ if (!s || typeof s !== 'object' || Array.isArray(s) || s.v !== 2 || !Number.isSafeInteger(s.seq) || (s.seq as number) <= 0 || typeof s.text !== 'string') return null;
+ const kind = s.kind as SubtitleKind, user = (USER_KINDS as readonly string[]).includes(kind);
+ if (!user && !(HER_KINDS as readonly string[]).includes(kind)) return null;
+ if (s.role !== (user ? 'user' : 'assistant')) return null;
+ const out: Subtitle = { seq: s.seq as number, kind, role: user ? 'user' : 'assistant', text: s.text, speech: user ? '' : id(s.speech_id) };
+ if (!user && !out.speech) return null;
+ if (s.turn_id !== undefined && s.turn_id !== null) { if (!id(s.turn_id)) return null; out.turnId = s.turn_id as string; }
+ if (kind === 'assistant_final' && !out.turnId) return null;
+ if (kind.startsWith('proactive_')) { if (!Number.isSafeInteger(s.message_id) || (s.message_id as number) <= 0) return null; out.messageId = s.message_id as number; }
+ if (typeof s.reason === 'string') out.reason = s.reason;
+ return out;
- if (!s || typeof s !== 'object' || typeof s.text !== 'string' || !['interim', 'user_final', 'assistant_final', 'error'].includes(s.kind as string)) return null;
- return { kind: s.kind as Subtitle['kind'], role: s.role === 'assistant' ? 'assistant' : 'user', text: s.text };
}
// ── What the controller needs from a real-time client. livekit.ts implements it with livekit-client; tests use fakes. ──
@@ -77,7 +99,10 @@
secure?: () => boolean;
}
+// seq: the highest subtitle sequence applied in this call (one worker job, one sequence). speech: her speech open now;
+// closed: speeches that ended, were cancelled or were superseded here; nothing for them is shown again.
+interface Live { gen: number; abort: AbortController; mic?: MicTrack; room?: VoiceRoom; session?: string; agent: boolean; stated: boolean; hushed: boolean; timer?: ReturnType<typeof setTimeout>; seq: number; speech: string; pro: boolean; message?: number; closed: string[] }
+const CLOSED_KEPT = 64;
-interface Live { gen: number; abort: AbortController; mic?: MicTrack; room?: VoiceRoom; session?: string; agent: boolean; stated: boolean; hushed: boolean; timer?: ReturnType<typeof setTimeout> }
const AGENT_PHASE: Record<string, VoicePhase> = { initializing: 'joining', listening: 'listening', thinking: 'thinking', speaking: 'speaking' };
@@ -127,7 +152,7 @@
// The generation is taken before anything is awaited, so a start that wakes up after a newer one never takes
// the call over; the call it replaces is released first, as end() would.
const prev = this.live;
+ const gen = ++this.gen, live: Live = { gen, abort: new AbortController(), agent: false, stated: false, hushed: false, seq: 0, speech: '', pro: false, closed: [] };
- const gen = ++this.gen, live: Live = { gen, abort: new AbortController(), agent: false, stated: false, hushed: false };
this.live = live;
this.view = { ...OFF, phase: 'opening' }; this.deps.onView(this.view);
const stale = () => gen !== this.gen;
@@ -189,11 +214,11 @@
},
subtitle: payload => {
if (stale()) return;
+ // Replays, late frames of an older turn and frames that overtook each other all sit at or below the
+ // highest sequence already applied in this call.
+ const s = subtitle(payload); if (!s || s.seq <= live.seq) return;
+ live.seq = s.seq;
+ this.hear(live, s);
- const s = subtitle(payload); if (!s) return;
- if (s.kind === 'interim') { this.show({ heard: s.text, spoken: '', ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); if (live.hushed && !live.stated) { live.hushed = false; live.room?.hush(false); } return; }
- if (s.kind === 'user_final') { this.show({ heard: s.text, spoken: '', ...(live.stated ? {} : { phase: 'thinking' as const }) }, gen); this.deps.onSpoke(); this.deps.onRecord(); return; }
- if (s.kind === 'assistant_final') { this.show({ heard: '', spoken: '', ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); this.deps.onRecord(); return; }
- this.show({ heard: '', spoken: s.text, ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); this.deps.onRecord();
},
dropped: () => { if (!stale()) void fail(WORDS.dropped); },
audioBlocked: blocked => this.show({ blocked }, gen),
@@ -224,10 +249,73 @@
if (live) void this.teardown(live, true);
}
+ // One subtitle that passed the sequence gate. Your draft and her line never share a slot: `heard` is yours, `her`
+ // is hers. Her frames act only on the speech open now; a speech once closed here is never shown again.
+ private hear(live: Live, s: Subtitle) {
+ const gen = live.gen, cur = this.view.her, honest = (phase: VoicePhase) => live.stated ? {} : { phase };
+ if (s.kind === 'interim') {
+ // Your draft. Her open line is left to the server's cancel (an SDK barge-in decides, not every sound).
+ this.show({ heard: s.text, spoken: '', ...honest('listening') }, gen);
+ if (live.hushed && !live.stated) { live.hushed = false; live.room?.hush(false); }
+ return;
+ }
+ if (s.kind === 'user_final') {
+ // A new turn of yours: nothing of hers from before may stand after it.
+ if (live.speech) this.close(live, live.speech);
+ this.show({ heard: s.text, spoken: '', her: null, ...honest('thinking') }, gen); this.deps.onSpoke(); this.deps.onRecord();
+ return;
+ }
+ const pro = s.kind.startsWith('proactive_');
+ if (s.kind === 'turn_start' || s.kind === 'proactive_start') {
+ if (live.closed.includes(s.speech) || live.speech === s.speech) return;
+ if (live.speech) this.close(live, live.speech);
+ live.speech = s.speech; live.pro = pro; live.message = s.messageId;
+ // A reach-out arrives whole (its stored text); a reply starts empty and grows.
+ this.show({ spoken: '', ...(pro ? { heard: '' } : {}), her: pro ? { speech: s.speech, text: s.text, state: 'open', messageId: s.messageId, proactive: true } : { speech: s.speech, text: '', state: 'open' }, ...honest(pro ? 'speaking' : 'thinking') }, gen);
+ return;
+ }
+ if (s.speech !== live.speech || pro !== live.pro || (pro && s.messageId !== live.message)) return;
+ const mine = cur?.speech === s.speech ? cur : null;
+ switch (s.kind) {
+ case 'assistant_interim':
+ // Replace, never append: each interim is the whole reply so far.
+ if (mine?.state === 'open' && mine.text !== s.text) this.show({ her: { ...mine, text: s.text } }, gen);
+ return;
+ case 'assistant_final':
+ if (mine?.state !== 'open') return;
+ // Her stored answer: your draft it answered has done its part (as before v2).
+ this.show({ heard: '', her: { ...mine, text: s.text, state: 'final', turnId: s.turnId }, ...honest('listening') }, gen); this.deps.onRecord();
+ return;
+ case 'error':
+ // φ: heard, not stored; her unfinished line goes at once. The cancel that follows closes the speech.
+ this.show({ heard: '', spoken: s.text, ...(mine ? { her: null } : {}), ...honest('listening') }, gen); this.deps.onRecord();
+ return;
+ case 'turn_end': case 'proactive_end':
+ this.close(live, s.speech);
+ // A reply closed without its final never became her reply; a reach-out read to the end stays until the
+ // record or the next turn takes its place.
+ this.show({ ...(mine ? { her: mine.state === 'final' || pro ? { ...mine, state: 'done' as const } : null } : {}), ...honest('listening') }, gen);
+ if (!pro) this.deps.onRecord();
+ return;
+ case 'turn_cancel': case 'proactive_cancel':
+ this.close(live, s.speech);
+ this.show({ ...(mine ? { her: null } : {}), ...honest('listening') }, gen);
+ if (!pro) this.deps.onRecord();
+ return;
+ }
+ }
+ private close(live: Live, speech: string) {
+ if (live.speech === speech) { live.speech = ''; live.pro = false; live.message = undefined; }
+ if (!live.closed.includes(speech)) { live.closed.push(speech); if (live.closed.length > CLOSED_KEPT) live.closed.shift(); }
+ }
+
// You sent text while she was answering by voice: the server cancels her voice turn; keep its last audio silent.
+ // Her line goes at once, whatever state it was in: your words come next on the stage.
hush() {
+ const live = this.live; if (!live) return;
+ if (live.speech) this.close(live, live.speech);
+ if (this.view.her) this.show({ her: null }, live.gen);
+ if (!live.room || (this.view.phase !== 'speaking' && this.view.phase !== 'thinking')) return;
- const live = this.live;
- if (!live?.room || (this.view.phase !== 'speaking' && this.view.phase !== 'thinking')) return;
live.hushed = true; live.room.hush(true);
this.show({ spoken: '' }, live.gen);
}
@@ -285,6 +373,21 @@
}
return next;
}
+// Her voice line as the stage shows it, in the place her text reply would take; null once it has nothing to add.
+// `since` is how many record rows there were when this speech was first seen. It gives way to the record: her reply
+// once its own row (same turn) is there, a reach-out once its stored row is the newest line; a finished line also as
+// soon as anything else came into the record after it, so it never stands in front of a newer turn.
+export function voiceLine(her: HerLine | null, messages: Message[], since: number, incoming: Message[] = []): ShownMessage | null {
+ if (!her || !her.text) return null;
+ const own = (m: Message) => her.proactive ? m.id === String(her.messageId) : !!her.turnId && m.turnId === her.turnId;
+ if (her.proactive ? messages.at(-1)?.id === String(her.messageId) : messages.some(m => m.role === 'companion' && own(m))) return null;
+ if (her.state !== 'open' && messages.slice(since).some(m => !own(m))) return null;
+ // A reach-out keeps its stored row's identity (id, turn), so the row taking over keeps the same lines on the stage.
+ const row = her.proactive ? [...messages, ...incoming].find(own) : undefined;
+ // Her reply still growing holds back its newest line, as a streamed text reply does.
+ return { id: row?.id ?? `voice-${her.speech}`, role: 'companion', text: her.text, lifeIds: [], turnId: row ? row.turnId : her.turnId, ...(her.proactive ? { proactive: true } : {}), ...(her.state === 'open' && !her.proactive ? { live: 'generating' as const } : {}) };
+}
+
// Newest rows down to `since` (exclusive), at most `pages` pages of 50. Voice turns are few per read; a longer gap is
// left to the next re-read rather than read without bound.
export async function recentRows(since: number, signal: AbortSignal, pages = 4): Promise<Message[]> {
diff --git a/web/tests/fake-livekit.js b/web/tests/fake-livekit.js
--- a/web/tests/fake-livekit.js
+++ b/web/tests/fake-livekit.js
@@ -26,7 +26,8 @@
return true;
};
lk.state = s => { const room = lk.rooms.at(-1), p = [...room.remoteParticipants.values()][0]; if (!p) return false; p.attributes['lk.agent.state'] = s; room.emit(RoomEvent.ParticipantAttributesChanged, { 'lk.agent.state': s }, p); return true; };
+ // Frames without an envelope get the worker's v2 one (next seq of this room); explicit frames are sent as given.
+ lk.say = obj => { const room = lk.rooms.at(-1), frame = 'v' in obj ? obj : { v: 2, seq: (room.seq || 0) + 1, ...obj }; if (Number.isSafeInteger(frame.seq)) room.seq = Math.max(room.seq || 0, frame.seq); room.emit(RoomEvent.DataReceived, new TextEncoder().encode(JSON.stringify(frame)), undefined, 0, 'cg.subtitle'); return true; };
- lk.say = obj => { const room = lk.rooms.at(-1); room.emit(RoomEvent.DataReceived, new TextEncoder().encode(JSON.stringify(obj)), undefined, 0, 'cg.subtitle'); return true; };
lk.drop = () => { const room = live(); room.state = 'gone'; room.emit(RoomEvent.Disconnected); return true; };
lk.block = () => { const room = live(); room.canPlaybackAudio = false; room.emit(RoomEvent.AudioPlaybackStatusChanged); return true; };
async function createLocalAudioTrack(constraints) {
diff --git a/web/tests/hel878_mock.mjs b/web/tests/hel878_mock.mjs
--- a/web/tests/hel878_mock.mjs
+++ b/web/tests/hel878_mock.mjs
@@ -4,7 +4,7 @@
// answered with tests/fake-livekit.js (a scripted stand-in, no WebRTC); the real vendored build stays reachable at
// /__real-sdk.js for the API-surface check only. Synthetic data only: no model, no LiveKit, no TTS, no network.
// Usage: node tests/hel878_mock.mjs <port>. Control GETs: /__voice?configured=&tts=, /__refuse/<code|0>,
+// /__voice-turn?stage=user|reply|cancel&text=, /__proactive?text=, /__cut, /__sessions, /__log, /__rows.
-// /__voice-turn?stage=user|reply|cancel&text=, /__cut, /__sessions, /__log, /__rows.
import { createServer } from 'node:http';
import { readFile } from 'node:fs/promises';
import { extname, join, normalize } from 'node:path';
@@ -41,6 +41,8 @@
if (stage === 'cancel') rows.forEach(r => { if (r.turn_id === `voice-${voiceTurn}`) r.status = 'cancelled'; });
return send(res, 200, { turn: voiceTurn });
}
+ // A reach-out the life runtime stored (HEL-898): the worker may read it aloud; the page sees it through the inbox.
+ if (p === '/__proactive') { const id = add({ role: 'companion', text: url.searchParams.get('text') || '', turn_id: `pro-${rows.length + 1}`, proactive: true }); return send(res, 200, { id }); }
// A voice turn took the server's turn gate: the text stream in flight ends without a final; its row is cancelled.
if (p === '/__cut') { const s = textStream; textStream = null; if (s) { rows.forEach(r => { if (r.turn_id === s.turn_id && r.role === 'user') r.status = 'cancelled'; }); s.cut = true; s.res.end(); } return send(res, 200, { cut: !!s }); }
if (p === '/__real-sdk.js') return send(res, 200, await readFile(join(dist, 'vendor/livekit-client-2.22.4.umd.min.js')), 'text/javascript');
diff --git a/web/tests/hel878_shots.mjs b/web/tests/hel878_shots.mjs
--- a/web/tests/hel878_shots.mjs
+++ b/web/tests/hel878_shots.mjs
@@ -114,7 +114,7 @@
await frames('m2-voice-speaking', 900);
await shot('08-moon-desktop-voice-speaking');
await ctl('/__voice-turn?stage=reply&text=' + encodeURIComponent('是橘色的那只吗?它最近总在桥下晒太阳。'));
+await lk(`__lk.say({ kind: 'turn_start', role: 'assistant', text: '', speech_id: 'sp-878-1' })`); await lk(`__lk.say({ kind: 'assistant_final', role: 'assistant', text: '是橘色的那只吗?它最近总在桥下晒太阳。', speech_id: 'sp-878-1', turn_id: 'voice-1' })`); await lk(`__lk.say({ kind: 'turn_end', role: 'assistant', text: '', speech_id: 'sp-878-1', turn_id: 'voice-1' })`); await lk(`__lk.state('listening')`);
-await lk(`__lk.say({ kind: 'assistant_final', role: 'assistant', text: '是橘色的那只吗?它最近总在桥下晒太阳。' })`); await lk(`__lk.state('listening')`);
await until(async () => (await said()).includes('晒太阳'), 6000);
check('her spoken answer reaches the stage once it is in the record', (await said()).includes('晒太阳'), await said());
await shot('09-moon-desktop-voice-reply-on-stage');
@@ -125,7 +125,7 @@
await click('.log-panel .btn-round'); await sleep(400);
// ── 6. φ: her failure line is heard and shown, never recorded. ──
+await lk(`__lk.say({ kind: 'turn_start', role: 'assistant', text: '', speech_id: 'sp-878-phi' })`); await lk(`__lk.say({ kind: 'error', role: 'assistant', text: '我刚刚没接上话,你再说一遍好吗?', speech_id: 'sp-878-phi' })`); await lk(`__lk.say({ kind: 'turn_cancel', role: 'assistant', text: '', speech_id: 'sp-878-phi', reason: 'error_phi' })`); await sleep(400);
-await lk(`__lk.say({ kind: 'error', role: 'assistant', text: '我刚刚没接上话,你再说一遍好吗?' })`); await sleep(400);
check('φ line shown as heard, marked not recorded; not in the record', (await row()).includes('没接上话') && (await row()).includes('未记入聊天') && !(await ctl('/__rows')).some(r => r.text.includes('没接上话')));
await shot('11-moon-desktop-voice-phi');
diff --git a/web/tests/hel898_shots.mjs b/web/tests/hel898_shots.mjs
new file mode 100644
--- /dev/null
+++ b/web/tests/hel898_shots.mjs
@@ -0,0 +1,172 @@
+// HEL-898 browser check: her words while she speaks (cg.subtitle contract v2) on the approved conversation. Drives
+// headless Edge over CDP against tests/hel878_mock.mjs, with tests/fake-livekit.js in place of the LiveKit client and
+// Chromium's fake audio device in place of a microphone: no model, no TTS, no real call. Frames are the worker's v2
+// envelopes (docs/voice-subtitle-contract-v2.md). Usage: node tests/hel898_shots.mjs <edge.exe> <outDir> [port].
+import { spawn } from 'node:child_process';
+import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+const [edge, out, portArg] = process.argv.slice(2);
+const port = Number(portArg || 8878), base = `http://127.0.0.1:${port}`, cdpPort = 9498;
+mkdirSync(out, { recursive: true });
+const profile = join(tmpdir(), `hel898-edge-${Date.now()}`);
+const browser = spawn(edge, ['--headless=new', `--remote-debugging-port=${cdpPort}`, `--user-data-dir=${profile}`, '--no-first-run', '--hide-scrollbars', '--force-color-profile=srgb', '--use-fake-device-for-media-stream', '--use-fake-ui-for-media-stream', '--autoplay-policy=no-user-gesture-required', 'about:blank'], { stdio: 'ignore' });
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+let targets;
+for (let i = 0; i < 60 && !targets; i++) { try { targets = await (await fetch(`http://127.0.0.1:${cdpPort}/json`)).json(); } catch { await sleep(250); } }
+const ws = new WebSocket(targets.find(t => t.type === 'page').webSocketDebuggerUrl);
+await new Promise(r => ws.addEventListener('open', r, { once: true }));
+let id = 0; const waiting = new Map();
+ws.addEventListener('message', ({ data }) => { const m = JSON.parse(data); if (m.id && waiting.has(m.id)) { const { ok, ko } = waiting.get(m.id); waiting.delete(m.id); m.error ? ko(new Error(m.error.message)) : ok(m.result); } });
+const cdp = (method, params = {}) => new Promise((ok, ko) => { const n = ++id; waiting.set(n, { ok, ko }); ws.send(JSON.stringify({ id: n, method, params })); });
+const js = async expr => { const r = await cdp('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }); if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'js error'); return r.result.value; };
+const results = [];
+const check = (name, ok, detail = '') => { results.push(ok); console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` — ${detail}` : ''}`); };
+const ctl = async path => (await fetch(`${base}${path}`)).json();
+const shot = async name => { const { data } = await cdp('Page.captureScreenshot', { format: 'png' }); writeFileSync(join(out, `${name}.png`), Buffer.from(data, 'base64')); };
+const frames = async (name, ms) => { const list = []; const end = Date.now() + ms; while (Date.now() < end) { const { data } = await cdp('Page.captureScreenshot', { format: 'jpeg', quality: 70 }); list.push(data); } list.forEach((d, i) => writeFileSync(join(out, `${name}-${String(i).padStart(2, '0')}.jpg`), Buffer.from(d, 'base64'))); return list.length; };
+const click = async (selector, text = '') => js(`(() => { const el = [...document.querySelectorAll(${JSON.stringify(selector)})].find(e => e.offsetParent !== null && e.textContent.includes(${JSON.stringify(text)})); if (!el) return false; el.click(); return true; })()`);
+const text = sel => js(`[...document.querySelectorAll(${JSON.stringify(sel)})].map(e => e.innerText).join('\\n')`);
+const exists = sel => js(`!!document.querySelector(${JSON.stringify(sel)})`);
+// The stage's own text (every line now on it) and the line read out to assistive tech.
+const stage = () => js(`document.querySelector('.conversation .stage')?.innerText || ''`);
+const said = () => js(`document.querySelector('.conversation .sr')?.textContent || ''`);
+const row = () => js(`document.querySelector('.voice-row.real')?.innerText || ''`);
+const phase = () => js(`document.querySelector('.voice-row.real')?.dataset.phase || 'off'`);
+const lk = expr => js(`window.__lk ? (${expr}) : null`);
+const say = frame => lk(`__lk.say(${JSON.stringify(frame)})`);
+const her = (kind, speech, extra = {}) => say({ kind, role: 'assistant', text: '', speech_id: speech, ...extra });
+const type = t => js(`(() => { const el = document.querySelector('textarea'); const set = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value').set; set.call(el, ${JSON.stringify(t)}); el.dispatchEvent(new Event('input', { bubbles: true })); return true; })()`);
+const until = async (fn, ms = 4000) => { const end = Date.now() + ms; while (Date.now() < end) { if (await fn()) return true; await sleep(100); } return false; };
+const views = { desktop: { width: 1440, height: 900, deviceScaleFactor: 1, mobile: false }, phone: { width: 390, height: 844, deviceScaleFactor: 2, mobile: true } };
+async function view(v) { await cdp('Emulation.setDeviceMetricsOverride', views[v]); await cdp('Emulation.setTouchEmulationEnabled', { enabled: views[v].mobile, maxTouchPoints: 5 }); }
+async function boot(theme) {
+ await cdp('Page.navigate', { url: `${base}/` }); await sleep(500);
+ await js(`localStorage.setItem('cg.theme', ${JSON.stringify(theme)}); sessionStorage.clear(); true`);
+ await cdp('Page.navigate', { url: `${base}/` }); await sleep(1600);
+}
+async function startVoice() {
+ await click('.voice-button'); await until(() => exists('.modal.open'));
+ await click('.modal.open .btn-primary', '开始语音');
+ await until(async () => (await phase()) === 'joining');
+ await until(async () => (await lk('__lk.rooms.at(-1)?.state')) === 'connected');
+ await lk('__lk.join()'); await sleep(200);
+}
+const calls = async () => ctl('/__log');
+// One voice turn of yours as the worker leaves it: the stored row, then user_final; waits for it on the stage.
+const youSay = async words => {
+ const { turn } = await ctl('/__voice-turn?stage=user&text=' + encodeURIComponent(words));
+ await say({ kind: 'user_final', role: 'user', text: words }); await lk(`__lk.state('thinking')`);
+ await until(async () => (await stage()).includes(words.slice(0, 4)), 6000);
+ return turn;
+};
+const rows = async () => ctl('/__rows');
+await cdp('Page.enable'); await cdp('Runtime.enable'); await cdp('Network.enable');
+
+const REPLY = '是橘色的那只吗?它最近总在桥下晒太阳,尾巴一甩一甩的,好像整座桥都是它的。你下次路过可以叫它一声,它会回头看你。';
+const STEPS = ['是橘色的那只吗?它最近总在桥下晒太阳,', '是橘色的那只吗?它最近总在桥下晒太阳,尾巴一甩一甩的,好像整座桥都是它的。', `${REPLY}`];
+
+for (const [theme, v] of [['moon', 'desktop'], ['fog', 'desktop'], ['moon', 'phone'], ['fog', 'phone']]) {
+ const tag = `${theme}-${v}`;
+ await view(v); await boot(theme); await click('.nav-item', '交谈'); await sleep(1400);
+ await startVoice(); await calls();
+
+ // ── 1. A voice turn: your draft and final, then her words growing on the stage before anything is stored. ──
+ await say({ kind: 'interim', role: 'user', text: '今天下班的路上' }); await sleep(200);
+ const prior = (await rows()).filter(r => r.text === REPLY).length;
+ const turn = await youSay('今天下班的路上看到一只猫');
+ await her('turn_start', `a-${tag}`); await lk(`__lk.state('speaking')`);
+ const shots = [];
+ for (const [i, step] of STEPS.entries()) {
+ await her('assistant_interim', `a-${tag}`, { text: step });
+ shots.push(frames(`${tag}-m1-growing-${i}`, 450));
+ await shots.at(-1);
+ }
+ await until(async () => (await stage()).includes('晒太阳'), 8000);
+ check(`${tag}: her reply shows on the stage while she speaks, before it is stored`, (await stage()).includes('晒太阳') && (await rows()).filter(r => r.text === REPLY).length === prior, await stage());
+ check(`${tag}: your draft stays in the voice line, her words never there`, !(await row()).includes('晒太阳'), await row());
+ await shot(`${tag}-01-her-words-growing`);
+
+ // ── 2. Pause and look back while she is still speaking: reading holds, nothing jumps. ──
+ await click('.reader-bar .flow'); await sleep(300);
+ check(`${tag}: pause while she speaks`, await exists('.conversation.reading-paused'));
+ await click('.reader-bar .icon[aria-label="上一句"]'); await sleep(400);
+ await shot(`${tag}-02-paused-looking-back`);
+ await ctl('/__voice-turn?stage=reply&text=' + encodeURIComponent(REPLY));
+ await her('assistant_final', `a-${tag}`, { text: REPLY, turn_id: `voice-${turn}` }); await her('turn_end', `a-${tag}`, { turn_id: `voice-${turn}` }); await lk(`__lk.state('listening')`);
+ await sleep(900);
+ await click('.reader-bar .latest'); await sleep(2500);
+ await click('.log-btn'); await sleep(700);
+ const rec = await text('.log-panel .scroll');
+ check(`${tag}: once stored, the record holds her reply exactly once (re-read, not written by the page)`, rec.split(REPLY).length - 1 === prior + 1, rec.slice(-160).replace(/\n/g, ' '));
+ await shot(`${tag}-03-record-after-final`);
+ await click('.log-panel .btn-round'); await sleep(400);
+ await click('.read-all'); await sleep(500); await shot(`${tag}-04-full-text`); await click('.record.full .btn-round'); await sleep(300);
+
+ // ── 3. Interrupted: no final; her line goes at once; late frames of the cut turn change nothing. ──
+ // Paused reading, then you speak: like sending, your new turn and her answer come in by themselves.
+ await click('.reader-bar .flow'); await sleep(200);
+ check(`${tag}: paused before speaking`, await exists('.conversation.reading-paused'));
+ await youSay('你今天去哪儿了');
+ check(`${tag}: speaking resumes reading for the new turn`, !(await exists('.conversation.reading-paused')));
+ await her('turn_start', `b-${tag}`); await lk(`__lk.state('speaking')`);
+ await her('assistant_interim', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,' });
+ check(`${tag}: second reply growing`, await until(async () => (await stage()).includes('白鹭'), 8000), await stage());
+ await shot(`${tag}-05-before-cut`);
+ await say({ kind: 'interim', role: 'user', text: '等一下' });
+ await her('turn_cancel', `b-${tag}`, { reason: 'interrupted' }); await lk(`__lk.state('listening')`); await ctl('/__voice-turn?stage=cancel');
+ await sleep(150);
+ check(`${tag}: interrupted: her unfinished line is gone at once`, !(await stage()).includes('白鹭'), await stage());
+ await her('assistant_interim', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,真好看。' });
+ await her('assistant_final', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,真好看。', turn_id: 'late' });
+ await sleep(400);
+ check(`${tag}: late frames of the cut turn stay out`, !(await stage()).includes('白鹭'));
+ await shot(`${tag}-06-after-cut`);
+
+ // ── 4. Text takes over while she speaks: her line goes, your typed words come next. ──
+ await youSay('我们聊点别的吧');
+ await her('turn_start', `c-${tag}`); await lk(`__lk.state('speaking')`);
+ await her('assistant_interim', `c-${tag}`, { text: '那我们换个话题吧,你今天晚饭吃了什么,好不好吃?我今天' });
+ check(`${tag}: her line before you type`, await until(async () => (await stage()).includes('换个话题'), 8000), await stage());
+ await type('先等等,我打字说'); await js(`document.querySelector('.send').click(), true`); await sleep(400);
+ check(`${tag}: text over voice: her line is gone, your words are on the stage`, !(await stage()).includes('换个话题') && (await stage()).includes('先等等'), await stage());
+ await until(async () => !(await exists('.reader-bar .stop')), 9000); await lk(`__lk.state('listening')`); await sleep(400);
+
+ // ── 5. A stored reach-out read aloud: its own text in her place, no request, no read mark, interruptible. ──
+ const { id: pro } = await ctl('/__proactive?text=' + encodeURIComponent('刚才路过花店,想起你说过喜欢白色的洋桔梗。'));
+ await calls();
+ await say({ kind: 'proactive_start', role: 'assistant', text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。', speech_id: `p-${tag}`, message_id: pro }); await lk(`__lk.state('speaking')`);
+ await sleep(1500);
+ check(`${tag}: reach-out shown in her place, word for word`, (await stage()).includes('洋桔梗'), await stage());
+ await shot(`${tag}-07-reach-out`);
+ await say({ kind: 'interim', role: 'user', text: '是吗' });
+ await say({ kind: 'proactive_cancel', role: 'assistant', text: '', speech_id: `p-${tag}`, message_id: pro, reason: 'interrupted' }); await lk(`__lk.state('listening')`);
+ await sleep(200);
+ const after = await calls();
+ check(`${tag}: reach-out cut: cleared; the page wrote nothing (no receipt, no read mark, no message)`, !after.some(l => !l.startsWith('GET ')) , after.join(' | '));
+
+ // ── 6. Hanging up mid-sentence clears her line with the call. ──
+ await youSay('我先去睡了');
+ await her('turn_start', `d-${tag}`); await her('assistant_interim', `d-${tag}`, { text: '晚安,明天见。今天说了好多好多话,你要早点睡哦。我会在窗边等你,' });
+ check(`${tag}: her line before hanging up`, await until(async () => (await stage()).includes('早点睡'), 8000), await stage());
+ await click('.voice-row.real .voice-end'); await sleep(400);
+ check(`${tag}: hung up: her line gone, voice off`, !(await stage()).includes('早点睡') && (await phase()) === 'off', await stage());
+ await shot(`${tag}-08-ended`);
+}
+
+// ── 7. Reduced motion: her words still arrive, without animated entrances. ──
+await view('desktop'); await cdp('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-reduced-motion', value: 'reduce' }] });
+await boot('moon'); await click('.nav-item', '交谈'); await sleep(1400); await startVoice();
+await youSay('慢一点说给我听');
+await her('turn_start', 'rm'); await lk(`__lk.state('speaking')`);
+for (const step of STEPS) { await her('assistant_interim', 'rm', { text: step }); await sleep(300); }
+check('reduced motion: her words arrive', await until(async () => (await stage()).includes('晒太阳'), 8000), await stage());
+await frames('reduced-motion-growing', 600);
+await click('.voice-row.real .voice-end'); await sleep(300);
+
+ws.close(); browser.kill();
+try { rmSync(profile, { recursive: true, force: true }); } catch { /* Edge may still hold the profile. */ }
+const failed = results.filter(r => !r).length;
+console.log(`${results.length - failed}/${results.length} passed`);
+process.exit(failed ? 1 : 0);
diff --git a/web/tests/voice.test.ts b/web/tests/voice.test.ts
--- a/web/tests/voice.test.ts
+++ b/web/tests/voice.test.ts
@@ -1,8 +1,14 @@
import { afterEach, describe, expect, it, vi } from 'vitest';
+import { mergeTurns, recentRows, VoiceController, voiceLine, voiceStatus, WORDS, type HerLine, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session';
+import { groupTurns, pagesOf, type ShownMessage } from '../src/chat/reading';
-import { mergeTurns, recentRows, VoiceController, voiceStatus, WORDS, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session';
import type { ServerMessage } from '../src/adapters/real';
import type { Message } from '../src/domain/model';
+import { readFileSync } from 'node:fs';
+// The backend's recorded cg.subtitle frames (contract v2, docs/rework/hel879/subtitle-samples, generated offline).
+type Frame = Record<string, unknown> & { seq: number };
+const sample = (name: string): Frame[] => readFileSync(new URL(`../../docs/rework/hel879/subtitle-samples/${name}.jsonl`, import.meta.url), 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l));
+
// Fake transport and media objects: no LiveKit, no microphone, no network. The fake server follows the HEL-877 voice
// router (src/cybergril/voice/router.py): GET /api/voice/status, POST /api/voice/session -> 201 grant, DELETE
// /api/voice/session/{id} -> 200 or 404.
@@ -176,7 +182,7 @@
const r = await connected(), old = r.transport.rooms[0];
await r.c.end(); r.record.mockClear();
const seen = r.views.length;
+ old.on.agentState('speaking'); old.on.subtitle({ v: 2, seq: 1, kind: 'user_final', role: 'user', text: '旧的话' }); old.on.subtitle({ v: 2, seq: 2, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'old' }); old.on.agentJoined(); old.on.dropped(); old.on.audioBlocked(true);
- old.on.agentState('speaking'); old.on.subtitle({ kind: 'user_final', role: 'user', text: '旧的话' }); old.on.agentJoined(); old.on.dropped(); old.on.audioBlocked(true);
r.transport.mics[0].ended?.();
await flush();
expect(r.views.length).toBe(seen);
@@ -243,10 +249,12 @@
});
it('never confirms a receipt or posts chat itself: the worker owns both', async () => {
const r = await connected(), room = r.transport.rooms[0];
+ for (const line of sample('normal-turn')) room.on.subtitle(line);
+ for (const line of sample('proactive')) room.on.subtitle({ ...line, seq: line.seq + 100 });
- room.on.subtitle({ kind: 'user_final', role: 'user', text: '今天好冷' });
- room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '那多穿一件。' });
await r.c.end();
expect(r.server.calls.some(c => c.url.startsWith('/api/chat'))).toBe(false);
+ // No receipt, no read mark, no message write: the page only talks to the voice routes.
+ expect(r.server.calls.every(c => c.url.startsWith('/api/voice/'))).toBe(true);
});
});
@@ -338,24 +346,32 @@
});
});
+// A v2 frame as the worker's SubtitleBus emits it (src/cybergril/voice/seam.py).
+const frame = (seq: number, kind: string, extra: Record<string, unknown> = {}) => ({ v: 2, seq, kind, role: kind === 'interim' || kind === 'user_final' ? 'user' : 'assistant', text: '', ...extra });
describe('subtitles and the two channels', () => {
it('interim is a draft; user_final cancels a text reply and asks for the record; φ is heard but not recorded', async () => {
const r = await connected(), room = r.transport.rooms[0];
+ room.on.subtitle(frame(1, 'interim', { text: '今天' }));
- room.on.subtitle({ kind: 'interim', role: 'user', text: '今天' });
expect(r.view().heard).toBe('今天'); expect(r.record).not.toHaveBeenCalled();
+ room.on.subtitle(frame(2, 'user_final', { text: '今天好冷' }));
- room.on.subtitle({ kind: 'user_final', role: 'user', text: '今天好冷' });
expect(r.spoke).toHaveBeenCalledTimes(1); expect(r.record).toHaveBeenCalledTimes(1);
+ room.on.subtitle(frame(3, 'turn_start', { speech_id: 'a' }));
+ room.on.subtitle(frame(4, 'assistant_interim', { speech_id: 'a', text: '我刚' }));
+ room.on.subtitle(frame(5, 'error', { speech_id: 'a', text: '我刚才走神了,你再说一遍?' }));
+ expect(r.view()).toMatchObject({ spoken: '我刚才走神了,你再说一遍?', her: null });
+ room.on.subtitle(frame(6, 'turn_cancel', { speech_id: 'a', reason: 'error_phi' }));
+ expect(r.view()).toMatchObject({ spoken: '我刚才走神了,你再说一遍?', her: null });
+ room.on.subtitle(frame(7, 'assistant_interim', { speech_id: 'a', text: '我刚才走神了' })); // after its cancel
+ expect(r.view().her).toBeNull();
- room.on.subtitle({ kind: 'error', role: 'assistant', text: '我刚才走神了,你再说一遍?' });
- expect(r.view().spoken).toBe('我刚才走神了,你再说一遍?');
- room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '那多穿一件。' });
- expect(r.view()).toMatchObject({ heard: '', spoken: '' });
room.on.subtitle({ kind: 'bogus', text: 1 }); room.on.subtitle(null);
expect(r.record).toHaveBeenCalledTimes(3);
+ expect(r.server.calls.some(c => c.url.startsWith('/api/chat') || c.url.startsWith('/api/messages'))).toBe(false);
});
it('without the agent state attribute, subtitles still move the state honestly', async () => {
const r = await connected(), room = r.transport.rooms[0];
+ room.on.subtitle(frame(1, 'user_final', { text: '在吗' })); expect(r.view().phase).toBe('thinking');
+ room.on.subtitle(frame(2, 'turn_start', { speech_id: 'a' })); expect(r.view().phase).toBe('thinking');
+ room.on.subtitle(frame(3, 'assistant_final', { speech_id: 'a', turn_id: 't', text: '在。' })); expect(r.view().phase).toBe('listening');
- room.on.subtitle({ kind: 'user_final', role: 'user', text: '在吗' }); expect(r.view().phase).toBe('thinking');
- room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '在。' }); expect(r.view().phase).toBe('listening');
});
it('typing over her spoken answer silences what is left of it until she is listening again', async () => {
const r = await connected(), room = r.transport.rooms[0];
@@ -367,6 +383,187 @@
});
});
+// HEL-898: her words while she speaks (contract v2). The page shows them; it never stores, confirms or marks them.
+describe('her voice line (cg.subtitle v2)', () => {
+ const play = (room: Room, frames: Frame[], after?: (f: Frame) => void) => { for (const f of frames) { room.on.subtitle(f); after?.(f); } };
+ it('rejects anything that is not exactly a v2 frame', async () => {
+ const r = await connected(), room = r.transport.rooms[0], seen = r.views.length;
+ const bad = [
+ { kind: 'user_final', role: 'user', text: 'v1 没有信封' },
+ { ...frame(1, 'user_final', { text: '版本不对' }), v: 3 },
+ { ...frame(1, 'user_final', { text: '序号不对' }), seq: 0 },
+ { ...frame(1, 'user_final', { text: '序号不对' }), seq: 1.5 },
+ { ...frame(1, 'user_final', { text: '序号不对' }), seq: '1' },
+ frame(1, 'turn_start'), // no speech_id
+ { ...frame(1, 'turn_start', { speech_id: 'a' }), role: 'user' }, // her kind, your role
+ { ...frame(1, 'interim', { text: '草稿' }), role: 'assistant' }, // your kind, her role
+ frame(1, 'assistant_final', { speech_id: 'a', text: '无归属' }), // final without turn_id
+ frame(1, 'proactive_start', { speech_id: 'p', text: '无行号' }), // reach-out without message_id
+ frame(1, 'proactive_start', { speech_id: 'p', text: '行号不对', message_id: '17' }),
+ frame(1, 'assistant_text', { speech_id: 'a', text: '未知种类' }),
+ [frame(1, 'user_final', { text: '数组' })],
+ ];
+ for (const b of bad) room.on.subtitle(b);
+ expect(r.views.length).toBe(seen); expect(r.record).not.toHaveBeenCalled(); expect(r.spoke).not.toHaveBeenCalled();
+ // A rejected frame does not move the sequence: the first good one still lands.
+ room.on.subtitle(frame(1, 'interim', { text: '好的' })); expect(r.view().heard).toBe('好的');
+ });
+ it('normal turn: your draft and final, her growing reply replaced in place, then her stored final', async () => {
+ const r = await connected(), room = r.transport.rooms[0], frames = sample('normal-turn');
+ const steps: (VoiceView['her'] | string)[] = [];
+ play(room, frames, () => steps.push(r.view().her ? { ...r.view().her! } : r.view().heard));
+ expect(steps).toEqual([
+ '你在听的', '你在听吗?',
+ { speech: 'speech_normal', text: '', state: 'open' },
+ { speech: 'speech_normal', text: '在听呢,', state: 'open' },
+ { speech: 'speech_normal', text: '在听呢,在听呢,你刚才说到一半的那件事我还在想。', state: 'open' },
+ { speech: 'speech_normal', text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'final', turnId: '0e2f-demo-turn' },
+ { speech: 'speech_normal', text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'done', turnId: '0e2f-demo-turn' },
+ ]);
+ expect(r.view().heard).toBe(''); // her stored answer retires the draft it answered
+ // Her words never land in your draft slot, and yours never in hers.
+ expect(r.views.every(v => !v.her || !v.heard.includes('在听呢'))).toBe(true);
+ expect(r.spoke).toHaveBeenCalledTimes(1);
+ expect(r.record).toHaveBeenCalledTimes(3); // your final, her final, her turn's end: re-read, never written
+ // The whole recording replayed (same seq) changes nothing.
+ const seen = r.views.length; play(room, frames);
+ expect(r.views.length).toBe(seen);
+ // Your next turn: her finished line does not stay in front of it.
+ room.on.subtitle(frame(8, 'interim', { text: '那' })); expect(r.view().her?.state).toBe('done');
+ room.on.subtitle(frame(9, 'user_final', { text: '那你想到了吗' })); expect(r.view().her).toBeNull();
+ });
+ it('interrupted: no final, her line goes at once, its late frames never reach the new turn', async () => {
+ const r = await connected(), room = r.transport.rooms[0], frames = sample('interrupt');
+ play(room, frames.slice(0, 2));
+ expect(r.view().her).toMatchObject({ speech: 'speech_cut', text: '我正想跟你说,今天下午', state: 'open' });
+ play(room, frames.slice(2, 3));
+ expect(r.view().her).toBeNull();
+ // Late increments of the cut turn, even with a higher seq, and a replay of its start.
+ room.on.subtitle(frame(4, 'assistant_interim', { speech_id: 'speech_cut', text: '我正想跟你说,今天下午去了' }));
+ room.on.subtitle(frame(5, 'assistant_final', { speech_id: 'speech_cut', turn_id: 'old', text: '我正想跟你说,今天下午去了河边。' }));
+ room.on.subtitle(frame(6, 'turn_start', { speech_id: 'speech_cut' }));
+ expect(r.view().her).toBeNull();
+ play(room, frames.slice(3).map(f => ({ ...f, seq: f.seq + 10 })));
+ expect(r.view()).toMatchObject({ heard: '', her: { speech: 'speech_next', text: '好,你先说。', state: 'done', turnId: '0e2f-demo-next' } });
+ });
+ it('frames that overtook each other: a lower seq is stale; an older speech never touches or closes the new one', async () => {
+ const r = await connected(), room = r.transport.rooms[0];
+ room.on.subtitle(frame(1, 'turn_start', { speech_id: 'a' }));
+ room.on.subtitle(frame(3, 'assistant_interim', { speech_id: 'a', text: '一二三' }));
+ room.on.subtitle(frame(2, 'assistant_interim', { speech_id: 'a', text: '一二' })); // overtaken: dropped
+ expect(r.view().her?.text).toBe('一二三');
+ // Two handles: b starts while a is still open (queued then generated). a is closed here by b.
+ room.on.subtitle(frame(4, 'turn_start', { speech_id: 'b' }));
+ room.on.subtitle(frame(5, 'assistant_interim', { speech_id: 'a', text: '一二三四' }));
+ room.on.subtitle(frame(6, 'turn_end', { speech_id: 'a', turn_id: 'ta' }));
+ room.on.subtitle(frame(7, 'turn_cancel', { speech_id: 'a', reason: 'interrupted' }));
+ room.on.subtitle(frame(8, 'proactive_cancel', { speech_id: 'b', message_id: 3, reason: 'interrupted' })); // wrong family
+ expect(r.view().her).toEqual({ speech: 'b', text: '', state: 'open' });
+ room.on.subtitle(frame(9, 'assistant_interim', { speech_id: 'b', text: '好' }));
+ room.on.subtitle(frame(10, 'turn_end', { speech_id: 'b', turn_id: 'tb' })); // closed without final
+ expect(r.view().her).toBeNull();
+ room.on.subtitle(frame(11, 'assistant_final', { speech_id: 'b', turn_id: 'tb', text: '好。' }));
+ expect(r.view().her).toBeNull();
+ });
+ it('reach-out read aloud: shown whole with its stored message id, interruptible, nothing marked or stored', async () => {
+ const r = await connected(), room = r.transport.rooms[0], frames = sample('proactive');
+ play(room, frames.slice(0, 1));
+ expect(r.view()).toMatchObject({ phase: 'speaking', her: { speech: 'speech_pro1', text: '我看到一句很适合现在的话,先念给你听。', state: 'open', messageId: 17, proactive: true } });
+ room.on.subtitle(frame(2, 'proactive_end', { speech_id: 'speech_pro1', message_id: 99 })); // another row's end
+ expect(r.view().her?.state).toBe('open');
+ play(room, frames.slice(1, 2).map(f => ({ ...f, seq: 3 })));
+ expect(r.view().her).toMatchObject({ messageId: 17, state: 'done' });
+ play(room, frames.slice(2).map(f => ({ ...f, seq: f.seq + 10 })));
+ expect(r.view().her).toBeNull();
+ // Reaching out is not a turn of yours: no "you spoke", no record write, no request at all.
+ expect(r.spoke).not.toHaveBeenCalled(); expect(r.record).not.toHaveBeenCalled();
+ expect(r.server.calls.filter(c => !c.url.startsWith('/api/voice/'))).toEqual([]);
+ });
+ it('a reach-out cut by your voice: the cancel clears it, your words follow', async () => {
+ const r = await connected(), room = r.transport.rooms[0];
+ room.on.subtitle(frame(1, 'proactive_start', { speech_id: 'p', message_id: 18, text: '还有一件事想告诉你——' }));
+ room.on.subtitle(frame(2, 'interim', { text: '等等' }));
+ expect(r.view()).toMatchObject({ heard: '等等', her: { speech: 'p', state: 'open' } });
+ room.on.subtitle(frame(3, 'proactive_cancel', { speech_id: 'p', message_id: 18, reason: 'interrupted' }));
+ expect(r.view()).toMatchObject({ heard: '等等', her: null });
+ });
+ it('text taking over, hanging up and starting again each clear her line at once', async () => {
+ const r = await connected(), room = r.transport.rooms[0];
+ room.on.agentState('speaking');
+ room.on.subtitle(frame(1, 'turn_start', { speech_id: 'a' }));
+ room.on.subtitle(frame(2, 'assistant_interim', { speech_id: 'a', text: '那我们' }));
+ r.c.hush();
+ expect(r.view().her).toBeNull(); expect(room.hushed).toEqual([true]);
+ room.on.subtitle(frame(3, 'assistant_interim', { speech_id: 'a', text: '那我们明天' }));
+ expect(r.view().her).toBeNull();
+ // A finished line is cleared by text too: your typed line comes next on the stage.
+ room.on.subtitle(frame(4, 'turn_start', { speech_id: 'b' }));
+ room.on.subtitle(frame(5, 'assistant_interim', { speech_id: 'b', text: '好。' }));
+ room.on.subtitle(frame(6, 'assistant_final', { speech_id: 'b', turn_id: 'tb', text: '好。' }));
+ room.on.subtitle(frame(7, 'turn_end', { speech_id: 'b', turn_id: 'tb' }));
+ expect(r.view().her?.state).toBe('done');
+ r.c.hush(); expect(r.view().her).toBeNull();
+ // Hanging up mid-sentence.
+ room.on.subtitle(frame(8, 'turn_start', { speech_id: 'c' }));
+ room.on.subtitle(frame(9, 'assistant_interim', { speech_id: 'c', text: '嗯' }));
+ await r.c.end(); expect(r.view().her).toBeNull();
+ room.on.subtitle(frame(10, 'assistant_interim', { speech_id: 'c', text: '嗯,晚安' }));
+ expect(r.view().her).toBeNull();
+ // A new call is a new worker job: its sequence starts again; the old room's frames stay dead.
+ await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined();
+ next.on.subtitle(frame(1, 'turn_start', { speech_id: 'n' }));
+ next.on.subtitle(frame(2, 'assistant_interim', { speech_id: 'n', text: '又见面了' }));
+ room.on.subtitle(frame(11, 'turn_start', { speech_id: 'z' }));
+ room.on.subtitle(frame(12, 'assistant_interim', { speech_id: 'n', text: '旧房间' }));
+ expect(r.view().her).toEqual({ speech: 'n', text: '又见面了', state: 'open' });
+ await r.c.end();
+ });
+ it('the room dropping mid-sentence clears her line with the call', async () => {
+ const r = await connected(), room = r.transport.rooms[0];
+ room.on.subtitle(frame(1, 'turn_start', { speech_id: 'a' }));
+ room.on.subtitle(frame(2, 'assistant_interim', { speech_id: 'a', text: '我' }));
+ room.on.dropped(); await flush();
+ expect(r.view()).toMatchObject({ phase: 'off', her: null, notice: WORDS.dropped });
+ });
+});
+
+describe('her voice line on the stage', () => {
+ const M = (id: number, role: Message['role'], text: string, turnId?: string, extra: Partial<Message> = {}): Message => ({ id: String(id), role, text, lifeIds: [], turnId, status: 'sent', ...extra });
+ const metrics = { width: 900, font: 23, compact: false };
+ const stage = (list: ShownMessage[]) => pagesOf(groupTurns(list).at(-1), metrics).map(p => `${p.key}=${p.text}`);
+ const you = M(4, 'user', '你在听吗?', 'T');
+ it('grows in her slot after your voice turn; the stored row takes over under the same line keys', () => {
+ const open: HerLine = { speech: 's', text: '在听呢,你刚才说到一半的那件事我一直还在心里想着。后来呢?', state: 'open' };
+ const growing = voiceLine(open, [you], 1)!;
+ expect(growing).toMatchObject({ role: 'companion', live: 'generating' });
+ // The newest line of a reply still growing is held back, as in a streamed text reply.
+ expect(stage([you, growing])).toEqual(['T:0:0=你在听吗?', 'T:1:0=在听呢,你刚才说到一半的那件事我一直还在心里想着。']);
+ const done: HerLine = { ...open, state: 'done', turnId: 'T' };
+ const shown = stage([you, voiceLine(done, [you], 1)!]);
+ expect(shown).toEqual(['T:0:0=你在听吗?', 'T:1:0=在听呢,你刚才说到一半的那件事我一直还在心里想着。', 'T:1:1=后来呢?']);
+ const stored = [you, M(5, 'companion', done.text, 'T')];
+ expect(voiceLine(done, stored, 1)).toBeNull();
+ expect(stage(stored)).toEqual(shown);
+ });
+ it('a finished line never stands in front of a newer turn; an open one waits for its own end', () => {
+ const done: HerLine = { speech: 's', text: '好。', state: 'done', turnId: 'T' };
+ expect(voiceLine(done, [you, M(6, 'user', '还有', 'U')], 1)).toBeNull();
+ expect(voiceLine({ ...done, state: 'open' }, [you, M(6, 'user', '还有', 'U')], 1)).not.toBeNull();
+ expect(voiceLine({ ...done, text: '' }, [you], 1)).toBeNull();
+ });
+ it('a reach-out read aloud keeps its stored row identity and gives way when that row is the newest line', () => {
+ const line: HerLine = { speech: 'p', text: '我看到一句很适合现在的话,先念给你听。', state: 'open', messageId: 17, proactive: true };
+ const row = M(17, 'companion', line.text, 'R', { proactive: true });
+ const before = [you, M(5, 'companion', '在。', 'T')];
+ const waiting = voiceLine(line, before, 2, [row])!;
+ expect(waiting).toMatchObject({ id: '17', turnId: 'R', proactive: true }); expect(waiting.live).toBeUndefined();
+ expect(stage([...before, waiting])).toEqual(stage([...before, row]));
+ expect(voiceLine({ ...line, state: 'done' }, [...before, row], 2)).toBeNull();
+ // Not on the page at all: shown under its own key; nothing is created for the record.
+ expect(voiceLine(line, before, 2)).toMatchObject({ id: 'voice-p', proactive: true });
+ });
+});
+
describe('record re-read', () => {
const msg = (id: number, role: Message['role'], turnId: string, extra: Partial<Message> = {}): Message => ({ id: String(id), role, text: `${role}${id}`, lifeIds: [], turnId, status: 'sent', ...extra });
it('adds voice turns once, in id order; leaves reach-outs to the inbox and text turns to the stream', () => {