864 lines
77 KiB
Diff
864 lines
77 KiB
Diff
diff --git a/docs/requirements/handoff-HEL-898.json b/docs/requirements/handoff-HEL-898.json
|
||
new file mode 100644
|
||
--- /dev/null
|
||
+++ b/docs/requirements/handoff-HEL-898.json
|
||
@@ -0,0 +1,39 @@
|
||
+{
|
||
+ "baseline_version": "1.11",
|
||
+ "read_commit": "3228b1e0d7f822b7030f945e285eac74126771aa",
|
||
+ "task": "HEL-898(HEL-879 接线卡,视觉总监本人编码):在已认可的交谈界面接上她的增量字幕(cg.subtitle 契约 v2)与语音内主动开口呈现,处理打断/文字接管/错误/挂断收尾;不另做聊天页、不改主题与整体动效",
|
||
+ "requirement_ids": [
|
||
+ "CG-004", "CG-006", "CG-010", "CG-011", "CG-018", "CG-020",
|
||
+ "VIS-001", "VIS-002", "VIS-003", "VIS-004", "VIS-005", "VIS-006", "VIS-007", "VIS-008", "VIS-009", "VIS-010", "VIS-011", "VIS-012", "VIS-013",
|
||
+ "UX-001", "UX-002", "UX-005", "UX-006",
|
||
+ "AM-001",
|
||
+ "GOV-001", "GOV-002", "GOV-003", "GOV-004", "GOV-005", "GOV-006",
|
||
+ "DEL-001"
|
||
+ ],
|
||
+ "deliverable": "自总管输入包 3228b1e(INPUT-MANIFEST 逐文件 SHA-256)起的前端补丁,只动 web/src/voice/session.ts、web/src/chat/ChatPanel.tsx 与测试:1) session.ts 严格解析 v2 信封(v===2、正整数 seq、kind 白名单、角色与种类对应、她的事件必带 speech_id、assistant_final 必带 turn_id、proactive_* 必带正整数 message_id;不合格整条丢弃且不推进序号);每次通话(控制器代次)独立 seq 高水位,seq<=已应用即丢;按 speech 生命周期只接受当前打开的一轮,已关闭/被取代的 speech 不再显示,旧 speech 的高 seq 事件不能关闭新轮。VoiceView 新增 her(speech/text/state open|final|done/turnId/messageId/proactive),与用户 ASR 草稿 heard 分槽。2) turn_start 开轮,assistant_interim 整句替换,assistant_final 原子替换为落库文本并请页面回读 /api/messages,turn_end 关轮(无 final 的关轮按未成为正式回复即清),turn_cancel/error(φ)/user_final/文字接管 hush/结束/掉线/卸载/重开即时清;φ 原句仍在语音行以“未记入聊天”显示。3) proactive_start 以落库原文、同源 message_id 显示在她的位置,proactive_end 保留至记录或下一轮接替,proactive_cancel 即清;不发任何请求、不标已读、不开麦、不重 TTS、不进入正式记录。4) ChatPanel:她的语音行进入与文字回复相同的舞台槽位(当前一两句、自动续读、暂停/回看、全文),生长中沿用文字流的“最新一句先压住”规则,逐分片不重启动画;落库行到来时同一 turn 同一位置接替,行键不变不重放;被撤回的语音行以 instant 方式立即离场(不淡出);减少动态设置沿用既有逻辑。你用语音开始新一轮时与发送文字一致恢复自动呈现(原先暂停状态下她的回答不会进来)。5) 测试:voice.test.ts 改为 v2 帧并新增 11 项(录制样例三场景回放、信封负例、乱序/重放/迟到/双 handle、φ、主动开口及打断、文字接管/挂断/重开、舞台行键接替);fake-livekit.js 未带信封的帧自动套 v2;hel878_mock.mjs 增 /__proactive;hel878_shots.mjs 的她的 final 与 φ 改为 v2 生命周期;新增 hel898_shots.mjs(两主题×电脑/手机)",
|
||
+ "acceptance": [
|
||
+ "假 LiveKit 有序回放 normal-turn.jsonl:用户草稿→用户终稿→turn_start→两次增量(整句替换)→final(带 turn_id)→turn_end;她的字在舞台出现时 /__rows 尚无该回复;落库后记录中只出现一次(页面只回读,不写)",
|
||
+ "interrupt.jsonl 与负例:turn_cancel 后她的未完成行立即离场;被取消轮之后的高 seq 增量/final/重复 turn_start 均不显示;整段录制重放不改变界面;双 handle 时旧 speech 的 turn_end/turn_cancel 不关闭新轮;无 final 的 turn_end 视为未成为正式回复并清除",
|
||
+ "φ:error 原句在语音行显示并标“未记入聊天”,她的增量行立即清除,随后 turn_cancel(error_phi) 不抹掉 φ 原句;不落正式记录",
|
||
+ "proactive.jsonl:原文一字不差显示在她的位置并保留落库行身份(id/turn),可被你的语音打断(proactive_cancel 即清);全程页面无任何非 GET 请求(无回执、无已读、无消息写入),不调用 onSpoke/onRecord",
|
||
+ "文字接管、结束语音、掉线、新通话:她的行即时清;旧房间迟到帧被代次挡住;新通话序号从 1 重新开始并正常显示",
|
||
+ "两主题(月光/雾光)电脑 1440×900 与手机 390×844 截图与连续帧:沿用已认可位置与组件,无新增组件或样式;减少动态设置下她的字照常出现、无入场动画",
|
||
+ "工程:tsc 0 错;eslint 0 错 1 条既有警告(ChatPanel fullButton ref,HEL-897 起即有);vitest 8 文件 99/99;vite build 通过;浏览器 hel898 61/61,HEL-878 回归 49/49;修前对照:新 voice.test.ts 在原 session.ts/ChatPanel 上 12 项失败"
|
||
+ ],
|
||
+ "deferred": [
|
||
+ "真实通话未测:真实 LiveKit/WebRTC、真实 worker 与 SDK 回调时序、真机/Safari/iOS、可信 HTTPS+WSS、R5 连续三轮/1 秒打断/停麦无残音/有限预算,均仍由 HEL-797 主卡承接;本卡全部为假 LiveKit、假音频设备与 Chromium 无头 Edge",
|
||
+ "CG-011 语音内主动开口仅是“会话开时念一条已落库旧留言”的最小接缝(后台默认关,本卡不启用);会话中持续主动联系、激活节奏、未读联动策略未完成,不得记为 CG-011 完成",
|
||
+ "后台并行小修(建房延迟后的清理唤醒、主动 say 的 start 发得过晚、无完整 receipt 错误发 final)由原总工在 HEL-879 处理;本卡按 v2 正确语义实现,最终后台 commit 出来后需按其统一复核(特别是 proactive_start 晚于音频开始时的呈现)",
|
||
+ "已知边界:seq 按“一次 worker job 一个序列”实现;若同一房间内 worker 重连且序号归零,前端会按契约丢弃其帧直到超过旧高水位——需后台确认 job 与房间一一对应",
|
||
+ "原版 Python 需求检查(scripts/check_requirements.py --base-ref origin/main,含本交接)未在本运行时执行(无 Python、无 git),须总管对实际目标分支复跑",
|
||
+ "D1 音色、cap=0、R6/R7、整体沉浸体验由主卡与白栖知决定;工程自测不是用户体验验收"
|
||
+ ],
|
||
+ "requirement_changes": [],
|
||
+ "verification": [
|
||
+ "读取:HEL-898 卡(启动交接与原说明)、输入包 3228b1e:docs/voice-subtitle-contract-v2.md、docs/rework/hel879/voice-seam-interfaces.md、subtitle-samples/*.jsonl、src/cybergril/voice/seam.py 与 worker.py(事件发出点)、AGENTS.md、docs/requirements/README.md/current.md(v1.11)/coverage.md/changes.md、sources/integration-plan-v1.md R5",
|
||
+ "工具:沿用本机既有离线工具链(HEL-834 留存 node v22.11.0;HEL-897 留存 web/node_modules,package-lock 哈希与本包一致,目录联接使用);未下载、未安装",
|
||
+ "web:tsc --noEmit 0;eslint src 0 错 1 既有警告;vitest run 8 文件 99/99(原 88 + 新增);vite build 通过;日志随交付包",
|
||
+ "浏览器:node tests/hel878_mock.mjs + node tests/hel898_shots.mjs(无头 Edge、脚本化 LiveKit 替身、Chromium 假音频设备)61/61;node tests/hel878_shots.mjs 回归 49/49;截图与连续帧随交付包",
|
||
+ "无模型、嵌入、ASR、TTS 请求,无真实麦克风采集,无部署、合并、服务或绑定改动;未改后台、总装分支或 PR#11"
|
||
+ ]
|
||
+}
|
||
diff --git a/web/src/chat/ChatPanel.tsx b/web/src/chat/ChatPanel.tsx
|
||
--- a/web/src/chat/ChatPanel.tsx
|
||
+++ b/web/src/chat/ChatPanel.tsx
|
||
@@ -10,7 +10,7 @@
|
||
import { motionOff } from '../stage/roomScene';
|
||
import { LineStage, type PresentHow } from './lineStage';
|
||
import { anchorOf, cursorAt, groupTurns, pagesOf, ReadingClock, type Anchor, type Metrics, type ShownMessage } from './reading';
|
||
+import { mergeTurns, OFF, recentRows, VoiceController, voiceLine, voiceStatus, WORDS, type VoiceView } from '../voice/session';
|
||
-import { mergeTurns, OFF, recentRows, VoiceController, voiceStatus, WORDS, type VoiceView } from '../voice/session';
|
||
import { loadTransport } from '../voice/livekit';
|
||
import { newestId, serverId } from '../adapters/inbox';
|
||
|
||
@@ -19,6 +19,8 @@
|
||
const typingTarget = (el: Element | null) => !!el && (el.matches('textarea,input,select,[contenteditable="true"]'));
|
||
|
||
const NO_IDS: string[] = [], NO_MESSAGES: Message[] = [];
|
||
+// Marks "your next turn is one you said by voice" until its row is read back and its turn key is known.
|
||
+const SPOKEN = '\u0000spoken';
|
||
export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation, onRecord, offline, fail, log, setLog, focusLife, focusIds = NO_IDS, real = false, onReceipt, incoming = NO_MESSAGES, onAdmit }: {
|
||
real?: boolean;
|
||
// Real only: the server accepted a complete-receipt confirmation (the life it may unlock is decided later).
|
||
@@ -68,7 +70,8 @@
|
||
};
|
||
const spoke = useRef(() => {});
|
||
// Speaking cancels a text reply still streaming (the server's turn gate already did); it is kept as a cut fragment.
|
||
+ // Like sending, it starts a new turn that comes in by itself, even if you had paused reading.
|
||
+ spoke.current = () => { if (busyNow.current) stop(true); sentTurn.current = SPOKEN; if (!flow) setFlash('新的一轮会自动呈现,随时可以暂停'); };
|
||
- spoke.current = () => { if (busyNow.current) stop(true); };
|
||
useEffect(() => {
|
||
if (!real) return;
|
||
const sync = recordSync.current;
|
||
@@ -99,17 +102,27 @@
|
||
// Real messages are trimmed by the server before transmission; their lifeIds point at records this page has not
|
||
// necessarily loaded, so the demo's local filter would wrongly hide them.
|
||
const view = useMemo(() => real ? state : publicView(state), [state, real]);
|
||
+ // What she says by voice takes the place her text reply would take (HEL-898). It is shown, never stored: once the
|
||
+ // record holds it (same turn, or the same stored reach-out as the newest line) the row takes over under the same
|
||
+ // line keys; a finished line also gives way as soon as anything else comes into the record after it.
|
||
+ const her = real ? voiceView.her : null, herBase = useRef({ speech: '', at: 0 });
|
||
+ if (her && herBase.current.speech !== her.speech) herBase.current = { speech: her.speech, at: view.messages.length };
|
||
+ const herLine = useMemo(() => reply ? null : voiceLine(her, view.messages, herBase.current.at, incoming), [her, reply, view.messages, incoming]);
|
||
+ const herOpen = !!herLine && her?.state === 'open', talking = busy || herOpen;
|
||
// The cut-off part of a stopped reply stays with its own turn (it is shown, never stored).
|
||
const shown = useMemo<ShownMessage[]>(() => {
|
||
const list: ShownMessage[] = [...view.messages];
|
||
if (reply) list.push({ id: 'live-reply', role: 'companion', text: reply, lifeIds: [], turnId: activeTurn, live: 'generating' });
|
||
+ else {
|
||
+ if (interrupted) {
|
||
+ const at = list.map(m => m.turnId).lastIndexOf(interrupted.turn);
|
||
+ list.splice(at < 0 ? list.length : at + 1, 0, { id: 'cut-reply', role: 'companion', text: interrupted.text, lifeIds: [], turnId: interrupted.turn, live: 'cancelled' });
|
||
+ }
|
||
+ if (herLine) list.push(herLine);
|
||
- else if (interrupted) {
|
||
- const at = list.map(m => m.turnId).lastIndexOf(interrupted.turn);
|
||
- list.splice(at < 0 ? list.length : at + 1, 0, { id: 'cut-reply', role: 'companion', text: interrupted.text, lifeIds: [], turnId: interrupted.turn, live: 'cancelled' });
|
||
}
|
||
return list;
|
||
+ }, [view.messages, reply, activeTurn, interrupted, herLine]);
|
||
+ const turn = useMemo(() => groupTurns(shown).at(-1), [shown]), opener = useRef(''); opener.current = turn?.items[0]?.role || '';
|
||
- }, [view.messages, reply, activeTurn, interrupted]);
|
||
- const turn = useMemo(() => groupTurns(shown).at(-1), [shown]);
|
||
const stageRef = useRef<HTMLDivElement>(null), lines = useRef<LineStage | null>(null), meter = useRef<HTMLElement>(null);
|
||
const [metrics, setMetrics] = useState<Metrics>(() => metricsFor(Math.min(900, innerWidth * .88)));
|
||
const pages = useMemo(() => pagesOf(turn, metrics), [turn, metrics]);
|
||
@@ -126,7 +139,7 @@
|
||
// Her own message that arrives while you are here waits its turn: never while she is answering you, a sheet is
|
||
// open, you paused or went back, or the line on screen has not had its time. Then it comes in as a new turn.
|
||
const waiting = useRef({ count: 0, busy: false, sheet: false }), admit = useRef(() => {});
|
||
+ waiting.current = { count: incoming.length, busy: talking || voiceView.phase === 'thinking' || voiceView.phase === 'speaking', sheet: log || full };
|
||
- waiting.current = { count: incoming.length, busy: busy || voiceView.phase === 'thinking' || voiceView.phase === 'speaking', sheet: log || full };
|
||
admit.current = () => { const add = incoming.filter(m => !live.current.some(k => k.id === m.id)); onAdmit?.(incoming.map(m => m.id)); if (add.length) change([...live.current, ...add]); };
|
||
|
||
function stop(quiet = false) { gate.current.cancel(); if (replyLive.current) setInterrupted({ text: replyLive.current, turn: turnLive.current }); replyLive.current = ''; setBusy(false); setReply(''); if (!quiet) onPresence('停下来了'); change(live.current.map(m => m.status === 'pending' ? { ...m, status: 'cancelled' } : m)); }
|
||
@@ -154,8 +167,12 @@
|
||
const key = turn?.key || '';
|
||
if (key === turnKey.current) return;
|
||
turnKey.current = key; how.current = 'jump'; head.current = START; setAnchor(START);
|
||
+ // Your own new turn (sent, or said by voice: its server turn id is only known once the row is read back).
|
||
+ if (key && (key === sentTurn.current || (sentTurn.current === SPOKEN && opener.current === 'user'))) { sentTurn.current = key; setFlowState(true); }
|
||
- if (key && key === sentTurn.current) setFlowState(true);
|
||
}, [turn?.key]);
|
||
+ // Her voice line withdrawn (cut, φ, text took over, you spoke, the call ended): it leaves at once, no fade.
|
||
+ const hadLine = useRef(false);
|
||
+ useLayoutEffect(() => { if (hadLine.current && !her) how.current = 'instant'; hadLine.current = !!herLine; }, [her, herLine]);
|
||
const signature = pages.map(p => p.key).join('|');
|
||
useLayoutEffect(() => {
|
||
lines.current?.present(pages, cursor, how.current, name, !motionOff());
|
||
@@ -165,8 +182,8 @@
|
||
useLayoutEffect(() => {
|
||
if (!page) return;
|
||
const mine = page.message, cancelled = mine.live === 'cancelled' || (mine.status === 'cancelled' && mine.role === 'companion');
|
||
+ const text = cancelled && page.last ? '未完成 · 已取消' : mine.role === 'user' && mine.status === 'failed' ? '没发出去' : mine.role === 'user' && mine.status === 'cancelled' && atEnd ? '本轮已取消' : atEnd && talking && mine.role === 'companion' ? '还在说' : '';
|
||
+ lines.current?.mark({ text, waiting: atEnd && talking && mine.role === 'companion' });
|
||
- const text = cancelled && page.last ? '未完成 · 已取消' : mine.role === 'user' && mine.status === 'failed' ? '没发出去' : mine.role === 'user' && mine.status === 'cancelled' && atEnd ? '本轮已取消' : atEnd && busy && mine.role === 'companion' ? '还在说' : '';
|
||
- lines.current?.mark({ text, waiting: atEnd && busy && mine.role === 'companion' });
|
||
});
|
||
useEffect(() => { lines.current?.rename(name); }, [name]);
|
||
|
||
@@ -306,7 +323,7 @@
|
||
useLayoutEffect(() => { const t = textarea.current; if (!t) return; t.style.height = 'auto'; t.style.height = Math.min(t.scrollHeight, 88) + 'px'; }, [draft]);
|
||
const typing = (on: boolean) => document.documentElement.classList.toggle('typing', on);
|
||
useEffect(() => () => { document.documentElement.classList.remove('typing'); }, []);
|
||
+ const record = shown.filter(m => m !== herLine && (!reply || !m.live));
|
||
- const record = reply ? shown.filter(m => !m.live) : shown;
|
||
// While a reading sheet is open the table words are out of reach (the sheet holds them all); layout effect so the
|
||
// focus hand-back after closing never lands on an inert button.
|
||
const conversation = useRef<HTMLElement>(null);
|
||
@@ -327,7 +344,7 @@
|
||
<div className="reader-bar">
|
||
<button className="flow" aria-pressed={flow} onClick={() => setFlow(!flow)}><Icon name={flow ? 'pause' : 'play'}/><span>{flow ? '暂停' : '继续'}</span></button>
|
||
<button className="icon" aria-label="上一句" disabled={cursor <= 0} onClick={() => back()}><Icon name="back"/></button>
|
||
+ <span className="count" aria-label={pages.length ? `第 ${cursor + 1} 句,共 ${pages.length} 句${talking ? ',还在说' : ''}` : '还没有对话'}><span>{pages.length ? `${cursor + 1} / ${pages.length}${talking ? '…' : ''}` : '0 / 0'}</span><i ref={meter}/></span>
|
||
- <span className="count" aria-label={pages.length ? `第 ${cursor + 1} 句,共 ${pages.length} 句${busy ? ',还在说' : ''}` : '还没有对话'}><span>{pages.length ? `${cursor + 1} / ${pages.length}${busy ? '…' : ''}` : '0 / 0'}</span><i ref={meter}/></span>
|
||
<button className="icon" aria-label="下一句" disabled={atEnd} onClick={() => forward()}><Icon name="next"/></button>
|
||
<button className="read-all" ref={fullButton} disabled={!page} onClick={() => setFull(true)}>全文</button>
|
||
{busy && <button className="stop" onClick={() => stop()}>停止回复</button>}
|
||
@@ -341,7 +358,7 @@
|
||
</div>
|
||
</section>
|
||
<aside className="record log-panel glass read" hidden={!log} role="dialog" aria-modal="false" aria-labelledby="record-title"><header><div><p className="eyebrow">{real ? '服务器聊天记录' : '完整记录 · 本机演示'}</p><h2 id="record-title">你们说过的话</h2></div><button className="btn-round sm" ref={logClose} aria-label="关闭记录" onClick={() => setLog(false)}><Icon name="x"/></button></header>
|
||
+ <div className="scroll" ref={logRef}><p className="guide">内容完整保留;关闭后回到刚才那一句,自动呈现会接着走。</p>{record.map(m => <article key={m.id} data-life={m.lifeIds.join(' ') || undefined} data-mid={m.id} data-role={m.role} className={`message ${m.status === 'failed' ? 'fail' : ''} ${(focusLife && m.lifeIds.includes(focusLife)) || focusIds.includes(m.id) ? 'hl' : ''}`}><div className="author"><i/>{m.role === 'user' ? '你' : name}{m.proactive && <span className="tagm">她主动说的</span>}</div><p>{m.text}</p>{m.status === 'failed' && <button className="btn-text" onClick={() => void send(m)}>没发出去 · 重试</button>}{m.status === 'cancelled' && <span className="incomplete">已取消</span>}{m.live === 'cancelled' && <span className="incomplete">已停止 · 这是未完成片段,未存入聊天记录</span>}</article>)}{reply && <article className="message" data-role="companion"><div className="author"><i/>{name}</div><p>{reply}</p><span className="incomplete">还在说…</span></article>}{herLine && !herLine.proactive && <article className="message" data-role="companion"><div className="author"><i/>{name}</div><p>{herLine.text}</p><span className="incomplete">{herOpen ? '还在说…' : '语音刚说完 · 正在读回聊天记录'}</span></article>}</div></aside>
|
||
- <div className="scroll" ref={logRef}><p className="guide">内容完整保留;关闭后回到刚才那一句,自动呈现会接着走。</p>{record.map(m => <article key={m.id} data-life={m.lifeIds.join(' ') || undefined} data-mid={m.id} data-role={m.role} className={`message ${m.status === 'failed' ? 'fail' : ''} ${(focusLife && m.lifeIds.includes(focusLife)) || focusIds.includes(m.id) ? 'hl' : ''}`}><div className="author"><i/>{m.role === 'user' ? '你' : name}{m.proactive && <span className="tagm">她主动说的</span>}</div><p>{m.text}</p>{m.status === 'failed' && <button className="btn-text" onClick={() => void send(m)}>没发出去 · 重试</button>}{m.status === 'cancelled' && <span className="incomplete">已取消</span>}{m.live === 'cancelled' && <span className="incomplete">已停止 · 这是未完成片段,未存入聊天记录</span>}</article>)}{reply && <article className="message" data-role="companion"><div className="author"><i/>{name}</div><p>{reply}</p><span className="incomplete">还在说…</span></article>}</div></aside>
|
||
{full && fullMessage && <aside className="record full glass read" role="dialog" aria-modal="false" aria-labelledby="full-title"><header><div><p className="eyebrow">{fullMessage.role === 'user' ? '你' : name} · 全文</p><h2 id="full-title">这一段,慢慢看</h2></div><button className="btn-round sm" ref={fullClose} aria-label="关闭全文" onClick={() => setFull(false)}><Icon name="x"/></button></header>
|
||
<div className="scroll"><p className="guide">整段完整可读。关闭后回到刚才那一句。</p><article className="message" data-role={fullMessage.role}><div className="author"><i/>{fullMessage.role === 'user' ? '你' : name}</div><p>{fullMessage.text}</p>{fullMessage.live === 'generating' && <span className="incomplete">还在说…</span>}{fullMessage.live === 'cancelled' && <span className="incomplete">这段回复已停止,内容未完成,未存入聊天记录。</span>}</article></div></aside>}
|
||
{real && permission === 'mic' && <Modal title={`开始和${name}语音?`} onClose={() => { setPermission(''); setNotice('没有开启语音,文字仍可使用。'); }} confirm="开始语音" onConfirm={() => { setPermission(''); setNotice(''); void voiceCtl.current?.start(); }}>浏览器会请求麦克风权限。你的声音只送到这台服务器上{name}的语音服务,她用同一份记忆、同一段聊天回应你;你说的话和她的回答照常存进聊天记录。直接开口就能打断她。结束语音、离开交谈或关上页面时,麦克风立即关闭。<br/><small className="voice-note">语音通路还没有经过真机验收,效果可能不稳定。</small></Modal>}
|
||
diff --git a/web/src/voice/session.ts b/web/src/voice/session.ts
|
||
--- a/web/src/voice/session.ts
|
||
+++ b/web/src/voice/session.ts
|
||
@@ -7,6 +7,7 @@
|
||
// One rule holds the whole lifecycle together: every call starts a new generation, and anything that answers for an
|
||
// older generation (a late token, a late join, a late microphone, a late room event) is cleaned up, never shown.
|
||
import type { Message } from '../domain/model';
|
||
+import type { ShownMessage } from '../chat/reading';
|
||
import { api, toMessage, within, type MessagePage } from '../adapters/real';
|
||
import { serverId } from '../adapters/inbox';
|
||
|
||
@@ -17,17 +18,38 @@
|
||
heard: string;
|
||
// Her line spoken after a failure (φ): heard, not part of the record.
|
||
spoken: string;
|
||
+ // What she is saying right now, shown on the stage where her text replies appear; never stored by the page.
|
||
+ her: HerLine | null;
|
||
// Why voice is off or what went wrong; empty while it simply runs.
|
||
notice: string;
|
||
// The browser held her audio back until a tap (autoplay rules); `resume` lets it out.
|
||
blocked: boolean;
|
||
}
|
||
+// One "she speaks" (an SDK speech): her reply to your voice turn, or a stored reach-out read aloud. `open` grows by
|
||
+// replacement (each interim is the whole text so far); `final` is the stored reply's exact text; `done` is closed and
|
||
+// stays only until the record (or the next turn) takes its place.
|
||
+export interface HerLine { speech: string; text: string; state: 'open' | 'final' | 'done'; turnId?: string; messageId?: number; proactive?: boolean }
|
||
+export const OFF: VoiceView = { phase: 'off', heard: '', spoken: '', her: null, notice: '', blocked: false };
|
||
+
|
||
+// ── cg.subtitle contract v2 (docs/voice-subtitle-contract-v2.md). Anything that is not exactly v2 is dropped. ──
|
||
+const USER_KINDS = ['interim', 'user_final'] as const;
|
||
+const HER_KINDS = ['turn_start', 'assistant_interim', 'assistant_final', 'turn_end', 'turn_cancel', 'error', 'proactive_start', 'proactive_end', 'proactive_cancel'] as const;
|
||
+export type SubtitleKind = typeof USER_KINDS[number] | typeof HER_KINDS[number];
|
||
+export interface Subtitle { seq: number; kind: SubtitleKind; role: 'user' | 'assistant'; text: string; speech: string; turnId?: string; messageId?: number; reason?: string }
|
||
+const id = (v: unknown) => typeof v === 'string' && v.length > 0 && v.length <= 200 ? v : '';
|
||
-export const OFF: VoiceView = { phase: 'off', heard: '', spoken: '', notice: '', blocked: false };
|
||
-export interface Subtitle { kind: 'interim' | 'user_final' | 'assistant_final' | 'error'; role: 'user' | 'assistant'; text: string }
|
||
export function subtitle(v: unknown): Subtitle | null {
|
||
const s = v as Record<string, unknown> | null;
|
||
+ if (!s || typeof s !== 'object' || Array.isArray(s) || s.v !== 2 || !Number.isSafeInteger(s.seq) || (s.seq as number) <= 0 || typeof s.text !== 'string') return null;
|
||
+ const kind = s.kind as SubtitleKind, user = (USER_KINDS as readonly string[]).includes(kind);
|
||
+ if (!user && !(HER_KINDS as readonly string[]).includes(kind)) return null;
|
||
+ if (s.role !== (user ? 'user' : 'assistant')) return null;
|
||
+ const out: Subtitle = { seq: s.seq as number, kind, role: user ? 'user' : 'assistant', text: s.text, speech: user ? '' : id(s.speech_id) };
|
||
+ if (!user && !out.speech) return null;
|
||
+ if (s.turn_id !== undefined && s.turn_id !== null) { if (!id(s.turn_id)) return null; out.turnId = s.turn_id as string; }
|
||
+ if (kind === 'assistant_final' && !out.turnId) return null;
|
||
+ if (kind.startsWith('proactive_')) { if (!Number.isSafeInteger(s.message_id) || (s.message_id as number) <= 0) return null; out.messageId = s.message_id as number; }
|
||
+ if (typeof s.reason === 'string') out.reason = s.reason;
|
||
+ return out;
|
||
- if (!s || typeof s !== 'object' || typeof s.text !== 'string' || !['interim', 'user_final', 'assistant_final', 'error'].includes(s.kind as string)) return null;
|
||
- return { kind: s.kind as Subtitle['kind'], role: s.role === 'assistant' ? 'assistant' : 'user', text: s.text };
|
||
}
|
||
|
||
// ── What the controller needs from a real-time client. livekit.ts implements it with livekit-client; tests use fakes. ──
|
||
@@ -77,7 +99,10 @@
|
||
secure?: () => boolean;
|
||
}
|
||
|
||
+// seq: the highest subtitle sequence applied in this call (one worker job, one sequence). speech: her speech open now;
|
||
+// closed: speeches that ended, were cancelled or were superseded here; nothing for them is shown again.
|
||
+interface Live { gen: number; abort: AbortController; mic?: MicTrack; room?: VoiceRoom; session?: string; agent: boolean; stated: boolean; hushed: boolean; timer?: ReturnType<typeof setTimeout>; seq: number; speech: string; pro: boolean; message?: number; closed: string[] }
|
||
+const CLOSED_KEPT = 64;
|
||
-interface Live { gen: number; abort: AbortController; mic?: MicTrack; room?: VoiceRoom; session?: string; agent: boolean; stated: boolean; hushed: boolean; timer?: ReturnType<typeof setTimeout> }
|
||
|
||
const AGENT_PHASE: Record<string, VoicePhase> = { initializing: 'joining', listening: 'listening', thinking: 'thinking', speaking: 'speaking' };
|
||
|
||
@@ -127,7 +152,7 @@
|
||
// The generation is taken before anything is awaited, so a start that wakes up after a newer one never takes
|
||
// the call over; the call it replaces is released first, as end() would.
|
||
const prev = this.live;
|
||
+ const gen = ++this.gen, live: Live = { gen, abort: new AbortController(), agent: false, stated: false, hushed: false, seq: 0, speech: '', pro: false, closed: [] };
|
||
- const gen = ++this.gen, live: Live = { gen, abort: new AbortController(), agent: false, stated: false, hushed: false };
|
||
this.live = live;
|
||
this.view = { ...OFF, phase: 'opening' }; this.deps.onView(this.view);
|
||
const stale = () => gen !== this.gen;
|
||
@@ -189,11 +214,11 @@
|
||
},
|
||
subtitle: payload => {
|
||
if (stale()) return;
|
||
+ // Replays, late frames of an older turn and frames that overtook each other all sit at or below the
|
||
+ // highest sequence already applied in this call.
|
||
+ const s = subtitle(payload); if (!s || s.seq <= live.seq) return;
|
||
+ live.seq = s.seq;
|
||
+ this.hear(live, s);
|
||
- const s = subtitle(payload); if (!s) return;
|
||
- if (s.kind === 'interim') { this.show({ heard: s.text, spoken: '', ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); if (live.hushed && !live.stated) { live.hushed = false; live.room?.hush(false); } return; }
|
||
- if (s.kind === 'user_final') { this.show({ heard: s.text, spoken: '', ...(live.stated ? {} : { phase: 'thinking' as const }) }, gen); this.deps.onSpoke(); this.deps.onRecord(); return; }
|
||
- if (s.kind === 'assistant_final') { this.show({ heard: '', spoken: '', ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); this.deps.onRecord(); return; }
|
||
- this.show({ heard: '', spoken: s.text, ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); this.deps.onRecord();
|
||
},
|
||
dropped: () => { if (!stale()) void fail(WORDS.dropped); },
|
||
audioBlocked: blocked => this.show({ blocked }, gen),
|
||
@@ -224,10 +249,73 @@
|
||
if (live) void this.teardown(live, true);
|
||
}
|
||
|
||
+ // One subtitle that passed the sequence gate. Your draft and her line never share a slot: `heard` is yours, `her`
|
||
+ // is hers. Her frames act only on the speech open now; a speech once closed here is never shown again.
|
||
+ private hear(live: Live, s: Subtitle) {
|
||
+ const gen = live.gen, cur = this.view.her, honest = (phase: VoicePhase) => live.stated ? {} : { phase };
|
||
+ if (s.kind === 'interim') {
|
||
+ // Your draft. Her open line is left to the server's cancel (an SDK barge-in decides, not every sound).
|
||
+ this.show({ heard: s.text, spoken: '', ...honest('listening') }, gen);
|
||
+ if (live.hushed && !live.stated) { live.hushed = false; live.room?.hush(false); }
|
||
+ return;
|
||
+ }
|
||
+ if (s.kind === 'user_final') {
|
||
+ // A new turn of yours: nothing of hers from before may stand after it.
|
||
+ if (live.speech) this.close(live, live.speech);
|
||
+ this.show({ heard: s.text, spoken: '', her: null, ...honest('thinking') }, gen); this.deps.onSpoke(); this.deps.onRecord();
|
||
+ return;
|
||
+ }
|
||
+ const pro = s.kind.startsWith('proactive_');
|
||
+ if (s.kind === 'turn_start' || s.kind === 'proactive_start') {
|
||
+ if (live.closed.includes(s.speech) || live.speech === s.speech) return;
|
||
+ if (live.speech) this.close(live, live.speech);
|
||
+ live.speech = s.speech; live.pro = pro; live.message = s.messageId;
|
||
+ // A reach-out arrives whole (its stored text); a reply starts empty and grows.
|
||
+ this.show({ spoken: '', ...(pro ? { heard: '' } : {}), her: pro ? { speech: s.speech, text: s.text, state: 'open', messageId: s.messageId, proactive: true } : { speech: s.speech, text: '', state: 'open' }, ...honest(pro ? 'speaking' : 'thinking') }, gen);
|
||
+ return;
|
||
+ }
|
||
+ if (s.speech !== live.speech || pro !== live.pro || (pro && s.messageId !== live.message)) return;
|
||
+ const mine = cur?.speech === s.speech ? cur : null;
|
||
+ switch (s.kind) {
|
||
+ case 'assistant_interim':
|
||
+ // Replace, never append: each interim is the whole reply so far.
|
||
+ if (mine?.state === 'open' && mine.text !== s.text) this.show({ her: { ...mine, text: s.text } }, gen);
|
||
+ return;
|
||
+ case 'assistant_final':
|
||
+ if (mine?.state !== 'open') return;
|
||
+ // Her stored answer: your draft it answered has done its part (as before v2).
|
||
+ this.show({ heard: '', her: { ...mine, text: s.text, state: 'final', turnId: s.turnId }, ...honest('listening') }, gen); this.deps.onRecord();
|
||
+ return;
|
||
+ case 'error':
|
||
+ // φ: heard, not stored; her unfinished line goes at once. The cancel that follows closes the speech.
|
||
+ this.show({ heard: '', spoken: s.text, ...(mine ? { her: null } : {}), ...honest('listening') }, gen); this.deps.onRecord();
|
||
+ return;
|
||
+ case 'turn_end': case 'proactive_end':
|
||
+ this.close(live, s.speech);
|
||
+ // A reply closed without its final never became her reply; a reach-out read to the end stays until the
|
||
+ // record or the next turn takes its place.
|
||
+ this.show({ ...(mine ? { her: mine.state === 'final' || pro ? { ...mine, state: 'done' as const } : null } : {}), ...honest('listening') }, gen);
|
||
+ if (!pro) this.deps.onRecord();
|
||
+ return;
|
||
+ case 'turn_cancel': case 'proactive_cancel':
|
||
+ this.close(live, s.speech);
|
||
+ this.show({ ...(mine ? { her: null } : {}), ...honest('listening') }, gen);
|
||
+ if (!pro) this.deps.onRecord();
|
||
+ return;
|
||
+ }
|
||
+ }
|
||
+ private close(live: Live, speech: string) {
|
||
+ if (live.speech === speech) { live.speech = ''; live.pro = false; live.message = undefined; }
|
||
+ if (!live.closed.includes(speech)) { live.closed.push(speech); if (live.closed.length > CLOSED_KEPT) live.closed.shift(); }
|
||
+ }
|
||
+
|
||
// You sent text while she was answering by voice: the server cancels her voice turn; keep its last audio silent.
|
||
+ // Her line goes at once, whatever state it was in: your words come next on the stage.
|
||
hush() {
|
||
+ const live = this.live; if (!live) return;
|
||
+ if (live.speech) this.close(live, live.speech);
|
||
+ if (this.view.her) this.show({ her: null }, live.gen);
|
||
+ if (!live.room || (this.view.phase !== 'speaking' && this.view.phase !== 'thinking')) return;
|
||
- const live = this.live;
|
||
- if (!live?.room || (this.view.phase !== 'speaking' && this.view.phase !== 'thinking')) return;
|
||
live.hushed = true; live.room.hush(true);
|
||
this.show({ spoken: '' }, live.gen);
|
||
}
|
||
@@ -285,6 +373,21 @@
|
||
}
|
||
return next;
|
||
}
|
||
+// Her voice line as the stage shows it, in the place her text reply would take; null once it has nothing to add.
|
||
+// `since` is how many record rows there were when this speech was first seen. It gives way to the record: her reply
|
||
+// once its own row (same turn) is there, a reach-out once its stored row is the newest line; a finished line also as
|
||
+// soon as anything else came into the record after it, so it never stands in front of a newer turn.
|
||
+export function voiceLine(her: HerLine | null, messages: Message[], since: number, incoming: Message[] = []): ShownMessage | null {
|
||
+ if (!her || !her.text) return null;
|
||
+ const own = (m: Message) => her.proactive ? m.id === String(her.messageId) : !!her.turnId && m.turnId === her.turnId;
|
||
+ if (her.proactive ? messages.at(-1)?.id === String(her.messageId) : messages.some(m => m.role === 'companion' && own(m))) return null;
|
||
+ if (her.state !== 'open' && messages.slice(since).some(m => !own(m))) return null;
|
||
+ // A reach-out keeps its stored row's identity (id, turn), so the row taking over keeps the same lines on the stage.
|
||
+ const row = her.proactive ? [...messages, ...incoming].find(own) : undefined;
|
||
+ // Her reply still growing holds back its newest line, as a streamed text reply does.
|
||
+ return { id: row?.id ?? `voice-${her.speech}`, role: 'companion', text: her.text, lifeIds: [], turnId: row ? row.turnId : her.turnId, ...(her.proactive ? { proactive: true } : {}), ...(her.state === 'open' && !her.proactive ? { live: 'generating' as const } : {}) };
|
||
+}
|
||
+
|
||
// Newest rows down to `since` (exclusive), at most `pages` pages of 50. Voice turns are few per read; a longer gap is
|
||
// left to the next re-read rather than read without bound.
|
||
export async function recentRows(since: number, signal: AbortSignal, pages = 4): Promise<Message[]> {
|
||
diff --git a/web/tests/fake-livekit.js b/web/tests/fake-livekit.js
|
||
--- a/web/tests/fake-livekit.js
|
||
+++ b/web/tests/fake-livekit.js
|
||
@@ -26,7 +26,8 @@
|
||
return true;
|
||
};
|
||
lk.state = s => { const room = lk.rooms.at(-1), p = [...room.remoteParticipants.values()][0]; if (!p) return false; p.attributes['lk.agent.state'] = s; room.emit(RoomEvent.ParticipantAttributesChanged, { 'lk.agent.state': s }, p); return true; };
|
||
+ // Frames without an envelope get the worker's v2 one (next seq of this room); explicit frames are sent as given.
|
||
+ lk.say = obj => { const room = lk.rooms.at(-1), frame = 'v' in obj ? obj : { v: 2, seq: (room.seq || 0) + 1, ...obj }; if (Number.isSafeInteger(frame.seq)) room.seq = Math.max(room.seq || 0, frame.seq); room.emit(RoomEvent.DataReceived, new TextEncoder().encode(JSON.stringify(frame)), undefined, 0, 'cg.subtitle'); return true; };
|
||
- lk.say = obj => { const room = lk.rooms.at(-1); room.emit(RoomEvent.DataReceived, new TextEncoder().encode(JSON.stringify(obj)), undefined, 0, 'cg.subtitle'); return true; };
|
||
lk.drop = () => { const room = live(); room.state = 'gone'; room.emit(RoomEvent.Disconnected); return true; };
|
||
lk.block = () => { const room = live(); room.canPlaybackAudio = false; room.emit(RoomEvent.AudioPlaybackStatusChanged); return true; };
|
||
async function createLocalAudioTrack(constraints) {
|
||
diff --git a/web/tests/hel878_mock.mjs b/web/tests/hel878_mock.mjs
|
||
--- a/web/tests/hel878_mock.mjs
|
||
+++ b/web/tests/hel878_mock.mjs
|
||
@@ -4,7 +4,7 @@
|
||
// answered with tests/fake-livekit.js (a scripted stand-in, no WebRTC); the real vendored build stays reachable at
|
||
// /__real-sdk.js for the API-surface check only. Synthetic data only: no model, no LiveKit, no TTS, no network.
|
||
// Usage: node tests/hel878_mock.mjs <port>. Control GETs: /__voice?configured=&tts=, /__refuse/<code|0>,
|
||
+// /__voice-turn?stage=user|reply|cancel&text=, /__proactive?text=, /__cut, /__sessions, /__log, /__rows.
|
||
-// /__voice-turn?stage=user|reply|cancel&text=, /__cut, /__sessions, /__log, /__rows.
|
||
import { createServer } from 'node:http';
|
||
import { readFile } from 'node:fs/promises';
|
||
import { extname, join, normalize } from 'node:path';
|
||
@@ -41,6 +41,8 @@
|
||
if (stage === 'cancel') rows.forEach(r => { if (r.turn_id === `voice-${voiceTurn}`) r.status = 'cancelled'; });
|
||
return send(res, 200, { turn: voiceTurn });
|
||
}
|
||
+ // A reach-out the life runtime stored (HEL-898): the worker may read it aloud; the page sees it through the inbox.
|
||
+ if (p === '/__proactive') { const id = add({ role: 'companion', text: url.searchParams.get('text') || '', turn_id: `pro-${rows.length + 1}`, proactive: true }); return send(res, 200, { id }); }
|
||
// A voice turn took the server's turn gate: the text stream in flight ends without a final; its row is cancelled.
|
||
if (p === '/__cut') { const s = textStream; textStream = null; if (s) { rows.forEach(r => { if (r.turn_id === s.turn_id && r.role === 'user') r.status = 'cancelled'; }); s.cut = true; s.res.end(); } return send(res, 200, { cut: !!s }); }
|
||
if (p === '/__real-sdk.js') return send(res, 200, await readFile(join(dist, 'vendor/livekit-client-2.22.4.umd.min.js')), 'text/javascript');
|
||
diff --git a/web/tests/hel878_shots.mjs b/web/tests/hel878_shots.mjs
|
||
--- a/web/tests/hel878_shots.mjs
|
||
+++ b/web/tests/hel878_shots.mjs
|
||
@@ -114,7 +114,7 @@
|
||
await frames('m2-voice-speaking', 900);
|
||
await shot('08-moon-desktop-voice-speaking');
|
||
await ctl('/__voice-turn?stage=reply&text=' + encodeURIComponent('是橘色的那只吗?它最近总在桥下晒太阳。'));
|
||
+await lk(`__lk.say({ kind: 'turn_start', role: 'assistant', text: '', speech_id: 'sp-878-1' })`); await lk(`__lk.say({ kind: 'assistant_final', role: 'assistant', text: '是橘色的那只吗?它最近总在桥下晒太阳。', speech_id: 'sp-878-1', turn_id: 'voice-1' })`); await lk(`__lk.say({ kind: 'turn_end', role: 'assistant', text: '', speech_id: 'sp-878-1', turn_id: 'voice-1' })`); await lk(`__lk.state('listening')`);
|
||
-await lk(`__lk.say({ kind: 'assistant_final', role: 'assistant', text: '是橘色的那只吗?它最近总在桥下晒太阳。' })`); await lk(`__lk.state('listening')`);
|
||
await until(async () => (await said()).includes('晒太阳'), 6000);
|
||
check('her spoken answer reaches the stage once it is in the record', (await said()).includes('晒太阳'), await said());
|
||
await shot('09-moon-desktop-voice-reply-on-stage');
|
||
@@ -125,7 +125,7 @@
|
||
await click('.log-panel .btn-round'); await sleep(400);
|
||
|
||
// ── 6. φ: her failure line is heard and shown, never recorded. ──
|
||
+await lk(`__lk.say({ kind: 'turn_start', role: 'assistant', text: '', speech_id: 'sp-878-phi' })`); await lk(`__lk.say({ kind: 'error', role: 'assistant', text: '我刚刚没接上话,你再说一遍好吗?', speech_id: 'sp-878-phi' })`); await lk(`__lk.say({ kind: 'turn_cancel', role: 'assistant', text: '', speech_id: 'sp-878-phi', reason: 'error_phi' })`); await sleep(400);
|
||
-await lk(`__lk.say({ kind: 'error', role: 'assistant', text: '我刚刚没接上话,你再说一遍好吗?' })`); await sleep(400);
|
||
check('φ line shown as heard, marked not recorded; not in the record', (await row()).includes('没接上话') && (await row()).includes('未记入聊天') && !(await ctl('/__rows')).some(r => r.text.includes('没接上话')));
|
||
await shot('11-moon-desktop-voice-phi');
|
||
|
||
diff --git a/web/tests/hel898_shots.mjs b/web/tests/hel898_shots.mjs
|
||
new file mode 100644
|
||
--- /dev/null
|
||
+++ b/web/tests/hel898_shots.mjs
|
||
@@ -0,0 +1,172 @@
|
||
+// HEL-898 browser check: her words while she speaks (cg.subtitle contract v2) on the approved conversation. Drives
|
||
+// headless Edge over CDP against tests/hel878_mock.mjs, with tests/fake-livekit.js in place of the LiveKit client and
|
||
+// Chromium's fake audio device in place of a microphone: no model, no TTS, no real call. Frames are the worker's v2
|
||
+// envelopes (docs/voice-subtitle-contract-v2.md). Usage: node tests/hel898_shots.mjs <edge.exe> <outDir> [port].
|
||
+import { spawn } from 'node:child_process';
|
||
+import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||
+import { join } from 'node:path';
|
||
+import { tmpdir } from 'node:os';
|
||
+
|
||
+const [edge, out, portArg] = process.argv.slice(2);
|
||
+const port = Number(portArg || 8878), base = `http://127.0.0.1:${port}`, cdpPort = 9498;
|
||
+mkdirSync(out, { recursive: true });
|
||
+const profile = join(tmpdir(), `hel898-edge-${Date.now()}`);
|
||
+const browser = spawn(edge, ['--headless=new', `--remote-debugging-port=${cdpPort}`, `--user-data-dir=${profile}`, '--no-first-run', '--hide-scrollbars', '--force-color-profile=srgb', '--use-fake-device-for-media-stream', '--use-fake-ui-for-media-stream', '--autoplay-policy=no-user-gesture-required', 'about:blank'], { stdio: 'ignore' });
|
||
+const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||
+let targets;
|
||
+for (let i = 0; i < 60 && !targets; i++) { try { targets = await (await fetch(`http://127.0.0.1:${cdpPort}/json`)).json(); } catch { await sleep(250); } }
|
||
+const ws = new WebSocket(targets.find(t => t.type === 'page').webSocketDebuggerUrl);
|
||
+await new Promise(r => ws.addEventListener('open', r, { once: true }));
|
||
+let id = 0; const waiting = new Map();
|
||
+ws.addEventListener('message', ({ data }) => { const m = JSON.parse(data); if (m.id && waiting.has(m.id)) { const { ok, ko } = waiting.get(m.id); waiting.delete(m.id); m.error ? ko(new Error(m.error.message)) : ok(m.result); } });
|
||
+const cdp = (method, params = {}) => new Promise((ok, ko) => { const n = ++id; waiting.set(n, { ok, ko }); ws.send(JSON.stringify({ id: n, method, params })); });
|
||
+const js = async expr => { const r = await cdp('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }); if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'js error'); return r.result.value; };
|
||
+const results = [];
|
||
+const check = (name, ok, detail = '') => { results.push(ok); console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` — ${detail}` : ''}`); };
|
||
+const ctl = async path => (await fetch(`${base}${path}`)).json();
|
||
+const shot = async name => { const { data } = await cdp('Page.captureScreenshot', { format: 'png' }); writeFileSync(join(out, `${name}.png`), Buffer.from(data, 'base64')); };
|
||
+const frames = async (name, ms) => { const list = []; const end = Date.now() + ms; while (Date.now() < end) { const { data } = await cdp('Page.captureScreenshot', { format: 'jpeg', quality: 70 }); list.push(data); } list.forEach((d, i) => writeFileSync(join(out, `${name}-${String(i).padStart(2, '0')}.jpg`), Buffer.from(d, 'base64'))); return list.length; };
|
||
+const click = async (selector, text = '') => js(`(() => { const el = [...document.querySelectorAll(${JSON.stringify(selector)})].find(e => e.offsetParent !== null && e.textContent.includes(${JSON.stringify(text)})); if (!el) return false; el.click(); return true; })()`);
|
||
+const text = sel => js(`[...document.querySelectorAll(${JSON.stringify(sel)})].map(e => e.innerText).join('\\n')`);
|
||
+const exists = sel => js(`!!document.querySelector(${JSON.stringify(sel)})`);
|
||
+// The stage's own text (every line now on it) and the line read out to assistive tech.
|
||
+const stage = () => js(`document.querySelector('.conversation .stage')?.innerText || ''`);
|
||
+const said = () => js(`document.querySelector('.conversation .sr')?.textContent || ''`);
|
||
+const row = () => js(`document.querySelector('.voice-row.real')?.innerText || ''`);
|
||
+const phase = () => js(`document.querySelector('.voice-row.real')?.dataset.phase || 'off'`);
|
||
+const lk = expr => js(`window.__lk ? (${expr}) : null`);
|
||
+const say = frame => lk(`__lk.say(${JSON.stringify(frame)})`);
|
||
+const her = (kind, speech, extra = {}) => say({ kind, role: 'assistant', text: '', speech_id: speech, ...extra });
|
||
+const type = t => js(`(() => { const el = document.querySelector('textarea'); const set = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value').set; set.call(el, ${JSON.stringify(t)}); el.dispatchEvent(new Event('input', { bubbles: true })); return true; })()`);
|
||
+const until = async (fn, ms = 4000) => { const end = Date.now() + ms; while (Date.now() < end) { if (await fn()) return true; await sleep(100); } return false; };
|
||
+const views = { desktop: { width: 1440, height: 900, deviceScaleFactor: 1, mobile: false }, phone: { width: 390, height: 844, deviceScaleFactor: 2, mobile: true } };
|
||
+async function view(v) { await cdp('Emulation.setDeviceMetricsOverride', views[v]); await cdp('Emulation.setTouchEmulationEnabled', { enabled: views[v].mobile, maxTouchPoints: 5 }); }
|
||
+async function boot(theme) {
|
||
+ await cdp('Page.navigate', { url: `${base}/` }); await sleep(500);
|
||
+ await js(`localStorage.setItem('cg.theme', ${JSON.stringify(theme)}); sessionStorage.clear(); true`);
|
||
+ await cdp('Page.navigate', { url: `${base}/` }); await sleep(1600);
|
||
+}
|
||
+async function startVoice() {
|
||
+ await click('.voice-button'); await until(() => exists('.modal.open'));
|
||
+ await click('.modal.open .btn-primary', '开始语音');
|
||
+ await until(async () => (await phase()) === 'joining');
|
||
+ await until(async () => (await lk('__lk.rooms.at(-1)?.state')) === 'connected');
|
||
+ await lk('__lk.join()'); await sleep(200);
|
||
+}
|
||
+const calls = async () => ctl('/__log');
|
||
+// One voice turn of yours as the worker leaves it: the stored row, then user_final; waits for it on the stage.
|
||
+const youSay = async words => {
|
||
+ const { turn } = await ctl('/__voice-turn?stage=user&text=' + encodeURIComponent(words));
|
||
+ await say({ kind: 'user_final', role: 'user', text: words }); await lk(`__lk.state('thinking')`);
|
||
+ await until(async () => (await stage()).includes(words.slice(0, 4)), 6000);
|
||
+ return turn;
|
||
+};
|
||
+const rows = async () => ctl('/__rows');
|
||
+await cdp('Page.enable'); await cdp('Runtime.enable'); await cdp('Network.enable');
|
||
+
|
||
+const REPLY = '是橘色的那只吗?它最近总在桥下晒太阳,尾巴一甩一甩的,好像整座桥都是它的。你下次路过可以叫它一声,它会回头看你。';
|
||
+const STEPS = ['是橘色的那只吗?它最近总在桥下晒太阳,', '是橘色的那只吗?它最近总在桥下晒太阳,尾巴一甩一甩的,好像整座桥都是它的。', `${REPLY}`];
|
||
+
|
||
+for (const [theme, v] of [['moon', 'desktop'], ['fog', 'desktop'], ['moon', 'phone'], ['fog', 'phone']]) {
|
||
+ const tag = `${theme}-${v}`;
|
||
+ await view(v); await boot(theme); await click('.nav-item', '交谈'); await sleep(1400);
|
||
+ await startVoice(); await calls();
|
||
+
|
||
+ // ── 1. A voice turn: your draft and final, then her words growing on the stage before anything is stored. ──
|
||
+ await say({ kind: 'interim', role: 'user', text: '今天下班的路上' }); await sleep(200);
|
||
+ const prior = (await rows()).filter(r => r.text === REPLY).length;
|
||
+ const turn = await youSay('今天下班的路上看到一只猫');
|
||
+ await her('turn_start', `a-${tag}`); await lk(`__lk.state('speaking')`);
|
||
+ const shots = [];
|
||
+ for (const [i, step] of STEPS.entries()) {
|
||
+ await her('assistant_interim', `a-${tag}`, { text: step });
|
||
+ shots.push(frames(`${tag}-m1-growing-${i}`, 450));
|
||
+ await shots.at(-1);
|
||
+ }
|
||
+ await until(async () => (await stage()).includes('晒太阳'), 8000);
|
||
+ check(`${tag}: her reply shows on the stage while she speaks, before it is stored`, (await stage()).includes('晒太阳') && (await rows()).filter(r => r.text === REPLY).length === prior, await stage());
|
||
+ check(`${tag}: your draft stays in the voice line, her words never there`, !(await row()).includes('晒太阳'), await row());
|
||
+ await shot(`${tag}-01-her-words-growing`);
|
||
+
|
||
+ // ── 2. Pause and look back while she is still speaking: reading holds, nothing jumps. ──
|
||
+ await click('.reader-bar .flow'); await sleep(300);
|
||
+ check(`${tag}: pause while she speaks`, await exists('.conversation.reading-paused'));
|
||
+ await click('.reader-bar .icon[aria-label="上一句"]'); await sleep(400);
|
||
+ await shot(`${tag}-02-paused-looking-back`);
|
||
+ await ctl('/__voice-turn?stage=reply&text=' + encodeURIComponent(REPLY));
|
||
+ await her('assistant_final', `a-${tag}`, { text: REPLY, turn_id: `voice-${turn}` }); await her('turn_end', `a-${tag}`, { turn_id: `voice-${turn}` }); await lk(`__lk.state('listening')`);
|
||
+ await sleep(900);
|
||
+ await click('.reader-bar .latest'); await sleep(2500);
|
||
+ await click('.log-btn'); await sleep(700);
|
||
+ const rec = await text('.log-panel .scroll');
|
||
+ check(`${tag}: once stored, the record holds her reply exactly once (re-read, not written by the page)`, rec.split(REPLY).length - 1 === prior + 1, rec.slice(-160).replace(/\n/g, ' '));
|
||
+ await shot(`${tag}-03-record-after-final`);
|
||
+ await click('.log-panel .btn-round'); await sleep(400);
|
||
+ await click('.read-all'); await sleep(500); await shot(`${tag}-04-full-text`); await click('.record.full .btn-round'); await sleep(300);
|
||
+
|
||
+ // ── 3. Interrupted: no final; her line goes at once; late frames of the cut turn change nothing. ──
|
||
+ // Paused reading, then you speak: like sending, your new turn and her answer come in by themselves.
|
||
+ await click('.reader-bar .flow'); await sleep(200);
|
||
+ check(`${tag}: paused before speaking`, await exists('.conversation.reading-paused'));
|
||
+ await youSay('你今天去哪儿了');
|
||
+ check(`${tag}: speaking resumes reading for the new turn`, !(await exists('.conversation.reading-paused')));
|
||
+ await her('turn_start', `b-${tag}`); await lk(`__lk.state('speaking')`);
|
||
+ await her('assistant_interim', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,' });
|
||
+ check(`${tag}: second reply growing`, await until(async () => (await stage()).includes('白鹭'), 8000), await stage());
|
||
+ await shot(`${tag}-05-before-cut`);
|
||
+ await say({ kind: 'interim', role: 'user', text: '等一下' });
|
||
+ await her('turn_cancel', `b-${tag}`, { reason: 'interrupted' }); await lk(`__lk.state('listening')`); await ctl('/__voice-turn?stage=cancel');
|
||
+ await sleep(150);
|
||
+ check(`${tag}: interrupted: her unfinished line is gone at once`, !(await stage()).includes('白鹭'), await stage());
|
||
+ await her('assistant_interim', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,真好看。' });
|
||
+ await her('assistant_final', `b-${tag}`, { text: '对了,我今天下午去了河边,看到一群白鹭从水面上飞过去了。它们飞得很低,真好看。', turn_id: 'late' });
|
||
+ await sleep(400);
|
||
+ check(`${tag}: late frames of the cut turn stay out`, !(await stage()).includes('白鹭'));
|
||
+ await shot(`${tag}-06-after-cut`);
|
||
+
|
||
+ // ── 4. Text takes over while she speaks: her line goes, your typed words come next. ──
|
||
+ await youSay('我们聊点别的吧');
|
||
+ await her('turn_start', `c-${tag}`); await lk(`__lk.state('speaking')`);
|
||
+ await her('assistant_interim', `c-${tag}`, { text: '那我们换个话题吧,你今天晚饭吃了什么,好不好吃?我今天' });
|
||
+ check(`${tag}: her line before you type`, await until(async () => (await stage()).includes('换个话题'), 8000), await stage());
|
||
+ await type('先等等,我打字说'); await js(`document.querySelector('.send').click(), true`); await sleep(400);
|
||
+ check(`${tag}: text over voice: her line is gone, your words are on the stage`, !(await stage()).includes('换个话题') && (await stage()).includes('先等等'), await stage());
|
||
+ await until(async () => !(await exists('.reader-bar .stop')), 9000); await lk(`__lk.state('listening')`); await sleep(400);
|
||
+
|
||
+ // ── 5. A stored reach-out read aloud: its own text in her place, no request, no read mark, interruptible. ──
|
||
+ const { id: pro } = await ctl('/__proactive?text=' + encodeURIComponent('刚才路过花店,想起你说过喜欢白色的洋桔梗。'));
|
||
+ await calls();
|
||
+ await say({ kind: 'proactive_start', role: 'assistant', text: '刚才路过花店,想起你说过喜欢白色的洋桔梗。', speech_id: `p-${tag}`, message_id: pro }); await lk(`__lk.state('speaking')`);
|
||
+ await sleep(1500);
|
||
+ check(`${tag}: reach-out shown in her place, word for word`, (await stage()).includes('洋桔梗'), await stage());
|
||
+ await shot(`${tag}-07-reach-out`);
|
||
+ await say({ kind: 'interim', role: 'user', text: '是吗' });
|
||
+ await say({ kind: 'proactive_cancel', role: 'assistant', text: '', speech_id: `p-${tag}`, message_id: pro, reason: 'interrupted' }); await lk(`__lk.state('listening')`);
|
||
+ await sleep(200);
|
||
+ const after = await calls();
|
||
+ check(`${tag}: reach-out cut: cleared; the page wrote nothing (no receipt, no read mark, no message)`, !after.some(l => !l.startsWith('GET ')) , after.join(' | '));
|
||
+
|
||
+ // ── 6. Hanging up mid-sentence clears her line with the call. ──
|
||
+ await youSay('我先去睡了');
|
||
+ await her('turn_start', `d-${tag}`); await her('assistant_interim', `d-${tag}`, { text: '晚安,明天见。今天说了好多好多话,你要早点睡哦。我会在窗边等你,' });
|
||
+ check(`${tag}: her line before hanging up`, await until(async () => (await stage()).includes('早点睡'), 8000), await stage());
|
||
+ await click('.voice-row.real .voice-end'); await sleep(400);
|
||
+ check(`${tag}: hung up: her line gone, voice off`, !(await stage()).includes('早点睡') && (await phase()) === 'off', await stage());
|
||
+ await shot(`${tag}-08-ended`);
|
||
+}
|
||
+
|
||
+// ── 7. Reduced motion: her words still arrive, without animated entrances. ──
|
||
+await view('desktop'); await cdp('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-reduced-motion', value: 'reduce' }] });
|
||
+await boot('moon'); await click('.nav-item', '交谈'); await sleep(1400); await startVoice();
|
||
+await youSay('慢一点说给我听');
|
||
+await her('turn_start', 'rm'); await lk(`__lk.state('speaking')`);
|
||
+for (const step of STEPS) { await her('assistant_interim', 'rm', { text: step }); await sleep(300); }
|
||
+check('reduced motion: her words arrive', await until(async () => (await stage()).includes('晒太阳'), 8000), await stage());
|
||
+await frames('reduced-motion-growing', 600);
|
||
+await click('.voice-row.real .voice-end'); await sleep(300);
|
||
+
|
||
+ws.close(); browser.kill();
|
||
+try { rmSync(profile, { recursive: true, force: true }); } catch { /* Edge may still hold the profile. */ }
|
||
+const failed = results.filter(r => !r).length;
|
||
+console.log(`${results.length - failed}/${results.length} passed`);
|
||
+process.exit(failed ? 1 : 0);
|
||
diff --git a/web/tests/voice.test.ts b/web/tests/voice.test.ts
|
||
--- a/web/tests/voice.test.ts
|
||
+++ b/web/tests/voice.test.ts
|
||
@@ -1,8 +1,14 @@
|
||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||
+import { mergeTurns, recentRows, VoiceController, voiceLine, voiceStatus, WORDS, type HerLine, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session';
|
||
+import { groupTurns, pagesOf, type ShownMessage } from '../src/chat/reading';
|
||
-import { mergeTurns, recentRows, VoiceController, voiceStatus, WORDS, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session';
|
||
import type { ServerMessage } from '../src/adapters/real';
|
||
import type { Message } from '../src/domain/model';
|
||
+import { readFileSync } from 'node:fs';
|
||
|
||
+// The backend's recorded cg.subtitle frames (contract v2, docs/rework/hel879/subtitle-samples, generated offline).
|
||
+type Frame = Record<string, unknown> & { seq: number };
|
||
+const sample = (name: string): Frame[] => readFileSync(new URL(`../../docs/rework/hel879/subtitle-samples/${name}.jsonl`, import.meta.url), 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l));
|
||
+
|
||
// Fake transport and media objects: no LiveKit, no microphone, no network. The fake server follows the HEL-877 voice
|
||
// router (src/cybergril/voice/router.py): GET /api/voice/status, POST /api/voice/session -> 201 grant, DELETE
|
||
// /api/voice/session/{id} -> 200 or 404.
|
||
@@ -176,7 +182,7 @@
|
||
const r = await connected(), old = r.transport.rooms[0];
|
||
await r.c.end(); r.record.mockClear();
|
||
const seen = r.views.length;
|
||
+ old.on.agentState('speaking'); old.on.subtitle({ v: 2, seq: 1, kind: 'user_final', role: 'user', text: '旧的话' }); old.on.subtitle({ v: 2, seq: 2, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'old' }); old.on.agentJoined(); old.on.dropped(); old.on.audioBlocked(true);
|
||
- old.on.agentState('speaking'); old.on.subtitle({ kind: 'user_final', role: 'user', text: '旧的话' }); old.on.agentJoined(); old.on.dropped(); old.on.audioBlocked(true);
|
||
r.transport.mics[0].ended?.();
|
||
await flush();
|
||
expect(r.views.length).toBe(seen);
|
||
@@ -243,10 +249,12 @@
|
||
});
|
||
it('never confirms a receipt or posts chat itself: the worker owns both', async () => {
|
||
const r = await connected(), room = r.transport.rooms[0];
|
||
+ for (const line of sample('normal-turn')) room.on.subtitle(line);
|
||
+ for (const line of sample('proactive')) room.on.subtitle({ ...line, seq: line.seq + 100 });
|
||
- room.on.subtitle({ kind: 'user_final', role: 'user', text: '今天好冷' });
|
||
- room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '那多穿一件。' });
|
||
await r.c.end();
|
||
expect(r.server.calls.some(c => c.url.startsWith('/api/chat'))).toBe(false);
|
||
+ // No receipt, no read mark, no message write: the page only talks to the voice routes.
|
||
+ expect(r.server.calls.every(c => c.url.startsWith('/api/voice/'))).toBe(true);
|
||
});
|
||
});
|
||
|
||
@@ -338,24 +346,32 @@
|
||
});
|
||
});
|
||
|
||
+// A v2 frame as the worker's SubtitleBus emits it (src/cybergril/voice/seam.py).
|
||
+const frame = (seq: number, kind: string, extra: Record<string, unknown> = {}) => ({ v: 2, seq, kind, role: kind === 'interim' || kind === 'user_final' ? 'user' : 'assistant', text: '', ...extra });
|
||
describe('subtitles and the two channels', () => {
|
||
it('interim is a draft; user_final cancels a text reply and asks for the record; φ is heard but not recorded', async () => {
|
||
const r = await connected(), room = r.transport.rooms[0];
|
||
+ room.on.subtitle(frame(1, 'interim', { text: '今天' }));
|
||
- room.on.subtitle({ kind: 'interim', role: 'user', text: '今天' });
|
||
expect(r.view().heard).toBe('今天'); expect(r.record).not.toHaveBeenCalled();
|
||
+ room.on.subtitle(frame(2, 'user_final', { text: '今天好冷' }));
|
||
- room.on.subtitle({ kind: 'user_final', role: 'user', text: '今天好冷' });
|
||
expect(r.spoke).toHaveBeenCalledTimes(1); expect(r.record).toHaveBeenCalledTimes(1);
|
||
+ room.on.subtitle(frame(3, 'turn_start', { speech_id: 'a' }));
|
||
+ room.on.subtitle(frame(4, 'assistant_interim', { speech_id: 'a', text: '我刚' }));
|
||
+ room.on.subtitle(frame(5, 'error', { speech_id: 'a', text: '我刚才走神了,你再说一遍?' }));
|
||
+ expect(r.view()).toMatchObject({ spoken: '我刚才走神了,你再说一遍?', her: null });
|
||
+ room.on.subtitle(frame(6, 'turn_cancel', { speech_id: 'a', reason: 'error_phi' }));
|
||
+ expect(r.view()).toMatchObject({ spoken: '我刚才走神了,你再说一遍?', her: null });
|
||
+ room.on.subtitle(frame(7, 'assistant_interim', { speech_id: 'a', text: '我刚才走神了' })); // after its cancel
|
||
+ expect(r.view().her).toBeNull();
|
||
- room.on.subtitle({ kind: 'error', role: 'assistant', text: '我刚才走神了,你再说一遍?' });
|
||
- expect(r.view().spoken).toBe('我刚才走神了,你再说一遍?');
|
||
- room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '那多穿一件。' });
|
||
- expect(r.view()).toMatchObject({ heard: '', spoken: '' });
|
||
room.on.subtitle({ kind: 'bogus', text: 1 }); room.on.subtitle(null);
|
||
expect(r.record).toHaveBeenCalledTimes(3);
|
||
+ expect(r.server.calls.some(c => c.url.startsWith('/api/chat') || c.url.startsWith('/api/messages'))).toBe(false);
|
||
});
|
||
it('without the agent state attribute, subtitles still move the state honestly', async () => {
|
||
const r = await connected(), room = r.transport.rooms[0];
|
||
+ room.on.subtitle(frame(1, 'user_final', { text: '在吗' })); expect(r.view().phase).toBe('thinking');
|
||
+ room.on.subtitle(frame(2, 'turn_start', { speech_id: 'a' })); expect(r.view().phase).toBe('thinking');
|
||
+ room.on.subtitle(frame(3, 'assistant_final', { speech_id: 'a', turn_id: 't', text: '在。' })); expect(r.view().phase).toBe('listening');
|
||
- room.on.subtitle({ kind: 'user_final', role: 'user', text: '在吗' }); expect(r.view().phase).toBe('thinking');
|
||
- room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '在。' }); expect(r.view().phase).toBe('listening');
|
||
});
|
||
it('typing over her spoken answer silences what is left of it until she is listening again', async () => {
|
||
const r = await connected(), room = r.transport.rooms[0];
|
||
@@ -367,6 +383,187 @@
|
||
});
|
||
});
|
||
|
||
+// HEL-898: her words while she speaks (contract v2). The page shows them; it never stores, confirms or marks them.
|
||
+describe('her voice line (cg.subtitle v2)', () => {
|
||
+ const play = (room: Room, frames: Frame[], after?: (f: Frame) => void) => { for (const f of frames) { room.on.subtitle(f); after?.(f); } };
|
||
+ it('rejects anything that is not exactly a v2 frame', async () => {
|
||
+ const r = await connected(), room = r.transport.rooms[0], seen = r.views.length;
|
||
+ const bad = [
|
||
+ { kind: 'user_final', role: 'user', text: 'v1 没有信封' },
|
||
+ { ...frame(1, 'user_final', { text: '版本不对' }), v: 3 },
|
||
+ { ...frame(1, 'user_final', { text: '序号不对' }), seq: 0 },
|
||
+ { ...frame(1, 'user_final', { text: '序号不对' }), seq: 1.5 },
|
||
+ { ...frame(1, 'user_final', { text: '序号不对' }), seq: '1' },
|
||
+ frame(1, 'turn_start'), // no speech_id
|
||
+ { ...frame(1, 'turn_start', { speech_id: 'a' }), role: 'user' }, // her kind, your role
|
||
+ { ...frame(1, 'interim', { text: '草稿' }), role: 'assistant' }, // your kind, her role
|
||
+ frame(1, 'assistant_final', { speech_id: 'a', text: '无归属' }), // final without turn_id
|
||
+ frame(1, 'proactive_start', { speech_id: 'p', text: '无行号' }), // reach-out without message_id
|
||
+ frame(1, 'proactive_start', { speech_id: 'p', text: '行号不对', message_id: '17' }),
|
||
+ frame(1, 'assistant_text', { speech_id: 'a', text: '未知种类' }),
|
||
+ [frame(1, 'user_final', { text: '数组' })],
|
||
+ ];
|
||
+ for (const b of bad) room.on.subtitle(b);
|
||
+ expect(r.views.length).toBe(seen); expect(r.record).not.toHaveBeenCalled(); expect(r.spoke).not.toHaveBeenCalled();
|
||
+ // A rejected frame does not move the sequence: the first good one still lands.
|
||
+ room.on.subtitle(frame(1, 'interim', { text: '好的' })); expect(r.view().heard).toBe('好的');
|
||
+ });
|
||
+ it('normal turn: your draft and final, her growing reply replaced in place, then her stored final', async () => {
|
||
+ const r = await connected(), room = r.transport.rooms[0], frames = sample('normal-turn');
|
||
+ const steps: (VoiceView['her'] | string)[] = [];
|
||
+ play(room, frames, () => steps.push(r.view().her ? { ...r.view().her! } : r.view().heard));
|
||
+ expect(steps).toEqual([
|
||
+ '你在听的', '你在听吗?',
|
||
+ { speech: 'speech_normal', text: '', state: 'open' },
|
||
+ { speech: 'speech_normal', text: '在听呢,', state: 'open' },
|
||
+ { speech: 'speech_normal', text: '在听呢,在听呢,你刚才说到一半的那件事我还在想。', state: 'open' },
|
||
+ { speech: 'speech_normal', text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'final', turnId: '0e2f-demo-turn' },
|
||
+ { speech: 'speech_normal', text: '在听呢,你刚才说到一半的那件事我还在想。', state: 'done', turnId: '0e2f-demo-turn' },
|
||
+ ]);
|
||
+ expect(r.view().heard).toBe(''); // her stored answer retires the draft it answered
|
||
+ // Her words never land in your draft slot, and yours never in hers.
|
||
+ expect(r.views.every(v => !v.her || !v.heard.includes('在听呢'))).toBe(true);
|
||
+ expect(r.spoke).toHaveBeenCalledTimes(1);
|
||
+ expect(r.record).toHaveBeenCalledTimes(3); // your final, her final, her turn's end: re-read, never written
|
||
+ // The whole recording replayed (same seq) changes nothing.
|
||
+ const seen = r.views.length; play(room, frames);
|
||
+ expect(r.views.length).toBe(seen);
|
||
+ // Your next turn: her finished line does not stay in front of it.
|
||
+ room.on.subtitle(frame(8, 'interim', { text: '那' })); expect(r.view().her?.state).toBe('done');
|
||
+ room.on.subtitle(frame(9, 'user_final', { text: '那你想到了吗' })); expect(r.view().her).toBeNull();
|
||
+ });
|
||
+ it('interrupted: no final, her line goes at once, its late frames never reach the new turn', async () => {
|
||
+ const r = await connected(), room = r.transport.rooms[0], frames = sample('interrupt');
|
||
+ play(room, frames.slice(0, 2));
|
||
+ expect(r.view().her).toMatchObject({ speech: 'speech_cut', text: '我正想跟你说,今天下午', state: 'open' });
|
||
+ play(room, frames.slice(2, 3));
|
||
+ expect(r.view().her).toBeNull();
|
||
+ // Late increments of the cut turn, even with a higher seq, and a replay of its start.
|
||
+ room.on.subtitle(frame(4, 'assistant_interim', { speech_id: 'speech_cut', text: '我正想跟你说,今天下午去了' }));
|
||
+ room.on.subtitle(frame(5, 'assistant_final', { speech_id: 'speech_cut', turn_id: 'old', text: '我正想跟你说,今天下午去了河边。' }));
|
||
+ room.on.subtitle(frame(6, 'turn_start', { speech_id: 'speech_cut' }));
|
||
+ expect(r.view().her).toBeNull();
|
||
+ play(room, frames.slice(3).map(f => ({ ...f, seq: f.seq + 10 })));
|
||
+ expect(r.view()).toMatchObject({ heard: '', her: { speech: 'speech_next', text: '好,你先说。', state: 'done', turnId: '0e2f-demo-next' } });
|
||
+ });
|
||
+ it('frames that overtook each other: a lower seq is stale; an older speech never touches or closes the new one', async () => {
|
||
+ const r = await connected(), room = r.transport.rooms[0];
|
||
+ room.on.subtitle(frame(1, 'turn_start', { speech_id: 'a' }));
|
||
+ room.on.subtitle(frame(3, 'assistant_interim', { speech_id: 'a', text: '一二三' }));
|
||
+ room.on.subtitle(frame(2, 'assistant_interim', { speech_id: 'a', text: '一二' })); // overtaken: dropped
|
||
+ expect(r.view().her?.text).toBe('一二三');
|
||
+ // Two handles: b starts while a is still open (queued then generated). a is closed here by b.
|
||
+ room.on.subtitle(frame(4, 'turn_start', { speech_id: 'b' }));
|
||
+ room.on.subtitle(frame(5, 'assistant_interim', { speech_id: 'a', text: '一二三四' }));
|
||
+ room.on.subtitle(frame(6, 'turn_end', { speech_id: 'a', turn_id: 'ta' }));
|
||
+ room.on.subtitle(frame(7, 'turn_cancel', { speech_id: 'a', reason: 'interrupted' }));
|
||
+ room.on.subtitle(frame(8, 'proactive_cancel', { speech_id: 'b', message_id: 3, reason: 'interrupted' })); // wrong family
|
||
+ expect(r.view().her).toEqual({ speech: 'b', text: '', state: 'open' });
|
||
+ room.on.subtitle(frame(9, 'assistant_interim', { speech_id: 'b', text: '好' }));
|
||
+ room.on.subtitle(frame(10, 'turn_end', { speech_id: 'b', turn_id: 'tb' })); // closed without final
|
||
+ expect(r.view().her).toBeNull();
|
||
+ room.on.subtitle(frame(11, 'assistant_final', { speech_id: 'b', turn_id: 'tb', text: '好。' }));
|
||
+ expect(r.view().her).toBeNull();
|
||
+ });
|
||
+ it('reach-out read aloud: shown whole with its stored message id, interruptible, nothing marked or stored', async () => {
|
||
+ const r = await connected(), room = r.transport.rooms[0], frames = sample('proactive');
|
||
+ play(room, frames.slice(0, 1));
|
||
+ expect(r.view()).toMatchObject({ phase: 'speaking', her: { speech: 'speech_pro1', text: '我看到一句很适合现在的话,先念给你听。', state: 'open', messageId: 17, proactive: true } });
|
||
+ room.on.subtitle(frame(2, 'proactive_end', { speech_id: 'speech_pro1', message_id: 99 })); // another row's end
|
||
+ expect(r.view().her?.state).toBe('open');
|
||
+ play(room, frames.slice(1, 2).map(f => ({ ...f, seq: 3 })));
|
||
+ expect(r.view().her).toMatchObject({ messageId: 17, state: 'done' });
|
||
+ play(room, frames.slice(2).map(f => ({ ...f, seq: f.seq + 10 })));
|
||
+ expect(r.view().her).toBeNull();
|
||
+ // Reaching out is not a turn of yours: no "you spoke", no record write, no request at all.
|
||
+ expect(r.spoke).not.toHaveBeenCalled(); expect(r.record).not.toHaveBeenCalled();
|
||
+ expect(r.server.calls.filter(c => !c.url.startsWith('/api/voice/'))).toEqual([]);
|
||
+ });
|
||
+ it('a reach-out cut by your voice: the cancel clears it, your words follow', async () => {
|
||
+ const r = await connected(), room = r.transport.rooms[0];
|
||
+ room.on.subtitle(frame(1, 'proactive_start', { speech_id: 'p', message_id: 18, text: '还有一件事想告诉你——' }));
|
||
+ room.on.subtitle(frame(2, 'interim', { text: '等等' }));
|
||
+ expect(r.view()).toMatchObject({ heard: '等等', her: { speech: 'p', state: 'open' } });
|
||
+ room.on.subtitle(frame(3, 'proactive_cancel', { speech_id: 'p', message_id: 18, reason: 'interrupted' }));
|
||
+ expect(r.view()).toMatchObject({ heard: '等等', her: null });
|
||
+ });
|
||
+ it('text taking over, hanging up and starting again each clear her line at once', async () => {
|
||
+ const r = await connected(), room = r.transport.rooms[0];
|
||
+ room.on.agentState('speaking');
|
||
+ room.on.subtitle(frame(1, 'turn_start', { speech_id: 'a' }));
|
||
+ room.on.subtitle(frame(2, 'assistant_interim', { speech_id: 'a', text: '那我们' }));
|
||
+ r.c.hush();
|
||
+ expect(r.view().her).toBeNull(); expect(room.hushed).toEqual([true]);
|
||
+ room.on.subtitle(frame(3, 'assistant_interim', { speech_id: 'a', text: '那我们明天' }));
|
||
+ expect(r.view().her).toBeNull();
|
||
+ // A finished line is cleared by text too: your typed line comes next on the stage.
|
||
+ room.on.subtitle(frame(4, 'turn_start', { speech_id: 'b' }));
|
||
+ room.on.subtitle(frame(5, 'assistant_interim', { speech_id: 'b', text: '好。' }));
|
||
+ room.on.subtitle(frame(6, 'assistant_final', { speech_id: 'b', turn_id: 'tb', text: '好。' }));
|
||
+ room.on.subtitle(frame(7, 'turn_end', { speech_id: 'b', turn_id: 'tb' }));
|
||
+ expect(r.view().her?.state).toBe('done');
|
||
+ r.c.hush(); expect(r.view().her).toBeNull();
|
||
+ // Hanging up mid-sentence.
|
||
+ room.on.subtitle(frame(8, 'turn_start', { speech_id: 'c' }));
|
||
+ room.on.subtitle(frame(9, 'assistant_interim', { speech_id: 'c', text: '嗯' }));
|
||
+ await r.c.end(); expect(r.view().her).toBeNull();
|
||
+ room.on.subtitle(frame(10, 'assistant_interim', { speech_id: 'c', text: '嗯,晚安' }));
|
||
+ expect(r.view().her).toBeNull();
|
||
+ // A new call is a new worker job: its sequence starts again; the old room's frames stay dead.
|
||
+ await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined();
|
||
+ next.on.subtitle(frame(1, 'turn_start', { speech_id: 'n' }));
|
||
+ next.on.subtitle(frame(2, 'assistant_interim', { speech_id: 'n', text: '又见面了' }));
|
||
+ room.on.subtitle(frame(11, 'turn_start', { speech_id: 'z' }));
|
||
+ room.on.subtitle(frame(12, 'assistant_interim', { speech_id: 'n', text: '旧房间' }));
|
||
+ expect(r.view().her).toEqual({ speech: 'n', text: '又见面了', state: 'open' });
|
||
+ await r.c.end();
|
||
+ });
|
||
+ it('the room dropping mid-sentence clears her line with the call', async () => {
|
||
+ const r = await connected(), room = r.transport.rooms[0];
|
||
+ room.on.subtitle(frame(1, 'turn_start', { speech_id: 'a' }));
|
||
+ room.on.subtitle(frame(2, 'assistant_interim', { speech_id: 'a', text: '我' }));
|
||
+ room.on.dropped(); await flush();
|
||
+ expect(r.view()).toMatchObject({ phase: 'off', her: null, notice: WORDS.dropped });
|
||
+ });
|
||
+});
|
||
+
|
||
+describe('her voice line on the stage', () => {
|
||
+ const M = (id: number, role: Message['role'], text: string, turnId?: string, extra: Partial<Message> = {}): Message => ({ id: String(id), role, text, lifeIds: [], turnId, status: 'sent', ...extra });
|
||
+ const metrics = { width: 900, font: 23, compact: false };
|
||
+ const stage = (list: ShownMessage[]) => pagesOf(groupTurns(list).at(-1), metrics).map(p => `${p.key}=${p.text}`);
|
||
+ const you = M(4, 'user', '你在听吗?', 'T');
|
||
+ it('grows in her slot after your voice turn; the stored row takes over under the same line keys', () => {
|
||
+ const open: HerLine = { speech: 's', text: '在听呢,你刚才说到一半的那件事我一直还在心里想着。后来呢?', state: 'open' };
|
||
+ const growing = voiceLine(open, [you], 1)!;
|
||
+ expect(growing).toMatchObject({ role: 'companion', live: 'generating' });
|
||
+ // The newest line of a reply still growing is held back, as in a streamed text reply.
|
||
+ expect(stage([you, growing])).toEqual(['T:0:0=你在听吗?', 'T:1:0=在听呢,你刚才说到一半的那件事我一直还在心里想着。']);
|
||
+ const done: HerLine = { ...open, state: 'done', turnId: 'T' };
|
||
+ const shown = stage([you, voiceLine(done, [you], 1)!]);
|
||
+ expect(shown).toEqual(['T:0:0=你在听吗?', 'T:1:0=在听呢,你刚才说到一半的那件事我一直还在心里想着。', 'T:1:1=后来呢?']);
|
||
+ const stored = [you, M(5, 'companion', done.text, 'T')];
|
||
+ expect(voiceLine(done, stored, 1)).toBeNull();
|
||
+ expect(stage(stored)).toEqual(shown);
|
||
+ });
|
||
+ it('a finished line never stands in front of a newer turn; an open one waits for its own end', () => {
|
||
+ const done: HerLine = { speech: 's', text: '好。', state: 'done', turnId: 'T' };
|
||
+ expect(voiceLine(done, [you, M(6, 'user', '还有', 'U')], 1)).toBeNull();
|
||
+ expect(voiceLine({ ...done, state: 'open' }, [you, M(6, 'user', '还有', 'U')], 1)).not.toBeNull();
|
||
+ expect(voiceLine({ ...done, text: '' }, [you], 1)).toBeNull();
|
||
+ });
|
||
+ it('a reach-out read aloud keeps its stored row identity and gives way when that row is the newest line', () => {
|
||
+ const line: HerLine = { speech: 'p', text: '我看到一句很适合现在的话,先念给你听。', state: 'open', messageId: 17, proactive: true };
|
||
+ const row = M(17, 'companion', line.text, 'R', { proactive: true });
|
||
+ const before = [you, M(5, 'companion', '在。', 'T')];
|
||
+ const waiting = voiceLine(line, before, 2, [row])!;
|
||
+ expect(waiting).toMatchObject({ id: '17', turnId: 'R', proactive: true }); expect(waiting.live).toBeUndefined();
|
||
+ expect(stage([...before, waiting])).toEqual(stage([...before, row]));
|
||
+ expect(voiceLine({ ...line, state: 'done' }, [...before, row], 2)).toBeNull();
|
||
+ // Not on the page at all: shown under its own key; nothing is created for the record.
|
||
+ expect(voiceLine(line, before, 2)).toMatchObject({ id: 'voice-p', proactive: true });
|
||
+ });
|
||
+});
|
||
+
|
||
describe('record re-read', () => {
|
||
const msg = (id: number, role: Message['role'], turnId: string, extra: Partial<Message> = {}): Message => ({ id: String(id), role, text: `${role}${id}`, lifeIds: [], turnId, status: 'sent', ...extra });
|
||
it('adds voice turns once, in id order; leaves reach-outs to the inbox and text turns to the stream', () => {
|