diff --git a/docs/requirements/handoff-HEL-879.json b/docs/requirements/handoff-HEL-879.json index 1c75eea..cf3cf32 100644 --- a/docs/requirements/handoff-HEL-879.json +++ b/docs/requirements/handoff-HEL-879.json @@ -1,7 +1,7 @@ { "baseline_version": "1.11", "read_commit": "c121c42e39174e0f627f129e7f7e5700f33acf26", - "task": "HEL-879 双职责一次交付：A) HEL-896 发布工具四项有限收口（总管独立复核清单）——①归档一致性强制：checkpoint busy 或源/快照行数不一致一律 CLI 非0、不形成可交付包，verify 拒绝 rows_match=False 旧包并按源路径映射做真实回退演练（解包隔离目录、实开各库核行数与哈希），自测 9/9 先复现旧失败形状再证修复；②目录内真实 SQLite（history_root/mem0_public_history.db 等）停服口径递归走 checkpoint+逐表核对，在线模式仅显式单库 backup API、含 SQLite 目录直接拒绝；③提前封账可执行：PersistentBudget.sealed()/seal() 原子写封账标记（真实用量保留不伪造不清零），acquire 首查封账（budget_sealed:51/55），跨重启/backup 归档恢复均拒绝、重复 seal 幂等、无解封路径，tools/seal_budget_hel879.py 为显式关闭命令，S6 负例（51/55 提前收尾、误重启、归档恢复、日顶有余量仍拒）并入 verify_budget_split 10/10；④安装入口唯一：hel895-release-candidate.md §2 全 offline 自足安装、§4 五步固定 SDK 入口（tarball 哈希→固定源构建→cp311 备产→仅本地轮→co-import），hel874-release.md §2/§4 标注被取代，mem0-stack.lock 的 mem0ai 行只作版本记录，audit 工具 head 动态化+证据目录可重定向，新证据 docs/rework/hel879/{sdk-audit,release-artifacts.json}。B) 统一语音候选隔离整合：hel-879-assembly 分支 = c121c42 + 发布收口(01ff919) + 语音后台 aba2aeb + 语音前端 c61d9c9（原作者 web/src 逐字节保留，CSS 哈希变化仅原作者 components.css +21 行），语音适配器测试对齐主线 HEL-882 硬闸（engineering_budget，不改产品门），全电池在该 head 实跑：pytest 324/324（299 主线+25 语音）、ruff 干净、web typecheck 0 错/lint 0 错1既有警告/vitest 88/88/build 通过、真实服务浏览器 9/9（零权限调用、零非回环出站，证据 docs/rework/hel879/browser/）、verify_hel877_voice_chain 8/8、.12 真SDK stdin 检查 8/8 ALL PASS（零写入零网络）、check_release_offline R1-R7、固定SDK回归 16/16+12/12+6/6+7/7+12/12。零真实 LLM/嵌入/ASR/TTS、零下载、零麦克风、未启动/停止任何服务、不部署、不合并 PR、PR#11 与 hel-870-assembly 未动、权威账与冻结未写、真实 cap 保持 0；2026-10-27 剩余收口（总管 b4dadd8 复核三项）：A) 房间自动到期与共享上限——清理所有者观察活跃会话截止时间与欠账，无人再发请求也会到期转债真实删房；显式挂断与后台共用同一持久尝试/年龄上限与状态（_sessions 不再旁路），超限 CleanupAbandoned→503+准确状态、绝不回完成 404，成功同清 sessions/debts，重启续同一上限，关闭不留协程；B) 排队话轮确切归属——CyberGrillLLM.chat 读 livekit-agents 1.8.8 自身 agents_speech_handle ContextVar（_create_speech_task 源级核实），流绑定到创建它的那个 SpeechHandle，无归属/未登记 fail-closed 不绑，新 speech 不再退役旧 pending（总管交错反例 H1/H2/S1/S2 修正），say/主动生成不吞用户轮流；C) 交前端的版本化字幕契约 v2——信封 v/seq、确切 speech_id/turn_id/message_id、turn_start/interim/final/turn_end/turn_cancel 起止与取消、迟到/乱序双规则保护、主动播报同源 message_id 自带 proactive_start/end/cancel（不借回执分支），契约 docs/voice-subtitle-contract-v2.md + 录制样例 subtitle-samples/，总工未改 web/src（接线归 HEL-898 原视觉作者）；另修 test_r2 lifespan 旧测试的双 portal 调用竞态确定性（原子冻结+阻塞，不抬超时不删断言）；2026-10-27 契约核准后 3 处实现时机收口（总管 3228b1e 复核，仅修实现时机，不改 v2 信封字段/事件种类、不动 web/src）：①open_session 在真实建房 await 让出事件循环后于注册成功并持久化时重新唤醒/重建清理所有者（旧代码 pre-dispatch kick 的 cleaner 已空转退出：建房 20ms 真实 await、TTL25ms、等 120ms 无任何请求时 DELETE=0/过期登记仍在/cleaner=None；修后 DELETE=1/登记清/无悬空协程，新增真实让出建房负例与更短 TTL 唤醒负例，保留自动截止与共享上限机制不变）；②speak_proactive_row 先取 say 同步返回的 SpeechHandle、立即发布 proactive_start 并注册该 handle 的 end/cancel 再按需等待（真实 SDK AgentSession.say 为同步函数、await handle=等整段播完，旧代码播完才发 start；.12 真SDK 验证播放中即可见 start、打断只有 cancel 不误发 end，v2 字段不变）；③_after_playout 的 assistant_final 前置完整/合法同轮回执门（流未完整或无有效同轮 receipt/旧 epoch 一律只 turn_cancel，不再先发 final 后决定不确认；完整+合法回执路径 final 与 ack 均保留，φ/打断/失败/无归属 fail-closed 不变）；.12 真SDK stdin 检查扩至 14/14 ALL PASS（新增：主动播报时机、主动播报打断、无回执不 final 三负例，实际 worker 模块回调路径），前后对照证据 docs/rework/hel879/timing-fix/ 与 docs/rework/hel879/sdk-check-hel879-timing.log；本轮回归：pytest 352/352（新增 2 项负例先在旧代码复现失败）、ruff(src/tests/tools) 0 错、verify_hel877_voice_chain 8/8、需求检查 4/4（origin/main 与 c121c42 各带/不带本 handoff）；零真实 LLM/嵌入/ASR/TTS、零下载零安装、未部署未合并、PR#11 与 hel-870-assembly 未动、真实 cap 仍 0", + "task": "HEL-879 双职责一次交付：A) HEL-896 发布工具四项有限收口（总管独立复核清单）——①归档一致性强制：checkpoint busy 或源/快照行数不一致一律 CLI 非0、不形成可交付包，verify 拒绝 rows_match=False 旧包并按源路径映射做真实回退演练（解包隔离目录、实开各库核行数与哈希），自测 9/9 先复现旧失败形状再证修复；②目录内真实 SQLite（history_root/mem0_public_history.db 等）停服口径递归走 checkpoint+逐表核对，在线模式仅显式单库 backup API、含 SQLite 目录直接拒绝；③提前封账可执行：PersistentBudget.sealed()/seal() 原子写封账标记（真实用量保留不伪造不清零），acquire 首查封账（budget_sealed:51/55），跨重启/backup 归档恢复均拒绝、重复 seal 幂等、无解封路径，tools/seal_budget_hel879.py 为显式关闭命令，S6 负例（51/55 提前收尾、误重启、归档恢复、日顶有余量仍拒）并入 verify_budget_split 10/10；④安装入口唯一：hel895-release-candidate.md §2 全 offline 自足安装、§4 五步固定 SDK 入口（tarball 哈希→固定源构建→cp311 备产→仅本地轮→co-import），hel874-release.md §2/§4 标注被取代，mem0-stack.lock 的 mem0ai 行只作版本记录，audit 工具 head 动态化+证据目录可重定向，新证据 docs/rework/hel879/{sdk-audit,release-artifacts.json}。B) 统一语音候选隔离整合：hel-879-assembly 分支 = c121c42 + 发布收口(01ff919) + 语音后台 aba2aeb + 语音前端 c61d9c9（原作者 web/src 逐字节保留，CSS 哈希变化仅原作者 components.css +21 行），语音适配器测试对齐主线 HEL-882 硬闸（engineering_budget，不改产品门），全电池在该 head 实跑：pytest 324/324（299 主线+25 语音）、ruff 干净、web typecheck 0 错/lint 0 错1既有警告/vitest 88/88/build 通过、真实服务浏览器 9/9（零权限调用、零非回环出站，证据 docs/rework/hel879/browser/）、verify_hel877_voice_chain 8/8、.12 真SDK stdin 检查 8/8 ALL PASS（零写入零网络）、check_release_offline R1-R7、固定SDK回归 16/16+12/12+6/6+7/7+12/12。零真实 LLM/嵌入/ASR/TTS、零下载、零麦克风、未启动/停止任何服务、不部署、不合并 PR、PR#11 与 hel-870-assembly 未动、权威账与冻结未写、真实 cap 保持 0；2026-10-27 剩余收口（总管 b4dadd8 复核三项）：A) 房间自动到期与共享上限——清理所有者观察活跃会话截止时间与欠账，无人再发请求也会到期转债真实删房；显式挂断与后台共用同一持久尝试/年龄上限与状态（_sessions 不再旁路），超限 CleanupAbandoned→503+准确状态、绝不回完成 404，成功同清 sessions/debts，重启续同一上限，关闭不留协程；B) 排队话轮确切归属——CyberGrillLLM.chat 读 livekit-agents 1.8.8 自身 agents_speech_handle ContextVar（_create_speech_task 源级核实），流绑定到创建它的那个 SpeechHandle，无归属/未登记 fail-closed 不绑，新 speech 不再退役旧 pending（总管交错反例 H1/H2/S1/S2 修正），say/主动生成不吞用户轮流；C) 交前端的版本化字幕契约 v2——信封 v/seq、确切 speech_id/turn_id/message_id、turn_start/interim/final/turn_end/turn_cancel 起止与取消、迟到/乱序双规则保护、主动播报同源 message_id 自带 proactive_start/end/cancel（不借回执分支），契约 docs/voice-subtitle-contract-v2.md + 录制样例 subtitle-samples/，总工未改 web/src（接线归 HEL-898 原视觉作者）；另修 test_r2 lifespan 旧测试的双 portal 调用竞态确定性（原子冻结+阻塞，不抬超时不删断言）；2026-10-27 契约核准后 3 处实现时机收口（总管 3228b1e 复核，仅修实现时机，不改 v2 信封字段/事件种类、不动 web/src）：①open_session 在真实建房 await 让出事件循环后于注册成功并持久化时重新唤醒/重建清理所有者（旧代码 pre-dispatch kick 的 cleaner 已空转退出：建房 20ms 真实 await、TTL25ms、等 120ms 无任何请求时 DELETE=0/过期登记仍在/cleaner=None；修后 DELETE=1/登记清/无悬空协程，新增真实让出建房负例与更短 TTL 唤醒负例，保留自动截止与共享上限机制不变）；②speak_proactive_row 先取 say 同步返回的 SpeechHandle、立即发布 proactive_start 并注册该 handle 的 end/cancel 再按需等待（真实 SDK AgentSession.say 为同步函数、await handle=等整段播完，旧代码播完才发 start；.12 真SDK 验证播放中即可见 start、打断只有 cancel 不误发 end，v2 字段不变）；③_after_playout 的 assistant_final 前置完整/合法同轮回执门（流未完整或无有效同轮 receipt/旧 epoch 一律只 turn_cancel，不再先发 final 后决定不确认；完整+合法回执路径 final 与 ack 均保留，φ/打断/失败/无归属 fail-closed 不变）；.12 真SDK stdin 检查扩至 14/14 ALL PASS（新增：主动播报时机、主动播报打断、无回执不 final 三负例，实际 worker 模块回调路径），前后对照证据 docs/rework/hel879/timing-fix/ 与 docs/rework/hel879/sdk-check-hel879-timing.log；本轮回归：pytest 352/352（新增 2 项负例先在旧代码复现失败）、ruff(src/tests/tools) 0 错、verify_hel877_voice_chain 8/8、需求检查 4/4（origin/main 与 c121c42 各带/不带本 handoff）；零真实 LLM/嵌入/ASR/TTS、零下载零安装、未部署未合并、PR#11 与 hel-870-assembly 未动、真实 cap 仍 0 || 统一整合审核追加（2026-10-27，整合目标 fb7ea84=fc905fe+HEL-898保全）：以 .12 真实 SDK（livekit-agents 1.8.8/livekit-api 1.2.1，SSH 只读 stdin）驱动实际 worker（make_voice_llm+bind_voice_session+speak_proactive_row）生成 v2 帧（normal/interrupt/phi/no_receipt/proactive，主动 start 播放中可见、无回执只 cancel 无 final），经 scripts/gen_hel879_unified_frames.py 与 web/tests/fixtures/hel879/ 固化，喂入当前前端：web/tests/hel879_frames.test.ts 9/9（增量→final→落库回读接替、φ/cancel/文字接管/用户开口/掉线重开清残影、replay/迟到/乱序帧全拦）。seq 命名空间核清（docs/rework/hel879/unified-review/seq-job-analysis.md）：一 job 一 entrypoint、重连不重跑、后端唯一派发点每会话新房，job=房间=控制器会话一一对应，无同房新 job 序号归零路径；残余缺口=agent 离场僵尸界面（归原作者前端）。closed 64 上限核清：64+ 后续关闭挤出退休 speech，重复高 seq turn_start 可重进（web/tests/hel879_frames.test.ts DOCUMENTED GAP 用例），现 worker 无法产生该帧（speech_id 唯一、begin 每流一次），列防御性建议归原作者。本机缺 GUI 系统库且零安装授权，浏览器层未独立复跑（web/tests/hel879_unified_shots.mjs 已备，供有 Edge 环境回放实际 worker帧）；作者 61/61 与两主题截图仍是界面证据。回归 pytest 352/352、vitest 108/108、typecheck/build/lint（0错1既有警告）通过；R5 真机测试包材料见 docs/rework/hel879/unified-review/r5-test-package.md，未执行任何真实调用/下载/安装/部署/合并。", "requirement_ids": [ "CG-006", "CG-008", diff --git a/docs/rework/hel879/unified-review/frames/sidecar.json b/docs/rework/hel879/unified-review/frames/sidecar.json new file mode 100644 index 0000000..3867795 --- /dev/null +++ b/docs/rework/hel879/unified-review/frames/sidecar.json @@ -0,0 +1,85 @@ +{ + "sdk": { + "livekit_agents": "1.8.8", + "livekit_api": "1.2.1" + }, + "source": "actual fc905fe+fb7ea84 worker driven on .12 real SDK (livekit-agents 1.8.8, livekit-api 1.2.1) via SSH stdin, zero network/writes", + "scenarios": { + "normal": { + "kinds": [ + "interim", + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "assistant_interim", + "assistant_final", + "turn_end" + ], + "frames": 8, + "acks": [ + "382c41d9-0d72-433e-b370-f838fa5d0bd0" + ], + "turn_id": "382c41d9-0d72-433e-b370-f838fa5d0bd0" + }, + "interrupt": { + "kinds": [ + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "turn_cancel", + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "assistant_final", + "turn_end" + ], + "frames": 11, + "acks": [ + "edbcb837-f19f-47e7-8d5c-057010da43eb" + ], + "turn_ids": { + "cut": "09d05c0f-3058-4ff8-a56e-4e94bd17d7b7", + "next": "edbcb837-f19f-47e7-8d5c-057010da43eb" + } + }, + "phi": { + "kinds": [ + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "assistant_interim", + "error", + "turn_cancel" + ], + "frames": 7, + "acks": [], + "turn_id": "31e70e52-1471-4e50-ac49-cce89979a9bf" + }, + "no_receipt": { + "kinds": [ + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "turn_cancel" + ], + "frames": 5, + "acks": [], + "turn_id": "36d160e3-e365-46ff-ad57-c2878c00720f" + }, + "proactive": { + "kinds": [ + "proactive_start", + "proactive_end", + "proactive_start", + "proactive_cancel" + ], + "frames": 4, + "start_while_playing": true + } + } +} \ No newline at end of file diff --git a/docs/rework/hel879/unified-review/frames/worker-frames-raw.json b/docs/rework/hel879/unified-review/frames/worker-frames-raw.json new file mode 100644 index 0000000..12c80b0 --- /dev/null +++ b/docs/rework/hel879/unified-review/frames/worker-frames-raw.json @@ -0,0 +1 @@ +{"sdk": {"livekit_agents": "1.8.8", "livekit_api": "1.2.1"}, "scenarios": {"normal": {"frames": [{"v": 2, "seq": 1, "kind": "interim", "role": "user", "text": "你在听"}, {"v": 2, "seq": 2, "kind": "user_final", "role": "user", "text": "你在听吗"}, {"v": 2, "seq": 3, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_17d892eb3dc3"}, {"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "在听呢，", "speech_id": "speech_17d892eb3dc3"}, {"v": 2, "seq": 5, "kind": "assistant_interim", "role": "assistant", "text": "在听呢，你刚才说到一半的那件事", "speech_id": "speech_17d892eb3dc3"}, {"v": 2, "seq": 6, "kind": "assistant_interim", "role": "assistant", "text": "在听呢，你刚才说到一半的那件事我还在想。", "speech_id": "speech_17d892eb3dc3"}, {"v": 2, "seq": 7, "kind": "assistant_final", "role": "assistant", "text": "在听呢，你刚才说到一半的那件事我还在想。", "speech_id": "speech_17d892eb3dc3", "turn_id": "382c41d9-0d72-433e-b370-f838fa5d0bd0"}, {"v": 2, "seq": 8, "kind": "turn_end", "role": "assistant", "text": "", "speech_id": "speech_17d892eb3dc3", "turn_id": "382c41d9-0d72-433e-b370-f838fa5d0bd0"}], "turn_id": "382c41d9-0d72-433e-b370-f838fa5d0bd0", "acks": ["382c41d9-0d72-433e-b370-f838fa5d0bd0"], "kinds": ["interim", "user_final", "turn_start", "assistant_interim", "assistant_interim", "assistant_interim", "assistant_final", "turn_end"]}, "interrupt": {"frames": [{"v": 2, "seq": 1, "kind": "user_final", "role": "user", "text": "讲讲你下午做了什么"}, {"v": 2, "seq": 2, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_dd1c4c382512"}, {"v": 2, "seq": 3, "kind": "assistant_interim", "role": "assistant", "text": "我下午去了河边，", "speech_id": "speech_dd1c4c382512"}, {"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "我下午去了河边，看到一群白鹭从水面上飞过去了，", "speech_id": "speech_dd1c4c382512"}, {"v": 2, "seq": 5, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_dd1c4c382512", "reason": "interrupted"}, {"v": 2, "seq": 6, "kind": "user_final", "role": "user", "text": "那后来呢"}, {"v": 2, "seq": 7, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_55ef57a410cf"}, {"v": 2, "seq": 8, "kind": "assistant_interim", "role": "assistant", "text": "后来我在桥上站了一会儿，", "speech_id": "speech_55ef57a410cf"}, {"v": 2, "seq": 9, "kind": "assistant_interim", "role": "assistant", "text": "后来我在桥上站了一会儿，风把水面吹皱了。", "speech_id": "speech_55ef57a410cf"}, {"v": 2, "seq": 10, "kind": "assistant_final", "role": "assistant", "text": "后来我在桥上站了一会儿，风把水面吹皱了。", "speech_id": "speech_55ef57a410cf", "turn_id": "edbcb837-f19f-47e7-8d5c-057010da43eb"}, {"v": 2, "seq": 11, "kind": "turn_end", "role": "assistant", "text": "", "speech_id": "speech_55ef57a410cf", "turn_id": "edbcb837-f19f-47e7-8d5c-057010da43eb"}], "turn_ids": {"cut": "09d05c0f-3058-4ff8-a56e-4e94bd17d7b7", "next": "edbcb837-f19f-47e7-8d5c-057010da43eb"}, "acks": ["edbcb837-f19f-47e7-8d5c-057010da43eb"], "kinds": ["user_final", "turn_start", "assistant_interim", "assistant_interim", "turn_cancel", "user_final", "turn_start", "assistant_interim", "assistant_interim", "assistant_final", "turn_end"]}, "phi": {"frames": [{"v": 2, "seq": 1, "kind": "user_final", "role": "user", "text": "说说那件事吧"}, {"v": 2, "seq": 2, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_7a5630ba8cc7"}, {"v": 2, "seq": 3, "kind": "assistant_interim", "role": "assistant", "text": "（她顿了一下）", "speech_id": "speech_7a5630ba8cc7"}, {"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "（她顿了一下）这会儿我有点说不出来话……", "speech_id": "speech_7a5630ba8cc7"}, {"v": 2, "seq": 5, "kind": "assistant_interim", "role": "assistant", "text": "（她顿了一下）这会儿我有点说不出来话……（她顿了一下）这会儿我有点说不出来话……", "speech_id": "speech_7a5630ba8cc7"}, {"v": 2, "seq": 6, "kind": "error", "role": "assistant", "text": "（她顿了一下）这会儿我有点说不出来话……", "speech_id": "speech_7a5630ba8cc7"}, {"v": 2, "seq": 7, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_7a5630ba8cc7", "reason": "error_phi"}], "turn_id": "31e70e52-1471-4e50-ac49-cce89979a9bf", "acks": [], "kinds": ["user_final", "turn_start", "assistant_interim", "assistant_interim", "assistant_interim", "error", "turn_cancel"]}, "no_receipt": {"frames": [{"v": 2, "seq": 1, "kind": "user_final", "role": "user", "text": "再讲讲那座桥"}, {"v": 2, "seq": 2, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_20d31bfb084a"}, {"v": 2, "seq": 3, "kind": "assistant_interim", "role": "assistant", "text": "那座桥是石头砌的，", "speech_id": "speech_20d31bfb084a"}, {"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "那座桥是石头砌的，下雨的时候水声很好听。", "speech_id": "speech_20d31bfb084a"}, {"v": 2, "seq": 5, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_20d31bfb084a", "reason": "no-receipt"}], "turn_id": "36d160e3-e365-46ff-ad57-c2878c00720f", "acks": [], "kinds": ["user_final", "turn_start", "assistant_interim", "assistant_interim", "turn_cancel"]}, "proactive": {"frames": [{"v": 2, "seq": 1, "kind": "proactive_start", "role": "assistant", "text": "刚才路过花店，想起你说过喜欢白色的洋桔梗。", "speech_id": "speech_172abc64e23c", "message_id": 301}, {"v": 2, "seq": 2, "kind": "proactive_end", "role": "assistant", "text": "", "speech_id": "speech_172abc64e23c", "message_id": 301}, {"v": 2, "seq": 3, "kind": "proactive_start", "role": "assistant", "text": "晚上风大，窗台的那盆薄荷记得搬进来。", "speech_id": "speech_835f928b989d", "message_id": 302}, {"v": 2, "seq": 4, "kind": "proactive_cancel", "role": "assistant", "text": "", "speech_id": "speech_835f928b989d", "message_id": 302, "reason": "interrupted"}], "start_while_playing": true, "kinds": ["proactive_start", "proactive_end", "proactive_start", "proactive_cancel"], "message_ids": [301, 302]}}} diff --git a/docs/rework/hel879/unified-review/logs/requirements-fc905fe-handoff.log b/docs/rework/hel879/unified-review/logs/requirements-fc905fe-handoff.log new file mode 100644 index 0000000..8d7e4a5 --- /dev/null +++ b/docs/rework/hel879/unified-review/logs/requirements-fc905fe-handoff.log @@ -0,0 +1 @@ +需求结构检查通过：60项；来源和覆盖完整。仍须人工核对授权语义与实际体验。 diff --git a/docs/rework/hel879/unified-review/logs/requirements-fc905fe.log b/docs/rework/hel879/unified-review/logs/requirements-fc905fe.log new file mode 100644 index 0000000..8d7e4a5 --- /dev/null +++ b/docs/rework/hel879/unified-review/logs/requirements-fc905fe.log @@ -0,0 +1 @@ +需求结构检查通过：60项；来源和覆盖完整。仍须人工核对授权语义与实际体验。 diff --git a/docs/rework/hel879/unified-review/logs/requirements-main-handoff.log b/docs/rework/hel879/unified-review/logs/requirements-main-handoff.log new file mode 100644 index 0000000..8d7e4a5 --- /dev/null +++ b/docs/rework/hel879/unified-review/logs/requirements-main-handoff.log @@ -0,0 +1 @@ +需求结构检查通过：60项；来源和覆盖完整。仍须人工核对授权语义与实际体验。 diff --git a/docs/rework/hel879/unified-review/logs/requirements-main.log b/docs/rework/hel879/unified-review/logs/requirements-main.log new file mode 100644 index 0000000..8d7e4a5 --- /dev/null +++ b/docs/rework/hel879/unified-review/logs/requirements-main.log @@ -0,0 +1 @@ +需求结构检查通过：60项；来源和覆盖完整。仍须人工核对授权语义与实际体验。 diff --git a/docs/rework/hel879/unified-review/r5-test-package.md b/docs/rework/hel879/unified-review/r5-test-package.md new file mode 100644 index 0000000..8256f69 --- /dev/null +++ b/docs/rework/hel879/unified-review/r5-test-package.md @@ -0,0 +1,73 @@ +# HEL-879 · R5 真机测试包（供主卡拍板的集中材料，未执行） + +> 性质：**准备材料**。零真实模型/嵌入/ASR/TTS/麦克风请求、零下载/安装/部署/合并、真实 cap=0。所有端口/显存数据为 2026-10-27 只读实测（见文末）。供总管在 HEL-797 一次性向白栖知申请授权，不替用户选内部算法。 + +## 1. 依赖清单：已有 / 可复用 / 真缺 + +| 类别 | 状态 | 明细 | +| --- | --- | --- | +| LiveKit SFU | ✅ 已在运行 | .12:7880/7881（docker `voice-livekit`，实测 200） | +| FunASR 流式 ASR | ✅ 已在运行 | .12:10095（funasr-runtime-sdk-online-cpu-0.1.13，实测 426=WS 存活） | +| silero VAD / agent venv | ✅ 已在 | `/opt/voice-agent/agent/venv`（livekit-agents 1.8.8、livekit-plugins-silero 1.8.8、livekit 1.1.20） | +| CosyVoice3 TTS | ⛔ 停机待重启 | .12:9880 停机中；权重在位 `/opt/models/CosyVoice3-0.5B-2512`（9.1GB）；显存已够（见 §2）；重启=运维动作非下载 | +| 统一 CyberGril 实例（语音入口） | ⛔ 待部署 | 端口 8360 实测空闲；部署物=本候选（hel-879-assembly 整合头）+ 已过发布包流程；`config/voice.yaml` 需按 .12 现值填写（键名：livekit_url/livekit_internal/api_key/api_secret） | +| worker 部署目录 | ⛔ 待部署 | `/opt/voice-agent-cybergril/`（新建独立目录，复用 agent venv；健康端口默认 8082 需确认不与现行冲突，可改 `CYBERGRIL_WORKER_HEALTH_PORT`） | +| Python 缺项（3.11 venv 41 项） | 与语音无关 | 属发布/mem0 线（sdk-audit）；R5 语音不依赖它们 | +| 下载需求 | **零** | R5 语音链全部部件已在机（代码+venv+权重+端口） | + +## 2. 资源现状（2026-10-27 只读实测） + +- GPU：RTX 5090D 32607MiB，**空闲 30520miB**（ComfyUI 仍处已授权暂停，未恢复）——CosyVoice 常驻约 5.6GB 无障碍。 +- 内存 94GB / 盘 916GB：充足（沿用 HEL-876 实测）。 + +## 3. 部署顺序（获批后执行；含回滚） + +1. 统一实例（8360，隔离数据目录，**不动 8300/8310/8320/8340/8350/8090/8081**）——同进程挂语音路由（现有 router 已实现 status/session/DELETE）。 +2. TTS 9880 重启：**先核实 9880 无监听再启动**（`cd /opt/voice-agent/cosyvoice && nohup venv/bin/python server.py > tts.log 2>&1 &`；历史 `fuser -k` 命令禁止照抄；当前实测 9880 已停，无需杀任何进程）。 +3. cybergril worker：`/opt/voice-agent-cybergril/` 放 worker（源码即本候选 src/cybergril/voice/worker.py），用 agent venv 启动，`CYBERGRIL_API_BASE=http://127.0.0.1:8360`。 +4. HTTPS 入口：统一实例第二端口（如 8361，自签证书，流程同 8090 已验证路径）；**语音入口必须与文字同一个她、同一进程**（勿另起写库实例）。 +- 回滚：停 worker → 停 8360/8361 → TTS 回到停机或原状（均为独立进程）；Git 回退=本分支回退到 fc905fe 重新部署；不触碰既有五实例与通用 worker。 + +## 4. D1 音色待选（不替用户定，仅供选择） + +现有中文女声 6 支：`default`（官方兜底）、`female_soft`（沉稳知性）、`xiaoyun`（俏皮）、`rain`、`jiaojiao`（亲密向）、`bailu`（阳光开朗；当前 persona_state 恰有用户自存「晚晚/白鹿」角色）。试听入口=`https://192.168.200.12:8090` 音色面板（TTS 重启后可用）；均不满意可录 5–20s 中文参考克隆。worker 缺省回落 `default`。 + +## 5. 可信 HTTPS+WSS 设备步骤（D3，用户动作） + +每台设备首次打开 `https://192.168.200.12:<语音端口>`：浏览器「高级→继续访问」信任自签证书（同时覆盖页面 HTTPS 与 LiveKit WSS 7880 的证书提示）→ 允许麦克风。不建内网 CA、不买公网证书、不改防火墙。手机与桌面同流程；Safari/Chrome 均需实测（列入验收项，未测即注明）。 + +## 6. 有限预算与停止条件（逐类计数，到线即停） + +| 类别 | 建议上限 | 计数方式 | 停止条件 | +| --- | --- | --- | --- | +| LLM（/api/chat 真实轮） | 30 轮 | 服务端既有逐笔账（chat_message 轮数） | 到 30 轮或出现异常扣费风险 | +| ASR（FunASR 流） | 同 LLM 轮数（随轮产生） | worker 日志/job 数 | 随 LLM 停 | +| TTS（CosyVoice 合成） | 同 LLM 轮数 | tts.log 合成行 | 随 LLM 停 | +| 嵌入 | 0（R5 不激活 mem0） | — | 不启用 | +| 语音会话总数 | 3 次完整通话 | /api/voice/session 建房数 | 3 次或验收完成 | + +冻结账不动；预算到线立即停并汇报实际用量。以上数字为建议，最终由白栖知定。 + +## 7. 验收清单（R5 真机，integration-plan 原四条 + 补充） + +1. 连续交谈 ≥3 轮（同一她、同档案、记忆延续，音色一致）。 +2. 开口打断约 1 秒停声（VAD→TTS 截断；字幕同步消失）。 +3. 挂断立即停麦（页面 track.stop + 房间删除；麦克风指示灯灭）。 +4. 无旧残音/残影（打断与掉线后旧帧不回显；本轮离线已证帧防护，真机复验）。 +5. 网络失败/重开：掉线诚实提示、重开仍是同一位她（本轮离线已证 UI 收口）。 +6. HTTPS 页面连 WSS（自签信任链一次覆盖）；手机实测（Safari/Chrome 至少其一）。 +7. 字幕全生命周期（本轮实际 worker 帧已离线全验；真机抽查增量→final→落库一致）。 +8. API 重启释放（LiveKit 房间与 worker job 真实回收）——R5 范围内做一次受控重启验证。 +9. 口型**不在本包**（不伪装接通）。 + +## 8. 未决前置（需要总管/白栖知拍板的项） + +1. D1 音色选择（§4 六选一或克隆）。 +2. 预算上限确认/调整（§6）。 +3. 8360/8361 端口与 `/opt/voice-agent-cybergril/` 目录名确认。 +4. 测试时段（TTS 重启占显存 ~5.6GB；ComfyUI 现处暂停，是否保持到测试后由用户定）。 +5. worker 健康端口（8082 或改）确认不与现行冲突。 + +## 9. 本包不做 + +- 不合并 PR#11/任何 PR；不替换 8300/8310 正式入口；不改队员运行时/模型/认证；不动冻结账与既有五实例；CG-011 完整会话中主动表达仍承接（默认关的「开场念一条旧留言」不冒充完整项）；R6/R7 继续承接。 diff --git a/docs/rework/hel879/unified-review/seq-job-analysis.md b/docs/rework/hel879/unified-review/seq-job-analysis.md new file mode 100644 index 0000000..c172eb5 --- /dev/null +++ b/docs/rework/hel879/unified-review/seq-job-analysis.md @@ -0,0 +1,42 @@ +# HEL-879 统一整合审核 · 焦点2：seq 命名空间与 job/房间/控制器代次（2026-10-27） + +问题（原作者 HEL-898 提出）：`SubtitleBus` 的 `seq` 按 worker job 从 1 计（`src/cybergril/voice/worker.py` `entrypoint` 内建 bus），而前端 `VoiceController` 按**控制器会话**（一个 `Live`，web/src/voice/session.ts `live.seq`）记高水位。若同一房间在控制器会话存活期间换了新 job（worker 重连/新派发），新 job 的 seq 归零，前端要么误丢新帧、要么误收旧帧。本文件用 .12 真实 SDK 源码（livekit-agents 1.8.8 / livekit-api 1.2.1，`/opt/voice-agent/agent/venv/lib/python3.12/site-packages/`，SSH 只读取证，零写入零网络）与仓库代码逐条核清。 + +## 1. 一个 job 只跑一次 entrypoint（seq 计数器只生一次） + +真实源码 `livekit/agents/ipc/job_proc_lazy_main.py`（默认进程执行器路径）： + +- L241–246：进程内已有运行中 job 时再收到 `StartJobRequest` 只告警跳过（`"trying to start a new job while one is already running"` → `continue`）——**一个 job 进程最多承载一个 job**。 +- L318–323：`_start_job` 为该 job 建 `JobContext` 并启动**唯一** `job_task`。 +- `_run_job_task`（L327 起）里 `self._job_entrypoint_fnc(job_ctx)` 只被创建为一个 task（`job_user_entrypoint`）；job 在以下任一时刻结束：entrypoint 返回、`_shutdown_fut` 置位（room disconnected / shutdown 请求 / job 崩溃），随后 `_shutdown_job` → `_on_cleanup`，进程随 `_exit_proc_flag` 退出。 + +结论：worker.py 的 `entrypoint`（含 `SubtitleBus` 实例）在**一个 job 内恰好执行一次**，job 结束即进程结束，不存在「同 job 二次 entrypoint 导致 seq 归零」。 + +## 2. 断线重连不重跑 entrypoint（seq 不归零） + +真实源码 `livekit/rtc/room.py`： + +- L85–86 / L1130–1133：瞬断走 `reconnecting` / `reconnected` 房间事件——这是 `rtc.Room` 内部的信号/媒体重连，job 任务不重建，entrypoint 不重跑。 +- 只有房间真正关闭才发 `disconnected`；job_proc_lazy_main L288–293 将其置入 `_shutdown_fut`（reason=`room disconnected`）→ job 结束。此时房间已不存在，前端收到 `dropped`（livekit.ts L75），整个 `Live` 连同 `live.seq` 一起终结。 + +结论：**同一 job 内的 worker 重连，`SubtitleBus` 实例保留，seq 连续递增**；房间没了 = job 没了 = 前端这次通话也没了，不存在跨 job 的旧高水位。 + +## 3. 本产品没有「同房间第二个 job」的路径 + +- 后端唯一派发点：`src/cybergril/voice/session.py:396` `open_session` 内 `livekit_wire.create_dispatch(...)`，房间名 `cg-voice-` + 10 位随机（L394），每次 POST /api/voice/session 都是**新房间新派发**；`create_dispatch` 在 src/ 下仅此一处（grep 证据在库）。没有任何对既有房间补发/重发 job 的代码路径。 +- 前端 join 的就是该房间（grant.room），一个控制器会话 ↔ 一个房间 ↔ 一个显式派发 ↔ 一个 job ↔ 一个 `SubtitleBus`，五者一一对应。 +- 服务器侧 job 重投（resend）只发生在 job 从未启动时（无可供 worker 接单/启动超时）：那时第一个 entrypoint 从未运行、从未发过任何帧，前端 `live.seq` 仍为 0，重投 job 从 1 计与前端水位天然一致。SDK 源码中不存在「已运行的 job 迁移到另一进程」的机制（job 进程即 job 本体，崩溃即终止；livekit-agents 侧无 resend 代码，grep `resend` 仅命中无关模块）。 + +## 4. 真实残余缺口（非 seq，但同场景）：worker job 中途死亡 = 前端僵尸「listening」 + +job 死亡（worker 进程崩溃/被运维重启）时：agent participant 离开房间，但**房间仍在**（用户还在），前端收不到 `dropped`（它只挂 `Disconnected`，web/src/voice/livekit.ts L69–75 只处理 `ParticipantConnected`/`DataReceived`/`Disconnected`/`AudioPlaybackStatusChanged`），页面停在最后相位（多为 listening），再无任何帧。`RoomEvents` 无 agentLeft。这不破坏 seq 或旧帧防护（没有第二命名空间），但用户会面对一个没有她的「通话中」。有限修复建议（归原作者，前端）：`livekit.ts` 增加 `ParticipantDisconnected`（agent 身份）→ `RoomEvents.agentLeft` → `VoiceController` 以诚实话术收场（与 dropped 同路径、不删她的话轮）。后端无需改动。 + +## 5. 可执行复核方式 + +- 证据 A（本轮）：`scripts/gen_hel879_unified_frames.py` 在 .12 真 SDK 上驱动实际 worker 回调生成全部 v2 帧（见 frames/），各场景 seq 严格递增、每 speech 恰一次 `turn_start`。 +- 证据 B（源码引用，可重复）：本文第 1/2/3 节的 file:line 均可在 .12 venv 直接核对；`.12` 只读 SSH stdin。 +- 证据 C（前端侧）：`web/tests/hel879_frames.test.ts` 用实际 worker 帧验证 replay/迟到/乱序帧全部被 seq+speech 双门拦下；「新通话 = 新房间 = 新序列」在 drop+重开用例中验证（旧房间帧不再影响新 `Live`）。 + +## 6. 结论 + +**现网代码内，seq 命名空间 = job = 房间 = 控制器会话，一一对应；不存在「同房间新 job 序号归零」的产品路径或 SDK 机制。** 前端以控制器会话记高水位是安全的，且对任何假设的第二命名空间表现为 fail-closed（旧水位丢弃全部低 seq 新帧，不会把旧帧当新帧收）。唯一真实缺口是 §4 的 agent 离场僵尸界面（🟡 建议项，归原作者）。 diff --git a/docs/rework/hel879/unified-review/summary.md b/docs/rework/hel879/unified-review/summary.md new file mode 100644 index 0000000..fdc50db --- /dev/null +++ b/docs/rework/hel879/unified-review/summary.md @@ -0,0 +1,44 @@ +# HEL-879 统一整合审核记录（2026-10-27） + +整合对象：`hel-879-assembly` @ fc905fe（后台时机收口）+ `origin/hel-898-preserved` @ fb7ea84（总管逐字节保全的 HEL-898 前端接线，作者代码 46ac0a7）。本审核不开新串、不重审已过内核（预算/备份/恢复/绑定/三处时机/总管 898 六项检查均沿用其通过记录）。 + +## 统一 head + +`hel-879-assembly` 快进至 **fb7ea84 + 本轮审核提交**（审核物：帧生成脚本、fixtures、前端用例、浏览器回放 harness、SDK 取证、R5 测试包、handoff 追加）。与 fc905fe 的差异 = HEL-898 的 8 文件（6 web/tests+docs/handoff-898+2 web/src：ChatPanel.tsx/session.ts，总管已核逐字节=作者附件）+ 本轮审核新增文件；`web/src` 除作者两文件外零改动，`src/` 零改动，PR#11 与 hel-870-assembly 未动。 + +## 焦点1 · 实际 worker 帧 → 当前前端（功能） + +**帧来源**：`scripts/gen_hel879_unified_frames.py` 在 .12 真实 SDK（livekit-agents 1.8.8 / livekit-api 1.2.1，SSH 只读 stdin，零写入零网络）驱动**实际 worker 接线**（`make_voice_llm` 归属绑定 + `bind_voice_session` 真回调 + 真 SpeechHandle 生命周期 + `speak_proactive_row`）生成五套 v2 帧（frames/ + web/tests/fixtures/hel879/）。唯一替身=HTTP 客户端（离线注入 /api/chat SSE）与播放时钟（`_mark_done`/`interrupt` 定时器）。 + +场景与结果（vitest `web/tests/hel879_frames.test.ts` 9/9，实际帧逐帧喂 `VoiceController`）： + +| 场景 | 帧序列（实测） | 前端行为（断言） | +| --- | --- | --- | +| normal | interim→user_final→turn_start→3×interim→**final→end**（receipt ack ✓） | 增量替换生长；final=存稿原文；落库回读接替：`voice-` 键→存档行 id（同 turn），mergeTurns 恰加一次 | +| interrupt | cut 轮 **无 final 只有 cancel(interrupted)**；下一轮干净 final+ack | 被切行立即清；迟到帧（新 seq、死 speech）不回显；下一轮独立落库一次 | +| φ | interim(φ)→error(φ 原句)→cancel(error_phi)，无 ack | spoken 槽可闻不入档；her 即清 | +| no_receipt | **cancel(no-receipt)，无 final**（fc905fe 门实测生效） | 无任何「她的回复」残留 | +| proactive | start **播放中**（sidecar `start_while_playing=true`）→end；打断变体→cancel(interrupted) | 她的位置整句显示、绑定存档行 id；打断即清；页面零写请求 | + +打断/文字接管/掉线重开（vitest 全过）：user_final 立即关她当前 speech；interim 不抢清（由服务端 cancel 定夺）；`hush()` 文字接管即清行+静音；`dropped` 整通话收场清残影；重开=新房新序列，旧房间帧无效。舞台身份与键接替（voiceLine/mergeTurns）用实际帧 turn_id 验证。 + +**界面层**：本机缺浏览器 GUI 系统库（libatk/libgbm 等十项，chromium/chrome-headless-shell 均无法启动），零安装授权下**未独立复跑浏览器**。`web/tests/hel879_unified_shots.mjs`（自持 mock+CDP，回放实际 worker 帧、双主题双尺寸）已入库，供有 Edge/Chromium 环境一键回放。界面证据仍=作者 61/61 假 LiveKit 浏览器与两主题截图（总管保全，未复跑，不称独立通过）。**录屏：本轮无新连续帧/视频产出，如实列未交。** + +## 焦点2 · seq 命名空间与 job/房间/控制器代次 + +结论（详见 `seq-job-analysis.md`，全部 file:line 指向 .12 真 SDK 源码）：**job=房间=派发=控制器会话一一对应，无「同房间新 job 序号归零」路径**。一 job 恰一次 entrypoint（job_proc_lazy_main L241-246/L318-323）；RTC 瞬断走 reconnecting/reconnected 不重跑（rtc/room.py L85-86/L1130-1133），房间真断=job 结束=前端 Live 同终；后端唯一派发点 session.py:396 每次新房（10 位随机名）；服务器重投仅限从未启动的 job（从未发帧，水位天然一致）。若假设性第二命名空间出现，前端表现为 fail-closed（丢帧而非收旧帧）。**残余缺口（🟡 归原作者前端）**：worker job 中途死亡（进程崩溃/重启）时 agent participant 离场但房间仍在，livekit.ts 无 agentLeft → 页面僵尸「listening」。有限修复：`ParticipantDisconnected`（agent 身份）→ 诚实话术收场；后端无需改。 + +## 焦点3 · closed 64 上限与长会话/清理边界 + +- **DOCUMENTED GAP（🟡 归原作者，非现网可触发）**：64+ 后续关闭把退休 speech 挤出窗口后，**重复高 seq turn_start 可重进**（vitest 用例以对抗帧实证：重进→重放终稿可再次立为她的行）。现 worker 无法产生该帧（SDK speech_id 每话唯一、`interim.begin` 每流一次、无重发路径），触发需第二/失常来源——与焦点2结论一致。防御性修复建议：`closed` 改无界 `Set`（每通话 speech 数有 TTL 上界，内存可忽略）或对 turn_start 增加按 job 内序号窗的约束。 +- 新用户 interim→cancel/终稿清理边界（vitest 全过）：interim 只更新 heard 不清她的行（打断由服务端定夺）；user_final 立即关闭当前 speech 并清行；cancel/final/turn_end 的即时清理由实际帧验证。 +- 舞台回读/身份与键接替：实际帧 turn_id 驱动（焦点1 表），非单测数字。 + +## 回归与需求检查 + +- pytest **352/352**（py3.11 venv）；vitest **108/108**（作者 99+本轮 9）；typecheck/build 通过；lint 0 错 1 既有 fullButton 警告；ruff(src/tests/tools) 0 错。 +- 需求检查 4/4：origin/main 与 fc905fe 各带/不带更新后的 handoff-HEL-879.json（本目录 requirements-*.log）。v1.11/60 项无需求变更；总管 898 四项检查沿用。 + +## 边界 + +零真实 LLM/嵌入/ASR/TTS/麦克风、零下载/安装/部署/合并；.12 仅 SSH 只读取证（含一次 nvidia-smi/端口探活供 R5 包）；真实 cap=0；不选音色、不改绑定、不碰旧服务/档案/冻结账。R5 真机包材料（`r5-test-package.md`）仅备齐待批，未执行。完整 CG-011、R6/R7 继续承接。 diff --git a/scripts/gen_hel879_unified_frames.py b/scripts/gen_hel879_unified_frames.py new file mode 100644 index 0000000..a653b31 --- /dev/null +++ b/scripts/gen_hel879_unified_frames.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Generate cg.subtitle v2 frames from the ACTUAL fc905fe worker on the .12 real SDK. + +Unified offline review (HEL-879 + HEL-898 integration). Same discipline as +scripts/gen_hel879_timing_probe.py: the repo voice-module sources are embedded +into a self-contained script that runs on .12's agent venv (livekit-agents +1.8.8) via SSH stdin — zero writes on .12, zero network, no service touched. +The only stand-ins are the HTTP client (the unified /api/chat SSE turn is fed +in-process) and the playout clock (SpeechHandle._mark_done / .interrupt on a +timer); every frame is produced by the real worker wiring: +make_voice_llm + bind_voice_session callbacks on a real AgentSession, +real SpeechHandle lifecycle, real SubtitleBus envelopes. + +Scenarios (each -> web/tests/fixtures/hel879/worker-*.jsonl + sidecar json): + normal interim/user_final -> turn_start -> interims -> final -> end + ack + interrupt barge-in: cancel without final, then a clean next turn + phi chat.error phi line: error frame + cancel, no final + no_receipt incomplete/no-receipt stream: cancel only (fc905fe timing gate) + proactive proactive_start DURING playout -> end; and an interrupted variant +""" +from __future__ import annotations + +import base64 +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MODULES = ["chat_api_client", "receipt", "seam", "llm_chat_api", + "stt_funasr", "tts_cosyvoice", "worker"] + +TEMPLATE = r''' +import asyncio, base64, json, sys, types + +SOURCES = json.loads(base64.b64decode("__B64__")) + +pkg = types.ModuleType("cybergril"); pkg.__path__ = [] +voice = types.ModuleType("cybergril.voice"); voice.__path__ = [] +sys.modules["cybergril"] = pkg +sys.modules["cybergril.voice"] = voice +for name, src in SOURCES.items(): + mod = types.ModuleType(f"cybergril.voice.{name}") + mod.__package__ = "cybergril.voice" + sys.modules[f"cybergril.voice.{name}"] = mod + exec(compile(src, f"", "exec"), mod.__dict__) + setattr(voice, name, mod) + +from livekit.agents import AgentSession +from livekit.agents.llm import ChatContext, ChatMessage +from livekit.agents.voice import agent_activity +from livekit.agents.voice.events import SpeechCreatedEvent, UserInputTranscribedEvent +from livekit.agents.voice.speech_handle import SpeechHandle + +w = voice.worker +out = {"sdk": {}, "scenarios": {}} + + +class FakeClient: + """In-process /api/chat: the worker's only model/LLM path, fed offline.""" + + def __init__(self): + self.epoch = 1 + self.acks = [] + + async def current_epoch(self): + return self.epoch + + async def send_turn(self, text, *, turn_id=None, epoch=None, on_delta=None): + reply = text and dict(TEXTS)[text] or None + if reply is None: + raise RuntimeError("unknown scripted utterance") + for part in reply["deltas"]: + if on_delta: + on_delta(part) + receipt = None + if reply.get("receipt"): + receipt = receipt_for(turn_id) + t = voice.chat_api_client.ChatApiTurn( + turn_id=turn_id, epoch=self.epoch, text=reply["text"], + receipt=receipt, error_phi=reply.get("error_phi"), + completed=reply.get("completed", False)) + return t + + async def acknowledge(self, receipt): + self.acks.append(receipt) + return True + + +TEXTS = {} + + +def script_turn(user_text, deltas, *, text="", receipt=False, error_phi=None, completed=False): + TEXTS[user_text] = {"deltas": deltas, "text": text or "".join(deltas), + "receipt": receipt, "error_phi": error_phi, + "completed": completed} + + +def receipt_for(turn_id, message_id=101): + return {"message_id": message_id, "turn_id": turn_id, "epoch": 1, + "token": "r" * 48, "text_hash": "h" * 64} + + +def rig(): + frames = [] + session = AgentSession() + ledger = voice.receipt.TurnLedger() + bus = voice.seam.SubtitleBus(frames.append) + interim = voice.seam.AssistantInterimPublisher(bus) + client = FakeClient() + llm = w.make_voice_llm("http://offline", ledger=ledger, interim=interim, client=client) + receipts = voice.receipt.ReceiptSender(client) + w.bind_voice_session(session, llm=llm, ledger=ledger, interim=interim, + bus=bus, stt=types.SimpleNamespace(), receipts=receipts) + return frames, session, ledger, interim, client, llm + + +async def drive_assistant_turn(session, llm, user_text, interrupt_at_delta=None): + """One real speech: SpeechCreatedEvent -> ContextVar attribution -> real + CyberGrillLLM.chat -> scripted /api/chat stream -> playout done.""" + h = SpeechHandle.create() + session.emit("speech_created", SpeechCreatedEvent( + user_initiated=True, source="generate_reply", speech_handle=h, created_at=1.0)) + token = agent_activity._SpeechHandleContextVar.set(h) + stream = llm.chat(chat_ctx=ChatContext([ChatMessage(role="user", content=[user_text])])) + agent_activity._SpeechHandleContextVar.reset(token) + # consume the chunks exactly as the SDK pipeline would; the deltas have + # already fed the subtitles through the adapter's own callback. + async for _ in stream: + pass + if interrupt_at_delta is not None: + h.interrupt() + h._mark_done() + for _ in range(20): + await asyncio.sleep(0) + await asyncio.sleep(0.05) + return h, stream + + +def user_says(session, text, final): + session.emit("user_input_transcribed", UserInputTranscribedEvent( + transcript=text, is_final=final)) + + +# --- scenario: normal turn ---------------------------------------------------- +async def normal(): + script_turn("你在听吗", ["在听呢，", "你刚才说到一半的那件事", "我还在想。"], + text="在听呢，你刚才说到一半的那件事我还在想。", receipt=True, completed=True) + frames, session, ledger, interim, client, llm = rig() + user_says(session, "你在听", False) + user_says(session, "你在听吗", True) + h, stream = await drive_assistant_turn(session, llm, "你在听吗") + out["scenarios"]["normal"] = { + "frames": frames, "turn_id": stream.turn_id, + "acks": [a["turn_id"] for a in client.acks], + "kinds": [f["kind"] for f in frames]} + + +# --- scenario: barge-in interrupt + clean next turn --------------------------- +async def interrupt(): + script_turn("讲讲你下午做了什么", ["我下午去了河边，", "看到一群白鹭从水面上飞过去了，"], + completed=False) + script_turn("那后来呢", ["后来我在桥上站了一会儿，", "风把水面吹皱了。"], receipt=True, completed=True) + frames, session, ledger, interim, client, llm = rig() + user_says(session, "讲讲你下午做了什么", True) + h, stream = await drive_assistant_turn(session, llm, "讲讲你下午做了什么", interrupt_at_delta=True) + user_says(session, "那后来呢", True) + h2, stream2 = await drive_assistant_turn(session, llm, "那后来呢") + out["scenarios"]["interrupt"] = { + "frames": frames, + "turn_ids": {"cut": stream.turn_id, "next": stream2.turn_id}, + "acks": [a["turn_id"] for a in client.acks], + "kinds": [f["kind"] for f in frames]} + + +# --- scenario: chat.error phi line ------------------------------------------- +async def phi(): + script_turn("说说那件事吧", ["（她顿了一下）", "这会儿我有点说不出来话……"], + text="", error_phi="（她顿了一下）这会儿我有点说不出来话……", completed=False) + frames, session, ledger, interim, client, llm = rig() + user_says(session, "说说那件事吧", True) + h, stream = await drive_assistant_turn(session, llm, "说说那件事吧") + out["scenarios"]["phi"] = { + "frames": frames, "turn_id": stream.turn_id, + "acks": [a["turn_id"] for a in client.acks], + "kinds": [f["kind"] for f in frames]} + + +# --- scenario: complete text but no legal receipt (fc905fe gate) -------------- +async def no_receipt(): + script_turn("再讲讲那座桥", ["那座桥是石头砌的，", "下雨的时候水声很好听。"], + text="那座桥是石头砌的，下雨的时候水声很好听。", completed=True) + frames, session, ledger, interim, client, llm = rig() + user_says(session, "再讲讲那座桥", True) + h, stream = await drive_assistant_turn(session, llm, "再讲讲那座桥") + out["scenarios"]["no_receipt"] = { + "frames": frames, "turn_id": stream.turn_id, + "acks": [a["turn_id"] for a in client.acks], + "kinds": [f["kind"] for f in frames]} + + +# --- scenario: proactive reach-out rows (real SpeechHandle playout) ----------- +async def proactive(): + frames = [] + bus = voice.seam.SubtitleBus(frames.append) + done_flags = {} + + class Say: + def __init__(self, interrupt=False): + self.interrupt = interrupt + + def say(self, *args, **kwargs): # real SDK: sync, returns SpeechHandle + h = SpeechHandle.create() + loop = asyncio.get_running_loop() + if self.interrupt: + loop.call_later(0.04, h.interrupt) + loop.call_later(0.08, h._mark_done) + else: + loop.call_later(0.09, h._mark_done) + done_flags[h.id] = h + return h + + said = Say() + task = asyncio.create_task(w.speak_proactive_row( + said, bus, {"id": 301, "text": "刚才路过花店，想起你说过喜欢白色的洋桔梗。"})) + await asyncio.sleep(0.02) + start_frames_early = [f for f in frames if f["kind"] == "proactive_start"] + handle_ids = list(done_flags) + start_while_playing = bool(start_frames_early) and not any( + done_flags[i].done() for i in handle_ids) + await task + await asyncio.sleep(0.02) + + task2 = asyncio.create_task(w.speak_proactive_row( + Say(interrupt=True), bus, {"id": 302, "text": "晚上风大，窗台的那盆薄荷记得搬进来。"})) + await task2 + await asyncio.sleep(0.02) + out["scenarios"]["proactive"] = { + "frames": frames, + "start_while_playing": start_while_playing, + "kinds": [f["kind"] for f in frames], + "message_ids": sorted({f["message_id"] for f in frames if "message_id" in f})} + + +async def main(): + import livekit.agents, livekit.api, livekit + out["sdk"] = {"livekit_agents": livekit.agents.__version__, + "livekit_api": livekit.api.__version__} + for fn in (normal, interrupt, phi, no_receipt, proactive): + await fn() + print(json.dumps(out, ensure_ascii=False)) + +asyncio.run(main()) +''' + + +def main() -> int: + sources = {} + for name in MODULES: + path = ROOT / "src/cybergril/voice" / f"{name}.py" + sources[name] = path.read_text(encoding="utf-8") + payload = base64.b64encode(json.dumps(sources).encode()).decode() + script = TEMPLATE.replace("__B64__", payload) + out = ROOT / "scripts/_hel879_unified_frames_payload.py" + out.write_text(script, encoding="utf-8") + print(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/web/tests/fixtures/hel879/adversarial-gates.jsonl b/web/tests/fixtures/hel879/adversarial-gates.jsonl new file mode 100644 index 0000000..eb286c9 --- /dev/null +++ b/web/tests/fixtures/hel879/adversarial-gates.jsonl @@ -0,0 +1,149 @@ +{"v": 2, "seq": 1, "kind": "interim", "role": "user", "text": "上次说的那只猫"} +{"v": 2, "seq": 2, "kind": "user_final", "role": "user", "text": "上次说的那只猫你见到了吗"} +{"v": 2, "seq": 3, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_A"} +{"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "见到了，它还在桥下", "speech_id": "speech_A"} +{"v": 2, "seq": 5, "kind": "assistant_interim", "role": "assistant", "text": "见到了，它还在桥下晒太阳。", "speech_id": "speech_A"} +{"v": 2, "seq": 6, "kind": "assistant_final", "role": "assistant", "text": "见到了，它还在桥下晒太阳。", "speech_id": "speech_A", "turn_id": "gate-turn-1"} +{"v": 2, "seq": 7, "kind": "turn_end", "role": "assistant", "text": "", "speech_id": "speech_A", "turn_id": "gate-turn-1"} +{"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "见到了，它还在桥下", "speech_id": "speech_A"} +{"v": 2, "seq": 5, "kind": "assistant_interim", "role": "assistant", "text": "见到了，它还在桥下晒太阳。", "speech_id": "speech_A"} +{"v": 2, "seq": 8, "kind": "assistant_final", "role": "assistant", "text": "迟到的旧稿", "speech_id": "speech_A", "turn_id": "late"} +{"v": 2, "seq": 9, "kind": "interim", "role": "user", "text": "再说点别的"} +{"v": 2, "seq": 8, "kind": "interim", "role": "user", "text": "乱序旧帧"} +{"v": 2, "seq": 10, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_A"} +{"v": 2, "seq": 11, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s00"} +{"v": 2, "seq": 12, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s00", "reason": "interrupted"} +{"v": 2, "seq": 13, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s01"} +{"v": 2, "seq": 14, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s01", "reason": "interrupted"} +{"v": 2, "seq": 15, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s02"} +{"v": 2, "seq": 16, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s02", "reason": "interrupted"} +{"v": 2, "seq": 17, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s03"} +{"v": 2, "seq": 18, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s03", "reason": "interrupted"} +{"v": 2, "seq": 19, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s04"} +{"v": 2, "seq": 20, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s04", "reason": "interrupted"} +{"v": 2, "seq": 21, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s05"} +{"v": 2, "seq": 22, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s05", "reason": "interrupted"} +{"v": 2, "seq": 23, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s06"} +{"v": 2, "seq": 24, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s06", "reason": "interrupted"} +{"v": 2, "seq": 25, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s07"} +{"v": 2, "seq": 26, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s07", "reason": "interrupted"} +{"v": 2, "seq": 27, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s08"} +{"v": 2, "seq": 28, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s08", "reason": "interrupted"} +{"v": 2, "seq": 29, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s09"} +{"v": 2, "seq": 30, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s09", "reason": "interrupted"} +{"v": 2, "seq": 31, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s10"} +{"v": 2, "seq": 32, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s10", "reason": "interrupted"} +{"v": 2, "seq": 33, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s11"} +{"v": 2, "seq": 34, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s11", "reason": "interrupted"} +{"v": 2, "seq": 35, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s12"} +{"v": 2, "seq": 36, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s12", "reason": "interrupted"} +{"v": 2, "seq": 37, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s13"} +{"v": 2, "seq": 38, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s13", "reason": "interrupted"} +{"v": 2, "seq": 39, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s14"} +{"v": 2, "seq": 40, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s14", "reason": "interrupted"} +{"v": 2, "seq": 41, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s15"} +{"v": 2, "seq": 42, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s15", "reason": "interrupted"} +{"v": 2, "seq": 43, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s16"} +{"v": 2, "seq": 44, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s16", "reason": "interrupted"} +{"v": 2, "seq": 45, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s17"} +{"v": 2, "seq": 46, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s17", "reason": "interrupted"} +{"v": 2, "seq": 47, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s18"} +{"v": 2, "seq": 48, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s18", "reason": "interrupted"} +{"v": 2, "seq": 49, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s19"} +{"v": 2, "seq": 50, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s19", "reason": "interrupted"} +{"v": 2, "seq": 51, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s20"} +{"v": 2, "seq": 52, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s20", "reason": "interrupted"} +{"v": 2, "seq": 53, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s21"} +{"v": 2, "seq": 54, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s21", "reason": "interrupted"} +{"v": 2, "seq": 55, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s22"} +{"v": 2, "seq": 56, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s22", "reason": "interrupted"} +{"v": 2, "seq": 57, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s23"} +{"v": 2, "seq": 58, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s23", "reason": "interrupted"} +{"v": 2, "seq": 59, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s24"} +{"v": 2, "seq": 60, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s24", "reason": "interrupted"} +{"v": 2, "seq": 61, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s25"} +{"v": 2, "seq": 62, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s25", "reason": "interrupted"} +{"v": 2, "seq": 63, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s26"} +{"v": 2, "seq": 64, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s26", "reason": "interrupted"} +{"v": 2, "seq": 65, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s27"} +{"v": 2, "seq": 66, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s27", "reason": "interrupted"} +{"v": 2, "seq": 67, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s28"} +{"v": 2, "seq": 68, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s28", "reason": "interrupted"} +{"v": 2, "seq": 69, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s29"} +{"v": 2, "seq": 70, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s29", "reason": "interrupted"} +{"v": 2, "seq": 71, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s30"} +{"v": 2, "seq": 72, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s30", "reason": "interrupted"} +{"v": 2, "seq": 73, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s31"} +{"v": 2, "seq": 74, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s31", "reason": "interrupted"} +{"v": 2, "seq": 75, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s32"} +{"v": 2, "seq": 76, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s32", "reason": "interrupted"} +{"v": 2, "seq": 77, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s33"} +{"v": 2, "seq": 78, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s33", "reason": "interrupted"} +{"v": 2, "seq": 79, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s34"} +{"v": 2, "seq": 80, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s34", "reason": "interrupted"} +{"v": 2, "seq": 81, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s35"} +{"v": 2, "seq": 82, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s35", "reason": "interrupted"} +{"v": 2, "seq": 83, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s36"} +{"v": 2, "seq": 84, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s36", "reason": "interrupted"} +{"v": 2, "seq": 85, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s37"} +{"v": 2, "seq": 86, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s37", "reason": "interrupted"} +{"v": 2, "seq": 87, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s38"} +{"v": 2, "seq": 88, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s38", "reason": "interrupted"} +{"v": 2, "seq": 89, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s39"} +{"v": 2, "seq": 90, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s39", "reason": "interrupted"} +{"v": 2, "seq": 91, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s40"} +{"v": 2, "seq": 92, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s40", "reason": "interrupted"} +{"v": 2, "seq": 93, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s41"} +{"v": 2, "seq": 94, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s41", "reason": "interrupted"} +{"v": 2, "seq": 95, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s42"} +{"v": 2, "seq": 96, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s42", "reason": "interrupted"} +{"v": 2, "seq": 97, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s43"} +{"v": 2, "seq": 98, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s43", "reason": "interrupted"} +{"v": 2, "seq": 99, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s44"} +{"v": 2, "seq": 100, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s44", "reason": "interrupted"} +{"v": 2, "seq": 101, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s45"} +{"v": 2, "seq": 102, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s45", "reason": "interrupted"} +{"v": 2, "seq": 103, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s46"} +{"v": 2, "seq": 104, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s46", "reason": "interrupted"} +{"v": 2, "seq": 105, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s47"} +{"v": 2, "seq": 106, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s47", "reason": "interrupted"} +{"v": 2, "seq": 107, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s48"} +{"v": 2, "seq": 108, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s48", "reason": "interrupted"} +{"v": 2, "seq": 109, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s49"} +{"v": 2, "seq": 110, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s49", "reason": "interrupted"} +{"v": 2, "seq": 111, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s50"} +{"v": 2, "seq": 112, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s50", "reason": "interrupted"} +{"v": 2, "seq": 113, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s51"} +{"v": 2, "seq": 114, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s51", "reason": "interrupted"} +{"v": 2, "seq": 115, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s52"} +{"v": 2, "seq": 116, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s52", "reason": "interrupted"} +{"v": 2, "seq": 117, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s53"} +{"v": 2, "seq": 118, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s53", "reason": "interrupted"} +{"v": 2, "seq": 119, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s54"} +{"v": 2, "seq": 120, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s54", "reason": "interrupted"} +{"v": 2, "seq": 121, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s55"} +{"v": 2, "seq": 122, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s55", "reason": "interrupted"} +{"v": 2, "seq": 123, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s56"} +{"v": 2, "seq": 124, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s56", "reason": "interrupted"} +{"v": 2, "seq": 125, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s57"} +{"v": 2, "seq": 126, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s57", "reason": "interrupted"} +{"v": 2, "seq": 127, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s58"} +{"v": 2, "seq": 128, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s58", "reason": "interrupted"} +{"v": 2, "seq": 129, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s59"} +{"v": 2, "seq": 130, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s59", "reason": "interrupted"} +{"v": 2, "seq": 131, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s60"} +{"v": 2, "seq": 132, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s60", "reason": "interrupted"} +{"v": 2, "seq": 133, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s61"} +{"v": 2, "seq": 134, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s61", "reason": "interrupted"} +{"v": 2, "seq": 135, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s62"} +{"v": 2, "seq": 136, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s62", "reason": "interrupted"} +{"v": 2, "seq": 137, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s63"} +{"v": 2, "seq": 138, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s63", "reason": "interrupted"} +{"v": 2, "seq": 139, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s64"} +{"v": 2, "seq": 140, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s64", "reason": "interrupted"} +{"v": 2, "seq": 141, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_s65"} +{"v": 2, "seq": 142, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_s65", "reason": "interrupted"} +{"v": 2, "seq": 143, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_A"} +{"v": 2, "seq": 144, "kind": "assistant_interim", "role": "assistant", "text": "被挤出后的重放增量", "speech_id": "speech_A"} +{"v": 2, "seq": 145, "kind": "assistant_final", "role": "assistant", "text": "被挤出后的重放终稿", "speech_id": "speech_A", "turn_id": "gate-turn-1"} +{"v": 2, "seq": 146, "kind": "turn_end", "role": "assistant", "text": "", "speech_id": "speech_A", "turn_id": "gate-turn-1"} diff --git a/web/tests/fixtures/hel879/sidecar.json b/web/tests/fixtures/hel879/sidecar.json new file mode 100644 index 0000000..3867795 --- /dev/null +++ b/web/tests/fixtures/hel879/sidecar.json @@ -0,0 +1,85 @@ +{ + "sdk": { + "livekit_agents": "1.8.8", + "livekit_api": "1.2.1" + }, + "source": "actual fc905fe+fb7ea84 worker driven on .12 real SDK (livekit-agents 1.8.8, livekit-api 1.2.1) via SSH stdin, zero network/writes", + "scenarios": { + "normal": { + "kinds": [ + "interim", + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "assistant_interim", + "assistant_final", + "turn_end" + ], + "frames": 8, + "acks": [ + "382c41d9-0d72-433e-b370-f838fa5d0bd0" + ], + "turn_id": "382c41d9-0d72-433e-b370-f838fa5d0bd0" + }, + "interrupt": { + "kinds": [ + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "turn_cancel", + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "assistant_final", + "turn_end" + ], + "frames": 11, + "acks": [ + "edbcb837-f19f-47e7-8d5c-057010da43eb" + ], + "turn_ids": { + "cut": "09d05c0f-3058-4ff8-a56e-4e94bd17d7b7", + "next": "edbcb837-f19f-47e7-8d5c-057010da43eb" + } + }, + "phi": { + "kinds": [ + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "assistant_interim", + "error", + "turn_cancel" + ], + "frames": 7, + "acks": [], + "turn_id": "31e70e52-1471-4e50-ac49-cce89979a9bf" + }, + "no_receipt": { + "kinds": [ + "user_final", + "turn_start", + "assistant_interim", + "assistant_interim", + "turn_cancel" + ], + "frames": 5, + "acks": [], + "turn_id": "36d160e3-e365-46ff-ad57-c2878c00720f" + }, + "proactive": { + "kinds": [ + "proactive_start", + "proactive_end", + "proactive_start", + "proactive_cancel" + ], + "frames": 4, + "start_while_playing": true + } + } +} \ No newline at end of file diff --git a/web/tests/fixtures/hel879/worker-interrupt.jsonl b/web/tests/fixtures/hel879/worker-interrupt.jsonl new file mode 100644 index 0000000..a35f59f --- /dev/null +++ b/web/tests/fixtures/hel879/worker-interrupt.jsonl @@ -0,0 +1,11 @@ +{"v": 2, "seq": 1, "kind": "user_final", "role": "user", "text": "讲讲你下午做了什么"} +{"v": 2, "seq": 2, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_dd1c4c382512"} +{"v": 2, "seq": 3, "kind": "assistant_interim", "role": "assistant", "text": "我下午去了河边，", "speech_id": "speech_dd1c4c382512"} +{"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "我下午去了河边，看到一群白鹭从水面上飞过去了，", "speech_id": "speech_dd1c4c382512"} +{"v": 2, "seq": 5, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_dd1c4c382512", "reason": "interrupted"} +{"v": 2, "seq": 6, "kind": "user_final", "role": "user", "text": "那后来呢"} +{"v": 2, "seq": 7, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_55ef57a410cf"} +{"v": 2, "seq": 8, "kind": "assistant_interim", "role": "assistant", "text": "后来我在桥上站了一会儿，", "speech_id": "speech_55ef57a410cf"} +{"v": 2, "seq": 9, "kind": "assistant_interim", "role": "assistant", "text": "后来我在桥上站了一会儿，风把水面吹皱了。", "speech_id": "speech_55ef57a410cf"} +{"v": 2, "seq": 10, "kind": "assistant_final", "role": "assistant", "text": "后来我在桥上站了一会儿，风把水面吹皱了。", "speech_id": "speech_55ef57a410cf", "turn_id": "edbcb837-f19f-47e7-8d5c-057010da43eb"} +{"v": 2, "seq": 11, "kind": "turn_end", "role": "assistant", "text": "", "speech_id": "speech_55ef57a410cf", "turn_id": "edbcb837-f19f-47e7-8d5c-057010da43eb"} diff --git a/web/tests/fixtures/hel879/worker-no_receipt.jsonl b/web/tests/fixtures/hel879/worker-no_receipt.jsonl new file mode 100644 index 0000000..34bc482 --- /dev/null +++ b/web/tests/fixtures/hel879/worker-no_receipt.jsonl @@ -0,0 +1,5 @@ +{"v": 2, "seq": 1, "kind": "user_final", "role": "user", "text": "再讲讲那座桥"} +{"v": 2, "seq": 2, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_20d31bfb084a"} +{"v": 2, "seq": 3, "kind": "assistant_interim", "role": "assistant", "text": "那座桥是石头砌的，", "speech_id": "speech_20d31bfb084a"} +{"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "那座桥是石头砌的，下雨的时候水声很好听。", "speech_id": "speech_20d31bfb084a"} +{"v": 2, "seq": 5, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_20d31bfb084a", "reason": "no-receipt"} diff --git a/web/tests/fixtures/hel879/worker-normal.jsonl b/web/tests/fixtures/hel879/worker-normal.jsonl new file mode 100644 index 0000000..59c0c82 --- /dev/null +++ b/web/tests/fixtures/hel879/worker-normal.jsonl @@ -0,0 +1,8 @@ +{"v": 2, "seq": 1, "kind": "interim", "role": "user", "text": "你在听"} +{"v": 2, "seq": 2, "kind": "user_final", "role": "user", "text": "你在听吗"} +{"v": 2, "seq": 3, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_17d892eb3dc3"} +{"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "在听呢，", "speech_id": "speech_17d892eb3dc3"} +{"v": 2, "seq": 5, "kind": "assistant_interim", "role": "assistant", "text": "在听呢，你刚才说到一半的那件事", "speech_id": "speech_17d892eb3dc3"} +{"v": 2, "seq": 6, "kind": "assistant_interim", "role": "assistant", "text": "在听呢，你刚才说到一半的那件事我还在想。", "speech_id": "speech_17d892eb3dc3"} +{"v": 2, "seq": 7, "kind": "assistant_final", "role": "assistant", "text": "在听呢，你刚才说到一半的那件事我还在想。", "speech_id": "speech_17d892eb3dc3", "turn_id": "382c41d9-0d72-433e-b370-f838fa5d0bd0"} +{"v": 2, "seq": 8, "kind": "turn_end", "role": "assistant", "text": "", "speech_id": "speech_17d892eb3dc3", "turn_id": "382c41d9-0d72-433e-b370-f838fa5d0bd0"} diff --git a/web/tests/fixtures/hel879/worker-phi.jsonl b/web/tests/fixtures/hel879/worker-phi.jsonl new file mode 100644 index 0000000..81c88f4 --- /dev/null +++ b/web/tests/fixtures/hel879/worker-phi.jsonl @@ -0,0 +1,7 @@ +{"v": 2, "seq": 1, "kind": "user_final", "role": "user", "text": "说说那件事吧"} +{"v": 2, "seq": 2, "kind": "turn_start", "role": "assistant", "text": "", "speech_id": "speech_7a5630ba8cc7"} +{"v": 2, "seq": 3, "kind": "assistant_interim", "role": "assistant", "text": "（她顿了一下）", "speech_id": "speech_7a5630ba8cc7"} +{"v": 2, "seq": 4, "kind": "assistant_interim", "role": "assistant", "text": "（她顿了一下）这会儿我有点说不出来话……", "speech_id": "speech_7a5630ba8cc7"} +{"v": 2, "seq": 5, "kind": "assistant_interim", "role": "assistant", "text": "（她顿了一下）这会儿我有点说不出来话……（她顿了一下）这会儿我有点说不出来话……", "speech_id": "speech_7a5630ba8cc7"} +{"v": 2, "seq": 6, "kind": "error", "role": "assistant", "text": "（她顿了一下）这会儿我有点说不出来话……", "speech_id": "speech_7a5630ba8cc7"} +{"v": 2, "seq": 7, "kind": "turn_cancel", "role": "assistant", "text": "", "speech_id": "speech_7a5630ba8cc7", "reason": "error_phi"} diff --git a/web/tests/fixtures/hel879/worker-proactive.jsonl b/web/tests/fixtures/hel879/worker-proactive.jsonl new file mode 100644 index 0000000..ad0c28b --- /dev/null +++ b/web/tests/fixtures/hel879/worker-proactive.jsonl @@ -0,0 +1,4 @@ +{"v": 2, "seq": 1, "kind": "proactive_start", "role": "assistant", "text": "刚才路过花店，想起你说过喜欢白色的洋桔梗。", "speech_id": "speech_172abc64e23c", "message_id": 301} +{"v": 2, "seq": 2, "kind": "proactive_end", "role": "assistant", "text": "", "speech_id": "speech_172abc64e23c", "message_id": 301} +{"v": 2, "seq": 3, "kind": "proactive_start", "role": "assistant", "text": "晚上风大，窗台的那盆薄荷记得搬进来。", "speech_id": "speech_835f928b989d", "message_id": 302} +{"v": 2, "seq": 4, "kind": "proactive_cancel", "role": "assistant", "text": "", "speech_id": "speech_835f928b989d", "message_id": 302, "reason": "interrupted"} diff --git a/web/tests/hel878_mock.mjs b/web/tests/hel878_mock.mjs index 1b4ad53..287dc75 100644 --- a/web/tests/hel878_mock.mjs +++ b/web/tests/hel878_mock.mjs @@ -34,15 +34,26 @@ createServer(async (req, res) => { if (p === '/__voice') { voice = { configured: url.searchParams.get('configured') !== '0', tts: url.searchParams.get('tts') !== '0' }; return send(res, 200, voice); } if (p.startsWith('/__refuse/')) { refuse = Number(p.slice(10)); return send(res, 200, { refuse }); } // What the worker's /api/chat call leaves in the record, one stage at a time (the browser never posts it). + // `&turn=` (HEL-879 unified review) lets a recorded worker fixture drive rows under its own + // turn id (the real worker's turn ids are uuids, not voice-N); without it behavior is unchanged. if (p === '/__voice-turn') { const stage = url.searchParams.get('stage'), text = url.searchParams.get('text') || ''; - if (stage === 'user') { voiceTurn++; add({ role: 'user', text, turn_id: `voice-${voiceTurn}`, status: 'pending' }); } - if (stage === 'reply') { add({ role: 'companion', text, turn_id: `voice-${voiceTurn}` }); rows.forEach(r => { if (r.turn_id === `voice-${voiceTurn}`) r.status = 'sent'; }); } - if (stage === 'cancel') rows.forEach(r => { if (r.turn_id === `voice-${voiceTurn}`) r.status = 'cancelled'; }); - return send(res, 200, { turn: voiceTurn }); + if (stage === 'user' && !url.searchParams.has('turn')) voiceTurn++; // default path: same counter semantics as before + const key = url.searchParams.get('turn') || `voice-${voiceTurn}`; + if (stage === 'user') add({ role: 'user', text, turn_id: key, status: 'pending' }); + if (stage === 'reply') { add({ role: 'companion', text, turn_id: key }); rows.forEach(r => { if (r.turn_id === key) r.status = 'sent'; }); } + if (stage === 'cancel') rows.forEach(r => { if (r.turn_id === key) r.status = 'cancelled'; }); + return send(res, 200, { turn: key }); } // A reach-out the life runtime stored (HEL-898): the worker may read it aloud; the page sees it through the inbox. - if (p === '/__proactive') { const id = add({ role: 'companion', text: url.searchParams.get('text') || '', turn_id: `pro-${rows.length + 1}`, proactive: true }); return send(res, 200, { id }); } + // `&id=` (HEL-879 unified review) pins the stored row id a fixture's proactive frames carry; without it + // behavior is unchanged. + if (p === '/__proactive') { + const text = url.searchParams.get('text') || '', want = Number(url.searchParams.get('id') || 0); + const id = want || add({ role: 'companion', text, turn_id: `pro-${rows.length + 1}`, proactive: true }); + if (want && !rows.some(r => r.id === want)) rows.push({ id: want, ts: T0 + want * 60e3 + 1, status: 'sent', lifeIds: [], role: 'companion', text, turn_id: `pro-${want}`, proactive: true }); + return send(res, 200, { id }); + } // A voice turn took the server's turn gate: the text stream in flight ends without a final; its row is cancelled. if (p === '/__cut') { const s = textStream; textStream = null; if (s) { rows.forEach(r => { if (r.turn_id === s.turn_id && r.role === 'user') r.status = 'cancelled'; }); s.cut = true; s.res.end(); } return send(res, 200, { cut: !!s }); } if (p === '/__real-sdk.js') return send(res, 200, await readFile(join(dist, 'vendor/livekit-client-2.22.4.umd.min.js')), 'text/javascript'); diff --git a/web/tests/hel879_frames.test.ts b/web/tests/hel879_frames.test.ts new file mode 100644 index 0000000..f76efdd --- /dev/null +++ b/web/tests/hel879_frames.test.ts @@ -0,0 +1,229 @@ +import { describe, expect, it, vi, afterEach } from 'vitest'; +import { mergeTurns, VoiceController, voiceLine, WORDS, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session'; +import { readFileSync } from 'node:fs'; + +// HEL-879 unified review: the frames here are the ACTUAL worker output, generated by +// scripts/gen_hel879_unified_frames.py driving the fc905fe worker wiring (make_voice_llm + +// bind_voice_session + speak_proactive_row) on the .12 real SDK (livekit-agents 1.8.8) — see +// docs/rework/hel879/unified-review/frames/. adversarial-gates.jsonl is hand-built (page-guard +// probes), never claimed as worker output. +type Frame = Record & { seq: number }; +const load = (name: string): Frame[] => readFileSync(new URL(`./fixtures/hel879/${name}.jsonl`, import.meta.url), 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l)); +const sidecar = JSON.parse(readFileSync(new URL('./fixtures/hel879/sidecar.json', import.meta.url), 'utf8')) as { scenarios: Record; acks?: string[]; start_while_playing?: boolean; kinds: string[] }> }; + +const gate = () => { let open!: () => void; const promise = new Promise(r => { open = r; }); return { promise, open }; }; +const flush = async (n = 8) => { for (let i = 0; i < n; i++) await Promise.resolve(); await new Promise(r => setTimeout(r, 0)); }; +class Mic implements MicTrack { stopped = 0; ended?: () => void; stop() { this.stopped++; } onEnded(cb: () => void) { this.ended = cb; } } +class Room implements VoiceRoom { + closed = 0; published: MicTrack[] = []; hushed: boolean[] = []; on!: RoomEvents; + async publish(mic: MicTrack) { this.published.push(mic); } + hush(on: boolean) { this.hushed.push(on); } + async resumeAudio() {} + async close() { this.closed++; } +} +class Transport { + rooms: Room[] = []; mics: Mic[] = []; + async capture() { const m = new Mic(); this.mics.push(m); return m; } + async join(_url: string, _token: string, on: RoomEvents) { const r = new Room(); r.on = on; this.rooms.push(r); return r; } +} +class Server { + open = new Set(); made = 0; calls: { method: string; url: string }[] = []; + fetch = vi.fn(async (url: string, init: RequestInit = {}) => { + const method = init.method || 'GET'; + this.calls.push({ method, url }); + if (url === '/api/voice/status') return new Response(JSON.stringify({ configured: true, tts_ok: true })); + if (url === '/api/voice/session' && method === 'POST') { const id = `s${++this.made}`; this.open.add(id); return new Response(JSON.stringify({ url: 'wss://lk.test', token: `t-${id}`, room: `cg-voice-${id}`, session_id: id, epoch: 3, expires_at: 0 }), { status: 201 }); } + const m = /^\/api\/voice\/session\/(.+)$/u.exec(url); + if (m && method === 'DELETE') return this.open.delete(decodeURIComponent(m[1])) ? new Response('{"ended":true}') : new Response('{"detail":"gone"}', { status: 404 }); + return new Response('{"detail":"Not Found"}', { status: 404 }); + }); +} +function rig() { + const server = new Server(), transport = new Transport(), views: VoiceView[] = [], record = vi.fn(), spoke = vi.fn(); + const c = new VoiceController({ transport: async () => transport, onView: v => views.push(v), onRecord: record, onSpoke: spoke, fetch: server.fetch as unknown as typeof fetch, secure: () => true, agentWaitMs: 1000 }); + const view = () => views.at(-1)!; + return { server, transport, views, view, record, spoke, c }; +} +async function connected() { + const r = rig(); await r.c.start(); r.transport.rooms[0].on.agentJoined(); return r; +} +const play = (room: Room, frames: Frame[]) => { for (const f of frames) room.on.subtitle(f); }; +// Rebase a fixture's seq namespace (a second recording replayed into the same call must +// continue past every seq the page has already applied). +const rebase = (frames: Frame[], by: number): Frame[] => frames.map(f => ({ ...f, seq: f.seq + by })); +afterEach(() => { vi.restoreAllMocks(); }); + +describe('actual-worker frames (fc905fe on the real SDK) through the page', () => { + it('normal turn: draft, growing reply, stored final, ack — and the record takes over the stage line', async () => { + const r = await connected(), room = r.transport.rooms[0], turn = sidecar.scenarios.normal.turn_id!; + const frames = load('worker-normal'); + play(room, frames.slice(0, 2)); + expect(r.view().heard).toBe('你在听吗'); + const speech = (frames.find(f => f.kind === 'turn_start') as { speech_id: string }).speech_id; + play(room, frames.slice(2, 3)); + expect(r.view().her).toEqual({ speech, text: '', state: 'open' }); + play(room, frames.slice(3, 6)); + expect(r.view().her).toEqual({ speech, text: '在听呢，你刚才说到一半的那件事我还在想。', state: 'open' }); + play(room, frames.slice(6, 7)); + expect(r.view().her).toEqual({ speech, text: '在听呢，你刚才说到一半的那件事我还在想。', state: 'final', turnId: turn }); + play(room, frames.slice(7, 8)); + expect(r.view().her).toEqual({ speech, text: '在听呢，你刚才说到一半的那件事我还在想。', state: 'done', turnId: turn }); + // The worker confirmed the receipt of exactly this turn (sidecar acks). + expect(sidecar.scenarios.normal.acks).toEqual([turn]); + // Stage takeover: while open, the subtitle line stands in with a voice- key; once the + // stored row (same turn id) is re-read, the line IS that row — no duplicate, no jump. + const open = { speech, text: '在听呢，你刚才说到一半的那件事', state: 'open' as const, turnId: undefined }; + expect(voiceLine(open, [], 0)?.id).toBe(`voice-${speech}`); + const rows = [ + { id: '501', role: 'user' as const, text: '你在听吗', lifeIds: [], turnId: turn, status: 'sent' as const }, + { id: '502', role: 'companion' as const, text: '在听呢，你刚才说到一半的那件事我还在想。', lifeIds: [], turnId: turn, status: 'sent' as const }, + ]; + expect(voiceLine({ ...open, state: 'final', turnId: turn, text: rows[1].text }, rows, 0)).toBeNull(); + const known = [rows[0]]; + const merged = mergeTurns(known, rows)!; + expect(merged.filter(m => m.turnId === turn)).toHaveLength(2); + expect(merged.at(-1)?.id).toBe('502'); + }); + + it('barge-in: the cut turn never gets a final; the next turn lands clean with its own receipt', async () => { + const r = await connected(), room = r.transport.rooms[0]; + const frames = load('worker-interrupt'); + play(room, frames); + const cut = (frames.find(f => f.kind === 'turn_cancel') as { speech_id: string }).speech_id; + const nextTurn = sidecar.scenarios.interrupt.turn_ids!.next; + expect(r.view().her?.turnId).toBe(nextTurn); + expect(r.view().her?.state).toBe('done'); + expect(sidecar.scenarios.interrupt.acks).toEqual([nextTurn]); // the cut turn was never acked + expect(frames.filter(f => f.kind === 'assistant_final')).toHaveLength(1); + expect(frames.find(f => f.kind === 'turn_cancel')?.speech_id).toBe(cut); + }); + + it('phi: her failed line is heard (spoken), not stored; the speech closes without a final', async () => { + const r = await connected(), room = r.transport.rooms[0]; + play(room, load('worker-phi')); + expect(r.view().spoken).toContain('说不出来话'); + expect(r.view().her).toBeNull(); + expect(sidecar.scenarios.phi.acks).toEqual([]); + }); + + it('no legal receipt: cancel only — the fc905fe gate keeps assistant_final off the wire', async () => { + const r = await connected(), room = r.transport.rooms[0]; + const frames = load('worker-no_receipt'); + play(room, frames); + expect(frames.some(f => f.kind === 'assistant_final')).toBe(false); + expect(frames.at(-1)?.kind).toBe('turn_cancel'); + expect((frames.at(-1) as { reason?: string }).reason).toBe('no-receipt'); + expect(r.view().her).toBeNull(); + expect(sidecar.scenarios.no_receipt.acks).toEqual([]); + }); + + it('proactive: start is published while the handle is still playing (sidecar), end closes the row-bound line', async () => { + const r = await connected(), room = r.transport.rooms[0]; + const frames = load('worker-proactive'); + expect(sidecar.scenarios.proactive.start_while_playing).toBe(true); + play(room, frames.slice(0, 2)); + expect(r.view().her).toMatchObject({ state: 'done', proactive: true, messageId: 301, text: '刚才路过花店，想起你说过喜欢白色的洋桔梗。' }); + // The reach-out keeps the stored row's identity on the stage while it is being read; + // once that row is the newest line of the record, the subtitle line gives way to it. + const row = { id: '301', role: 'companion' as const, text: '刚才路过花店，想起你说过喜欢白色的洋桔梗。', lifeIds: [], turnId: 'pro-1', proactive: true, status: 'sent' as const }; + const line = voiceLine({ speech: 'p1', text: row.text, state: 'open', messageId: 301, proactive: true }, [], 0, [row]); + expect(line?.id).toBe('301'); + expect(line?.proactive).toBe(true); + expect(voiceLine({ speech: 'p1', text: row.text, state: 'done', messageId: 301, proactive: true }, [row], 0)).toBeNull(); + // The interrupted variant clears without a trace. + play(room, frames.slice(2)); + expect(r.view().her).toBeNull(); + expect(r.spoke).not.toHaveBeenCalled(); + }); +}); + +describe('page guards against bad frames (adversarial, hand-built)', () => { + it('replays, late frames of a closed speech and out-of-order frames are all dropped', async () => { + const r = await connected(), room = r.transport.rooms[0]; + const frames = load('adversarial-gates'); + // Everything up to and including the later user interim, plus the replayed interim + // (old seq), the closed speech's late frames (fresh seqs, dead speech) and an + // out-of-order lower seq — none of the bad ones may move anything. + play(room, frames.slice(0, 11)); + expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' }); + expect(r.view().heard).toBe('再说点别的'); + // The duplicate turn_start for speech_A (still inside the 64-entry window) is ignored. + play(room, frames.slice(12, 13)); + expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' }); + expect(r.view().her?.text).not.toContain('迟到'); + // The out-of-order lower-seq frame (a lower seq arriving after a higher one) is + // already in slice(0, 11): it never overwrote the newer draft. + const outOfOrder = frames.find(f => f.kind === 'interim' && f.text === '乱序旧帧')!; + expect(outOfOrder.seq).toBeLessThan(9); + expect(r.view().heard).toBe('再说点别的'); + // A replay of the whole first turn (start of speech_A again, still inside the + // window) keeps being ignored. + const dup = frames.filter(f => f.kind === 'turn_start' && f.speech_id === 'speech_A'); + expect(dup.length).toBeGreaterThan(1); + play(room, dup.slice(1)); + expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' }); + }); + + it('DOCUMENTED GAP: after 64+ later closures a retired speech falls out of the window and a duplicate start with a fresh seq re-enters', async () => { + const r = await connected(), room = r.transport.rooms[0]; + const frames = load('adversarial-gates'); + // Up to the end of the 66 squeeze speeches: speech_A is long closed (and by now pushed + // out of the 64-entry window); nothing of it is on the page. + const dupStart = frames.filter(f => f.kind === 'turn_start' && f.speech_id === 'speech_A').at(-1)!; + play(room, frames.filter(f => f.seq < dupStart.seq)); + expect(r.view().her).toBeNull(); + expect(dupStart.seq).toBeGreaterThan(140); + // The current worker never re-emits a speech id (each SDK speech gets a fresh id and + // interim.begin runs once per stream), so this needs a misbehaving/second source — + // but if it happens, the page DOES re-open the long-retired speech... + play(room, frames.filter(f => f.seq >= dupStart.seq)); + expect(r.view().her).toMatchObject({ speech: 'speech_A', state: 'done', turnId: 'gate-turn-1' }); + // ...whose replayed line stands as her words again. + expect(r.view().her?.text).toBe('被挤出后的重放终稿'); + }); + + it('your interim leaves her open line to the server\'s cancel; your final closes it at once', async () => { + const r = await connected(), room = r.transport.rooms[0]; + const frames = load('worker-interrupt'); + play(room, frames.slice(0, 4)); + expect(r.view().her?.state).toBe('open'); + // A barge-in is the SDK's decision, not any sound: the worker's cancel clears her line. + play(room, frames.slice(4, 5)); + expect(r.view().her).toBeNull(); + // A user_final closes the current speech immediately (the server owns the turn gate). + play(room, frames.slice(5, 6)); + expect(r.view().her).toBeNull(); + expect(r.view().heard).toBe('那后来呢'); + expect(r.spoke).toHaveBeenCalled(); + // Your interim alone (a draft, not a turn) leaves her open line alone. + room.on.subtitle({ v: 2, seq: 99, kind: 'interim', role: 'user', text: '嗯' }); + expect(r.view().heard).toBe('嗯'); + }); + + it('text takeover while she speaks: her line and its audio go at once; a drop clears everything and a new call ignores the old room', async () => { + const r = await connected(), room = r.transport.rooms[0]; + room.on.agentState('speaking'); + const frames = load('worker-normal'); + play(room, frames.slice(2, 6)); + expect(r.view().her?.state).toBe('open'); + r.c.hush(); + expect(r.view().her).toBeNull(); + expect(room.hushed).toEqual([true]); + play(room, frames.slice(6)); // her late final from the cut turn + expect(r.view().her).toBeNull(); + // The room drops mid-sentence: everything clears with the call. + room.on.agentState('speaking'); + play(room, rebase(load('worker-phi').slice(1, 3), 20)); + expect(r.view().her?.state).toBe('open'); + room.on.dropped(); await flush(); + expect(r.view()).toMatchObject({ phase: 'off', her: null, notice: WORDS.dropped }); + // A new call is a new room and a new sequence; stale frames of the old room change nothing. + await r.c.start(); const next = r.transport.rooms[1]; next.on.agentJoined(); + next.on.subtitle(frames.find(f => f.kind === 'turn_start')!); + next.on.subtitle(frames.find(f => f.kind === 'assistant_interim')!); + expect(r.view().her).toEqual({ speech: (frames.find(f => f.kind === 'turn_start') as { speech_id: string }).speech_id, text: '在听呢，', state: 'open' }); + room.on.subtitle({ v: 2, seq: 99, kind: 'turn_start', role: 'assistant', text: '', speech_id: 'stale' }); + expect(r.view().her?.speech).not.toBe('stale'); + await r.c.end(); + }); +}); diff --git a/web/tests/hel879_unified_shots.mjs b/web/tests/hel879_unified_shots.mjs new file mode 100644 index 0000000..b8cef5d --- /dev/null +++ b/web/tests/hel879_unified_shots.mjs @@ -0,0 +1,192 @@ +// HEL-879 unified review: the ACTUAL worker's v2 frames (web/tests/fixtures/hel879, generated on +// the .12 real SDK by scripts/gen_hel879_unified_frames.py) replayed through the current page. +// Headless Chromium over CDP against tests/hel878_mock.mjs + tests/fake-livekit.js (no WebRTC, no +// model, no TTS; fake audio device as the microphone). Everything on the subtitle side is the +// worker's own output — turn ids, speech ids, seqs, kinds; the mock's record rows are keyed to the +// fixture turn ids so the stage takeover is exercised end to end. Usage: +// node tests/hel879_unified_shots.mjs [port] +import { spawn } from 'node:child_process'; +import { mkdirSync, writeFileSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +const [chrome, out, portArg] = process.argv.slice(2); +const port = Number(portArg || 8879), base = `http://127.0.0.1:${port}`, cdpPort = 9499; +mkdirSync(out, { recursive: true }); +// The mock rides along (child of this run, killed at the end): one self-contained command. +const mock = spawn(process.execPath, [new URL('./hel878_mock.mjs', import.meta.url).pathname, String(port)], { stdio: 'ignore' }); +await new Promise((ok, ko) => { const t = setTimeout(() => ok(undefined), 3000); mock.once('spawn', () => setTimeout(ok, 400)); }); +const profile = join(tmpdir(), `hel879-unified-${Date.now()}`); +const browser = spawn(chrome, ['--headless=new', `--remote-debugging-port=${cdpPort}`, `--user-data-dir=${profile}`, '--no-first-run', '--hide-scrollbars', '--force-color-profile=srgb', '--use-fake-device-for-media-stream', '--use-fake-ui-for-media-stream', '--autoplay-policy=no-user-gesture-required', 'about:blank'], { stdio: 'ignore' }); +const sleep = ms => new Promise(r => setTimeout(r, ms)); +let targets; +for (let i = 0; i < 60 && !targets; i++) { try { targets = await (await fetch(`http://127.0.0.1:${cdpPort}/json`)).json(); } catch { await sleep(250); } } +const ws = new WebSocket(targets.find(t => t.type === 'page').webSocketDebuggerUrl); +await new Promise(r => ws.addEventListener('open', r, { once: true })); +let id = 0; const waiting = new Map(); +ws.addEventListener('message', ({ data }) => { const m = JSON.parse(data); if (m.id && waiting.has(m.id)) { const { ok, ko } = waiting.get(m.id); waiting.delete(m.id); m.error ? ko(new Error(m.error.message)) : ok(m.result); } }); +const cdp = (method, params = {}) => new Promise((ok, ko) => { const n = ++id; waiting.set(n, { ok, ko }); ws.send(JSON.stringify({ id: n, method, params })); }); +const js = async expr => { const r = await cdp('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }); if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'js error'); return r.result.value; }; +const results = []; +const check = (name, ok, detail = '') => { results.push(ok); console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` — ${detail}` : ''}`); }; +const ctl = async path => (await fetch(`${base}${path}`)).json(); +const shot = async name => { const { data } = await cdp('Page.captureScreenshot', { format: 'png' }); writeFileSync(join(out, `${name}.png`), Buffer.from(data, 'base64')); }; +const frames = async (name, ms) => { const list = []; const end = Date.now() + ms; while (Date.now() < end) { const { data } = await cdp('Page.captureScreenshot', { format: 'jpeg', quality: 70 }); list.push(data); } list.forEach((d, i) => writeFileSync(join(out, `${name}-${String(i).padStart(2, '0')}.jpg`), Buffer.from(d, 'base64'))); return list.length; }; +const click = async (selector, text = '') => js(`(() => { const el = [...document.querySelectorAll(${JSON.stringify(selector)})].find(e => e.offsetParent !== null && e.textContent.includes(${JSON.stringify(text)})); if (!el) return false; el.click(); return true; })()`); +const type = t => js(`(() => { const el = document.querySelector('textarea'); const set = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value').set; set.call(el, ${JSON.stringify(t)}); el.dispatchEvent(new Event('input', { bubbles: true })); return true; })()`); +const until = async (fn, ms = 4000) => { const end = Date.now() + ms; while (Date.now() < end) { if (await fn()) return true; await sleep(100); } return false; }; +const stage = () => js(`document.querySelector('.conversation .stage')?.innerText || ''`); +const said = () => js(`document.querySelector('.conversation .sr')?.textContent || ''`); +const row = () => js(`document.querySelector('.voice-row.real')?.innerText || ''`); +const phase = () => js(`document.querySelector('.voice-row.real')?.dataset.phase || 'off'`); +const lk = expr => js(`window.__lk ? (${expr}) : null`); +const say = frame => lk(`__lk.say(${JSON.stringify(frame)})`); +const state = s => lk(`__lk.state(${JSON.stringify(s)})`); +const exists = sel => js(`!!document.querySelector(${JSON.stringify(sel)})`); +const views = { desktop: { width: 1440, height: 900, deviceScaleFactor: 1, mobile: false }, phone: { width: 390, height: 844, deviceScaleFactor: 2, mobile: true } }; +async function view(v) { await cdp('Emulation.setDeviceMetricsOverride', views[v]); await cdp('Emulation.setTouchEmulationEnabled', { enabled: views[v].mobile, maxTouchPoints: 5 }); } +async function boot(theme) { + await cdp('Page.navigate', { url: `${base}/` }); await sleep(500); + await js(`localStorage.setItem('cg.theme', ${JSON.stringify(theme)}); sessionStorage.clear(); true`); + await cdp('Page.navigate', { url: `${base}/` }); await sleep(1600); +} +async function startVoice() { + await click('.voice-button'); await until(() => exists('.modal.open')); + await click('.modal.open .btn-primary', '开始语音'); + await until(async () => (await phase()) === 'joining'); + await until(async () => (await lk('__lk.rooms.at(-1)?.state')) === 'connected'); + await lk('__lk.join()'); await sleep(250); +} +// The actual worker fixtures + the sidecar that carries their turn ids / receipts. +const load = name => readFileSync(new URL(`./fixtures/hel879/${name}.jsonl`, import.meta.url), 'utf8').split('\n').filter(l => l.trim()).map(l => JSON.parse(l)); +const sidecar = JSON.parse(readFileSync(new URL('./fixtures/hel879/sidecar.json', import.meta.url), 'utf8')); +const of = (frames, kind, nth = 0) => frames.filter(f => f.kind === kind)[nth]; +const sayAll = async frames => { for (const f of frames) await say(f); }; +const calls = async () => ctl('/__log'); +const rows = async () => ctl('/__rows'); +await cdp('Page.enable'); await cdp('Runtime.enable'); await cdp('Network.enable'); + +// ── The full run on moon/desktop: every worker scenario through the real page. ── +await view('desktop'); await boot('moon'); await click('.nav-item', '交谈'); await sleep(1400); +await startVoice(); await calls(); + +// 1. Normal turn (worker-normal.jsonl): her words grow on the stage before anything is stored. +const normal = load('worker-normal'), turn = sidecar.scenarios.normal.turn_id; +await ctl(`/__voice-turn?stage=user&turn=${encodeURIComponent(turn)}&text=${encodeURIComponent('你在听吗')}`); +await sayAll(normal.filter(f => f.seq <= 2)); await state('thinking'); +for (const f of normal.filter(f => f.kind === 'turn_start' || f.kind === 'assistant_interim')) { + await say(f); await state('speaking'); + if (f.text.length > 8) { await until(async () => (await stage()).includes(f.text.slice(0, 6)), 6000); await frames(`w-normal-grow-${f.seq}`, 350); } +} +check('worker frames: her reply grows on the stage while she speaks, before it is stored', (await stage()).includes('在听呢') && (await rows()).every(r => r.role !== 'companion' || r.turn_id !== turn), (await stage())); +check('worker frames: your final is your line, her words never in the voice line', !(await row()).includes('在听呢'), await row()); +await shot('w1-her-words-growing'); +// The stored row takes over: same turn id, exactly once, stage keeps its place. +await ctl(`/__voice-turn?stage=reply&turn=${encodeURIComponent(turn)}&text=${encodeURIComponent('在听呢，你刚才说到一半的那件事我还在想。')}`); +await sayAll(normal.filter(f => f.kind === 'assistant_final' || f.kind === 'turn_end')); await state('listening'); await sleep(900); +await click('.log-btn'); await sleep(700); +const record1 = await js(`document.querySelector('.log-panel .scroll')?.innerText || ''`); +check('worker frames: the stored row (same turn id) takes over exactly once — re-read, never written by the page', record1.split('在听呢，你刚才说到一半的那件事我还在想。').length - 1 === 1, record1.slice(-140).replace(/\n/g, ' ')); +await shot('w2-record-after-final'); +await click('.log-panel .btn-round'); await sleep(400); await click('.read-all'); await sleep(400); await click('.record.full .btn-round'); await sleep(300); + +// 2. Barge-in (worker-interrupt.jsonl): the cut turn never gets a final; the next turn lands clean. +const it = load('worker-interrupt'), cutTurn = sidecar.scenarios.interrupt.turn_ids.cut, nextTurn = sidecar.scenarios.interrupt.turn_ids.next; +await ctl(`/__voice-turn?stage=user&turn=${encodeURIComponent(cutTurn)}&text=${encodeURIComponent('讲讲你下午做了什么')}`); +await sayAll(it.slice(0, 4)); await state('speaking'); +await until(async () => (await stage()).includes('白鹭'), 6000); +await shot('w3-before-cut'); +await sayAll(it.slice(4, 5)); await state('listening'); await ctl(`/__voice-turn?stage=cancel&turn=${encodeURIComponent(cutTurn)}`); await sleep(250); +check('worker frames: interrupted — her unfinished line is gone at once, no final exists for the cut turn', !(await stage()).includes('白鹭') && it.filter(f => f.kind === 'assistant_final' && f.speech_id === of(it, 'turn_cancel').speech_id).length === 0, await stage()); +// Late frames of the cut speech (fresh seqs, dead speech) change nothing. +const cutSpeech = of(it, 'turn_cancel').speech_id; +await say({ v: 2, seq: 90, kind: 'assistant_interim', role: 'assistant', speech_id: cutSpeech, text: '白鹭又飞回来了' }); +await say({ v: 2, seq: 91, kind: 'assistant_final', role: 'assistant', speech_id: cutSpeech, turn_id: 'late', text: '白鹭又飞回来了。' }); +await sleep(350); +check('worker frames: late frames of the cut speech stay out', !(await stage()).includes('飞回来')); +// The next clean turn of the same recording lands with its own receipt. +await ctl(`/__voice-turn?stage=user&turn=${encodeURIComponent(nextTurn)}&text=${encodeURIComponent('那后来呢')}`); +await ctl(`/__voice-turn?stage=reply&turn=${encodeURIComponent(nextTurn)}&text=${encodeURIComponent('后来我在桥上站了一会儿，风把水面吹皱了。')}`); +await sayAll(it.slice(5)); await state('speaking'); await until(async () => (await stage()).includes('桥上'), 6000); await state('listening'); await sleep(700); +await click('.log-btn'); await sleep(600); +const record2 = await js(`document.querySelector('.log-panel .scroll')?.innerText || ''`); +check('worker frames: the next turn lands clean (one stored row of its own)', record2.includes('风把水面吹皱了') && record2.split('风把水面吹皱了').length - 1 === 1, record2.slice(-120).replace(/\n/g, ' ')); +await click('.log-panel .btn-round'); await sleep(300); + +// 3. phi (worker-phi.jsonl): her failed line is heard, not stored. +const phi = load('worker-phi'); +await ctl(`/__voice-turn?stage=user&turn=${encodeURIComponent(sidecar.scenarios.phi.turn_id)}&text=${encodeURIComponent('说说那件事吧')}`); +await sayAll(phi); await state('speaking'); await sleep(300); await state('listening'); await sleep(300); +check('worker frames: phi is heard in the voice line, not stored as her reply', (await row()).includes('说不出来话') && !(await stage()).includes('说不出来话'), await row()); +await shot('w4-phi-heard'); +await sleep(600); + +// 4. No legal receipt (worker-no_receipt.jsonl): cancel only, nothing on the stage. +const nr = load('worker-no_receipt'); +await ctl(`/__voice-turn?stage=user&turn=${encodeURIComponent(sidecar.scenarios.no_receipt.turn_id)}&text=${encodeURIComponent('再讲讲那座桥')}`); +await sayAll(nr.filter(f => f.kind !== 'turn_cancel')); await state('speaking'); +await until(async () => (await stage()).includes('水声很好听'), 6000); +await say(of(nr, 'turn_cancel')); await state('listening'); await sleep(300); +check('worker frames: no legal receipt — the page never shows a final (cancel only)', !(await stage()).includes('水声很好听') && nr.every(f => f.kind !== 'assistant_final'), await stage()); + +// 5. Proactive (worker-proactive.jsonl): start DURING playout, end/cancel with the row's own id. +await ctl(`/__proactive?id=301&text=${encodeURIComponent('刚才路过花店，想起你说过喜欢白色的洋桔梗。')}`); await calls(); +const pro = load('worker-proactive'); +await say(of(pro.filter(f => f.message_id === 301), 'proactive_start')); await state('speaking'); await sleep(600); +check('worker frames: reach-out read aloud is shown word for word while she speaks it', (await stage()).includes('洋桔梗'), await stage()); +await shot('w5-reach-out'); +await say(of(pro.filter(f => f.message_id === 301), 'proactive_end')); await state('listening'); await sleep(400); +// The interrupted variant clears without a trace and the page wrote nothing. +const before = await calls(); +await say(pro.filter(f => f.message_id === 302)[0]); await state('speaking'); await sleep(250); +await say(pro.filter(f => f.message_id === 302)[1]); await state('listening'); await sleep(250); +const after = await calls(); +check('worker frames: an interrupted reach-out clears; the page still wrote nothing (no receipt, no read mark)', !after.some(l => !l.startsWith('GET ')), after.join(' | ')); + +// 6. Text takes over while she speaks (the worker's own next turn): typed words come next. +const nt = load('worker-normal'), nt2 = { ...normal[2], seq: 100, speech_id: 'speech_text_takeover' }; +await say(nt2); await state('speaking'); await sleep(200); +await type('先等等，我打字跟你说'); await js(`document.querySelector('.send').click(), true`); await sleep(400); +check('text over voice: her worker line is gone at once, your typed words are on the stage', !(await stage()).includes('在听呢，') && (await stage()).includes('先等等'), await stage()); +await until(async () => !(await exists('.reader-bar .stop')), 9000); await state('listening'); await sleep(400); + +// 7. The room drops mid-sentence: everything clears with the call; starting again works. +await say({ v: 2, seq: 120, kind: 'turn_start', role: 'assistant', speech_id: 'speech_drop', text: '' }); +await say({ v: 2, seq: 121, kind: 'assistant_interim', role: 'assistant', speech_id: 'speech_drop', text: '晚安，明天见。' }); await state('speaking'); +await until(async () => (await stage()).includes('晚安'), 6000); +await lk('__lk.drop()'); await sleep(400); +check('dropped mid-sentence: her line and the call clear together', !(await stage()).includes('晚安') && (await phase()) === 'off', await stage()); +await shot('w6-after-drop'); +await startVoice(); await calls(); +const fresh = { v: 2, seq: 1, kind: 'turn_start', role: 'assistant', speech_id: 'speech_fresh', text: '' }; +const freshText = { v: 2, seq: 2, kind: 'assistant_interim', role: 'assistant', speech_id: 'speech_fresh', text: '又见面了。' }; +await say(fresh); await say(freshText); await state('speaking'); +await until(async () => (await stage()).includes('又见面了'), 6000); +check('a new call is a new room and a new sequence: her words flow again, nothing of the old call remains', (await stage()).includes('又见面了') && !(await stage()).includes('晚安'), await stage()); +await shot('w7-reopened'); +await click('.voice-row.real .voice-end'); await sleep(300); + +// ── Second theme + phone: the same worker frames on the approved visuals. ── +for (const [theme, v, tag] of [['fog', 'desktop', 'fog'], ['moon', 'phone', 'phone']]) { + await view(v); await boot(theme); await click('.nav-item', '交谈'); await sleep(1400); await startVoice(); await calls(); + const t = load('worker-normal'), tid = sidecar.scenarios.normal.turn_id; + await ctl(`/__voice-turn?stage=user&turn=${encodeURIComponent(tid)}&text=${encodeURIComponent('你在听吗')}`); + await sayAll(t.filter(f => f.seq <= 2)); + await say(of(t, 'turn_start')); await state('speaking'); + for (const f of t.filter(f => f.kind === 'assistant_interim')) { await say(f); await sleep(260); } + check(`${tag}: worker frames grow her reply on the approved ${theme} ${v} layout`, await until(async () => (await stage()).includes('在听呢，你刚才说到一半的那件事'), 8000), await stage()); + if (tag === 'phone') await frames('phone-growing', 700); + await shot(`${tag}-her-words-growing`); + await ctl(`/__voice-turn?stage=reply&turn=${encodeURIComponent(tid)}&text=${encodeURIComponent('在听呢，你刚才说到一半的那件事我还在想。')}`); + await sayAll(t.filter(f => f.kind === 'assistant_final' || f.kind === 'turn_end')); await state('listening'); await sleep(800); + await click('.log-btn'); await sleep(600); + const rec = await js(`document.querySelector('.log-panel .scroll')?.innerText || ''`); + check(`${tag}: the stored row takes over once on ${theme} ${v}`, rec.split('在听呢，你刚才说到一半的那件事我还在想。').length - 1 === 1, ''); + await click('.log-panel .btn-round'); await sleep(300); + await click('.voice-row.real .voice-end'); await sleep(300); +} + +ws.close(); browser.kill(); mock.kill(); +const failed = results.filter(r => !r).length; +console.log(`${results.length - failed}/${results.length} passed`); +process.exit(failed ? 1 : 0);