HEL-878: preserve original author continuous voice wiring
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
multica-agent
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"baseline_version": "1.11",
|
||||
"read_commit": "bf234570c0b6595a72f8dbc79e42c63685e08e01",
|
||||
"task": "HEL-878（R5 前端接线，视觉总监本人设计并编码）：在已认可的交谈界面上由用户主动开启/结束连续语音；按 HEL-877 后台契约（aba2aeb docs/runbook-hel877-voice.md §3）接 GET /api/voice/status、POST/DELETE /api/voice/session 与 cg.subtitle 字幕，复用 .12 :8090 既有 livekit-client 2.22.4 浏览器客户端；文字与语音同一位她、同一聊天记录，轮次从 /api/messages 回读不串台；所有退出路径立即停麦并清理；零真实模型/嵌入/合成/识别/采集/部署",
|
||||
"requirement_ids": [
|
||||
"CG-006", "CG-010", "CG-011", "CG-018", "CG-020",
|
||||
"VIS-001", "VIS-002", "VIS-003", "VIS-004", "VIS-005", "VIS-006", "VIS-007", "VIS-008", "VIS-009", "VIS-010", "VIS-011", "VIS-012", "VIS-013",
|
||||
"UX-001", "UX-002", "UX-005", "UX-006",
|
||||
"AM-001",
|
||||
"GOV-001", "GOV-002", "GOV-003", "GOV-004", "GOV-005", "GOV-006",
|
||||
"DEL-001"
|
||||
],
|
||||
"deliverable": "自前端原稿 hel-875-proactive-wiring @ bf23457 起的补丁（仅 web/ 与本交接）：1) web/src/voice/session.ts——VoiceController 生命周期（代次计数：任何迟到的 token/房间/麦克风/房间事件只清理不显示）；顺序为诚实状态（只 GET status，不建房不采麦）→ 加载客户端 → 采麦（拒绝则服务端什么都不开）→ POST session（结束时已在途的请求等其返回并 DELETE，不留孤儿房）→ 加入并发布麦克风 → 等 worker 加入（20 s 未加入如实失败）；状态取 worker 的 lk.agent.state，缺属性时按字幕保守推断；interim 为草稿、user_final 取消在途文字回复并触发回读、assistant_final 触发回读、error(φ) 只显示「未记入聊天」不入记录；文字盖过她的语音时静音残余音频直到她回到聆听；结束顺序为停麦→关房间与音频→DELETE；断网时本地立即停采，远端 DELETE 记入本浏览器 cg.voice.owed，恢复网络/再次进入/再次开启时补发（404 视为已清）；页面卸载用 keepalive DELETE；mergeTurns/recentRows 只合并非主动留言的服务器行（主动留言仍走原 inbox 轮询），按 turn_id 与已有文字轮去重，pending 行回读至落定。2) web/src/voice/livekit.ts——同源懒加载 /vendor/livekit-client-2.22.4.umd.min.js（来源 .12:8090/static，SHA-256 见 web/VENDOR.md，未改 package.json/锁文件），回声消除采集、她的音频集中在隐藏容器（结束即暂停/清空/移除）、自动播放受阻时提供「点一下听她的声音」、加入可被结束即时中止。3) ChatPanel.tsx——真实模式麦克风按钮按状态诚实标注（不可用时淡化并说明原因），确认框说明麦克风、去向、同一份记忆、如何打断与停止、未经真机验收；沿用原演示 voice-row 位置与视觉语言做成玻璃单行（连接/等待为呼吸、聆听用你的颜色、她说话用她的光色，文字始终写明状态）；文字流在无 final 时先查服务器行：被新轮取消即按已取消处理，不报连接中断、不回填草稿；离开交谈/Esc/结束交谈/导航卸载面板即结束语音；在途语音轮期间她的主动留言照原规则排队。4) RealApp.tsx——语音进行中顶栏写「语音（未经真机验收）」。5) components.css 新增 .voice-row.real 等样式（含手机）。6) 测试：web/tests/voice.test.ts 26 项；web/tests/hel878_mock.mjs（HEL-877 路由契约 mock）+ fake-livekit.js（脚本化替身，浏览器假音频设备）+ hel878_shots.mjs 浏览器检查 45 项与截图；web/VENDOR.md、README 语音段落。",
|
||||
"acceptance": [
|
||||
"到达交谈页只发一次 GET /api/voice/status：不建房、不申请麦克风、不加载 LiveKit 客户端；configured=false / tts_ok=false / 无路由各有诚实说明，按钮淡化且不弹确认；voice=false 不被当作已接通（浏览器 1–2、单测 honest switch）",
|
||||
"用户确认后：一个会话、一个麦克风、发布到该房间；等待 worker 时显示等待而非已接通；聆听/思考/说话随 lk.agent.state 切换；字幕草稿、φ 原句（标未记入聊天且不入记录）均按契约（浏览器 4–6）",
|
||||
"语音轮只从 /api/messages 进入舞台与完整记录，各一次，与文字同一段记录（浏览器 5）；前端从不 POST /api/chat 或 /api/chat/received（单测）",
|
||||
"双通道：文字盖过她说话时残余音频静音、回到聆听再恢复；语音打断在途文字回复时该轮按已取消处理，无“连接中断/重试”、不回填草稿，记录显示已取消（浏览器 7–8）",
|
||||
"结束语音/导航离开/Esc/结束交谈/房间掉线/麦克风被撤销/服务端拒绝/重复开关后：本地麦克风 track 全部 ended、她的音频与容器移除、服务端房间删除；迟到事件不复活旧会话（浏览器 10–14，单测 lifecycle 15 项）",
|
||||
"断网：本地立即停麦停音频，远端房间如实记为欠删（不冒充已删）；恢复网络后补发 DELETE 并清除记录；页面卸载 keepalive DELETE 到达（浏览器 14）；欠删跨页面保留、404 视为已清（单测）",
|
||||
"电脑 1440×900 与手机 390×844（2x）、月光/雾光两主题截图；手机语音行不出屏（浏览器 15）",
|
||||
"真实 livekit-client 2.22.4 构建中，适配器用到的全部名称存在（RoomEvent 七项、Track.Source.Microphone、ParticipantKind.AGENT、createLocalAudioTrack、Room 方法/属性）；仅 API 面核对，未连接",
|
||||
"演示入口零语音请求；工程：tsc 0 错，eslint 0 错（1 条既有警告），vitest 7 文件 81/81，vite build 通过"
|
||||
],
|
||||
"deferred": [
|
||||
"真实通话全部未测（R5 真机验收交 HEL-879 统一审核）：连续 3 轮、开口打断约 1 s 停声、结束停麦无残音、回声、首包延迟、移动端 Safari/Chrome 麦克风与自动播放、自签 HTTPS 信任；浏览器检查用的是脚本化 LiveKit 替身与 Chromium 假音频设备，不是 WebRTC 通话",
|
||||
"lk.agent.state 属性名按 livekit-agents 1.x 惯例接入，并在缺失时用字幕推断；worker 实际是否发布该属性须真机确认",
|
||||
"断网时服务端房间的即时删除在物理上做不到：依赖 worker 在浏览器参与者离开后结束任务与会话登记过期（runbook §3.4/3.5，后台行为未在本卡实测），本页恢复网络或下次进入时补发 DELETE",
|
||||
"CG-011 语音内主动开口：worker._maybe_open_proactive 仍是空操作，前端未虚构；后台最小需求见 runbook §3.6（会话内取已落库 proactive 行 session.say 原文、可打断、无新回执通道），呈现节奏待总管/视觉总监另定；目前语音开启时她的主动留言仍以文字进入并在她思考/说话时排队",
|
||||
"她说话时没有逐字字幕（契约只在播放完成后发 assistant_final）；舞台在落库后呈现全文。若要边说边显示需后台发增量字幕，属契约变更",
|
||||
"D1 音色、D3 设备信任、测试额度由总管与白栖知沟通；本卡未接触任何服务、凭据或部署",
|
||||
"原版 Python 需求检查未在本运行时执行（无 Python、无 git），须总管/总工对实际目标分支与 origin/main 含/不含本交接复跑；需与 HEL-882 最新母线同步后再审",
|
||||
"R6 天气/素材/备份、R7 视觉观察、CG-028/029 口型与真实视频仍由主卡 HEL-797 承接；整体沉浸体验由白栖知验收"
|
||||
],
|
||||
"requirement_changes": [],
|
||||
"verification": [
|
||||
"读取：前端 bf234570（总管输入包 frontend/，HEL-875/880 原作者原稿）；后台契约 aba2aeb（输入包 voice-backend/：runbook-hel877-voice.md、voice/router.py、session.py、worker.py、handoff-HEL-877.json、requirements v1.11 含 comment-evidence-full-experience-20261026.json）；已读 AGENTS.md、README/current（CG-006/010/011/018/020、UX-005/006、VIS-013）/coverage/changes、integration-plan-v1 R5 与 D1/D3、voice-r5-readiness-HEL876.md",
|
||||
"web：tsc --noEmit 0；eslint src 0 错；vitest run 7 文件 81/81（原 55 + 语音 26，含变异检查：去掉迟到授权回收或代次判定各使对应用例失败）；vite build 通过（node_modules 沿用 HEL-875 同一锁文件离线树，无新增依赖、无下载）",
|
||||
"浏览器：node tests/hel878_mock.mjs 8878 + node tests/hel878_shots.mjs <msedge.exe> <out> 8878，45/45（无头 Edge，--use-fake-device-for-media-stream；脚本化 LiveKit 替身；本机回环）；中断前一轮为 39/43，失败的 4 项（语音打断文字误报、断网远端清理、卸载检查受其连带）经本轮修复后通过",
|
||||
"唯一外部读取：一次只读 GET .12:8090 页面及其 /static/livekit-client.umd.min.js 以复用既有客户端；无模型、嵌入、合成、识别请求，无真实麦克风/摄像头，无部署或服务改动",
|
||||
"需求结构检查：本运行时无 Python/git，未执行原版脚本，不声称通过；仅用 HEL-875 留下的 Node 移植版辅助：以 bf23457 为基含本交接通过（60 项）；以后台 aba2aeb 的需求文件为基失败「历史来源必须保留」，原因是该线已追加 comment-evidence-full-experience-20261026.json 而前端原稿尚未包含，与本补丁无关，合入母线前须先同步该需求目录"
|
||||
]
|
||||
}
|
||||
@@ -32,4 +32,6 @@ python3 -m pytest -q # 现有 Python 核心，仓库根目录
|
||||
|
||||
脚本在随机 loopback 端口同步测试静态产物，结束关闭浏览器和服务；截图写入 `docs/frontend/screenshots/`（不进入前端产物）。没有持续预览服务或部署。真实手机、Safari、真实麦克风/摄像头、ASR/TTS、后台生活和口型均未测试。
|
||||
|
||||
语音（HEL-878，R5 前端接线）：交谈页麦克风按钮按 `GET /api/voice/status` 诚实显示；用户确认后才申请麦克风、`POST /api/voice/session` 取短时 token，用 `public/vendor/` 下既有 livekit-client 加入房间（来源见 `VENDOR.md`）。状态、字幕、结束与清理逻辑在 `src/voice/`。离线检查：`npm test`（含 `tests/voice.test.ts`），构建后 `node tests/hel878_mock.mjs 8878` + `node tests/hel878_shots.mjs <msedge.exe> <outDir> 8878`（假 LiveKit + 浏览器假音频设备，不是真实通话）。真实通话、真机、Safari、回声与打断时延均未测。
|
||||
|
||||
接线边界、交付与测试记录见 `docs/frontend/IMPLEMENTATION.md`。
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
Vendored browser client (HEL-878)
|
||||
|
||||
`public/vendor/livekit-client-2.22.4.umd.min.js`
|
||||
|
||||
- What: livekit-client 2.22.4, UMD build (global `LivekitClient`), Apache-2.0. The file header names its origin: jsdelivr `/npm/livekit-client@2.22.4/dist/livekit-client.umd.js`.
|
||||
- Where it came from: the existing .12 call page, `https://192.168.200.12:8090/static/livekit-client.umd.min.js` (the page that verified LAN microphone calls), read once with a plain GET on 2026-10-26. Not downloaded from npm or a CDN, nothing installed.
|
||||
- SHA-256: `040DB4C0DB72E7F7599CD0E06B0F58F95E0082A395CDDE410CA26224A79A0626` (598 796 bytes).
|
||||
- Loaded only when the user switches voice on (`src/voice/livekit.ts`, same origin, no CDN at run time). Text and demo paths never fetch it.
|
||||
- Not an npm dependency: package.json and the lock file are unchanged. Upgrading means replacing this file and the name in `LIVEKIT_SRC`.
|
||||
File diff suppressed because one or more lines are too long
@@ -79,7 +79,7 @@ export function RealApp() {
|
||||
useEffect(() => { if (fresh && !unread.some(m => m.id === fresh.id)) setFresh(null); }, [fresh, unread]);
|
||||
// Reply to her: one new message plays on the stage; several open the record at the first one you have not seen.
|
||||
const openUnread = () => { setFresh(null); const first = unread[0]; leaveSettingsThen(() => { setArea('talk'); setFocusLife(''); setLog(unread.length > 1); setFocusIds(unread.length > 1 && first ? [first.id] : []); }); };
|
||||
const ignoreVoice = useCallback(() => {}, []);
|
||||
const ignoreVoice = useCallback(() => {}, []), [voiceOn, setVoiceOn] = useState(false);
|
||||
// Status words follow the server, re-read when you open a page that shows them; a failed read keeps the last one.
|
||||
const life = lifeStatus(health);
|
||||
const refreshHealth = useCallback(() => { api<Health>('health').then(setHealth, () => {}); }, []);
|
||||
@@ -100,12 +100,12 @@ export function RealApp() {
|
||||
const lastReply = state ? [...state.messages].reverse().find(m => m.role === 'companion') : undefined;
|
||||
const talking = area === 'talk' && !!state && !creating;
|
||||
return <><Presenter real scene={{ mode: area === 'talk' ? 'talk' : 'daily', activity: '文字已接通', time_band: '', weather: '', outfit: '' }} space={space} theme={theme} instant={instant} observation="" now={state && !creating ? life.now : ''} lifeLabel={life.wired ? life.label : undefined} onApproach={() => navigate('talk')}/>
|
||||
<header className="topbar"><div className="who"><i className="dot"/><span className="name" title={state?.identity.name}>{state?.identity.name || 'CyberGril'}</span>{talking && <span className="presence">{presence}</span>}<span className="state">{ready ? '真实档案 · 文字对话' : '连接真实服务'}</span></div><div className="env"><span className="chip glass weather">天气尚未接通</span><span className="theme-switch glass">{(['moon', 'fog'] as Theme[]).map(t => <button key={t} data-t={t} aria-label={t === 'moon' ? '靛蓝月光' : '蓝紫雾光'} aria-pressed={theme === t} onClick={() => setTheme(t)}><i/><span>{t === 'moon' ? '月光' : '雾光'}</span></button>)}</span>{talking && <button className="end-talk" onClick={() => navigate('now')}><Icon name="x"/><span>结束交谈</span></button>}</div></header>
|
||||
<header className="topbar"><div className="who"><i className="dot"/><span className="name" title={state?.identity.name}>{state?.identity.name || 'CyberGril'}</span>{talking && <span className="presence">{presence}</span>}<span className="state">{!ready ? '连接真实服务' : talking && voiceOn ? '真实档案 · 语音（未经真机验收）' : '真实档案 · 文字对话'}</span></div><div className="env"><span className="chip glass weather">天气尚未接通</span><span className="theme-switch glass">{(['moon', 'fog'] as Theme[]).map(t => <button key={t} data-t={t} aria-label={t === 'moon' ? '靛蓝月光' : '蓝紫雾光'} aria-pressed={theme === t} onClick={() => setTheme(t)}><i/><span>{t === 'moon' ? '月光' : '雾光'}</span></button>)}</span>{talking && <button className="end-talk" onClick={() => navigate('now')}><Icon name="x"/><span>结束交谈</span></button>}</div></header>
|
||||
{!ready ? <main className="entry"><div className="card glass"><h1>一扇亮着的窗。</h1><Status text={error || '正在读取服务器档案…'} error={!!error}/>{error && <><p>请检查真实服务是否可达。资料仍在服务器保存。</p><Button primary onClick={() => void initialize()}>重新连接</Button></>}</div></main> : <>
|
||||
{creating && <CreationWizard real fail={false} onCreate={setState} onEnter={() => setCreating(false)}/>}
|
||||
{!creating && state && <>
|
||||
{area === 'now' && <main className="home"><div className="eyebrow">一扇亮着的窗</div><h1 className="whisper">慢慢说，我在这里。</h1><div className="meta"><span><Icon name="heart"/>{state.identity.relation} · 慢慢熟悉</span></div><button className="away" onClick={() => navigate('days')}>日子与回忆 · {life.label} <Icon name="next"/></button>{unread.length ? <button className="letter glass" onClick={openUnread}><span className="lh"><b>她留给你的</b><span>{unread.length > 1 ? `${unread.length} 句还没看` : '还没看'}</span></span><p>{unread[unread.length - 1].text}</p></button> : lastReply && <button className="letter glass read" onClick={() => { navigate('talk'); setLog(true); }}><span className="lh"><b>最近说过的话</b><span>打开完整记录</span></span><p>{lastReply.text}</p></button>}<div className="cta"><Button primary small={false} onClick={() => navigate('talk')}><Icon name="chat"/>和她说话</Button></div><p className="first-note">{memoryNote(health, life)}</p>{health && !health.llm.ok && <Status error text="模型暂时不可达，档案和历史仍可查看。"/>}</main>}
|
||||
{area === 'talk' && <ChatPanel real state={state} onChange={changeMessages} onPresence={setPresence} onVoice={ignoreVoice} onObservation={ignoreVoice} onRecord={setRecord} offline={false} fail={false} log={log} setLog={setLog} focusLife={focusLife} focusIds={focusIds} onReceipt={acknowledged} incoming={incoming} onAdmit={admitted}/>}
|
||||
{area === 'talk' && <ChatPanel real state={state} onChange={changeMessages} onPresence={setPresence} onVoice={setVoiceOn} onObservation={ignoreVoice} onRecord={setRecord} offline={false} fail={false} log={log} setLog={setLog} focusLife={focusLife} focusIds={focusIds} onReceipt={acknowledged} incoming={incoming} onAdmit={admitted}/>}
|
||||
<LifePanel real state={state} open={area === 'days'} now={life.now} life={life} ackAt={ackAt} onClose={() => navigate('now')} onChat={openTold} onSaid={openSaid} onSettings={() => { navigate('settings'); setHerFocus(n => n + 1); }}/>
|
||||
<ManagePanel real state={state} open={area === 'settings'} now={life.now} life={life} focus={herFocus} onClose={() => navigate('now')} onSave={setState} fail={false} theme={theme} setTheme={setTheme}/>
|
||||
{/* Same card as the approved demo reach-out, now with her real words; it never opens the conversation by itself. */}
|
||||
|
||||
@@ -10,6 +10,9 @@ import { snap } from '../shared/motion';
|
||||
import { motionOff } from '../stage/roomScene';
|
||||
import { LineStage, type PresentHow } from './lineStage';
|
||||
import { anchorOf, cursorAt, groupTurns, pagesOf, ReadingClock, type Anchor, type Metrics, type ShownMessage } from './reading';
|
||||
import { mergeTurns, OFF, recentRows, VoiceController, voiceStatus, WORDS, type VoiceView } from '../voice/session';
|
||||
import { loadTransport } from '../voice/livekit';
|
||||
import { newestId, serverId } from '../adapters/inbox';
|
||||
|
||||
const START: Anchor = { mi: 0, offset: 0 };
|
||||
const metricsFor = (width: number): Metrics => { const compact = document.documentElement.classList.contains('compact'); return { width, font: compact ? 16 : innerWidth <= 720 ? 18 : 23, compact }; };
|
||||
@@ -47,6 +50,51 @@ export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation,
|
||||
const name = state.identity.name;
|
||||
const change = (messages: Message[]) => { live.current = messages; onChange(messages); };
|
||||
|
||||
// ── Real voice (R5). The controller owns the call; this panel shows its state and keeps the record in step. ──
|
||||
const [voiceView, setVoiceView] = useState<VoiceView>(OFF), [voiceOff, setVoiceOff] = useState('');
|
||||
const voiceCtl = useRef<VoiceController | null>(null), busyNow = useRef(busy); busyNow.current = busy;
|
||||
// Your voice turn and her spoken answer reach the record only as server rows (subtitles carry no message id).
|
||||
// Rows still pending on the page are read again until they settle.
|
||||
const recordSync = useRef({ running: false, again: false, timer: 0 as ReturnType<typeof setTimeout> | 0 });
|
||||
const reread = useRef(() => {});
|
||||
reread.current = () => {
|
||||
const sync = recordSync.current;
|
||||
if (sync.running) { sync.again = true; return; }
|
||||
sync.running = true; sync.again = false;
|
||||
const pending = live.current.filter(m => m.status === 'pending').map(serverId).filter(Boolean);
|
||||
const since = Math.min(newestId(live.current), ...pending.map(id => id - 1));
|
||||
recentRows(since, new AbortController().signal).then(rows => { const next = mergeTurns(live.current, rows); if (next) change(next); }, () => { /* The next turn or the inbox read tries again. */ })
|
||||
.finally(() => { sync.running = false; if (sync.again) reread.current(); });
|
||||
};
|
||||
const spoke = useRef(() => {});
|
||||
// Speaking cancels a text reply still streaming (the server's turn gate already did); it is kept as a cut fragment.
|
||||
spoke.current = () => { if (busyNow.current) stop(true); };
|
||||
useEffect(() => {
|
||||
if (!real) return;
|
||||
const sync = recordSync.current;
|
||||
const c = new VoiceController({ transport: loadTransport, onView: setVoiceView, onSpoke: () => spoke.current(), onRecord: () => { clearTimeout(sync.timer); sync.timer = setTimeout(() => reread.current(), 250); } });
|
||||
voiceCtl.current = c;
|
||||
const gone = () => c.unload(), offline = () => { if (c.active) void c.end(WORDS.offline); }, online = () => void c.settle();
|
||||
addEventListener('pagehide', gone); addEventListener('offline', offline); addEventListener('online', online); online();
|
||||
// Leaving the conversation (nav, Esc, 结束交谈) unmounts this panel: the call ends with it.
|
||||
return () => { removeEventListener('pagehide', gone); removeEventListener('offline', offline); removeEventListener('online', online); voiceCtl.current = null; void c.end(); };
|
||||
}, [real]);
|
||||
// The switch reads the server's honest state once on arrival; it never opens a room or touches the microphone.
|
||||
useEffect(() => {
|
||||
if (!real) return;
|
||||
const c = new AbortController();
|
||||
voiceStatus(c.signal).then(s => setVoiceOff(s.available ? '' : s.reason), () => {});
|
||||
return () => c.abort();
|
||||
}, [real]);
|
||||
const voiceOn = real ? voiceView.phase !== 'off' : voice;
|
||||
const voiceLabel = { off: '', opening: '正在接通…', joining: `在等${name}接起…`, listening: '在听你说', thinking: `${name}在想`, speaking: `${name}在说 · 开口就能打断` }[voiceView.phase];
|
||||
useEffect(() => {
|
||||
if (!real) return;
|
||||
onVoice(voiceView.phase !== 'off');
|
||||
if (voiceView.phase !== 'off') onPresence(voiceLabel);
|
||||
else if (voiceView.notice) { onPresence('在你面前'); setNotice(voiceView.notice); }
|
||||
}, [real, voiceView.phase, voiceView.notice, voiceLabel, onVoice, onPresence]);
|
||||
|
||||
// ── What the stage shows: the newest turn of the (already privacy-filtered) record, plus the live reply. ──
|
||||
// Real messages are trimmed by the server before transmission; their lifeIds point at records this page has not
|
||||
// necessarily loaded, so the demo's local filter would wrongly hide them.
|
||||
@@ -78,10 +126,10 @@ export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation,
|
||||
// Her own message that arrives while you are here waits its turn: never while she is answering you, a sheet is
|
||||
// open, you paused or went back, or the line on screen has not had its time. Then it comes in as a new turn.
|
||||
const waiting = useRef({ count: 0, busy: false, sheet: false }), admit = useRef(() => {});
|
||||
waiting.current = { count: incoming.length, busy, sheet: log || full };
|
||||
waiting.current = { count: incoming.length, busy: busy || voiceView.phase === 'thinking' || voiceView.phase === 'speaking', sheet: log || full };
|
||||
admit.current = () => { const add = incoming.filter(m => !live.current.some(k => k.id === m.id)); onAdmit?.(incoming.map(m => m.id)); if (add.length) change([...live.current, ...add]); };
|
||||
|
||||
function stop() { gate.current.cancel(); if (replyLive.current) setInterrupted({ text: replyLive.current, turn: turnLive.current }); replyLive.current = ''; setBusy(false); setReply(''); onPresence('停下来了'); change(live.current.map(m => m.status === 'pending' ? { ...m, status: 'cancelled' } : m)); }
|
||||
function stop(quiet = false) { gate.current.cancel(); if (replyLive.current) setInterrupted({ text: replyLive.current, turn: turnLive.current }); replyLive.current = ''; setBusy(false); setReply(''); if (!quiet) onPresence('停下来了'); change(live.current.map(m => m.status === 'pending' ? { ...m, status: 'cancelled' } : m)); }
|
||||
function endVoice() { stop(); voiceController.current?.abort(); setVoice(false); onVoice(false); onPresence('在你面前'); }
|
||||
useEffect(() => { const current = gate.current; return () => { current.cancel(); if (real) onChange(live.current.map(m => m.status === 'pending' ? { ...m, status: 'cancelled' } : m)); voiceController.current?.abort(); onVoice(false); onObservation(''); }; }, [onVoice, onObservation, onChange, real]);
|
||||
useEffect(() => { try { sessionStorage.setItem(draftKey, draft); } catch { /* Draft remains in memory. */ } }, [draft, draftKey]);
|
||||
@@ -184,6 +232,8 @@ export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation,
|
||||
if (!retry && real) retry = [...live.current].reverse().find(m => m.role === 'user' && m.status === 'failed' && m.text === draft.trim());
|
||||
const text = retry?.text || draft.trim(); if (!text || offline) return;
|
||||
stop();
|
||||
// Typing over her spoken answer: the server cancels that voice turn; what is left of its audio stays silent.
|
||||
voiceCtl.current?.hush();
|
||||
const turnId = retry?.turnId || newId(), signal = gate.current.begin(turnId);
|
||||
const msg: Message = { id: retry?.id || newId(), role: 'user', text, lifeIds: [], turnId, status: 'pending' };
|
||||
sentTurn.current = turnId;
|
||||
@@ -207,12 +257,26 @@ export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation,
|
||||
replyLive.current = ''; setInterrupted(null); setReply('');
|
||||
if (real) setDraft(current => current === text ? '' : current);
|
||||
setBusy(false);
|
||||
onPresence(voice ? '在听你说（模拟）' : '在你面前');
|
||||
onPresence(real ? (voiceCtl.current?.active ? '在听你说' : '在你面前') : voice ? '在听你说（模拟）' : '在你面前');
|
||||
}
|
||||
}, fail);
|
||||
}
|
||||
catch (e) {
|
||||
if (signal.aborted) return;
|
||||
// A stream that ends without its final is not always a broken connection: a newer turn (your voice, or
|
||||
// another page) takes the server's turn gate and cancels this one. The server's record says which.
|
||||
if (real) {
|
||||
const row = await recentRows(0, new AbortController().signal, 1).then(rows => rows.find(m => m.role === 'user' && m.turnId === turnId), () => undefined);
|
||||
if (signal.aborted) return;
|
||||
if (row?.status === 'cancelled') {
|
||||
gate.current.cancel();
|
||||
if (replyLive.current) setInterrupted({ text: replyLive.current, turn: turnId });
|
||||
replyLive.current = ''; setReply(''); setBusy(false);
|
||||
change(live.current.map(m => m.id === msg.id ? { ...m, status: 'cancelled' } : m));
|
||||
if (!voiceCtl.current?.active) onPresence('在你面前');
|
||||
return;
|
||||
}
|
||||
}
|
||||
change(live.current.map(m => m.id === msg.id ? { ...m, status: 'failed' } : m));
|
||||
setBusy(false);
|
||||
if (replyLive.current) setInterrupted({ text: replyLive.current, turn: turnId });
|
||||
@@ -222,7 +286,17 @@ export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation,
|
||||
onPresence('在你面前');
|
||||
}
|
||||
}
|
||||
const setVoiceMode = () => { if (real) { setNotice('语音尚未接通，文字仍可使用。'); return; } if (voice) { endVoice(); return; } setPermission('mic'); };
|
||||
const setVoiceMode = () => {
|
||||
if (real) {
|
||||
const c = voiceCtl.current; if (!c) return;
|
||||
if (c.active) { void c.end(); setNotice('语音已结束，麦克风已关闭。'); return; }
|
||||
// Ask the server again: its state may have changed since the page opened (no room, no microphone yet).
|
||||
setNotice('');
|
||||
voiceStatus(AbortSignal.timeout(8000)).then(s => { setVoiceOff(s.available ? '' : s.reason); if (s.available) { setDenied(false); setPermission('mic'); } else setNotice(s.reason); }, () => setNotice(WORDS.unknown));
|
||||
return;
|
||||
}
|
||||
if (voice) { endVoice(); return; } setPermission('mic');
|
||||
};
|
||||
const lastUser = [...view.messages].reverse().find(m => m.role === 'user');
|
||||
// Only says why her new words wait when it is your own reading that holds them; otherwise they simply come in.
|
||||
const queued = incoming.length && !busy ? (!flow ? `${name}刚又说了一句 · 继续后呈现` : !atEnd ? `${name}刚又说了一句 · 读完这段呈现` : '') : '';
|
||||
@@ -239,7 +313,14 @@ export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation,
|
||||
useLayoutEffect(() => { conversation.current?.toggleAttribute('inert', log || full); }, [log, full]);
|
||||
|
||||
return <><section ref={conversation} className={`conversation ${single ? 'single-page' : ''} ${flow ? '' : 'reading-paused'} ${holding && flow ? 'reading-held' : ''}`} aria-label="当前对话">
|
||||
{voice && <div className="voice-row"><span className="voice-wave" aria-hidden="true"><i/><i/><i/><i/></span><span>{busy ? '她在想（模拟）' : '在听你说（模拟，无收音）'}</span><button onClick={() => { stop(); setNotice(fail ? '刚才没听清，你再说一遍？（模拟识别失败）' : '她停下来了，在等你。'); }}>打断她</button><button onClick={endVoice}>结束语音</button></div>}
|
||||
{real && voiceOn && <div className="voice-row real" data-phase={voiceView.phase} role="status" aria-live="polite">
|
||||
<span className="voice-wave" aria-hidden="true"><i/><i/><i/><i/></span>
|
||||
<span className="voice-state">{voiceLabel}</span>
|
||||
{voiceView.spoken ? <span className="voice-heard said" title={voiceView.spoken}>{name}：{voiceView.spoken}<em>未记入聊天</em></span> : voiceView.heard && <span className="voice-heard" title={voiceView.heard}>「{voiceView.heard}」</span>}
|
||||
{voiceView.blocked && <button onClick={() => void voiceCtl.current?.resumeAudio()}>点一下听{name}的声音</button>}
|
||||
<button className="voice-end" onClick={() => { void voiceCtl.current?.end(); setNotice('语音已结束，麦克风已关闭。'); }}>结束语音</button>
|
||||
</div>}
|
||||
{!real && voice && <div className="voice-row"><span className="voice-wave" aria-hidden="true"><i/><i/><i/><i/></span><span>{busy ? '她在想（模拟）' : '在听你说（模拟，无收音）'}</span><button onClick={() => { stop(); setNotice(fail ? '刚才没听清，你再说一遍？（模拟识别失败）' : '她停下来了，在等你。'); }}>打断她</button><button onClick={endVoice}>结束语音</button></div>}
|
||||
<div className="stage" ref={stageRef} aria-hidden="true" onPointerEnter={e => { if (e.pointerType === 'mouse') hold('hover', true); }} onPointerLeave={() => hold('hover', false)}/>
|
||||
<p className="sr" aria-live="polite" aria-atomic="true">{page ? `${page.message.role === 'user' ? '你' : name}：${page.text}` : ''}</p>
|
||||
{status && <div className="talk-status" role="status"><span>{status}</span>{lastUser?.status === 'failed' && !busy && <button className="retry" onClick={() => void send(lastUser)}>重试</button>}</div>}
|
||||
@@ -249,20 +330,21 @@ export function ChatPanel({ state, onChange, onPresence, onVoice, onObservation,
|
||||
<span className="count" aria-label={pages.length ? `第 ${cursor + 1} 句，共 ${pages.length} 句${busy ? '，还在说' : ''}` : '还没有对话'}><span>{pages.length ? `${cursor + 1} / ${pages.length}${busy ? '…' : ''}` : '0 / 0'}</span><i ref={meter}/></span>
|
||||
<button className="icon" aria-label="下一句" disabled={atEnd} onClick={() => forward()}><Icon name="next"/></button>
|
||||
<button className="read-all" ref={fullButton} disabled={!page} onClick={() => setFull(true)}>全文</button>
|
||||
{busy && <button className="stop" onClick={stop}>停止回复</button>}
|
||||
{busy && <button className="stop" onClick={() => stop()}>停止回复</button>}
|
||||
{!flow && cursor < cursorAt(pages, head.current) && <button className="latest" onClick={toLatest}><Icon name="down"/><span>{unread > 0 ? `回到读到的地方 · 后面还有 ${unread} 句` : '回到读到的地方'}</span></button>}
|
||||
</div>
|
||||
<div className="composer"><button className={`round log-btn ${log ? 'on' : ''}`} ref={historyButton} aria-label="完整记录" aria-expanded={log} onClick={() => setLog(!log)}><Icon name="lines"/></button>
|
||||
<textarea ref={textarea} aria-label="想和她说的话" rows={1} maxLength={4000} value={draft} placeholder="想和她说点什么…" onFocus={() => typing(true)} onBlur={() => typing(false)} onChange={e => setDraft(e.target.value)} onKeyDown={e => { if (e.key === 'Enter' && !e.shiftKey && !e.nativeEvent.isComposing && e.keyCode !== 229) { e.preventDefault(); void send(); } }}/>
|
||||
<button className={`round cam ${vision ? 'on' : ''}`} aria-label={real ? '视觉观察尚未接通' : '视觉观察（模拟）'} onClick={() => { if (real) { setNotice('视觉尚未接通，不采集影像。'); return; } if (vision) { setVision(false); onObservation('窗户关上了'); } else setPermission('vision'); }}><Icon name="eye"/></button>
|
||||
<button className="round send" disabled={!draft.trim() || offline} aria-label="发送" onClick={() => void send()}><Icon name="up"/></button>
|
||||
<button className={`voice-button ${denied ? 'denied' : ''}`} aria-label={real ? '语音尚未接通' : voice ? '停止语音' : '开始模拟语音'} disabled={offline} onClick={setVoiceMode}><Icon name={voice ? 'stop' : 'mic'}/></button>
|
||||
<button className={`voice-button ${denied || (real && voiceOff && !voiceOn) ? 'denied' : ''} ${real && voiceOn ? 'on' : ''}`} aria-label={real ? voiceOn ? '结束语音' : voiceOff ? `语音不可用：${voiceOff}` : '开始语音' : voice ? '停止语音' : '开始模拟语音'} aria-pressed={real ? voiceOn : undefined} disabled={offline} onClick={setVoiceMode}><Icon name={voiceOn ? 'stop' : 'mic'}/></button>
|
||||
</div>
|
||||
</section>
|
||||
<aside className="record log-panel glass read" hidden={!log} role="dialog" aria-modal="false" aria-labelledby="record-title"><header><div><p className="eyebrow">{real ? '服务器聊天记录' : '完整记录 · 本机演示'}</p><h2 id="record-title">你们说过的话</h2></div><button className="btn-round sm" ref={logClose} aria-label="关闭记录" onClick={() => setLog(false)}><Icon name="x"/></button></header>
|
||||
<div className="scroll" ref={logRef}><p className="guide">内容完整保留；关闭后回到刚才那一句，自动呈现会接着走。</p>{record.map(m => <article key={m.id} data-life={m.lifeIds.join(' ') || undefined} data-mid={m.id} data-role={m.role} className={`message ${m.status === 'failed' ? 'fail' : ''} ${(focusLife && m.lifeIds.includes(focusLife)) || focusIds.includes(m.id) ? 'hl' : ''}`}><div className="author"><i/>{m.role === 'user' ? '你' : name}{m.proactive && <span className="tagm">她主动说的</span>}</div><p>{m.text}</p>{m.status === 'failed' && <button className="btn-text" onClick={() => void send(m)}>没发出去 · 重试</button>}{m.status === 'cancelled' && <span className="incomplete">已取消</span>}{m.live === 'cancelled' && <span className="incomplete">已停止 · 这是未完成片段，未存入聊天记录</span>}</article>)}{reply && <article className="message" data-role="companion"><div className="author"><i/>{name}</div><p>{reply}</p><span className="incomplete">还在说…</span></article>}</div></aside>
|
||||
{full && fullMessage && <aside className="record full glass read" role="dialog" aria-modal="false" aria-labelledby="full-title"><header><div><p className="eyebrow">{fullMessage.role === 'user' ? '你' : name} · 全文</p><h2 id="full-title">这一段，慢慢看</h2></div><button className="btn-round sm" ref={fullClose} aria-label="关闭全文" onClick={() => setFull(false)}><Icon name="x"/></button></header>
|
||||
<div className="scroll"><p className="guide">整段完整可读。关闭后回到刚才那一句。</p><article className="message" data-role={fullMessage.role}><div className="author"><i/>{fullMessage.role === 'user' ? '你' : name}</div><p>{fullMessage.text}</p>{fullMessage.live === 'generating' && <span className="incomplete">还在说…</span>}{fullMessage.live === 'cancelled' && <span className="incomplete">这段回复已停止，内容未完成，未存入聊天记录。</span>}</article></div></aside>}
|
||||
{permission && <Modal title={permission === 'mic' ? '开启模拟语音？' : '打开模拟视觉窗？'} onClose={() => { setPermission(''); setDenied(true); setNotice('没有开启。真实服务尚未接通，文字仍可使用。'); }} confirm="开始模拟" onConfirm={() => { if (permission === 'mic') { setDenied(false); setVoice(true); onVoice(true); voiceController.current = new AbortController(); onPresence('在听你说（模拟）'); setNotice('模拟语音没有收音，可继续用文字输入。'); } else { if (fail) { onObservation('模拟 · 看不清，视觉服务未接通'); setNotice('模拟观察失败，文字仍可使用。'); } else setVision(true); } setPermission(''); }}>本次仅模拟界面状态，不申请麦克风或摄像头权限，不采集声音或影像，也不向外接服务发送请求。</Modal>}
|
||||
{real && permission === 'mic' && <Modal title={`开始和${name}语音？`} onClose={() => { setPermission(''); setNotice('没有开启语音，文字仍可使用。'); }} confirm="开始语音" onConfirm={() => { setPermission(''); setNotice(''); void voiceCtl.current?.start(); }}>浏览器会请求麦克风权限。你的声音只送到这台服务器上{name}的语音服务，她用同一份记忆、同一段聊天回应你；你说的话和她的回答照常存进聊天记录。直接开口就能打断她。结束语音、离开交谈或关上页面时，麦克风立即关闭。<br/><small className="voice-note">语音通路还没有经过真机验收，效果可能不稳定。</small></Modal>}
|
||||
{!real && permission && <Modal title={permission === 'mic' ? '开启模拟语音？' : '打开模拟视觉窗？'} onClose={() => { setPermission(''); setDenied(true); setNotice('没有开启。真实服务尚未接通，文字仍可使用。'); }} confirm="开始模拟" onConfirm={() => { if (permission === 'mic') { setDenied(false); setVoice(true); onVoice(true); voiceController.current = new AbortController(); onPresence('在听你说（模拟）'); setNotice('模拟语音没有收音，可继续用文字输入。'); } else { if (fail) { onObservation('模拟 · 看不清，视觉服务未接通'); setNotice('模拟观察失败，文字仍可使用。'); } else setVision(true); } setPermission(''); }}>本次仅模拟界面状态，不申请麦克风或摄像头权限，不采集声音或影像，也不向外接服务发送请求。</Modal>}
|
||||
</>;
|
||||
}
|
||||
|
||||
@@ -389,6 +389,26 @@ body.offline .composer .send{opacity:.4;pointer-events:none}
|
||||
.voice-row{display:flex;align-items:center;justify-content:center;gap:12px;font-size:12px;color:var(--text-2);margin-bottom:6px;text-shadow:0 1px 10px var(--pool)}.voice-row button{color:var(--accent);padding:4px 2px}
|
||||
.voice-wave{display:inline-flex;gap:3px;align-items:center;height:14px}.voice-wave i{width:2px;height:100%;background:var(--accent);border-radius:2px;animation:r-wave 1.1s var(--ease-in-out) infinite}.voice-wave i:nth-child(2){animation-delay:.15s}.voice-wave i:nth-child(3){animation-delay:.3s}.voice-wave i:nth-child(4){animation-delay:.45s}
|
||||
@keyframes r-wave{50%{transform:scaleY(.3)}}
|
||||
/* Real voice (HEL-878): one quiet glass line above the stage, same width language as the composer. The wave says who
|
||||
has the floor without relying on colour alone (the words beside it always name the state): your open-ring colour
|
||||
while she listens, her lit accent while she speaks, a slow breath while connecting or thinking. */
|
||||
.voice-row.real{max-width:min(640px,100%);min-height:34px;padding:0 6px 0 14px;border-radius:17px;gap:10px;background:color-mix(in srgb,var(--surface) 34%,transparent);border:1px solid var(--glass-line);-webkit-backdrop-filter:blur(16px);backdrop-filter:blur(16px);animation:voice-in 280ms var(--ease-out) both}
|
||||
@keyframes voice-in{from{opacity:0;transform:translateY(6px)}}
|
||||
.voice-row.real .voice-state{white-space:nowrap;color:var(--text-1);letter-spacing:.04em}
|
||||
.voice-row.real .voice-heard{min-width:0;max-width:24em;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:var(--user);opacity:.82}
|
||||
.voice-row.real .voice-heard.said{color:var(--text-2)}
|
||||
.voice-row.real .voice-heard em{font-style:normal;font-size:11px;color:var(--warn);margin-left:8px}
|
||||
.voice-row.real button{white-space:nowrap;height:28px;padding:0 10px;border-radius:14px;transition:background-color 160ms var(--ease-out)}
|
||||
.voice-row.real .voice-end{color:var(--text-1);box-shadow:inset 0 0 0 1px var(--glass-line)}
|
||||
.voice-row.real .voice-wave i{transition:background-color var(--dur-ui) var(--ease-out),opacity var(--dur-ui) var(--ease-out)}
|
||||
.voice-row[data-phase="opening"] .voice-wave i,.voice-row[data-phase="joining"] .voice-wave i,.voice-row[data-phase="thinking"] .voice-wave i{animation:r-breath 1.4s var(--ease-in-out) infinite;transform:scaleY(.35)}
|
||||
.voice-row[data-phase="opening"] .voice-wave i,.voice-row[data-phase="joining"] .voice-wave i{background:var(--text-3)}
|
||||
.voice-row[data-phase="listening"] .voice-wave i{background:var(--user);animation-duration:1.6s;opacity:.85}
|
||||
.voice-row[data-phase="speaking"] .voice-wave i{box-shadow:0 0 6px var(--accent-glow)}
|
||||
@keyframes r-breath{50%{opacity:.3}}
|
||||
.voice-button.on{box-shadow:0 0 0 4px var(--accent-soft),0 0 18px var(--accent-glow)}
|
||||
.voice-note{display:block;margin-top:10px;color:var(--warn);font-size:12px}
|
||||
@media(hover:hover) and (pointer:fine){.voice-row.real button:hover{background:#ffffff0d}}
|
||||
|
||||
/* 完整记录与全文：房间里的阅读面，她仍在桌对面；不是模态框，导航始终可以直接切换 */
|
||||
.record{position:fixed;z-index:20;top:calc(var(--view-top) + 84px);left:4vw;width:min(470px,36vw);height:calc(var(--view-h) - 150px - var(--safe-bottom));border-radius:26px;display:flex;flex-direction:column;overflow:hidden;box-shadow:0 25px 100px #02041270,inset 0 1px 0 #ffffff10;animation:sheet-in 240ms var(--ease-out)}
|
||||
@@ -703,6 +723,7 @@ body.creating .create{opacity:1;pointer-events:auto}
|
||||
.reader-bar .latest span{max-width:9em;overflow:hidden;text-overflow:ellipsis}
|
||||
.composer{min-height:48px;padding:0 4px}.composer textarea{font-size:16px}
|
||||
.composer .round{width:36px;height:36px}
|
||||
.voice-row.real{width:100%;gap:8px;padding:0 4px 0 12px}.voice-row.real .voice-heard{flex:1;max-width:none}.voice-row.real button{height:34px}
|
||||
|
||||
.nav{padding:3px}
|
||||
.nav-item{width:70px;height:40px}
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
// The existing LiveKit browser client (livekit-client 2.22.4, the same UMD build the .12 call page at :8090 serves from
|
||||
// /static/). It is loaded from this site only when voice is switched on: the text and demo paths never fetch it.
|
||||
// See web/VENDOR.md for its source and checksum.
|
||||
import type { MicTrack, RoomEvents, VoiceRoom, VoiceTransport } from './session';
|
||||
|
||||
export const LIVEKIT_SRC = '/vendor/livekit-client-2.22.4.umd.min.js';
|
||||
const SUBTITLES = 'cg.subtitle', AGENT_STATE = 'lk.agent.state';
|
||||
|
||||
// The few parts of the client used here (typed locally; the bundle carries no type package).
|
||||
interface LkTrack { kind: string; mediaStreamTrack: MediaStreamTrack; attach(): HTMLMediaElement; detach(): HTMLMediaElement[]; stop(): void }
|
||||
interface LkParticipant { identity: string; kind?: number; attributes?: Record<string, string> }
|
||||
interface LkRoom {
|
||||
on(event: string, cb: (...args: never[]) => void): LkRoom;
|
||||
connect(url: string, token: string, options?: { autoSubscribe?: boolean }): Promise<void>;
|
||||
disconnect(stopTracks?: boolean): Promise<void>;
|
||||
startAudio(): Promise<void>;
|
||||
canPlaybackAudio: boolean;
|
||||
remoteParticipants: Map<string, LkParticipant>;
|
||||
localParticipant: { publishTrack(track: LkTrack, options?: { source?: string }): Promise<unknown> };
|
||||
}
|
||||
export interface Livekit {
|
||||
Room: new (options?: Record<string, unknown>) => LkRoom;
|
||||
RoomEvent: Record<'TrackSubscribed' | 'TrackUnsubscribed' | 'ParticipantConnected' | 'ParticipantAttributesChanged' | 'DataReceived' | 'Disconnected' | 'AudioPlaybackStatusChanged', string>;
|
||||
Track: { Source: { Microphone: string } };
|
||||
ParticipantKind?: { AGENT?: number };
|
||||
createLocalAudioTrack(options?: MediaTrackConstraints): Promise<LkTrack>;
|
||||
}
|
||||
declare global { interface Window { LivekitClient?: Livekit } }
|
||||
|
||||
let loading: Promise<Livekit> | null = null;
|
||||
export function loadLivekit(src = LIVEKIT_SRC): Promise<Livekit> {
|
||||
if (window.LivekitClient) return Promise.resolve(window.LivekitClient);
|
||||
loading ??= new Promise<Livekit>((ok, ko) => {
|
||||
const script = document.createElement('script');
|
||||
script.src = src; script.async = true;
|
||||
script.onload = () => window.LivekitClient ? ok(window.LivekitClient) : ko(new Error('livekit-client missing'));
|
||||
script.onerror = () => { script.remove(); ko(new Error('livekit-client failed to load')); };
|
||||
document.head.append(script);
|
||||
}).catch(e => { loading = null; throw e; });
|
||||
return loading;
|
||||
}
|
||||
|
||||
const tracks = new WeakMap<MicTrack, LkTrack>();
|
||||
export function livekitTransport(lk: Livekit): VoiceTransport {
|
||||
const isAgent = (p: LkParticipant) => (lk.ParticipantKind?.AGENT !== undefined && p.kind === lk.ParticipantKind.AGENT) || AGENT_STATE in (p.attributes || {}) || p.identity.startsWith('agent-');
|
||||
return {
|
||||
async capture() {
|
||||
// Same capture defaults as the :8090 call page: echo cancellation keeps her own voice out of the microphone.
|
||||
const track = await lk.createLocalAudioTrack({ echoCancellation: true, noiseSuppression: true, autoGainControl: true });
|
||||
const mic: MicTrack = { stop: () => { track.stop(); track.mediaStreamTrack.stop(); }, onEnded: cb => track.mediaStreamTrack.addEventListener('ended', cb, { once: true }) };
|
||||
tracks.set(mic, track);
|
||||
return mic;
|
||||
},
|
||||
async join(url, token, on: RoomEvents, signal) {
|
||||
const room = new lk.Room({ adaptiveStream: true, dynacast: true });
|
||||
// Her audio lives in one hidden holder; closing empties it, so nothing of an old call can keep playing.
|
||||
const holder = document.createElement('div'), E = lk.RoomEvent;
|
||||
holder.hidden = true; holder.dataset.voice = 'her'; document.body.append(holder);
|
||||
let closing = false, hushed = false;
|
||||
const players = () => [...holder.querySelectorAll<HTMLMediaElement>('audio,video')];
|
||||
const close = async () => {
|
||||
if (closing) return; closing = true;
|
||||
players().forEach(el => { el.muted = true; el.pause(); el.srcObject = null; });
|
||||
holder.remove();
|
||||
await room.disconnect(true).catch(() => {});
|
||||
};
|
||||
room.on(E.TrackSubscribed, ((track: LkTrack) => { if (closing || track.kind !== 'audio') return; const el = track.attach(); el.muted = hushed; holder.append(el); }) as (...a: never[]) => void);
|
||||
room.on(E.TrackUnsubscribed, ((track: LkTrack) => { track.detach().forEach(el => el.remove()); }) as (...a: never[]) => void);
|
||||
room.on(E.ParticipantConnected, ((p: LkParticipant) => { if (!closing && isAgent(p)) on.agentJoined(); }) as (...a: never[]) => void);
|
||||
room.on(E.ParticipantAttributesChanged, ((changed: Record<string, string>, p: LkParticipant) => { if (!closing && AGENT_STATE in changed && isAgent(p)) on.agentState(changed[AGENT_STATE]); }) as (...a: never[]) => void);
|
||||
room.on(E.DataReceived, ((payload: Uint8Array, _p: unknown, _kind: unknown, topic?: string) => {
|
||||
if (closing || topic !== SUBTITLES) return;
|
||||
try { on.subtitle(JSON.parse(new TextDecoder().decode(payload))); } catch { /* A broken frame is dropped, never shown. */ }
|
||||
}) as (...a: never[]) => void);
|
||||
room.on(E.Disconnected, (() => { if (!closing) { void close(); on.dropped(); } }) as (...a: never[]) => void);
|
||||
room.on(E.AudioPlaybackStatusChanged, (() => { if (!closing) on.audioBlocked(!room.canPlaybackAudio); }) as (...a: never[]) => void);
|
||||
const abort = () => void close();
|
||||
signal.addEventListener('abort', abort, { once: true });
|
||||
try { await room.connect(url, token, { autoSubscribe: true }); }
|
||||
catch (e) { await close(); throw e; }
|
||||
if (signal.aborted) { await close(); throw new DOMException('ended', 'AbortError'); }
|
||||
signal.removeEventListener('abort', abort);
|
||||
room.remoteParticipants.forEach(p => { if (!isAgent(p)) return; on.agentJoined(); const state = p.attributes?.[AGENT_STATE]; if (state) on.agentState(state); });
|
||||
void room.startAudio().catch(() => {}).then(() => { if (!closing && !room.canPlaybackAudio) on.audioBlocked(true); });
|
||||
const voiceRoom: VoiceRoom = {
|
||||
async publish(mic) { const track = tracks.get(mic); if (!track) throw new Error('unknown microphone'); await room.localParticipant.publishTrack(track, { source: lk.Track.Source.Microphone }); },
|
||||
hush(quiet) { hushed = quiet; players().forEach(el => { el.muted = quiet; }); },
|
||||
resumeAudio: () => room.startAudio(),
|
||||
close,
|
||||
};
|
||||
return voiceRoom;
|
||||
},
|
||||
};
|
||||
}
|
||||
export const loadTransport = async (): Promise<VoiceTransport> => livekitTransport(await loadLivekit());
|
||||
@@ -0,0 +1,287 @@
|
||||
// ── Continuous voice on the approved conversation (R5, HEL-878). ──
|
||||
// The server opens a room and hands out a short join token (POST /api/voice/session, keys never reach the page); the
|
||||
// browser joins with the existing LiveKit client and publishes the microphone. Her words are spoken by the dedicated
|
||||
// worker, which goes through the same /api/chat as text; the page never confirms a receipt for voice (the worker does,
|
||||
// only after real playout) and never turns subtitles into chat rows: the record is re-read from /api/messages.
|
||||
//
|
||||
// One rule holds the whole lifecycle together: every call starts a new generation, and anything that answers for an
|
||||
// older generation (a late token, a late join, a late microphone, a late room event) is cleaned up, never shown.
|
||||
import type { Message } from '../domain/model';
|
||||
import { api, toMessage, within, type MessagePage } from '../adapters/real';
|
||||
import { serverId } from '../adapters/inbox';
|
||||
|
||||
export type VoicePhase = 'off' | 'opening' | 'joining' | 'listening' | 'thinking' | 'speaking';
|
||||
export interface VoiceView {
|
||||
phase: VoicePhase;
|
||||
// What the recogniser heard of you so far (a draft; it is replaced, never stored).
|
||||
heard: string;
|
||||
// Her line spoken after a failure (φ): heard, not part of the record.
|
||||
spoken: string;
|
||||
// Why voice is off or what went wrong; empty while it simply runs.
|
||||
notice: string;
|
||||
// The browser held her audio back until a tap (autoplay rules); `resume` lets it out.
|
||||
blocked: boolean;
|
||||
}
|
||||
export const OFF: VoiceView = { phase: 'off', heard: '', spoken: '', notice: '', blocked: false };
|
||||
export interface Subtitle { kind: 'interim' | 'user_final' | 'assistant_final' | 'error'; role: 'user' | 'assistant'; text: string }
|
||||
export function subtitle(v: unknown): Subtitle | null {
|
||||
const s = v as Record<string, unknown> | null;
|
||||
if (!s || typeof s !== 'object' || typeof s.text !== 'string' || !['interim', 'user_final', 'assistant_final', 'error'].includes(s.kind as string)) return null;
|
||||
return { kind: s.kind as Subtitle['kind'], role: s.role === 'assistant' ? 'assistant' : 'user', text: s.text };
|
||||
}
|
||||
|
||||
// ── What the controller needs from a real-time client. livekit.ts implements it with livekit-client; tests use fakes. ──
|
||||
export interface MicTrack { stop(): void; onEnded(cb: () => void): void }
|
||||
export interface RoomEvents {
|
||||
agentJoined(): void;
|
||||
// The worker's own state attribute (lk.agent.state): initializing / listening / thinking / speaking.
|
||||
agentState(state: string): void;
|
||||
subtitle(payload: unknown): void;
|
||||
// The room went away without us closing it (server ended it, network, kicked).
|
||||
dropped(): void;
|
||||
audioBlocked(blocked: boolean): void;
|
||||
}
|
||||
export interface VoiceRoom { publish(mic: MicTrack): Promise<void>; hush(on: boolean): void; resumeAudio(): Promise<void>; close(): Promise<void> }
|
||||
export interface VoiceTransport { capture(): Promise<MicTrack>; join(url: string, token: string, on: RoomEvents, signal: AbortSignal): Promise<VoiceRoom> }
|
||||
|
||||
export interface VoiceAvailability { available: boolean; reason: string }
|
||||
export interface SessionGrant { url: string; token: string; room: string; session_id: string; epoch: number; expires_at: number }
|
||||
|
||||
export const WORDS = {
|
||||
notConfigured: '语音还没有在这台服务上配置，文字仍可使用。',
|
||||
ttsDown: '她的声音服务这会儿不在，稍后再试试。文字仍可使用。',
|
||||
noRoute: '这台服务还没有语音通路，文字仍可使用。',
|
||||
unknown: '暂时读不到语音状态，文字仍可使用。',
|
||||
insecure: '这个地址不是安全连接，浏览器不允许使用麦克风。需要从 HTTPS 入口打开。',
|
||||
denied: '没有拿到麦克风权限，语音没有开始。文字仍可使用。',
|
||||
noMic: '没有找到可用的麦克风，语音没有开始。',
|
||||
micBusy: '麦克风被别的程序占用了，语音没有开始。',
|
||||
client: '语音组件没有加载出来，文字仍可使用。',
|
||||
unreachable: '通话服务暂时连不上，稍后再试试。',
|
||||
noAgent: '她没有接上这次通话，麦克风已经关闭。文字仍可使用。',
|
||||
dropped: '通话断开了，麦克风已经关闭。文字仍可使用。',
|
||||
micGone: '麦克风被关掉了，语音已结束。',
|
||||
offline: '网络断开了，语音已结束，麦克风已关闭。',
|
||||
} as const;
|
||||
|
||||
export interface VoiceDeps {
|
||||
transport: () => Promise<VoiceTransport>;
|
||||
onView: (view: VoiceView) => void;
|
||||
// The record may have changed (a voice turn began, ended or was cut): re-read /api/messages.
|
||||
onRecord: () => void;
|
||||
// You finished saying something by voice: a text reply still streaming has been cancelled by the server.
|
||||
onSpoke: () => void;
|
||||
fetch?: typeof fetch;
|
||||
// How long the room may wait for her worker to join before giving up honestly.
|
||||
agentWaitMs?: number;
|
||||
secure?: () => boolean;
|
||||
}
|
||||
|
||||
interface Live { gen: number; abort: AbortController; mic?: MicTrack; room?: VoiceRoom; session?: string; agent: boolean; stated: boolean; hushed: boolean; timer?: ReturnType<typeof setTimeout> }
|
||||
|
||||
const AGENT_PHASE: Record<string, VoicePhase> = { initializing: 'joining', listening: 'listening', thinking: 'thinking', speaking: 'speaking' };
|
||||
|
||||
export async function voiceStatus(signal: AbortSignal, f: typeof fetch = fetch): Promise<VoiceAvailability> {
|
||||
let response: Response;
|
||||
try { response = await f('/api/voice/status', { signal }); } catch (e) { if (signal.aborted) throw e; return { available: false, reason: WORDS.unknown }; }
|
||||
// A server from before R5 has no such route: that is "not wired", not a failure.
|
||||
if (response.status === 404 || response.status === 405) return { available: false, reason: WORDS.noRoute };
|
||||
const data = await response.json().catch(() => null) as { configured?: unknown; tts_ok?: unknown } | null;
|
||||
if (!response.ok || !data) return { available: false, reason: WORDS.unknown };
|
||||
if (data.configured !== true) return { available: false, reason: WORDS.notConfigured };
|
||||
if (data.tts_ok !== true) return { available: false, reason: WORDS.ttsDown };
|
||||
return { available: true, reason: '' };
|
||||
}
|
||||
|
||||
function micFailure(e: unknown, secure: boolean) {
|
||||
const name = (e as { name?: string } | null)?.name || '';
|
||||
if (!secure) return WORDS.insecure;
|
||||
if (name === 'NotAllowedError' || name === 'SecurityError' || name === 'PermissionDeniedError') return WORDS.denied;
|
||||
if (name === 'NotFoundError' || name === 'DevicesNotFoundError' || name === 'OverconstrainedError') return WORDS.noMic;
|
||||
if (name === 'NotReadableError' || name === 'AbortError') return WORDS.micBusy;
|
||||
return WORDS.denied;
|
||||
}
|
||||
|
||||
const OWED = 'cg.voice.owed';
|
||||
function owedSessions(): string[] {
|
||||
try { const v = JSON.parse(localStorage.getItem(OWED) || '[]'); return Array.isArray(v) ? v.filter((s): s is string => typeof s === 'string' && !!s && s.length < 100) : []; } catch { return []; }
|
||||
}
|
||||
|
||||
export class VoiceController {
|
||||
private gen = 0;
|
||||
private live: Live | null = null;
|
||||
private view: VoiceView = OFF;
|
||||
private f: typeof fetch;
|
||||
constructor(private deps: VoiceDeps) { this.f = deps.fetch ?? ((...a) => fetch(...a)); }
|
||||
|
||||
get phase() { return this.view.phase; }
|
||||
get active() { return this.live !== null; }
|
||||
|
||||
private show(next: Partial<VoiceView>, gen: number) {
|
||||
if (gen !== this.gen) return;
|
||||
this.view = { ...this.view, ...next };
|
||||
this.deps.onView(this.view);
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.end();
|
||||
void this.settle();
|
||||
const gen = ++this.gen, live: Live = { gen, abort: new AbortController(), agent: false, stated: false, hushed: false };
|
||||
this.live = live;
|
||||
this.view = { ...OFF, phase: 'opening' }; this.deps.onView(this.view);
|
||||
const stale = () => gen !== this.gen;
|
||||
const fail = async (notice: string) => { if (stale()) return; this.live = null; this.gen++; await this.teardown(live); this.view = { ...OFF, notice }; this.deps.onView(this.view); };
|
||||
const secure = (this.deps.secure ?? (() => window.isSecureContext && !!navigator.mediaDevices?.getUserMedia))();
|
||||
if (!secure) return fail(WORDS.insecure);
|
||||
|
||||
// 1. The honest switch: configured and her voice service reachable (no room, no microphone yet).
|
||||
let status: VoiceAvailability;
|
||||
try { status = await voiceStatus(live.abort.signal, this.f); } catch { return; }
|
||||
if (stale()) return;
|
||||
if (!status.available) return fail(status.reason);
|
||||
|
||||
// 2. The client, then the microphone. A refusal here opens nothing on the server.
|
||||
let transport: VoiceTransport;
|
||||
try { transport = await this.deps.transport(); } catch { return fail(WORDS.client); }
|
||||
if (stale()) return;
|
||||
let mic: MicTrack;
|
||||
try { mic = await transport.capture(); } catch (e) { return fail(micFailure(e, true)); }
|
||||
if (stale()) { mic.stop(); return; }
|
||||
live.mic = mic;
|
||||
mic.onEnded(() => { if (!stale()) void fail(WORDS.micGone); });
|
||||
|
||||
// 3. A room of her own for this call. This request is not cut short by ending: if it was already under way, its
|
||||
// answer still owns a room on the server, so it is waited for and given back.
|
||||
let response: Response, data: (SessionGrant & { detail?: unknown }) | null;
|
||||
try {
|
||||
response = await this.f('/api/voice/session', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}', signal: AbortSignal.timeout(15000) });
|
||||
data = await response.json().catch(() => null);
|
||||
} catch { return fail(WORDS.unreachable); }
|
||||
if (response.ok && typeof data?.session_id === 'string') live.session = data.session_id;
|
||||
if (stale()) { await this.teardown(live); return; }
|
||||
if (!response.ok || !data?.token || !data.url || !live.session) return fail(typeof data?.detail === 'string' && data.detail ? data.detail : WORDS.unreachable);
|
||||
const grant = data;
|
||||
|
||||
// 4. Join and speak.
|
||||
this.show({ phase: 'joining' }, gen);
|
||||
const events: RoomEvents = {
|
||||
agentJoined: () => { if (stale() || live.agent) return; live.agent = true; clearTimeout(live.timer); if (!live.stated) this.show({ phase: 'listening' }, gen); },
|
||||
agentState: state => {
|
||||
if (stale()) return;
|
||||
const phase = AGENT_PHASE[state]; if (!phase) return;
|
||||
live.stated = true;
|
||||
if (phase !== 'joining' && !live.agent) events.agentJoined();
|
||||
// After text took over, her old voice stays silent until she is back to listening.
|
||||
if (live.hushed && phase !== 'speaking') { live.hushed = false; live.room?.hush(false); }
|
||||
this.show({ phase, ...(phase === 'thinking' ? { spoken: '' } : {}) }, gen);
|
||||
},
|
||||
subtitle: payload => {
|
||||
if (stale()) return;
|
||||
const s = subtitle(payload); if (!s) return;
|
||||
if (s.kind === 'interim') { this.show({ heard: s.text, spoken: '', ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); if (live.hushed && !live.stated) { live.hushed = false; live.room?.hush(false); } return; }
|
||||
if (s.kind === 'user_final') { this.show({ heard: s.text, spoken: '', ...(live.stated ? {} : { phase: 'thinking' as const }) }, gen); this.deps.onSpoke(); this.deps.onRecord(); return; }
|
||||
if (s.kind === 'assistant_final') { this.show({ heard: '', spoken: '', ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); this.deps.onRecord(); return; }
|
||||
this.show({ heard: '', spoken: s.text, ...(live.stated ? {} : { phase: 'listening' as const }) }, gen); this.deps.onRecord();
|
||||
},
|
||||
dropped: () => { if (!stale()) void fail(WORDS.dropped); },
|
||||
audioBlocked: blocked => this.show({ blocked }, gen),
|
||||
};
|
||||
let room: VoiceRoom;
|
||||
try { room = await transport.join(grant.url, grant.token, events, live.abort.signal); } catch { return fail(WORDS.unreachable); }
|
||||
if (stale()) { live.room = room; await this.teardown(live); return; }
|
||||
live.room = room;
|
||||
try { await room.publish(mic); } catch { return fail(WORDS.unreachable); }
|
||||
if (stale()) { await this.teardown(live); return; }
|
||||
if (!live.agent) live.timer = setTimeout(() => { if (!stale() && !live.agent) void fail(WORDS.noAgent); }, this.deps.agentWaitMs ?? 20000);
|
||||
}
|
||||
|
||||
// Stop everything now: microphone first, then the room and its audio, then the server-side room. Safe to call twice.
|
||||
async end(notice = ''): Promise<void> {
|
||||
const live = this.live;
|
||||
this.gen++; this.live = null;
|
||||
const was = this.view.phase !== 'off' || this.view.notice !== notice;
|
||||
this.view = { ...OFF, notice };
|
||||
if (was) this.deps.onView(this.view);
|
||||
if (live) { await this.teardown(live); this.deps.onRecord(); }
|
||||
}
|
||||
|
||||
// Page is going away: same order, without waiting; the DELETE survives the unload.
|
||||
unload() {
|
||||
const live = this.live;
|
||||
this.gen++; this.live = null; this.view = OFF;
|
||||
if (live) void this.teardown(live, true);
|
||||
}
|
||||
|
||||
// You sent text while she was answering by voice: the server cancels her voice turn; keep its last audio silent.
|
||||
hush() {
|
||||
const live = this.live;
|
||||
if (!live?.room || (this.view.phase !== 'speaking' && this.view.phase !== 'thinking')) return;
|
||||
live.hushed = true; live.room.hush(true);
|
||||
this.show({ spoken: '' }, live.gen);
|
||||
}
|
||||
|
||||
async resumeAudio() { const live = this.live; if (!live?.room) return; try { await live.room.resumeAudio(); this.show({ blocked: false }, live.gen); } catch { /* Still blocked; the tap stays offered. */ } }
|
||||
|
||||
private async teardown(live: Live, keepalive = false) {
|
||||
live.abort.abort(); clearTimeout(live.timer);
|
||||
try { live.mic?.stop(); } catch { /* Already stopped. */ }
|
||||
const room = live.room; live.room = undefined;
|
||||
const session = live.session; live.session = undefined;
|
||||
const closing = room ? room.close().catch(() => {}) : Promise.resolve();
|
||||
// The server deletes the room: her worker leaves and its job ends even if the page could not say goodbye itself.
|
||||
// A DELETE that could not get through (offline) is kept and sent again when the network is back.
|
||||
const ending = session ? this.release(session, keepalive) : Promise.resolve();
|
||||
await Promise.all([closing, ending]);
|
||||
}
|
||||
// Kept in this browser (not only in memory), so leaving the conversation or reloading does not forget them. Until
|
||||
// one gets through, the server side still ends on its own: the worker's job ends when the page's participant is
|
||||
// gone from the room, and the session registry expires (runbook-hel877-voice §3.4/§3.5).
|
||||
private owe(session: string, on: boolean) {
|
||||
const list = new Set(owedSessions());
|
||||
if (on) list.add(session); else list.delete(session);
|
||||
try { if (list.size) localStorage.setItem(OWED, JSON.stringify([...list].slice(-20))); else localStorage.removeItem(OWED); } catch { /* Memory only for this visit. */ }
|
||||
if (on) this.owed.add(session); else this.owed.delete(session);
|
||||
}
|
||||
private owed = new Set<string>();
|
||||
private release(session: string, keepalive = false) {
|
||||
// Noted before sending: if the page is gone before the answer, the next visit sends it again (404 = done).
|
||||
this.owe(session, true);
|
||||
return this.f(`/api/voice/session/${encodeURIComponent(session)}`, { method: 'DELETE', keepalive })
|
||||
.then(r => { this.owe(session, !(r.ok || r.status === 404)); }, () => { this.owe(session, true); });
|
||||
}
|
||||
// Back online, arriving, or starting again: give back rooms an earlier end could not reach the server to delete.
|
||||
async settle() { await Promise.all([...new Set([...this.owed, ...owedSessions()])].map(s => this.release(s))); }
|
||||
}
|
||||
|
||||
// The record's view of voice: rows of your and her voice turns, re-read from the server. Her reach-outs (proactive)
|
||||
// are left to the inbox reader so they keep their own entrance; text turns already on the page are matched by turn id.
|
||||
export function mergeTurns(known: Message[], rows: Message[]): Message[] | null {
|
||||
const byId = new Map(known.map((m, i) => [m.id, i])), turns = new Set(known.flatMap(m => m.turnId ? [`${m.role}:${m.turnId}`] : []));
|
||||
let next: Message[] | null = null;
|
||||
for (const m of [...rows].sort((a, b) => serverId(a) - serverId(b))) {
|
||||
if (m.proactive || !serverId(m)) continue;
|
||||
const at = byId.get(m.id);
|
||||
if (at !== undefined) {
|
||||
const old = (next ?? known)[at];
|
||||
if (old.status !== m.status || old.text !== m.text) { next ??= [...known]; next[at] = { ...old, status: m.status, text: m.text }; }
|
||||
continue;
|
||||
}
|
||||
if (m.turnId && turns.has(`${m.role}:${m.turnId}`)) continue;
|
||||
next ??= [...known]; next.push(m); byId.set(m.id, next.length - 1); if (m.turnId) turns.add(`${m.role}:${m.turnId}`);
|
||||
}
|
||||
return next;
|
||||
}
|
||||
// Newest rows down to `since` (exclusive), at most `pages` pages of 50. Voice turns are few per read; a longer gap is
|
||||
// left to the next re-read rather than read without bound.
|
||||
export async function recentRows(since: number, signal: AbortSignal, pages = 4): Promise<Message[]> {
|
||||
const rows: Message[] = [];
|
||||
let before: number | null = null;
|
||||
for (let i = 0; i < pages; i++) {
|
||||
const page: MessagePage = await api(`messages?limit=50${before ? `&before=${before}` : ''}`, { signal: within(signal, 10000) });
|
||||
const list = (Array.isArray(page.messages) ? page.messages : []).filter(m => Number.isSafeInteger(m?.id) && m.id > 0 && typeof m.text === 'string');
|
||||
rows.push(...list.filter(m => m.id > since).map(toMessage));
|
||||
if (list.some(m => m.id <= since) || !page.next_before || !Number.isSafeInteger(page.next_before)) break;
|
||||
before = page.next_before;
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
// HEL-878 browser check only: a scripted stand-in for the livekit-client UMD global, served by tests/hel878_mock.mjs in
|
||||
// place of the vendored build. Same names the page uses (Room, RoomEvent, Track.Source, ParticipantKind,
|
||||
// createLocalAudioTrack); no WebRTC, no signalling. The microphone is a real MediaStreamTrack from the browser's fake
|
||||
// device (--use-fake-device-for-media-stream), so stopping it can be observed. window.__lk drives her side.
|
||||
(() => {
|
||||
const RoomEvent = { TrackSubscribed: 'trackSubscribed', TrackUnsubscribed: 'trackUnsubscribed', ParticipantConnected: 'participantConnected', ParticipantAttributesChanged: 'participantAttributesChanged', DataReceived: 'dataReceived', Disconnected: 'disconnected', AudioPlaybackStatusChanged: 'audioPlaybackChanged' };
|
||||
const lk = window.__lk = { rooms: [], connects: [], published: [], mics: [], connectDelay: 200 };
|
||||
class Room {
|
||||
constructor() { this.handlers = {}; this.remoteParticipants = new Map(); this.canPlaybackAudio = true; this.state = 'new'; const self = this; this.localParticipant = { tracks: [], async publishTrack(t) { if (self.state !== 'connected') throw new Error('not connected'); self.localParticipant.tracks.push(t); lk.published.push(t); } }; lk.rooms.push(this); }
|
||||
on(e, cb) { (this.handlers[e] ||= []).push(cb); return this; }
|
||||
emit(e, ...a) { (this.handlers[e] || []).forEach(cb => cb(...a)); }
|
||||
async connect(url, token) { lk.connects.push({ url, token }); this.state = 'connecting'; await new Promise(r => setTimeout(r, lk.connectDelay)); if (this.state !== 'connecting') throw new Error('aborted'); this.state = 'connected'; }
|
||||
async disconnect(stop = true) { if (this.state === 'gone') return; this.state = 'gone'; if (stop) this.localParticipant.tracks.forEach(t => t.stop()); this.emit(RoomEvent.Disconnected); }
|
||||
async startAudio() { this.canPlaybackAudio = true; }
|
||||
}
|
||||
const live = () => lk.rooms.filter(r => r.state === 'connected').at(-1);
|
||||
// Her worker joins: an agent participant with a looping tone as her voice track.
|
||||
lk.join = () => {
|
||||
const room = live(); if (!room) return false;
|
||||
const p = { identity: 'agent-AJ_fake', kind: 4, attributes: { 'lk.agent.state': 'listening' } };
|
||||
room.remoteParticipants.set(p.identity, p); room.emit(RoomEvent.ParticipantConnected, p);
|
||||
const ctx = new AudioContext(), osc = ctx.createOscillator(), dest = ctx.createMediaStreamDestination(); osc.connect(dest); osc.start();
|
||||
const mst = dest.stream.getAudioTracks()[0];
|
||||
const track = { kind: 'audio', mediaStreamTrack: mst, els: [], attach() { const el = document.createElement('audio'); el.srcObject = new MediaStream([mst]); el.autoplay = true; this.els.push(el); return el; }, detach() { const e = this.els; this.els = []; return e; }, stop() { mst.stop(); } };
|
||||
room.emit(RoomEvent.TrackSubscribed, track, {}, p);
|
||||
return true;
|
||||
};
|
||||
lk.state = s => { const room = lk.rooms.at(-1), p = [...room.remoteParticipants.values()][0]; if (!p) return false; p.attributes['lk.agent.state'] = s; room.emit(RoomEvent.ParticipantAttributesChanged, { 'lk.agent.state': s }, p); return true; };
|
||||
lk.say = obj => { const room = lk.rooms.at(-1); room.emit(RoomEvent.DataReceived, new TextEncoder().encode(JSON.stringify(obj)), undefined, 0, 'cg.subtitle'); return true; };
|
||||
lk.drop = () => { const room = live(); room.state = 'gone'; room.emit(RoomEvent.Disconnected); return true; };
|
||||
lk.block = () => { const room = live(); room.canPlaybackAudio = false; room.emit(RoomEvent.AudioPlaybackStatusChanged); return true; };
|
||||
async function createLocalAudioTrack(constraints) {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: constraints || true }), mst = stream.getAudioTracks()[0];
|
||||
lk.mics.push(mst);
|
||||
return { kind: 'audio', mediaStreamTrack: mst, attach() { return document.createElement('audio'); }, detach() { return []; }, stop() { mst.stop(); } };
|
||||
}
|
||||
window.LivekitClient = { Room, RoomEvent, Track: { Source: { Microphone: 'microphone' } }, ParticipantKind: { AGENT: 4 }, createLocalAudioTrack };
|
||||
})();
|
||||
@@ -0,0 +1,84 @@
|
||||
// HEL-878 isolated contract mock for real voice on the approved conversation. Serves web/dist, the persisted-message
|
||||
// contract (as tests/hel875_mock.mjs) and the HEL-877 voice router (src/cybergril/voice/router.py): GET
|
||||
// /api/voice/status, POST /api/voice/session -> 201 grant, DELETE /api/voice/session/{id}. The LiveKit client path is
|
||||
// answered with tests/fake-livekit.js (a scripted stand-in, no WebRTC); the real vendored build stays reachable at
|
||||
// /__real-sdk.js for the API-surface check only. Synthetic data only: no model, no LiveKit, no TTS, no network.
|
||||
// Usage: node tests/hel878_mock.mjs <port>. Control GETs: /__voice?configured=&tts=, /__refuse/<code|0>,
|
||||
// /__voice-turn?stage=user|reply|cancel&text=, /__cut, /__sessions, /__log, /__rows.
|
||||
import { createServer } from 'node:http';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { extname, join, normalize } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const dist = fileURLToPath(new URL('../dist/', import.meta.url)), fake = fileURLToPath(new URL('./fake-livekit.js', import.meta.url));
|
||||
const port = Number(process.argv[2] || 8878);
|
||||
const T0 = Date.UTC(2026, 9, 26, 12, 0);
|
||||
const identity = { id: '1', name: '栖', birth: '2000-03-14', hour: '子', relation: '朋友', nickname: '', boundary: '', color: { pillars: { year: '庚辰', month: '己卯', day: '丙午', hour: '戊子' }, traits: '合成测试档案' } };
|
||||
const epoch = 3, log = [], rows = [], sessions = new Map();
|
||||
let voice = { configured: true, tts: true }, refuse = 0, made = 0, voiceTurn = 0, textStream = null;
|
||||
const add = row => { const id = rows.length + 1; rows.push({ id, ts: T0 + id * 60e3, status: 'sent', lifeIds: [], ...row }); return id; };
|
||||
add({ role: 'user', text: '下雨天的时候，你会想听什么？', turn_id: '00000000-0000-4000-8000-000000000001' });
|
||||
add({ role: 'companion', text: '会想把窗开一条缝，听雨落在铁皮檐上。你呢，今天累不累？', turn_id: '00000000-0000-4000-8000-000000000001' });
|
||||
const dto = r => ({ id: r.id, ts: r.ts, role: r.role, text: r.text, turn_id: r.turn_id, status: r.status, lifeIds: r.lifeIds, ...(r.proactive ? { proactive: true } : {}) });
|
||||
const send = (res, status, body, type = 'application/json; charset=utf-8') => { res.writeHead(status, { 'Content-Type': type, 'Cache-Control': 'no-store' }); res.end(typeof body === 'string' || Buffer.isBuffer(body) ? body : JSON.stringify(body)); };
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||||
const REFUSALS = { 503: '她的声音服务这会儿不在，稍后再试试。', 409: '还没有创建真实伴侣，语音也不会开始。' };
|
||||
|
||||
createServer(async (req, res) => {
|
||||
const url = new URL(req.url, 'http://x'), p = url.pathname;
|
||||
if (!p.startsWith('/__') && !p.startsWith('/assets/') && !p.startsWith('/vendor/')) log.push(`${req.method} ${p}${url.search}`);
|
||||
if (p === '/__log') return send(res, 200, log.splice(0));
|
||||
if (p === '/__rows') return send(res, 200, rows.map(dto));
|
||||
if (p === '/__sessions') return send(res, 200, { open: [...sessions.keys()], made });
|
||||
if (p === '/__voice') { voice = { configured: url.searchParams.get('configured') !== '0', tts: url.searchParams.get('tts') !== '0' }; return send(res, 200, voice); }
|
||||
if (p.startsWith('/__refuse/')) { refuse = Number(p.slice(10)); return send(res, 200, { refuse }); }
|
||||
// What the worker's /api/chat call leaves in the record, one stage at a time (the browser never posts it).
|
||||
if (p === '/__voice-turn') {
|
||||
const stage = url.searchParams.get('stage'), text = url.searchParams.get('text') || '';
|
||||
if (stage === 'user') { voiceTurn++; add({ role: 'user', text, turn_id: `voice-${voiceTurn}`, status: 'pending' }); }
|
||||
if (stage === 'reply') { add({ role: 'companion', text, turn_id: `voice-${voiceTurn}` }); rows.forEach(r => { if (r.turn_id === `voice-${voiceTurn}`) r.status = 'sent'; }); }
|
||||
if (stage === 'cancel') rows.forEach(r => { if (r.turn_id === `voice-${voiceTurn}`) r.status = 'cancelled'; });
|
||||
return send(res, 200, { turn: voiceTurn });
|
||||
}
|
||||
// A voice turn took the server's turn gate: the text stream in flight ends without a final; its row is cancelled.
|
||||
if (p === '/__cut') { const s = textStream; textStream = null; if (s) { rows.forEach(r => { if (r.turn_id === s.turn_id && r.role === 'user') r.status = 'cancelled'; }); s.cut = true; s.res.end(); } return send(res, 200, { cut: !!s }); }
|
||||
if (p === '/__real-sdk.js') return send(res, 200, await readFile(join(dist, 'vendor/livekit-client-2.22.4.umd.min.js')), 'text/javascript');
|
||||
if (p.startsWith('/vendor/livekit-client')) return send(res, 200, await readFile(fake), 'text/javascript');
|
||||
if (p === '/api/health') return send(res, 200, { ok: true, version: 'R5-candidate', epoch, llm: { ok: true, model: 'mock' }, backend: { life: true, persona: true, receipt: true, web_connected: true }, life_runtime: { state: 'running', timezone: 'Asia/Shanghai', budget_warning: false, budget_stopped: false, last: null }, capabilities: { text: true, proactive: true, voice: false, memory: true } });
|
||||
if (p === '/api/companion') return send(res, 200, identity);
|
||||
if (p === '/api/messages') {
|
||||
const limit = Math.min(200, Number(url.searchParams.get('limit') || 200)), before = Number(url.searchParams.get('before') || Number.MAX_SAFE_INTEGER);
|
||||
const desc = rows.filter(r => r.id < before).sort((a, b) => b.id - a.id).slice(0, limit + 1), more = desc.length > limit, page = desc.slice(0, limit);
|
||||
return send(res, 200, { messages: [...page].reverse().map(dto), next_before: more ? page[page.length - 1].id : null, epoch });
|
||||
}
|
||||
if (p === '/api/persona') return send(res, 200, { state: 'ready', interpretation: null, seeds: [], growth: [] });
|
||||
if (p === '/api/life') return send(res, 200, { events: [], next_before: null, epoch });
|
||||
if (p === '/api/memories') return send(res, 200, { memories: [], next_before: null, epoch });
|
||||
if (p === '/api/voice/status') return send(res, 200, voice.configured ? { configured: true, tts_ok: voice.tts, agent_name: 'cybergril-voice-agent', voice: false, livekit_url: 'wss://192.168.200.12:7880' } : { configured: false, tts_ok: false, agent_name: null, voice: false });
|
||||
if (p === '/api/voice/session' && req.method === 'POST') {
|
||||
if (!voice.configured) return send(res, 503, { detail: '语音还没有在这台服务上配置，不能假装能通话。' });
|
||||
if (refuse) return send(res, refuse, { detail: REFUSALS[refuse] || '通话服务暂时连不上，稍后再试试。' });
|
||||
await sleep(120);
|
||||
const id = `sess-${++made}`; sessions.set(id, { room: `cg-voice-${made}` });
|
||||
return send(res, 201, { url: 'wss://lk.invalid', token: `token-${id}`, room: `cg-voice-${made}`, session_id: id, agent_name: 'cybergril-voice-agent', epoch, expires_at: Date.now() + 2 * 3600e3 });
|
||||
}
|
||||
if (p.startsWith('/api/voice/session/') && req.method === 'DELETE') { const id = decodeURIComponent(p.slice(19)); return sessions.delete(id) ? send(res, 200, { ended: true }) : send(res, 404, { detail: '这次通话已经不在了。' }); }
|
||||
if (p === '/api/chat' && req.method === 'POST') {
|
||||
let body = ''; for await (const chunk of req) body += chunk;
|
||||
const { text: said, turn_id } = JSON.parse(body), reply = '我在。今天的雨停了没有？要是还在下，就慢点回家。';
|
||||
add({ role: 'user', text: said, turn_id, status: 'pending' });
|
||||
res.writeHead(200, { 'Content-Type': 'text/event-stream' });
|
||||
const stream = { res, turn_id, cut: false }; textStream = stream;
|
||||
for (const part of reply.match(/.{1,4}/gu)) { if (stream.cut) return; res.write(`data: ${JSON.stringify({ t: 'chat.delta', epoch, turn_id, delta: part })}\n\n`); await sleep(400); }
|
||||
if (stream.cut) return;
|
||||
textStream = null;
|
||||
add({ role: 'companion', text: reply, turn_id }); rows.forEach(r => { if (r.turn_id === turn_id) r.status = 'sent'; });
|
||||
res.end(`data: ${JSON.stringify({ t: 'chat.final', epoch, turn_id, utterance: { text: reply, speech_tags: [] }, receipt: null })}\n\n`);
|
||||
return;
|
||||
}
|
||||
if (p.startsWith('/api/')) return send(res, 404, { detail: 'Not Found' });
|
||||
const file = normalize(join(dist, p === '/' ? 'index.html' : p));
|
||||
if (!file.startsWith(normalize(dist))) return send(res, 403, 'no', 'text/plain');
|
||||
try { send(res, 200, await readFile(file), { '.js': 'text/javascript', '.css': 'text/css', '.html': 'text/html; charset=utf-8', '.svg': 'image/svg+xml', '.png': 'image/png', '.jpg': 'image/jpeg', '.webp': 'image/webp' }[extname(file)] || 'application/octet-stream'); }
|
||||
catch { send(res, 200, await readFile(join(dist, 'index.html')), 'text/html; charset=utf-8'); }
|
||||
}).listen(port, '127.0.0.1', () => console.log(`hel878 mock on ${port}`));
|
||||
@@ -0,0 +1,245 @@
|
||||
// HEL-878 browser check: drives headless Edge over CDP against tests/hel878_mock.mjs (real voice on the approved
|
||||
// conversation, with tests/fake-livekit.js in place of the LiveKit client and Chromium's fake audio device in place of
|
||||
// a microphone). Usage: node tests/hel878_shots.mjs <edge.exe> <outDir> [port]. One PASS/FAIL line per check.
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
const [edge, out, portArg] = process.argv.slice(2);
|
||||
const port = Number(portArg || 8878), base = `http://127.0.0.1:${port}`, cdpPort = 9478;
|
||||
mkdirSync(out, { recursive: true });
|
||||
const profile = join(tmpdir(), `hel878-edge-${Date.now()}`);
|
||||
const browser = spawn(edge, ['--headless=new', `--remote-debugging-port=${cdpPort}`, `--user-data-dir=${profile}`, '--no-first-run', '--hide-scrollbars', '--force-color-profile=srgb', '--use-fake-device-for-media-stream', '--use-fake-ui-for-media-stream', '--autoplay-policy=no-user-gesture-required', 'about:blank'], { stdio: 'ignore' });
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||||
let targets;
|
||||
for (let i = 0; i < 60 && !targets; i++) { try { targets = await (await fetch(`http://127.0.0.1:${cdpPort}/json`)).json(); } catch { await sleep(250); } }
|
||||
const ws = new WebSocket(targets.find(t => t.type === 'page').webSocketDebuggerUrl);
|
||||
await new Promise(r => ws.addEventListener('open', r, { once: true }));
|
||||
let seq = 0; const waiting = new Map();
|
||||
ws.addEventListener('message', ({ data }) => { const m = JSON.parse(data); if (m.id && waiting.has(m.id)) { const { ok, ko } = waiting.get(m.id); waiting.delete(m.id); m.error ? ko(new Error(m.error.message)) : ok(m.result); } });
|
||||
const cdp = (method, params = {}) => new Promise((ok, ko) => { const id = ++seq; waiting.set(id, { ok, ko }); ws.send(JSON.stringify({ id, method, params })); });
|
||||
const js = async expr => { const r = await cdp('Runtime.evaluate', { expression: expr, returnByValue: true, awaitPromise: true }); if (r.exceptionDetails) throw new Error(r.exceptionDetails.exception?.description || 'js error'); return r.result.value; };
|
||||
const results = [];
|
||||
const check = (name, ok, detail = '') => { results.push(ok); console.log(`${ok ? 'PASS' : 'FAIL'} ${name}${detail ? ` — ${detail}` : ''}`); };
|
||||
const ctl = async path => (await fetch(`${base}${path}`)).json();
|
||||
const shot = async name => { const { data } = await cdp('Page.captureScreenshot', { format: 'png' }); writeFileSync(join(out, `${name}.png`), Buffer.from(data, 'base64')); };
|
||||
const click = async (selector, text = '') => js(`(() => { const el = [...document.querySelectorAll(${JSON.stringify(selector)})].find(e => e.offsetParent !== null && e.textContent.includes(${JSON.stringify(text)})); if (!el) return false; el.click(); return true; })()`);
|
||||
const text = sel => js(`[...document.querySelectorAll(${JSON.stringify(sel)})].map(e => e.innerText).join('\\n')`);
|
||||
const exists = sel => js(`!!document.querySelector(${JSON.stringify(sel)})`);
|
||||
const said = () => js(`document.querySelector('.conversation .sr')?.textContent || ''`);
|
||||
const status = () => js(`document.querySelector('.talk-status')?.innerText || ''`);
|
||||
const row = () => js(`document.querySelector('.voice-row.real')?.innerText || ''`);
|
||||
const phase = () => js(`document.querySelector('.voice-row.real')?.dataset.phase || 'off'`);
|
||||
const lk = expr => js(`window.__lk ? (${expr}) : null`);
|
||||
const type = t => js(`(() => { const el = document.querySelector('textarea'); const set = Object.getOwnPropertyDescriptor(HTMLTextAreaElement.prototype, 'value').set; set.call(el, ${JSON.stringify(t)}); el.dispatchEvent(new Event('input', { bubbles: true })); return true; })()`);
|
||||
const until = async (fn, ms = 4000) => { const end = Date.now() + ms; while (Date.now() < end) { if (await fn()) return true; await sleep(100); } return false; };
|
||||
const views = { desktop: { width: 1440, height: 900, deviceScaleFactor: 1, mobile: false }, phone: { width: 390, height: 844, deviceScaleFactor: 2, mobile: true } };
|
||||
async function view(v) { await cdp('Emulation.setDeviceMetricsOverride', views[v]); await cdp('Emulation.setTouchEmulationEnabled', { enabled: views[v].mobile, maxTouchPoints: 5 }); }
|
||||
async function boot(theme) {
|
||||
await cdp('Page.navigate', { url: `${base}/` }); await sleep(500);
|
||||
await js(`localStorage.setItem('cg.theme', ${JSON.stringify(theme)}); sessionStorage.clear(); true`);
|
||||
await cdp('Page.navigate', { url: `${base}/` }); await sleep(1600);
|
||||
}
|
||||
async function frames(name, ms) { const list = []; const end = Date.now() + ms; while (Date.now() < end) { const { data } = await cdp('Page.captureScreenshot', { format: 'jpeg', quality: 70 }); list.push(data); } list.forEach((d, i) => writeFileSync(join(out, `${name}-${String(i).padStart(2, '0')}.jpg`), Buffer.from(d, 'base64'))); return list.length; }
|
||||
// Every microphone request is counted (the fake device still answers it); a test can make the next one refuse.
|
||||
await cdp('Page.enable'); await cdp('Runtime.enable'); await cdp('Network.enable');
|
||||
await cdp('Page.addScriptToEvaluateOnNewDocument', { source: `window.__asked = 0; window.__refuseMic = ''; try { const md = navigator.mediaDevices, real = md.getUserMedia.bind(md); md.getUserMedia = c => { window.__asked++; if (window.__refuseMic) return Promise.reject(new DOMException('refused', window.__refuseMic)); return real(c); }; } catch {}` });
|
||||
const micsLive = () => lk(`__lk.mics.filter(t => t.readyState === 'live').length`);
|
||||
const herAudio = () => js(`document.querySelectorAll('[data-voice="her"] audio').length`);
|
||||
const calls = async () => (await ctl('/__log'));
|
||||
async function startVoice() {
|
||||
await click('.voice-button'); await until(() => exists('.modal.open'));
|
||||
await click('.modal.open .btn-primary', '开始语音');
|
||||
await until(async () => (await phase()) === 'joining');
|
||||
await until(async () => (await lk('__lk.rooms.at(-1)?.state')) === 'connected');
|
||||
await lk('__lk.join()'); await sleep(200);
|
||||
}
|
||||
const cleanEnd = async label => {
|
||||
await sleep(400);
|
||||
const s = await ctl('/__sessions'), mics = await micsLive(), audio = await herAudio(), holder = await exists('[data-voice="her"]');
|
||||
check(`${label}: microphone stopped, her audio and room gone, server room deleted`, mics === 0 && audio === 0 && !holder && s.open.length === 0 && (await phase()) === 'off', `mics=${mics} audio=${audio} holder=${holder} open=${s.open.length}`);
|
||||
};
|
||||
|
||||
// ── 1. Arriving at the conversation: one honest status read; no room, no microphone, no client download. ──
|
||||
await view('desktop'); await boot('moon'); await calls();
|
||||
await click('.nav-item', '交谈'); await sleep(1400);
|
||||
let log = await calls();
|
||||
check('status read on arrival only: no session, no microphone, no LiveKit client', log.filter(l => l.includes('/api/voice')).join() === 'GET /api/voice/status' && await js('window.__asked') === 0 && await js('!window.LivekitClient'), log.filter(l => l.includes('voice')).join(' | '));
|
||||
check('voice button offers to start, honestly labelled', await js(`document.querySelector('.voice-button').getAttribute('aria-label')`) === '开始语音');
|
||||
await shot('01-moon-desktop-talk-voice-off');
|
||||
|
||||
// ── 2. Not configured / TTS down: the button says so and nothing starts. ──
|
||||
await ctl('/__voice?configured=0'); await boot('moon'); await click('.nav-item', '交谈'); await sleep(1400); await calls();
|
||||
await click('.voice-button'); await sleep(500);
|
||||
check('not configured: refusal in her own words, no dialog, no session, no microphone', (await status()).includes('语音还没有在这台服务上配置') && !(await exists('.modal.open')) && !(await calls()).some(l => l.startsWith('POST')) && await js('window.__asked') === 0 && await exists('.voice-button.denied'), await status());
|
||||
await shot('02-moon-desktop-voice-not-configured');
|
||||
await ctl('/__voice?configured=1&tts=0'); await click('.voice-button'); await sleep(500);
|
||||
check('TTS down: says her voice service is away', (await status()).includes('她的声音服务这会儿不在'), await status());
|
||||
await ctl('/__voice?configured=1&tts=1');
|
||||
|
||||
// ── 3. The ask: what happens, where the sound goes, that it is not yet real-device accepted. Cancel asks nothing. ──
|
||||
await click('.voice-button'); await until(() => exists('.modal.open'));
|
||||
const ask = await text('.modal.open');
|
||||
check('the dialog explains microphone, destination, same memory, how to stop, and the unverified path', ask.includes('麦克风') && ask.includes('这台服务器') && ask.includes('同一份记忆') && ask.includes('立即关闭') && ask.includes('还没有经过真机验收'), ask.replace(/\n/g, ' '));
|
||||
await shot('03-moon-desktop-voice-ask');
|
||||
await click('.modal.open .btn-ghost', '取消'); await sleep(300);
|
||||
check('cancel: no microphone request, no session', await js('window.__asked') === 0 && (await ctl('/__sessions')).made === 0);
|
||||
|
||||
// ── 4. Start: opening → waiting for her → listening. ──
|
||||
await click('.voice-button'); await until(() => exists('.modal.open')); await calls();
|
||||
await click('.modal.open .btn-primary', '开始语音');
|
||||
await until(async () => (await phase()) === 'joining');
|
||||
await shot('04-moon-desktop-voice-joining');
|
||||
check('waiting for her is shown as waiting, not as connected', (await row()).includes('在等栖接起') && (await text('.topbar .state')).includes('未经真机验收'), await row());
|
||||
await until(async () => (await lk('__lk.rooms.at(-1)?.state')) === 'connected');
|
||||
log = await calls();
|
||||
check('one session opened with a server token; one microphone; mic published to that room', log.filter(l => l === 'POST /api/voice/session').length === 1 && (await lk('__lk.connects.at(-1).token')).startsWith('token-sess-') && await js('window.__asked') === 1 && await lk('__lk.published.length') === 1);
|
||||
await lk('__lk.join()'); await sleep(300);
|
||||
check('her worker joined: listening, wave in your colour, her audio attached', (await phase()) === 'listening' && (await row()).includes('在听你说') && await herAudio() === 1, await row());
|
||||
await frames('m1-voice-in', 700);
|
||||
await shot('05-moon-desktop-voice-listening');
|
||||
|
||||
// ── 5. A voice turn: your draft, the record row, her thinking and speaking, then her words on the stage. ──
|
||||
await lk(`__lk.say({ kind: 'interim', role: 'user', text: '今天下班的路上' })`); await sleep(200);
|
||||
check('interim words show as a draft line beside the state', (await row()).includes('今天下班的路上'));
|
||||
await shot('06-moon-desktop-voice-heard');
|
||||
await ctl('/__voice-turn?stage=user&text=' + encodeURIComponent('今天下班的路上看到一只猫'));
|
||||
await lk(`__lk.say({ kind: 'user_final', role: 'user', text: '今天下班的路上看到一只猫' })`); await lk(`__lk.state('thinking')`);
|
||||
await until(async () => (await said()).includes('看到一只猫'));
|
||||
check('your spoken words enter the stage from the server record (not from the subtitle)', (await said()).includes('看到一只猫') && (await phase()) === 'thinking', await said());
|
||||
await shot('07-moon-desktop-voice-thinking');
|
||||
await lk(`__lk.state('speaking')`); await sleep(250);
|
||||
check('speaking: her name, and that you can cut in by speaking', (await row()).includes('栖在说') && (await row()).includes('开口就能打断'));
|
||||
await frames('m2-voice-speaking', 900);
|
||||
await shot('08-moon-desktop-voice-speaking');
|
||||
await ctl('/__voice-turn?stage=reply&text=' + encodeURIComponent('是橘色的那只吗？它最近总在桥下晒太阳。'));
|
||||
await lk(`__lk.say({ kind: 'assistant_final', role: 'assistant', text: '是橘色的那只吗？它最近总在桥下晒太阳。' })`); await lk(`__lk.state('listening')`);
|
||||
await until(async () => (await said()).includes('晒太阳'), 6000);
|
||||
check('her spoken answer reaches the stage once it is in the record', (await said()).includes('晒太阳'), await said());
|
||||
await shot('09-moon-desktop-voice-reply-on-stage');
|
||||
await click('.log-btn'); await sleep(700);
|
||||
const rec = await text('.log-panel .scroll');
|
||||
check('the record holds the voice turn once, same conversation as text', rec.split('看到一只猫').length === 2 && rec.split('晒太阳').length === 2 && rec.includes('下雨天'));
|
||||
await shot('10-moon-desktop-voice-record');
|
||||
await click('.log-panel .btn-round'); await sleep(400);
|
||||
|
||||
// ── 6. φ: her failure line is heard and shown, never recorded. ──
|
||||
await lk(`__lk.say({ kind: 'error', role: 'assistant', text: '我刚刚没接上话，你再说一遍好吗？' })`); await sleep(400);
|
||||
check('φ line shown as heard, marked not recorded; not in the record', (await row()).includes('没接上话') && (await row()).includes('未记入聊天') && !(await ctl('/__rows')).some(r => r.text.includes('没接上话')));
|
||||
await shot('11-moon-desktop-voice-phi');
|
||||
|
||||
// ── 7. Typing over her spoken answer: what is left of it goes silent until she listens again. ──
|
||||
await lk(`__lk.state('speaking')`); await sleep(150);
|
||||
await type('等等，我先打字说'); await js(`document.querySelector('.send').click(), true`); await sleep(300);
|
||||
check('text over voice: her remaining audio is muted', await js(`[...document.querySelectorAll('[data-voice="her"] audio')].every(a => a.muted)`));
|
||||
await lk(`__lk.state('listening')`); await sleep(150);
|
||||
check('…and unmuted once she is listening again', await js(`[...document.querySelectorAll('[data-voice="her"] audio')].every(a => !a.muted)`));
|
||||
await until(async () => !(await exists('.reader-bar .stop')), 8000);
|
||||
|
||||
// ── 8. Speaking over a text reply: the text turn is cut (not "failed"), your words stay as said. ──
|
||||
await type('再给我讲讲那只猫'); await js(`document.querySelector('.send').click(), true`); await sleep(1300);
|
||||
check('text reply streaming', await exists('.reader-bar .stop'));
|
||||
await ctl('/__cut'); await ctl('/__voice-turn?stage=user&text=' + encodeURIComponent('算了先不说了'));
|
||||
await lk(`__lk.say({ kind: 'user_final', role: 'user', text: '算了先不说了' })`); await sleep(1500);
|
||||
const cutStatus = await status();
|
||||
check('voice over text: no failure notice, no retry, draft not refilled', !cutStatus.includes('连接中断') && !(await exists('.talk-status .retry')) && await js(`document.querySelector('textarea').value`) === '', cutStatus);
|
||||
await click('.log-btn'); await sleep(700);
|
||||
const cutRec = await text('.log-panel .scroll');
|
||||
check('…the cut text turn is marked cancelled in the record; the voice words are there once', cutRec.includes('已取消') && cutRec.split('算了先不说了').length === 2, cutRec.slice(-200).replace(/\n/g, ' '));
|
||||
await click('.log-panel .btn-round'); await sleep(300);
|
||||
|
||||
// ── 9. Autoplay held back: a tap lets her voice out. ──
|
||||
await lk('__lk.block()'); await sleep(200);
|
||||
check('blocked audio offers a tap', (await row()).includes('点一下听栖的声音'));
|
||||
await shot('12-moon-desktop-voice-audio-blocked');
|
||||
await click('.voice-row.real button', '点一下'); await sleep(200);
|
||||
check('…and the offer goes away after the tap', !(await row()).includes('点一下'));
|
||||
|
||||
// ── 10. 结束语音: everything stops at once; late events from the old room change nothing. ──
|
||||
await calls();
|
||||
await click('.voice-row.real .voice-end');
|
||||
await cleanEnd('结束语音');
|
||||
check('ended: says the microphone is closed; DELETE sent', (await status()).includes('麦克风已关闭') && (await calls()).some(l => l.startsWith('DELETE /api/voice/session/')), await status());
|
||||
await shot('13-moon-desktop-voice-ended');
|
||||
await lk(`__lk.state('speaking')`); await lk(`__lk.say({ kind: 'user_final', role: 'user', text: '幽灵' })`); await sleep(500);
|
||||
check('late events after the end do not revive anything', (await phase()) === 'off' && !(await exists('.voice-row.real')) && !(await calls()).some(l => l.startsWith('POST')));
|
||||
|
||||
// ── 11. Microphone permission refused: nothing opened. ──
|
||||
await js(`window.__refuseMic = 'NotAllowedError', true`); await calls();
|
||||
await click('.voice-button'); await until(() => exists('.modal.open')); await click('.modal.open .btn-primary', '开始语音'); await sleep(800);
|
||||
check('permission refused: told plainly, no session opened, text still there', (await status()).includes('没有拿到麦克风权限') && !(await calls()).some(l => l.startsWith('POST')) && await exists('textarea'), await status());
|
||||
await shot('14-moon-desktop-voice-denied');
|
||||
await js(`window.__refuseMic = '', true`);
|
||||
|
||||
// ── 12. Permission revoked mid-call / room dropped / server refusal. ──
|
||||
await startVoice();
|
||||
await lk(`(__lk.mics.at(-1).dispatchEvent(new Event('ended')), true)`);
|
||||
await sleep(300);
|
||||
check('microphone revoked mid-call: call ends, said so', (await status()).includes('麦克风被关掉了'), await status());
|
||||
await cleanEnd('revoked');
|
||||
await startVoice(); await lk('__lk.drop()');
|
||||
await sleep(300);
|
||||
check('room dropped: call ends, said so', (await status()).includes('通话断开了'), await status());
|
||||
await cleanEnd('dropped');
|
||||
await ctl('/__refuse/503'); await click('.voice-button'); await until(() => exists('.modal.open')); await click('.modal.open .btn-primary', '开始语音'); await sleep(900);
|
||||
check('server refusal shows its detail; microphone released', (await status()).includes('她的声音服务这会儿不在') && await micsLive() === 0, await status());
|
||||
await ctl('/__refuse/0');
|
||||
|
||||
// ── 13. Repeated on/off: one call at most, nothing left behind. ──
|
||||
await click('.voice-button'); await until(() => exists('.modal.open')); await click('.modal.open .btn-primary', '开始语音'); await sleep(60);
|
||||
await click('.voice-row.real .voice-end'); await sleep(40);
|
||||
await click('.voice-button'); await until(() => exists('.modal.open')); await click('.modal.open .btn-primary', '开始语音');
|
||||
await until(async () => (await lk('__lk.rooms.at(-1)?.state')) === 'connected'); await sleep(600);
|
||||
const sess = await ctl('/__sessions');
|
||||
check('rapid off/on: exactly one open session and one live microphone', sess.open.length === 1 && await micsLive() === 1, JSON.stringify(sess));
|
||||
await click('.voice-row.real .voice-end'); await cleanEnd('rapid toggle');
|
||||
|
||||
// ── 14. Leaving: nav, Esc, 结束交谈, page unload, network loss — each ends the call. ──
|
||||
await startVoice(); await click('.nav-item', '此刻'); await cleanEnd('nav away');
|
||||
await click('.nav-item', '交谈'); await sleep(1300); await startVoice();
|
||||
await js(`document.body.dispatchEvent(new KeyboardEvent('keydown', { key: 'Escape', bubbles: true })), true`); await cleanEnd('Esc');
|
||||
await click('.nav-item', '交谈'); await sleep(1300); await startVoice();
|
||||
await click('.end-talk'); await cleanEnd('结束交谈');
|
||||
await click('.nav-item', '交谈'); await sleep(1300); await startVoice();
|
||||
await cdp('Network.emulateNetworkConditions', { offline: true, latency: 0, downloadThroughput: -1, uploadThroughput: -1 }); await sleep(500);
|
||||
check('network lost: call ends with its own words', (await status()).includes('网络断开了'), await status());
|
||||
const offMics = await micsLive(), offAudio = await herAudio(), offOpen = (await ctl('/__sessions')).open.length;
|
||||
check('network lost: microphone and her audio stop at once; the server room is owed, not pretended deleted', offMics === 0 && offAudio === 0 && offOpen === 1 && (await js(`localStorage.getItem('cg.voice.owed')`) || '').includes('sess-'), `mics=${offMics} audio=${offAudio} open=${offOpen}`);
|
||||
await cdp('Network.emulateNetworkConditions', { offline: false, latency: 0, downloadThroughput: -1, uploadThroughput: -1 });
|
||||
await cleanEnd('offline, after the network is back');
|
||||
check('…the owed DELETE is sent once back online and forgotten', !(await js(`localStorage.getItem('cg.voice.owed')`)));
|
||||
await startVoice(); await calls();
|
||||
await cdp('Page.navigate', { url: 'about:blank' }); await sleep(800);
|
||||
log = await calls();
|
||||
check('page unload: the server room is deleted (keepalive)', log.some(l => l.startsWith('DELETE /api/voice/session/')) && (await ctl('/__sessions')).open.length === 0, log.join(' | '));
|
||||
|
||||
// ── 15. Phone and the fog theme: the same line, the same states. ──
|
||||
for (const [theme, v] of [['fog', 'desktop'], ['moon', 'phone'], ['fog', 'phone']]) {
|
||||
await view(v); await boot(theme); await click('.nav-item', '交谈'); await sleep(1400);
|
||||
await startVoice();
|
||||
await lk(`__lk.say({ kind: 'interim', role: 'user', text: '今天下班的路上看到一只很胖的橘猫' })`); await sleep(250);
|
||||
await shot(`15-${theme}-${v}-voice-listening`);
|
||||
await lk(`__lk.state('speaking')`); await sleep(300);
|
||||
await shot(`16-${theme}-${v}-voice-speaking`);
|
||||
if (v === 'phone') check(`${theme}/${v}: voice line fits the screen`, await js(`(() => { const r = document.querySelector('.voice-row.real').getBoundingClientRect(); return r.left >= 0 && r.right <= innerWidth; })()`));
|
||||
await click('.voice-row.real .voice-end'); await sleep(400);
|
||||
}
|
||||
await shot('17-fog-phone-voice-ended');
|
||||
|
||||
// ── 16. The real vendored client: same API names the page uses (no connection is made). ──
|
||||
await view('desktop');
|
||||
await cdp('Page.navigate', { url: `${base}/__nothing` }); await sleep(800);
|
||||
const sdk = await js(`new Promise(ok => { delete window.LivekitClient; const s = document.createElement('script'); s.src = '/__real-sdk.js'; s.onload = () => { const L = window.LivekitClient, E = L.RoomEvent; const room = new L.Room({ adaptiveStream: true, dynacast: true }); ok({ events: ['TrackSubscribed','TrackUnsubscribed','ParticipantConnected','ParticipantAttributesChanged','DataReceived','Disconnected','AudioPlaybackStatusChanged'].every(k => typeof E[k] === 'string'), values: [E.ParticipantAttributesChanged, E.DataReceived, E.AudioPlaybackStatusChanged], mic: L.Track.Source.Microphone, agent: L.ParticipantKind && L.ParticipantKind.AGENT, create: typeof L.createLocalAudioTrack, methods: ['connect','disconnect','startAudio','on'].every(m => typeof room[m] === 'function'), playback: typeof room.canPlaybackAudio, remote: room.remoteParticipants instanceof Map, publish: typeof room.localParticipant.publishTrack, version: L.version }); }; s.onerror = () => ok(null); document.head.append(s); })`);
|
||||
check('real livekit-client 2.22.4: every name the adapter uses exists', !!sdk && sdk.events && sdk.mic === 'microphone' && typeof sdk.agent === 'number' && sdk.create === 'function' && sdk.methods && sdk.playback === 'boolean' && sdk.remote && sdk.publish === 'function', JSON.stringify(sdk));
|
||||
|
||||
// ── 17. The demo entry stays separate: no voice route, no client. ──
|
||||
await calls();
|
||||
await cdp('Page.navigate', { url: `${base}/?demo=1` }); await sleep(2500);
|
||||
check('demo entry makes no voice requests', !(await calls()).some(l => l.includes('/api/voice')));
|
||||
|
||||
ws.close(); browser.kill();
|
||||
try { rmSync(profile, { recursive: true, force: true }); } catch { /* Edge may still hold the profile. */ }
|
||||
const failed = results.filter(r => !r).length;
|
||||
console.log(`${results.length - failed}/${results.length} passed`);
|
||||
process.exit(failed ? 1 : 0);
|
||||
@@ -0,0 +1,309 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { mergeTurns, recentRows, VoiceController, voiceStatus, WORDS, type MicTrack, type RoomEvents, type VoiceRoom, type VoiceView } from '../src/voice/session';
|
||||
import type { ServerMessage } from '../src/adapters/real';
|
||||
import type { Message } from '../src/domain/model';
|
||||
|
||||
// Fake transport and media objects: no LiveKit, no microphone, no network. The fake server follows the HEL-877 voice
|
||||
// router (src/cybergril/voice/router.py): GET /api/voice/status, POST /api/voice/session -> 201 grant, DELETE
|
||||
// /api/voice/session/{id} -> 200 or 404.
|
||||
const gate = () => { let open!: () => void; const promise = new Promise<void>(r => { open = r; }); return { promise, open }; };
|
||||
const flush = async (n = 8) => { for (let i = 0; i < n; i++) await Promise.resolve(); await new Promise(r => setTimeout(r, 0)); };
|
||||
class Mic implements MicTrack { stopped = 0; ended?: () => void; stop() { this.stopped++; } onEnded(cb: () => void) { this.ended = cb; } }
|
||||
class Room implements VoiceRoom {
|
||||
closed = 0; published: MicTrack[] = []; hushed: boolean[] = []; on!: RoomEvents;
|
||||
async publish(mic: MicTrack) { this.published.push(mic); }
|
||||
hush(on: boolean) { this.hushed.push(on); }
|
||||
async resumeAudio() {}
|
||||
async close() { this.closed++; }
|
||||
}
|
||||
class Transport {
|
||||
mics: Mic[] = []; rooms: Room[] = []; signals: AbortSignal[] = [];
|
||||
micGate?: ReturnType<typeof gate>; joinGate?: ReturnType<typeof gate>; micError?: Error; joinError?: Error;
|
||||
async capture() { const g = this.micGate; if (g) await g.promise; if (this.micError) throw this.micError; const m = new Mic(); this.mics.push(m); return m; }
|
||||
async join(_url: string, _token: string, on: RoomEvents, signal: AbortSignal) { const r = new Room(); r.on = on; this.rooms.push(r); this.signals.push(signal); const g = this.joinGate; if (g) await g.promise; if (this.joinError) throw this.joinError; return r; }
|
||||
}
|
||||
class Server {
|
||||
status: Record<string, unknown> | number = { configured: true, tts_ok: true, agent_name: 'cybergril-voice-agent', voice: false };
|
||||
open = new Set<string>(); made = 0; calls: { method: string; url: string; keepalive?: boolean }[] = [];
|
||||
postGate?: ReturnType<typeof gate>; refuse?: { status: number; detail: string };
|
||||
fetch = vi.fn(async (url: string, init: RequestInit = {}) => {
|
||||
const method = init.method || 'GET';
|
||||
this.calls.push({ method, url, keepalive: init.keepalive });
|
||||
if (url === '/api/voice/status') return typeof this.status === 'number' ? new Response('{"detail":"Not Found"}', { status: this.status }) : new Response(JSON.stringify(this.status));
|
||||
if (url === '/api/voice/session' && method === 'POST') {
|
||||
const g = this.postGate; if (g) await g.promise;
|
||||
if (this.refuse) return new Response(JSON.stringify({ detail: this.refuse.detail }), { status: this.refuse.status });
|
||||
const id = `s${++this.made}`; this.open.add(id);
|
||||
return new Response(JSON.stringify({ url: 'wss://lk.test', token: `tok-${id}`, room: `cg-voice-${id}`, session_id: id, agent_name: 'cybergril-voice-agent', epoch: 3, expires_at: 0 }), { status: 201 });
|
||||
}
|
||||
const m = /^\/api\/voice\/session\/(.+)$/u.exec(url);
|
||||
if (m && method === 'DELETE') return this.open.delete(decodeURIComponent(m[1])) ? new Response('{"ended":true}') : new Response('{"detail":"这次通话已经不在了。"}', { status: 404 });
|
||||
return new Response('{"detail":"Not Found"}', { status: 404 });
|
||||
});
|
||||
}
|
||||
function rig() {
|
||||
const server = new Server(), transport = new Transport(), views: VoiceView[] = [], record = vi.fn(), spoke = vi.fn();
|
||||
const c = new VoiceController({ transport: async () => transport, onView: v => views.push(v), onRecord: record, onSpoke: spoke, fetch: server.fetch as unknown as typeof fetch, secure: () => true, agentWaitMs: 1000 });
|
||||
const view = () => views.at(-1)!;
|
||||
return { server, transport, views, view, record, spoke, c };
|
||||
}
|
||||
async function connected() {
|
||||
const r = rig(); await r.c.start(); r.transport.rooms[0].on.agentJoined(); return r;
|
||||
}
|
||||
afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); });
|
||||
|
||||
describe('honest switch', () => {
|
||||
it('reads status only: no room, no microphone', async () => {
|
||||
const s = new Server();
|
||||
expect(await voiceStatus(new AbortController().signal, s.fetch as unknown as typeof fetch)).toEqual({ available: true, reason: '' });
|
||||
s.status = { configured: false, tts_ok: false, voice: false };
|
||||
expect((await voiceStatus(new AbortController().signal, s.fetch as unknown as typeof fetch)).reason).toBe(WORDS.notConfigured);
|
||||
s.status = { configured: true, tts_ok: false, voice: false };
|
||||
expect((await voiceStatus(new AbortController().signal, s.fetch as unknown as typeof fetch)).reason).toBe(WORDS.ttsDown);
|
||||
s.status = 404;
|
||||
expect((await voiceStatus(new AbortController().signal, s.fetch as unknown as typeof fetch)).reason).toBe(WORDS.noRoute);
|
||||
const down = vi.fn(async () => { throw new TypeError('Failed to fetch'); });
|
||||
expect((await voiceStatus(new AbortController().signal, down as unknown as typeof fetch)).reason).toBe(WORDS.unknown);
|
||||
expect(s.calls.every(c => c.method === 'GET' && c.url === '/api/voice/status')).toBe(true);
|
||||
});
|
||||
it('configured && tts_ok never means "verified": voice:false is not read as connected', async () => {
|
||||
const r = rig(); r.server.status = { configured: true, tts_ok: true, voice: false };
|
||||
await r.c.start();
|
||||
// Starting is allowed (the user asked), but the page only says what the room is doing, never "已接通".
|
||||
expect(r.views.map(v => v.phase)).toEqual(['opening', 'joining']);
|
||||
});
|
||||
it('refuses honestly before touching the microphone when the server cannot speak', async () => {
|
||||
const r = rig(); r.server.status = { configured: true, tts_ok: false };
|
||||
await r.c.start();
|
||||
expect(r.view()).toMatchObject({ phase: 'off', notice: WORDS.ttsDown });
|
||||
expect(r.transport.mics).toHaveLength(0);
|
||||
expect(r.server.calls.some(c => c.method === 'POST')).toBe(false);
|
||||
});
|
||||
it('an insecure page is told why, without asking for the microphone', async () => {
|
||||
const r = rig(); const c = new VoiceController({ transport: async () => r.transport, onView: v => r.views.push(v), onRecord: r.record, onSpoke: r.spoke, fetch: r.server.fetch as unknown as typeof fetch, secure: () => false });
|
||||
await c.start();
|
||||
expect(r.view().notice).toBe(WORDS.insecure);
|
||||
expect(r.server.calls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('lifecycle', () => {
|
||||
it('opens, joins, publishes the microphone, follows her state; ending stops everything', async () => {
|
||||
const r = await connected(), room = r.transport.rooms[0], mic = r.transport.mics[0];
|
||||
expect(room.published).toEqual([mic]);
|
||||
expect(r.view().phase).toBe('listening');
|
||||
room.on.agentState('thinking'); expect(r.view().phase).toBe('thinking');
|
||||
room.on.agentState('speaking'); expect(r.view().phase).toBe('speaking');
|
||||
room.on.agentState('listening'); expect(r.view().phase).toBe('listening');
|
||||
expect(r.server.open.size).toBe(1);
|
||||
await r.c.end();
|
||||
expect(mic.stopped).toBeGreaterThan(0);
|
||||
expect(room.closed).toBe(1);
|
||||
expect(r.server.open.size).toBe(0);
|
||||
expect(r.view()).toMatchObject({ phase: 'off', notice: '' });
|
||||
expect(r.c.active).toBe(false);
|
||||
});
|
||||
it('permission denied: nothing opened on the server, nothing left running', async () => {
|
||||
const r = rig(); r.transport.micError = Object.assign(new Error('denied'), { name: 'NotAllowedError' });
|
||||
await r.c.start();
|
||||
expect(r.view()).toMatchObject({ phase: 'off', notice: WORDS.denied });
|
||||
expect(r.server.calls.filter(c => c.method !== 'GET')).toHaveLength(0);
|
||||
r.transport.micError = Object.assign(new Error('none'), { name: 'NotFoundError' });
|
||||
await r.c.start(); expect(r.view().notice).toBe(WORDS.noMic);
|
||||
});
|
||||
it('server refusal shows its own words and releases the microphone', async () => {
|
||||
const r = rig(); r.server.refuse = { status: 409, detail: '还没有创建真实伴侣，语音也不会开始。' };
|
||||
await r.c.start();
|
||||
expect(r.view().notice).toBe('还没有创建真实伴侣，语音也不会开始。');
|
||||
expect(r.transport.mics[0].stopped).toBeGreaterThan(0);
|
||||
expect(r.transport.rooms).toHaveLength(0);
|
||||
});
|
||||
it('a failed join releases the microphone and gives the room back', async () => {
|
||||
const r = rig(); r.transport.joinError = new Error('ws failed');
|
||||
await r.c.start();
|
||||
expect(r.view().notice).toBe(WORDS.unreachable);
|
||||
expect(r.transport.mics[0].stopped).toBeGreaterThan(0);
|
||||
expect(r.server.open.size).toBe(0);
|
||||
});
|
||||
it('ending while the microphone prompt is open: the late microphone is stopped, no room is asked for', async () => {
|
||||
const r = rig(); r.transport.micGate = gate();
|
||||
const starting = r.c.start(); await flush();
|
||||
await r.c.end();
|
||||
r.transport.micGate.open(); await starting; await flush();
|
||||
expect(r.transport.mics[0].stopped).toBeGreaterThan(0);
|
||||
expect(r.server.calls.some(c => c.method === 'POST')).toBe(false);
|
||||
expect(r.view().phase).toBe('off');
|
||||
});
|
||||
it('ending while the room is being opened: the late grant is given back, never joined', async () => {
|
||||
const r = rig(); r.server.postGate = gate();
|
||||
const starting = r.c.start(); await flush();
|
||||
await r.c.end();
|
||||
expect(r.transport.mics[0].stopped).toBeGreaterThan(0);
|
||||
r.server.postGate.open(); await starting; await flush();
|
||||
expect(r.server.made).toBe(1);
|
||||
expect(r.server.open.size).toBe(0);
|
||||
expect(r.transport.rooms).toHaveLength(0);
|
||||
expect(r.view().phase).toBe('off');
|
||||
});
|
||||
it('ending while joining: the join is aborted, a late room is closed, the server room deleted', async () => {
|
||||
const r = rig(); r.transport.joinGate = gate();
|
||||
const starting = r.c.start(); await flush();
|
||||
expect(r.view().phase).toBe('joining');
|
||||
await r.c.end();
|
||||
expect(r.transport.signals[0].aborted).toBe(true);
|
||||
r.transport.joinGate.open(); await starting; await flush();
|
||||
expect(r.transport.rooms[0].closed).toBe(1);
|
||||
expect(r.server.open.size).toBe(0);
|
||||
expect(r.transport.mics[0].stopped).toBeGreaterThan(0);
|
||||
expect(r.view().phase).toBe('off');
|
||||
});
|
||||
it('rapid on/off/on: only the last call stays, every earlier microphone and room is released', async () => {
|
||||
const r = rig();
|
||||
const a = r.c.start(); void r.c.end(); const b = r.c.start(); const c = r.c.start();
|
||||
await Promise.all([a, b, c]); await flush();
|
||||
expect(r.server.open.size).toBe(1);
|
||||
const live = r.transport.mics.filter(m => !m.stopped);
|
||||
expect(live).toHaveLength(1);
|
||||
expect(r.transport.rooms.filter(x => !x.closed)).toHaveLength(1);
|
||||
await r.c.end(); await flush();
|
||||
expect(r.server.open.size).toBe(0);
|
||||
expect(r.transport.mics.every(m => m.stopped)).toBe(true);
|
||||
});
|
||||
it('late events from an ended call never bring it back', async () => {
|
||||
const r = await connected(), old = r.transport.rooms[0];
|
||||
await r.c.end(); r.record.mockClear();
|
||||
const seen = r.views.length;
|
||||
old.on.agentState('speaking'); old.on.subtitle({ kind: 'user_final', role: 'user', text: '旧的话' }); old.on.agentJoined(); old.on.dropped(); old.on.audioBlocked(true);
|
||||
r.transport.mics[0].ended?.();
|
||||
await flush();
|
||||
expect(r.views.length).toBe(seen);
|
||||
expect(r.record).not.toHaveBeenCalled(); expect(r.spoke).not.toHaveBeenCalled();
|
||||
expect(r.c.active).toBe(false);
|
||||
});
|
||||
it('the room dropping on its own (network, server ended it) cleans up and says so', async () => {
|
||||
const r = await connected();
|
||||
r.transport.rooms[0].on.dropped(); await flush();
|
||||
expect(r.view()).toMatchObject({ phase: 'off', notice: WORDS.dropped });
|
||||
expect(r.transport.mics[0].stopped).toBeGreaterThan(0);
|
||||
expect(r.server.open.size).toBe(0);
|
||||
});
|
||||
it('microphone permission revoked mid-call ends the call', async () => {
|
||||
const r = await connected();
|
||||
r.transport.mics[0].ended?.(); await flush();
|
||||
expect(r.view()).toMatchObject({ phase: 'off', notice: WORDS.micGone });
|
||||
expect(r.server.open.size).toBe(0);
|
||||
expect(r.transport.rooms[0].closed).toBe(1);
|
||||
});
|
||||
it('her worker never joining is a failure, not an endless wait', async () => {
|
||||
vi.useFakeTimers();
|
||||
const r = rig(); await r.c.start();
|
||||
expect(r.view().phase).toBe('joining');
|
||||
await vi.advanceTimersByTimeAsync(1100);
|
||||
expect(r.view()).toMatchObject({ phase: 'off', notice: WORDS.noAgent });
|
||||
expect(r.server.open.size).toBe(0);
|
||||
});
|
||||
it('offline ends with its own words; unload releases at once and the DELETE survives the page', async () => {
|
||||
const r = await connected();
|
||||
await r.c.end(WORDS.offline);
|
||||
expect(r.view().notice).toBe(WORDS.offline);
|
||||
const u = await connected();
|
||||
u.c.unload();
|
||||
expect(u.transport.mics[0].stopped).toBeGreaterThan(0);
|
||||
await flush();
|
||||
expect(u.server.calls.find(c => c.method === 'DELETE')?.keepalive).toBe(true);
|
||||
expect(u.server.open.size).toBe(0);
|
||||
});
|
||||
it('a DELETE that cannot get through (offline) is owed, survives a new page, and is sent once back online', async () => {
|
||||
const data = new Map<string, string>();
|
||||
vi.stubGlobal('localStorage', { getItem: (k: string) => data.get(k) ?? null, setItem: (k: string, v: string) => void data.set(k, v), removeItem: (k: string) => void data.delete(k) });
|
||||
const r = await connected(), real = r.server.fetch.getMockImplementation()!;
|
||||
r.server.fetch.mockImplementation(async (url: string, init: RequestInit = {}) => { if (init.method === 'DELETE') { r.server.calls.push({ method: 'DELETE', url }); throw new TypeError('Failed to fetch'); } return real(url, init); });
|
||||
await r.c.end(WORDS.offline);
|
||||
expect(r.transport.mics[0].stopped).toBeGreaterThan(0); // local capture stops regardless
|
||||
expect(r.server.open.size).toBe(1); // the server room is not pretended gone
|
||||
expect(JSON.parse(data.get('cg.voice.owed')!)).toEqual(['s1']);
|
||||
r.server.fetch.mockImplementation(real);
|
||||
// A new page (new controller) settles it: once, then forgets.
|
||||
const again = new VoiceController({ transport: async () => r.transport, onView: () => {}, onRecord: () => {}, onSpoke: () => {}, fetch: r.server.fetch as unknown as typeof fetch, secure: () => true });
|
||||
await again.settle();
|
||||
expect(r.server.open.size).toBe(0);
|
||||
expect(data.has('cg.voice.owed')).toBe(false);
|
||||
const before = r.server.calls.length; await again.settle();
|
||||
expect(r.server.calls.length).toBe(before);
|
||||
});
|
||||
it('an owed room already gone on the server (404) counts as settled', async () => {
|
||||
const data = new Map<string, string>([['cg.voice.owed', '["gone-1"]']]);
|
||||
vi.stubGlobal('localStorage', { getItem: (k: string) => data.get(k) ?? null, setItem: (k: string, v: string) => void data.set(k, v), removeItem: (k: string) => void data.delete(k) });
|
||||
const r = rig(); await r.c.settle();
|
||||
expect(r.server.calls.at(-1)).toMatchObject({ method: 'DELETE', url: '/api/voice/session/gone-1' });
|
||||
expect(data.has('cg.voice.owed')).toBe(false);
|
||||
});
|
||||
it('never confirms a receipt or posts chat itself: the worker owns both', async () => {
|
||||
const r = await connected(), room = r.transport.rooms[0];
|
||||
room.on.subtitle({ kind: 'user_final', role: 'user', text: '今天好冷' });
|
||||
room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '那多穿一件。' });
|
||||
await r.c.end();
|
||||
expect(r.server.calls.some(c => c.url.startsWith('/api/chat'))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('subtitles and the two channels', () => {
|
||||
it('interim is a draft; user_final cancels a text reply and asks for the record; φ is heard but not recorded', async () => {
|
||||
const r = await connected(), room = r.transport.rooms[0];
|
||||
room.on.subtitle({ kind: 'interim', role: 'user', text: '今天' });
|
||||
expect(r.view().heard).toBe('今天'); expect(r.record).not.toHaveBeenCalled();
|
||||
room.on.subtitle({ kind: 'user_final', role: 'user', text: '今天好冷' });
|
||||
expect(r.spoke).toHaveBeenCalledTimes(1); expect(r.record).toHaveBeenCalledTimes(1);
|
||||
room.on.subtitle({ kind: 'error', role: 'assistant', text: '我刚才走神了，你再说一遍？' });
|
||||
expect(r.view().spoken).toBe('我刚才走神了，你再说一遍？');
|
||||
room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '那多穿一件。' });
|
||||
expect(r.view()).toMatchObject({ heard: '', spoken: '' });
|
||||
room.on.subtitle({ kind: 'bogus', text: 1 }); room.on.subtitle(null);
|
||||
expect(r.record).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
it('without the agent state attribute, subtitles still move the state honestly', async () => {
|
||||
const r = await connected(), room = r.transport.rooms[0];
|
||||
room.on.subtitle({ kind: 'user_final', role: 'user', text: '在吗' }); expect(r.view().phase).toBe('thinking');
|
||||
room.on.subtitle({ kind: 'assistant_final', role: 'assistant', text: '在。' }); expect(r.view().phase).toBe('listening');
|
||||
});
|
||||
it('typing over her spoken answer silences what is left of it until she is listening again', async () => {
|
||||
const r = await connected(), room = r.transport.rooms[0];
|
||||
r.c.hush(); expect(room.hushed).toEqual([]); // listening: nothing to silence
|
||||
room.on.agentState('speaking'); r.c.hush();
|
||||
expect(room.hushed).toEqual([true]);
|
||||
room.on.agentState('speaking'); expect(room.hushed).toEqual([true]);
|
||||
room.on.agentState('listening'); expect(room.hushed).toEqual([true, false]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('record re-read', () => {
|
||||
const msg = (id: number, role: Message['role'], turnId: string, extra: Partial<Message> = {}): Message => ({ id: String(id), role, text: `${role}${id}`, lifeIds: [], turnId, status: 'sent', ...extra });
|
||||
it('adds voice turns once, in id order; leaves reach-outs to the inbox and text turns to the stream', () => {
|
||||
const known = [msg(1, 'user', 't1'), msg(2, 'companion', 't1'), { ...msg(0, 'user', 'local-turn'), id: 'abc-local', status: 'pending' as const }];
|
||||
const rows = [msg(5, 'companion', 'v1'), msg(4, 'user', 'v1'), msg(6, 'companion', 'r6', { proactive: true }), msg(7, 'user', 'local-turn'), msg(2, 'companion', 't1')];
|
||||
const next = mergeTurns(known, rows)!;
|
||||
expect(next.map(m => m.id)).toEqual(['1', '2', 'abc-local', '4', '5']);
|
||||
expect(mergeTurns(next, rows)).toBeNull();
|
||||
});
|
||||
it('a pending voice row settles in place (sent or cancelled), never duplicated', () => {
|
||||
const known = [msg(4, 'user', 'v1', { status: 'pending' })];
|
||||
const next = mergeTurns(known, [msg(4, 'user', 'v1', { status: 'cancelled' })])!;
|
||||
expect(next).toHaveLength(1); expect(next[0].status).toBe('cancelled');
|
||||
});
|
||||
it('reads newest pages down to what the page holds, bounded', async () => {
|
||||
const rows: ServerMessage[] = Array.from({ length: 130 }, (_, i) => ({ id: i + 1, role: i % 2 ? 'companion' : 'user', text: `第${i + 1}句`, turn_id: `t${i}`, status: 'sent' }));
|
||||
const calls: string[] = [];
|
||||
vi.stubGlobal('fetch', vi.fn(async (url: string) => {
|
||||
calls.push(url);
|
||||
const u = new URL(url, 'http://x'), limit = Number(u.searchParams.get('limit')), before = Number(u.searchParams.get('before') || Infinity);
|
||||
const desc = rows.filter(r => r.id < before).sort((a, b) => b.id - a.id).slice(0, limit + 1), more = desc.length > limit, page = desc.slice(0, limit);
|
||||
return new Response(JSON.stringify({ messages: [...page].reverse(), next_before: more ? page.at(-1)!.id : null }));
|
||||
}));
|
||||
expect((await recentRows(120, new AbortController().signal)).map(m => m.id)).toEqual(['121', '122', '123', '124', '125', '126', '127', '128', '129', '130']);
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(await recentRows(60, new AbortController().signal)).toHaveLength(70);
|
||||
calls.length = 0;
|
||||
expect(await recentRows(0, new AbortController().signal, 2)).toHaveLength(100);
|
||||
expect(calls).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user