From 6a5e155258429cbbf9699fcf9011097190bb52a6 Mon Sep 17 00:00:00 2001 From: mobai Date: Tue, 20 Oct 2026 23:24:31 +0800 Subject: [PATCH 1/6] feat(hel715): LiveTalking+MuseTalk verification kit (report, material spec, e2e scripts) Co-authored-by: multica-agent --- experiments/hel715/MATERIAL-SPEC.md | 72 ++++ experiments/hel715/README.md | 42 ++ experiments/hel715/REPORT.md | 73 ++++ .../livetalking-patch/adapter-and-diff.txt | 82 ++++ .../hel715/livetalking-patch/cosyvoice.py | 69 ++++ .../enable-cosyvoice-registration.diff | 13 + experiments/hel715/scripts/e2e_client.py | 375 ++++++++++++++++++ experiments/hel715/scripts/hel715-chain.html | 129 ++++++ experiments/hel715/scripts/llm_chain2.py | 124 ++++++ experiments/hel715/scripts/record_chain.py | 59 +++ 10 files changed, 1038 insertions(+) create mode 100644 experiments/hel715/MATERIAL-SPEC.md create mode 100644 experiments/hel715/README.md create mode 100644 experiments/hel715/REPORT.md create mode 100644 experiments/hel715/livetalking-patch/adapter-and-diff.txt create mode 100644 experiments/hel715/livetalking-patch/cosyvoice.py create mode 100644 experiments/hel715/livetalking-patch/enable-cosyvoice-registration.diff create mode 100644 experiments/hel715/scripts/e2e_client.py create mode 100644 experiments/hel715/scripts/hel715-chain.html create mode 100644 experiments/hel715/scripts/llm_chain2.py create mode 100644 experiments/hel715/scripts/record_chain.py diff --git a/experiments/hel715/MATERIAL-SPEC.md b/experiments/hel715/MATERIAL-SPEC.md new file mode 100644 index 0000000..7ca55a0 --- /dev/null +++ b/experiments/hel715/MATERIAL-SPEC.md @@ -0,0 +1,72 @@ +# HEL-715 素材包规范与施工方案(可交施工员) + +依据 CG-029(日常用成品视频、交谈用同一人物实时口型)、CG-005/006(换外观保留人格/记忆/关系)、CG-030(首批素材小队制作)。原则:单图不自动生成整套;每份素材可独立替换、可校验、可回滚。 + +## 1. 素材包目录规范 + +``` +assets/personas// # persona=外观方案,如 wanwan-v2;换外观=新目录 + manifest.json # 见下 + daily/ # 日常成品视频(CG-029 主路线) + .mp4 # H.264 25fps ≥6s 可无缝循环;按时段/天气/活动命名 + talk/ # 交谈底板(近景,同房间同脸同服装) + .mp4 # H.264 25fps 5~15s,人物目视镜头、口部微闭或自然微动 + voice/ # CosyVoice 音色登记名(.12 voices.json 中的 voice 名) + ref/ # 定妆照/场景照 PNG(身份对照用) +``` + +`manifest.json` 必填字段:`persona_id`、`identity_ref`(ref/ 内主定妆图)、`daily_clips[{id, timeofday, weather, activity, loop_points}]`、`talk_clips[{id, avatar_id, framing}]`、`created_by`、`created_at`、`pipeline_version`(如 hel715-v1)。 + +## 2. 身份/服装/光线一致性标记(导入校验项) + +- 同一 persona 内所有 daily/talk 片段必须与 `identity_ref` 同人物、同发型、同套服装;换服装=新 persona 目录。 +- 每段导入时人工对照 ref 图打勾(首版人工,不做自动人脸embedding校验——留待后续)。 +- 光线:同一 persona 的交谈底板光线需与常用日常场景一致(近景底板从日常成品裁切放大可天然保证,如本轮 talk-close = loop 1.75× 裁切)。 + +## 3. 可切节点与缺素材回退 + +- 日常片循环点:`loop_points` 记录首尾衔接帧(硬循环跳变处需美术确认或选首尾近同帧的段)。 +- 交谈进入点:任意时刻可切(预连接流,实测切换 0~3ms);退出点同。 +- 缺素材回退顺序:请求场景无 daily 片 → 用同 persona 默认 daily 片;无 talk 底板 → 禁用交谈入口并提示,不得用其他人物素材顶替;TTS 失败 → 显示重试按钮,不静默。 + +## 4. 导入与回滚流程(本轮实测命令) + +``` +# 1) 预处理:任意源 → H.264 25fps(示例见 scripts/,PyAV,秒级) +# 2) 生成交谈 avatar(47~60s GPU/份,独立目录,无硬编码): +cd /opt/hel715-verify/LiveTalking +venv/bin/python -m avatars.musetalk.genavatar --file --avatar_id - +# 3) 校验:data/avatars// 含 avator_info.json/coords.pkl/latents.pt/mask;avator_info.json 的 video_path 指向素材包内文件 +# 4) 服务挂载:app.py --avatar_id (或 custom_config 多 avatar 预载) +``` + +回滚:素材包按目录整体保留旧版本(`.bak-`),avatar 目录同理;manifest 记录版本,恢复=改回引用。模型权重不入 Git,assets 二进制走附件/对象存储,仓库只放 manifest 与脚本。 + +## 5. 换一套外观的实测成本(CG-005/006 输入) + +| 项 | 实测耗时 | 说明 | +| --- | --- | --- | +| 交谈底板由既有成品裁切 | <1min(PyAV 转码)+ 47~60s genavatar | 本轮 talk-close 路线,推荐 | +| 交谈底板由 Wan2.2 全新生成 | ~57s/3s 片段 + 挑选重试(3~5 倍余量) | HEL-712 实测单次推理 | +| 日常成品视频生成 | 同上量级/段 | 小队批量制作 | +| 照片需求 | 定妆/场景各 ≥1 张 | 用于身份对照与 i2v 起点 | +| 预处理 | 自动,无人工参数 | bbox_shift=0 默认可用 | + +## 6. 首版最小验收矩阵(交谈链路) + +| 项 | 通过标准 | +| --- | --- | +| 日常循环播放 | ≥3 周期无肉眼跳变(或美术确认衔接) | +| 进交谈 | 切换 ≤200ms,同人物同服装目视确认 | +| 口型 | 说话段口部动作肉眼可辨,静息段无明显异常 | +| 打断 | interrupt 后 ≤1s 静音,不等播完 | +| 再开口 | 打断后可立即再次发话 | +| 回日常 | 切回 ≤200ms,日常从暂停点续播 | +| 持续 | 单次会话 ≥10 分钟不崩、显存不涨 | +| 失败路径 | TTS 断连时有可见错误提示,可恢复 | + +## 7. 职责划分(衔接 HEL-714 施工) + +- 客户端(web):日常选片/循环、交谈视图挂载(预连接 WebRTC)、打断信号(先按钮/快捷键,VAD 后置)、失败回退 UI。 +- 服务端:LiveTalking(交谈渲染 + TTS 适配,本分支 patch);日常视频静态文件由现有 web 服务出。 +- 素材:美术/施工按第 1 节规范产出,总工验收导入。 diff --git a/experiments/hel715/README.md b/experiments/hel715/README.md new file mode 100644 index 0000000..f0c227f --- /dev/null +++ b/experiments/hel715/README.md @@ -0,0 +1,42 @@ +# HEL-715 实验复现说明 + +实验环境(.12,全部隔离在 `/opt/hel715-verify/`,未改系统 Python/CUDA/ASR/TTS): + +- LiveTalking c726943(官方 GitHub)+ MuseTalk v1.5 权重 3.8GB(`LiveTalking/models/`) +- venv:`/opt/hel715-verify/venv`(torch 2.8.0+cu128、aiortc 1.15.0、PyAV 17.1.0) +- 既有服务复用:CosyVoice TTS `http://127.0.0.1:9880`(音色 xiaoyun)、LLM `http://192.168.200.11:8731`(halogen-qwen3.8-flash-next) +- 对上游的两处改动:`livetalking-patch/cosyvoice.py`(新增,对接 .12 CosyVoice)、`livetalking-patch/enable-cosyvoice-registration.diff`(base_avatar.py 启用注册) + +## 启动 + +```bash +cd /opt/hel715-verify/LiveTalking +nohup /opt/hel715-verify/venv/bin/python app.py --model musetalk \ + --avatar_id hel715-talk-close --transport webrtc \ + --tts cosyvoice --REF_FILE xiaoyun \ + --TTS_SERVER http://127.0.0.1:9880 --listenport 8010 \ + > /opt/hel715-verify/livetalking.log 2>&1 & +``` + +注意:仓库内 `config.yaml` 的 `TTS_SERVER: ''` 会覆盖 argparse 默认值,必须显式传 `--TTS_SERVER`(本轮曾因此空跑过一次)。 + +已生成 avatar(`LiveTalking/data/avatars/`):`hel715-loop-wide`(loop-25fps.mp4,196 帧全景)、`hel715-talk-close`(talk-close.mp4,196 帧近景=loop 1.75× 裁切)、`hel715-return`(return-25fps.mp4,77 帧独立源)。 + +## 测试脚本 + +- `scripts/e2e_client.py`:aiortc 收流客户端,四句时间线(含打断),产出 mp4+stats.json+events.json+contact-sheet.png。在 .12 上运行:`E2E_OUT=... venv/bin/python e2e_client.py`。 +- `scripts/llm_chain2.py`:LLM→TTS→数字人单链路采样。 +- `scripts/hel715-chain.html`:日常→交谈→打断→回日常→再交谈演示页(部署于 LiveTalking `web/`,`?auto=1` 自动演示;依赖 `web/hel715-daily.webm`)。 +- `scripts/record_chain.py`:本机(192.168.200.13)Playwright+Chromium 录制上述页面,需 LD_LIBRARY_PATH 指向解包的运行库(脚本内已写死 /tmp 路径,迁移时改)。 + +## 清理 + +```bash +# 停实验服务(先对 PID:pgrep -af 'app.py --model musetalk') +pkill -f 'app.py --model musetalk' +# 删除整个实验目录(含 venv/权重/素材,约 12GB) +rm -rf /opt/hel715-verify +# 演示页随目录一起消失;18080/18081 旧演示与 ASR/TTS 未动 +``` + +本轮收尾状态:LiveTalking 已停止;目录与权重保留供 HEL-714 复用。 diff --git a/experiments/hel715/REPORT.md b/experiments/hel715/REPORT.md new file mode 100644 index 0000000..8680b8e --- /dev/null +++ b/experiments/hel715/REPORT.md @@ -0,0 +1,73 @@ +# HEL-715 技术验证报告:本地真人交谈与日常视频衔接 + +日期:2026-10-20 | 环境:192.168.200.12(RTX 5090D 32GB,torch 2.8.0+cu128) +候选:LiveTalking @ c726943(官方 GitHub)+ MuseTalk v1.5 + .12 既有 CosyVoice(9880) + .11 LLM(halogen-qwen3.8-flash-next) + +## 一句话结论 + +**有条件可用。** 日常↔交谈切换、打断、25fps 连续口型视频、素材流水线复用均实测通过;条件是接受 ①发话到出声约 4.5~5.5s 的起步延迟,②TTS 流式按标点分句带来的句内 0.4~1.1s 停顿,③交谈驱动只改口型(表情/目光来自底板视频)。三项都需要在首版验收时向用户如实展示,其中 ①② 可在施工阶段继续压。 + +## 四个问题的实测回答 + +### 1. 主候选 LiveTalking+MuseTalk 实际跑通了吗?——跑通 + +- 安装:官方仓库 clone 至 `/opt/hel715-verify/LiveTalking`,独立 venv(不改系统 Python/CUDA/ASR/TTS);权重 musetalkV15/face-parse/sd-vae/whisper 共约 3.8GB,已就位。 +- 适配:仅两处改动(见 `livetalking-patch/`):`tts/cosyvoice.py` 新增(对接 .12 既有 CosyVoice 的 POST /tts 24kHz PCM 流)+ `avatars/base_avatar.py` 启用该 TTS 注册。其余代码未动。 +- 能力边界(CG-028 相关,明确来源): + - **口型**:实时音频驱动(whisper 特征→unet 逐帧生成下半脸)——实测(口部区差分:静息中位 0.74 vs 说话峰值 13~14)。 + - **表情/目光/头姿/身体**:全部来自底板视频循环(预处理时逐帧 landmark+face-parse 固化),**不是**实时生成——来源:代码结构+录像复核。交谈底板选片时必须挑"目视镜头、微动作"段。 + - 不要求也不提供逐句新肩部手势;与 CG-027/028 的预期一致,未发现必须换 Ditto 等替代的实证。 + +### 2. 同一人物实时交谈可否成立?——可以,起步延迟与句内停顿需接受 + +实测(aiortc 客户端走真实 WebRTC 通道,含网络传输;e2e-session.mp4 / stats.json): + +| 指标 | 实测值 | +| --- | --- | +| 连接→首帧视频 | 3.6s(含 ICE 协商 1.7s) | +| 文本→首次出声 | 4.6~5.5s(CosyVoice 首包 3.4s + LiveTalking 音频缓冲约 1.2s) | +| 连续帧率 | 25.05fps 恒定(27s 全程,说话与静息一致) | +| 音画同步 | 容器内音视频齐备;口型动作较出声晚约 0.5~1.3s(MuseTalk 音频前瞻缓冲所致)。自动互相关度量噪声大,最终以录像人工复核为准 | +| 句内停顿 | TTS 按标点分句合成,句内间隙 0.45~1.05s("抱歉"与"你刚想说什么"之间 0.66s)——CosyVoice 流式模式固有,非口型管线问题 | +| 打断→静音 | **0.35s**(interrupt_talk 后队列即清) | +| 打断后再开口 | 正常(S3/S4 均正常播出) | +| 峰值资源 | 总显存 18.9GB(基线 5.8GB + CosyVoice 4.2GB + LiveTalking 空载 11.2GB/说话峰值约 13.1GB);GPU 利用率瞬时 100% 但帧率不跌 | +| 失败行为 | 本轮未触发服务崩溃;语音名不存在时 TTS 适配层回退 default 音色 | + +LLM 链路(llm-chain.mp4):.11 真实模型 halogen-qwen3.8-flash-next 回复 1.33s → 递给数字人出声共 4.63s,全程录音正常。 + +### 3. 日常→交谈→打断→回日常衔接?——信号级链路实测通过 + +两份证据: + +- **浏览器真实链路**(chain/page@*.webm,Playwright+Chromium 走 LAN WebRTC):日常成品视频循环 → 切交谈视图(预连接流,切换 0~3ms)→ 发文本 → 语音进行中注入 interrupt_talk(HTTP 2~5ms)→ 立即切回日常(不等她说完、不等日常片尾)→ 再次进入交谈并发第二句 → 回日常。全程 33.5s,页面零 JS 错误。注:Playwright 录屏不含音轨,音频证据见下条。 +- **媒体级连续音画**(e2e-session.mp4):同一 WebRTC 通道收到的连续音视频原样落盘,覆盖 静息→说话→打断→静默→再说话 全程,有声有画。 + +边界说明:麦克风/VAD 未接入,打断由信号注入触发,**只是信号级验证**,不冒充自然语音抢话已通过。转场为视图级切换(同人物同房间同服装的近景底板 vs 日常全景),自然转场(如淡入淡出)属施工项,不计入本轮结论。 + +### 4. 素材能否持续制作/替换/维护?——流水线已验证可复用 + +- 同一流水线已跑通 **3 份素材、3 个 avatar**:loop 全景(196 帧)、面部近景(196 帧,经核实为 loop 素材 1.75× 放大裁切,同人物同房间同服装)、回程片段(77 帧,独立源文件)。 +- 新素材仅需:转码 25fps mp4 → `python -m avatars.musetalk.genavatar --file X --avatar_id Y`,47~60s/份(GPU),零代码改动、目录隔离、无硬编码人脸。 +- 成本实测:交谈底板若由 Wan2.2 生成约 57s/3s 片段(HEL-712 实测);由既有成品裁切仅需秒级(PyAV)。详见 MATERIAL-SPEC.md。 +- 限制:本轮三份素材同属一个人物;跨人物泛化未实测(离线环境无第二真人素材可用),算法层面为通用 landmark/face-parse 管线,无按脸硬编码逻辑。 + +## CG-029/028 能力矩阵(来源/实测/未测) + +| 能力 | 状态 | 依据 | +| --- | --- | --- | +| 成品日常视频按时段/活动选片播放 | 实测(浏览器循环播放) | chain webm | +| 同人物近景实时口型交谈 | 实测 | e2e-session.mp4 / llm-chain.mp4 | +| 口型由音频实时驱动 | 实测 | 口部差分 15~19 倍 | +| 表情/目光实时生成 | 未测(来源=底板视频) | 代码结构 | +| 语音打断(信号级) | 实测 0.35s 停止 | events.json | +| 自然语音抢话(VAD) | 未测 | 麦克风未接 | +| 日常↔交谈视图切换 | 实测 0~3ms | chain webm + 日志 | +| LLM→TTS→口型全链 | 实测 | llm-chain.mp4 | +| 素材替换保留人格记忆 | 设计已验证路径(素材目录隔离) | genavatar 复用 | + +## 风险与建议(给总管的决策输入) + +1. **起步延迟 4.5~5.5s** 是当前体验最大短板:可在施工时用"思考中"微动作/状态提示掩盖,或后续换更低首包延迟的 TTS 配置(CosyVoice 首包 3.4s 占大头)。无已确认延迟门槛,不编造 SLA;建议向用户展示实测录像后再定验收线。 +2. **句内停顿**可通过 TTS 侧整句缓冲或标点预处理缓解,代价是首包更慢——二者权衡留施工阶段。 +3. LiveTalking 空载即占 11.2GB 显存,与 ComfyUI 重生成任务并存时需调度(先交谈后生成的队列约定)。 diff --git a/experiments/hel715/livetalking-patch/adapter-and-diff.txt b/experiments/hel715/livetalking-patch/adapter-and-diff.txt new file mode 100644 index 0000000..4c783c1 --- /dev/null +++ b/experiments/hel715/livetalking-patch/adapter-and-diff.txt @@ -0,0 +1,82 @@ +import time +import numpy as np +import resampy +import requests +from io import BytesIO +from typing import Iterator + +from utils.logger import logger +from .base_tts import BaseTTS, State +from registry import register + + +@register("tts", "cosyvoice") +class CosyVoiceTTS(BaseTTS): + """适配本机既有 CosyVoice3 流式 TTS (POST /tts -> 24kHz int16 PCM 流).""" + + def txt_to_audio(self, msg: tuple[str, dict]): + text, textevent = msg + voice = textevent.get('tts', {}).get('voice', self.opt.REF_FILE or 'default') + self.stream_tts(self.cosyvoice(text, voice), msg) + + def cosyvoice(self, text: str, voice: str) -> Iterator[bytes]: + start = time.perf_counter() + try: + res = requests.post( + f"{self.opt.TTS_SERVER}/tts", + json={"text": text, "voice": voice}, + stream=True, + timeout=(5, 300), + ) + if res.status_code != 200: + logger.error("cosyvoice http %s: %s", res.status_code, res.text[:200]) + return + first = True + for chunk in res.iter_content(chunk_size=4800): # 100ms @24k int16 + if first: + logger.info("cosyvoice time to first chunk: %.3fs", time.perf_counter() - start) + first = False + if chunk and self.state == State.RUNNING: + yield chunk + elif self.state != State.RUNNING: + logger.info("cosyvoice interrupted, stop consuming") + res.close() + return + except Exception: + logger.exception('cosyvoice tts error') + + def stream_tts(self, audio_stream: Iterator[bytes], msg: tuple[str, dict]): + text, textevent = msg + first = True + for chunk in audio_stream: + if not chunk: + continue + pcm24 = np.frombuffer(chunk, dtype=np.int16).astype(np.float32) / 32767.0 + stream = resampy.resample(pcm24, sr_orig=24000, sr_new=self.sample_rate) + streamlen = stream.shape[0] + idx = 0 + while streamlen >= self.chunk: + eventpoint = {} + if first: + eventpoint = {'status': 'start', 'text': text} + first = False + eventpoint.update(**textevent) + self.parent.put_audio_frame(stream[idx:idx + self.chunk], eventpoint) + streamlen -= self.chunk + idx += self.chunk + eventpoint = {'status': 'end', 'text': text} + eventpoint.update(**textevent) + self.parent.put_audio_frame(np.zeros(self.chunk, np.float32), eventpoint) +diff --git a/avatars/base_avatar.py b/avatars/base_avatar.py +index 28c826a..a9744e6 100644 +--- a/avatars/base_avatar.py ++++ b/avatars/base_avatar.py +@@ -90,7 +90,7 @@ class BaseAvatar: + 'edgetts': 'tts.edge', + 'gpt-sovits': 'tts.sovits', + 'xtts': 'tts.xtts', +- # 'cosyvoice': 'tts.cosyvoice', ++ 'cosyvoice': 'tts.cosyvoice', + # 'fishtts': 'tts.fish', + 'tencent': 'tts.tencent', + 'doubao': 'tts.doubao', diff --git a/experiments/hel715/livetalking-patch/cosyvoice.py b/experiments/hel715/livetalking-patch/cosyvoice.py new file mode 100644 index 0000000..e7d120e --- /dev/null +++ b/experiments/hel715/livetalking-patch/cosyvoice.py @@ -0,0 +1,69 @@ +import time +import numpy as np +import resampy +import requests +from io import BytesIO +from typing import Iterator + +from utils.logger import logger +from .base_tts import BaseTTS, State +from registry import register + + +@register("tts", "cosyvoice") +class CosyVoiceTTS(BaseTTS): + """适配本机既有 CosyVoice3 流式 TTS (POST /tts -> 24kHz int16 PCM 流).""" + + def txt_to_audio(self, msg: tuple[str, dict]): + text, textevent = msg + voice = textevent.get('tts', {}).get('voice', self.opt.REF_FILE or 'default') + self.stream_tts(self.cosyvoice(text, voice), msg) + + def cosyvoice(self, text: str, voice: str) -> Iterator[bytes]: + start = time.perf_counter() + try: + res = requests.post( + f"{self.opt.TTS_SERVER}/tts", + json={"text": text, "voice": voice}, + stream=True, + timeout=(5, 300), + ) + if res.status_code != 200: + logger.error("cosyvoice http %s: %s", res.status_code, res.text[:200]) + return + first = True + for chunk in res.iter_content(chunk_size=4800): # 100ms @24k int16 + if first: + logger.info("cosyvoice time to first chunk: %.3fs", time.perf_counter() - start) + first = False + if chunk and self.state == State.RUNNING: + yield chunk + elif self.state != State.RUNNING: + logger.info("cosyvoice interrupted, stop consuming") + res.close() + return + except Exception: + logger.exception('cosyvoice tts error') + + def stream_tts(self, audio_stream: Iterator[bytes], msg: tuple[str, dict]): + text, textevent = msg + first = True + for chunk in audio_stream: + if not chunk: + continue + pcm24 = np.frombuffer(chunk, dtype=np.int16).astype(np.float32) / 32767.0 + stream = resampy.resample(pcm24, sr_orig=24000, sr_new=self.sample_rate) + streamlen = stream.shape[0] + idx = 0 + while streamlen >= self.chunk: + eventpoint = {} + if first: + eventpoint = {'status': 'start', 'text': text} + first = False + eventpoint.update(**textevent) + self.parent.put_audio_frame(stream[idx:idx + self.chunk], eventpoint) + streamlen -= self.chunk + idx += self.chunk + eventpoint = {'status': 'end', 'text': text} + eventpoint.update(**textevent) + self.parent.put_audio_frame(np.zeros(self.chunk, np.float32), eventpoint) diff --git a/experiments/hel715/livetalking-patch/enable-cosyvoice-registration.diff b/experiments/hel715/livetalking-patch/enable-cosyvoice-registration.diff new file mode 100644 index 0000000..adfb04b --- /dev/null +++ b/experiments/hel715/livetalking-patch/enable-cosyvoice-registration.diff @@ -0,0 +1,13 @@ +diff --git a/avatars/base_avatar.py b/avatars/base_avatar.py +index 28c826a..a9744e6 100644 +--- a/avatars/base_avatar.py ++++ b/avatars/base_avatar.py +@@ -90,7 +90,7 @@ class BaseAvatar: + 'edgetts': 'tts.edge', + 'gpt-sovits': 'tts.sovits', + 'xtts': 'tts.xtts', +- # 'cosyvoice': 'tts.cosyvoice', ++ 'cosyvoice': 'tts.cosyvoice', + # 'fishtts': 'tts.fish', + 'tencent': 'tts.tencent', + 'doubao': 'tts.doubao', diff --git a/experiments/hel715/scripts/e2e_client.py b/experiments/hel715/scripts/e2e_client.py new file mode 100644 index 0000000..47583f3 --- /dev/null +++ b/experiments/hel715/scripts/e2e_client.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python +"""HEL-715 end-to-end verification client for LiveTalking(MuseTalk) @ 127.0.0.1:8010. + +Connects as a WebRTC receiver (same path a browser takes), drives the avatar via +/human + /interrupt_talk, records the received A/V to mp4 and logs timing stats. +""" +import asyncio +import json +import os +import time + +import aiohttp +import av +import numpy as np +from aiortc import RTCPeerConnection, RTCSessionDescription + +BASE = os.environ.get("LT_BASE", "http://127.0.0.1:8010") +OUT = os.environ.get("E2E_OUT", "/opt/hel715-verify/e2e") +os.makedirs(OUT, exist_ok=True) + +S1 = "在呢。你回来啦,今天在外面累不累?" +S2 = "还有,我跟你说,今天下午我在窗边看了好久的天空,云跑得特别快,有一朵特别像小兔子,我还给它拍了张照片,等下拿给你看。" +S3 = "抱歉,你刚想说什么?我在听。" +S4 = "嗯,我在。" + +# approx mouth region (lower third of face bbox 218,203,324,333 from avatar prep) +MOUTH = (222, 268, 320, 331) + +log_lines = [] + + +def log(msg): + line = f"[{time.strftime('%H:%M:%S')}] {msg}" + print(line, flush=True) + log_lines.append(line) + + +class Recorder: + def __init__(self, path, w, h): + self.container = av.open(path, "w") + self.vstream = self.container.add_stream("libx264", rate=25) + self.vstream.width = w + self.vstream.height = h + self.vstream.pix_fmt = "yuv420p" + self.vstream.options = {"crf": "20", "preset": "veryfast"} + self.astream = self.container.add_stream("aac", rate=48000) + self.astream.layout = "stereo" + + def write_video(self, img_bgr): + frame = av.VideoFrame.from_ndarray(img_bgr, format="bgr24") + for p in self.vstream.encode(frame): + self.container.mux(p) + + def write_audio(self, pcm_i16, sr, ch): + arr = pcm_i16.astype(np.int16) + if ch == 1: + arr = np.repeat(arr[:, None], 2, axis=1).reshape(-1) + elif ch == 2: + arr = arr.reshape(-1) + else: + arr = arr[: len(arr) // ch * ch].reshape(-1, ch)[:, :2].reshape(-1) + if sr != 48000: + n = len(arr) // 2 + idx = (np.arange(n) * (sr / 48000.0)).astype(int) + idx = idx[idx < len(arr) // 2] + arr = arr.reshape(-1, 2)[idx].reshape(-1) + frame = av.AudioFrame.from_ndarray( + np.ascontiguousarray(arr.reshape(1, -1)), format="s16", layout="stereo" + ) + frame.sample_rate = 48000 + for p in self.astream.encode(frame): + self.container.mux(p) + + def close(self): + for p in self.vstream.encode(): + self.container.mux(p) + for p in self.astream.encode(): + self.container.mux(p) + self.container.close() + + +class Session: + def __init__(self): + self.events = {"video": [], "audio": [], "speak": [], "actions": []} + self.imgs_sample = [] + self.sid = None + self.http = None + + async def post(self, path, payload): + t0 = time.perf_counter() + async with self.http.post(BASE + path, json=payload) as r: + body = await r.json() + return time.perf_counter() - t0, body + + async def poll_speaking(self, stop_evt): + while not stop_evt.is_set(): + try: + dt, body = await self.post("/is_speaking", {"sessionid": self.sid}) + self.events["speak"].append( + [round(time.perf_counter(), 3), body.get("data", False)] + ) + except Exception: + pass + await asyncio.sleep(0.25) + + +def mouth_motion(prev, cur): + if prev is None or cur is None: + return 0.0 + x0, y0, x1, y1 = MOUTH + d = np.abs( + cur[y0:y1, x0:x1].astype(np.int16) - prev[y0:y1, x0:x1].astype(np.int16) + ).mean() + return float(d) + + +async def main(): + stats = {"sentences": {}, "meta": {}} + rec = None + pc = RTCPeerConnection() + sess = Session() + t_start = time.perf_counter() + + done = asyncio.Event() + + @pc.on("track") + def on_track(track): + log(f"track: {track.kind}") + if track.kind == "video": + asyncio.ensure_future(video_loop(track)) + else: + asyncio.ensure_future(audio_loop(track)) + + async def video_loop(track): + nonlocal rec + prev = None + while True: + try: + frame = await track.recv() + except Exception: + break + img = frame.to_ndarray(format="bgr24") + if rec is None: + rec = Recorder(os.path.join(OUT, "e2e-session.mp4"), img.shape[1], img.shape[0]) + log(f"recorder ready {img.shape[1]}x{img.shape[0]}") + t = time.perf_counter() + m = mouth_motion(prev, img) + sess.events["video"].append([round(t, 3), round(m, 2)]) + prev = img + rec.write_video(img) + if len(sess.imgs_sample) < 400: + sess.imgs_sample.append((round(t - t_start, 2), img)) + + async def audio_loop(track): + while True: + try: + frame = await track.recv() + except Exception: + break + t = time.perf_counter() + arr = frame.to_ndarray() + if arr.dtype != np.int16: + if arr.dtype in (np.float32, np.float64): + arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) + else: + arr = arr.astype(np.int16) + ch = len(frame.layout.channels) + if arr.ndim == 2 and arr.shape[0] == 1: + pcm = arr.reshape(-1) # packed s16 + elif arr.ndim == 2 and arr.shape[0] == ch: + pcm = arr.T.reshape(-1) # planar + else: + pcm = arr.reshape(-1) + ch = 1 + rms = float(np.sqrt((pcm.astype(np.float64) ** 2).mean()) / 32767.0) + sess.events["audio"].append([round(t, 3), round(rms, 5)]) + if rec is not None: + rec.write_audio(pcm, frame.sample_rate, ch) + + async with aiohttp.ClientSession() as http: + sess.http = http + pc.addTransceiver("video", direction="recvonly") + pc.addTransceiver("audio", direction="recvonly") + offer = await pc.createOffer() + await pc.setLocalDescription(offer) + while pc.iceGatheringState != "complete": + await asyncio.sleep(0.1) + t0 = time.perf_counter() + async with http.post( + BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"} + ) as r: + answer = await r.json() + log(f"offer round-trip {time.perf_counter()-t0:.3f}s sessionid={answer.get('sessionid')}") + sess.sid = answer["sessionid"] + await pc.setRemoteDescription( + RTCSessionDescription(sdp=answer["sdp"], type=answer["type"]) + ) + stats["meta"]["sessionid"] = sess.sid + + stop_evt = asyncio.Event() + poller = asyncio.ensure_future(sess.poll_speaking(stop_evt)) + + def mark(name): + t = time.perf_counter() + sess.events["actions"].append([round(t, 3), name]) + log(f"ACTION {name}") + return t + + # wait for first video frame / idle + for _ in range(200): + if sess.events["video"]: + break + await asyncio.sleep(0.1) + first_v = sess.events["video"][0][0] if sess.events["video"] else -1 + first_a = sess.events["audio"][0][0] if sess.events["audio"] else -1 + stats["meta"]["first_video_s"] = round(first_v - t_start, 3) + stats["meta"]["first_audio_s"] = round(first_a - t_start, 3) + log(f"first video {stats['meta']['first_video_s']}s audio {stats['meta']['first_audio_s']}s") + await asyncio.sleep(3.0) # idle baseline + + async def speak(name, text, wait_end=True, interrupt_after=None, next_line=None): + t_send = mark(f"send:{name}") + dt, body = await sess.post( + "/human", {"sessionid": sess.sid, "type": "echo", "text": text} + ) + log(f"/human {name} ack {dt:.3f}s {body}") + t_interrupt = None + if interrupt_after is not None: + await asyncio.sleep(interrupt_after) + t_interrupt = mark(f"interrupt:{name}") + dt2, b2 = await sess.post("/interrupt_talk", {"sessionid": sess.sid}) + log(f"/interrupt_talk ack {dt2:.3f}s {b2}") + if wait_end: + # wait until speaking False and audio silent + silent = 0.0 + last_speak_true = time.perf_counter() + while True: + await asyncio.sleep(0.1) + now = time.perf_counter() + sp = sess.events["speak"][-1][1] if sess.events["speak"] else None + if sp: + last_speak_true = now + recent = [a for a in sess.events["audio"] if a[0] > now - 0.6] + loud = any(a[1] > 0.02 for a in recent) + if not sp and not loud: + silent = now - last_speak_true + if silent > 1.2: + break + else: + last_speak_true = max(last_speak_true, now) + return t_send, t_interrupt + + # S1 short + s, _ = await speak("S1", S1) + stats["sentences"]["S1"] = {"text": S1, "sent_at": round(s - t_start, 2)} + await asyncio.sleep(1.5) + # S2 long with interrupt 3s after send + s2, ti = await speak("S2", S2, wait_end=True, interrupt_after=3.0) + stats["sentences"]["S2"] = { + "text": S2, "sent_at": round(s2 - t_start, 2), + "interrupt_at": round(ti - t_start, 2), + } + await asyncio.sleep(2.0) + # S3 re-open after interrupt + s3, _ = await speak("S3", S3) + stats["sentences"]["S3"] = {"text": S3, "sent_at": round(s3 - t_start, 2)} + await asyncio.sleep(3.0) # pause test + s4, _ = await speak("S4", S4) + stats["sentences"]["S4"] = {"text": S4, "sent_at": round(s4 - t_start, 2)} + mark("end") + await asyncio.sleep(2.0) + stop_evt.set() + poller.cancel() + await pc.close() + + if rec: + rec.close() + log("recording closed") + + # analysis + def rel(ts): + return round(ts - t_start, 3) + + ev_a = sess.events["audio"] + ev_v = sess.events["video"] + for name, info in stats["sentences"].items(): + sent = info.get("sent_at", 0) + t_start + aud = [a for a in ev_a if a[0] >= sent and a[1] > 0.02] + info["first_audible_rel_s"] = rel(aud[0][0]) if aud else None + if aud: + last = sent + for a in aud: + if a[0] - last < 1.5: + last = a[0] + info["last_audible_rel_s"] = rel(last) + if "interrupt_at" in info: + it = info["interrupt_at"] + t_start + after = [a for a in ev_a if a[0] >= it and a[1] > 0.02] + info["audible_after_interrupt_s"] = rel(after[0][0]) if after else None + tail = [a for a in ev_a if a[0] >= it] + tl = [a for a in tail if a[1] > 0.02] + info["last_audible_after_interrupt_rel_s"] = rel(tl[-1][0]) if tl else None + + # video motion around each sentence + for name, info in stats["sentences"].items(): + sent = info.get("sent_at", 0) + t_start + window = [v for v in ev_v if sent - 1.0 <= v[0] <= sent + 12] + base = [v for v in ev_v if v[0] < sent - 1.0][-60:] + if window and base: + b = np.median([v[1] for v in base]) + info["mouth_motion_idle"] = round(float(b), 2) + peak = max(v[1] for v in window) + info["mouth_motion_peak"] = round(float(peak), 2) + + # fps over speech windows + if ev_v: + dur = ev_v[-1][0] - ev_v[0][0] + stats["meta"]["video_frames"] = len(ev_v) + stats["meta"]["video_span_s"] = round(dur, 2) + stats["meta"]["avg_recv_fps"] = round(len(ev_v) / dur, 2) if dur > 0 else None + if ev_a: + dur = ev_a[-1][0] - ev_a[0][0] + stats["meta"]["audio_frames"] = len(ev_a) + stats["meta"]["audio_span_s"] = round(dur, 2) + + stats["meta"]["wall_total_s"] = round(time.perf_counter() - t_start, 2) + + with open(os.path.join(OUT, "stats.json"), "w") as f: + json.dump(stats, f, ensure_ascii=False, indent=1) + with open(os.path.join(OUT, "events.json"), "w") as f: + json.dump( + { + "t_start": t_start, + "video": sess.events["video"], + "audio": sess.events["audio"], + "speak": sess.events["speak"], + "actions": sess.events["actions"], + }, + f, + ) + with open(os.path.join(OUT, "run.log"), "w") as f: + f.write("\n".join(log_lines)) + + # contact sheet: pick frames around S1 onset, S2 mid, interrupt, S3 + try: + from PIL import Image + + picks = [] + s1 = stats["sentences"]["S1"].get("first_audible_rel_s") or 5 + s2i = stats["sentences"]["S2"].get("interrupt_at") + picks += [max(0, s1 - 1.5), s1 + 0.5, s1 + 2.5] + if s2i: + picks += [s2i - 1.0, s2i + 0.3, s2i + 1.5] + picks += [stats["sentences"]["S3"]["sent_at"] + 2.0] + tiles = [] + for p in picks: + cand = min(sess.imgs_sample, key=lambda x: abs(x[0] - p)) + im = Image.fromarray(cand[1][:, :, ::-1]).resize((276, 276)) + tiles.append((cand[0], im)) + sheet = Image.new("RGB", (276 * len(tiles), 276 + 18), "black") + from PIL import ImageDraw + + d = ImageDraw.Draw(sheet) + for i, (ts, im) in enumerate(tiles): + sheet.paste(im, (i * 276, 18)) + d.text((i * 276 + 4, 2), f"t={ts:.1f}s", fill="white") + sheet.save(os.path.join(OUT, "contact-sheet.png")) + except Exception as e: + log(f"contact sheet failed: {e}") + + log("DONE") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/experiments/hel715/scripts/hel715-chain.html b/experiments/hel715/scripts/hel715-chain.html new file mode 100644 index 0000000..61949b0 --- /dev/null +++ b/experiments/hel715/scripts/hel715-chain.html @@ -0,0 +1,129 @@ + + + + +HEL-715 日常→交谈→打断→日常 链路演示 + + + +
+ + +
0.0s
+
+
初始化…
成品日常视频循环播放 · 交谈流后台预连接
+
+ + + diff --git a/experiments/hel715/scripts/llm_chain2.py b/experiments/hel715/scripts/llm_chain2.py new file mode 100644 index 0000000..3d7e302 --- /dev/null +++ b/experiments/hel715/scripts/llm_chain2.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +"""LLM-chained conversation sample: .11 LLM -> .12 CosyVoice -> MuseTalk avatar. +Same receiver architecture as e2e_client.py (proven to capture A/V).""" +import asyncio, json, os, time +import aiohttp, av, numpy as np +from aiortc import RTCPeerConnection, RTCSessionDescription + +BASE = "http://127.0.0.1:8010" +LLM = "http://192.168.200.11:8731/v1/chat/completions" +OUT = "/opt/hel715-verify/e2e" +os.makedirs(OUT, exist_ok=True) + + +class Recorder: + def __init__(self, path, w, h): + self.container = av.open(path, "w") + self.v = self.container.add_stream("libx264", rate=25) + self.v.width, self.v.height, self.v.pix_fmt = w, h, "yuv420p" + self.v.options = {"crf": "20", "preset": "veryfast"} + self.a = self.container.add_stream("aac", rate=48000) + self.a.layout = "stereo" + + def video(self, img): + for p in self.v.encode(av.VideoFrame.from_ndarray(img, format="bgr24")): + self.container.mux(p) + + def audio(self, frame): + arr = frame.to_ndarray() + if arr.dtype != np.int16: + arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) + ch = len(frame.layout.channels) + if arr.ndim == 2 and arr.shape[0] == 1: + pcm = arr.reshape(-1) + elif arr.ndim == 2 and arr.shape[0] == ch: + pcm = arr.T.reshape(-1) + else: + pcm = arr.reshape(-1) + fr = av.AudioFrame.from_ndarray(np.ascontiguousarray(pcm.reshape(1, -1)), format="s16", layout="stereo") + fr.sample_rate = 48000 + for p in self.a.encode(fr): + self.container.mux(p) + + def close(self): + for p in self.v.encode(): self.container.mux(p) + for p in self.a.encode(): self.container.mux(p) + self.container.close() + + +async def main(): + t0 = time.perf_counter() + result = {} + async with aiohttp.ClientSession() as http: + async with http.post(LLM, json={ + "model": "halogen-qwen3.8-flash-next", + "messages": [{"role": "user", "content": "你是住在高层公寓卧室里的伴侣晚晚,用一两句自然中文回应:我回来了,今天好累啊。"}], + "max_tokens": 80, + }) as r: + body = await r.json() + reply = body["choices"][0]["message"]["content"] + result["llm_model"] = body["model"] + result["llm_reply"] = reply + result["llm_latency_s"] = round(time.perf_counter() - t0, 2) + print(f"[{result['llm_latency_s']}s] LLM: {reply}", flush=True) + + pc = RTCPeerConnection() + rec = {"r": None} + audio_marks = [] + + @pc.on("track") + def on_track(track): + async def drain(): + while True: + try: + f = await track.recv() + except Exception: + break + t = time.perf_counter() + if track.kind == "video": + img = f.to_ndarray(format="bgr24") + if rec["r"] is None: + rec["r"] = Recorder(f"{OUT}/llm-chain.mp4", img.shape[1], img.shape[0]) + rec["r"].video(img) + else: + arr = f.to_ndarray() + if arr.dtype != np.int16: + arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) + rms = float(np.sqrt((arr.astype(np.float64) ** 2).mean()) / 32767) + audio_marks.append([round(t - t0, 3), round(rms, 5)]) + if rec["r"] is not None: + rec["r"].audio(f) + asyncio.ensure_future(drain()) + + pc.addTransceiver("video", direction="recvonly") + pc.addTransceiver("audio", direction="recvonly") + offer = await pc.createOffer() + await pc.setLocalDescription(offer) + while pc.iceGatheringState != "complete": + await asyncio.sleep(0.1) + async with http.post(BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r: + answer = await r.json() + await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"])) + sid = answer["sessionid"] + await asyncio.sleep(3) + t_send = time.perf_counter() - t0 + async with http.post(BASE + "/human", json={"sessionid": sid, "type": "echo", "text": reply}) as r: + await r.json() + print(f"[{t_send:.2f}s] sent to avatar", flush=True) + await asyncio.sleep(15) + await pc.close() + + if rec["r"]: + rec["r"].close() + loud = [m for _, m in audio_marks if _ and m > 0.02] + first_loud = next((t for t, m in audio_marks if m > 0.02), None) + result["sent_at_s"] = round(t_send, 2) + result["first_audible_s"] = round(first_loud, 2) if first_loud else None + result["send_to_audible_s"] = round(first_loud - t_send, 2) if first_loud else None + with open(f"{OUT}/llm-chain-stats.json", "w") as f: + json.dump(result, f, ensure_ascii=False, indent=1) + print(json.dumps(result, ensure_ascii=False), flush=True) + print("DONE", flush=True) + + +asyncio.run(main()) diff --git a/experiments/hel715/scripts/record_chain.py b/experiments/hel715/scripts/record_chain.py new file mode 100644 index 0000000..0a32873 --- /dev/null +++ b/experiments/hel715/scripts/record_chain.py @@ -0,0 +1,59 @@ +import asyncio +import json +import os +import pathlib + +os.environ["LD_LIBRARY_PATH"] = ( + "/tmp/multica-task-4070005680/debs/root/usr/lib/x86_64-linux-gnu" + + (":" + os.environ["LD_LIBRARY_PATH"] if os.environ.get("LD_LIBRARY_PATH") else "") +) + +from playwright.async_api import async_playwright + +OUT = pathlib.Path("/tmp/multica-task-4070005680/opencode/chain") +OUT.mkdir(exist_ok=True) +URL = "http://192.168.200.12:8010/hel715-chain.html?auto=1" + + +async def main(): + async with async_playwright() as p: + browser = await p.chromium.launch( + args=["--autoplay-policy=no-user-gesture-required", "--no-sandbox"] + ) + ctx = await browser.new_context( + viewport={"width": 1280, "height": 800}, + record_video_dir=str(OUT), + record_video_size={"width": 1280, "height": 800}, + ) + page = await ctx.new_page() + errors = [] + page.on("pageerror", lambda e: errors.append(str(e))) + await page.goto(URL, wait_until="domcontentloaded") + # wait for demo end marker (auto demo ~37s) + try: + await page.wait_for_function( + "() => window.__chainLog && window.__chainLog.some(l => l.includes('AUTO-DEMO-END'))", + timeout=60000, + ) + except Exception as e: + errors.append(f"wait_for_function: {e}") + await asyncio.sleep(1.5) + logs = await page.evaluate("() => window.__chainLog") + metrics = { + "url": URL, + "chain_log": logs, + "pageerrors": errors, + } + (OUT / "chain-browser-log.json").write_text( + json.dumps(metrics, ensure_ascii=False, indent=1) + ) + video = page.video + await ctx.close() + path = await video.path() + await browser.close() + print("video:", path) + print("errors:", errors) + print(json.dumps(logs, ensure_ascii=False, indent=1)) + + +asyncio.run(main()) -- 2.54.0 From bcc0eb7036aaa9203d307d82e7a6326a41e01a29 Mon Sep 17 00:00:00 2001 From: mobai Date: Wed, 21 Oct 2026 00:14:18 +0800 Subject: [PATCH 2/6] feat(HEL-714): add isolated daily-video and local talking sample Co-authored-by: multica-agent --- samples/companion/.gitignore | 14 + samples/companion/README.md | 80 ++ samples/companion/REPORT.md | 71 + samples/companion/assets/active.json | 3 + .../wanwan-v2-cream-night/manifest.json | 60 + .../wanwan-v2-cream-night/voice/voice.json | 4 + samples/companion/index.html | 26 + samples/companion/package-lock.json | 1166 +++++++++++++++++ samples/companion/package.json | 19 + samples/companion/public/pcm-worklet.js | 23 + samples/companion/runtime/config.json | 8 + samples/companion/runtime/server.py | 253 ++++ samples/companion/src/controls.css | 70 + samples/companion/src/main.ts | 323 +++++ samples/companion/src/style.css | 20 + samples/companion/src/tokens.css | 73 ++ samples/companion/tools/analyze_sync.py | 91 ++ samples/companion/tools/baseline.py | 50 + samples/companion/tools/check_browser.py | 136 ++ samples/companion/tools/check_gateway.py | 127 ++ samples/companion/tools/check_microphone.py | 89 ++ samples/companion/tools/check_races.py | 52 + .../companion/tools/generate_transition.py | 251 ++++ samples/companion/tools/pack_assets.py | 41 + samples/companion/tools/patch_driver.py | 47 + samples/companion/tools/prepare_driver.py | 26 + samples/companion/tools/prepare_material.py | 37 + samples/companion/tools/soak.py | 85 ++ samples/companion/tools/stage_assets.py | 24 + samples/companion/tsconfig.json | 20 + .../versions/hel714-v1/manifest.json | 51 + 31 files changed, 3340 insertions(+) create mode 100644 samples/companion/.gitignore create mode 100644 samples/companion/README.md create mode 100644 samples/companion/REPORT.md create mode 100644 samples/companion/assets/active.json create mode 100644 samples/companion/assets/personas/wanwan-v2-cream-night/manifest.json create mode 100644 samples/companion/assets/personas/wanwan-v2-cream-night/voice/voice.json create mode 100644 samples/companion/index.html create mode 100644 samples/companion/package-lock.json create mode 100644 samples/companion/package.json create mode 100644 samples/companion/public/pcm-worklet.js create mode 100644 samples/companion/runtime/config.json create mode 100644 samples/companion/runtime/server.py create mode 100644 samples/companion/src/controls.css create mode 100644 samples/companion/src/main.ts create mode 100644 samples/companion/src/style.css create mode 100644 samples/companion/src/tokens.css create mode 100644 samples/companion/tools/analyze_sync.py create mode 100644 samples/companion/tools/baseline.py create mode 100644 samples/companion/tools/check_browser.py create mode 100644 samples/companion/tools/check_gateway.py create mode 100644 samples/companion/tools/check_microphone.py create mode 100644 samples/companion/tools/check_races.py create mode 100644 samples/companion/tools/generate_transition.py create mode 100644 samples/companion/tools/pack_assets.py create mode 100644 samples/companion/tools/patch_driver.py create mode 100644 samples/companion/tools/prepare_driver.py create mode 100644 samples/companion/tools/prepare_material.py create mode 100644 samples/companion/tools/soak.py create mode 100644 samples/companion/tools/stage_assets.py create mode 100644 samples/companion/tsconfig.json create mode 100644 samples/companion/versions/hel714-v1/manifest.json diff --git a/samples/companion/.gitignore b/samples/companion/.gitignore new file mode 100644 index 0000000..03d8cb0 --- /dev/null +++ b/samples/companion/.gitignore @@ -0,0 +1,14 @@ +node_modules/ +dist/ +public/assets/*.mp4 +public/assets/*.png +runtime/*.log +runtime/*.pid +__pycache__/ +evidence/ + +packs/ + +public/assets/ +assets/personas/**/*.mp4 +assets/personas/**/*.png diff --git a/samples/companion/README.md b/samples/companion/README.md new file mode 100644 index 0000000..7141d8a --- /dev/null +++ b/samples/companion/README.md @@ -0,0 +1,80 @@ +HEL-714 日常成片+本地半身交谈样机,基于远端 main `1e9b08a` 和 P10 `6a5e155`。独立交付,不合并、不部署。PR #1 仍是旧分层小样,目标 main;本目录不依赖其尚未合并代码。 + +## 模块边界 + +- `src/main.ts`:日常选片、可切点、真实进退素材、WebRTC、麦克风/能量 VAD、原生 PCM AudioWorklet、取消、恢复和浏览器音画录制。 +- `runtime/server.py`:aiohttp 同源网关;既有 FunASR WebSocket、LLM HTTP、CosyVoice HTTP、LiveTalking WebRTC 信令与音频上传。只读素材 manifest。不写人格、八字、记忆、关系或生活数据。 +- `src/tokens.css` 原样复用 HEL-711 的唯一 token 表;`controls.css` 从已确认雨夜界面的基础控件抽取,未另造主题。页面布局在 `style.css`,无追加覆盖旧样式。没有引入动画库;Emil animate 模块的目的为避免场景跳变,主体过渡依赖真实正向素材,减少动态偏好下 UI 无按压位移。 +- `tools/patch_driver.py` 仅修改独立 LiveTalking 代码副本,给 peer 记录 session id,并增加精确关闭该 peer 的端点。模型权重复用,不改 P10 安装、18080/18081、ASR/TTS 或系统环境。 + +## 运行候选 + +先解压本卡复核包,恢复 `assets/personas/wanwan-v2-cream-night`(整个外观包)。素材二进制不入 Git。依赖锁来自既有小样,没有新增 npm 依赖。 + +```bash +npm ci --ignore-scripts --no-audit --no-fund +python3 tools/pack_assets.py +npm run build +``` + +在 .12 复用 `/opt/hel715-verify/venv`(aiohttp 3.14.5、aiortc 1.15.0、PyAV 17.1.0、torch 2.8.0+cu128、numpy 2.5.4、resampy 0.4.3),不安装或升级系统依赖。先检查 8010、8014 没有其他监听者。 + +```bash +python3 tools/prepare_driver.py --destination /opt/hel714-review/LiveTalking +python3 tools/patch_driver.py /opt/hel714-review/LiveTalking +``` + +把构建结果、runtime 和素材放在独立样机目录后,在独立 LiveTalking 目录生成本包 avatar(仓库中没有权重或预处理张量): + +```bash +cd /opt/hel714-review/LiveTalking +/opt/hel715-verify/venv/bin/python -m avatars.musetalk.genavatar \ + --file /path/to/sample/dist/assets/talk/gaze.mp4 --avatar_id hel714-gaze-v1 +/opt/hel715-verify/venv/bin/python app.py --model musetalk \ + --avatar_id hel714-gaze-v1 --batch_size 8 --transport webrtc --tts cosyvoice \ + --REF_FILE xiaoyun --TTS_SERVER http://127.0.0.1:9880 \ + --listenport 8010 --stun '' +``` + +另一个前台终端在本目录启动网关;Ctrl-C 后确认两个自测进程都退出。禁止 pkill、fuser -k 或停止共享服务。`runtime/config.json` 是本样机唯一模型/端点配置。 + +```bash +/opt/hel715-verify/venv/bin/python runtime/server.py --port 8014 +``` + +浏览器在 .12 本机可打开 `http://localhost:8014`。远程麦克风需要 HTTPS;本次自测只读复用已有 `/opt/voice-agent/web/certs/cert.pem`、`key.pem`,给网关传 `--host 0.0.0.0 --cert ... --key ...`。未改系统证书信任、防火墙或开机启动。自动化使用 Chromium 的忽略证书检查,仅作用于该测试浏览器;普通浏览器是否能授权须实际确认。 + +## 行为和取消契约 + +日常片目前只有一份雨夜阅读。活动、手动天气、时段全部匹配才选中;无匹配时明确回退默认片。没有接真实天气,没有批量换装或跨人物验证。 + +进入时等阅读第 96 帧,播真实正向进场,再交谈;返回用独立正向生成回程,续播原日常切点。v2 新底板来自已确认的 `04-交谈近景-v2.png`,有微动作和目视镜头候选;表情/身体/目光来自底板,只有口型实时驱动。全部仍待总工人工观感审核,不能据“可播放”判断自然。 + +每轮回复拥有独立 peer。取消首先断开浏览器音频图、静音/暂停旧视频、关闭 peer、递增轮次;后端取消 LLM/TTS HTTP 任务,关闭上游响应,再精确销毁该 driver session(含其推理/输出队列)。旧 peer 的 RTP 和旧轮异步错误不进入新轮。这个选择增加建连成本,换来不复用难以完整清理的上游队列。共享 CosyVoice 服务不由本样机管理:客户端断开不等于已证明共享模型内部 GPU 生成即时停止。 + +LLM 和整句 TTS 生成与 ICE/转场并行。网关等浏览器 `/api/ready` 才把音频送入驱动器,防止转场期间提前说话。整句缓冲减少标点分句空隙,代价是长回复要等整句生成;未声称首包 SLA 或流式生成性能通过。前端的“等待首声”是真实等待。 + +麦克风请求 AEC/降噪,原生 Worklet 输出 16k PCM;150ms 连续能量启动会立即停止旧回复,650ms 静音发送 ASR 端点。FunASR `2pass-online` 仅展示,`2pass-offline` 收集终稿后提交。失败不伪造文字。长停顿、轻声、扬声器回声仍需真人校准;请用耳机。按钮和合成 WAV 结果均不冒充真人自然抢话通过。 + +## 素材版本和回滚 + +`assets/personas/wanwan-v2-cream-night/manifest.json`:稳定 `companion_id=wanwan`、外观 `persona_id`、服装/光线/天气标签、avatar id、帧级切点、v2 版本、v1 前版和逐文件 SHA256。换包不创建伴侣或写入身份状态。 + +先把候选完整包解压到新目录,用 `tools/pack_assets.py --assets <目录>` 校验,再整体替换 `assets/personas/`,在 `assets/active.json` 选择它,再构建、重启独立测试服务。禁止把不同版本的单文件混放。avatar 按 manifest 的 id 重新生成或复用对应缓存;网关从 manifest 指定 avatar,不使用别的脸兜底。构建先校验再把所选整包挂载到生成目录 `public/assets`,该目录可由构建重建,勿作为用户素材唯一副本。回滚对旧完整包执行同样步骤。复核包另附 `packs/hel714-v1`,仓库只保留它的 manifest 元数据;当前包为 hel714-v2。 + +## 检查 + +```bash +npm run build +python3 -m ruff check samples/companion/runtime samples/companion/tools # 仓库根目录 +python3 -m pytest -q # 仓库根目录,原78项 +/opt/hel715-verify/venv/bin/python tools/check_gateway.py # .12,5项取消/音色/模型故障检查 +LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/check_browser.py +LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/check_races.py +LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/check_microphone.py +LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/soak.py +``` + +浏览器脚本依赖运行机已安装 Playwright/Chromium;麦克风脚本需复核包内的 `evidence/synthetic-microphone.wav`。它是 CosyVoice 合成的测试句,经 Chromium 假麦克风送入实际 Worklet→ASR→LLM→TTS→口型链路,绝不是人现场说话。录制用页面实际绘制画面和实际播放的 WebAudio 输出、同一个 MediaRecorder 时钟,不按帧数重建时间戳;静音时钟从开头持续输出零样本,保留进入前的日常画面。连续音画是浏览器播放图的合成采集,不是物理屏幕/扬声器校准。 + +实测结果和未通过项见 `REPORT.md`。生成工作流、history、原始候选片、浏览器证据和资源采样放复核附件,不在主仓库存大文件。 diff --git a/samples/companion/REPORT.md b/samples/companion/REPORT.md new file mode 100644 index 0000000..a76b6d3 --- /dev/null +++ b/samples/companion/REPORT.md @@ -0,0 +1,71 @@ +HEL-714 候选交付记录,2026-10-20~21(上海时间)。基线 main `1e9b08a`,引入已选 P10 实验提交 `6a5e155`,独立样机位于 `samples/companion`。 + +## 结论 + +代码、构建、基础自测和连续音画已交候选;**同步与自然观感、真人自然抢话、手机真机仍未通过体验验收,禁止据此发布或宣称首版完成**。后续由总管组织总工审查。当前需要继续处理的首要问题是口部起点提前的来源;没有用“思考中”掩盖,也未盲加固定延迟。 + +已实现:单组日常片按活动/手动天气/时段匹配及明确回退;已确认 v2 近景图生成目视微动作候选;真实正向进场/回程和第 96 帧可切点;文字与麦克风入口;FunASR→LLM→TTS→MuseTalk;停止、倾听、回日常、再进入;旧轮隔离;缺片、网络、模型与音色错误提示;素材版本/校验/回滚。 + +## 同步:实际播放证据、修正和未通过项 + +P10 附件 REPORT、MATERIAL-SPEC、e2e-session、chain-demo-browser、llm-chain 均经鉴权 CLI 下载,文档与远端实验分支一致。先读了原始适配器、音频特征/输出队列和录制脚本。 + +旧采样脚本按音频样本/视频帧数重建容器时间轴,首个视频 recorder 建立前的音频也会丢弃;因此不把原报告的“口型晚 0.5–1.3 秒”当作已标定真实浏览器偏差。本轮改为同一 Chromium 绘制的画面+同一 WebAudio 实际播放图输出,交给一个 MediaRecorder;不重新按帧数生成时间戳。 + +还修正了两项采集问题:Chromium 从 WebRTC video 元素建立 MediaElementSource 时分析输出全零,改用同一接收流的 MediaStreamSource,播放和录音共用输出;无 peer 时音频图缺少活动源会裁掉录像开头,增加恒为零的音频时钟后,最终录像视频从 0 秒开始、首个静音音频包 0.03 秒,完整保留日常和进场。无物理扬声器/屏幕的硬件时钟校准。 + +| 样本 | 口部变化起点−声音起点 | 含义 | +| --- | ---: | --- | +| P10 同驱动、原裁切底板,重新在浏览器采集 | −0.081s | 只比较起点,不能证明逐音节通过;没有复现报告的明显起步滞后 | +| 新目视底板,batch 16,第二轮完整静息段校准 | −0.635s | 口部先变化,体验验收仍有缺口 | +| 新目视底板,batch 8,单份对照 | −0.622s | 减半批量未稳定消除差异 | +| 最终包 batch 8,复测第二轮 | −0.424s | 仍有提前;不能据单个数值说同步已修好 | + +这些数值来自源片相对口部残差与音频 RMS>0.015,使用录制文件本身的 PTS。新底板先按整段无语音静息片校准,再查口部残差越阈;坐标与窗口保存在脚本/CSV/JSON 中。头部微动、生成器改动嘴周和噪声会影响检测;它不是音素对齐或人工听看结论。各段不是相同台词、相同负载的受控基准,不能将上表差值解释为修复幅度。连续音画、口部帧条和原始数值一并提供。 + +下一条最小路径:固定同一段含清晰闭唇/开口音节的音频,在同一浏览器做原生 video 音频播放与 WebAudio 输出的 A/B,采集轨道/播放时钟并逐音节听看,区分浏览器播放图与 Whisper 前瞻/音频帧关联;确认稳定偏移来源后再改时间基准或特征对齐。当前没有证明消除了明显错位,列为体验验收阻塞。 + +## 素材和自然往返 + +复用已有 Wan2.2 生成工具,独立前台任务生成并停止自己的 ComfyUI PID 687113。无新模型、无新增依赖,未访问 4080。新 avatar 预处理只用原 MuseTalk 权重,新缓存独立保存在样机副本。三个任务 history 均 success,实际生成耗时约 65.5s、37.6s、34.4s。 + +| 素材 | 来源和长度 | 审核状态 | +| --- | --- | --- | +| 日常 | P10 loop,196帧/25fps | 单份雨夜阅读;其他天气/活动缺片,明确回退 | +| 倾听/交谈底板 | 确认过的04近景v2中裁,81个新生成帧→127帧/25fps,5.08s | 目视、眨眼、呼吸候选;身体/目光来自片段 | +| 进场 | 日常f96→近景v2,49帧→77帧/25fps,3.08s | 真正正向生成,非淡化/倒放;仍待人工检查形变和镜头连续性 | +| 回程 | 近景v2→日常f96,独立49帧→77帧/25fps,3.08s | 与进场分开生成,取消声音不等回程 | + +页面等到日常 f96 再进场;回来续该位置。连续录屏覆盖完整往返、说话中取消、再次说话和准备期取消。联系表已核对同人物/发型/服装/房间的方向一致性,不代表动态自然性通过。镜头变焦、书本/手部、坐垫形变、循环衔接必须由总工看连续片人工裁定。 + +canonical 素材放 `assets/personas/wanwan-v2-cream-night`,含 daily/talk/ref/voice/transitions 和 manifest。版本 hel714-v2,前版 hel714-v1;6 个文件逐一 SHA256。构建只在全部校验后挂载选中整包至生成目录 public/assets。v1/v2 均校验通过,篡改 talk 文件会拒绝;复核包提供 v1 完整回滚包。固定 companion_id=wanwan 与 persona/服装/版本分离;代码不写人格、八字、记忆或关系。没有跨人物、跨服装或真实天气验证。 + +## 真实链路、取消与恢复 + +既有 ASR 协议经只读检查:FunASR 2pass WebSocket,16k PCM,online 展示、offline 终稿,端点发送 is_speaking=false;没有通过端口猜协议。既有 LLM halogen-qwen3.8-flash-next(.11:8731)、CosyVoice xiaoyun(.12:9880)保持原服务。 + +每轮独立 peer。前端立即断开播放音频图、暂停/静音并丢弃视频、递增轮次;后端取消请求、关闭上游响应并精确关闭 driver peer/session,不复用旧媒体队列。旧 ICE/offer 迟到和超时异常按轮次拒绝;取消连接时也立即唤醒等待,避免旧超时误停新轮。LLM/TTS 生成与 ICE/进场并行,ready 门控防止画面到位前发声。TTS 整句缓冲减少分句空隙,长句首声较慢;共享 TTS 内部 GPU 生成是否随客户端断开即时停止未获证明,未改共享服务。 + +- 原78项 Python测试通过;TypeScript严格检查+Vite构建通过;新增Python代码ruff通过。 +- 5项网关检查通过:取消LLM、取消TTS、等待画面ready时取消、错误音色不回退、模型不可达。 +- Chromium真实WebRTC回归通过:文字回复、说话轮取消、准备期取消、再次说话、回阅读续播、暂停、无匹配场景回退、offer失败、断网/恢复重连、缺片提示。未停止共享模型模拟故障。 +- ICE/offer建立中快速取消再发送的真实浏览器回归通过,旧超时没有停止新轮。 +- 合成WAV经Chromium假麦克风→Worklet/VAD→真实ASR终稿→LLM/TTS/口型通过;旧回复已出声,检测到开口后停止,新终稿“你好,请听我说一个问题。”正常产生新回复。附有声连续录像。**输入不是人现场说话,不计真人自然抢话通过**;长停顿可拆轮、轻声阈值和扬声器回声仍待耳机/真麦克风校准。 +- 1280×800 和390×844视口可操作,无横向溢出、无pageerror。390px只是桌面Chromium视口,手机真机/iOS/Safari未测。 +- 页面隐藏处理器停止音视频/麦克风并要求主动恢复;测试用合成visibilitychange,真实系统切后台/锁屏未测。 + +## 设备、延迟和长时观察 + +推理 .12:RTX5090D 32GB,Linux7.0.0,复用 torch2.8.0+cu128、LiveTalking c726943、MuseTalk v1.5、aiortc1.15.0、PyAV17.1.0、aiohttp3.14.5。浏览器在 .13 用已安装 Playwright/Chromium,headless;没有真实耳机/麦克风或手机接入。 + +测量口径:同一轮用户提交完成(或ASR终稿提交)→播放图RMS首次>0.015,包含该轮建连、等待可切点、进场和模型处理。不是模型首包、不是物理耳朵校准,也不拿不同样本相加。LLM/queued字段只是同轮分段记录。视频“首次可见”只有媒体ready标记,逐音节首个可见口型仍需同步复核,不与声音口径混用。 + +最终连续浏览器音画的两轮:首次从日常进入7.43s,已在近景的下一轮5.74s;本轮记录期间有长测客户端并行,不是单用户SLA。此前batch16连续618.41秒、15次真实模型回复、15次首声全部返回、无pageerror/错误状态;P50 5.62s,范围4.93~7.57s,10秒采样总显存最大20248MiB,末尾18277MiB。前段曾并行功能/麦克风自测,不能说这是纯单会话显存峰值。 + +batch8最终长测结果与清理记录在交付前补齐。总显存采样包含既有TTS等进程,10秒采样可能漏过瞬时峰值;不声称已测得绝对峰值或无泄漏的长期保证。 + +## 隔离与交接 + +代码/模型不合并、不部署、不改开机启动。现有18080/18081、8090、9880、10095等共享服务保持。临时服务使用8010/8014,仅本轮自测;启动前确认空闲,按精确PID记录和停止,不使用pkill/fuser清理。权重和用户素材保留。 + +最终PR、分支、提交与本轮进程清理证据随卡片评论和复核包交接。当前同步/自然观感与真人、真机缺口均显式列出,不能据构建和基础测试通过自审上线。 diff --git a/samples/companion/assets/active.json b/samples/companion/assets/active.json new file mode 100644 index 0000000..2bbe2d6 --- /dev/null +++ b/samples/companion/assets/active.json @@ -0,0 +1,3 @@ +{ + "persona_id": "wanwan-v2-cream-night" +} diff --git a/samples/companion/assets/personas/wanwan-v2-cream-night/manifest.json b/samples/companion/assets/personas/wanwan-v2-cream-night/manifest.json new file mode 100644 index 0000000..66b9c42 --- /dev/null +++ b/samples/companion/assets/personas/wanwan-v2-cream-night/manifest.json @@ -0,0 +1,60 @@ +{ + "companion_id": "wanwan", + "persona_id": "wanwan-v2-cream-night", + "version": "hel714-v2", + "previous_version": "hel714-v1", + "identity_ref": "ref/identity.png", + "created_by": "HEL-714 / P10 reused assets", + "created_at": "2026-10-20", + "pipeline_version": "hel714-wan-first-last-v1", + "outfit": "cream", + "light": "warm-bedroom-night", + "weather_source": "manual-test", + "default_daily": "reading-rain-night", + "daily_clips": [ + { + "id": "reading-rain-night", + "file": "daily/reading-rain-night.mp4", + "timeofday": "night", + "weather": "rain", + "activity": "reading", + "loop_points": [ + 0, + 195 + ], + "loop_review": "pending-human-review" + } + ], + "talk_clips": [ + { + "id": "gaze", + "file": "talk/gaze.mp4", + "avatar_id": "hel714-gaze-v1", + "framing": "approved v2 near reference, center square crop", + "approved": false, + "gaze_review": "new 81-frame subtle-motion candidate; pending human review", + "source_sha256": "d8028b938096ed02d03f727c3723cf423d8dd2ab794d84b532c778a2cde118d5", + "driver_revision": "c726943" + } + ], + "transition": { + "enter": "transitions/enter.mp4", + "return": "transitions/return.mp4", + "cut_frame": 96, + "available": true, + "approved": false, + "cut_points": [ + 96 + ], + "reason": "forward-generated first/last conditioned motion, not reversed or crossfaded" + }, + "manual_consistency_review": "candidate contact sheet checked; final human motion review pending", + "sha256": { + "ref/identity.png": "e40b72c44611cc6f0a9651ad24cc4b0985e2bb8efa19cba963d515a7a6cf4c8f", + "daily/reading-rain-night.mp4": "776801d38db4a4c3c0afa7488b380987fd9541c699dcfa6e8b26aa492d8255cf", + "talk/gaze.mp4": "d8028b938096ed02d03f727c3723cf423d8dd2ab794d84b532c778a2cde118d5", + "transitions/enter.mp4": "d014fc07f0f1c4ba6686f51380cc99f472119c70527a65b8f37c7992dded8828", + "transitions/return.mp4": "ca5cdfd8a3dd1267a5e95b587fb1d5fc921421993742707e9fe931bedeee9c7c", + "voice/voice.json": "dc235f2eccba209fdb63873b9f74ee99a899fe9e8883e497704d6ebd75b3ee2f" + } +} diff --git a/samples/companion/assets/personas/wanwan-v2-cream-night/voice/voice.json b/samples/companion/assets/personas/wanwan-v2-cream-night/voice/voice.json new file mode 100644 index 0000000..80588b4 --- /dev/null +++ b/samples/companion/assets/personas/wanwan-v2-cream-night/voice/voice.json @@ -0,0 +1,4 @@ +{ + "registered_voice": "xiaoyun", + "silent_fallback": false +} diff --git a/samples/companion/index.html b/samples/companion/index.html new file mode 100644 index 0000000..0388721 --- /dev/null +++ b/samples/companion/index.html @@ -0,0 +1,26 @@ + +晚晚 · 交谈样机 +
+

晚晚

本地交谈样机 · 人物 v2
+
+ + + + +

日常成片 · 阅读

+
+
+

正在检查素材…

+ +
+
+
+

+
场景测试
+ + + +

真实天气未接入。素材替换不修改身份、八字、记忆或关系;本页未接入这些模块。
+ +
+
diff --git a/samples/companion/package-lock.json b/samples/companion/package-lock.json new file mode 100644 index 0000000..90d8f3f --- /dev/null +++ b/samples/companion/package-lock.json @@ -0,0 +1,1166 @@ +{ + "name": "cybergril-companion-sample", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cybergril-companion-sample", + "version": "0.1.0", + "dependencies": { + "pixi.js": "^8.0.0" + }, + "devDependencies": { + "typescript": "~5.9.3", + "vite": "^5.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", + "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", + "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", + "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", + "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", + "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", + "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", + "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", + "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", + "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", + "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", + "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", + "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", + "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", + "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", + "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", + "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", + "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", + "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", + "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", + "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", + "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", + "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", + "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=12" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@pixi/colord": { + "version": "2.9.6", + "resolved": "https://registry.npmjs.org/@pixi/colord/-/colord-2.9.6.tgz", + "integrity": "sha512-nezytU2pw587fQstUu1AsJZDVEynjskwOL+kibwcdxsMBFqPsFFNA7xl0ii/gXuDi6M0xj3mfRJj8pBSc2jCfA==", + "license": "MIT" + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.64.10.tgz", + "integrity": "sha512-oJ9ZfVIZjSjts0ZIYA48+5T0vurE5fQfX/Liad2U52Y6EN99x/OCdjGGuKMDP6jSk2i2lY6jwwA3X0drEc4PBg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.64.10.tgz", + "integrity": "sha512-LylBINgYYwgHNz8OMCy5+Y0w/LEgRYE0PcAIZfEuTuLNMf0S40GSLX6SFwqZsBLplS7RIHHy4zOAbl8Y5cQEvg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.64.10.tgz", + "integrity": "sha512-vbuT3VT1GBI2fAOoUJUrsHVoOg9nbrFXTM7e3DSnb9PYf4n6jZCN8v4Vo8O6x4u5B67+/c803b8BtCZe2nIeqA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.64.10.tgz", + "integrity": "sha512-qYUp/qYvM3T4cF+VWalPl+przBUESILxi+qZ1m1MVRHpg06vS+jHaxXubjCaRpXjLJPDFdXzOZ0FABDghHOhNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.64.10.tgz", + "integrity": "sha512-v9NRQG+JztbP4DP/0yuhSyDcEJk047lYbqeNnqIgbXITcVDjGDS7OpkW5nadSY3tJ7MY8W1qIMMqP4Q3Q8BDjQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.64.10.tgz", + "integrity": "sha512-CHc9LBHdxhn6/NXfb9IegXCWoz5VlvcDOx9QrzZ9q+XYhFlfzwymSWJjOlZcEMMtC1zgMV7qxDe6U18AI1SG0A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.64.10.tgz", + "integrity": "sha512-ble2WR7kWfzZFJUd3ODMGCJn1XDszptfRC0QvWc0h9/EA6MxnrQ1mvbjPZifEdgHyfHhDrwWeKeJONDITcrHSg==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.64.10.tgz", + "integrity": "sha512-DMCYegihxBMKvLpODdk4OsvIgxtHTi+Oddf1Pm3d6wMVNjcBiG2pqyXjMPy55HgG1gCdEFr9VfhB1CmvuKn1lw==", + "cpu": [ + "arm" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.64.10.tgz", + "integrity": "sha512-HQ6HuVH9TmhvPWIas7MA/KeAROgdwqH3REpUU6hZq47NW7m1b1QCrHgM+M3+N/B8DfMt88x1/z79ppRc9RflCQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.64.10.tgz", + "integrity": "sha512-avBhxba1DdctX0kxKkN3JWdTSg72QsxSn6FdfhZ9MkL3TTYDOXqxY1GqbC7VNttlgviK7tCe2No5btfRkFgeIg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.64.10.tgz", + "integrity": "sha512-UtQMex7fiXq0W/GqcWRpy6XKqUbXgQeGKB+Bd5rJngmHzpI+3kjwpWFLAhKg02onFsIMSh+Zx0zp9JIMQ7/05A==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.64.10.tgz", + "integrity": "sha512-0Aum2pUsCSuOopi6oka+nR1DTj/nfvc1xjL979lvh6kXUjNa30TCX7sKFjyMsrX7bK3bNVs0FsJwQcE2HCWX1Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.64.10.tgz", + "integrity": "sha512-4dgsouKal3qfMFORgtNigLPt7K3HXiO6jTr7+nFJv1GWu1OkZx0kvQIXbK4sjrIGnaLBk0s80eQCQ9hPIelLrw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.64.10.tgz", + "integrity": "sha512-d7NE89try3jisYzBD7J9M5iwfgcrXYC4dwHEjQ3HIWBftvwch1U/cy3iZ26SthtUOyClKiu7iqYiFjQEwiOlqQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.64.10.tgz", + "integrity": "sha512-dqB5ChSBp91JWXxpCKAS5ZVAE38esKQvdfyqANm0lF5JPiQrXnYwgG3oVBQmuNRcQ1WHkA3j5qYLsrl2CtfhCA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.64.10.tgz", + "integrity": "sha512-7GMf3vbQQslorWLmfsbi5fdt9RVnglk205XpOMFvW6UY5jE5oJc6JLieQJPHtsYlZ2EO0PtC06nY9Hz/IG8guw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.64.10.tgz", + "integrity": "sha512-tuKbkB23DKc5LG+irPSUnZZztRHqC3CNmyxeWloueBteHB+i+1HkRCMzAE41lseLDlarybeLhqfibFitKOxKMg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.64.10.tgz", + "integrity": "sha512-ntHtQOMw7oIshHRF1swV3vyBw+beQt9PeZ04O3D32EeDZUMoHMJuZKp4/fDllH/+xogeK0GUCONVfjjKusqKLg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.64.10.tgz", + "integrity": "sha512-Dlh8q8r145aYgh+N3gNIG1dz35Fz/FCxe5hs/43orDsffjP8S1P4Xz5I4FpSJsDS4LpQw1QmswZkQlUbkCldjQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.64.10.tgz", + "integrity": "sha512-VBJhHWz/55u45CWFv9OmfSRDgJDVENESgFxSAn13gIKJpTuFJbP4/7ymMZhDfeKFx1zGn1XBsLnNB7Ryy71qww==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.64.10.tgz", + "integrity": "sha512-Vjcazg+EEzUfIz+LcaxS6YtbQHo+sv0sCD+5VbwhY504bCKNW4OOreVgKq61pHT2esRQ/+AXYVPStDwxSvAOdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.64.10.tgz", + "integrity": "sha512-jJAaP4El+HWxYkM0jeJghASaFgfnluY0FRHhyHXxU6STs/nBhKMIxIMDrJEFg2ftKz6orX9WAo4diVan5NugNA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.64.10.tgz", + "integrity": "sha512-bXXdRUfRx9PaHCToKt8R/7wjDV6Tbk/hmSHK0uQBKmD3rj4jkuYndct8uGX77ZuZsePrVkQbWUgPR2jJhCTMpQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.64.10.tgz", + "integrity": "sha512-57Idx+VkEOQeG6Ys6sZOBFT7FfbeyxGv9A3k1m+Lf/z2SwqT40rNWbAINzjPHSMoCCBvY7Fnb9RGZWQin/HHAw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.64.10.tgz", + "integrity": "sha512-vrvuasIpm/xf+BpZbzgirku0WnrQZMAc+H2Ai26jzqBxYEmQQE4DdJb7h8erkJMfRRipx0ma2Cty8jJwUm9ocQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/earcut": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/earcut/-/earcut-3.0.0.tgz", + "integrity": "sha512-k/9fOUGO39yd2sCjrbAJvGDEQvRwRnQIZlBz43roGwUZo5SHAmyVvSFyaVVZkicRVCaDXPKlbxrUcBuJoSWunQ==", + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@webgpu/types": { + "version": "0.1.74", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.74.tgz", + "integrity": "sha512-lgiI4hbuLcI9unnm2cL/tvCaQU45dp0xcLWJh5uB/9MBGvIA4F8XIk7nSiBeg+K4xWGYwgZKn80LmERI5CxoTA==", + "license": "BSD-3-Clause" + }, + "node_modules/@xmldom/xmldom": { + "version": "0.8.15", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.15.tgz", + "integrity": "sha512-/5NV/vDALVFDXgLmfsy9TRCBlKwO2LNBFzpzvb9iIj+jR+eSc6DLYYvVOdivT/jm7MtU6TebYuRmzEOI7w40UA==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/earcut": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/earcut/-/earcut-3.2.4.tgz", + "integrity": "sha512-FHc6yNpnTORB0tM5kmZ03iD2mDuNakDE2lvf4Hw1XkfxtEHIGVi01B/GqUSwx+NcC1WGjidAfpTXzeALtPtfpg==", + "license": "ISC" + }, + "node_modules/esbuild": { + "version": "0.21.5", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", + "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.21.5", + "@esbuild/android-arm": "0.21.5", + "@esbuild/android-arm64": "0.21.5", + "@esbuild/android-x64": "0.21.5", + "@esbuild/darwin-arm64": "0.21.5", + "@esbuild/darwin-x64": "0.21.5", + "@esbuild/freebsd-arm64": "0.21.5", + "@esbuild/freebsd-x64": "0.21.5", + "@esbuild/linux-arm": "0.21.5", + "@esbuild/linux-arm64": "0.21.5", + "@esbuild/linux-ia32": "0.21.5", + "@esbuild/linux-loong64": "0.21.5", + "@esbuild/linux-mips64el": "0.21.5", + "@esbuild/linux-ppc64": "0.21.5", + "@esbuild/linux-riscv64": "0.21.5", + "@esbuild/linux-s390x": "0.21.5", + "@esbuild/linux-x64": "0.21.5", + "@esbuild/netbsd-x64": "0.21.5", + "@esbuild/openbsd-x64": "0.21.5", + "@esbuild/sunos-x64": "0.21.5", + "@esbuild/win32-arm64": "0.21.5", + "@esbuild/win32-ia32": "0.21.5", + "@esbuild/win32-x64": "0.21.5" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gifuct-js": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/gifuct-js/-/gifuct-js-2.1.2.tgz", + "integrity": "sha512-rI2asw77u0mGgwhV3qA+OEgYqaDn5UNqgs+Bx0FGwSpuqfYn+Ir6RQY5ENNQ8SbIiG/m5gVa7CD5RriO4f4Lsg==", + "license": "MIT", + "dependencies": { + "js-binary-schema-parser": "^2.0.3" + } + }, + "node_modules/ismobilejs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ismobilejs/-/ismobilejs-1.1.1.tgz", + "integrity": "sha512-VaFW53yt8QO61k2WJui0dHf4SlL8lxBofUuUmwBo0ljPk0Drz2TiuDW4jo3wDcv41qy/SxrJ+VAzJ/qYqsmzRw==", + "license": "MIT" + }, + "node_modules/js-binary-schema-parser": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/js-binary-schema-parser/-/js-binary-schema-parser-2.0.3.tgz", + "integrity": "sha512-xezGJmOb4lk/M1ZZLTR/jaBHQ4gG/lqQnJqdIv4721DMggsa1bDVlHXNeHYogaIEHD9vCRv0fcL4hMA+Coarkg==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/parse-svg-path": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/parse-svg-path/-/parse-svg-path-0.2.0.tgz", + "integrity": "sha512-Tf7FFIrguPKQwzD4pWnYkR2VOv3raoHeKED80Bm+BYHI3KxC8KsgsGC5+fSMzAGDA6UEk4bHvmi+RsjmL3khpg==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/pixi.js": { + "version": "8.22.0", + "resolved": "https://registry.npmjs.org/pixi.js/-/pixi.js-8.22.0.tgz", + "integrity": "sha512-QbTANHMZ751MIjLiLSUSM8a7LNMWwMIf0KbRCqlW+SPS9hNLfSVS+cl1O2SEp9g1K/wyLzj/OJHv+1TUfiPr/w==", + "license": "MIT", + "workspaces": [ + "examples", + "playground" + ], + "dependencies": { + "@pixi/colord": "^2.9.6", + "@types/earcut": "^3.0.0", + "@webgpu/types": "^0.1.69", + "@xmldom/xmldom": "^0.8.15", + "earcut": "^3.0.2", + "eventemitter3": "^5.0.1", + "gifuct-js": "^2.1.2", + "ismobilejs": "^1.1.1", + "parse-svg-path": "^0.2.0", + "tiny-lru": "^11.4.7" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/pixijs" + } + }, + "node_modules/postcss": { + "version": "8.5.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.31.tgz", + "integrity": "sha512-YUbWYCgErwnMAVkjUnYa+KlMBzGpHwiIt/DE7f0G33AuX2LH9UcTk6KgWxd3zRU+pN3u/C+Xl3EnMLM4Hea7Sg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rollup": { + "version": "4.64.10", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.64.10.tgz", + "integrity": "sha512-8N52YH1B5JzBP+B1rkcBERG7rfRTGAM0RmUpyd76yyRfTljuG452GCNbVm6MPzs1+gXBha6Rw5/oHRFohOpfDg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.64.10", + "@rollup/rollup-android-arm64": "4.64.10", + "@rollup/rollup-darwin-arm64": "4.64.10", + "@rollup/rollup-darwin-x64": "4.64.10", + "@rollup/rollup-freebsd-arm64": "4.64.10", + "@rollup/rollup-freebsd-x64": "4.64.10", + "@rollup/rollup-linux-arm-gnueabihf": "4.64.10", + "@rollup/rollup-linux-arm-musleabihf": "4.64.10", + "@rollup/rollup-linux-arm64-gnu": "4.64.10", + "@rollup/rollup-linux-arm64-musl": "4.64.10", + "@rollup/rollup-linux-loong64-gnu": "4.64.10", + "@rollup/rollup-linux-loong64-musl": "4.64.10", + "@rollup/rollup-linux-ppc64-gnu": "4.64.10", + "@rollup/rollup-linux-ppc64-musl": "4.64.10", + "@rollup/rollup-linux-riscv64-gnu": "4.64.10", + "@rollup/rollup-linux-riscv64-musl": "4.64.10", + "@rollup/rollup-linux-s390x-gnu": "4.64.10", + "@rollup/rollup-linux-x64-gnu": "4.64.10", + "@rollup/rollup-linux-x64-musl": "4.64.10", + "@rollup/rollup-openbsd-x64": "4.64.10", + "@rollup/rollup-openharmony-arm64": "4.64.10", + "@rollup/rollup-win32-arm64-msvc": "4.64.10", + "@rollup/rollup-win32-ia32-msvc": "4.64.10", + "@rollup/rollup-win32-x64-gnu": "4.64.10", + "@rollup/rollup-win32-x64-msvc": "4.64.10", + "fsevents": "~2.3.2" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tiny-lru": { + "version": "11.4.7", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.4.7.tgz", + "integrity": "sha512-w/Te7uMUVeH0CR8vZIjr+XiN41V+30lkDdK+NRIDCUYKKuL9VcmaUEmaPISuwGhLlrTGh5yu18lENtR9axSxYw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/vite": { + "version": "5.4.21", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", + "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.21.3", + "postcss": "^8.4.43", + "rollup": "^4.20.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || >=20.0.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.4.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + } + } + } + } +} diff --git a/samples/companion/package.json b/samples/companion/package.json new file mode 100644 index 0000000..0cbac42 --- /dev/null +++ b/samples/companion/package.json @@ -0,0 +1,19 @@ +{ + "name": "cybergril-companion-sample", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite --host 127.0.0.1", + "typecheck": "tsc --noEmit", + "build": "python3 tools/stage_assets.py && tsc --noEmit && vite build", + "preview": "vite preview --host 127.0.0.1" + }, + "dependencies": { + "pixi.js": "^8.0.0" + }, + "devDependencies": { + "typescript": "~5.9.3", + "vite": "^5.4.0" + } +} diff --git a/samples/companion/public/pcm-worklet.js b/samples/companion/public/pcm-worklet.js new file mode 100644 index 0000000..5f5b389 --- /dev/null +++ b/samples/companion/public/pcm-worklet.js @@ -0,0 +1,23 @@ +/* Native AudioWorklet: 100ms PCM packets. AudioContext requests 16kHz. */ +class PCM extends AudioWorkletProcessor { + constructor() { super(); this.buffer = []; } + process(inputs) { + const channel = inputs[0]?.[0]; + if (!channel) return true; + this.buffer.push(...channel); + const count = Math.round(sampleRate / 10); + while (this.buffer.length >= count) { + const input = this.buffer.splice(0, count); + const output = new Int16Array(1600); + let sum = 0; + for (let i = 0; i < 1600; i++) { + const value = input[Math.min(count - 1, Math.floor(i * count / 1600))]; + output[i] = Math.round(Math.max(-1, Math.min(1, value)) * 32767); + sum += value * value; + } + this.port.postMessage({pcm:output.buffer, rms:Math.sqrt(sum/1600), ms:100}, [output.buffer]); + } + return true; + } +} +registerProcessor('pcm', PCM); diff --git a/samples/companion/runtime/config.json b/samples/companion/runtime/config.json new file mode 100644 index 0000000..985bb92 --- /dev/null +++ b/samples/companion/runtime/config.json @@ -0,0 +1,8 @@ +{ + "driver": "http://127.0.0.1:8010", + "llm": "http://192.168.200.11:8731/v1/chat/completions", + "model": "halogen-qwen3.8-flash-next", + "tts": "http://127.0.0.1:9880", + "voice": "xiaoyun", + "asr": "ws://127.0.0.1:10095" +} diff --git a/samples/companion/runtime/server.py b/samples/companion/runtime/server.py new file mode 100644 index 0000000..0b17128 --- /dev/null +++ b/samples/companion/runtime/server.py @@ -0,0 +1,253 @@ +"""Isolated sample gateway. Reuses existing aiohttp; never mutates companion state. + +Each speaking turn owns a WebRTC peer. Cancellation retires that peer, cancels +upstream HTTP work, and closes the driver session. A late packet cannot be played +by the next peer. TTS is buffered as one utterance to avoid punctuation gaps. +""" + +import argparse +import asyncio +import contextlib +import io +import json +import secrets +import ssl +import time +import wave +from pathlib import Path + +import aiohttp +from aiohttp import web + +ROOT = Path(__file__).resolve().parents[1] + + +class Gateway: + def __init__(self, config): + self.config = config + self.sessions = {} + self.http = None + + async def start(self, app): + self.http = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=90)) + + async def stop(self, app): + for token in list(self.sessions): + await self.retire(token) + await self.http.close() + + async def driver(self, path, data): + async with self.http.post(self.config["driver"] + path, json=data) as response: + response.raise_for_status() + result = await response.json() + if result.get("code", 0) != 0: + raise RuntimeError(result.get("msg", "driver failed")) + return result + + async def retire(self, token): + session = self.sessions.pop(token, None) + if not session: + return + task = session.get("task") + if task and not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + # Flush and destroy the peer, including inference/output queues. The + # isolated driver extension closes only this peer, never other services. + await self.driver("/hel714/close", {"sessionid": session["sid"]}) + + async def offer(self, request): + params = await request.json() + manifest = json.loads((ROOT / "dist/assets/manifest.json").read_text()) + clips = manifest.get("talk_clips", []) + if not clips: + raise RuntimeError("当前外观包没有交谈底板,未改用其他人物") + answer = await self.driver( + "/offer", {"sdp": params["sdp"], "type": "offer", "avatar": clips[0]["avatar_id"]} + ) + token = secrets.token_urlsafe(24) + self.sessions[token] = {"sid": answer["sessionid"], "task": None, "ready": asyncio.Event()} + return web.json_response({**answer, "token": token}) + + async def ready(self, request): + session = self.sessions.get((await request.json())["token"]) + if not session: + raise web.HTTPConflict(text="会话已取消") + session["ready"].set() + return web.json_response({"ready": True}) + + async def cancel(self, request): + await self.retire((await request.json())["token"]) + return web.json_response({"cancelled": True}) + + async def reply(self, session, text): + started = time.monotonic() + cfg = self.config + async with self.http.get(cfg["tts"] + "/health") as response: + response.raise_for_status() + health = await response.json() + if not health.get("ok") or cfg["voice"] not in health.get("voices", []): + raise RuntimeError("指定音色不可用;已停止,未回退其他音色") + async with self.http.post( + cfg["llm"], + json={ + "model": cfg["model"], + "messages": [ + { + "role": "system", + "content": "你是晚晚。当前画面是你在卧室阅读,书名未知。用一两句简短自然的中文回应。不编造书名,不声称有未接入的记忆、天气或感知。", + }, + {"role": "user", "content": text}, + ], + "max_tokens": 100, + }, + ) as response: + response.raise_for_status() + body = await response.json() + reply = body["choices"][0]["message"]["content"] + llm_ms = round((time.monotonic() - started) * 1000) + pcm = bytearray() + async with self.http.post( + cfg["tts"] + "/tts", + json={ + "voice": cfg["voice"], + "text": reply, + }, + ) as response: + response.raise_for_status() + async for chunk in response.content.iter_chunked(4800): + pcm.extend(chunk) + if len(pcm) > 24000 * 2 * 45: + raise RuntimeError("语音超过样机上限,已停止") + if not pcm: + raise RuntimeError("语音服务未返回音频") + # Generate while the camera transition/ICE handshake runs, but never + # enqueue speech until the browser confirms the media view is ready. + if session.get("ready"): + await asyncio.wait_for(session["ready"].wait(), timeout=20) + wav = io.BytesIO() + with wave.open(wav, "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(24000) + output.writeframes(pcm) + form = aiohttp.FormData() + form.add_field("sessionid", session["sid"]) + form.add_field("file", wav.getvalue(), filename="utterance.wav", content_type="audio/wav") + async with self.http.post(cfg["driver"] + "/humanaudio", data=form) as response: + response.raise_for_status() + result = await response.json() + if result.get("code") != 0: + raise RuntimeError(result.get("msg", "audio enqueue failed")) + return { + "text": reply, + "llm_ms": llm_ms, + "queued_ms": round((time.monotonic() - started) * 1000), + "audio_seconds": len(pcm) / 48000, + } + + async def turn(self, request): + params = await request.json() + session = self.sessions.get(params["token"]) + if not session: + raise web.HTTPConflict(text="会话已取消,请重新进入") + if session["task"]: + raise web.HTTPConflict(text="本轮已提交,请先取消") + text = str(params.get("text", "")).strip() + if not text or len(text) > 500: + raise web.HTTPBadRequest(text="请输入1至500字") + task = session["task"] = asyncio.create_task(self.reply(session, text)) + try: + result = await task + except asyncio.CancelledError: + raise web.HTTPConflict(text="本轮已取消") from None + return web.json_response(result) + + async def asr(self, request): + browser = web.WebSocketResponse(max_msg_size=1024 * 1024) + await browser.prepare(request) + try: + async with self.http.ws_connect(self.config["asr"]) as upstream: + await upstream.send_json( + { + "mode": "2pass", + "wav_name": "hel714", + "is_speaking": True, + "wav_format": "pcm", + "chunk_size": [5, 10, 5], + "audio_fs": 16000, + "itn": True, + } + ) + + async def receive(): + async for message in upstream: + if message.type == aiohttp.WSMsgType.TEXT: + data = json.loads(message.data) + await browser.send_json(data) + await browser.close() + + reader = asyncio.create_task(receive()) + try: + async for message in browser: + if message.type == aiohttp.WSMsgType.BINARY: + await upstream.send_bytes(message.data) + elif message.type == aiohttp.WSMsgType.TEXT: + # Only endpoint signals are accepted from the client. + await upstream.send_json({"is_speaking": False}) + finally: + reader.cancel() + with contextlib.suppress(asyncio.CancelledError): + await reader + except (aiohttp.ClientError, TimeoutError) as error: + if not browser.closed: + await browser.send_json({"error": f"ASR不可达: {type(error).__name__}"}) + finally: + await browser.close() + return browser + + +@web.middleware +async def guard(request, handler): + origin = request.headers.get("Origin") + if origin and origin != f"{request.scheme}://{request.host}": + raise web.HTTPForbidden(text="仅接受同源请求") + try: + return await handler(request) + except (aiohttp.ClientError, TimeoutError, RuntimeError, KeyError) as error: + return web.json_response({"error": str(error)}, status=502) + + +def make_app(config): + gateway = Gateway(config) + app = web.Application(middlewares=[guard], client_max_size=1024 * 1024) + app.on_startup.append(gateway.start) + app.on_cleanup.append(gateway.stop) + app.router.add_post("/api/offer", gateway.offer) + app.router.add_post("/api/cancel", gateway.cancel) + app.router.add_post("/api/turn", gateway.turn) + app.router.add_post("/api/ready", gateway.ready) + app.router.add_get("/api/asr", gateway.asr) + app.router.add_get("/", lambda _: web.FileResponse(ROOT / "dist/index.html")) + app.router.add_static("/", ROOT / "dist", show_index=False) + return app + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8014) + parser.add_argument("--cert") + parser.add_argument("--key") + args = parser.parse_args() + tls = None + if args.cert: + tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls.load_cert_chain(args.cert, args.key) + web.run_app( + make_app(json.loads((ROOT / "runtime/config.json").read_text())), + host=args.host, + port=args.port, + ssl_context=tls, + ) diff --git a/samples/companion/src/controls.css b/samples/companion/src/controls.css new file mode 100644 index 0000000..df76d14 --- /dev/null +++ b/samples/companion/src/controls.css @@ -0,0 +1,70 @@ +/* Existing rain-room base controls, extracted without overrides. */ +* { + box-sizing:border-box +} +[hidden] { + display:none!important +} +html,body { + margin:0; + min-height:100%; + background:var(--scene-night-deep); + color:var(--ink-primary); + font-family:var(--font-sans); + font-size:var(--fs-body); + letter-spacing:.02em +} +body { + height:100dvh; + overflow:hidden +} +button,input,textarea,select { + font:inherit; + color:inherit +} +button { + cursor:pointer; + border:1px solid transparent; + background:transparent; + border-radius:var(--r-control); + padding:var(--space-2) var(--space-3); + transition:transform 160ms var(--ease-out),opacity 160ms var(--ease-out) +} +button:active { + transform:scale(.94); + opacity:.7 +} +button:disabled { + color:var(--ink-disabled); + cursor:default +} +button[aria-disabled=true] { + color:var(--ink-disabled) +} +button:focus-visible,input:focus-visible,select:focus-visible,textarea:focus-visible { + outline:2px solid var(--accent); + outline-offset:3px +} +p,h1,h2 { + margin:0 +} +small { + font-size:var(--fs-caption); + color:var(--ink-secondary) +} +h1 { + font-size:var(--fs-title); + font-weight:500 +} +h2 { + font-size:var(--fs-title); + font-weight:500 +} +.glass { + background:var(--glass-bg); + border:1px solid var(--glass-border); + border-radius:var(--r-panel); + backdrop-filter:var(--glass-blur); + -webkit-backdrop-filter:var(--glass-blur); + box-shadow:inset 0 1px 0 var(--glass-highlight),var(--shadow-panel) +} diff --git a/samples/companion/src/main.ts b/samples/companion/src/main.ts new file mode 100644 index 0000000..665103b --- /dev/null +++ b/samples/companion/src/main.ts @@ -0,0 +1,323 @@ +import './style.css'; + +type Clip = { id:string; file:string; activity:string; weather:string; timeofday:string }; +type Manifest = { companion_id:string; version:string; default_daily:string; daily_clips:Clip[]; talk_clips:{file:string;avatar_id:string; approved:boolean}[]; transition?:{enter:string;return:string;cut_frame:number;available:boolean} }; +const el = (id:string) => document.getElementById(id) as T; +const daily=el('daily'), talk=el('talk'); +const listening=el('listening'), bridge=el('bridge'); +let view:'daily'|'talk'|'listening'|'bridge'='daily'; +const status=el('status'); +let manifest:Manifest, clipId='', inTalk=false, paused=false, generation=0; +let dailyMissing=false; +let pc:RTCPeerConnection|null=null, token='', abort:AbortController|null=null; +let serial:Promise=Promise.resolve(); +let mic:MediaStream|null=null, micContext:AudioContext|null=null, asr:WebSocket|null=null; +let utterance=false, loud=0, quiet=0, submitted=false, micEpoch=0, finalTimer=0; +let asrTimeout=0, endpointSent=false; +let partials:string[]=[]; +let mediaSource:MediaStreamAudioSourceNode|null=null, audioContext:AudioContext|null=null; +let destination:MediaStreamAudioDestinationNode|null=null, analyser:AnalyserNode|null=null; +let recording:MediaRecorder|null=null, recordingRaf=0, downloadUrl=''; +const events:{at:number; event:string; detail:unknown}[]=[]; +function mark(event:string, detail:unknown={}) { events.push({at:performance.now(),event,detail}); } +Object.assign(window,{sampleEvents:events}); +function message(text:string) { status.textContent=text; mark('status',text); } +function setView(next:typeof view) { + view=next; + for(const [name,video] of [['daily',daily],['talk',talk],['listening',listening],['bridge',bridge]] as const) { + video.hidden=name!==next; + if(name!==next)video.pause(); + } + if(next==='listening')void listening.play().catch(()=>message('倾听底板不可用,请回到日常或重试。')); +} +async function playBridge(file:string, epoch:number) { + bridge.src=`/assets/${file}`;setView('bridge');await bridge.play(); + const deadline=performance.now()+15000; + while(!bridge.ended&&epoch===generation) { + if(bridge.error||performance.now()>deadline)throw new Error('转场素材无法播放'); + await new Promise(r=>setTimeout(r,40)); + } +} +async function enterBridge(epoch:number) { + const transition=manifest.transition; + if(view!=='daily'||!transition?.available)return; + const cut=transition.cut_frame/25; + message('等待阅读可切点,再抬头交谈…'); + if(paused){paused=false;el('pause').textContent='暂停日常';} + await daily.play(); + const deadline=performance.now()+12000; + let previous=daily.currentTime; + while(epoch===generation) { + const now=daily.currentTime; + if(previous<=cut&&now>=cut)break; + if(performance.now()>deadline)throw new Error('日常视频未到达可切点'); + previous=now;await new Promise(r=>setTimeout(r,16)); + } + if(epoch!==generation)return; + daily.pause();daily.currentTime=cut;mark('enter-cut',{frame:transition.cut_frame}); + await playBridge(transition.enter,epoch); +} + +async function api(path:string, data:unknown, signal?:AbortSignal) { + const response=await fetch(`/api/${path}`,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(data),signal}); + if(!response.ok) throw new Error((await response.text()).slice(0,200)); + return response.json(); +} +function enqueue(fn:()=>Promise):Promise { + const next=serial.then(fn,fn); serial=next.catch(()=>{}); return next; +} +function stopMic() { + micEpoch++; clearTimeout(finalTimer); clearTimeout(asrTimeout); asr?.close(); asr=null; + mic?.getTracks().forEach(t=>t.stop());mic=null; + void micContext?.close();micContext=null; + utterance=false;partials=[]; loud=quiet=0; + el('mic').setAttribute('aria-pressed','false');el('mic').textContent='开启麦克风'; +} +// Immediate local silence is independent of network/transition progress. +function silence(reason:string, keepMic=false) { + generation++; abort?.abort();abort=null; + mediaSource?.disconnect();mediaSource=null; + talk.muted=true;talk.pause();talk.srcObject=null; + pc?.close();pc=null;setView(inTalk?'listening':'daily'); + const oldToken=token;token=''; + if(!keepMic)stopMic(); + if(oldToken)void enqueue(()=>api('cancel',{token:oldToken})).catch(e=>message(`已本地停止;服务端取消失败:${e.message}`)); + mark('cancel',{reason,generation}); +} +function fail(error:unknown) { + inTalk=false;silence('error'); + message(dailyMissing?'日常素材丢失或无法解码,交谈入口已禁用。请恢复素材包后刷新。':`已停止交谈,保留日常。${error instanceof Error?error.message:String(error)};可点击重连。`); +} +async function audioGraph() { + if(!audioContext) { + audioContext=new AudioContext();destination=audioContext.createMediaStreamDestination(); + analyser=audioContext.createAnalyser(); + // Keep the recording clock producing actual silence before the first peer + // arrives. Otherwise Chromium may omit the opening daily/transition frames + // while waiting for the first audio packet. + const clock=audioContext.createConstantSource();clock.offset.value=0; + clock.connect(destination);clock.connect(audioContext.destination);clock.start(); + } + await audioContext.resume(); +} +async function connect(epoch:number, onOffer?:(token:string)=>void) { + if(pc&&token) return; + const peer=new RTCPeerConnection({iceServers:[]});pc=peer; + const stream=new MediaStream();talk.srcObject=stream;talk.muted=true; + peer.ontrack=event=>{if(epoch===generation)stream.addTrack(event.track)}; + peer.onconnectionstatechange=()=>{ + if(peer!==pc)return; + if(['failed','disconnected'].includes(peer.connectionState))fail(new Error('媒体连接中断')); + }; + peer.addTransceiver('audio',{direction:'recvonly'});peer.addTransceiver('video',{direction:'recvonly'}); + await peer.setLocalDescription(await peer.createOffer()); + await new Promise((resolve,reject)=>{ + const timeout=setTimeout(()=>reject(new Error('媒体协商超时')),8000); + const check=()=>{if(peer.iceGatheringState==='complete'){clearTimeout(timeout);resolve()}}; + peer.addEventListener('icegatheringstatechange',check);check(); + peer.addEventListener('connectionstatechange',()=>{ + if(peer.connectionState==='closed'){clearTimeout(timeout);reject(new Error('本轮媒体已取消'));} + }); + }); + const answer=await api('offer',peer.localDescription); + if(epoch!==generation) { peer.close();await api('cancel',{token:answer.token});return; } + token=answer.token; + onOffer?.(token); + await peer.setRemoteDescription(answer); + await talk.play(); + await new Promise((resolve,reject)=>{ + const timeout=setTimeout(()=>reject(new Error('媒体未到达,HTTP成功不代表媒体已通')),10000); + const check=()=>{if(talk.readyState>=2){clearTimeout(timeout);resolve()}}; + talk.addEventListener('loadeddata',check,{once:true});check(); + }); + if(epoch!==generation)return; + await enterBridge(epoch); + if(epoch!==generation)return; + setView('talk');await talk.play(); + // New peer contains only this turn's media, so unmuting cannot reveal an old queue. + mediaSource=audioContext!.createMediaStreamSource(stream); + mediaSource.connect(audioContext!.destination);mediaSource.connect(destination!);mediaSource.connect(analyser!); + talk.muted=true;el('scene-label').textContent='本地实时口型 · 目光与身体来自底板'; + await api('ready',{token}); + mark('media-ready'); +} +async function enter() { + if(!manifest?.talk_clips.length)throw new Error('交谈素材缺失,入口已禁用'); + inTalk=true;message('正在连接本地交谈…');await audioGraph(); + const epoch=generation; + try { await enqueue(()=>connect(epoch)); } + catch(error) { if(epoch===generation)throw error;return; } + if(epoch===generation)message('正在听。可以输入文字,或开启麦克风。'); +} +async function send(text:string) { + if(!text.trim())return; + const finishedAt=performance.now(); + silence('new-turn',true);inTalk=true; + const epoch=generation; + message('正在连接本轮媒体…');await audioGraph(); + abort=new AbortController(); + const signal=abort.signal; + let pending:Promise<{result:any;error:unknown}>|undefined; + mark('input-finished',{text,finishedAt}); + try { + await enqueue(()=>connect(epoch,currentToken=>{ + pending=api('turn',{token:currentToken,text},signal).then(result=>({result,error:null}),error=>({result:null,error})); + })); + } catch(error) { if(epoch===generation)throw error;return; } + if(epoch!==generation)return; + message('正在准备回复与语音,等待首声…'); + const analysis=analyser!, values=new Float32Array(analysis.fftSize); + let firstAudio=false, monitor=0; + function measure() { + if(epoch!==generation)return; + analysis.getFloatTimeDomainData(values); + const rms=Math.sqrt(values.reduce((s,v)=>s+v*v,0)/values.length); + if(!firstAudio&&rms>0.015){firstAudio=true;mark('first-audible',{latencyMs:performance.now()-finishedAt});message('正在回应…');} + monitor=requestAnimationFrame(measure); + } measure(); + try { + if(!pending)throw new Error('本轮请求未创建'); + const response=await pending; + if(response.error)throw response.error; + const result=response.result; + if(epoch!==generation)return; + el('reply').textContent=result.text;mark('audio-queued',result); + // Duration covers utterance playback plus driver lookahead; cancellation + // still independently retires the peer at any point during this wait. + const deadline=performance.now()+(result.audio_seconds+8)*1000; + while(epoch===generation&&performance.now()setTimeout(r,100)); + if(epoch===generation){silence('completed',true);message('正在听。');} + } catch(error) { if(epoch===generation)throw error; } + finally {cancelAnimationFrame(monitor);} +} +async function selectClip() { + const activity=el('activity').value, weather=el('weather').value, timeofday=el('timeofday').value; + const exact=manifest.daily_clips.find(c=>c.activity===activity&&c.weather===weather&&c.timeofday===timeofday); + const clip=exact??manifest.daily_clips.find(c=>c.id===manifest.default_daily); + if(!clip)throw new Error('默认日常素材缺失'); + el('selection').textContent=exact?'已匹配活动/手动天气/时段。':`缺少所选场景,回退默认阅读片(${clip.weather}/${clip.timeofday})。`; + if(clip.id!==clipId){daily.src=`/assets/${clip.file}`;clipId=clip.id;} + if(!paused&&!inTalk&&!document.hidden)await daily.play(); +} + +async function startMic() { + if(mic){stopMic();return;} + if(!isSecureContext||!navigator.mediaDevices)throw new Error('麦克风需要HTTPS或localhost安全上下文;文字仍可用'); + mic=await navigator.mediaDevices.getUserMedia({audio:{echoCancellation:true,noiseSuppression:true,autoGainControl:true},video:false}); + micContext=new AudioContext({sampleRate:16000}); + await micContext.audioWorklet.addModule('/pcm-worklet.js'); + const source=micContext.createMediaStreamSource(mic), processor=new AudioWorkletNode(micContext,'pcm'); + const gain=micContext.createGain();gain.gain.value=0; + source.connect(processor);processor.connect(gain);gain.connect(micContext.destination); + const epoch=++micEpoch; + const preRoll:ArrayBuffer[]=[]; + const commit=(socket:WebSocket)=>{ + if(!endpointSent||!partials.length)return; + clearTimeout(finalTimer); + finalTimer=window.setTimeout(()=>{ + if(epoch!==micEpoch||socket!==asr||submitted)return; + clearTimeout(asrTimeout);submitted=true;const text=partials.join('');socket.close();asr=null; + utterance=false;loud=quiet=0;preRoll.length=0; + void send(text).catch(fail); + },600); + }; + processor.port.onmessage=({data}:{data:{pcm:ArrayBuffer;rms:number;ms:number}})=>{ + if(epoch!==micEpoch)return; + preRoll.push(data.pcm);if(preRoll.length>3)preRoll.shift(); + if(data.rms>0.025){loud+=data.ms;quiet=0;}else{quiet+=data.ms;if(!utterance)loud=0;} + if(!utterance&&loud>=150){ + silence('microphone-onset',true);inTalk=true;message('检测到开口,已停止旧回复,正在识别…'); + utterance=true;submitted=false;partials=[];endpointSent=false; + asr=new WebSocket(`${location.protocol==='https:'?'wss':'ws'}://${location.host}/api/asr`); + const socket=asr; + socket.onopen=()=>{if(socket===asr)preRoll.forEach(packet=>socket.send(packet))}; + socket.onmessage=event=>{ + if(epoch!==micEpoch||socket!==asr||submitted)return; + const response=JSON.parse(event.data); + mark('asr-result',response); + if(response.error){fail(new Error(response.error));return;} + if(response.text)el('transcript').textContent=`识别:${response.text}`; + if(String(response.mode).endsWith('offline')&&response.text){ + partials.push(response.text);commit(socket); + } + }; + socket.onerror=()=>fail(new Error('ASR连接失败')); + socket.onclose=()=>{if(socket===asr&&!submitted){asr=null;utterance=false;message('识别连接结束,请重新开口或用文字。');}}; + } + if(utterance&&!endpointSent&&asr?.readyState===WebSocket.OPEN)asr.send(data.pcm); + if(utterance&&!endpointSent&&quiet>=650&&asr?.readyState===WebSocket.OPEN){ + endpointSent=true;asr.send(JSON.stringify({is_speaking:false}));commit(asr); + mark('microphone-endpoint'); + asrTimeout=window.setTimeout(()=>fail(new Error('ASR未返回终稿,请重新开麦或输入文字')),6000); + // ASR must return an actual final; never fabricate one from silence. + } + }; + await micContext.resume();el('mic').setAttribute('aria-pressed','true');el('mic').textContent='关闭麦克风'; + message('麦克风已开启。开口会停止旧回复;建议佩戴耳机。'); +} + +// Record the rendered canvas and the same media element's audible output on a +// shared browser MediaRecorder clock, instead of rebuilding timestamps by count. +async function record() { + if(recording?.state==='recording'){recording.stop();return;} + await audioGraph(); + const canvas=document.createElement('canvas');canvas.width=960;canvas.height=720; + const context=canvas.getContext('2d')!; + const theme=getComputedStyle(document.documentElement); + const background=theme.getPropertyValue('--scene-night-deep').trim(); + const ink=theme.getPropertyValue('--ink-primary').trim(); + const draw=()=>{ + context.fillStyle=background;context.fillRect(0,0,960,720); + const video=view==='bridge'?bridge:view==='listening'?listening:view==='talk'?talk:daily; + if(video.readyState>=2){const scale=Math.min(960/video.videoWidth,640/video.videoHeight);context.drawImage(video,(960-video.videoWidth*scale)/2,0,video.videoWidth*scale,video.videoHeight*scale);} + context.fillStyle=ink;context.font='18px sans-serif';context.fillText(status.textContent??'',12,675);context.fillText(`${(performance.now()/1000).toFixed(3)}s · 浏览器播放音画`,12,705); + recordingRaf=requestAnimationFrame(draw); + };draw(); + const stream=canvas.captureStream(25);destination!.stream.getAudioTracks().forEach(t=>stream.addTrack(t)); + const chunks:Blob[]=[]; + recording=new MediaRecorder(stream,{mimeType:'video/webm;codecs=vp8,opus'}); + recording.ondataavailable=e=>chunks.push(e.data); + recording.onstop=()=>{ + cancelAnimationFrame(recordingRaf);stream.getVideoTracks().forEach(t=>t.stop()); + if(downloadUrl)URL.revokeObjectURL(downloadUrl); + downloadUrl=URL.createObjectURL(new Blob(chunks,{type:'video/webm'})); + const link=el('download');link.href=downloadUrl;link.download='wanwan-browser-av.webm';link.hidden=false; + el('record').textContent='录制音画证据';mark('record-stop'); + }; + recording.start(1000);el('record').textContent='停止录制';mark('record-start'); +} + +el('enter').onclick=()=>void enter().catch(fail); +el('retry').onclick=()=>{silence('reconnect');void enter().catch(fail)}; +el('cancel').onclick=()=>{silence('button',true);message('已停下,正在听。')}; +el('back').onclick=()=>{ + const needsReturn=view!=='daily';inTalk=false;silence('daily');const epoch=generation; + void (async()=>{ + if(needsReturn&&manifest.transition?.available) { + message('已停止声音,正在回到阅读…'); + await playBridge(manifest.transition.return,epoch); + } + if(epoch!==generation)return; + setView('daily');el('scene-label').textContent='日常成片 · 阅读';await selectClip();message('已回到阅读原位置。'); + })().catch(fail); +}; +el('pause').onclick=()=>{paused=!paused;daily.pause();if(!paused)void selectClip().catch(fail);el('pause').textContent=paused?'继续日常':'暂停日常'}; +el('mic').onclick=()=>void startMic().catch(fail); +el('record').onclick=()=>void record().catch(fail); +el('form').onsubmit=event=>{event.preventDefault();const input=el('text');const text=input.value;input.value='';void send(text).catch(error=>{if(error?.name!=='AbortError')fail(error)})}; +for(const id of ['activity','weather','timeofday'])el(id).onchange=()=>void selectClip().catch(fail); +document.addEventListener('visibilitychange',()=>{ + if(document.hidden){inTalk=false;silence('hidden');daily.pause();message('页面已隐藏,音视频与麦克风均已停止;恢复后需主动进入。');} +}); +window.addEventListener('pagehide',()=>{const old=token;silence('pagehide');if(old)void fetch('/api/cancel',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({token:old}),keepalive:true})}); +window.addEventListener('offline',()=>fail(new Error('网络已断开'))); +daily.onerror=()=>{dailyMissing=true;el('enter').disabled=true;fail(new Error('日常素材丢失或无法解码'))}; +talk.onerror=()=>fail(new Error('实时视频无法解码')); +void (async()=>{ + const response=await fetch('/assets/manifest.json');if(!response.ok)throw new Error('素材manifest不可达'); + manifest=await response.json();await selectClip(); + if(manifest.talk_clips.length)listening.src=`/assets/${manifest.talk_clips[0].file}`; + if(manifest.transition?.available)el('warning').textContent='已补目视底板与真实往返候选;自然衔接、同脸和同步仍待人工观感验收。'; + el('enter').disabled=!manifest.talk_clips.length; + message('日常视频已就绪。交谈入口会连接本地实时口型。'); +})().catch(fail); diff --git a/samples/companion/src/style.css b/samples/companion/src/style.css new file mode 100644 index 0000000..66e9e77 --- /dev/null +++ b/samples/companion/src/style.css @@ -0,0 +1,20 @@ +@import './tokens.css'; +@import './controls.css'; +main { height:100dvh; overflow:auto; padding:var(--space-6); display:grid; grid-template-columns:minmax(0,1fr) minmax(320px,420px); grid-template-rows:auto 1fr; gap:var(--space-4); } +header { grid-column:1/-1; display:flex; align-items:baseline; gap:var(--space-4); } +.stage { position:relative; overflow:hidden; min-height:280px; } +.stage video { width:100%; height:100%; object-fit:contain; position:absolute; inset:0; } +#scene-label { position:absolute; bottom:var(--space-4); left:var(--space-4); padding:var(--space-2); background:var(--glass-bg-strong); border-radius:var(--r-control); } +.controls { padding:var(--space-5); display:flex; flex-direction:column; gap:var(--space-4); overflow:auto; } +.row { display:flex; gap:var(--space-2); flex-wrap:wrap; align-items:center; } +.controls button { border-color:var(--glass-border); background:var(--bubble-her); min-height:44px; } +#enter,button[type=submit] { background:var(--accent); color:var(--accent-ink); } +label { display:block; color:var(--ink-secondary); font-size:var(--fs-small); } +input,select { border:1px solid var(--glass-border); border-radius:var(--r-control); background:var(--glass-bg-strong); padding:var(--space-3); min-width:0; } +input { flex:1; width:100%; } +#warning { color:var(--state-danger); font-size:var(--fs-small); } +#selection, #transcript { color:var(--ink-secondary); } +summary { cursor:pointer; padding-block:var(--space-2); } +a { color:var(--accent); } +@media(max-width:760px) { main { grid-template-columns:1fr; grid-template-rows:auto minmax(260px,45dvh) auto; padding:var(--space-3); } .controls { overflow:visible; } } +@media(prefers-reduced-motion:reduce) { button { transition:none; } button:active { transform:none; } } diff --git a/samples/companion/src/tokens.css b/samples/companion/src/tokens.css new file mode 100644 index 0000000..e42ddd6 --- /dev/null +++ b/samples/companion/src/tokens.css @@ -0,0 +1,73 @@ +/* Reused verbatim from hel-700-rain-room; no alternative product theme. */ +:root { + --scene-night-deep:#16233c; + --scene-city-blue:#305481; + --scene-city-mid:#5a76a2; + --scene-tower:#2b456b; + --scene-glass-blue:#718fc2; + --scene-amber:#e8a45c; + --scene-amber-soft:#f2c894; + --scene-wood:#59361d; + --scene-cream:#e8d9c8; + --scene-blanket:#362928; + --ink-primary:#f2efe9; + --ink-secondary:rgba(242,239,233,.64); + --ink-tertiary:rgba(242,239,233,.4); + --ink-disabled:rgba(242,239,233,.28); + --accent:#e8a45c; + --accent-hover:#f2b879; + --accent-pressed:#c9853f; + --accent-ink:#1a1208; + --glass-bg:rgba(14,18,30,.55); + --glass-bg-strong:rgba(14,18,30,.72); + --glass-border:rgba(255,255,255,.08); + --glass-highlight:rgba(255,255,255,.14); + --bubble-user:rgba(120,144,180,.2); + --bubble-user-border:rgba(255,255,255,.07); + --bubble-her:rgba(255,255,255,.06); + --bubble-failed:rgba(200,90,80,.14); + --state-danger:#e07a6a; + --font-sans:"PingFang SC","HarmonyOS Sans SC","MiSans","Source Han Sans SC",system-ui,-apple-system,sans-serif; + --fs-caption:12px; + --fs-small:13px; + --fs-body:14px; + --fs-chat:15px; + --fs-nav:15px; + --fs-title:20px; + --fs-brand:16px; + --fs-time:13px; + --space-1:4px; + --space-2:8px; + --space-3:12px; + --space-4:16px; + --space-5:20px; + --space-6:24px; + --space-8:32px; + --space-10:40px; + --r-panel:24px; + --r-bubble:16px; + --r-card:16px; + --r-control:12px; + --r-pill:999px; + --shadow-panel:0 24px 80px rgba(0,0,0,.45); + --shadow-pop:0 8px 24px rgba(0,0,0,.35); + --glow-accent:0 0 12px rgba(232,164,92,.45); + --glass-blur:blur(28px) saturate(1.2); + --z-scene-far:0; + --z-scene-rain:1; + --z-scene-near:2; + --z-caption:10; + --z-nav:10; + --z-topbar:20; + --z-panel:30; + --z-toast:40; + --ease-out:cubic-bezier(.23,1,.32,1); + --ease-in-out:cubic-bezier(.77,0,.175,1); + --ease-drawer:cubic-bezier(.32,.72,0,1); + --lamp-opacity:.12; + --panel-width:clamp(360px,30vw,460px); + --panel-top:64px; + --panel-inset:24px; + --scene-height:330px; + --drawer-height:30dvh; +} \ No newline at end of file diff --git a/samples/companion/tools/analyze_sync.py b/samples/companion/tools/analyze_sync.py new file mode 100644 index 0000000..638b4d9 --- /dev/null +++ b/samples/companion/tools/analyze_sync.py @@ -0,0 +1,91 @@ +"""Source-relative lip residual + audio RMS using browser MediaRecorder PTS. +Heuristic evidence only: no automatic perceptual-sync acceptance. +""" + +import argparse +import csv +import json +from pathlib import Path + +import av +import cv2 +import numpy as np + +p = argparse.ArgumentParser() +p.add_argument("video", type=Path) +p.add_argument("--source", type=Path, default=Path("/opt/hel715-verify/talk-close.mp4")) +p.add_argument("--size", type=int, default=768) +p.add_argument("--offset-x", type=int, default=0) +p.add_argument("--offset-y", type=int, default=0) +p.add_argument("--render-size", type=int, default=768) +p.add_argument("--start", type=float, default=0) +p.add_argument("--stop", type=float, default=100000) +p.add_argument("--roi", default="0.4492,0.6185,0.4753,0.6315") +p.add_argument("--idle-until", type=float) +a = p.parse_args() +source = [f.to_ndarray(format="bgr24") for f in av.open(str(a.source)).decode(video=0)] +size = a.size +source = [cv2.resize(f, (size, size)) for f in source] +# Broad lower-face region; source matching uses pixels outside it and outside +# caption/watermark so normal head movement is not mistaken for synthesized lips. +x0, x1, y0, y1 = [round(v * size) for v in map(float, a.roi.split(","))] +mask = np.ones((48, 48), dtype=bool) +mask[0:3] = False +mask[43:] = False +mask[ + round(y0 * 48 / size) : round(y1 * 48 / size), round(x0 * 48 / size) : round(x1 * 48 / size) +] = False +features = np.asarray([cv2.resize(f, (48, 48))[mask].astype(np.float32) for f in source]) +rows = [] +for f in av.open(str(a.video)).decode(video=0): + if not a.start <= float(f.time) <= a.stop: + continue + image = f.to_ndarray(format="bgr24") + image = image[a.offset_y : a.offset_y + a.render_size, a.offset_x : a.offset_x + a.render_size] + image = cv2.resize(image, (size, size)) + feature = cv2.resize(image, (48, 48))[mask].astype(np.float32) + index = np.argmin(((features - feature) ** 2).mean(axis=(1, 2))) + residual = np.abs(image.astype(np.float32) - source[index]).mean(axis=2) + rows.append([float(f.time), float(residual[y0:y1, x0:x1].mean()), int(index)]) +audio = [] +for f in av.open(str(a.video)).decode(audio=0): + if not a.start <= float(f.time) <= a.stop: + continue + data = f.to_ndarray().astype(np.float64) + if f.format.is_planar and f.format.name.startswith("fl"): + pass + elif data.max() > 1: + data /= 32768 + audio.append([float(f.time), float(np.sqrt((data**2).mean()))]) +with a.video.with_suffix(".sync.csv").open("w") as out: + w = csv.writer(out) + w.writerow(["time_seconds", "lip_residual", "source_index", "audio_rms"]) + for t, residual, idx in rows: + nearest = min(audio, key=lambda x: abs(x[0] - t)) + w.writerow([t, residual, idx, nearest[1]]) +first_audio = next((t for t, rms in audio if rms > 0.015), None) +baseline = [ + r + for t, r, _ in rows + if first_audio and t < (a.idle_until if a.idle_until is not None else first_audio - 0.5) +] +threshold = max(4, float(np.quantile(baseline, 0.99)) * 1.5) if baseline else 4 +first_lip = next( + ( + t + for t, r, _ in rows + if r > threshold + and first_audio + and t >= (a.idle_until if a.idle_until is not None else first_audio - 1) + ), + None, +) +result = dict( + first_audio=first_audio, + first_lip=first_lip, + residual_threshold=threshold, + onset_difference_seconds=first_lip - first_audio if first_lip and first_audio else None, + caveat="Source-relative residual heuristic, not phoneme alignment or perceptual acceptance", +) +a.video.with_suffix(".sync.json").write_text(json.dumps(result, indent=2)) +print(result) diff --git a/samples/companion/tools/baseline.py b/samples/companion/tools/baseline.py new file mode 100644 index 0000000..036880d --- /dev/null +++ b/samples/companion/tools/baseline.py @@ -0,0 +1,50 @@ +import asyncio +import json +import pathlib + +from playwright.async_api import async_playwright + +OUT = pathlib.Path(__file__).resolve().parents[1] / "evidence" +OUT.mkdir(exist_ok=True) + + +async def main(): + async with async_playwright() as p: + browser = await p.chromium.launch( + args=["--no-sandbox", "--autoplay-policy=no-user-gesture-required"] + ) + page = await browser.new_page() + await page.goto("http://192.168.200.12:8010") + await page.evaluate("""async () => { + document.body.innerHTML=''; + const v=document.querySelector('video'), c=document.querySelector('canvas'), ctx=c.getContext('2d'); + const ac=new AudioContext(), dest=ac.createMediaStreamDestination(); + const pc=window.pc=new RTCPeerConnection({iceServers:[]}); + const stream=new MediaStream(); v.srcObject=stream; + pc.ontrack=e=>stream.addTrack(e.track); + pc.addTransceiver('audio',{direction:'recvonly'}); pc.addTransceiver('video',{direction:'recvonly'}); + await pc.setLocalDescription(await pc.createOffer()); + await new Promise(r=>{if(pc.iceGatheringState==='complete')r();else pc.onicegatheringstatechange=()=>{if(pc.iceGatheringState==='complete')r()}}); + const a=await (await fetch('/offer',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(pc.localDescription)})).json(); + window.sid=a.sessionid; await pc.setRemoteDescription(a); + await new Promise(r=>v.onloadeddata=r); await v.play(); + v.muted=true; const source=ac.createMediaStreamSource(stream); source.connect(ac.destination); source.connect(dest); + await ac.resume(); const recorded=c.captureStream(25); dest.stream.getAudioTracks().forEach(t=>recorded.addTrack(t)); + window.metrics=[]; const analyser=ac.createAnalyser();source.connect(analyser);let buffer=new Float32Array(analyser.fftSize); + function draw(){ctx.drawImage(v,0,0,768,768);ctx.fillStyle='white';ctx.font='20px sans-serif';ctx.fillText((performance.now()/1000).toFixed(3),20,750);analyser.getFloatTimeDomainData(buffer);metrics.push([performance.now(),Math.sqrt(buffer.reduce((s,x)=>s+x*x,0)/buffer.length)]);window.raf=requestAnimationFrame(draw)}draw(); + window.chunks=[];window.rec=new MediaRecorder(recorded,{mimeType:'video/webm;codecs=vp8,opus'});rec.ondataavailable=e=>chunks.push(e.data);rec.start(1000); + }""") + await page.wait_for_timeout(1500) + await page.evaluate( + """async()=>{window.sent=performance.now();await fetch('/human',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({sessionid:sid,type:'echo',text:'你好。八百标兵奔北坡,炮兵并排北边跑。现在我停下来听你说话。'})})}""" + ) + await page.wait_for_timeout(18000) + data = await page.evaluate( + """async()=>{await new Promise(r=>{rec.onstop=r;rec.stop()});const b=new Blob(chunks);return Array.from(new Uint8Array(await b.arrayBuffer()))}""" + ) + (OUT / "baseline-browser-av.webm").write_bytes(bytes(data)) + (OUT / "baseline.json").write_text(json.dumps(await page.evaluate("({sent,metrics})"))) + await browser.close() + + +asyncio.run(main()) diff --git a/samples/companion/tools/check_browser.py b/samples/companion/tools/check_browser.py new file mode 100644 index 0000000..0d1542c --- /dev/null +++ b/samples/companion/tools/check_browser.py @@ -0,0 +1,136 @@ +"""Real Chromium/WebRTC check; fake devices are explicitly synthetic fixtures.""" + +import asyncio +import json +import pathlib +import time + +from playwright.async_api import async_playwright + +ROOT = pathlib.Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" +OUT.mkdir(exist_ok=True) +URL = "https://192.168.200.12:8014" + + +async def download(page, path): + await page.click("#record") + await page.wait_for_selector("#download:not([hidden])") + async with page.expect_download() as pending: + await page.click("#download") + await (await pending.value).save_as(path) + + +async def main(): + async with async_playwright() as p: + browser = await p.chromium.launch( + args=["--no-sandbox", "--autoplay-policy=no-user-gesture-required"] + ) + context = await browser.new_context( + ignore_https_errors=True, viewport={"width": 1280, "height": 800} + ) + page = await context.new_page() + errors = [] + page.on("pageerror", lambda e: errors.append(str(e))) + await page.goto(URL) + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('就绪')" + ) + await page.click("#record") + await page.wait_for_timeout(1000) + await page.fill("#text", "我今天下班了,有点累。请只用一句简短的话回应。") + await page.click("button[type=submit]") + await page.wait_for_function( + "window.sampleEvents.some(e=>e.event==='first-audible')", timeout=60000 + ) + await page.wait_for_timeout(1500) + await page.click("#cancel") + await page.wait_for_timeout(1500) + assert await page.evaluate("document.querySelector('#talk').srcObject===null") + await page.fill("#text", "你在看什么书?只说一句话。") + await page.click("button[type=submit]") + await page.wait_for_function( + "window.sampleEvents.filter(e=>e.event==='first-audible').length>=2", timeout=60000 + ) + await page.wait_for_timeout(6000) + await page.click("#back") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('已回到阅读')", timeout=15000 + ) + assert not await page.locator("#daily").evaluate("(v)=>v.paused") + await page.click("#pause") + assert await page.locator("#daily").evaluate("(v)=>v.paused") + await page.click("#pause") + await page.locator("summary").click() + await page.select_option("#weather", "clear") + assert "回退" in await page.locator("#selection").inner_text() + await page.click("#enter") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('正在听')", timeout=30000 + ) + await page.fill("#text", "这是取消准备阶段的测试,请简短回复。") + await page.click("button[type=submit]") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('准备回复')", timeout=30000 + ) + await page.wait_for_timeout(1000) + await page.click("#cancel") + await page.wait_for_timeout(6000) + assert await page.locator("#talk").evaluate("(v)=>v.srcObject===null") + await page.click("#back") + (OUT / "sample-events.json").write_text( + json.dumps(await page.evaluate("window.sampleEvents"), ensure_ascii=False, indent=2) + ) + await download(page, OUT / "sample-browser-av.webm") + await page.screenshot(path=str(OUT / "desktop.png")) + await page.set_viewport_size({"width": 390, "height": 844}) + await page.screenshot(path=str(OUT / "mobile.png"), full_page=True) + mobile_overflow = await page.evaluate("document.documentElement.scrollWidth > innerWidth") + # Failure tests use browser request routing, never stop shared models. + await page.route("**/api/offer", lambda route: route.abort()) + await page.click("#retry") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('已停止交谈')", timeout=30000 + ) + await page.unroute("**/api/offer") + await page.click("#retry") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('正在听')", timeout=30000 + ) + await context.set_offline(True) + await page.wait_for_timeout(500) + assert await page.locator("#talk").evaluate("(v)=>v.srcObject===null") + await context.set_offline(False) + await page.reload() + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('就绪')" + ) + # This is a handler test, not a claim of real OS background testing. + await page.evaluate( + "Object.defineProperty(document,'hidden',{configurable:true,value:true});document.dispatchEvent(new Event('visibilitychange'))" + ) + assert await page.locator("#daily").evaluate("(v)=>v.paused") + await page.evaluate( + "Object.defineProperty(document,'hidden',{configurable:true,value:false})" + ) + await page.route("**/reading-rain-night.mp4", lambda route: route.abort()) + await page.reload() + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('素材丢失')", timeout=30000 + ) + result = { + "pageerrors": errors, + "mobile_overflow": mobile_overflow, + "visibility_test": "synthetic event only; real OS background not tested", + "cancel": "speaking and preparing", + "network": "offer abort, offline, recovery", + "missing_asset": "daily video request abort", + "time": time.time(), + } + # Events from the main continuous recording were captured before reload. + (OUT / "browser-check.json").write_text(json.dumps(result, ensure_ascii=False, indent=2)) + await browser.close() + print(json.dumps(result, ensure_ascii=False), flush=True) + + +asyncio.run(main()) diff --git a/samples/companion/tools/check_gateway.py b/samples/companion/tools/check_gateway.py new file mode 100644 index 0000000..3bb1294 --- /dev/null +++ b/samples/companion/tools/check_gateway.py @@ -0,0 +1,127 @@ +"""Cancellation/voice checks against local fake HTTP services, no paid APIs.""" + +import asyncio +import importlib.util +import pathlib +import unittest + +from aiohttp import web +from aiohttp.test_utils import TestServer + +spec = importlib.util.spec_from_file_location( + "sample_server", pathlib.Path(__file__).resolve().parents[1] / "runtime/server.py" +) +server = importlib.util.module_from_spec(spec) +spec.loader.exec_module(server) + + +class GatewayChecks(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.started = asyncio.Event() + self.release = asyncio.Event() + self.mode = "llm" + self.voice = "xiaoyun" + self.queued = 0 + self.closed = [] + app = web.Application() + + async def health(request): + return web.json_response({"ok": True, "voices": [self.voice]}) + + async def llm(request): + if self.mode == "llm": + self.started.set() + await self.release.wait() + return web.json_response({"choices": [{"message": {"content": "你好。"}}]}) + + async def tts(request): + self.started.set() + await self.release.wait() + return web.Response(body=b"\0\0" * 2400) + + async def audio(request): + self.queued += 1 + return web.json_response({"code": 0}) + + async def close(request): + self.closed.append((await request.json())["sessionid"]) + return web.json_response({"closed": True}) + + app.router.add_get("/health", health) + app.router.add_post("/llm", llm) + app.router.add_post("/tts", tts) + app.router.add_post("/humanaudio", audio) + app.router.add_post("/hel714/close", close) + self.upstream = TestServer(app) + await self.upstream.start_server() + base = str(self.upstream.make_url("")).rstrip("/") + self.gateway = server.Gateway( + { + "driver": base, + "llm": base + "/llm", + "model": "fixture", + "tts": base, + "voice": "xiaoyun", + } + ) + await self.gateway.start(None) + + async def asyncTearDown(self): + self.release.set() + await self.gateway.stop(None) + await self.upstream.close() + + async def cancel_at(self, mode): + self.mode = mode + session = {"sid": "old-session", "task": None} + self.gateway.sessions["old-token"] = session + task = session["task"] = asyncio.create_task(self.gateway.reply(session, "测试")) + await asyncio.wait_for(self.started.wait(), 2) + await self.gateway.retire("old-token") + self.assertTrue(task.cancelled()) + self.release.set() + await asyncio.sleep(0.05) + self.assertEqual(self.queued, 0) + self.assertEqual(self.closed, ["old-session"]) + self.assertNotIn("old-token", self.gateway.sessions) + # Idempotence: a late duplicate cancel cannot close a new session. + self.gateway.sessions["new-token"] = {"sid": "new-session", "task": None} + await self.gateway.retire("old-token") + self.assertIn("new-token", self.gateway.sessions) + + async def test_cancel_during_llm(self): + await self.cancel_at("llm") + + async def test_cancel_during_tts(self): + await self.cancel_at("tts") + + async def test_missing_voice_is_explicit(self): + self.voice = "default" + with self.assertRaisesRegex(RuntimeError, "未回退"): + await self.gateway.reply({"sid": "test"}, "测试") + self.assertEqual(self.queued, 0) + + async def test_cancel_while_waiting_for_media_ready(self): + self.mode = "tts" + session = {"sid": "old-session", "task": None, "ready": asyncio.Event()} + self.gateway.sessions["old-token"] = session + task = session["task"] = asyncio.create_task(self.gateway.reply(session, "测试")) + await asyncio.wait_for(self.started.wait(), 2) + self.release.set() + await asyncio.sleep(0.05) + self.assertEqual(self.queued, 0) + await self.gateway.retire("old-token") + session["ready"].set() + await asyncio.sleep(0.05) + self.assertTrue(task.cancelled()) + self.assertEqual(self.queued, 0) + + async def test_model_unreachable(self): + self.gateway.config["llm"] = "http://127.0.0.1:1/unavailable" + with self.assertRaises(server.aiohttp.ClientError): + await self.gateway.reply({"sid": "unreachable"}, "测试") + self.assertEqual(self.queued, 0) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/samples/companion/tools/check_microphone.py b/samples/companion/tools/check_microphone.py new file mode 100644 index 0000000..56d74ec --- /dev/null +++ b/samples/companion/tools/check_microphone.py @@ -0,0 +1,89 @@ +"""Synthetic WAV through Chromium's microphone device, not a human VAD pass.""" + +import asyncio +import json +import pathlib + +from playwright.async_api import async_playwright + +ROOT = pathlib.Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" + + +async def main(): + async with async_playwright() as p: + browser = await p.chromium.launch( + args=[ + "--no-sandbox", + "--autoplay-policy=no-user-gesture-required", + "--use-fake-device-for-media-stream", + "--use-fake-ui-for-media-stream", + f"--use-file-for-fake-audio-capture={OUT / 'synthetic-microphone.wav'}%noloop", + ] + ) + context = await browser.new_context(ignore_https_errors=True, permissions=["microphone"]) + page = await context.new_page() + errors = [] + page.on("pageerror", lambda error: errors.append(str(error))) + await page.goto("https://192.168.200.12:8014") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('就绪')" + ) + await page.click("#enter") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('正在听')", timeout=40000 + ) + await page.click("#record") + await page.click("#mic") + await page.wait_for_function( + "document.querySelector('#mic').getAttribute('aria-pressed')==='true'" + ) + await page.fill("#text", "请用三句话描述你眼前卧室的温暖氛围,大约六十个字。") + await page.click("button[type=submit]") + await page.wait_for_function( + "window.sampleEvents.some(e=>e.event==='cancel'&&e.detail.reason==='microphone-onset')", + timeout=25000, + ) + await page.wait_for_function( + "document.querySelector('#transcript').textContent.includes('识别:')", timeout=15000 + ) + await page.wait_for_function( + "window.sampleEvents.filter(e=>e.event==='input-finished').length>=2", timeout=15000 + ) + await page.wait_for_timeout(18000) + await page.click("#record") + await page.wait_for_selector("#download:not([hidden])") + async with page.expect_download() as pending: + await page.click("#download") + await (await pending.value).save_as(OUT / "synthetic-microphone-av.webm") + events = await page.evaluate("window.sampleEvents") + onset = next( + e["at"] + for e in events + if e["event"] == "cancel" and e["detail"]["reason"] == "microphone-onset" + ) + audible_before = [e for e in events if e["event"] == "first-audible" and e["at"] < onset] + audible_after = [e for e in events if e["event"] == "first-audible" and e["at"] > onset] + result = dict( + fixture="CosyVoice synthesized WAV through fake Chromium microphone; no human recording", + secure_context=await page.evaluate("isSecureContext"), + transcript=await page.locator("#transcript").inner_text(), + speech_was_audible_before_onset=bool(audible_before), + new_reply_after_asr=bool(audible_after), + events=events, + pageerrors=errors, + ) + (OUT / "synthetic-microphone.json").write_text( + json.dumps(result, ensure_ascii=False, indent=2) + ) + await page.click("#mic") + await page.click("#back") + await page.wait_for_timeout(4000) + await browser.close() + print( + json.dumps({k: v for k, v in result.items() if k != "events"}, ensure_ascii=False), + flush=True, + ) + + +asyncio.run(main()) diff --git a/samples/companion/tools/check_races.py b/samples/companion/tools/check_races.py new file mode 100644 index 0000000..6e96ccc --- /dev/null +++ b/samples/companion/tools/check_races.py @@ -0,0 +1,52 @@ +"""Cancel during ICE/offer, then verify a late old error cannot stop a new turn.""" + +import asyncio +import json +import pathlib + +from playwright.async_api import async_playwright + +OUT = pathlib.Path(__file__).resolve().parents[1] / "evidence" + + +async def main(): + async with async_playwright() as p: + browser = await p.chromium.launch( + args=["--no-sandbox", "--autoplay-policy=no-user-gesture-required"] + ) + context = await browser.new_context(ignore_https_errors=True) + page = await context.new_page() + errors = [] + page.on("pageerror", lambda e: errors.append(str(e))) + await page.goto("https://192.168.200.12:8014") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('就绪')" + ) + await page.click("#enter") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('正在听')", timeout=40000 + ) + await page.fill("#text", "这一轮马上取消。") + await page.click("button[type=submit]") + await page.wait_for_timeout(100) + await page.click("#cancel") + await page.fill("#text", "新一轮只回复:你好。") + await page.click("button[type=submit]") + await page.wait_for_function( + "window.sampleEvents.some(e=>e.event==='first-audible')", timeout=45000 + ) + await page.wait_for_timeout(10000) + events = await page.evaluate("window.sampleEvents") + failures = [ + e for e in events if e["event"] == "status" and "媒体协商超时" in str(e["detail"]) + ] + result = {"pageerrors": errors, "late_error_failures": failures, "events": events} + (OUT / "race-check.json").write_text(json.dumps(result, ensure_ascii=False, indent=2)) + assert not failures + await page.click("#back") + await page.wait_for_timeout(4000) + await browser.close() + print("ICE cancellation race passed", flush=True) + + +asyncio.run(main()) diff --git a/samples/companion/tools/generate_transition.py b/samples/companion/tools/generate_transition.py new file mode 100644 index 0000000..3126a99 --- /dev/null +++ b/samples/companion/tools/generate_transition.py @@ -0,0 +1,251 @@ +"""Generate one v2 gaze idle and two forward first/last-frame transitions. + +Owns and stops its exact server PID in finally. Does not modify the installation. +""" + +import json +import os +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path("/media/lucian/Downloads/ComfyUI") +TASK = Path("/opt/hel714-material") +URL = "http://127.0.0.1:18189" + + +def request(path, payload=None): + data = None if payload is None else json.dumps(payload).encode() + req = urllib.request.Request( + URL + path, data=data, headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(req, timeout=20) as response: + return json.load(response) + + +def graph(kind): + nodes = {} + + def add(key, kind, **inputs): + nodes[str(key)] = dict(class_type=kind, inputs=inputs) + + for key, mode in [(1, "high"), (2, "low")]: + add( + key, + "UNETLoader", + unet_name=f"wan2.2_i2v_{mode}_noise_14B_fp8_scaled.safetensors", + weight_dtype="default", + ) + add( + key + 2, + "LoraLoaderModelOnly", + model=[str(key), 0], + lora_name=f"wan2.2_i2v_lightx2v_4steps_lora_v1_{mode}_noise.safetensors", + strength_model=1.0, + ) + add(key + 4, "ModelSamplingSD3", model=[str(key + 2), 0], shift=5.0) + add( + 7, + "CLIPLoader", + clip_name="umt5_xxl_fp8_e4m3fn_scaled.safetensors", + type="wan", + device="cpu", + ) + add(8, "VAELoader", vae_name="wan_2.1_vae.safetensors") + add(9, "LoadImage", image="start.png") + add(17, "LoadImage", image="end.png") + add( + 10, + "CLIPTextEncode", + clip=["7", 0], + text=( + "Locked off camera, photorealistic young adult woman with short dark hair, " + "cream cardigan and dark trousers seated on the same sofa in the same bedroom. " + "She gently raises her gaze from the book to meet the viewer, then holds a calm listening gaze. " + "Her hands keep holding the open book without moving it. Natural continuous breathing. " + "Fixed seat position, stationary room and furniture, identical face and outfit." + ), + ) + add( + 11, + "CLIPTextEncode", + clip=["7", 0], + text="camera movement, zoom, changing face, changing clothes, standing up, " + "extra fingers, distorted hands, deformed face, moving furniture, subtitles", + ) + add( + 12, + "WanFirstLastFrameToVideo", + positive=["10", 0], + negative=["11", 0], + vae=["8", 0], + width=512, + height=512, + length=49, + batch_size=1, + start_image=["9", 0], + end_image=["17", 0], + ) + add( + 13, + "KSamplerAdvanced", + model=["5", 0], + add_noise="enable", + noise_seed=714, + steps=4, + cfg=1.0, + sampler_name="euler", + scheduler="simple", + positive=["12", 0], + negative=["12", 1], + latent_image=["12", 2], + start_at_step=0, + end_at_step=2, + return_with_leftover_noise="enable", + ) + add( + 14, + "KSamplerAdvanced", + model=["6", 0], + add_noise="disable", + noise_seed=714, + steps=4, + cfg=1.0, + sampler_name="euler", + scheduler="simple", + positive=["12", 0], + negative=["12", 1], + latent_image=["13", 0], + start_at_step=2, + end_at_step=4, + return_with_leftover_noise="disable", + ) + add(15, "VAEDecode", samples=["14", 0], vae=["8", 0]) + add(16, "SaveWEBM", images=["15", 0], codec="vp9", fps=16.0, filename_prefix="enter", crf=20.0) + nodes["9"]["inputs"]["image"] = "daily-cut.png" if kind == "enter" else "near.png" + nodes["17"]["inputs"]["image"] = "daily-cut.png" if kind == "return" else "near.png" + nodes["10"]["inputs"]["text"] = { + "enter": "Photorealistic adult woman, same short dark hair, cream cardigan, same bedroom. Starting from reading, she gently raises her head to meet the viewer. Smooth slow camera push toward her half body, ending at the exact reference framing. Hands keep holding the same book. Continuous physically plausible movement, stable face, stable clothing and light.", + "return": "Photorealistic adult woman, same short dark hair, cream cardigan, same bedroom. She gently lowers her gaze to the book and returns to reading, with a smooth slow camera pull back to the exact final reference. Same hands holding the same book, continuous physically plausible movement, stable face, stable clothing and light.", + "idle": "Locked camera, photorealistic adult woman in cream cardigan in the same bedroom. Calmly looks toward the viewer, lips gently closed, one subtle natural blink and breathing. Keeps holding the open book still. Stable face, clothing, seat, camera, light and room.", + }[kind] + nodes["11"]["inputs"]["text"] = ( + "changing identity, changing clothes, extra fingers, distorted hands, deformed face, melting furniture, subtitles, speaking mouth" + ) + nodes["12"]["inputs"]["length"] = 81 if kind == "idle" else 49 + nodes["16"]["inputs"]["filename_prefix"] = kind + return nodes + + +def main(): + (TASK / "assets").mkdir(parents=True, exist_ok=True) + # Explicit first/last frames are prepared from the approved v2 source. + assert (TASK / "assets/daily-cut.png").exists() + assert (TASK / "assets/near.png").exists() + if list(TASK.glob("*-history.json")): + raise RuntimeError("Use a new task directory; existing generation evidence is preserved") + for name in ["user", "temp"]: + (TASK / name).mkdir(exist_ok=True) + os.environ["HF_HUB_OFFLINE"] = "1" + os.environ["TRANSFORMERS_OFFLINE"] = "1" + command = [ + str(ROOT / "venv/bin/python"), + str(ROOT / "main.py"), + "--listen", + "127.0.0.1", + "--port", + "18189", + "--disable-all-custom-nodes", + "--offline", + "--disable-manager-ui", + "--disable-assets", + "--reserve-vram", + "8", + "--disable-dynamic-vram", + "--input-directory", + str(TASK / "assets"), + "--output-directory", + str(TASK), + "--user-directory", + str(TASK / "user"), + "--temp-directory", + str(TASK / "temp"), + "--database-url", + "sqlite:///:memory:", + ] + # Refuse to attach to an unrelated listener. + import socket + + with socket.socket() as sock: + if sock.connect_ex(("127.0.0.1", 18189)) == 0: + raise RuntimeError("Task port already occupied; no process started") + with open(TASK / "server.log", "w") as log: + server = subprocess.Popen(command, cwd=ROOT, stdout=log, stderr=subprocess.STDOUT) + (TASK / "process.json").write_text(json.dumps(dict(pid=server.pid, command=command))) + try: + deadline = time.monotonic() + 90 + while True: + if server.poll() is not None: + raise RuntimeError(f"ComfyUI exited: {server.returncode}; see server.log") + try: + info = request("/object_info") + break + except (OSError, urllib.error.URLError): + if time.monotonic() > deadline: + raise TimeoutError("Startup exceeded 90 seconds") from None + time.sleep(1) + for kind in ["idle", "enter", "return"]: + workflow = graph(kind) + (TASK / f"{kind}-workflow.json").write_text(json.dumps(workflow, indent=2)) + required = {n["class_type"]: info.get(n["class_type"]) for n in workflow.values()} + (TASK / f"{kind}-nodes.json").write_text(json.dumps(required, indent=2)) + if any(v is None for v in required.values()): + raise RuntimeError("Required native node unavailable; see nodes.json") + result = request("/prompt", {"prompt": workflow}) + (TASK / f"{kind}-submission.json").write_text(json.dumps(result)) + prompt_id = result["prompt_id"] + print( + f"Generating one clip: {prompt_id}; owned server PID {server.pid}", flush=True + ) + deadline = time.monotonic() + 2400 + last_report = 0 + while time.monotonic() < deadline: + history = request("/history/" + prompt_id) + if prompt_id in history: + (TASK / f"{kind}-history.json").write_text(json.dumps(history, indent=2)) + print(json.dumps(history[prompt_id]["status"]), flush=True) + if history[prompt_id]["status"]["status_str"] != "success": + raise RuntimeError("Generation failed; see history.json") + break + if time.monotonic() - last_report > 30: + print( + "Single generation still active; existing services untouched.", + flush=True, + ) + last_report = time.monotonic() + time.sleep(3) + else: + request("/interrupt", {}) + raise TimeoutError("Single generation exceeded 40 minutes") + except urllib.error.HTTPError as error: + (TASK / "api-error.json").write_bytes(error.read()) + raise + finally: + server.terminate() + try: + server.wait(timeout=20) + except subprocess.TimeoutExpired: + server.kill() + server.wait() + print(f"Stopped owned server PID {server.pid}", flush=True) + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--task", type=Path, default=TASK) + TASK = parser.parse_args().task + main() diff --git a/samples/companion/tools/pack_assets.py b/samples/companion/tools/pack_assets.py new file mode 100644 index 0000000..8553192 --- /dev/null +++ b/samples/companion/tools/pack_assets.py @@ -0,0 +1,41 @@ +"""Verify a whole versioned pack before building or rolling back. + +Material versions and outfit labels are independent of the stable companion id. +This command never writes identity, bazi, memory or relationship state. +""" + +import argparse +import hashlib +import json +from pathlib import Path + + +def verify(root): + root = root.resolve() + manifest = json.loads((root / "manifest.json").read_text()) + assert manifest["version"] and manifest["persona_id"] + assert manifest["companion_id"] == "wanwan", "Companion identity cannot change with assets" + files = [manifest["identity_ref"]] + files += [c["file"] for c in manifest["daily_clips"] + manifest["talk_clips"]] + transition = manifest.get("transition", {}) + if transition.get("available"): + files += [transition["enter"], transition["return"]] + for name in files: + assert name in manifest["sha256"], f"No checksum: {name}" + for name, checksum in manifest["sha256"].items(): + path = (root / name).resolve() + assert path.is_relative_to(root), f"Unsafe asset path: {name}" + assert hashlib.sha256(path.read_bytes()).hexdigest() == checksum, name + return manifest + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--assets", + type=Path, + default=Path(__file__).resolve().parents[1] / "assets/personas/wanwan-v2-cream-night", + ) + args = parser.parse_args() + manifest = verify(args.assets) + print("Verified asset pack:", manifest["version"], len(manifest["sha256"]), "files") diff --git a/samples/companion/tools/patch_driver.py b/samples/companion/tools/patch_driver.py new file mode 100644 index 0000000..da24c6d --- /dev/null +++ b/samples/companion/tools/patch_driver.py @@ -0,0 +1,47 @@ +"""Apply only to a private c726943 code copy, never the reused installation. + +The extra endpoint destroys one owned peer rather than reusing partially flushed +queues. The caller must retain the randomly generated session id. +""" + +import argparse +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument("copy", type=Path) +args = parser.parse_args() +root = args.copy.resolve() +if root == Path("/opt/hel715-verify/LiveTalking"): + raise SystemExit("Refusing to edit reused P10 installation") +p = root / "server/rtc_manager.py" +s = p.read_text() +needle = " self.pcs.add(pc)" +replacement = needle + "\n pc.hel714_sessionid = sessionid" +if replacement not in s: + assert s.count(needle) >= 1 + p.write_text(s.replace(needle, replacement, 1)) +p = root / "app.py" +s = p.read_text() +needle = " setup_routes(appasync)" +replacement = ( + """ async def hel714_close(request): + params = await request.json() + sid = params['sessionid'] + session = session_manager.get_session(sid) + if session: + session.flush_talk() + for peer in list(rtc_manager.pcs): + if getattr(peer, 'hel714_sessionid', None) == sid: + await peer.close() + rtc_manager.pcs.discard(peer) + session_manager.remove_session(sid) + return web.json_response({'closed': sid}) + + appasync.router.add_post('/hel714/close', hel714_close) +""" + + needle +) +if replacement not in s: + assert s.count(needle) >= 1 + p.write_text(s.replace(needle, replacement, 1)) +print("Installed HEL-714 peer retirement endpoint in private copy:", root) diff --git a/samples/companion/tools/prepare_driver.py b/samples/companion/tools/prepare_driver.py new file mode 100644 index 0000000..bfac834 --- /dev/null +++ b/samples/companion/tools/prepare_driver.py @@ -0,0 +1,26 @@ +"""Create a private code copy; symlink only existing weights, never overwrite.""" + +import argparse +import shutil +from pathlib import Path + +parser = argparse.ArgumentParser() +parser.add_argument("--source", type=Path, default=Path("/opt/hel715-verify/LiveTalking")) +parser.add_argument("--destination", type=Path, required=True) +args = parser.parse_args() +source, destination = args.source.resolve(), args.destination.resolve() +assert source != destination and not destination.exists(), "Destination must be new" + + +def ignore(directory, names): + excluded = {n for n in names if n == "__pycache__" or n.endswith(".log")} + if Path(directory).resolve() == source: + excluded.update({"models", "data", ".git"}) + return excluded + + +shutil.copytree(source, destination, ignore=ignore) +(destination / "models").symlink_to(source / "models", target_is_directory=True) +(destination / "data/avatars").mkdir(parents=True) +print("Private driver code:", destination) +print("Weights reused read-only by convention:", source / "models") diff --git a/samples/companion/tools/prepare_material.py b/samples/companion/tools/prepare_material.py new file mode 100644 index 0000000..409295d --- /dev/null +++ b/samples/companion/tools/prepare_material.py @@ -0,0 +1,37 @@ +"""Convert the three generated candidates to 25fps; retain real forward motion.""" + +import json +from pathlib import Path + +import av +import numpy as np +from PIL import Image, ImageDraw + +root = Path("/opt/hel714-material") +assets = Path("/opt/hel714-sample/dist/assets") +metrics = {} +sheet = Image.new("RGB", (7 * 220, 3 * 245)) +draw = ImageDraw.Draw(sheet) +for row, (kind, name) in enumerate([("idle", "talk"), ("enter", "enter"), ("return", "return")]): + source = next(root.glob(f"{kind}_*.webm")) + frames = [f.to_image() for f in av.open(str(source)).decode(video=0)] + out = av.open(str(assets / f"{name}.mp4"), "w") + stream = out.add_stream("libx264", rate=25) + stream.width = stream.height = 512 + stream.pix_fmt = "yuv420p" + stream.options = {"crf": "18", "preset": "fast"} + count = round(len(frames) * 25 / 16) + for i in range(count): + frame = av.VideoFrame.from_image(frames[min(len(frames) - 1, int(i * 16 / 25))]) + for packet in stream.encode(frame): + out.mux(packet) + for packet in stream.encode(): + out.mux(packet) + out.close() + for column, index in enumerate(np.linspace(0, len(frames) - 1, 7).astype(int)): + sheet.paste(frames[index].resize((220, 220)), (column * 220, row * 245)) + draw.text((column * 220, row * 245 + 222), f"{kind} f{index}", fill="white") + metrics[kind] = {"input_frames": len(frames), "output_frames": count, "seconds": count / 25} +sheet.save(root / "candidate-contact.png") +(root / "candidate-metrics.json").write_text(json.dumps(metrics, indent=2)) +print(metrics) diff --git a/samples/companion/tools/soak.py b/samples/companion/tools/soak.py new file mode 100644 index 0000000..f0c6c40 --- /dev/null +++ b/samples/companion/tools/soak.py @@ -0,0 +1,85 @@ +"""Ten-minute real browser / local model session, sampled resource evidence.""" + +import asyncio +import json +import pathlib +import subprocess +import time + +from playwright.async_api import async_playwright + +OUT = pathlib.Path(__file__).resolve().parents[1] / "evidence" + + +async def main(): + async with async_playwright() as p: + browser = await p.chromium.launch( + args=["--no-sandbox", "--autoplay-policy=no-user-gesture-required"] + ) + context = await browser.new_context(ignore_https_errors=True) + page = await context.new_page() + errors = [] + page.on("pageerror", lambda error: errors.append(str(error))) + await page.goto("https://192.168.200.12:8014") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('就绪')" + ) + await page.click("#enter") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('正在听')", timeout=40000 + ) + start = time.monotonic() + samples = [] + for index in range(15): + text = f"这是第{index + 1}次交谈。请只回复一句不超过十五个字的自然中文。" + await page.fill("#text", text) + await page.click("button[type=submit]") + for _tick in range(4): + await page.wait_for_timeout(10000) + state = await page.evaluate("""()=>({status:document.querySelector('#status').textContent, + videoTime:document.querySelector('#talk').currentTime, + frames:document.querySelector('#talk').getVideoPlaybackQuality().totalVideoFrames, + events:window.sampleEvents.slice(-3)})""") + gpu = subprocess.run( + [ + "ssh", + "root@192.168.200.12", + "nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + sample = dict( + elapsed=round(time.monotonic() - start, 2), turn=index + 1, state=state, gpu=gpu + ) + samples.append(sample) + (OUT / "soak-progress.json").write_text( + json.dumps(samples, ensure_ascii=False, indent=2) + ) + print( + "soak", + index + 1, + "elapsed", + round(time.monotonic() - start), + "status", + state["status"], + "gpu", + gpu, + flush=True, + ) + result = dict( + duration_seconds=round(time.monotonic() - start, 2), + samples=samples, + events=await page.evaluate("window.sampleEvents"), + pageerrors=errors, + scope="One logical session, peer retired each turn by cancellation design; 15 real LLM/TTS/avatar turns", + ) + (OUT / "soak.json").write_text(json.dumps(result, ensure_ascii=False, indent=2)) + await page.click("#back") + await page.wait_for_timeout(4000) + await browser.close() + print("SOAK COMPLETE", result["duration_seconds"], len(errors), flush=True) + + +asyncio.run(main()) diff --git a/samples/companion/tools/stage_assets.py b/samples/companion/tools/stage_assets.py new file mode 100644 index 0000000..0a2b0b0 --- /dev/null +++ b/samples/companion/tools/stage_assets.py @@ -0,0 +1,24 @@ +"""Mount the verified active persona pack at the sample's /assets URL. + +public/assets is generated build output. Canonical user-replaceable packs live +under assets/personas; no identity or companion-state file is touched. +""" + +import json +import shutil +from pathlib import Path + +from pack_assets import verify + +root = Path(__file__).resolve().parents[1] +active = json.loads((root / "assets/active.json").read_text()) +source = (root / "assets/personas" / active["persona_id"]).resolve() +assert source.is_relative_to((root / "assets/personas").resolve()) +manifest = verify(source) +assert manifest["persona_id"] == active["persona_id"] +destination = root / "public/assets" +assert not destination.is_symlink(), "Generated asset mount must not be a symlink" +if destination.exists(): + shutil.rmtree(destination) +shutil.copytree(source, destination) +print("Staged complete asset pack:", manifest["persona_id"], manifest["version"]) diff --git a/samples/companion/tsconfig.json b/samples/companion/tsconfig.json new file mode 100644 index 0000000..6fd0535 --- /dev/null +++ b/samples/companion/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable" + ], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "noUnusedLocals": true, + "noUnusedParameters": true + }, + "include": [ + "src" + ] +} diff --git a/samples/companion/versions/hel714-v1/manifest.json b/samples/companion/versions/hel714-v1/manifest.json new file mode 100644 index 0000000..aa01e2a --- /dev/null +++ b/samples/companion/versions/hel714-v1/manifest.json @@ -0,0 +1,51 @@ +{ + "companion_id": "wanwan", + "persona_id": "wanwan-v2-cream-night", + "version": "hel714-v1", + "previous_version": null, + "identity_ref": "identity.png", + "created_by": "HEL-714 / P10 reused assets", + "created_at": "2026-10-20", + "pipeline_version": "hel715-v1", + "outfit": "cream", + "light": "warm-bedroom-night", + "weather_source": "manual-test", + "default_daily": "reading-rain-night", + "daily_clips": [ + { + "id": "reading-rain-night", + "file": "daily.mp4", + "timeofday": "night", + "weather": "rain", + "activity": "reading", + "loop_points": [ + 0, + 195 + ], + "loop_review": "pending-human-review" + } + ], + "talk_clips": [ + { + "id": "close", + "file": "talk.mp4", + "avatar_id": "hel715-talk-close", + "framing": "1.75x reading crop", + "approved": false, + "gaze_review": "not a newly generated eye-contact plate" + } + ], + "transition": { + "file": "return.mp4", + "approved": false, + "cut_points": [], + "reason": "independent source; continuity not accepted" + }, + "manual_consistency_review": "same P10 source; not an accepted natural transition", + "sha256": { + "identity.png": "e40b72c44611cc6f0a9651ad24cc4b0985e2bb8efa19cba963d515a7a6cf4c8f", + "daily.mp4": "776801d38db4a4c3c0afa7488b380987fd9541c699dcfa6e8b26aa492d8255cf", + "talk.mp4": "daf5af9e08efc8580338064b4095e7451e6df531697d4400b49e7e481b1ba62a", + "return.mp4": "f86541a4d141b05c4b2fa3270e22d3d086d5fc128fe0524f385e140455c60875" + } +} -- 2.54.0 From ae4fd5c49ddf81608332fd200c5a5d45db84e5dd Mon Sep 17 00:00:00 2001 From: mobai Date: Wed, 21 Oct 2026 00:21:23 +0800 Subject: [PATCH 3/6] docs(HEL-714): record final soak, sync blocker and cleanup evidence Co-authored-by: multica-agent --- samples/companion/.gitignore | 2 -- samples/companion/REPORT.md | 8 +++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/samples/companion/.gitignore b/samples/companion/.gitignore index 03d8cb0..7f4f59e 100644 --- a/samples/companion/.gitignore +++ b/samples/companion/.gitignore @@ -1,7 +1,5 @@ node_modules/ dist/ -public/assets/*.mp4 -public/assets/*.png runtime/*.log runtime/*.pid __pycache__/ diff --git a/samples/companion/REPORT.md b/samples/companion/REPORT.md index a76b6d3..cb08139 100644 --- a/samples/companion/REPORT.md +++ b/samples/companion/REPORT.md @@ -44,7 +44,7 @@ canonical 素材放 `assets/personas/wanwan-v2-cream-night`,含 daily/talk/ref 既有 ASR 协议经只读检查:FunASR 2pass WebSocket,16k PCM,online 展示、offline 终稿,端点发送 is_speaking=false;没有通过端口猜协议。既有 LLM halogen-qwen3.8-flash-next(.11:8731)、CosyVoice xiaoyun(.12:9880)保持原服务。 -每轮独立 peer。前端立即断开播放音频图、暂停/静音并丢弃视频、递增轮次;后端取消请求、关闭上游响应并精确关闭 driver peer/session,不复用旧媒体队列。旧 ICE/offer 迟到和超时异常按轮次拒绝;取消连接时也立即唤醒等待,避免旧超时误停新轮。LLM/TTS 生成与 ICE/进场并行,ready 门控防止画面到位前发声。TTS 整句缓冲减少分句空隙,长句首声较慢;共享 TTS 内部 GPU 生成是否随客户端断开即时停止未获证明,未改共享服务。 +每轮独立 peer。前端立即断开播放音频图、暂停/静音并丢弃视频、递增轮次;后端取消请求、关闭上游响应并精确关闭 driver peer/session,不复用旧媒体队列。最终连续录像中,两次按钮取消100ms后的1.1秒窗口内,无RMS>0.015的残留声音帧(最大0.000145/0);这只是数字播放图的观测上界。旧 ICE/offer 迟到和超时异常按轮次拒绝;取消连接时也立即唤醒等待,避免旧超时误停新轮。LLM/TTS 生成与 ICE/进场并行,ready 门控防止画面到位前发声。TTS 整句缓冲减少分句空隙,长句首声较慢;共享 TTS 内部 GPU 生成是否随客户端断开即时停止未获证明,未改共享服务。 - 原78项 Python测试通过;TypeScript严格检查+Vite构建通过;新增Python代码ruff通过。 - 5项网关检查通过:取消LLM、取消TTS、等待画面ready时取消、错误音色不回退、模型不可达。 @@ -62,10 +62,12 @@ canonical 素材放 `assets/personas/wanwan-v2-cream-night`,含 daily/talk/ref 最终连续浏览器音画的两轮:首次从日常进入7.43s,已在近景的下一轮5.74s;本轮记录期间有长测客户端并行,不是单用户SLA。此前batch16连续618.41秒、15次真实模型回复、15次首声全部返回、无pageerror/错误状态;P50 5.62s,范围4.93~7.57s,10秒采样总显存最大20248MiB,末尾18277MiB。前段曾并行功能/麦克风自测,不能说这是纯单会话显存峰值。 -batch8最终长测结果与清理记录在交付前补齐。总显存采样包含既有TTS等进程,10秒采样可能漏过瞬时峰值;不声称已测得绝对峰值或无泄漏的长期保证。 +batch8最终代码与新素材目录的长测 619.89 秒,15轮提交均返回实际声音,P50 5.19s、范围4.49~7.79s;无pageerror或错误状态。10秒采样总显存最大16168MiB、末尾15110MiB。前段并行了功能、麦克风和取消竞态测试。与batch16样本非严格受控对照,不能把差值当作SLA改善保证。总显存采样包含既有TTS等进程,10秒采样可能漏过瞬时峰值;不声称已测得绝对峰值或无泄漏的长期保证。 ## 隔离与交接 代码/模型不合并、不部署、不改开机启动。现有18080/18081、8090、9880、10095等共享服务保持。临时服务使用8010/8014,仅本轮自测;启动前确认空闲,按精确PID记录和停止,不使用pkill/fuser清理。权重和用户素材保留。 -最终PR、分支、提交与本轮进程清理证据随卡片评论和复核包交接。当前同步/自然观感与真人、真机缺口均显式列出,不能据构建和基础测试通过自审上线。 +分支 `hel-714-daily-talk`,实际[草稿PR #2](http://192.168.200.36:3200/leefer/CyberGril/pulls/2),目标main。通过Gitea现有SSH的AGit创建,无需Token;PR和远端分支head均已核对。交付时Multica的PR链接表仍为空,CLI只有查询功能,没有盲改标题或调用平台私有API。最终提交号随卡片评论交接。 + +本轮最后的网关PID706044、driver PID737284已按精确PID/命令/cwd核对后停止,先核对其不等于Multica daemon PID44286。8010/8014均无监听;18080/18081仍为原PID538167/608898,9880仍为445461,10095仍为原docker代理。停止后总显存约5701MiB。权重、生成素材和新avatar保留,未部署长期站点。故障注入的HTTPS客户端断开曾产生Python/aiohttp sendfile清理traceback,页面随后恢复且不影响长测;原始网关日志保留,没有升级共享依赖掩盖它。当前同步/自然观感与真人、真机缺口均显式列出,不能据构建和基础测试通过自审上线。 -- 2.54.0 From 415f8372e3759da3fd6e9d362bbb4eebd00a3144 Mon Sep 17 00:00:00 2001 From: mobai Date: Wed, 21 Oct 2026 01:18:41 +0800 Subject: [PATCH 4/6] fix(HEL-718): bound synthetic idle audio and preserve RTP cadence --- samples/companion/README.md | 37 ++++ samples/companion/REPORT-HEL718.md | 64 ++++++ samples/companion/tools/check_driver_audio.py | 92 +++++++++ samples/companion/tools/driver_audio.py | 35 ++++ samples/companion/tools/graph_server.py | 41 ++++ samples/companion/tools/measure_sync.py | 192 ++++++++++++++++++ samples/companion/tools/patch_driver.py | 72 +++++++ samples/companion/tools/record_driver.py | 149 ++++++++++++++ samples/companion/tools/record_graph.html | 89 ++++++++ samples/companion/tools/record_graph.py | 32 +++ samples/companion/tools/record_product.py | 59 ++++++ samples/companion/tools/trace_driver.py | 40 ++++ 12 files changed, 902 insertions(+) create mode 100644 samples/companion/REPORT-HEL718.md create mode 100644 samples/companion/tools/check_driver_audio.py create mode 100644 samples/companion/tools/driver_audio.py create mode 100644 samples/companion/tools/graph_server.py create mode 100644 samples/companion/tools/measure_sync.py create mode 100644 samples/companion/tools/record_driver.py create mode 100644 samples/companion/tools/record_graph.html create mode 100644 samples/companion/tools/record_graph.py create mode 100644 samples/companion/tools/record_product.py create mode 100644 samples/companion/tools/trace_driver.py diff --git a/samples/companion/README.md b/samples/companion/README.md index 7141d8a..b212710 100644 --- a/samples/companion/README.md +++ b/samples/companion/README.md @@ -78,3 +78,40 @@ LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/ 浏览器脚本依赖运行机已安装 Playwright/Chromium;麦克风脚本需复核包内的 `evidence/synthetic-microphone.wav`。它是 CosyVoice 合成的测试句,经 Chromium 假麦克风送入实际 Worklet→ASR→LLM→TTS→口型链路,绝不是人现场说话。录制用页面实际绘制画面和实际播放的 WebAudio 输出、同一个 MediaRecorder 时钟,不按帧数重建时间戳;静音时钟从开头持续输出零样本,保留进入前的日常画面。连续音画是浏览器播放图的合成采集,不是物理屏幕/扬声器校准。 实测结果和未通过项见 `REPORT.md`。生成工作流、history、原始候选片、浏览器证据和资源采样放复核附件,不在主仓库存大文件。 + +## HEL-718 驱动器补丁(候选,尚未通过同步门槛) + +`tools/patch_driver.py` 追加类型透传:`AudioFrameData.type` 经 +`BaseAvatar.process_frames → WebRTCOutput → HumanPlayer` 到未赋 PTS 的 +`PendingAudio`。仅合成空闲 `type=1` 且无事件的块限至3块(60ms); +真实语音(包括零值 PCM)、自定义音频和事件保留,100块队列提供背压。 +首次音频随真实首帧建立时间原点,此后轨道每20ms从队列取最新待播块, +饥饿时补320样本静音;所有已输出包保持320步长,不在 RTP 队列跳时间戳。 +时钟改用 monotonic,`warm_up` 排空左右全部上下文,不移动 Whisper 特征窗。 +取消仍销毁本轮 peer,补丁不改变前后端取消协议。 + +先用 `prepare_driver.py` 新建私有副本,再运行补丁;源为已选 c726943。 +已验证从 P10 干净副本首次及重复应用,拒绝直接修改 P10 复用目录。 +`driver_audio.py` 必须与 `patch_driver.py` 一起携带。回滚时停止自己的候选 +服务,重建全新 c726943 副本,使用 `git show ae4fd5c:samples/companion/tools/patch_driver.py` +导出的旧版补丁恢复 HEL-714 endpoint,再指向原 avatar/权重;不要原地混用新旧文件。 + +新增有效检查(使用原有 P10 venv,无新增依赖): + +```bash +/opt/hel715-verify/venv/bin/python tools/check_driver_audio.py /path/to/private/LiveTalking +# 按真实 RTP PTS 录制刚进入、空闲15秒、连续多轮、打断后再次送入相同音频 +/opt/hel715-verify/venv/bin/python tools/record_driver.py \ + --wav fixed.wav --out server-rounds.mp4 --rounds 0,15,1,1 --post 9 +``` + +`trace_driver.py` 仅用于隔离测试,以 `PYTHONPATH=私有驱动器目录` 从该目录运行, +首参为 JSONL 路径,其余为原 `app.py` 参数。记录逐块生产/接收 RMS、类型、深度、 +丢弃合成静音计数、PTS 和 monotonic 墙钟,不随产品默认开启。 +`record_graph.html/graph_server.py/record_graph.py` 复现浏览器产品音频图; +`record_product.py` 记录三轮日常→进场→交谈→说话中取消→正向回程→续读。 +`measure_sync.py` 固定 ROI 和阈值并保存检测器 SHA256、完整命令、offset、窗口、 +idle-until、输入/底板哈希、检测误差及逐 lag 相关性。相关性很低时不能把最佳 lag 当作可靠对齐。 + +本轮仍未达到服务端绝对偏差≤0.15秒/浏览器≤0.25秒,详见 `REPORT-HEL718.md`。 +本补丁解决的是可确认的空闲队列/时钟问题,不能据此宣称音画同步已修好。 diff --git a/samples/companion/REPORT-HEL718.md b/samples/companion/REPORT-HEL718.md new file mode 100644 index 0000000..62a72ff --- /dev/null +++ b/samples/companion/REPORT-HEL718.md @@ -0,0 +1,64 @@ +HEL-718 返工记录,2026-10-21。候选沿草稿 PR #2 / `hel-714-daily-talk`,base `main@1e9b08a`,施工基线 `ae4fd5c`。结论:**blocked,同步未修好,不发布、不合并、不转正式 PR。** + +## 实现与实测结论 + +- `tools/patch_driver.py`、`tools/driver_audio.py`:在私有 c726943 驱动器透传 `AudioFrameData.type`,未赋 PTS 的合成空闲块最多3块;真实语音(包括零 PCM)、自定义音频和事件不丢,100块总队列背压。音频饥饿补零保持320样本/20ms时钟;时钟使用 monotonic;warm_up 排空左右全部上下文。 +- 追加4项有效回归:2000次空闲/语音混入顺序与事件保留、真实音频满队列背压、warm_up 后首块关联、饥饿期间 PTS/节奏与首块 PCM 字节哈希。P10 干净副本首次及重复应用通过,旧取消 endpoint 保留。 +- 不改前端播放延迟、不移动 Whisper 特征窗、不改转场、素材、接口或结束检测。其他输出 transport 保持原来的 `push_audio_frame`,新增类型入口仅作用于 WebRTC。 +- 自测与分析工具纳入 `tools/`;复现和回滚步骤见 README 的 HEL-718 节。 + +静音积压与时钟问题已得到有效限制,**残余起点仍超门槛**。独立单会话约66秒:最大音频队列27块(含语音),删除合成静音1206块;四个完整轮次各338块真实音频均按顺序匹配到输出 RMS,首块排队19/37/9/75ms。RMS顺序匹配不是全 PCM 哈希证明,单元检查另核对首块原始字节;没有靠裁掉弱音/首音节改善数值。 + +## 固定口径与原始指标 + +固定音频“八百标兵奔北坡,炮兵并排北边跑。宝宝抱抱爸爸,妈妈慢慢说。”,xiaoyun,24kHz,6.76秒,SHA256 `8c0c1ec943882d4aafbb4f8cda488f138a19231dabe2a58335777c0c42fff3dc`。本轮直接复制审核留存文件,未重新合成。强度门槛0.015下,第一个20ms RMS达标块在源音频0.56秒,首次样本幅度达标0.169秒;审核所述11ms并非同一强度检测口径,不应直接与当前起点混用。真实语音开头的低强度部分仍全部保留。 + +所有主比较使用 `measure_sync.py` 的 `hel718-v2-mask-fix`,修正旧 source mask 的 x1 索引错误,并用新版对修前录像重新计算。ROI固定为512尺度的 `(271,341,180,220)`;服务端不裁切,浏览器/产品canvas裁切 `offset=(160,0), render-size=640` 后缩放512。服务端运动/残差门槛固定 `3.1718642664 / 7.693739183`;浏览器与产品固定 `4.294028549194336 / 10.492400400638578`。音频RMS门槛0.015。每个窗口 JSON 持久化完整命令、输入/底板哈希、检测器脚本哈希、ROI/offset/idle-until/start/stop、门槛、逐lag相关性;每个窗口各自保留CSV,避免被下一轮覆盖。 + +数值为口部起点减声音起点,**负号表示嘴提前**。合格意图为绝对值≤0.15秒(服务端)、≤0.25秒(浏览器)。视频帧量化约40ms,音频包约22ms,起点阈值/弱音/口型残差另有未定标误差,数值不替代真人听看。 + +| 实验 | 运动起点差(s) | 残差起点差(s) | 稳态最佳有符号lag(s) / 相关系数 | +| --- | ---: | ---: | ---: | +| 修前真 PTS | -0.576 | -0.576 | -0.06 / 0.209 | +| 修前浏览器图 | -0.699 | -0.665 | +0.52 / 0.231 | +| 修后真 PTS 第1轮 | -0.433 | -0.433 | -0.58 / 0.191 | +| 修后真 PTS 第2轮 | -0.320 | -0.320 | -0.44 / 0.189 | +| 修后真 PTS 第3轮 | -0.273 | -0.273 | -0.40 / 0.192 | +| 修后真 PTS 第4轮 | -0.313 | -0.313 | +0.34 / 0.173 | +| 修后真 PTS 第5轮 | -0.343 | -0.343 | +0.80 / 0.897 | +| 修后真 PTS 第6轮 | -0.296 | -0.296 | -0.44 / 0.339 | +| 修后浏览器图 第1轮 | -0.442 | -0.442 | -0.56 / 0.183 | +| 修后浏览器图 第2轮 | -0.360 | -0.360 | +0.32 / 0.171 | +| 修后浏览器图 第3轮 | -0.448 | -0.448 | -0.52 / 0.184 | +| 修后浏览器图 第4轮 | -0.324 | -0.324 | -0.40 / 0.206 | +| 修前产品端到端(不同短句) | -0.382 | -0.382 | -0.46 / 0.391 | +| 修后产品端到端首轮 | -0.507 | -0.507 | -0.26 / 0.203 | +| 修后产品端到端被打断轮 | -0.241 | -0.218 | -0.34 / 0.593 | + +服务端第1轮刚进入;第2轮前空闲15秒;第3/4轮连续播放后空闲1秒;第5轮在2秒处发 `/interrupt_talk`;第6轮同peer再次送固定音频(仅驱动诊断,不能替代产品取消隔离)。浏览器图覆盖前四个完整场景,产品回归另覆盖打断后新peer重入。修前固定音频只复用审核单轮原始录像,**未重建修前完整多轮矩阵**;此限制不能写成完整A/B验收已过。 + +产品端到端使用实际LLM/TTS,不以固定音频替换上游;首轮虽同文本,实际TTS为9.52秒,第二轮7.96秒,修前为1.96秒短句,**不能将这些行当成同音频因果对照**。受控同音频的因果比较以真PTS/浏览器图为准。 + +稳态lag使用运动能量与声音包络相关:定义 `lip_minus_audio = -shift`,修正旧xcorr脚本打印标签与相关计算方向相反的问题。四个完整受控轮次相关系数只有约0.17~0.21,最佳lag不稳定,**无法证明稳态音素同步或排除感知漂移**;第5轮只有截断短窗,其高相关与边界lag也不具验收意义。不能拿“某个最佳lag接近零”冒充通过。 + +音频PTS逐包20ms、视频40ms,无跳步;独立单会话末端墙钟减PTS分别约+1.07ms/+0.22ms;瞬时墙钟停顿最大约110/129ms,累计偏移峰值约90ms后回落。没有观察到持续时钟漂移,未机械重跑620秒。并行诊断曾有约1秒停顿(关闭其他peer同步join时),相应原始trace保留;主表使用独立单会话重测,不能把并发记录的卡顿抹掉。 + +## 回归、完整录像及未测项 + +- 前端构建通过;ruff通过;原78项pytest通过;5项网关检查通过;新增4项队列/时钟检查通过。日志见复核包 `checks/` 与 `remote/evidence/*checks.log`。 +- Chromium真实WebRTC回归通过:说话中取消、准备中取消、回日常、offer故障、缺素材、断网重连、快速取消再重入。快速重入首次与多轮采集并行时等待首声45秒超时,独立复跑通过,无pageerror或迟到旧轮错误;首轮未自动保存页面状态,不能追认已定位该偶发问题。 +- `product-continuous.webm` 约66.7秒:日常片7.84秒,录制后等待24.52秒(至少三轮)→f96切点→正向进场→两轮实际LLM/TTS→说话中按钮取消→独立正向回程→阅读续播。无pageerror;末态“已回到阅读原位置”。两轮首声8.49/7.91秒(本次样本,不是SLA)。取消后200~1000ms窗口最大RMS约0.0000041,低于0.015,无旧声复播证据。 +- 真人走查仍逐项待确认:日常循环接缝、两个切点、近景暖色、书本/手部/坐垫形变、微动作自然性、物理扬声器听感。没有用像素统计代替真人通过。 +- 真人自然抢话、手机真机/iOS/Safari、真实系统后台/锁屏、共享TTS内部GPU即停、多用户并发仍未测;桌面390px与synthetic visibility仅是处理器回归。 + +## 阻塞与最小后续路径 + +目前证据只能确认空闲积压受限、真实音频未被队列裁切、PTS连续;不能确认“积压修复足以消除同步故障”。残余服务端约−0.27~−0.43秒、浏览器约−0.32~−0.45秒,产品首轮约−0.51秒,均不足以放行。 + +建议下一步在同一固定音频的弱起音/闭唇音附近,保存每个推理批次的输入采样区间、Whisper特征区间和实际paste-back视频帧对应关系,以原始PCM对齐/人工音素标注核实残余来自特征关联还是起点检测;再由总工审定最小修法。不要盲加固定视频延迟、删除真实静音段或移动特征窗来压低指标。 + +## 隔离与交接 + +复用 c726943、MuseTalk v1.5 及原P10 venv/权重/avatar,无新依赖。工作仅在 `/opt/hel718-fix` 私有目录;18080/18081/9880/10095、系统Python/CUDA/驱动、证书信任、防火墙与开机配置未改。精确PID 1018154(驱动)、1018618(网关)、1098378(浏览器图代理)已核对cmd/cwd并在比较Multica daemon PID 44286后停止;8010/8014/8933已释放;共享服务PID 538167/608898/445461/220889保持。详见 `cleanup.json`。没有部署长期站点。 + +草稿 PR #2 沿原分支更新head,不另建PR。候选交付不等于样机通过;本卡保持blocked,由总管组织后续归因与真人检查。 diff --git a/samples/companion/tools/check_driver_audio.py b/samples/companion/tools/check_driver_audio.py new file mode 100644 index 0000000..20bdb91 --- /dev/null +++ b/samples/companion/tools/check_driver_audio.py @@ -0,0 +1,92 @@ +"""Run with the pinned driver venv: python check_driver_audio.py PRIVATE_DRIVER.""" + +import hashlib +import importlib.util +import pathlib +import sys +import time +import unittest +from types import SimpleNamespace + +import numpy as np + +ROOT = pathlib.Path(sys.argv.pop(1)).resolve() +sys.path.insert(0, str(ROOT)) +spec = importlib.util.spec_from_file_location('driver_webrtc', ROOT / 'server/webrtc.py') +driver = importlib.util.module_from_spec(spec) +spec.loader.exec_module(driver) +PendingAudio = driver.PendingAudio + + +class QueueChecks(unittest.TestCase): + def test_idle_is_bounded_speech_and_events_are_lossless(self): + pending = PendingAudio() + expected = [] + for i in range(2000): + if i % 100 == 0: + # Real zero PCM and custom audio are not synthetic silence. + item = (i, {'event': i} if i % 200 else {}, 0 if i % 200 else 2) + expected.append(item) + else: + item = (i, None, 1) + pending.put(item) + actual = list(pending.queue) + self.assertEqual([x for x in actual if x[2] != 1], expected) + self.assertEqual(sum(x[2] == 1 for x in actual), 3) + self.assertLessEqual(pending.high_water, len(expected) + 3) + self.assertGreater(pending.dropped_silence, 1900) + marker = ('event', {'end': True}, 1) + pending.put(marker) + for i in range(50): + pending.put((i, None, 1)) + self.assertIn(marker, list(pending.queue)) + + def test_real_audio_has_bounded_backpressure(self): + pending = PendingAudio() + for i in range(100): + pending.put((i, None, 0)) + import queue + with self.assertRaises(queue.Full): + pending.put((101, None, 0), block=False) + self.assertEqual([x[0] for x in pending.queue], list(range(100))) + + def test_warmup_drains_all_context(self): + from avatars.audio_features.base_asr import BaseASR + asr = BaseASR(SimpleNamespace(fps=25, batch_size=8, l=10, r=10)) + asr.warm_up() + self.assertEqual(len(asr.frames), 20) + self.assertTrue(asr.output_queue.empty()) + first = np.arange(320, dtype=np.float32) + asr.put_audio_frame(first, {'first': True}) + frame = asr.get_audio_frame() + np.testing.assert_array_equal(frame.data, first) + self.assertEqual(frame.type, 0) + self.assertEqual(frame.userdata, {'first': True}) + + +class ClockChecks(unittest.IsolatedAsyncioTestCase): + async def test_starvation_preserves_cadence_and_first_pcm(self): + player = SimpleNamespace(_start=lambda _: None, _stop=lambda _: None, notify=lambda _: None) + track = driver.PlayerStreamTrack(player, 'audio') + frame = driver.AudioFrame.from_ndarray(np.ones((1, 320), np.int16), format='s16', layout='mono') + frame.sample_rate = 16000 + track._queue.put((frame, None, 0)) + start = time.monotonic() + frames = [await track.recv() for _ in range(15)] + self.assertEqual([f.pts for f in frames], list(range(0, 15 * 320, 320))) + self.assertGreater(time.monotonic() - start, 0.25) + self.assertLess(time.monotonic() - start, 0.5) + self.assertTrue(all(not f.to_ndarray().any() for f in frames[1:])) + # Resume with exact first speech PCM, without trimming any syllable. + pcm = np.arange(320, dtype=np.int16).reshape(1, -1) + voice = driver.AudioFrame.from_ndarray(pcm, format='s16', layout='mono') + voice.sample_rate = 16000 + track._queue.put((voice, {'start': True}, 0)) + out = await track.recv() + self.assertEqual(out.pts, 15 * 320) + self.assertEqual(hashlib.sha256(out.to_ndarray().tobytes()).digest(), hashlib.sha256(pcm.tobytes()).digest()) + track.stop() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/samples/companion/tools/driver_audio.py b/samples/companion/tools/driver_audio.py new file mode 100644 index 0000000..5004997 --- /dev/null +++ b/samples/companion/tools/driver_audio.py @@ -0,0 +1,35 @@ +"""Pending audio before RTP timestamps: only type=1 synthetic idle is expendable.""" + +import queue + + +class PendingAudio(queue.Queue): + """Keep at most three un-timestamped synthetic blocks; preserve speech/events. + + One render producer, one RTP consumer. Real speech (including zero PCM and + custom type>1 audio) uses bounded Queue backpressure, never a drop policy. + """ + + silence_limit = 3 + + def __init__(self): + super().__init__(maxsize=100) + self.dropped_silence = 0 + self.high_water = 0 + + def put(self, item, block=True, timeout=None): + with self.not_full: + # Trim before assigning pts, never inside the RTP consumer. Event + # markers remain observable even if a producer puts one on silence. + idle = [i for i, (_, event, kind) in enumerate(self.queue) if kind == 1 and not event] + incoming_idle = item[2] == 1 and not item[1] + excess = len(idle) + int(incoming_idle) - self.silence_limit + for i in reversed(idle[:max(0, excess)]): + del self.queue[i] + self.unfinished_tasks -= 1 + self.dropped_silence += 1 + if excess > 0: + self.not_full.notify_all() + super().put(item, block=block, timeout=timeout) + with self.mutex: + self.high_water = max(self.high_water, len(self.queue)) diff --git a/samples/companion/tools/graph_server.py b/samples/companion/tools/graph_server.py new file mode 100644 index 0000000..331f0ba --- /dev/null +++ b/samples/companion/tools/graph_server.py @@ -0,0 +1,41 @@ +import argparse +import asyncio +from pathlib import Path + +import aiohttp +from aiohttp import web + +p = argparse.ArgumentParser() +p.add_argument('--wav', type=Path, required=True) +p.add_argument('--index', type=Path, required=True) +a = p.parse_args() +INDEX = a.index.read_bytes() +WAV = a.wav.read_bytes() + + +async def proxy(request): + data = await request.read() + async with aiohttp.ClientSession() as http, http.request( + request.method, + "http://127.0.0.1:8010" + request.path[len("/lt") :], + data=data if request.method == "POST" else None, + headers={"Content-Type": request.headers.get("Content-Type", "")} if request.method == "POST" else {}, + ) as r: + body = await r.read() + return web.Response(status=r.status, body=body, content_type=r.content_type) + + +async def main(): + app = web.Application() + app.router.add_get("/", lambda _: web.Response(body=INDEX, content_type="text/html")) + app.router.add_get("/fixed.wav", lambda _: web.Response(body=WAV, content_type="audio/wav")) + app.router.add_route("*", "/lt/{tail:.*}", proxy) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "0.0.0.0", 8933) + await site.start() + print("serving on 8933") + await asyncio.Event().wait() + + +asyncio.run(main()) diff --git a/samples/companion/tools/measure_sync.py b/samples/companion/tools/measure_sync.py new file mode 100644 index 0000000..c91acdc --- /dev/null +++ b/samples/companion/tools/measure_sync.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +"""HEL-717 onset analysis: audio RMS onset vs lip-motion onset, multiple detectors. + +Works on any muxed recording (browser webm or true-pts mp4). Mouth ROI is +derived from the recording itself (temporal variance), coordinates are saved. +""" +import argparse +import csv +import hashlib +import json +import sys +from pathlib import Path + +import av +import cv2 +import numpy as np + +p = argparse.ArgumentParser() +p.add_argument("video", type=Path) +p.add_argument("--source", type=Path, default=Path("/opt/hel714-sample/dist/assets/talk/gaze.mp4")) +p.add_argument("--size", type=int, default=512) +p.add_argument("--idle", type=float, required=True, help="seconds of pre-speech idle to calibrate") +p.add_argument("--roi", help="x0,x1,y0,y1 in pixels at --size; default derive from variance") +p.add_argument("--strip", type=Path, help="write a contact strip of the ROI around onset") +p.add_argument("--offset-x", type=int, default=0) +p.add_argument("--offset-y", type=int, default=0) +p.add_argument("--render-size", type=int, default=0, help="crop window size before resize (canvas recordings)") +p.add_argument("--start", type=float, default=0) +p.add_argument("--stop", type=float, default=100000) +p.add_argument("--out", type=Path) +p.add_argument("--idle-until", type=float) +p.add_argument("--motion-threshold", type=float) +p.add_argument("--residual-threshold", type=float) +a = p.parse_args() + +frames = [] +rs = a.render_size or None +for f in av.open(str(a.video)).decode(video=0): + if not a.start <= float(f.time) <= a.stop: + continue + img = f.to_ndarray(format="bgr24") + if rs: + img = img[a.offset_y : a.offset_y + rs, a.offset_x : a.offset_x + rs] + img = cv2.resize(img, (a.size, a.size)) + frames.append((float(f.time), img)) +audio = [] +for f in av.open(str(a.video)).decode(audio=0): + if not a.start <= float(f.time) <= a.stop: + continue + data = f.to_ndarray().astype(np.float64) + if data.max() > 1: + data /= 32768 + audio.append([float(f.time), float(np.sqrt((data**2).mean()))]) + +stack = np.stack([img for _, img in frames]).astype(np.float32) +std = stack.std(axis=0).mean(axis=2) +if a.roi: + roi = tuple(int(v) for v in map(float, a.roi.split(","))) + print("using explicit mouth ROI (x0,x1,y0,y1):", roi) +else: + region = np.zeros_like(std, dtype=bool) + h = a.size + region[int(h * 0.35) : int(h * 0.80), int(h * 0.25) : int(h * 0.75)] = True + stdm = np.where(region, std, 0) + # smooth, then take the top-percentile blob + k = np.ones((15, 15), np.float32) / 225 + stds = cv2.filter2D(stdm, -1, k) + thresh = np.quantile(stds[stds > 0], 0.995) + ys, xs = np.where(stds >= thresh) + x0, x1, y0, y1 = int(xs.min()), int(xs.max()), int(ys.min()), int(ys.max()) + pad = 6 + roi = (max(0, x0 - pad), min(a.size, x1 + pad), max(0, y0 - pad), min(a.size, y1 + pad)) + print("derived mouth ROI (x0,x1,y0,y1):", roi, "fractions:", [v / a.size for v in roi]) + +# --- detector 1: frame-to-frame motion inside ROI (independent of source board) +motion = [] +for i in range(1, len(frames)): + t0, i0 = frames[i - 1] + t1, i1 = frames[i] + m = np.abs(i1[roi[2] : roi[3], roi[0] : roi[1]].astype(np.float32) - i0[roi[2] : roi[3], roi[0] : roi[1]].astype(np.float32)).mean() + motion.append([t1, float(m)]) + +# --- detector 2: source-matching residual (same as construction analyzer) +source = [cv2.resize(f.to_ndarray(format="bgr24"), (a.size, a.size)) for f in av.open(str(a.source)).decode(video=0)] +mask = np.ones((48, 48), dtype=bool) +mask[0:3] = False +mask[43:] = False +mr = np.zeros((48, 48), dtype=bool) +my0, my1 = int(roi[2] * 48 / a.size), int(roi[3] * 48 / a.size) +mx0, mx1 = int(roi[0] * 48 / a.size), int(roi[1] * 48 / a.size) +mr[my0:my1, mx0:mx1] = True +mask = mask & ~mr +feats = np.asarray([cv2.resize(f, (48, 48))[mask].astype(np.float32) for f in source]) +residuals = [] +for t, img in frames: + feat = cv2.resize(img, (48, 48))[mask].astype(np.float32) + idx = int(np.argmin(((feats - feat) ** 2).mean(axis=(1, 2)))) + res = np.abs(img.astype(np.float32) - source[idx]).mean(axis=2) + residuals.append([t, float(res[roi[2] : roi[3], roi[0] : roi[1]].mean()), idx]) + +first_audio = next((t for t, r in audio if r > 0.015), None) + + +def onset(rows, idle_end, floor): + base = [m for t, m, * _ in [tuple(r) for r in rows] if t <= idle_end] + if not base: + return None, None + th = max(floor, float(np.quantile(base, 0.99)) * 1.5) + hit = next((t for t, m, *_ in [tuple(r) for r in rows] if m > th and t > idle_end), None) + return hit, th + + +idle_end = a.idle_until if a.idle_until is not None else (first_audio - 0.7 if first_audio else a.idle) +lip_motion_onset, motion_th = onset(motion, idle_end, 1.2) +if a.motion_threshold is not None: + motion_th = a.motion_threshold + lip_motion_onset = next((t for t, m in motion if t > idle_end and m > motion_th), None) +lip_residual_onset, residual_th = onset(residuals, idle_end, 4) +if a.residual_threshold is not None: + residual_th = a.residual_threshold + lip_residual_onset = next((t for t, m, _ in residuals if t > idle_end and m > residual_th), None) + +result = { + "detector_version": "hel718-v2-mask-fix", + "detector_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "command": [sys.executable, *sys.argv], + "parameters": {k: str(v) if isinstance(v, Path) else v for k, v in vars(a).items()}, + "effective_idle_until": idle_end, + "audio_threshold": 0.015, + "input_sha256": hashlib.sha256(a.video.read_bytes()).hexdigest(), + "source_sha256": hashlib.sha256(a.source.read_bytes()).hexdigest(), + "detection_uncertainty_seconds": {"video": 0.04, "audio": 0.022}, + "video": str(a.video), + "frames": len(frames), + "audio_packets": len(audio), + "first_audio": first_audio, + "lip_motion_onset": lip_motion_onset, + "lip_motion_threshold": motion_th, + "motion_delta_seconds": (lip_motion_onset - first_audio) if lip_motion_onset and first_audio else None, + "lip_residual_onset": lip_residual_onset, + "lip_residual_threshold": residual_th, + "residual_delta_seconds": (lip_residual_onset - first_audio) if lip_residual_onset and first_audio else None, + "derived_roi": [int(v) for v in roi], + "roi_fractions": [round(v / a.size, 4) for v in roi], + "note": "delta<0 means lips move before sound; ROI derived from temporal variance, coordinates saved", +} +# Same envelope detector for each window; positive result means lips later. +if motion and audio: + grid = np.arange(max(motion[0][0], audio[0][0]), min(motion[-1][0], audio[-1][0]), 0.02) + m = np.interp(grid, np.asarray(motion)[:, 0], np.asarray(motion)[:, 1]) + au = np.interp(grid, np.asarray(audio)[:, 0], np.asarray(audio)[:, 1]) + m = np.convolve(m, np.ones(5) / 5, 'same') + au = np.convolve(au, np.ones(5) / 5, 'same') + active = grid[au > 0.02] + if len(active): + selected = (grid >= active.min() - 0.8) & (grid <= active.max() + 0.8) + m, au = m[selected], au[selected] + correlations = [] + for shift in range(-40, 41): + x = m[:len(m)-shift or None] if shift >= 0 else m[-shift:] + y = au[shift:] if shift >= 0 else au[:len(au)+shift] + if len(x) > 2 and np.std(x) > 0 and np.std(y) > 0: + correlations.append({"lip_minus_audio_seconds": -shift * 0.02, + "correlation": float(np.corrcoef(x, y)[0, 1])}) + result["steady_envelope"] = {"best": max(correlations, key=lambda c: c["correlation"]), + "all_lags": correlations, + "smooth_seconds": 0.1, "active_rms_threshold": 0.02, + "caveat": "motion energy heuristic, not phoneme alignment"} +out = a.out or a.video.with_suffix(".onset.json") +out.write_text(json.dumps(result, indent=2)) + +if a.strip and first_audio: + # contact strip: frames every 100ms from first_audio-1.2s to +1.6s + times = [first_audio - 1.2 + 0.1 * i for i in range(30)] + crops = [] + for tt in times: + best = min(frames, key=lambda fr: abs(fr[0] - tt))[1] + crop = best[roi[2] - 20 : roi[3] + 20, roi[0] - 30 : roi[1] + 30] + crops.append(cv2.resize(crop, (120, 80))) + rows = [np.hstack(crops[i : i + 10]) for i in range(0, 30, 10)] + cv2.imwrite(str(a.strip), np.vstack(rows)) + print("strip:", a.strip) +with out.with_suffix(".csv").open("w") as f: + w = csv.writer(f) + w.writerow(["time_seconds", "motion", "lip_residual", "source_index", "audio_rms"]) + mmap = {t: m for t, m in motion} + amap = {t: r for t, r in audio} + for t, res, idx in residuals: + near_audio = min(audio, key=lambda x: abs(x[0] - t))[1] if audio else "" + near_motion = min(motion, key=lambda x: abs(x[0] - t))[1] if motion else "" + w.writerow([t, near_motion, res, idx, near_audio]) +print(json.dumps(result, indent=2)) diff --git a/samples/companion/tools/patch_driver.py b/samples/companion/tools/patch_driver.py index da24c6d..8603237 100644 --- a/samples/companion/tools/patch_driver.py +++ b/samples/companion/tools/patch_driver.py @@ -45,3 +45,75 @@ if replacement not in s: assert s.count(needle) >= 1 p.write_text(s.replace(needle, replacement, 1)) print("Installed HEL-714 peer retirement endpoint in private copy:", root) + +# HEL-718: all edits remain reproducible on a clean pinned driver copy. +def replace_once(path, before, after): + text = path.read_text() + if after in text: + return + assert text.count(before) == 1, f"Unexpected c726943 source: {path}" + path.write_text(text.replace(before, after, 1)) + + +(root / "server/hel718_audio.py").write_text( + Path(__file__).with_name("driver_audio.py").read_text() +) +p = root / "server/webrtc.py" +replace_once(p, "import queue\n", "import queue\nfrom server.hel718_audio import PendingAudio\n") +replace_once(p, "self._queue = queue.Queue(maxsize=100)", + "self._queue = PendingAudio() if kind == 'audio' else queue.Queue(maxsize=100)") +# Preserve the initial common audio/video origin. Once started, pace BEFORE +# selecting audio, so fresh speech cannot wait behind a dequeued idle block. +s = p.read_text() +start = s.find(" # frame = self.frames[self.counter % 30]") +end = s.index(" frame.pts = pts", max(0, start)) +replacement = ''' self._player._start(self) + if self.kind == 'audio' and hasattr(self, '_timestamp'): + pts, time_base = await self.next_timestamp() + try: + frame, eventpoint, _ = self._queue.get_nowait() + except queue.Empty: + frame = AudioFrame.from_ndarray( + np.zeros((1, 320), dtype=np.int16), layout='mono', format='s16') + frame.sample_rate = SAMPLE_RATE + eventpoint = None + else: + while self.readyState == 'live': + try: + frame, eventpoint, *_ = self._queue.get_nowait() + break + except queue.Empty: + await asyncio.sleep(0.005) + else: + raise Exception('Track stopped') + pts, time_base = await self.next_timestamp() +''' +if "np.zeros((1, 320), dtype=np.int16), layout='mono'" not in s: + assert start >= 0, "Unexpected c726943 recv implementation" + p.write_text(s[:start] + replacement + s[end:]) +replace_once(p, "def push_audio(self, frame, eventpoint=None):", + "def push_audio(self, frame, eventpoint=None, audio_type=0):") +replace_once(p, "self.__audio._queue.put((new_frame, eventpoint))", + "self.__audio._queue.put((new_frame, eventpoint, audio_type))") +# Wall-clock adjustments must not change packet cadence. +s = p.read_text() +start, end = s.index(" async def next_timestamp"), s.index(" async def recv") +s = s[:start] + s[start:end].replace("time.time()", "time.monotonic()") + s[end:] +p.write_text(s) + +p = root / "streamout/webrtc.py" +replace_once(p, " def get_buffer_size(self) -> int:", ''' def push_audio_data(self, frame, eventpoint, audio_type): + if self._player: + self._player.push_audio(frame, eventpoint, audio_type) + + def get_buffer_size(self) -> int:''') +p = root / "avatars/base_avatar.py" +replace_once(p, " self.output.push_audio_frame(frame, audio_frame.userdata)", +''' if hasattr(self.output, 'push_audio_data'): + self.output.push_audio_data(frame, audio_frame.userdata, audio_frame.type) + else: + self.output.push_audio_frame(frame, audio_frame.userdata)''') +p = root / "avatars/audio_features/base_asr.py" +replace_once(p, " for _ in range(self.stride_left_size):\n self.output_queue.get()", + " for _ in range(self.stride_left_size + self.stride_right_size):\n self.output_queue.get()") +print("Installed HEL-718 bounded synthetic idle, continuous RTP audio, and warm-up drain") diff --git a/samples/companion/tools/record_driver.py b/samples/companion/tools/record_driver.py new file mode 100644 index 0000000..d9ab76b --- /dev/null +++ b/samples/companion/tools/record_driver.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +"""HEL-717 diagnostic: record the driver's own WebRTC output with TRUE RTP pts. + +No browser involved. If the lips-early offset appears here, it originates +upstream (feature lookahead / frame association); if not, it is introduced by +the browser playback/recording graph. +""" +import argparse +import asyncio +import hashlib +import json +import sys +import time +from fractions import Fraction +from pathlib import Path + +import aiohttp +import av +import numpy as np +from aiortc import RTCPeerConnection, RTCSessionDescription + +p = argparse.ArgumentParser() +p.add_argument("--base", default="http://127.0.0.1:8010") +p.add_argument("--wav", required=True) +p.add_argument("--out", required=True) +p.add_argument("--post", type=float, default=9.0) +p.add_argument("--rounds", default="0,15,1,1", help="Idle seconds before each full utterance; then interrupt and restart") +a = p.parse_args() + +events = [] + + +def log(name, **kw): + row = {"t": time.time(), "rel": time.perf_counter(), "event": name, **kw} + events.append(row) + print(json.dumps({k: v for k, v in row.items() if k != "t"}), flush=True) + + +async def main(): + pc = RTCPeerConnection() + container = av.open(a.out, "w") + vstream = container.add_stream("libx264", rate=25) + vstream.time_base = Fraction(1, 90000) + vstream.pix_fmt = "yuv420p" + vstream.options = {"crf": "18", "preset": "veryfast"} + astream = container.add_stream("aac", rate=48000) + astream.time_base = Fraction(1, 48000) + astream.layout = "mono" + + stats = {"v": 0, "a": 0} + timeline = [] + + def video_track(track): + async def loop(): + while True: + try: + frame = await track.recv() + except Exception: + break + stats["v"] += 1 + timeline.append({"kind": "v", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter()}) + img = frame.to_ndarray(format="bgr24") + out = av.VideoFrame.from_ndarray(img, format="bgr24") + out.pts = frame.pts + out.time_base = frame.time_base + for pkt in vstream.encode(out): + container.mux(pkt) + return asyncio.ensure_future(loop()) + + def audio_track(track): + async def loop(): + while True: + try: + frame = await track.recv() + except Exception: + break + stats["a"] += 1 + data = frame.to_ndarray() + # aiortc delivers s16 mono 48k planar [1, n] + pcm = data.reshape(-1).astype(np.int16) + timeline.append({"kind": "a", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter(), + "rms": float(np.sqrt((pcm.astype(np.float64) / 32768.0) ** 2).mean())}) + out = av.AudioFrame.from_ndarray(pcm.reshape(1, -1), format="s16", layout="mono") + out.sample_rate = frame.sample_rate + out.pts = frame.pts + out.time_base = frame.time_base + for pkt in astream.encode(out): + container.mux(pkt) + return asyncio.ensure_future(loop()) + + @pc.on("track") + def on_track(track): + log("track", kind=track.kind) + if track.kind == "video": + video_track(track) + else: + audio_track(track) + + pc.addTransceiver("video", direction="recvonly") + pc.addTransceiver("audio", direction="recvonly") + offer = await pc.createOffer() + await pc.setLocalDescription(offer) + for _ in range(100): + if pc.iceGatheringState == "complete": + break + await asyncio.sleep(0.05) + async with aiohttp.ClientSession() as http: + async with http.post(a.base + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r: + answer = await r.json() + sid = answer["sessionid"] + log("offer-ok", sid=sid) + await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"])) + for _ in range(200): + if stats["v"]: + break + await asyncio.sleep(0.05) + log("first-video", v=stats["v"], a=stats["a"]) + wav_bytes = Path(a.wav).read_bytes() + async def send(label): + log("send", label=label, v=stats["v"], a=stats["a"]) + form = aiohttp.FormData() + form.add_field("sessionid", sid) + form.add_field("file", wav_bytes, filename="utterance.wav", content_type="audio/wav") + async with http.post(a.base + "/humanaudio", data=form) as r: + log("humanaudio-ok", label=label, body=await r.json()) + for index, idle in enumerate(map(float, a.rounds.split(","))): + await asyncio.sleep(idle) + await send(f"full-{index+1}-idle-{idle}") + await asyncio.sleep(a.post) + await send("interrupted") + await asyncio.sleep(2) + async with http.post(a.base + "/interrupt_talk", json={"sessionid": sid}) as r: + log("interrupt", body=await r.json()) + await asyncio.sleep(1) + await send("restart-same-peer-diagnostic") + await asyncio.sleep(a.post) + await pc.close() + async with http.post(a.base + "/hel714/close", json={"sessionid": sid}) as r: + log("closed", body=await r.json()) + for pkt in vstream.encode(): + container.mux(pkt) + for pkt in astream.encode(): + container.mux(pkt) + container.close() + Path(a.out + ".timeline.json").write_text(json.dumps({"command": [sys.executable, *sys.argv], "wav_sha256": hashlib.sha256(Path(a.wav).read_bytes()).hexdigest(), "events": events, "timeline": timeline})) + log("done", v=stats["v"], a=stats["a"]) + + +asyncio.run(main()) diff --git a/samples/companion/tools/record_graph.html b/samples/companion/tools/record_graph.html new file mode 100644 index 0000000..30faac9 --- /dev/null +++ b/samples/companion/tools/record_graph.html @@ -0,0 +1,89 @@ + + +HEL-717 remote-stream graph + + + + + + diff --git a/samples/companion/tools/record_graph.py b/samples/companion/tools/record_graph.py new file mode 100644 index 0000000..6c2bd4f --- /dev/null +++ b/samples/companion/tools/record_graph.py @@ -0,0 +1,32 @@ +import asyncio +import json +import sys + +from playwright.async_api import async_playwright + +OUT = sys.argv[1] if len(sys.argv) > 1 else "." + + +async def main(): + async with async_playwright() as pw: + browser = await pw.chromium.launch( + headless=True, + args=["--autoplay-policy=no-user-gesture-required", "--no-sandbox"], + ) + page = await browser.new_page(viewport={"width": 1280, "height": 800}) + await page.goto("http://192.168.200.12:8933/") + await page.evaluate("diagRun()") + await page.wait_for_function("document.title === 'done'", timeout=180000) + from pathlib import Path + out = Path(OUT) + out.mkdir(parents=True, exist_ok=True) + events = await page.evaluate("window.diagResult.events") + async with page.expect_download() as pending: + await page.evaluate("""() => {const a=document.createElement('a'); a.href=window.diagResult.blobUrl;a.download='graph.webm';a.click()}""") + await (await pending.value).save_as(out / 'graph-after.webm') + (out / 'graph-after.events.json').write_text(json.dumps(events, indent=2)) + print('saved graph audio/video', flush=True) + await browser.close() + + +asyncio.run(main()) diff --git a/samples/companion/tools/record_product.py b/samples/companion/tools/record_product.py new file mode 100644 index 0000000..3d175ef --- /dev/null +++ b/samples/companion/tools/record_product.py @@ -0,0 +1,59 @@ +"""Continuous real product recording: three daily loops, talk, cancel, return.""" + +import asyncio +import json +import pathlib +import sys + +from playwright.async_api import async_playwright + +OUT = pathlib.Path(sys.argv[1]) +OUT.mkdir(parents=True, exist_ok=True) + + +async def main(): + async with async_playwright() as pw: + browser = await pw.chromium.launch(args=['--no-sandbox', '--autoplay-policy=no-user-gesture-required']) + context = await browser.new_context(ignore_https_errors=True, viewport={'width': 1280, 'height': 800}) + page = await context.new_page() + errors = [] + page.on('pageerror', lambda error: errors.append(str(error))) + await page.goto('https://192.168.200.12:8014') + await page.wait_for_function("document.querySelector('#status').textContent.includes('就绪')") + await page.click('#record') + duration = await page.locator('#daily').evaluate('(v)=>v.duration') + await page.wait_for_timeout((duration * 3 + 1) * 1000) + await page.click('#enter') + await page.wait_for_function("document.querySelector('#status').textContent.includes('正在听')", timeout=40000) + await page.fill('#text', '请原样念这一句,不添加内容:八百标兵奔北坡,炮兵并排北边跑。宝宝抱抱爸爸,妈妈慢慢说。') + await page.click('button[type=submit]') + await page.wait_for_function("window.sampleEvents.some(e=>e.event==='first-audible')", timeout=60000) + await page.wait_for_timeout(8000) + # Actual second turn is cancelled while speaking. + await page.fill('#text', '请再念一遍:八百标兵奔北坡,炮兵并排北边跑。宝宝抱抱爸爸,妈妈慢慢说。') + await page.click('button[type=submit]') + await page.wait_for_function("window.sampleEvents.filter(e=>e.event==='first-audible').length>=2", timeout=60000) + await page.wait_for_timeout(1200) + await page.click('#cancel') + await page.wait_for_timeout(1200) + await page.click('#back') + await page.wait_for_function("document.querySelector('#status').textContent.includes('已回到阅读')", timeout=20000) + await page.wait_for_timeout(6000) + assert not await page.locator('#daily').evaluate('(v)=>v.paused') + await page.click('#record') + await page.wait_for_selector('#download:not([hidden])') + async with page.expect_download() as pending: + await page.click('#download') + await (await pending.value).save_as(OUT / 'product-continuous.webm') + (OUT / 'product-continuous.json').write_text(json.dumps({ + 'command': [sys.executable, *sys.argv], 'daily_duration': duration, + 'daily_recording_seconds_before_enter': duration * 3 + 1, + 'events': await page.evaluate('window.sampleEvents'), 'pageerrors': errors, + 'final_status': await page.locator('#status').inner_text(), + 'human_review': 'pending: seams, cuts, warmth, book/hands/cushion; real barge-in, phone, OS background untested', + }, ensure_ascii=False, indent=2)) + await browser.close() + print('Saved continuous product audio/video', flush=True) + + +asyncio.run(main()) diff --git a/samples/companion/tools/trace_driver.py b/samples/companion/tools/trace_driver.py new file mode 100644 index 0000000..9226a3b --- /dev/null +++ b/samples/companion/tools/trace_driver.py @@ -0,0 +1,40 @@ +"""Optional test-only trace wrapper; pass app.py arguments after the trace path.""" + +import json +import runpy +import sys +import time + +import numpy as np +from server import webrtc + +out = open(sys.argv.pop(1), 'w', buffering=1) # noqa: SIM115 -- process-lifetime trace + + +def write(event, player, **fields): + out.write(json.dumps({'wall': time.monotonic(), 'event': event, 'player': id(player), **fields}) + '\n') + + +push = webrtc.HumanPlayer.push_audio +recv = webrtc.PlayerStreamTrack.recv + + +def push_audio(self, frame, eventpoint=None, audio_type=0): + push(self, frame, eventpoint, audio_type) + q = self.audio._queue + write('push', self, kind=audio_type, rms=float(np.sqrt(np.mean((frame.astype(float) / 32768) ** 2))), + depth=q.qsize(), dropped=q.dropped_silence, high_water=q.high_water) + + +async def receive(self): + frame = await recv(self) + fields = {'kind': self.kind, 'pts': frame.pts, 'tb': str(frame.time_base), 'depth': self._queue.qsize()} + if self.kind == 'audio': + fields['rms'] = float(np.sqrt(np.mean((frame.to_ndarray().astype(float) / 32768) ** 2))) + write('recv', self._player, **fields) + return frame + + +webrtc.HumanPlayer.push_audio = push_audio +webrtc.PlayerStreamTrack.recv = receive +runpy.run_path('app.py', run_name='__main__') -- 2.54.0 From 77b76083d05536893744d50ccf2a01c4a3c85a27 Mon Sep 17 00:00:00 2001 From: mobai Date: Wed, 21 Oct 2026 02:03:02 +0800 Subject: [PATCH 5/6] fix(HEL-720): gate only leading quiet PCM render frames Co-authored-by: multica-agent --- samples/companion/README.md | 38 +++++++ samples/companion/REPORT-HEL720.md | 84 +++++++++++++++ samples/companion/tools/check_driver_head.py | 103 +++++++++++++++++++ samples/companion/tools/check_races.py | 71 +++++++------ samples/companion/tools/driver_head.py | 42 ++++++++ samples/companion/tools/patch_driver.py | 27 +++++ samples/companion/tools/trace_head.py | 80 ++++++++++++++ 7 files changed, 416 insertions(+), 29 deletions(-) create mode 100644 samples/companion/REPORT-HEL720.md create mode 100644 samples/companion/tools/check_driver_head.py create mode 100644 samples/companion/tools/driver_head.py create mode 100644 samples/companion/tools/trace_head.py diff --git a/samples/companion/README.md b/samples/companion/README.md index b212710..3770a1f 100644 --- a/samples/companion/README.md +++ b/samples/companion/README.md @@ -115,3 +115,41 @@ idle-until、输入/底板哈希、检测误差及逐 lag 相关性。相关性 本轮仍未达到服务端绝对偏差≤0.15秒/浏览器≤0.25秒,详见 `REPORT-HEL718.md`。 本补丁解决的是可确认的空闲队列/时钟问题,不能据此宣称音画同步已修好。 + +## HEL-720 头部闭唇候选(阈值待真人确认,仍未达同步门槛) + +`patch_driver.py` 携带 `driver_head.py`,安装到私有驱动的 +`avatars/hel720_head.py`。在 `BaseASR.put_audio_frame` 的原始16k PCM入口逐块 +标记头部闭唇;20ms块不修改、不改类型或事件,仍进入原特征及RTP路径。 +首个RMS超过候选值的块结束本话语的门控,其后弱音保留口型。 +`start`/`end`事件和`flush_talk`重置状态,新peer各有独立状态;合成空闲不重新 +开启头部规则。标记随原`AudioFrameData`传递,旧帧的决定不会因新话语重置改变。 + +每个视频帧关联两块PCM;仅当对应两块均不需口型时走既有闭唇底板。 +混合批次仍推理全部8帧,再逐帧选择底板/预测结果,不以首块决定整批画面。 +自定义音频的类型与底板选择、真实语音的`is_speaking`及事件语义保留。 +Whisper的`[2,3]`特征窗、队列、时钟、转场及前后端取消协议不变。 + +候选进程启动环境变量 `HEL720_HEAD_RMS=0.001` 为默认试验值;`0`关闭规则。 +负值或非有限值拒绝启动会话。不要把该值写入共享服务环境。 +固定音频0.16秒处RMS约0.01333,规则已在这里结束,不能继续压制其后弱段。 +阈值未经真人听看批准,本轮未继续调参。详见 `REPORT-HEL720.md`。 + +```bash +# 原有P10 venv;不装新依赖,PRIVATE_DRIVER必须为新建私有副本 +python3 tools/prepare_driver.py --source /path/to/clean-c726943 --destination "$PRIVATE_DRIVER" +python3 tools/patch_driver.py "$PRIVATE_DRIVER" +python3 tools/patch_driver.py "$PRIVATE_DRIVER" # 幂等重复应用 +/opt/hel715-verify/venv/bin/python tools/check_driver_head.py "$PRIVATE_DRIVER" +/opt/hel715-verify/venv/bin/python tools/check_driver_audio.py "$PRIVATE_DRIVER" +``` + +`trace_head.py`仅在隔离诊断时显式调用,参数与`trace_driver.py`一致;记录原始 +PCM哈希/RMS/门控、两块对应的渲染帧及RTP输出哈希/PTS,不由产品默认加载。 +`check_races.py [证据目录]`在失败时也保留页面状态、事件和截图,仍不把一次通过 +视为已定位历史超时。 + +回滚:停止自己的候选进程,新建同一c726943私有副本,使用 +`git show 415f837:samples/companion/tools/patch_driver.py`和同提交的 +`driver_audio.py`重装HEL-718补丁,复用原avatar/权重。也可在候选启动前设 +`HEL720_HEAD_RMS=0`关闭策略;该开关回滚保留代码但不闭唇。不要原地覆盖共享安装。 diff --git a/samples/companion/REPORT-HEL720.md b/samples/companion/REPORT-HEL720.md new file mode 100644 index 0000000..d6a9607 --- /dev/null +++ b/samples/companion/REPORT-HEL720.md @@ -0,0 +1,84 @@ +HEL-720 最小返工记录,2026-10-21。施工基线 `hel-714-daily-talk@415f837`,原草稿 PR #2,base `main@1e9b08a`。**结论:blocked;已完成一轮限定修复和验证,指标仍未全过,不发布、不合并。0.001候选阈值待真人听看确认。** + +## 实现、粒度与边界 + +- `tools/patch_driver.py`、`driver_head.py`:在ASR入口对原始16k/320样本/20ms块计算RMS并携带独立渲染标记。`type=0`、PCM内容、事件原样保留;未用合成静音类型绕过HEL-718队列保护。 +- 每个视频帧对应`audio_frames[i*2:i*2+2]`,每个批次8帧/16块。混合批次照常推理,仅把应闭唇的单帧预测替换为既有底板,不整批闭嘴。全闭唇批次复用既有跳过推理分支。画面选择仍沿原三元组队列,不按墙钟门控。 +- 状态在输入端顺序计算,`start`重启头部、首个超过门槛的块锁定开口、`end`/`flush_talk`重置;新peer有独立实例。合成空闲不重启话语,避免流式饥饿后吞掉弱音。标记写在帧对象上,重置不追改旧帧。产品取消仍销毁peer,本卡未另造轮次隔离。 +- 保留自定义音频类型和静态视频选择;画面闭唇期间真实音频的speaking/通知语义保持。可配置环境变量`HEL720_HEAD_RMS`,默认0.001,0关闭,非有限值/负值拒绝。 +- `check_driver_head.py`检查边界、禁用、混合先弱后强、话语中弱音、事件、取消/新peer及实际推理→输出循环的PCM字节/类型/事件保留。`trace_head.py`是显式调用的诊断工具。`check_races.py`新增无论成功失败都保存页面状态/事件/截图。 +- 未改窗口、队列、时钟、视频裁剪、固定延迟、transition、检测器或UI/素材。没有新依赖。 + +Whisper源码粒度核对:`run_step`每次追加16块,保留`l+r=20`块上下文;`_feature2chunks`的`start=l/2=5`、`feature_idx_multiplier=2`、`audio_feat_win=[2,3]`,特征索引窗口为`[2*i+6, 2*i+16)`。本轮仅在同一批次的每帧对应PCM对上选择画面,未改变这些参数。**没有逐特征张量转储,特征窗口说明来自固定驱动源码;实录trace直接覆盖PCM→渲染对→RTP,不能据此声称完整音素关联已验证。** + +## 同固定音频的逐轮结果 + +沿用原fixed.wav,24kHz、6.76秒,SHA256 `8c0c1ec943882d4aafbb4f8cda488f138a19231dabe2a58335777c0c42fff3dc`。没有重新合成或裁掉头部。与HEL-718同`hel718-v2-mask-fix`检测器,脚本SHA256 `081e40486793d9c57dbd9d682aacbefe58670eb07a9b22675953cd6b883f0ec6`,ROI `(271,341,180,220)`;服务端不裁切,浏览器/产品`offset=(160,0), render-size=640`后缩放512。服务端motion/residual门槛 `3.1718642664/7.693739183`,浏览器 `4.294028549194336/10.492400400638578`,音频RMS门槛0.015,均未放宽。 + +所有差值为“口部起点-相应声音起点”,秒,负号为嘴提前。主验收是响亮RMS起点绝对差服务端≤0.15、浏览器≤0.25;另列弱瞬态的采样幅度起点和原始PCM首块口径,不能互换。样本幅度检测是独立辅助报告,非主检测器变更。原音频首次样本幅度绝对值>0.015在0.169208秒,首次20ms RMS>0.015在0.56秒;原PCM首块包含此前全部弱头。 + +| 路径/轮 | HEL-718响亮差 | 本轮响亮差 | 本轮残差差 | 样本幅度起点差 | 原PCM首块差 | +| --- | ---: | ---: | ---: | ---: | ---: | +| server/1 | -0.433 | -0.640 | -0.640 | -0.288 | -0.080 | +| server/2 | -0.320 | -0.159 | -0.159 | +0.212 | +0.420 | +| server/3 | -0.273 | -0.199 | -0.199 | +0.252 | +0.440 | +| server/4 | -0.313 | -0.068 | -0.068 | +0.312 | +0.500 | +| server/5 | -0.343 | -0.183 | -0.183 | +0.172 | +0.580 | +| server/6 | -0.296 | -0.103 | -0.103 | +0.252 | +0.720 | +| graph/1 | -0.442 | -0.350 | -0.350 | +0.024 | 未标定 | +| graph/2 | -0.360 | -0.307 | -0.307 | +0.068 | 未标定 | +| graph/3 | -0.448 | -0.220 | -0.220 | +0.153 | 未标定 | +| graph/4 | -0.324 | -0.208 | -0.208 | +0.166 | 未标定 | +| product/1 | 未标定 | -0.314 | -0.314 | +0.019 | 未标定 | +| product/2 | 未标定 | -0.322 | -0.288 | -0.318 | 未标定 | +0 3 + +server 1刚进入、2空闲15秒后、3/4连续轮次、5在2秒处打断(非完整轮)、6同peer重新送音频诊断。graph覆盖前四场景。product为实际LLM/TTS的另两轮,9.56/9.16秒,不是固定音频对照,不与HEL-718产品数值作因果比较。 + +浏览器“原PCM首块”未单独标定;表中的样本幅度起点直接从其录音解码采样测得,另受有损编码和图时钟误差影响。服务端原PCM首块用实际RTP PTS锚定。视频量化约40ms、主音频检测约22ms,阈值/弱瞬态的感知误差未标定。弱瞬态口径中服务端出现+0.21~+0.31秒反向滞后,不能只报告嘴提前缩小。稳态包络相关在完整受控轮中约0.139~0.368,仍不足以证明音素同步或感知漂移已通过;被截断短窗不用于稳态验收。 + +本轮所有JSON/CSV保留各自完整命令、窗口、idle-until、阈值、源/录像/检测器哈希。分析窗口在各轮开始前保留空闲段;第6轮排除上一被打断轮的尾音。首轮若把idle-until放在原PCM首块之前仅40ms,会把更早的真实口部变化藏掉;最终保留口部1.12秒、响亮1.76秒的−0.640秒结果,没有用后移检测窗口美化首轮。 + +## 直接证据与最小剩余问题 + +固定音频输入的每一轮都仅前8块闭唇,第8号块(0.16秒)RMS `0.0133345156` 超过0.001,结束头部规则;后面的弱块继续有口型。`head-policy-summary.json`显示所有渲染帧选择与对应两块标记一致(0个冲突),实录有3个一弱一强的混合帧,均正确保留预测帧。 + +服务端四个完整轮及打断后第6轮各338块,浏览器四轮各338块,均逐块SHA256按顺序匹配到编码前RTP输出;5号打断轮已输出104块后主动取消,其余未输出是取消,不冒充338块完整播放。音频320样本/20ms、视频3600tick/40ms PTS连续,无跳步。`trace-summary.json`含各peer原始计数;原始`trace.jsonl`记录门控、原PCM float/int16哈希、渲染对、RTP音频哈希/PTS和视频PTS。 + +首轮还有独立的时序疑点:`head-frame-map.json`中第0/1块的闭唇画面在视频0.96秒,RTP第0块音频在1.20秒;第8/9块的首次预测画面在1.12秒,对应音频第8块在1.36秒。该轮画面与对应音频PTS相差−0.24秒,不能把剩余偏差全部归因弱音;其余轮并非同一偏移。该图的video_pts按render输出顺序与无丢帧的video recv顺序配对,trace可复核。需要总工复核首轮供给停顿/两路PTS推进与输出关联;本卡未扩展到视频修剪或新时钟方案。 + +因此0.001方案只覆盖最前0.16秒,**未满足总体目标**。下一步先由总管安排真人听原音频0.00~0.60秒,确认0.169秒瞬态应否触发口型,并由总工审视上述首轮PTS证据。不要将“近无声均不可闻”“阈值应调到更高”写为已证事实,本轮不循环盲调。 + +## 构建、自测、录像和未验项 + +- 前端`npm run build`通过;完整样机runtime/tools的ruff通过;新增4项门控检查、受影响的原4项队列/时钟检查通过。原78项核心测试/5项网关检查与本次未改动模块无关,沿用HEL-718已通过记录,未机械重跑。 +- 私有clean-driver首次/重复应用通过,最终再次应用所有Python文件哈希完全相同(`reapply.json`)。保留c726943/MuseTalk v1.5、原P10 venv、avatar和素材。 +- 本轮Chromium实际WebRTC检查:说话中取消、准备中取消、回阅读、offer故障、断网/恢复通过;快速重入独立首测通过,页面无pageerror。已保存本次完整页面状态/事件/截图。HEL-718快速重入并行首测超时仍未定位,本次通过不能关闭历史疑点。网关故障注入期间有SSL断连回调日志,浏览器恢复通过;未据此扩展修改网关。 +- 新`product-continuous.webm`约68秒,日常三轮→f96切点→正向进场→两轮交谈→说话中取消→独立正向回程→续读。取消后200~1000ms最大录音RMS `0.00000605`,无旧声复播证据。页面最终“已回到阅读原位置”,无pageerror。录制是浏览器实际画面/音频图的共同MediaRecorder,非物理扬声器校准。 +- 供真人:fixed.wav的0.00~0.60秒(重点0.16~0.18瞬态、0.54~0.58响亮起点);新完整录像约26.3~30.5秒进场、38.2~39.6秒首轮弱头/开口/响亮起点、55.0~56.3秒第二轮开头、57.51秒取消、58.76~61.89秒正向回程、61.89~67.97秒续读。事件时间已减record-start,实际容器有一帧左右误差,以音画原件为准。 +- 真人循环接缝/切点/暖色/书本/手部/坐垫/微动作、真人自然抢话、手机真机、真实后台/锁屏、共享TTS内部GPU即停逐项待验。390px桌面视口和synthetic visibility不等同真机/系统测试。0.001阈值、弱瞬态听感和稳态音素同步待验。任何脚本通过都不构成发布许可。 + +## 可复现命令与回滚 + +代码运行见README的HEL-720节。附件包含本轮实际`remote_supervisor.py`、`analyze_rounds.py`、`analyze_graph.py`、`weak_onsets.py`和`summarize.py`,各分析JSON含完整展开命令,`services.json`含实际启动参数/cwd/日志。诊断使用独立`/opt/hel720-fix`,复用材料不原地改写。 + +```bash +# 先确认8010/8014/8933空闲;PRIVATE_DRIVER为全新c726943副本 +python3 tools/prepare_driver.py --source /path/to/clean-c726943 --destination "$PRIVATE_DRIVER" +python3 tools/patch_driver.py "$PRIVATE_DRIVER" +/opt/hel715-verify/venv/bin/python tools/check_driver_head.py "$PRIVATE_DRIVER" +/opt/hel715-verify/venv/bin/python tools/check_driver_audio.py "$PRIVATE_DRIVER" +# 在私有驱动cwd启动trace_head.py,后跟原app.py参数;原始命令见services.json +# 以下驱动录制是单会话顺序执行;浏览器各脚本也逐项运行,不并发制造peer关闭卡顿 +/opt/hel715-verify/venv/bin/python tools/record_driver.py --wav fixed.wav --out server.mp4 --rounds 0,15,1,1 --post 9 +LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/record_graph.py evidence +LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/record_product.py evidence +LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/check_races.py evidence/race +LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/check_browser.py +python3 -m ruff check samples/companion/runtime samples/companion/tools +# 样机目录:npm ci --ignore-scripts --no-audit --no-fund;npm run build +``` + +回滚可在候选启动前`HEL720_HEAD_RMS=0`关闭策略;完全回滚则在新私有c726943副本应用415f837版本的`patch_driver.py`和`driver_audio.py`。不覆盖共享安装,不修改现网配置。 + +本轮对照Multica daemon PID 44286后停止自己的1425456/1425457/1425458,实际cmd/cwd/退出码见`cleanup.json`。8010/8014/8933已释放,18080/18081/9880/10095共享PID保持。没有运行中诊断遗留。提交/推送沿原分支和草稿PR,最终提交号见本卡结果评论。 diff --git a/samples/companion/tools/check_driver_head.py b/samples/companion/tools/check_driver_head.py new file mode 100644 index 0000000..f944baa --- /dev/null +++ b/samples/companion/tools/check_driver_head.py @@ -0,0 +1,103 @@ +"""Exercise the patched ingress, mixed inference batch and real output loop.""" + +import hashlib +import pathlib +import queue +import sys +import unittest +from threading import Event +from types import SimpleNamespace + +import numpy as np + +ROOT = pathlib.Path(sys.argv.pop(1)).resolve() +sys.path.insert(0, str(ROOT)) +from avatars.audio_features.base_asr import BaseASR # noqa: E402 +from avatars.base_avatar import AudioFrameData, BaseAvatar # noqa: E402 +from avatars.hel720_head import HeadGate # noqa: E402 + + +def frame(value, status=None, kind=0): + return AudioFrameData(np.full(320, value, dtype=np.float64), kind, + {'status': status} if status else {}) + + +class HeadChecks(unittest.TestCase): + def test_boundary_disabled_and_invalid(self): + gate = HeadGate(.001) + self.assertTrue(gate.classify(frame(.001, 'start'))) + self.assertFalse(gate.classify(frame(.0010001))) + self.assertFalse(gate.classify(frame(0))) + self.assertFalse(HeadGate(0).classify(frame(0, 'start'))) + for value in [-1, float('nan'), float('inf')]: + with self.assertRaises(ValueError): + HeadGate(value) + + def test_lifecycle_starvation_custom_and_end(self): + gate = HeadGate(.001) + self.assertTrue(gate.classify(frame(0, 'start'))) + self.assertFalse(gate.classify(frame(.1))) + self.assertFalse(gate.classify(frame(0, kind=1))) + self.assertFalse(gate.classify(frame(0, kind=2))) + self.assertFalse(gate.classify(frame(0))) # weak mid-utterance remains speech + self.assertFalse(gate.classify(frame(0, 'end'))) + self.assertTrue(gate.classify(frame(0))) + self.assertFalse(gate.classify(frame(.1))) + self.assertTrue(gate.classify(frame(0, 'start'))) + + def test_cancel_and_new_peer_keep_old_decisions_isolated(self): + opt = SimpleNamespace(fps=25, batch_size=8, l=10, r=10) + asr = BaseASR(opt) + asr.put_audio_frame(np.ones(320), {'status': 'start'}) + old = asr.queue.get() + asr.flush_talk() + asr.put_audio_frame(np.zeros(320), {}) + self.assertTrue(asr.queue.get().hel720_head_closed) + self.assertFalse(old.hel720_head_closed) + fresh = BaseASR(opt) + fresh.put_audio_frame(np.zeros(320), {}) + self.assertTrue(fresh.queue.get().hel720_head_closed) + + def test_mixed_batch_render_and_exact_audio_events_types(self): + asr = BaseASR(SimpleNamespace(fps=25, batch_size=8, l=10, r=10)) + values = [0] * 5 + [.02] + [0] * 10 + expected = [] + for i, value in enumerate(values): + data = np.full(320, value, np.float32) + event = {'status': 'start'} if i == 0 else {'status': 'end'} if i == 15 else {} + asr.put_audio_frame(data, event) + expected.append((hashlib.sha256((data * 32767).astype(np.int16).tobytes()).hexdigest(), event, 0)) + frames = [asr.queue.get() for _ in values] + self.assertEqual([f.hel720_head_closed for f in frames], [True]*5 + [False]*11) + for f in frames: + asr.output_queue.put(f) + asr.feat_queue.put(['feature']*8) + avatar = BaseAvatar.__new__(BaseAvatar) + avatar.asr, avatar.batch_size = asr, 8 + avatar.frame_list_cycle = [np.zeros((32, 32, 3), np.uint8) for _ in range(20)] + avatar.custom_index = {} + avatar.res_frame_queue = queue.Queue() + avatar.inference_batch = lambda index, features: [np.ones((32, 32, 3), np.uint8)*100 for _ in features] + avatar.inference(SimpleNamespace(is_set=lambda: avatar.res_frame_queue.qsize() == 8)) + results = list(avatar.res_frame_queue.queue) + self.assertEqual([x[0] is None for x in results], [True, True] + [False]*6) + quit_event = Event() + videos, audios = [], [] + def video(image): + videos.append(image.copy()) + if len(videos) == 8: + quit_event.set() + avatar.output = SimpleNamespace(start=lambda: None, stop=lambda: None, + push_video_frame=video, + push_audio_data=lambda pcm, event, kind: audios.append( + (hashlib.sha256(pcm.tobytes()).hexdigest(), event, kind))) + avatar.record_video_data = lambda image: None + avatar.record_audio_data = lambda pcm: None + avatar.paste_back_frame = lambda image, index: image + avatar.process_frames(quit_event) + self.assertEqual([int(v[-1, -1, 0]) for v in videos], [0, 0] + [100]*6) + self.assertEqual(audios, expected) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/samples/companion/tools/check_races.py b/samples/companion/tools/check_races.py index 6e96ccc..dba62b0 100644 --- a/samples/companion/tools/check_races.py +++ b/samples/companion/tools/check_races.py @@ -3,10 +3,12 @@ import asyncio import json import pathlib +import sys from playwright.async_api import async_playwright -OUT = pathlib.Path(__file__).resolve().parents[1] / "evidence" +OUT = pathlib.Path(sys.argv[1]) if len(sys.argv) > 1 else pathlib.Path(__file__).resolve().parents[1] / "evidence" +OUT.mkdir(parents=True, exist_ok=True) async def main(): @@ -18,34 +20,45 @@ async def main(): page = await context.new_page() errors = [] page.on("pageerror", lambda e: errors.append(str(e))) - await page.goto("https://192.168.200.12:8014") - await page.wait_for_function( - "document.querySelector('#status').textContent.includes('就绪')" - ) - await page.click("#enter") - await page.wait_for_function( - "document.querySelector('#status').textContent.includes('正在听')", timeout=40000 - ) - await page.fill("#text", "这一轮马上取消。") - await page.click("button[type=submit]") - await page.wait_for_timeout(100) - await page.click("#cancel") - await page.fill("#text", "新一轮只回复:你好。") - await page.click("button[type=submit]") - await page.wait_for_function( - "window.sampleEvents.some(e=>e.event==='first-audible')", timeout=45000 - ) - await page.wait_for_timeout(10000) - events = await page.evaluate("window.sampleEvents") - failures = [ - e for e in events if e["event"] == "status" and "媒体协商超时" in str(e["detail"]) - ] - result = {"pageerrors": errors, "late_error_failures": failures, "events": events} - (OUT / "race-check.json").write_text(json.dumps(result, ensure_ascii=False, indent=2)) - assert not failures - await page.click("#back") - await page.wait_for_timeout(4000) - await browser.close() + try: + await page.goto("https://192.168.200.12:8014") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('就绪')" + ) + await page.click("#enter") + await page.wait_for_function( + "document.querySelector('#status').textContent.includes('正在听')", timeout=40000 + ) + await page.fill("#text", "这一轮马上取消。") + await page.click("button[type=submit]") + await page.wait_for_timeout(100) + await page.click("#cancel") + await page.fill("#text", "新一轮只回复:你好。") + await page.click("button[type=submit]") + await page.wait_for_function( + "window.sampleEvents.some(e=>e.event==='first-audible')", timeout=45000 + ) + await page.wait_for_timeout(10000) + events = await page.evaluate("window.sampleEvents") + failures = [ + e for e in events if e["event"] == "status" and "媒体协商超时" in str(e["detail"]) + ] + result = {"pageerrors": errors, "late_error_failures": failures, "events": events} + (OUT / "race-check.json").write_text(json.dumps(result, ensure_ascii=False, indent=2)) + assert not failures + await page.click("#back") + await page.wait_for_timeout(4000) + finally: + (OUT / "race-page-state.json").write_text(json.dumps({ + "url": page.url, + "status": await page.locator('#status').inner_text(), + "events": await page.evaluate('window.sampleEvents'), + "pageerrors": errors, + "media": await page.locator('video').evaluate_all( + '(vs)=>vs.map(v=>({id:v.id, ready:v.readyState, paused:v.paused, time:v.currentTime}))'), + }, ensure_ascii=False, indent=2)) + await page.screenshot(path=str(OUT / 'race-final.png')) + await browser.close() print("ICE cancellation race passed", flush=True) diff --git a/samples/companion/tools/driver_head.py b/samples/companion/tools/driver_head.py new file mode 100644 index 0000000..392e5b5 --- /dev/null +++ b/samples/companion/tools/driver_head.py @@ -0,0 +1,42 @@ +"""Candidate head-only render policy; audio bytes, types and events stay intact.""" + +import math +import os +import threading + +import numpy as np + + +class HeadGate: + def __init__(self, threshold=None): + self.threshold = float(os.environ.get('HEL720_HEAD_RMS', '0.001') + if threshold is None else threshold) + if not math.isfinite(self.threshold) or self.threshold < 0: + raise ValueError('HEL720_HEAD_RMS must be finite and >= 0 (0 disables)') + self._lock = threading.Lock() + self.leading = True + + def reset(self): + with self._lock: + self.leading = True + + def classify(self, frame): + # Called on original 16k PCM at ASR ingress, BEFORE feature batching. + # Synthetic idle never rearms a running utterance (stream starvation). + with self._lock: + if frame.type != 0: + return False + status = (frame.userdata or {}).get('status') + if status == 'start': + self.leading = True + rms = float(np.sqrt(np.mean(np.square(frame.data, dtype=np.float64)))) + closed = self.threshold > 0 and self.leading and rms <= self.threshold + if not closed: + self.leading = False + if status == 'end': + self.leading = True + return closed + + +def render_speech(frames): + return any(f.type == 0 and not getattr(f, 'hel720_head_closed', False) for f in frames) diff --git a/samples/companion/tools/patch_driver.py b/samples/companion/tools/patch_driver.py index 8603237..37b382d 100644 --- a/samples/companion/tools/patch_driver.py +++ b/samples/companion/tools/patch_driver.py @@ -117,3 +117,30 @@ p = root / "avatars/audio_features/base_asr.py" replace_once(p, " for _ in range(self.stride_left_size):\n self.output_queue.get()", " for _ in range(self.stride_left_size + self.stride_right_size):\n self.output_queue.get()") print("Installed HEL-718 bounded synthetic idle, continuous RTP audio, and warm-up drain") + +# HEL-720: tag original PCM at ingress; never change type or feature windows. +(root / "avatars/hel720_head.py").write_text( + Path(__file__).with_name("driver_head.py").read_text() +) +p = root / "avatars/audio_features/base_asr.py" +replace_once(p, "from avatars.base_avatar import BaseAvatar,AudioFrameData", + "from avatars.base_avatar import BaseAvatar,AudioFrameData\nfrom avatars.hel720_head import HeadGate") +replace_once(p, " self.parent = parent", " self.parent = parent\n self.hel720_head = HeadGate()") +replace_once(p, " self.queue.queue.clear()", + " self.queue.queue.clear()\n self.hel720_head.reset()") +replace_once(p, " self.queue.put(AudioFrameData(data=audio_chunk,type=0,userdata=datainfo))", ''' frame = AudioFrameData(data=audio_chunk,type=0,userdata=datainfo) + frame.hel720_head_closed = self.hel720_head.classify(frame) + self.queue.put(frame)''') +p = root / "avatars/base_avatar.py" +replace_once(p, "from utils.logger import logger", + "from utils.logger import logger\nfrom avatars.hel720_head import render_speech") +replace_once(p, " if audioframe.type == 0:", + " if render_speech([audioframe]):") +replace_once(p, " self.res_frame_queue.put((res_frame, audio_frames[i*2:i*2+2], mirror_index(length, index)))", + ''' pair = audio_frames[i*2:i*2+2] + self.res_frame_queue.put((res_frame if render_speech(pair) else None, pair, mirror_index(length, index)))''') +replace_once(p, " if audio_frames[0].type!=0 and audio_frames[1].type!=0: #全为静音数据,只需要取fullimg", + " if res_frame is None: # synthetic idle OR head-only closed-lip render") +replace_once(p, " self.speaking = False\n audiotype = audio_frames[0].type", + " self.speaking = current_speaking\n audiotype = audio_frames[0].type or 1") +print("Installed HEL-720 per-PCM head-only render flags (candidate RMS 0.001; 0 disables)") diff --git a/samples/companion/tools/trace_head.py b/samples/companion/tools/trace_head.py new file mode 100644 index 0000000..72c08ef --- /dev/null +++ b/samples/companion/tools/trace_head.py @@ -0,0 +1,80 @@ +"""Test-only trace: original PCM -> batched features -> rendered pairs -> RTP.""" + +import hashlib +import json +import runpy +import sys +import time + +import numpy as np +from avatars.base_avatar import BaseAvatar +from avatars.hel720_head import HeadGate +from server import webrtc + +out = open(sys.argv.pop(1), 'w', buffering=1) # noqa: SIM115 + + +def digest(data): + return hashlib.sha256(data.tobytes()).hexdigest() + + +def write(event, **fields): + out.write(json.dumps({'wall': time.monotonic(), 'event': event, **fields}) + '\n') + + +classify = HeadGate.classify +init = BaseAvatar.__init__ +push = webrtc.HumanPlayer.push_audio +recv = webrtc.PlayerStreamTrack.recv + + +def classify_trace(self, frame): + closed = classify(self, frame) + seq = getattr(self, 'trace_seq', 0) + self.trace_seq = seq + 1 + frame.hel720_trace = [id(self), seq] + write('pcm', key=frame.hel720_trace, samples=[seq*320, (seq+1)*320], + rms=float(np.sqrt(np.mean(frame.data.astype(float)**2))), closed=closed, + threshold=self.threshold, status=frame.userdata, kind=frame.type, + float_sha256=digest(frame.data), pcm_sha256=digest((frame.data*32767).astype(np.int16))) + return closed + + +def init_trace(self, opt): + init(self, opt) + get = self.res_frame_queue.get + def render_get(*args, **kw): + result = get(*args, **kw) + image, frames, index = result + write('render', sid=self.sessionid, index=index, closed=image is None, + keys=[getattr(f, 'hel720_trace', None) for f in frames], + flags=[getattr(f, 'hel720_head_closed', False) for f in frames], + kinds=[f.type for f in frames]) + return result + self.res_frame_queue.get = render_get + + +def push_audio(self, frame, eventpoint=None, audio_type=0): + push(self, frame, eventpoint, audio_type) + q = self.audio._queue + write('push', player=id(self), kind=audio_type, pcm_sha256=digest(frame), status=eventpoint, + rms=float(np.sqrt(np.mean((frame.astype(float)/32768)**2))), + depth=q.qsize(), dropped=q.dropped_silence, high_water=q.high_water) + + +async def receive(self): + frame = await recv(self) + fields = {'player': id(self._player), 'kind': self.kind, 'pts': frame.pts, + 'tb': str(frame.time_base), 'depth': self._queue.qsize()} + if self.kind == 'audio': + fields['pcm_sha256'] = digest(frame.to_ndarray()) + fields['rms'] = float(np.sqrt(np.mean((frame.to_ndarray().astype(float)/32768)**2))) + write('recv', **fields) + return frame + + +HeadGate.classify = classify_trace +BaseAvatar.__init__ = init_trace +webrtc.HumanPlayer.push_audio = push_audio +webrtc.PlayerStreamTrack.recv = receive +runpy.run_path('app.py', run_name='__main__') -- 2.54.0 From 20cd586670667d58fb918e29e27f9a011c238174 Mon Sep 17 00:00:00 2001 From: mobai Date: Wed, 21 Oct 2026 02:48:36 +0800 Subject: [PATCH 6/6] fix(HEL-722): prepare first audio and avatar inference before accepting peers Co-authored-by: multica-agent --- samples/companion/README.md | 30 +++++++ samples/companion/REPORT-HEL722.md | 80 +++++++++++++++++++ samples/companion/tools/check_driver_ready.py | 47 +++++++++++ samples/companion/tools/driver_ready.py | 64 +++++++++++++++ samples/companion/tools/patch_driver.py | 14 ++++ samples/companion/tools/trace_head.py | 20 +++++ 6 files changed, 255 insertions(+) create mode 100644 samples/companion/REPORT-HEL722.md create mode 100644 samples/companion/tools/check_driver_ready.py create mode 100644 samples/companion/tools/driver_ready.py diff --git a/samples/companion/README.md b/samples/companion/README.md index 3770a1f..038a00d 100644 --- a/samples/companion/README.md +++ b/samples/companion/README.md @@ -153,3 +153,33 @@ PCM哈希/RMS/门控、两块对应的渲染帧及RTP输出哈希/PTS,不由 `git show 415f837:samples/companion/tools/patch_driver.py`和同提交的 `driver_audio.py`重装HEL-718补丁,复用原avatar/权重。也可在候选启动前设 `HEL720_HEAD_RMS=0`关闭策略;该开关回滚保留代码但不闭唇。不要原地覆盖共享安装。 + +## HEL-722 启动准备(局部通过,整体同步仍未通过) + +`driver_ready.py` 与补丁一起携带。MuseTalk 在监听 HTTP 前依次执行真实文件 +解码/24k→16k 重采样、Whisper 特征提取/既有切窗、真实 avatar latent 的 +`inference_batch`。替换原固定张量的 `warm_up` 调用,不重复加载模型。 +探针仅引用已加载模型与 avatar,不构造 session/ASR/HeadGate/输出队列,不调用 +`put_audio_file`、TTS 或网络。预测结果丢弃,不生成真实话语的 start/end。 + +日志明确报告 preparing/ready/failed、各阶段和总耗时;失败抛异常并退出,HTTP +不会开始监听。成功后 `GET /hel722/ready` 返回本次准备状态与时间。这个状态只 +表示启动路径已准备,不表示音画同步验收通过。首次工作线程仍有约0.18秒模型 +调用,暖轮约0.08秒;已有批处理/排队/补静音的残差未在本轮修改。 + +```bash +python3 tools/prepare_driver.py --source /path/to/clean-c726943 --destination "$PRIVATE_DRIVER" +python3 tools/patch_driver.py "$PRIVATE_DRIVER" +/opt/hel715-verify/venv/bin/python tools/check_driver_ready.py "$PRIVATE_DRIVER" +/opt/hel715-verify/venv/bin/python tools/check_driver_head.py "$PRIVATE_DRIVER" +/opt/hel715-verify/venv/bin/python tools/check_driver_audio.py "$PRIVATE_DRIVER" +``` + +`trace_head.py` 诊断额外把原 PCM key/type/event 放在编码前 AVFrame 的 opaque +元数据中,区分真实全零 PCM 与合成静音;不改 PCM、事件、类型、队列策略或 RTP。 +仅显式诊断时加载,元数据不会进入网络编码。阶段计时包装和本轮完整命令随证据包。 + +回滚:停止自己的候选进程,在新建的 c726943 私有副本应用 +`77b7608` 的 `patch_driver.py`、`driver_audio.py`、`driver_head.py`,复用同 avatar +及权重,即恢复 HEL-720。不要原地去掉 readiness 而留其他混合版本。 +详见 `REPORT-HEL722.md`,不部署、不发布、不合并。 diff --git a/samples/companion/REPORT-HEL722.md b/samples/companion/REPORT-HEL722.md new file mode 100644 index 0000000..c31f611 --- /dev/null +++ b/samples/companion/REPORT-HEL722.md @@ -0,0 +1,80 @@ +HEL-722 一轮最小施工与验证,2026-10-21。基线 `hel-714-daily-talk@77b76083d05536893744d50ccf2a01c4a3c85a27`,原草稿 PR #2,base `main@1e9b08a`。 + +**结论:启动局部目标通过,交 in_review 复审;整体音画同步仍未通过,继续禁止部署、发布、合并。** 未调整门槛、PCM、特征窗、空闲视频队列或任何固定延迟。 + +## 已有预热缺口与最小修复 + +独立重现旧候选后,修正 HEL-721 的一项因果归因:约0.55秒并非全部是 MuseTalk 推理。首次 `/humanaudio` 在 aiohttp 主事件循环同步执行文件解码/重采样;`resampy.resample` 首次调用实测0.557秒,期间 WebRTC recv 停顿。首批真实模型推理另为0.201秒。旧启动 `warm_up` 只用固定 ones 特征和 latent 跑 pe/UNet/VAE,未跑上传重采样及 Whisper;会话的 `BaseASR.warm_up` 只填左右上下文,完全不提特征、不推理。固定张量与真实张量尺寸实际上均为 `(8,50,384)` / `(8,8,32,32)`,不是尺寸失配,不再沿用猜测。 + +旧候选本次实测:视频 recv 空洞0.574秒、render间隔0.632秒、视频时钟落后0.257秒,随后3个连续小于5ms的追赶间隔;视频深度39、音频深度93。首口部帧相对原第8块音频墙钟 vlag=-0.142秒,文件口径=-0.260秒。独立录端归零确实夸大偏差,但不能解释真实停顿。 + +`tools/patch_driver.py` 仅在 MuseTalk 的 HTTP 监听之前,改调用新 `driver_ready.prepare`: + +1. 用内部短 WAV 走原私有解码函数,覆盖相同24k→16k首次重采样路径。 +2. 使用同一已加载 Whisper、相同上下文长度及 `_feature2chunks([2,3],l/2,2)`。 +3. 用真实 avatar latent 调用原 `MuseReal.inference_batch`,丢弃预测。 + +没有重复加载模型。没有会话/ASR构造器、HeadGate、音频输入/输出队列、TTS、网络或 start/end,内部样本不进入真实话语。`preparing/ready/failed`、阶段和总耗时写日志;异常明确抛出,未监听 HTTP。成功后 `/hel722/ready` 返回状态。两次冷进程准备耗时0.968/0.973秒(重采样0.461/0.455、特征0.198/0.201、推理0.309/0.316);第二进程从创建到 ready 标志约13.0秒,包含导入、模型和avatar加载,监听观测另有1秒轮询量化,没有把准备耗时冒充完整启动耗时。 + +## 测量口径与关联可信度 + +固定音频原件24kHz/6.76秒、SHA256 `8c0c1ec943882d4aafbb4f8cda488f138a19231dabe2a58335777c0c42fff3dc`,未重合成、未裁头。检测脚本 SHA256 `081e40486793d9c57dbd9d682aacbefe58670eb07a9b22675953cd6b883f0ec6`,ROI `(271,341,180,220)`、服务端 motion/residual `3.1718642664/7.693739183`、浏览器 `4.294028549194336/10.492400400638578`,浏览器裁切 offset160、size640→512,全保留。 + +原PCM key→render两块配对→视频recv序号,使用同一主机 `time.monotonic/perf_counter`,原PCM SHA顺序匹配到编码前音频recv;接收端独立记录实际 recv 的同一单调时钟与两轨PTS。文件指标另列,不混作墙钟。服务端视频步进3600、音频320,序号配对无中途负延迟;未入RTP的尾部render分别17/18帧,均为关闭时未拉取的队尾;这些帧可在关闭前已入队,不能写成“全在teardown后才render”,详见原trace。冷2进一步在AVFrame opaque携带原key/type/event,338块/338唯一key全部哈希、类型、事件一致,含最后5块原始全零PCM,预热期间输入PCM数=0。 + +冷1与旧基线用顺序哈希;原块0/8/28均非零、可直接追踪。尾部333~337号全零块与合成静音字节相同,旧格式不能独立证明这5块身份,因此没有用零/低相关性的 placement 结果证明定位。新opaque证据补足冷2该风险;冷1的关键起点不依赖全零尾部。录端视频关联是顺序及连续性核验,不是逐帧像素指纹。没有宣称完整音素对应已验证。 + +## 六轮服务端原门槛 + +秒;差值=口部-响亮声音,负=嘴提前,绝对值≤0.15达标。S4为2秒打断的局部轮,不能当完整稳态样本。弱瞬态栏是原第8块内幅度>0.015的采样起点,按16k重采样后0.16925秒定位;原24k为0.169208秒,量化/录制误差约40ms视频、22ms主RMS。主RMS仍为0.015,不替换验收口径。 + +|轮/场景|文件口部-响亮|发送墙钟口部-响亮|弱瞬态差|原PCM首块差|墙钟/文件达标| +|---|---:|---:|---:|---:|---| +|S1 冷进程A首轮|-0.251|-0.240|+0.151|+0.319|否/否| +|S2 同peer空闲15秒|-0.148|-0.140|+0.270|+0.440|是/是| +|S3 暖连续轮|-0.091|-0.079|+0.312|+0.482|是/是| +|S4 说话中取消(104块)|-0.160|-0.159|+0.232|+0.560|否/否| +|S5 同peer取消后再送(诊断)|-0.091|-0.079|+0.311|+0.481|是/是| +|S6 冷进程B首轮|-0.193|-0.178|+0.272|+0.441|否/否| + +发送墙钟、文件均3/6;两次冷首轮仍未达整体同步门槛。接收端实际收帧墙钟口部-响亮依次为 -0.317/-0.198/-0.148/-0.212/-0.149/-0.259,记录的是 aiortc 解码接收时刻,不是浏览器扬声器时间,也不与发送端验收行偷换。 + +|轮|首口部帧vlag(相对所配原块)|首次推理|上传重采样|视频recv最大间隔*|render最大间隔*|音频补位块/内容跨度拉伸| +|---|---:|---:|---:|---:|---:|---| +|旧基线冷首轮|-0.142|0.201|0.557|0.574|0.632|0 / 0.00| +|S1|+0.160|0.178|0.083|0.134|0.342|0 / 0.00| +|S2|+0.280|0.080|0.085|0.128|0.426|2 / 0.04| +|S3|+0.322|0.082|0.086|0.116|0.339|6 / 0.12| +|S4|+0.241|0.084|0.092|0.148|0.385|8 / 0.16| +|S5|+0.320|0.079|0.084|0.133|0.342|8 / 0.16| +|S6|+0.281|0.178|0.090|0.111|0.343|3 / 0.06| + +*窗口为上传前0.2秒至后2秒,包含跨窗口起点的间隔,避免漏掉上传时的停顿。原批处理按8帧突发生产仍存在;没有宣称逐帧生产间距40ms。两次冷首轮时钟最大落后0.093/0.071秒,连续<5ms追赶间隔最多1个,已与暖轮0~1个同构,原0.24秒异常及连续追赶消失。冷首轮视频深度峰值16/14(暖轮约14~16),音频深度25/21,不是通过新增一秒排队取得结果。 + +冷工作线程首推仍约0.178秒,暖轮约0.08秒;预热没有声称覆盖所有线程局部初始化。原预渲染队列相位、每批边界和合成音频补位仍使冷暖/轮间差异浮动:例如S1第一口部render→recv为0.321秒,S6为0.506秒,S6弱到响亮之间另插入0.06秒音频补位。已有逐块墙钟表可以分解这些差异,不能将全部差异继续归咎弱头,也没有调阈值或补固定延迟抵消它们。 + +## 暖驱动新peer浏览器对照(单列) + +同驱动A已暖,新建peer、相同固定音频四轮。共同MediaRecorder文件的口部-响亮为 -0.234/-0.211/-0.220/-0.221秒,4/4过浏览器0.25门槛。驱动侧首口部对应原PCM对vlag为 +0.320/+0.301/+0.242/+0.321;第2/4轮是第7/8块混合对,保留预测画面符合既有规则。它们不证明冷驱动已热,故不冒充独立冷启动。浏览器文件弱瞬态口径另为 +0.144/+0.133/+0.114/+0.215秒(有损解码采样幅度>0.015,不代替主RMS门槛);原PCM首块在浏览器播放端未独立定位,不能用低相关性推算。浏览器音频通路额外延迟仍未独立标定,不批准固定补偿;文件采集也不冒充物理扬声器/屏幕或真人听感。 + +## HEL-721 统计纠错与残余风险 + +按原列值独立取绝对值,服务端S4/S6达标(2/6),浏览器第3/4轮达标(2/4);实时修正列同样是S4/S6,不是“仅S4”或“浏览器全超线”。附件 `HEL721-corrected-gates.json`逐轮列出,原复审zip原封保留,不沿用其错误gate文字。原结论整体未通过仍成立。 + +全部固定音频完整轮仅前8块闭唇、首个超门槛块后弱音保持;本轮全trace渲染标记0冲突。S4已输入338块,取消前实际输出104块,其余234块取消;S5新话语从新start重置,产品路径仍用销毁peer隔离。首轮嘴不再早于配对块,但相对响亮/弱瞬态仍有正负残差;稳态包络只是运动能量启发式,音素与真人感知未验。本卡只交局部修复复审,不盲调残差。 + +## 检查、隔离、重现与回滚 + +新增2项检查:禁止探针构造真实会话/写PCM/通知/触门控,准备成功只返回状态;注入解码/推理失败必须抛异常。另实跑进程级重采样故障,启动exit=1、明确failed日志,8010未监听。原4项HeadGate和4项队列/时钟检查全过。干净c726943副本应用及重复应用116个Python文件哈希一致,和私有运行候选一致。 + +`npm run build`、runtime/tools的ruff、diff检查通过。实际Chromium准备中/说话中取消、快速重入、断网/恢复/offer故障通过,pageerror为空,有页面状态、事件、截图和连续录像。历史快速重入并行首测超时仍未定位;本轮通过不关闭历史疑点。网关故障注入中仍有既有SSL断连回调异常,页面恢复成功,未扩范围修共享服务。核心人格/记忆模块未改,沿用既有核心检查,未机械重跑。 + +私有工作目录 `/opt/hel722-fix`,源代码、模型/权重/avatar只读复用材料,新建代码副本;不修改共享安装/配置/端口/数据。仓库CLI因未配置该项目资源拒绝checkout,改用明确仓库URL普通git clone,确认工作树干净、head/base与无并行运行;不新建PR。 + +实际完整命令在证据包 `diagnostic/supervisor.py`、services.json、逐轮analysis JSON及脚本中。baseline在打HEL-722补丁之前跑;cold1为 `record_driver.py --rounds 0,15,1 --post 9`;cold2为相同录制器移除打断段的 `record_once.py --rounds 0 --post 9`。先查8010/8014/8933空闲,再前台运行supervisor,发送record命令,收齐结果后stop,禁止留下运行中诊断。浏览器脚本按原README命令运行,需已有Playwright/Chromium库路径;全部文件随附件,无新依赖。`diagnostic/summarize.py`在证据包根运行产生逐轮JSON。`analyze.py`在私有远端执行检测,统一窗口和参数,固定检测器未改。 + +前台监督进程均已收回;对照Multica daemon PID44286后,只停止记录的自有精确PID;cleanup.json记录实际cmd/cwd/exit。8010/8014/8933释放,18080/18081/9880/10095等共享服务PID保持。无诊断留运行。 + +完整回滚:停止自己的候选,另建c726943私有副本,取 `77b7608:samples/companion/tools/{patch_driver.py,driver_audio.py,driver_head.py}` 应用,复用同avatar/权重;不要编辑共享安装。仓库本次提交可用 `git revert <本次提交>`撤回,提交号见本卡结果评论。只推原候选分支,不合并。 + +真人弱音听感、画面循环/切点/暖色/书本手部坐垫、自然抢话、手机真机、后台锁屏、共享TTS内部GPU即停仍逐项待验。本轮不访问4080、不新增依赖、不部署、不发布。 diff --git a/samples/companion/tools/check_driver_ready.py b/samples/companion/tools/check_driver_ready.py new file mode 100644 index 0000000..96cdbc2 --- /dev/null +++ b/samples/companion/tools/check_driver_ready.py @@ -0,0 +1,47 @@ +"""Startup failure must propagate; preparation must never touch live I/O or gates.""" + +import pathlib +import sys +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import numpy as np + +root = pathlib.Path(sys.argv.pop(1)).resolve() +sys.path.insert(0, str(root)) +from avatars.base_avatar import BaseAvatar # noqa: E402 +from avatars.hel720_head import HeadGate # noqa: E402 +from avatars.hel722_ready import prepare # noqa: E402 +from avatars.musetalk_avatar import MuseReal # noqa: E402 + + +class ReadyChecks(unittest.TestCase): + def test_decoder_failure_is_fatal_without_constructing_session(self): + with patch.object(MuseReal, '__init__', side_effect=AssertionError('live session')), \ + patch('resampy.resample', side_effect=ValueError('injected resampler failure')), \ + self.assertRaisesRegex(RuntimeError, 'decode_resample'): + prepare(SimpleNamespace(), None, None) + + def test_discard_predictions_and_no_pcm_events_or_gate_mutation(self): + opt = SimpleNamespace(fps=25, batch_size=8, l=10, r=10) + processor = SimpleNamespace(audio2feat=lambda x: np.zeros((1500, 5, 384), np.float16)) + avatar = [None, None, None, None, [np.zeros((1, 8, 32, 32))]] + model = [None, None, None, None, processor] + # Real decode/resample and feature slicing; deny all live-session entrypoints. + with patch.object(MuseReal, '__init__', side_effect=AssertionError('live session')), \ + patch.object(HeadGate, 'classify', side_effect=AssertionError('gate')), \ + patch.object(BaseAvatar, 'put_audio_file', side_effect=AssertionError('input')), \ + patch.object(BaseAvatar, 'notify', side_effect=AssertionError('event')), \ + patch.object(MuseReal, 'inference_batch', return_value=np.zeros((8, 4, 4, 3))) as infer: + status = prepare(opt, model, avatar) + self.assertEqual(status['state'], 'ready') + self.assertEqual(status['feature_shape'], [8, 50, 384]) + self.assertEqual(infer.call_count, 1) + self.assertNotIn('prediction', status) + with patch.object(MuseReal, 'inference_batch', side_effect=RuntimeError('GPU failed')), \ + self.assertRaisesRegex(RuntimeError, 'inference'): + prepare(opt, model, avatar) + + +unittest.main() diff --git a/samples/companion/tools/driver_ready.py b/samples/companion/tools/driver_ready.py new file mode 100644 index 0000000..4fdea64 --- /dev/null +++ b/samples/companion/tools/driver_ready.py @@ -0,0 +1,64 @@ +"""Warm the pinned MuseTalk file/feature/inference path before HTTP accepts peers. + +No avatar constructor, input/output queue, HeadGate, TTS or network is used. +The probe owns only model references and a private ASR helper; discard predictions. +""" + +import json +import time +from io import BytesIO + +import numpy as np +import soundfile as sf + + +def prepare(opt, model, avatar): + from avatars.audio_features.whisper import WhisperASR + from avatars.base_avatar import BaseAvatar + from avatars.musetalk_avatar import MuseReal + from utils.logger import logger + + started = time.monotonic() + state = {'state': 'preparing', 'started_wall': started, 'stages': {}} + logger.info('HEL-722 readiness %s', json.dumps(state)) + stage = 'decode_resample' + try: + # Match the actual upload decoder, including its first-use resampler JIT. + # These bytes are local, never passed to put_audio_file/put_audio_frame. + wav = BytesIO() + sf.write(wav, np.zeros(2400, np.float32), 24000, format='WAV', subtype='PCM_16') + wav.seek(0) + probe = MuseReal.__new__(MuseReal) + probe.sample_rate = 16000 + t = time.monotonic() + BaseAvatar._BaseAvatar__create_bytes_stream(probe, wav) + state['stages'][stage] = time.monotonic() - t + + probe.batch_size = opt.batch_size + probe.vae, probe.unet, probe.pe, probe.timesteps, probe.audio_processor = model + probe.input_latent_list_cycle = avatar[-1] + # __new__ avoids ASR queues/gates as well as session/output construction. + asr = WhisperASR.__new__(WhisperASR) + stage = 'features' + t = time.monotonic() + inputs = np.zeros((opt.l + opt.r + opt.batch_size * 2) * (16000 // (opt.fps * 2)), + dtype=np.float32) + features = probe.audio_processor.audio2feat(inputs) + chunks = asr._feature2chunks(features, opt.batch_size, [2, 3], opt.l / 2, 2) + state['stages'][stage] = time.monotonic() - t + state['feature_shape'] = list(np.stack(chunks).shape) + state['latent_shape'] = list(avatar[-1][0].shape) + stage = 'inference' + t = time.monotonic() + prediction = probe.inference_batch(0, chunks) + if len(prediction) != opt.batch_size or not np.isfinite(prediction).all(): + raise RuntimeError('Invalid startup inference output') + state['stages'][stage] = time.monotonic() - t + state.update(state='ready', elapsed=time.monotonic() - started) + logger.info('HEL-722 readiness %s', json.dumps(state)) + return state + except Exception as exc: + state.update(state='failed', stage=stage, error=str(exc), + elapsed=time.monotonic() - started) + logger.exception('HEL-722 readiness %s', json.dumps(state)) + raise RuntimeError(f'HEL-722 startup warmup failed at {stage}') from exc diff --git a/samples/companion/tools/patch_driver.py b/samples/companion/tools/patch_driver.py index 37b382d..7d57ce6 100644 --- a/samples/companion/tools/patch_driver.py +++ b/samples/companion/tools/patch_driver.py @@ -144,3 +144,17 @@ replace_once(p, " if audio_frames[0].type!=0 and audio_frames[1].type replace_once(p, " self.speaking = False\n audiotype = audio_frames[0].type", " self.speaking = current_speaking\n audiotype = audio_frames[0].type or 1") print("Installed HEL-720 per-PCM head-only render flags (candidate RMS 0.001; 0 disables)") + +# HEL-722: prepare actual upload + feature + avatar inference before listening. +(root / "avatars/hel722_ready.py").write_text( + Path(__file__).with_name("driver_ready.py").read_text() +) +p = root / "app.py" +replace_once(p, " warm_up(opt.batch_size,model) ", ''' from avatars.hel722_ready import prepare + hel722_ready = prepare(opt, model, global_avatars[opt.avatar_id])''') +replace_once(p, ' appasync["rtc_manager"] = rtc_manager', ''' appasync["rtc_manager"] = rtc_manager + if opt.model == 'musetalk': + async def hel722_readiness(request): + return web.json_response(hel722_ready) + appasync.router.add_get('/hel722/ready', hel722_readiness)''') +print("Installed HEL-722 fail-closed startup preparation; /hel722/ready after success") diff --git a/samples/companion/tools/trace_head.py b/samples/companion/tools/trace_head.py index 72c08ef..405f2b3 100644 --- a/samples/companion/tools/trace_head.py +++ b/samples/companion/tools/trace_head.py @@ -26,6 +26,22 @@ classify = HeadGate.classify init = BaseAvatar.__init__ push = webrtc.HumanPlayer.push_audio recv = webrtc.PlayerStreamTrack.recv +player_init = webrtc.HumanPlayer.__init__ + + +def init_player(self, avatar_session, *args, **kw): + player_init(self, avatar_session, *args, **kw) + put = self.audio._queue.put + def tagged_put(item, *args, **kw): + frame, eventpoint, kind = item + keys = getattr(avatar_session, 'hel722_trace_pair', []) + cursor = getattr(avatar_session, 'hel722_trace_cursor', 0) + frame.opaque = {'key': keys[cursor] if cursor < len(keys) else None, + 'origin_kind': kind, 'status': eventpoint, + 'sid': avatar_session.sessionid} + avatar_session.hel722_trace_cursor = cursor + 1 + return put(item, *args, **kw) + self.audio._queue.put = tagged_put def classify_trace(self, frame): @@ -46,6 +62,8 @@ def init_trace(self, opt): def render_get(*args, **kw): result = get(*args, **kw) image, frames, index = result + self.hel722_trace_pair = [getattr(f, 'hel720_trace', None) for f in frames] + self.hel722_trace_cursor = 0 write('render', sid=self.sessionid, index=index, closed=image is None, keys=[getattr(f, 'hel720_trace', None) for f in frames], flags=[getattr(f, 'hel720_head_closed', False) for f in frames], @@ -67,12 +85,14 @@ async def receive(self): fields = {'player': id(self._player), 'kind': self.kind, 'pts': frame.pts, 'tb': str(frame.time_base), 'depth': self._queue.qsize()} if self.kind == 'audio': + fields['origin'] = frame.opaque fields['pcm_sha256'] = digest(frame.to_ndarray()) fields['rms'] = float(np.sqrt(np.mean((frame.to_ndarray().astype(float)/32768)**2))) write('recv', **fields) return frame +webrtc.HumanPlayer.__init__ = init_player HeadGate.classify = classify_trace BaseAvatar.__init__ = init_trace webrtc.HumanPlayer.push_audio = push_audio -- 2.54.0