From 6a5e155258429cbbf9699fcf9011097190bb52a6 Mon Sep 17 00:00:00 2001 From: mobai Date: Tue, 20 Oct 2026 23:24:31 +0800 Subject: [PATCH] feat(hel715): LiveTalking+MuseTalk verification kit (report, material spec, e2e scripts) Co-authored-by: multica-agent --- experiments/hel715/MATERIAL-SPEC.md | 72 ++++ experiments/hel715/README.md | 42 ++ experiments/hel715/REPORT.md | 73 ++++ .../livetalking-patch/adapter-and-diff.txt | 82 ++++ .../hel715/livetalking-patch/cosyvoice.py | 69 ++++ .../enable-cosyvoice-registration.diff | 13 + experiments/hel715/scripts/e2e_client.py | 375 ++++++++++++++++++ experiments/hel715/scripts/hel715-chain.html | 129 ++++++ experiments/hel715/scripts/llm_chain2.py | 124 ++++++ experiments/hel715/scripts/record_chain.py | 59 +++ 10 files changed, 1038 insertions(+) create mode 100644 experiments/hel715/MATERIAL-SPEC.md create mode 100644 experiments/hel715/README.md create mode 100644 experiments/hel715/REPORT.md create mode 100644 experiments/hel715/livetalking-patch/adapter-and-diff.txt create mode 100644 experiments/hel715/livetalking-patch/cosyvoice.py create mode 100644 experiments/hel715/livetalking-patch/enable-cosyvoice-registration.diff create mode 100644 experiments/hel715/scripts/e2e_client.py create mode 100644 experiments/hel715/scripts/hel715-chain.html create mode 100644 experiments/hel715/scripts/llm_chain2.py create mode 100644 experiments/hel715/scripts/record_chain.py diff --git a/experiments/hel715/MATERIAL-SPEC.md b/experiments/hel715/MATERIAL-SPEC.md new file mode 100644 index 0000000..7ca55a0 --- /dev/null +++ b/experiments/hel715/MATERIAL-SPEC.md @@ -0,0 +1,72 @@ +# HEL-715 素材包规范与施工方案（可交施工员） + +依据 CG-029（日常用成品视频、交谈用同一人物实时口型）、CG-005/006（换外观保留人格/记忆/关系）、CG-030（首批素材小队制作）。原则：单图不自动生成整套；每份素材可独立替换、可校验、可回滚。 + +## 1. 素材包目录规范 + +``` +assets/personas// # persona=外观方案，如 wanwan-v2；换外观=新目录 + manifest.json # 见下 + daily/ # 日常成品视频（CG-029 主路线） + .mp4 # H.264 25fps ≥6s 可无缝循环；按时段/天气/活动命名 + talk/ # 交谈底板（近景，同房间同脸同服装） + .mp4 # H.264 25fps 5~15s，人物目视镜头、口部微闭或自然微动 + voice/ # CosyVoice 音色登记名（.12 voices.json 中的 voice 名） + ref/ # 定妆照/场景照 PNG（身份对照用） +``` + +`manifest.json` 必填字段：`persona_id`、`identity_ref`（ref/ 内主定妆图）、`daily_clips[{id, timeofday, weather, activity, loop_points}]`、`talk_clips[{id, avatar_id, framing}]`、`created_by`、`created_at`、`pipeline_version`（如 hel715-v1）。 + +## 2. 身份/服装/光线一致性标记（导入校验项） + +- 同一 persona 内所有 daily/talk 片段必须与 `identity_ref` 同人物、同发型、同套服装；换服装=新 persona 目录。 +- 每段导入时人工对照 ref 图打勾（首版人工，不做自动人脸embedding校验——留待后续）。 +- 光线：同一 persona 的交谈底板光线需与常用日常场景一致（近景底板从日常成品裁切放大可天然保证，如本轮 talk-close = loop 1.75× 裁切）。 + +## 3. 可切节点与缺素材回退 + +- 日常片循环点：`loop_points` 记录首尾衔接帧（硬循环跳变处需美术确认或选首尾近同帧的段）。 +- 交谈进入点：任意时刻可切（预连接流，实测切换 0~3ms）；退出点同。 +- 缺素材回退顺序：请求场景无 daily 片 → 用同 persona 默认 daily 片；无 talk 底板 → 禁用交谈入口并提示，不得用其他人物素材顶替；TTS 失败 → 显示重试按钮，不静默。 + +## 4. 导入与回滚流程（本轮实测命令） + +``` +# 1) 预处理：任意源 → H.264 25fps（示例见 scripts/，PyAV，秒级） +# 2) 生成交谈 avatar（47~60s GPU/份，独立目录，无硬编码）： +cd /opt/hel715-verify/LiveTalking +venv/bin/python -m avatars.musetalk.genavatar --file --avatar_id - +# 3) 校验：data/avatars// 含 avator_info.json/coords.pkl/latents.pt/mask；avator_info.json 的 video_path 指向素材包内文件 +# 4) 服务挂载：app.py --avatar_id （或 custom_config 多 avatar 预载） +``` + +回滚：素材包按目录整体保留旧版本（`.bak-`），avatar 目录同理；manifest 记录版本，恢复=改回引用。模型权重不入 Git，assets 二进制走附件/对象存储，仓库只放 manifest 与脚本。 + +## 5. 换一套外观的实测成本（CG-005/006 输入） + +| 项 | 实测耗时 | 说明 | +| --- | --- | --- | +| 交谈底板由既有成品裁切 | <1min（PyAV 转码）+ 47~60s genavatar | 本轮 talk-close 路线，推荐 | +| 交谈底板由 Wan2.2 全新生成 | ~57s/3s 片段 + 挑选重试（3~5 倍余量） | HEL-712 实测单次推理 | +| 日常成品视频生成 | 同上量级/段 | 小队批量制作 | +| 照片需求 | 定妆/场景各 ≥1 张 | 用于身份对照与 i2v 起点 | +| 预处理 | 自动，无人工参数 | bbox_shift=0 默认可用 | + +## 6. 首版最小验收矩阵（交谈链路） + +| 项 | 通过标准 | +| --- | --- | +| 日常循环播放 | ≥3 周期无肉眼跳变（或美术确认衔接） | +| 进交谈 | 切换 ≤200ms，同人物同服装目视确认 | +| 口型 | 说话段口部动作肉眼可辨，静息段无明显异常 | +| 打断 | interrupt 后 ≤1s 静音，不等播完 | +| 再开口 | 打断后可立即再次发话 | +| 回日常 | 切回 ≤200ms，日常从暂停点续播 | +| 持续 | 单次会话 ≥10 分钟不崩、显存不涨 | +| 失败路径 | TTS 断连时有可见错误提示，可恢复 | + +## 7. 职责划分（衔接 HEL-714 施工） + +- 客户端（web）：日常选片/循环、交谈视图挂载（预连接 WebRTC）、打断信号（先按钮/快捷键，VAD 后置）、失败回退 UI。 +- 服务端：LiveTalking（交谈渲染 + TTS 适配，本分支 patch）；日常视频静态文件由现有 web 服务出。 +- 素材：美术/施工按第 1 节规范产出，总工验收导入。 diff --git a/experiments/hel715/README.md b/experiments/hel715/README.md new file mode 100644 index 0000000..f0c227f --- /dev/null +++ b/experiments/hel715/README.md @@ -0,0 +1,42 @@ +# HEL-715 实验复现说明 + +实验环境（.12，全部隔离在 `/opt/hel715-verify/`，未改系统 Python/CUDA/ASR/TTS）： + +- LiveTalking c726943（官方 GitHub）+ MuseTalk v1.5 权重 3.8GB（`LiveTalking/models/`） +- venv：`/opt/hel715-verify/venv`（torch 2.8.0+cu128、aiortc 1.15.0、PyAV 17.1.0） +- 既有服务复用：CosyVoice TTS `http://127.0.0.1:9880`（音色 xiaoyun）、LLM `http://192.168.200.11:8731`（halogen-qwen3.8-flash-next） +- 对上游的两处改动：`livetalking-patch/cosyvoice.py`（新增，对接 .12 CosyVoice）、`livetalking-patch/enable-cosyvoice-registration.diff`（base_avatar.py 启用注册） + +## 启动 + +```bash +cd /opt/hel715-verify/LiveTalking +nohup /opt/hel715-verify/venv/bin/python app.py --model musetalk \ + --avatar_id hel715-talk-close --transport webrtc \ + --tts cosyvoice --REF_FILE xiaoyun \ + --TTS_SERVER http://127.0.0.1:9880 --listenport 8010 \ + > /opt/hel715-verify/livetalking.log 2>&1 & +``` + +注意：仓库内 `config.yaml` 的 `TTS_SERVER: ''` 会覆盖 argparse 默认值，必须显式传 `--TTS_SERVER`（本轮曾因此空跑过一次）。 + +已生成 avatar（`LiveTalking/data/avatars/`）：`hel715-loop-wide`（loop-25fps.mp4，196 帧全景）、`hel715-talk-close`（talk-close.mp4，196 帧近景=loop 1.75× 裁切）、`hel715-return`（return-25fps.mp4，77 帧独立源）。 + +## 测试脚本 + +- `scripts/e2e_client.py`：aiortc 收流客户端，四句时间线（含打断），产出 mp4+stats.json+events.json+contact-sheet.png。在 .12 上运行：`E2E_OUT=... venv/bin/python e2e_client.py`。 +- `scripts/llm_chain2.py`：LLM→TTS→数字人单链路采样。 +- `scripts/hel715-chain.html`：日常→交谈→打断→回日常→再交谈演示页（部署于 LiveTalking `web/`，`?auto=1` 自动演示；依赖 `web/hel715-daily.webm`）。 +- `scripts/record_chain.py`：本机（192.168.200.13）Playwright+Chromium 录制上述页面，需 LD_LIBRARY_PATH 指向解包的运行库（脚本内已写死 /tmp 路径，迁移时改）。 + +## 清理 + +```bash +# 停实验服务（先对 PID：pgrep -af 'app.py --model musetalk'） +pkill -f 'app.py --model musetalk' +# 删除整个实验目录（含 venv/权重/素材，约 12GB） +rm -rf /opt/hel715-verify +# 演示页随目录一起消失；18080/18081 旧演示与 ASR/TTS 未动 +``` + +本轮收尾状态：LiveTalking 已停止；目录与权重保留供 HEL-714 复用。 diff --git a/experiments/hel715/REPORT.md b/experiments/hel715/REPORT.md new file mode 100644 index 0000000..8680b8e --- /dev/null +++ b/experiments/hel715/REPORT.md @@ -0,0 +1,73 @@ +# HEL-715 技术验证报告：本地真人交谈与日常视频衔接 + +日期：2026-10-20 ｜ 环境：192.168.200.12（RTX 5090D 32GB，torch 2.8.0+cu128） +候选：LiveTalking @ c726943（官方 GitHub）+ MuseTalk v1.5 + .12 既有 CosyVoice(9880) + .11 LLM(halogen-qwen3.8-flash-next) + +## 一句话结论 + +**有条件可用。** 日常↔交谈切换、打断、25fps 连续口型视频、素材流水线复用均实测通过；条件是接受 ①发话到出声约 4.5~5.5s 的起步延迟，②TTS 流式按标点分句带来的句内 0.4~1.1s 停顿，③交谈驱动只改口型（表情/目光来自底板视频）。三项都需要在首版验收时向用户如实展示，其中 ①② 可在施工阶段继续压。 + +## 四个问题的实测回答 + +### 1. 主候选 LiveTalking+MuseTalk 实际跑通了吗？——跑通 + +- 安装：官方仓库 clone 至 `/opt/hel715-verify/LiveTalking`，独立 venv（不改系统 Python/CUDA/ASR/TTS）；权重 musetalkV15/face-parse/sd-vae/whisper 共约 3.8GB，已就位。 +- 适配：仅两处改动（见 `livetalking-patch/`）：`tts/cosyvoice.py` 新增（对接 .12 既有 CosyVoice 的 POST /tts 24kHz PCM 流）+ `avatars/base_avatar.py` 启用该 TTS 注册。其余代码未动。 +- 能力边界（CG-028 相关，明确来源）： + - **口型**：实时音频驱动（whisper 特征→unet 逐帧生成下半脸）——实测（口部区差分：静息中位 0.74 vs 说话峰值 13~14）。 + - **表情/目光/头姿/身体**：全部来自底板视频循环（预处理时逐帧 landmark+face-parse 固化），**不是**实时生成——来源：代码结构+录像复核。交谈底板选片时必须挑"目视镜头、微动作"段。 + - 不要求也不提供逐句新肩部手势；与 CG-027/028 的预期一致，未发现必须换 Ditto 等替代的实证。 + +### 2. 同一人物实时交谈可否成立？——可以，起步延迟与句内停顿需接受 + +实测（aiortc 客户端走真实 WebRTC 通道，含网络传输；e2e-session.mp4 / stats.json）： + +| 指标 | 实测值 | +| --- | --- | +| 连接→首帧视频 | 3.6s（含 ICE 协商 1.7s） | +| 文本→首次出声 | 4.6~5.5s（CosyVoice 首包 3.4s + LiveTalking 音频缓冲约 1.2s） | +| 连续帧率 | 25.05fps 恒定（27s 全程，说话与静息一致） | +| 音画同步 | 容器内音视频齐备；口型动作较出声晚约 0.5~1.3s（MuseTalk 音频前瞻缓冲所致）。自动互相关度量噪声大，最终以录像人工复核为准 | +| 句内停顿 | TTS 按标点分句合成，句内间隙 0.45~1.05s（"抱歉"与"你刚想说什么"之间 0.66s）——CosyVoice 流式模式固有，非口型管线问题 | +| 打断→静音 | **0.35s**（interrupt_talk 后队列即清） | +| 打断后再开口 | 正常（S3/S4 均正常播出） | +| 峰值资源 | 总显存 18.9GB（基线 5.8GB + CosyVoice 4.2GB + LiveTalking 空载 11.2GB/说话峰值约 13.1GB）；GPU 利用率瞬时 100% 但帧率不跌 | +| 失败行为 | 本轮未触发服务崩溃；语音名不存在时 TTS 适配层回退 default 音色 | + +LLM 链路（llm-chain.mp4）：.11 真实模型 halogen-qwen3.8-flash-next 回复 1.33s → 递给数字人出声共 4.63s，全程录音正常。 + +### 3. 日常→交谈→打断→回日常衔接？——信号级链路实测通过 + +两份证据： + +- **浏览器真实链路**（chain/page@*.webm，Playwright+Chromium 走 LAN WebRTC）：日常成品视频循环 → 切交谈视图（预连接流，切换 0~3ms）→ 发文本 → 语音进行中注入 interrupt_talk（HTTP 2~5ms）→ 立即切回日常（不等她说完、不等日常片尾）→ 再次进入交谈并发第二句 → 回日常。全程 33.5s，页面零 JS 错误。注：Playwright 录屏不含音轨，音频证据见下条。 +- **媒体级连续音画**（e2e-session.mp4）：同一 WebRTC 通道收到的连续音视频原样落盘，覆盖 静息→说话→打断→静默→再说话 全程，有声有画。 + +边界说明：麦克风/VAD 未接入，打断由信号注入触发，**只是信号级验证**，不冒充自然语音抢话已通过。转场为视图级切换（同人物同房间同服装的近景底板 vs 日常全景），自然转场（如淡入淡出）属施工项，不计入本轮结论。 + +### 4. 素材能否持续制作/替换/维护？——流水线已验证可复用 + +- 同一流水线已跑通 **3 份素材、3 个 avatar**：loop 全景（196 帧）、面部近景（196 帧，经核实为 loop 素材 1.75× 放大裁切，同人物同房间同服装）、回程片段（77 帧，独立源文件）。 +- 新素材仅需：转码 25fps mp4 → `python -m avatars.musetalk.genavatar --file X --avatar_id Y`，47~60s/份（GPU），零代码改动、目录隔离、无硬编码人脸。 +- 成本实测：交谈底板若由 Wan2.2 生成约 57s/3s 片段（HEL-712 实测）；由既有成品裁切仅需秒级（PyAV）。详见 MATERIAL-SPEC.md。 +- 限制：本轮三份素材同属一个人物；跨人物泛化未实测（离线环境无第二真人素材可用），算法层面为通用 landmark/face-parse 管线，无按脸硬编码逻辑。 + +## CG-029/028 能力矩阵（来源/实测/未测） + +| 能力 | 状态 | 依据 | +| --- | --- | --- | +| 成品日常视频按时段/活动选片播放 | 实测（浏览器循环播放） | chain webm | +| 同人物近景实时口型交谈 | 实测 | e2e-session.mp4 / llm-chain.mp4 | +| 口型由音频实时驱动 | 实测 | 口部差分 15~19 倍 | +| 表情/目光实时生成 | 未测（来源=底板视频） | 代码结构 | +| 语音打断（信号级） | 实测 0.35s 停止 | events.json | +| 自然语音抢话（VAD） | 未测 | 麦克风未接 | +| 日常↔交谈视图切换 | 实测 0~3ms | chain webm + 日志 | +| LLM→TTS→口型全链 | 实测 | llm-chain.mp4 | +| 素材替换保留人格记忆 | 设计已验证路径（素材目录隔离） | genavatar 复用 | + +## 风险与建议（给总管的决策输入） + +1. **起步延迟 4.5~5.5s** 是当前体验最大短板：可在施工时用"思考中"微动作/状态提示掩盖，或后续换更低首包延迟的 TTS 配置（CosyVoice 首包 3.4s 占大头）。无已确认延迟门槛，不编造 SLA；建议向用户展示实测录像后再定验收线。 +2. **句内停顿**可通过 TTS 侧整句缓冲或标点预处理缓解，代价是首包更慢——二者权衡留施工阶段。 +3. LiveTalking 空载即占 11.2GB 显存，与 ComfyUI 重生成任务并存时需调度（先交谈后生成的队列约定）。 diff --git a/experiments/hel715/livetalking-patch/adapter-and-diff.txt b/experiments/hel715/livetalking-patch/adapter-and-diff.txt new file mode 100644 index 0000000..4c783c1 --- /dev/null +++ b/experiments/hel715/livetalking-patch/adapter-and-diff.txt @@ -0,0 +1,82 @@ +import time +import numpy as np +import resampy +import requests +from io import BytesIO +from typing import Iterator + +from utils.logger import logger +from .base_tts import BaseTTS, State +from registry import register + + +@register("tts", "cosyvoice") +class CosyVoiceTTS(BaseTTS): + """适配本机既有 CosyVoice3 流式 TTS (POST /tts -> 24kHz int16 PCM 流).""" + + def txt_to_audio(self, msg: tuple[str, dict]): + text, textevent = msg + voice = textevent.get('tts', {}).get('voice', self.opt.REF_FILE or 'default') + self.stream_tts(self.cosyvoice(text, voice), msg) + + def cosyvoice(self, text: str, voice: str) -> Iterator[bytes]: + start = time.perf_counter() + try: + res = requests.post( + f"{self.opt.TTS_SERVER}/tts", + json={"text": text, "voice": voice}, + stream=True, + timeout=(5, 300), + ) + if res.status_code != 200: + logger.error("cosyvoice http %s: %s", res.status_code, res.text[:200]) + return + first = True + for chunk in res.iter_content(chunk_size=4800): # 100ms @24k int16 + if first: + logger.info("cosyvoice time to first chunk: %.3fs", time.perf_counter() - start) + first = False + if chunk and self.state == State.RUNNING: + yield chunk + elif self.state != State.RUNNING: + logger.info("cosyvoice interrupted, stop consuming") + res.close() + return + except Exception: + logger.exception('cosyvoice tts error') + + def stream_tts(self, audio_stream: Iterator[bytes], msg: tuple[str, dict]): + text, textevent = msg + first = True + for chunk in audio_stream: + if not chunk: + continue + pcm24 = np.frombuffer(chunk, dtype=np.int16).astype(np.float32) / 32767.0 + stream = resampy.resample(pcm24, sr_orig=24000, sr_new=self.sample_rate) + streamlen = stream.shape[0] + idx = 0 + while streamlen >= self.chunk: + eventpoint = {} + if first: + eventpoint = {'status': 'start', 'text': text} + first = False + eventpoint.update(**textevent) + self.parent.put_audio_frame(stream[idx:idx + self.chunk], eventpoint) + streamlen -= self.chunk + idx += self.chunk + eventpoint = {'status': 'end', 'text': text} + eventpoint.update(**textevent) + self.parent.put_audio_frame(np.zeros(self.chunk, np.float32), eventpoint) +diff --git a/avatars/base_avatar.py b/avatars/base_avatar.py +index 28c826a..a9744e6 100644 +--- a/avatars/base_avatar.py ++++ b/avatars/base_avatar.py +@@ -90,7 +90,7 @@ class BaseAvatar: + 'edgetts': 'tts.edge', + 'gpt-sovits': 'tts.sovits', + 'xtts': 'tts.xtts', +- # 'cosyvoice': 'tts.cosyvoice', ++ 'cosyvoice': 'tts.cosyvoice', + # 'fishtts': 'tts.fish', + 'tencent': 'tts.tencent', + 'doubao': 'tts.doubao', diff --git a/experiments/hel715/livetalking-patch/cosyvoice.py b/experiments/hel715/livetalking-patch/cosyvoice.py new file mode 100644 index 0000000..e7d120e --- /dev/null +++ b/experiments/hel715/livetalking-patch/cosyvoice.py @@ -0,0 +1,69 @@ +import time +import numpy as np +import resampy +import requests +from io import BytesIO +from typing import Iterator + +from utils.logger import logger +from .base_tts import BaseTTS, State +from registry import register + + +@register("tts", "cosyvoice") +class CosyVoiceTTS(BaseTTS): + """适配本机既有 CosyVoice3 流式 TTS (POST /tts -> 24kHz int16 PCM 流).""" + + def txt_to_audio(self, msg: tuple[str, dict]): + text, textevent = msg + voice = textevent.get('tts', {}).get('voice', self.opt.REF_FILE or 'default') + self.stream_tts(self.cosyvoice(text, voice), msg) + + def cosyvoice(self, text: str, voice: str) -> Iterator[bytes]: + start = time.perf_counter() + try: + res = requests.post( + f"{self.opt.TTS_SERVER}/tts", + json={"text": text, "voice": voice}, + stream=True, + timeout=(5, 300), + ) + if res.status_code != 200: + logger.error("cosyvoice http %s: %s", res.status_code, res.text[:200]) + return + first = True + for chunk in res.iter_content(chunk_size=4800): # 100ms @24k int16 + if first: + logger.info("cosyvoice time to first chunk: %.3fs", time.perf_counter() - start) + first = False + if chunk and self.state == State.RUNNING: + yield chunk + elif self.state != State.RUNNING: + logger.info("cosyvoice interrupted, stop consuming") + res.close() + return + except Exception: + logger.exception('cosyvoice tts error') + + def stream_tts(self, audio_stream: Iterator[bytes], msg: tuple[str, dict]): + text, textevent = msg + first = True + for chunk in audio_stream: + if not chunk: + continue + pcm24 = np.frombuffer(chunk, dtype=np.int16).astype(np.float32) / 32767.0 + stream = resampy.resample(pcm24, sr_orig=24000, sr_new=self.sample_rate) + streamlen = stream.shape[0] + idx = 0 + while streamlen >= self.chunk: + eventpoint = {} + if first: + eventpoint = {'status': 'start', 'text': text} + first = False + eventpoint.update(**textevent) + self.parent.put_audio_frame(stream[idx:idx + self.chunk], eventpoint) + streamlen -= self.chunk + idx += self.chunk + eventpoint = {'status': 'end', 'text': text} + eventpoint.update(**textevent) + self.parent.put_audio_frame(np.zeros(self.chunk, np.float32), eventpoint) diff --git a/experiments/hel715/livetalking-patch/enable-cosyvoice-registration.diff b/experiments/hel715/livetalking-patch/enable-cosyvoice-registration.diff new file mode 100644 index 0000000..adfb04b --- /dev/null +++ b/experiments/hel715/livetalking-patch/enable-cosyvoice-registration.diff @@ -0,0 +1,13 @@ +diff --git a/avatars/base_avatar.py b/avatars/base_avatar.py +index 28c826a..a9744e6 100644 +--- a/avatars/base_avatar.py ++++ b/avatars/base_avatar.py +@@ -90,7 +90,7 @@ class BaseAvatar: + 'edgetts': 'tts.edge', + 'gpt-sovits': 'tts.sovits', + 'xtts': 'tts.xtts', +- # 'cosyvoice': 'tts.cosyvoice', ++ 'cosyvoice': 'tts.cosyvoice', + # 'fishtts': 'tts.fish', + 'tencent': 'tts.tencent', + 'doubao': 'tts.doubao', diff --git a/experiments/hel715/scripts/e2e_client.py b/experiments/hel715/scripts/e2e_client.py new file mode 100644 index 0000000..47583f3 --- /dev/null +++ b/experiments/hel715/scripts/e2e_client.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python +"""HEL-715 end-to-end verification client for LiveTalking(MuseTalk) @ 127.0.0.1:8010. + +Connects as a WebRTC receiver (same path a browser takes), drives the avatar via +/human + /interrupt_talk, records the received A/V to mp4 and logs timing stats. +""" +import asyncio +import json +import os +import time + +import aiohttp +import av +import numpy as np +from aiortc import RTCPeerConnection, RTCSessionDescription + +BASE = os.environ.get("LT_BASE", "http://127.0.0.1:8010") +OUT = os.environ.get("E2E_OUT", "/opt/hel715-verify/e2e") +os.makedirs(OUT, exist_ok=True) + +S1 = "在呢。你回来啦，今天在外面累不累？" +S2 = "还有，我跟你说，今天下午我在窗边看了好久的天空，云跑得特别快，有一朵特别像小兔子，我还给它拍了张照片，等下拿给你看。" +S3 = "抱歉，你刚想说什么？我在听。" +S4 = "嗯，我在。" + +# approx mouth region (lower third of face bbox 218,203,324,333 from avatar prep) +MOUTH = (222, 268, 320, 331) + +log_lines = [] + + +def log(msg): + line = f"[{time.strftime('%H:%M:%S')}] {msg}" + print(line, flush=True) + log_lines.append(line) + + +class Recorder: + def __init__(self, path, w, h): + self.container = av.open(path, "w") + self.vstream = self.container.add_stream("libx264", rate=25) + self.vstream.width = w + self.vstream.height = h + self.vstream.pix_fmt = "yuv420p" + self.vstream.options = {"crf": "20", "preset": "veryfast"} + self.astream = self.container.add_stream("aac", rate=48000) + self.astream.layout = "stereo" + + def write_video(self, img_bgr): + frame = av.VideoFrame.from_ndarray(img_bgr, format="bgr24") + for p in self.vstream.encode(frame): + self.container.mux(p) + + def write_audio(self, pcm_i16, sr, ch): + arr = pcm_i16.astype(np.int16) + if ch == 1: + arr = np.repeat(arr[:, None], 2, axis=1).reshape(-1) + elif ch == 2: + arr = arr.reshape(-1) + else: + arr = arr[: len(arr) // ch * ch].reshape(-1, ch)[:, :2].reshape(-1) + if sr != 48000: + n = len(arr) // 2 + idx = (np.arange(n) * (sr / 48000.0)).astype(int) + idx = idx[idx < len(arr) // 2] + arr = arr.reshape(-1, 2)[idx].reshape(-1) + frame = av.AudioFrame.from_ndarray( + np.ascontiguousarray(arr.reshape(1, -1)), format="s16", layout="stereo" + ) + frame.sample_rate = 48000 + for p in self.astream.encode(frame): + self.container.mux(p) + + def close(self): + for p in self.vstream.encode(): + self.container.mux(p) + for p in self.astream.encode(): + self.container.mux(p) + self.container.close() + + +class Session: + def __init__(self): + self.events = {"video": [], "audio": [], "speak": [], "actions": []} + self.imgs_sample = [] + self.sid = None + self.http = None + + async def post(self, path, payload): + t0 = time.perf_counter() + async with self.http.post(BASE + path, json=payload) as r: + body = await r.json() + return time.perf_counter() - t0, body + + async def poll_speaking(self, stop_evt): + while not stop_evt.is_set(): + try: + dt, body = await self.post("/is_speaking", {"sessionid": self.sid}) + self.events["speak"].append( + [round(time.perf_counter(), 3), body.get("data", False)] + ) + except Exception: + pass + await asyncio.sleep(0.25) + + +def mouth_motion(prev, cur): + if prev is None or cur is None: + return 0.0 + x0, y0, x1, y1 = MOUTH + d = np.abs( + cur[y0:y1, x0:x1].astype(np.int16) - prev[y0:y1, x0:x1].astype(np.int16) + ).mean() + return float(d) + + +async def main(): + stats = {"sentences": {}, "meta": {}} + rec = None + pc = RTCPeerConnection() + sess = Session() + t_start = time.perf_counter() + + done = asyncio.Event() + + @pc.on("track") + def on_track(track): + log(f"track: {track.kind}") + if track.kind == "video": + asyncio.ensure_future(video_loop(track)) + else: + asyncio.ensure_future(audio_loop(track)) + + async def video_loop(track): + nonlocal rec + prev = None + while True: + try: + frame = await track.recv() + except Exception: + break + img = frame.to_ndarray(format="bgr24") + if rec is None: + rec = Recorder(os.path.join(OUT, "e2e-session.mp4"), img.shape[1], img.shape[0]) + log(f"recorder ready {img.shape[1]}x{img.shape[0]}") + t = time.perf_counter() + m = mouth_motion(prev, img) + sess.events["video"].append([round(t, 3), round(m, 2)]) + prev = img + rec.write_video(img) + if len(sess.imgs_sample) < 400: + sess.imgs_sample.append((round(t - t_start, 2), img)) + + async def audio_loop(track): + while True: + try: + frame = await track.recv() + except Exception: + break + t = time.perf_counter() + arr = frame.to_ndarray() + if arr.dtype != np.int16: + if arr.dtype in (np.float32, np.float64): + arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) + else: + arr = arr.astype(np.int16) + ch = len(frame.layout.channels) + if arr.ndim == 2 and arr.shape[0] == 1: + pcm = arr.reshape(-1) # packed s16 + elif arr.ndim == 2 and arr.shape[0] == ch: + pcm = arr.T.reshape(-1) # planar + else: + pcm = arr.reshape(-1) + ch = 1 + rms = float(np.sqrt((pcm.astype(np.float64) ** 2).mean()) / 32767.0) + sess.events["audio"].append([round(t, 3), round(rms, 5)]) + if rec is not None: + rec.write_audio(pcm, frame.sample_rate, ch) + + async with aiohttp.ClientSession() as http: + sess.http = http + pc.addTransceiver("video", direction="recvonly") + pc.addTransceiver("audio", direction="recvonly") + offer = await pc.createOffer() + await pc.setLocalDescription(offer) + while pc.iceGatheringState != "complete": + await asyncio.sleep(0.1) + t0 = time.perf_counter() + async with http.post( + BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"} + ) as r: + answer = await r.json() + log(f"offer round-trip {time.perf_counter()-t0:.3f}s sessionid={answer.get('sessionid')}") + sess.sid = answer["sessionid"] + await pc.setRemoteDescription( + RTCSessionDescription(sdp=answer["sdp"], type=answer["type"]) + ) + stats["meta"]["sessionid"] = sess.sid + + stop_evt = asyncio.Event() + poller = asyncio.ensure_future(sess.poll_speaking(stop_evt)) + + def mark(name): + t = time.perf_counter() + sess.events["actions"].append([round(t, 3), name]) + log(f"ACTION {name}") + return t + + # wait for first video frame / idle + for _ in range(200): + if sess.events["video"]: + break + await asyncio.sleep(0.1) + first_v = sess.events["video"][0][0] if sess.events["video"] else -1 + first_a = sess.events["audio"][0][0] if sess.events["audio"] else -1 + stats["meta"]["first_video_s"] = round(first_v - t_start, 3) + stats["meta"]["first_audio_s"] = round(first_a - t_start, 3) + log(f"first video {stats['meta']['first_video_s']}s audio {stats['meta']['first_audio_s']}s") + await asyncio.sleep(3.0) # idle baseline + + async def speak(name, text, wait_end=True, interrupt_after=None, next_line=None): + t_send = mark(f"send:{name}") + dt, body = await sess.post( + "/human", {"sessionid": sess.sid, "type": "echo", "text": text} + ) + log(f"/human {name} ack {dt:.3f}s {body}") + t_interrupt = None + if interrupt_after is not None: + await asyncio.sleep(interrupt_after) + t_interrupt = mark(f"interrupt:{name}") + dt2, b2 = await sess.post("/interrupt_talk", {"sessionid": sess.sid}) + log(f"/interrupt_talk ack {dt2:.3f}s {b2}") + if wait_end: + # wait until speaking False and audio silent + silent = 0.0 + last_speak_true = time.perf_counter() + while True: + await asyncio.sleep(0.1) + now = time.perf_counter() + sp = sess.events["speak"][-1][1] if sess.events["speak"] else None + if sp: + last_speak_true = now + recent = [a for a in sess.events["audio"] if a[0] > now - 0.6] + loud = any(a[1] > 0.02 for a in recent) + if not sp and not loud: + silent = now - last_speak_true + if silent > 1.2: + break + else: + last_speak_true = max(last_speak_true, now) + return t_send, t_interrupt + + # S1 short + s, _ = await speak("S1", S1) + stats["sentences"]["S1"] = {"text": S1, "sent_at": round(s - t_start, 2)} + await asyncio.sleep(1.5) + # S2 long with interrupt 3s after send + s2, ti = await speak("S2", S2, wait_end=True, interrupt_after=3.0) + stats["sentences"]["S2"] = { + "text": S2, "sent_at": round(s2 - t_start, 2), + "interrupt_at": round(ti - t_start, 2), + } + await asyncio.sleep(2.0) + # S3 re-open after interrupt + s3, _ = await speak("S3", S3) + stats["sentences"]["S3"] = {"text": S3, "sent_at": round(s3 - t_start, 2)} + await asyncio.sleep(3.0) # pause test + s4, _ = await speak("S4", S4) + stats["sentences"]["S4"] = {"text": S4, "sent_at": round(s4 - t_start, 2)} + mark("end") + await asyncio.sleep(2.0) + stop_evt.set() + poller.cancel() + await pc.close() + + if rec: + rec.close() + log("recording closed") + + # analysis + def rel(ts): + return round(ts - t_start, 3) + + ev_a = sess.events["audio"] + ev_v = sess.events["video"] + for name, info in stats["sentences"].items(): + sent = info.get("sent_at", 0) + t_start + aud = [a for a in ev_a if a[0] >= sent and a[1] > 0.02] + info["first_audible_rel_s"] = rel(aud[0][0]) if aud else None + if aud: + last = sent + for a in aud: + if a[0] - last < 1.5: + last = a[0] + info["last_audible_rel_s"] = rel(last) + if "interrupt_at" in info: + it = info["interrupt_at"] + t_start + after = [a for a in ev_a if a[0] >= it and a[1] > 0.02] + info["audible_after_interrupt_s"] = rel(after[0][0]) if after else None + tail = [a for a in ev_a if a[0] >= it] + tl = [a for a in tail if a[1] > 0.02] + info["last_audible_after_interrupt_rel_s"] = rel(tl[-1][0]) if tl else None + + # video motion around each sentence + for name, info in stats["sentences"].items(): + sent = info.get("sent_at", 0) + t_start + window = [v for v in ev_v if sent - 1.0 <= v[0] <= sent + 12] + base = [v for v in ev_v if v[0] < sent - 1.0][-60:] + if window and base: + b = np.median([v[1] for v in base]) + info["mouth_motion_idle"] = round(float(b), 2) + peak = max(v[1] for v in window) + info["mouth_motion_peak"] = round(float(peak), 2) + + # fps over speech windows + if ev_v: + dur = ev_v[-1][0] - ev_v[0][0] + stats["meta"]["video_frames"] = len(ev_v) + stats["meta"]["video_span_s"] = round(dur, 2) + stats["meta"]["avg_recv_fps"] = round(len(ev_v) / dur, 2) if dur > 0 else None + if ev_a: + dur = ev_a[-1][0] - ev_a[0][0] + stats["meta"]["audio_frames"] = len(ev_a) + stats["meta"]["audio_span_s"] = round(dur, 2) + + stats["meta"]["wall_total_s"] = round(time.perf_counter() - t_start, 2) + + with open(os.path.join(OUT, "stats.json"), "w") as f: + json.dump(stats, f, ensure_ascii=False, indent=1) + with open(os.path.join(OUT, "events.json"), "w") as f: + json.dump( + { + "t_start": t_start, + "video": sess.events["video"], + "audio": sess.events["audio"], + "speak": sess.events["speak"], + "actions": sess.events["actions"], + }, + f, + ) + with open(os.path.join(OUT, "run.log"), "w") as f: + f.write("\n".join(log_lines)) + + # contact sheet: pick frames around S1 onset, S2 mid, interrupt, S3 + try: + from PIL import Image + + picks = [] + s1 = stats["sentences"]["S1"].get("first_audible_rel_s") or 5 + s2i = stats["sentences"]["S2"].get("interrupt_at") + picks += [max(0, s1 - 1.5), s1 + 0.5, s1 + 2.5] + if s2i: + picks += [s2i - 1.0, s2i + 0.3, s2i + 1.5] + picks += [stats["sentences"]["S3"]["sent_at"] + 2.0] + tiles = [] + for p in picks: + cand = min(sess.imgs_sample, key=lambda x: abs(x[0] - p)) + im = Image.fromarray(cand[1][:, :, ::-1]).resize((276, 276)) + tiles.append((cand[0], im)) + sheet = Image.new("RGB", (276 * len(tiles), 276 + 18), "black") + from PIL import ImageDraw + + d = ImageDraw.Draw(sheet) + for i, (ts, im) in enumerate(tiles): + sheet.paste(im, (i * 276, 18)) + d.text((i * 276 + 4, 2), f"t={ts:.1f}s", fill="white") + sheet.save(os.path.join(OUT, "contact-sheet.png")) + except Exception as e: + log(f"contact sheet failed: {e}") + + log("DONE") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/experiments/hel715/scripts/hel715-chain.html b/experiments/hel715/scripts/hel715-chain.html new file mode 100644 index 0000000..61949b0 --- /dev/null +++ b/experiments/hel715/scripts/hel715-chain.html @@ -0,0 +1,129 @@ + + + + +HEL-715 日常→交谈→打断→日常 链路演示 + + + +
+ + +
0.0s
+
+
初始化…
成品日常视频循环播放 · 交谈流后台预连接
+
+ + + diff --git a/experiments/hel715/scripts/llm_chain2.py b/experiments/hel715/scripts/llm_chain2.py new file mode 100644 index 0000000..3d7e302 --- /dev/null +++ b/experiments/hel715/scripts/llm_chain2.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +"""LLM-chained conversation sample: .11 LLM -> .12 CosyVoice -> MuseTalk avatar. +Same receiver architecture as e2e_client.py (proven to capture A/V).""" +import asyncio, json, os, time +import aiohttp, av, numpy as np +from aiortc import RTCPeerConnection, RTCSessionDescription + +BASE = "http://127.0.0.1:8010" +LLM = "http://192.168.200.11:8731/v1/chat/completions" +OUT = "/opt/hel715-verify/e2e" +os.makedirs(OUT, exist_ok=True) + + +class Recorder: + def __init__(self, path, w, h): + self.container = av.open(path, "w") + self.v = self.container.add_stream("libx264", rate=25) + self.v.width, self.v.height, self.v.pix_fmt = w, h, "yuv420p" + self.v.options = {"crf": "20", "preset": "veryfast"} + self.a = self.container.add_stream("aac", rate=48000) + self.a.layout = "stereo" + + def video(self, img): + for p in self.v.encode(av.VideoFrame.from_ndarray(img, format="bgr24")): + self.container.mux(p) + + def audio(self, frame): + arr = frame.to_ndarray() + if arr.dtype != np.int16: + arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) + ch = len(frame.layout.channels) + if arr.ndim == 2 and arr.shape[0] == 1: + pcm = arr.reshape(-1) + elif arr.ndim == 2 and arr.shape[0] == ch: + pcm = arr.T.reshape(-1) + else: + pcm = arr.reshape(-1) + fr = av.AudioFrame.from_ndarray(np.ascontiguousarray(pcm.reshape(1, -1)), format="s16", layout="stereo") + fr.sample_rate = 48000 + for p in self.a.encode(fr): + self.container.mux(p) + + def close(self): + for p in self.v.encode(): self.container.mux(p) + for p in self.a.encode(): self.container.mux(p) + self.container.close() + + +async def main(): + t0 = time.perf_counter() + result = {} + async with aiohttp.ClientSession() as http: + async with http.post(LLM, json={ + "model": "halogen-qwen3.8-flash-next", + "messages": [{"role": "user", "content": "你是住在高层公寓卧室里的伴侣晚晚，用一两句自然中文回应：我回来了，今天好累啊。"}], + "max_tokens": 80, + }) as r: + body = await r.json() + reply = body["choices"][0]["message"]["content"] + result["llm_model"] = body["model"] + result["llm_reply"] = reply + result["llm_latency_s"] = round(time.perf_counter() - t0, 2) + print(f"[{result['llm_latency_s']}s] LLM: {reply}", flush=True) + + pc = RTCPeerConnection() + rec = {"r": None} + audio_marks = [] + + @pc.on("track") + def on_track(track): + async def drain(): + while True: + try: + f = await track.recv() + except Exception: + break + t = time.perf_counter() + if track.kind == "video": + img = f.to_ndarray(format="bgr24") + if rec["r"] is None: + rec["r"] = Recorder(f"{OUT}/llm-chain.mp4", img.shape[1], img.shape[0]) + rec["r"].video(img) + else: + arr = f.to_ndarray() + if arr.dtype != np.int16: + arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) + rms = float(np.sqrt((arr.astype(np.float64) ** 2).mean()) / 32767) + audio_marks.append([round(t - t0, 3), round(rms, 5)]) + if rec["r"] is not None: + rec["r"].audio(f) + asyncio.ensure_future(drain()) + + pc.addTransceiver("video", direction="recvonly") + pc.addTransceiver("audio", direction="recvonly") + offer = await pc.createOffer() + await pc.setLocalDescription(offer) + while pc.iceGatheringState != "complete": + await asyncio.sleep(0.1) + async with http.post(BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r: + answer = await r.json() + await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"])) + sid = answer["sessionid"] + await asyncio.sleep(3) + t_send = time.perf_counter() - t0 + async with http.post(BASE + "/human", json={"sessionid": sid, "type": "echo", "text": reply}) as r: + await r.json() + print(f"[{t_send:.2f}s] sent to avatar", flush=True) + await asyncio.sleep(15) + await pc.close() + + if rec["r"]: + rec["r"].close() + loud = [m for _, m in audio_marks if _ and m > 0.02] + first_loud = next((t for t, m in audio_marks if m > 0.02), None) + result["sent_at_s"] = round(t_send, 2) + result["first_audible_s"] = round(first_loud, 2) if first_loud else None + result["send_to_audible_s"] = round(first_loud - t_send, 2) if first_loud else None + with open(f"{OUT}/llm-chain-stats.json", "w") as f: + json.dump(result, f, ensure_ascii=False, indent=1) + print(json.dumps(result, ensure_ascii=False), flush=True) + print("DONE", flush=True) + + +asyncio.run(main()) diff --git a/experiments/hel715/scripts/record_chain.py b/experiments/hel715/scripts/record_chain.py new file mode 100644 index 0000000..0a32873 --- /dev/null +++ b/experiments/hel715/scripts/record_chain.py @@ -0,0 +1,59 @@ +import asyncio +import json +import os +import pathlib + +os.environ["LD_LIBRARY_PATH"] = ( + "/tmp/multica-task-4070005680/debs/root/usr/lib/x86_64-linux-gnu" + + (":" + os.environ["LD_LIBRARY_PATH"] if os.environ.get("LD_LIBRARY_PATH") else "") +) + +from playwright.async_api import async_playwright + +OUT = pathlib.Path("/tmp/multica-task-4070005680/opencode/chain") +OUT.mkdir(exist_ok=True) +URL = "http://192.168.200.12:8010/hel715-chain.html?auto=1" + + +async def main(): + async with async_playwright() as p: + browser = await p.chromium.launch( + args=["--autoplay-policy=no-user-gesture-required", "--no-sandbox"] + ) + ctx = await browser.new_context( + viewport={"width": 1280, "height": 800}, + record_video_dir=str(OUT), + record_video_size={"width": 1280, "height": 800}, + ) + page = await ctx.new_page() + errors = [] + page.on("pageerror", lambda e: errors.append(str(e))) + await page.goto(URL, wait_until="domcontentloaded") + # wait for demo end marker (auto demo ~37s) + try: + await page.wait_for_function( + "() => window.__chainLog && window.__chainLog.some(l => l.includes('AUTO-DEMO-END'))", + timeout=60000, + ) + except Exception as e: + errors.append(f"wait_for_function: {e}") + await asyncio.sleep(1.5) + logs = await page.evaluate("() => window.__chainLog") + metrics = { + "url": URL, + "chain_log": logs, + "pageerrors": errors, + } + (OUT / "chain-browser-log.json").write_text( + json.dumps(metrics, ensure_ascii=False, indent=1) + ) + video = page.video + await ctx.close() + path = await video.path() + await browser.close() + print("video:", path) + print("errors:", errors) + print(json.dumps(logs, ensure_ascii=False, indent=1)) + + +asyncio.run(main())