From 415f8372e3759da3fd6e9d362bbb4eebd00a3144 Mon Sep 17 00:00:00 2001 From: mobai Date: Wed, 21 Oct 2026 01:18:41 +0800 Subject: [PATCH] fix(HEL-718): bound synthetic idle audio and preserve RTP cadence --- samples/companion/README.md | 37 ++++ samples/companion/REPORT-HEL718.md | 64 ++++++ samples/companion/tools/check_driver_audio.py | 92 +++++++++ samples/companion/tools/driver_audio.py | 35 ++++ samples/companion/tools/graph_server.py | 41 ++++ samples/companion/tools/measure_sync.py | 192 ++++++++++++++++++ samples/companion/tools/patch_driver.py | 72 +++++++ samples/companion/tools/record_driver.py | 149 ++++++++++++++ samples/companion/tools/record_graph.html | 89 ++++++++ samples/companion/tools/record_graph.py | 32 +++ samples/companion/tools/record_product.py | 59 ++++++ samples/companion/tools/trace_driver.py | 40 ++++ 12 files changed, 902 insertions(+) create mode 100644 samples/companion/REPORT-HEL718.md create mode 100644 samples/companion/tools/check_driver_audio.py create mode 100644 samples/companion/tools/driver_audio.py create mode 100644 samples/companion/tools/graph_server.py create mode 100644 samples/companion/tools/measure_sync.py create mode 100644 samples/companion/tools/record_driver.py create mode 100644 samples/companion/tools/record_graph.html create mode 100644 samples/companion/tools/record_graph.py create mode 100644 samples/companion/tools/record_product.py create mode 100644 samples/companion/tools/trace_driver.py diff --git a/samples/companion/README.md b/samples/companion/README.md index 7141d8a..b212710 100644 --- a/samples/companion/README.md +++ b/samples/companion/README.md @@ -78,3 +78,40 @@ LD_LIBRARY_PATH=/home/xiaomo/pwlibs/root/usr/lib/x86_64-linux-gnu python3 tools/ 浏览器脚本依赖运行机已安装 Playwright/Chromium；麦克风脚本需复核包内的 `evidence/synthetic-microphone.wav`。它是 CosyVoice 合成的测试句，经 Chromium 假麦克风送入实际 Worklet→ASR→LLM→TTS→口型链路，绝不是人现场说话。录制用页面实际绘制画面和实际播放的 WebAudio 输出、同一个 MediaRecorder 时钟，不按帧数重建时间戳；静音时钟从开头持续输出零样本，保留进入前的日常画面。连续音画是浏览器播放图的合成采集，不是物理屏幕/扬声器校准。 实测结果和未通过项见 `REPORT.md`。生成工作流、history、原始候选片、浏览器证据和资源采样放复核附件，不在主仓库存大文件。 + +## HEL-718 驱动器补丁（候选，尚未通过同步门槛） + +`tools/patch_driver.py` 追加类型透传：`AudioFrameData.type` 经 +`BaseAvatar.process_frames → WebRTCOutput → HumanPlayer` 到未赋 PTS 的 +`PendingAudio`。仅合成空闲 `type=1` 且无事件的块限至3块（60ms）； +真实语音（包括零值 PCM）、自定义音频和事件保留，100块队列提供背压。 +首次音频随真实首帧建立时间原点，此后轨道每20ms从队列取最新待播块， +饥饿时补320样本静音；所有已输出包保持320步长，不在 RTP 队列跳时间戳。 +时钟改用 monotonic，`warm_up` 排空左右全部上下文，不移动 Whisper 特征窗。 +取消仍销毁本轮 peer，补丁不改变前后端取消协议。 + +先用 `prepare_driver.py` 新建私有副本，再运行补丁；源为已选 c726943。 +已验证从 P10 干净副本首次及重复应用，拒绝直接修改 P10 复用目录。 +`driver_audio.py` 必须与 `patch_driver.py` 一起携带。回滚时停止自己的候选 +服务，重建全新 c726943 副本，使用 `git show ae4fd5c:samples/companion/tools/patch_driver.py` +导出的旧版补丁恢复 HEL-714 endpoint，再指向原 avatar/权重；不要原地混用新旧文件。 + +新增有效检查（使用原有 P10 venv，无新增依赖）： + +```bash +/opt/hel715-verify/venv/bin/python tools/check_driver_audio.py /path/to/private/LiveTalking +# 按真实 RTP PTS 录制刚进入、空闲15秒、连续多轮、打断后再次送入相同音频 +/opt/hel715-verify/venv/bin/python tools/record_driver.py \ + --wav fixed.wav --out server-rounds.mp4 --rounds 0,15,1,1 --post 9 +``` + +`trace_driver.py` 仅用于隔离测试，以 `PYTHONPATH=私有驱动器目录` 从该目录运行， +首参为 JSONL 路径，其余为原 `app.py` 参数。记录逐块生产/接收 RMS、类型、深度、 +丢弃合成静音计数、PTS 和 monotonic 墙钟，不随产品默认开启。 +`record_graph.html/graph_server.py/record_graph.py` 复现浏览器产品音频图； +`record_product.py` 记录三轮日常→进场→交谈→说话中取消→正向回程→续读。 +`measure_sync.py` 固定 ROI 和阈值并保存检测器 SHA256、完整命令、offset、窗口、 +idle-until、输入/底板哈希、检测误差及逐 lag 相关性。相关性很低时不能把最佳 lag 当作可靠对齐。 + +本轮仍未达到服务端绝对偏差≤0.15秒/浏览器≤0.25秒，详见 `REPORT-HEL718.md`。 +本补丁解决的是可确认的空闲队列/时钟问题，不能据此宣称音画同步已修好。 diff --git a/samples/companion/REPORT-HEL718.md b/samples/companion/REPORT-HEL718.md new file mode 100644 index 0000000..62a72ff --- /dev/null +++ b/samples/companion/REPORT-HEL718.md @@ -0,0 +1,64 @@ +HEL-718 返工记录，2026-10-21。候选沿草稿 PR #2 / `hel-714-daily-talk`，base `main@1e9b08a`，施工基线 `ae4fd5c`。结论：**blocked，同步未修好，不发布、不合并、不转正式 PR。** + +## 实现与实测结论 + +- `tools/patch_driver.py`、`tools/driver_audio.py`：在私有 c726943 驱动器透传 `AudioFrameData.type`，未赋 PTS 的合成空闲块最多3块；真实语音（包括零 PCM）、自定义音频和事件不丢，100块总队列背压。音频饥饿补零保持320样本/20ms时钟；时钟使用 monotonic；warm_up 排空左右全部上下文。 +- 追加4项有效回归：2000次空闲/语音混入顺序与事件保留、真实音频满队列背压、warm_up 后首块关联、饥饿期间 PTS/节奏与首块 PCM 字节哈希。P10 干净副本首次及重复应用通过，旧取消 endpoint 保留。 +- 不改前端播放延迟、不移动 Whisper 特征窗、不改转场、素材、接口或结束检测。其他输出 transport 保持原来的 `push_audio_frame`，新增类型入口仅作用于 WebRTC。 +- 自测与分析工具纳入 `tools/`；复现和回滚步骤见 README 的 HEL-718 节。 + +静音积压与时钟问题已得到有效限制，**残余起点仍超门槛**。独立单会话约66秒：最大音频队列27块（含语音），删除合成静音1206块；四个完整轮次各338块真实音频均按顺序匹配到输出 RMS，首块排队19/37/9/75ms。RMS顺序匹配不是全 PCM 哈希证明，单元检查另核对首块原始字节；没有靠裁掉弱音/首音节改善数值。 + +## 固定口径与原始指标 + +固定音频“八百标兵奔北坡，炮兵并排北边跑。宝宝抱抱爸爸，妈妈慢慢说。”，xiaoyun，24kHz，6.76秒，SHA256 `8c0c1ec943882d4aafbb4f8cda488f138a19231dabe2a58335777c0c42fff3dc`。本轮直接复制审核留存文件，未重新合成。强度门槛0.015下，第一个20ms RMS达标块在源音频0.56秒，首次样本幅度达标0.169秒；审核所述11ms并非同一强度检测口径，不应直接与当前起点混用。真实语音开头的低强度部分仍全部保留。 + +所有主比较使用 `measure_sync.py` 的 `hel718-v2-mask-fix`，修正旧 source mask 的 x1 索引错误，并用新版对修前录像重新计算。ROI固定为512尺度的 `(271,341,180,220)`；服务端不裁切，浏览器/产品canvas裁切 `offset=(160,0), render-size=640` 后缩放512。服务端运动/残差门槛固定 `3.1718642664 / 7.693739183`；浏览器与产品固定 `4.294028549194336 / 10.492400400638578`。音频RMS门槛0.015。每个窗口 JSON 持久化完整命令、输入/底板哈希、检测器脚本哈希、ROI/offset/idle-until/start/stop、门槛、逐lag相关性；每个窗口各自保留CSV，避免被下一轮覆盖。 + +数值为口部起点减声音起点，**负号表示嘴提前**。合格意图为绝对值≤0.15秒（服务端）、≤0.25秒（浏览器）。视频帧量化约40ms，音频包约22ms，起点阈值/弱音/口型残差另有未定标误差，数值不替代真人听看。 + +| 实验 | 运动起点差(s) | 残差起点差(s) | 稳态最佳有符号lag(s) / 相关系数 | +| --- | ---: | ---: | ---: | +| 修前真 PTS | -0.576 | -0.576 | -0.06 / 0.209 | +| 修前浏览器图 | -0.699 | -0.665 | +0.52 / 0.231 | +| 修后真 PTS 第1轮 | -0.433 | -0.433 | -0.58 / 0.191 | +| 修后真 PTS 第2轮 | -0.320 | -0.320 | -0.44 / 0.189 | +| 修后真 PTS 第3轮 | -0.273 | -0.273 | -0.40 / 0.192 | +| 修后真 PTS 第4轮 | -0.313 | -0.313 | +0.34 / 0.173 | +| 修后真 PTS 第5轮 | -0.343 | -0.343 | +0.80 / 0.897 | +| 修后真 PTS 第6轮 | -0.296 | -0.296 | -0.44 / 0.339 | +| 修后浏览器图 第1轮 | -0.442 | -0.442 | -0.56 / 0.183 | +| 修后浏览器图 第2轮 | -0.360 | -0.360 | +0.32 / 0.171 | +| 修后浏览器图 第3轮 | -0.448 | -0.448 | -0.52 / 0.184 | +| 修后浏览器图 第4轮 | -0.324 | -0.324 | -0.40 / 0.206 | +| 修前产品端到端（不同短句） | -0.382 | -0.382 | -0.46 / 0.391 | +| 修后产品端到端首轮 | -0.507 | -0.507 | -0.26 / 0.203 | +| 修后产品端到端被打断轮 | -0.241 | -0.218 | -0.34 / 0.593 | + +服务端第1轮刚进入；第2轮前空闲15秒；第3/4轮连续播放后空闲1秒；第5轮在2秒处发 `/interrupt_talk`；第6轮同peer再次送固定音频（仅驱动诊断，不能替代产品取消隔离）。浏览器图覆盖前四个完整场景，产品回归另覆盖打断后新peer重入。修前固定音频只复用审核单轮原始录像，**未重建修前完整多轮矩阵**；此限制不能写成完整A/B验收已过。 + +产品端到端使用实际LLM/TTS，不以固定音频替换上游；首轮虽同文本，实际TTS为9.52秒，第二轮7.96秒，修前为1.96秒短句，**不能将这些行当成同音频因果对照**。受控同音频的因果比较以真PTS/浏览器图为准。 + +稳态lag使用运动能量与声音包络相关：定义 `lip_minus_audio = -shift`，修正旧xcorr脚本打印标签与相关计算方向相反的问题。四个完整受控轮次相关系数只有约0.17～0.21，最佳lag不稳定，**无法证明稳态音素同步或排除感知漂移**；第5轮只有截断短窗，其高相关与边界lag也不具验收意义。不能拿“某个最佳lag接近零”冒充通过。 + +音频PTS逐包20ms、视频40ms，无跳步；独立单会话末端墙钟减PTS分别约+1.07ms/+0.22ms；瞬时墙钟停顿最大约110/129ms，累计偏移峰值约90ms后回落。没有观察到持续时钟漂移，未机械重跑620秒。并行诊断曾有约1秒停顿（关闭其他peer同步join时），相应原始trace保留；主表使用独立单会话重测，不能把并发记录的卡顿抹掉。 + +## 回归、完整录像及未测项 + +- 前端构建通过；ruff通过；原78项pytest通过；5项网关检查通过；新增4项队列/时钟检查通过。日志见复核包 `checks/` 与 `remote/evidence/*checks.log`。 +- Chromium真实WebRTC回归通过：说话中取消、准备中取消、回日常、offer故障、缺素材、断网重连、快速取消再重入。快速重入首次与多轮采集并行时等待首声45秒超时，独立复跑通过，无pageerror或迟到旧轮错误；首轮未自动保存页面状态，不能追认已定位该偶发问题。 +- `product-continuous.webm` 约66.7秒：日常片7.84秒，录制后等待24.52秒（至少三轮）→f96切点→正向进场→两轮实际LLM/TTS→说话中按钮取消→独立正向回程→阅读续播。无pageerror；末态“已回到阅读原位置”。两轮首声8.49/7.91秒（本次样本，不是SLA）。取消后200～1000ms窗口最大RMS约0.0000041，低于0.015，无旧声复播证据。 +- 真人走查仍逐项待确认：日常循环接缝、两个切点、近景暖色、书本/手部/坐垫形变、微动作自然性、物理扬声器听感。没有用像素统计代替真人通过。 +- 真人自然抢话、手机真机/iOS/Safari、真实系统后台/锁屏、共享TTS内部GPU即停、多用户并发仍未测；桌面390px与synthetic visibility仅是处理器回归。 + +## 阻塞与最小后续路径 + +目前证据只能确认空闲积压受限、真实音频未被队列裁切、PTS连续；不能确认“积压修复足以消除同步故障”。残余服务端约−0.27～−0.43秒、浏览器约−0.32～−0.45秒，产品首轮约−0.51秒，均不足以放行。 + +建议下一步在同一固定音频的弱起音/闭唇音附近，保存每个推理批次的输入采样区间、Whisper特征区间和实际paste-back视频帧对应关系，以原始PCM对齐/人工音素标注核实残余来自特征关联还是起点检测；再由总工审定最小修法。不要盲加固定视频延迟、删除真实静音段或移动特征窗来压低指标。 + +## 隔离与交接 + +复用 c726943、MuseTalk v1.5 及原P10 venv/权重/avatar，无新依赖。工作仅在 `/opt/hel718-fix` 私有目录；18080/18081/9880/10095、系统Python/CUDA/驱动、证书信任、防火墙与开机配置未改。精确PID 1018154（驱动）、1018618（网关）、1098378（浏览器图代理）已核对cmd/cwd并在比较Multica daemon PID 44286后停止；8010/8014/8933已释放；共享服务PID 538167/608898/445461/220889保持。详见 `cleanup.json`。没有部署长期站点。 + +草稿 PR #2 沿原分支更新head，不另建PR。候选交付不等于样机通过；本卡保持blocked，由总管组织后续归因与真人检查。 diff --git a/samples/companion/tools/check_driver_audio.py b/samples/companion/tools/check_driver_audio.py new file mode 100644 index 0000000..20bdb91 --- /dev/null +++ b/samples/companion/tools/check_driver_audio.py @@ -0,0 +1,92 @@ +"""Run with the pinned driver venv: python check_driver_audio.py PRIVATE_DRIVER.""" + +import hashlib +import importlib.util +import pathlib +import sys +import time +import unittest +from types import SimpleNamespace + +import numpy as np + +ROOT = pathlib.Path(sys.argv.pop(1)).resolve() +sys.path.insert(0, str(ROOT)) +spec = importlib.util.spec_from_file_location('driver_webrtc', ROOT / 'server/webrtc.py') +driver = importlib.util.module_from_spec(spec) +spec.loader.exec_module(driver) +PendingAudio = driver.PendingAudio + + +class QueueChecks(unittest.TestCase): + def test_idle_is_bounded_speech_and_events_are_lossless(self): + pending = PendingAudio() + expected = [] + for i in range(2000): + if i % 100 == 0: + # Real zero PCM and custom audio are not synthetic silence. + item = (i, {'event': i} if i % 200 else {}, 0 if i % 200 else 2) + expected.append(item) + else: + item = (i, None, 1) + pending.put(item) + actual = list(pending.queue) + self.assertEqual([x for x in actual if x[2] != 1], expected) + self.assertEqual(sum(x[2] == 1 for x in actual), 3) + self.assertLessEqual(pending.high_water, len(expected) + 3) + self.assertGreater(pending.dropped_silence, 1900) + marker = ('event', {'end': True}, 1) + pending.put(marker) + for i in range(50): + pending.put((i, None, 1)) + self.assertIn(marker, list(pending.queue)) + + def test_real_audio_has_bounded_backpressure(self): + pending = PendingAudio() + for i in range(100): + pending.put((i, None, 0)) + import queue + with self.assertRaises(queue.Full): + pending.put((101, None, 0), block=False) + self.assertEqual([x[0] for x in pending.queue], list(range(100))) + + def test_warmup_drains_all_context(self): + from avatars.audio_features.base_asr import BaseASR + asr = BaseASR(SimpleNamespace(fps=25, batch_size=8, l=10, r=10)) + asr.warm_up() + self.assertEqual(len(asr.frames), 20) + self.assertTrue(asr.output_queue.empty()) + first = np.arange(320, dtype=np.float32) + asr.put_audio_frame(first, {'first': True}) + frame = asr.get_audio_frame() + np.testing.assert_array_equal(frame.data, first) + self.assertEqual(frame.type, 0) + self.assertEqual(frame.userdata, {'first': True}) + + +class ClockChecks(unittest.IsolatedAsyncioTestCase): + async def test_starvation_preserves_cadence_and_first_pcm(self): + player = SimpleNamespace(_start=lambda _: None, _stop=lambda _: None, notify=lambda _: None) + track = driver.PlayerStreamTrack(player, 'audio') + frame = driver.AudioFrame.from_ndarray(np.ones((1, 320), np.int16), format='s16', layout='mono') + frame.sample_rate = 16000 + track._queue.put((frame, None, 0)) + start = time.monotonic() + frames = [await track.recv() for _ in range(15)] + self.assertEqual([f.pts for f in frames], list(range(0, 15 * 320, 320))) + self.assertGreater(time.monotonic() - start, 0.25) + self.assertLess(time.monotonic() - start, 0.5) + self.assertTrue(all(not f.to_ndarray().any() for f in frames[1:])) + # Resume with exact first speech PCM, without trimming any syllable. + pcm = np.arange(320, dtype=np.int16).reshape(1, -1) + voice = driver.AudioFrame.from_ndarray(pcm, format='s16', layout='mono') + voice.sample_rate = 16000 + track._queue.put((voice, {'start': True}, 0)) + out = await track.recv() + self.assertEqual(out.pts, 15 * 320) + self.assertEqual(hashlib.sha256(out.to_ndarray().tobytes()).digest(), hashlib.sha256(pcm.tobytes()).digest()) + track.stop() + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/samples/companion/tools/driver_audio.py b/samples/companion/tools/driver_audio.py new file mode 100644 index 0000000..5004997 --- /dev/null +++ b/samples/companion/tools/driver_audio.py @@ -0,0 +1,35 @@ +"""Pending audio before RTP timestamps: only type=1 synthetic idle is expendable.""" + +import queue + + +class PendingAudio(queue.Queue): + """Keep at most three un-timestamped synthetic blocks; preserve speech/events. + + One render producer, one RTP consumer. Real speech (including zero PCM and + custom type>1 audio) uses bounded Queue backpressure, never a drop policy. + """ + + silence_limit = 3 + + def __init__(self): + super().__init__(maxsize=100) + self.dropped_silence = 0 + self.high_water = 0 + + def put(self, item, block=True, timeout=None): + with self.not_full: + # Trim before assigning pts, never inside the RTP consumer. Event + # markers remain observable even if a producer puts one on silence. + idle = [i for i, (_, event, kind) in enumerate(self.queue) if kind == 1 and not event] + incoming_idle = item[2] == 1 and not item[1] + excess = len(idle) + int(incoming_idle) - self.silence_limit + for i in reversed(idle[:max(0, excess)]): + del self.queue[i] + self.unfinished_tasks -= 1 + self.dropped_silence += 1 + if excess > 0: + self.not_full.notify_all() + super().put(item, block=block, timeout=timeout) + with self.mutex: + self.high_water = max(self.high_water, len(self.queue)) diff --git a/samples/companion/tools/graph_server.py b/samples/companion/tools/graph_server.py new file mode 100644 index 0000000..331f0ba --- /dev/null +++ b/samples/companion/tools/graph_server.py @@ -0,0 +1,41 @@ +import argparse +import asyncio +from pathlib import Path + +import aiohttp +from aiohttp import web + +p = argparse.ArgumentParser() +p.add_argument('--wav', type=Path, required=True) +p.add_argument('--index', type=Path, required=True) +a = p.parse_args() +INDEX = a.index.read_bytes() +WAV = a.wav.read_bytes() + + +async def proxy(request): + data = await request.read() + async with aiohttp.ClientSession() as http, http.request( + request.method, + "http://127.0.0.1:8010" + request.path[len("/lt") :], + data=data if request.method == "POST" else None, + headers={"Content-Type": request.headers.get("Content-Type", "")} if request.method == "POST" else {}, + ) as r: + body = await r.read() + return web.Response(status=r.status, body=body, content_type=r.content_type) + + +async def main(): + app = web.Application() + app.router.add_get("/", lambda _: web.Response(body=INDEX, content_type="text/html")) + app.router.add_get("/fixed.wav", lambda _: web.Response(body=WAV, content_type="audio/wav")) + app.router.add_route("*", "/lt/{tail:.*}", proxy) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, "0.0.0.0", 8933) + await site.start() + print("serving on 8933") + await asyncio.Event().wait() + + +asyncio.run(main()) diff --git a/samples/companion/tools/measure_sync.py b/samples/companion/tools/measure_sync.py new file mode 100644 index 0000000..c91acdc --- /dev/null +++ b/samples/companion/tools/measure_sync.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python +"""HEL-717 onset analysis: audio RMS onset vs lip-motion onset, multiple detectors. + +Works on any muxed recording (browser webm or true-pts mp4). Mouth ROI is +derived from the recording itself (temporal variance), coordinates are saved. +""" +import argparse +import csv +import hashlib +import json +import sys +from pathlib import Path + +import av +import cv2 +import numpy as np + +p = argparse.ArgumentParser() +p.add_argument("video", type=Path) +p.add_argument("--source", type=Path, default=Path("/opt/hel714-sample/dist/assets/talk/gaze.mp4")) +p.add_argument("--size", type=int, default=512) +p.add_argument("--idle", type=float, required=True, help="seconds of pre-speech idle to calibrate") +p.add_argument("--roi", help="x0,x1,y0,y1 in pixels at --size; default derive from variance") +p.add_argument("--strip", type=Path, help="write a contact strip of the ROI around onset") +p.add_argument("--offset-x", type=int, default=0) +p.add_argument("--offset-y", type=int, default=0) +p.add_argument("--render-size", type=int, default=0, help="crop window size before resize (canvas recordings)") +p.add_argument("--start", type=float, default=0) +p.add_argument("--stop", type=float, default=100000) +p.add_argument("--out", type=Path) +p.add_argument("--idle-until", type=float) +p.add_argument("--motion-threshold", type=float) +p.add_argument("--residual-threshold", type=float) +a = p.parse_args() + +frames = [] +rs = a.render_size or None +for f in av.open(str(a.video)).decode(video=0): + if not a.start <= float(f.time) <= a.stop: + continue + img = f.to_ndarray(format="bgr24") + if rs: + img = img[a.offset_y : a.offset_y + rs, a.offset_x : a.offset_x + rs] + img = cv2.resize(img, (a.size, a.size)) + frames.append((float(f.time), img)) +audio = [] +for f in av.open(str(a.video)).decode(audio=0): + if not a.start <= float(f.time) <= a.stop: + continue + data = f.to_ndarray().astype(np.float64) + if data.max() > 1: + data /= 32768 + audio.append([float(f.time), float(np.sqrt((data**2).mean()))]) + +stack = np.stack([img for _, img in frames]).astype(np.float32) +std = stack.std(axis=0).mean(axis=2) +if a.roi: + roi = tuple(int(v) for v in map(float, a.roi.split(","))) + print("using explicit mouth ROI (x0,x1,y0,y1):", roi) +else: + region = np.zeros_like(std, dtype=bool) + h = a.size + region[int(h * 0.35) : int(h * 0.80), int(h * 0.25) : int(h * 0.75)] = True + stdm = np.where(region, std, 0) + # smooth, then take the top-percentile blob + k = np.ones((15, 15), np.float32) / 225 + stds = cv2.filter2D(stdm, -1, k) + thresh = np.quantile(stds[stds > 0], 0.995) + ys, xs = np.where(stds >= thresh) + x0, x1, y0, y1 = int(xs.min()), int(xs.max()), int(ys.min()), int(ys.max()) + pad = 6 + roi = (max(0, x0 - pad), min(a.size, x1 + pad), max(0, y0 - pad), min(a.size, y1 + pad)) + print("derived mouth ROI (x0,x1,y0,y1):", roi, "fractions:", [v / a.size for v in roi]) + +# --- detector 1: frame-to-frame motion inside ROI (independent of source board) +motion = [] +for i in range(1, len(frames)): + t0, i0 = frames[i - 1] + t1, i1 = frames[i] + m = np.abs(i1[roi[2] : roi[3], roi[0] : roi[1]].astype(np.float32) - i0[roi[2] : roi[3], roi[0] : roi[1]].astype(np.float32)).mean() + motion.append([t1, float(m)]) + +# --- detector 2: source-matching residual (same as construction analyzer) +source = [cv2.resize(f.to_ndarray(format="bgr24"), (a.size, a.size)) for f in av.open(str(a.source)).decode(video=0)] +mask = np.ones((48, 48), dtype=bool) +mask[0:3] = False +mask[43:] = False +mr = np.zeros((48, 48), dtype=bool) +my0, my1 = int(roi[2] * 48 / a.size), int(roi[3] * 48 / a.size) +mx0, mx1 = int(roi[0] * 48 / a.size), int(roi[1] * 48 / a.size) +mr[my0:my1, mx0:mx1] = True +mask = mask & ~mr +feats = np.asarray([cv2.resize(f, (48, 48))[mask].astype(np.float32) for f in source]) +residuals = [] +for t, img in frames: + feat = cv2.resize(img, (48, 48))[mask].astype(np.float32) + idx = int(np.argmin(((feats - feat) ** 2).mean(axis=(1, 2)))) + res = np.abs(img.astype(np.float32) - source[idx]).mean(axis=2) + residuals.append([t, float(res[roi[2] : roi[3], roi[0] : roi[1]].mean()), idx]) + +first_audio = next((t for t, r in audio if r > 0.015), None) + + +def onset(rows, idle_end, floor): + base = [m for t, m, * _ in [tuple(r) for r in rows] if t <= idle_end] + if not base: + return None, None + th = max(floor, float(np.quantile(base, 0.99)) * 1.5) + hit = next((t for t, m, *_ in [tuple(r) for r in rows] if m > th and t > idle_end), None) + return hit, th + + +idle_end = a.idle_until if a.idle_until is not None else (first_audio - 0.7 if first_audio else a.idle) +lip_motion_onset, motion_th = onset(motion, idle_end, 1.2) +if a.motion_threshold is not None: + motion_th = a.motion_threshold + lip_motion_onset = next((t for t, m in motion if t > idle_end and m > motion_th), None) +lip_residual_onset, residual_th = onset(residuals, idle_end, 4) +if a.residual_threshold is not None: + residual_th = a.residual_threshold + lip_residual_onset = next((t for t, m, _ in residuals if t > idle_end and m > residual_th), None) + +result = { + "detector_version": "hel718-v2-mask-fix", + "detector_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), + "command": [sys.executable, *sys.argv], + "parameters": {k: str(v) if isinstance(v, Path) else v for k, v in vars(a).items()}, + "effective_idle_until": idle_end, + "audio_threshold": 0.015, + "input_sha256": hashlib.sha256(a.video.read_bytes()).hexdigest(), + "source_sha256": hashlib.sha256(a.source.read_bytes()).hexdigest(), + "detection_uncertainty_seconds": {"video": 0.04, "audio": 0.022}, + "video": str(a.video), + "frames": len(frames), + "audio_packets": len(audio), + "first_audio": first_audio, + "lip_motion_onset": lip_motion_onset, + "lip_motion_threshold": motion_th, + "motion_delta_seconds": (lip_motion_onset - first_audio) if lip_motion_onset and first_audio else None, + "lip_residual_onset": lip_residual_onset, + "lip_residual_threshold": residual_th, + "residual_delta_seconds": (lip_residual_onset - first_audio) if lip_residual_onset and first_audio else None, + "derived_roi": [int(v) for v in roi], + "roi_fractions": [round(v / a.size, 4) for v in roi], + "note": "delta<0 means lips move before sound; ROI derived from temporal variance, coordinates saved", +} +# Same envelope detector for each window; positive result means lips later. +if motion and audio: + grid = np.arange(max(motion[0][0], audio[0][0]), min(motion[-1][0], audio[-1][0]), 0.02) + m = np.interp(grid, np.asarray(motion)[:, 0], np.asarray(motion)[:, 1]) + au = np.interp(grid, np.asarray(audio)[:, 0], np.asarray(audio)[:, 1]) + m = np.convolve(m, np.ones(5) / 5, 'same') + au = np.convolve(au, np.ones(5) / 5, 'same') + active = grid[au > 0.02] + if len(active): + selected = (grid >= active.min() - 0.8) & (grid <= active.max() + 0.8) + m, au = m[selected], au[selected] + correlations = [] + for shift in range(-40, 41): + x = m[:len(m)-shift or None] if shift >= 0 else m[-shift:] + y = au[shift:] if shift >= 0 else au[:len(au)+shift] + if len(x) > 2 and np.std(x) > 0 and np.std(y) > 0: + correlations.append({"lip_minus_audio_seconds": -shift * 0.02, + "correlation": float(np.corrcoef(x, y)[0, 1])}) + result["steady_envelope"] = {"best": max(correlations, key=lambda c: c["correlation"]), + "all_lags": correlations, + "smooth_seconds": 0.1, "active_rms_threshold": 0.02, + "caveat": "motion energy heuristic, not phoneme alignment"} +out = a.out or a.video.with_suffix(".onset.json") +out.write_text(json.dumps(result, indent=2)) + +if a.strip and first_audio: + # contact strip: frames every 100ms from first_audio-1.2s to +1.6s + times = [first_audio - 1.2 + 0.1 * i for i in range(30)] + crops = [] + for tt in times: + best = min(frames, key=lambda fr: abs(fr[0] - tt))[1] + crop = best[roi[2] - 20 : roi[3] + 20, roi[0] - 30 : roi[1] + 30] + crops.append(cv2.resize(crop, (120, 80))) + rows = [np.hstack(crops[i : i + 10]) for i in range(0, 30, 10)] + cv2.imwrite(str(a.strip), np.vstack(rows)) + print("strip:", a.strip) +with out.with_suffix(".csv").open("w") as f: + w = csv.writer(f) + w.writerow(["time_seconds", "motion", "lip_residual", "source_index", "audio_rms"]) + mmap = {t: m for t, m in motion} + amap = {t: r for t, r in audio} + for t, res, idx in residuals: + near_audio = min(audio, key=lambda x: abs(x[0] - t))[1] if audio else "" + near_motion = min(motion, key=lambda x: abs(x[0] - t))[1] if motion else "" + w.writerow([t, near_motion, res, idx, near_audio]) +print(json.dumps(result, indent=2)) diff --git a/samples/companion/tools/patch_driver.py b/samples/companion/tools/patch_driver.py index da24c6d..8603237 100644 --- a/samples/companion/tools/patch_driver.py +++ b/samples/companion/tools/patch_driver.py @@ -45,3 +45,75 @@ if replacement not in s: assert s.count(needle) >= 1 p.write_text(s.replace(needle, replacement, 1)) print("Installed HEL-714 peer retirement endpoint in private copy:", root) + +# HEL-718: all edits remain reproducible on a clean pinned driver copy. +def replace_once(path, before, after): + text = path.read_text() + if after in text: + return + assert text.count(before) == 1, f"Unexpected c726943 source: {path}" + path.write_text(text.replace(before, after, 1)) + + +(root / "server/hel718_audio.py").write_text( + Path(__file__).with_name("driver_audio.py").read_text() +) +p = root / "server/webrtc.py" +replace_once(p, "import queue\n", "import queue\nfrom server.hel718_audio import PendingAudio\n") +replace_once(p, "self._queue = queue.Queue(maxsize=100)", + "self._queue = PendingAudio() if kind == 'audio' else queue.Queue(maxsize=100)") +# Preserve the initial common audio/video origin. Once started, pace BEFORE +# selecting audio, so fresh speech cannot wait behind a dequeued idle block. +s = p.read_text() +start = s.find(" # frame = self.frames[self.counter % 30]") +end = s.index(" frame.pts = pts", max(0, start)) +replacement = ''' self._player._start(self) + if self.kind == 'audio' and hasattr(self, '_timestamp'): + pts, time_base = await self.next_timestamp() + try: + frame, eventpoint, _ = self._queue.get_nowait() + except queue.Empty: + frame = AudioFrame.from_ndarray( + np.zeros((1, 320), dtype=np.int16), layout='mono', format='s16') + frame.sample_rate = SAMPLE_RATE + eventpoint = None + else: + while self.readyState == 'live': + try: + frame, eventpoint, *_ = self._queue.get_nowait() + break + except queue.Empty: + await asyncio.sleep(0.005) + else: + raise Exception('Track stopped') + pts, time_base = await self.next_timestamp() +''' +if "np.zeros((1, 320), dtype=np.int16), layout='mono'" not in s: + assert start >= 0, "Unexpected c726943 recv implementation" + p.write_text(s[:start] + replacement + s[end:]) +replace_once(p, "def push_audio(self, frame, eventpoint=None):", + "def push_audio(self, frame, eventpoint=None, audio_type=0):") +replace_once(p, "self.__audio._queue.put((new_frame, eventpoint))", + "self.__audio._queue.put((new_frame, eventpoint, audio_type))") +# Wall-clock adjustments must not change packet cadence. +s = p.read_text() +start, end = s.index(" async def next_timestamp"), s.index(" async def recv") +s = s[:start] + s[start:end].replace("time.time()", "time.monotonic()") + s[end:] +p.write_text(s) + +p = root / "streamout/webrtc.py" +replace_once(p, " def get_buffer_size(self) -> int:", ''' def push_audio_data(self, frame, eventpoint, audio_type): + if self._player: + self._player.push_audio(frame, eventpoint, audio_type) + + def get_buffer_size(self) -> int:''') +p = root / "avatars/base_avatar.py" +replace_once(p, " self.output.push_audio_frame(frame, audio_frame.userdata)", +''' if hasattr(self.output, 'push_audio_data'): + self.output.push_audio_data(frame, audio_frame.userdata, audio_frame.type) + else: + self.output.push_audio_frame(frame, audio_frame.userdata)''') +p = root / "avatars/audio_features/base_asr.py" +replace_once(p, " for _ in range(self.stride_left_size):\n self.output_queue.get()", + " for _ in range(self.stride_left_size + self.stride_right_size):\n self.output_queue.get()") +print("Installed HEL-718 bounded synthetic idle, continuous RTP audio, and warm-up drain") diff --git a/samples/companion/tools/record_driver.py b/samples/companion/tools/record_driver.py new file mode 100644 index 0000000..d9ab76b --- /dev/null +++ b/samples/companion/tools/record_driver.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +"""HEL-717 diagnostic: record the driver's own WebRTC output with TRUE RTP pts. + +No browser involved. If the lips-early offset appears here, it originates +upstream (feature lookahead / frame association); if not, it is introduced by +the browser playback/recording graph. +""" +import argparse +import asyncio +import hashlib +import json +import sys +import time +from fractions import Fraction +from pathlib import Path + +import aiohttp +import av +import numpy as np +from aiortc import RTCPeerConnection, RTCSessionDescription + +p = argparse.ArgumentParser() +p.add_argument("--base", default="http://127.0.0.1:8010") +p.add_argument("--wav", required=True) +p.add_argument("--out", required=True) +p.add_argument("--post", type=float, default=9.0) +p.add_argument("--rounds", default="0,15,1,1", help="Idle seconds before each full utterance; then interrupt and restart") +a = p.parse_args() + +events = [] + + +def log(name, **kw): + row = {"t": time.time(), "rel": time.perf_counter(), "event": name, **kw} + events.append(row) + print(json.dumps({k: v for k, v in row.items() if k != "t"}), flush=True) + + +async def main(): + pc = RTCPeerConnection() + container = av.open(a.out, "w") + vstream = container.add_stream("libx264", rate=25) + vstream.time_base = Fraction(1, 90000) + vstream.pix_fmt = "yuv420p" + vstream.options = {"crf": "18", "preset": "veryfast"} + astream = container.add_stream("aac", rate=48000) + astream.time_base = Fraction(1, 48000) + astream.layout = "mono" + + stats = {"v": 0, "a": 0} + timeline = [] + + def video_track(track): + async def loop(): + while True: + try: + frame = await track.recv() + except Exception: + break + stats["v"] += 1 + timeline.append({"kind": "v", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter()}) + img = frame.to_ndarray(format="bgr24") + out = av.VideoFrame.from_ndarray(img, format="bgr24") + out.pts = frame.pts + out.time_base = frame.time_base + for pkt in vstream.encode(out): + container.mux(pkt) + return asyncio.ensure_future(loop()) + + def audio_track(track): + async def loop(): + while True: + try: + frame = await track.recv() + except Exception: + break + stats["a"] += 1 + data = frame.to_ndarray() + # aiortc delivers s16 mono 48k planar [1, n] + pcm = data.reshape(-1).astype(np.int16) + timeline.append({"kind": "a", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter(), + "rms": float(np.sqrt((pcm.astype(np.float64) / 32768.0) ** 2).mean())}) + out = av.AudioFrame.from_ndarray(pcm.reshape(1, -1), format="s16", layout="mono") + out.sample_rate = frame.sample_rate + out.pts = frame.pts + out.time_base = frame.time_base + for pkt in astream.encode(out): + container.mux(pkt) + return asyncio.ensure_future(loop()) + + @pc.on("track") + def on_track(track): + log("track", kind=track.kind) + if track.kind == "video": + video_track(track) + else: + audio_track(track) + + pc.addTransceiver("video", direction="recvonly") + pc.addTransceiver("audio", direction="recvonly") + offer = await pc.createOffer() + await pc.setLocalDescription(offer) + for _ in range(100): + if pc.iceGatheringState == "complete": + break + await asyncio.sleep(0.05) + async with aiohttp.ClientSession() as http: + async with http.post(a.base + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r: + answer = await r.json() + sid = answer["sessionid"] + log("offer-ok", sid=sid) + await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"])) + for _ in range(200): + if stats["v"]: + break + await asyncio.sleep(0.05) + log("first-video", v=stats["v"], a=stats["a"]) + wav_bytes = Path(a.wav).read_bytes() + async def send(label): + log("send", label=label, v=stats["v"], a=stats["a"]) + form = aiohttp.FormData() + form.add_field("sessionid", sid) + form.add_field("file", wav_bytes, filename="utterance.wav", content_type="audio/wav") + async with http.post(a.base + "/humanaudio", data=form) as r: + log("humanaudio-ok", label=label, body=await r.json()) + for index, idle in enumerate(map(float, a.rounds.split(","))): + await asyncio.sleep(idle) + await send(f"full-{index+1}-idle-{idle}") + await asyncio.sleep(a.post) + await send("interrupted") + await asyncio.sleep(2) + async with http.post(a.base + "/interrupt_talk", json={"sessionid": sid}) as r: + log("interrupt", body=await r.json()) + await asyncio.sleep(1) + await send("restart-same-peer-diagnostic") + await asyncio.sleep(a.post) + await pc.close() + async with http.post(a.base + "/hel714/close", json={"sessionid": sid}) as r: + log("closed", body=await r.json()) + for pkt in vstream.encode(): + container.mux(pkt) + for pkt in astream.encode(): + container.mux(pkt) + container.close() + Path(a.out + ".timeline.json").write_text(json.dumps({"command": [sys.executable, *sys.argv], "wav_sha256": hashlib.sha256(Path(a.wav).read_bytes()).hexdigest(), "events": events, "timeline": timeline})) + log("done", v=stats["v"], a=stats["a"]) + + +asyncio.run(main()) diff --git a/samples/companion/tools/record_graph.html b/samples/companion/tools/record_graph.html new file mode 100644 index 0000000..30faac9 --- /dev/null +++ b/samples/companion/tools/record_graph.html @@ -0,0 +1,89 @@ + + +HEL-717 remote-stream graph + + + + + + diff --git a/samples/companion/tools/record_graph.py b/samples/companion/tools/record_graph.py new file mode 100644 index 0000000..6c2bd4f --- /dev/null +++ b/samples/companion/tools/record_graph.py @@ -0,0 +1,32 @@ +import asyncio +import json +import sys + +from playwright.async_api import async_playwright + +OUT = sys.argv[1] if len(sys.argv) > 1 else "." + + +async def main(): + async with async_playwright() as pw: + browser = await pw.chromium.launch( + headless=True, + args=["--autoplay-policy=no-user-gesture-required", "--no-sandbox"], + ) + page = await browser.new_page(viewport={"width": 1280, "height": 800}) + await page.goto("http://192.168.200.12:8933/") + await page.evaluate("diagRun()") + await page.wait_for_function("document.title === 'done'", timeout=180000) + from pathlib import Path + out = Path(OUT) + out.mkdir(parents=True, exist_ok=True) + events = await page.evaluate("window.diagResult.events") + async with page.expect_download() as pending: + await page.evaluate("""() => {const a=document.createElement('a'); a.href=window.diagResult.blobUrl;a.download='graph.webm';a.click()}""") + await (await pending.value).save_as(out / 'graph-after.webm') + (out / 'graph-after.events.json').write_text(json.dumps(events, indent=2)) + print('saved graph audio/video', flush=True) + await browser.close() + + +asyncio.run(main()) diff --git a/samples/companion/tools/record_product.py b/samples/companion/tools/record_product.py new file mode 100644 index 0000000..3d175ef --- /dev/null +++ b/samples/companion/tools/record_product.py @@ -0,0 +1,59 @@ +"""Continuous real product recording: three daily loops, talk, cancel, return.""" + +import asyncio +import json +import pathlib +import sys + +from playwright.async_api import async_playwright + +OUT = pathlib.Path(sys.argv[1]) +OUT.mkdir(parents=True, exist_ok=True) + + +async def main(): + async with async_playwright() as pw: + browser = await pw.chromium.launch(args=['--no-sandbox', '--autoplay-policy=no-user-gesture-required']) + context = await browser.new_context(ignore_https_errors=True, viewport={'width': 1280, 'height': 800}) + page = await context.new_page() + errors = [] + page.on('pageerror', lambda error: errors.append(str(error))) + await page.goto('https://192.168.200.12:8014') + await page.wait_for_function("document.querySelector('#status').textContent.includes('就绪')") + await page.click('#record') + duration = await page.locator('#daily').evaluate('(v)=>v.duration') + await page.wait_for_timeout((duration * 3 + 1) * 1000) + await page.click('#enter') + await page.wait_for_function("document.querySelector('#status').textContent.includes('正在听')", timeout=40000) + await page.fill('#text', '请原样念这一句，不添加内容：八百标兵奔北坡，炮兵并排北边跑。宝宝抱抱爸爸，妈妈慢慢说。') + await page.click('button[type=submit]') + await page.wait_for_function("window.sampleEvents.some(e=>e.event==='first-audible')", timeout=60000) + await page.wait_for_timeout(8000) + # Actual second turn is cancelled while speaking. + await page.fill('#text', '请再念一遍：八百标兵奔北坡，炮兵并排北边跑。宝宝抱抱爸爸，妈妈慢慢说。') + await page.click('button[type=submit]') + await page.wait_for_function("window.sampleEvents.filter(e=>e.event==='first-audible').length>=2", timeout=60000) + await page.wait_for_timeout(1200) + await page.click('#cancel') + await page.wait_for_timeout(1200) + await page.click('#back') + await page.wait_for_function("document.querySelector('#status').textContent.includes('已回到阅读')", timeout=20000) + await page.wait_for_timeout(6000) + assert not await page.locator('#daily').evaluate('(v)=>v.paused') + await page.click('#record') + await page.wait_for_selector('#download:not([hidden])') + async with page.expect_download() as pending: + await page.click('#download') + await (await pending.value).save_as(OUT / 'product-continuous.webm') + (OUT / 'product-continuous.json').write_text(json.dumps({ + 'command': [sys.executable, *sys.argv], 'daily_duration': duration, + 'daily_recording_seconds_before_enter': duration * 3 + 1, + 'events': await page.evaluate('window.sampleEvents'), 'pageerrors': errors, + 'final_status': await page.locator('#status').inner_text(), + 'human_review': 'pending: seams, cuts, warmth, book/hands/cushion; real barge-in, phone, OS background untested', + }, ensure_ascii=False, indent=2)) + await browser.close() + print('Saved continuous product audio/video', flush=True) + + +asyncio.run(main()) diff --git a/samples/companion/tools/trace_driver.py b/samples/companion/tools/trace_driver.py new file mode 100644 index 0000000..9226a3b --- /dev/null +++ b/samples/companion/tools/trace_driver.py @@ -0,0 +1,40 @@ +"""Optional test-only trace wrapper; pass app.py arguments after the trace path.""" + +import json +import runpy +import sys +import time + +import numpy as np +from server import webrtc + +out = open(sys.argv.pop(1), 'w', buffering=1) # noqa: SIM115 -- process-lifetime trace + + +def write(event, player, **fields): + out.write(json.dumps({'wall': time.monotonic(), 'event': event, 'player': id(player), **fields}) + '\n') + + +push = webrtc.HumanPlayer.push_audio +recv = webrtc.PlayerStreamTrack.recv + + +def push_audio(self, frame, eventpoint=None, audio_type=0): + push(self, frame, eventpoint, audio_type) + q = self.audio._queue + write('push', self, kind=audio_type, rms=float(np.sqrt(np.mean((frame.astype(float) / 32768) ** 2))), + depth=q.qsize(), dropped=q.dropped_silence, high_water=q.high_water) + + +async def receive(self): + frame = await recv(self) + fields = {'kind': self.kind, 'pts': frame.pts, 'tb': str(frame.time_base), 'depth': self._queue.qsize()} + if self.kind == 'audio': + fields['rms'] = float(np.sqrt(np.mean((frame.to_ndarray().astype(float) / 32768) ** 2))) + write('recv', self._player, **fields) + return frame + + +webrtc.HumanPlayer.push_audio = push_audio +webrtc.PlayerStreamTrack.recv = receive +runpy.run_path('app.py', run_name='__main__')