Files
CyberGril/scripts/gen_hel911_wait_repro.py
T
施工员2号andmultica-agent 4a38106eef HEL-911: fix AgentHandoff/shutdown seams, attribute & fix the 3.66s pre-generation wait
- worker _on_item: filter conversation_item_added by the real 1.8.8 item
  discriminator (ChatMessage | AgentHandoff | _TypeDiscriminator); only
  assistant str-part body feeds the echo filter (HEL-910 AttributeError)
- bind_interrupt_channel unload: async (JobContext shutdown callbacks are
  awaited), idempotent, disconnect-safe; interrupt/receipt/budget paths
  untouched (HEL-910 TypeError await None)
- TURN_HANDLING: explicit turn_detection=None — 1.8.8 silently installs an
  English EOU TurnDetector whose unlikely verdicts on Chinese finals push
  endpointing to max_delay=6.0 (measured 3.667/3.654s HEL-910 call2; offline
  A/B on the real SDK: baseline 3.704s vs fixed 0.004s). Endpointing
  0.8/6.0 kept verbatim; no VAD/buffer_turn/semantic change
- scripts/gen_hel911_wait_repro.py: offline reproduction harness (real
  AgentSession + real silero VAD + HEL-910 wav and measured timings)
- tests: +3 real-SDK checks (11/11 on .12 agent venv; prior 8 regressions
  kept, unload call sites await the new protocol)
- hel910_call1.mjs reliability: segmented probe/recorder dumps to disk,
  timing-A trigger requires the server-committed voice turn row (not a
  we-sent-it proxy); node --check only, no real run
- window-record.md §9: appended factual corrections (companion 9 not 18,
  turn_start +6.099 not +4.10, assistant_final is the playout/receipt gate,
  post-window ListRooms token failure is not empty-room evidence); original
  record untouched
- docs/rework/hel911/summary.md + handoff-HEL-911.json; requirement checks
  pass vs origin/main and 407fc81

Co-authored-by: multica-agent <github@multica.ai>
2026-10-29 21:25:25 +08:00

433 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""HEL-911 离线等待链复现:真 AgentSession + 真 silero VAD + 脚本化 STT/LLM/TTS.
目的:把 HEL-910 call2 实测的「识别终稿 → 生成开始」3.667s 间隙(q1
+2.432→+6.099、q2 +48.525→+52.179,两轮同为 3.65~3.67s)在完全离线的
真实 SDK 1.8.8 里复现或排除,从而把 9.5s 首声等待的分段归因落到可检验
的事实上。
做法(全部本地计算,零网络,零 LLM/ASR/TTS 外呼):
- 输入:HEL-910 真实回放用过的 /opt/cybergril-hel910/etc/trim/u_bajie.trim.wav
按 100ms 帧、真实时钟节奏推给 session.input.audio(与 hel910_client 的
发布节奏一致),播完继续推真零样本静音帧(对应 asr-usage.jsonl 证实
的连续帧流 ~100.5% 实时);
- STT:脚本化 stt.STT,按实测时间发 INTERIM(语音期间)与 FINAL(回放
结束后 +2.432s,即 FunASR 离线纠错 + 适配器 0.6s 去抖的实测结果),
不设 end_time(与 FunASRSTT 的 SpeechData 形状一致);
- LLM:脚本化 LLM,记录 chat() 调用时刻(= 生产链路里 on_stream_started
→ 字幕 turn_start 的同一时刻),1.415s 后发两条 delta(对应实测
turn_start→assistant_interim 的服务端整轮生成时延);
- TTS:脚本化 ChunkedStream,1.614s 后出首帧(对应实测
[tts] first audio after 1.614s,worker.log:220);
- 会话参数与 worker.py entrypoint 完全一致(真 VAD.load()、
endpointing min 0.8 / max 6.0)。
输出各里程碑相对回放结束(= wav 末帧后第一帧静音的时刻)的秒数,与
HEL-910 实测列对照。本脚本不修改任何服务,只读 wav。
运行(.12 agent venv):
/opt/voice-agent/agent/venv/bin/python scripts/gen_hel911_wait_repro.py
"""
from __future__ import annotations
import asyncio
import sys
import time
import wave
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from livekit import rtc
from livekit.agents import stt, tts, llm as lk_llm
from livekit.agents.llm import ChatContext, LLM, LLMStream, Tool
from livekit.agents.stt import RecognizeStream, SpeechData, SpeechEvent, SpeechEventType
from livekit.agents.tts import AudioEmitter, ChunkedStream
from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS, APIConnectOptions
from livekit.agents.voice import Agent, AgentSession
from livekit.plugins.silero import VAD
WAV = Path("/opt/cybergril-hel910/etc/trim/u_bajie.trim.wav")
# HEL-910 call2 实测时延(相对回放结束/各起点)
FINAL_AFTER_PLAYBACK_END = 2.432 # user_final(VAD+FunASR 离线+0.6s 去抖)
GEN_AFTER_TURN_START = 1.415 # turn_start -> assistant_interim(服务端整轮)
TTS_FIRST_AUDIO = 1.614 # [tts] first audio after(worker.log:220)
TAIL_SILENCE = 14.0 # 回放结束后继续推的静音时长
def _now() -> float:
return time.perf_counter()
class Timeline:
def __init__(self) -> None:
self.t0: float | None = None
self.playback_end: float | None = None
self.marks: list[tuple[str, float]] = []
def mark(self, name: str) -> None:
self.marks.append((name, _now()))
class ScriptedSTT(stt.STT):
"""按 HEL-910 实测时间发 FINAL 的 STT;不连任何识别服务。"""
def __init__(self, tl: Timeline) -> None:
super().__init__(capabilities=stt.STTCapabilities(streaming=True, interim_results=True))
self._tl = tl
@property
def provider(self) -> str:
return "scripted"
@property
def model(self) -> str:
return "hel910-timeline"
async def _recognize_impl(self, buffer, *, language=None,
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
) -> SpeechEvent:
raise NotImplementedError("scripted STT only supports streaming")
def stream(self, *, language=None,
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
) -> RecognizeStream:
return ScriptedRecognizeStream(stt=self, conn_options=conn_options, tl=self._tl)
class ScriptedRecognizeStream(RecognizeStream):
def __init__(self, *, stt: ScriptedSTT, conn_options: APIConnectOptions,
tl: Timeline) -> None:
super().__init__(stt=stt, conn_options=conn_options)
self._tl = tl
def _ev(self, ev_type: SpeechEventType, text: str) -> SpeechEvent:
return SpeechEvent(type=ev_type,
alternatives=[SpeechData(language="zh", text=text)])
async def _run(self) -> None:
tl = self._tl
async def schedule_final() -> None:
# 等回放结束锚点(由输入源标记),再按实测延迟发终稿
while tl.playback_end is None:
await asyncio.sleep(0.02)
await asyncio.sleep(FINAL_AFTER_PLAYBACK_END)
tl.mark("stt_final_emitted")
self._event_ch.send_nowait(
self._ev(SpeechEventType.FINAL_TRANSCRIPT,
"师傅,哎,这取经路比高,老庄种地还苦,俺老猪饿的能吞下石龙素"
"包子,猴哥,你筋斗云块,要不先去化个缘"))
final_task = asyncio.create_task(schedule_final())
sent_interim = False
try:
async for ev in self._input_ch:
if ev is None:
break
if isinstance(ev, rtc.AudioFrame) and not sent_interim:
sent_interim = True
self._event_ch.send_nowait(
self._ev(SpeechEventType.INTERIM_TRANSCRIPT, "师傅"))
await asyncio.sleep(0)
finally:
final_task.cancel()
class ProbeLLM(LLM):
def __init__(self, tl: Timeline) -> None:
super().__init__()
self._tl = tl
@property
def provider(self) -> str:
return "probe"
@property
def model(self) -> str:
return "hel911-wait-probe"
def chat(self, *, chat_ctx: ChatContext, tools: list[Tool] | None = None,
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, **_
) -> LLMStream:
self._tl.mark("llm_chat_called") # = 生产链路的 turn_start 时刻
return ProbeLLMStream(llm=self, chat_ctx=chat_ctx, tools=tools or [],
conn_options=conn_options, tl=self._tl)
class ProbeLLMStream(LLMStream):
def __init__(self, *, llm: ProbeLLM, chat_ctx: ChatContext, tools: list[Tool],
conn_options: APIConnectOptions, tl: Timeline) -> None:
super().__init__(llm, chat_ctx=chat_ctx, tools=tools, conn_options=conn_options)
self._tl = tl
async def _run(self) -> None:
from livekit.agents.llm import ChatChunk, ChoiceDelta
await asyncio.sleep(GEN_AFTER_TURN_START)
self._tl.mark("first_delta")
self._event_ch.send_nowait(ChatChunk(
id="hel911-probe",
delta=ChoiceDelta(role="assistant", content="好,小十。去化缘")))
self._event_ch.send_nowait(ChatChunk(
id="hel911-probe", delta=ChoiceDelta(role="assistant", content="")))
self._event_ch.send_nowait(ChatChunk(id="hel911-probe"))
class ProbeTTS(tts.TTS):
def __init__(self, tl: Timeline) -> None:
super().__init__(capabilities=tts.TTSCapabilities(streaming=False),
sample_rate=24000, num_channels=1)
self._tl = tl
@property
def provider(self) -> str:
return "probe"
@property
def model(self) -> str:
return "hel911-wait-probe"
def synthesize(self, text: str, *,
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
) -> ChunkedStream:
return ProbeChunkedStream(tts=self, input_text=text,
conn_options=conn_options, tl=self._tl)
class ProbeChunkedStream(ChunkedStream):
def __init__(self, *, tts: ProbeTTS, input_text: str,
conn_options: APIConnectOptions, tl: Timeline) -> None:
super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
self._tl = tl
async def _run(self, output_emitter: AudioEmitter) -> None:
output_emitter.initialize(request_id="hel911", sample_rate=24000,
num_channels=1, mime_type="audio/pcm")
await asyncio.sleep(TTS_FIRST_AUDIO) # 实测 CosyVoice3 首包
self._tl.mark("tts_first_audio")
# ~1s 的可听音频(20ms 帧),足够走完 playout/回执路径
frame = b"\x10\x00" * 480
for _ in range(50):
output_emitter.push_frame(rtc.AudioFrame(frame, 24000, 1, 480))
await asyncio.sleep(0.02)
output_emitter.flush()
def make_sink(tl: Timeline):
from livekit.agents.voice.io import AudioOutput, AudioOutputCapabilities
class _Sink(AudioOutput):
def __init__(self) -> None:
super().__init__(label="probe-sink",
capabilities=AudioOutputCapabilities(pause=False))
self.first_frame_at: float | None = None
async def capture_frame(self, frame: rtc.AudioFrame) -> None:
if self.first_frame_at is None:
tl.mark("sink_first_frame")
self.first_frame_at = _now()
def flush(self) -> None:
self.on_playback_finished(interrupted=False)
def clear_buffer(self) -> None:
self.on_playback_finished(interrupted=True)
return _Sink()
class MicSource:
"""wav 实时回放 + 尾部连续静音,节奏与 hel910_client._publish_pcm 一致。"""
def __init__(self, tl: Timeline) -> None:
self._tl = tl
def make(self):
tl = self._tl
from livekit.agents.voice.io import AudioInput
with wave.open(str(WAV), "rb") as w:
raw = w.readframes(w.getnframes())
rate = w.getframerate()
class _Source(AudioInput):
def __init__(self) -> None:
super().__init__(label="hel911-mic")
self._i = 0
self._phase = "wav"
async def __anext__(self) -> rtc.AudioFrame:
step = 1600 * 2 # 100ms @16k,同 hel910_client
if self._phase == "wav":
chunk = raw[self._i:self._i + step]
if tl.t0 is None:
tl.t0 = _now()
if len(chunk) < step:
chunk = chunk + b"\x00" * (step - len(chunk))
self._i += step
if self._i >= len(raw):
self._phase = "silence"
tl.playback_end = _now()
tl.mark("playback_end")
await asyncio.sleep(0.1)
return rtc.AudioFrame(chunk, rate, 1, 1600)
# 静音段:真实时钟节奏(帧流连续,asr-usage 证实)
await asyncio.sleep(0.1)
return rtc.AudioFrame(b"\x00" * step, rate, 1, 1600)
return _Source()
async def main() -> None:
if not WAV.exists():
raise SystemExit(f"wav not found: {WAV}")
import logging
logging.basicConfig(level=logging.INFO) # eot prediction 走 debug 时再降
logging.getLogger("livekit.agents.voice.audio_recognition").setLevel(logging.DEBUG)
import os
# 默认复刻生产接线(worker.TURN_HANDLING,已显式 turn_detection=None);
# HEL911_BASELINE=1 复现缺陷态:1.8.8 对缺省键默认补英文 TurnDetector
baseline = os.environ.get("HEL911_BASELINE") == "1"
from cybergril.voice.worker import TURN_HANDLING as WORKER_TURN_HANDLING
turn_handling = (
{k: v for k, v in WORKER_TURN_HANDLING.items() if k != "turn_detection"}
if baseline else dict(WORKER_TURN_HANDLING)
)
tl = Timeline()
session = AgentSession(
stt=ScriptedSTT(tl),
llm=ProbeLLM(tl),
tts=ProbeTTS(tl),
vad=VAD.load(),
turn_handling=turn_handling,
)
session.input.audio = MicSource(tl).make() # type: ignore[assignment]
session.output.audio = make_sink(tl) # type: ignore[assignment]
@session.on("user_state_changed")
def _ustate(ev):
tl.mark(f"user_state_{getattr(ev, 'new_state', '?')}")
@session.on("agent_state_changed")
def _astate(ev):
tl.mark(f"agent_state_{getattr(ev, 'new_state', '?')}")
@session.on("user_input_transcribed")
def _tr(ev):
if ev.is_final:
tl.mark("sdk_user_final")
await session.start(agent=Agent(instructions="probe"))
tl.mark("session_started")
# 定位 FINAL→llm.chat 间隙:给真 SDK 的提交/调度内部打点(只读包装,不改行为)
from livekit.agents.voice import agent_activity
activity = session._activity
_orig_eou = activity.on_end_of_turn
def _logged_eou(info):
committed = _orig_eou(info)
tl.mark(f"on_end_of_turn(committed={committed})")
return committed
activity.on_end_of_turn = _logged_eou # type: ignore[method-assign]
_orig_gen = activity._generate_reply
def _logged_gen(**kwargs):
handle = _orig_gen(**kwargs)
tl.mark("_generate_reply")
return handle
activity._generate_reply = _logged_gen # type: ignore[method-assign]
_orig_sched = activity._schedule_speech
def _logged_sched(speech, priority, force=False):
tl.mark("_schedule_speech")
return _orig_sched(speech, priority, force=force)
activity._schedule_speech = _logged_sched # type: ignore[method-assign]
_orig_run_eou = type(activity._audio_recognition)._run_eou_detection
def _logged_run_eou(self, chat_ctx, *, trigger, skip_reply=False):
tl.mark(f"_run_eou_detection({trigger})")
# EOU 锚点体检:提交前的 extra_sleep 由这些值决定
lst = self._last_speaking_time
print(f" [eou:{trigger}] speaking={self._speaking} "
f"last_speaking_time={'none' if lst is None else f'{time.time() - lst:+.3f}s ago'} "
f"final_at={'none' if self._last_final_transcript_time is None else f'{time.time() - self._last_final_transcript_time:+.3f}s ago'} "
f"endpointing_min={self._endpointing.min_delay} "
f"turn_detector={type(self._turn_detector).__name__ if self._turn_detector else None} "
f"td_stream={type(self._turn_detector_stream).__name__ if self._turn_detector_stream else None} "
f"mode={self._turn_detection_mode}", flush=True)
return _orig_run_eou(self, chat_ctx, trigger=trigger, skip_reply=skip_reply)
type(activity._audio_recognition)._run_eou_detection = _logged_run_eou
ar = activity._audio_recognition
_orig_span = ar._ensure_eou_wait_span
def _logged_span(user_turn_span, **kwargs):
tl.mark("_ensure_eou_wait_span")
out = _orig_span(user_turn_span, **kwargs)
print(f" [eou-span] args={kwargs}", flush=True)
return out
ar._ensure_eou_wait_span = _logged_span # type: ignore[method-assign]
import livekit.agents.voice.audio_recognition as ar_mod
_orig_metrics = ar_mod._compute_end_of_turn_metrics
def _logged_metrics(**kwargs):
tl.mark("eou_metrics_start")
out = _orig_metrics(**kwargs)
tl.mark("eou_metrics_end")
return out
ar_mod._compute_end_of_turn_metrics = _logged_metrics
_orig_auth = type(activity)._scheduling_task
async def _logged_sched_task(self_inner):
tl.mark("_scheduling_task_start")
await _orig_auth(self_inner)
type(activity)._scheduling_task = _logged_sched_task
# 跑完一轮:回放 13.26s + 尾部静音(覆盖 FINAL→生成→TTS→播放收尾)
await asyncio.sleep(13.26 + TAIL_SILENCE)
await session.aclose()
pe = tl.playback_end
if pe is None:
raise SystemExit("playback never ended?")
print(f"\nwav={WAV.name} marks 相对回放结束(秒):")
print(f"{'mark':28s} {'t-playback_end':>15s}")
for name, at in tl.marks:
print(f"{name:28s} {at - pe:>+15.3f}")
# 与 HEL-910 实测对照
stt_final = next((at for n, at in tl.marks if n == "stt_final_emitted"), None)
chat = next((at for n, at in tl.marks if n == "llm_chat_called"), None)
if stt_final and chat:
gap = chat - stt_final
print(f"\nFINAL->llm.chat 间隙: {gap:+.3f}s "
f"(HEL-910 实测 user_final->turn_start = +3.667s / +3.654s)")
baseline_note = ("BASELINE(默认英文 TurnDetector 在)" if baseline
else "生产 TURN_HANDLING(turn_detection=None)")
print(f"模式: {baseline_note}")
if __name__ == "__main__":
asyncio.run(main())