- worker _on_item: filter conversation_item_added by the real 1.8.8 item
discriminator (ChatMessage | AgentHandoff | _TypeDiscriminator); only
assistant str-part body feeds the echo filter (HEL-910 AttributeError)
- bind_interrupt_channel unload: async (JobContext shutdown callbacks are
awaited), idempotent, disconnect-safe; interrupt/receipt/budget paths
untouched (HEL-910 TypeError await None)
- TURN_HANDLING: explicit turn_detection=None — 1.8.8 silently installs an
English EOU TurnDetector whose unlikely verdicts on Chinese finals push
endpointing to max_delay=6.0 (measured 3.667/3.654s HEL-910 call2; offline
A/B on the real SDK: baseline 3.704s vs fixed 0.004s). Endpointing
0.8/6.0 kept verbatim; no VAD/buffer_turn/semantic change
- scripts/gen_hel911_wait_repro.py: offline reproduction harness (real
AgentSession + real silero VAD + HEL-910 wav and measured timings)
- tests: +3 real-SDK checks (11/11 on .12 agent venv; prior 8 regressions
kept, unload call sites await the new protocol)
- hel910_call1.mjs reliability: segmented probe/recorder dumps to disk,
timing-A trigger requires the server-committed voice turn row (not a
we-sent-it proxy); node --check only, no real run
- window-record.md §9: appended factual corrections (companion 9 not 18,
turn_start +6.099 not +4.10, assistant_final is the playout/receipt gate,
post-window ListRooms token failure is not empty-room evidence); original
record untouched
- docs/rework/hel911/summary.md + handoff-HEL-911.json; requirement checks
pass vs origin/main and 407fc81
Co-authored-by: multica-agent <github@multica.ai>
433 lines
17 KiB
Python
433 lines
17 KiB
Python
"""HEL-911 离线等待链复现:真 AgentSession + 真 silero VAD + 脚本化 STT/LLM/TTS.
|
||
|
||
目的:把 HEL-910 call2 实测的「识别终稿 → 生成开始」3.667s 间隙(q1
|
||
+2.432→+6.099、q2 +48.525→+52.179,两轮同为 3.65~3.67s)在完全离线的
|
||
真实 SDK 1.8.8 里复现或排除,从而把 9.5s 首声等待的分段归因落到可检验
|
||
的事实上。
|
||
|
||
做法(全部本地计算,零网络,零 LLM/ASR/TTS 外呼):
|
||
- 输入:HEL-910 真实回放用过的 /opt/cybergril-hel910/etc/trim/u_bajie.trim.wav
|
||
按 100ms 帧、真实时钟节奏推给 session.input.audio(与 hel910_client 的
|
||
发布节奏一致),播完继续推真零样本静音帧(对应 asr-usage.jsonl 证实
|
||
的连续帧流 ~100.5% 实时);
|
||
- STT:脚本化 stt.STT,按实测时间发 INTERIM(语音期间)与 FINAL(回放
|
||
结束后 +2.432s,即 FunASR 离线纠错 + 适配器 0.6s 去抖的实测结果),
|
||
不设 end_time(与 FunASRSTT 的 SpeechData 形状一致);
|
||
- LLM:脚本化 LLM,记录 chat() 调用时刻(= 生产链路里 on_stream_started
|
||
→ 字幕 turn_start 的同一时刻),1.415s 后发两条 delta(对应实测
|
||
turn_start→assistant_interim 的服务端整轮生成时延);
|
||
- TTS:脚本化 ChunkedStream,1.614s 后出首帧(对应实测
|
||
[tts] first audio after 1.614s,worker.log:220);
|
||
- 会话参数与 worker.py entrypoint 完全一致(真 VAD.load()、
|
||
endpointing min 0.8 / max 6.0)。
|
||
|
||
输出各里程碑相对回放结束(= wav 末帧后第一帧静音的时刻)的秒数,与
|
||
HEL-910 实测列对照。本脚本不修改任何服务,只读 wav。
|
||
|
||
运行(.12 agent venv):
|
||
/opt/voice-agent/agent/venv/bin/python scripts/gen_hel911_wait_repro.py
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import sys
|
||
import time
|
||
import wave
|
||
from pathlib import Path
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||
|
||
from livekit import rtc
|
||
from livekit.agents import stt, tts, llm as lk_llm
|
||
from livekit.agents.llm import ChatContext, LLM, LLMStream, Tool
|
||
from livekit.agents.stt import RecognizeStream, SpeechData, SpeechEvent, SpeechEventType
|
||
from livekit.agents.tts import AudioEmitter, ChunkedStream
|
||
from livekit.agents.types import DEFAULT_API_CONNECT_OPTIONS, APIConnectOptions
|
||
from livekit.agents.voice import Agent, AgentSession
|
||
from livekit.plugins.silero import VAD
|
||
|
||
WAV = Path("/opt/cybergril-hel910/etc/trim/u_bajie.trim.wav")
|
||
# HEL-910 call2 实测时延(相对回放结束/各起点)
|
||
FINAL_AFTER_PLAYBACK_END = 2.432 # user_final(VAD+FunASR 离线+0.6s 去抖)
|
||
GEN_AFTER_TURN_START = 1.415 # turn_start -> assistant_interim(服务端整轮)
|
||
TTS_FIRST_AUDIO = 1.614 # [tts] first audio after(worker.log:220)
|
||
TAIL_SILENCE = 14.0 # 回放结束后继续推的静音时长
|
||
|
||
|
||
def _now() -> float:
|
||
return time.perf_counter()
|
||
|
||
|
||
class Timeline:
|
||
def __init__(self) -> None:
|
||
self.t0: float | None = None
|
||
self.playback_end: float | None = None
|
||
self.marks: list[tuple[str, float]] = []
|
||
|
||
def mark(self, name: str) -> None:
|
||
self.marks.append((name, _now()))
|
||
|
||
|
||
class ScriptedSTT(stt.STT):
|
||
"""按 HEL-910 实测时间发 FINAL 的 STT;不连任何识别服务。"""
|
||
|
||
def __init__(self, tl: Timeline) -> None:
|
||
super().__init__(capabilities=stt.STTCapabilities(streaming=True, interim_results=True))
|
||
self._tl = tl
|
||
|
||
@property
|
||
def provider(self) -> str:
|
||
return "scripted"
|
||
|
||
@property
|
||
def model(self) -> str:
|
||
return "hel910-timeline"
|
||
|
||
async def _recognize_impl(self, buffer, *, language=None,
|
||
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
|
||
) -> SpeechEvent:
|
||
raise NotImplementedError("scripted STT only supports streaming")
|
||
|
||
def stream(self, *, language=None,
|
||
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
|
||
) -> RecognizeStream:
|
||
return ScriptedRecognizeStream(stt=self, conn_options=conn_options, tl=self._tl)
|
||
|
||
|
||
class ScriptedRecognizeStream(RecognizeStream):
|
||
def __init__(self, *, stt: ScriptedSTT, conn_options: APIConnectOptions,
|
||
tl: Timeline) -> None:
|
||
super().__init__(stt=stt, conn_options=conn_options)
|
||
self._tl = tl
|
||
|
||
def _ev(self, ev_type: SpeechEventType, text: str) -> SpeechEvent:
|
||
return SpeechEvent(type=ev_type,
|
||
alternatives=[SpeechData(language="zh", text=text)])
|
||
|
||
async def _run(self) -> None:
|
||
tl = self._tl
|
||
|
||
async def schedule_final() -> None:
|
||
# 等回放结束锚点(由输入源标记),再按实测延迟发终稿
|
||
while tl.playback_end is None:
|
||
await asyncio.sleep(0.02)
|
||
await asyncio.sleep(FINAL_AFTER_PLAYBACK_END)
|
||
tl.mark("stt_final_emitted")
|
||
self._event_ch.send_nowait(
|
||
self._ev(SpeechEventType.FINAL_TRANSCRIPT,
|
||
"师傅,哎,这取经路比高,老庄种地还苦,俺老猪饿的能吞下石龙素"
|
||
"包子,猴哥,你筋斗云块,要不先去化个缘"))
|
||
|
||
final_task = asyncio.create_task(schedule_final())
|
||
sent_interim = False
|
||
try:
|
||
async for ev in self._input_ch:
|
||
if ev is None:
|
||
break
|
||
if isinstance(ev, rtc.AudioFrame) and not sent_interim:
|
||
sent_interim = True
|
||
self._event_ch.send_nowait(
|
||
self._ev(SpeechEventType.INTERIM_TRANSCRIPT, "师傅"))
|
||
await asyncio.sleep(0)
|
||
finally:
|
||
final_task.cancel()
|
||
|
||
|
||
class ProbeLLM(LLM):
|
||
def __init__(self, tl: Timeline) -> None:
|
||
super().__init__()
|
||
self._tl = tl
|
||
|
||
@property
|
||
def provider(self) -> str:
|
||
return "probe"
|
||
|
||
@property
|
||
def model(self) -> str:
|
||
return "hel911-wait-probe"
|
||
|
||
def chat(self, *, chat_ctx: ChatContext, tools: list[Tool] | None = None,
|
||
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, **_
|
||
) -> LLMStream:
|
||
self._tl.mark("llm_chat_called") # = 生产链路的 turn_start 时刻
|
||
return ProbeLLMStream(llm=self, chat_ctx=chat_ctx, tools=tools or [],
|
||
conn_options=conn_options, tl=self._tl)
|
||
|
||
|
||
class ProbeLLMStream(LLMStream):
|
||
def __init__(self, *, llm: ProbeLLM, chat_ctx: ChatContext, tools: list[Tool],
|
||
conn_options: APIConnectOptions, tl: Timeline) -> None:
|
||
super().__init__(llm, chat_ctx=chat_ctx, tools=tools, conn_options=conn_options)
|
||
self._tl = tl
|
||
|
||
async def _run(self) -> None:
|
||
from livekit.agents.llm import ChatChunk, ChoiceDelta
|
||
|
||
await asyncio.sleep(GEN_AFTER_TURN_START)
|
||
self._tl.mark("first_delta")
|
||
self._event_ch.send_nowait(ChatChunk(
|
||
id="hel911-probe",
|
||
delta=ChoiceDelta(role="assistant", content="好,小十。去化缘")))
|
||
self._event_ch.send_nowait(ChatChunk(
|
||
id="hel911-probe", delta=ChoiceDelta(role="assistant", content="")))
|
||
self._event_ch.send_nowait(ChatChunk(id="hel911-probe"))
|
||
|
||
|
||
class ProbeTTS(tts.TTS):
|
||
def __init__(self, tl: Timeline) -> None:
|
||
super().__init__(capabilities=tts.TTSCapabilities(streaming=False),
|
||
sample_rate=24000, num_channels=1)
|
||
self._tl = tl
|
||
|
||
@property
|
||
def provider(self) -> str:
|
||
return "probe"
|
||
|
||
@property
|
||
def model(self) -> str:
|
||
return "hel911-wait-probe"
|
||
|
||
def synthesize(self, text: str, *,
|
||
conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS
|
||
) -> ChunkedStream:
|
||
return ProbeChunkedStream(tts=self, input_text=text,
|
||
conn_options=conn_options, tl=self._tl)
|
||
|
||
|
||
class ProbeChunkedStream(ChunkedStream):
|
||
def __init__(self, *, tts: ProbeTTS, input_text: str,
|
||
conn_options: APIConnectOptions, tl: Timeline) -> None:
|
||
super().__init__(tts=tts, input_text=input_text, conn_options=conn_options)
|
||
self._tl = tl
|
||
|
||
async def _run(self, output_emitter: AudioEmitter) -> None:
|
||
output_emitter.initialize(request_id="hel911", sample_rate=24000,
|
||
num_channels=1, mime_type="audio/pcm")
|
||
await asyncio.sleep(TTS_FIRST_AUDIO) # 实测 CosyVoice3 首包
|
||
self._tl.mark("tts_first_audio")
|
||
# ~1s 的可听音频(20ms 帧),足够走完 playout/回执路径
|
||
frame = b"\x10\x00" * 480
|
||
for _ in range(50):
|
||
output_emitter.push_frame(rtc.AudioFrame(frame, 24000, 1, 480))
|
||
await asyncio.sleep(0.02)
|
||
output_emitter.flush()
|
||
|
||
|
||
def make_sink(tl: Timeline):
|
||
from livekit.agents.voice.io import AudioOutput, AudioOutputCapabilities
|
||
|
||
class _Sink(AudioOutput):
|
||
def __init__(self) -> None:
|
||
super().__init__(label="probe-sink",
|
||
capabilities=AudioOutputCapabilities(pause=False))
|
||
self.first_frame_at: float | None = None
|
||
|
||
async def capture_frame(self, frame: rtc.AudioFrame) -> None:
|
||
if self.first_frame_at is None:
|
||
tl.mark("sink_first_frame")
|
||
self.first_frame_at = _now()
|
||
|
||
def flush(self) -> None:
|
||
self.on_playback_finished(interrupted=False)
|
||
|
||
def clear_buffer(self) -> None:
|
||
self.on_playback_finished(interrupted=True)
|
||
|
||
return _Sink()
|
||
|
||
|
||
class MicSource:
|
||
"""wav 实时回放 + 尾部连续静音,节奏与 hel910_client._publish_pcm 一致。"""
|
||
|
||
def __init__(self, tl: Timeline) -> None:
|
||
self._tl = tl
|
||
|
||
def make(self):
|
||
tl = self._tl
|
||
from livekit.agents.voice.io import AudioInput
|
||
|
||
with wave.open(str(WAV), "rb") as w:
|
||
raw = w.readframes(w.getnframes())
|
||
rate = w.getframerate()
|
||
|
||
class _Source(AudioInput):
|
||
def __init__(self) -> None:
|
||
super().__init__(label="hel911-mic")
|
||
self._i = 0
|
||
self._phase = "wav"
|
||
|
||
async def __anext__(self) -> rtc.AudioFrame:
|
||
step = 1600 * 2 # 100ms @16k,同 hel910_client
|
||
if self._phase == "wav":
|
||
chunk = raw[self._i:self._i + step]
|
||
if tl.t0 is None:
|
||
tl.t0 = _now()
|
||
if len(chunk) < step:
|
||
chunk = chunk + b"\x00" * (step - len(chunk))
|
||
self._i += step
|
||
if self._i >= len(raw):
|
||
self._phase = "silence"
|
||
tl.playback_end = _now()
|
||
tl.mark("playback_end")
|
||
await asyncio.sleep(0.1)
|
||
return rtc.AudioFrame(chunk, rate, 1, 1600)
|
||
# 静音段:真实时钟节奏(帧流连续,asr-usage 证实)
|
||
await asyncio.sleep(0.1)
|
||
return rtc.AudioFrame(b"\x00" * step, rate, 1, 1600)
|
||
|
||
return _Source()
|
||
|
||
|
||
async def main() -> None:
|
||
if not WAV.exists():
|
||
raise SystemExit(f"wav not found: {WAV}")
|
||
|
||
import logging
|
||
logging.basicConfig(level=logging.INFO) # eot prediction 走 debug 时再降
|
||
logging.getLogger("livekit.agents.voice.audio_recognition").setLevel(logging.DEBUG)
|
||
|
||
import os
|
||
# 默认复刻生产接线(worker.TURN_HANDLING,已显式 turn_detection=None);
|
||
# HEL911_BASELINE=1 复现缺陷态:1.8.8 对缺省键默认补英文 TurnDetector
|
||
baseline = os.environ.get("HEL911_BASELINE") == "1"
|
||
|
||
from cybergril.voice.worker import TURN_HANDLING as WORKER_TURN_HANDLING
|
||
|
||
turn_handling = (
|
||
{k: v for k, v in WORKER_TURN_HANDLING.items() if k != "turn_detection"}
|
||
if baseline else dict(WORKER_TURN_HANDLING)
|
||
)
|
||
|
||
tl = Timeline()
|
||
session = AgentSession(
|
||
stt=ScriptedSTT(tl),
|
||
llm=ProbeLLM(tl),
|
||
tts=ProbeTTS(tl),
|
||
vad=VAD.load(),
|
||
turn_handling=turn_handling,
|
||
)
|
||
session.input.audio = MicSource(tl).make() # type: ignore[assignment]
|
||
session.output.audio = make_sink(tl) # type: ignore[assignment]
|
||
|
||
@session.on("user_state_changed")
|
||
def _ustate(ev):
|
||
tl.mark(f"user_state_{getattr(ev, 'new_state', '?')}")
|
||
|
||
@session.on("agent_state_changed")
|
||
def _astate(ev):
|
||
tl.mark(f"agent_state_{getattr(ev, 'new_state', '?')}")
|
||
|
||
@session.on("user_input_transcribed")
|
||
def _tr(ev):
|
||
if ev.is_final:
|
||
tl.mark("sdk_user_final")
|
||
|
||
await session.start(agent=Agent(instructions="probe"))
|
||
tl.mark("session_started")
|
||
|
||
# 定位 FINAL→llm.chat 间隙:给真 SDK 的提交/调度内部打点(只读包装,不改行为)
|
||
from livekit.agents.voice import agent_activity
|
||
|
||
activity = session._activity
|
||
|
||
_orig_eou = activity.on_end_of_turn
|
||
|
||
def _logged_eou(info):
|
||
committed = _orig_eou(info)
|
||
tl.mark(f"on_end_of_turn(committed={committed})")
|
||
return committed
|
||
|
||
activity.on_end_of_turn = _logged_eou # type: ignore[method-assign]
|
||
|
||
_orig_gen = activity._generate_reply
|
||
|
||
def _logged_gen(**kwargs):
|
||
handle = _orig_gen(**kwargs)
|
||
tl.mark("_generate_reply")
|
||
return handle
|
||
|
||
activity._generate_reply = _logged_gen # type: ignore[method-assign]
|
||
|
||
_orig_sched = activity._schedule_speech
|
||
|
||
def _logged_sched(speech, priority, force=False):
|
||
tl.mark("_schedule_speech")
|
||
return _orig_sched(speech, priority, force=force)
|
||
|
||
activity._schedule_speech = _logged_sched # type: ignore[method-assign]
|
||
|
||
_orig_run_eou = type(activity._audio_recognition)._run_eou_detection
|
||
|
||
def _logged_run_eou(self, chat_ctx, *, trigger, skip_reply=False):
|
||
tl.mark(f"_run_eou_detection({trigger})")
|
||
# EOU 锚点体检:提交前的 extra_sleep 由这些值决定
|
||
lst = self._last_speaking_time
|
||
print(f" [eou:{trigger}] speaking={self._speaking} "
|
||
f"last_speaking_time={'none' if lst is None else f'{time.time() - lst:+.3f}s ago'} "
|
||
f"final_at={'none' if self._last_final_transcript_time is None else f'{time.time() - self._last_final_transcript_time:+.3f}s ago'} "
|
||
f"endpointing_min={self._endpointing.min_delay} "
|
||
f"turn_detector={type(self._turn_detector).__name__ if self._turn_detector else None} "
|
||
f"td_stream={type(self._turn_detector_stream).__name__ if self._turn_detector_stream else None} "
|
||
f"mode={self._turn_detection_mode}", flush=True)
|
||
return _orig_run_eou(self, chat_ctx, trigger=trigger, skip_reply=skip_reply)
|
||
|
||
type(activity._audio_recognition)._run_eou_detection = _logged_run_eou
|
||
|
||
ar = activity._audio_recognition
|
||
_orig_span = ar._ensure_eou_wait_span
|
||
|
||
def _logged_span(user_turn_span, **kwargs):
|
||
tl.mark("_ensure_eou_wait_span")
|
||
out = _orig_span(user_turn_span, **kwargs)
|
||
print(f" [eou-span] args={kwargs}", flush=True)
|
||
return out
|
||
|
||
ar._ensure_eou_wait_span = _logged_span # type: ignore[method-assign]
|
||
|
||
import livekit.agents.voice.audio_recognition as ar_mod
|
||
|
||
_orig_metrics = ar_mod._compute_end_of_turn_metrics
|
||
|
||
def _logged_metrics(**kwargs):
|
||
tl.mark("eou_metrics_start")
|
||
out = _orig_metrics(**kwargs)
|
||
tl.mark("eou_metrics_end")
|
||
return out
|
||
|
||
ar_mod._compute_end_of_turn_metrics = _logged_metrics
|
||
|
||
_orig_auth = type(activity)._scheduling_task
|
||
|
||
async def _logged_sched_task(self_inner):
|
||
tl.mark("_scheduling_task_start")
|
||
await _orig_auth(self_inner)
|
||
|
||
type(activity)._scheduling_task = _logged_sched_task
|
||
|
||
# 跑完一轮:回放 13.26s + 尾部静音(覆盖 FINAL→生成→TTS→播放收尾)
|
||
await asyncio.sleep(13.26 + TAIL_SILENCE)
|
||
await session.aclose()
|
||
|
||
pe = tl.playback_end
|
||
if pe is None:
|
||
raise SystemExit("playback never ended?")
|
||
print(f"\nwav={WAV.name} marks 相对回放结束(秒):")
|
||
print(f"{'mark':28s} {'t-playback_end':>15s}")
|
||
for name, at in tl.marks:
|
||
print(f"{name:28s} {at - pe:>+15.3f}")
|
||
# 与 HEL-910 实测对照
|
||
stt_final = next((at for n, at in tl.marks if n == "stt_final_emitted"), None)
|
||
chat = next((at for n, at in tl.marks if n == "llm_chat_called"), None)
|
||
if stt_final and chat:
|
||
gap = chat - stt_final
|
||
print(f"\nFINAL->llm.chat 间隙: {gap:+.3f}s "
|
||
f"(HEL-910 实测 user_final->turn_start = +3.667s / +3.654s)")
|
||
baseline_note = ("BASELINE(默认英文 TurnDetector 在)" if baseline
|
||
else "生产 TURN_HANDLING(turn_detection=None)")
|
||
print(f"模式: {baseline_note}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(main())
|