Files
CyberGril/scripts/gen_hel879_unified_frames.py

271 lines
11 KiB
Python

#!/usr/bin/env python3
"""Generate cg.subtitle v2 frames from the ACTUAL fc905fe worker on the .12 real SDK.
Unified offline review (HEL-879 + HEL-898 integration). Same discipline as
scripts/gen_hel879_timing_probe.py: the repo voice-module sources are embedded
into a self-contained script that runs on .12's agent venv (livekit-agents
1.8.8) via SSH stdin — zero writes on .12, zero network, no service touched.
The only stand-ins are the HTTP client (the unified /api/chat SSE turn is fed
in-process) and the playout clock (SpeechHandle._mark_done / .interrupt on a
timer); every frame is produced by the real worker wiring:
make_voice_llm + bind_voice_session callbacks on a real AgentSession,
real SpeechHandle lifecycle, real SubtitleBus envelopes.
Scenarios (each -> web/tests/fixtures/hel879/worker-*.jsonl + sidecar json):
normal interim/user_final -> turn_start -> interims -> final -> end + ack
interrupt barge-in: cancel without final, then a clean next turn
phi chat.error phi line: error frame + cancel, no final
no_receipt incomplete/no-receipt stream: cancel only (fc905fe timing gate)
proactive proactive_start DURING playout -> end; and an interrupted variant
"""
from __future__ import annotations
import base64
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
MODULES = ["chat_api_client", "receipt", "seam", "llm_chat_api",
"stt_funasr", "tts_cosyvoice", "worker"]
TEMPLATE = r'''
import asyncio, base64, json, sys, types
SOURCES = json.loads(base64.b64decode("__B64__"))
pkg = types.ModuleType("cybergril"); pkg.__path__ = []
voice = types.ModuleType("cybergril.voice"); voice.__path__ = []
sys.modules["cybergril"] = pkg
sys.modules["cybergril.voice"] = voice
for name, src in SOURCES.items():
mod = types.ModuleType(f"cybergril.voice.{name}")
mod.__package__ = "cybergril.voice"
sys.modules[f"cybergril.voice.{name}"] = mod
exec(compile(src, f"<cybergril.voice.{name}>", "exec"), mod.__dict__)
setattr(voice, name, mod)
from livekit.agents import AgentSession
from livekit.agents.llm import ChatContext, ChatMessage
from livekit.agents.voice import agent_activity
from livekit.agents.voice.events import SpeechCreatedEvent, UserInputTranscribedEvent
from livekit.agents.voice.speech_handle import SpeechHandle
w = voice.worker
out = {"sdk": {}, "scenarios": {}}
class FakeClient:
"""In-process /api/chat: the worker's only model/LLM path, fed offline."""
def __init__(self):
self.epoch = 1
self.acks = []
async def current_epoch(self):
return self.epoch
async def send_turn(self, text, *, turn_id=None, epoch=None, on_delta=None):
reply = text and dict(TEXTS)[text] or None
if reply is None:
raise RuntimeError("unknown scripted utterance")
for part in reply["deltas"]:
if on_delta:
on_delta(part)
receipt = None
if reply.get("receipt"):
receipt = receipt_for(turn_id)
t = voice.chat_api_client.ChatApiTurn(
turn_id=turn_id, epoch=self.epoch, text=reply["text"],
receipt=receipt, error_phi=reply.get("error_phi"),
completed=reply.get("completed", False))
return t
async def acknowledge(self, receipt):
self.acks.append(receipt)
return True
TEXTS = {}
def script_turn(user_text, deltas, *, text="", receipt=False, error_phi=None, completed=False):
TEXTS[user_text] = {"deltas": deltas, "text": text or "".join(deltas),
"receipt": receipt, "error_phi": error_phi,
"completed": completed}
def receipt_for(turn_id, message_id=101):
return {"message_id": message_id, "turn_id": turn_id, "epoch": 1,
"token": "r" * 48, "text_hash": "h" * 64}
def rig():
frames = []
session = AgentSession()
ledger = voice.receipt.TurnLedger()
bus = voice.seam.SubtitleBus(frames.append)
interim = voice.seam.AssistantInterimPublisher(bus)
client = FakeClient()
llm = w.make_voice_llm("http://offline", ledger=ledger, interim=interim, client=client)
receipts = voice.receipt.ReceiptSender(client)
w.bind_voice_session(session, llm=llm, ledger=ledger, interim=interim,
bus=bus, stt=types.SimpleNamespace(), receipts=receipts)
return frames, session, ledger, interim, client, llm
async def drive_assistant_turn(session, llm, user_text, interrupt_at_delta=None):
"""One real speech: SpeechCreatedEvent -> ContextVar attribution -> real
CyberGrillLLM.chat -> scripted /api/chat stream -> playout done."""
h = SpeechHandle.create()
session.emit("speech_created", SpeechCreatedEvent(
user_initiated=True, source="generate_reply", speech_handle=h, created_at=1.0))
token = agent_activity._SpeechHandleContextVar.set(h)
stream = llm.chat(chat_ctx=ChatContext([ChatMessage(role="user", content=[user_text])]))
agent_activity._SpeechHandleContextVar.reset(token)
# consume the chunks exactly as the SDK pipeline would; the deltas have
# already fed the subtitles through the adapter's own callback.
async for _ in stream:
pass
if interrupt_at_delta is not None:
h.interrupt()
h._mark_done()
for _ in range(20):
await asyncio.sleep(0)
await asyncio.sleep(0.05)
return h, stream
def user_says(session, text, final):
session.emit("user_input_transcribed", UserInputTranscribedEvent(
transcript=text, is_final=final))
# --- scenario: normal turn ----------------------------------------------------
async def normal():
script_turn("你在听吗", ["在听呢,", "你刚才说到一半的那件事", "我还在想。"],
text="在听呢,你刚才说到一半的那件事我还在想。", receipt=True, completed=True)
frames, session, ledger, interim, client, llm = rig()
user_says(session, "你在听", False)
user_says(session, "你在听吗", True)
h, stream = await drive_assistant_turn(session, llm, "你在听吗")
out["scenarios"]["normal"] = {
"frames": frames, "turn_id": stream.turn_id,
"acks": [a["turn_id"] for a in client.acks],
"kinds": [f["kind"] for f in frames]}
# --- scenario: barge-in interrupt + clean next turn ---------------------------
async def interrupt():
script_turn("讲讲你下午做了什么", ["我下午去了河边,", "看到一群白鹭从水面上飞过去了,"],
completed=False)
script_turn("那后来呢", ["后来我在桥上站了一会儿,", "风把水面吹皱了。"], receipt=True, completed=True)
frames, session, ledger, interim, client, llm = rig()
user_says(session, "讲讲你下午做了什么", True)
h, stream = await drive_assistant_turn(session, llm, "讲讲你下午做了什么", interrupt_at_delta=True)
user_says(session, "那后来呢", True)
h2, stream2 = await drive_assistant_turn(session, llm, "那后来呢")
out["scenarios"]["interrupt"] = {
"frames": frames,
"turn_ids": {"cut": stream.turn_id, "next": stream2.turn_id},
"acks": [a["turn_id"] for a in client.acks],
"kinds": [f["kind"] for f in frames]}
# --- scenario: chat.error phi line -------------------------------------------
async def phi():
script_turn("说说那件事吧", ["(她顿了一下)", "这会儿我有点说不出来话……"],
text="", error_phi="(她顿了一下)这会儿我有点说不出来话……", completed=False)
frames, session, ledger, interim, client, llm = rig()
user_says(session, "说说那件事吧", True)
h, stream = await drive_assistant_turn(session, llm, "说说那件事吧")
out["scenarios"]["phi"] = {
"frames": frames, "turn_id": stream.turn_id,
"acks": [a["turn_id"] for a in client.acks],
"kinds": [f["kind"] for f in frames]}
# --- scenario: complete text but no legal receipt (fc905fe gate) --------------
async def no_receipt():
script_turn("再讲讲那座桥", ["那座桥是石头砌的,", "下雨的时候水声很好听。"],
text="那座桥是石头砌的,下雨的时候水声很好听。", completed=True)
frames, session, ledger, interim, client, llm = rig()
user_says(session, "再讲讲那座桥", True)
h, stream = await drive_assistant_turn(session, llm, "再讲讲那座桥")
out["scenarios"]["no_receipt"] = {
"frames": frames, "turn_id": stream.turn_id,
"acks": [a["turn_id"] for a in client.acks],
"kinds": [f["kind"] for f in frames]}
# --- scenario: proactive reach-out rows (real SpeechHandle playout) -----------
async def proactive():
frames = []
bus = voice.seam.SubtitleBus(frames.append)
done_flags = {}
class Say:
def __init__(self, interrupt=False):
self.interrupt = interrupt
def say(self, *args, **kwargs): # real SDK: sync, returns SpeechHandle
h = SpeechHandle.create()
loop = asyncio.get_running_loop()
if self.interrupt:
loop.call_later(0.04, h.interrupt)
loop.call_later(0.08, h._mark_done)
else:
loop.call_later(0.09, h._mark_done)
done_flags[h.id] = h
return h
said = Say()
task = asyncio.create_task(w.speak_proactive_row(
said, bus, {"id": 301, "text": "刚才路过花店,想起你说过喜欢白色的洋桔梗。"}))
await asyncio.sleep(0.02)
start_frames_early = [f for f in frames if f["kind"] == "proactive_start"]
handle_ids = list(done_flags)
start_while_playing = bool(start_frames_early) and not any(
done_flags[i].done() for i in handle_ids)
await task
await asyncio.sleep(0.02)
task2 = asyncio.create_task(w.speak_proactive_row(
Say(interrupt=True), bus, {"id": 302, "text": "晚上风大,窗台的那盆薄荷记得搬进来。"}))
await task2
await asyncio.sleep(0.02)
out["scenarios"]["proactive"] = {
"frames": frames,
"start_while_playing": start_while_playing,
"kinds": [f["kind"] for f in frames],
"message_ids": sorted({f["message_id"] for f in frames if "message_id" in f})}
async def main():
import livekit.agents, livekit.api, livekit
out["sdk"] = {"livekit_agents": livekit.agents.__version__,
"livekit_api": livekit.api.__version__}
for fn in (normal, interrupt, phi, no_receipt, proactive):
await fn()
print(json.dumps(out, ensure_ascii=False))
asyncio.run(main())
'''
def main() -> int:
sources = {}
for name in MODULES:
path = ROOT / "src/cybergril/voice" / f"{name}.py"
sources[name] = path.read_text(encoding="utf-8")
payload = base64.b64encode(json.dumps(sources).encode()).decode()
script = TEMPLATE.replace("__B64__", payload)
out = ROOT / "scripts/_hel879_unified_frames_payload.py"
out.write_text(script, encoding="utf-8")
print(out)
return 0
if __name__ == "__main__":
sys.exit(main())