271 lines
11 KiB
Python
271 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate cg.subtitle v2 frames from the ACTUAL fc905fe worker on the .12 real SDK.
|
|
|
|
Unified offline review (HEL-879 + HEL-898 integration). Same discipline as
|
|
scripts/gen_hel879_timing_probe.py: the repo voice-module sources are embedded
|
|
into a self-contained script that runs on .12's agent venv (livekit-agents
|
|
1.8.8) via SSH stdin — zero writes on .12, zero network, no service touched.
|
|
The only stand-ins are the HTTP client (the unified /api/chat SSE turn is fed
|
|
in-process) and the playout clock (SpeechHandle._mark_done / .interrupt on a
|
|
timer); every frame is produced by the real worker wiring:
|
|
make_voice_llm + bind_voice_session callbacks on a real AgentSession,
|
|
real SpeechHandle lifecycle, real SubtitleBus envelopes.
|
|
|
|
Scenarios (each -> web/tests/fixtures/hel879/worker-*.jsonl + sidecar json):
|
|
normal interim/user_final -> turn_start -> interims -> final -> end + ack
|
|
interrupt barge-in: cancel without final, then a clean next turn
|
|
phi chat.error phi line: error frame + cancel, no final
|
|
no_receipt incomplete/no-receipt stream: cancel only (fc905fe timing gate)
|
|
proactive proactive_start DURING playout -> end; and an interrupted variant
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
MODULES = ["chat_api_client", "receipt", "seam", "llm_chat_api",
|
|
"stt_funasr", "tts_cosyvoice", "worker"]
|
|
|
|
TEMPLATE = r'''
|
|
import asyncio, base64, json, sys, types
|
|
|
|
SOURCES = json.loads(base64.b64decode("__B64__"))
|
|
|
|
pkg = types.ModuleType("cybergril"); pkg.__path__ = []
|
|
voice = types.ModuleType("cybergril.voice"); voice.__path__ = []
|
|
sys.modules["cybergril"] = pkg
|
|
sys.modules["cybergril.voice"] = voice
|
|
for name, src in SOURCES.items():
|
|
mod = types.ModuleType(f"cybergril.voice.{name}")
|
|
mod.__package__ = "cybergril.voice"
|
|
sys.modules[f"cybergril.voice.{name}"] = mod
|
|
exec(compile(src, f"<cybergril.voice.{name}>", "exec"), mod.__dict__)
|
|
setattr(voice, name, mod)
|
|
|
|
from livekit.agents import AgentSession
|
|
from livekit.agents.llm import ChatContext, ChatMessage
|
|
from livekit.agents.voice import agent_activity
|
|
from livekit.agents.voice.events import SpeechCreatedEvent, UserInputTranscribedEvent
|
|
from livekit.agents.voice.speech_handle import SpeechHandle
|
|
|
|
w = voice.worker
|
|
out = {"sdk": {}, "scenarios": {}}
|
|
|
|
|
|
class FakeClient:
|
|
"""In-process /api/chat: the worker's only model/LLM path, fed offline."""
|
|
|
|
def __init__(self):
|
|
self.epoch = 1
|
|
self.acks = []
|
|
|
|
async def current_epoch(self):
|
|
return self.epoch
|
|
|
|
async def send_turn(self, text, *, turn_id=None, epoch=None, on_delta=None):
|
|
reply = text and dict(TEXTS)[text] or None
|
|
if reply is None:
|
|
raise RuntimeError("unknown scripted utterance")
|
|
for part in reply["deltas"]:
|
|
if on_delta:
|
|
on_delta(part)
|
|
receipt = None
|
|
if reply.get("receipt"):
|
|
receipt = receipt_for(turn_id)
|
|
t = voice.chat_api_client.ChatApiTurn(
|
|
turn_id=turn_id, epoch=self.epoch, text=reply["text"],
|
|
receipt=receipt, error_phi=reply.get("error_phi"),
|
|
completed=reply.get("completed", False))
|
|
return t
|
|
|
|
async def acknowledge(self, receipt):
|
|
self.acks.append(receipt)
|
|
return True
|
|
|
|
|
|
TEXTS = {}
|
|
|
|
|
|
def script_turn(user_text, deltas, *, text="", receipt=False, error_phi=None, completed=False):
|
|
TEXTS[user_text] = {"deltas": deltas, "text": text or "".join(deltas),
|
|
"receipt": receipt, "error_phi": error_phi,
|
|
"completed": completed}
|
|
|
|
|
|
def receipt_for(turn_id, message_id=101):
|
|
return {"message_id": message_id, "turn_id": turn_id, "epoch": 1,
|
|
"token": "r" * 48, "text_hash": "h" * 64}
|
|
|
|
|
|
def rig():
|
|
frames = []
|
|
session = AgentSession()
|
|
ledger = voice.receipt.TurnLedger()
|
|
bus = voice.seam.SubtitleBus(frames.append)
|
|
interim = voice.seam.AssistantInterimPublisher(bus)
|
|
client = FakeClient()
|
|
llm = w.make_voice_llm("http://offline", ledger=ledger, interim=interim, client=client)
|
|
receipts = voice.receipt.ReceiptSender(client)
|
|
w.bind_voice_session(session, llm=llm, ledger=ledger, interim=interim,
|
|
bus=bus, stt=types.SimpleNamespace(), receipts=receipts)
|
|
return frames, session, ledger, interim, client, llm
|
|
|
|
|
|
async def drive_assistant_turn(session, llm, user_text, interrupt_at_delta=None):
|
|
"""One real speech: SpeechCreatedEvent -> ContextVar attribution -> real
|
|
CyberGrillLLM.chat -> scripted /api/chat stream -> playout done."""
|
|
h = SpeechHandle.create()
|
|
session.emit("speech_created", SpeechCreatedEvent(
|
|
user_initiated=True, source="generate_reply", speech_handle=h, created_at=1.0))
|
|
token = agent_activity._SpeechHandleContextVar.set(h)
|
|
stream = llm.chat(chat_ctx=ChatContext([ChatMessage(role="user", content=[user_text])]))
|
|
agent_activity._SpeechHandleContextVar.reset(token)
|
|
# consume the chunks exactly as the SDK pipeline would; the deltas have
|
|
# already fed the subtitles through the adapter's own callback.
|
|
async for _ in stream:
|
|
pass
|
|
if interrupt_at_delta is not None:
|
|
h.interrupt()
|
|
h._mark_done()
|
|
for _ in range(20):
|
|
await asyncio.sleep(0)
|
|
await asyncio.sleep(0.05)
|
|
return h, stream
|
|
|
|
|
|
def user_says(session, text, final):
|
|
session.emit("user_input_transcribed", UserInputTranscribedEvent(
|
|
transcript=text, is_final=final))
|
|
|
|
|
|
# --- scenario: normal turn ----------------------------------------------------
|
|
async def normal():
|
|
script_turn("你在听吗", ["在听呢,", "你刚才说到一半的那件事", "我还在想。"],
|
|
text="在听呢,你刚才说到一半的那件事我还在想。", receipt=True, completed=True)
|
|
frames, session, ledger, interim, client, llm = rig()
|
|
user_says(session, "你在听", False)
|
|
user_says(session, "你在听吗", True)
|
|
h, stream = await drive_assistant_turn(session, llm, "你在听吗")
|
|
out["scenarios"]["normal"] = {
|
|
"frames": frames, "turn_id": stream.turn_id,
|
|
"acks": [a["turn_id"] for a in client.acks],
|
|
"kinds": [f["kind"] for f in frames]}
|
|
|
|
|
|
# --- scenario: barge-in interrupt + clean next turn ---------------------------
|
|
async def interrupt():
|
|
script_turn("讲讲你下午做了什么", ["我下午去了河边,", "看到一群白鹭从水面上飞过去了,"],
|
|
completed=False)
|
|
script_turn("那后来呢", ["后来我在桥上站了一会儿,", "风把水面吹皱了。"], receipt=True, completed=True)
|
|
frames, session, ledger, interim, client, llm = rig()
|
|
user_says(session, "讲讲你下午做了什么", True)
|
|
h, stream = await drive_assistant_turn(session, llm, "讲讲你下午做了什么", interrupt_at_delta=True)
|
|
user_says(session, "那后来呢", True)
|
|
h2, stream2 = await drive_assistant_turn(session, llm, "那后来呢")
|
|
out["scenarios"]["interrupt"] = {
|
|
"frames": frames,
|
|
"turn_ids": {"cut": stream.turn_id, "next": stream2.turn_id},
|
|
"acks": [a["turn_id"] for a in client.acks],
|
|
"kinds": [f["kind"] for f in frames]}
|
|
|
|
|
|
# --- scenario: chat.error phi line -------------------------------------------
|
|
async def phi():
|
|
script_turn("说说那件事吧", ["(她顿了一下)", "这会儿我有点说不出来话……"],
|
|
text="", error_phi="(她顿了一下)这会儿我有点说不出来话……", completed=False)
|
|
frames, session, ledger, interim, client, llm = rig()
|
|
user_says(session, "说说那件事吧", True)
|
|
h, stream = await drive_assistant_turn(session, llm, "说说那件事吧")
|
|
out["scenarios"]["phi"] = {
|
|
"frames": frames, "turn_id": stream.turn_id,
|
|
"acks": [a["turn_id"] for a in client.acks],
|
|
"kinds": [f["kind"] for f in frames]}
|
|
|
|
|
|
# --- scenario: complete text but no legal receipt (fc905fe gate) --------------
|
|
async def no_receipt():
|
|
script_turn("再讲讲那座桥", ["那座桥是石头砌的,", "下雨的时候水声很好听。"],
|
|
text="那座桥是石头砌的,下雨的时候水声很好听。", completed=True)
|
|
frames, session, ledger, interim, client, llm = rig()
|
|
user_says(session, "再讲讲那座桥", True)
|
|
h, stream = await drive_assistant_turn(session, llm, "再讲讲那座桥")
|
|
out["scenarios"]["no_receipt"] = {
|
|
"frames": frames, "turn_id": stream.turn_id,
|
|
"acks": [a["turn_id"] for a in client.acks],
|
|
"kinds": [f["kind"] for f in frames]}
|
|
|
|
|
|
# --- scenario: proactive reach-out rows (real SpeechHandle playout) -----------
|
|
async def proactive():
|
|
frames = []
|
|
bus = voice.seam.SubtitleBus(frames.append)
|
|
done_flags = {}
|
|
|
|
class Say:
|
|
def __init__(self, interrupt=False):
|
|
self.interrupt = interrupt
|
|
|
|
def say(self, *args, **kwargs): # real SDK: sync, returns SpeechHandle
|
|
h = SpeechHandle.create()
|
|
loop = asyncio.get_running_loop()
|
|
if self.interrupt:
|
|
loop.call_later(0.04, h.interrupt)
|
|
loop.call_later(0.08, h._mark_done)
|
|
else:
|
|
loop.call_later(0.09, h._mark_done)
|
|
done_flags[h.id] = h
|
|
return h
|
|
|
|
said = Say()
|
|
task = asyncio.create_task(w.speak_proactive_row(
|
|
said, bus, {"id": 301, "text": "刚才路过花店,想起你说过喜欢白色的洋桔梗。"}))
|
|
await asyncio.sleep(0.02)
|
|
start_frames_early = [f for f in frames if f["kind"] == "proactive_start"]
|
|
handle_ids = list(done_flags)
|
|
start_while_playing = bool(start_frames_early) and not any(
|
|
done_flags[i].done() for i in handle_ids)
|
|
await task
|
|
await asyncio.sleep(0.02)
|
|
|
|
task2 = asyncio.create_task(w.speak_proactive_row(
|
|
Say(interrupt=True), bus, {"id": 302, "text": "晚上风大,窗台的那盆薄荷记得搬进来。"}))
|
|
await task2
|
|
await asyncio.sleep(0.02)
|
|
out["scenarios"]["proactive"] = {
|
|
"frames": frames,
|
|
"start_while_playing": start_while_playing,
|
|
"kinds": [f["kind"] for f in frames],
|
|
"message_ids": sorted({f["message_id"] for f in frames if "message_id" in f})}
|
|
|
|
|
|
async def main():
|
|
import livekit.agents, livekit.api, livekit
|
|
out["sdk"] = {"livekit_agents": livekit.agents.__version__,
|
|
"livekit_api": livekit.api.__version__}
|
|
for fn in (normal, interrupt, phi, no_receipt, proactive):
|
|
await fn()
|
|
print(json.dumps(out, ensure_ascii=False))
|
|
|
|
asyncio.run(main())
|
|
'''
|
|
|
|
|
|
def main() -> int:
|
|
sources = {}
|
|
for name in MODULES:
|
|
path = ROOT / "src/cybergril/voice" / f"{name}.py"
|
|
sources[name] = path.read_text(encoding="utf-8")
|
|
payload = base64.b64encode(json.dumps(sources).encode()).decode()
|
|
script = TEMPLATE.replace("__B64__", payload)
|
|
out = ROOT / "scripts/_hel879_unified_frames_payload.py"
|
|
out.write_text(script, encoding="utf-8")
|
|
print(out)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|