Files
CyberGril/experiments/hel715/scripts/llm_chain2.py
T

125 lines
5.1 KiB
Python

#!/usr/bin/env python
"""LLM-chained conversation sample: .11 LLM -> .12 CosyVoice -> MuseTalk avatar.
Same receiver architecture as e2e_client.py (proven to capture A/V)."""
import asyncio, json, os, time
import aiohttp, av, numpy as np
from aiortc import RTCPeerConnection, RTCSessionDescription
BASE = "http://127.0.0.1:8010"
LLM = "http://192.168.200.11:8731/v1/chat/completions"
OUT = "/opt/hel715-verify/e2e"
os.makedirs(OUT, exist_ok=True)
class Recorder:
def __init__(self, path, w, h):
self.container = av.open(path, "w")
self.v = self.container.add_stream("libx264", rate=25)
self.v.width, self.v.height, self.v.pix_fmt = w, h, "yuv420p"
self.v.options = {"crf": "20", "preset": "veryfast"}
self.a = self.container.add_stream("aac", rate=48000)
self.a.layout = "stereo"
def video(self, img):
for p in self.v.encode(av.VideoFrame.from_ndarray(img, format="bgr24")):
self.container.mux(p)
def audio(self, frame):
arr = frame.to_ndarray()
if arr.dtype != np.int16:
arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16)
ch = len(frame.layout.channels)
if arr.ndim == 2 and arr.shape[0] == 1:
pcm = arr.reshape(-1)
elif arr.ndim == 2 and arr.shape[0] == ch:
pcm = arr.T.reshape(-1)
else:
pcm = arr.reshape(-1)
fr = av.AudioFrame.from_ndarray(np.ascontiguousarray(pcm.reshape(1, -1)), format="s16", layout="stereo")
fr.sample_rate = 48000
for p in self.a.encode(fr):
self.container.mux(p)
def close(self):
for p in self.v.encode(): self.container.mux(p)
for p in self.a.encode(): self.container.mux(p)
self.container.close()
async def main():
t0 = time.perf_counter()
result = {}
async with aiohttp.ClientSession() as http:
async with http.post(LLM, json={
"model": "halogen-qwen3.8-flash-next",
"messages": [{"role": "user", "content": "你是住在高层公寓卧室里的伴侣晚晚,用一两句自然中文回应:我回来了,今天好累啊。"}],
"max_tokens": 80,
}) as r:
body = await r.json()
reply = body["choices"][0]["message"]["content"]
result["llm_model"] = body["model"]
result["llm_reply"] = reply
result["llm_latency_s"] = round(time.perf_counter() - t0, 2)
print(f"[{result['llm_latency_s']}s] LLM: {reply}", flush=True)
pc = RTCPeerConnection()
rec = {"r": None}
audio_marks = []
@pc.on("track")
def on_track(track):
async def drain():
while True:
try:
f = await track.recv()
except Exception:
break
t = time.perf_counter()
if track.kind == "video":
img = f.to_ndarray(format="bgr24")
if rec["r"] is None:
rec["r"] = Recorder(f"{OUT}/llm-chain.mp4", img.shape[1], img.shape[0])
rec["r"].video(img)
else:
arr = f.to_ndarray()
if arr.dtype != np.int16:
arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16)
rms = float(np.sqrt((arr.astype(np.float64) ** 2).mean()) / 32767)
audio_marks.append([round(t - t0, 3), round(rms, 5)])
if rec["r"] is not None:
rec["r"].audio(f)
asyncio.ensure_future(drain())
pc.addTransceiver("video", direction="recvonly")
pc.addTransceiver("audio", direction="recvonly")
offer = await pc.createOffer()
await pc.setLocalDescription(offer)
while pc.iceGatheringState != "complete":
await asyncio.sleep(0.1)
async with http.post(BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r:
answer = await r.json()
await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"]))
sid = answer["sessionid"]
await asyncio.sleep(3)
t_send = time.perf_counter() - t0
async with http.post(BASE + "/human", json={"sessionid": sid, "type": "echo", "text": reply}) as r:
await r.json()
print(f"[{t_send:.2f}s] sent to avatar", flush=True)
await asyncio.sleep(15)
await pc.close()
if rec["r"]:
rec["r"].close()
loud = [m for _, m in audio_marks if _ and m > 0.02]
first_loud = next((t for t, m in audio_marks if m > 0.02), None)
result["sent_at_s"] = round(t_send, 2)
result["first_audible_s"] = round(first_loud, 2) if first_loud else None
result["send_to_audible_s"] = round(first_loud - t_send, 2) if first_loud else None
with open(f"{OUT}/llm-chain-stats.json", "w") as f:
json.dump(result, f, ensure_ascii=False, indent=1)
print(json.dumps(result, ensure_ascii=False), flush=True)
print("DONE", flush=True)
asyncio.run(main())