#!/usr/bin/env python """LLM-chained conversation sample: .11 LLM -> .12 CosyVoice -> MuseTalk avatar. Same receiver architecture as e2e_client.py (proven to capture A/V).""" import asyncio, json, os, time import aiohttp, av, numpy as np from aiortc import RTCPeerConnection, RTCSessionDescription BASE = "http://127.0.0.1:8010" LLM = "http://192.168.200.11:8731/v1/chat/completions" OUT = "/opt/hel715-verify/e2e" os.makedirs(OUT, exist_ok=True) class Recorder: def __init__(self, path, w, h): self.container = av.open(path, "w") self.v = self.container.add_stream("libx264", rate=25) self.v.width, self.v.height, self.v.pix_fmt = w, h, "yuv420p" self.v.options = {"crf": "20", "preset": "veryfast"} self.a = self.container.add_stream("aac", rate=48000) self.a.layout = "stereo" def video(self, img): for p in self.v.encode(av.VideoFrame.from_ndarray(img, format="bgr24")): self.container.mux(p) def audio(self, frame): arr = frame.to_ndarray() if arr.dtype != np.int16: arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) ch = len(frame.layout.channels) if arr.ndim == 2 and arr.shape[0] == 1: pcm = arr.reshape(-1) elif arr.ndim == 2 and arr.shape[0] == ch: pcm = arr.T.reshape(-1) else: pcm = arr.reshape(-1) fr = av.AudioFrame.from_ndarray(np.ascontiguousarray(pcm.reshape(1, -1)), format="s16", layout="stereo") fr.sample_rate = 48000 for p in self.a.encode(fr): self.container.mux(p) def close(self): for p in self.v.encode(): self.container.mux(p) for p in self.a.encode(): self.container.mux(p) self.container.close() async def main(): t0 = time.perf_counter() result = {} async with aiohttp.ClientSession() as http: async with http.post(LLM, json={ "model": "halogen-qwen3.8-flash-next", "messages": [{"role": "user", "content": "你是住在高层公寓卧室里的伴侣晚晚,用一两句自然中文回应:我回来了,今天好累啊。"}], "max_tokens": 80, }) as r: body = await r.json() reply = body["choices"][0]["message"]["content"] result["llm_model"] = body["model"] result["llm_reply"] = reply result["llm_latency_s"] = round(time.perf_counter() - t0, 2) print(f"[{result['llm_latency_s']}s] LLM: {reply}", flush=True) pc = RTCPeerConnection() rec = {"r": None} audio_marks = [] @pc.on("track") def on_track(track): async def drain(): while True: try: f = await track.recv() except Exception: break t = time.perf_counter() if track.kind == "video": img = f.to_ndarray(format="bgr24") if rec["r"] is None: rec["r"] = Recorder(f"{OUT}/llm-chain.mp4", img.shape[1], img.shape[0]) rec["r"].video(img) else: arr = f.to_ndarray() if arr.dtype != np.int16: arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) rms = float(np.sqrt((arr.astype(np.float64) ** 2).mean()) / 32767) audio_marks.append([round(t - t0, 3), round(rms, 5)]) if rec["r"] is not None: rec["r"].audio(f) asyncio.ensure_future(drain()) pc.addTransceiver("video", direction="recvonly") pc.addTransceiver("audio", direction="recvonly") offer = await pc.createOffer() await pc.setLocalDescription(offer) while pc.iceGatheringState != "complete": await asyncio.sleep(0.1) async with http.post(BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r: answer = await r.json() await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"])) sid = answer["sessionid"] await asyncio.sleep(3) t_send = time.perf_counter() - t0 async with http.post(BASE + "/human", json={"sessionid": sid, "type": "echo", "text": reply}) as r: await r.json() print(f"[{t_send:.2f}s] sent to avatar", flush=True) await asyncio.sleep(15) await pc.close() if rec["r"]: rec["r"].close() loud = [m for _, m in audio_marks if _ and m > 0.02] first_loud = next((t for t, m in audio_marks if m > 0.02), None) result["sent_at_s"] = round(t_send, 2) result["first_audible_s"] = round(first_loud, 2) if first_loud else None result["send_to_audible_s"] = round(first_loud - t_send, 2) if first_loud else None with open(f"{OUT}/llm-chain-stats.json", "w") as f: json.dump(result, f, ensure_ascii=False, indent=1) print(json.dumps(result, ensure_ascii=False), flush=True) print("DONE", flush=True) asyncio.run(main())