125 lines
5.1 KiB
Python
125 lines
5.1 KiB
Python
#!/usr/bin/env python
|
|
"""LLM-chained conversation sample: .11 LLM -> .12 CosyVoice -> MuseTalk avatar.
|
|
Same receiver architecture as e2e_client.py (proven to capture A/V)."""
|
|
import asyncio, json, os, time
|
|
import aiohttp, av, numpy as np
|
|
from aiortc import RTCPeerConnection, RTCSessionDescription
|
|
|
|
BASE = "http://127.0.0.1:8010"
|
|
LLM = "http://192.168.200.11:8731/v1/chat/completions"
|
|
OUT = "/opt/hel715-verify/e2e"
|
|
os.makedirs(OUT, exist_ok=True)
|
|
|
|
|
|
class Recorder:
|
|
def __init__(self, path, w, h):
|
|
self.container = av.open(path, "w")
|
|
self.v = self.container.add_stream("libx264", rate=25)
|
|
self.v.width, self.v.height, self.v.pix_fmt = w, h, "yuv420p"
|
|
self.v.options = {"crf": "20", "preset": "veryfast"}
|
|
self.a = self.container.add_stream("aac", rate=48000)
|
|
self.a.layout = "stereo"
|
|
|
|
def video(self, img):
|
|
for p in self.v.encode(av.VideoFrame.from_ndarray(img, format="bgr24")):
|
|
self.container.mux(p)
|
|
|
|
def audio(self, frame):
|
|
arr = frame.to_ndarray()
|
|
if arr.dtype != np.int16:
|
|
arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16)
|
|
ch = len(frame.layout.channels)
|
|
if arr.ndim == 2 and arr.shape[0] == 1:
|
|
pcm = arr.reshape(-1)
|
|
elif arr.ndim == 2 and arr.shape[0] == ch:
|
|
pcm = arr.T.reshape(-1)
|
|
else:
|
|
pcm = arr.reshape(-1)
|
|
fr = av.AudioFrame.from_ndarray(np.ascontiguousarray(pcm.reshape(1, -1)), format="s16", layout="stereo")
|
|
fr.sample_rate = 48000
|
|
for p in self.a.encode(fr):
|
|
self.container.mux(p)
|
|
|
|
def close(self):
|
|
for p in self.v.encode(): self.container.mux(p)
|
|
for p in self.a.encode(): self.container.mux(p)
|
|
self.container.close()
|
|
|
|
|
|
async def main():
|
|
t0 = time.perf_counter()
|
|
result = {}
|
|
async with aiohttp.ClientSession() as http:
|
|
async with http.post(LLM, json={
|
|
"model": "halogen-qwen3.8-flash-next",
|
|
"messages": [{"role": "user", "content": "你是住在高层公寓卧室里的伴侣晚晚,用一两句自然中文回应:我回来了,今天好累啊。"}],
|
|
"max_tokens": 80,
|
|
}) as r:
|
|
body = await r.json()
|
|
reply = body["choices"][0]["message"]["content"]
|
|
result["llm_model"] = body["model"]
|
|
result["llm_reply"] = reply
|
|
result["llm_latency_s"] = round(time.perf_counter() - t0, 2)
|
|
print(f"[{result['llm_latency_s']}s] LLM: {reply}", flush=True)
|
|
|
|
pc = RTCPeerConnection()
|
|
rec = {"r": None}
|
|
audio_marks = []
|
|
|
|
@pc.on("track")
|
|
def on_track(track):
|
|
async def drain():
|
|
while True:
|
|
try:
|
|
f = await track.recv()
|
|
except Exception:
|
|
break
|
|
t = time.perf_counter()
|
|
if track.kind == "video":
|
|
img = f.to_ndarray(format="bgr24")
|
|
if rec["r"] is None:
|
|
rec["r"] = Recorder(f"{OUT}/llm-chain.mp4", img.shape[1], img.shape[0])
|
|
rec["r"].video(img)
|
|
else:
|
|
arr = f.to_ndarray()
|
|
if arr.dtype != np.int16:
|
|
arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16)
|
|
rms = float(np.sqrt((arr.astype(np.float64) ** 2).mean()) / 32767)
|
|
audio_marks.append([round(t - t0, 3), round(rms, 5)])
|
|
if rec["r"] is not None:
|
|
rec["r"].audio(f)
|
|
asyncio.ensure_future(drain())
|
|
|
|
pc.addTransceiver("video", direction="recvonly")
|
|
pc.addTransceiver("audio", direction="recvonly")
|
|
offer = await pc.createOffer()
|
|
await pc.setLocalDescription(offer)
|
|
while pc.iceGatheringState != "complete":
|
|
await asyncio.sleep(0.1)
|
|
async with http.post(BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r:
|
|
answer = await r.json()
|
|
await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"]))
|
|
sid = answer["sessionid"]
|
|
await asyncio.sleep(3)
|
|
t_send = time.perf_counter() - t0
|
|
async with http.post(BASE + "/human", json={"sessionid": sid, "type": "echo", "text": reply}) as r:
|
|
await r.json()
|
|
print(f"[{t_send:.2f}s] sent to avatar", flush=True)
|
|
await asyncio.sleep(15)
|
|
await pc.close()
|
|
|
|
if rec["r"]:
|
|
rec["r"].close()
|
|
loud = [m for _, m in audio_marks if _ and m > 0.02]
|
|
first_loud = next((t for t, m in audio_marks if m > 0.02), None)
|
|
result["sent_at_s"] = round(t_send, 2)
|
|
result["first_audible_s"] = round(first_loud, 2) if first_loud else None
|
|
result["send_to_audible_s"] = round(first_loud - t_send, 2) if first_loud else None
|
|
with open(f"{OUT}/llm-chain-stats.json", "w") as f:
|
|
json.dump(result, f, ensure_ascii=False, indent=1)
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
print("DONE", flush=True)
|
|
|
|
|
|
asyncio.run(main())
|