#!/usr/bin/env python """HEL-715 end-to-end verification client for LiveTalking(MuseTalk) @ 127.0.0.1:8010. Connects as a WebRTC receiver (same path a browser takes), drives the avatar via /human + /interrupt_talk, records the received A/V to mp4 and logs timing stats. """ import asyncio import json import os import time import aiohttp import av import numpy as np from aiortc import RTCPeerConnection, RTCSessionDescription BASE = os.environ.get("LT_BASE", "http://127.0.0.1:8010") OUT = os.environ.get("E2E_OUT", "/opt/hel715-verify/e2e") os.makedirs(OUT, exist_ok=True) S1 = "在呢。你回来啦,今天在外面累不累?" S2 = "还有,我跟你说,今天下午我在窗边看了好久的天空,云跑得特别快,有一朵特别像小兔子,我还给它拍了张照片,等下拿给你看。" S3 = "抱歉,你刚想说什么?我在听。" S4 = "嗯,我在。" # approx mouth region (lower third of face bbox 218,203,324,333 from avatar prep) MOUTH = (222, 268, 320, 331) log_lines = [] def log(msg): line = f"[{time.strftime('%H:%M:%S')}] {msg}" print(line, flush=True) log_lines.append(line) class Recorder: def __init__(self, path, w, h): self.container = av.open(path, "w") self.vstream = self.container.add_stream("libx264", rate=25) self.vstream.width = w self.vstream.height = h self.vstream.pix_fmt = "yuv420p" self.vstream.options = {"crf": "20", "preset": "veryfast"} self.astream = self.container.add_stream("aac", rate=48000) self.astream.layout = "stereo" def write_video(self, img_bgr): frame = av.VideoFrame.from_ndarray(img_bgr, format="bgr24") for p in self.vstream.encode(frame): self.container.mux(p) def write_audio(self, pcm_i16, sr, ch): arr = pcm_i16.astype(np.int16) if ch == 1: arr = np.repeat(arr[:, None], 2, axis=1).reshape(-1) elif ch == 2: arr = arr.reshape(-1) else: arr = arr[: len(arr) // ch * ch].reshape(-1, ch)[:, :2].reshape(-1) if sr != 48000: n = len(arr) // 2 idx = (np.arange(n) * (sr / 48000.0)).astype(int) idx = idx[idx < len(arr) // 2] arr = arr.reshape(-1, 2)[idx].reshape(-1) frame = av.AudioFrame.from_ndarray( np.ascontiguousarray(arr.reshape(1, -1)), format="s16", layout="stereo" ) frame.sample_rate = 48000 for p in self.astream.encode(frame): self.container.mux(p) def close(self): for p in self.vstream.encode(): self.container.mux(p) for p in self.astream.encode(): self.container.mux(p) self.container.close() class Session: def __init__(self): self.events = {"video": [], "audio": [], "speak": [], "actions": []} self.imgs_sample = [] self.sid = None self.http = None async def post(self, path, payload): t0 = time.perf_counter() async with self.http.post(BASE + path, json=payload) as r: body = await r.json() return time.perf_counter() - t0, body async def poll_speaking(self, stop_evt): while not stop_evt.is_set(): try: dt, body = await self.post("/is_speaking", {"sessionid": self.sid}) self.events["speak"].append( [round(time.perf_counter(), 3), body.get("data", False)] ) except Exception: pass await asyncio.sleep(0.25) def mouth_motion(prev, cur): if prev is None or cur is None: return 0.0 x0, y0, x1, y1 = MOUTH d = np.abs( cur[y0:y1, x0:x1].astype(np.int16) - prev[y0:y1, x0:x1].astype(np.int16) ).mean() return float(d) async def main(): stats = {"sentences": {}, "meta": {}} rec = None pc = RTCPeerConnection() sess = Session() t_start = time.perf_counter() done = asyncio.Event() @pc.on("track") def on_track(track): log(f"track: {track.kind}") if track.kind == "video": asyncio.ensure_future(video_loop(track)) else: asyncio.ensure_future(audio_loop(track)) async def video_loop(track): nonlocal rec prev = None while True: try: frame = await track.recv() except Exception: break img = frame.to_ndarray(format="bgr24") if rec is None: rec = Recorder(os.path.join(OUT, "e2e-session.mp4"), img.shape[1], img.shape[0]) log(f"recorder ready {img.shape[1]}x{img.shape[0]}") t = time.perf_counter() m = mouth_motion(prev, img) sess.events["video"].append([round(t, 3), round(m, 2)]) prev = img rec.write_video(img) if len(sess.imgs_sample) < 400: sess.imgs_sample.append((round(t - t_start, 2), img)) async def audio_loop(track): while True: try: frame = await track.recv() except Exception: break t = time.perf_counter() arr = frame.to_ndarray() if arr.dtype != np.int16: if arr.dtype in (np.float32, np.float64): arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16) else: arr = arr.astype(np.int16) ch = len(frame.layout.channels) if arr.ndim == 2 and arr.shape[0] == 1: pcm = arr.reshape(-1) # packed s16 elif arr.ndim == 2 and arr.shape[0] == ch: pcm = arr.T.reshape(-1) # planar else: pcm = arr.reshape(-1) ch = 1 rms = float(np.sqrt((pcm.astype(np.float64) ** 2).mean()) / 32767.0) sess.events["audio"].append([round(t, 3), round(rms, 5)]) if rec is not None: rec.write_audio(pcm, frame.sample_rate, ch) async with aiohttp.ClientSession() as http: sess.http = http pc.addTransceiver("video", direction="recvonly") pc.addTransceiver("audio", direction="recvonly") offer = await pc.createOffer() await pc.setLocalDescription(offer) while pc.iceGatheringState != "complete": await asyncio.sleep(0.1) t0 = time.perf_counter() async with http.post( BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"} ) as r: answer = await r.json() log(f"offer round-trip {time.perf_counter()-t0:.3f}s sessionid={answer.get('sessionid')}") sess.sid = answer["sessionid"] await pc.setRemoteDescription( RTCSessionDescription(sdp=answer["sdp"], type=answer["type"]) ) stats["meta"]["sessionid"] = sess.sid stop_evt = asyncio.Event() poller = asyncio.ensure_future(sess.poll_speaking(stop_evt)) def mark(name): t = time.perf_counter() sess.events["actions"].append([round(t, 3), name]) log(f"ACTION {name}") return t # wait for first video frame / idle for _ in range(200): if sess.events["video"]: break await asyncio.sleep(0.1) first_v = sess.events["video"][0][0] if sess.events["video"] else -1 first_a = sess.events["audio"][0][0] if sess.events["audio"] else -1 stats["meta"]["first_video_s"] = round(first_v - t_start, 3) stats["meta"]["first_audio_s"] = round(first_a - t_start, 3) log(f"first video {stats['meta']['first_video_s']}s audio {stats['meta']['first_audio_s']}s") await asyncio.sleep(3.0) # idle baseline async def speak(name, text, wait_end=True, interrupt_after=None, next_line=None): t_send = mark(f"send:{name}") dt, body = await sess.post( "/human", {"sessionid": sess.sid, "type": "echo", "text": text} ) log(f"/human {name} ack {dt:.3f}s {body}") t_interrupt = None if interrupt_after is not None: await asyncio.sleep(interrupt_after) t_interrupt = mark(f"interrupt:{name}") dt2, b2 = await sess.post("/interrupt_talk", {"sessionid": sess.sid}) log(f"/interrupt_talk ack {dt2:.3f}s {b2}") if wait_end: # wait until speaking False and audio silent silent = 0.0 last_speak_true = time.perf_counter() while True: await asyncio.sleep(0.1) now = time.perf_counter() sp = sess.events["speak"][-1][1] if sess.events["speak"] else None if sp: last_speak_true = now recent = [a for a in sess.events["audio"] if a[0] > now - 0.6] loud = any(a[1] > 0.02 for a in recent) if not sp and not loud: silent = now - last_speak_true if silent > 1.2: break else: last_speak_true = max(last_speak_true, now) return t_send, t_interrupt # S1 short s, _ = await speak("S1", S1) stats["sentences"]["S1"] = {"text": S1, "sent_at": round(s - t_start, 2)} await asyncio.sleep(1.5) # S2 long with interrupt 3s after send s2, ti = await speak("S2", S2, wait_end=True, interrupt_after=3.0) stats["sentences"]["S2"] = { "text": S2, "sent_at": round(s2 - t_start, 2), "interrupt_at": round(ti - t_start, 2), } await asyncio.sleep(2.0) # S3 re-open after interrupt s3, _ = await speak("S3", S3) stats["sentences"]["S3"] = {"text": S3, "sent_at": round(s3 - t_start, 2)} await asyncio.sleep(3.0) # pause test s4, _ = await speak("S4", S4) stats["sentences"]["S4"] = {"text": S4, "sent_at": round(s4 - t_start, 2)} mark("end") await asyncio.sleep(2.0) stop_evt.set() poller.cancel() await pc.close() if rec: rec.close() log("recording closed") # analysis def rel(ts): return round(ts - t_start, 3) ev_a = sess.events["audio"] ev_v = sess.events["video"] for name, info in stats["sentences"].items(): sent = info.get("sent_at", 0) + t_start aud = [a for a in ev_a if a[0] >= sent and a[1] > 0.02] info["first_audible_rel_s"] = rel(aud[0][0]) if aud else None if aud: last = sent for a in aud: if a[0] - last < 1.5: last = a[0] info["last_audible_rel_s"] = rel(last) if "interrupt_at" in info: it = info["interrupt_at"] + t_start after = [a for a in ev_a if a[0] >= it and a[1] > 0.02] info["audible_after_interrupt_s"] = rel(after[0][0]) if after else None tail = [a for a in ev_a if a[0] >= it] tl = [a for a in tail if a[1] > 0.02] info["last_audible_after_interrupt_rel_s"] = rel(tl[-1][0]) if tl else None # video motion around each sentence for name, info in stats["sentences"].items(): sent = info.get("sent_at", 0) + t_start window = [v for v in ev_v if sent - 1.0 <= v[0] <= sent + 12] base = [v for v in ev_v if v[0] < sent - 1.0][-60:] if window and base: b = np.median([v[1] for v in base]) info["mouth_motion_idle"] = round(float(b), 2) peak = max(v[1] for v in window) info["mouth_motion_peak"] = round(float(peak), 2) # fps over speech windows if ev_v: dur = ev_v[-1][0] - ev_v[0][0] stats["meta"]["video_frames"] = len(ev_v) stats["meta"]["video_span_s"] = round(dur, 2) stats["meta"]["avg_recv_fps"] = round(len(ev_v) / dur, 2) if dur > 0 else None if ev_a: dur = ev_a[-1][0] - ev_a[0][0] stats["meta"]["audio_frames"] = len(ev_a) stats["meta"]["audio_span_s"] = round(dur, 2) stats["meta"]["wall_total_s"] = round(time.perf_counter() - t_start, 2) with open(os.path.join(OUT, "stats.json"), "w") as f: json.dump(stats, f, ensure_ascii=False, indent=1) with open(os.path.join(OUT, "events.json"), "w") as f: json.dump( { "t_start": t_start, "video": sess.events["video"], "audio": sess.events["audio"], "speak": sess.events["speak"], "actions": sess.events["actions"], }, f, ) with open(os.path.join(OUT, "run.log"), "w") as f: f.write("\n".join(log_lines)) # contact sheet: pick frames around S1 onset, S2 mid, interrupt, S3 try: from PIL import Image picks = [] s1 = stats["sentences"]["S1"].get("first_audible_rel_s") or 5 s2i = stats["sentences"]["S2"].get("interrupt_at") picks += [max(0, s1 - 1.5), s1 + 0.5, s1 + 2.5] if s2i: picks += [s2i - 1.0, s2i + 0.3, s2i + 1.5] picks += [stats["sentences"]["S3"]["sent_at"] + 2.0] tiles = [] for p in picks: cand = min(sess.imgs_sample, key=lambda x: abs(x[0] - p)) im = Image.fromarray(cand[1][:, :, ::-1]).resize((276, 276)) tiles.append((cand[0], im)) sheet = Image.new("RGB", (276 * len(tiles), 276 + 18), "black") from PIL import ImageDraw d = ImageDraw.Draw(sheet) for i, (ts, im) in enumerate(tiles): sheet.paste(im, (i * 276, 18)) d.text((i * 276 + 4, 2), f"t={ts:.1f}s", fill="white") sheet.save(os.path.join(OUT, "contact-sheet.png")) except Exception as e: log(f"contact sheet failed: {e}") log("DONE") if __name__ == "__main__": asyncio.run(main())