Files
CyberGril/experiments/hel715/scripts/e2e_client.py
T

376 lines
14 KiB
Python

#!/usr/bin/env python
"""HEL-715 end-to-end verification client for LiveTalking(MuseTalk) @ 127.0.0.1:8010.
Connects as a WebRTC receiver (same path a browser takes), drives the avatar via
/human + /interrupt_talk, records the received A/V to mp4 and logs timing stats.
"""
import asyncio
import json
import os
import time
import aiohttp
import av
import numpy as np
from aiortc import RTCPeerConnection, RTCSessionDescription
BASE = os.environ.get("LT_BASE", "http://127.0.0.1:8010")
OUT = os.environ.get("E2E_OUT", "/opt/hel715-verify/e2e")
os.makedirs(OUT, exist_ok=True)
S1 = "在呢。你回来啦,今天在外面累不累?"
S2 = "还有,我跟你说,今天下午我在窗边看了好久的天空,云跑得特别快,有一朵特别像小兔子,我还给它拍了张照片,等下拿给你看。"
S3 = "抱歉,你刚想说什么?我在听。"
S4 = "嗯,我在。"
# approx mouth region (lower third of face bbox 218,203,324,333 from avatar prep)
MOUTH = (222, 268, 320, 331)
log_lines = []
def log(msg):
line = f"[{time.strftime('%H:%M:%S')}] {msg}"
print(line, flush=True)
log_lines.append(line)
class Recorder:
def __init__(self, path, w, h):
self.container = av.open(path, "w")
self.vstream = self.container.add_stream("libx264", rate=25)
self.vstream.width = w
self.vstream.height = h
self.vstream.pix_fmt = "yuv420p"
self.vstream.options = {"crf": "20", "preset": "veryfast"}
self.astream = self.container.add_stream("aac", rate=48000)
self.astream.layout = "stereo"
def write_video(self, img_bgr):
frame = av.VideoFrame.from_ndarray(img_bgr, format="bgr24")
for p in self.vstream.encode(frame):
self.container.mux(p)
def write_audio(self, pcm_i16, sr, ch):
arr = pcm_i16.astype(np.int16)
if ch == 1:
arr = np.repeat(arr[:, None], 2, axis=1).reshape(-1)
elif ch == 2:
arr = arr.reshape(-1)
else:
arr = arr[: len(arr) // ch * ch].reshape(-1, ch)[:, :2].reshape(-1)
if sr != 48000:
n = len(arr) // 2
idx = (np.arange(n) * (sr / 48000.0)).astype(int)
idx = idx[idx < len(arr) // 2]
arr = arr.reshape(-1, 2)[idx].reshape(-1)
frame = av.AudioFrame.from_ndarray(
np.ascontiguousarray(arr.reshape(1, -1)), format="s16", layout="stereo"
)
frame.sample_rate = 48000
for p in self.astream.encode(frame):
self.container.mux(p)
def close(self):
for p in self.vstream.encode():
self.container.mux(p)
for p in self.astream.encode():
self.container.mux(p)
self.container.close()
class Session:
def __init__(self):
self.events = {"video": [], "audio": [], "speak": [], "actions": []}
self.imgs_sample = []
self.sid = None
self.http = None
async def post(self, path, payload):
t0 = time.perf_counter()
async with self.http.post(BASE + path, json=payload) as r:
body = await r.json()
return time.perf_counter() - t0, body
async def poll_speaking(self, stop_evt):
while not stop_evt.is_set():
try:
dt, body = await self.post("/is_speaking", {"sessionid": self.sid})
self.events["speak"].append(
[round(time.perf_counter(), 3), body.get("data", False)]
)
except Exception:
pass
await asyncio.sleep(0.25)
def mouth_motion(prev, cur):
if prev is None or cur is None:
return 0.0
x0, y0, x1, y1 = MOUTH
d = np.abs(
cur[y0:y1, x0:x1].astype(np.int16) - prev[y0:y1, x0:x1].astype(np.int16)
).mean()
return float(d)
async def main():
stats = {"sentences": {}, "meta": {}}
rec = None
pc = RTCPeerConnection()
sess = Session()
t_start = time.perf_counter()
done = asyncio.Event()
@pc.on("track")
def on_track(track):
log(f"track: {track.kind}")
if track.kind == "video":
asyncio.ensure_future(video_loop(track))
else:
asyncio.ensure_future(audio_loop(track))
async def video_loop(track):
nonlocal rec
prev = None
while True:
try:
frame = await track.recv()
except Exception:
break
img = frame.to_ndarray(format="bgr24")
if rec is None:
rec = Recorder(os.path.join(OUT, "e2e-session.mp4"), img.shape[1], img.shape[0])
log(f"recorder ready {img.shape[1]}x{img.shape[0]}")
t = time.perf_counter()
m = mouth_motion(prev, img)
sess.events["video"].append([round(t, 3), round(m, 2)])
prev = img
rec.write_video(img)
if len(sess.imgs_sample) < 400:
sess.imgs_sample.append((round(t - t_start, 2), img))
async def audio_loop(track):
while True:
try:
frame = await track.recv()
except Exception:
break
t = time.perf_counter()
arr = frame.to_ndarray()
if arr.dtype != np.int16:
if arr.dtype in (np.float32, np.float64):
arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16)
else:
arr = arr.astype(np.int16)
ch = len(frame.layout.channels)
if arr.ndim == 2 and arr.shape[0] == 1:
pcm = arr.reshape(-1) # packed s16
elif arr.ndim == 2 and arr.shape[0] == ch:
pcm = arr.T.reshape(-1) # planar
else:
pcm = arr.reshape(-1)
ch = 1
rms = float(np.sqrt((pcm.astype(np.float64) ** 2).mean()) / 32767.0)
sess.events["audio"].append([round(t, 3), round(rms, 5)])
if rec is not None:
rec.write_audio(pcm, frame.sample_rate, ch)
async with aiohttp.ClientSession() as http:
sess.http = http
pc.addTransceiver("video", direction="recvonly")
pc.addTransceiver("audio", direction="recvonly")
offer = await pc.createOffer()
await pc.setLocalDescription(offer)
while pc.iceGatheringState != "complete":
await asyncio.sleep(0.1)
t0 = time.perf_counter()
async with http.post(
BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}
) as r:
answer = await r.json()
log(f"offer round-trip {time.perf_counter()-t0:.3f}s sessionid={answer.get('sessionid')}")
sess.sid = answer["sessionid"]
await pc.setRemoteDescription(
RTCSessionDescription(sdp=answer["sdp"], type=answer["type"])
)
stats["meta"]["sessionid"] = sess.sid
stop_evt = asyncio.Event()
poller = asyncio.ensure_future(sess.poll_speaking(stop_evt))
def mark(name):
t = time.perf_counter()
sess.events["actions"].append([round(t, 3), name])
log(f"ACTION {name}")
return t
# wait for first video frame / idle
for _ in range(200):
if sess.events["video"]:
break
await asyncio.sleep(0.1)
first_v = sess.events["video"][0][0] if sess.events["video"] else -1
first_a = sess.events["audio"][0][0] if sess.events["audio"] else -1
stats["meta"]["first_video_s"] = round(first_v - t_start, 3)
stats["meta"]["first_audio_s"] = round(first_a - t_start, 3)
log(f"first video {stats['meta']['first_video_s']}s audio {stats['meta']['first_audio_s']}s")
await asyncio.sleep(3.0) # idle baseline
async def speak(name, text, wait_end=True, interrupt_after=None, next_line=None):
t_send = mark(f"send:{name}")
dt, body = await sess.post(
"/human", {"sessionid": sess.sid, "type": "echo", "text": text}
)
log(f"/human {name} ack {dt:.3f}s {body}")
t_interrupt = None
if interrupt_after is not None:
await asyncio.sleep(interrupt_after)
t_interrupt = mark(f"interrupt:{name}")
dt2, b2 = await sess.post("/interrupt_talk", {"sessionid": sess.sid})
log(f"/interrupt_talk ack {dt2:.3f}s {b2}")
if wait_end:
# wait until speaking False and audio silent
silent = 0.0
last_speak_true = time.perf_counter()
while True:
await asyncio.sleep(0.1)
now = time.perf_counter()
sp = sess.events["speak"][-1][1] if sess.events["speak"] else None
if sp:
last_speak_true = now
recent = [a for a in sess.events["audio"] if a[0] > now - 0.6]
loud = any(a[1] > 0.02 for a in recent)
if not sp and not loud:
silent = now - last_speak_true
if silent > 1.2:
break
else:
last_speak_true = max(last_speak_true, now)
return t_send, t_interrupt
# S1 short
s, _ = await speak("S1", S1)
stats["sentences"]["S1"] = {"text": S1, "sent_at": round(s - t_start, 2)}
await asyncio.sleep(1.5)
# S2 long with interrupt 3s after send
s2, ti = await speak("S2", S2, wait_end=True, interrupt_after=3.0)
stats["sentences"]["S2"] = {
"text": S2, "sent_at": round(s2 - t_start, 2),
"interrupt_at": round(ti - t_start, 2),
}
await asyncio.sleep(2.0)
# S3 re-open after interrupt
s3, _ = await speak("S3", S3)
stats["sentences"]["S3"] = {"text": S3, "sent_at": round(s3 - t_start, 2)}
await asyncio.sleep(3.0) # pause test
s4, _ = await speak("S4", S4)
stats["sentences"]["S4"] = {"text": S4, "sent_at": round(s4 - t_start, 2)}
mark("end")
await asyncio.sleep(2.0)
stop_evt.set()
poller.cancel()
await pc.close()
if rec:
rec.close()
log("recording closed")
# analysis
def rel(ts):
return round(ts - t_start, 3)
ev_a = sess.events["audio"]
ev_v = sess.events["video"]
for name, info in stats["sentences"].items():
sent = info.get("sent_at", 0) + t_start
aud = [a for a in ev_a if a[0] >= sent and a[1] > 0.02]
info["first_audible_rel_s"] = rel(aud[0][0]) if aud else None
if aud:
last = sent
for a in aud:
if a[0] - last < 1.5:
last = a[0]
info["last_audible_rel_s"] = rel(last)
if "interrupt_at" in info:
it = info["interrupt_at"] + t_start
after = [a for a in ev_a if a[0] >= it and a[1] > 0.02]
info["audible_after_interrupt_s"] = rel(after[0][0]) if after else None
tail = [a for a in ev_a if a[0] >= it]
tl = [a for a in tail if a[1] > 0.02]
info["last_audible_after_interrupt_rel_s"] = rel(tl[-1][0]) if tl else None
# video motion around each sentence
for name, info in stats["sentences"].items():
sent = info.get("sent_at", 0) + t_start
window = [v for v in ev_v if sent - 1.0 <= v[0] <= sent + 12]
base = [v for v in ev_v if v[0] < sent - 1.0][-60:]
if window and base:
b = np.median([v[1] for v in base])
info["mouth_motion_idle"] = round(float(b), 2)
peak = max(v[1] for v in window)
info["mouth_motion_peak"] = round(float(peak), 2)
# fps over speech windows
if ev_v:
dur = ev_v[-1][0] - ev_v[0][0]
stats["meta"]["video_frames"] = len(ev_v)
stats["meta"]["video_span_s"] = round(dur, 2)
stats["meta"]["avg_recv_fps"] = round(len(ev_v) / dur, 2) if dur > 0 else None
if ev_a:
dur = ev_a[-1][0] - ev_a[0][0]
stats["meta"]["audio_frames"] = len(ev_a)
stats["meta"]["audio_span_s"] = round(dur, 2)
stats["meta"]["wall_total_s"] = round(time.perf_counter() - t_start, 2)
with open(os.path.join(OUT, "stats.json"), "w") as f:
json.dump(stats, f, ensure_ascii=False, indent=1)
with open(os.path.join(OUT, "events.json"), "w") as f:
json.dump(
{
"t_start": t_start,
"video": sess.events["video"],
"audio": sess.events["audio"],
"speak": sess.events["speak"],
"actions": sess.events["actions"],
},
f,
)
with open(os.path.join(OUT, "run.log"), "w") as f:
f.write("\n".join(log_lines))
# contact sheet: pick frames around S1 onset, S2 mid, interrupt, S3
try:
from PIL import Image
picks = []
s1 = stats["sentences"]["S1"].get("first_audible_rel_s") or 5
s2i = stats["sentences"]["S2"].get("interrupt_at")
picks += [max(0, s1 - 1.5), s1 + 0.5, s1 + 2.5]
if s2i:
picks += [s2i - 1.0, s2i + 0.3, s2i + 1.5]
picks += [stats["sentences"]["S3"]["sent_at"] + 2.0]
tiles = []
for p in picks:
cand = min(sess.imgs_sample, key=lambda x: abs(x[0] - p))
im = Image.fromarray(cand[1][:, :, ::-1]).resize((276, 276))
tiles.append((cand[0], im))
sheet = Image.new("RGB", (276 * len(tiles), 276 + 18), "black")
from PIL import ImageDraw
d = ImageDraw.Draw(sheet)
for i, (ts, im) in enumerate(tiles):
sheet.paste(im, (i * 276, 18))
d.text((i * 276 + 4, 2), f"t={ts:.1f}s", fill="white")
sheet.save(os.path.join(OUT, "contact-sheet.png"))
except Exception as e:
log(f"contact sheet failed: {e}")
log("DONE")
if __name__ == "__main__":
asyncio.run(main())