376 lines
14 KiB
Python
376 lines
14 KiB
Python
#!/usr/bin/env python
|
|
"""HEL-715 end-to-end verification client for LiveTalking(MuseTalk) @ 127.0.0.1:8010.
|
|
|
|
Connects as a WebRTC receiver (same path a browser takes), drives the avatar via
|
|
/human + /interrupt_talk, records the received A/V to mp4 and logs timing stats.
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import time
|
|
|
|
import aiohttp
|
|
import av
|
|
import numpy as np
|
|
from aiortc import RTCPeerConnection, RTCSessionDescription
|
|
|
|
BASE = os.environ.get("LT_BASE", "http://127.0.0.1:8010")
|
|
OUT = os.environ.get("E2E_OUT", "/opt/hel715-verify/e2e")
|
|
os.makedirs(OUT, exist_ok=True)
|
|
|
|
S1 = "在呢。你回来啦,今天在外面累不累?"
|
|
S2 = "还有,我跟你说,今天下午我在窗边看了好久的天空,云跑得特别快,有一朵特别像小兔子,我还给它拍了张照片,等下拿给你看。"
|
|
S3 = "抱歉,你刚想说什么?我在听。"
|
|
S4 = "嗯,我在。"
|
|
|
|
# approx mouth region (lower third of face bbox 218,203,324,333 from avatar prep)
|
|
MOUTH = (222, 268, 320, 331)
|
|
|
|
log_lines = []
|
|
|
|
|
|
def log(msg):
|
|
line = f"[{time.strftime('%H:%M:%S')}] {msg}"
|
|
print(line, flush=True)
|
|
log_lines.append(line)
|
|
|
|
|
|
class Recorder:
|
|
def __init__(self, path, w, h):
|
|
self.container = av.open(path, "w")
|
|
self.vstream = self.container.add_stream("libx264", rate=25)
|
|
self.vstream.width = w
|
|
self.vstream.height = h
|
|
self.vstream.pix_fmt = "yuv420p"
|
|
self.vstream.options = {"crf": "20", "preset": "veryfast"}
|
|
self.astream = self.container.add_stream("aac", rate=48000)
|
|
self.astream.layout = "stereo"
|
|
|
|
def write_video(self, img_bgr):
|
|
frame = av.VideoFrame.from_ndarray(img_bgr, format="bgr24")
|
|
for p in self.vstream.encode(frame):
|
|
self.container.mux(p)
|
|
|
|
def write_audio(self, pcm_i16, sr, ch):
|
|
arr = pcm_i16.astype(np.int16)
|
|
if ch == 1:
|
|
arr = np.repeat(arr[:, None], 2, axis=1).reshape(-1)
|
|
elif ch == 2:
|
|
arr = arr.reshape(-1)
|
|
else:
|
|
arr = arr[: len(arr) // ch * ch].reshape(-1, ch)[:, :2].reshape(-1)
|
|
if sr != 48000:
|
|
n = len(arr) // 2
|
|
idx = (np.arange(n) * (sr / 48000.0)).astype(int)
|
|
idx = idx[idx < len(arr) // 2]
|
|
arr = arr.reshape(-1, 2)[idx].reshape(-1)
|
|
frame = av.AudioFrame.from_ndarray(
|
|
np.ascontiguousarray(arr.reshape(1, -1)), format="s16", layout="stereo"
|
|
)
|
|
frame.sample_rate = 48000
|
|
for p in self.astream.encode(frame):
|
|
self.container.mux(p)
|
|
|
|
def close(self):
|
|
for p in self.vstream.encode():
|
|
self.container.mux(p)
|
|
for p in self.astream.encode():
|
|
self.container.mux(p)
|
|
self.container.close()
|
|
|
|
|
|
class Session:
|
|
def __init__(self):
|
|
self.events = {"video": [], "audio": [], "speak": [], "actions": []}
|
|
self.imgs_sample = []
|
|
self.sid = None
|
|
self.http = None
|
|
|
|
async def post(self, path, payload):
|
|
t0 = time.perf_counter()
|
|
async with self.http.post(BASE + path, json=payload) as r:
|
|
body = await r.json()
|
|
return time.perf_counter() - t0, body
|
|
|
|
async def poll_speaking(self, stop_evt):
|
|
while not stop_evt.is_set():
|
|
try:
|
|
dt, body = await self.post("/is_speaking", {"sessionid": self.sid})
|
|
self.events["speak"].append(
|
|
[round(time.perf_counter(), 3), body.get("data", False)]
|
|
)
|
|
except Exception:
|
|
pass
|
|
await asyncio.sleep(0.25)
|
|
|
|
|
|
def mouth_motion(prev, cur):
|
|
if prev is None or cur is None:
|
|
return 0.0
|
|
x0, y0, x1, y1 = MOUTH
|
|
d = np.abs(
|
|
cur[y0:y1, x0:x1].astype(np.int16) - prev[y0:y1, x0:x1].astype(np.int16)
|
|
).mean()
|
|
return float(d)
|
|
|
|
|
|
async def main():
|
|
stats = {"sentences": {}, "meta": {}}
|
|
rec = None
|
|
pc = RTCPeerConnection()
|
|
sess = Session()
|
|
t_start = time.perf_counter()
|
|
|
|
done = asyncio.Event()
|
|
|
|
@pc.on("track")
|
|
def on_track(track):
|
|
log(f"track: {track.kind}")
|
|
if track.kind == "video":
|
|
asyncio.ensure_future(video_loop(track))
|
|
else:
|
|
asyncio.ensure_future(audio_loop(track))
|
|
|
|
async def video_loop(track):
|
|
nonlocal rec
|
|
prev = None
|
|
while True:
|
|
try:
|
|
frame = await track.recv()
|
|
except Exception:
|
|
break
|
|
img = frame.to_ndarray(format="bgr24")
|
|
if rec is None:
|
|
rec = Recorder(os.path.join(OUT, "e2e-session.mp4"), img.shape[1], img.shape[0])
|
|
log(f"recorder ready {img.shape[1]}x{img.shape[0]}")
|
|
t = time.perf_counter()
|
|
m = mouth_motion(prev, img)
|
|
sess.events["video"].append([round(t, 3), round(m, 2)])
|
|
prev = img
|
|
rec.write_video(img)
|
|
if len(sess.imgs_sample) < 400:
|
|
sess.imgs_sample.append((round(t - t_start, 2), img))
|
|
|
|
async def audio_loop(track):
|
|
while True:
|
|
try:
|
|
frame = await track.recv()
|
|
except Exception:
|
|
break
|
|
t = time.perf_counter()
|
|
arr = frame.to_ndarray()
|
|
if arr.dtype != np.int16:
|
|
if arr.dtype in (np.float32, np.float64):
|
|
arr = (np.clip(arr, -1, 1) * 32767).astype(np.int16)
|
|
else:
|
|
arr = arr.astype(np.int16)
|
|
ch = len(frame.layout.channels)
|
|
if arr.ndim == 2 and arr.shape[0] == 1:
|
|
pcm = arr.reshape(-1) # packed s16
|
|
elif arr.ndim == 2 and arr.shape[0] == ch:
|
|
pcm = arr.T.reshape(-1) # planar
|
|
else:
|
|
pcm = arr.reshape(-1)
|
|
ch = 1
|
|
rms = float(np.sqrt((pcm.astype(np.float64) ** 2).mean()) / 32767.0)
|
|
sess.events["audio"].append([round(t, 3), round(rms, 5)])
|
|
if rec is not None:
|
|
rec.write_audio(pcm, frame.sample_rate, ch)
|
|
|
|
async with aiohttp.ClientSession() as http:
|
|
sess.http = http
|
|
pc.addTransceiver("video", direction="recvonly")
|
|
pc.addTransceiver("audio", direction="recvonly")
|
|
offer = await pc.createOffer()
|
|
await pc.setLocalDescription(offer)
|
|
while pc.iceGatheringState != "complete":
|
|
await asyncio.sleep(0.1)
|
|
t0 = time.perf_counter()
|
|
async with http.post(
|
|
BASE + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}
|
|
) as r:
|
|
answer = await r.json()
|
|
log(f"offer round-trip {time.perf_counter()-t0:.3f}s sessionid={answer.get('sessionid')}")
|
|
sess.sid = answer["sessionid"]
|
|
await pc.setRemoteDescription(
|
|
RTCSessionDescription(sdp=answer["sdp"], type=answer["type"])
|
|
)
|
|
stats["meta"]["sessionid"] = sess.sid
|
|
|
|
stop_evt = asyncio.Event()
|
|
poller = asyncio.ensure_future(sess.poll_speaking(stop_evt))
|
|
|
|
def mark(name):
|
|
t = time.perf_counter()
|
|
sess.events["actions"].append([round(t, 3), name])
|
|
log(f"ACTION {name}")
|
|
return t
|
|
|
|
# wait for first video frame / idle
|
|
for _ in range(200):
|
|
if sess.events["video"]:
|
|
break
|
|
await asyncio.sleep(0.1)
|
|
first_v = sess.events["video"][0][0] if sess.events["video"] else -1
|
|
first_a = sess.events["audio"][0][0] if sess.events["audio"] else -1
|
|
stats["meta"]["first_video_s"] = round(first_v - t_start, 3)
|
|
stats["meta"]["first_audio_s"] = round(first_a - t_start, 3)
|
|
log(f"first video {stats['meta']['first_video_s']}s audio {stats['meta']['first_audio_s']}s")
|
|
await asyncio.sleep(3.0) # idle baseline
|
|
|
|
async def speak(name, text, wait_end=True, interrupt_after=None, next_line=None):
|
|
t_send = mark(f"send:{name}")
|
|
dt, body = await sess.post(
|
|
"/human", {"sessionid": sess.sid, "type": "echo", "text": text}
|
|
)
|
|
log(f"/human {name} ack {dt:.3f}s {body}")
|
|
t_interrupt = None
|
|
if interrupt_after is not None:
|
|
await asyncio.sleep(interrupt_after)
|
|
t_interrupt = mark(f"interrupt:{name}")
|
|
dt2, b2 = await sess.post("/interrupt_talk", {"sessionid": sess.sid})
|
|
log(f"/interrupt_talk ack {dt2:.3f}s {b2}")
|
|
if wait_end:
|
|
# wait until speaking False and audio silent
|
|
silent = 0.0
|
|
last_speak_true = time.perf_counter()
|
|
while True:
|
|
await asyncio.sleep(0.1)
|
|
now = time.perf_counter()
|
|
sp = sess.events["speak"][-1][1] if sess.events["speak"] else None
|
|
if sp:
|
|
last_speak_true = now
|
|
recent = [a for a in sess.events["audio"] if a[0] > now - 0.6]
|
|
loud = any(a[1] > 0.02 for a in recent)
|
|
if not sp and not loud:
|
|
silent = now - last_speak_true
|
|
if silent > 1.2:
|
|
break
|
|
else:
|
|
last_speak_true = max(last_speak_true, now)
|
|
return t_send, t_interrupt
|
|
|
|
# S1 short
|
|
s, _ = await speak("S1", S1)
|
|
stats["sentences"]["S1"] = {"text": S1, "sent_at": round(s - t_start, 2)}
|
|
await asyncio.sleep(1.5)
|
|
# S2 long with interrupt 3s after send
|
|
s2, ti = await speak("S2", S2, wait_end=True, interrupt_after=3.0)
|
|
stats["sentences"]["S2"] = {
|
|
"text": S2, "sent_at": round(s2 - t_start, 2),
|
|
"interrupt_at": round(ti - t_start, 2),
|
|
}
|
|
await asyncio.sleep(2.0)
|
|
# S3 re-open after interrupt
|
|
s3, _ = await speak("S3", S3)
|
|
stats["sentences"]["S3"] = {"text": S3, "sent_at": round(s3 - t_start, 2)}
|
|
await asyncio.sleep(3.0) # pause test
|
|
s4, _ = await speak("S4", S4)
|
|
stats["sentences"]["S4"] = {"text": S4, "sent_at": round(s4 - t_start, 2)}
|
|
mark("end")
|
|
await asyncio.sleep(2.0)
|
|
stop_evt.set()
|
|
poller.cancel()
|
|
await pc.close()
|
|
|
|
if rec:
|
|
rec.close()
|
|
log("recording closed")
|
|
|
|
# analysis
|
|
def rel(ts):
|
|
return round(ts - t_start, 3)
|
|
|
|
ev_a = sess.events["audio"]
|
|
ev_v = sess.events["video"]
|
|
for name, info in stats["sentences"].items():
|
|
sent = info.get("sent_at", 0) + t_start
|
|
aud = [a for a in ev_a if a[0] >= sent and a[1] > 0.02]
|
|
info["first_audible_rel_s"] = rel(aud[0][0]) if aud else None
|
|
if aud:
|
|
last = sent
|
|
for a in aud:
|
|
if a[0] - last < 1.5:
|
|
last = a[0]
|
|
info["last_audible_rel_s"] = rel(last)
|
|
if "interrupt_at" in info:
|
|
it = info["interrupt_at"] + t_start
|
|
after = [a for a in ev_a if a[0] >= it and a[1] > 0.02]
|
|
info["audible_after_interrupt_s"] = rel(after[0][0]) if after else None
|
|
tail = [a for a in ev_a if a[0] >= it]
|
|
tl = [a for a in tail if a[1] > 0.02]
|
|
info["last_audible_after_interrupt_rel_s"] = rel(tl[-1][0]) if tl else None
|
|
|
|
# video motion around each sentence
|
|
for name, info in stats["sentences"].items():
|
|
sent = info.get("sent_at", 0) + t_start
|
|
window = [v for v in ev_v if sent - 1.0 <= v[0] <= sent + 12]
|
|
base = [v for v in ev_v if v[0] < sent - 1.0][-60:]
|
|
if window and base:
|
|
b = np.median([v[1] for v in base])
|
|
info["mouth_motion_idle"] = round(float(b), 2)
|
|
peak = max(v[1] for v in window)
|
|
info["mouth_motion_peak"] = round(float(peak), 2)
|
|
|
|
# fps over speech windows
|
|
if ev_v:
|
|
dur = ev_v[-1][0] - ev_v[0][0]
|
|
stats["meta"]["video_frames"] = len(ev_v)
|
|
stats["meta"]["video_span_s"] = round(dur, 2)
|
|
stats["meta"]["avg_recv_fps"] = round(len(ev_v) / dur, 2) if dur > 0 else None
|
|
if ev_a:
|
|
dur = ev_a[-1][0] - ev_a[0][0]
|
|
stats["meta"]["audio_frames"] = len(ev_a)
|
|
stats["meta"]["audio_span_s"] = round(dur, 2)
|
|
|
|
stats["meta"]["wall_total_s"] = round(time.perf_counter() - t_start, 2)
|
|
|
|
with open(os.path.join(OUT, "stats.json"), "w") as f:
|
|
json.dump(stats, f, ensure_ascii=False, indent=1)
|
|
with open(os.path.join(OUT, "events.json"), "w") as f:
|
|
json.dump(
|
|
{
|
|
"t_start": t_start,
|
|
"video": sess.events["video"],
|
|
"audio": sess.events["audio"],
|
|
"speak": sess.events["speak"],
|
|
"actions": sess.events["actions"],
|
|
},
|
|
f,
|
|
)
|
|
with open(os.path.join(OUT, "run.log"), "w") as f:
|
|
f.write("\n".join(log_lines))
|
|
|
|
# contact sheet: pick frames around S1 onset, S2 mid, interrupt, S3
|
|
try:
|
|
from PIL import Image
|
|
|
|
picks = []
|
|
s1 = stats["sentences"]["S1"].get("first_audible_rel_s") or 5
|
|
s2i = stats["sentences"]["S2"].get("interrupt_at")
|
|
picks += [max(0, s1 - 1.5), s1 + 0.5, s1 + 2.5]
|
|
if s2i:
|
|
picks += [s2i - 1.0, s2i + 0.3, s2i + 1.5]
|
|
picks += [stats["sentences"]["S3"]["sent_at"] + 2.0]
|
|
tiles = []
|
|
for p in picks:
|
|
cand = min(sess.imgs_sample, key=lambda x: abs(x[0] - p))
|
|
im = Image.fromarray(cand[1][:, :, ::-1]).resize((276, 276))
|
|
tiles.append((cand[0], im))
|
|
sheet = Image.new("RGB", (276 * len(tiles), 276 + 18), "black")
|
|
from PIL import ImageDraw
|
|
|
|
d = ImageDraw.Draw(sheet)
|
|
for i, (ts, im) in enumerate(tiles):
|
|
sheet.paste(im, (i * 276, 18))
|
|
d.text((i * 276 + 4, 2), f"t={ts:.1f}s", fill="white")
|
|
sheet.save(os.path.join(OUT, "contact-sheet.png"))
|
|
except Exception as e:
|
|
log(f"contact sheet failed: {e}")
|
|
|
|
log("DONE")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|