Files
CyberGril/samples/companion/tools/record_driver.py
T

150 lines
5.7 KiB
Python

#!/usr/bin/env python
"""HEL-717 diagnostic: record the driver's own WebRTC output with TRUE RTP pts.
No browser involved. If the lips-early offset appears here, it originates
upstream (feature lookahead / frame association); if not, it is introduced by
the browser playback/recording graph.
"""
import argparse
import asyncio
import hashlib
import json
import sys
import time
from fractions import Fraction
from pathlib import Path
import aiohttp
import av
import numpy as np
from aiortc import RTCPeerConnection, RTCSessionDescription
p = argparse.ArgumentParser()
p.add_argument("--base", default="http://127.0.0.1:8010")
p.add_argument("--wav", required=True)
p.add_argument("--out", required=True)
p.add_argument("--post", type=float, default=9.0)
p.add_argument("--rounds", default="0,15,1,1", help="Idle seconds before each full utterance; then interrupt and restart")
a = p.parse_args()
events = []
def log(name, **kw):
row = {"t": time.time(), "rel": time.perf_counter(), "event": name, **kw}
events.append(row)
print(json.dumps({k: v for k, v in row.items() if k != "t"}), flush=True)
async def main():
pc = RTCPeerConnection()
container = av.open(a.out, "w")
vstream = container.add_stream("libx264", rate=25)
vstream.time_base = Fraction(1, 90000)
vstream.pix_fmt = "yuv420p"
vstream.options = {"crf": "18", "preset": "veryfast"}
astream = container.add_stream("aac", rate=48000)
astream.time_base = Fraction(1, 48000)
astream.layout = "mono"
stats = {"v": 0, "a": 0}
timeline = []
def video_track(track):
async def loop():
while True:
try:
frame = await track.recv()
except Exception:
break
stats["v"] += 1
timeline.append({"kind": "v", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter()})
img = frame.to_ndarray(format="bgr24")
out = av.VideoFrame.from_ndarray(img, format="bgr24")
out.pts = frame.pts
out.time_base = frame.time_base
for pkt in vstream.encode(out):
container.mux(pkt)
return asyncio.ensure_future(loop())
def audio_track(track):
async def loop():
while True:
try:
frame = await track.recv()
except Exception:
break
stats["a"] += 1
data = frame.to_ndarray()
# aiortc delivers s16 mono 48k planar [1, n]
pcm = data.reshape(-1).astype(np.int16)
timeline.append({"kind": "a", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter(),
"rms": float(np.sqrt((pcm.astype(np.float64) / 32768.0) ** 2).mean())})
out = av.AudioFrame.from_ndarray(pcm.reshape(1, -1), format="s16", layout="mono")
out.sample_rate = frame.sample_rate
out.pts = frame.pts
out.time_base = frame.time_base
for pkt in astream.encode(out):
container.mux(pkt)
return asyncio.ensure_future(loop())
@pc.on("track")
def on_track(track):
log("track", kind=track.kind)
if track.kind == "video":
video_track(track)
else:
audio_track(track)
pc.addTransceiver("video", direction="recvonly")
pc.addTransceiver("audio", direction="recvonly")
offer = await pc.createOffer()
await pc.setLocalDescription(offer)
for _ in range(100):
if pc.iceGatheringState == "complete":
break
await asyncio.sleep(0.05)
async with aiohttp.ClientSession() as http:
async with http.post(a.base + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r:
answer = await r.json()
sid = answer["sessionid"]
log("offer-ok", sid=sid)
await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"]))
for _ in range(200):
if stats["v"]:
break
await asyncio.sleep(0.05)
log("first-video", v=stats["v"], a=stats["a"])
wav_bytes = Path(a.wav).read_bytes()
async def send(label):
log("send", label=label, v=stats["v"], a=stats["a"])
form = aiohttp.FormData()
form.add_field("sessionid", sid)
form.add_field("file", wav_bytes, filename="utterance.wav", content_type="audio/wav")
async with http.post(a.base + "/humanaudio", data=form) as r:
log("humanaudio-ok", label=label, body=await r.json())
for index, idle in enumerate(map(float, a.rounds.split(","))):
await asyncio.sleep(idle)
await send(f"full-{index+1}-idle-{idle}")
await asyncio.sleep(a.post)
await send("interrupted")
await asyncio.sleep(2)
async with http.post(a.base + "/interrupt_talk", json={"sessionid": sid}) as r:
log("interrupt", body=await r.json())
await asyncio.sleep(1)
await send("restart-same-peer-diagnostic")
await asyncio.sleep(a.post)
await pc.close()
async with http.post(a.base + "/hel714/close", json={"sessionid": sid}) as r:
log("closed", body=await r.json())
for pkt in vstream.encode():
container.mux(pkt)
for pkt in astream.encode():
container.mux(pkt)
container.close()
Path(a.out + ".timeline.json").write_text(json.dumps({"command": [sys.executable, *sys.argv], "wav_sha256": hashlib.sha256(Path(a.wav).read_bytes()).hexdigest(), "events": events, "timeline": timeline}))
log("done", v=stats["v"], a=stats["a"])
asyncio.run(main())