150 lines
5.7 KiB
Python
150 lines
5.7 KiB
Python
#!/usr/bin/env python
|
|
"""HEL-717 diagnostic: record the driver's own WebRTC output with TRUE RTP pts.
|
|
|
|
No browser involved. If the lips-early offset appears here, it originates
|
|
upstream (feature lookahead / frame association); if not, it is introduced by
|
|
the browser playback/recording graph.
|
|
"""
|
|
import argparse
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
import time
|
|
from fractions import Fraction
|
|
from pathlib import Path
|
|
|
|
import aiohttp
|
|
import av
|
|
import numpy as np
|
|
from aiortc import RTCPeerConnection, RTCSessionDescription
|
|
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--base", default="http://127.0.0.1:8010")
|
|
p.add_argument("--wav", required=True)
|
|
p.add_argument("--out", required=True)
|
|
p.add_argument("--post", type=float, default=9.0)
|
|
p.add_argument("--rounds", default="0,15,1,1", help="Idle seconds before each full utterance; then interrupt and restart")
|
|
a = p.parse_args()
|
|
|
|
events = []
|
|
|
|
|
|
def log(name, **kw):
|
|
row = {"t": time.time(), "rel": time.perf_counter(), "event": name, **kw}
|
|
events.append(row)
|
|
print(json.dumps({k: v for k, v in row.items() if k != "t"}), flush=True)
|
|
|
|
|
|
async def main():
|
|
pc = RTCPeerConnection()
|
|
container = av.open(a.out, "w")
|
|
vstream = container.add_stream("libx264", rate=25)
|
|
vstream.time_base = Fraction(1, 90000)
|
|
vstream.pix_fmt = "yuv420p"
|
|
vstream.options = {"crf": "18", "preset": "veryfast"}
|
|
astream = container.add_stream("aac", rate=48000)
|
|
astream.time_base = Fraction(1, 48000)
|
|
astream.layout = "mono"
|
|
|
|
stats = {"v": 0, "a": 0}
|
|
timeline = []
|
|
|
|
def video_track(track):
|
|
async def loop():
|
|
while True:
|
|
try:
|
|
frame = await track.recv()
|
|
except Exception:
|
|
break
|
|
stats["v"] += 1
|
|
timeline.append({"kind": "v", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter()})
|
|
img = frame.to_ndarray(format="bgr24")
|
|
out = av.VideoFrame.from_ndarray(img, format="bgr24")
|
|
out.pts = frame.pts
|
|
out.time_base = frame.time_base
|
|
for pkt in vstream.encode(out):
|
|
container.mux(pkt)
|
|
return asyncio.ensure_future(loop())
|
|
|
|
def audio_track(track):
|
|
async def loop():
|
|
while True:
|
|
try:
|
|
frame = await track.recv()
|
|
except Exception:
|
|
break
|
|
stats["a"] += 1
|
|
data = frame.to_ndarray()
|
|
# aiortc delivers s16 mono 48k planar [1, n]
|
|
pcm = data.reshape(-1).astype(np.int16)
|
|
timeline.append({"kind": "a", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter(),
|
|
"rms": float(np.sqrt((pcm.astype(np.float64) / 32768.0) ** 2).mean())})
|
|
out = av.AudioFrame.from_ndarray(pcm.reshape(1, -1), format="s16", layout="mono")
|
|
out.sample_rate = frame.sample_rate
|
|
out.pts = frame.pts
|
|
out.time_base = frame.time_base
|
|
for pkt in astream.encode(out):
|
|
container.mux(pkt)
|
|
return asyncio.ensure_future(loop())
|
|
|
|
@pc.on("track")
|
|
def on_track(track):
|
|
log("track", kind=track.kind)
|
|
if track.kind == "video":
|
|
video_track(track)
|
|
else:
|
|
audio_track(track)
|
|
|
|
pc.addTransceiver("video", direction="recvonly")
|
|
pc.addTransceiver("audio", direction="recvonly")
|
|
offer = await pc.createOffer()
|
|
await pc.setLocalDescription(offer)
|
|
for _ in range(100):
|
|
if pc.iceGatheringState == "complete":
|
|
break
|
|
await asyncio.sleep(0.05)
|
|
async with aiohttp.ClientSession() as http:
|
|
async with http.post(a.base + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r:
|
|
answer = await r.json()
|
|
sid = answer["sessionid"]
|
|
log("offer-ok", sid=sid)
|
|
await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"]))
|
|
for _ in range(200):
|
|
if stats["v"]:
|
|
break
|
|
await asyncio.sleep(0.05)
|
|
log("first-video", v=stats["v"], a=stats["a"])
|
|
wav_bytes = Path(a.wav).read_bytes()
|
|
async def send(label):
|
|
log("send", label=label, v=stats["v"], a=stats["a"])
|
|
form = aiohttp.FormData()
|
|
form.add_field("sessionid", sid)
|
|
form.add_field("file", wav_bytes, filename="utterance.wav", content_type="audio/wav")
|
|
async with http.post(a.base + "/humanaudio", data=form) as r:
|
|
log("humanaudio-ok", label=label, body=await r.json())
|
|
for index, idle in enumerate(map(float, a.rounds.split(","))):
|
|
await asyncio.sleep(idle)
|
|
await send(f"full-{index+1}-idle-{idle}")
|
|
await asyncio.sleep(a.post)
|
|
await send("interrupted")
|
|
await asyncio.sleep(2)
|
|
async with http.post(a.base + "/interrupt_talk", json={"sessionid": sid}) as r:
|
|
log("interrupt", body=await r.json())
|
|
await asyncio.sleep(1)
|
|
await send("restart-same-peer-diagnostic")
|
|
await asyncio.sleep(a.post)
|
|
await pc.close()
|
|
async with http.post(a.base + "/hel714/close", json={"sessionid": sid}) as r:
|
|
log("closed", body=await r.json())
|
|
for pkt in vstream.encode():
|
|
container.mux(pkt)
|
|
for pkt in astream.encode():
|
|
container.mux(pkt)
|
|
container.close()
|
|
Path(a.out + ".timeline.json").write_text(json.dumps({"command": [sys.executable, *sys.argv], "wav_sha256": hashlib.sha256(Path(a.wav).read_bytes()).hexdigest(), "events": events, "timeline": timeline}))
|
|
log("done", v=stats["v"], a=stats["a"])
|
|
|
|
|
|
asyncio.run(main())
|