#!/usr/bin/env python """HEL-717 diagnostic: record the driver's own WebRTC output with TRUE RTP pts. No browser involved. If the lips-early offset appears here, it originates upstream (feature lookahead / frame association); if not, it is introduced by the browser playback/recording graph. """ import argparse import asyncio import hashlib import json import sys import time from fractions import Fraction from pathlib import Path import aiohttp import av import numpy as np from aiortc import RTCPeerConnection, RTCSessionDescription p = argparse.ArgumentParser() p.add_argument("--base", default="http://127.0.0.1:8010") p.add_argument("--wav", required=True) p.add_argument("--out", required=True) p.add_argument("--post", type=float, default=9.0) p.add_argument("--rounds", default="0,15,1,1", help="Idle seconds before each full utterance; then interrupt and restart") a = p.parse_args() events = [] def log(name, **kw): row = {"t": time.time(), "rel": time.perf_counter(), "event": name, **kw} events.append(row) print(json.dumps({k: v for k, v in row.items() if k != "t"}), flush=True) async def main(): pc = RTCPeerConnection() container = av.open(a.out, "w") vstream = container.add_stream("libx264", rate=25) vstream.time_base = Fraction(1, 90000) vstream.pix_fmt = "yuv420p" vstream.options = {"crf": "18", "preset": "veryfast"} astream = container.add_stream("aac", rate=48000) astream.time_base = Fraction(1, 48000) astream.layout = "mono" stats = {"v": 0, "a": 0} timeline = [] def video_track(track): async def loop(): while True: try: frame = await track.recv() except Exception: break stats["v"] += 1 timeline.append({"kind": "v", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter()}) img = frame.to_ndarray(format="bgr24") out = av.VideoFrame.from_ndarray(img, format="bgr24") out.pts = frame.pts out.time_base = frame.time_base for pkt in vstream.encode(out): container.mux(pkt) return asyncio.ensure_future(loop()) def audio_track(track): async def loop(): while True: try: frame = await track.recv() except Exception: break stats["a"] += 1 data = frame.to_ndarray() # aiortc delivers s16 mono 48k planar [1, n] pcm = data.reshape(-1).astype(np.int16) timeline.append({"kind": "a", "pts": frame.pts, "tb": str(frame.time_base), "wall": time.perf_counter(), "rms": float(np.sqrt((pcm.astype(np.float64) / 32768.0) ** 2).mean())}) out = av.AudioFrame.from_ndarray(pcm.reshape(1, -1), format="s16", layout="mono") out.sample_rate = frame.sample_rate out.pts = frame.pts out.time_base = frame.time_base for pkt in astream.encode(out): container.mux(pkt) return asyncio.ensure_future(loop()) @pc.on("track") def on_track(track): log("track", kind=track.kind) if track.kind == "video": video_track(track) else: audio_track(track) pc.addTransceiver("video", direction="recvonly") pc.addTransceiver("audio", direction="recvonly") offer = await pc.createOffer() await pc.setLocalDescription(offer) for _ in range(100): if pc.iceGatheringState == "complete": break await asyncio.sleep(0.05) async with aiohttp.ClientSession() as http: async with http.post(a.base + "/offer", json={"sdp": pc.localDescription.sdp, "type": "offer"}) as r: answer = await r.json() sid = answer["sessionid"] log("offer-ok", sid=sid) await pc.setRemoteDescription(RTCSessionDescription(sdp=answer["sdp"], type=answer["type"])) for _ in range(200): if stats["v"]: break await asyncio.sleep(0.05) log("first-video", v=stats["v"], a=stats["a"]) wav_bytes = Path(a.wav).read_bytes() async def send(label): log("send", label=label, v=stats["v"], a=stats["a"]) form = aiohttp.FormData() form.add_field("sessionid", sid) form.add_field("file", wav_bytes, filename="utterance.wav", content_type="audio/wav") async with http.post(a.base + "/humanaudio", data=form) as r: log("humanaudio-ok", label=label, body=await r.json()) for index, idle in enumerate(map(float, a.rounds.split(","))): await asyncio.sleep(idle) await send(f"full-{index+1}-idle-{idle}") await asyncio.sleep(a.post) await send("interrupted") await asyncio.sleep(2) async with http.post(a.base + "/interrupt_talk", json={"sessionid": sid}) as r: log("interrupt", body=await r.json()) await asyncio.sleep(1) await send("restart-same-peer-diagnostic") await asyncio.sleep(a.post) await pc.close() async with http.post(a.base + "/hel714/close", json={"sessionid": sid}) as r: log("closed", body=await r.json()) for pkt in vstream.encode(): container.mux(pkt) for pkt in astream.encode(): container.mux(pkt) container.close() Path(a.out + ".timeline.json").write_text(json.dumps({"command": [sys.executable, *sys.argv], "wav_sha256": hashlib.sha256(Path(a.wav).read_bytes()).hexdigest(), "events": events, "timeline": timeline})) log("done", v=stats["v"], a=stats["a"]) asyncio.run(main())