"""Source-relative lip residual + audio RMS using browser MediaRecorder PTS. Heuristic evidence only: no automatic perceptual-sync acceptance. """ import argparse import csv import json from pathlib import Path import av import cv2 import numpy as np p = argparse.ArgumentParser() p.add_argument("video", type=Path) p.add_argument("--source", type=Path, default=Path("/opt/hel715-verify/talk-close.mp4")) p.add_argument("--size", type=int, default=768) p.add_argument("--offset-x", type=int, default=0) p.add_argument("--offset-y", type=int, default=0) p.add_argument("--render-size", type=int, default=768) p.add_argument("--start", type=float, default=0) p.add_argument("--stop", type=float, default=100000) p.add_argument("--roi", default="0.4492,0.6185,0.4753,0.6315") p.add_argument("--idle-until", type=float) a = p.parse_args() source = [f.to_ndarray(format="bgr24") for f in av.open(str(a.source)).decode(video=0)] size = a.size source = [cv2.resize(f, (size, size)) for f in source] # Broad lower-face region; source matching uses pixels outside it and outside # caption/watermark so normal head movement is not mistaken for synthesized lips. x0, x1, y0, y1 = [round(v * size) for v in map(float, a.roi.split(","))] mask = np.ones((48, 48), dtype=bool) mask[0:3] = False mask[43:] = False mask[ round(y0 * 48 / size) : round(y1 * 48 / size), round(x0 * 48 / size) : round(x1 * 48 / size) ] = False features = np.asarray([cv2.resize(f, (48, 48))[mask].astype(np.float32) for f in source]) rows = [] for f in av.open(str(a.video)).decode(video=0): if not a.start <= float(f.time) <= a.stop: continue image = f.to_ndarray(format="bgr24") image = image[a.offset_y : a.offset_y + a.render_size, a.offset_x : a.offset_x + a.render_size] image = cv2.resize(image, (size, size)) feature = cv2.resize(image, (48, 48))[mask].astype(np.float32) index = np.argmin(((features - feature) ** 2).mean(axis=(1, 2))) residual = np.abs(image.astype(np.float32) - source[index]).mean(axis=2) rows.append([float(f.time), float(residual[y0:y1, x0:x1].mean()), int(index)]) audio = [] for f in av.open(str(a.video)).decode(audio=0): if not a.start <= float(f.time) <= a.stop: continue data = f.to_ndarray().astype(np.float64) if f.format.is_planar and f.format.name.startswith("fl"): pass elif data.max() > 1: data /= 32768 audio.append([float(f.time), float(np.sqrt((data**2).mean()))]) with a.video.with_suffix(".sync.csv").open("w") as out: w = csv.writer(out) w.writerow(["time_seconds", "lip_residual", "source_index", "audio_rms"]) for t, residual, idx in rows: nearest = min(audio, key=lambda x: abs(x[0] - t)) w.writerow([t, residual, idx, nearest[1]]) first_audio = next((t for t, rms in audio if rms > 0.015), None) baseline = [ r for t, r, _ in rows if first_audio and t < (a.idle_until if a.idle_until is not None else first_audio - 0.5) ] threshold = max(4, float(np.quantile(baseline, 0.99)) * 1.5) if baseline else 4 first_lip = next( ( t for t, r, _ in rows if r > threshold and first_audio and t >= (a.idle_until if a.idle_until is not None else first_audio - 1) ), None, ) result = dict( first_audio=first_audio, first_lip=first_lip, residual_threshold=threshold, onset_difference_seconds=first_lip - first_audio if first_lip and first_audio else None, caveat="Source-relative residual heuristic, not phoneme alignment or perceptual acceptance", ) a.video.with_suffix(".sync.json").write_text(json.dumps(result, indent=2)) print(result)