Files
CyberGril/samples/companion/tools/analyze_sync.py
T

92 lines
3.6 KiB
Python

"""Source-relative lip residual + audio RMS using browser MediaRecorder PTS.
Heuristic evidence only: no automatic perceptual-sync acceptance.
"""
import argparse
import csv
import json
from pathlib import Path
import av
import cv2
import numpy as np
p = argparse.ArgumentParser()
p.add_argument("video", type=Path)
p.add_argument("--source", type=Path, default=Path("/opt/hel715-verify/talk-close.mp4"))
p.add_argument("--size", type=int, default=768)
p.add_argument("--offset-x", type=int, default=0)
p.add_argument("--offset-y", type=int, default=0)
p.add_argument("--render-size", type=int, default=768)
p.add_argument("--start", type=float, default=0)
p.add_argument("--stop", type=float, default=100000)
p.add_argument("--roi", default="0.4492,0.6185,0.4753,0.6315")
p.add_argument("--idle-until", type=float)
a = p.parse_args()
source = [f.to_ndarray(format="bgr24") for f in av.open(str(a.source)).decode(video=0)]
size = a.size
source = [cv2.resize(f, (size, size)) for f in source]
# Broad lower-face region; source matching uses pixels outside it and outside
# caption/watermark so normal head movement is not mistaken for synthesized lips.
x0, x1, y0, y1 = [round(v * size) for v in map(float, a.roi.split(","))]
mask = np.ones((48, 48), dtype=bool)
mask[0:3] = False
mask[43:] = False
mask[
round(y0 * 48 / size) : round(y1 * 48 / size), round(x0 * 48 / size) : round(x1 * 48 / size)
] = False
features = np.asarray([cv2.resize(f, (48, 48))[mask].astype(np.float32) for f in source])
rows = []
for f in av.open(str(a.video)).decode(video=0):
if not a.start <= float(f.time) <= a.stop:
continue
image = f.to_ndarray(format="bgr24")
image = image[a.offset_y : a.offset_y + a.render_size, a.offset_x : a.offset_x + a.render_size]
image = cv2.resize(image, (size, size))
feature = cv2.resize(image, (48, 48))[mask].astype(np.float32)
index = np.argmin(((features - feature) ** 2).mean(axis=(1, 2)))
residual = np.abs(image.astype(np.float32) - source[index]).mean(axis=2)
rows.append([float(f.time), float(residual[y0:y1, x0:x1].mean()), int(index)])
audio = []
for f in av.open(str(a.video)).decode(audio=0):
if not a.start <= float(f.time) <= a.stop:
continue
data = f.to_ndarray().astype(np.float64)
if f.format.is_planar and f.format.name.startswith("fl"):
pass
elif data.max() > 1:
data /= 32768
audio.append([float(f.time), float(np.sqrt((data**2).mean()))])
with a.video.with_suffix(".sync.csv").open("w") as out:
w = csv.writer(out)
w.writerow(["time_seconds", "lip_residual", "source_index", "audio_rms"])
for t, residual, idx in rows:
nearest = min(audio, key=lambda x: abs(x[0] - t))
w.writerow([t, residual, idx, nearest[1]])
first_audio = next((t for t, rms in audio if rms > 0.015), None)
baseline = [
r
for t, r, _ in rows
if first_audio and t < (a.idle_until if a.idle_until is not None else first_audio - 0.5)
]
threshold = max(4, float(np.quantile(baseline, 0.99)) * 1.5) if baseline else 4
first_lip = next(
(
t
for t, r, _ in rows
if r > threshold
and first_audio
and t >= (a.idle_until if a.idle_until is not None else first_audio - 1)
),
None,
)
result = dict(
first_audio=first_audio,
first_lip=first_lip,
residual_threshold=threshold,
onset_difference_seconds=first_lip - first_audio if first_lip and first_audio else None,
caveat="Source-relative residual heuristic, not phoneme alignment or perceptual acceptance",
)
a.video.with_suffix(".sync.json").write_text(json.dumps(result, indent=2))
print(result)