Files
CyberGril/samples/companion/tools/measure_sync.py
T

193 lines
8.7 KiB
Python

#!/usr/bin/env python
"""HEL-717 onset analysis: audio RMS onset vs lip-motion onset, multiple detectors.
Works on any muxed recording (browser webm or true-pts mp4). Mouth ROI is
derived from the recording itself (temporal variance), coordinates are saved.
"""
import argparse
import csv
import hashlib
import json
import sys
from pathlib import Path
import av
import cv2
import numpy as np
p = argparse.ArgumentParser()
p.add_argument("video", type=Path)
p.add_argument("--source", type=Path, default=Path("/opt/hel714-sample/dist/assets/talk/gaze.mp4"))
p.add_argument("--size", type=int, default=512)
p.add_argument("--idle", type=float, required=True, help="seconds of pre-speech idle to calibrate")
p.add_argument("--roi", help="x0,x1,y0,y1 in pixels at --size; default derive from variance")
p.add_argument("--strip", type=Path, help="write a contact strip of the ROI around onset")
p.add_argument("--offset-x", type=int, default=0)
p.add_argument("--offset-y", type=int, default=0)
p.add_argument("--render-size", type=int, default=0, help="crop window size before resize (canvas recordings)")
p.add_argument("--start", type=float, default=0)
p.add_argument("--stop", type=float, default=100000)
p.add_argument("--out", type=Path)
p.add_argument("--idle-until", type=float)
p.add_argument("--motion-threshold", type=float)
p.add_argument("--residual-threshold", type=float)
a = p.parse_args()
frames = []
rs = a.render_size or None
for f in av.open(str(a.video)).decode(video=0):
if not a.start <= float(f.time) <= a.stop:
continue
img = f.to_ndarray(format="bgr24")
if rs:
img = img[a.offset_y : a.offset_y + rs, a.offset_x : a.offset_x + rs]
img = cv2.resize(img, (a.size, a.size))
frames.append((float(f.time), img))
audio = []
for f in av.open(str(a.video)).decode(audio=0):
if not a.start <= float(f.time) <= a.stop:
continue
data = f.to_ndarray().astype(np.float64)
if data.max() > 1:
data /= 32768
audio.append([float(f.time), float(np.sqrt((data**2).mean()))])
stack = np.stack([img for _, img in frames]).astype(np.float32)
std = stack.std(axis=0).mean(axis=2)
if a.roi:
roi = tuple(int(v) for v in map(float, a.roi.split(",")))
print("using explicit mouth ROI (x0,x1,y0,y1):", roi)
else:
region = np.zeros_like(std, dtype=bool)
h = a.size
region[int(h * 0.35) : int(h * 0.80), int(h * 0.25) : int(h * 0.75)] = True
stdm = np.where(region, std, 0)
# smooth, then take the top-percentile blob
k = np.ones((15, 15), np.float32) / 225
stds = cv2.filter2D(stdm, -1, k)
thresh = np.quantile(stds[stds > 0], 0.995)
ys, xs = np.where(stds >= thresh)
x0, x1, y0, y1 = int(xs.min()), int(xs.max()), int(ys.min()), int(ys.max())
pad = 6
roi = (max(0, x0 - pad), min(a.size, x1 + pad), max(0, y0 - pad), min(a.size, y1 + pad))
print("derived mouth ROI (x0,x1,y0,y1):", roi, "fractions:", [v / a.size for v in roi])
# --- detector 1: frame-to-frame motion inside ROI (independent of source board)
motion = []
for i in range(1, len(frames)):
t0, i0 = frames[i - 1]
t1, i1 = frames[i]
m = np.abs(i1[roi[2] : roi[3], roi[0] : roi[1]].astype(np.float32) - i0[roi[2] : roi[3], roi[0] : roi[1]].astype(np.float32)).mean()
motion.append([t1, float(m)])
# --- detector 2: source-matching residual (same as construction analyzer)
source = [cv2.resize(f.to_ndarray(format="bgr24"), (a.size, a.size)) for f in av.open(str(a.source)).decode(video=0)]
mask = np.ones((48, 48), dtype=bool)
mask[0:3] = False
mask[43:] = False
mr = np.zeros((48, 48), dtype=bool)
my0, my1 = int(roi[2] * 48 / a.size), int(roi[3] * 48 / a.size)
mx0, mx1 = int(roi[0] * 48 / a.size), int(roi[1] * 48 / a.size)
mr[my0:my1, mx0:mx1] = True
mask = mask & ~mr
feats = np.asarray([cv2.resize(f, (48, 48))[mask].astype(np.float32) for f in source])
residuals = []
for t, img in frames:
feat = cv2.resize(img, (48, 48))[mask].astype(np.float32)
idx = int(np.argmin(((feats - feat) ** 2).mean(axis=(1, 2))))
res = np.abs(img.astype(np.float32) - source[idx]).mean(axis=2)
residuals.append([t, float(res[roi[2] : roi[3], roi[0] : roi[1]].mean()), idx])
first_audio = next((t for t, r in audio if r > 0.015), None)
def onset(rows, idle_end, floor):
base = [m for t, m, * _ in [tuple(r) for r in rows] if t <= idle_end]
if not base:
return None, None
th = max(floor, float(np.quantile(base, 0.99)) * 1.5)
hit = next((t for t, m, *_ in [tuple(r) for r in rows] if m > th and t > idle_end), None)
return hit, th
idle_end = a.idle_until if a.idle_until is not None else (first_audio - 0.7 if first_audio else a.idle)
lip_motion_onset, motion_th = onset(motion, idle_end, 1.2)
if a.motion_threshold is not None:
motion_th = a.motion_threshold
lip_motion_onset = next((t for t, m in motion if t > idle_end and m > motion_th), None)
lip_residual_onset, residual_th = onset(residuals, idle_end, 4)
if a.residual_threshold is not None:
residual_th = a.residual_threshold
lip_residual_onset = next((t for t, m, _ in residuals if t > idle_end and m > residual_th), None)
result = {
"detector_version": "hel718-v2-mask-fix",
"detector_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
"command": [sys.executable, *sys.argv],
"parameters": {k: str(v) if isinstance(v, Path) else v for k, v in vars(a).items()},
"effective_idle_until": idle_end,
"audio_threshold": 0.015,
"input_sha256": hashlib.sha256(a.video.read_bytes()).hexdigest(),
"source_sha256": hashlib.sha256(a.source.read_bytes()).hexdigest(),
"detection_uncertainty_seconds": {"video": 0.04, "audio": 0.022},
"video": str(a.video),
"frames": len(frames),
"audio_packets": len(audio),
"first_audio": first_audio,
"lip_motion_onset": lip_motion_onset,
"lip_motion_threshold": motion_th,
"motion_delta_seconds": (lip_motion_onset - first_audio) if lip_motion_onset and first_audio else None,
"lip_residual_onset": lip_residual_onset,
"lip_residual_threshold": residual_th,
"residual_delta_seconds": (lip_residual_onset - first_audio) if lip_residual_onset and first_audio else None,
"derived_roi": [int(v) for v in roi],
"roi_fractions": [round(v / a.size, 4) for v in roi],
"note": "delta<0 means lips move before sound; ROI derived from temporal variance, coordinates saved",
}
# Same envelope detector for each window; positive result means lips later.
if motion and audio:
grid = np.arange(max(motion[0][0], audio[0][0]), min(motion[-1][0], audio[-1][0]), 0.02)
m = np.interp(grid, np.asarray(motion)[:, 0], np.asarray(motion)[:, 1])
au = np.interp(grid, np.asarray(audio)[:, 0], np.asarray(audio)[:, 1])
m = np.convolve(m, np.ones(5) / 5, 'same')
au = np.convolve(au, np.ones(5) / 5, 'same')
active = grid[au > 0.02]
if len(active):
selected = (grid >= active.min() - 0.8) & (grid <= active.max() + 0.8)
m, au = m[selected], au[selected]
correlations = []
for shift in range(-40, 41):
x = m[:len(m)-shift or None] if shift >= 0 else m[-shift:]
y = au[shift:] if shift >= 0 else au[:len(au)+shift]
if len(x) > 2 and np.std(x) > 0 and np.std(y) > 0:
correlations.append({"lip_minus_audio_seconds": -shift * 0.02,
"correlation": float(np.corrcoef(x, y)[0, 1])})
result["steady_envelope"] = {"best": max(correlations, key=lambda c: c["correlation"]),
"all_lags": correlations,
"smooth_seconds": 0.1, "active_rms_threshold": 0.02,
"caveat": "motion energy heuristic, not phoneme alignment"}
out = a.out or a.video.with_suffix(".onset.json")
out.write_text(json.dumps(result, indent=2))
if a.strip and first_audio:
# contact strip: frames every 100ms from first_audio-1.2s to +1.6s
times = [first_audio - 1.2 + 0.1 * i for i in range(30)]
crops = []
for tt in times:
best = min(frames, key=lambda fr: abs(fr[0] - tt))[1]
crop = best[roi[2] - 20 : roi[3] + 20, roi[0] - 30 : roi[1] + 30]
crops.append(cv2.resize(crop, (120, 80)))
rows = [np.hstack(crops[i : i + 10]) for i in range(0, 30, 10)]
cv2.imwrite(str(a.strip), np.vstack(rows))
print("strip:", a.strip)
with out.with_suffix(".csv").open("w") as f:
w = csv.writer(f)
w.writerow(["time_seconds", "motion", "lip_residual", "source_index", "audio_rms"])
mmap = {t: m for t, m in motion}
amap = {t: r for t, r in audio}
for t, res, idx in residuals:
near_audio = min(audio, key=lambda x: abs(x[0] - t))[1] if audio else ""
near_motion = min(motion, key=lambda x: abs(x[0] - t))[1] if motion else ""
w.writerow([t, near_motion, res, idx, near_audio])
print(json.dumps(result, indent=2))