92 lines
3.6 KiB
Python
92 lines
3.6 KiB
Python
"""Source-relative lip residual + audio RMS using browser MediaRecorder PTS.
|
|
Heuristic evidence only: no automatic perceptual-sync acceptance.
|
|
"""
|
|
|
|
import argparse
|
|
import csv
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import av
|
|
import cv2
|
|
import numpy as np
|
|
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("video", type=Path)
|
|
p.add_argument("--source", type=Path, default=Path("/opt/hel715-verify/talk-close.mp4"))
|
|
p.add_argument("--size", type=int, default=768)
|
|
p.add_argument("--offset-x", type=int, default=0)
|
|
p.add_argument("--offset-y", type=int, default=0)
|
|
p.add_argument("--render-size", type=int, default=768)
|
|
p.add_argument("--start", type=float, default=0)
|
|
p.add_argument("--stop", type=float, default=100000)
|
|
p.add_argument("--roi", default="0.4492,0.6185,0.4753,0.6315")
|
|
p.add_argument("--idle-until", type=float)
|
|
a = p.parse_args()
|
|
source = [f.to_ndarray(format="bgr24") for f in av.open(str(a.source)).decode(video=0)]
|
|
size = a.size
|
|
source = [cv2.resize(f, (size, size)) for f in source]
|
|
# Broad lower-face region; source matching uses pixels outside it and outside
|
|
# caption/watermark so normal head movement is not mistaken for synthesized lips.
|
|
x0, x1, y0, y1 = [round(v * size) for v in map(float, a.roi.split(","))]
|
|
mask = np.ones((48, 48), dtype=bool)
|
|
mask[0:3] = False
|
|
mask[43:] = False
|
|
mask[
|
|
round(y0 * 48 / size) : round(y1 * 48 / size), round(x0 * 48 / size) : round(x1 * 48 / size)
|
|
] = False
|
|
features = np.asarray([cv2.resize(f, (48, 48))[mask].astype(np.float32) for f in source])
|
|
rows = []
|
|
for f in av.open(str(a.video)).decode(video=0):
|
|
if not a.start <= float(f.time) <= a.stop:
|
|
continue
|
|
image = f.to_ndarray(format="bgr24")
|
|
image = image[a.offset_y : a.offset_y + a.render_size, a.offset_x : a.offset_x + a.render_size]
|
|
image = cv2.resize(image, (size, size))
|
|
feature = cv2.resize(image, (48, 48))[mask].astype(np.float32)
|
|
index = np.argmin(((features - feature) ** 2).mean(axis=(1, 2)))
|
|
residual = np.abs(image.astype(np.float32) - source[index]).mean(axis=2)
|
|
rows.append([float(f.time), float(residual[y0:y1, x0:x1].mean()), int(index)])
|
|
audio = []
|
|
for f in av.open(str(a.video)).decode(audio=0):
|
|
if not a.start <= float(f.time) <= a.stop:
|
|
continue
|
|
data = f.to_ndarray().astype(np.float64)
|
|
if f.format.is_planar and f.format.name.startswith("fl"):
|
|
pass
|
|
elif data.max() > 1:
|
|
data /= 32768
|
|
audio.append([float(f.time), float(np.sqrt((data**2).mean()))])
|
|
with a.video.with_suffix(".sync.csv").open("w") as out:
|
|
w = csv.writer(out)
|
|
w.writerow(["time_seconds", "lip_residual", "source_index", "audio_rms"])
|
|
for t, residual, idx in rows:
|
|
nearest = min(audio, key=lambda x: abs(x[0] - t))
|
|
w.writerow([t, residual, idx, nearest[1]])
|
|
first_audio = next((t for t, rms in audio if rms > 0.015), None)
|
|
baseline = [
|
|
r
|
|
for t, r, _ in rows
|
|
if first_audio and t < (a.idle_until if a.idle_until is not None else first_audio - 0.5)
|
|
]
|
|
threshold = max(4, float(np.quantile(baseline, 0.99)) * 1.5) if baseline else 4
|
|
first_lip = next(
|
|
(
|
|
t
|
|
for t, r, _ in rows
|
|
if r > threshold
|
|
and first_audio
|
|
and t >= (a.idle_until if a.idle_until is not None else first_audio - 1)
|
|
),
|
|
None,
|
|
)
|
|
result = dict(
|
|
first_audio=first_audio,
|
|
first_lip=first_lip,
|
|
residual_threshold=threshold,
|
|
onset_difference_seconds=first_lip - first_audio if first_lip and first_audio else None,
|
|
caveat="Source-relative residual heuristic, not phoneme alignment or perceptual acceptance",
|
|
)
|
|
a.video.with_suffix(".sync.json").write_text(json.dumps(result, indent=2))
|
|
print(result)
|