193 lines
8.7 KiB
Python
193 lines
8.7 KiB
Python
#!/usr/bin/env python
|
|
"""HEL-717 onset analysis: audio RMS onset vs lip-motion onset, multiple detectors.
|
|
|
|
Works on any muxed recording (browser webm or true-pts mp4). Mouth ROI is
|
|
derived from the recording itself (temporal variance), coordinates are saved.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import av
|
|
import cv2
|
|
import numpy as np
|
|
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("video", type=Path)
|
|
p.add_argument("--source", type=Path, default=Path("/opt/hel714-sample/dist/assets/talk/gaze.mp4"))
|
|
p.add_argument("--size", type=int, default=512)
|
|
p.add_argument("--idle", type=float, required=True, help="seconds of pre-speech idle to calibrate")
|
|
p.add_argument("--roi", help="x0,x1,y0,y1 in pixels at --size; default derive from variance")
|
|
p.add_argument("--strip", type=Path, help="write a contact strip of the ROI around onset")
|
|
p.add_argument("--offset-x", type=int, default=0)
|
|
p.add_argument("--offset-y", type=int, default=0)
|
|
p.add_argument("--render-size", type=int, default=0, help="crop window size before resize (canvas recordings)")
|
|
p.add_argument("--start", type=float, default=0)
|
|
p.add_argument("--stop", type=float, default=100000)
|
|
p.add_argument("--out", type=Path)
|
|
p.add_argument("--idle-until", type=float)
|
|
p.add_argument("--motion-threshold", type=float)
|
|
p.add_argument("--residual-threshold", type=float)
|
|
a = p.parse_args()
|
|
|
|
frames = []
|
|
rs = a.render_size or None
|
|
for f in av.open(str(a.video)).decode(video=0):
|
|
if not a.start <= float(f.time) <= a.stop:
|
|
continue
|
|
img = f.to_ndarray(format="bgr24")
|
|
if rs:
|
|
img = img[a.offset_y : a.offset_y + rs, a.offset_x : a.offset_x + rs]
|
|
img = cv2.resize(img, (a.size, a.size))
|
|
frames.append((float(f.time), img))
|
|
audio = []
|
|
for f in av.open(str(a.video)).decode(audio=0):
|
|
if not a.start <= float(f.time) <= a.stop:
|
|
continue
|
|
data = f.to_ndarray().astype(np.float64)
|
|
if data.max() > 1:
|
|
data /= 32768
|
|
audio.append([float(f.time), float(np.sqrt((data**2).mean()))])
|
|
|
|
stack = np.stack([img for _, img in frames]).astype(np.float32)
|
|
std = stack.std(axis=0).mean(axis=2)
|
|
if a.roi:
|
|
roi = tuple(int(v) for v in map(float, a.roi.split(",")))
|
|
print("using explicit mouth ROI (x0,x1,y0,y1):", roi)
|
|
else:
|
|
region = np.zeros_like(std, dtype=bool)
|
|
h = a.size
|
|
region[int(h * 0.35) : int(h * 0.80), int(h * 0.25) : int(h * 0.75)] = True
|
|
stdm = np.where(region, std, 0)
|
|
# smooth, then take the top-percentile blob
|
|
k = np.ones((15, 15), np.float32) / 225
|
|
stds = cv2.filter2D(stdm, -1, k)
|
|
thresh = np.quantile(stds[stds > 0], 0.995)
|
|
ys, xs = np.where(stds >= thresh)
|
|
x0, x1, y0, y1 = int(xs.min()), int(xs.max()), int(ys.min()), int(ys.max())
|
|
pad = 6
|
|
roi = (max(0, x0 - pad), min(a.size, x1 + pad), max(0, y0 - pad), min(a.size, y1 + pad))
|
|
print("derived mouth ROI (x0,x1,y0,y1):", roi, "fractions:", [v / a.size for v in roi])
|
|
|
|
# --- detector 1: frame-to-frame motion inside ROI (independent of source board)
|
|
motion = []
|
|
for i in range(1, len(frames)):
|
|
t0, i0 = frames[i - 1]
|
|
t1, i1 = frames[i]
|
|
m = np.abs(i1[roi[2] : roi[3], roi[0] : roi[1]].astype(np.float32) - i0[roi[2] : roi[3], roi[0] : roi[1]].astype(np.float32)).mean()
|
|
motion.append([t1, float(m)])
|
|
|
|
# --- detector 2: source-matching residual (same as construction analyzer)
|
|
source = [cv2.resize(f.to_ndarray(format="bgr24"), (a.size, a.size)) for f in av.open(str(a.source)).decode(video=0)]
|
|
mask = np.ones((48, 48), dtype=bool)
|
|
mask[0:3] = False
|
|
mask[43:] = False
|
|
mr = np.zeros((48, 48), dtype=bool)
|
|
my0, my1 = int(roi[2] * 48 / a.size), int(roi[3] * 48 / a.size)
|
|
mx0, mx1 = int(roi[0] * 48 / a.size), int(roi[1] * 48 / a.size)
|
|
mr[my0:my1, mx0:mx1] = True
|
|
mask = mask & ~mr
|
|
feats = np.asarray([cv2.resize(f, (48, 48))[mask].astype(np.float32) for f in source])
|
|
residuals = []
|
|
for t, img in frames:
|
|
feat = cv2.resize(img, (48, 48))[mask].astype(np.float32)
|
|
idx = int(np.argmin(((feats - feat) ** 2).mean(axis=(1, 2))))
|
|
res = np.abs(img.astype(np.float32) - source[idx]).mean(axis=2)
|
|
residuals.append([t, float(res[roi[2] : roi[3], roi[0] : roi[1]].mean()), idx])
|
|
|
|
first_audio = next((t for t, r in audio if r > 0.015), None)
|
|
|
|
|
|
def onset(rows, idle_end, floor):
|
|
base = [m for t, m, * _ in [tuple(r) for r in rows] if t <= idle_end]
|
|
if not base:
|
|
return None, None
|
|
th = max(floor, float(np.quantile(base, 0.99)) * 1.5)
|
|
hit = next((t for t, m, *_ in [tuple(r) for r in rows] if m > th and t > idle_end), None)
|
|
return hit, th
|
|
|
|
|
|
idle_end = a.idle_until if a.idle_until is not None else (first_audio - 0.7 if first_audio else a.idle)
|
|
lip_motion_onset, motion_th = onset(motion, idle_end, 1.2)
|
|
if a.motion_threshold is not None:
|
|
motion_th = a.motion_threshold
|
|
lip_motion_onset = next((t for t, m in motion if t > idle_end and m > motion_th), None)
|
|
lip_residual_onset, residual_th = onset(residuals, idle_end, 4)
|
|
if a.residual_threshold is not None:
|
|
residual_th = a.residual_threshold
|
|
lip_residual_onset = next((t for t, m, _ in residuals if t > idle_end and m > residual_th), None)
|
|
|
|
result = {
|
|
"detector_version": "hel718-v2-mask-fix",
|
|
"detector_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
|
"command": [sys.executable, *sys.argv],
|
|
"parameters": {k: str(v) if isinstance(v, Path) else v for k, v in vars(a).items()},
|
|
"effective_idle_until": idle_end,
|
|
"audio_threshold": 0.015,
|
|
"input_sha256": hashlib.sha256(a.video.read_bytes()).hexdigest(),
|
|
"source_sha256": hashlib.sha256(a.source.read_bytes()).hexdigest(),
|
|
"detection_uncertainty_seconds": {"video": 0.04, "audio": 0.022},
|
|
"video": str(a.video),
|
|
"frames": len(frames),
|
|
"audio_packets": len(audio),
|
|
"first_audio": first_audio,
|
|
"lip_motion_onset": lip_motion_onset,
|
|
"lip_motion_threshold": motion_th,
|
|
"motion_delta_seconds": (lip_motion_onset - first_audio) if lip_motion_onset and first_audio else None,
|
|
"lip_residual_onset": lip_residual_onset,
|
|
"lip_residual_threshold": residual_th,
|
|
"residual_delta_seconds": (lip_residual_onset - first_audio) if lip_residual_onset and first_audio else None,
|
|
"derived_roi": [int(v) for v in roi],
|
|
"roi_fractions": [round(v / a.size, 4) for v in roi],
|
|
"note": "delta<0 means lips move before sound; ROI derived from temporal variance, coordinates saved",
|
|
}
|
|
# Same envelope detector for each window; positive result means lips later.
|
|
if motion and audio:
|
|
grid = np.arange(max(motion[0][0], audio[0][0]), min(motion[-1][0], audio[-1][0]), 0.02)
|
|
m = np.interp(grid, np.asarray(motion)[:, 0], np.asarray(motion)[:, 1])
|
|
au = np.interp(grid, np.asarray(audio)[:, 0], np.asarray(audio)[:, 1])
|
|
m = np.convolve(m, np.ones(5) / 5, 'same')
|
|
au = np.convolve(au, np.ones(5) / 5, 'same')
|
|
active = grid[au > 0.02]
|
|
if len(active):
|
|
selected = (grid >= active.min() - 0.8) & (grid <= active.max() + 0.8)
|
|
m, au = m[selected], au[selected]
|
|
correlations = []
|
|
for shift in range(-40, 41):
|
|
x = m[:len(m)-shift or None] if shift >= 0 else m[-shift:]
|
|
y = au[shift:] if shift >= 0 else au[:len(au)+shift]
|
|
if len(x) > 2 and np.std(x) > 0 and np.std(y) > 0:
|
|
correlations.append({"lip_minus_audio_seconds": -shift * 0.02,
|
|
"correlation": float(np.corrcoef(x, y)[0, 1])})
|
|
result["steady_envelope"] = {"best": max(correlations, key=lambda c: c["correlation"]),
|
|
"all_lags": correlations,
|
|
"smooth_seconds": 0.1, "active_rms_threshold": 0.02,
|
|
"caveat": "motion energy heuristic, not phoneme alignment"}
|
|
out = a.out or a.video.with_suffix(".onset.json")
|
|
out.write_text(json.dumps(result, indent=2))
|
|
|
|
if a.strip and first_audio:
|
|
# contact strip: frames every 100ms from first_audio-1.2s to +1.6s
|
|
times = [first_audio - 1.2 + 0.1 * i for i in range(30)]
|
|
crops = []
|
|
for tt in times:
|
|
best = min(frames, key=lambda fr: abs(fr[0] - tt))[1]
|
|
crop = best[roi[2] - 20 : roi[3] + 20, roi[0] - 30 : roi[1] + 30]
|
|
crops.append(cv2.resize(crop, (120, 80)))
|
|
rows = [np.hstack(crops[i : i + 10]) for i in range(0, 30, 10)]
|
|
cv2.imwrite(str(a.strip), np.vstack(rows))
|
|
print("strip:", a.strip)
|
|
with out.with_suffix(".csv").open("w") as f:
|
|
w = csv.writer(f)
|
|
w.writerow(["time_seconds", "motion", "lip_residual", "source_index", "audio_rms"])
|
|
mmap = {t: m for t, m in motion}
|
|
amap = {t: r for t, r in audio}
|
|
for t, res, idx in residuals:
|
|
near_audio = min(audio, key=lambda x: abs(x[0] - t))[1] if audio else ""
|
|
near_motion = min(motion, key=lambda x: abs(x[0] - t))[1] if motion else ""
|
|
w.writerow([t, near_motion, res, idx, near_audio])
|
|
print(json.dumps(result, indent=2))
|