Files
aigc-home/docs/hel-892/director-replace-reference/t8__audio_ops.py
T
2026-11-01 10:50:03 +08:00

144 lines
5.9 KiB
Python

from __future__ import annotations
import json
import torch
import torch.nn.functional as torch_functional
import torchaudio
from comfy_extras.nodes_audio import vae_decode_audio
from .core import encode_audio_once, nested_av_parts, replace_audio_latent, validate_audio
def inject_audio_latent(av_latent: dict, source_audio: dict, audio_vae, mode: str, strength: float):
mode = mode.lower()
if mode not in {"lock", "remix"}:
raise ValueError("Audio latent control mode must be lock or remix")
if not 0.0 <= strength <= 1.0:
raise ValueError("strength must be between 0 and 1")
encoded = encode_audio_once(audio_vae, source_audio)
denoise = 0.0 if mode == "lock" else strength
return replace_audio_latent(av_latent, encoded, denoise), source_audio
def decode_av_latent(av_latent: dict, video_vae, audio_vae):
video, audio = nested_av_parts(av_latent)
images = video_vae.decode(video)
if images.ndim == 5:
images = images.reshape(-1, *images.shape[-3:])
decoded_audio = vae_decode_audio(audio_vae, {"samples": audio})
video_latent = {key: value for key, value in av_latent.items() if key not in {"samples", "noise_mask"}}
audio_latent = video_latent.copy()
video_latent["samples"] = video
audio_latent["samples"] = audio
masks = av_latent.get("noise_mask")
if getattr(masks, "is_nested", False):
video_mask, audio_mask = masks.unbind()
video_latent["noise_mask"] = video_mask
audio_latent["noise_mask"] = audio_mask
return images, decoded_audio, video_latent, audio_latent
def _resample(waveform: torch.Tensor, source_rate: int, target_rate: int) -> torch.Tensor:
if source_rate == target_rate:
return waveform
return torchaudio.functional.resample(waveform, source_rate, target_rate)
def _match_channels(waveform: torch.Tensor, target_channels: int) -> torch.Tensor:
channels = waveform.shape[1]
if channels == target_channels:
return waveform
if channels == 1:
return waveform.expand(-1, target_channels, -1)
if target_channels == 1:
return waveform.mean(dim=1, keepdim=True)
# For uncommon multichannel inputs, make a deterministic stereo-compatible downmix.
mono = waveform.mean(dim=1, keepdim=True)
return mono.expand(-1, target_channels, -1)
def mix_audio(
source_audio: dict,
generated_audio: dict,
source_gain_db: float = 0.0,
generated_gain_db: float = -6.0,
duck_generated: float = 0.5,
output_sample_rate: str = "source",
peak_limit_dbfs: float = -1.0,
):
source, source_rate = validate_audio(source_audio, "source_audio")
generated, generated_rate = validate_audio(generated_audio, "generated_audio")
if output_sample_rate == "source":
target_rate = source_rate
elif output_sample_rate == "generated":
target_rate = generated_rate
else:
target_rate = int(output_sample_rate)
source = _resample(source, source_rate, target_rate)
generated = _resample(generated, generated_rate, target_rate)
target_channels = max(source.shape[1], generated.shape[1])
source = _match_channels(source, target_channels)
generated = _match_channels(generated, target_channels)
target_samples = max(source.shape[-1], generated.shape[-1])
source = torch_functional.pad(source, (0, target_samples - source.shape[-1]))
generated = torch_functional.pad(generated, (0, target_samples - generated.shape[-1]))
source = source * (10.0 ** (source_gain_db / 20.0))
generated = generated * (10.0 ** (generated_gain_db / 20.0))
if duck_generated > 0:
envelope = source.abs().mean(dim=1, keepdim=True)
window = max(1, round(target_rate * 0.02))
envelope = torch_functional.avg_pool1d(envelope, window, stride=1, padding=window // 2)
envelope = envelope[..., :target_samples]
activity = (envelope / 0.05).clamp(0.0, 1.0)
generated = generated * (1.0 - float(duck_generated) * activity)
mixed = source + generated
limit = 10.0 ** (peak_limit_dbfs / 20.0)
peak = mixed.abs().amax(dim=(1, 2), keepdim=True).clamp_min(1e-8)
scale = torch.minimum(torch.ones_like(peak), torch.full_like(peak, limit) / peak)
mixed = mixed * scale
return {"waveform": mixed, "sample_rate": target_rate}
def trim_av_output(frames: torch.Tensor, start_seconds: float, duration_seconds: float, audio=None, fps: float = 24.0):
if frames.ndim != 4:
raise ValueError(f"frames must be IMAGE [N,H,W,C], got {tuple(frames.shape)}")
if start_seconds < 0 or duration_seconds <= 0 or fps <= 0:
raise ValueError("trim start must be nonnegative; duration and fps must be positive")
start_frame = round(start_seconds * fps)
frame_count = max(1, round(duration_seconds * fps))
end_frame = start_frame + frame_count
if end_frame > frames.shape[0]:
raise ValueError(
f"Requested output frames [{start_frame}:{end_frame}] exceed decoded frame count {frames.shape[0]}"
)
trimmed_frames = frames[start_frame:end_frame]
trimmed_audio = None
if audio is not None:
waveform, sample_rate = validate_audio(audio)
start_sample = round(start_seconds * sample_rate)
sample_count = round(duration_seconds * sample_rate)
sliced = waveform[..., start_sample : start_sample + sample_count]
if sliced.shape[-1] < sample_count:
sliced = torch_functional.pad(sliced, (0, sample_count - sliced.shape[-1]))
trimmed_audio = {"waveform": sliced, "sample_rate": sample_rate}
report = json.dumps(
{
"start_seconds": start_seconds,
"requested_duration_seconds": duration_seconds,
"fps": fps,
"start_frame": start_frame,
"frame_count": frame_count,
"actual_video_duration_seconds": frame_count / fps,
},
ensure_ascii=False,
indent=2,
)
return trimmed_frames, trimmed_audio, report