Files
aigc-home/docs/hel-892/separation-reference/plugin-5291831d__src__separation.py
T

172 lines
6.1 KiB
Python

"""Credit: https://pytorch.org/audio/stable/tutorials/hybrid_demucs_tutorial.html"""
import torch
from torchaudio.transforms import Fade, Resample
from torchaudio.pipelines import HDEMUCS_HIGH_MUSDB_PLUS
import comfy.model_management
from typing import Dict, Tuple
from ._types import AUDIO
from .utils import ensure_stereo
class AudioSeparation:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"audio": ("AUDIO",),
},
"optional": {
"chunk_fade_shape": (
[
"linear",
"half_sine",
"logarithmic",
"exponential",
],
{
"default": "linear",
"tooltip": "Audio is split into segments (chunks) with overlapping areas to ensure smooth transitions. This setting controls the fade effect at these overlaps. Choose Linear for even fading, Half-Sine for a smooth curve, Logarithmic for a quick fade out and slow fade in, or Exponential for a slow fade out and quick fade in.",
},
),
"chunk_length": (
"FLOAT",
{
"default": 10.0,
"tooltip": "The length of each segment (chunk) in seconds. Longer chunks may require more memory and MIGHT produce better results.",
},
),
"chunk_overlap": (
"FLOAT",
{
"default": 0.1,
"tooltip": "The overlap between each segment (chunk) in seconds. A higher overlap may be necessary if chunks are too short or the audio changes rapidly.",
},
),
},
}
FUNCTION = "main"
RETURN_TYPES = ("AUDIO", "AUDIO", "AUDIO", "AUDIO")
RETURN_NAMES = ("Bass", "Drums", "Other", "Vocals")
CATEGORY = "audio"
DESCRIPTION = "Separate audio into four sources: bass, drums, other, and vocals."
def main(
self,
audio: AUDIO,
chunk_fade_shape: str = "linear",
chunk_length: float = 10.0,
chunk_overlap: float = 0.1,
) -> Tuple[AUDIO, AUDIO, AUDIO, AUDIO]:
device: torch.device = comfy.model_management.get_torch_device()
waveform: torch.Tensor = audio["waveform"]
waveform = waveform.squeeze(0).to(device)
self.input_sample_rate_: int = audio["sample_rate"]
bundle = HDEMUCS_HIGH_MUSDB_PLUS
model: torch.nn.Module = bundle.get_model().to(device)
self.model_sample_rate = bundle.sample_rate
waveform = ensure_stereo(waveform)
# Resample to model's expected sample rate
if self.input_sample_rate_ != self.model_sample_rate:
resample = Resample(self.input_sample_rate_, self.model_sample_rate).to(
device
)
waveform = resample(waveform)
ref = waveform.mean(0)
waveform = (waveform - ref.mean()) / ref.std() # Zs
sources = self.separate_sources(
model,
waveform[None],
self.model_sample_rate,
segment=chunk_length,
overlap=chunk_overlap,
device=device,
chunk_fade_shape=chunk_fade_shape,
)[0]
sources = sources * ref.std() + ref.mean()
sources_list = model.sources
sources = list(sources)
return self.sources_to_tuple(dict(zip(sources_list, sources)))
def sources_to_tuple(
self, sources: Dict[str, torch.Tensor]
) -> Tuple[AUDIO, AUDIO, AUDIO, AUDIO]:
output_order = ["bass", "drums", "other", "vocals"]
outputs = []
for source in output_order:
if source not in sources:
raise ValueError(f"Missing source {source} in the output")
outputs.append(
{
"waveform": sources[source].cpu().unsqueeze(0),
"sample_rate": self.model_sample_rate,
}
)
return tuple(outputs)
def separate_sources(
self,
model: torch.nn.Module,
mix: torch.Tensor,
sample_rate: int,
segment: float = 10.0,
overlap: float = 0.1,
device: torch.device = None,
chunk_fade_shape: str = "linear",
) -> torch.Tensor:
"""
From: https://pytorch.org/audio/stable/tutorials/hybrid_demucs_tutorial.html
Apply model to a given mixture. Use fade, and add segments together in order to add model segment by segment.
Args:
segment (int): segment length in seconds
device (torch.device, str, or None): if provided, device on which to
execute the computation, otherwise `mix.device` is assumed.
When `device` is different from `mix.device`, only local computations will
be on `device`, while the entire tracks will be stored on `mix.device`.
"""
if device is None:
device = mix.device
else:
device = torch.device(device)
batch, channels, length = mix.shape
chunk_len = int(sample_rate * segment * (1 + overlap))
start = 0
end = chunk_len
overlap_frames = overlap * sample_rate
fade = Fade(
fade_in_len=0, fade_out_len=int(overlap_frames), fade_shape=chunk_fade_shape
)
final = torch.zeros(batch, len(model.sources), channels, length, device=device)
while start < length - overlap_frames:
chunk = mix[:, :, start:end]
with torch.no_grad():
out = model.forward(chunk)
out = fade(out)
final[:, :, :, start:end] += out
if start == 0:
fade.fade_in_len = int(overlap_frames)
start += int(chunk_len - overlap_frames)
else:
start += chunk_len
end += chunk_len
if end >= length:
fade.fade_out_len = 0
return final