172 lines
6.1 KiB
Python
172 lines
6.1 KiB
Python
"""Credit: https://pytorch.org/audio/stable/tutorials/hybrid_demucs_tutorial.html"""
|
|
|
|
import torch
|
|
from torchaudio.transforms import Fade, Resample
|
|
from torchaudio.pipelines import HDEMUCS_HIGH_MUSDB_PLUS
|
|
|
|
import comfy.model_management
|
|
|
|
from typing import Dict, Tuple
|
|
from ._types import AUDIO
|
|
from .utils import ensure_stereo
|
|
|
|
|
|
class AudioSeparation:
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {
|
|
"required": {
|
|
"audio": ("AUDIO",),
|
|
},
|
|
"optional": {
|
|
"chunk_fade_shape": (
|
|
[
|
|
"linear",
|
|
"half_sine",
|
|
"logarithmic",
|
|
"exponential",
|
|
],
|
|
{
|
|
"default": "linear",
|
|
"tooltip": "Audio is split into segments (chunks) with overlapping areas to ensure smooth transitions. This setting controls the fade effect at these overlaps. Choose Linear for even fading, Half-Sine for a smooth curve, Logarithmic for a quick fade out and slow fade in, or Exponential for a slow fade out and quick fade in.",
|
|
},
|
|
),
|
|
"chunk_length": (
|
|
"FLOAT",
|
|
{
|
|
"default": 10.0,
|
|
"tooltip": "The length of each segment (chunk) in seconds. Longer chunks may require more memory and MIGHT produce better results.",
|
|
},
|
|
),
|
|
"chunk_overlap": (
|
|
"FLOAT",
|
|
{
|
|
"default": 0.1,
|
|
"tooltip": "The overlap between each segment (chunk) in seconds. A higher overlap may be necessary if chunks are too short or the audio changes rapidly.",
|
|
},
|
|
),
|
|
},
|
|
}
|
|
|
|
FUNCTION = "main"
|
|
RETURN_TYPES = ("AUDIO", "AUDIO", "AUDIO", "AUDIO")
|
|
RETURN_NAMES = ("Bass", "Drums", "Other", "Vocals")
|
|
CATEGORY = "audio"
|
|
DESCRIPTION = "Separate audio into four sources: bass, drums, other, and vocals."
|
|
|
|
def main(
|
|
self,
|
|
audio: AUDIO,
|
|
chunk_fade_shape: str = "linear",
|
|
chunk_length: float = 10.0,
|
|
chunk_overlap: float = 0.1,
|
|
) -> Tuple[AUDIO, AUDIO, AUDIO, AUDIO]:
|
|
|
|
device: torch.device = comfy.model_management.get_torch_device()
|
|
waveform: torch.Tensor = audio["waveform"]
|
|
waveform = waveform.squeeze(0).to(device)
|
|
self.input_sample_rate_: int = audio["sample_rate"]
|
|
|
|
bundle = HDEMUCS_HIGH_MUSDB_PLUS
|
|
model: torch.nn.Module = bundle.get_model().to(device)
|
|
self.model_sample_rate = bundle.sample_rate
|
|
|
|
waveform = ensure_stereo(waveform)
|
|
|
|
# Resample to model's expected sample rate
|
|
if self.input_sample_rate_ != self.model_sample_rate:
|
|
resample = Resample(self.input_sample_rate_, self.model_sample_rate).to(
|
|
device
|
|
)
|
|
waveform = resample(waveform)
|
|
|
|
ref = waveform.mean(0)
|
|
waveform = (waveform - ref.mean()) / ref.std() # Zs
|
|
|
|
sources = self.separate_sources(
|
|
model,
|
|
waveform[None],
|
|
self.model_sample_rate,
|
|
segment=chunk_length,
|
|
overlap=chunk_overlap,
|
|
device=device,
|
|
chunk_fade_shape=chunk_fade_shape,
|
|
)[0]
|
|
sources = sources * ref.std() + ref.mean()
|
|
sources_list = model.sources
|
|
sources = list(sources)
|
|
|
|
return self.sources_to_tuple(dict(zip(sources_list, sources)))
|
|
|
|
def sources_to_tuple(
|
|
self, sources: Dict[str, torch.Tensor]
|
|
) -> Tuple[AUDIO, AUDIO, AUDIO, AUDIO]:
|
|
|
|
output_order = ["bass", "drums", "other", "vocals"]
|
|
outputs = []
|
|
for source in output_order:
|
|
if source not in sources:
|
|
raise ValueError(f"Missing source {source} in the output")
|
|
outputs.append(
|
|
{
|
|
"waveform": sources[source].cpu().unsqueeze(0),
|
|
"sample_rate": self.model_sample_rate,
|
|
}
|
|
)
|
|
return tuple(outputs)
|
|
|
|
def separate_sources(
|
|
self,
|
|
model: torch.nn.Module,
|
|
mix: torch.Tensor,
|
|
sample_rate: int,
|
|
segment: float = 10.0,
|
|
overlap: float = 0.1,
|
|
device: torch.device = None,
|
|
chunk_fade_shape: str = "linear",
|
|
) -> torch.Tensor:
|
|
"""
|
|
From: https://pytorch.org/audio/stable/tutorials/hybrid_demucs_tutorial.html
|
|
|
|
Apply model to a given mixture. Use fade, and add segments together in order to add model segment by segment.
|
|
|
|
Args:
|
|
segment (int): segment length in seconds
|
|
device (torch.device, str, or None): if provided, device on which to
|
|
execute the computation, otherwise `mix.device` is assumed.
|
|
When `device` is different from `mix.device`, only local computations will
|
|
be on `device`, while the entire tracks will be stored on `mix.device`.
|
|
"""
|
|
if device is None:
|
|
device = mix.device
|
|
else:
|
|
device = torch.device(device)
|
|
|
|
batch, channels, length = mix.shape
|
|
|
|
chunk_len = int(sample_rate * segment * (1 + overlap))
|
|
start = 0
|
|
end = chunk_len
|
|
overlap_frames = overlap * sample_rate
|
|
fade = Fade(
|
|
fade_in_len=0, fade_out_len=int(overlap_frames), fade_shape=chunk_fade_shape
|
|
)
|
|
|
|
final = torch.zeros(batch, len(model.sources), channels, length, device=device)
|
|
|
|
while start < length - overlap_frames:
|
|
chunk = mix[:, :, start:end]
|
|
with torch.no_grad():
|
|
out = model.forward(chunk)
|
|
out = fade(out)
|
|
final[:, :, :, start:end] += out
|
|
if start == 0:
|
|
fade.fade_in_len = int(overlap_frames)
|
|
start += int(chunk_len - overlap_frames)
|
|
else:
|
|
start += chunk_len
|
|
end += chunk_len
|
|
if end >= length:
|
|
fade.fade_out_len = 0
|
|
return final
|