Files
aigc-home/docs/hel-892/director-two-reference/mediamatch__nodes__basic.py
T

3675 lines
143 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import json
import math
import os
import re
import tempfile
from enum import Enum
from pathlib import Path
import folder_paths
import nodes as comfy_nodes
import torch
import torch.nn.functional as F
from comfy_api.latest import InputImpl, Types, io
from comfy_execution.graph_utils import GraphBuilder, is_link
from comfy.utils import ProgressBar
from ..utils import (
FFMPEG_RESIZE_METHODS,
audio_db_to_gain,
audio_is_muted,
audio_volume_db,
build_multitrack_data_from_prompt_override,
burn_subtitles_with_ffmpeg,
collect_multitrack_subtitle_segments,
default_subtitle_filename,
equirectangular_to_perspective,
frames_to_seconds,
load_audio_waveform,
load_image_tensor,
log_node_info,
iter_valid_audio_inputs,
merge_audio_inputs,
audio_data_uris,
image_tensor_data_uris,
LLAMACPP_MODEL,
MINIMAX_MODEL,
PROMPT_ENHANCER_MAX_TOKENS,
PROMPT_ENHANCER_MODELS,
PromptEnhancerApiError,
PromptEnhancerClient,
prompt_enhancer_video_inputs,
minimax_length_to_seconds,
split_list_outputs,
merge_video_track_with_ffmpeg,
parse_subtitle_text,
parse_override_segments,
prompt_override_has_frame_ranges,
prompt_override_has_value,
resize_image,
resize_video_with_ffmpeg,
resolve_video_path,
silence,
trim_audio,
video_input_to_local_file,
video_data_uris,
write_ass_file,
write_srt_file,
)
from ..utils.prompt_builder import build_llm_prompt, build_prompt_request
# ---------------------------------------------------------------------------
# Resolution combo setup
# ---------------------------------------------------------------------------
class AspectRatio(str, Enum):
SQUARE = "1:1 (Square)"
PHOTO_V = "2:3 (Portrait Photo)"
PHOTO_H = "3:2 (Photo)"
STANDARD_V = "3:4 (Portrait Standard)"
STANDARD_H = "4:3 (Standard)"
WIDESCREEN_V = "9:16 (Portrait Widescreen)"
WIDESCREEN_H = "16:9 (Widescreen)"
ULTRAWIDE_H = "21:9 (Ultrawide)"
ASPECT_RATIOS: dict[AspectRatio, tuple[int, int]] = {
AspectRatio.SQUARE: (1, 1),
AspectRatio.PHOTO_V: (2, 3),
AspectRatio.PHOTO_H: (3, 2),
AspectRatio.STANDARD_V: (3, 4),
AspectRatio.STANDARD_H: (4, 3),
AspectRatio.WIDESCREEN_V: (9, 16),
AspectRatio.WIDESCREEN_H: (16, 9),
AspectRatio.ULTRAWIDE_H: (21, 9),
}
BASE_RESOLUTIONS = [
["width", "height", "auto"],
["width", "height", "shortest"],
["width", "height", "longest"],
["width", "height", "custom"],
["width", "height", "megapixels"],
[480, 832, "9:16"],
[544, 960, "9:16"],
[576, 1024, "9:16"],
[720, 1280, "9:16"],
[768, 1024, "3:4"],
[768, 1344, "9:16"],
[816, 1456, "9:16"],
[817, 1920, "1:2.35"],
[864, 1536, "9:16"],
[1080, 1920, "9:16"],
[1920, 1080, "16:9"],
[1920, 817, "2.35:1"],
[1536, 864, "16:9"],
[1456, 816, "16:9"],
[1344, 768, "16:9"],
[1280, 720, "16:9"],
[1024, 768, "4:3"],
[1024, 576, "16:9"],
[960, 544, "16:9"],
[832, 480, "16:9"],
]
VIDEO_FORMATS = {
'None': {},
'AnimateDiff': {'target_rate': 8, 'dim': (8,0,512,512)},
'Mochi': {'target_rate': 24, 'dim': (16,0,848,480), 'frames':(6,1)},
'LTXV': {'target_rate': 24, 'dim': (32,0,768,512), 'frames':(8,1)},
'Hunyuan': {'target_rate': 24, 'dim': (16,0,848,480), 'frames':(4,1)},
'Cosmos': {'target_rate': 24, 'dim': (16,0,1280,704), 'frames':(8,1)},
'Wan': {'target_rate': 16, 'dim': (8,0,832,480), 'frames':(4,1)},
'MiniMax': {'target_rate': 24, 'dim': (32,0,1344,768), 'frames':(17,5)},
}
resolution_strings = [f"{w} x {h} ({r})" for w, h, r in BASE_RESOLUTIONS]
resize_method_input = io.Combo.Input(
"resize_method",
default="stretch",
options=["stretch", "resize", "pad", "pad (white)", "pad_edge", "pad_edge_pixel", "crop", "pillarbox_blur"],
)
megapixels_input = [
io.Combo.Input(
"aspect_ratio",
options=AspectRatio,
default=AspectRatio.SQUARE,
tooltip="The aspect ratio for the output dimensions.",
),
io.Float.Input(
"megapixels",
default=1.0,
min=0.1,
max=16.0,
step=0.1,
tooltip="Target total megapixels. 1.0 MP ≈ 1024x1024 for square.",
),
]
resolution_combo_options = [
io.DynamicCombo.Option(
s,
[
io.Int.Input("width", default=544, min=64, max=8096, step=8),
io.Int.Input("height", default=960, min=64, max=8096, step=8),
resize_method_input,
]
if "custom" in s
else (
[
io.Int.Input("resize_to_pixel", default=960, min=64, max=8096, step=8),
resize_method_input
]
if "shortest" in s or "longest" in s
else (
megapixels_input if "megapixels" in s
else [resize_method_input]
)
),
)
for s in resolution_strings
]
# ---------------------------------------------------------------------------
# Custom types
# ---------------------------------------------------------------------------
TYPE_TIMELINE = io.Custom(io_type="TIMELINE")
TYPE_TIMELINE_INFO = io.Custom(io_type="TIMELINE_INFO")
TYPE_TRACK_DATA = io.Custom(io_type="TRACK_DATA")
TYPE_TRACKS_INFO = io.Custom(io_type="TRACKS_INFO")
TYPE_LLAMACPP_MODEL = io.Custom(io_type="LLAMACPPMODEL")
TYPE_LLAMACPP_MODEL_CONFIG = io.Custom(io_type="LLAMACPPMODEL_CONFIG")
TYPE_PROMPT_ENHANCER_ACCOUNT = io.Custom(io_type="EASY_API_ACCOUNT")
CATEGORY_MEDIA = "EasyUse/Media"
CATEGORY_TIMELINE = "EasyUse/TimelineEditor"
CATEGORY_MULTITRACK = "EasyUse/MultiTrackEditor"
CATEGORY_AUDIO = "EasyUse/Audio"
CATEGORY_LOGIC = "EasyUse/Logic"
CATEGORY_VIDEO = "EasyUse/Video"
PROMPT_FORMAT_OPTIONS = ["default", "promptRelay"]
LLAMA_CPP_INSTRUCT_NODE_ID = "llama_cpp_instruct_adv"
LLAMA_CPP_IMAGE_LIST_BRIDGE_NODE_ID = "easy multiTrackPromptEnhancerImageListBridge"
STRING_COMPARE_NODE_ID = "StringCompare"
STRING_REPLACE_NODE_ID = "StringReplace"
STRING_TRIM_NODE_ID = "StringTrim"
SWITCH_NODE_ID = "ComfySwitchNode"
LLAMA_CPP_INSTALL_URL = "https://github.com/lihaoyun6/ComfyUI-llama-cpp_vlm"
PROMPT_ENHANCER_RATIO_OPTIONS = [
"adaptive",
"21:9",
"16:9",
"4:3",
"1:1",
"3:4",
"9:16",
]
def _align_video_frame_count(frame_count: int, format_name: str) -> int:
frame_grid = VIDEO_FORMATS.get(format_name, {}).get("frames")
if not frame_grid:
return frame_count
step, remainder = (int(value) for value in frame_grid)
if step <= 0:
return frame_count
return frame_count + (remainder - frame_count) % step
def _nearest_video_frame_count(frame_count: int | float, format_name: str) -> int:
frame_grid = VIDEO_FORMATS.get(format_name, {}).get("frames")
if not frame_grid:
return max(0, math.floor(float(frame_count) + 0.5))
step, remainder = (int(value) for value in frame_grid)
if step <= 0:
return max(0, math.floor(float(frame_count) + 0.5))
normalized_count = max(float(remainder), float(frame_count))
grid_index = math.floor((normalized_count - remainder) / step + 0.5)
return remainder + grid_index * step
def _video_frame_count_from_duration(
duration_frames: int | float,
source_frame_rate: int | float,
format_name: str,
) -> int:
if format_name == "MiniMax":
# Keep the timeline's native FPS; only snap its frame count to 17k+5.
return _nearest_video_frame_count(duration_frames, format_name)
format_info = VIDEO_FORMATS.get(format_name, {})
target_frame_rate = float(format_info.get("target_rate", source_frame_rate))
safe_source_rate = float(source_frame_rate)
if safe_source_rate <= 0 or target_frame_rate <= 0:
target_frames = max(0.0, float(duration_frames))
else:
target_frames = max(0.0, float(duration_frames)) * target_frame_rate / safe_source_rate
return _align_video_frame_count(math.ceil(target_frames), format_name)
def _contains_workflow_metadata(value: object) -> bool:
if isinstance(value, dict):
if "workflow" in value and value["workflow"] is not None:
return True
return any(_contains_workflow_metadata(item) for item in value.values())
if isinstance(value, list):
return any(_contains_workflow_metadata(item) for item in value)
return False
def _is_workflow_format(extra_pnginfo: object) -> bool:
return _contains_workflow_metadata(extra_pnginfo)
# ---------------------------------------------------------------------------
# prompt_override parsing helpers
# ---------------------------------------------------------------------------
_SLOT_ONE_BASED_INDEX_RE = re.compile(r'(?:image|audio|video)(\d+)$', re.IGNORECASE)
_parse_override_segments = parse_override_segments
def _is_valid_audio(audio) -> bool:
if not isinstance(audio, dict):
return False
waveform = audio.get('waveform')
if not isinstance(waveform, torch.Tensor):
return False
try:
return bool(waveform.any())
except (RuntimeError, TypeError, ValueError):
return False
def _single_valid_audio(audio_input) -> 'dict | None':
"""Return the only valid audio dict from input, ignoring empty list items."""
if audio_input is None:
return None
if _is_valid_audio(audio_input):
return audio_input
if not isinstance(audio_input, list):
return None
valid = [
audio
for audio in audio_input
if _is_valid_audio(audio)
]
return valid[0] if len(valid) == 1 else None
def _slot_index(slot_name: str | None) -> int:
if not slot_name:
return 0
slot_text = str(slot_name)
m = _SLOT_ONE_BASED_INDEX_RE.search(slot_text)
if m:
return max(0, int(m.group(1)) - 1)
return 0
def _unwrap_slot_input(value):
if isinstance(value, list) and len(value) == 1 and isinstance(value[0], list):
return value[0]
return value
def _parse_track_data(track_data: str | dict) -> dict:
if isinstance(track_data, str):
try:
parsed = json.loads(track_data)
except json.JSONDecodeError as exc:
raise ValueError("Invalid TRACK_DATA JSON.") from exc
if not isinstance(parsed, dict):
raise ValueError("TRACK_DATA must decode to an object.")
return parsed
if isinstance(track_data, dict):
return dict(track_data)
if track_data is None:
return {}
raise ValueError("TRACK_DATA must be a JSON string or object.")
def _resolve_configured_dimensions(
resolution: str | dict,
format_name: str,
source_dimensions: tuple[int, int] | None = None,
) -> tuple[int, int]:
if isinstance(resolution, dict):
resolution_label = resolution.get("resolution", "")
width_value = resolution.get("width")
height_value = resolution.get("height")
resize_to_pixel_value = resolution.get("resize_to_pixel")
aspect_ratio_value = resolution.get("aspect_ratio")
megapixels_value = resolution.get("megapixels")
else:
resolution_label = resolution
width_value = None
height_value = None
resize_to_pixel_value = None
aspect_ratio_value = None
megapixels_value = None
if isinstance(resolution_label, list):
resolution_label = resolution_label[0] if resolution_label else ""
if isinstance(width_value, list):
width_value = width_value[0] if width_value else None
if isinstance(height_value, list):
height_value = height_value[0] if height_value else None
if isinstance(resize_to_pixel_value, list):
resize_to_pixel_value = resize_to_pixel_value[0] if resize_to_pixel_value else None
if isinstance(aspect_ratio_value, list):
aspect_ratio_value = aspect_ratio_value[0] if aspect_ratio_value else None
if isinstance(megapixels_value, list):
megapixels_value = megapixels_value[0] if megapixels_value else None
resolution_text = str(resolution_label)
normalized_resolution = resolution_text.lower()
divisor = _video_format_dimension_multiple(format_name)
if "megapixels" in normalized_resolution:
width, height = _resolve_megapixel_dimensions(
aspect_ratio_value,
megapixels_value,
divisor,
)
elif "custom" in normalized_resolution:
width = int(width_value) if width_value else 544
height = int(height_value) if height_value else 960
elif ("shortest" in normalized_resolution or "longest" in normalized_resolution) and source_dimensions:
source_width, source_height = source_dimensions
resize_to_pixel = int(resize_to_pixel_value) if resize_to_pixel_value else 960
aspect = source_width / source_height
if "longest" in normalized_resolution:
if source_width >= source_height:
width, height = resize_to_pixel, round(resize_to_pixel / aspect)
else:
width, height = round(resize_to_pixel * aspect), resize_to_pixel
elif source_width <= source_height:
width, height = resize_to_pixel, round(resize_to_pixel / aspect)
else:
width, height = round(resize_to_pixel * aspect), resize_to_pixel
else:
preset = re.search(r"(\d+)\s*x\s*(\d+)", resolution_text)
if preset:
width = int(preset.group(1))
height = int(preset.group(2))
elif "auto" in normalized_resolution and source_dimensions:
width, height = source_dimensions
else:
width, height = 544, 960
if divisor > 1 and "megapixels" not in normalized_resolution:
width = max(divisor, ((width + divisor - 1) // divisor) * divisor)
height = max(divisor, ((height + divisor - 1) // divisor) * divisor)
return width, height
def _video_format_dimension_multiple(format_name: str) -> int:
format_info = VIDEO_FORMATS.get(format_name, {})
return max(1, int(format_info.get("dim", [1])[0]) if format_info else 1)
def _resolve_megapixel_dimensions(
aspect_ratio: object,
megapixels: object,
multiple: int,
) -> tuple[int, int]:
ratio = ASPECT_RATIOS.get(
str(aspect_ratio or AspectRatio.SQUARE.value),
ASPECT_RATIOS[AspectRatio.SQUARE],
)
megapixel_value = float(megapixels) if megapixels else 1.0
total_pixels = megapixel_value * 1024 * 1024
scale = math.sqrt(total_pixels / (ratio[0] * ratio[1]))
width = round(ratio[0] * scale / multiple) * multiple
height = round(ratio[1] * scale / multiple) * multiple
return width, height
def _configured_resize_method(resolution: str | dict) -> str:
if not isinstance(resolution, dict):
return "stretch"
resize_method = resolution.get("resize_method", "stretch")
if isinstance(resize_method, list):
resize_method = resize_method[0] if resize_method else "stretch"
return str(resize_method)
def _as_list_input(value) -> list:
if value is None:
return []
if isinstance(value, list):
if len(value) == 1 and isinstance(value[0], list):
return value[0]
return value
return [value]
def _media_output_for_index(items: list, index: int):
if index < 0 or index >= len(items):
return None
return items[index]
def _index_slot_video(video_input, slot_name: str | None):
items = _as_list_input(video_input)
return _media_output_for_index(items, _slot_index(slot_name))
def _resolve_multitrack_video(content: dict, video_input):
source_type = str(content.get("source_type", "input"))
if source_type == "preset":
return None
if source_type == "slot":
return _index_slot_video(video_input, content.get("slot_name") or content.get("file_name"))
source = resolve_video_path(
source_type,
content.get("file_path"),
content.get("local_path"),
content.get("url"),
)
return InputImpl.VideoFromFile(source)
def _resolve_multitrack_audio(content: dict, audio_input, sample_rate: int = 44100) -> 'dict | None':
if content.get("source_type") == "slot":
return _index_slot_audio(audio_input, content.get("slot_name") or content.get("file_name"))
waveform = load_audio_waveform(
content.get("source_type", "input"),
content.get("file_path"),
content.get("local_path"),
content.get("url"),
sample_rate,
)
if waveform is None:
return None
return {"waveform": waveform, "sample_rate": sample_rate}
def _video_resize_cache_key(video, width: int, height: int, resize_method: str) -> tuple:
source = _video_stream_source(video)
identity = ("source", source) if isinstance(source, str) else ("object", id(video))
return identity, width, height, resize_method
def _resize_multitrack_video(
video,
width: int,
height: int,
resize_method: str,
cache: dict[tuple, object],
progress_callback,
):
if tuple(video.get_dimensions()) == (width, height):
progress_callback(1.0)
return video
cache_key = _video_resize_cache_key(video, width, height, resize_method)
cached = cache.get(cache_key)
if cached is not None:
progress_callback(1.0)
return cached
source = cache_key[0][1] if cache_key[0][0] == "source" else None
if resize_method in FFMPEG_RESIZE_METHODS and isinstance(source, str):
resized_path = resize_video_with_ffmpeg(
source,
width,
height,
resize_method,
progress_callback=progress_callback,
)
if resized_path is not None:
resized_video = InputImpl.VideoFromFile(resized_path)
cache[cache_key] = resized_video
return resized_video
components = video.get_components()
progress_callback(0.5)
resized_frames = resize_image(components.images, width, height, resize_method)
resized_video = InputImpl.VideoFromComponents(
Types.VideoComponents(
images=resized_frames,
audio=components.audio,
frame_rate=components.frame_rate,
)
)
cache[cache_key] = resized_video
progress_callback(1.0)
return resized_video
def _resample_audio_waveform(
waveform: torch.Tensor,
source_rate: int,
target_rate: int,
) -> torch.Tensor:
if source_rate == target_rate:
return waveform
target_samples = max(1, round(waveform.shape[-1] * target_rate / source_rate))
return F.interpolate(waveform, size=target_samples, mode="linear", align_corners=False)
def _merge_audio_track(
segments: list[tuple[dict, dict]],
total_length: int,
frame_rate: float,
base_volume_db: float = 0.0,
muted: bool = False,
) -> dict:
sample_rate = 44100
channels = 2
for _segment, audio in segments:
waveform = audio.get("waveform")
if isinstance(waveform, torch.Tensor):
sample_rate = int(audio.get("sample_rate", sample_rate))
channels = int(waveform.shape[1])
break
total_samples = max(1, round(total_length * sample_rate / frame_rate))
merged = torch.zeros(1, channels, total_samples)
if muted:
return {"waveform": merged, "sample_rate": sample_rate}
for segment, audio in sorted(segments, key=lambda item: int(item[0].get("start_frame", 0))):
waveform = audio.get("waveform")
if not isinstance(waveform, torch.Tensor):
continue
source_rate = int(audio.get("sample_rate", sample_rate))
waveform = _resample_audio_waveform(waveform, source_rate, sample_rate)
if waveform.shape[1] == 1 and channels > 1:
waveform = waveform.expand(-1, channels, -1)
elif waveform.shape[1] != channels:
waveform = waveform[:, :channels]
content = segment.get("content", {})
if not isinstance(content, dict):
content = {}
if audio_is_muted(content):
continue
gain = audio_db_to_gain(base_volume_db + audio_volume_db(content))
start_frame = max(0, int(segment.get("start_frame", 0)))
end_frame = min(total_length, max(start_frame, int(segment.get("end_frame", start_frame))))
start_sample = round(start_frame * sample_rate / frame_rate)
segment_samples = max(0, round((end_frame - start_frame) * sample_rate / frame_rate))
origin_start = int(segment.get("origin_start_frame", start_frame))
source_start_sample = max(0, round((start_frame - origin_start) * sample_rate / frame_rate))
copy_samples = min(
segment_samples,
max(0, waveform.shape[-1] - source_start_sample),
total_samples - start_sample,
)
if copy_samples > 0:
merged[..., start_sample:start_sample + copy_samples] = (
waveform[..., source_start_sample:source_start_sample + copy_samples] * gain
)
return {"waveform": merged, "sample_rate": sample_rate}
def _video_stream_source(video) -> 'str | None':
try:
trim_start, trim_duration = video.get_active_trim_window()
if float(trim_start) != 0.0 or float(trim_duration) != 0.0:
return None
except (AttributeError, NotImplementedError, RuntimeError, TypeError, ValueError):
pass
try:
source = video.get_stream_source()
except (AttributeError, NotImplementedError, RuntimeError, ValueError):
return None
return source if isinstance(source, str) else None
def _merge_video_track_tensor(
segments: list[tuple[dict, object]],
total_length: int,
frame_rate: float,
width: int,
height: int,
base_volume_db: float = 0.0,
audio_muted: bool = False,
):
merged_frames = torch.zeros(total_length, height, width, 3)
embedded_audio_segments: list[tuple[dict, dict]] = []
components_cache: dict[int, object] = {}
for segment, video in sorted(segments, key=lambda item: int(item[0].get("start_frame", 0))):
components = components_cache.get(id(video))
if components is None:
components = video.get_components()
components_cache[id(video)] = components
frames = components.images
start_frame = max(0, int(segment.get("start_frame", 0)))
end_frame = min(total_length, max(start_frame, int(segment.get("end_frame", start_frame))))
segment_frames = end_frame - start_frame
source_rate = float(components.frame_rate)
origin_start = int(segment.get("origin_start_frame", start_frame))
source_start_frame = max(0, math.floor((start_frame - origin_start) * source_rate / frame_rate))
available_frames = (
min(
segment_frames,
max(0, int((frames.shape[0] - source_start_frame) * frame_rate / source_rate)),
)
if frames.shape[0] > 0 and source_rate > 0
else 0
)
if available_frames > 0:
indices = source_start_frame + torch.floor(
torch.arange(available_frames, device=frames.device) * source_rate / frame_rate
).long().clamp(max=frames.shape[0] - 1)
merged_frames[start_frame:start_frame + available_frames] = frames[indices].cpu()
if isinstance(components.audio, dict):
embedded_audio_segments.append((segment, components.audio))
merged_audio = (
_merge_audio_track(
embedded_audio_segments,
total_length,
frame_rate,
base_volume_db,
audio_muted,
)
if embedded_audio_segments
else None
)
return InputImpl.VideoFromComponents(
Types.VideoComponents(
images=merged_frames,
audio=merged_audio,
frame_rate=frame_rate,
)
)
def _merge_video_track(
segments: list[tuple[dict, object]],
total_length: int,
frame_rate: float,
width: int,
height: int,
base_volume_db: float = 0.0,
audio_muted: bool = False,
):
file_segments: list[dict] = []
for segment, video in segments:
source = _video_stream_source(video)
if source is None:
break
content = segment.get("content", {})
if not isinstance(content, dict):
content = {}
file_segment = {
"source": source,
"start_frame": int(segment.get("start_frame", 0)),
"end_frame": int(segment.get("end_frame", 0)),
"audio_volume_db": base_volume_db + audio_volume_db(content),
"audio_muted": audio_muted or audio_is_muted(content),
}
origin_start = int(segment.get("origin_start_frame", file_segment["start_frame"]))
source_start_frame = max(0, file_segment["start_frame"] - origin_start)
if source_start_frame > 0:
file_segment["source_start_frame"] = source_start_frame
file_segments.append(file_segment)
else:
merged_path = merge_video_track_with_ffmpeg(
file_segments,
total_length,
frame_rate,
width,
height,
)
if merged_path is not None:
return InputImpl.VideoFromFile(merged_path)
return _merge_video_track_tensor(
segments,
total_length,
frame_rate,
width,
height,
base_volume_db,
audio_muted,
)
def _build_tracks_info_and_media_outputs(
data: dict,
image_input,
audio_input,
video_input,
resolution: str | dict,
format_name: str,
) -> tuple[dict, list, list, list]:
tracks = data.get("tracks", [])
if not isinstance(tracks, list):
raise ValueError("TRACK_DATA.tracks must be a list.")
frame_rate = float(data.get("frame_rate", 24.0))
total_length_is_final = data.get("_total_length_is_final") is True
total_length = int(data.get("total_length", 0))
segment_timeline_end = max(
(
max(0, int(segment.get("end_frame", 0)))
for track in tracks
if isinstance(track, dict)
for segment in track.get("segments", [])
if isinstance(segment, dict)
),
default=0,
)
task_duration_length = sum(
max(
0,
int(segment.get("end_frame", 0)) - int(segment.get("start_frame", 0)),
)
for track in tracks
if isinstance(track, dict) and track.get("type") == "task"
for segment in track.get("segments", [])
if isinstance(segment, dict)
)
if segment_timeline_end > 0:
timeline_total_length = segment_timeline_end
elif format_name == "MiniMax":
timeline_total_length = (
max(0, total_length - 1) if total_length_is_final else total_length
)
else:
timeline_total_length = total_length
effective_total_length = task_duration_length or timeline_total_length
if format_name == "MiniMax":
output_total_length = _video_frame_count_from_duration(
effective_total_length,
frame_rate,
format_name,
)
elif task_duration_length > 0 or segment_timeline_end > 0:
output_total_length = effective_total_length + 1
else:
output_total_length = total_length if total_length_is_final else total_length + 1
global_volume_db = audio_volume_db(data)
global_muted = audio_is_muted(data)
has_solo_track = any(
isinstance(track, dict) and
track.get("type") in ("video", "audio") and
track.get("solo") is True
for track in tracks
)
images_out: list[torch.Tensor] = []
audio_out: list[dict] = []
video_out: list = []
video_segments: list[tuple[int, int, dict]] = []
for track_index, track in enumerate(tracks):
if not isinstance(track, dict) or track.get("type") != "video":
continue
for segment_index, segment in enumerate(track.get("segments", [])):
if not isinstance(segment, dict):
continue
content = segment.get("content", {})
if isinstance(content, dict) and content.get("media_type") == "video":
video_segments.append((track_index, segment_index, content))
progress = ProgressBar(max(1, len(video_segments) * 3)) if video_segments else None
progress_value = 0
if progress is not None:
progress.update_absolute(0)
resolved_videos: dict[tuple[int, int], object] = {}
for track_index, segment_index, content in video_segments:
video = _resolve_multitrack_video(content, video_input)
if video is not None:
resolved_videos[(track_index, segment_index)] = video
progress_value += 1
if progress is not None:
progress.update_absolute(progress_value)
first_video = next(iter(resolved_videos.values()), None)
source_dimensions = first_video.get_dimensions() if first_video is not None else None
width, height = _resolve_configured_dimensions(resolution, format_name, source_dimensions)
resize_method = _configured_resize_method(resolution)
resized_video_cache: dict[tuple, object] = {}
normalized_tracks: list[dict] = []
for track_index, track in enumerate(tracks):
if not isinstance(track, dict):
continue
track_type = track.get("type")
track_volume_db = global_volume_db + audio_volume_db(track)
track_muted = (
global_muted or
audio_is_muted(track) or
(has_solo_track and track.get("solo") is not True)
)
normalized_segments: list[dict] = []
track_audio_segments: list[tuple[dict, dict]] = []
track_video_segments: list[tuple[dict, object]] = []
for segment_index, segment in enumerate(track.get("segments", [])):
if not isinstance(segment, dict):
continue
if track_type == "subtitle" and track.get("visible") is False:
continue
content = segment.get("content", {})
if not isinstance(content, dict):
content = {}
normalized_content = dict(content)
normalized_content.pop("volume", None)
if track_type == "task":
normalized_images: list[dict] = []
raw_images = content.get("images", [])
if isinstance(raw_images, list):
for image_item in raw_images:
if not isinstance(image_item, dict):
continue
normalized_image = dict(image_item)
image = _resolve_timeline_image_item(image_item, image_input)
if image is not None:
panorama_view = image_item.get("panorama_view")
if panorama_view is not None:
try:
image = equirectangular_to_perspective(
image,
panorama_view,
width,
height,
)
except (TypeError, ValueError, RuntimeError) as exc:
image_id = image_item.get("id", "")
raise ValueError(
f"Failed to project panorama image {image_id!r}: {exc}"
) from exc
media_index = len(images_out)
images_out.append(image)
normalized_image["media_index"] = media_index
normalized_images.append(normalized_image)
normalized_content["images"] = normalized_images
elif track_type == "audio" and content.get("media_type") == "audio":
audio = _resolve_multitrack_audio(content, audio_input)
if audio is not None:
track_audio_segments.append((segment, audio))
elif track_type == "video" and content.get("media_type") == "video":
video = resolved_videos.get((track_index, segment_index))
if video is not None:
progress_start = progress_value
def update_video_progress(ratio: float) -> None:
if progress is not None:
progress.update_absolute(progress_start + min(1.0, max(0.0, ratio)) * 2)
rebuilt_video = _resize_multitrack_video(
video,
width,
height,
resize_method,
resized_video_cache,
update_video_progress,
)
progress_value = progress_start + 2
if progress is not None:
progress.update_absolute(progress_value)
track_video_segments.append((segment, rebuilt_video))
normalized_segment = dict(segment)
normalized_segment.pop("volume", None)
normalized_segment["content"] = normalized_content
normalized_segments.append(normalized_segment)
normalized_track = dict(track)
normalized_track.pop("volume", None)
normalized_track["segments"] = normalized_segments
track_total_length = timeline_total_length
if format_name == "MiniMax" and track_type in ("audio", "video"):
track_end_frame = _track_media_end_frame(normalized_track)
if track_end_frame is not None:
track_total_length = max(0, track_end_frame)
if track_type == "audio" and (format_name != "MiniMax" or track_audio_segments):
media_index = len(audio_out)
audio_out.append(_merge_audio_track(
track_audio_segments,
track_total_length,
frame_rate,
track_volume_db,
track_muted,
))
normalized_track["media_index"] = media_index
for normalized_segment in normalized_segments:
content = normalized_segment.get("content", {})
if content.get("media_type") == "audio":
content["media_index"] = media_index
elif track_type == "video" and (format_name != "MiniMax" or track_video_segments):
media_index = len(video_out)
video_out.append(
_merge_video_track(
track_video_segments,
track_total_length,
frame_rate,
width,
height,
track_volume_db,
track_muted,
)
)
normalized_track["media_index"] = media_index
for normalized_segment in normalized_segments:
content = normalized_segment.get("content", {})
if content.get("media_type") == "video":
content["media_index"] = media_index
normalized_tracks.append(normalized_track)
if progress is not None and progress_value < progress.total:
progress.update_absolute(progress.total)
tracks_info = {
# UI track data stores an exclusive timeline end, while prompt_override
# data has already normalized total_length to the final output value.
"total_length": output_total_length,
"timeline_total_length": timeline_total_length,
"frame_rate": frame_rate,
"target_frame_rate": frame_rate,
"format": format_name,
"muted": global_muted,
"volume_db": global_volume_db,
"width": width,
"height": height,
"task_markers": [
dict(marker)
for marker in data.get("task_markers", [])
if isinstance(marker, dict)
] if isinstance(data.get("task_markers", []), list) else [],
"tracks": normalized_tracks,
}
audio_result = (audio_out or [None]) if format_name == "MiniMax" else audio_out
video_result = (video_out or [None]) if format_name == "MiniMax" else video_out
return (
tracks_info,
images_out,
audio_result,
video_result,
)
def _index_slot_image(image_input, slot_name: str | None) -> 'torch.Tensor | None':
idx = _slot_index(slot_name)
image_input = _unwrap_slot_input(image_input)
if image_input is None:
return None
candidates = image_input if isinstance(image_input, list) else [image_input]
flattened: list[torch.Tensor] = []
for candidate in candidates:
if not isinstance(candidate, torch.Tensor):
continue
tensor = _normalize_image_tensor(candidate)
if tensor is None:
continue
if _is_empty_slot_image(tensor):
continue
flattened.extend(tensor[i:i + 1] for i in range(tensor.shape[0]))
return flattened[idx] if idx < len(flattened) else None
def _normalize_image_tensor(tensor: torch.Tensor) -> 'torch.Tensor | None':
if tensor.dim() == 3:
if tensor.shape[0] in (1, 3, 4) and tensor.shape[-1] not in (1, 3, 4):
tensor = tensor.permute(1, 2, 0)
tensor = tensor.unsqueeze(0)
elif tensor.dim() == 4:
if tensor.shape[1] in (1, 3, 4) and tensor.shape[-1] not in (1, 3, 4):
tensor = tensor.permute(0, 2, 3, 1)
else:
return None
return tensor
def _is_empty_slot_image(tensor: torch.Tensor) -> bool:
if tensor.dim() == 3:
return tensor.shape[0] == 1 and tensor.shape[1] == 1
if tensor.dim() == 4:
return tensor.shape[1] == 1 and tensor.shape[2] == 1
return False
def _index_slot_audio(audio_input, slot_name: str | None) -> 'dict | None':
idx = _slot_index(slot_name)
audio_input = _unwrap_slot_input(audio_input)
if audio_input is None:
return None
if isinstance(audio_input, list):
if idx < len(audio_input):
audio = audio_input[idx]
return audio if isinstance(audio, dict) and 'waveform' in audio else None
return None
if isinstance(audio_input, dict) and 'waveform' in audio_input and idx == 0:
return audio_input
return None
def _resolve_timeline_image_item(item: dict, image_input, image_loader=load_image_tensor) -> 'torch.Tensor | None':
if item.get("source_type") == "slot":
return _index_slot_image(image_input, item.get("slot_name") or item.get("file_name"))
return image_loader(
item.get("source_type", "input"),
item.get("file_path"),
item.get("local_path"),
item.get("url"),
)
def _sort_timeline_images(images: list[dict]) -> list[dict]:
return sorted(
images,
key=lambda item: int(item.get("start_frame", 0) or 0),
)
def _collect_timeline_image_items(maintain_segs: list[dict]) -> list[dict]:
all_image_items: list[dict] = []
for seg in maintain_segs:
all_image_items.extend(_sort_timeline_images(seg.get("images", [])))
return all_image_items
def _select_dimension_image_item(image_items: list[dict]) -> 'dict | None':
for item in image_items:
if item.get("source_type") != "slot":
return item
return image_items[0] if image_items else None
def _count_images(image_input) -> int:
"""Return the number of images in image_input."""
if image_input is None:
return 0
if isinstance(image_input, list):
return len(image_input)
if isinstance(image_input, torch.Tensor):
return image_input.shape[0] if image_input.dim() == 4 else (1 if image_input.dim() == 3 else 0)
return 0
def _index_image(image_input, idx_one_based: int) -> 'torch.Tensor | None':
"""Return a [1, H, W, C] tensor for the 1-based image index, or None."""
i = idx_one_based - 1
if image_input is None:
return None
if isinstance(image_input, list):
if i < len(image_input):
t = image_input[i]
if isinstance(t, torch.Tensor):
return t if t.dim() == 4 else t.unsqueeze(0)
return None
if isinstance(image_input, torch.Tensor):
if image_input.dim() == 4 and i < image_input.shape[0]:
return image_input[i : i + 1]
if image_input.dim() == 3 and i == 0:
return image_input.unsqueeze(0)
return None
def _index_audio(audio_input, idx_one_based: int) -> 'dict | None':
"""Return the audio dict for the 1-based index, or None."""
i = idx_one_based - 1
if audio_input is None:
return None
if isinstance(audio_input, list):
if i < len(audio_input):
a = audio_input[i]
return a if isinstance(a, dict) and 'waveform' in a else None
return None
if isinstance(audio_input, dict) and 'waveform' in audio_input:
return audio_input if i == 0 else None
return None
def _merge_audio_batches(audio_input) -> 'dict | None':
"""With is_input_list=True, a single audio source is split into N batch items.
Concatenate all items along the time axis to reconstruct the full audio."""
if audio_input is None:
return None
if isinstance(audio_input, dict) and 'waveform' in audio_input:
return audio_input # already a single audio dict
if not isinstance(audio_input, list) or not audio_input:
return None
valid = [a for a in audio_input if isinstance(a, dict) and 'waveform' in a
and isinstance(a['waveform'], torch.Tensor)]
if not valid:
return None
_raw_sr = valid[0].get('sample_rate', 44100)
sr = int(_raw_sr[0] if isinstance(_raw_sr, (list, tuple)) else _raw_sr)
waveforms = [a['waveform'] for a in valid] # each [1, C, T_i]
# Normalize channel count: up-mix mono to stereo if mixed
max_ch = max(w.shape[1] for w in waveforms)
if max_ch > 1:
waveforms = [w.expand(-1, max_ch, -1) if w.shape[1] < max_ch else w for w in waveforms]
combined = torch.cat(waveforms, dim=-1) # [1, C, sum(T_i)]
return {'waveform': combined, 'sample_rate': sr}
# ---------------------------------------------------------------------------
# Node
# ---------------------------------------------------------------------------
class TimelineEditor(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy timelineEditor",
display_name="Timeline Editor",
category=CATEGORY_TIMELINE,
description="Load a timeline of media items (prompt, image, audio tracks) and outputs structured data.",
is_input_list=True,
inputs=[
io.DynamicCombo.Input(
"resolution",
options=resolution_combo_options,
tooltip="Select a resolution or choose 'Custom' to specify your own width and height.",
),
io.Combo.Input("format", options=list(VIDEO_FORMATS.keys()), default="LTXV", tooltip="Choose a video format to automatically set resolution and frame rate."),
TYPE_TIMELINE.Input(
"timeline_data",
),
io.AnyType.Input("prompt_override", optional=True, tooltip="If provided, overrides all segment prompts in the timeline.",),
io.Image.Input("image", optional=True, tooltip="List of images to override images in the timeline."),
io.Audio.Input("audio", optional=True, tooltip="List of audio clips to override audio in the timeline."),
],
outputs=[
TYPE_TIMELINE_INFO.Output("TIMELINE_INFO"),
io.Image.Output("IMAGES"),
io.Audio.Output("AUDIO"),
],
)
@classmethod
def execute(
cls,
resolution: str | dict,
format: str,
timeline_data: str | dict,
**kwargs: object,
) -> io.NodeOutput:
# is_input_list=True: every param arrives as a list; unwrap scalars here
if isinstance(resolution, list):
resolution = resolution[0]
if isinstance(format, list):
format = format[0]
if isinstance(timeline_data, list):
timeline_data = timeline_data[0]
prompt_override = kwargs.get('prompt_override')
if isinstance(prompt_override, list) and len(prompt_override) == 1:
prompt_override = prompt_override[0]
image_input = kwargs.get('image') # kept as list
audio_input = kwargs.get('audio') # kept as list
# Unwrap double-wrapped list from is_input_list (list of audio lists)
# When audio comes from MakeAudioList (is_output_list=True), it's already a list.
# With is_input_list=True, that list gets wrapped again → [[audio1, audio2, ...]]
# We need to unwrap to get the original list of audio dicts.
if isinstance(audio_input, list) and len(audio_input) == 1:
inner = audio_input[0]
if isinstance(inner, list):
audio_input = inner
# Segment parsing override: only needs non-empty prompt_override
use_prompt_override = prompt_override_has_value(prompt_override)
# Audio override: prompt_override + audio (image is NOT required)
audio_override = (
use_prompt_override
and audio_input is not None
and (not isinstance(audio_input, list) or len(audio_input) > 0)
)
# Keep use_override as alias for image-loading context (prompt_override active)
use_override = use_prompt_override
# ---- Parse data source ----
if use_prompt_override:
# Still read frame_rate from timeline_data metadata if available
if isinstance(timeline_data, str):
try:
_td = json.loads(timeline_data)
except json.JSONDecodeError:
_td = {}
else:
_td = dict(timeline_data) if timeline_data else {}
frame_rate: int = int(_td.get('frame_rate', 24))
total_length = int(_td.get('total_length', 121))
override_segs = _parse_override_segments(
prompt_override,
total_length=total_length,
frame_rate=frame_rate,
)
if prompt_override_has_frame_ranges(prompt_override):
max_override_end = max((s['end_frame'] for s in override_segs), default=120)
total_length = max_override_end + 1
# Build maintain_segs — images stored as slot refs with _tensor_idx
maintain_segs: list[dict] = []
for s in override_segs:
n_img = len(s['image_indices'])
seg_start = s['start_frame']
seg_end = s['end_frame']
seg_duration = seg_end - seg_start
images: list[dict] = []
for i, idx_1based in enumerate(s['image_indices']):
img_entry: dict = {
'source_type': 'slot',
'file_name': f'image_{idx_1based}',
'_tensor_idx': idx_1based,
}
if n_img > 1:
img_entry['start_frame'] = round(seg_start + i * seg_duration / n_img)
img_entry['end_frame'] = round(seg_start + (i + 1) * seg_duration / n_img)
images.append(img_entry)
maintain_segs.append({
'start_frame': seg_start,
'end_frame': seg_end,
'text': s['text'],
'images': images,
'type': s['type'],
'_audio_indices': s['audio_indices'],
})
tracks: list = [] # not used in override path; defined for audio else-branch
else:
# ---- Normal path: Parse timeline_data ----
if isinstance(timeline_data, str):
try:
data = json.loads(timeline_data)
except json.JSONDecodeError:
data = {}
else:
data = dict(timeline_data) if timeline_data else {}
tracks = data.get("tracks", [])
total_length: int = int(data.get("total_length", 121))
frame_rate: int = int(data.get("frame_rate", 24))
# =========================================================
# Collect maintain (main) track segments
# =========================================================
maintain_segs: list[dict] = []
for track in tracks:
if track.get("type") != "maintain":
continue
for seg in sorted(track.get("segments", []), key=lambda s: s.get("start_frame", 0)):
content = seg.get("content", {})
maintain_segs.append({
"start_frame": int(seg.get("start_frame", 0)),
"end_frame": int(seg.get("end_frame", 0)),
"text": content.get("text", ""),
"images": _sort_timeline_images(content.get("images", [])), # list of ImageItem dicts
"type": content.get("type", "flf"),
})
if format == "MiniMax":
output_total_length = _video_frame_count_from_duration(
max(0, total_length - 1),
frame_rate,
format,
)
else:
output_total_length = _align_video_frame_count(total_length, format)
# Flat list of all image items from maintain segments, in order
all_image_items = _collect_timeline_image_items(maintain_segs)
# =========================================================
# Resolve target dimensions
# =========================================================
def _unwrap(v, default=None):
"""If DynamicCombo sub-value is wrapped as a list (is_input_list side-effect), unwrap it."""
if isinstance(v, list):
return v[0] if v else default
return v if v is not None else default
_resolution: str = _unwrap(resolution.get("resolution"), "")
resize_method: str = _unwrap(resolution.get("resize_method"), "stretch")
resize_to_pixel: int | None = _unwrap(resolution.get("resize_to_pixel"), None)
width_custom: int | None = _unwrap(resolution.get("width"), None)
height_custom: int | None = _unwrap(resolution.get("height"), None)
aspect_ratio: str = str(_unwrap(resolution.get("aspect_ratio"), AspectRatio.SQUARE.value))
megapixels: float = float(_unwrap(resolution.get("megapixels"), 1.0))
# Detect mode from resolution string
if "auto" in _resolution:
mode = "auto"
elif "longest" in _resolution:
mode = "longest"
elif "shortest" in _resolution:
mode = "shortest"
elif "custom" in _resolution:
mode = "custom"
elif "megapixels" in _resolution:
mode = "megapixels"
else:
mode = "preset"
# image_override: True whenever image input is connected, regardless of full override mode
image_override = (
image_input is not None
and (not isinstance(image_input, list) or len(image_input) > 0)
)
# Load one image for dimension inference (auto / longest / shortest)
dimension_image_tensor: torch.Tensor | None = None
if mode in ("auto", "longest", "shortest"):
if use_override and image_override:
dimension_image_tensor = _index_image(image_input, 1)
elif all_image_items:
dimension_item = _select_dimension_image_item(all_image_items)
if dimension_item is not None:
dimension_image_tensor = _resolve_timeline_image_item(dimension_item, image_input)
target_w: int
target_h: int
div = _video_format_dimension_multiple(format)
if mode == "megapixels":
target_w, target_h = _resolve_megapixel_dimensions(aspect_ratio, megapixels, div)
elif mode == "preset":
target_w, target_h = 544, 960
for entry in BASE_RESOLUTIONS:
w, h = entry[0], entry[1]
if isinstance(w, int) and f"{w} x {h}" in _resolution:
target_w, target_h = int(w), int(h)
break
elif mode == "auto":
if dimension_image_tensor is not None:
target_h = dimension_image_tensor.shape[1]
target_w = dimension_image_tensor.shape[2]
else:
target_w, target_h = 544, 960
elif mode in ("longest", "shortest"):
if dimension_image_tensor is not None:
img_h = dimension_image_tensor.shape[1]
img_w = dimension_image_tensor.shape[2]
pix = int(resize_to_pixel) if resize_to_pixel else 960
aspect = img_w / img_h # width / height
if mode == "longest":
if img_w >= img_h:
target_w = pix
target_h = round(pix / aspect)
else:
target_h = pix
target_w = round(pix * aspect)
else: # shortest
if img_w <= img_h:
target_w = pix
target_h = round(pix / aspect)
else:
target_h = pix
target_w = round(pix * aspect)
else:
target_w, target_h = 544, 960
else: # custom
target_w = int(width_custom) if width_custom else 544
target_h = int(height_custom) if height_custom else 960
# Apply format divisibility to finalise target dimensions
if div > 1 and mode != "megapixels":
target_w = max(div, ((target_w + div - 1) // div) * div)
target_h = max(div, ((target_h + div - 1) // div) * div)
# =========================================================
# Load and resize images from maintain segments
# =========================================================
image_tensors: list[torch.Tensor] = []
for idx, item in enumerate(all_image_items):
if use_override and image_override:
# Use connected image input (positional for normal path, _tensor_idx for override)
tensor_idx = item.get('_tensor_idx', idx + 1)
if idx == 0 and dimension_image_tensor is not None:
t = dimension_image_tensor
else:
t = _index_image(image_input, tensor_idx)
else:
t = _resolve_timeline_image_item(item, image_input)
if t is None:
continue
t = resize_image(t, target_w, target_h, resize_method)
# Normalize to RGB (3 channels) — drop alpha channel if present
if t.shape[-1] == 1:
t = t.expand(-1, -1, -1, 3)
elif t.shape[-1] == 4:
t = t[..., :3]
elif t.shape[-1] != 3:
continue
image_tensors.append(t)
if image_tensors:
images_out = torch.cat(image_tensors, dim=0)
else:
images_out = torch.zeros(1, target_h, target_w, 3)
# =========================================================
# Audio track processing
# =========================================================
default_sr = 44100
merged_waveform: torch.Tensor | None = None
# ---- Single audio as whole timeline: clip/pad to total duration ----
single_timeline_audio = _single_valid_audio(audio_input) if prompt_override else None
if prompt_override and prompt_override != '' and "@audio" not in prompt_override and "@音频" not in prompt_override and single_timeline_audio is not None:
a = single_timeline_audio
channels = a['waveform'].shape[1] if 'waveform' in a else 2
_raw_sr = a.get('sample_rate', default_sr)
sr = int(_raw_sr[0] if isinstance(_raw_sr, (list, tuple)) else _raw_sr)
if sr != default_sr:
default_sr = sr
total_sec = (total_length - 1) / frame_rate
wav = a['waveform'][0] # [C, T]
target_samples = max(1, int(total_sec * sr))
chunk = wav[:, :target_samples]
if chunk.shape[-1] < target_samples:
chunk = torch.cat([
chunk,
torch.zeros(channels, target_samples - chunk.shape[-1],
dtype=chunk.dtype, device=chunk.device)
], dim=-1)
merged_waveform = chunk.unsqueeze(0)
elif audio_override:
# ---- Override audio: build from audio input per segment ----
# audio_input is a list from MakeAudioList (is_output_list) where
# index N-1 corresponds to @audioN reference in prompt_override.
# Detect channel count from first non-silent real audio clip.
channels = 2
for _probe_idx in range(1, 11):
_probe = _index_audio(audio_input, _probe_idx)
if _probe is not None and _probe['waveform'].any():
channels = _probe['waveform'].shape[1]
_raw_sr = _probe.get('sample_rate', default_sr)
default_sr = int(_raw_sr[0] if isinstance(_raw_sr, (list, tuple)) else _raw_sr)
break
def _extract_clip(a: dict, duration_sec: float) -> torch.Tensor:
"""Extract from the beginning of audio clip `a`, clip/pad to duration_sec. Returns [C, T]."""
_raw = a.get('sample_rate', default_sr)
sr = int(_raw[0] if isinstance(_raw, (list, tuple)) else _raw)
wav = a['waveform'][0] # [C, T]
# Up-mix mono to stereo if needed
if wav.shape[0] < channels:
wav = wav.expand(channels, -1)
n_ch = wav.shape[0]
target_samples = max(1, int(duration_sec * sr))
chunk = wav[:, :target_samples]
if chunk.shape[-1] < target_samples:
chunk = torch.cat(
[chunk, torch.zeros(n_ch, target_samples - chunk.shape[-1],
dtype=chunk.dtype, device=chunk.device)],
dim=-1,
)
return chunk[:, :target_samples]
audio_parts: list[torch.Tensor] = []
prev_end_sec = 0.0
for seg in maintain_segs:
start_sec = seg['start_frame'] / frame_rate
end_sec = seg['end_frame'] / frame_rate
duration_sec = max(0.0, end_sec - start_sec)
# Gap silence before this segment
if start_sec > prev_end_sec + 1e-6:
audio_parts.append(silence(default_sr, start_sec - prev_end_sec, channels))
audio_indices = seg.get('_audio_indices', [])
if audio_indices:
# @audioN present → try list indexing first, fall back to single audio
a = _index_audio(audio_input, audio_indices[0])
if a is None and not isinstance(audio_input, list):
# Single audio input (not a list) — use it directly
a = audio_input
if a is not None:
chunk = _extract_clip(a, duration_sec)
audio_parts.append(chunk.unsqueeze(0))
else:
audio_parts.append(silence(default_sr, duration_sec, channels))
else:
# No @audio reference → mute this segment
audio_parts.append(silence(default_sr, duration_sec, channels))
prev_end_sec = end_sec
if audio_parts:
merged_waveform = torch.cat(audio_parts, dim=-1)
else:
# ---- Normal path: audio from timeline tracks ----
for track in tracks:
if track.get("type") != "audio":
continue
track_parts: list[torch.Tensor] = []
prev_end_sec = 0.0
channels = 2
for seg in sorted(track.get("segments", []), key=lambda s: s.get("start_frame", 0)):
start = int(seg.get("start_frame", 0))
end = min(int(seg.get("end_frame", 0)), total_length - 1)
start_sec = max(0.0, frames_to_seconds(start, frame_rate))
end_sec = frames_to_seconds(end, frame_rate)
duration_sec = max(0.0, end_sec - start_sec)
# Trim offset: how far into the source audio this segment starts
origin_start = int(seg.get("origin_start_frame", start))
# Use plain frame-count division (not frames_to_seconds which applies a -1 offset for indices)
trim_offset_sec = max(0.0, (start - origin_start) / frame_rate) if start > origin_start else 0.0
content = seg.get("content", {})
slot_audio = None
if content.get("source_type") == "slot":
slot_audio = _index_slot_audio(audio_input, content.get("slot_name") or content.get("file_name"))
waveform = (
slot_audio.get("waveform")
if slot_audio is not None
else load_audio_waveform(
content.get("source_type", "input"),
content.get("file_path"),
content.get("local_path"),
content.get("url"),
default_sr,
)
)
if slot_audio is not None:
_raw_sr = slot_audio.get('sample_rate', default_sr)
default_sr = int(_raw_sr[0] if isinstance(_raw_sr, (list, tuple)) else _raw_sr)
# Determine channel count from loaded audio before adding gap silence,
# so the silence tensor has matching channels and torch.cat won't fail.
if waveform is not None:
channels = waveform.shape[1]
# Silence gap before this segment (inserted after channel count is known)
if start_sec > prev_end_sec + 1e-6:
track_parts.append(silence(default_sr, start_sec - prev_end_sec, channels))
if waveform is not None:
wav = waveform[0] # [C,T]
# Apply trim offset — skip samples from the start of the source
if trim_offset_sec > 0.0:
offset_samples = int(default_sr * trim_offset_sec)
wav = wav[:, offset_samples:]
target_samples = max(1, int(default_sr * duration_sec))
if wav.shape[-1] > target_samples:
wav = wav[:, :target_samples]
elif wav.shape[-1] < target_samples:
wav = torch.cat([wav, torch.zeros(channels, target_samples - wav.shape[-1])], dim=-1)
track_parts.append(wav.unsqueeze(0))
else:
track_parts.append(silence(default_sr, duration_sec, channels))
prev_end_sec = end_sec
if track_parts:
merged_waveform = torch.cat(track_parts, dim=-1)
total_sec = (total_length - 1) / frame_rate
channels = 2
if merged_waveform is not None:
channels = merged_waveform.shape[1]
total_samples = max(1, int(default_sr * total_sec))
wav = merged_waveform[0]
if wav.shape[-1] > total_samples:
wav = wav[:, :total_samples]
elif wav.shape[-1] < total_samples:
wav = torch.cat([wav, torch.zeros(channels, total_samples - wav.shape[-1])], dim=-1)
merged_waveform = wav.unsqueeze(0)
else:
merged_waveform = silence(default_sr, total_sec, channels)
audio_out = {"waveform": merged_waveform, "sample_rate": default_sr}
# =========================================================
# Build audio segment info from maintain segment boundaries
# Collect audio sources from tracks for output in timeline_info
# =========================================================
audio_seg_info: list[dict] = []
audio_sources: list[dict] = [] # Track audio sources with their frame ranges
for track in tracks:
if track.get("type") != "audio":
continue
for seg in sorted(track.get("segments", []), key=lambda s: s.get("start_frame", 0)):
content = seg.get("content", {})
audio_sources.append({
"start_frame": int(seg.get("start_frame", 0)),
"end_frame": int(seg.get("end_frame", 0)),
"source_type": content.get("source_type", "input"),
"file_path": content.get("file_path", ""),
"local_path": content.get("local_path", ""),
"url": content.get("url", ""),
"file_name": content.get("file_name", ""),
})
for i, seg in enumerate(maintain_segs):
start_sec = seg["start_frame"] / frame_rate
if i < len(maintain_segs) - 1:
end_sec = maintain_segs[i + 1]["start_frame"] / frame_rate
else:
end_sec = min(seg["end_frame"], total_length - 1) / frame_rate
audio_entry: dict = {
"start_sec": round(start_sec, 4),
"end_sec": round(end_sec, 4),
"duration": round(end_sec - start_sec, 4),
}
# Find audio source that overlaps with this maintain segment
for src in audio_sources:
if (src["start_frame"] >= seg["start_frame"] and src["start_frame"] <= seg["end_frame"]) or \
(src["end_frame"] >= seg["start_frame"] and src["end_frame"] <= seg["end_frame"]):
if src.get("file_path"):
audio_entry["file_path"] = src["file_path"]
if src.get("source_type"):
audio_entry["source_type"] = src["source_type"]
break
audio_seg_info.append(audio_entry)
# =========================================================
# Build per-segment info for timeline_info
# =========================================================
seg_infos: list[dict] = []
for seg in maintain_segs:
images_info: list[dict] = []
for img in seg["images"]:
entry: dict = {
"source_type": img.get("source_type", "input"),
"file_name": img.get("file_name", ""),
}
if img.get("file_path"):
entry["file_path"] = img["file_path"]
if img.get("start_frame") is not None:
entry["start_frame"] = img["start_frame"]
if img.get("end_frame") is not None:
entry["end_frame"] = img["end_frame"]
images_info.append(entry)
seg_info: dict = {
"start_frame": seg["start_frame"],
"end_frame": seg["end_frame"],
"prompt": seg["text"],
"images": images_info,
}
if images_info:
seg_info["type"] = seg["type"]
seg_infos.append(seg_info)
# =========================================================
# timeline_info output
# =========================================================
timeline_info = {
"total_length": output_total_length,
"timeline_total_length": total_length,
"frame_rate": frame_rate,
"target_frame_rate": frame_rate,
"format": format,
"width": target_w,
"height": target_h,
"segments": seg_infos,
"audio": {
"segments": audio_seg_info,
},
}
return io.NodeOutput(timeline_info, images_out, audio_out)
class MultiTrackEditor(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy multiTrackEditor",
display_name="MultiTrack Editor",
category=CATEGORY_MULTITRACK,
description=(
"Edit and pass through multitrack media data. Outputs tracks info "
"and list-style image, audio, and video media outputs."
),
is_input_list=True,
inputs=[
io.DynamicCombo.Input(
"resolution",
options=resolution_combo_options,
tooltip="Select a resolution or choose 'Custom' to specify your own width and height.",
),
io.Combo.Input("format", options=list(VIDEO_FORMATS.keys()), default="Wan", tooltip="Choose a video format to automatically set resolution and frame rate."),
TYPE_TRACK_DATA.Input("track_data"),
io.AnyType.Input("prompt_override", optional=True, tooltip="If provided, overrides all segment prompts in the timeline.",),
io.Image.Input("image", optional=True, tooltip="Optional image media list for multitrack segments."),
io.Audio.Input("audio", optional=True, tooltip="Optional audio media list for multitrack segments."),
io.Video.Input("video", optional=True, tooltip="Optional video media list for multitrack segments."),
],
outputs=[
TYPE_TRACKS_INFO.Output("TRACKS_INFO"),
io.Image.Output("IMAGES", is_output_list=True),
io.Audio.Output("AUDIO", is_output_list=True),
io.Video.Output("VIDEO", is_output_list=True),
],
)
@classmethod
def execute(
cls,
resolution: str | dict,
format: str,
track_data: str | dict,
**kwargs: object,
) -> io.NodeOutput:
if isinstance(resolution, list):
resolution = resolution[0]
if isinstance(format, list):
format = format[0]
if isinstance(track_data, list):
track_data = track_data[0]
prompt_override = kwargs.get('prompt_override')
if isinstance(prompt_override, list) and len(prompt_override) == 1:
prompt_override = prompt_override[0]
data = _parse_track_data(track_data)
if prompt_override_has_value(prompt_override):
data = build_multitrack_data_from_prompt_override(data, prompt_override)
tracks_info, images_out, audio_out, video_out = _build_tracks_info_and_media_outputs(
data,
kwargs.get("image"),
kwargs.get("audio"),
kwargs.get("video"),
resolution,
format,
)
return io.NodeOutput(tracks_info, images_out, audio_out, video_out)
class TimelineInfoOutput(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy timelineInfoOutput",
display_name="Timeline Info Output",
category=CATEGORY_TIMELINE,
description="Output timeline info including formatted prompt, dimensions, and image indexes.",
inputs=[
TYPE_TIMELINE_INFO.Input("timeline_info"),
io.Combo.Input(
"prompt_format",
options=PROMPT_FORMAT_OPTIONS,
default="default",
tooltip="Choose prompt format. promptRelay formats prompts with frame ranges.",
),
],
outputs=[
io.String.Output("PROMPT"),
io.Int.Output("WIDTH"),
io.Int.Output("HEIGHT"),
io.Int.Output("TOTAL_FRAMES"),
io.Float.Output("FPS"),
io.String.Output("IMAGE_INDEXES"),
],
)
@classmethod
def execute(
cls,
timeline_info: str | dict,
prompt_format: str,
**kwargs: object,
) -> io.NodeOutput:
if isinstance(timeline_info, str):
try:
info = json.loads(timeline_info)
except json.JSONDecodeError:
info = {}
else:
info = dict(timeline_info) if timeline_info else {}
total_length: int = info.get("total_length", 121)
frame_rate: int = info.get("target_frame_rate", info.get("frame_rate", 24))
width: int = info.get("width", 544)
height: int = info.get("height", 960)
segments: list[dict] = info.get("segments", [])
# Build image_indexes: comma-separated string of starting frames
image_indexes: str = ",".join(str(int(seg.get("start_frame", 0))) for seg in segments if seg.get("images", []))
def normalize_prompt(value: str | list | None) -> str:
if value is None:
return ""
if isinstance(value, list):
return "\n".join(v for v in value if isinstance(v, str))
return str(value).strip()
# Build prompt string
if prompt_format == "promptRelay":
prompt_parts: list[str] = []
for seg in segments:
seg_text = normalize_prompt(seg.get("prompt"))
if seg_text:
start = int(seg.get("start_frame", 0))
end = int(seg.get("end_frame", 0))
prompt_parts.append(f"{seg_text} [{start}-{end}]")
prompt_str = " | ".join(prompt_parts)
else:
prompt_str = [seg.get("prompt").strip() for seg in segments]
return io.NodeOutput(
prompt_str,
width,
height,
total_length,
float(frame_rate),
image_indexes,
)
class MultiTrackInfoOutput(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy multiTrackInfoOutput",
display_name="MultiTrack Info Output",
category=CATEGORY_MULTITRACK,
description="Output multitrack dimensions, duration, frame rate, and task count.",
inputs=[
TYPE_TRACKS_INFO.Input("tracks_info"),
],
outputs=[
io.Int.Output("WIDTH"),
io.Int.Output("HEIGHT"),
io.Int.Output("TOTAL_FRAMES"),
io.Float.Output("FPS"),
io.Int.Output("TASK_COUNT"),
],
)
@classmethod
def execute(cls, tracks_info: str | dict) -> io.NodeOutput:
if isinstance(tracks_info, str):
try:
info = json.loads(tracks_info)
except json.JSONDecodeError:
info = {}
else:
info = dict(tracks_info) if tracks_info else {}
task_count = len(_multitrack_task_entries(info))
return io.NodeOutput(
int(info.get("width", 544)),
int(info.get("height", 960)),
int(info.get("total_length", 121)),
float(info.get("target_frame_rate", info.get("frame_rate", 24))),
task_count,
)
def _multitrack_task_segments(info: dict) -> list[dict]:
tracks = info.get("tracks", [])
if not isinstance(tracks, list):
return []
return sorted(
[
segment
for track in tracks
if isinstance(track, dict) and track.get("type") == "task"
for segment in track.get("segments", [])
if isinstance(segment, dict)
],
key=lambda segment: _multitrack_frame_value(segment.get("start_frame")),
)
def _multitrack_frame_value(value: object, default: int = 0) -> int:
try:
return int(value) if value is not None else default
except (TypeError, ValueError, OverflowError):
return default
def _task_for_marker_range(tasks: list[dict], start_frame: int, end_frame: int) -> dict:
if not tasks:
return {}
return max(
tasks,
key=lambda task: max(
0,
min(end_frame, _multitrack_frame_value(task.get("end_frame"))) -
max(start_frame, _multitrack_frame_value(task.get("start_frame"))),
),
)
def _multitrack_task_entries(info: dict) -> list[dict]:
tasks = _multitrack_task_segments(info)
markers = info.get("task_markers", [])
if not isinstance(markers, list) or not markers:
return [
{
"task": task,
"start_frame": max(0, _multitrack_frame_value(task.get("start_frame"))),
"end_frame": max(
max(0, _multitrack_frame_value(task.get("start_frame"))),
_multitrack_frame_value(task.get("end_frame")),
),
}
for task in tasks
]
range_start = 0
total_length = max(0, _multitrack_frame_value(
info.get("timeline_total_length", info.get("total_length")),
))
marker_end = max(
(
_multitrack_frame_value(marker.get("frame"))
for marker in markers
if isinstance(marker, dict)
and 0 < _multitrack_frame_value(marker.get("frame")) <= total_length
),
default=0,
)
range_end = max(0, total_length - 1, marker_end)
if range_end <= range_start and tasks:
range_end = max(_multitrack_frame_value(task.get("end_frame"), range_start) for task in tasks)
if range_end <= range_start:
return []
marker_frames: set[int] = set()
for marker in markers:
if not isinstance(marker, dict):
continue
try:
frame = int(marker.get("frame"))
except (TypeError, ValueError, OverflowError):
continue
if range_start < frame <= range_end:
marker_frames.add(frame)
if not marker_frames:
return [
{
"task": task,
"start_frame": max(0, _multitrack_frame_value(task.get("start_frame"))),
"end_frame": max(0, _multitrack_frame_value(task.get("end_frame"))),
}
for task in tasks
]
boundaries = [range_start, *sorted(marker_frames)]
if boundaries[-1] < range_end:
boundaries.append(range_end)
return [
{
"task": _task_for_marker_range(tasks, start_frame, end_frame),
"start_frame": start_frame,
"end_frame": end_frame,
"marker_mode": True,
}
for start_frame, end_frame in zip(boundaries, boundaries[1:])
if end_frame > start_frame
]
def _audio_track_frame_range(track: object, frame_rate: float) -> tuple[int, int] | None:
if not isinstance(track, dict):
return None
segments = track.get("segments")
if not isinstance(segments, list):
return None
starts: list[int] = []
ends: list[int] = []
for segment in segments:
if not isinstance(segment, dict):
continue
try:
if segment.get("start_frame") is not None:
starts.append(int(segment["start_frame"]))
elif segment.get("start_time") is not None:
starts.append(round(float(segment["start_time"]) * frame_rate))
if segment.get("end_frame") is not None:
ends.append(int(segment["end_frame"]))
elif segment.get("end_time") is not None:
ends.append(round(float(segment["end_time"]) * frame_rate))
except (TypeError, ValueError, OverflowError):
continue
if not starts or not ends:
return None
start_frame = min(starts)
end_frame = max(ends)
return (start_frame, end_frame) if end_frame > start_frame else None
def _trim_audio_to_track(audio: dict | None, frame_range: tuple[int, int] | None, frame_rate: float) -> dict | None:
if audio is None or frame_range is None:
return None
start_frame, end_frame = frame_range
try:
return trim_audio(audio, start_frame / frame_rate, (end_frame - start_frame) / frame_rate)
except (KeyError, TypeError, ValueError, ZeroDivisionError):
return None
def _audio_track_range_within_task(
track: object,
task_range: tuple[int, int] | None,
frame_rate: float,
) -> tuple[int, int] | None:
if not isinstance(track, dict) or task_range is None:
return None
segments = track.get("segments")
if not isinstance(segments, list):
return None
task_start, task_end = task_range
intersections: list[tuple[int, int]] = []
for segment in segments:
segment_range = _audio_track_frame_range({"segments": [segment]}, frame_rate)
if segment_range is None:
continue
start_frame = max(task_start, segment_range[0])
end_frame = min(task_end, segment_range[1])
if end_frame > start_frame:
intersections.append((start_frame, end_frame))
if not intersections:
return None
return min(start for start, _end in intersections), max(end for _start, end in intersections)
def _silent_audio_for_range(
audio: dict | None,
frame_range: tuple[int, int] | None,
frame_rate: float,
) -> dict | None:
if frame_range is None:
return None
sample_rate = 44100
channels = 2
if isinstance(audio, dict):
try:
sample_rate = int(audio.get("sample_rate", sample_rate))
waveform = audio.get("waveform")
if isinstance(waveform, torch.Tensor) and waveform.ndim >= 2:
channels = int(waveform.shape[-2])
except (TypeError, ValueError, OverflowError):
sample_rate = 44100
channels = 2
start_frame, end_frame = frame_range
duration = (end_frame - start_frame) / frame_rate
return {"waveform": silence(sample_rate, duration, channels), "sample_rate": sample_rate}
class MultiTrackAudioOutput(io.ComfyNode):
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id="easy multiTrackAudioOutput",
display_name="MultiTrack Audio Output",
category=CATEGORY_MULTITRACK,
description="Merge all audio tracks and output the first two tracks in full or cropped for S2V.",
is_input_list=True,
inputs=[
TYPE_TRACKS_INFO.Input("tracks_info"),
io.Audio.Input("audio"),
io.Combo.Input(
"mode",
options=["default", "crop"],
default="default",
socketless=True,
),
io.Int.Input(
"task_index",
default=0,
min=0,
step=1,
tooltip="Select a zero-based task segment range.",
),
],
outputs=[
io.Audio.Output("combine_audio"),
io.Audio.Output("audio_0"),
io.Int.Output("audio_0_start"),
io.Audio.Output("audio_1"),
io.Int.Output("audio_1_start"),
],
)
@classmethod
def execute(
cls,
tracks_info: list | dict | str,
audio: list | dict | None,
mode: str | list[str] = "default",
task_index: int | list[int] = 0,
) -> io.NodeOutput:
raw_info = _unwrap_list_scalar(tracks_info, {})
info = _parse_track_data(raw_info)
frame_rate = max(0.001, float(info.get("frame_rate", 24)))
tracks = info.get("tracks", [])
audio_tracks = [
track for track in tracks
if isinstance(track, dict) and track.get("type") == "audio"
] if isinstance(tracks, list) else []
audios = iter_valid_audio_inputs(audio)
combined_audio = merge_audio_inputs(audios, "add")
selected_mode = str(_unwrap_list_scalar(mode, "default"))
if selected_mode == "default":
return io.NodeOutput(
combined_audio,
audios[0] if audios else None,
0,
audios[1] if len(audios) > 1 else None,
0,
)
selected_task_index = int(_unwrap_list_scalar(task_index, 0))
if selected_task_index >= 0:
task_entries = _multitrack_task_entries(info)
task_range = (
(
int(task_entries[selected_task_index]["start_frame"]),
int(task_entries[selected_task_index]["end_frame"]),
)
if selected_task_index < len(task_entries)
else None
)
first_range = _audio_track_range_within_task(
audio_tracks[0] if audio_tracks else None,
task_range,
frame_rate,
)
second_range = _audio_track_range_within_task(
audio_tracks[1] if len(audio_tracks) > 1 else None,
task_range,
frame_rate,
)
task_start = task_range[0] if task_range is not None else -1
first_input = audios[0] if audios else None
second_input = audios[1] if len(audios) > 1 else None
return io.NodeOutput(
combined_audio,
_trim_audio_to_track(first_input, first_range, frame_rate)
or _silent_audio_for_range(first_input, task_range, frame_rate),
first_range[0] - task_start if first_range is not None else -1,
_trim_audio_to_track(second_input, second_range, frame_rate)
or _silent_audio_for_range(second_input, task_range, frame_rate),
second_range[0] - task_start if second_range is not None else -1,
)
first_range = _audio_track_frame_range(audio_tracks[0], frame_rate) if audio_tracks else None
second_range = _audio_track_frame_range(audio_tracks[1], frame_rate) if len(audio_tracks) > 1 else None
first_audio = _trim_audio_to_track(audios[0] if audios else None, first_range, frame_rate)
second_audio = _trim_audio_to_track(audios[1] if len(audios) > 1 else None, second_range, frame_rate)
first_start = first_range[0] if first_range is not None else -1
second_start = second_range[0] if second_range is not None else -1
return io.NodeOutput(
combined_audio,
first_audio,
first_start,
second_audio,
second_start,
)
def _subtitle_base_name(video_path: str | None) -> str:
if video_path:
stem = Path(video_path).stem.strip()
if stem:
return default_subtitle_filename(stem)
return default_subtitle_filename()
def _add_subtitle_segments_to_video(
video: object,
subtitle_segments: list[object],
srt_save: str,
) -> io.NodeOutput:
if not subtitle_segments:
return io.NodeOutput(video)
save_mode = str(_unwrap_list_scalar(srt_save, "temp"))
if save_mode not in {"temp", "output"}:
save_mode = "temp"
width, height = video.get_dimensions()
input_path, temp_files = video_input_to_local_file(
video,
suffix=".mp4",
save_kwargs={
"format": Types.VideoContainer.AUTO,
"codec": Types.VideoCodec.AUTO,
},
)
ass_path: Path | None = None
try:
base_name = _subtitle_base_name(input_path)
if save_mode == "output":
srt_dir = Path(folder_paths.get_output_directory()) / "srt"
else:
srt_dir = Path(folder_paths.get_temp_directory())
write_srt_file(subtitle_segments, srt_dir / f"{base_name}.srt")
ass_fd, ass_raw_path = tempfile.mkstemp(
prefix=f"{base_name}_",
suffix=".ass",
dir=folder_paths.get_temp_directory(),
)
os.close(ass_fd)
ass_path = write_ass_file(subtitle_segments, Path(ass_raw_path), width, height)
output_fd, output_path = tempfile.mkstemp(
prefix=f"{base_name}_subtitled_",
suffix=".mp4",
dir=folder_paths.get_temp_directory(),
)
os.close(output_fd)
burn_subtitles_with_ffmpeg(input_path, str(ass_path), output_path)
return io.NodeOutput(InputImpl.VideoFromFile(output_path))
finally:
for path in temp_files:
try:
os.unlink(path)
except OSError:
pass
if ass_path is not None:
try:
ass_path.unlink(missing_ok=True)
except OSError:
pass
class AddSubtitleToVideo(io.ComfyNode):
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id="easy addSubtitleToVideo",
display_name="Add Subtitle To Video",
category=CATEGORY_VIDEO,
description=(
"Burn multiline SRT, timestamp, or bracket-formatted subtitle text "
"into a VIDEO and save a normalized SRT file."
),
inputs=[
io.String.Input(
"subtitle_text",
multiline=True,
default="",
placeholder=(
"1\n00:00:00,000 --> 00:00:02,000\nSubtitle text\n\n"
"or [00:02.000 --> 00:04.000] Subtitle text"
),
dynamic_prompts=False,
),
io.Video.Input("video"),
io.Combo.Input(
"srt_save",
options=["temp", "output"],
default="temp",
tooltip="Save the normalized SRT in temp or output/srt.",
),
io.Int.Input("font_size", default=16, min=8, max=96, step=1),
],
outputs=[io.Video.Output("VIDEO")],
)
@classmethod
def execute(
cls,
subtitle_text: str,
video: object,
srt_save: str = "temp",
font_size: int = 16,
) -> io.NodeOutput:
segments = parse_subtitle_text(
str(_unwrap_list_scalar(subtitle_text, "")),
style={"font_size": int(_unwrap_list_scalar(font_size, 16))},
)
return _add_subtitle_segments_to_video(video, segments, srt_save)
class MultiTrackAddSubtitleToVideo(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy multiTrackAddSubtitleToVIdeo",
display_name="MultiTrack Add Subtitle To Video",
category=CATEGORY_MULTITRACK,
description="Burn all subtitle track segments from TRACKS_INFO into a VIDEO and save an SRT file.",
inputs=[
TYPE_TRACKS_INFO.Input("tracks_info"),
io.Video.Input("video"),
io.Combo.Input(
"srt_save",
options=["temp", "output"],
default="temp",
tooltip="Save the generated SRT in temp or output/srt.",
),
],
outputs=[
io.Video.Output("VIDEO"),
],
)
@classmethod
def execute(cls, tracks_info: str | dict, video, srt_save: str = "temp") -> io.NodeOutput:
if isinstance(tracks_info, list):
tracks_info = tracks_info[0] if tracks_info else {}
info = _parse_track_data(tracks_info)
subtitle_segments = collect_multitrack_subtitle_segments(info)
return _add_subtitle_segments_to_video(video, subtitle_segments, srt_save)
def _unwrap_list_scalar(value, default=None):
if isinstance(value, list):
return value[0] if value else default
return value if value is not None else default
def _unwrap_singleton_container(value, default=None):
while isinstance(value, (list, tuple)):
if not value:
return default
if len(value) != 1:
return value
value = value[0]
return value if value is not None else default
def _track_output_index(track: dict) -> 'int | None':
raw_index = track.get("media_index")
if raw_index is None:
for segment in track.get("segments", []):
if isinstance(segment, dict):
content = segment.get("content", {})
if isinstance(content, dict) and content.get("media_index") is not None:
raw_index = content["media_index"]
break
try:
return int(raw_index) if raw_index is not None else None
except (TypeError, ValueError):
return None
def _track_media_end_frame(
track: dict,
start_frame: int | None = None,
end_frame: int | None = None,
) -> 'int | None':
track_type = track.get("type")
valid_ends: list[int] = []
for segment in track.get("segments", []):
if not isinstance(segment, dict):
continue
content = segment.get("content")
if not isinstance(content, dict) or content.get("media_type") != track_type:
continue
segment_start = _multitrack_frame_value(segment.get("start_frame"))
segment_end = _multitrack_frame_value(segment.get("end_frame"), -1)
if segment_end < 0:
continue
if start_frame is not None and segment_end <= start_frame:
continue
if end_frame is not None and segment_start >= end_frame:
continue
valid_ends.append(min(segment_end, end_frame) if end_frame is not None else segment_end)
return max(valid_ends) if valid_ends else None
def _ranges_overlap(start: int, end: int, segment: dict) -> bool:
return int(segment.get("start_frame", 0)) < end and int(segment.get("end_frame", 0)) > start
def _multitrack_task_type(task: dict, image_count: int, has_video: bool) -> str:
content = task.get("content", {})
explicit_task_type = content.get("task_type") if isinstance(content, dict) else None
if isinstance(explicit_task_type, str) and explicit_task_type.strip():
return explicit_task_type.strip()
mode = content.get("task_mode", "default") if isinstance(content, dict) else "default"
if mode == "l2v":
return "l2v"
if mode == "ref":
return "rv2v" if has_video else "r2v"
if mode == "edit":
return "vi2v" if image_count > 0 else "v2v"
return "i2v" if image_count > 0 else "t2v"
def _trim_track_audio(
audio: dict,
start_frame: int,
length: int | None,
frame_rate: float,
) -> dict:
waveform = audio.get("waveform")
sample_rate = int(audio.get("sample_rate", 44100))
if not isinstance(waveform, torch.Tensor):
return {"waveform": torch.zeros(1, 1, 1), "sample_rate": sample_rate}
start_sample = max(0, round(start_frame * sample_rate / frame_rate))
sample_count = (
max(1, waveform.shape[-1] - start_sample)
if length is None
else max(1, round(length * sample_rate / frame_rate))
)
end_sample = min(waveform.shape[-1], start_sample + sample_count)
trimmed = waveform[..., start_sample:end_sample]
if trimmed.shape[-1] < sample_count:
trimmed = F.pad(trimmed, (0, sample_count - trimmed.shape[-1]))
return {"waveform": trimmed, "sample_rate": sample_rate}
# code based on https://github.com/RH-RunningHub/ComfyUI-RH-Bernini/blob/main/nodes_bernini.py
def _build_chat_prompts(system_prompt, api_prompt, original_prompt):
system_prompt = (system_prompt or "").strip()
api_prompt = (api_prompt or "").strip()
original_prompt = (original_prompt or "").strip()
if not api_prompt or api_prompt == original_prompt:
return system_prompt, original_prompt
text = api_prompt
match = re.search(
r"\n\s*(?P<label>Original (?:instruction|description)):\s*\n(?P<user>.*?)\s*$",
text,
flags=re.DOTALL,
)
if match:
return text[: match.start()].strip(), match.group("user").strip()
match = re.search(
r"(?m)^\s*-?\s*User's (?:raw instruction|editing instruction|instruction|prompt):\s*\"(?P<user>.*?)\"\s*$",
text,
)
if match:
cleaned = (text[: match.start()] + text[match.end() :]).strip()
return cleaned, match.group("user").strip()
return api_prompt, original_prompt
def _format_multitrack_prompt_relay(
prompt: str,
start_frame: int,
end_frame: int,
image_count: int,
) -> str:
prompt = (prompt or "").strip()
if not prompt or end_frame <= start_frame:
return prompt
if image_count <= 0:
return f"{prompt} [{start_frame}-{end_frame}]"
parts = [part.strip() for part in prompt.split("|") if part.strip()]
frame_count = end_frame - start_frame
formatted: list[str] = []
for index, part in enumerate(parts[:image_count]):
range_start = start_frame + math.ceil(index * frame_count / image_count)
range_end = start_frame + math.ceil((index + 1) * frame_count / image_count)
formatted.append(f"{part} [{range_start}-{range_end}]")
return " | ".join(formatted)
def _selected_multitrack_user_prompt(content: dict) -> str:
if str(content.get("user_prompt_variant", "a")).lower() == "b":
prompt = str(content.get("user_prompt_b") or "")
else:
prompt = str(content.get("user_prompt") or content.get("text") or "")
return prompt.replace("@", "")
def _format_marker_task_prompt_relay(
tasks: list[dict],
start_frame: int,
end_frame: int,
) -> str:
formatted: list[str] = []
for task in tasks:
task_start = max(start_frame, _multitrack_frame_value(task.get("start_frame")))
task_end = min(end_frame, _multitrack_frame_value(task.get("end_frame")))
if task_end <= task_start:
continue
content = task.get("content", {})
if not isinstance(content, dict):
continue
prompt = _selected_multitrack_user_prompt(content).strip()
if prompt:
formatted.append(_format_multitrack_prompt_relay(prompt, task_start, task_end, 0))
return " | ".join(formatted)
def _evenly_distributed_image_indexes(image_count: int, duration_frames: int) -> str:
if image_count <= 0:
return ""
if image_count == 1:
return "0"
indexes = [0]
indexes.extend(
math.ceil(index * max(0, duration_frames) / (image_count - 1))
for index in range(1, image_count - 1)
)
indexes.append(-1)
return ",".join(str(index) for index in indexes)
class MultiTrackTaskOutput(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy multiTrackTaskOutput",
display_name="MultiTrack Task Output",
category=CATEGORY_MULTITRACK,
description="Output prompts and task-range media for a multitrack task segment.",
is_input_list=True,
inputs=[
TYPE_TRACKS_INFO.Input("tracks_info"),
io.Image.Input("images", optional=True),
io.Audio.Input("audio", optional=True),
io.Video.Input("video", optional=True),
io.Int.Input("task_index", default=0, min=0),
io.Combo.Input(
"prompt_format",
options=PROMPT_FORMAT_OPTIONS + ["api", "llm"],
default="api",
tooltip="Choose prompt format.",
),
],
outputs=[
io.String.Output("SYSTEM_PROMPT"),
io.String.Output("USER_PROMPT"),
io.String.Output("TYPE"),
io.Int.Output("LENGTH"),
io.Image.Output("IMAGES", is_output_list=True),
io.Audio.Output("AUDIO", is_output_list=True),
io.Video.Output("VIDEO", is_output_list=True),
io.String.Output("IMAGE_INDEXES"),
],
)
@classmethod
def execute(
cls,
tracks_info: list | dict | str,
images: list | torch.Tensor | None = None,
audio: list | dict | None = None,
video: list | object | None = None,
task_index: list[int] | int | None = None,
prompt_format: list[str] | str | None = None,
) -> io.NodeOutput:
raw_info = _unwrap_list_scalar(tracks_info, {})
info = _parse_track_data(raw_info)
image_items = _as_list_input(images)
audio_items = _as_list_input(audio)
video_items = _as_list_input(video)
index = max(0, int(_unwrap_list_scalar(task_index, 0)))
selected_prompt_format = str(_unwrap_list_scalar(prompt_format, "default"))
tracks = info.get("tracks", [])
task_entries = _multitrack_task_entries(info)
task_entry_index = min(index, len(task_entries) - 1) if task_entries else -1
task_entry = task_entries[task_entry_index] if task_entry_index >= 0 else {}
task = task_entry.get("task", {}) if isinstance(task_entry, dict) else {}
content = task.get("content", {}) if isinstance(task.get("content", {}), dict) else {}
start_frame = max(0, int(task_entry.get("start_frame", task.get("start_frame", 0))))
end_frame = max(start_frame, int(task_entry.get("end_frame", task.get("end_frame", start_frame))))
duration_frames = end_frame - start_frame
frame_rate = float(info.get("frame_rate", 24))
is_minimax = info.get("format") == "MiniMax"
next_task_start = None
if is_minimax and task_entry_index + 1 < len(task_entries):
next_task_start = max(
start_frame,
_multitrack_frame_value(task_entries[task_entry_index + 1].get("start_frame")),
)
media_duration_frames = (
next_task_start - start_frame if next_task_start is not None else None
)
if not task_entry:
length = 0
elif is_minimax:
length = _video_frame_count_from_duration(duration_frames, frame_rate, "MiniMax")
else:
length = duration_frames + 1
task_content_entries = [(content, start_frame)]
if task_entry.get("marker_mode"):
task_content_entries = [
(
candidate_content,
max(start_frame, _multitrack_frame_value(candidate.get("start_frame"))),
)
for candidate in _multitrack_task_segments(info)
if _ranges_overlap(start_frame, end_frame, candidate)
for candidate_content in [candidate.get("content", {})]
if isinstance(candidate_content, dict)
]
selected_images: list[torch.Tensor] = []
selected_image_indexes: set[int] = set()
marker_image_frames: list[int] = []
for task_content, task_content_start in task_content_entries:
for image_info in task_content.get("images", []):
if not isinstance(image_info, dict):
continue
try:
media_index = int(image_info.get("media_index"))
except (TypeError, ValueError):
continue
if (
media_index not in selected_image_indexes
and 0 <= media_index < len(image_items)
and isinstance(image_items[media_index], torch.Tensor)
):
selected_image_indexes.add(media_index)
selected_images.append(image_items[media_index])
if task_entry.get("marker_mode"):
marker_image_frames.append(max(0, task_content_start - start_frame))
if task_entry.get("marker_mode"):
image_indexes = ",".join(str(frame) for frame in marker_image_frames)
else:
image_indexes = _evenly_distributed_image_indexes(
len(selected_images),
duration_frames,
)
selected_audio: list[dict] = []
selected_video: list = []
has_video = False
for track in tracks[1:] if isinstance(tracks, list) else []:
if not isinstance(track, dict):
continue
media_index = _track_output_index(track)
track_media_duration_frames = media_duration_frames
if is_minimax:
track_media_end = _track_media_end_frame(
track,
start_frame,
next_task_start,
)
if track_media_end is not None:
available_frames = max(0, track_media_end - start_frame)
track_media_duration_frames = (
min(track_media_duration_frames, available_frames)
if track_media_duration_frames is not None
else available_frames
)
if track_media_duration_frames is not None and track_media_duration_frames <= 0:
continue
if track.get("type") == "audio" and media_index is not None and 0 <= media_index < len(audio_items):
track_audio = audio_items[media_index]
if isinstance(track_audio, dict):
selected_audio.append(
_trim_track_audio(
track_audio,
start_frame,
track_media_duration_frames if is_minimax else duration_frames,
frame_rate,
)
)
elif track.get("type") == "video" and media_index is not None and 0 <= media_index < len(video_items):
track_video = video_items[media_index]
video_duration = duration_frames / frame_rate
if is_minimax:
video_duration = (
track_media_duration_frames / frame_rate
if track_media_duration_frames is not None
else 0.0
)
trimmed = track_video.as_trimmed(
start_time=start_frame / frame_rate,
duration=video_duration,
strict_duration=False,
)
if trimmed is not None:
selected_video.append(trimmed)
has_video = has_video or any(
isinstance(segment, dict)
and isinstance(segment.get("content"), dict)
and _ranges_overlap(start_frame, end_frame, segment)
for segment in track.get("segments", [])
)
task_type = _multitrack_task_type(task, len(selected_images), has_video)
prompt = _selected_multitrack_user_prompt(content)
system_prompt, api_prompt, json_mode = build_prompt_request(
task_type,
prompt,
images=selected_images,
video=selected_video,
custom_system_prompt=(
str(content.get("system_prompt")).replace("@", "")
if content.get("system_prompt")
else None
),
video_format=info.get("format"),
task_mode=content.get("task_mode", "default"),
)
chat_system_prompt, chat_user_prompt = _build_chat_prompts(system_prompt, api_prompt, prompt)
llm_prompt = build_llm_prompt(chat_system_prompt, chat_user_prompt, json_mode)
if selected_prompt_format == "promptRelay":
if task_entry.get("marker_mode"):
user_prompt = _format_marker_task_prompt_relay(
_multitrack_task_segments(info),
start_frame,
end_frame,
)
else:
user_prompt = _format_multitrack_prompt_relay(
chat_user_prompt,
start_frame,
end_frame,
len(selected_images),
)
elif selected_prompt_format == "api":
user_prompt = chat_user_prompt
elif selected_prompt_format == "llm":
user_prompt = llm_prompt
else:
user_prompt = chat_user_prompt
output_system_prompt = (
"" if selected_prompt_format in {"default", "promptRelay"} else chat_system_prompt
)
return io.NodeOutput(
output_system_prompt,
user_prompt,
task_type,
length,
selected_images,
(selected_audio or [None]) if is_minimax else selected_audio,
(selected_video or [None]) if is_minimax else selected_video,
image_indexes,
)
class MultiTrackPromptEnhancer(io.ComfyNode):
@staticmethod
def _api_key_input() -> object:
return io.String.Input(
"apikey",
default="",
tooltip=(
"Provider API key. If empty, the matching key is read from "
"config.yaml or the environment."
),
)
@classmethod
def _model_options(cls) -> list:
options: list = []
for model_name in PROMPT_ENHANCER_MODELS:
inputs: list = []
if model_name != LLAMACPP_MODEL:
inputs.append(cls._api_key_input())
if model_name == MINIMAX_MODEL:
inputs.extend(
[
io.Combo.Input(
"ratio",
options=PROMPT_ENHANCER_RATIO_OPTIONS,
default="adaptive",
tooltip=(
"MiniMax official API ratio. Text-only adaptive "
"requests use 16:9."
),
),
io.Boolean.Input(
"return_async",
default=False,
tooltip=(
"Only effective for h3-context-ir. When enabled, "
"return the task ID without polling the task status."
),
),
]
)
elif model_name in PROMPT_ENHANCER_MAX_TOKENS:
if model_name == LLAMACPP_MODEL:
inputs.extend(
[
io.Combo.Input(
"inference_mode",
options=["one by one", "images", "video"],
default="images",
tooltip=(
"one by one: process every list item separately; "
"a multi-image batch inside one item is inferred "
"together. images: combine all images from every "
"list item into one prompt. video: treat each image "
"list item as a separate video clip."
),
),
io.Boolean.Input(
"force_offload",
default=True,
tooltip="Unload the local llama.cpp model after inference.",
),
]
)
default, maximum = PROMPT_ENHANCER_MAX_TOKENS[model_name]
inputs.append(
io.Int.Input(
"max_size",
default=default,
min=128 if model_name == LLAMACPP_MODEL else 1,
max=maximum,
step=64 if model_name == LLAMACPP_MODEL else 1,
tooltip=(
"Maximum input image size for all llama.cpp inference modes."
if model_name == LLAMACPP_MODEL
else "Maximum number of output tokens generated by the model."
),
)
)
options.append(io.DynamicCombo.Option(model_name, inputs))
return options
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id="easy multiTrackPromptEnhancer",
display_name="MultiTrack Prompt Enhancer",
category=CATEGORY_MULTITRACK,
description=(
"Use MiniMax H3-Context-IR, a configured third-party multimodal LLM, "
"or a local llama.cpp model to enhance a multitrack video prompt."
),
is_input_list=True,
not_idempotent=True,
enable_expand=True,
inputs=[
io.String.Input(
"system_prompt",
default="",
multiline=True,
force_input=True,
tooltip="Optional system instructions from MultiTrack Task Output.",
),
io.String.Input(
"user_prompt",
default="",
multiline=True,
force_input=True,
tooltip="Optional user prompt from MultiTrack Task Output.",
),
io.String.Input(
"type",
default="t2v",
force_input=True,
tooltip=(
"Task type used to select H3 t2va, i2va, or r2va and "
"assemble provider-specific media roles."
),
),
io.Int.Input(
"length",
default=124,
min=1,
max=0x7FFFFFFF,
force_input=True,
tooltip="MiniMax-aligned frame length; converted back to 4–15 integer seconds.",
),
io.Image.Input(
"images",
optional=True,
tooltip=(
"Optional image, batch, or list. H3 uploads each selected image; "
"third-party inputs are limited to 2 megapixels."
),
),
io.Audio.Input(
"audio",
optional=True,
tooltip=(
"Optional H3 r2va audio input or list. Audio is uploaded to "
"MiniMax and omitted for third-party models."
),
),
io.Video.Input(
"video",
optional=True,
tooltip=(
"Optional video input or list. A supported public video URL "
"or local video data is sent natively when the provider accepts "
"it; otherwise up to 24 resized frames are sent per video. "
"Native video uploads are limited to 15 seconds; RunningHub "
"videos are additionally limited to 10MB."
),
),
io.AnyType.Input(
"files",
optional=True,
tooltip="Reserved file input for a future provider implementation.",
),
TYPE_LLAMACPP_MODEL.Input(
"llama_model",
optional=True,
lazy=True,
raw_link=True,
tooltip=(
"Local llama.cpp model input; evaluated only when the local "
"provider is selected. Requires llama_cpp_instruct_adv from "
f"{LLAMA_CPP_INSTALL_URL}."
),
),
io.DynamicCombo.Input(
"model",
options=cls._model_options(),
tooltip="Provider and model used to enhance the prompt.",
),
io.Int.Input(
"seed",
default=0,
min=0,
max=0xFFFFFFFFFFFFFFFF,
control_after_generate=True,
tooltip="ComfyUI generation seed; used by compatible third-party LLM APIs.",
),
io.Boolean.Input(
"enabled",
default=True,
tooltip=(
"Enhance the prompt when enabled. When disabled, return "
"user_prompt unchanged without calling the selected model."
),
),
TYPE_PROMPT_ENHANCER_ACCOUNT.Input(
"api_account",
tooltip=(
"Provider balance and API key management. This widget is for "
"account status only and does not affect prompt enhancement."
),
),
],
outputs=[
io.String.Output("PROMPT", tooltip="Enhanced video prompt."),
io.String.Output(
"TASK_ID",
tooltip="MiniMax task ID; empty for all other providers.",
),
io.String.Output(
"FILE_IDS",
tooltip=(
"Comma-separated MiniMax file IDs for uploaded images, "
"videos, and audio; empty when no media was uploaded."
),
),
],
)
@classmethod
def execute(
cls,
system_prompt: list[str] | str | None = None,
user_prompt: list[str] | str | None = None,
type: list[str] | str | None = None,
length: list[int] | int | None = None,
images: list | torch.Tensor | None = None,
video: list | object | None = None,
audio: list | dict | None = None,
files: list | object | None = None,
llama_model: list | object | None = None,
model: list[dict] | dict | None = None,
seed: list[int] | int | None = None,
enabled: list[bool] | bool | None = None,
api_account: list[str] | str | None = None,
) -> io.NodeOutput:
system_text = str(_unwrap_list_scalar(system_prompt, ""))
user_text = str(_unwrap_list_scalar(user_prompt, ""))
if not bool(_unwrap_list_scalar(enabled, True)):
return io.NodeOutput(user_text, "", "")
model_config = _unwrap_list_scalar(model, {})
if not isinstance(model_config, dict):
raise TypeError("model must be a DynamicCombo configuration dictionary.")
selected_model = str(
_unwrap_list_scalar(model_config.get("model"), MINIMAX_MODEL)
)
selected_api_key = str(
_unwrap_list_scalar(model_config.get("apikey"), "")
)
selected_max_size = model_config.get("max_size")
selected_inference_mode = model_config.get(
"inference_mode", "one by one"
)
selected_force_offload = model_config.get("force_offload", True)
selected_ratio_value = model_config.get("ratio", "adaptive")
selected_return_async = model_config.get("return_async", False)
progress_total = 100
process_bar = ProgressBar(progress_total)
process_bar.update_absolute(0, progress_total)
selected_seed = int(_unwrap_list_scalar(seed, 0))
if selected_model == LLAMACPP_MODEL:
if LLAMA_CPP_INSTRUCT_NODE_ID not in getattr(
comfy_nodes, "NODE_CLASS_MAPPINGS", {}
):
raise RuntimeError(
f"Missing node '{LLAMA_CPP_INSTRUCT_NODE_ID}'. 未找到该节点。请前往 "
f"{LLAMA_CPP_INSTALL_URL} 下载并安装 ComfyUI-llama-cpp_vlm,"
"然后重启 ComfyUI。"
)
selected_llama_model = _unwrap_singleton_container(llama_model, None)
if selected_llama_model is None:
raise RuntimeError(
"llama_model must be connected when model is llama.cpp (本地)."
)
if not isinstance(selected_llama_model, dict) and not is_link(
selected_llama_model
):
raise TypeError(
"llama_model must resolve to a graph link or llama.cpp "
"configuration dictionary; "
f"received {type(selected_llama_model).__name__}."
)
graph = GraphBuilder()
local_max_size = min(
int(_unwrap_list_scalar(selected_max_size, 512)),
PROMPT_ENHANCER_MAX_TOKENS[LLAMACPP_MODEL][1],
)
node_inputs: dict[str, object] = {
"llama_model": selected_llama_model,
"preset_prompt": "Empty - Nothing",
"custom_prompt": user_text,
"system_prompt": system_text,
"inference_mode": str(
_unwrap_list_scalar(selected_inference_mode, "one by one")
),
"max_frames": 24,
"max_size": local_max_size,
"seed": selected_seed,
"force_offload": bool(
_unwrap_list_scalar(selected_force_offload, True)
),
"save_states": False,
}
image_inputs = _as_list_input(images)
if image_inputs:
image_bridge = graph.node(
LLAMA_CPP_IMAGE_LIST_BRIDGE_NODE_ID,
id="local_llama_images",
images=image_inputs,
inference_mode=str(
_unwrap_list_scalar(selected_inference_mode, "one by one")
),
max_size=local_max_size,
)
node_inputs["images"] = image_bridge.out(0)
enhancer = graph.node(
LLAMA_CPP_INSTRUCT_NODE_ID,
id="local_llama_prompt_enhancer",
**node_inputs,
)
trimmed = graph.node(
STRING_TRIM_NODE_ID,
id="local_llama_prompt_trim",
string=enhancer.out(0),
mode="Both",
)
starts_with_text_fence = graph.node(
STRING_COMPARE_NODE_ID,
id="local_llama_prompt_starts_with_text_fence",
string_a=trimmed.out(0),
string_b="```text",
mode="Starts With",
case_sensitive=True,
)
ends_with_fence = graph.node(
STRING_COMPARE_NODE_ID,
id="local_llama_prompt_ends_with_fence",
string_a=trimmed.out(0),
string_b="```",
mode="Ends With",
case_sensitive=True,
)
without_text_fence = graph.node(
STRING_REPLACE_NODE_ID,
id="local_llama_prompt_remove_text_fence",
string=trimmed.out(0),
find="```text",
replace="",
)
without_closing_fence = graph.node(
STRING_REPLACE_NODE_ID,
id="local_llama_prompt_remove_closing_fence",
string=without_text_fence.out(0),
find="```",
replace="",
)
cleaned = graph.node(
STRING_TRIM_NODE_ID,
id="local_llama_prompt_cleaned_trim",
string=without_closing_fence.out(0),
mode="Both",
)
cleaned_if_ending_matches = graph.node(
SWITCH_NODE_ID,
id="local_llama_prompt_end_switch",
switch=ends_with_fence.out(0),
on_false=trimmed.out(0),
on_true=cleaned.out(0),
)
final_prompt = graph.node(
SWITCH_NODE_ID,
id="local_llama_prompt_start_switch",
switch=starts_with_text_fence.out(0),
on_false=trimmed.out(0),
on_true=cleaned_if_ending_matches.out(0),
)
process_bar.update_absolute(progress_total, progress_total)
return io.NodeOutput(
final_prompt.out(0),
"",
"",
expand=graph.finalize(),
)
try:
client = PromptEnhancerClient(
selected_model,
selected_api_key,
)
is_minimax = selected_model == MINIMAX_MODEL
image_urls = image_tensor_data_uris(
_as_list_input(images),
max_pixels=None if is_minimax else 2_000_000,
)
video_urls = (
video_data_uris(_as_list_input(video), max_duration=15)
if is_minimax
else prompt_enhancer_video_inputs(
selected_model,
_as_list_input(video),
)
)
audio_urls = audio_data_uris(_as_list_input(audio)) if is_minimax else []
file_items = [item for item in _as_list_input(files) if item is not None]
process_bar.update_absolute(10, progress_total)
# The file socket is reserved until provider uploads are implemented.
duration = minimax_length_to_seconds(_unwrap_list_scalar(length, 124))
task_type = str(_unwrap_list_scalar(type, "t2v"))
selected_ratio = str(_unwrap_list_scalar(selected_ratio_value, "adaptive"))
async_requested = bool(
_unwrap_list_scalar(selected_return_async, False)
)
poll_progress = 20
def on_poll(_status: str) -> None:
nonlocal poll_progress
try:
import comfy.model_management as model_management
model_management.throw_exception_if_processing_interrupted()
except ImportError:
pass
poll_progress = min(95, poll_progress + 5)
process_bar.update_absolute(poll_progress, progress_total)
process_bar.update_absolute(poll_progress, progress_total)
result = client.enhance(
system_prompt=system_text,
user_prompt=user_text,
task_type=task_type,
duration=duration,
ratio=selected_ratio,
seed=selected_seed,
image_urls=image_urls,
video_urls=video_urls,
audio_urls=audio_urls,
max_size=(
None
if selected_max_size is None
else int(_unwrap_list_scalar(selected_max_size, 512))
),
return_async=async_requested,
poll_interval=5.0,
poll_callback=on_poll,
file_count=len(file_items),
request_logger=log_node_info,
)
except (
PromptEnhancerApiError,
NotImplementedError,
ValueError,
TypeError,
OSError,
) as exc:
process_bar.update_absolute(progress_total, progress_total)
raise RuntimeError(f"Prompt enhancement failed: {exc}") from exc
process_bar.update_absolute(progress_total, progress_total)
return io.NodeOutput(
result.prompt,
result.task_id if selected_model == MINIMAX_MODEL else "",
result.file_ids if selected_model == MINIMAX_MODEL else "",
)
class MultiTrackPromptEnhancerImageListBridge(io.ComfyNode):
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id=LLAMA_CPP_IMAGE_LIST_BRIDGE_NODE_ID,
display_name="MultiTrack Prompt Enhancer Image List Bridge",
category="_easy_media/internal",
description=(
"Internal bridge that preserves list-style image inputs for the local "
"llama.cpp expansion graph."
),
is_dev_only=True,
inputs=[
io.AnyType.Input(
"images",
tooltip="Image values forwarded by MultiTrack Prompt Enhancer.",
),
io.Int.Input(
"max_size",
default=512,
min=128,
max=PROMPT_ENHANCER_MAX_TOKENS[LLAMACPP_MODEL][1],
step=64,
tooltip=(
"Maximum image long-edge size used before local llama.cpp "
"vision inference."
),
),
io.Combo.Input(
"inference_mode",
options=["one by one", "images", "video"],
default="one by one",
tooltip=(
"Preserves separate references and distributes the visual-size "
"budget across images or sampled video frames."
),
),
],
outputs=[
io.Image.Output(
"IMAGES",
is_output_list=True,
tooltip="Images forwarded as a ComfyUI output list.",
)
],
)
@classmethod
def execute(
cls,
images: list | torch.Tensor | None = None,
max_size: int = 512,
inference_mode: str = "one by one",
) -> io.NodeOutput:
image_inputs = _as_list_input(images)
safe_max_size = min(
max_size,
PROMPT_ENHANCER_MAX_TOKENS[LLAMACPP_MODEL][1],
)
total_frame_count = sum(
image.shape[0]
if isinstance(image, torch.Tensor) and image.ndim == 4
else 1
for image in image_inputs
)
resized_images: list[torch.Tensor] = []
for image in image_inputs:
if not isinstance(image, torch.Tensor):
raise TypeError("llama.cpp image inputs must be torch.Tensor values.")
if image.ndim not in (3, 4):
raise ValueError(
"llama.cpp image inputs must have shape [H,W,C] or [B,H,W,C]."
)
batched_image = image.unsqueeze(0) if image.ndim == 3 else image
if inference_mode == "images":
budget_frame_count = total_frame_count
elif inference_mode == "video":
budget_frame_count = min(batched_image.shape[0], 24)
else:
budget_frame_count = batched_image.shape[0]
effective_max_size = max(
128,
int(safe_max_size / math.sqrt(max(1, budget_frame_count))),
)
height, width = batched_image.shape[1:3]
long_edge = max(height, width)
if long_edge > effective_max_size:
scale = effective_max_size / long_edge
target_width = max(1, round(width * scale))
target_height = max(1, round(height * scale))
batched_image = F.interpolate(
batched_image.movedim(-1, 1),
size=(target_height, target_width),
mode="bilinear",
align_corners=False,
).movedim(1, -1)
resized_images.append(
batched_image.squeeze(0) if image.ndim == 3 else batched_image
)
return io.NodeOutput(resized_images)
TYPE_MAP = {"flf": 0, "fmlf": 1, "ref": 2}
class TimelineSegmentOutput(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy timelineSegmentOutput",
display_name="Timeline Segment Output",
category=CATEGORY_TIMELINE,
description="Output data for a specific segment from the timeline.",
inputs=[
TYPE_TIMELINE_INFO.Input("timeline_info"),
io.Combo.Input(
"prompt_format",
options=PROMPT_FORMAT_OPTIONS,
default="default",
tooltip="Choose prompt format. promptRelay formats prompts with frame ranges.",
),
io.Image.Input("images", optional=True),
io.Audio.Input("audio", optional=True),
io.Int.Input("segment_index", default=0, min=0),
],
outputs=[
io.String.Output("PROMPT"),
io.Int.Output("TYPE"),
io.Boolean.Output("NO_IMAGES"),
io.String.Output("IMAGE_INDEXES"),
io.Int.Output("LENGTH"),
io.Image.Output("IMAGES"),
io.Audio.Output("AUDIO"),
],
)
@classmethod
def execute(
cls,
timeline_info: str | dict,
prompt_format: str,
segment_index: int,
images: 'torch.Tensor | None' = None,
audio: dict | None = None,
) -> io.NodeOutput:
if isinstance(timeline_info, str):
try:
info = json.loads(timeline_info)
except json.JSONDecodeError:
info = {}
else:
info = dict(timeline_info) if timeline_info else {}
segments: list[dict] = info.get("segments", [])
height = info.get("height", 960)
width = info.get("width", 544)
# Clamp index to valid range
segment_index = max(0, min(segment_index, len(segments) - 1))
seg = segments[segment_index] if segments else {}
seg_images = seg.get("images", [])
start_frame = seg.get("start_frame", 0)
end_frame = seg.get("end_frame", 0)
no_images = len(seg_images) == 0
seg_type_str = seg.get("type", "flf")
seg_type = TYPE_MAP.get(seg_type_str, 0)
raw_prompt = seg.get("prompt", "") or ""
if prompt_format == "promptRelay" and raw_prompt.strip():
parts = [p.strip() for p in raw_prompt.split("|") if p.strip()]
prompt_parts: list[str] = []
for i, p in enumerate(parts):
if i < len(seg_images):
img = seg_images[i]
img_start = img.get("start_frame")
img_end = img.get("end_frame")
if img_start is not None and img_end is not None:
prompt_parts.append(f"{p} [{int(img_start)}-{int(img_end)}]")
prompt = " | ".join(prompt_parts)
else:
prompt = raw_prompt.split('|') if len(seg_images) == 1 and seg_type <= 1 and "|" in raw_prompt else raw_prompt
audio_segments = info.get("audio", {}).get("segments", [])
frame_rate = info.get("frame_rate", 30)
# Calculate segment length (frame count)
if seg_images:
duration_frames = max(0, end_frame - start_frame)
segment_length = duration_frames + 1
elif segment_index < len(audio_segments):
duration_frames = max(
0.0,
float(audio_segments[segment_index].get("duration", 0.0)) * frame_rate,
)
segment_length = int(duration_frames)
else:
duration_frames = None
segment_length = 0
if info.get("format") == "MiniMax" and duration_frames is not None:
segment_length = _video_frame_count_from_duration(
duration_frames,
frame_rate,
"MiniMax",
)
# Output images from segment (based on images array in segment)
num_seg_images = len(seg_images)
if images is not None and isinstance(images, torch.Tensor) and num_seg_images > 0:
# Calculate offset: sum of images in all previous segments
offset = sum(len(segments[i].get("images", [])) for i in range(segment_index))
if offset + num_seg_images <= images.shape[0]:
images_out = images[offset:offset + num_seg_images]
else:
images_out = images[offset:]
images_indexes_str = ",".join(str(int(img.get("start_frame", 0))) for img in seg_images)
else:
images_out = torch.zeros(1, height, width, 3)
images_indexes_str = ""
# Output audio from segment (trimmed by segment index)
if audio is not None and isinstance(audio, dict):
waveform = audio.get("waveform")
sample_rate = audio.get("sample_rate", 44100)
if waveform is not None and isinstance(waveform, torch.Tensor):
if segment_index < len(audio_segments):
seg_audio = audio_segments[segment_index]
audio_out = trim_audio(
{"waveform": waveform, "sample_rate": sample_rate},
seg_audio["start_sec"],
seg_audio["duration"],
)
else:
audio_out = {"waveform": waveform, "sample_rate": sample_rate}
else:
audio_out = {"waveform": None, "sample_rate": sample_rate}
else:
audio_out = {"waveform": None, "sample_rate": 44100}
return io.NodeOutput(
prompt,
seg_type,
no_images,
images_indexes_str,
segment_length,
images_out,
audio_out,
)
class TimelineSegmentCount(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy timelineSegmentCount",
display_name="Timeline Segment Count",
category=CATEGORY_TIMELINE,
description="Output the total number of segments in the timeline.",
inputs=[
TYPE_TIMELINE_INFO.Input("timeline_info"),
],
outputs=[
io.Int.Output("COUNT"),
],
)
@classmethod
def execute(cls, timeline_info: str | dict) -> io.NodeOutput:
if isinstance(timeline_info, str):
try:
info = json.loads(timeline_info)
except json.JSONDecodeError:
info = {}
else:
info = dict(timeline_info) if timeline_info else {}
count: int = len(info.get("segments", []))
return io.NodeOutput(count)
# ---------------------------------------------------------------------------
# Tuple builder nodes
# ---------------------------------------------------------------------------
class MakeAudioList(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy makeAudioList",
display_name="Make Audio List",
category=CATEGORY_AUDIO,
description="Combine up to 10 optional audio inputs into an audio list.",
inputs=[
io.Boolean.Input("skip_empty", default=False, label_on="Skip", label_off="Fill"),
io.Audio.Input("audio1", optional=True),
io.Audio.Input("audio2", optional=True),
io.Audio.Input("audio3", optional=True),
io.Audio.Input("audio4", optional=True),
io.Audio.Input("audio5", optional=True),
io.Audio.Input("audio6", optional=True),
io.Audio.Input("audio7", optional=True),
io.Audio.Input("audio8", optional=True),
io.Audio.Input("audio9", optional=True),
io.Audio.Input("audio10", optional=True),
],
outputs=[
io.Audio.Output("AUDIO", is_output_list=True),
],
)
@classmethod
def execute(cls, skip_empty: bool, **kwargs: object) -> io.NodeOutput:
audios: list[dict] = []
for i in range(1, 11):
key = f"audio{i}"
v = kwargs.get(key)
if v is not None:
audios.append(v)
elif not skip_empty:
empty = silence(16000, 0.001, 1)
audios.append({"waveform": empty, "sample_rate": 16000})
return io.NodeOutput(audios)
class SplitAudios(io.ComfyNode):
@classmethod
def define_schema(cls) -> io.Schema:
return io.Schema(
node_id="easy splitAudios",
display_name="Split Audios",
category=CATEGORY_AUDIO,
description="Split an audio list into 10 single-audio outputs.",
is_input_list=True,
inputs=[
io.Audio.Input("audios"),
],
outputs=[
io.Audio.Output(f"AUDIO{i}") for i in range(0, 10)
],
)
@classmethod
def execute(cls, audios: list[dict[str, object]]) -> io.NodeOutput:
if not audios:
raise ValueError("audios must contain at least one audio.")
if not all(isinstance(audio, dict) for audio in audios):
raise TypeError("audios must contain only audio dictionaries.")
return io.NodeOutput(*split_list_outputs(audios))
class MatchLine(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy matchLine",
display_name="Match Line",
category=CATEGORY_LOGIC,
description="Return the zero-based index of the first line containing the match text.",
inputs=[
io.String.Input("text", default="", multiline=True),
io.String.Input("match", default=""),
],
outputs=[
io.Int.Output("LINE_INDEX"),
],
)
@classmethod
def execute(cls, text: str, match: str) -> io.NodeOutput:
if not match:
return io.NodeOutput(-1)
line_index = next(
(index for index, line in enumerate(text.splitlines()) if match in line),
-1,
)
return io.NodeOutput(line_index)
class APIWorkflowGate(io.ComfyNode):
@classmethod
def define_schema(cls):
return io.Schema(
node_id="easy apiWorkflowGate",
display_name="API Workflow Gate",
category=CATEGORY_LOGIC,
description=(
"Pass the input through for API workflow execution; return None without "
"evaluating the input when execution includes workflow metadata."
),
inputs=[
io.AnyType.Input(
"value",
optional=True,
lazy=True,
tooltip="Any input to evaluate only when the execution prompt is API workflow format.",
),
],
hidden=[io.Hidden.extra_pnginfo],
outputs=[
io.AnyType.Output("VALUE"),
io.AnyType.Output("VALUES", is_output_list=True),
],
)
@classmethod
def _is_workflow_format(cls) -> bool:
hidden = getattr(cls, "hidden", None)
extra_pnginfo = getattr(hidden, "extra_pnginfo", None)
return _is_workflow_format(extra_pnginfo)
@classmethod
def check_lazy_status(cls, value: object | None = None) -> list[str]:
if cls._is_workflow_format():
return []
if value is None:
return ["value"]
return []
@classmethod
def execute(cls, value: object | None = None) -> io.NodeOutput:
if cls._is_workflow_format():
return io.NodeOutput(None, [])
if isinstance(value, list):
return io.NodeOutput(None, value)
return io.NodeOutput(value, [])