Files
aigc-home/calliope-backend/tests/test_vhs_reference.py
T

389 lines
16 KiB
Python

"""Pinned source bytes and synthetic graphs only; no plugins/media are executed."""
from __future__ import annotations
import ast
import copy
import hashlib
import json
from pathlib import Path
import pytest
from scripts import audit_reference_conversion as audit
from scripts.verify_frontend_sources import verify
from calliope.comfyui.serialization_reference import (
VHS_COMBINE_COMMIT,
VHS_LOAD_COMMIT,
VHS_PROFILES,
serialize_vhs_node,
)
from calliope.comfyui.ui2api import convert, convert_reference
REPO = Path(__file__).resolve().parents[2]
ROOT = REPO / "docs/hel-892/vhs-reference"
PREFIX = "example_ComfyUI_workflows/0-基础处理:图像合并-音频分离-视频截取合并-图像视频放大/"
LOAD = PREFIX + "多视频合成分屏展示.json"
COMBINE = (
"example_ComfyUI_workflows/4-视频生成/Minimax-h3/"
"Minimax_H3_T8双时钟采样_参考生视频-8步高质量极速版.json"
)
def real_node(path, nid):
return next(n for n in json.loads((REPO / path).read_bytes())["nodes"] if n["id"] == nid)
def synthetic():
# This pair is not a source workflow, target export or encoding/history test.
load, combine = real_node(LOAD, 17), real_node(COMBINE, 12)
for n in (load, combine):
n.pop("widgets_values_named", None)
for s in n["inputs"]:
s["link"] = None
for s in n["outputs"]:
s["links"] = []
load["outputs"][0]["links"] = [1]
load["outputs"][2]["links"] = [2]
combine["inputs"][0]["link"] = 1
combine["inputs"][1]["link"] = 2
return {
"nodes": [load, combine],
"links": [[1, 17, 0, 12, 0, "IMAGE"], [2, 17, 2, 12, 1, "AUDIO"]],
}
def test_pinned_bytes_registration_schema_and_js_boundaries():
assert len(verify(ROOT)["sources_verified"]) == len(
json.loads((ROOT / "sources.json").read_bytes())["sources"]
)
for commit, ct, clsname, filename in (
(VHS_COMBINE_COMMIT, "VHS_VideoCombine", "VideoCombine", "nodes.py"),
(VHS_LOAD_COMMIT, "VHS_LoadVideo", "LoadVideoUpload", "load_video_nodes.py"),
):
base = ROOT / commit[:8]
tree = ast.parse((base / "videohelpersuite" / filename).read_bytes())
cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == clsname)
schema = next(
n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "INPUT_TYPES"
)
assert schema.lineno > 0
mappings = (base / "videohelpersuite/nodes.py").read_text()
assert f'"{ct}": {clsname},' in mappings
js = (base / "web/js/VHS.core.js").read_text()
assert "w.value = widgetDict[w.name];" in js
assert "w.callback?.(w.value)" in js
assert "info.widgets_values[w.name] = w.value;" in js
assert "uploadWidget.options.serialize = false;" in js
assert 'this.addDOMWidget("videopreview", "preview", element, {' in js
assert "serialize: false," in js
assert (
'VHS_VideoCombine : ["frame_rate", "loop_count", "filename_prefix", '
'"format", "pingpong", "save_image"]' in js
)
fmt = json.loads((ROOT / "4ee72c06/video_formats/h264-mp4.json").read_bytes())
assert fmt["main_pass"][4] == ["pix_fmt", ["yuv420p", "yuv420p10le"]]
assert fmt["main_pass"][6] == ["crf", "INT", {"default": 19, "min": 0, "max": 100, "step": 1}]
assert fmt["save_metadata"] == ["save_metadata", "BOOLEAN", {"default": True}]
assert fmt["trim_to_audio"] == ["trim_to_audio", "BOOLEAN", {"default": False}]
exe = (ROOT / "frontend-1.48.6/src/utils/executionUtil.ts").read_text()
assert "widget.options?.serialize === false" in exe
assert "await widget.serializeValue(node, i)" in exe
wrapper = (ROOT / "frontend-1.48.6/src/scripts/utils.ts").read_text()
assert "return _applyTextReplacements(app.rootGraph, value)" in wrapper
@pytest.mark.parametrize("profile", VHS_PROFILES)
def test_synthetic_complete_typed_fields_nonzero_audio_slot_and_ui_omission(profile):
source = synthetic()
if VHS_PROFILES[profile] == "1.48.6":
assert convert_reference(source, profile).prompt is None
assert audit.profile_attempt(source, profile)["status"] == "not_applicable"
source["extra"] = {"frontendVersion": VHS_PROFILES[profile]}
result = convert_reference(source, profile)
assert result.prompt is not None and result.reference_only
assert result.prompt["12"]["inputs"]["audio"] == ["17", 2]
assert result.prompt["12"]["inputs"]["images"] == ["17", 0]
assert len(audit.vhs_field_evidence(source, result.prompt)) == 20
assert not any("videopreview" in n["inputs"] for n in result.prompt.values())
assert not convert(source).prompt
before = copy.deepcopy(result.prompt)
source["nodes"][0]["widgets_values"]["videopreview"]["params"]["filename"] = "other.mp4"
source["nodes"][0]["widgets_values"]["choose video to upload"] = "image"
assert convert_reference(source, profile).prompt == before
source["nodes"][0]["widgets_values"]["video"] = "第二个.mp4"
source["nodes"][0]["widgets_values"]["skip_first_frames"] = 41
source["nodes"][1]["widgets_values"]["crf"] = 37
source["nodes"][1]["widgets_values"]["pix_fmt"] = "yuv420p10le"
prompt = convert_reference(source, profile).prompt
assert prompt["17"]["inputs"]["video"] == "第二个.mp4"
assert prompt["17"]["inputs"]["skip_first_frames"] == 41
assert prompt["12"]["inputs"]["crf"] == 37
assert prompt["12"]["inputs"]["pix_fmt"] == "yuv420p10le"
@pytest.mark.parametrize(
"index,field,value",
[
(0, "force_rate", True),
(0, "force_rate", None),
(0, "force_rate", float("inf")),
(0, "force_rate", -1),
(0, "force_rate", 61),
(0, "video", 12),
(0, "custom_width", 513),
(0, "custom_height", 8193),
(0, "custom_width", True),
(0, "frame_load_cap", 1.2),
(0, "frame_load_cap", 2**53),
(0, "skip_first_frames", -1),
(0, "select_every_nth", 0),
(0, "format", "Wan"),
(0, "format", "custom"),
(1, "format", "image/webp"),
(1, "format", "video/custom"),
(1, "pix_fmt", "rgb24"),
(1, "crf", 101),
(1, "crf", True),
(1, "frame_rate", float("nan")),
(1, "frame_rate", 0),
(1, "loop_count", -1),
(1, "loop_count", 1.0),
(1, "pingpong", 1),
(1, "save_output", None),
(1, "trim_to_audio", "false"),
(1, "filename_prefix", "%date:yyyy%"),
(1, "filename_prefix", "{unknown}"),
],
)
def test_invalid_fields_fail_whole_graph_without_partial_prompt(index, field, value):
source = synthetic()
source["nodes"][index]["widgets_values"][field] = value
result = convert_reference(source, next(iter(VHS_PROFILES)))
assert result.prompt is None and result.issues
@pytest.mark.parametrize(
"change",
[
"provider",
"release",
"other_commit",
"aux",
"array",
"extra_value",
"missing_value",
"mirror_type",
"mirror_value",
"binding",
"socket_type",
"extra_socket",
"missing_socket",
"mode",
"bool_mode",
"ue",
"override",
"prop_override",
"vae",
"batch",
"extra_output",
"output_type",
"output_index",
"bool_index",
"bad_source_slot",
"bad_target_slot",
"bad_link_type",
"missing_reciprocal",
"wrong_frontend",
"unknown_node",
"preview_shape",
],
)
def test_contract_mutations_reject(change):
source = synthetic()
n = source["nodes"][0]
if change == "provider":
n["properties"]["cnr_id"] = "same-name-plugin"
elif change == "release":
n["properties"]["ver"] = "1.7.7"
elif change == "other_commit":
n["properties"]["ver"] = VHS_COMBINE_COMMIT
elif change == "aux":
n["properties"]["aux_id"] = "other"
elif change == "array":
n["widgets_values"] = list(n["widgets_values"].values())
elif change == "extra_value":
n["widgets_values"]["custom"] = 1
elif change == "missing_value":
del n["widgets_values"]["format"]
elif change.startswith("mirror"):
n["widgets_values_named"] = copy.deepcopy(n["widgets_values"])
n["widgets_values_named"]["force_rate"] = 16.0 if change == "mirror_type" else 20
elif change == "binding":
n["inputs"][2]["widget"] = {"name": "wrong"}
elif change == "socket_type":
n["inputs"][2]["type"] = "STRING"
elif change == "extra_socket":
n["inputs"].append(copy.deepcopy(n["inputs"][0]))
elif change == "missing_socket":
n["inputs"].pop()
elif change == "mode":
n["mode"] = 4
elif change == "bool_mode":
n["mode"] = False
elif change == "ue":
n["properties"]["ue_properties"]["widget_ue_connectable"] = {"video": True}
elif change == "override":
n["serializeValue"] = "custom"
elif change == "prop_override":
n["properties"]["widgets"] = []
elif change == "vae":
n["inputs"][1]["link"] = 1
elif change == "batch":
n["inputs"][0]["link"] = 1
elif change == "extra_output":
n["outputs"].append({"type": "IMAGE", "links": []})
elif change == "output_type":
n["outputs"][0]["type"] = "LATENT"
elif change == "output_index":
n["outputs"][2]["slot_index"] = 0
elif change == "bool_index":
n["outputs"][0]["slot_index"] = False
elif change == "bad_source_slot":
source["links"][0][2] = 1
elif change == "bad_target_slot":
source["links"][0][4] = 2
elif change == "bad_link_type":
source["links"][0][5] = "LATENT"
elif change == "missing_reciprocal":
n["outputs"][0]["links"] = []
elif change == "wrong_frontend":
source["extra"] = {"frontendVersion": "1.7.9"}
elif change == "unknown_node":
source["nodes"].append({"id": 100, "type": "Unknown", "inputs": [], "outputs": []})
elif change == "preview_shape":
n["widgets_values"]["videopreview"] = "server/path.mp4"
assert convert_reference(source, next(iter(VHS_PROFILES))).prompt is None
def test_real_five_load_instances_are_distinct_and_modes_not_dropped():
source = json.loads((REPO / LOAD).read_bytes())
nodes = [n for n in source["nodes"] if n["type"] == "VHS_LoadVideo"]
assert len(nodes) == 5
for n in nodes:
if n["mode"] == 4:
with pytest.raises(ValueError):
serialize_vhs_node(n)
else:
values = serialize_vhs_node(n)
assert all(
type(v) is type(n["widgets_values"][k]) and v == n["widgets_values"][k]
for k, v in values.items()
)
assert convert_reference(source, next(iter(VHS_PROFILES))).prompt is None
def test_old_sixteen_attempts_gaps_candidates_and_six_comparisons_protected():
old = json.loads((REPO / "docs/hel-892/crop-split-evidence/library.json").read_bytes())
# Freeze this batch's 18-profile history; Set/Get has separate current protection.
historical_profiles = tuple(
p
for p in audit.REFERENCE_PROFILES[:26]
if p
not in (
*audit.SETGET_PROFILES,
*audit.VIDEOINFO_PROFILES,
*audit.SCAIL_LOAD_PROFILES,
audit.GRID_PROFILE,
audit.H3_PROFILE,
)
)
profiles = tuple(p for p in historical_profiles if p not in VHS_PROFILES)
baseline = audit.audit_library(REPO, profiles)
now = audit.audit_library(REPO, historical_profiles, baseline_profiles=profiles)
assert len(profiles) == 16 and len(now["profiles"]) == 18
assert old["gap_groups"] == baseline["gap_groups"]
for previous, before, after in zip(old["cases"], baseline["cases"], now["cases"]):
assert previous["attempts"] == before["attempts"] == after["attempts"][:-2]
assert previous.get("comparisons") == after.get("comparisons")
assert previous["status"] == after["status"]
assert previous["candidate_hashes"] == after["candidate_hashes"]
assert previous["comparison_status"] == after["comparison_status"]
assert len(now["coverage_delta"]["current_complete_paths"]) == 8
assert now["coverage_delta"]["new_complete_paths"] == []
assert now["coverage_delta"]["baseline_profiles"] == list(profiles)
def unavailable(r):
return {
c["source_path"]
for c in r["cases"]
if all(a["status"] == "not_applicable" for a in c["attempts"])
}
assert unavailable(old) == unavailable(now) and len(unavailable(now)) == 98
for raw in json.loads((ROOT / "sources.json").read_bytes())["sources"]:
assert hashlib.sha256((ROOT / raw["local_file"]).read_bytes()).hexdigest() == raw["sha256"]
def test_disjoint_instances_values_ids_links_not_shared():
first, second = synthetic(), synthetic()
for n in second["nodes"]:
n["id"] += 100
for socket in n["inputs"]:
if socket.get("link") is not None:
socket["link"] += 100
for socket in n["outputs"]:
socket["links"] = [lid + 100 for lid in socket.get("links", [])]
for link in second["links"]:
for i in (0, 1, 3):
link[i] += 100
second["nodes"][0]["widgets_values"]["video"] = "other-file.mp4"
second["nodes"][0]["widgets_values"]["force_rate"] = 29.97
second["nodes"][1]["widgets_values"]["frame_rate"] = 29.97
source = {"nodes": first["nodes"] + second["nodes"], "links": first["links"] + second["links"]}
prompt = convert_reference(source, next(iter(VHS_PROFILES))).prompt
assert set(prompt) == {"17", "12", "117", "112"}
assert prompt["112"]["inputs"]["audio"] == ["117", 2]
assert prompt["12"]["inputs"]["audio"] == ["17", 2]
assert prompt["117"]["inputs"]["video"] == "other-file.mp4"
assert prompt["17"]["inputs"]["video"] != "other-file.mp4"
assert prompt["112"]["inputs"]["frame_rate"] == 29.97
assert len(audit.vhs_field_evidence(source, prompt)) == 40
def test_all_five_real_combines_have_node_contract_even_when_other_nodes_close_graph():
seen = []
for path in (REPO / "example_ComfyUI_workflows").rglob("*"):
if not path.is_file():
continue
source = json.loads(path.read_bytes())
for n in source.get("nodes", []):
if n["type"] == "VHS_VideoCombine" and n["properties"].get("ver") == VHS_COMBINE_COMMIT:
frontend = source.get("extra", {}).get("frontendVersion") or "1.37.11"
if frontend not in VHS_PROFILES.values():
assert all(
audit.profile_attempt(source, p)["status"] == "not_applicable"
for p in VHS_PROFILES
)
continue
values = serialize_vhs_node(n)
assert values["crf"] == n["widgets_values"]["crf"]
assert values["frame_rate"] == n["widgets_values"]["frame_rate"]
seen.append((path, n["id"]))
profile = next(
p
for p, v in VHS_PROFILES.items()
if v == (source.get("extra", {}).get("frontendVersion") or "1.37.11")
)
result = convert_reference(source, profile)
assert result.prompt is None
node_issues = [i for i in result.issues if i.get("node_id") == str(n["id"])]
# The source upstream classes still have no schema in this
# minimal profile; preserve those edge errors, not a partial prompt.
assert node_issues
assert {i["code"] for i in node_issues} <= {
"invalid_output_slot",
"missing_required_input",
}
assert len(seen) == 5