276 lines
11 KiB
Python
276 lines
11 KiB
Python
"""Real saved-source checks and explicitly synthetic mutations; no target execution."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import ast
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from scripts import audit_reference_conversion as audit
|
|
|
|
from calliope.comfyui.serialization_reference import ASR_PROFILE, PREVIEW_PROFILE
|
|
from calliope.comfyui.ui2api import Conversion, convert, convert_reference
|
|
|
|
REPO = Path(__file__).resolve().parents[2]
|
|
SOURCE = "example_ComfyUI_workflows/1-语音生成/Qwen3-ASR-语音转文字.json"
|
|
|
|
|
|
def graph():
|
|
return json.loads((REPO / SOURCE).read_bytes())
|
|
|
|
|
|
def test_real_source_matches_fixed_original_contract_independently():
|
|
source = graph()
|
|
original = copy.deepcopy(source)
|
|
result = convert_reference(source, ASR_PROFILE)
|
|
assert result.prompt is not None and result.reference_only
|
|
assert result.serialization_stage == "restored_saved_state_before_queue_hooks"
|
|
assert source == original and convert(source).prompt is None
|
|
# Parse syntax only: never import or execute upstream plugin.
|
|
tree = ast.parse((REPO / "docs/hel-892/structure-reference/qwen-b037d7be.py").read_text())
|
|
cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "AILab_Qwen3ASR")
|
|
method = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "INPUT_TYPES")
|
|
schema = next(n.value for n in ast.walk(method) if isinstance(n, ast.Return))
|
|
optional = next(v for k, v in zip(schema.keys, schema.values) if k.value == "optional")
|
|
fields = [k.value for k in optional.keys]
|
|
assert fields == ["model", "precision", "language", "hints", "unload_models"]
|
|
assert any(
|
|
isinstance(n, ast.Assign)
|
|
and any(isinstance(t, ast.Name) and t.id == "RETURN_TYPES" for t in n.targets)
|
|
and ast.literal_eval(n.value) == ("STRING",)
|
|
for n in cls.body
|
|
)
|
|
for node in source["nodes"]:
|
|
nid = str(node["id"])
|
|
if node["type"] == "MarkdownNote":
|
|
assert nid not in result.prompt
|
|
continue
|
|
actual = result.prompt[nid]
|
|
assert actual["class_type"] == node["type"]
|
|
assert actual["_meta"] == {"title": node.get("title") or node["type"]}
|
|
if node["type"] == "AILab_Qwen3ASR":
|
|
for index, name in enumerate(fields):
|
|
assert actual["inputs"][name] == node["widgets_values"][index]
|
|
assert type(actual["inputs"][name]) is type(node["widgets_values"][index])
|
|
if node["type"] == "LoadAudio":
|
|
assert actual["inputs"] == {"audio": node["widgets_values"][0], "audioUI": ""}
|
|
if node["type"] == "PreviewAny":
|
|
assert actual["inputs"]["preview"] == ""
|
|
assert actual["inputs"]["previewMode"] is None
|
|
for _, upstream, slot, target, input_slot, _ in source["links"]:
|
|
target_node = next(n for n in source["nodes"] if n["id"] == target)
|
|
name = target_node["inputs"][input_slot]["name"]
|
|
assert result.prompt[str(target)]["inputs"][name] == [str(upstream), slot]
|
|
|
|
|
|
@pytest.mark.parametrize("hints,unload", [("", False), ("第一行\n第二行", True), ("Qwen", False)])
|
|
def test_two_instances_propagate_changed_values_without_cross_talk(hints, unload):
|
|
source = graph()
|
|
source["nodes"][0]["widgets_values"][3:] = [hints, unload]
|
|
source["nodes"][3]["widgets_values"][3:] = ["other\nvalue", not unload]
|
|
prompt = convert_reference(source, ASR_PROFILE).prompt
|
|
assert prompt["32"]["inputs"]["hints"] == hints
|
|
assert prompt["32"]["inputs"]["unload_models"] is unload
|
|
assert prompt["34"]["inputs"]["hints"] == "other\nvalue"
|
|
assert prompt["34"]["inputs"]["unload_models"] is not unload
|
|
assert "normalize_text" not in prompt["32"]["inputs"]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"mutation",
|
|
[
|
|
"extra",
|
|
"missing",
|
|
"boolean_integer",
|
|
"text_null",
|
|
"precision",
|
|
"language",
|
|
"provider",
|
|
"version",
|
|
"mode",
|
|
"widget_binding",
|
|
"input_type",
|
|
"missing_socket",
|
|
"extra_socket",
|
|
"output",
|
|
"custom",
|
|
"frontend",
|
|
"missing_frontend",
|
|
"bad_link",
|
|
"audio_extra",
|
|
"preview_extra",
|
|
"subtitle",
|
|
],
|
|
)
|
|
def test_invalid_assumptions_never_emit_partial_prompt(mutation):
|
|
source = graph()
|
|
node = source["nodes"][0]
|
|
if mutation == "extra":
|
|
node["widgets_values"].append(True)
|
|
elif mutation == "missing":
|
|
node["widgets_values"].pop()
|
|
elif mutation == "boolean_integer":
|
|
node["widgets_values"][4] = 1
|
|
elif mutation == "text_null":
|
|
node["widgets_values"][3] = None
|
|
elif mutation == "precision":
|
|
node["widgets_values"][1] = "unknown"
|
|
elif mutation == "language":
|
|
node["widgets_values"][2] = "unknown"
|
|
elif mutation == "provider":
|
|
node["properties"]["cnr_id"] = "other"
|
|
elif mutation == "version":
|
|
node["properties"]["ver"] = "new"
|
|
elif mutation == "mode":
|
|
node["mode"] = 2
|
|
elif mutation == "widget_binding":
|
|
node["inputs"][4]["widget"] = {"name": "model"}
|
|
elif mutation == "input_type":
|
|
node["inputs"][4]["type"] = "COMBO"
|
|
elif mutation == "missing_socket":
|
|
node["inputs"].pop()
|
|
elif mutation == "extra_socket":
|
|
node["inputs"].append(copy.deepcopy(node["inputs"][1]))
|
|
elif mutation == "output":
|
|
node["outputs"].append({"type": "STRING", "links": []})
|
|
elif mutation == "custom":
|
|
node["serializeValue"] = "unknown"
|
|
elif mutation == "frontend":
|
|
source["extra"]["frontendVersion"] = "1.53.6"
|
|
elif mutation == "missing_frontend":
|
|
source["extra"].pop("frontendVersion")
|
|
elif mutation == "bad_link":
|
|
source["links"][0][2] = 1
|
|
elif mutation == "audio_extra":
|
|
source["nodes"][1]["widgets_values"].append("extra")
|
|
elif mutation == "preview_extra":
|
|
source["nodes"][2]["widgets_values"].append(None)
|
|
elif mutation == "subtitle":
|
|
node["type"] = "AILab_Qwen3ASRSubtitle"
|
|
result = convert_reference(source, ASR_PROFILE)
|
|
assert result.prompt is None and result.issues
|
|
|
|
|
|
def test_actual_library_partition_delta_and_no_partial_candidates():
|
|
report = audit.audit_library(REPO)
|
|
old = audit.audit_library(REPO, tuple(p for p in audit.REFERENCE_PROFILES if p != ASR_PROFILE))
|
|
paths = {r["source_path"] for r in report["cases"]}
|
|
assert report["source_total"] == len(paths) + len(report["api_paths"]) + len(report["excluded"])
|
|
assert not paths.intersection(report["excluded"] + report["api_paths"])
|
|
newly_complete = {r["source_path"] for r in report["cases"] if r["complete_profiles"]} - {
|
|
r["source_path"] for r in old["cases"] if r["complete_profiles"]
|
|
}
|
|
assert newly_complete == {SOURCE}
|
|
assert len([r for r in report["cases"] if r["cohort"] == "with_companion"]) == 6
|
|
for row in report["cases"]:
|
|
assert (
|
|
hashlib.sha256((REPO / row["source_path"]).read_bytes()).hexdigest()
|
|
== row["source_sha256"]
|
|
)
|
|
assert not row["production_prompt_available"]
|
|
if row["cohort"] == "without_companion":
|
|
assert row["comparison_status"] == "comparison_unavailable"
|
|
assert "comparisons" not in row
|
|
for attempt in row["attempts"]:
|
|
assert ("candidate_prompt" in attempt) == (
|
|
attempt["status"] == "complete_reference_candidate"
|
|
)
|
|
if "candidate_prompt" in attempt:
|
|
assert (
|
|
audit.canonical_sha(attempt["candidate_prompt"]) == attempt["candidate_sha256"]
|
|
)
|
|
assert all(
|
|
r["status"] in ("blocked", "complete_reference_candidate", "candidate_conflict")
|
|
for r in report["cases"]
|
|
)
|
|
|
|
|
|
def synthetic_repo(tmp_path, monkeypatch):
|
|
"""Explicitly synthetic tiny manifest, not a real export/history."""
|
|
(tmp_path / "docs/hel-892/offline").mkdir(parents=True)
|
|
(tmp_path / "example_ComfyUI_workflows").mkdir()
|
|
raw = json.dumps(graph()).encode()
|
|
(tmp_path / "example_ComfyUI_workflows/synthetic.json").write_bytes(raw)
|
|
(tmp_path / "docs/hel-892/offline/manifest.json").write_text(
|
|
json.dumps(
|
|
[
|
|
{
|
|
"path": "example_ComfyUI_workflows/synthetic.json",
|
|
"source_sha": hashlib.sha1(raw).hexdigest(),
|
|
},
|
|
]
|
|
)
|
|
)
|
|
monkeypatch.setattr(audit, "load_companions", lambda _: {})
|
|
return tmp_path
|
|
|
|
|
|
def test_source_manifest_sha_mismatch_rejected(tmp_path, monkeypatch):
|
|
repo = synthetic_repo(tmp_path, monkeypatch)
|
|
(repo / "example_ComfyUI_workflows/synthetic.json").write_text("{}")
|
|
with pytest.raises(ValueError, match="SHA"):
|
|
audit.audit_library(repo)
|
|
|
|
|
|
def test_profile_conflicts_are_reported_not_selected(tmp_path, monkeypatch):
|
|
repo = synthetic_repo(tmp_path, monkeypatch)
|
|
|
|
def conflict(source, profile):
|
|
return Conversion({"synthetic": {"class_type": profile, "inputs": {}}}, reference_only=True)
|
|
|
|
monkeypatch.setattr(audit, "convert_reference", conflict)
|
|
monkeypatch.setattr(audit, "asr_field_evidence", lambda *_: [])
|
|
row = audit.audit_library(repo, (ASR_PROFILE, PREVIEW_PROFILE))["cases"][0]
|
|
assert row["status"] == "candidate_conflict"
|
|
assert len(row["candidate_hashes"]) == 2
|
|
assert len(row["complete_profiles"]) == 2
|
|
|
|
|
|
def test_explicit_frontend_conflict_skips_conversion():
|
|
source = graph()
|
|
source["extra"]["frontendVersion"] = "unknown"
|
|
result = audit.profile_attempt(source, ASR_PROFILE)
|
|
assert result["status"] == "not_applicable" and "candidate_prompt" not in result
|
|
|
|
|
|
def test_identical_candidates_keep_both_assumptions(tmp_path, monkeypatch):
|
|
repo = synthetic_repo(tmp_path, monkeypatch)
|
|
monkeypatch.setattr(
|
|
audit,
|
|
"convert_reference",
|
|
lambda *_: Conversion(
|
|
{"synthetic": {"class_type": "same", "inputs": {}}}, reference_only=True
|
|
),
|
|
)
|
|
monkeypatch.setattr(audit, "asr_field_evidence", lambda *_: [])
|
|
row = audit.audit_library(repo, (ASR_PROFILE, PREVIEW_PROFILE))["cases"][0]
|
|
assert row["status"] == "complete_reference_candidate"
|
|
assert len(row["complete_profiles"]) == 2 and len(row["candidate_hashes"]) == 1
|
|
|
|
|
|
def test_changed_dynamic_model_string_and_socket_order_follow_fixed_constructor():
|
|
source = graph()
|
|
source["nodes"][0]["widgets_values"] = ["unverified/model", "fp32", "Chinese", "", False]
|
|
# Keep the connected audio socket first; reorder only named widget sockets.
|
|
node = source["nodes"][0]
|
|
node["inputs"] = node["inputs"][:1] + list(reversed(node["inputs"][1:]))
|
|
result = audit.profile_attempt(source, ASR_PROFILE)
|
|
assert result["status"] == "complete_reference_candidate"
|
|
values = result["candidate_prompt"]["32"]["inputs"]
|
|
assert values["model"] == "unverified/model"
|
|
assert values["precision"] == "fp32" and values["language"] == "Chinese"
|
|
assert values["unload_models"] is False
|
|
assert result["assumptions"]["dynamic_options_and_files_available"] == "unknown"
|
|
evidence = result["field_evidence"]
|
|
assert len(evidence) == sum(len(n["inputs"]) for n in result["candidate_prompt"].values())
|
|
|
|
|
|
def test_manifest_cannot_hide_new_source_file(tmp_path, monkeypatch):
|
|
repo = synthetic_repo(tmp_path, monkeypatch)
|
|
(repo / "example_ComfyUI_workflows/unlisted.json").write_text("{}")
|
|
with pytest.raises(ValueError, match="集合"):
|
|
audit.audit_library(repo)
|