Files
aigc-home/calliope-backend/tests/test_asr_library_reference.py
T

276 lines
11 KiB
Python

"""Real saved-source checks and explicitly synthetic mutations; no target execution."""
from __future__ import annotations
import ast
import copy
import hashlib
import json
from pathlib import Path
import pytest
from scripts import audit_reference_conversion as audit
from calliope.comfyui.serialization_reference import ASR_PROFILE, PREVIEW_PROFILE
from calliope.comfyui.ui2api import Conversion, convert, convert_reference
REPO = Path(__file__).resolve().parents[2]
SOURCE = "example_ComfyUI_workflows/1-语音生成/Qwen3-ASR-语音转文字.json"
def graph():
return json.loads((REPO / SOURCE).read_bytes())
def test_real_source_matches_fixed_original_contract_independently():
source = graph()
original = copy.deepcopy(source)
result = convert_reference(source, ASR_PROFILE)
assert result.prompt is not None and result.reference_only
assert result.serialization_stage == "restored_saved_state_before_queue_hooks"
assert source == original and convert(source).prompt is None
# Parse syntax only: never import or execute upstream plugin.
tree = ast.parse((REPO / "docs/hel-892/structure-reference/qwen-b037d7be.py").read_text())
cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == "AILab_Qwen3ASR")
method = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "INPUT_TYPES")
schema = next(n.value for n in ast.walk(method) if isinstance(n, ast.Return))
optional = next(v for k, v in zip(schema.keys, schema.values) if k.value == "optional")
fields = [k.value for k in optional.keys]
assert fields == ["model", "precision", "language", "hints", "unload_models"]
assert any(
isinstance(n, ast.Assign)
and any(isinstance(t, ast.Name) and t.id == "RETURN_TYPES" for t in n.targets)
and ast.literal_eval(n.value) == ("STRING",)
for n in cls.body
)
for node in source["nodes"]:
nid = str(node["id"])
if node["type"] == "MarkdownNote":
assert nid not in result.prompt
continue
actual = result.prompt[nid]
assert actual["class_type"] == node["type"]
assert actual["_meta"] == {"title": node.get("title") or node["type"]}
if node["type"] == "AILab_Qwen3ASR":
for index, name in enumerate(fields):
assert actual["inputs"][name] == node["widgets_values"][index]
assert type(actual["inputs"][name]) is type(node["widgets_values"][index])
if node["type"] == "LoadAudio":
assert actual["inputs"] == {"audio": node["widgets_values"][0], "audioUI": ""}
if node["type"] == "PreviewAny":
assert actual["inputs"]["preview"] == ""
assert actual["inputs"]["previewMode"] is None
for _, upstream, slot, target, input_slot, _ in source["links"]:
target_node = next(n for n in source["nodes"] if n["id"] == target)
name = target_node["inputs"][input_slot]["name"]
assert result.prompt[str(target)]["inputs"][name] == [str(upstream), slot]
@pytest.mark.parametrize("hints,unload", [("", False), ("第一行\n第二行", True), ("Qwen", False)])
def test_two_instances_propagate_changed_values_without_cross_talk(hints, unload):
source = graph()
source["nodes"][0]["widgets_values"][3:] = [hints, unload]
source["nodes"][3]["widgets_values"][3:] = ["other\nvalue", not unload]
prompt = convert_reference(source, ASR_PROFILE).prompt
assert prompt["32"]["inputs"]["hints"] == hints
assert prompt["32"]["inputs"]["unload_models"] is unload
assert prompt["34"]["inputs"]["hints"] == "other\nvalue"
assert prompt["34"]["inputs"]["unload_models"] is not unload
assert "normalize_text" not in prompt["32"]["inputs"]
@pytest.mark.parametrize(
"mutation",
[
"extra",
"missing",
"boolean_integer",
"text_null",
"precision",
"language",
"provider",
"version",
"mode",
"widget_binding",
"input_type",
"missing_socket",
"extra_socket",
"output",
"custom",
"frontend",
"missing_frontend",
"bad_link",
"audio_extra",
"preview_extra",
"subtitle",
],
)
def test_invalid_assumptions_never_emit_partial_prompt(mutation):
source = graph()
node = source["nodes"][0]
if mutation == "extra":
node["widgets_values"].append(True)
elif mutation == "missing":
node["widgets_values"].pop()
elif mutation == "boolean_integer":
node["widgets_values"][4] = 1
elif mutation == "text_null":
node["widgets_values"][3] = None
elif mutation == "precision":
node["widgets_values"][1] = "unknown"
elif mutation == "language":
node["widgets_values"][2] = "unknown"
elif mutation == "provider":
node["properties"]["cnr_id"] = "other"
elif mutation == "version":
node["properties"]["ver"] = "new"
elif mutation == "mode":
node["mode"] = 2
elif mutation == "widget_binding":
node["inputs"][4]["widget"] = {"name": "model"}
elif mutation == "input_type":
node["inputs"][4]["type"] = "COMBO"
elif mutation == "missing_socket":
node["inputs"].pop()
elif mutation == "extra_socket":
node["inputs"].append(copy.deepcopy(node["inputs"][1]))
elif mutation == "output":
node["outputs"].append({"type": "STRING", "links": []})
elif mutation == "custom":
node["serializeValue"] = "unknown"
elif mutation == "frontend":
source["extra"]["frontendVersion"] = "1.53.6"
elif mutation == "missing_frontend":
source["extra"].pop("frontendVersion")
elif mutation == "bad_link":
source["links"][0][2] = 1
elif mutation == "audio_extra":
source["nodes"][1]["widgets_values"].append("extra")
elif mutation == "preview_extra":
source["nodes"][2]["widgets_values"].append(None)
elif mutation == "subtitle":
node["type"] = "AILab_Qwen3ASRSubtitle"
result = convert_reference(source, ASR_PROFILE)
assert result.prompt is None and result.issues
def test_actual_library_partition_delta_and_no_partial_candidates():
report = audit.audit_library(REPO)
old = audit.audit_library(REPO, tuple(p for p in audit.REFERENCE_PROFILES if p != ASR_PROFILE))
paths = {r["source_path"] for r in report["cases"]}
assert report["source_total"] == len(paths) + len(report["api_paths"]) + len(report["excluded"])
assert not paths.intersection(report["excluded"] + report["api_paths"])
newly_complete = {r["source_path"] for r in report["cases"] if r["complete_profiles"]} - {
r["source_path"] for r in old["cases"] if r["complete_profiles"]
}
assert newly_complete == {SOURCE}
assert len([r for r in report["cases"] if r["cohort"] == "with_companion"]) == 6
for row in report["cases"]:
assert (
hashlib.sha256((REPO / row["source_path"]).read_bytes()).hexdigest()
== row["source_sha256"]
)
assert not row["production_prompt_available"]
if row["cohort"] == "without_companion":
assert row["comparison_status"] == "comparison_unavailable"
assert "comparisons" not in row
for attempt in row["attempts"]:
assert ("candidate_prompt" in attempt) == (
attempt["status"] == "complete_reference_candidate"
)
if "candidate_prompt" in attempt:
assert (
audit.canonical_sha(attempt["candidate_prompt"]) == attempt["candidate_sha256"]
)
assert all(
r["status"] in ("blocked", "complete_reference_candidate", "candidate_conflict")
for r in report["cases"]
)
def synthetic_repo(tmp_path, monkeypatch):
"""Explicitly synthetic tiny manifest, not a real export/history."""
(tmp_path / "docs/hel-892/offline").mkdir(parents=True)
(tmp_path / "example_ComfyUI_workflows").mkdir()
raw = json.dumps(graph()).encode()
(tmp_path / "example_ComfyUI_workflows/synthetic.json").write_bytes(raw)
(tmp_path / "docs/hel-892/offline/manifest.json").write_text(
json.dumps(
[
{
"path": "example_ComfyUI_workflows/synthetic.json",
"source_sha": hashlib.sha1(raw).hexdigest(),
},
]
)
)
monkeypatch.setattr(audit, "load_companions", lambda _: {})
return tmp_path
def test_source_manifest_sha_mismatch_rejected(tmp_path, monkeypatch):
repo = synthetic_repo(tmp_path, monkeypatch)
(repo / "example_ComfyUI_workflows/synthetic.json").write_text("{}")
with pytest.raises(ValueError, match="SHA"):
audit.audit_library(repo)
def test_profile_conflicts_are_reported_not_selected(tmp_path, monkeypatch):
repo = synthetic_repo(tmp_path, monkeypatch)
def conflict(source, profile):
return Conversion({"synthetic": {"class_type": profile, "inputs": {}}}, reference_only=True)
monkeypatch.setattr(audit, "convert_reference", conflict)
monkeypatch.setattr(audit, "asr_field_evidence", lambda *_: [])
row = audit.audit_library(repo, (ASR_PROFILE, PREVIEW_PROFILE))["cases"][0]
assert row["status"] == "candidate_conflict"
assert len(row["candidate_hashes"]) == 2
assert len(row["complete_profiles"]) == 2
def test_explicit_frontend_conflict_skips_conversion():
source = graph()
source["extra"]["frontendVersion"] = "unknown"
result = audit.profile_attempt(source, ASR_PROFILE)
assert result["status"] == "not_applicable" and "candidate_prompt" not in result
def test_identical_candidates_keep_both_assumptions(tmp_path, monkeypatch):
repo = synthetic_repo(tmp_path, monkeypatch)
monkeypatch.setattr(
audit,
"convert_reference",
lambda *_: Conversion(
{"synthetic": {"class_type": "same", "inputs": {}}}, reference_only=True
),
)
monkeypatch.setattr(audit, "asr_field_evidence", lambda *_: [])
row = audit.audit_library(repo, (ASR_PROFILE, PREVIEW_PROFILE))["cases"][0]
assert row["status"] == "complete_reference_candidate"
assert len(row["complete_profiles"]) == 2 and len(row["candidate_hashes"]) == 1
def test_changed_dynamic_model_string_and_socket_order_follow_fixed_constructor():
source = graph()
source["nodes"][0]["widgets_values"] = ["unverified/model", "fp32", "Chinese", "", False]
# Keep the connected audio socket first; reorder only named widget sockets.
node = source["nodes"][0]
node["inputs"] = node["inputs"][:1] + list(reversed(node["inputs"][1:]))
result = audit.profile_attempt(source, ASR_PROFILE)
assert result["status"] == "complete_reference_candidate"
values = result["candidate_prompt"]["32"]["inputs"]
assert values["model"] == "unverified/model"
assert values["precision"] == "fp32" and values["language"] == "Chinese"
assert values["unload_models"] is False
assert result["assumptions"]["dynamic_options_and_files_available"] == "unknown"
evidence = result["field_evidence"]
assert len(evidence) == sum(len(n["inputs"]) for n in result["candidate_prompt"].values())
def test_manifest_cannot_hide_new_source_file(tmp_path, monkeypatch):
repo = synthetic_repo(tmp_path, monkeypatch)
(repo / "example_ComfyUI_workflows/unlisted.json").write_text("{}")
with pytest.raises(ValueError, match="集合"):
audit.audit_library(repo)