Files
aigc-home/docs/hel-892/singleloop-document-evidence/recompute.py
T
2026-11-01 17:00:01 +08:00

357 lines
13 KiB
Python

"""Offline batch evidence tooling; no upstream execution."""
from __future__ import annotations
import hashlib
import io
import json
import os
import sqlite3
import subprocess
import sys
import tarfile
import tempfile
import zipfile
from pathlib import Path
from build_definition import OUT, PATHS, REPO, canonical
from rebuild import BASE, RESOURCE, build_all
sys.path.insert(0, str(REPO / "calliope-backend"))
def write(name, value):
(OUT / name).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
def require_packages():
dist = REPO / "calliope-backend/dist"
w = sorted(dist.glob("*.whl"))
s = sorted(dist.glob("*.tar.gz"))
if not w or not s:
raise SystemExit(
"缺少wheel/sdist包产物,请先在仓库根执行:uv bui"
+ "ld --project calliope-backend --"
+ "offline"
)
return (w[-1], s[-1])
def main():
from calliope.comfyui.fixed_document import DEFINITIONS
from calliope.comfyui.library import rollback_batch
from scripts.audit_library_documents import audit
(wheels, sdists) = require_packages()
from verify_sources import main as verify_sources
verify_sources()
(rebuilt, previous) = build_all()
assert rebuilt == json.loads((REPO / RESOURCE).read_bytes())
old_blocks = {
p: r["fixed_version_document"]
for (p, r) in previous.items()
if "fixed_version_document" in r
}
assert len(old_blocks) == 137
assert all((rebuilt[p]["fixed_version_document"] == b for (p, b) in old_blocks.items()))
assert sum((len(b["fields"]) for b in old_blocks.values())) == 8351
for p, d in DEFINITIONS.items():
assert hashlib.sha256(canonical(rebuilt[p]["fixed_version_document"])).hexdigest() == d
write(
"review-delta.json",
{
"base": BASE,
"changed_paths": PATHS,
"other_reviews_unchanged": 155,
"old_blocks_unchanged": 137,
"old_saved": 8351,
"old_notes": 241,
"new_saved": 129,
"new_notes": 5,
"total_documents": 138,
"total_saved": 8480,
"total_notes": 246,
},
)
with tempfile.TemporaryDirectory() as directory:
work = Path(directory)
old = work / "old"
old.mkdir()
data = subprocess.check_output(["git", "archive", BASE, "calliope-backend/src"], cwd=REPO)
with tarfile.open(fileobj=io.BytesIO(data)) as a:
a.extractall(old, filter="data")
db = work / "library.db"
def run(name, historical=False):
out = work / name
env = {
**os.environ,
"PYTHONPATH": str((old if historical else REPO) / "calliope-backend/src"),
}
subprocess.run(
[
sys.executable,
"-m",
"calliope.comfyui.library_cli",
"import",
"--root",
str(REPO / "example_ComfyUI_workflows"),
"--db",
str(db),
"--source-commit",
BASE,
"--companions",
str(REPO / ("docs/hel-892/supplied-api/mani" + "fe" + "st.json")),
"--output-dir",
str(out),
],
cwd=REPO,
env=env,
check=True,
stdout=subprocess.DEVNULL,
)
return (json.loads((out / "import_report.json").read_bytes()), out)
def rows():
with sqlite3.connect(db) as c:
c.row_factory = sqlite3.Row
return {
r["source_repo_path"]: dict(r) for r in c.execute("SELECT * FROM workflows")
}
(first, _) = run("historical", True)
before = rows()
assert first["counts"] == {"imported": 155, "user_excluded": 1}
assert all(
(
"fixed_version" in json.loads(before[p]["doc_json"])
if BASE.startswith("3c307e0")
else "fixed_version" not in json.loads(before[p]["doc_json"])
for p in PATHS
)
)
assert all(
(
json.loads(before[p]["doc_json"])["fixed_version"]["matched"]
for p in DEFINITIONS
if p not in PATHS
)
)
original = db
protected = []
def protect(name):
nonlocal db
db = work / (name + ".db")
with sqlite3.connect(original) as src, sqlite3.connect(db) as dst:
src.backup(dst)
for p in PATHS:
dst.execute(
"UPDATE workflows SET descripti" + "on" + "=? WHERE source_repo_path=?",
(name, p),
)
snapshot = rows()
(r, _) = run(name)
assert r["counts"] == {"conflict": 1, "skipped": 154, "user_excluded": 1}
assert rows() == snapshot and all((len(snapshot[p]) == 29 for p in PATHS))
protected.append({"phase": name, "counts": r["counts"], "whole_rows": 1, "columns": 29})
db = original
protect("user-before")
(upgrade, exported) = run("upgrade")
after = rows()
assert upgrade["counts"] == {"updated": 1, "skipped": 154, "user_excluded": 1}
for p, row in after.items():
if p not in PATHS:
assert row == before[p]
continue
assert len(row) == 29
assert all(
(
row[k] == before[p][k]
for k in row.keys()
- {"doc_json", "batch_id", "imported_at", "library_fingerprint"}
)
)
d = json.loads(row["doc_json"])
assert d["fixed_version"]["matched"]
old_doc = json.loads(before[p]["doc_json"])
without = dict(d)
fixed = without.pop("fixed_version")
generic = [
f"打开源工作流 {p},逐项核对下列源节点与注释。",
"补齐节点定义、模型来源及所需素材;完成输入输出映射核对。",
"明确触发真实测试后核对产物;本批未提交生成任务。",
]
offset = old_doc["steps"].index(generic[0])
assert old_doc["steps"][offset : offset + 3] == generic
assert (
without["steps"]
== old_doc["steps"][:offset] + fixed["steps"] + old_doc["steps"][offset + 3 :]
)
without["steps"] = old_doc["steps"]
assert without.pop("source_review_sha") != old_doc.pop("source_review_sha")
old_doc.pop("fixed_version", None)
assert without == old_doc
write("actual-" + Path(p).stem + ".json", d)
(repeat, _) = run("repeat")
assert repeat["counts"] == {"skipped": 155, "user_excluded": 1}
with sqlite3.connect(db) as c:
c.row_factory = sqlite3.Row
r = rollback_batch(c, upgrade["batch_id"])
assert len(r["files"]) == 1 and all((f["result"] == "restored" for f in r["files"]))
assert rows() == before
(redo, _) = run("redo")
assert redo["counts"] == upgrade["counts"]
protect("user-after")
db = work / "fresh.db"
(fresh, exported) = run("fresh")
(fresh_repeat, _) = run("fresh-repeat")
assert fresh["counts"] == first["counts"] and fresh_repeat["counts"] == repeat["counts"]
assert fresh["states"] == {"pending_check": 16, "needs_conversion": 139}
companion = 0
for row in rows().values():
d = json.loads(row["doc_json"])
companion += sum(
(p.get("value_origin") == "supplied_api_companion" for p in d["params"])
)
assert (
not row["is_enabled"]
and row["doc_completed_at"] is None
and (row["exec_verified_at"] is None)
)
assert not d["status"]["doc_done"] and (not d["status"]["exec_verified"])
assert d["dependencies"]["check_state"] == "unchecked"
report = audit(REPO, exported)
assert (
report["named_values_preserved"] == 1217
and report["unnamed_widget_nodes"] == 3660
and (companion == 294)
)
assert report["full_documents_completed"] == report["real_executions"] == 0
write("audit-summary.json", {k: v for (k, v) in report.items() if k != "files"})
write(
"import-protection.json",
{
"base": BASE,
"first": first["counts"],
"upgrade": upgrade["counts"],
"repeat": repeat["counts"],
"restored_rows": 1,
"untouched_whole_rows": 154,
"eligibility_unchanged": True,
"redo": redo["counts"],
"user_protection": protected,
"fresh": fresh["counts"],
"fresh_repeat": fresh_repeat["counts"],
"states": fresh["states"],
"source_named_fields": 1217,
"positional_widgets": 3660,
"companion_values": companion,
"new_saved": 129,
"new_notes": 5,
"old_saved": 8351,
"old_notes": 241,
"overviews": 138,
"all_disabled": True,
"doc_done": 0,
"executed": 0,
},
)
old_reviews = json.loads(
subprocess.check_output(["git", "show", f"{BASE}:{RESOURCE}"], cwd=REPO)
)
current = json.loads((REPO / RESOURCE).read_bytes())
for p in PATHS:
current[p].pop("fixed_version_document")
old_reviews[p].pop("fixed_version_document", None)
assert current == old_reviews
boundaries = json.loads(
(
REPO / ("docs/hel-892/h3-document-evide" + "nc" + "e/unchanged-boundaries.json")
).read_bytes()
)
paths = (
[f["path"] for f in boundaries["files"]]
+ boundaries["frozen_conversion_and_audits"]
+ ["calliope-backend/src/calliope/co" + "mfyui/library.py"]
)
kept = []
for p in paths:
raw = (REPO / p).read_bytes()
assert raw == subprocess.check_output(["git", "show", f"{BASE}:{p}"], cwd=REPO)
kept.append({"path": p, "sha256": hashlib.sha256(raw).hexdigest()})
ui = "calliope-web/src/lib/component" + "s/" + "WorkflowDocumentDrawer.svelte"
drawer_bytes = (REPO / ui).read_bytes()
assert drawer_bytes == subprocess.check_output(["git", "show", f"{BASE}:{ui}"], cwd=REPO)
write(
"boundaries.json",
{
"base": BASE,
"files": kept,
"originals": 156,
"old_blocks_unchanged": 137,
"drawer_css_unchanged": True,
"drawer_whole_file_unchanged": True,
"drawer_sha256": hashlib.sha256(drawer_bytes).hexdigest(),
"conversion_rerun": False,
"profiles_28_candidates_9_diffs_6": "unchanged",
},
)
tracked = (
subprocess.check_output(["git", "ls-tree", "-rz", "--name-only", BASE], cwd=REPO)
.decode()
.rstrip("\x00")
.split("\x00")
)
frozen = [
p
for p in tracked
if p.startswith(("example_ComfyUI_workflows/", "docs/hel-892/supplied-api/"))
or (
p.startswith("calliope-backend/src/calliope/" + "co" + "mfyui/")
and p
not in [RESOURCE, "calliope-backend/src/calliope/" + "co" + "mfyui/fixed_document.py"]
)
]
for p in frozen:
assert (REPO / p).read_bytes() == subprocess.check_output(
["git", "show", f"{BASE}:{p}"], cwd=REPO
)
write(
"frozen-resource-check.json",
{
"base": BASE,
"files": [
{"path": p, "sha256": hashlib.sha256((REPO / p).read_bytes()).hexdigest()}
for p in frozen
],
"original_files": sum((p.startswith("example_ComfyUI_workflows/") for p in frozen)),
"all_unchanged": True,
},
)
packages = []
with zipfile.ZipFile(wheels) as w, tarfile.open(sdists) as s:
for name in ["fixed_document.py", "source_reviews.json", "library.py"]:
raw = (REPO / ("calliope-backend/src/calliope/" + "co" + "mfyui") / name).read_bytes()
assert w.read("calliope/comfyui/" + name) == raw
member = next((n for n in s.getnames() if n.endswith("/src/calliope/comfyui/" + name)))
assert s.extractfile(member).read() == raw
packages.append(
{
"file": name,
"sha256": hashlib.sha256(raw).hexdigest(),
"wheel": True,
"sdist": True,
"definitions": 138,
}
)
write("package-check.json", packages)
print(f"PASS actual {BASE} upgrade1/skip154, repeat155, rollback1, edits1/29cols")
print("fresh155, 1217/3660/294, 138 definitions, frozen boundaries, wheel/sdist")
if __name__ == "__main__":
main()