Files
aigc-home/docs/hel-892/frames-document-evidence/recompute.py
T

383 lines
14 KiB
Python

"""Actual 9db8857 upgrade, one-row protection/rollback, frozen boundaries and packages."""
from __future__ import annotations
import hashlib
import io
import json
import os
import sqlite3
import subprocess
import sys
import tarfile
import tempfile
import zipfile
from pathlib import Path
from build_definition import OUT, PATHS, REPO, canonical
from rebuild import BASE, RESOURCE, build_all
sys.path.insert(0, str(REPO / "calliope-backend"))
def write(name, value):
(OUT / name).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
def require_packages():
dist = REPO / "calliope-backend/dist"
w = sorted(dist.glob("*.whl"))
s = sorted(dist.glob("*.tar.gz"))
if not w or not s:
raise SystemExit(
(
"缺少wheel/sdist包产物,请先在仓库根执行:"
"uv build --project calliop"
"e-backend --offline"
)
)
return w[-1], s[-1]
def main():
from calliope.comfyui.fixed_document import DEFINITIONS
from calliope.comfyui.library import rollback_batch
from scripts.audit_library_documents import audit
wheels, sdists = require_packages()
reference = OUT.parent / "frames-reference"
entries = json.loads((reference / "sources.json").read_bytes())["sources"]
checked_packages = []
for folder, version, key in [
("reverse-reference", "1.7.7", "vhs177"),
("openpose-reference", "1.7.9", "vhs179"),
]:
root = OUT.parent / folder
package = json.loads((root / "vhs-package-index.json").read_bytes())
archive = (root / ("vhs-" + version + ".zip")).read_bytes()
assert hashlib.sha256(archive).hexdigest() == package["sha256"]
with zipfile.ZipFile(io.BytesIO(archive)) as z:
for e in package["members"]:
assert hashlib.sha256(z.read(e["file"])).hexdigest() == e["sha256"]
for e in entries:
if e["id"] == key:
assert z.read(e["file"]) == (reference / e["local_file"]).read_bytes()
checked_packages.append(
dict(
version=version,
members=len(package["members"]),
sha256=package["sha256"],
git_commit_binding="unknown",
)
)
for e in entries:
assert hashlib.sha256((reference / e["local_file"]).read_bytes()).hexdigest() == e["sha256"]
write(
"source-check.json",
dict(
fixed_originals=len(entries),
packages=checked_packages,
local_file_base="docs/hel-892/frames-reference",
imported_upstream=False,
target_installation=False,
),
)
assert (reference / "core-tags.txt").read_text().split() == [
"1c218282369a6cc80651d878fc51fa33d7bf34e2",
"refs/tags/v0.16.3",
"c55fd7481626d8bee8044ea7512ea996d13a1b90",
"refs/tags/v0.3.71",
]
graphs = []
for path in PATHS:
raw = (REPO / path).read_bytes()
g = json.loads(raw)
by_id = {n["id"]: n for n in g["nodes"]}
assert len(g["nodes"]) == 6 and len(g["links"]) == 6
assert sum(len(n["widgets_values"]) for n in g["nodes"]) == 31
assert "frontendVersion" not in g["extra"]
assert {n["id"]: n["mode"] for n in g["nodes"]} == {11: 0, 13: 0, 8: 4, 9: 0, 1: 0, 10: 0}
assert by_id[11]["widgets_values"] == [] and by_id[8]["widgets_values"] == []
assert by_id[13]["widgets_values"] == [2, None]
assert by_id[1]["properties"]["ver"] == by_id[10]["properties"]["ver"] == "1.7.7"
assert by_id[9]["properties"]["ver"] == "1.7.9"
assert (
by_id[11]["properties"]["ver"] == "0.16.3" and by_id[8]["properties"]["ver"] == "0.3.71"
)
assert g["links"] == [
[5, 1, 0, 8, 0, "IMAGE"],
[7, 1, 2, 9, 1, "AUDIO"],
[8, 10, 0, 11, 0, "IMAGE"],
[10, 10, 0, 13, 0, "IMAGE"],
[11, 1, 0, 13, 1, "IMAGE"],
[12, 13, 0, 9, 0, "IMAGE"],
]
for eid, origin, slot, target, inp, typ in g["links"]:
assert eid in by_id[origin]["outputs"][slot]["links"]
assert by_id[origin]["outputs"][slot]["type"] == typ
assert by_id[target]["inputs"][inp]["link"] == eid
assert by_id[target]["inputs"][inp]["type"] == typ
graphs.append(
dict(
source_path=path,
sha1=hashlib.sha1(raw).hexdigest(),
raw_sha256=hashlib.sha256(raw).hexdigest(),
canonical_sha256=hashlib.sha256(canonical(g)).hexdigest(),
nodes=g["nodes"],
links=g["links"],
saved_fields=31,
history_objects=3,
)
)
write("source-graphs.json", graphs)
rebuilt, previous = build_all()
assert rebuilt == json.loads((REPO / RESOURCE).read_bytes())
old_blocks = {
p: r["fixed_version_document"] for p, r in previous.items() if "fixed_version_document" in r
}
assert len(old_blocks) == 21
assert all(rebuilt[p]["fixed_version_document"] == b for p, b in old_blocks.items())
write(
"review-delta.json",
dict(
base=BASE,
changed_paths=PATHS,
other_reviews_unchanged=len(previous) - 1,
old_blocks_unchanged=21,
old_saved=618,
old_notes=15,
new_saved=31,
new_notes=0,
total_documents=22,
total_saved=649,
total_notes=15,
),
)
for p, d in DEFINITIONS.items():
assert hashlib.sha256(canonical(rebuilt[p]["fixed_version_document"])).hexdigest() == d
with tempfile.TemporaryDirectory() as directory:
work = Path(directory)
old = work / "old"
old.mkdir()
data = subprocess.check_output(["git", "archive", BASE, "calliope-backend/src"], cwd=REPO)
with tarfile.open(fileobj=io.BytesIO(data)) as a:
a.extractall(old, filter="data")
db = work / "library.db"
def run(name, historical=False):
out = work / name
env = {
**os.environ,
"PYTHONPATH": str((old if historical else REPO) / "calliope-backend/src"),
}
subprocess.run(
[
sys.executable,
"-m",
"calliope.comfyui.library_cli",
"import",
"--root",
str(REPO / "example_ComfyUI_workflows"),
"--db",
str(db),
"--source-commit",
BASE,
"--companions",
str(REPO / "docs/hel-892/supplied-api/manifest.json"),
"--output-dir",
str(out),
],
cwd=REPO,
env=env,
check=True,
stdout=subprocess.DEVNULL,
)
return json.loads((out / "import_report.json").read_bytes()), out
def rows():
with sqlite3.connect(db) as c:
c.row_factory = sqlite3.Row
return {
r["source_repo_path"]: dict(r) for r in c.execute("SELECT * FROM workflows")
}
first, _ = run("historical", True)
before = rows()
assert first["counts"] == {"imported": 155, "user_excluded": 1}
assert all("fixed_version" not in json.loads(before[p]["doc_json"]) for p in PATHS)
assert all(
json.loads(before[p]["doc_json"])["fixed_version"]["matched"]
for p in DEFINITIONS
if p not in PATHS
)
original = db
protected = []
def protect(name):
nonlocal db
db = work / (name + ".db")
with sqlite3.connect(original) as src, sqlite3.connect(db) as dst:
src.backup(dst)
for p in PATHS:
dst.execute(
"UPDATE workflows SET description=? WHERE source_repo_path=?",
(name, p),
)
snapshot = rows()
r, _ = run(name)
assert r["counts"] == {"conflict": 1, "skipped": 154, "user_excluded": 1}
assert rows() == snapshot and all(len(snapshot[p]) == 29 for p in PATHS)
protected.append(dict(phase=name, counts=r["counts"], whole_rows=1, columns=29))
db = original
protect("user-before")
upgrade, exported = run("upgrade")
after = rows()
assert upgrade["counts"] == {"updated": 1, "skipped": 154, "user_excluded": 1}
for p, row in after.items():
if p not in PATHS:
assert row == before[p]
continue
assert len(row) == 29
assert all(
row[k] == before[p][k]
for k in row.keys() - {"doc_json", "batch_id", "imported_at", "library_fingerprint"}
)
d = json.loads(row["doc_json"])
assert d["fixed_version"]["matched"]
old_doc = json.loads(before[p]["doc_json"])
without = dict(d)
fixed = without.pop("fixed_version")
assert without["steps"] == old_doc["steps"][:-3] + fixed["steps"]
without["steps"] = old_doc["steps"]
assert without.pop("source_review_sha") != old_doc.pop("source_review_sha")
assert without == old_doc
write("actual-" + Path(p).stem + ".json", d)
repeat, _ = run("repeat")
assert repeat["counts"] == {"skipped": 155, "user_excluded": 1}
with sqlite3.connect(db) as c:
c.row_factory = sqlite3.Row
r = rollback_batch(c, upgrade["batch_id"])
assert len(r["files"]) == 1 and all(f["result"] == "restored" for f in r["files"])
assert rows() == before
redo, _ = run("redo")
assert redo["counts"] == upgrade["counts"]
protect("user-after")
db = work / "fresh.db"
fresh, exported = run("fresh")
fresh_repeat, _ = run("fresh-repeat")
assert fresh["counts"] == first["counts"] and fresh_repeat["counts"] == repeat["counts"]
assert fresh["states"] == {"pending_check": 16, "needs_conversion": 139}
companion = 0
for row in rows().values():
d = json.loads(row["doc_json"])
companion += sum(p.get("value_origin") == "supplied_api_companion" for p in d["params"])
assert (
not row["is_enabled"]
and row["doc_completed_at"] is None
and row["exec_verified_at"] is None
)
assert not d["status"]["doc_done"] and not d["status"]["exec_verified"]
assert d["dependencies"]["check_state"] == "unchecked"
report = audit(REPO, exported)
assert (
report["named_values_preserved"] == 1217
and report["unnamed_widget_nodes"] == 3660
and companion == 294
)
assert report["full_documents_completed"] == report["real_executions"] == 0
write("audit-summary.json", {k: v for k, v in report.items() if k != "files"})
write(
"import-protection.json",
dict(
base=BASE,
first=first["counts"],
upgrade=upgrade["counts"],
repeat=repeat["counts"],
restored_rows=1,
untouched_whole_rows=154,
eligibility_unchanged=True,
redo=redo["counts"],
user_protection=protected,
fresh=fresh["counts"],
fresh_repeat=fresh_repeat["counts"],
states=fresh["states"],
source_named_fields=1217,
positional_widgets=3660,
companion_values=companion,
new_saved=31,
new_notes=0,
old_saved=618,
old_notes=15,
overviews=22,
all_disabled=True,
doc_done=0,
executed=0,
),
)
old_reviews = json.loads(
subprocess.check_output(["git", "show", f"{BASE}:{RESOURCE}"], cwd=REPO)
)
current = json.loads((REPO / RESOURCE).read_bytes())
for p in PATHS:
current[p].pop("fixed_version_document")
assert current == old_reviews
boundaries = json.loads(
(REPO / "docs/hel-892/h3-document-evidence/unchanged-boundaries.json").read_bytes()
)
paths = (
[f["path"] for f in boundaries["files"]]
+ boundaries["frozen_conversion_and_audits"]
+ ["calliope-backend/src/calliope/comfyui/library.py"]
)
kept = []
for p in paths:
raw = (REPO / p).read_bytes()
assert raw == subprocess.check_output(["git", "show", f"{BASE}:{p}"], cwd=REPO)
kept.append(dict(path=p, sha256=hashlib.sha256(raw).hexdigest()))
ui = "calliope-web/src/lib/components/WorkflowDocumentDrawer.svelte"
drawer_bytes = (REPO / ui).read_bytes()
assert drawer_bytes == subprocess.check_output(["git", "show", f"{BASE}:{ui}"], cwd=REPO)
write(
"boundaries.json",
dict(
base=BASE,
files=kept,
originals=163,
old_twenty_one_blocks_unchanged=True,
drawer_css_unchanged=True,
drawer_whole_file_unchanged=True,
drawer_sha256=hashlib.sha256(drawer_bytes).hexdigest(),
conversion_rerun=False,
profiles_28_candidates_9_diffs_6="unchanged",
),
)
packages = []
with zipfile.ZipFile(wheels) as w, tarfile.open(sdists) as s:
for name in ["fixed_document.py", "source_reviews.json", "library.py"]:
raw = (REPO / "calliope-backend/src/calliope/comfyui" / name).read_bytes()
assert w.read("calliope/comfyui/" + name) == raw
member = next(n for n in s.getnames() if n.endswith("/src/calliope/comfyui/" + name))
assert s.extractfile(member).read() == raw
packages.append(
dict(
file=name,
sha256=hashlib.sha256(raw).hexdigest(),
wheel=True,
sdist=True,
definitions=22,
)
)
write("package-check.json", packages)
print(
"PASS actual 9db8857 upgrade1/skip154, repe"
"at155, rollback1, edits1/29cols, fresh155,"
" 1217/3660/294, 22 definitions, frozen bou"
"ndaries, wheel/sdist"
)
if __name__ == "__main__":
main()