Files
aigc-home/docs/hel-892/reverse-document-evidence/recompute.py
T

370 lines
14 KiB
Python

"""Actual ab18cd3 database upgrade, one-row protection/rollback, frozen boundaries and packages."""
from __future__ import annotations
import hashlib
import io
import json
import os
import sqlite3
import subprocess
import sys
import tarfile
import tempfile
import zipfile
from pathlib import Path
from build_definition import OUT, PATHS, REPO, canonical
from rebuild import BASE, RESOURCE, build_all
sys.path.insert(0, str(REPO / "calliope-backend"))
def write(name, value):
(OUT / name).write_text(json.dumps(value, ensure_ascii=False, indent=2) + "\n")
def require_packages():
dist = REPO / "calliope-backend/dist"
w = sorted(dist.glob("*.whl"))
s = sorted(dist.glob("*.tar.gz"))
if not w or not s:
raise SystemExit(
(
"缺少wheel/sdist包产物,请先在仓库根执行:"
"uv build --project calliop"
"e-backend --offline"
)
)
return w[-1], s[-1]
def main():
from calliope.comfyui.fixed_document import DEFINITIONS
from calliope.comfyui.library import rollback_batch
from scripts.audit_library_documents import audit
wheels, sdists = require_packages()
reference = OUT.parent / "reverse-reference"
source_records = json.loads((reference / "sources.json").read_bytes())["sources"]
for e in source_records:
raw = (reference / e["local_file"]).read_bytes()
assert hashlib.sha256(raw).hexdigest() == e["sha256"]
assert len(e["commit"]) == 40 or e["commit"].startswith("发布包SHA256:")
package = json.loads((reference / "vhs-package-index.json").read_bytes())
archive = (reference / "vhs-1.7.7.zip").read_bytes()
assert hashlib.sha256(archive).hexdigest() == package["sha256"]
with zipfile.ZipFile(io.BytesIO(archive)) as z:
for e in package["members"]:
assert hashlib.sha256(z.read(e["file"])).hexdigest() == e["sha256"]
for e in source_records:
if e["id"] == "vhs":
assert z.read(e["file"]) == (reference / e["local_file"]).read_bytes()
assert (reference / "core-tag.txt").read_text().split() == [
"8402c8700a29a97bc5d706d6a0b14c41bc2c2d8a",
"refs/tags/v0.3.75",
]
write(
"source-check.json",
dict(
fixed_originals=len(source_records),
checked_sha256=True,
package_sha256=package["sha256"],
package_members=len(package["members"]),
git_commit_binding="unknown",
frontend_not_declared=True,
imported_upstream=False,
target_installation=False,
local_file_base="docs/hel-892/reverse-reference",
),
)
graph_raw = (REPO / PATHS[0]).read_bytes()
graph = json.loads(graph_raw)
nodes = graph["nodes"]
assert len(nodes) == 4 and len(graph["links"]) == 4
assert "definitions" not in graph and "frontendVersion" not in graph["extra"]
assert sum(len(n["widgets_values"]) for n in nodes) == 21
by_id = {n["id"]: n for n in nodes}
assert all(n["mode"] == 0 for n in nodes)
assert by_id[20]["widgets_values"] == by_id[15]["widgets_values"] == []
assert graph["links"] == [
[15, 1, 0, 20, 0, "IMAGE"],
[16, 20, 0, 15, 0, "IMAGE"],
[17, 20, 0, 21, 0, "IMAGE"],
[18, 1, 2, 21, 1, "AUDIO"],
]
for eid, origin, slot, target, inp, typ in graph["links"]:
assert eid in by_id[origin]["outputs"][slot]["links"]
assert by_id[origin]["outputs"][slot]["type"] == typ
assert by_id[target]["inputs"][inp]["link"] == eid
assert by_id[target]["inputs"][inp]["type"] == typ
write(
"source-graph.json",
dict(
source_path=PATHS[0],
raw_sha256=hashlib.sha256(graph_raw).hexdigest(),
canonical_sha256=hashlib.sha256(canonical(graph)).hexdigest(),
nodes=nodes,
links=graph["links"],
no_subgraphs=True,
saved_fields=21,
scalar_positions=19,
history_objects=2,
notes=0,
extra_sha256=hashlib.sha256(canonical(graph["extra"])).hexdigest(),
),
)
rebuilt, previous = build_all()
assert rebuilt == json.loads((REPO / RESOURCE).read_bytes())
old_blocks = {
p: r["fixed_version_document"] for p, r in previous.items() if "fixed_version_document" in r
}
assert len(old_blocks) == 17
assert all(rebuilt[p]["fixed_version_document"] == b for p, b in old_blocks.items())
write(
"review-delta.json",
dict(
base=BASE,
changed_paths=PATHS,
other_reviews_unchanged=len(previous) - 1,
old_blocks_unchanged=17,
old_saved=495,
old_notes=15,
new_saved=21,
new_notes=0,
total_documents=18,
total_saved=516,
total_notes=15,
),
)
for p, d in DEFINITIONS.items():
assert hashlib.sha256(canonical(rebuilt[p]["fixed_version_document"])).hexdigest() == d
with tempfile.TemporaryDirectory() as directory:
work = Path(directory)
old = work / "old"
old.mkdir()
data = subprocess.check_output(["git", "archive", BASE, "calliope-backend/src"], cwd=REPO)
with tarfile.open(fileobj=io.BytesIO(data)) as a:
a.extractall(old, filter="data")
db = work / "library.db"
def run(name, historical=False):
out = work / name
env = {
**os.environ,
"PYTHONPATH": str((old if historical else REPO) / "calliope-backend/src"),
}
subprocess.run(
[
sys.executable,
"-m",
"calliope.comfyui.library_cli",
"import",
"--root",
str(REPO / "example_ComfyUI_workflows"),
"--db",
str(db),
"--source-commit",
BASE,
"--companions",
str(REPO / "docs/hel-892/supplied-api/manifest.json"),
"--output-dir",
str(out),
],
cwd=REPO,
env=env,
check=True,
stdout=subprocess.DEVNULL,
)
return json.loads((out / "import_report.json").read_bytes()), out
def rows():
with sqlite3.connect(db) as c:
c.row_factory = sqlite3.Row
return {
r["source_repo_path"]: dict(r) for r in c.execute("SELECT * FROM workflows")
}
first, _ = run("historical", True)
before = rows()
assert first["counts"] == {"imported": 155, "user_excluded": 1}
assert all("fixed_version" not in json.loads(before[p]["doc_json"]) for p in PATHS)
assert all(
json.loads(before[p]["doc_json"])["fixed_version"]["matched"]
for p in DEFINITIONS
if p not in PATHS
)
original = db
protected = []
def protect(name):
nonlocal db
db = work / (name + ".db")
with sqlite3.connect(original) as src, sqlite3.connect(db) as dst:
src.backup(dst)
for p in PATHS:
dst.execute(
"UPDATE workflows SET description=? WHERE source_repo_path=?",
(name, p),
)
snapshot = rows()
r, _ = run(name)
assert r["counts"] == {"conflict": 1, "skipped": 154, "user_excluded": 1}
assert rows() == snapshot and all(len(snapshot[p]) == 29 for p in PATHS)
protected.append(dict(phase=name, counts=r["counts"], whole_rows=1, columns=29))
db = original
protect("user-before")
upgrade, exported = run("upgrade")
after = rows()
assert upgrade["counts"] == {"updated": 1, "skipped": 154, "user_excluded": 1}
for p, row in after.items():
if p not in PATHS:
assert row == before[p]
continue
assert len(row) == 29
assert all(
row[k] == before[p][k]
for k in row.keys() - {"doc_json", "batch_id", "imported_at", "library_fingerprint"}
)
d = json.loads(row["doc_json"])
assert d["fixed_version"]["matched"]
old_doc = json.loads(before[p]["doc_json"])
without = dict(d)
fixed = without.pop("fixed_version")
assert without["steps"] == old_doc["steps"][:-3] + fixed["steps"]
without["steps"] = old_doc["steps"]
assert without.pop("source_review_sha") != old_doc.pop("source_review_sha")
assert without == old_doc
write("actual-" + Path(p).stem + ".json", d)
repeat, _ = run("repeat")
assert repeat["counts"] == {"skipped": 155, "user_excluded": 1}
with sqlite3.connect(db) as c:
c.row_factory = sqlite3.Row
r = rollback_batch(c, upgrade["batch_id"])
assert len(r["files"]) == 1 and all(f["result"] == "restored" for f in r["files"])
assert rows() == before
redo, _ = run("redo")
assert redo["counts"] == upgrade["counts"]
protect("user-after")
db = work / "fresh.db"
fresh, exported = run("fresh")
fresh_repeat, _ = run("fresh-repeat")
assert fresh["counts"] == first["counts"] and fresh_repeat["counts"] == repeat["counts"]
assert fresh["states"] == {"pending_check": 16, "needs_conversion": 139}
companion = 0
for row in rows().values():
d = json.loads(row["doc_json"])
companion += sum(p.get("value_origin") == "supplied_api_companion" for p in d["params"])
assert (
not row["is_enabled"]
and row["doc_completed_at"] is None
and row["exec_verified_at"] is None
)
assert not d["status"]["doc_done"] and not d["status"]["exec_verified"]
assert d["dependencies"]["check_state"] == "unchecked"
report = audit(REPO, exported)
assert (
report["named_values_preserved"] == 1217
and report["unnamed_widget_nodes"] == 3660
and companion == 294
)
assert report["full_documents_completed"] == report["real_executions"] == 0
write("audit-summary.json", {k: v for k, v in report.items() if k != "files"})
write(
"import-protection.json",
dict(
base=BASE,
first=first["counts"],
upgrade=upgrade["counts"],
repeat=repeat["counts"],
restored_rows=1,
untouched_whole_rows=154,
eligibility_unchanged=True,
redo=redo["counts"],
user_protection=protected,
fresh=fresh["counts"],
fresh_repeat=fresh_repeat["counts"],
states=fresh["states"],
source_named_fields=1217,
positional_widgets=3660,
companion_values=companion,
new_saved=21,
new_notes=0,
old_saved=495,
old_notes=15,
overviews=18,
all_disabled=True,
doc_done=0,
executed=0,
),
)
old_reviews = json.loads(
subprocess.check_output(["git", "show", f"{BASE}:{RESOURCE}"], cwd=REPO)
)
current = json.loads((REPO / RESOURCE).read_bytes())
for p in PATHS:
current[p].pop("fixed_version_document")
assert current == old_reviews
boundaries = json.loads(
(REPO / "docs/hel-892/h3-document-evidence/unchanged-boundaries.json").read_bytes()
)
paths = (
[f["path"] for f in boundaries["files"]]
+ boundaries["frozen_conversion_and_audits"]
+ ["calliope-backend/src/calliope/comfyui/library.py"]
)
kept = []
for p in paths:
raw = (REPO / p).read_bytes()
assert raw == subprocess.check_output(["git", "show", f"{BASE}:{p}"], cwd=REPO)
kept.append(dict(path=p, sha256=hashlib.sha256(raw).hexdigest()))
ui = "calliope-web/src/lib/components/WorkflowDocumentDrawer.svelte"
drawer_bytes = (REPO / ui).read_bytes()
assert drawer_bytes == subprocess.check_output(["git", "show", f"{BASE}:{ui}"], cwd=REPO)
write(
"boundaries.json",
dict(
base=BASE,
files=kept,
originals=163,
old_seventeen_blocks_unchanged=True,
drawer_css_unchanged=True,
drawer_whole_file_unchanged=True,
drawer_sha256=hashlib.sha256(drawer_bytes).hexdigest(),
conversion_rerun=False,
profiles_28_candidates_9_diffs_6="unchanged",
),
)
packages = []
with zipfile.ZipFile(wheels) as w, tarfile.open(sdists) as s:
for name in ["fixed_document.py", "source_reviews.json", "library.py"]:
raw = (REPO / "calliope-backend/src/calliope/comfyui" / name).read_bytes()
assert w.read("calliope/comfyui/" + name) == raw
member = next(n for n in s.getnames() if n.endswith("/src/calliope/comfyui/" + name))
assert s.extractfile(member).read() == raw
packages.append(
dict(
file=name,
sha256=hashlib.sha256(raw).hexdigest(),
wheel=True,
sdist=True,
definitions=18,
)
)
write("package-check.json", packages)
print(
(
"PASS actual ab18cd3 upgrade1/skip154, "
"repeat155, rollback1, edits1/29cols, f"
"resh155, 1217/3660/294, 18 definitions"
", frozen boundaries, wheel/sdist"
)
)
if __name__ == "__main__":
main()