Files
aigc-home/calliope-backend/scripts/audit_supplied_api.py
T

129 lines
5.4 KiB
Python

"""Audit received API companions against source bytes and an isolated imported DB."""
from __future__ import annotations
import argparse
import hashlib
import json
from collections import Counter
from pathlib import Path
from calliope.comfyui.companions import load_companions, resolve_companion
from calliope.comfyui.delivery_scope import exclusion_for
from calliope.comfyui.library import _json
from calliope.db import get_db
def audit(repo: Path, manifest: Path, db: Path, output: Path):
records = load_companions(manifest)
conn = get_db(db)
rows = []
output.mkdir(parents=True, exist_ok=True)
try:
for path, record in records.items():
excluded = exclusion_for(path)
if excluded:
history = conn.execute(
"SELECT id FROM workflows WHERE source_repo_path=?", (path,)
).fetchone()
rows.append(
{
"id": record["binding"]["id"],
"source_path": path,
"state": "user_excluded",
"adopted": False,
"new_api_literal_params": 0,
"exclusion": excluded,
"historical_row_preserved": history is not None,
}
)
continue
raw = (repo / path).read_bytes()
prompt, evidence, issues = resolve_companion(record, path, raw, json.loads(raw))
row = conn.execute(
"SELECT * FROM workflows WHERE source_repo_path=?", (path,)
).fetchone()
assert row is not None and not row["is_enabled"]
assert row["doc_completed_at"] is row["exec_verified_at"] is None
assert row["source_sha"] == hashlib.sha1(raw).hexdigest()
assert json.loads(row["raw_source_json"]) == json.loads(raw)
assert json.loads(row["workflow_json"]) == (prompt or {})
doc = json.loads(row["doc_json"])
assert doc["api_companion"] == evidence
params = [p for p in doc["params"] if p.get("value_origin") == "supplied_api_companion"]
assert len(params) == sum(
not isinstance(value, list)
for node in (prompt or {}).values()
for value in node["inputs"].values()
)
for param in params:
assert _json(param["default"]) == _json(
prompt[param["api_node_id"]]["inputs"][param["field"]]
)
assert param["unit"] is param["min"] is param["max"] is None
details = evidence["comparison"]
fields = [f for n in details["nodes"] for f in n["fields"]]
api = json.loads(record["raw"].lstrip("\ufeff"))
assert len(fields) == sum(len(n.get("inputs", {})) for n in api.values())
case = {
"id": evidence["binding"]["id"],
"source_path": path,
"state": row["import_state"],
"adopted": evidence["adopted"],
"api_nodes": len(api),
"api_inputs": len(fields),
"new_api_literal_params": len(params),
"field_comparison": dict(Counter(f["status"] for f in fields)),
"source_only_ids": [n["source_id"] for n in details["source_only_nodes"]],
"api_only_ids": [
n["api_id"] for n in details["nodes"] if n["status"].startswith("api_only")
],
"named_evidence_differences": [
{"node_id": n["api_id"], **f}
for n in details["nodes"]
for f in n["fields"]
if f.get("named_evidence", {}).get("status") == "different"
],
"output_map": json.loads(row["output_map"]),
"kind": row["kind"],
"issues": issues,
"general_conversion_verified": False,
"execution_verified": False,
"evidence_file": evidence["binding"]["id"] + ".json",
}
(output / case["evidence_file"]).write_text(
json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
rows.append(case)
finally:
conn.close()
report = {
"cases": rows,
"adopted": sum(r["adopted"] for r in rows),
"user_excluded": sum(r["state"] == "user_excluded" for r in rows),
"source_files": 156,
"target_files": 155,
"general_conversion_verified": 0,
"execution_verified": 0,
"new_api_literal_params": sum(r["new_api_literal_params"] for r in rows),
}
(output / "summary.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
return report
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo", type=Path, required=True)
parser.add_argument("--manifest", type=Path, required=True)
parser.add_argument("--db", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
report = audit(args.repo, args.manifest, args.db, args.output)
print(json.dumps({k: v for k, v in report.items() if k != "cases"}, ensure_ascii=False))
if __name__ == "__main__":
main()