Files
aigc-home/docs/hel-892/ace-document-evidence/fetch_sources.py
T
2026-10-31 04:31:41 +08:00

163 lines
5.6 KiB
Python

"""Fetch only pinned upstream text; never import upstream code."""
import datetime
import hashlib
import json
import urllib.request
from pathlib import Path
OUT = Path(__file__).resolve().parents[1] / "ace-reference"
SPECS = {}
def get(url):
with urllib.request.urlopen(
urllib.request.Request(url, headers={"User-Agent": "offline-source-review"}), timeout=40
) as r:
return r.read()
def fetch(spec):
key, (repo, rev, files) = spec
base = f"https://api.github.com/repos/{repo}/git/"
tree_path = OUT / (key + "-tree.json")
tree = (
tree_path.read_bytes()
if tree_path.exists()
else get(base + f"trees/{rev}?recursive=1&review=hel892ace")
)
t = json.loads(tree)
assert not t["truncated"]
(OUT / (key + "-tree.json")).write_bytes(tree)
commit_path = OUT / (key + "-commit.json")
commit = (
commit_path.read_bytes()
if commit_path.exists()
else get(base + f"commits/{rev}?review=hel892ace")
)
(OUT / (key + "-commit.json")).write_bytes(commit)
rows = []
previous = (
json.loads((OUT / "sources.json").read_bytes())["sources"]
if (OUT / "sources.json").exists()
else []
)
for f in files:
url = f"https://raw.githubusercontent.com/{repo}/{rev}/{f}"
prior = next((e for e in previous if e["id"] == key and e["file"] == f), None)
name = prior["local_file"] if prior else key + "__" + f.replace("/", "__")
cached = OUT / name
raw = cached.read_bytes() if cached.exists() else get(url)
blob = next(e["sha"] for e in t["tree"] if e["path"] == f)
assert hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).hexdigest() == blob
if not cached.exists():
cached.write_bytes(raw)
rows.append(
dict(
id=key,
repo=repo,
file=f,
local_file=name,
commit=rev,
tree=json.loads(commit)["tree"]["sha"],
url=url,
sha256=hashlib.sha256(raw).hexdigest(),
git_blob_sha1=blob,
verified_at=datetime.datetime.now(datetime.timezone.utc).isoformat(),
acquisition_time=(
prior["acquisition_time"]
if prior
else datetime.datetime.fromtimestamp(
cached.stat().st_mtime, datetime.timezone.utc
).isoformat()
),
acquisition_time_basis=(
"local raw-cache write immediately after acquisition; "
"HTTP receipt time not separately recorded"
),
cached_bytes_reverified=True,
license_file=next(
(
e["local_file"]
for e in previous
if e["id"] == key and e["file"] == "LICENSE"
),
key + "__LICENSE"
if any(x["path"] == "LICENSE" for x in t["tree"])
else "unknown: pyproject declares LICENSE but exact tree has no such file",
),
applicability="源声明准确提交;发布包/目标安装绑定未知",
transport="GitHub pinned raw text; blob verified",
)
)
return rows
def main():
OUT.mkdir(parents=True, exist_ok=True)
definitions = {
"core": (
"Comfy-Org/ComfyUI",
"0.11.1",
[
"nodes.py",
"folder_paths.py",
"comfy_extras/nodes_ace.py",
"comfy_extras/nodes_audio.py",
"comfy_extras/nodes_model_advanced.py",
"comfy/sd.py",
"comfy/samplers.py",
"comfy/sample.py",
"comfy/model_sampling.py",
"comfy/text_encoders/ace.py",
"comfy_api/latest/_ui.py",
"comfy_api/latest/_ui_public.py",
"LICENSE",
],
),
"primitive": (
"Comfy-Org/ComfyUI",
"0.18.1",
["comfy_extras/nodes_primitive.py", "comfy_api/latest/_io.py", "nodes.py", "LICENSE"],
),
"frontend": (
"Comfy-Org/ComfyUI_frontend",
"1.44.0",
[
"package.json",
"pnpm-lock.yaml",
"src/scripts/widgets.ts",
"src/scripts/app.ts",
"src/extensions/core/widgetInputs.ts",
"src/extensions/core/uploadAudio.ts",
"src/renderer/extensions/vueNodes/widgets/composables/useIntWidget.ts",
"src/renderer/extensions/vueNodes/widgets/composables/useFloatWidget.ts",
"src/schemas/nodeDef/migration.ts",
"src/lib/litegraph/src/LGraphNode.ts",
"LICENSE",
],
),
}
for key, (repo, tag, files) in definitions.items():
p = OUT / (key + "-tag.json")
raw = (
p.read_bytes()
if p.exists()
else get(f"https://api.github.com/repos/{repo}/git/ref/tags/v{tag}")
)
p.write_bytes(raw)
obj = json.loads(raw)["object"]
assert obj["type"] == "commit", obj
SPECS[key] = (repo, obj["sha"], files)
rows = []
for spec in SPECS.items():
rows += fetch(spec)
(OUT / "sources.json").write_text(
json.dumps({"sources": rows}, ensure_ascii=False, indent=2) + "\n"
)
if __name__ == "__main__":
main()