Files
aigc-home/docs/hel-892/moge3-document-evidence/fetch_sources.py
T
2026-10-31 03:13:25 +08:00

121 lines
4.1 KiB
Python

"""Fetch exact tags and bounded text blobs; never checkout or execute upstream."""
import datetime
import hashlib
import json
import subprocess
from pathlib import Path
REPO = Path(__file__).resolve().parents[3]
REF = Path(__file__).resolve().parents[1] / "moge3-reference"
SPECS = {
"core": (
"Comfy-Org/ComfyUI",
"v0.36.0",
"core036",
[
"LICENSE",
"nodes.py",
"comfy_extras/nodes_moge.py",
"comfy_extras/nodes_images.py",
"comfy_extras/nodes_post_processing.py",
"comfy_api/latest/_io.py",
"comfy/ldm/moge/model.py",
"folder_paths.py",
],
),
"frontend": (
"Comfy-Org/ComfyUI_frontend",
"v1.52.7",
"front1527",
[
"LICENSE",
"src/lib/litegraph/LICENSE",
"src/lib/litegraph/src/LGraphNode.ts",
("src/lib/litegraph/src/subgraph/S" + "ubgraphNode.ts"),
("src/lib/litegraph/src/subgraph/E" + "xecutableNodeDTO.ts"),
"src/scripts/widgets.ts",
"src/extensions/core/uploadImage.ts",
"src/extensions/core/noteNode.ts",
"src/core/graph/widgets/dynamicWidgets.ts",
"src/utils/executionUtil.ts",
],
),
}
def main():
REF.mkdir(parents=True, exist_ok=True)
rows = []
identities = []
for key, (repo, tag, cache, files) in SPECS.items():
gitdir = REPO / ".local-evidence" / cache
if not gitdir.exists():
subprocess.run(
[
"git",
"clone",
"--depth",
"1",
"--filter=blob:none",
"--no-checkout",
"--branch",
tag,
f"https://github.com/{repo}.git",
str(gitdir),
],
check=True,
)
def git(*args, gitdir=gitdir):
return subprocess.check_output(["git", "-C", str(gitdir), *args])
rev = git("rev-parse", tag + "^{commit}").decode().strip()
tree = []
for line in git("ls-tree", "-r", rev).decode().splitlines():
head, path = line.split("\t", 1)
mode, kind, sha = head.split()
tree.append({"path": path, "mode": mode, "type": kind, "sha": sha})
(REF / (key + "-tree.json")).write_text(
json.dumps({"sha": rev, "truncated": False, "tree": tree}, indent=2) + "\n"
)
(REF / (key + "-tag.txt")).write_bytes(git("show-ref", "--tags", tag))
(REF / (key + "-commit.txt")).write_bytes(git("cat-file", "commit", rev))
identities.append(
{
"id": key,
"repo": repo,
"tag": tag,
"commit": rev,
"tree": git("rev-parse", rev + "^{tree}").decode().strip(),
}
)
for file in files:
raw = git("show", rev + ":" + file)
name = key + "__" + file.replace("/", "__")
blob = next(e["sha"] for e in tree if e["path"] == file)
assert hashlib.sha1(b"blob " + str(len(raw)).encode() + b"\0" + raw).hexdigest() == blob
(REF / name).write_bytes(raw)
rows.append(
{
"id": key,
"file": file,
"local_file": name,
"commit": rev,
"tag": tag,
"repo": repo,
"url": f"https://raw.githubusercontent.com/{repo}/{rev}/{file}",
"sha256": hashlib.sha256(raw).hexdigest(),
"git_blob_sha1": blob,
"fetched_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"license_file": key + "__LICENSE",
"transport": "git show exact tag; no checkout",
}
)
(REF / "sources.json").write_text(json.dumps({"sources": rows}, indent=2) + "\n")
(REF / "identities.json").write_text(json.dumps(identities, indent=2) + "\n")
if __name__ == "__main__":
main()