Files
aigc-home/docs/hel-892/batch181-parameter-plan/package.py
T

311 lines
13 KiB
Python

"""Isolated transport and completed-stream verification; sources remain inert."""
import datetime
import json
import subprocess
import sys
import urllib.request
import zipfile
from pathlib import Path
from acquire import DOC, ROOT, save, sha
LOCAL = ROOT / ".local-evidence/batch181-parameter-plan"
DELIVERY = LOCAL / "delivery"
PREFIX = "docs/hel-892/batch181-parameter-plan/"
BASE = "2fa818d62b6ecefa7a144437a535fcd062dfa1e5"
BRANCH = "agent/agent/a64a969a6a44"
ACTIVE_KEY = "package"
def put(path, value):
with path.open("x") as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
stream.write("\n")
def file_info(path):
data = path.read_bytes()
return dict(bytes=len(data), sha256=sha(data))
def git(*args):
return subprocess.check_output(["git", *args], cwd=ROOT).decode().strip()
def prepare():
rows = []
# Only the isolated new directory; never cross-batch globbing.
for path in sorted(DOC.rglob("*")):
if path.is_file():
relative = str(path.relative_to(ROOT))
rows.append(
dict(
path=relative,
purpose=(
"exact upstream original"
if "/source-text/" in relative
else "original HTTP evidence"
if "/network/" in relative
else "static audit, provenance, report or tool"
),
**file_info(path),
)
)
save("file-purpose-sha256.json", rows)
print(len(rows), "new files indexed; purpose index self-excluded")
def package():
assert not git("status", "--porcelain"), "Worktree must be clean"
baseline = json.loads((DOC / "baseline.json").read_text())
for entry in baseline["files"]:
assert file_info(ROOT / entry["path"]) == {k: entry[k] for k in ("bytes", "sha256")}
for entry in json.loads((DOC / "old-local-artifacts.json").read_text())["files"]:
assert file_info(ROOT / entry["path"]) == {k: entry[k] for k in ("bytes", "sha256")}
assert (ROOT / entry["path"]).stat().st_mtime_ns == entry["observed_mtime_ns"]
head = git("rev-parse", "HEAD")
assert git("branch", "--show-current") == BRANCH
remote_raw = subprocess.check_output(
["git", "ls-remote", "origin", "refs/heads/" + BRANCH, "refs/heads/main"], cwd=ROOT
)
remote = dict(line.split()[::-1] for line in remote_raw.decode().splitlines())
assert remote["refs/heads/" + BRANCH] == head
(DELIVERY / "remote-refs.stdout.txt").write_bytes(remote_raw)
(DELIVERY / "remote-refs.stderr.txt").write_bytes(b"")
url = "https://gitea.xbay.cc/api/v1/repos/leefer/aigc-home/pulls/25"
started = datetime.datetime.now(datetime.timezone.utc).isoformat()
with urllib.request.urlopen(url, timeout=30) as response:
raw = response.read(524289)
status = response.status
assert len(raw) <= 524288 and status == 200
with (DELIVERY / "pr-response.body").open("xb") as stream:
stream.write(raw)
pr = json.loads(raw)
assert pr["head"]["sha"] == head and pr["head"]["ref"] == BRANCH
assert pr["base"]["ref"] == "main"
put(
DELIVERY / "pr-check.json",
dict(
url=url,
status=status,
response_cap=524288,
response_sha256=sha(raw),
started_at_utc=started,
completed_at_utc=datetime.datetime.now(datetime.timezone.utc).isoformat(),
head=head,
branch=BRANCH,
base="main",
pr_url=pr["html_url"],
),
)
put(
DELIVERY / "final-head.json",
dict(
head=head,
base=BASE,
branch=BRANCH,
remote=remote,
worktree_clean=True,
pr=pr["html_url"],
),
)
patch = LOCAL / "logs/batch181-parameter-plan-patch.stdout.txt"
with (DELIVERY / "complete.patch").open("xb") as stream:
stream.write(patch.read_bytes())
files = [
x.decode()
for x in subprocess.check_output(
["git", "diff", "--name-only", "-z", BASE, head], cwd=ROOT
).split(b"\0")
if x
]
assert files and all(path.startswith(PREFIX) for path in files)
members = {path: (ROOT / path).read_bytes() for path in files}
needed = {
"docs/hel-892/batch179-target-snapshots/incoming/ssh-snapshot.stdout.json",
"docs/hel-892/batch179-target-snapshots/incoming/target179-access.zip",
"docs/hel-892/batch179-target-snapshots/installation-matrix.json",
"docs/hel-892/batch179-correction/selection-fact.json",
"docs/hel-892/batch177-controlnet-provenance/class-schema-matrix.json",
"docs/hel-892/batch177-controlnet-provenance/field-matrix.json",
"docs/hel-892/batch177-controlnet-provenance/release-member-proofs.json",
"docs/hel-892/batch177-controlnet-provenance/release-directory.json",
"docs/hel-892/batch177-controlnet-provenance/network/registry-version.body",
"docs/hel-892/batch177-controlnet-provenance/network/registry-version.json",
"docs/hel-892/batch177-controlnet-provenance/release-members/LICENSE.txt",
"docs/hel-892/batch177-controlnet-provenance/release-members/utils.py",
"docs/hel-892/supplied-api/01.api.json",
}
reuse = json.loads((DOC / "metadata-license-reuse.json").read_text())
needed.update(item["file"] for item in reuse)
aux = json.loads((DOC / "reused-aux-schema.json").read_text())
prefix = "docs/hel-892/batch177-controlnet-provenance/"
for row in aux.values():
proof = row["source_proof"]
needed.add(prefix + proof["file"])
for key in ("header_request", "name_request", "data_request"):
for extension in (".body", ".json"):
needed.add(prefix + "network/" + proof[key] + extension)
old_core = json.loads((ROOT / (prefix + "class-schema-matrix.json")).read_text())
for name in ("LoadImage", "PreviewImage", "Canny", "ImageStitch"):
proof = old_core[name]["source_proof"]
needed.add(proof["retained_path"])
fields = json.loads((DOC / "parameter-matrix.json").read_text())
needed.add(fields[0]["original_document"])
for path in sorted(needed):
members[path] = (ROOT / path).read_bytes()
register_path = ".local-evidence/batch180-exact-engine/delivery/execution-register.json"
register = json.loads((ROOT / register_path).read_text())
members["historical/" + register_path] = (ROOT / register_path).read_bytes()
for entry in register["records"]:
record = entry["record"]
if record["returncode"] != 0:
path = ".local-evidence/batch180-exact-engine/records/" + record["key"] + ".json"
members["historical/" + path] = (ROOT / path).read_bytes()
for log in record["logs"]:
members["historical/" + log["path"]] = (ROOT / log["path"]).read_bytes()
for directory, names in (
("exclusive-contract", ("stdout.txt",)),
("exclusive-contract-verification-final", ("stdout.txt", "stderr.txt", "record.json")),
("exclusive-contract-verification-delivery", ("stdout.txt", "stderr.txt", "record.json")),
):
for name in names:
path = LOCAL / directory / name
members["offline-contract-artifacts/" + directory + "/" + name] = path.read_bytes()
records = []
for path in sorted((LOCAL / "records").iterdir()):
assert path.suffix == ".json"
record = json.loads(path.read_text())
records.append(record)
members["execution/records/" + path.name] = path.read_bytes()
for log in record["logs"]:
data = (ROOT / log["path"]).read_bytes()
assert len(data) == log["bytes"] and sha(data) == log["sha256"]
members["execution/logs/" + Path(log["path"]).name] = data
put(DELIVERY / "completed-command-index.json", records)
delivery_files = [
"complete.patch",
"mapping.bundle",
"pr-response.body",
"pr-check.json",
"final-head.json",
"remote-refs.stdout.txt",
"remote-refs.stderr.txt",
"completed-command-index.json",
]
for name in delivery_files:
members["delivery/" + name] = (DELIVERY / name).read_bytes()
manifest = [
dict(
path=name,
bytes=len(data),
sha256=sha(data),
purpose=(
"new batch180 deliverable"
if name.startswith(PREFIX)
else "completed command record/log"
if name.startswith("execution/")
else "historical source/snapshot/failure or transport"
),
)
for name, data in sorted(members.items())
]
put(DELIVERY / "purpose-sha256.json", manifest)
members["delivery/purpose-sha256.json"] = (DELIVERY / "purpose-sha256.json").read_bytes()
archive = DELIVERY / "HEL-892-batch181-parameter-plan.zip"
with zipfile.ZipFile(archive, "x", zipfile.ZIP_DEFLATED) as stream:
for name, data in sorted(members.items()):
stream.writestr(name, data)
with zipfile.ZipFile(archive) as stream:
assert len(stream.namelist()) == len(set(stream.namelist())) == len(members)
assert stream.testzip() is None
for name, expected in members.items():
actual = stream.read(name)
assert len(actual) == len(expected) and sha(actual) == sha(expected), name
# Every record's two raw streams are read from the finished ZIP, not merely filesystem.
for record in records:
archived = json.loads(stream.read("execution/records/" + record["key"] + ".json"))
assert archived == record and len(record["logs"]) == 2
for log in record["logs"]:
actual = stream.read("execution/logs/" + Path(log["path"]).name)
assert len(actual) == log["bytes"] and sha(actual) == log["sha256"]
put(
DELIVERY / "archive-verification.json",
dict(
**file_info(archive),
members=len(members),
crc="all pass",
member_bytes_sha256="all pass",
completed_commands=len(records),
archived_streams=len(records) * 2,
excluded_active_command=(
"package; final record and two raw streams delivered separately"
),
),
)
print(archive.name, len(members), file_info(archive))
def finalize():
"""Read-only verification of finished formal commands; no new formal subprocesses."""
archive = DELIVERY / "HEL-892-batch181-parameter-plan.zip"
records = []
with zipfile.ZipFile(archive) as stream:
assert stream.testzip() is None
for path in sorted((LOCAL / "records").iterdir()):
record = json.loads(path.read_text())
assert len(record["logs"]) == 2
location = (
"separately attached final package record/logs"
if record["key"] == ACTIVE_KEY
else "ZIP"
)
if record["key"] != ACTIVE_KEY:
assert stream.read("execution/records/" + path.name) == path.read_bytes()
for log in record["logs"]:
actual = (ROOT / log["path"]).read_bytes()
assert len(actual) == log["bytes"] and sha(actual) == log["sha256"]
if record["key"] != ACTIVE_KEY:
assert actual == stream.read("execution/logs/" + Path(log["path"]).name)
records.append(dict(record=record, coverage=location))
manifest = json.loads(stream.read("delivery/purpose-sha256.json"))
for item in manifest:
raw = stream.read(item["path"])
assert len(raw) == item["bytes"] and sha(raw) == item["sha256"]
assert sum(x["record"]["key"] == ACTIVE_KEY for x in records) == 1
package_record = LOCAL / "records" / (ACTIVE_KEY + ".json")
with (DELIVERY / "package-record.json").open("xb") as output:
output.write(package_record.read_bytes())
coverage = dict(
archive=file_info(archive),
formal_commands=len(records),
raw_streams=2 * len(records),
failed_keys=[x["record"]["key"] for x in records if x["record"]["returncode"]],
records=records,
complete_merged_coverage=True,
observer=(
"This finalizer only reads completed logs/ZIP and writes this receipt; "
"no source/test subprocess"
),
)
put(DELIVERY / "execution-register.json", coverage)
print(
json.dumps(
{
k: coverage[k]
for k in (
"formal_commands",
"raw_streams",
"failed_keys",
"complete_merged_coverage",
)
},
ensure_ascii=False,
)
)
if __name__ == "__main__":
{"prepare": prepare, "package": package, "finalize": finalize}[sys.argv[1]]()