311 lines
13 KiB
Python
311 lines
13 KiB
Python
"""Isolated transport and completed-stream verification; sources remain inert."""
|
|
|
|
import datetime
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import urllib.request
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
from acquire import DOC, ROOT, save, sha
|
|
|
|
LOCAL = ROOT / ".local-evidence/batch181-parameter-plan"
|
|
DELIVERY = LOCAL / "delivery"
|
|
PREFIX = "docs/hel-892/batch181-parameter-plan/"
|
|
BASE = "2fa818d62b6ecefa7a144437a535fcd062dfa1e5"
|
|
BRANCH = "agent/agent/a64a969a6a44"
|
|
ACTIVE_KEY = "package"
|
|
|
|
|
|
def put(path, value):
|
|
with path.open("x") as stream:
|
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
|
stream.write("\n")
|
|
|
|
|
|
def file_info(path):
|
|
data = path.read_bytes()
|
|
return dict(bytes=len(data), sha256=sha(data))
|
|
|
|
|
|
def git(*args):
|
|
return subprocess.check_output(["git", *args], cwd=ROOT).decode().strip()
|
|
|
|
|
|
def prepare():
|
|
rows = []
|
|
# Only the isolated new directory; never cross-batch globbing.
|
|
for path in sorted(DOC.rglob("*")):
|
|
if path.is_file():
|
|
relative = str(path.relative_to(ROOT))
|
|
rows.append(
|
|
dict(
|
|
path=relative,
|
|
purpose=(
|
|
"exact upstream original"
|
|
if "/source-text/" in relative
|
|
else "original HTTP evidence"
|
|
if "/network/" in relative
|
|
else "static audit, provenance, report or tool"
|
|
),
|
|
**file_info(path),
|
|
)
|
|
)
|
|
save("file-purpose-sha256.json", rows)
|
|
print(len(rows), "new files indexed; purpose index self-excluded")
|
|
|
|
|
|
def package():
|
|
assert not git("status", "--porcelain"), "Worktree must be clean"
|
|
baseline = json.loads((DOC / "baseline.json").read_text())
|
|
for entry in baseline["files"]:
|
|
assert file_info(ROOT / entry["path"]) == {k: entry[k] for k in ("bytes", "sha256")}
|
|
for entry in json.loads((DOC / "old-local-artifacts.json").read_text())["files"]:
|
|
assert file_info(ROOT / entry["path"]) == {k: entry[k] for k in ("bytes", "sha256")}
|
|
assert (ROOT / entry["path"]).stat().st_mtime_ns == entry["observed_mtime_ns"]
|
|
head = git("rev-parse", "HEAD")
|
|
assert git("branch", "--show-current") == BRANCH
|
|
remote_raw = subprocess.check_output(
|
|
["git", "ls-remote", "origin", "refs/heads/" + BRANCH, "refs/heads/main"], cwd=ROOT
|
|
)
|
|
remote = dict(line.split()[::-1] for line in remote_raw.decode().splitlines())
|
|
assert remote["refs/heads/" + BRANCH] == head
|
|
(DELIVERY / "remote-refs.stdout.txt").write_bytes(remote_raw)
|
|
(DELIVERY / "remote-refs.stderr.txt").write_bytes(b"")
|
|
url = "https://gitea.xbay.cc/api/v1/repos/leefer/aigc-home/pulls/25"
|
|
started = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
with urllib.request.urlopen(url, timeout=30) as response:
|
|
raw = response.read(524289)
|
|
status = response.status
|
|
assert len(raw) <= 524288 and status == 200
|
|
with (DELIVERY / "pr-response.body").open("xb") as stream:
|
|
stream.write(raw)
|
|
pr = json.loads(raw)
|
|
assert pr["head"]["sha"] == head and pr["head"]["ref"] == BRANCH
|
|
assert pr["base"]["ref"] == "main"
|
|
put(
|
|
DELIVERY / "pr-check.json",
|
|
dict(
|
|
url=url,
|
|
status=status,
|
|
response_cap=524288,
|
|
response_sha256=sha(raw),
|
|
started_at_utc=started,
|
|
completed_at_utc=datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
head=head,
|
|
branch=BRANCH,
|
|
base="main",
|
|
pr_url=pr["html_url"],
|
|
),
|
|
)
|
|
put(
|
|
DELIVERY / "final-head.json",
|
|
dict(
|
|
head=head,
|
|
base=BASE,
|
|
branch=BRANCH,
|
|
remote=remote,
|
|
worktree_clean=True,
|
|
pr=pr["html_url"],
|
|
),
|
|
)
|
|
patch = LOCAL / "logs/batch181-parameter-plan-patch.stdout.txt"
|
|
with (DELIVERY / "complete.patch").open("xb") as stream:
|
|
stream.write(patch.read_bytes())
|
|
files = [
|
|
x.decode()
|
|
for x in subprocess.check_output(
|
|
["git", "diff", "--name-only", "-z", BASE, head], cwd=ROOT
|
|
).split(b"\0")
|
|
if x
|
|
]
|
|
assert files and all(path.startswith(PREFIX) for path in files)
|
|
members = {path: (ROOT / path).read_bytes() for path in files}
|
|
needed = {
|
|
"docs/hel-892/batch179-target-snapshots/incoming/ssh-snapshot.stdout.json",
|
|
"docs/hel-892/batch179-target-snapshots/incoming/target179-access.zip",
|
|
"docs/hel-892/batch179-target-snapshots/installation-matrix.json",
|
|
"docs/hel-892/batch179-correction/selection-fact.json",
|
|
"docs/hel-892/batch177-controlnet-provenance/class-schema-matrix.json",
|
|
"docs/hel-892/batch177-controlnet-provenance/field-matrix.json",
|
|
"docs/hel-892/batch177-controlnet-provenance/release-member-proofs.json",
|
|
"docs/hel-892/batch177-controlnet-provenance/release-directory.json",
|
|
"docs/hel-892/batch177-controlnet-provenance/network/registry-version.body",
|
|
"docs/hel-892/batch177-controlnet-provenance/network/registry-version.json",
|
|
"docs/hel-892/batch177-controlnet-provenance/release-members/LICENSE.txt",
|
|
"docs/hel-892/batch177-controlnet-provenance/release-members/utils.py",
|
|
"docs/hel-892/supplied-api/01.api.json",
|
|
}
|
|
reuse = json.loads((DOC / "metadata-license-reuse.json").read_text())
|
|
needed.update(item["file"] for item in reuse)
|
|
aux = json.loads((DOC / "reused-aux-schema.json").read_text())
|
|
prefix = "docs/hel-892/batch177-controlnet-provenance/"
|
|
for row in aux.values():
|
|
proof = row["source_proof"]
|
|
needed.add(prefix + proof["file"])
|
|
for key in ("header_request", "name_request", "data_request"):
|
|
for extension in (".body", ".json"):
|
|
needed.add(prefix + "network/" + proof[key] + extension)
|
|
old_core = json.loads((ROOT / (prefix + "class-schema-matrix.json")).read_text())
|
|
for name in ("LoadImage", "PreviewImage", "Canny", "ImageStitch"):
|
|
proof = old_core[name]["source_proof"]
|
|
needed.add(proof["retained_path"])
|
|
fields = json.loads((DOC / "parameter-matrix.json").read_text())
|
|
needed.add(fields[0]["original_document"])
|
|
for path in sorted(needed):
|
|
members[path] = (ROOT / path).read_bytes()
|
|
register_path = ".local-evidence/batch180-exact-engine/delivery/execution-register.json"
|
|
register = json.loads((ROOT / register_path).read_text())
|
|
members["historical/" + register_path] = (ROOT / register_path).read_bytes()
|
|
for entry in register["records"]:
|
|
record = entry["record"]
|
|
if record["returncode"] != 0:
|
|
path = ".local-evidence/batch180-exact-engine/records/" + record["key"] + ".json"
|
|
members["historical/" + path] = (ROOT / path).read_bytes()
|
|
for log in record["logs"]:
|
|
members["historical/" + log["path"]] = (ROOT / log["path"]).read_bytes()
|
|
for directory, names in (
|
|
("exclusive-contract", ("stdout.txt",)),
|
|
("exclusive-contract-verification-final", ("stdout.txt", "stderr.txt", "record.json")),
|
|
("exclusive-contract-verification-delivery", ("stdout.txt", "stderr.txt", "record.json")),
|
|
):
|
|
for name in names:
|
|
path = LOCAL / directory / name
|
|
members["offline-contract-artifacts/" + directory + "/" + name] = path.read_bytes()
|
|
records = []
|
|
for path in sorted((LOCAL / "records").iterdir()):
|
|
assert path.suffix == ".json"
|
|
record = json.loads(path.read_text())
|
|
records.append(record)
|
|
members["execution/records/" + path.name] = path.read_bytes()
|
|
for log in record["logs"]:
|
|
data = (ROOT / log["path"]).read_bytes()
|
|
assert len(data) == log["bytes"] and sha(data) == log["sha256"]
|
|
members["execution/logs/" + Path(log["path"]).name] = data
|
|
put(DELIVERY / "completed-command-index.json", records)
|
|
delivery_files = [
|
|
"complete.patch",
|
|
"mapping.bundle",
|
|
"pr-response.body",
|
|
"pr-check.json",
|
|
"final-head.json",
|
|
"remote-refs.stdout.txt",
|
|
"remote-refs.stderr.txt",
|
|
"completed-command-index.json",
|
|
]
|
|
for name in delivery_files:
|
|
members["delivery/" + name] = (DELIVERY / name).read_bytes()
|
|
manifest = [
|
|
dict(
|
|
path=name,
|
|
bytes=len(data),
|
|
sha256=sha(data),
|
|
purpose=(
|
|
"new batch180 deliverable"
|
|
if name.startswith(PREFIX)
|
|
else "completed command record/log"
|
|
if name.startswith("execution/")
|
|
else "historical source/snapshot/failure or transport"
|
|
),
|
|
)
|
|
for name, data in sorted(members.items())
|
|
]
|
|
put(DELIVERY / "purpose-sha256.json", manifest)
|
|
members["delivery/purpose-sha256.json"] = (DELIVERY / "purpose-sha256.json").read_bytes()
|
|
archive = DELIVERY / "HEL-892-batch181-parameter-plan.zip"
|
|
with zipfile.ZipFile(archive, "x", zipfile.ZIP_DEFLATED) as stream:
|
|
for name, data in sorted(members.items()):
|
|
stream.writestr(name, data)
|
|
with zipfile.ZipFile(archive) as stream:
|
|
assert len(stream.namelist()) == len(set(stream.namelist())) == len(members)
|
|
assert stream.testzip() is None
|
|
for name, expected in members.items():
|
|
actual = stream.read(name)
|
|
assert len(actual) == len(expected) and sha(actual) == sha(expected), name
|
|
# Every record's two raw streams are read from the finished ZIP, not merely filesystem.
|
|
for record in records:
|
|
archived = json.loads(stream.read("execution/records/" + record["key"] + ".json"))
|
|
assert archived == record and len(record["logs"]) == 2
|
|
for log in record["logs"]:
|
|
actual = stream.read("execution/logs/" + Path(log["path"]).name)
|
|
assert len(actual) == log["bytes"] and sha(actual) == log["sha256"]
|
|
put(
|
|
DELIVERY / "archive-verification.json",
|
|
dict(
|
|
**file_info(archive),
|
|
members=len(members),
|
|
crc="all pass",
|
|
member_bytes_sha256="all pass",
|
|
completed_commands=len(records),
|
|
archived_streams=len(records) * 2,
|
|
excluded_active_command=(
|
|
"package; final record and two raw streams delivered separately"
|
|
),
|
|
),
|
|
)
|
|
print(archive.name, len(members), file_info(archive))
|
|
|
|
|
|
def finalize():
|
|
"""Read-only verification of finished formal commands; no new formal subprocesses."""
|
|
archive = DELIVERY / "HEL-892-batch181-parameter-plan.zip"
|
|
records = []
|
|
with zipfile.ZipFile(archive) as stream:
|
|
assert stream.testzip() is None
|
|
for path in sorted((LOCAL / "records").iterdir()):
|
|
record = json.loads(path.read_text())
|
|
assert len(record["logs"]) == 2
|
|
location = (
|
|
"separately attached final package record/logs"
|
|
if record["key"] == ACTIVE_KEY
|
|
else "ZIP"
|
|
)
|
|
if record["key"] != ACTIVE_KEY:
|
|
assert stream.read("execution/records/" + path.name) == path.read_bytes()
|
|
for log in record["logs"]:
|
|
actual = (ROOT / log["path"]).read_bytes()
|
|
assert len(actual) == log["bytes"] and sha(actual) == log["sha256"]
|
|
if record["key"] != ACTIVE_KEY:
|
|
assert actual == stream.read("execution/logs/" + Path(log["path"]).name)
|
|
records.append(dict(record=record, coverage=location))
|
|
manifest = json.loads(stream.read("delivery/purpose-sha256.json"))
|
|
for item in manifest:
|
|
raw = stream.read(item["path"])
|
|
assert len(raw) == item["bytes"] and sha(raw) == item["sha256"]
|
|
assert sum(x["record"]["key"] == ACTIVE_KEY for x in records) == 1
|
|
package_record = LOCAL / "records" / (ACTIVE_KEY + ".json")
|
|
with (DELIVERY / "package-record.json").open("xb") as output:
|
|
output.write(package_record.read_bytes())
|
|
coverage = dict(
|
|
archive=file_info(archive),
|
|
formal_commands=len(records),
|
|
raw_streams=2 * len(records),
|
|
failed_keys=[x["record"]["key"] for x in records if x["record"]["returncode"]],
|
|
records=records,
|
|
complete_merged_coverage=True,
|
|
observer=(
|
|
"This finalizer only reads completed logs/ZIP and writes this receipt; "
|
|
"no source/test subprocess"
|
|
),
|
|
)
|
|
put(DELIVERY / "execution-register.json", coverage)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
k: coverage[k]
|
|
for k in (
|
|
"formal_commands",
|
|
"raw_streams",
|
|
"failed_keys",
|
|
"complete_merged_coverage",
|
|
)
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
{"prepare": prepare, "package": package, "finalize": finalize}[sys.argv[1]]()
|