660 lines
35 KiB
Python
660 lines
35 KiB
Python
#!/usr/bin/env python3
|
||
"""HEL-879 · 发布归档一致性快照与回退证据(HEL-896 有限收口版;不冒充 R6 完整备份)。
|
||
|
||
HEL-896 工具的总管独立复审发现四个失败边界,v2 全部收口;v3 收口总管对 v2 的
|
||
独立复核(HEL-879 edges):
|
||
|
||
1. checkpoint 返回值必须核对:`PRAGMA wal_checkpoint(TRUNCATE)` 返回 (busy, log,
|
||
checkpointed),busy≠0 即未完成——旧版照抄复制并在 manifest 记 rows_match=False
|
||
仍打包成功。busy≠0 / 源表行数≠快照行数一律**明确失败**:CLI 非 0、不产生
|
||
可交付归档(tar 不生成,半成品目录清除)。「先停服」是操作前提,不是忽略实际
|
||
失败结果的理由。
|
||
2. 目录内真实 SQLite 走一致性路径:history_root/vector_root 等数据目录里的
|
||
`mem0_public_history.db`、qdrant 存储等 SQLite 文件按文件头魔数递归识别
|
||
(`semantic.py` 的 history_db_path 就在 history_root 内)。停服口径对它们逐个
|
||
checkpoint+复制+逐表核对;**在线口径只允许显式单库 --db(backup API),传入含
|
||
SQLite 的目录直接拒绝**——在线目录裸复制正是丢 WAL 的不安全口径。
|
||
3. verify 拒绝 rows_match=False/缺失的旧包;三个计数(manifest 源行数、manifest
|
||
快照行数、解包实测)必须直接相等——v2 的 `rows_match is True or ...` 允许记录
|
||
标记绕过源计数比较(源 2/快照 1 + rows_match=true 仍 ok,总管实测)。回退演练
|
||
按源路径映射**实际恢复**到归档旁的持久独立目标树并打开恢复库核行数与哈希,
|
||
输出 rollback_plan(源路径 → 真实恢复路径)。
|
||
4. 归档侧同名覆盖:不同目录的同名 `ledger.db` 在 v2 都写入 `databases/ledger.db`,
|
||
后写覆盖前写、快照先交成功包再由 verify 发现。v3 对归档目标名做无冲突映射
|
||
(同名源追加解析路径摘要),同一源重复传入直接拒绝。
|
||
|
||
完整实例归档只支持停服/释放全部写入方;在线模式仅显式单库 backup API。
|
||
|
||
产物:`<out>/<name>/` 下的 `databases/`(一致性快照库,含目录内发现的 SQLite)、
|
||
`files/`(其余数据文件原样)、`MANIFEST.json`(逐文件 SHA256、源路径、代码精确
|
||
提交、逐表行数核对结果)、`<name>.tar.gz`。归档永不删除;本工具不实现 R6。
|
||
|
||
自测(`--selftest`,临时目录,零产品数据)复现:全部旧失败形状(busy checkpoint
|
||
仍打包、rows_match=False 旧包 verify 仍 ok、在线目录含 SQLite、v2 的 OR 绕过与
|
||
同名覆盖)与本版修复,输出 `docs/rework/hel879/release-snapshot/selftest.json`。
|
||
|
||
用法:
|
||
python3 tools/release_snapshot_hel896.py --selftest [outDir]
|
||
python3 tools/release_snapshot_hel896.py snapshot --name hel8xx-release \
|
||
--db path/a.db --db path/b.sqlite3 --file path/vecdir --file path/hist \
|
||
--out archives/ # 停服口径(推荐,目录内 SQLite 一并一致性处理)
|
||
python3 tools/release_snapshot_hel896.py snapshot --name ... --db path/a.db \
|
||
--out archives/ --online # 在线口径:仅显式单库 backup API,含 SQLite 的
|
||
# --file 目录会被拒绝
|
||
python3 tools/release_snapshot_hel896.py verify --archive archives/hel8xx-release.tar.gz \
|
||
[--restore-dir DIR] # 回退演练默认恢复到归档旁 verify-restore-<name>-<时间>/
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import shutil
|
||
import sqlite3
|
||
import subprocess
|
||
import tarfile
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
|
||
REPO = Path(__file__).resolve().parents[1]
|
||
SQLITE_MAGIC = b"SQLite format 3\x00"
|
||
TOOL_VERSION = 3
|
||
|
||
|
||
class SnapshotError(RuntimeError):
|
||
"""快照/验证明确失败:CLI 非 0,不形成可交付成功包。"""
|
||
|
||
|
||
def sha256(path: Path) -> str:
|
||
h = hashlib.sha256()
|
||
with open(path, "rb") as f:
|
||
for chunk in iter(lambda: f.read(1 << 20), b""):
|
||
h.update(chunk)
|
||
return h.hexdigest()
|
||
|
||
|
||
def is_sqlite(path: Path) -> bool:
|
||
"""按文件头魔数识别真实 SQLite 库(覆盖 .db/.sqlite3/无后缀;qdrant 本地存储)。"""
|
||
try:
|
||
with open(path, "rb") as f:
|
||
return f.read(16) == SQLITE_MAGIC
|
||
except OSError:
|
||
return False
|
||
|
||
|
||
def find_sqlite(root: Path) -> list[Path]:
|
||
"""递归找出目录内全部真实 SQLite 文件(按魔数,不按扩展名)。"""
|
||
if root.is_file():
|
||
return [root] if is_sqlite(root) else []
|
||
found: list[Path] = []
|
||
for p in sorted(root.rglob("*")):
|
||
if p.is_file() and is_sqlite(p):
|
||
found.append(p)
|
||
return found
|
||
|
||
|
||
def table_counts(db: Path) -> dict[str, int]:
|
||
con = sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||
try:
|
||
tables = [r[0] for r in con.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")]
|
||
return {t: con.execute(f"SELECT count(*) FROM \"{t}\"").fetchone()[0] for t in tables}
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def checkpoint(db: Path) -> tuple[int, int, int]:
|
||
"""TRUNCATE checkpoint,返回 (busy, log, checkpointed)。busy≠0 = 未完成。"""
|
||
con = sqlite3.connect(str(db))
|
||
try:
|
||
row = con.execute("PRAGMA wal_checkpoint(TRUNCATE)").fetchone()
|
||
return (row[0], row[1], row[2]) if row else (1, -1, -1)
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def snapshot_db(src: Path, dst: Path, *, online: bool) -> str:
|
||
"""一致性快照单库。online=False 先 checkpoint(必须完成);True 用 backup API。"""
|
||
if not online:
|
||
busy, log, ckpt = checkpoint(src)
|
||
if busy != 0:
|
||
raise SnapshotError(
|
||
f"checkpoint 未完成(busy={busy}, log={log}, checkpointed={ckpt}):"
|
||
f"{src} 仍有读取/写入方持有 WAL。停服口径要求先停止全部相关写入方,"
|
||
f"不得忽略该失败继续打包。")
|
||
shutil.copy2(src, dst)
|
||
return "stop+checkpoint+copy"
|
||
with sqlite3.connect(str(src)) as source, sqlite3.connect(str(dst)) as target:
|
||
source.backup(target)
|
||
return "sqlite-backup-api"
|
||
|
||
|
||
def git_head() -> str:
|
||
try:
|
||
return subprocess.run(["git", "rev-parse", "HEAD"], cwd=REPO, capture_output=True,
|
||
text=True, check=True).stdout.strip()
|
||
except Exception:
|
||
return "unknown"
|
||
|
||
|
||
def _unique_name(taken: set[str], display: str, key: str) -> str:
|
||
"""Archive-side name that cannot collide: two different source DBs with
|
||
the same basename must never overwrite each other in `databases/`
|
||
(HEL-879 总管复核第 4 项). Same-basename sources get a short digest of
|
||
their resolved source path appended."""
|
||
if display not in taken:
|
||
taken.add(display)
|
||
return display
|
||
digest = hashlib.sha256(key.encode()).hexdigest()[:8]
|
||
stem, suffix = Path(display).stem, Path(display).suffix
|
||
unique = f"{stem}__{digest}{suffix}"
|
||
while unique in taken:
|
||
unique = f"{stem}__{digest}{len(taken)}{suffix}"
|
||
taken.add(unique)
|
||
return unique
|
||
|
||
|
||
def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, *, online: bool) -> dict:
|
||
"""做一致性快照。任一环节失败:不产生 tar、清除半成品、抛 SnapshotError。
|
||
|
||
收口语义(HEL-879):
|
||
- 全部 --db 与目录内发现的 SQLite 都走一致性路径并逐表核对源/快照行数;
|
||
- 行数不一致或 checkpoint busy = 明确失败(旧版记 rows_match=False 仍打包);
|
||
- 在线模式含 SQLite 的 --file 目录直接拒绝(在线目录裸复制丢 WAL);
|
||
- 同一源重复传入或归档侧目标名冲突 = 明确失败/唯一化(同名 ledger.db
|
||
后写覆盖前写、快照先报成功包再由 verify 发现——旧版真实缺陷)。
|
||
"""
|
||
resolved = [str(Path(db).resolve()) for db in dbs]
|
||
duplicates = sorted({p for p in resolved if resolved.count(p) > 1})
|
||
if duplicates:
|
||
raise SnapshotError(f"--db 同一源重复传入,拒绝:{duplicates}")
|
||
stamp = time.strftime("%Y%m%d-%H%M%S")
|
||
root = out_dir / f"{name}-{stamp}"
|
||
manifest: dict = {"name": name, "created": stamp, "code_commit": git_head(),
|
||
"tool": "release_snapshot_hel896.py", "tool_version": TOOL_VERSION,
|
||
"mode": "online-backup-api" if online else "stop-checkpoint-copy",
|
||
"databases": [], "files": [], "verify": "not-run"}
|
||
taken_db_names: set[str] = set()
|
||
taken_file_names: set[str] = set()
|
||
try:
|
||
(root / "databases").mkdir(parents=True, exist_ok=True)
|
||
for db in dbs:
|
||
if not Path(db).is_file():
|
||
raise SnapshotError(f"--db 不是文件:{db}")
|
||
dst = root / "databases" / _unique_name(
|
||
taken_db_names, Path(db).name, str(Path(db).resolve()))
|
||
method = snapshot_db(Path(db), dst, online=online)
|
||
src_counts = table_counts(Path(db))
|
||
dst_counts = table_counts(dst)
|
||
if src_counts != dst_counts:
|
||
raise SnapshotError(
|
||
f"源/快照行数不一致,不打包:{db} source={src_counts} snapshot={dst_counts}")
|
||
manifest["databases"].append({"source": str(db), "snapshot": str(dst.relative_to(root)),
|
||
"sha256": sha256(dst), "method": method,
|
||
"source_table_counts": src_counts,
|
||
"snapshot_table_counts": dst_counts,
|
||
"rows_match": True})
|
||
for f in files:
|
||
src = Path(f)
|
||
if not src.exists():
|
||
raise SnapshotError(f"--file 不存在:{src}")
|
||
embedded = find_sqlite(src)
|
||
if online and embedded:
|
||
raise SnapshotError(
|
||
f"在线模式只允许显式单库 backup API(--db);--file {src} 内含 SQLite:"
|
||
f"{[str(e) for e in embedded]}。含活动库的目录在线裸复制会丢 WAL,"
|
||
f"拒绝。完整实例归档请用停服口径(默认模式)。")
|
||
for emb in embedded:
|
||
# 停服口径:目录内真实 SQLite 一并走 checkpoint+复制+逐表核对。
|
||
rel = emb.relative_to(src) if emb.is_relative_to(src) else Path(emb.name)
|
||
display = f"{src.name}__{rel}".replace("/", "__")
|
||
dst = root / "databases" / _unique_name(
|
||
taken_db_names, display, str(emb.resolve()))
|
||
method = snapshot_db(emb, dst, online=False)
|
||
src_counts = table_counts(emb)
|
||
dst_counts = table_counts(dst)
|
||
if src_counts != dst_counts:
|
||
raise SnapshotError(
|
||
f"目录内 SQLite 源/快照行数不一致,不打包:{emb} "
|
||
f"source={src_counts} snapshot={dst_counts}")
|
||
manifest["databases"].append({"source": str(emb), "snapshot": str(dst.relative_to(root)),
|
||
"sha256": sha256(dst), "method": method + "+dir-scan",
|
||
"source_table_counts": src_counts,
|
||
"snapshot_table_counts": dst_counts,
|
||
"rows_match": True})
|
||
dst = root / "files" / _unique_name(taken_file_names, src.name, str(src.resolve()))
|
||
if src.is_dir():
|
||
shutil.copytree(src, dst)
|
||
entries = {str(p.relative_to(dst)): sha256(p) for p in sorted(dst.rglob("*")) if p.is_file()}
|
||
else:
|
||
shutil.copy2(src, dst)
|
||
entries = {src.name: sha256(dst)}
|
||
manifest["files"].append({"source": str(src), "snapshot": str(dst.relative_to(root)),
|
||
"entries": entries})
|
||
if not manifest["databases"]:
|
||
raise SnapshotError("没有任何数据库进入一致性路径(--db 为空)")
|
||
manifest["verify"] = "rows_match=True (all, enforced; mismatch aborts packaging)"
|
||
(root / "MANIFEST.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
tar = out_dir / f"{name}-{stamp}.tar.gz"
|
||
with tarfile.open(tar, "w:gz") as tf:
|
||
tf.add(root, arcname=root.name)
|
||
manifest["archive"] = str(tar)
|
||
manifest["archive_sha256"] = sha256(tar)
|
||
return manifest
|
||
except Exception:
|
||
# 失败不形成可交付包:清除半成品目录与可能已生成的 tar。
|
||
shutil.rmtree(root, ignore_errors=True)
|
||
for partial in out_dir.glob(f"{name}-{stamp}.tar.gz"):
|
||
partial.unlink(missing_ok=True)
|
||
raise
|
||
|
||
|
||
def _restore_target(restore_root: Path, source: str) -> Path:
|
||
"""Map a recorded source path into the isolated restore tree, preserving
|
||
the full source hierarchy so same-basename DBs can never collide there."""
|
||
sanitized = source.strip("/").replace("/", "__")
|
||
return restore_root / "by-source" / sanitized
|
||
|
||
|
||
def verify_archive(archive: Path, restore_root: Path | None = None) -> dict:
|
||
"""回退演练:隔离新目录解包,按源路径映射**实际恢复**到独立目标树,
|
||
打开恢复出的每个库核对行数与哈希(不是只打开 tar 内布局自证)。
|
||
|
||
收口语义(HEL-879):
|
||
- 三个计数必须直接相等:manifest 源行数 = manifest 快照行数 = 解包实测;
|
||
`rows_match` 只是额外约束,缺失或矛盾明确拒绝(旧版 OR 逻辑允许记录值
|
||
绕过源计数比较);
|
||
- 恢复目标树默认落在归档旁的持久目录,rollback_plan 指向真实存在的恢复
|
||
文件(旧版输出随 TemporaryDirectory 消失的路径)。
|
||
"""
|
||
archive = Path(archive)
|
||
if restore_root is None:
|
||
restore_root = archive.parent / f"verify-restore-{archive.stem}-{time.strftime('%Y%m%d-%H%M%S')}"
|
||
result: dict = {"archive": str(archive), "sha256": sha256(archive),
|
||
"restore_root": str(restore_root), "checks": []}
|
||
with tempfile.TemporaryDirectory(prefix="hel879-verify-") as tmp:
|
||
with tarfile.open(archive) as tf:
|
||
tf.extractall(tmp)
|
||
root = next(p for p in Path(tmp).iterdir() if p.is_dir())
|
||
manifest = json.loads((root / "MANIFEST.json").read_text(encoding="utf-8"))
|
||
result["code_commit"] = manifest.get("code_commit")
|
||
result["mode"] = manifest.get("mode")
|
||
result["tool_version"] = manifest.get("tool_version", 1)
|
||
ok_all = True
|
||
rollback_plan = []
|
||
for d in manifest["databases"]:
|
||
snap = root / d["snapshot"]
|
||
# The actual restore step: copy into the isolated target tree at
|
||
# the recorded source path's mapped location, then verify THERE.
|
||
restored = _restore_target(restore_root, str(d.get("source") or d["snapshot"]))
|
||
restored.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy2(snap, restored)
|
||
digest = sha256(restored)
|
||
try:
|
||
counts = table_counts(restored)
|
||
except sqlite3.Error as exc:
|
||
result["checks"].append({"db": d["snapshot"], "restored": str(restored),
|
||
"sha256_ok": False, "rows_ok": False, "error": str(exc),
|
||
"rows_match_recorded": d.get("rows_match")})
|
||
ok_all = False
|
||
continue
|
||
recorded_rows_match = d.get("rows_match")
|
||
src_counts = d.get("source_table_counts")
|
||
snap_counts = d.get("snapshot_table_counts")
|
||
# Three-way equality is the only passing shape; the recorded flag
|
||
# is an extra constraint, never a substitute for the comparison.
|
||
counts_agree = (
|
||
isinstance(src_counts, dict) and isinstance(snap_counts, dict)
|
||
and counts == src_counts and counts == snap_counts
|
||
)
|
||
rows_ok = counts_agree
|
||
legacy_reject = recorded_rows_match is not True or not counts_agree
|
||
ok = digest == d["sha256"] and rows_ok and not legacy_reject
|
||
ok_all &= ok
|
||
result["checks"].append({"db": d["snapshot"], "restored": str(restored),
|
||
"sha256_ok": digest == d["sha256"],
|
||
"rows_ok": rows_ok, "counts": counts,
|
||
"source_counts": src_counts,
|
||
"snapshot_counts": snap_counts,
|
||
"rows_match_recorded": recorded_rows_match,
|
||
"rejected_legacy_package": legacy_reject})
|
||
rollback_plan.append({"source": d.get("source"), "restored_path": str(restored),
|
||
"restore_action": "停服后复制回源路径(行数与哈希已在恢复树实际核对)"})
|
||
for f in manifest["files"]:
|
||
base = root / f["snapshot"]
|
||
restored_dir = _restore_target(restore_root, str(f.get("source") or f["snapshot"]))
|
||
restored_dir.parent.mkdir(parents=True, exist_ok=True)
|
||
if base.is_dir():
|
||
shutil.copytree(base, restored_dir, dirs_exist_ok=True)
|
||
else:
|
||
shutil.copy2(base, restored_dir)
|
||
for rel, want in f["entries"].items():
|
||
p = (restored_dir / rel) if restored_dir.is_dir() else restored_dir
|
||
ok = sha256(p) == want
|
||
ok_all &= ok
|
||
result["checks"].append({"file": rel, "restored": str(p), "sha256_ok": ok})
|
||
rollback_plan.append({"source": f.get("source"), "restored_path": str(restored_dir),
|
||
"restore_action": "停服后按相对结构复制回源目录"})
|
||
result["rollback_plan"] = rollback_plan
|
||
result["ok"] = ok_all
|
||
return result
|
||
|
||
|
||
def _make_wal_db(path: Path) -> sqlite3.Connection:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
con = sqlite3.connect(str(path))
|
||
con.execute("PRAGMA journal_mode=WAL")
|
||
con.execute("CREATE TABLE companion(id INTEGER PRIMARY KEY, payload TEXT)")
|
||
con.execute("CREATE TABLE chat_message(id INTEGER PRIMARY KEY, text TEXT)")
|
||
con.execute("CREATE TABLE events(id INTEGER PRIMARY KEY, kind TEXT)")
|
||
con.commit()
|
||
con.execute("INSERT INTO companion VALUES(1,'her')")
|
||
con.execute("INSERT INTO chat_message VALUES(1,'hello')")
|
||
con.commit()
|
||
return con
|
||
|
||
|
||
def selftest(out_dir: Path) -> int:
|
||
"""临时库复现全部失败边界并证明修复(零产品数据、零网络)。"""
|
||
checks: list[dict] = []
|
||
with tempfile.TemporaryDirectory(prefix="hel879-snapshot-selftest-") as tmp:
|
||
tmp = Path(tmp)
|
||
db = tmp / "instance" / "companion.db"
|
||
con = _make_wal_db(db)
|
||
|
||
# ── 旧失败形状 1 复现:WAL 持有(读者未放)时 naive 复制丢已提交行。 ──
|
||
naive = tmp / "naive-copy.db"
|
||
shutil.copy2(db, naive)
|
||
try:
|
||
row = sqlite3.connect(f"file:{naive}?mode=ro", uri=True).execute(
|
||
"SELECT count(*) FROM companion").fetchone()[0]
|
||
naive_result = f"rows={row}"
|
||
except sqlite3.OperationalError as exc:
|
||
naive_result = f"error={exc}"
|
||
checks.append({"name": "naive_online_copy_loses_wal_reproduced",
|
||
"ok": "rows=1" not in naive_result, "detail": naive_result})
|
||
|
||
# ── 旧失败形状 2 复现(总管 edges 场景):读者先持读事务,随后写入方提交
|
||
# events=1;TRUNCATE checkpoint 返回 busy=[1,1,0],naive 快照丢该行,
|
||
# 旧工具此时仍打包成功。 ──
|
||
reader = sqlite3.connect(str(db))
|
||
reader.execute("BEGIN")
|
||
reader.execute("SELECT count(*) FROM companion").fetchone()
|
||
con.execute("INSERT INTO events VALUES(1,'reach_out')")
|
||
con.commit()
|
||
busy_state = checkpoint(db)
|
||
naive2 = tmp / "naive2.db"
|
||
shutil.copy2(db, naive2)
|
||
try:
|
||
naive2_events = sqlite3.connect(f"file:{naive2}?mode=ro", uri=True).execute(
|
||
"SELECT count(*) FROM events").fetchone()[0]
|
||
except sqlite3.Error as exc:
|
||
naive2_events = f"error={type(exc).__name__}:{exc}"
|
||
checks.append({"name": "busy_checkpoint_state_reproduced",
|
||
"ok": busy_state[0] != 0 and naive2_events != 1,
|
||
"detail": f"checkpoint={busy_state} naive_snapshot_events={naive2_events}"})
|
||
|
||
# ── 修复 1:busy checkpoint 必须明确失败,不形成归档。 ──
|
||
out_busy = tmp / "archives-busy"
|
||
out_busy.mkdir()
|
||
failed_busy = None
|
||
try:
|
||
make_snapshot("busy-must-fail", [db], [], out_busy, online=False)
|
||
except SnapshotError as exc:
|
||
failed_busy = str(exc)
|
||
leftovers = list(out_busy.iterdir())
|
||
checks.append({"name": "fix1_busy_checkpoint_fails_no_archive",
|
||
"ok": failed_busy is not None and "busy" in failed_busy and not leftovers,
|
||
"detail": f"error={'yes' if failed_busy else 'none'} leftovers={len(leftovers)}"})
|
||
|
||
reader.rollback()
|
||
reader.close()
|
||
|
||
# ── 修复 1b:源/快照行数不一致(快照停在某刻、源继续前进的形状)必须
|
||
# 明确失败、不形成归档。用停当快照替换法直接触发比对分支。 ──
|
||
out_mis = tmp / "archives-mismatch"
|
||
out_mis.mkdir()
|
||
stale = out_mis / "stale.db"
|
||
with sqlite3.connect(str(db)) as source, sqlite3.connect(str(stale)) as target:
|
||
source.backup(target)
|
||
con.execute("INSERT INTO events VALUES(2,'after')")
|
||
con.commit()
|
||
real_snapshot_db = snapshot_db
|
||
_self = globals()
|
||
|
||
def _stale_snapshot(src: Path, dst: Path, *, online: bool) -> str:
|
||
shutil.copy2(stale, dst)
|
||
return "stop+checkpoint+copy(stale-standin)"
|
||
|
||
_self["snapshot_db"] = _stale_snapshot
|
||
failed_mis = None
|
||
try:
|
||
make_snapshot("mismatch-must-fail", [db], [], out_mis, online=False)
|
||
except SnapshotError as exc:
|
||
failed_mis = str(exc)
|
||
finally:
|
||
_self["snapshot_db"] = real_snapshot_db
|
||
mis_leftovers = list(out_mis.glob("mismatch-must-fail*"))
|
||
checks.append({"name": "fix1b_row_mismatch_fails_no_archive",
|
||
"ok": failed_mis is not None and "不一致" in failed_mis
|
||
and not [p for p in mis_leftovers if p.suffix == ".gz"],
|
||
"detail": f"error={failed_mis[:60] if failed_mis else 'none'} "
|
||
f"artifacts={[p.name for p in mis_leftovers]}"})
|
||
con.execute("DELETE FROM events WHERE id=2")
|
||
con.commit()
|
||
|
||
# ── 旧失败形状 3 复现:手工构造 rows_match=False 的旧包,旧版 verify 仍 ok。 ──
|
||
legacy_root = tmp / "legacy-pkg"
|
||
legacy_root.mkdir()
|
||
(legacy_root / "databases").mkdir()
|
||
legacy_snap = legacy_root / "databases" / "companion.db"
|
||
with sqlite3.connect(str(db)) as source, sqlite3.connect(str(legacy_snap)) as target:
|
||
source.backup(target)
|
||
# 漏行形状:快照内删掉 events 行,manifest 记 rows_match=False(旧版真实输出)。
|
||
with sqlite3.connect(str(legacy_snap)) as lk:
|
||
lk.execute("DELETE FROM events")
|
||
legacy_manifest = {
|
||
"name": "legacy", "code_commit": "legacy-shape", "tool": "release_snapshot_hel896.py",
|
||
"tool_version": 1, "mode": "stop-checkpoint-copy",
|
||
"databases": [{"source": str(db), "snapshot": "databases/companion.db",
|
||
"sha256": sha256(legacy_snap), "method": "stop+checkpoint+copy",
|
||
"source_table_counts": {"companion": 1, "chat_message": 1, "events": 1},
|
||
"snapshot_table_counts": {"companion": 1, "chat_message": 1, "events": 0},
|
||
"rows_match": False}],
|
||
"files": [], "verify": "rows_match=False",
|
||
}
|
||
(legacy_root / "MANIFEST.json").write_text(
|
||
json.dumps(legacy_manifest, ensure_ascii=False), encoding="utf-8")
|
||
legacy_tar = tmp / "legacy-pkg.tar.gz"
|
||
with tarfile.open(legacy_tar, "w:gz") as tf:
|
||
tf.add(legacy_root, arcname=legacy_root.name)
|
||
legacy_verify = verify_archive(legacy_tar)
|
||
checks.append({"name": "fix_verify_rejects_rows_match_false_legacy_package",
|
||
"ok": legacy_verify["ok"] is False,
|
||
"detail": f"ok={legacy_verify['ok']} checks={legacy_verify['checks'][:1]}"})
|
||
|
||
# ── 修复 2:目录内 SQLite(history_root 形状)。 ──
|
||
hist = tmp / "instance" / "history_root"
|
||
hist.mkdir(parents=True, exist_ok=True)
|
||
mem_hist = hist / "mem0_public_history.db"
|
||
mem_con = sqlite3.connect(str(mem_hist))
|
||
mem_con.execute("PRAGMA journal_mode=WAL")
|
||
mem_con.execute("CREATE TABLE history(id INTEGER PRIMARY KEY, payload TEXT)")
|
||
mem_con.execute("INSERT INTO history VALUES(1,'mem')")
|
||
mem_con.commit()
|
||
mem_reader = sqlite3.connect(str(mem_hist))
|
||
mem_reader.execute("BEGIN")
|
||
mem_reader.execute("SELECT count(*) FROM history").fetchone()
|
||
# 在线模式 + 含 SQLite 的目录 → 拒绝。
|
||
out_online = tmp / "archives-online"
|
||
out_online.mkdir()
|
||
refused = None
|
||
try:
|
||
make_snapshot("online-dir-refused", [db], [hist], out_online, online=True)
|
||
except SnapshotError as exc:
|
||
refused = str(exc)
|
||
checks.append({"name": "fix2_online_mode_refuses_dir_with_sqlite",
|
||
"ok": refused is not None and "拒绝" in refused and not list(out_online.iterdir()),
|
||
"detail": refused[:120] if refused else "not refused"})
|
||
mem_reader.rollback()
|
||
mem_reader.close()
|
||
# 停服口径:目录内 SQLite 走 checkpoint+复制+逐表核对(mem_reader 已放,
|
||
# 但 mem_con 仍持有 WAL——checkpoint 必须真实完成才算过)。
|
||
mem_con.close()
|
||
|
||
# 主库也关闭,进入停服状态。
|
||
con.close()
|
||
|
||
vec = tmp / "instance" / "vec"
|
||
vec.mkdir(exist_ok=True)
|
||
(vec / "vectors.bin").write_bytes(b"\x00\x01\x02cg")
|
||
out = tmp / "archives"
|
||
out.mkdir()
|
||
manifest = make_snapshot("hel879-selftest", [db], [vec, hist], out, online=False)
|
||
# 从解包侧核对 manifest(目录内 SQLite 是否真走了逐表核对)。
|
||
with tarfile.open(manifest["archive"]) as tf:
|
||
tf.extractall(out / "unpack")
|
||
unpack_root = out / "unpack" / Path(manifest["archive"]).name.replace(".tar.gz", "")
|
||
unpack_manifest = json.loads((unpack_root / "MANIFEST.json").read_text(encoding="utf-8"))
|
||
embedded_dbs = [d for d in unpack_manifest["databases"] if "+dir-scan" in d["method"]]
|
||
hist_ok = any("mem0_public_history.db" in d["source"] and d["rows_match"] is True
|
||
and d["source_table_counts"].get("history") == 1 for d in embedded_dbs)
|
||
checks.append({"name": "fix2_stop_mode_embeds_sqlite_via_consistency_path",
|
||
"ok": bool(embedded_dbs) and hist_ok,
|
||
"detail": f"embedded={[d['source'] for d in embedded_dbs]}"})
|
||
|
||
# ── 修复 3:回退演练按源路径映射、实测行数与哈希、旧包拒绝已验。 ──
|
||
verify = verify_archive(Path(manifest["archive"]))
|
||
plan_sources = {entry.get("source") for entry in verify.get("rollback_plan", [])}
|
||
mapped = str(db) in plan_sources and any(
|
||
"mem0_public_history.db" in (s or "") for s in plan_sources)
|
||
checks.append({"name": "fix3_verify_maps_source_paths_and_passes",
|
||
"ok": verify["ok"] and mapped,
|
||
"detail": f"ok={verify['ok']} commit={verify.get('code_commit')} "
|
||
f"plan_entries={len(verify.get('rollback_plan', []))}"})
|
||
|
||
# backup API(在线口径,仅显式单库)依旧完整。
|
||
with sqlite3.connect(str(db)) as source, sqlite3.connect(str(tmp / "via-api.db")) as target:
|
||
source.backup(target)
|
||
api_counts = table_counts(tmp / "via-api.db")
|
||
checks.append({"name": "backup_api_snapshot_complete",
|
||
"ok": api_counts.get("companion") == 1 and api_counts.get("chat_message") == 1,
|
||
"detail": json.dumps(api_counts)})
|
||
|
||
# ── 总管对 v2 的 edges(HEL-879 独立复核)──
|
||
# Edge A:manifest 记 rows_match=true 但源计数被改为 events=2(快照=1)。
|
||
# v2 的 `rows_match is True or ...` 绕过源计数比较 → 仍 ok;v3 必须拒绝。
|
||
clean = make_snapshot("edge-counts", [db], [], out, online=False)
|
||
edge_unpack = out / "edge-unpack"
|
||
with tarfile.open(clean["archive"]) as tf:
|
||
tf.extractall(edge_unpack)
|
||
edge_pkg = next(edge_unpack.iterdir())
|
||
edge_manifest_path = edge_pkg / "MANIFEST.json"
|
||
edge_manifest = json.loads(edge_manifest_path.read_text(encoding="utf-8"))
|
||
edge_manifest["databases"][0]["source_table_counts"]["events"] = 2
|
||
edge_manifest_path.write_text(json.dumps(edge_manifest, ensure_ascii=False))
|
||
edge_tar = tmp / "edge-counts-tampered.tar.gz"
|
||
with tarfile.open(edge_tar, "w:gz") as tf:
|
||
tf.add(edge_pkg, arcname=edge_pkg.name)
|
||
edge_verify = verify_archive(edge_tar, restore_root=tmp / "edge-restore")
|
||
checks.append({"name": "fix_rows_match_true_cannot_bypass_source_counts",
|
||
"ok": edge_verify["ok"] is False and all(
|
||
c.get("rejected_legacy_package") for c in edge_verify["checks"]
|
||
if "rejected_legacy_package" in c),
|
||
"detail": f"ok={edge_verify['ok']} "
|
||
f"counts={edge_verify['checks'][0].get('counts')} "
|
||
f"source={edge_verify['checks'][0].get('source_counts')}"})
|
||
|
||
# Edge B:两份不同目录的同名 ledger.db。v2 后写覆盖前写、快照仍交成功包;
|
||
# v3 无冲突映射 + verify 全过 + 恢复树里两个源路径各自落位。
|
||
la = tmp / "ledger-a" / "ledger.db"
|
||
lb = tmp / "ledger-b" / "ledger.db"
|
||
for path, value in [(la, "first"), (lb, "second")]:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
with sqlite3.connect(str(path)) as conn:
|
||
conn.execute("CREATE TABLE item(value TEXT)")
|
||
conn.execute("INSERT INTO item VALUES(?)", (value,))
|
||
same = make_snapshot("same-basename", [la, lb], [], out, online=False)
|
||
snap_names = [Path(d["snapshot"]).name for d in same["databases"]]
|
||
same_restore = tmp / "same-restore"
|
||
same_verify = verify_archive(Path(same["archive"]), restore_root=same_restore)
|
||
plan = same_verify.get("rollback_plan", [])
|
||
restored_paths = [e["restored_path"] for e in plan if e.get("source", "").endswith("ledger.db")]
|
||
distinct = len(set(restored_paths)) == 2 and all(
|
||
Path(p).is_file() for p in restored_paths)
|
||
rows = sorted(sqlite3.connect(f"file:{p}?mode=ro", uri=True)
|
||
.execute("SELECT value FROM item").fetchone()[0]
|
||
for p in restored_paths)
|
||
dup_failed = None
|
||
try:
|
||
make_snapshot("dup-source", [la, la], [], out, online=False)
|
||
except SnapshotError as exc:
|
||
dup_failed = str(exc)
|
||
checks.append({"name": "fix_same_basename_no_overwrite_and_duplicate_refused",
|
||
"ok": (len(set(snap_names)) == 2 and same_verify["ok"] is True
|
||
and distinct and rows == ["first", "second"]
|
||
and dup_failed is not None),
|
||
"detail": f"snap_names={snap_names} verify_ok={same_verify['ok']} "
|
||
f"restored_rows={rows} dup_refused={'yes' if dup_failed else 'no'}"})
|
||
# 恢复路径必须真实存在于恢复树(不是随 TemporaryDirectory 消失的路径)。
|
||
checks.append({"name": "restore_drill_paths_persist",
|
||
"ok": all(Path(p).is_file() for p in restored_paths)
|
||
and same_restore.is_dir(),
|
||
"detail": f"restore_root={same_restore}"})
|
||
ok = all(c["ok"] for c in checks)
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
(out_dir / "selftest.json").write_text(
|
||
json.dumps({"ok": ok, "code_commit": git_head(),
|
||
"tool_version": TOOL_VERSION, "checks": checks}, ensure_ascii=False, indent=2),
|
||
encoding="utf-8")
|
||
for c in checks:
|
||
print(f"{'PASS' if c['ok'] else 'FAIL'} {c['name']} — {c['detail']}")
|
||
print(json.dumps({"ok": ok}, ensure_ascii=False))
|
||
return 0 if ok else 1
|
||
|
||
|
||
def main() -> int:
|
||
ap = argparse.ArgumentParser(description=__doc__)
|
||
ap.add_argument("--selftest", action="store_true")
|
||
ap.add_argument("command", nargs="?", choices=["snapshot", "verify"])
|
||
ap.add_argument("--name")
|
||
ap.add_argument("--db", action="append", default=[], type=Path)
|
||
ap.add_argument("--file", action="append", default=[], type=Path)
|
||
ap.add_argument("--out", type=Path, default=REPO / "deploy" / "archives")
|
||
ap.add_argument("--online", action="store_true",
|
||
help="在线口径:仅显式单库 backup API(默认停服+checkpoint;含 SQLite 的 --file 目录会被拒绝)")
|
||
ap.add_argument("--archive", type=Path)
|
||
ap.add_argument("--restore-dir", type=Path,
|
||
help="回退演练的恢复目标树(默认:归档旁 verify-restore-<name>-<时间>/,持久保留供回滚使用)")
|
||
args = ap.parse_args()
|
||
if args.selftest:
|
||
return selftest(REPO / "docs/rework/hel879/release-snapshot")
|
||
if args.command == "snapshot":
|
||
if not args.name or not args.db:
|
||
ap.error("snapshot 需要 --name 与至少一个 --db")
|
||
out_dir = args.out
|
||
out_dir.mkdir(parents=True, exist_ok=True)
|
||
try:
|
||
manifest = make_snapshot(args.name, args.db, args.file, out_dir, online=args.online)
|
||
except SnapshotError as exc:
|
||
print(json.dumps({"ok": False, "error": str(exc),
|
||
"deliverable_archive": None}, ensure_ascii=False, indent=1))
|
||
return 1
|
||
print(json.dumps({k: manifest[k] for k in ("archive", "archive_sha256", "code_commit", "mode", "verify")},
|
||
ensure_ascii=False, indent=1))
|
||
return 0
|
||
if args.command == "verify":
|
||
if not args.archive:
|
||
ap.error("verify 需要 --archive")
|
||
result = verify_archive(args.archive, restore_root=args.restore_dir)
|
||
print(json.dumps(result, ensure_ascii=False, indent=1))
|
||
return 0 if result["ok"] else 1
|
||
ap.print_help()
|
||
return 2
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|