migration: preserve market insights slice

2026-07-31 02:41:56 +08:00
parent 814e75730a
commit cf2aad28ec
30 changed files with 2487 additions and 1759 deletions
@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import http.cookiejar
+import json
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+
+def request_json(
+    opener: urllib.request.OpenerDirector,
+    url: str,
+    payload: dict[str, Any] | None = None,
+) -> tuple[int, Any]:
+    data = None
+    headers = {"Accept": "application/json"}
+    if payload is not None:
+        data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
+        headers["Content-Type"] = "application/json"
+    request = urllib.request.Request(url, data=data, headers=headers)
+    try:
+        with opener.open(request, timeout=90) as response:
+            return response.status, json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        return exc.code, json.loads(exc.read().decode("utf-8"))
+
+
+def session(base_url: str, username: str, password: str) -> urllib.request.OpenerDirector:
+    opener = urllib.request.build_opener(
+        urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar())
+    )
+    status, body = request_json(
+        opener,
+        f"{base_url.rstrip('/')}/api/auth/login",
+        {"username": username, "password": password},
+    )
+    if status != 200 or not body.get("ok"):
+        raise RuntimeError(f"Login failed for {base_url}: HTTP {status} {body}")
+    return opener
+
+
+def digest(value: Any) -> str:
+    content = json.dumps(
+        value, ensure_ascii=False, sort_keys=True, separators=(",", ":")
+    ).encode("utf-8")
+    return hashlib.sha256(content).hexdigest()
+
+
+def comparable(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {
+            key: comparable(item)
+            for key, item in value.items()
+            if key != "request_id"
+        }
+    if isinstance(value, list):
+        return [comparable(item) for item in value]
+    return value
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Compare authenticated preservation APIs")
+    parser.add_argument("--original", required=True)
+    parser.add_argument("--migrated", required=True)
+    parser.add_argument("--username", required=True)
+    parser.add_argument("--password", required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("endpoints", nargs="+")
+    args = parser.parse_args()
+
+    original = session(args.original, args.username, args.password)
+    migrated = session(args.migrated, args.username, args.password)
+    rows = []
+    all_equal = True
+    for endpoint in args.endpoints:
+        original_status, original_body = request_json(
+            original, f"{args.original.rstrip('/')}{endpoint}"
+        )
+        migrated_status, migrated_body = request_json(
+            migrated, f"{args.migrated.rstrip('/')}{endpoint}"
+        )
+        original_comparable = comparable(original_body)
+        migrated_comparable = comparable(migrated_body)
+        equal = original_status == migrated_status and original_comparable == migrated_comparable
+        all_equal = all_equal and equal
+        rows.append(
+            {
+                "endpoint": endpoint,
+                "original_status": original_status,
+                "migrated_status": migrated_status,
+                "original_sha256": digest(original_comparable),
+                "migrated_sha256": digest(migrated_comparable),
+                "equal": equal,
+            }
+        )
+
+    result = {"all_equal": all_equal, "endpoints": rows}
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(
+        json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+    )
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    if not all_equal:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+
+def digest(value: Any) -> str:
+    content = json.dumps(
+        value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str
+    ).encode("utf-8")
+    return hashlib.sha256(content).hexdigest()
+
+
+def schema(connection: sqlite3.Connection) -> list[dict[str, Any]]:
+    rows = connection.execute(
+        """
+        SELECT type, name, tbl_name, sql
+        FROM sqlite_master
+        WHERE name NOT LIKE 'sqlite_%'
+        ORDER BY type, name
+        """
+    ).fetchall()
+    return [dict(row) for row in rows]
+
+
+def table_rows(connection: sqlite3.Connection, table: str) -> list[dict[str, Any]]:
+    quoted = '"' + table.replace('"', '""') + '"'
+    rows = [dict(row) for row in connection.execute(f"SELECT * FROM {quoted}").fetchall()]
+    return sorted(rows, key=lambda row: json.dumps(row, ensure_ascii=False, sort_keys=True, default=str))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Compare preservation SQLite databases")
+    parser.add_argument("--original", type=Path, required=True)
+    parser.add_argument("--migrated", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("tables", nargs="+")
+    args = parser.parse_args()
+
+    original = sqlite3.connect(args.original)
+    migrated = sqlite3.connect(args.migrated)
+    original.row_factory = sqlite3.Row
+    migrated.row_factory = sqlite3.Row
+    try:
+        original_schema = schema(original)
+        migrated_schema = schema(migrated)
+        tables = []
+        all_equal = original_schema == migrated_schema
+        for table in args.tables:
+            original_rows = table_rows(original, table)
+            migrated_rows = table_rows(migrated, table)
+            equal = original_rows == migrated_rows
+            all_equal = all_equal and equal
+            tables.append(
+                {
+                    "table": table,
+                    "original_count": len(original_rows),
+                    "migrated_count": len(migrated_rows),
+                    "original_sha256": digest(original_rows),
+                    "migrated_sha256": digest(migrated_rows),
+                    "equal": equal,
+                }
+            )
+        result = {
+            "all_equal": all_equal,
+            "schema": {
+                "object_count": len(original_schema),
+                "original_sha256": digest(original_schema),
+                "migrated_sha256": digest(migrated_schema),
+                "equal": original_schema == migrated_schema,
+            },
+            "tables": tables,
+        }
+    finally:
+        original.close()
+        migrated.close()
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(
+        json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+    )
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    if not all_equal:
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from http.server import ThreadingHTTPServer
+from pathlib import Path
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run an isolated preservation runtime")
+    parser.add_argument("--runtime-root", type=Path, required=True)
+    parser.add_argument("--data-dir", type=Path, required=True)
+    parser.add_argument("--port", type=int, required=True)
+    args = parser.parse_args()
+
+    runtime_root = args.runtime_root.resolve()
+    data_dir = args.data_dir.resolve()
+    data_dir.mkdir(parents=True, exist_ok=True)
+    sys.path.insert(0, str(runtime_root))
+
+    if (runtime_root / "backend" / "bootstrap" / "config.py").is_file():
+        from backend.bootstrap import config
+
+        config.DATA_DIR = data_dir
+        config.PRIVATE_MENTOR_SKILLS_DIR = data_dir / "private-mentor-skills"
+    else:
+        import app_config as config
+
+        config.DATA_DIR = data_dir
+        config.PRIVATE_MENTOR_SKILLS_DIR = data_dir / "private-mentor-skills"
+
+    from server import RequestHandler, SERVICE
+
+    server = ThreadingHTTPServer(("127.0.0.1", args.port), RequestHandler)
+    print(f"Preservation runtime is running at http://127.0.0.1:{args.port}", flush=True)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:
+        pass
+    finally:
+        SERVICE._background_stop.set()
+        server.server_close()
+
+
+if __name__ == "__main__":
+    main()