diff --git a/deploy/voice/serve_dual_8360_8361.py b/deploy/voice/serve_dual_8360_8361.py index 567827f..d87c39c 100644 --- a/deploy/voice/serve_dual_8360_8361.py +++ b/deploy/voice/serve_dual_8360_8361.py @@ -1,9 +1,20 @@ """统一 CyberGril 实例单进程双入口启动器(HEL-877 部署草案,本卡未执行). HTTP 8360 与 HTTPS 8361(语音页需要 secure context 才能拿麦克风)必须共用 -同一个 API 进程、同一份 app.state(epoch / chat_lock / store / 语音会话登记): -本启动器在一个事件循环里跑两个 uvicorn Server,指向同一个 FastAPI 应用对象, -不是两个实例。自签证书签发命令与回滚步骤见 docs/runbook-hel877-voice.md。 +同一个 API 进程、同一份 app.state(epoch / chat_lock / store / 语音会话登记)。 + +HEL-879 收口(总管复核第 1 项):uvicorn 两个 Server 默认各自进入一次 FastAPI +lifespan,导致 epoch 1→2、Store/life/chat_lock 全部被第二次初始化替换——同一个 +app 对象并不保证只初始化一次。本启动器改为: + +- 应用生命周期由本启动器**独占管理且只进入一次**(`app.router.lifespan_context`); +- 两个 uvicorn Server 以 `lifespan="off"` 只负责监听,共享同一份状态; +- 任一监听退出(失败/取消/信号)先把另一个也停掉,**再**退出生命周期: + 绝不出现一边还在接请求、一边已被另一个监听提前关库;循环与资源只关闭一次; +- `create_app` 的 lifespan 自带重入门卫:任何二次进入直接报错(fail-closed), + 防止未来改回 auto 双 Server 时再次静默替换状态。 + +自签证书签发命令与回滚步骤见 docs/runbook-hel877-voice.md。 用法(部署卡执行,先 `cd` 到仓库根并激活 3.11 venv): PYTHONPATH=src python deploy/voice/serve_dual_8360_8361.py \ @@ -28,24 +39,52 @@ import uvicorn from cybergril.api.app import create_app -async def serve(http_port: int, https_port: int, cert: str, key: str) -> None: - # ONE app object: both servers share state, locks, epoch and the store. - app = create_app() +async def serve(http_port: int, https_port: int, cert: str, key: str, app=None) -> None: + # ONE app object, ONE lifespan owner: both listeners share state, locks, + # epoch and the store. The uvicorn servers never run the app lifespan + # themselves (lifespan="off") — the context manager below is its only + # owner, so initialization and shutdown each happen exactly once. + if app is None: + app = create_app() http_server = uvicorn.Server( - uvicorn.Config(app, host="0.0.0.0", port=http_port, log_level="info") + uvicorn.Config(app, host="0.0.0.0", port=http_port, log_level="info", + lifespan="off") ) https_server = uvicorn.Server( uvicorn.Config(app, host="0.0.0.0", port=https_port, ssl_certfile=cert, - ssl_keyfile=key, log_level="info") + ssl_keyfile=key, log_level="info", lifespan="off") ) - servers = [asyncio.create_task(http_server.serve()), asyncio.create_task(https_server.serve())] - done, pending = await asyncio.wait(servers, return_when=asyncio.FIRST_COMPLETED) - for task in pending: - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - for task in done: - task.result() + async with app.router.lifespan_context(app): + servers = [asyncio.create_task(http_server.serve()), + asyncio.create_task(https_server.serve())] + try: + done, pending = await asyncio.wait(servers, + return_when=asyncio.FIRST_COMPLETED) + except asyncio.CancelledError: + # The launcher itself was cancelled: stop both listeners first + # (graceful should_exit, hard cancel only as a last resort), await + # their full exit, only then let the lifespan close (and re-raise). + # No listener is ever left running without its state. + for server in (http_server, https_server): + server.should_exit = True + for task in servers: + with contextlib.suppress(asyncio.CancelledError, Exception): + await asyncio.wait_for(asyncio.shield(task), timeout=5.0) + for task in servers: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + raise + # Any listener ending stops the other FIRST; only after both listeners + # have fully exited does the lifespan context above close the loop's + # resources and the store — never the other way around. + for server in (http_server, https_server): + server.should_exit = True + for task in pending: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + for task in done: + task.result() # surface the first listener's real error, after cleanup def main() -> int: diff --git a/docs/rework/hel879/release-snapshot/selftest.json b/docs/rework/hel879/release-snapshot/selftest.json index 39015ec..9b42433 100644 --- a/docs/rework/hel879/release-snapshot/selftest.json +++ b/docs/rework/hel879/release-snapshot/selftest.json @@ -1,7 +1,7 @@ { "ok": true, - "code_commit": "7f66015ac854b7f513a35a42b680a143058561bb", - "tool_version": 2, + "code_commit": "7f0a739213e1cd8010d6e7118827884e973ea7c9", + "tool_version": 3, "checks": [ { "name": "naive_online_copy_loses_wal_reproduced", @@ -21,32 +21,47 @@ { "name": "fix1b_row_mismatch_fails_no_archive", "ok": true, - "detail": "error=源/快照行数不一致,不打包:/tmp/multica-task-29494715/hel879-snapshot-sel artifacts=[]" + "detail": "error=源/快照行数不一致,不打包:/tmp/multica-task-3939683741/hel879-snapshot-s artifacts=[]" }, { "name": "fix_verify_rejects_rows_match_false_legacy_package", "ok": true, - "detail": "ok=False checks=[{'db': 'databases/companion.db', 'sha256_ok': True, 'rows_ok': False, 'counts': {'companion': 1, 'chat_message': 1, 'events': 0}, 'source_counts': {'companion': 1, 'chat_message': 1, 'events': 1}, 'rows_match_recorded': False, 'rejected_legacy_package': True}]" + "detail": "ok=False checks=[{'db': 'databases/companion.db', 'restored': '/tmp/multica-task-3939683741/hel879-snapshot-selftest-jc481l71/verify-restore-legacy-pkg.tar-20261027-035938/by-source/tmp__multica-task-3939683741__hel879-snapshot-selftest-jc481l71__instance__companion.db', 'sha256_ok': True, 'rows_ok': False, 'counts': {'companion': 1, 'chat_message': 1, 'events': 1}, 'source_counts': {'companion': 1, 'chat_message': 1, 'events': 1}, 'snapshot_counts': {'companion': 1, 'chat_message': 1, 'events': 0}, 'rows_match_recorded': False, 'rejected_legacy_package': True}]" }, { "name": "fix2_online_mode_refuses_dir_with_sqlite", "ok": true, - "detail": "在线模式只允许显式单库 backup API(--db);--file /tmp/multica-task-29494715/hel879-snapshot-selftest-pxbo6xw1/instance/history_root 内" + "detail": "在线模式只允许显式单库 backup API(--db);--file /tmp/multica-task-3939683741/hel879-snapshot-selftest-jc481l71/instance/history_root" }, { "name": "fix2_stop_mode_embeds_sqlite_via_consistency_path", "ok": true, - "detail": "embedded=['/tmp/multica-task-29494715/hel879-snapshot-selftest-pxbo6xw1/instance/history_root/mem0_public_history.db']" + "detail": "embedded=['/tmp/multica-task-3939683741/hel879-snapshot-selftest-jc481l71/instance/history_root/mem0_public_history.db']" }, { "name": "fix3_verify_maps_source_paths_and_passes", "ok": true, - "detail": "ok=True commit=7f66015ac854b7f513a35a42b680a143058561bb plan_entries=4" + "detail": "ok=True commit=7f0a739213e1cd8010d6e7118827884e973ea7c9 plan_entries=4" }, { "name": "backup_api_snapshot_complete", "ok": true, "detail": "{\"companion\": 1, \"chat_message\": 1, \"events\": 1}" + }, + { + "name": "fix_rows_match_true_cannot_bypass_source_counts", + "ok": true, + "detail": "ok=False counts={'companion': 1, 'chat_message': 1, 'events': 1} source={'companion': 1, 'chat_message': 1, 'events': 2}" + }, + { + "name": "fix_same_basename_no_overwrite_and_duplicate_refused", + "ok": true, + "detail": "snap_names=['ledger.db', 'ledger__0e56d727.db'] verify_ok=True restored_rows=['first', 'second'] dup_refused=yes" + }, + { + "name": "restore_drill_paths_persist", + "ok": true, + "detail": "restore_root=/tmp/multica-task-3939683741/hel879-snapshot-selftest-jc481l71/same-restore" } ] } \ No newline at end of file diff --git a/src/cybergril/api/app.py b/src/cybergril/api/app.py index 5361ed0..6b99275 100644 --- a/src/cybergril/api/app.py +++ b/src/cybergril/api/app.py @@ -210,73 +210,90 @@ def create_app( @asynccontextmanager async def lifespan(app): - store = Store(db_path) - app.state.store = store - with store.conn: - store.execute( - "INSERT INTO api_meta(key,value) VALUES('epoch',1) ON CONFLICT(key) DO UPDATE SET value=value+1" - ) - store.execute("UPDATE chat_message SET status='cancelled' WHERE status='pending'") - app.state.epoch = store.query_one("SELECT value FROM api_meta WHERE key='epoch'")["value"] - app.state.active = None - app.state.chat_lock = asyncio.Lock() - app.state.health_cache = (0, False) - app.state.health_lock = asyncio.Lock() - router.guard = PersistentBudget(store, router.guard) - app.state.sharing = Sharing(store, router, models) - app.state.persona = Persona(store, router, models, enabled=life_config.enabled) - app.state.public = PublicView(store) - app.state.mood = MoodRuntime(store, character) - app.state.dyad = DyadRuntime(store, character, router=router, models=models, - timezone=life_config.timezone) - app.state.semantic = build_semantic(store, mode=semantic_mode, config=semantic_config, - request_guard=router.guard) - # Original positional contract preserved; R4 collaborators attach after assembly. - app.state.life = LifeRuntime(store, router, models, life_config, app.state.sharing, - app.state.persona) - app.state.life.mood = app.state.mood - app.state.life.semantic = app.state.semantic - app.state.life.dyad = app.state.dyad - app.state.life.reach_out = ReachOut(store, router, models, app.state.persona, - app.state.mood, config=character.reach_out, - timezone=life_config.timezone) - # Preserve the original runtime factory contract. Assemble only after its - # clock is known, before any recovery or background work can run. - app.state.memories = Memories(store, router, models, timezone=life_config.timezone, - clock=lambda: app.state.life.clock()) - app.state.life.memories = app.state.memories - app.state.memories.recover() - app.state.mood.recover() - if app.state.semantic: - app.state.semantic.recover() - if app.state.life.reach_out: - app.state.life.reach_out.recover() - life_task = asyncio.create_task(app.state.life.run()) + # HEL-879 dual-entry guard: a second concurrent entry (e.g. two uvicorn + # servers both running the default lifespan against this app) must fail + # loudly, never silently re-initialize Store/epoch/life/chat_lock under + # live requests. Re-entry after a completed exit stays legitimate. + if getattr(app.state, "lifespan_active", False): + raise RuntimeError( + "application lifespan entered twice — the lifecycle has exactly " + "one owner (deploy/voice/serve_dual_8360_8361.py runs uvicorn " + "with lifespan='off' and owns it once)") + app.state.lifespan_active = True try: - yield - finally: - app.state.life.stop.set() - if life_task: - life_task.cancel() - with suppress(asyncio.CancelledError): - await life_task - if app.state.active: - app.state.active["cancel"].set() - await app.state.active["finished"].wait() + store = Store(db_path) + app.state.store = store + with store.conn: + store.execute( + "INSERT INTO api_meta(key,value) VALUES('epoch',1) ON CONFLICT(key) DO UPDATE SET value=value+1" + ) + store.execute("UPDATE chat_message SET status='cancelled' WHERE status='pending'") + app.state.epoch = store.query_one("SELECT value FROM api_meta WHERE key='epoch'")["value"] + app.state.active = None + app.state.chat_lock = asyncio.Lock() + app.state.health_cache = (0, False) + app.state.health_lock = asyncio.Lock() + router.guard = PersistentBudget(store, router.guard) + app.state.sharing = Sharing(store, router, models) + app.state.persona = Persona(store, router, models, enabled=life_config.enabled) + app.state.public = PublicView(store) + app.state.mood = MoodRuntime(store, character) + app.state.dyad = DyadRuntime(store, character, router=router, models=models, + timezone=life_config.timezone) + app.state.semantic = build_semantic(store, mode=semantic_mode, config=semantic_config, + request_guard=router.guard) + # Original positional contract preserved; R4 collaborators attach after assembly. + app.state.life = LifeRuntime(store, router, models, life_config, app.state.sharing, + app.state.persona) + app.state.life.mood = app.state.mood + app.state.life.semantic = app.state.semantic + app.state.life.dyad = app.state.dyad + app.state.life.reach_out = ReachOut(store, router, models, app.state.persona, + app.state.mood, config=character.reach_out, + timezone=life_config.timezone) + # Preserve the original runtime factory contract. Assemble only after its + # clock is known, before any recovery or background work can run. + app.state.memories = Memories(store, router, models, timezone=life_config.timezone, + clock=lambda: app.state.life.clock()) + app.state.life.memories = app.state.memories + app.state.memories.recover() + app.state.mood.recover() if app.state.semantic: - # HEL-889/894 shutdown ownership: aclose returns only after - # every live batch worker has actually exited (drain_timeout - # bounds the polite phase, never a licence to close under a - # worker). If this await is cancelled after the resource - # close began, the close still lands exactly once before the - # cancellation propagates (HEL-894) — the provider clients - # and the store below stay safe in either order. - await app.state.semantic.aclose() - for provider in set(router.provider_for(t) for t in router.available_tiers()): - await provider.aclose() + app.state.semantic.recover() + if app.state.life.reach_out: + app.state.life.reach_out.recover() + # HEL-879: bounded restart compensation for this instance's own + # voice rooms (persisted cleanup ledger; own rooms only). if getattr(app.state, "voice_service", None): - await app.state.voice_service.aclose() - store.close() + await app.state.voice_service.recover() + life_task = asyncio.create_task(app.state.life.run()) + try: + yield + finally: + app.state.life.stop.set() + if life_task: + life_task.cancel() + with suppress(asyncio.CancelledError): + await life_task + if app.state.active: + app.state.active["cancel"].set() + await app.state.active["finished"].wait() + if app.state.semantic: + # HEL-889/894 shutdown ownership: aclose returns only after + # every live batch worker has actually exited (drain_timeout + # bounds the polite phase, never a licence to close under a + # worker). If this await is cancelled after the resource + # close began, the close still lands exactly once before the + # cancellation propagates (HEL-894) — the provider clients + # and the store below stay safe in either order. + await app.state.semantic.aclose() + for provider in set(router.provider_for(t) for t in router.available_tiers()): + await provider.aclose() + if getattr(app.state, "voice_service", None): + await app.state.voice_service.aclose() + store.close() + finally: + app.state.lifespan_active = False app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None) @@ -288,8 +305,15 @@ def create_app( voice_config = load_voice_config() except Exception: voice_config = None + # Cleanup ledger beside the instance database: rooms this instance + # opened stay owned (persisted) until their bounded delete lands — + # a restarted process compensates its OWN rooms only (HEL-879). voice_service = ( - VoiceSessionService(voice_config) if voice_config is not None else None + VoiceSessionService( + voice_config, + state_path=None if str(db_path) == ":memory:" else Path(db_path).with_name( + Path(db_path).stem + "-voice-sessions.json"), + ) if voice_config is not None else None ) app.state.voice_service = voice_service app.include_router(build_voice_router(voice_service)) diff --git a/src/cybergril/voice/livekit_wire.py b/src/cybergril/voice/livekit_wire.py index bd65eaf..97cbeca 100644 --- a/src/cybergril/voice/livekit_wire.py +++ b/src/cybergril/voice/livekit_wire.py @@ -149,6 +149,21 @@ async def create_dispatch( ) +class RoomDeleteError(RuntimeError): + """LiveKit refused a room delete with a status other than 2xx/404. + + Raised (not returned) so callers cannot mistake a failed delete for a + settled one — the caller must keep the room owned for a bounded retry + (HEL-879). A 404 answer means the room is already gone and is reported + as success instead: the delete's goal (room no longer exists) holds. + """ + + def __init__(self, status_code: int, room: str) -> None: + super().__init__(f"delete room {room!r} failed with HTTP {status_code}") + self.status_code = status_code + self.room = room + + async def delete_room( http: httpx.AsyncClient, *, @@ -162,6 +177,10 @@ async def delete_room( Deleting the room disconnects every participant, ends the dispatched agent job and releases the browser microphone server-side. + + Returns True when the room is gone: 2xx, or 404 (already deleted — the + retry after a lost response stays idempotent). Any other status raises + RoomDeleteError so ownership is retained until the delete really lands. """ response = await http.post( f"{internal_url.rstrip('/')}{TWIRP_PREFIX}/{ROOM_SERVICE}/DeleteRoom", @@ -172,8 +191,10 @@ async def delete_room( }, timeout=timeout, ) - # LiveKit answers DeleteRoom with an empty protobuf body; treat 2xx as done. - return response.status_code // 100 == 2 + # LiveKit answers DeleteRoom with an empty protobuf body; 404 = already gone. + if response.status_code // 100 == 2 or response.status_code == 404: + return True + raise RoomDeleteError(response.status_code, room) def decode_token(token: str) -> tuple[dict[str, Any], bytes]: diff --git a/src/cybergril/voice/receipt.py b/src/cybergril/voice/receipt.py index c2bb857..ea37ea6 100644 --- a/src/cybergril/voice/receipt.py +++ b/src/cybergril/voice/receipt.py @@ -7,9 +7,13 @@ ended in chat.error (the φ line is spoken verbatim but is not her archived reply), missing receipt, or a receipt from an epoch the server has since left (the server fail-closes those anyway — this is the client-side half). -Speech-handle-to-stream binding lives in TurnLedger: AgentSession emits -speech_created before the LLM stream starts and runs one user-turn generation -at a time, so the first stream created at/after a handle belongs to it. +Speech-handle-to-stream binding lives in TurnLedger as an EXPLICIT +association, not a time-window guess (HEL-879 总管复核第 3 项): AgentSession +runs one user-turn generation at a time, so the LLM stream a generation starts +is bound to the oldest handle still awaiting its generation; a handle whose +generation never starts (cancelled/interrupted before the LLM call) is retired +the moment a newer speech appears — it can never consume a later turn's +stream, and a failed/interrupted handle only ever releases its own binding. """ from __future__ import annotations @@ -48,31 +52,46 @@ class _HasHandle(Protocol): class TurnLedger: - """Binds livekit SpeechHandles to the ChatApiLLMStream of their generation.""" + """Binds livekit SpeechHandles to the ChatApiLLMStream of their generation. + + The binding is exact, never inferred from ordering timestamps: a stream is + bound to the oldest handle that has not yet been retired (its generation is + the one the SDK is starting), and each stream is consumed exactly once by + exactly its own handle. + """ def __init__(self) -> None: - self._handles: deque[tuple[float, Any]] = deque() # (created_at, handle) - self._streams: deque[tuple[float, Any]] = deque() # (created_at, stream) + self._pending: deque[Any] = deque() # handles awaiting their generation + self._bound: dict[int, Any] = {} # id(handle) -> its stream + self._retired: set[int] = set() # ended without a generation - def note_handle(self, handle: _HasHandle, created_at: float) -> None: - self._handles.append((created_at, handle)) + def note_handle(self, handle: _HasHandle, created_at: float | None = None) -> None: + """A new user speech started: every older handle still awaiting its + generation will never get one (the SDK moves on to the new turn), so it + is retired instead of staying eligible to steal the next stream.""" + while self._pending: + self._retired.add(id(self._pending.popleft())) + self._pending.append(handle) - def note_stream(self, stream: Any, created_at: float) -> None: - self._streams.append((created_at, stream)) + def note_stream(self, stream: Any, created_at: float | None = None) -> None: + """The generation the SDK just started belongs to the oldest handle + still awaiting one. A stream with no pending handle (proactive speech + outside a user turn) is left unbound: no user turn may claim it.""" + if self._pending: + handle = self._pending.popleft() + self._bound[id(handle)] = stream def stream_for(self, handle: _HasHandle) -> Any | None: - """First not-yet-consumed stream created at/after the handle.""" - while self._handles and self._handles[0][1] is not handle: - self._handles.popleft() # older handles ended without a generation - if not self._handles: - return None - # (handle present below) - created = self._handles.popleft()[0] - while self._streams: - at, stream = self._streams.popleft() - if at >= created: - return stream - return None + """This handle's own stream, exactly once; None when its generation + never started or it was retired/interrupted without one.""" + return self._bound.pop(id(handle), None) + + def discard(self, handle: _HasHandle) -> None: + """Release a handle's binding without using it (fail-closed path for + interrupted/failed playout): the turn never acks, and the binding does + not leak into any later turn.""" + self._bound.pop(id(handle), None) + self._retired.discard(id(handle)) class ReceiptSender: diff --git a/src/cybergril/voice/router.py b/src/cybergril/voice/router.py index 98ded3a..bc401c7 100644 --- a/src/cybergril/voice/router.py +++ b/src/cybergril/voice/router.py @@ -8,6 +8,10 @@ POST /api/voice/session open a room, dispatch the dedicated worker, DELETE /api/voice/session/{id} end: delete the LiveKit room, which disconnects browser and worker, ends the agent job and releases the microphone. + A delete that cannot land keeps the room + in a bounded, persisted cleanup ledger + (503, retried); it never collapses into + a 404 while the room is still owned. GET /api/voice/status honest switch state for the frontend (configured / tts reachable / agent name). @@ -70,7 +74,13 @@ def build_voice_router(service: VoiceSessionService | None) -> APIRouter: async def end_session(session_id: str) -> dict: if service is None: raise HTTPException(503, REFUSAL_NOT_CONFIGURED) - session = await service.end_session(session_id) + try: + session = await service.end_session(session_id) + except Exception: + # The remote delete did not land: the room stays owned in the + # bounded cleanup ledger and will be retried; the caller must NOT + # treat this as done (a 404 would erase the browser's owed DELETE). + raise HTTPException(503, "这次通话还没有完全挂断,会自动重试清理;请稍后再试。") from None if session is None: raise HTTPException(404, "这次通话已经不在了。") return {"ended": True, "room": session.room} diff --git a/src/cybergril/voice/session.py b/src/cybergril/voice/session.py index 00a346f..1b07eaf 100644 --- a/src/cybergril/voice/session.py +++ b/src/cybergril/voice/session.py @@ -5,14 +5,25 @@ silero VAD) exactly as HEL-876 audited it: this module only creates a room dispatch for the dedicated ``cybergril-voice-agent`` worker and mints a short-lived join token server-side, so no LiveKit key ever reaches the browser. -Everything here is process-local and stateless apart from the small session -registry: chat state, persona, memory and locks stay in the single API process -that mounts this router (HTTP and the HTTPS voice entry share it). +Everything here is process-local apart from the small session registry and its +persisted cleanup ledger (HEL-879): chat state, persona, memory and locks stay +in the single API process that mounts this router (HTTP and the HTTPS voice +entry share it). + +Room ownership stays until the delete really lands (HEL-879 总管复核第 2 项): +a failed remote delete keeps the session in a bounded cleanup ledger — retried +on the next end call and by a bounded background sweep — and the ledger is +persisted beside the instance database so a restarted process compensates its +OWN rooms (never another instance's, never the generic worker). Retries are +idempotent (404 = already gone = settled). Every debt is bounded by attempts +and age, then recorded as abandoned instead of retried forever. """ from __future__ import annotations import asyncio +import contextlib +import json import os import secrets import string @@ -29,6 +40,10 @@ DEFAULT_AGENT_NAME = "cybergril-voice-agent" DEFAULT_TOKEN_TTL = 300.0 DEFAULT_SESSION_TTL = 2 * 3600.0 TTS_HEALTH_CACHE_SECONDS = 30.0 +CLEANUP_MAX_ATTEMPTS = 8 +CLEANUP_BACKOFF_SECONDS = 0.5 +CLEANUP_GRACE_SECONDS = 24 * 3600.0 +STATE_VERSION = 1 @dataclass @@ -103,6 +118,50 @@ class VoiceSession: epoch: int dispatch_id: str = "" + def as_state(self) -> dict: + return { + "session_id": self.session_id, "room": self.room, "identity": self.identity, + "created_at": self.created_at, "expires_at": self.expires_at, + "epoch": self.epoch, "dispatch_id": self.dispatch_id, + } + + @classmethod + def from_state(cls, data: dict) -> "VoiceSession": + return cls( + session_id=str(data["session_id"]), room=str(data["room"]), + identity=str(data.get("identity", "")), created_at=float(data.get("created_at", 0)), + expires_at=float(data.get("expires_at", 0)), epoch=int(data.get("epoch", 0)), + dispatch_id=str(data.get("dispatch_id", "")), + ) + + +@dataclass +class _CleanupDebt: + """One room this instance still owns: its remote delete has not landed.""" + + session: VoiceSession + attempts: int = 0 + first_debt_at: float = 0.0 + last_error: str = "" + abandoned: str = "" + eligible_at: float = 0.0 # background retries wait until this clock time + + def as_state(self) -> dict: + return {"session": self.session.as_state(), "attempts": self.attempts, + "first_debt_at": self.first_debt_at, "last_error": self.last_error, + "abandoned": self.abandoned, "eligible_at": self.eligible_at} + + @classmethod + def from_state(cls, data: dict) -> "_CleanupDebt": + return cls( + session=VoiceSession.from_state(data.get("session", {})), + attempts=int(data.get("attempts", 0)), + first_debt_at=float(data.get("first_debt_at", 0)), + last_error=str(data.get("last_error", "")), + abandoned=str(data.get("abandoned", "")), + eligible_at=float(data.get("eligible_at", 0)), + ) + @dataclass class _Health: @@ -116,11 +175,73 @@ class VoiceSessionService: http: httpx.AsyncClient = field(default_factory=httpx.AsyncClient) clock: object = time.time tts_health_cache_seconds: float = TTS_HEALTH_CACHE_SECONDS + state_path: Path | str | None = None + cleanup_max_attempts: int = CLEANUP_MAX_ATTEMPTS + cleanup_backoff_seconds: float = CLEANUP_BACKOFF_SECONDS + cleanup_grace_seconds: float = CLEANUP_GRACE_SECONDS def __post_init__(self) -> None: self._sessions: dict[str, VoiceSession] = {} + self._debts: dict[str, _CleanupDebt] = {} self._tts_health = _Health(checked=-1e9, ok=False) self._tts_lock = asyncio.Lock() + self._cleaner: asyncio.Task | None = None + self._load_state() + + # -- persisted cleanup ledger (own rooms only) --------------------------- + def _load_state(self) -> None: + """Restart compensation: rooms recorded by a previous process of THIS + instance become cleanup debts (bounded by the persisted attempt counts + and age); they are the only rooms ever touched.""" + if not self.state_path: + return + try: + data = json.loads(Path(self.state_path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return + if data.get("version") != STATE_VERSION: + return + now = self.clock() + for entry in data.get("debts", []): + try: + debt = _CleanupDebt.from_state(entry) + except (KeyError, TypeError, ValueError): + continue + if debt.session.session_id not in self._debts: + self._debts[debt.session.session_id] = debt + for entry in data.get("sessions", []): + try: + session = VoiceSession.from_state(entry) + except (KeyError, TypeError, ValueError): + continue + # A room that was live when the process went down: the call may + # legitimately survive an API restart (the room lives on the SFU), + # so background compensation waits for the session's own TTL — + # bounded, never leaked, and never a longer-lived claim. + if session.session_id not in self._debts and session.session_id not in self._sessions: + self._debts[session.session_id] = _CleanupDebt( + session=session, first_debt_at=now, + eligible_at=max(now, session.expires_at)) + self._prune() + + def _persist_state(self) -> None: + if not self.state_path: + return + payload = { + "version": STATE_VERSION, + "sessions": [s.as_state() for s in self._sessions.values()], + "debts": [d.as_state() for d in self._debts.values()], + } + path = Path(self.state_path) + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8") + tmp.replace(path) + except OSError: + # Best effort: losing the ledger never breaks a request, it only + # costs the restart compensation for still-owned rooms. + pass # -- honest readiness --------------------------------------------------- async def tts_available(self) -> bool: @@ -141,12 +262,94 @@ class VoiceSessionService: # -- lifecycle ---------------------------------------------------------- def _prune(self) -> None: + """Expired registrations move into the cleanup ledger — never silently + dropped: their room stays owned until its bounded delete lands.""" now = self.clock() - for session_id in [k for k, v in self._sessions.items() if v.expires_at < now]: - del self._sessions[session_id] + expired = [k for k, v in self._sessions.items() if v.expires_at < now] + for session_id in expired: + session = self._sessions.pop(session_id) + if session_id not in self._debts: + self._debts[session_id] = _CleanupDebt(session=session, first_debt_at=now) + if expired: + self._persist_state() + + def _abandon_reason(self, debt: _CleanupDebt) -> str: + if debt.abandoned: + return debt.abandoned + if debt.attempts >= self.cleanup_max_attempts: + return "attempts-exhausted" + if self.clock() - debt.session.expires_at > self.cleanup_grace_seconds: + return "aged-out" + return "" + + def _retryable_debts(self) -> list[str]: + now = self.clock() + return [sid for sid, debt in self._debts.items() + if not self._abandon_reason(debt) and now >= debt.eligible_at] + + async def _delete_room_once(self, session: VoiceSession) -> None: + """One remote delete attempt; 2xx/404 settle, anything else raises.""" + await livekit_wire.delete_room( + self.http, + internal_url=self.config.livekit_internal, + api_key=self.config.api_key, + api_secret=self.config.api_secret, + room=session.room, + timeout=self.config.dispatch_timeout_seconds, + ) + + async def _cleanup_loop(self) -> None: + try: + while True: + pending = self._retryable_debts() + if not pending: + waiting = [d for d in self._debts.values() + if not self._abandon_reason(d)] + if not waiting: + break + # Deferred debts (restart compensation waiting out a live + # session's TTL): re-check when the earliest becomes + # eligible, capped so the loop stays cheap and bounded. + now = self.clock() + delay = min(max(d.eligible_at - now for d in waiting), 60.0) + await asyncio.sleep(max(delay, 0.05) if delay > 0 else 0.05) + continue + for session_id in pending: + debt = self._debts.get(session_id) + if debt is None or self._abandon_reason(debt): + continue + try: + await self._delete_room_once(debt.session) + self._debts.pop(session_id, None) + except Exception as exc: # bounded retry, never a crash loop + debt.attempts += 1 + debt.last_error = f"{type(exc).__name__}: {exc}"[:200] + reason = self._abandon_reason(debt) + if reason: + debt.abandoned = reason + self._persist_state() + if not debt.abandoned and session_id in self._debts: + await asyncio.sleep(self.cleanup_backoff_seconds) + finally: + self._cleaner = None + + def _kick_cleanup(self) -> None: + if self._cleaner is not None and not self._cleaner.done(): + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + self._cleaner = loop.create_task(self._cleanup_loop()) + + async def recover(self) -> None: + """App-start hook: compensate rooms left over by a previous process of + this instance (bounded by the persisted ledger).""" + self._kick_cleanup() async def open_session(self, *, epoch: int) -> VoiceSession: self._prune() + self._kick_cleanup() alphabet = string.ascii_lowercase + string.digits room = "cg-voice-" + "".join(secrets.choice(alphabet) for _ in range(10)) identity = "user-" + "".join(secrets.choice(string.ascii_lowercase) for _ in range(8)) @@ -170,6 +373,7 @@ class VoiceSessionService: dispatch_id=dispatch.dispatch_id, ) self._sessions[session.session_id] = session + self._persist_state() return session def token_for(self, session: VoiceSession) -> str: @@ -187,23 +391,72 @@ class VoiceSessionService: ) async def end_session(self, session_id: str) -> VoiceSession | None: + """End a session by actually deleting its room (HEL-879). + + The registration is only removed AFTER the remote delete settles: + a failed delete keeps the session in the bounded cleanup ledger, so a + retry (this method again, the background sweep, or the next process + via the persisted ledger) still targets the same room — it can never + degrade into a 404 that silently drops the room. Unknown ids (never + seen here, or fully settled and forgotten) still return None → 404. + """ self._prune() - session = self._sessions.pop(session_id, None) + session = self._sessions.get(session_id) if session is None: - return None - await livekit_wire.delete_room( - self.http, - internal_url=self.config.livekit_internal, - api_key=self.config.api_key, - api_secret=self.config.api_secret, - room=session.room, - timeout=self.config.dispatch_timeout_seconds, - ) + debt = self._debts.get(session_id) + if debt is None or debt.abandoned: + return None + session = debt.session + try: + await self._delete_room_once(session) + except Exception as exc: + debt = self._debts.get(session_id) + if debt is None: + debt = _CleanupDebt(session=session, first_debt_at=self.clock()) + self._debts[session_id] = debt + debt.attempts += 1 + debt.last_error = f"{type(exc).__name__}: {exc}"[:200] + reason = self._abandon_reason(debt) + if reason: + debt.abandoned = reason + self._persist_state() + self._kick_cleanup() + raise + self._sessions.pop(session_id, None) + self._debts.pop(session_id, None) + self._persist_state() return session def get(self, session_id: str) -> VoiceSession | None: self._prune() return self._sessions.get(session_id) + def cleanup_debt(self, session_id: str) -> _CleanupDebt | None: + """Test/diagnostics view of the owned-but-not-deleted ledger.""" + return self._debts.get(session_id) + async def aclose(self) -> None: + if self._cleaner is not None: + self._cleaner.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._cleaner + self._cleaner = None + # Graceful shutdown: rooms still active here become debts, then one + # final bounded sweep runs before the transport closes; whatever + # remains stays in the persisted ledger for the next process. + self._prune() + for session_id in self._retryable_debts(): + debt = self._debts.get(session_id) + if debt is None or self._abandon_reason(debt): + continue + try: + await self._delete_room_once(debt.session) + self._debts.pop(session_id, None) + except Exception as exc: + debt.attempts += 1 + debt.last_error = f"{type(exc).__name__}: {exc}"[:200] + reason = self._abandon_reason(debt) + if reason: + debt.abandoned = reason + self._persist_state() await self.http.aclose() diff --git a/src/cybergril/voice/worker.py b/src/cybergril/voice/worker.py index aaa29fe..c3c736b 100644 --- a/src/cybergril/voice/worker.py +++ b/src/cybergril/voice/worker.py @@ -119,10 +119,15 @@ async def entrypoint(ctx: JobContext) -> None: async def _after_playout(handle) -> None: # SDK 真实播放完成(wait_for_playout 返回)之后才考虑回执。 await handle.wait_for_playout() + # HEL-879:先判定失败/打断并整单放弃(fail-closed)——被放弃的绑定 + # 只属于这一轮,绝不会吞掉下一轮的回执归属。 + failed = handle.exception() is not None + if handle.interrupted or failed: + ledger.discard(handle) + return stream = ledger.stream_for(handle) if stream is None: return - failed = handle.exception() is not None if stream.error_phi: subtitle({"kind": "error", "role": "assistant", "text": stream.error_phi}) return diff --git a/tests/test_voice_chat_adapter.py b/tests/test_voice_chat_adapter.py index b8f485e..1252834 100644 --- a/tests/test_voice_chat_adapter.py +++ b/tests/test_voice_chat_adapter.py @@ -302,13 +302,23 @@ async def test_receipt_sender_confirms_once_and_reports_server_refusal(): @pytest.fixture -def live_server(api): +def live_server(api, tmp_path): app, provider, _ = api + # HEL-879: an app's lifespan has exactly one owner — the api fixture's + # TestClient still holds its lifespan, so the live server gets its OWN app + # instance on the same file-backed database (companion row included) and + # the same provider; the old fixture entered one app's lifespan twice + # concurrently, the exact defect the app now refuses loudly. + router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}, + budget=engineering_budget()) + served = create_app(db_path=tmp_path / "adapter.db", router=router, + life_config=LifeConfig(enabled=True, poll_seconds=3600), + semantic_mode="stub") probe = socket.socket() probe.bind(("127.0.0.1", 0)) port = probe.getsockname()[1] probe.close() # free the port for uvicorn to bind - config = uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error") + config = uvicorn.Config(served, host="127.0.0.1", port=port, log_level="error") server = uvicorn.Server(config) thread = threading.Thread(target=server.run, daemon=True) thread.start() @@ -318,7 +328,7 @@ def live_server(api): break except Exception: time.sleep(0.1) - yield f"http://127.0.0.1:{port}", app, provider + yield f"http://127.0.0.1:{port}", served, provider server.should_exit = True thread.join(timeout=5) diff --git a/tests/test_voice_rework_hel879.py b/tests/test_voice_rework_hel879.py new file mode 100644 index 0000000..7160b43 --- /dev/null +++ b/tests/test_voice_rework_hel879.py @@ -0,0 +1,497 @@ +"""HEL-879 rework regressions for the 总管's independent-review edges. + +Four confirmed offline defects, each reproduced first and fixed: +1. dual uvicorn listeners both running the app lifespan (epoch 1→2, state + replaced, store closed under live listeners); +2. end_session popping the registry before the remote delete (lost retry + target; expiry pruning dropping rooms with zero DELETE attempts); +3. TurnLedger binding streams to handles by timestamp (a handle without a + generation steals the next turn's stream/receipt); +4. release-snapshot verify accepting self-contradictory success markers and + same-basename DBs overwriting each other (covered by the tool selftest). + +Zero network beyond 127.0.0.1; fake LiveKit/TTS transports throughout. +""" + +from __future__ import annotations + +import asyncio +import json +import socket +import ssl +import subprocess +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) + +from test_r2 import R2LLM # noqa: E402 + +from cybergril.api.app import create_app # noqa: E402 +from cybergril.life.runtime import LifeConfig # noqa: E402 +from cybergril.memory.store import Store # noqa: E402 +from cybergril.think.router import build_router # noqa: E402 +from cybergril.think.scenes import Tier # noqa: E402 +from cybergril.voice import livekit_wire # noqa: E402 +from cybergril.voice.receipt import TurnLedger # noqa: E402 +from cybergril.voice.session import VoiceConfig, VoiceSessionService # noqa: E402 + + +# ---------------------------------------------------------------- TurnLedger + +def _stream(turn_id: str) -> SimpleNamespace: + return SimpleNamespace(turn_id=turn_id, receipt={"message_id": hash(turn_id) % 1000}) + + +def test_ledger_binds_each_handle_to_its_own_stream(): + ledger = TurnLedger() + h1, h2 = object(), object() + s1, s2 = _stream("old"), _stream("new") + ledger.note_handle(h1, 1.0) + ledger.note_stream(s1, 1.1) + ledger.note_handle(h2, 2.0) + ledger.note_stream(s2, 2.1) + # A late cleanup of H2 must leave H1's own stream to H1 (old code: None). + assert ledger.stream_for(h2) is s2 + assert ledger.stream_for(h1) is s1 + + +def test_ledger_handle_without_generation_consumes_nothing(): + ledger = TurnLedger() + no_generation, real = object(), object() + s2 = _stream("new") + ledger.note_handle(no_generation, 10.0) + ledger.note_handle(real, 11.0) # retires the older, generation-less handle + ledger.note_stream(s2, 11.1) + assert ledger.stream_for(no_generation) is None + assert ledger.stream_for(real) is s2 + + +def test_ledger_interrupted_handle_discards_only_its_own_binding(): + ledger = TurnLedger() + h1, h2 = object(), object() + s1, s2 = _stream("one"), _stream("two") + ledger.note_handle(h1, 1.0) + ledger.note_stream(s1, 1.0) + ledger.note_handle(h2, 2.0) + ledger.note_stream(s2, 2.0) + ledger.discard(h1) # worker fail-closed path (interrupted/failed playout) + assert ledger.stream_for(h1) is None + assert ledger.stream_for(h2) is s2 + + +def test_ledger_stream_without_pending_handle_stays_unclaimed(): + ledger = TurnLedger() + stray = _stream("proactive") + ledger.note_stream(stray, 1.0) # no user turn awaited a generation + handle = object() + ledger.note_handle(handle, 2.0) + later = _stream("answer") + ledger.note_stream(later, 2.5) + assert ledger.stream_for(handle) is later + + +# --------------------------------------------------- session ownership ledger + +def _proto_field1(data: bytes) -> str: + """Decode varint-prefixed field 1 (room) from a livekit proto body.""" + if not data or (data[0] >> 3) != 1 or (data[0] & 7) != 2: + return "" + length, index, shift, value = 0, 1, 0, 0 + while True: + byte = data[index] + length |= (byte & 0x7F) << shift + index += 1 + if not byte & 0x80: + break + shift += 7 + return data[index:index + length].decode() + + +class StandinLiveKit: + """Fake Twirp transport: configurable delete failures per call.""" + + def __init__(self) -> None: + self.deleted: list[str] = [] + self.delete_errors: list[Exception | int] = [] + self._transport = httpx.MockTransport(self._handle) + + def _handle(self, request: httpx.Request) -> httpx.Response: + path = request.url.path + if path.endswith("/CreateDispatch"): + return httpx.Response(200, json={"id": "AD_1", "room": "r"}) + if path.endswith("/DeleteRoom"): + if self.delete_errors: + error = self.delete_errors.pop(0) + if isinstance(error, int): + return httpx.Response(error, text="room service error") + raise error + self.deleted.append(_proto_field1(request.content)) + return httpx.Response(200, content=b"") + return httpx.Response(404, text="unknown") + + @property + def transport(self) -> httpx.MockTransport: + return self._transport + + +def _service(standin: StandinLiveKit, *, clock=None, backoff=0.0, **kwargs) -> VoiceSessionService: + config = VoiceConfig( + livekit_url="ws://voice.test:7880", + livekit_internal="http://livekit.test", + api_key="devkey", + api_secret="test-secret-not-real", + tts_health_url="http://tts.test/health", + session_ttl_seconds=kwargs.pop("session_ttl_seconds", 5.0), + ) + return VoiceSessionService( + config=config, + http=httpx.AsyncClient(transport=standin.transport), + clock=clock or time.time, + cleanup_backoff_seconds=backoff, + **kwargs, + ) + + +async def _open(service: VoiceSessionService) -> str: + return (await service.open_session(epoch=1)).session_id + + +@pytest.mark.asyncio +async def test_delete_failure_keeps_room_owned_until_retry_lands(): + standin = StandinLiveKit() + standin.delete_errors = [RuntimeError("standin delete failed")] + service = _service(standin) + try: + session = await service.open_session(epoch=1) + with pytest.raises(RuntimeError): + await service.end_session(session.session_id) + # The room is retained for retry (old code: registry already popped, + # second call answered None and never deleted the room again). + debt = service.cleanup_debt(session.session_id) + assert debt is not None and debt.session.room == session.room + assert standin.deleted == [] + ended = await service.end_session(session.session_id) + assert ended.room == session.room + assert standin.deleted == [session.room] + assert service.cleanup_debt(session.session_id) is None + assert await service.end_session(session.session_id) is None + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_expired_session_room_is_actually_deleted(): + standin = StandinLiveKit() + now = [1000.0] + service = _service(standin, clock=lambda: now[0]) + try: + session = await service.open_session(epoch=1) + now[0] += service.config.session_ttl_seconds + 1 + # Old code: _prune dropped the registration silently — zero DELETEs. + ended = await service.end_session(session.session_id) + assert ended is not None and ended.room == session.room + assert standin.deleted == [session.room] + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_delete_already_gone_is_idempotent_success(): + standin = StandinLiveKit() + standin.delete_errors = [404] # room vanished server-side first + service = _service(standin) + try: + session = await service.open_session(epoch=1) + ended = await service.end_session(session.session_id) + assert ended is not None + assert service.cleanup_debt(session.session_id) is None + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_background_cleanup_settles_expiry_debt(): + standin = StandinLiveKit() + standin.delete_errors = [RuntimeError("flap"), RuntimeError("flap")] + now = [1000.0] + service = _service(standin, clock=lambda: now[0]) + try: + session = await service.open_session(epoch=1) + now[0] += service.config.session_ttl_seconds + 1 + with pytest.raises(RuntimeError): + await service.end_session(session.session_id) + now[0] += 1 + for _ in range(50): + if standin.deleted: + break + await asyncio.sleep(0.02) + assert standin.deleted == [session.room] + for _ in range(50): + if service.cleanup_debt(session.session_id) is None: + break + await asyncio.sleep(0.02) + assert service.cleanup_debt(session.session_id) is None + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_cleanup_is_bounded_then_recorded_abandoned(tmp_path): + standin = StandinLiveKit() + standin.delete_errors = [503] * 50 # never succeeds + now = [1000.0] + service = _service(standin, clock=lambda: now[0], + state_path=tmp_path / "voice-sessions.json", + cleanup_max_attempts=3) + try: + session = await service.open_session(epoch=1) + now[0] += service.config.session_ttl_seconds + 1 + with pytest.raises(Exception): + await service.end_session(session.session_id) + for _ in range(200): + debt = service.cleanup_debt(session.session_id) + if debt is not None and debt.abandoned: + break + await asyncio.sleep(0.02) + debt = service.cleanup_debt(session.session_id) + assert debt is not None and debt.abandoned == "attempts-exhausted" + # Bounded: after abandonment no further delete attempts happen. + attempts = len(standin.delete_errors) + await asyncio.sleep(0.2) + assert len(standin.delete_errors) == attempts + recorded = json.loads((tmp_path / "voice-sessions.json").read_text()) + assert recorded["debts"][0]["abandoned"] == "attempts-exhausted" + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_restart_compensation_deletes_previous_process_rooms(tmp_path): + standin = StandinLiveKit() + standin.delete_errors = [RuntimeError("down")] * 2 # end attempt + shutdown sweep + state = tmp_path / "voice-sessions.json" + now = [1000.0] + # big backoff: the background sweeper must not race this scenario + first = _service(standin, clock=lambda: now[0], state_path=state, backoff=3600.0) + session = await first.open_session(epoch=1) + with pytest.raises(RuntimeError): + await first.end_session(session.session_id) + await first.aclose() + assert standin.deleted == [] # LiveKit was down for process 1's whole life + persisted = json.loads(state.read_text()) + assert any(d["session"]["room"] == session.room for d in persisted["debts"]) + # "Restart": a new process of the same instance reads the same ledger and + # compensates only the rooms recorded there. + second = _service(standin, clock=lambda: now[0], state_path=state) + try: + await second.recover() + for _ in range(100): + if standin.deleted: + break + await asyncio.sleep(0.02) + assert standin.deleted == [session.room] + for _ in range(100): + settled = json.loads(state.read_text()) + if settled["debts"] == [] and settled["sessions"] == []: + break + await asyncio.sleep(0.02) + settled = json.loads(state.read_text()) + assert settled["debts"] == [] and settled["sessions"] == [] + finally: + await second.aclose() + + +@pytest.mark.asyncio +async def test_active_room_surives_restart_until_ttl_then_compensated(tmp_path): + standin = StandinLiveKit() + state = tmp_path / "voice-sessions.json" + now = [1000.0] + first = _service(standin, clock=lambda: now[0], state_path=state, + session_ttl_seconds=0.5) + session = await first.open_session(epoch=1) # never ended by the user + await first.aclose() # graceful shutdown must not hang up a live call + assert standin.deleted == [] + persisted = json.loads(state.read_text()) + assert any(s["room"] == session.room for s in persisted["sessions"]) + # Restart immediately: the call may still be alive on the SFU, so the + # compensation waits for the session's own TTL — then deletes, bounded. + second = _service(standin, clock=lambda: now[0], state_path=state, + session_ttl_seconds=0.5) + try: + await second.recover() + await asyncio.sleep(0.15) + assert standin.deleted == [] # still within TTL: untouched + now[0] += 1.0 + for _ in range(100): + if standin.deleted: + break + await asyncio.sleep(0.05) + assert standin.deleted == [session.room] + finally: + await second.aclose() + + +# ------------------------------------------------------ router-level semantics + +def _voice_app(standin: StandinLiveKit, tts_ok=True): + provider = R2LLM() + router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}) + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.host == "tts.test": + return httpx.Response(200 if tts_ok else 503) + return standin._handle(request) + + service = _service(standin, backoff=3600.0) # keep the sweep out of the race + service.http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + app = create_app(db_path=":memory:", router=router, + life_config=LifeConfig(enabled=False), semantic_mode="stub", + voice_service=service) + return app, service + + +def test_router_delete_failure_is_503_and_keeps_room_owned(): + from fastapi.testclient import TestClient + + standin = StandinLiveKit() + # interactive attempt + the background sweep's immediate first retry; + # the sweep then backs off (3600 s), leaving the debt visibly owned + standin.delete_errors = [RuntimeError("livekit down"), RuntimeError("livekit down")] + app, service = _voice_app(standin) + with TestClient(app) as client: + client.post("/api/companion", json={ + "name": "语音重试", "birth": "2000-10-18", "hour": "亥时", "relation": "朋友"}) + opened = client.post("/api/voice/session") + assert opened.status_code == 201 + session_id = opened.json()["session_id"] + failed = client.delete(f"/api/voice/session/{session_id}") + assert failed.status_code == 503 # NOT 404: the room is still owned + assert service.cleanup_debt(session_id) is not None + retried = client.delete(f"/api/voice/session/{session_id}") + assert retried.status_code == 200 and retried.json()["ended"] is True + assert standin.deleted == [opened.json()["room"]] + again = client.delete(f"/api/voice/session/{session_id}") + assert again.status_code == 404 + + +# ---------------------------------------------------- dual-listener lifecycle + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +@pytest.fixture +def self_signed_cert(tmp_path): + cert = tmp_path / "cert.pem" + key = tmp_path / "key.pem" + subprocess.run( + ["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(key), "-out", str(cert), "-days", "1", + "-subj", "/CN=127.0.0.1", + "-addext", "subjectAltName=IP:127.0.0.1"], + check=True, capture_output=True) + return str(cert), str(key) + + +@pytest.mark.asyncio +async def test_dual_listeners_share_one_lifecycle_and_close_in_order( + tmp_path, self_signed_cert, monkeypatch): + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "deploy" / "voice")) + import serve_dual_8360_8361 as launcher + + cert, key = self_signed_cert + events: list[str] = [] + store_inits: list[int] = [] + real_store_init = Store.__init__ + real_store_close = Store.close + + def counting_init(self, *args, **kwargs): + store_inits.append(id(self)) + real_store_init(self, *args, **kwargs) + + def recording_close(self): + events.append("store_close") + real_store_close(self) + + monkeypatch.setattr(Store, "__init__", counting_init) + monkeypatch.setattr(Store, "close", recording_close) + + import uvicorn + real_shutdown = uvicorn.Server.shutdown + + async def recording_shutdown(self, sockets=None): + events.append(f"listener_shutdown:{self.config.port}") + await real_shutdown(self, sockets=sockets) + + monkeypatch.setattr(uvicorn.Server, "shutdown", recording_shutdown) + + provider = R2LLM() + router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}) + app = create_app(db_path=tmp_path / "dual.db", router=router, + life_config=LifeConfig(enabled=False), semantic_mode="off") + http_port, https_port = _free_port(), _free_port() + serve_task = asyncio.get_running_loop().create_task( + launcher.serve(http_port, https_port, cert, key, app=app)) + + client = httpx.AsyncClient(timeout=5.0, verify=False) + try: + epochs = [] + for _ in range(100): + try: + for port, scheme in ((http_port, "http"), (https_port, "https")): + response = await client.get(f"{scheme}://127.0.0.1:{port}/api/health") + if response.status_code == 200: + epochs.append(response.json()["epoch"]) + if len(epochs) >= 2: + break + except (httpx.ConnectError, httpx.ConnectTimeout, ssl.SSLError): + await asyncio.sleep(0.05) + assert len(epochs) >= 2, "both listeners must come up" + # ONE lifecycle for both listeners: a second entry would be epoch 2. + assert epochs[0] == epochs[1] + assert len(set(store_inits)) == 1, "the app must be initialized exactly once" + finally: + serve_task.cancel() + try: + await serve_task + except asyncio.CancelledError: + pass + await client.aclose() + + # Exit boundary: both listeners fully shut down BEFORE the store closes, + # and the store closes exactly once (never closed under live listeners). + shutdowns = [e for e in events if e.startswith("listener_shutdown")] + assert len(shutdowns) == 2, events + assert events.index("store_close") > max(events.index(e) for e in shutdowns) + assert events.count("store_close") == 1 + assert len(set(store_inits)) == 1 + + +def test_second_lifespan_entry_fails_loudly(tmp_path): + from fastapi.testclient import TestClient + + provider = R2LLM() + router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}) + app = create_app(db_path=tmp_path / "guard.db", router=router, + life_config=LifeConfig(enabled=False), semantic_mode="off") + + async def _double_enter(): + async with app.router.lifespan_context(app): + pass + + with TestClient(app) as client: + epoch_before = client.get("/api/health").json()["epoch"] + # A second entry (the old dual-uvicorn defect: epoch 1→2, Store/life/ + # chat_lock silently replaced) must now fail closed. + with pytest.raises(RuntimeError, match="entered twice"): + asyncio.run(_double_enter()) + assert client.get("/api/health").json()["epoch"] == epoch_before diff --git a/tools/release_snapshot_hel896.py b/tools/release_snapshot_hel896.py index 08faa99..2458124 100644 --- a/tools/release_snapshot_hel896.py +++ b/tools/release_snapshot_hel896.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 """HEL-879 · 发布归档一致性快照与回退证据(HEL-896 有限收口版;不冒充 R6 完整备份)。 -HEL-896 工具的总管独立复审发现四个失败边界,本版全部收口: +HEL-896 工具的总管独立复审发现四个失败边界,v2 全部收口;v3 收口总管对 v2 的 +独立复核(HEL-879 edges): 1. checkpoint 返回值必须核对:`PRAGMA wal_checkpoint(TRUNCATE)` 返回 (busy, log, checkpointed),busy≠0 即未完成——旧版照抄复制并在 manifest 记 rows_match=False - 仍打包成功。本版 busy≠0 / 源表行数≠快照行数一律**明确失败**:CLI 非 0、不产生 + 仍打包成功。busy≠0 / 源表行数≠快照行数一律**明确失败**:CLI 非 0、不产生 可交付归档(tar 不生成,半成品目录清除)。「先停服」是操作前提,不是忽略实际 失败结果的理由。 2. 目录内真实 SQLite 走一致性路径:history_root/vector_root 等数据目录里的 @@ -13,29 +14,35 @@ HEL-896 工具的总管独立复审发现四个失败边界,本版全部收口 (`semantic.py` 的 history_db_path 就在 history_root 内)。停服口径对它们逐个 checkpoint+复制+逐表核对;**在线口径只允许显式单库 --db(backup API),传入含 SQLite 的目录直接拒绝**——在线目录裸复制正是丢 WAL 的不安全口径。 -3. verify 拒绝 rows_match=False 的旧包,并按源路径映射做回退演练:解包到隔离新 - 目录、实际打开每个库核对行数与哈希(对照 manifest 记录的源行数与快照行数, - 两者必须相等且与解包实测一致),非只打开 tar 中快照自证;输出 rollback_plan - (源路径 → 解包恢复路径)。 -4. 完整实例归档只支持停服/释放全部写入方;在线模式仅显式单库 backup API。 +3. verify 拒绝 rows_match=False/缺失的旧包;三个计数(manifest 源行数、manifest + 快照行数、解包实测)必须直接相等——v2 的 `rows_match is True or ...` 允许记录 + 标记绕过源计数比较(源 2/快照 1 + rows_match=true 仍 ok,总管实测)。回退演练 + 按源路径映射**实际恢复**到归档旁的持久独立目标树并打开恢复库核行数与哈希, + 输出 rollback_plan(源路径 → 真实恢复路径)。 +4. 归档侧同名覆盖:不同目录的同名 `ledger.db` 在 v2 都写入 `databases/ledger.db`, + 后写覆盖前写、快照先交成功包再由 verify 发现。v3 对归档目标名做无冲突映射 + (同名源追加解析路径摘要),同一源重复传入直接拒绝。 + +完整实例归档只支持停服/释放全部写入方;在线模式仅显式单库 backup API。 产物:`//` 下的 `databases/`(一致性快照库,含目录内发现的 SQLite)、 `files/`(其余数据文件原样)、`MANIFEST.json`(逐文件 SHA256、源路径、代码精确 提交、逐表行数核对结果)、`.tar.gz`。归档永不删除;本工具不实现 R6。 -自测(`--selftest`,临时目录,零产品数据)复现:旧版失败形状(busy checkpoint 仍 -打包、rows_match=False 旧包 verify 仍 ok、在线目录含 SQLite)与本版修复,输出 -`docs/rework/hel879/release-snapshot/selftest.json`。 +自测(`--selftest`,临时目录,零产品数据)复现:全部旧失败形状(busy checkpoint +仍打包、rows_match=False 旧包 verify 仍 ok、在线目录含 SQLite、v2 的 OR 绕过与 +同名覆盖)与本版修复,输出 `docs/rework/hel879/release-snapshot/selftest.json`。 用法: python3 tools/release_snapshot_hel896.py --selftest [outDir] python3 tools/release_snapshot_hel896.py snapshot --name hel8xx-release \ --db path/a.db --db path/b.sqlite3 --file path/vecdir --file path/hist \ - --out archives/ # 停服口径(推荐,目录内 SQLite 一并一致性处理) + --out archives/ # 停服口径(推荐,目录内 SQLite 一并一致性处理) python3 tools/release_snapshot_hel896.py snapshot --name ... --db path/a.db \ - --out archives/ --online # 在线口径:仅显式单库 backup API,含 SQLite 的 - # --file 目录会被拒绝 - python3 tools/release_snapshot_hel896.py verify --archive archives/hel8xx-release.tar.gz + --out archives/ --online # 在线口径:仅显式单库 backup API,含 SQLite 的 + # --file 目录会被拒绝 + python3 tools/release_snapshot_hel896.py verify --archive archives/hel8xx-release.tar.gz \ + [--restore-dir DIR] # 回退演练默认恢复到归档旁 verify-restore--<时间>/ """ from __future__ import annotations @@ -53,7 +60,7 @@ from pathlib import Path REPO = Path(__file__).resolve().parents[1] SQLITE_MAGIC = b"SQLite format 3\x00" -TOOL_VERSION = 2 +TOOL_VERSION = 3 class SnapshotError(RuntimeError): @@ -132,26 +139,52 @@ def git_head() -> str: return "unknown" +def _unique_name(taken: set[str], display: str, key: str) -> str: + """Archive-side name that cannot collide: two different source DBs with + the same basename must never overwrite each other in `databases/` + (HEL-879 总管复核第 4 项). Same-basename sources get a short digest of + their resolved source path appended.""" + if display not in taken: + taken.add(display) + return display + digest = hashlib.sha256(key.encode()).hexdigest()[:8] + stem, suffix = Path(display).stem, Path(display).suffix + unique = f"{stem}__{digest}{suffix}" + while unique in taken: + unique = f"{stem}__{digest}{len(taken)}{suffix}" + taken.add(unique) + return unique + + def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, *, online: bool) -> dict: """做一致性快照。任一环节失败:不产生 tar、清除半成品、抛 SnapshotError。 收口语义(HEL-879): - 全部 --db 与目录内发现的 SQLite 都走一致性路径并逐表核对源/快照行数; - 行数不一致或 checkpoint busy = 明确失败(旧版记 rows_match=False 仍打包); - - 在线模式含 SQLite 的 --file 目录直接拒绝(在线目录裸复制丢 WAL)。 + - 在线模式含 SQLite 的 --file 目录直接拒绝(在线目录裸复制丢 WAL); + - 同一源重复传入或归档侧目标名冲突 = 明确失败/唯一化(同名 ledger.db + 后写覆盖前写、快照先报成功包再由 verify 发现——旧版真实缺陷)。 """ + resolved = [str(Path(db).resolve()) for db in dbs] + duplicates = sorted({p for p in resolved if resolved.count(p) > 1}) + if duplicates: + raise SnapshotError(f"--db 同一源重复传入,拒绝:{duplicates}") stamp = time.strftime("%Y%m%d-%H%M%S") root = out_dir / f"{name}-{stamp}" manifest: dict = {"name": name, "created": stamp, "code_commit": git_head(), "tool": "release_snapshot_hel896.py", "tool_version": TOOL_VERSION, "mode": "online-backup-api" if online else "stop-checkpoint-copy", "databases": [], "files": [], "verify": "not-run"} + taken_db_names: set[str] = set() + taken_file_names: set[str] = set() try: (root / "databases").mkdir(parents=True, exist_ok=True) for db in dbs: if not Path(db).is_file(): raise SnapshotError(f"--db 不是文件:{db}") - dst = root / "databases" / db.name + dst = root / "databases" / _unique_name( + taken_db_names, Path(db).name, str(Path(db).resolve())) method = snapshot_db(Path(db), dst, online=online) src_counts = table_counts(Path(db)) dst_counts = table_counts(dst) @@ -176,7 +209,9 @@ def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, for emb in embedded: # 停服口径:目录内真实 SQLite 一并走 checkpoint+复制+逐表核对。 rel = emb.relative_to(src) if emb.is_relative_to(src) else Path(emb.name) - dst = root / "databases" / f"{src.name}__{rel}".replace("/", "__") + display = f"{src.name}__{rel}".replace("/", "__") + dst = root / "databases" / _unique_name( + taken_db_names, display, str(emb.resolve())) method = snapshot_db(emb, dst, online=False) src_counts = table_counts(emb) dst_counts = table_counts(dst) @@ -189,7 +224,7 @@ def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, "source_table_counts": src_counts, "snapshot_table_counts": dst_counts, "rows_match": True}) - dst = root / "files" / src.name + dst = root / "files" / _unique_name(taken_file_names, src.name, str(src.resolve())) if src.is_dir(): shutil.copytree(src, dst) entries = {str(p.relative_to(dst)): sha256(p) for p in sorted(dst.rglob("*")) if p.is_file()} @@ -216,15 +251,29 @@ def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, raise -def verify_archive(archive: Path) -> dict: - """回退演练:隔离新目录解包,实际打开各库核对行数与哈希,按源路径映射。 +def _restore_target(restore_root: Path, source: str) -> Path: + """Map a recorded source path into the isolated restore tree, preserving + the full source hierarchy so same-basename DBs can never collide there.""" + sanitized = source.strip("/").replace("/", "__") + return restore_root / "by-source" / sanitized + + +def verify_archive(archive: Path, restore_root: Path | None = None) -> dict: + """回退演练:隔离新目录解包,按源路径映射**实际恢复**到独立目标树, + 打开恢复出的每个库核对行数与哈希(不是只打开 tar 内布局自证)。 收口语义(HEL-879): - - manifest 记录 rows_match=False(旧版失败包)→ ok=False 拒绝; - - 行数同时对照 source_table_counts 与 snapshot_table_counts:两者必须相等, - 且与解包实测一致(旧版只比快照自证,源=1 行/快照=0 行的漏行包仍 ok)。 + - 三个计数必须直接相等:manifest 源行数 = manifest 快照行数 = 解包实测; + `rows_match` 只是额外约束,缺失或矛盾明确拒绝(旧版 OR 逻辑允许记录值 + 绕过源计数比较); + - 恢复目标树默认落在归档旁的持久目录,rollback_plan 指向真实存在的恢复 + 文件(旧版输出随 TemporaryDirectory 消失的路径)。 """ - result: dict = {"archive": str(archive), "sha256": sha256(archive), "checks": []} + archive = Path(archive) + if restore_root is None: + restore_root = archive.parent / f"verify-restore-{archive.stem}-{time.strftime('%Y%m%d-%H%M%S')}" + result: dict = {"archive": str(archive), "sha256": sha256(archive), + "restore_root": str(restore_root), "checks": []} with tempfile.TemporaryDirectory(prefix="hel879-verify-") as tmp: with tarfile.open(archive) as tf: tf.extractall(tmp) @@ -237,39 +286,56 @@ def verify_archive(archive: Path) -> dict: rollback_plan = [] for d in manifest["databases"]: snap = root / d["snapshot"] - digest = sha256(snap) + # The actual restore step: copy into the isolated target tree at + # the recorded source path's mapped location, then verify THERE. + restored = _restore_target(restore_root, str(d.get("source") or d["snapshot"])) + restored.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(snap, restored) + digest = sha256(restored) try: - counts = table_counts(snap) + counts = table_counts(restored) except sqlite3.Error as exc: - result["checks"].append({"db": d["snapshot"], "sha256_ok": digest == d["sha256"], - "rows_ok": False, "error": str(exc), + result["checks"].append({"db": d["snapshot"], "restored": str(restored), + "sha256_ok": False, "rows_ok": False, "error": str(exc), "rows_match_recorded": d.get("rows_match")}) ok_all = False continue recorded_rows_match = d.get("rows_match") src_counts = d.get("source_table_counts") snap_counts = d.get("snapshot_table_counts") - rows_ok = counts == snap_counts and ( - recorded_rows_match is True or (src_counts is not None and counts == src_counts)) - legacy_reject = recorded_rows_match is False or ( - recorded_rows_match is None and src_counts != snap_counts) + # Three-way equality is the only passing shape; the recorded flag + # is an extra constraint, never a substitute for the comparison. + counts_agree = ( + isinstance(src_counts, dict) and isinstance(snap_counts, dict) + and counts == src_counts and counts == snap_counts + ) + rows_ok = counts_agree + legacy_reject = recorded_rows_match is not True or not counts_agree ok = digest == d["sha256"] and rows_ok and not legacy_reject ok_all &= ok - result["checks"].append({"db": d["snapshot"], "sha256_ok": digest == d["sha256"], + result["checks"].append({"db": d["snapshot"], "restored": str(restored), + "sha256_ok": digest == d["sha256"], "rows_ok": rows_ok, "counts": counts, "source_counts": src_counts, + "snapshot_counts": snap_counts, "rows_match_recorded": recorded_rows_match, "rejected_legacy_package": legacy_reject}) - rollback_plan.append({"source": d.get("source"), "restored_path": str(snap), - "restore_action": "停服后复制回源路径(行数与哈希已在本演练核对)"}) + rollback_plan.append({"source": d.get("source"), "restored_path": str(restored), + "restore_action": "停服后复制回源路径(行数与哈希已在恢复树实际核对)"}) for f in manifest["files"]: base = root / f["snapshot"] + restored_dir = _restore_target(restore_root, str(f.get("source") or f["snapshot"])) + restored_dir.parent.mkdir(parents=True, exist_ok=True) + if base.is_dir(): + shutil.copytree(base, restored_dir, dirs_exist_ok=True) + else: + shutil.copy2(base, restored_dir) for rel, want in f["entries"].items(): - p = (base / rel) if base.is_dir() else base + p = (restored_dir / rel) if restored_dir.is_dir() else restored_dir ok = sha256(p) == want ok_all &= ok - result["checks"].append({"file": str(p.relative_to(root)), "sha256_ok": ok}) - rollback_plan.append({"source": f.get("source"), "restored_path": str(base), + result["checks"].append({"file": rel, "restored": str(p), "sha256_ok": ok}) + rollback_plan.append({"source": f.get("source"), "restored_path": str(restored_dir), "restore_action": "停服后按相对结构复制回源目录"}) result["rollback_plan"] = rollback_plan result["ok"] = ok_all @@ -476,6 +542,67 @@ def selftest(out_dir: Path) -> int: checks.append({"name": "backup_api_snapshot_complete", "ok": api_counts.get("companion") == 1 and api_counts.get("chat_message") == 1, "detail": json.dumps(api_counts)}) + + # ── 总管对 v2 的 edges(HEL-879 独立复核)── + # Edge A:manifest 记 rows_match=true 但源计数被改为 events=2(快照=1)。 + # v2 的 `rows_match is True or ...` 绕过源计数比较 → 仍 ok;v3 必须拒绝。 + clean = make_snapshot("edge-counts", [db], [], out, online=False) + edge_unpack = out / "edge-unpack" + with tarfile.open(clean["archive"]) as tf: + tf.extractall(edge_unpack) + edge_pkg = next(edge_unpack.iterdir()) + edge_manifest_path = edge_pkg / "MANIFEST.json" + edge_manifest = json.loads(edge_manifest_path.read_text(encoding="utf-8")) + edge_manifest["databases"][0]["source_table_counts"]["events"] = 2 + edge_manifest_path.write_text(json.dumps(edge_manifest, ensure_ascii=False)) + edge_tar = tmp / "edge-counts-tampered.tar.gz" + with tarfile.open(edge_tar, "w:gz") as tf: + tf.add(edge_pkg, arcname=edge_pkg.name) + edge_verify = verify_archive(edge_tar, restore_root=tmp / "edge-restore") + checks.append({"name": "fix_rows_match_true_cannot_bypass_source_counts", + "ok": edge_verify["ok"] is False and all( + c.get("rejected_legacy_package") for c in edge_verify["checks"] + if "rejected_legacy_package" in c), + "detail": f"ok={edge_verify['ok']} " + f"counts={edge_verify['checks'][0].get('counts')} " + f"source={edge_verify['checks'][0].get('source_counts')}"}) + + # Edge B:两份不同目录的同名 ledger.db。v2 后写覆盖前写、快照仍交成功包; + # v3 无冲突映射 + verify 全过 + 恢复树里两个源路径各自落位。 + la = tmp / "ledger-a" / "ledger.db" + lb = tmp / "ledger-b" / "ledger.db" + for path, value in [(la, "first"), (lb, "second")]: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(str(path)) as conn: + conn.execute("CREATE TABLE item(value TEXT)") + conn.execute("INSERT INTO item VALUES(?)", (value,)) + same = make_snapshot("same-basename", [la, lb], [], out, online=False) + snap_names = [Path(d["snapshot"]).name for d in same["databases"]] + same_restore = tmp / "same-restore" + same_verify = verify_archive(Path(same["archive"]), restore_root=same_restore) + plan = same_verify.get("rollback_plan", []) + restored_paths = [e["restored_path"] for e in plan if e.get("source", "").endswith("ledger.db")] + distinct = len(set(restored_paths)) == 2 and all( + Path(p).is_file() for p in restored_paths) + rows = sorted(sqlite3.connect(f"file:{p}?mode=ro", uri=True) + .execute("SELECT value FROM item").fetchone()[0] + for p in restored_paths) + dup_failed = None + try: + make_snapshot("dup-source", [la, la], [], out, online=False) + except SnapshotError as exc: + dup_failed = str(exc) + checks.append({"name": "fix_same_basename_no_overwrite_and_duplicate_refused", + "ok": (len(set(snap_names)) == 2 and same_verify["ok"] is True + and distinct and rows == ["first", "second"] + and dup_failed is not None), + "detail": f"snap_names={snap_names} verify_ok={same_verify['ok']} " + f"restored_rows={rows} dup_refused={'yes' if dup_failed else 'no'}"}) + # 恢复路径必须真实存在于恢复树(不是随 TemporaryDirectory 消失的路径)。 + checks.append({"name": "restore_drill_paths_persist", + "ok": all(Path(p).is_file() for p in restored_paths) + and same_restore.is_dir(), + "detail": f"restore_root={same_restore}"}) ok = all(c["ok"] for c in checks) out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "selftest.json").write_text( @@ -499,6 +626,8 @@ def main() -> int: ap.add_argument("--online", action="store_true", help="在线口径:仅显式单库 backup API(默认停服+checkpoint;含 SQLite 的 --file 目录会被拒绝)") ap.add_argument("--archive", type=Path) + ap.add_argument("--restore-dir", type=Path, + help="回退演练的恢复目标树(默认:归档旁 verify-restore--<时间>/,持久保留供回滚使用)") args = ap.parse_args() if args.selftest: return selftest(REPO / "docs/rework/hel879/release-snapshot") @@ -519,7 +648,7 @@ def main() -> int: if args.command == "verify": if not args.archive: ap.error("verify 需要 --archive") - result = verify_archive(args.archive) + result = verify_archive(args.archive, restore_root=args.restore_dir) print(json.dumps(result, ensure_ascii=False, indent=1)) return 0 if result["ok"] else 1 ap.print_help()