From 78a5cfd8d75e3cfd00c60dc5c3592eb08cc757a1 Mon Sep 17 00:00:00 2001 From: zonggong Date: Tue, 27 Oct 2026 04:10:23 +0800 Subject: [PATCH] HEL-879 rework: fix the four manager-review edges - (1) dual-port launcher owns the app lifecycle exactly once: uvicorn listeners run with lifespan=off, either listener ending stops the other BEFORE the shared store closes, cancellation cleans up listeners first, and create_app's lifespan gains a fail-closed double-entry guard (the old shape silently re-initialized Store/epoch/life/chat_lock on the second entry); (2) room ownership survives delete failures: end_session keeps the session in a bounded persisted cleanup ledger until the remote delete lands (retry/idempotent, LiveKit 404 = already gone = settled), expiry moves rooms into that ledger instead of silently dropping them (the manager measured zero DELETE attempts on expired ends), graceful shutdown does not hang up live calls, restart compensates only this instance's own recorded rooms bounded by attempts/TTL+grace with abandonment recorded, and the router answers 503 (not 404) while a room is still owned so the browser keeps its owed DELETE; (3) TurnLedger binds each speech handle to its own generation stream explicitly (oldest pending handle gets the stream the SDK starts; a newer speech retires an older generation-less handle) - a handle without a generation can no longer steal the next turn's stream/receipt, the worker discards interrupted/failed bindings fail-closed before consuming; (4) release snapshot v3: verify requires three-way count equality (manifest source = manifest snapshot = restored actual; rows_match is an extra constraint, never an OR bypass), the rollback drill actually restores by source path into a persistent isolated tree and opens the restored DBs there, same-basename DBs get collision-free archive names, duplicate sources are refused; new regression file tests/test_voice_rework_hel879.py 14/14 reproducing every edge first, full pytest 338/338 Co-authored-by: multica-agent --- deploy/voice/serve_dual_8360_8361.py | 71 ++- .../hel879/release-snapshot/selftest.json | 29 +- src/cybergril/api/app.py | 154 +++--- src/cybergril/voice/livekit_wire.py | 25 +- src/cybergril/voice/receipt.py | 63 ++- src/cybergril/voice/router.py | 12 +- src/cybergril/voice/session.py | 283 +++++++++- src/cybergril/voice/worker.py | 7 +- tests/test_voice_chat_adapter.py | 16 +- tests/test_voice_rework_hel879.py | 497 ++++++++++++++++++ tools/release_snapshot_hel896.py | 209 ++++++-- 11 files changed, 1194 insertions(+), 172 deletions(-) create mode 100644 tests/test_voice_rework_hel879.py diff --git a/deploy/voice/serve_dual_8360_8361.py b/deploy/voice/serve_dual_8360_8361.py index 567827f..d87c39c 100644 --- a/deploy/voice/serve_dual_8360_8361.py +++ b/deploy/voice/serve_dual_8360_8361.py @@ -1,9 +1,20 @@ """统一 CyberGril 实例单进程双入口启动器(HEL-877 部署草案,本卡未执行). HTTP 8360 与 HTTPS 8361(语音页需要 secure context 才能拿麦克风)必须共用 -同一个 API 进程、同一份 app.state(epoch / chat_lock / store / 语音会话登记): -本启动器在一个事件循环里跑两个 uvicorn Server,指向同一个 FastAPI 应用对象, -不是两个实例。自签证书签发命令与回滚步骤见 docs/runbook-hel877-voice.md。 +同一个 API 进程、同一份 app.state(epoch / chat_lock / store / 语音会话登记)。 + +HEL-879 收口(总管复核第 1 项):uvicorn 两个 Server 默认各自进入一次 FastAPI +lifespan,导致 epoch 1→2、Store/life/chat_lock 全部被第二次初始化替换——同一个 +app 对象并不保证只初始化一次。本启动器改为: + +- 应用生命周期由本启动器**独占管理且只进入一次**(`app.router.lifespan_context`); +- 两个 uvicorn Server 以 `lifespan="off"` 只负责监听,共享同一份状态; +- 任一监听退出(失败/取消/信号)先把另一个也停掉,**再**退出生命周期: + 绝不出现一边还在接请求、一边已被另一个监听提前关库;循环与资源只关闭一次; +- `create_app` 的 lifespan 自带重入门卫:任何二次进入直接报错(fail-closed), + 防止未来改回 auto 双 Server 时再次静默替换状态。 + +自签证书签发命令与回滚步骤见 docs/runbook-hel877-voice.md。 用法(部署卡执行,先 `cd` 到仓库根并激活 3.11 venv): PYTHONPATH=src python deploy/voice/serve_dual_8360_8361.py \ @@ -28,24 +39,52 @@ import uvicorn from cybergril.api.app import create_app -async def serve(http_port: int, https_port: int, cert: str, key: str) -> None: - # ONE app object: both servers share state, locks, epoch and the store. - app = create_app() +async def serve(http_port: int, https_port: int, cert: str, key: str, app=None) -> None: + # ONE app object, ONE lifespan owner: both listeners share state, locks, + # epoch and the store. The uvicorn servers never run the app lifespan + # themselves (lifespan="off") — the context manager below is its only + # owner, so initialization and shutdown each happen exactly once. + if app is None: + app = create_app() http_server = uvicorn.Server( - uvicorn.Config(app, host="0.0.0.0", port=http_port, log_level="info") + uvicorn.Config(app, host="0.0.0.0", port=http_port, log_level="info", + lifespan="off") ) https_server = uvicorn.Server( uvicorn.Config(app, host="0.0.0.0", port=https_port, ssl_certfile=cert, - ssl_keyfile=key, log_level="info") + ssl_keyfile=key, log_level="info", lifespan="off") ) - servers = [asyncio.create_task(http_server.serve()), asyncio.create_task(https_server.serve())] - done, pending = await asyncio.wait(servers, return_when=asyncio.FIRST_COMPLETED) - for task in pending: - task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await task - for task in done: - task.result() + async with app.router.lifespan_context(app): + servers = [asyncio.create_task(http_server.serve()), + asyncio.create_task(https_server.serve())] + try: + done, pending = await asyncio.wait(servers, + return_when=asyncio.FIRST_COMPLETED) + except asyncio.CancelledError: + # The launcher itself was cancelled: stop both listeners first + # (graceful should_exit, hard cancel only as a last resort), await + # their full exit, only then let the lifespan close (and re-raise). + # No listener is ever left running without its state. + for server in (http_server, https_server): + server.should_exit = True + for task in servers: + with contextlib.suppress(asyncio.CancelledError, Exception): + await asyncio.wait_for(asyncio.shield(task), timeout=5.0) + for task in servers: + task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + raise + # Any listener ending stops the other FIRST; only after both listeners + # have fully exited does the lifespan context above close the loop's + # resources and the store — never the other way around. + for server in (http_server, https_server): + server.should_exit = True + for task in pending: + with contextlib.suppress(asyncio.CancelledError, Exception): + await task + for task in done: + task.result() # surface the first listener's real error, after cleanup def main() -> int: diff --git a/docs/rework/hel879/release-snapshot/selftest.json b/docs/rework/hel879/release-snapshot/selftest.json index 39015ec..9b42433 100644 --- a/docs/rework/hel879/release-snapshot/selftest.json +++ b/docs/rework/hel879/release-snapshot/selftest.json @@ -1,7 +1,7 @@ { "ok": true, - "code_commit": "7f66015ac854b7f513a35a42b680a143058561bb", - "tool_version": 2, + "code_commit": "7f0a739213e1cd8010d6e7118827884e973ea7c9", + "tool_version": 3, "checks": [ { "name": "naive_online_copy_loses_wal_reproduced", @@ -21,32 +21,47 @@ { "name": "fix1b_row_mismatch_fails_no_archive", "ok": true, - "detail": "error=源/快照行数不一致,不打包:/tmp/multica-task-29494715/hel879-snapshot-sel artifacts=[]" + "detail": "error=源/快照行数不一致,不打包:/tmp/multica-task-3939683741/hel879-snapshot-s artifacts=[]" }, { "name": "fix_verify_rejects_rows_match_false_legacy_package", "ok": true, - "detail": "ok=False checks=[{'db': 'databases/companion.db', 'sha256_ok': True, 'rows_ok': False, 'counts': {'companion': 1, 'chat_message': 1, 'events': 0}, 'source_counts': {'companion': 1, 'chat_message': 1, 'events': 1}, 'rows_match_recorded': False, 'rejected_legacy_package': True}]" + "detail": "ok=False checks=[{'db': 'databases/companion.db', 'restored': '/tmp/multica-task-3939683741/hel879-snapshot-selftest-jc481l71/verify-restore-legacy-pkg.tar-20261027-035938/by-source/tmp__multica-task-3939683741__hel879-snapshot-selftest-jc481l71__instance__companion.db', 'sha256_ok': True, 'rows_ok': False, 'counts': {'companion': 1, 'chat_message': 1, 'events': 1}, 'source_counts': {'companion': 1, 'chat_message': 1, 'events': 1}, 'snapshot_counts': {'companion': 1, 'chat_message': 1, 'events': 0}, 'rows_match_recorded': False, 'rejected_legacy_package': True}]" }, { "name": "fix2_online_mode_refuses_dir_with_sqlite", "ok": true, - "detail": "在线模式只允许显式单库 backup API(--db);--file /tmp/multica-task-29494715/hel879-snapshot-selftest-pxbo6xw1/instance/history_root 内" + "detail": "在线模式只允许显式单库 backup API(--db);--file /tmp/multica-task-3939683741/hel879-snapshot-selftest-jc481l71/instance/history_root" }, { "name": "fix2_stop_mode_embeds_sqlite_via_consistency_path", "ok": true, - "detail": "embedded=['/tmp/multica-task-29494715/hel879-snapshot-selftest-pxbo6xw1/instance/history_root/mem0_public_history.db']" + "detail": "embedded=['/tmp/multica-task-3939683741/hel879-snapshot-selftest-jc481l71/instance/history_root/mem0_public_history.db']" }, { "name": "fix3_verify_maps_source_paths_and_passes", "ok": true, - "detail": "ok=True commit=7f66015ac854b7f513a35a42b680a143058561bb plan_entries=4" + "detail": "ok=True commit=7f0a739213e1cd8010d6e7118827884e973ea7c9 plan_entries=4" }, { "name": "backup_api_snapshot_complete", "ok": true, "detail": "{\"companion\": 1, \"chat_message\": 1, \"events\": 1}" + }, + { + "name": "fix_rows_match_true_cannot_bypass_source_counts", + "ok": true, + "detail": "ok=False counts={'companion': 1, 'chat_message': 1, 'events': 1} source={'companion': 1, 'chat_message': 1, 'events': 2}" + }, + { + "name": "fix_same_basename_no_overwrite_and_duplicate_refused", + "ok": true, + "detail": "snap_names=['ledger.db', 'ledger__0e56d727.db'] verify_ok=True restored_rows=['first', 'second'] dup_refused=yes" + }, + { + "name": "restore_drill_paths_persist", + "ok": true, + "detail": "restore_root=/tmp/multica-task-3939683741/hel879-snapshot-selftest-jc481l71/same-restore" } ] } \ No newline at end of file diff --git a/src/cybergril/api/app.py b/src/cybergril/api/app.py index 5361ed0..6b99275 100644 --- a/src/cybergril/api/app.py +++ b/src/cybergril/api/app.py @@ -210,73 +210,90 @@ def create_app( @asynccontextmanager async def lifespan(app): - store = Store(db_path) - app.state.store = store - with store.conn: - store.execute( - "INSERT INTO api_meta(key,value) VALUES('epoch',1) ON CONFLICT(key) DO UPDATE SET value=value+1" - ) - store.execute("UPDATE chat_message SET status='cancelled' WHERE status='pending'") - app.state.epoch = store.query_one("SELECT value FROM api_meta WHERE key='epoch'")["value"] - app.state.active = None - app.state.chat_lock = asyncio.Lock() - app.state.health_cache = (0, False) - app.state.health_lock = asyncio.Lock() - router.guard = PersistentBudget(store, router.guard) - app.state.sharing = Sharing(store, router, models) - app.state.persona = Persona(store, router, models, enabled=life_config.enabled) - app.state.public = PublicView(store) - app.state.mood = MoodRuntime(store, character) - app.state.dyad = DyadRuntime(store, character, router=router, models=models, - timezone=life_config.timezone) - app.state.semantic = build_semantic(store, mode=semantic_mode, config=semantic_config, - request_guard=router.guard) - # Original positional contract preserved; R4 collaborators attach after assembly. - app.state.life = LifeRuntime(store, router, models, life_config, app.state.sharing, - app.state.persona) - app.state.life.mood = app.state.mood - app.state.life.semantic = app.state.semantic - app.state.life.dyad = app.state.dyad - app.state.life.reach_out = ReachOut(store, router, models, app.state.persona, - app.state.mood, config=character.reach_out, - timezone=life_config.timezone) - # Preserve the original runtime factory contract. Assemble only after its - # clock is known, before any recovery or background work can run. - app.state.memories = Memories(store, router, models, timezone=life_config.timezone, - clock=lambda: app.state.life.clock()) - app.state.life.memories = app.state.memories - app.state.memories.recover() - app.state.mood.recover() - if app.state.semantic: - app.state.semantic.recover() - if app.state.life.reach_out: - app.state.life.reach_out.recover() - life_task = asyncio.create_task(app.state.life.run()) + # HEL-879 dual-entry guard: a second concurrent entry (e.g. two uvicorn + # servers both running the default lifespan against this app) must fail + # loudly, never silently re-initialize Store/epoch/life/chat_lock under + # live requests. Re-entry after a completed exit stays legitimate. + if getattr(app.state, "lifespan_active", False): + raise RuntimeError( + "application lifespan entered twice — the lifecycle has exactly " + "one owner (deploy/voice/serve_dual_8360_8361.py runs uvicorn " + "with lifespan='off' and owns it once)") + app.state.lifespan_active = True try: - yield - finally: - app.state.life.stop.set() - if life_task: - life_task.cancel() - with suppress(asyncio.CancelledError): - await life_task - if app.state.active: - app.state.active["cancel"].set() - await app.state.active["finished"].wait() + store = Store(db_path) + app.state.store = store + with store.conn: + store.execute( + "INSERT INTO api_meta(key,value) VALUES('epoch',1) ON CONFLICT(key) DO UPDATE SET value=value+1" + ) + store.execute("UPDATE chat_message SET status='cancelled' WHERE status='pending'") + app.state.epoch = store.query_one("SELECT value FROM api_meta WHERE key='epoch'")["value"] + app.state.active = None + app.state.chat_lock = asyncio.Lock() + app.state.health_cache = (0, False) + app.state.health_lock = asyncio.Lock() + router.guard = PersistentBudget(store, router.guard) + app.state.sharing = Sharing(store, router, models) + app.state.persona = Persona(store, router, models, enabled=life_config.enabled) + app.state.public = PublicView(store) + app.state.mood = MoodRuntime(store, character) + app.state.dyad = DyadRuntime(store, character, router=router, models=models, + timezone=life_config.timezone) + app.state.semantic = build_semantic(store, mode=semantic_mode, config=semantic_config, + request_guard=router.guard) + # Original positional contract preserved; R4 collaborators attach after assembly. + app.state.life = LifeRuntime(store, router, models, life_config, app.state.sharing, + app.state.persona) + app.state.life.mood = app.state.mood + app.state.life.semantic = app.state.semantic + app.state.life.dyad = app.state.dyad + app.state.life.reach_out = ReachOut(store, router, models, app.state.persona, + app.state.mood, config=character.reach_out, + timezone=life_config.timezone) + # Preserve the original runtime factory contract. Assemble only after its + # clock is known, before any recovery or background work can run. + app.state.memories = Memories(store, router, models, timezone=life_config.timezone, + clock=lambda: app.state.life.clock()) + app.state.life.memories = app.state.memories + app.state.memories.recover() + app.state.mood.recover() if app.state.semantic: - # HEL-889/894 shutdown ownership: aclose returns only after - # every live batch worker has actually exited (drain_timeout - # bounds the polite phase, never a licence to close under a - # worker). If this await is cancelled after the resource - # close began, the close still lands exactly once before the - # cancellation propagates (HEL-894) — the provider clients - # and the store below stay safe in either order. - await app.state.semantic.aclose() - for provider in set(router.provider_for(t) for t in router.available_tiers()): - await provider.aclose() + app.state.semantic.recover() + if app.state.life.reach_out: + app.state.life.reach_out.recover() + # HEL-879: bounded restart compensation for this instance's own + # voice rooms (persisted cleanup ledger; own rooms only). if getattr(app.state, "voice_service", None): - await app.state.voice_service.aclose() - store.close() + await app.state.voice_service.recover() + life_task = asyncio.create_task(app.state.life.run()) + try: + yield + finally: + app.state.life.stop.set() + if life_task: + life_task.cancel() + with suppress(asyncio.CancelledError): + await life_task + if app.state.active: + app.state.active["cancel"].set() + await app.state.active["finished"].wait() + if app.state.semantic: + # HEL-889/894 shutdown ownership: aclose returns only after + # every live batch worker has actually exited (drain_timeout + # bounds the polite phase, never a licence to close under a + # worker). If this await is cancelled after the resource + # close began, the close still lands exactly once before the + # cancellation propagates (HEL-894) — the provider clients + # and the store below stay safe in either order. + await app.state.semantic.aclose() + for provider in set(router.provider_for(t) for t in router.available_tiers()): + await provider.aclose() + if getattr(app.state, "voice_service", None): + await app.state.voice_service.aclose() + store.close() + finally: + app.state.lifespan_active = False app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None) @@ -288,8 +305,15 @@ def create_app( voice_config = load_voice_config() except Exception: voice_config = None + # Cleanup ledger beside the instance database: rooms this instance + # opened stay owned (persisted) until their bounded delete lands — + # a restarted process compensates its OWN rooms only (HEL-879). voice_service = ( - VoiceSessionService(voice_config) if voice_config is not None else None + VoiceSessionService( + voice_config, + state_path=None if str(db_path) == ":memory:" else Path(db_path).with_name( + Path(db_path).stem + "-voice-sessions.json"), + ) if voice_config is not None else None ) app.state.voice_service = voice_service app.include_router(build_voice_router(voice_service)) diff --git a/src/cybergril/voice/livekit_wire.py b/src/cybergril/voice/livekit_wire.py index bd65eaf..97cbeca 100644 --- a/src/cybergril/voice/livekit_wire.py +++ b/src/cybergril/voice/livekit_wire.py @@ -149,6 +149,21 @@ async def create_dispatch( ) +class RoomDeleteError(RuntimeError): + """LiveKit refused a room delete with a status other than 2xx/404. + + Raised (not returned) so callers cannot mistake a failed delete for a + settled one — the caller must keep the room owned for a bounded retry + (HEL-879). A 404 answer means the room is already gone and is reported + as success instead: the delete's goal (room no longer exists) holds. + """ + + def __init__(self, status_code: int, room: str) -> None: + super().__init__(f"delete room {room!r} failed with HTTP {status_code}") + self.status_code = status_code + self.room = room + + async def delete_room( http: httpx.AsyncClient, *, @@ -162,6 +177,10 @@ async def delete_room( Deleting the room disconnects every participant, ends the dispatched agent job and releases the browser microphone server-side. + + Returns True when the room is gone: 2xx, or 404 (already deleted — the + retry after a lost response stays idempotent). Any other status raises + RoomDeleteError so ownership is retained until the delete really lands. """ response = await http.post( f"{internal_url.rstrip('/')}{TWIRP_PREFIX}/{ROOM_SERVICE}/DeleteRoom", @@ -172,8 +191,10 @@ async def delete_room( }, timeout=timeout, ) - # LiveKit answers DeleteRoom with an empty protobuf body; treat 2xx as done. - return response.status_code // 100 == 2 + # LiveKit answers DeleteRoom with an empty protobuf body; 404 = already gone. + if response.status_code // 100 == 2 or response.status_code == 404: + return True + raise RoomDeleteError(response.status_code, room) def decode_token(token: str) -> tuple[dict[str, Any], bytes]: diff --git a/src/cybergril/voice/receipt.py b/src/cybergril/voice/receipt.py index c2bb857..ea37ea6 100644 --- a/src/cybergril/voice/receipt.py +++ b/src/cybergril/voice/receipt.py @@ -7,9 +7,13 @@ ended in chat.error (the φ line is spoken verbatim but is not her archived reply), missing receipt, or a receipt from an epoch the server has since left (the server fail-closes those anyway — this is the client-side half). -Speech-handle-to-stream binding lives in TurnLedger: AgentSession emits -speech_created before the LLM stream starts and runs one user-turn generation -at a time, so the first stream created at/after a handle belongs to it. +Speech-handle-to-stream binding lives in TurnLedger as an EXPLICIT +association, not a time-window guess (HEL-879 总管复核第 3 项): AgentSession +runs one user-turn generation at a time, so the LLM stream a generation starts +is bound to the oldest handle still awaiting its generation; a handle whose +generation never starts (cancelled/interrupted before the LLM call) is retired +the moment a newer speech appears — it can never consume a later turn's +stream, and a failed/interrupted handle only ever releases its own binding. """ from __future__ import annotations @@ -48,31 +52,46 @@ class _HasHandle(Protocol): class TurnLedger: - """Binds livekit SpeechHandles to the ChatApiLLMStream of their generation.""" + """Binds livekit SpeechHandles to the ChatApiLLMStream of their generation. + + The binding is exact, never inferred from ordering timestamps: a stream is + bound to the oldest handle that has not yet been retired (its generation is + the one the SDK is starting), and each stream is consumed exactly once by + exactly its own handle. + """ def __init__(self) -> None: - self._handles: deque[tuple[float, Any]] = deque() # (created_at, handle) - self._streams: deque[tuple[float, Any]] = deque() # (created_at, stream) + self._pending: deque[Any] = deque() # handles awaiting their generation + self._bound: dict[int, Any] = {} # id(handle) -> its stream + self._retired: set[int] = set() # ended without a generation - def note_handle(self, handle: _HasHandle, created_at: float) -> None: - self._handles.append((created_at, handle)) + def note_handle(self, handle: _HasHandle, created_at: float | None = None) -> None: + """A new user speech started: every older handle still awaiting its + generation will never get one (the SDK moves on to the new turn), so it + is retired instead of staying eligible to steal the next stream.""" + while self._pending: + self._retired.add(id(self._pending.popleft())) + self._pending.append(handle) - def note_stream(self, stream: Any, created_at: float) -> None: - self._streams.append((created_at, stream)) + def note_stream(self, stream: Any, created_at: float | None = None) -> None: + """The generation the SDK just started belongs to the oldest handle + still awaiting one. A stream with no pending handle (proactive speech + outside a user turn) is left unbound: no user turn may claim it.""" + if self._pending: + handle = self._pending.popleft() + self._bound[id(handle)] = stream def stream_for(self, handle: _HasHandle) -> Any | None: - """First not-yet-consumed stream created at/after the handle.""" - while self._handles and self._handles[0][1] is not handle: - self._handles.popleft() # older handles ended without a generation - if not self._handles: - return None - # (handle present below) - created = self._handles.popleft()[0] - while self._streams: - at, stream = self._streams.popleft() - if at >= created: - return stream - return None + """This handle's own stream, exactly once; None when its generation + never started or it was retired/interrupted without one.""" + return self._bound.pop(id(handle), None) + + def discard(self, handle: _HasHandle) -> None: + """Release a handle's binding without using it (fail-closed path for + interrupted/failed playout): the turn never acks, and the binding does + not leak into any later turn.""" + self._bound.pop(id(handle), None) + self._retired.discard(id(handle)) class ReceiptSender: diff --git a/src/cybergril/voice/router.py b/src/cybergril/voice/router.py index 98ded3a..bc401c7 100644 --- a/src/cybergril/voice/router.py +++ b/src/cybergril/voice/router.py @@ -8,6 +8,10 @@ POST /api/voice/session open a room, dispatch the dedicated worker, DELETE /api/voice/session/{id} end: delete the LiveKit room, which disconnects browser and worker, ends the agent job and releases the microphone. + A delete that cannot land keeps the room + in a bounded, persisted cleanup ledger + (503, retried); it never collapses into + a 404 while the room is still owned. GET /api/voice/status honest switch state for the frontend (configured / tts reachable / agent name). @@ -70,7 +74,13 @@ def build_voice_router(service: VoiceSessionService | None) -> APIRouter: async def end_session(session_id: str) -> dict: if service is None: raise HTTPException(503, REFUSAL_NOT_CONFIGURED) - session = await service.end_session(session_id) + try: + session = await service.end_session(session_id) + except Exception: + # The remote delete did not land: the room stays owned in the + # bounded cleanup ledger and will be retried; the caller must NOT + # treat this as done (a 404 would erase the browser's owed DELETE). + raise HTTPException(503, "这次通话还没有完全挂断,会自动重试清理;请稍后再试。") from None if session is None: raise HTTPException(404, "这次通话已经不在了。") return {"ended": True, "room": session.room} diff --git a/src/cybergril/voice/session.py b/src/cybergril/voice/session.py index 00a346f..1b07eaf 100644 --- a/src/cybergril/voice/session.py +++ b/src/cybergril/voice/session.py @@ -5,14 +5,25 @@ silero VAD) exactly as HEL-876 audited it: this module only creates a room dispatch for the dedicated ``cybergril-voice-agent`` worker and mints a short-lived join token server-side, so no LiveKit key ever reaches the browser. -Everything here is process-local and stateless apart from the small session -registry: chat state, persona, memory and locks stay in the single API process -that mounts this router (HTTP and the HTTPS voice entry share it). +Everything here is process-local apart from the small session registry and its +persisted cleanup ledger (HEL-879): chat state, persona, memory and locks stay +in the single API process that mounts this router (HTTP and the HTTPS voice +entry share it). + +Room ownership stays until the delete really lands (HEL-879 总管复核第 2 项): +a failed remote delete keeps the session in a bounded cleanup ledger — retried +on the next end call and by a bounded background sweep — and the ledger is +persisted beside the instance database so a restarted process compensates its +OWN rooms (never another instance's, never the generic worker). Retries are +idempotent (404 = already gone = settled). Every debt is bounded by attempts +and age, then recorded as abandoned instead of retried forever. """ from __future__ import annotations import asyncio +import contextlib +import json import os import secrets import string @@ -29,6 +40,10 @@ DEFAULT_AGENT_NAME = "cybergril-voice-agent" DEFAULT_TOKEN_TTL = 300.0 DEFAULT_SESSION_TTL = 2 * 3600.0 TTS_HEALTH_CACHE_SECONDS = 30.0 +CLEANUP_MAX_ATTEMPTS = 8 +CLEANUP_BACKOFF_SECONDS = 0.5 +CLEANUP_GRACE_SECONDS = 24 * 3600.0 +STATE_VERSION = 1 @dataclass @@ -103,6 +118,50 @@ class VoiceSession: epoch: int dispatch_id: str = "" + def as_state(self) -> dict: + return { + "session_id": self.session_id, "room": self.room, "identity": self.identity, + "created_at": self.created_at, "expires_at": self.expires_at, + "epoch": self.epoch, "dispatch_id": self.dispatch_id, + } + + @classmethod + def from_state(cls, data: dict) -> "VoiceSession": + return cls( + session_id=str(data["session_id"]), room=str(data["room"]), + identity=str(data.get("identity", "")), created_at=float(data.get("created_at", 0)), + expires_at=float(data.get("expires_at", 0)), epoch=int(data.get("epoch", 0)), + dispatch_id=str(data.get("dispatch_id", "")), + ) + + +@dataclass +class _CleanupDebt: + """One room this instance still owns: its remote delete has not landed.""" + + session: VoiceSession + attempts: int = 0 + first_debt_at: float = 0.0 + last_error: str = "" + abandoned: str = "" + eligible_at: float = 0.0 # background retries wait until this clock time + + def as_state(self) -> dict: + return {"session": self.session.as_state(), "attempts": self.attempts, + "first_debt_at": self.first_debt_at, "last_error": self.last_error, + "abandoned": self.abandoned, "eligible_at": self.eligible_at} + + @classmethod + def from_state(cls, data: dict) -> "_CleanupDebt": + return cls( + session=VoiceSession.from_state(data.get("session", {})), + attempts=int(data.get("attempts", 0)), + first_debt_at=float(data.get("first_debt_at", 0)), + last_error=str(data.get("last_error", "")), + abandoned=str(data.get("abandoned", "")), + eligible_at=float(data.get("eligible_at", 0)), + ) + @dataclass class _Health: @@ -116,11 +175,73 @@ class VoiceSessionService: http: httpx.AsyncClient = field(default_factory=httpx.AsyncClient) clock: object = time.time tts_health_cache_seconds: float = TTS_HEALTH_CACHE_SECONDS + state_path: Path | str | None = None + cleanup_max_attempts: int = CLEANUP_MAX_ATTEMPTS + cleanup_backoff_seconds: float = CLEANUP_BACKOFF_SECONDS + cleanup_grace_seconds: float = CLEANUP_GRACE_SECONDS def __post_init__(self) -> None: self._sessions: dict[str, VoiceSession] = {} + self._debts: dict[str, _CleanupDebt] = {} self._tts_health = _Health(checked=-1e9, ok=False) self._tts_lock = asyncio.Lock() + self._cleaner: asyncio.Task | None = None + self._load_state() + + # -- persisted cleanup ledger (own rooms only) --------------------------- + def _load_state(self) -> None: + """Restart compensation: rooms recorded by a previous process of THIS + instance become cleanup debts (bounded by the persisted attempt counts + and age); they are the only rooms ever touched.""" + if not self.state_path: + return + try: + data = json.loads(Path(self.state_path).read_text(encoding="utf-8")) + except (OSError, ValueError): + return + if data.get("version") != STATE_VERSION: + return + now = self.clock() + for entry in data.get("debts", []): + try: + debt = _CleanupDebt.from_state(entry) + except (KeyError, TypeError, ValueError): + continue + if debt.session.session_id not in self._debts: + self._debts[debt.session.session_id] = debt + for entry in data.get("sessions", []): + try: + session = VoiceSession.from_state(entry) + except (KeyError, TypeError, ValueError): + continue + # A room that was live when the process went down: the call may + # legitimately survive an API restart (the room lives on the SFU), + # so background compensation waits for the session's own TTL — + # bounded, never leaked, and never a longer-lived claim. + if session.session_id not in self._debts and session.session_id not in self._sessions: + self._debts[session.session_id] = _CleanupDebt( + session=session, first_debt_at=now, + eligible_at=max(now, session.expires_at)) + self._prune() + + def _persist_state(self) -> None: + if not self.state_path: + return + payload = { + "version": STATE_VERSION, + "sessions": [s.as_state() for s in self._sessions.values()], + "debts": [d.as_state() for d in self._debts.values()], + } + path = Path(self.state_path) + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(path.suffix + ".tmp") + tmp.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8") + tmp.replace(path) + except OSError: + # Best effort: losing the ledger never breaks a request, it only + # costs the restart compensation for still-owned rooms. + pass # -- honest readiness --------------------------------------------------- async def tts_available(self) -> bool: @@ -141,12 +262,94 @@ class VoiceSessionService: # -- lifecycle ---------------------------------------------------------- def _prune(self) -> None: + """Expired registrations move into the cleanup ledger — never silently + dropped: their room stays owned until its bounded delete lands.""" now = self.clock() - for session_id in [k for k, v in self._sessions.items() if v.expires_at < now]: - del self._sessions[session_id] + expired = [k for k, v in self._sessions.items() if v.expires_at < now] + for session_id in expired: + session = self._sessions.pop(session_id) + if session_id not in self._debts: + self._debts[session_id] = _CleanupDebt(session=session, first_debt_at=now) + if expired: + self._persist_state() + + def _abandon_reason(self, debt: _CleanupDebt) -> str: + if debt.abandoned: + return debt.abandoned + if debt.attempts >= self.cleanup_max_attempts: + return "attempts-exhausted" + if self.clock() - debt.session.expires_at > self.cleanup_grace_seconds: + return "aged-out" + return "" + + def _retryable_debts(self) -> list[str]: + now = self.clock() + return [sid for sid, debt in self._debts.items() + if not self._abandon_reason(debt) and now >= debt.eligible_at] + + async def _delete_room_once(self, session: VoiceSession) -> None: + """One remote delete attempt; 2xx/404 settle, anything else raises.""" + await livekit_wire.delete_room( + self.http, + internal_url=self.config.livekit_internal, + api_key=self.config.api_key, + api_secret=self.config.api_secret, + room=session.room, + timeout=self.config.dispatch_timeout_seconds, + ) + + async def _cleanup_loop(self) -> None: + try: + while True: + pending = self._retryable_debts() + if not pending: + waiting = [d for d in self._debts.values() + if not self._abandon_reason(d)] + if not waiting: + break + # Deferred debts (restart compensation waiting out a live + # session's TTL): re-check when the earliest becomes + # eligible, capped so the loop stays cheap and bounded. + now = self.clock() + delay = min(max(d.eligible_at - now for d in waiting), 60.0) + await asyncio.sleep(max(delay, 0.05) if delay > 0 else 0.05) + continue + for session_id in pending: + debt = self._debts.get(session_id) + if debt is None or self._abandon_reason(debt): + continue + try: + await self._delete_room_once(debt.session) + self._debts.pop(session_id, None) + except Exception as exc: # bounded retry, never a crash loop + debt.attempts += 1 + debt.last_error = f"{type(exc).__name__}: {exc}"[:200] + reason = self._abandon_reason(debt) + if reason: + debt.abandoned = reason + self._persist_state() + if not debt.abandoned and session_id in self._debts: + await asyncio.sleep(self.cleanup_backoff_seconds) + finally: + self._cleaner = None + + def _kick_cleanup(self) -> None: + if self._cleaner is not None and not self._cleaner.done(): + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + self._cleaner = loop.create_task(self._cleanup_loop()) + + async def recover(self) -> None: + """App-start hook: compensate rooms left over by a previous process of + this instance (bounded by the persisted ledger).""" + self._kick_cleanup() async def open_session(self, *, epoch: int) -> VoiceSession: self._prune() + self._kick_cleanup() alphabet = string.ascii_lowercase + string.digits room = "cg-voice-" + "".join(secrets.choice(alphabet) for _ in range(10)) identity = "user-" + "".join(secrets.choice(string.ascii_lowercase) for _ in range(8)) @@ -170,6 +373,7 @@ class VoiceSessionService: dispatch_id=dispatch.dispatch_id, ) self._sessions[session.session_id] = session + self._persist_state() return session def token_for(self, session: VoiceSession) -> str: @@ -187,23 +391,72 @@ class VoiceSessionService: ) async def end_session(self, session_id: str) -> VoiceSession | None: + """End a session by actually deleting its room (HEL-879). + + The registration is only removed AFTER the remote delete settles: + a failed delete keeps the session in the bounded cleanup ledger, so a + retry (this method again, the background sweep, or the next process + via the persisted ledger) still targets the same room — it can never + degrade into a 404 that silently drops the room. Unknown ids (never + seen here, or fully settled and forgotten) still return None → 404. + """ self._prune() - session = self._sessions.pop(session_id, None) + session = self._sessions.get(session_id) if session is None: - return None - await livekit_wire.delete_room( - self.http, - internal_url=self.config.livekit_internal, - api_key=self.config.api_key, - api_secret=self.config.api_secret, - room=session.room, - timeout=self.config.dispatch_timeout_seconds, - ) + debt = self._debts.get(session_id) + if debt is None or debt.abandoned: + return None + session = debt.session + try: + await self._delete_room_once(session) + except Exception as exc: + debt = self._debts.get(session_id) + if debt is None: + debt = _CleanupDebt(session=session, first_debt_at=self.clock()) + self._debts[session_id] = debt + debt.attempts += 1 + debt.last_error = f"{type(exc).__name__}: {exc}"[:200] + reason = self._abandon_reason(debt) + if reason: + debt.abandoned = reason + self._persist_state() + self._kick_cleanup() + raise + self._sessions.pop(session_id, None) + self._debts.pop(session_id, None) + self._persist_state() return session def get(self, session_id: str) -> VoiceSession | None: self._prune() return self._sessions.get(session_id) + def cleanup_debt(self, session_id: str) -> _CleanupDebt | None: + """Test/diagnostics view of the owned-but-not-deleted ledger.""" + return self._debts.get(session_id) + async def aclose(self) -> None: + if self._cleaner is not None: + self._cleaner.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._cleaner + self._cleaner = None + # Graceful shutdown: rooms still active here become debts, then one + # final bounded sweep runs before the transport closes; whatever + # remains stays in the persisted ledger for the next process. + self._prune() + for session_id in self._retryable_debts(): + debt = self._debts.get(session_id) + if debt is None or self._abandon_reason(debt): + continue + try: + await self._delete_room_once(debt.session) + self._debts.pop(session_id, None) + except Exception as exc: + debt.attempts += 1 + debt.last_error = f"{type(exc).__name__}: {exc}"[:200] + reason = self._abandon_reason(debt) + if reason: + debt.abandoned = reason + self._persist_state() await self.http.aclose() diff --git a/src/cybergril/voice/worker.py b/src/cybergril/voice/worker.py index aaa29fe..c3c736b 100644 --- a/src/cybergril/voice/worker.py +++ b/src/cybergril/voice/worker.py @@ -119,10 +119,15 @@ async def entrypoint(ctx: JobContext) -> None: async def _after_playout(handle) -> None: # SDK 真实播放完成(wait_for_playout 返回)之后才考虑回执。 await handle.wait_for_playout() + # HEL-879:先判定失败/打断并整单放弃(fail-closed)——被放弃的绑定 + # 只属于这一轮,绝不会吞掉下一轮的回执归属。 + failed = handle.exception() is not None + if handle.interrupted or failed: + ledger.discard(handle) + return stream = ledger.stream_for(handle) if stream is None: return - failed = handle.exception() is not None if stream.error_phi: subtitle({"kind": "error", "role": "assistant", "text": stream.error_phi}) return diff --git a/tests/test_voice_chat_adapter.py b/tests/test_voice_chat_adapter.py index b8f485e..1252834 100644 --- a/tests/test_voice_chat_adapter.py +++ b/tests/test_voice_chat_adapter.py @@ -302,13 +302,23 @@ async def test_receipt_sender_confirms_once_and_reports_server_refusal(): @pytest.fixture -def live_server(api): +def live_server(api, tmp_path): app, provider, _ = api + # HEL-879: an app's lifespan has exactly one owner — the api fixture's + # TestClient still holds its lifespan, so the live server gets its OWN app + # instance on the same file-backed database (companion row included) and + # the same provider; the old fixture entered one app's lifespan twice + # concurrently, the exact defect the app now refuses loudly. + router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}, + budget=engineering_budget()) + served = create_app(db_path=tmp_path / "adapter.db", router=router, + life_config=LifeConfig(enabled=True, poll_seconds=3600), + semantic_mode="stub") probe = socket.socket() probe.bind(("127.0.0.1", 0)) port = probe.getsockname()[1] probe.close() # free the port for uvicorn to bind - config = uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error") + config = uvicorn.Config(served, host="127.0.0.1", port=port, log_level="error") server = uvicorn.Server(config) thread = threading.Thread(target=server.run, daemon=True) thread.start() @@ -318,7 +328,7 @@ def live_server(api): break except Exception: time.sleep(0.1) - yield f"http://127.0.0.1:{port}", app, provider + yield f"http://127.0.0.1:{port}", served, provider server.should_exit = True thread.join(timeout=5) diff --git a/tests/test_voice_rework_hel879.py b/tests/test_voice_rework_hel879.py new file mode 100644 index 0000000..7160b43 --- /dev/null +++ b/tests/test_voice_rework_hel879.py @@ -0,0 +1,497 @@ +"""HEL-879 rework regressions for the 总管's independent-review edges. + +Four confirmed offline defects, each reproduced first and fixed: +1. dual uvicorn listeners both running the app lifespan (epoch 1→2, state + replaced, store closed under live listeners); +2. end_session popping the registry before the remote delete (lost retry + target; expiry pruning dropping rooms with zero DELETE attempts); +3. TurnLedger binding streams to handles by timestamp (a handle without a + generation steals the next turn's stream/receipt); +4. release-snapshot verify accepting self-contradictory success markers and + same-basename DBs overwriting each other (covered by the tool selftest). + +Zero network beyond 127.0.0.1; fake LiveKit/TTS transports throughout. +""" + +from __future__ import annotations + +import asyncio +import json +import socket +import ssl +import subprocess +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import httpx +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) + +from test_r2 import R2LLM # noqa: E402 + +from cybergril.api.app import create_app # noqa: E402 +from cybergril.life.runtime import LifeConfig # noqa: E402 +from cybergril.memory.store import Store # noqa: E402 +from cybergril.think.router import build_router # noqa: E402 +from cybergril.think.scenes import Tier # noqa: E402 +from cybergril.voice import livekit_wire # noqa: E402 +from cybergril.voice.receipt import TurnLedger # noqa: E402 +from cybergril.voice.session import VoiceConfig, VoiceSessionService # noqa: E402 + + +# ---------------------------------------------------------------- TurnLedger + +def _stream(turn_id: str) -> SimpleNamespace: + return SimpleNamespace(turn_id=turn_id, receipt={"message_id": hash(turn_id) % 1000}) + + +def test_ledger_binds_each_handle_to_its_own_stream(): + ledger = TurnLedger() + h1, h2 = object(), object() + s1, s2 = _stream("old"), _stream("new") + ledger.note_handle(h1, 1.0) + ledger.note_stream(s1, 1.1) + ledger.note_handle(h2, 2.0) + ledger.note_stream(s2, 2.1) + # A late cleanup of H2 must leave H1's own stream to H1 (old code: None). + assert ledger.stream_for(h2) is s2 + assert ledger.stream_for(h1) is s1 + + +def test_ledger_handle_without_generation_consumes_nothing(): + ledger = TurnLedger() + no_generation, real = object(), object() + s2 = _stream("new") + ledger.note_handle(no_generation, 10.0) + ledger.note_handle(real, 11.0) # retires the older, generation-less handle + ledger.note_stream(s2, 11.1) + assert ledger.stream_for(no_generation) is None + assert ledger.stream_for(real) is s2 + + +def test_ledger_interrupted_handle_discards_only_its_own_binding(): + ledger = TurnLedger() + h1, h2 = object(), object() + s1, s2 = _stream("one"), _stream("two") + ledger.note_handle(h1, 1.0) + ledger.note_stream(s1, 1.0) + ledger.note_handle(h2, 2.0) + ledger.note_stream(s2, 2.0) + ledger.discard(h1) # worker fail-closed path (interrupted/failed playout) + assert ledger.stream_for(h1) is None + assert ledger.stream_for(h2) is s2 + + +def test_ledger_stream_without_pending_handle_stays_unclaimed(): + ledger = TurnLedger() + stray = _stream("proactive") + ledger.note_stream(stray, 1.0) # no user turn awaited a generation + handle = object() + ledger.note_handle(handle, 2.0) + later = _stream("answer") + ledger.note_stream(later, 2.5) + assert ledger.stream_for(handle) is later + + +# --------------------------------------------------- session ownership ledger + +def _proto_field1(data: bytes) -> str: + """Decode varint-prefixed field 1 (room) from a livekit proto body.""" + if not data or (data[0] >> 3) != 1 or (data[0] & 7) != 2: + return "" + length, index, shift, value = 0, 1, 0, 0 + while True: + byte = data[index] + length |= (byte & 0x7F) << shift + index += 1 + if not byte & 0x80: + break + shift += 7 + return data[index:index + length].decode() + + +class StandinLiveKit: + """Fake Twirp transport: configurable delete failures per call.""" + + def __init__(self) -> None: + self.deleted: list[str] = [] + self.delete_errors: list[Exception | int] = [] + self._transport = httpx.MockTransport(self._handle) + + def _handle(self, request: httpx.Request) -> httpx.Response: + path = request.url.path + if path.endswith("/CreateDispatch"): + return httpx.Response(200, json={"id": "AD_1", "room": "r"}) + if path.endswith("/DeleteRoom"): + if self.delete_errors: + error = self.delete_errors.pop(0) + if isinstance(error, int): + return httpx.Response(error, text="room service error") + raise error + self.deleted.append(_proto_field1(request.content)) + return httpx.Response(200, content=b"") + return httpx.Response(404, text="unknown") + + @property + def transport(self) -> httpx.MockTransport: + return self._transport + + +def _service(standin: StandinLiveKit, *, clock=None, backoff=0.0, **kwargs) -> VoiceSessionService: + config = VoiceConfig( + livekit_url="ws://voice.test:7880", + livekit_internal="http://livekit.test", + api_key="devkey", + api_secret="test-secret-not-real", + tts_health_url="http://tts.test/health", + session_ttl_seconds=kwargs.pop("session_ttl_seconds", 5.0), + ) + return VoiceSessionService( + config=config, + http=httpx.AsyncClient(transport=standin.transport), + clock=clock or time.time, + cleanup_backoff_seconds=backoff, + **kwargs, + ) + + +async def _open(service: VoiceSessionService) -> str: + return (await service.open_session(epoch=1)).session_id + + +@pytest.mark.asyncio +async def test_delete_failure_keeps_room_owned_until_retry_lands(): + standin = StandinLiveKit() + standin.delete_errors = [RuntimeError("standin delete failed")] + service = _service(standin) + try: + session = await service.open_session(epoch=1) + with pytest.raises(RuntimeError): + await service.end_session(session.session_id) + # The room is retained for retry (old code: registry already popped, + # second call answered None and never deleted the room again). + debt = service.cleanup_debt(session.session_id) + assert debt is not None and debt.session.room == session.room + assert standin.deleted == [] + ended = await service.end_session(session.session_id) + assert ended.room == session.room + assert standin.deleted == [session.room] + assert service.cleanup_debt(session.session_id) is None + assert await service.end_session(session.session_id) is None + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_expired_session_room_is_actually_deleted(): + standin = StandinLiveKit() + now = [1000.0] + service = _service(standin, clock=lambda: now[0]) + try: + session = await service.open_session(epoch=1) + now[0] += service.config.session_ttl_seconds + 1 + # Old code: _prune dropped the registration silently — zero DELETEs. + ended = await service.end_session(session.session_id) + assert ended is not None and ended.room == session.room + assert standin.deleted == [session.room] + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_delete_already_gone_is_idempotent_success(): + standin = StandinLiveKit() + standin.delete_errors = [404] # room vanished server-side first + service = _service(standin) + try: + session = await service.open_session(epoch=1) + ended = await service.end_session(session.session_id) + assert ended is not None + assert service.cleanup_debt(session.session_id) is None + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_background_cleanup_settles_expiry_debt(): + standin = StandinLiveKit() + standin.delete_errors = [RuntimeError("flap"), RuntimeError("flap")] + now = [1000.0] + service = _service(standin, clock=lambda: now[0]) + try: + session = await service.open_session(epoch=1) + now[0] += service.config.session_ttl_seconds + 1 + with pytest.raises(RuntimeError): + await service.end_session(session.session_id) + now[0] += 1 + for _ in range(50): + if standin.deleted: + break + await asyncio.sleep(0.02) + assert standin.deleted == [session.room] + for _ in range(50): + if service.cleanup_debt(session.session_id) is None: + break + await asyncio.sleep(0.02) + assert service.cleanup_debt(session.session_id) is None + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_cleanup_is_bounded_then_recorded_abandoned(tmp_path): + standin = StandinLiveKit() + standin.delete_errors = [503] * 50 # never succeeds + now = [1000.0] + service = _service(standin, clock=lambda: now[0], + state_path=tmp_path / "voice-sessions.json", + cleanup_max_attempts=3) + try: + session = await service.open_session(epoch=1) + now[0] += service.config.session_ttl_seconds + 1 + with pytest.raises(Exception): + await service.end_session(session.session_id) + for _ in range(200): + debt = service.cleanup_debt(session.session_id) + if debt is not None and debt.abandoned: + break + await asyncio.sleep(0.02) + debt = service.cleanup_debt(session.session_id) + assert debt is not None and debt.abandoned == "attempts-exhausted" + # Bounded: after abandonment no further delete attempts happen. + attempts = len(standin.delete_errors) + await asyncio.sleep(0.2) + assert len(standin.delete_errors) == attempts + recorded = json.loads((tmp_path / "voice-sessions.json").read_text()) + assert recorded["debts"][0]["abandoned"] == "attempts-exhausted" + finally: + await service.aclose() + + +@pytest.mark.asyncio +async def test_restart_compensation_deletes_previous_process_rooms(tmp_path): + standin = StandinLiveKit() + standin.delete_errors = [RuntimeError("down")] * 2 # end attempt + shutdown sweep + state = tmp_path / "voice-sessions.json" + now = [1000.0] + # big backoff: the background sweeper must not race this scenario + first = _service(standin, clock=lambda: now[0], state_path=state, backoff=3600.0) + session = await first.open_session(epoch=1) + with pytest.raises(RuntimeError): + await first.end_session(session.session_id) + await first.aclose() + assert standin.deleted == [] # LiveKit was down for process 1's whole life + persisted = json.loads(state.read_text()) + assert any(d["session"]["room"] == session.room for d in persisted["debts"]) + # "Restart": a new process of the same instance reads the same ledger and + # compensates only the rooms recorded there. + second = _service(standin, clock=lambda: now[0], state_path=state) + try: + await second.recover() + for _ in range(100): + if standin.deleted: + break + await asyncio.sleep(0.02) + assert standin.deleted == [session.room] + for _ in range(100): + settled = json.loads(state.read_text()) + if settled["debts"] == [] and settled["sessions"] == []: + break + await asyncio.sleep(0.02) + settled = json.loads(state.read_text()) + assert settled["debts"] == [] and settled["sessions"] == [] + finally: + await second.aclose() + + +@pytest.mark.asyncio +async def test_active_room_surives_restart_until_ttl_then_compensated(tmp_path): + standin = StandinLiveKit() + state = tmp_path / "voice-sessions.json" + now = [1000.0] + first = _service(standin, clock=lambda: now[0], state_path=state, + session_ttl_seconds=0.5) + session = await first.open_session(epoch=1) # never ended by the user + await first.aclose() # graceful shutdown must not hang up a live call + assert standin.deleted == [] + persisted = json.loads(state.read_text()) + assert any(s["room"] == session.room for s in persisted["sessions"]) + # Restart immediately: the call may still be alive on the SFU, so the + # compensation waits for the session's own TTL — then deletes, bounded. + second = _service(standin, clock=lambda: now[0], state_path=state, + session_ttl_seconds=0.5) + try: + await second.recover() + await asyncio.sleep(0.15) + assert standin.deleted == [] # still within TTL: untouched + now[0] += 1.0 + for _ in range(100): + if standin.deleted: + break + await asyncio.sleep(0.05) + assert standin.deleted == [session.room] + finally: + await second.aclose() + + +# ------------------------------------------------------ router-level semantics + +def _voice_app(standin: StandinLiveKit, tts_ok=True): + provider = R2LLM() + router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}) + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.host == "tts.test": + return httpx.Response(200 if tts_ok else 503) + return standin._handle(request) + + service = _service(standin, backoff=3600.0) # keep the sweep out of the race + service.http = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + app = create_app(db_path=":memory:", router=router, + life_config=LifeConfig(enabled=False), semantic_mode="stub", + voice_service=service) + return app, service + + +def test_router_delete_failure_is_503_and_keeps_room_owned(): + from fastapi.testclient import TestClient + + standin = StandinLiveKit() + # interactive attempt + the background sweep's immediate first retry; + # the sweep then backs off (3600 s), leaving the debt visibly owned + standin.delete_errors = [RuntimeError("livekit down"), RuntimeError("livekit down")] + app, service = _voice_app(standin) + with TestClient(app) as client: + client.post("/api/companion", json={ + "name": "语音重试", "birth": "2000-10-18", "hour": "亥时", "relation": "朋友"}) + opened = client.post("/api/voice/session") + assert opened.status_code == 201 + session_id = opened.json()["session_id"] + failed = client.delete(f"/api/voice/session/{session_id}") + assert failed.status_code == 503 # NOT 404: the room is still owned + assert service.cleanup_debt(session_id) is not None + retried = client.delete(f"/api/voice/session/{session_id}") + assert retried.status_code == 200 and retried.json()["ended"] is True + assert standin.deleted == [opened.json()["room"]] + again = client.delete(f"/api/voice/session/{session_id}") + assert again.status_code == 404 + + +# ---------------------------------------------------- dual-listener lifecycle + +def _free_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +@pytest.fixture +def self_signed_cert(tmp_path): + cert = tmp_path / "cert.pem" + key = tmp_path / "key.pem" + subprocess.run( + ["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", + "-keyout", str(key), "-out", str(cert), "-days", "1", + "-subj", "/CN=127.0.0.1", + "-addext", "subjectAltName=IP:127.0.0.1"], + check=True, capture_output=True) + return str(cert), str(key) + + +@pytest.mark.asyncio +async def test_dual_listeners_share_one_lifecycle_and_close_in_order( + tmp_path, self_signed_cert, monkeypatch): + sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "deploy" / "voice")) + import serve_dual_8360_8361 as launcher + + cert, key = self_signed_cert + events: list[str] = [] + store_inits: list[int] = [] + real_store_init = Store.__init__ + real_store_close = Store.close + + def counting_init(self, *args, **kwargs): + store_inits.append(id(self)) + real_store_init(self, *args, **kwargs) + + def recording_close(self): + events.append("store_close") + real_store_close(self) + + monkeypatch.setattr(Store, "__init__", counting_init) + monkeypatch.setattr(Store, "close", recording_close) + + import uvicorn + real_shutdown = uvicorn.Server.shutdown + + async def recording_shutdown(self, sockets=None): + events.append(f"listener_shutdown:{self.config.port}") + await real_shutdown(self, sockets=sockets) + + monkeypatch.setattr(uvicorn.Server, "shutdown", recording_shutdown) + + provider = R2LLM() + router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}) + app = create_app(db_path=tmp_path / "dual.db", router=router, + life_config=LifeConfig(enabled=False), semantic_mode="off") + http_port, https_port = _free_port(), _free_port() + serve_task = asyncio.get_running_loop().create_task( + launcher.serve(http_port, https_port, cert, key, app=app)) + + client = httpx.AsyncClient(timeout=5.0, verify=False) + try: + epochs = [] + for _ in range(100): + try: + for port, scheme in ((http_port, "http"), (https_port, "https")): + response = await client.get(f"{scheme}://127.0.0.1:{port}/api/health") + if response.status_code == 200: + epochs.append(response.json()["epoch"]) + if len(epochs) >= 2: + break + except (httpx.ConnectError, httpx.ConnectTimeout, ssl.SSLError): + await asyncio.sleep(0.05) + assert len(epochs) >= 2, "both listeners must come up" + # ONE lifecycle for both listeners: a second entry would be epoch 2. + assert epochs[0] == epochs[1] + assert len(set(store_inits)) == 1, "the app must be initialized exactly once" + finally: + serve_task.cancel() + try: + await serve_task + except asyncio.CancelledError: + pass + await client.aclose() + + # Exit boundary: both listeners fully shut down BEFORE the store closes, + # and the store closes exactly once (never closed under live listeners). + shutdowns = [e for e in events if e.startswith("listener_shutdown")] + assert len(shutdowns) == 2, events + assert events.index("store_close") > max(events.index(e) for e in shutdowns) + assert events.count("store_close") == 1 + assert len(set(store_inits)) == 1 + + +def test_second_lifespan_entry_fails_loudly(tmp_path): + from fastapi.testclient import TestClient + + provider = R2LLM() + router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}) + app = create_app(db_path=tmp_path / "guard.db", router=router, + life_config=LifeConfig(enabled=False), semantic_mode="off") + + async def _double_enter(): + async with app.router.lifespan_context(app): + pass + + with TestClient(app) as client: + epoch_before = client.get("/api/health").json()["epoch"] + # A second entry (the old dual-uvicorn defect: epoch 1→2, Store/life/ + # chat_lock silently replaced) must now fail closed. + with pytest.raises(RuntimeError, match="entered twice"): + asyncio.run(_double_enter()) + assert client.get("/api/health").json()["epoch"] == epoch_before diff --git a/tools/release_snapshot_hel896.py b/tools/release_snapshot_hel896.py index 08faa99..2458124 100644 --- a/tools/release_snapshot_hel896.py +++ b/tools/release_snapshot_hel896.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 """HEL-879 · 发布归档一致性快照与回退证据(HEL-896 有限收口版;不冒充 R6 完整备份)。 -HEL-896 工具的总管独立复审发现四个失败边界,本版全部收口: +HEL-896 工具的总管独立复审发现四个失败边界,v2 全部收口;v3 收口总管对 v2 的 +独立复核(HEL-879 edges): 1. checkpoint 返回值必须核对:`PRAGMA wal_checkpoint(TRUNCATE)` 返回 (busy, log, checkpointed),busy≠0 即未完成——旧版照抄复制并在 manifest 记 rows_match=False - 仍打包成功。本版 busy≠0 / 源表行数≠快照行数一律**明确失败**:CLI 非 0、不产生 + 仍打包成功。busy≠0 / 源表行数≠快照行数一律**明确失败**:CLI 非 0、不产生 可交付归档(tar 不生成,半成品目录清除)。「先停服」是操作前提,不是忽略实际 失败结果的理由。 2. 目录内真实 SQLite 走一致性路径:history_root/vector_root 等数据目录里的 @@ -13,29 +14,35 @@ HEL-896 工具的总管独立复审发现四个失败边界,本版全部收口 (`semantic.py` 的 history_db_path 就在 history_root 内)。停服口径对它们逐个 checkpoint+复制+逐表核对;**在线口径只允许显式单库 --db(backup API),传入含 SQLite 的目录直接拒绝**——在线目录裸复制正是丢 WAL 的不安全口径。 -3. verify 拒绝 rows_match=False 的旧包,并按源路径映射做回退演练:解包到隔离新 - 目录、实际打开每个库核对行数与哈希(对照 manifest 记录的源行数与快照行数, - 两者必须相等且与解包实测一致),非只打开 tar 中快照自证;输出 rollback_plan - (源路径 → 解包恢复路径)。 -4. 完整实例归档只支持停服/释放全部写入方;在线模式仅显式单库 backup API。 +3. verify 拒绝 rows_match=False/缺失的旧包;三个计数(manifest 源行数、manifest + 快照行数、解包实测)必须直接相等——v2 的 `rows_match is True or ...` 允许记录 + 标记绕过源计数比较(源 2/快照 1 + rows_match=true 仍 ok,总管实测)。回退演练 + 按源路径映射**实际恢复**到归档旁的持久独立目标树并打开恢复库核行数与哈希, + 输出 rollback_plan(源路径 → 真实恢复路径)。 +4. 归档侧同名覆盖:不同目录的同名 `ledger.db` 在 v2 都写入 `databases/ledger.db`, + 后写覆盖前写、快照先交成功包再由 verify 发现。v3 对归档目标名做无冲突映射 + (同名源追加解析路径摘要),同一源重复传入直接拒绝。 + +完整实例归档只支持停服/释放全部写入方;在线模式仅显式单库 backup API。 产物:`//` 下的 `databases/`(一致性快照库,含目录内发现的 SQLite)、 `files/`(其余数据文件原样)、`MANIFEST.json`(逐文件 SHA256、源路径、代码精确 提交、逐表行数核对结果)、`.tar.gz`。归档永不删除;本工具不实现 R6。 -自测(`--selftest`,临时目录,零产品数据)复现:旧版失败形状(busy checkpoint 仍 -打包、rows_match=False 旧包 verify 仍 ok、在线目录含 SQLite)与本版修复,输出 -`docs/rework/hel879/release-snapshot/selftest.json`。 +自测(`--selftest`,临时目录,零产品数据)复现:全部旧失败形状(busy checkpoint +仍打包、rows_match=False 旧包 verify 仍 ok、在线目录含 SQLite、v2 的 OR 绕过与 +同名覆盖)与本版修复,输出 `docs/rework/hel879/release-snapshot/selftest.json`。 用法: python3 tools/release_snapshot_hel896.py --selftest [outDir] python3 tools/release_snapshot_hel896.py snapshot --name hel8xx-release \ --db path/a.db --db path/b.sqlite3 --file path/vecdir --file path/hist \ - --out archives/ # 停服口径(推荐,目录内 SQLite 一并一致性处理) + --out archives/ # 停服口径(推荐,目录内 SQLite 一并一致性处理) python3 tools/release_snapshot_hel896.py snapshot --name ... --db path/a.db \ - --out archives/ --online # 在线口径:仅显式单库 backup API,含 SQLite 的 - # --file 目录会被拒绝 - python3 tools/release_snapshot_hel896.py verify --archive archives/hel8xx-release.tar.gz + --out archives/ --online # 在线口径:仅显式单库 backup API,含 SQLite 的 + # --file 目录会被拒绝 + python3 tools/release_snapshot_hel896.py verify --archive archives/hel8xx-release.tar.gz \ + [--restore-dir DIR] # 回退演练默认恢复到归档旁 verify-restore--<时间>/ """ from __future__ import annotations @@ -53,7 +60,7 @@ from pathlib import Path REPO = Path(__file__).resolve().parents[1] SQLITE_MAGIC = b"SQLite format 3\x00" -TOOL_VERSION = 2 +TOOL_VERSION = 3 class SnapshotError(RuntimeError): @@ -132,26 +139,52 @@ def git_head() -> str: return "unknown" +def _unique_name(taken: set[str], display: str, key: str) -> str: + """Archive-side name that cannot collide: two different source DBs with + the same basename must never overwrite each other in `databases/` + (HEL-879 总管复核第 4 项). Same-basename sources get a short digest of + their resolved source path appended.""" + if display not in taken: + taken.add(display) + return display + digest = hashlib.sha256(key.encode()).hexdigest()[:8] + stem, suffix = Path(display).stem, Path(display).suffix + unique = f"{stem}__{digest}{suffix}" + while unique in taken: + unique = f"{stem}__{digest}{len(taken)}{suffix}" + taken.add(unique) + return unique + + def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, *, online: bool) -> dict: """做一致性快照。任一环节失败:不产生 tar、清除半成品、抛 SnapshotError。 收口语义(HEL-879): - 全部 --db 与目录内发现的 SQLite 都走一致性路径并逐表核对源/快照行数; - 行数不一致或 checkpoint busy = 明确失败(旧版记 rows_match=False 仍打包); - - 在线模式含 SQLite 的 --file 目录直接拒绝(在线目录裸复制丢 WAL)。 + - 在线模式含 SQLite 的 --file 目录直接拒绝(在线目录裸复制丢 WAL); + - 同一源重复传入或归档侧目标名冲突 = 明确失败/唯一化(同名 ledger.db + 后写覆盖前写、快照先报成功包再由 verify 发现——旧版真实缺陷)。 """ + resolved = [str(Path(db).resolve()) for db in dbs] + duplicates = sorted({p for p in resolved if resolved.count(p) > 1}) + if duplicates: + raise SnapshotError(f"--db 同一源重复传入,拒绝:{duplicates}") stamp = time.strftime("%Y%m%d-%H%M%S") root = out_dir / f"{name}-{stamp}" manifest: dict = {"name": name, "created": stamp, "code_commit": git_head(), "tool": "release_snapshot_hel896.py", "tool_version": TOOL_VERSION, "mode": "online-backup-api" if online else "stop-checkpoint-copy", "databases": [], "files": [], "verify": "not-run"} + taken_db_names: set[str] = set() + taken_file_names: set[str] = set() try: (root / "databases").mkdir(parents=True, exist_ok=True) for db in dbs: if not Path(db).is_file(): raise SnapshotError(f"--db 不是文件:{db}") - dst = root / "databases" / db.name + dst = root / "databases" / _unique_name( + taken_db_names, Path(db).name, str(Path(db).resolve())) method = snapshot_db(Path(db), dst, online=online) src_counts = table_counts(Path(db)) dst_counts = table_counts(dst) @@ -176,7 +209,9 @@ def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, for emb in embedded: # 停服口径:目录内真实 SQLite 一并走 checkpoint+复制+逐表核对。 rel = emb.relative_to(src) if emb.is_relative_to(src) else Path(emb.name) - dst = root / "databases" / f"{src.name}__{rel}".replace("/", "__") + display = f"{src.name}__{rel}".replace("/", "__") + dst = root / "databases" / _unique_name( + taken_db_names, display, str(emb.resolve())) method = snapshot_db(emb, dst, online=False) src_counts = table_counts(emb) dst_counts = table_counts(dst) @@ -189,7 +224,7 @@ def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, "source_table_counts": src_counts, "snapshot_table_counts": dst_counts, "rows_match": True}) - dst = root / "files" / src.name + dst = root / "files" / _unique_name(taken_file_names, src.name, str(src.resolve())) if src.is_dir(): shutil.copytree(src, dst) entries = {str(p.relative_to(dst)): sha256(p) for p in sorted(dst.rglob("*")) if p.is_file()} @@ -216,15 +251,29 @@ def make_snapshot(name: str, dbs: list[Path], files: list[Path], out_dir: Path, raise -def verify_archive(archive: Path) -> dict: - """回退演练:隔离新目录解包,实际打开各库核对行数与哈希,按源路径映射。 +def _restore_target(restore_root: Path, source: str) -> Path: + """Map a recorded source path into the isolated restore tree, preserving + the full source hierarchy so same-basename DBs can never collide there.""" + sanitized = source.strip("/").replace("/", "__") + return restore_root / "by-source" / sanitized + + +def verify_archive(archive: Path, restore_root: Path | None = None) -> dict: + """回退演练:隔离新目录解包,按源路径映射**实际恢复**到独立目标树, + 打开恢复出的每个库核对行数与哈希(不是只打开 tar 内布局自证)。 收口语义(HEL-879): - - manifest 记录 rows_match=False(旧版失败包)→ ok=False 拒绝; - - 行数同时对照 source_table_counts 与 snapshot_table_counts:两者必须相等, - 且与解包实测一致(旧版只比快照自证,源=1 行/快照=0 行的漏行包仍 ok)。 + - 三个计数必须直接相等:manifest 源行数 = manifest 快照行数 = 解包实测; + `rows_match` 只是额外约束,缺失或矛盾明确拒绝(旧版 OR 逻辑允许记录值 + 绕过源计数比较); + - 恢复目标树默认落在归档旁的持久目录,rollback_plan 指向真实存在的恢复 + 文件(旧版输出随 TemporaryDirectory 消失的路径)。 """ - result: dict = {"archive": str(archive), "sha256": sha256(archive), "checks": []} + archive = Path(archive) + if restore_root is None: + restore_root = archive.parent / f"verify-restore-{archive.stem}-{time.strftime('%Y%m%d-%H%M%S')}" + result: dict = {"archive": str(archive), "sha256": sha256(archive), + "restore_root": str(restore_root), "checks": []} with tempfile.TemporaryDirectory(prefix="hel879-verify-") as tmp: with tarfile.open(archive) as tf: tf.extractall(tmp) @@ -237,39 +286,56 @@ def verify_archive(archive: Path) -> dict: rollback_plan = [] for d in manifest["databases"]: snap = root / d["snapshot"] - digest = sha256(snap) + # The actual restore step: copy into the isolated target tree at + # the recorded source path's mapped location, then verify THERE. + restored = _restore_target(restore_root, str(d.get("source") or d["snapshot"])) + restored.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(snap, restored) + digest = sha256(restored) try: - counts = table_counts(snap) + counts = table_counts(restored) except sqlite3.Error as exc: - result["checks"].append({"db": d["snapshot"], "sha256_ok": digest == d["sha256"], - "rows_ok": False, "error": str(exc), + result["checks"].append({"db": d["snapshot"], "restored": str(restored), + "sha256_ok": False, "rows_ok": False, "error": str(exc), "rows_match_recorded": d.get("rows_match")}) ok_all = False continue recorded_rows_match = d.get("rows_match") src_counts = d.get("source_table_counts") snap_counts = d.get("snapshot_table_counts") - rows_ok = counts == snap_counts and ( - recorded_rows_match is True or (src_counts is not None and counts == src_counts)) - legacy_reject = recorded_rows_match is False or ( - recorded_rows_match is None and src_counts != snap_counts) + # Three-way equality is the only passing shape; the recorded flag + # is an extra constraint, never a substitute for the comparison. + counts_agree = ( + isinstance(src_counts, dict) and isinstance(snap_counts, dict) + and counts == src_counts and counts == snap_counts + ) + rows_ok = counts_agree + legacy_reject = recorded_rows_match is not True or not counts_agree ok = digest == d["sha256"] and rows_ok and not legacy_reject ok_all &= ok - result["checks"].append({"db": d["snapshot"], "sha256_ok": digest == d["sha256"], + result["checks"].append({"db": d["snapshot"], "restored": str(restored), + "sha256_ok": digest == d["sha256"], "rows_ok": rows_ok, "counts": counts, "source_counts": src_counts, + "snapshot_counts": snap_counts, "rows_match_recorded": recorded_rows_match, "rejected_legacy_package": legacy_reject}) - rollback_plan.append({"source": d.get("source"), "restored_path": str(snap), - "restore_action": "停服后复制回源路径(行数与哈希已在本演练核对)"}) + rollback_plan.append({"source": d.get("source"), "restored_path": str(restored), + "restore_action": "停服后复制回源路径(行数与哈希已在恢复树实际核对)"}) for f in manifest["files"]: base = root / f["snapshot"] + restored_dir = _restore_target(restore_root, str(f.get("source") or f["snapshot"])) + restored_dir.parent.mkdir(parents=True, exist_ok=True) + if base.is_dir(): + shutil.copytree(base, restored_dir, dirs_exist_ok=True) + else: + shutil.copy2(base, restored_dir) for rel, want in f["entries"].items(): - p = (base / rel) if base.is_dir() else base + p = (restored_dir / rel) if restored_dir.is_dir() else restored_dir ok = sha256(p) == want ok_all &= ok - result["checks"].append({"file": str(p.relative_to(root)), "sha256_ok": ok}) - rollback_plan.append({"source": f.get("source"), "restored_path": str(base), + result["checks"].append({"file": rel, "restored": str(p), "sha256_ok": ok}) + rollback_plan.append({"source": f.get("source"), "restored_path": str(restored_dir), "restore_action": "停服后按相对结构复制回源目录"}) result["rollback_plan"] = rollback_plan result["ok"] = ok_all @@ -476,6 +542,67 @@ def selftest(out_dir: Path) -> int: checks.append({"name": "backup_api_snapshot_complete", "ok": api_counts.get("companion") == 1 and api_counts.get("chat_message") == 1, "detail": json.dumps(api_counts)}) + + # ── 总管对 v2 的 edges(HEL-879 独立复核)── + # Edge A:manifest 记 rows_match=true 但源计数被改为 events=2(快照=1)。 + # v2 的 `rows_match is True or ...` 绕过源计数比较 → 仍 ok;v3 必须拒绝。 + clean = make_snapshot("edge-counts", [db], [], out, online=False) + edge_unpack = out / "edge-unpack" + with tarfile.open(clean["archive"]) as tf: + tf.extractall(edge_unpack) + edge_pkg = next(edge_unpack.iterdir()) + edge_manifest_path = edge_pkg / "MANIFEST.json" + edge_manifest = json.loads(edge_manifest_path.read_text(encoding="utf-8")) + edge_manifest["databases"][0]["source_table_counts"]["events"] = 2 + edge_manifest_path.write_text(json.dumps(edge_manifest, ensure_ascii=False)) + edge_tar = tmp / "edge-counts-tampered.tar.gz" + with tarfile.open(edge_tar, "w:gz") as tf: + tf.add(edge_pkg, arcname=edge_pkg.name) + edge_verify = verify_archive(edge_tar, restore_root=tmp / "edge-restore") + checks.append({"name": "fix_rows_match_true_cannot_bypass_source_counts", + "ok": edge_verify["ok"] is False and all( + c.get("rejected_legacy_package") for c in edge_verify["checks"] + if "rejected_legacy_package" in c), + "detail": f"ok={edge_verify['ok']} " + f"counts={edge_verify['checks'][0].get('counts')} " + f"source={edge_verify['checks'][0].get('source_counts')}"}) + + # Edge B:两份不同目录的同名 ledger.db。v2 后写覆盖前写、快照仍交成功包; + # v3 无冲突映射 + verify 全过 + 恢复树里两个源路径各自落位。 + la = tmp / "ledger-a" / "ledger.db" + lb = tmp / "ledger-b" / "ledger.db" + for path, value in [(la, "first"), (lb, "second")]: + path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(str(path)) as conn: + conn.execute("CREATE TABLE item(value TEXT)") + conn.execute("INSERT INTO item VALUES(?)", (value,)) + same = make_snapshot("same-basename", [la, lb], [], out, online=False) + snap_names = [Path(d["snapshot"]).name for d in same["databases"]] + same_restore = tmp / "same-restore" + same_verify = verify_archive(Path(same["archive"]), restore_root=same_restore) + plan = same_verify.get("rollback_plan", []) + restored_paths = [e["restored_path"] for e in plan if e.get("source", "").endswith("ledger.db")] + distinct = len(set(restored_paths)) == 2 and all( + Path(p).is_file() for p in restored_paths) + rows = sorted(sqlite3.connect(f"file:{p}?mode=ro", uri=True) + .execute("SELECT value FROM item").fetchone()[0] + for p in restored_paths) + dup_failed = None + try: + make_snapshot("dup-source", [la, la], [], out, online=False) + except SnapshotError as exc: + dup_failed = str(exc) + checks.append({"name": "fix_same_basename_no_overwrite_and_duplicate_refused", + "ok": (len(set(snap_names)) == 2 and same_verify["ok"] is True + and distinct and rows == ["first", "second"] + and dup_failed is not None), + "detail": f"snap_names={snap_names} verify_ok={same_verify['ok']} " + f"restored_rows={rows} dup_refused={'yes' if dup_failed else 'no'}"}) + # 恢复路径必须真实存在于恢复树(不是随 TemporaryDirectory 消失的路径)。 + checks.append({"name": "restore_drill_paths_persist", + "ok": all(Path(p).is_file() for p in restored_paths) + and same_restore.is_dir(), + "detail": f"restore_root={same_restore}"}) ok = all(c["ok"] for c in checks) out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "selftest.json").write_text( @@ -499,6 +626,8 @@ def main() -> int: ap.add_argument("--online", action="store_true", help="在线口径:仅显式单库 backup API(默认停服+checkpoint;含 SQLite 的 --file 目录会被拒绝)") ap.add_argument("--archive", type=Path) + ap.add_argument("--restore-dir", type=Path, + help="回退演练的恢复目标树(默认:归档旁 verify-restore--<时间>/,持久保留供回滚使用)") args = ap.parse_args() if args.selftest: return selftest(REPO / "docs/rework/hel879/release-snapshot") @@ -519,7 +648,7 @@ def main() -> int: if args.command == "verify": if not args.archive: ap.error("verify 需要 --archive") - result = verify_archive(args.archive) + result = verify_archive(args.archive, restore_root=args.restore_dir) print(json.dumps(result, ensure_ascii=False, indent=1)) return 0 if result["ok"] else 1 ap.print_help()