313 lines
14 KiB
Python
313 lines
14 KiB
Python
"""HEL-889 rework regressions: the manager's independent-review edge failures.
|
|
|
|
Two defects against the real product path (review_hel886_edges.py):
|
|
|
|
P1 — the shutdown drain TIMEOUT closed resources under a live worker:
|
|
``aclose`` gave up after ``drain_timeout``, the backend closed, the app closed
|
|
the store, and the still-running worker died on ``Cannot operate on a closed
|
|
database`` with its durable writes lost (reopened: pending / row_done 0 /
|
|
llm_used 1).
|
|
|
|
P2 — cancelling a tick whose worker was still QUEUED left a phantom registry
|
|
entry: ``_run`` (and its finally) never ran, so the batch stayed pending
|
|
forever with ``_live=1`` and only a restart could clear it.
|
|
|
|
These cases drive the REAL product path with barrier-blocked workers and a
|
|
single-worker executor (the manager's exact congestion shape), not preseeded
|
|
dead leases. The pinned real install is covered live by
|
|
tools/verify_shutdown_queued_hel889.py.
|
|
"""
|
|
import asyncio
|
|
import concurrent.futures
|
|
import json
|
|
import threading
|
|
import time
|
|
from contextlib import suppress
|
|
|
|
from test_mem0_backend import _make_ledger
|
|
from test_rework_hel885 import seed_public_two_row_turn
|
|
from test_rework_hel886 import batch_row, install_blocking, seed_user_row
|
|
from test_semantic_recovery_hel882 import config_for, install_fake
|
|
|
|
from cybergril.memory.semantic import Mem0Backend, SemanticMemory
|
|
from cybergril.memory.store import Store
|
|
|
|
|
|
def make_backend(tmp_path, name, *, embed_cap=50, llm_cap=10):
|
|
"""(backend, config): the SAME ledger/config must be reused for the
|
|
restart step — a second ledger file would fail the frozen-prefix check."""
|
|
ledger = tmp_path / f"{name}.sqlite3"
|
|
_make_ledger(ledger)
|
|
cfg = config_for(tmp_path, ledger, embed_cap=embed_cap, llm_cap=llm_cap)
|
|
return Mem0Backend(cfg), cfg
|
|
|
|
|
|
def single_worker_executor():
|
|
"""The manager's congestion shape: one worker thread, occupied by an
|
|
unrelated blocker, so the batch worker sits QUEUED behind it."""
|
|
return concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
|
|
|
|
|
async def wait_until(predicate, timeout=5.0):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return True
|
|
await asyncio.sleep(0.01) # never block the loop the tick task runs on
|
|
return False
|
|
|
|
|
|
async def test_queued_cancel_releases_and_recovers(tmp_path, monkeypatch):
|
|
"""P2 exact shape: the executor is busy, the worker is QUEUED (never
|
|
started); cancelling the tick must release the phantom registry entry and
|
|
return the batch to the queue — recover + tick then finish it in-process,
|
|
no restart, exactly one send."""
|
|
install_fake(monkeypatch)
|
|
backend, _cfg = make_backend(tmp_path, "queued")
|
|
store = Store(tmp_path / "queued.db")
|
|
seed_user_row(store, "t-queued", "排队取消恢复的合成原话")
|
|
clock = [1000.0]
|
|
executor = single_worker_executor()
|
|
entered, release = threading.Event(), threading.Event()
|
|
blocker = executor.submit(lambda: (entered.set(), release.wait(8)))
|
|
assert entered.wait(5)
|
|
live = SemanticMemory(store, backend, clock=lambda: clock[0], executor=executor)
|
|
live.enqueue_uncovered()
|
|
task = asyncio.create_task(live.tick())
|
|
# The batch is claimed and its worker sits in the executor queue.
|
|
assert await wait_until(lambda: live._live
|
|
and all(not h.started.is_set() for h in live._live.values()))
|
|
handle = next(iter(live._live.values()))
|
|
assert handle.future is not None and not handle.future.done()
|
|
|
|
task.cancel() # cancels the QUEUED work item — _run never runs
|
|
with suppress(asyncio.CancelledError):
|
|
await task
|
|
assert not live._live # the phantom entry is gone immediately
|
|
after = batch_row(store)
|
|
assert after["state"] == "queued" and after["owner"] is None
|
|
assert store.query_one("SELECT attempts FROM sem_batch")["attempts"] == 0
|
|
assert len(backend._memory.llm.calls) == 0 # nothing was ever sent
|
|
|
|
release.set()
|
|
blocker.result(timeout=5)
|
|
clock[0] += 41 # the manager's lease-expiry probe
|
|
live.recover() # nothing to fail: queued, not abandoned-pending
|
|
assert batch_row(store)["state"] == "queued"
|
|
await live.tick() # in-process recovery, no restart
|
|
final = batch_row(store)
|
|
rows = store.query("SELECT * FROM sem_row_done")
|
|
assert final["state"] == "done" and final["reason"] is None
|
|
assert len(backend._memory.llm.calls) == 1 # exactly one send, ever
|
|
assert len(rows) == 1 and json.loads(final["entry_ids"]) == json.loads(rows[0]["entry_ids"])
|
|
await live.aclose()
|
|
store.close()
|
|
executor.shutdown(wait=True)
|
|
|
|
|
|
async def test_shutdown_timeout_never_closes_under_live_worker(tmp_path, monkeypatch):
|
|
"""P1 exact shape: the first HTTP is accepted but unanswered; the drain
|
|
deadline (0.05s, the same branch as the default 120s) passes while the
|
|
worker is STILL alive. aclose must NOT return-and-close: the worker
|
|
finishes, flushes its durable writes, and only then resources close —
|
|
app-order store.close() afterwards is safe and nothing is lost."""
|
|
arrived, release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, release)
|
|
backend, _cfg = make_backend(tmp_path, "shut")
|
|
store = Store(tmp_path / "shut.db")
|
|
seed_user_row(store, "t-shut", "关闭超时排空的合成原话")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
task = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5)
|
|
|
|
closing = asyncio.create_task(live.aclose(drain_timeout=0.05))
|
|
await asyncio.sleep(0.3) # the deadline is long past
|
|
assert not closing.done() # a live worker is never treated as dead
|
|
assert batch_row(store)["state"] == "pending"
|
|
assert not backend._client.closed # nothing closed under it
|
|
|
|
release.set() # the wire answer arrives
|
|
await closing
|
|
final = batch_row(store)
|
|
rows = store.query("SELECT * FROM sem_row_done")
|
|
assert final["state"] == "done" and len(rows) == 1 # flushed BEFORE close
|
|
assert backend._client.closed # closed only after the drain
|
|
with suppress(asyncio.CancelledError, Exception):
|
|
await task
|
|
store.close() # exact app lifespan order — now safe
|
|
check = Store(tmp_path / "shut.db") # durable, no lost-write window
|
|
durable = check.query_one("SELECT state,entry_ids FROM sem_batch")
|
|
assert durable["state"] == "done" and json.loads(durable["entry_ids"])
|
|
assert len(check.query("SELECT * FROM sem_row_done")) == 1
|
|
check.close()
|
|
|
|
|
|
async def test_shutdown_between_rows_requeues_and_resume_sends_only_missing(
|
|
tmp_path, monkeypatch):
|
|
"""aclose arriving BETWEEN rows: finished rows keep their durable proofs,
|
|
the unsent remainder returns to the queue, and the next process sends ONLY
|
|
the missing row — never a resend of the proven one."""
|
|
arrived, release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, release)
|
|
backend, cfg = make_backend(tmp_path, "between")
|
|
db = tmp_path / "between.db"
|
|
store = Store(db)
|
|
seed_public_two_row_turn(store, "t-between", "行间关闭第一行", "行间关闭第二行")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
task = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5) # row 1 on the wire
|
|
|
|
closing = asyncio.create_task(live.aclose(drain_timeout=0.05))
|
|
await asyncio.sleep(0.2) # close signal set while row 1 is in flight
|
|
release.set()
|
|
await closing
|
|
after = batch_row(store)
|
|
assert after["state"] == "queued" and after["owner"] is None
|
|
assert "shutdown" in (after["reason"] or "")
|
|
assert len(store.query("SELECT * FROM sem_row_done")) == 1 # row 1 proven
|
|
with suppress(asyncio.CancelledError, Exception):
|
|
await task
|
|
store.close()
|
|
|
|
# next process: only the unproven row is sent (same ledger, same caps)
|
|
backend2 = Mem0Backend(cfg)
|
|
store2 = Store(db)
|
|
live2 = SemanticMemory(store2, backend2)
|
|
calls_before = len(backend2._memory.llm.calls)
|
|
await live2.tick()
|
|
final = batch_row(store2)
|
|
assert final["state"] == "done" and final["reason"] is None
|
|
assert len(backend2._memory.llm.calls) - calls_before == 1 # only row 2
|
|
assert len(json.loads(final["entry_ids"])) == 2 # both rows' entries
|
|
assert len(store2.query("SELECT * FROM sem_row_done")) == 2
|
|
await live2.aclose()
|
|
store2.close()
|
|
|
|
|
|
async def test_shutdown_before_start_zero_sends_and_resume(tmp_path, monkeypatch):
|
|
"""aclose racing a QUEUED worker cancels it out of the executor: provably
|
|
nothing was sent, the attempt is restored, and the next process completes
|
|
the batch normally."""
|
|
install_fake(monkeypatch)
|
|
backend, cfg = make_backend(tmp_path, "before")
|
|
db = tmp_path / "before.db"
|
|
store = Store(db)
|
|
seed_user_row(store, "t-before", "未启动即关闭的合成原话")
|
|
executor = single_worker_executor()
|
|
entered, release = threading.Event(), threading.Event()
|
|
blocker = executor.submit(lambda: (entered.set(), release.wait(8)))
|
|
assert entered.wait(5)
|
|
live = SemanticMemory(store, backend, executor=executor)
|
|
live.enqueue_uncovered()
|
|
task = asyncio.create_task(live.tick())
|
|
assert await wait_until(lambda: live._live
|
|
and all(not h.started.is_set() for h in live._live.values()))
|
|
|
|
await live.aclose(drain_timeout=0.05) # cancels the queued worker outright
|
|
assert not live._live
|
|
after = batch_row(store)
|
|
assert after["state"] == "queued" and "shutdown" in (after["reason"] or "")
|
|
assert store.query_one("SELECT attempts FROM sem_batch")["attempts"] == 0
|
|
assert len(backend._memory.llm.calls) == 0 # provably zero sends
|
|
with suppress(asyncio.CancelledError, Exception):
|
|
await task
|
|
store.close()
|
|
release.set()
|
|
blocker.result(timeout=5)
|
|
|
|
backend2 = Mem0Backend(cfg) # same ledger/config, fresh instance
|
|
store2 = Store(db)
|
|
live2 = SemanticMemory(store2, backend2)
|
|
await live2.tick()
|
|
final = batch_row(store2)
|
|
assert final["state"] == "done"
|
|
assert len(backend2._memory.llm.calls) == 1
|
|
await live2.aclose()
|
|
store2.close()
|
|
executor.shutdown(wait=True)
|
|
|
|
|
|
async def test_cancelled_aclose_leaves_everything_open(tmp_path, monkeypatch):
|
|
"""A close cancelled mid-drain closes NOTHING: the closing signal is
|
|
re-armed, the worker keeps running, and a later aclose finishes the job."""
|
|
arrived, release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, release)
|
|
backend, _cfg = make_backend(tmp_path, "cancelclose")
|
|
store = Store(tmp_path / "cancelclose.db")
|
|
seed_user_row(store, "t-cc", "关闭被取消的合成原话")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
task = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5)
|
|
|
|
closing = asyncio.create_task(live.aclose(drain_timeout=0.05))
|
|
await asyncio.sleep(0.2) # past the deadline, still draining
|
|
closing.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await closing
|
|
assert not backend._client.closed # nothing half-closed
|
|
assert batch_row(store)["state"] == "pending" # worker still owns it
|
|
assert not live._closing.is_set() # the layer is re-armed
|
|
await live.tick() # and still usable, not frozen
|
|
|
|
release.set()
|
|
with suppress(asyncio.CancelledError, Exception):
|
|
await task
|
|
await live.aclose() # a later aclose finishes cleanly
|
|
assert backend._client.closed
|
|
assert batch_row(store)["state"] == "done"
|
|
store.close()
|
|
|
|
|
|
async def test_concurrent_aclose_serializes_and_is_idempotent(tmp_path, monkeypatch):
|
|
"""Two aclose callers while a worker is live: both complete, the drain
|
|
happens once, resources close once, and a third call returns immediately."""
|
|
arrived, release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, release)
|
|
backend, _cfg = make_backend(tmp_path, "concurrent")
|
|
store = Store(tmp_path / "concurrent.db")
|
|
seed_user_row(store, "t-conc", "并发关闭的合成原话")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
task = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5)
|
|
|
|
first = asyncio.create_task(live.aclose(drain_timeout=0.05))
|
|
second = asyncio.create_task(live.aclose(drain_timeout=5))
|
|
await asyncio.sleep(0.2)
|
|
release.set()
|
|
await asyncio.gather(first, second)
|
|
assert backend._client.closed
|
|
assert batch_row(store)["state"] == "done"
|
|
assert len(store.query("SELECT * FROM sem_row_done")) == 1
|
|
await live.aclose() # idempotent no-op
|
|
assert backend._client.closed
|
|
with suppress(asyncio.CancelledError, Exception):
|
|
await task
|
|
store.close()
|
|
|
|
|
|
async def test_tick_refuses_claims_after_close(tmp_path, monkeypatch):
|
|
"""After aclose the layer is closed: a newly enqueued batch is left
|
|
queued — shutdown never accepts work it would have to drain again."""
|
|
install_fake(monkeypatch)
|
|
backend, _cfg = make_backend(tmp_path, "after")
|
|
store = Store(tmp_path / "after.db")
|
|
seed_user_row(store, "t-after-1", "关闭后不再领取的第一句")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
await live.tick()
|
|
assert batch_row(store)["state"] == "done"
|
|
await live.aclose()
|
|
|
|
seed_user_row(store, "t-after-2", "关闭后不再领取的第二句")
|
|
live.enqueue_uncovered()
|
|
await live.tick() # closed layer: no new claims
|
|
second = store.query_one(
|
|
"SELECT state FROM sem_batch WHERE turn_id='t-after-2'")
|
|
assert second["state"] == "queued"
|
|
assert len(backend._memory.llm.calls) == 1 # only the pre-close batch ran
|
|
store.close()
|