232 lines
10 KiB
Python
232 lines
10 KiB
Python
"""HEL-894 rework regressions: close ownership under cancellation.
|
|
|
|
Two defects from the manager's independent pinned-SDK review
|
|
(review_hel889_cancel.py):
|
|
|
|
D1 — cancelling aclose AFTER ``backend.aclose`` began released the lock,
|
|
cleared ``closing`` and left the shielded close running unowned in the
|
|
background: the layer looked usable, a new tick claimed a batch onto real
|
|
HTTP, then the orphaned finish closed Qdrant under it (batch failed /
|
|
row_done=0 / llm_used=1 on ``QdrantLocal instance is closed``; with the
|
|
caller closing the store too, the late worker died on ``Cannot operate on a
|
|
closed database`` and the reopened batch sat pending).
|
|
|
|
D2 — cancelling a SECOND aclose that was still WAITING for the lock cleared
|
|
the FIRST closer's ``closing`` flag: a new batch ran to ``done`` while the
|
|
real drain still held the original worker.
|
|
|
|
The close is now two-phase (HEL-894): a drain-phase cancel re-arms the
|
|
layer; once ``backend.aclose`` begins the close is irreversible — markers
|
|
stay, ticks stay refused, the layer-owned close task lands exactly once
|
|
(repeated cancels included), and only the protocol owner may reset the
|
|
signal. The pinned real install is covered live by
|
|
tools/verify_close_ownership_hel894.py (manager negatives rerun).
|
|
"""
|
|
import asyncio
|
|
import threading
|
|
from contextlib import suppress
|
|
|
|
from test_rework_hel886 import batch_row, install_blocking, seed_user_row
|
|
from test_rework_hel889 import make_backend, wait_until
|
|
from test_semantic_recovery_hel882 import install_fake
|
|
|
|
from cybergril.memory.semantic import SemanticMemory
|
|
from cybergril.memory.store import Store
|
|
|
|
|
|
def gate_client_close(backend, entered, release):
|
|
"""Gate ONLY the SDK client boundary — the manager's exact shape: the
|
|
REAL product aclose path runs and the underlying close still executes,
|
|
scheduling is just held at the irreversible boundary."""
|
|
calls = {"count": 0}
|
|
real_close = backend._client.close
|
|
|
|
def gated_close(*args, **kwargs):
|
|
calls["count"] += 1
|
|
entered.set()
|
|
assert release.wait(10)
|
|
return real_close(*args, **kwargs)
|
|
|
|
backend._client.close = gated_close
|
|
return calls
|
|
|
|
|
|
def batch_row_for(store, turn_id):
|
|
return dict(store.query_one(
|
|
"SELECT state,reason,owner,attempts,entry_ids FROM sem_batch WHERE turn_id=?",
|
|
(turn_id,)))
|
|
|
|
|
|
async def test_finish_cancel_keeps_ownership_and_refuses_new_work(
|
|
tmp_path, monkeypatch):
|
|
"""D1: cancel aclose after ``backend.aclose`` began. The cancelled caller
|
|
must NOT report the layer usable: markers stay, the new batch is never
|
|
claimed (zero wire work, zero budget), and the owned close lands exactly
|
|
once before the cancellation propagates."""
|
|
install_fake(monkeypatch)
|
|
backend, _cfg = make_backend(tmp_path, "finish")
|
|
store = Store(tmp_path / "finish.db")
|
|
live = SemanticMemory(store, backend)
|
|
entered, release = threading.Event(), threading.Event()
|
|
close_calls = gate_client_close(backend, entered, release)
|
|
|
|
closing = asyncio.create_task(live.aclose())
|
|
assert await asyncio.to_thread(entered.wait, 5) # the close BEGAN
|
|
closing.cancel()
|
|
await asyncio.sleep(0.05) # cancel delivered into the owned-close tail
|
|
assert not closing.done() # still landing the close it started
|
|
assert live._closing.is_set() # never half-closed-usable
|
|
assert not live._closed and not backend._client.closed
|
|
|
|
seed_user_row(store, "t-finish", "提交阶段取消后不应被领取的合成原话")
|
|
live.enqueue_uncovered()
|
|
await live.tick() # refused: the close owns the layer
|
|
row = batch_row_for(store, "t-finish")
|
|
assert row["state"] == "queued" and row["owner"] is None and row["attempts"] == 0
|
|
assert len(backend._memory.llm.calls) == 0 # zero sends, zero budget
|
|
|
|
release.set()
|
|
with suppress(asyncio.CancelledError):
|
|
await closing # the close lands, THEN the cancellation propagates
|
|
assert live._closed and backend._client.closed
|
|
assert close_calls["count"] == 1 # closed exactly once
|
|
assert live._closing.is_set() # a finished close never re-arms
|
|
await live.tick() # a closed layer keeps refusing new claims
|
|
assert batch_row_for(store, "t-finish")["state"] == "queued"
|
|
store.close() # caller order stays safe — no worker was ever started
|
|
|
|
|
|
async def test_repeated_cancel_during_finish_lands_close_once(
|
|
tmp_path, monkeypatch):
|
|
"""Repeated cancels while the finish task is gated only re-shield the
|
|
same owned close: it still lands exactly once."""
|
|
install_fake(monkeypatch)
|
|
backend, _cfg = make_backend(tmp_path, "repeat")
|
|
store = Store(tmp_path / "repeat.db")
|
|
live = SemanticMemory(store, backend)
|
|
entered, release = threading.Event(), threading.Event()
|
|
close_calls = gate_client_close(backend, entered, release)
|
|
|
|
closing = asyncio.create_task(live.aclose())
|
|
assert await asyncio.to_thread(entered.wait, 5)
|
|
for _ in range(3):
|
|
closing.cancel()
|
|
await asyncio.sleep(0.02)
|
|
assert not closing.done()
|
|
assert live._closing.is_set() and not live._closed
|
|
|
|
release.set()
|
|
with suppress(asyncio.CancelledError):
|
|
await closing
|
|
assert live._closed and close_calls["count"] == 1
|
|
store.close()
|
|
|
|
|
|
async def test_second_close_during_finish_waits_and_cancels_cleanly(
|
|
tmp_path, monkeypatch):
|
|
"""Same-phase second/third callers: an uncancelled second aclose waits
|
|
for the owner's finish and returns after ONE close; a third cancelled
|
|
while still waiting for the lock touches nothing (D2 at the finish
|
|
phase)."""
|
|
install_fake(monkeypatch)
|
|
backend, _cfg = make_backend(tmp_path, "second")
|
|
store = Store(tmp_path / "second.db")
|
|
live = SemanticMemory(store, backend)
|
|
entered, release = threading.Event(), threading.Event()
|
|
close_calls = gate_client_close(backend, entered, release)
|
|
|
|
first = asyncio.create_task(live.aclose())
|
|
assert await asyncio.to_thread(entered.wait, 5) # finish phase, gated
|
|
second = asyncio.create_task(live.aclose())
|
|
third = asyncio.create_task(live.aclose())
|
|
await asyncio.sleep(0.05) # both parked on the close lock
|
|
assert not second.done() and not third.done()
|
|
third.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await third
|
|
assert live._closing.is_set() # the waiter changed nothing
|
|
|
|
release.set()
|
|
await asyncio.gather(first, second) # both land after the ONE close
|
|
assert live._closed and close_calls["count"] == 1
|
|
assert live._closing.is_set()
|
|
store.close()
|
|
|
|
|
|
async def test_waiter_cancel_never_touches_active_close(tmp_path, monkeypatch):
|
|
"""D2: the first aclose is mid-drain with a live worker on the wire; a
|
|
second aclose cancelled while WAITING for the lock must not clear the
|
|
owner's signal — new batches stay queued, the original worker completes,
|
|
and the owner's close then lands exactly once."""
|
|
arrived, wire_release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, wire_release)
|
|
backend, _cfg = make_backend(tmp_path, "waiter")
|
|
store = Store(tmp_path / "waiter.db")
|
|
seed_user_row(store, "t-waiter", "等锁者取消期间仍在排空的合成原话")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
worker = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5) # original worker on the wire
|
|
|
|
close_one = asyncio.create_task(live.aclose(drain_timeout=0.05))
|
|
assert await wait_until(lambda: live._closing.is_set())
|
|
close_two = asyncio.create_task(live.aclose()) # still WAITING for the lock
|
|
await asyncio.sleep(0.05)
|
|
close_two.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await close_two
|
|
assert not close_one.done() # the real close is still draining
|
|
assert live._closing.is_set() # D2: the waiter cleared nothing
|
|
|
|
seed_user_row(store, "t-waiter-new", "关闭期间不应被领取的新批次")
|
|
live.enqueue_uncovered()
|
|
await live.tick() # refused: the owner's close still holds the layer
|
|
new_row = batch_row_for(store, "t-waiter-new")
|
|
assert new_row["state"] == "queued" and new_row["owner"] is None
|
|
assert len(backend._memory.llm.attempts) == 1 # only the original wire call
|
|
|
|
wire_release.set()
|
|
await close_one # drain completes, then the close lands
|
|
assert backend._client.closed and live._closed
|
|
assert batch_row_for(store, "t-waiter")["state"] == "done"
|
|
assert len(store.query("SELECT * FROM sem_row_done")) == 1
|
|
assert batch_row_for(store, "t-waiter-new")["state"] == "queued"
|
|
with suppress(asyncio.CancelledError, Exception):
|
|
await worker
|
|
store.close()
|
|
|
|
|
|
async def test_drain_cancel_still_rearms_for_reuse(tmp_path, monkeypatch):
|
|
"""The revocable window stays revocable: a cancel while only DRAINING
|
|
(nothing closed yet) re-arms the layer and later batches still run; a
|
|
subsequent aclose then closes for real."""
|
|
arrived, wire_release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, wire_release)
|
|
backend, _cfg = make_backend(tmp_path, "drain")
|
|
store = Store(tmp_path / "drain.db")
|
|
seed_user_row(store, "t-drain", "排空阶段取消应当可续用的合成原话")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
worker = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5)
|
|
|
|
closing = asyncio.create_task(live.aclose(drain_timeout=0.05))
|
|
await asyncio.sleep(0.2) # past the deadline, still only draining
|
|
closing.cancel()
|
|
with suppress(asyncio.CancelledError):
|
|
await closing
|
|
assert not live._closing.is_set() # re-armed
|
|
assert not backend._client.closed # nothing closed
|
|
|
|
wire_release.set()
|
|
with suppress(asyncio.CancelledError, Exception):
|
|
await worker
|
|
assert batch_row(store)["state"] == "done"
|
|
seed_user_row(store, "t-drain-2", "取消关闭后继续处理的合成原话")
|
|
live.enqueue_uncovered()
|
|
await live.tick() # the layer is USABLE again
|
|
assert batch_row_for(store, "t-drain-2")["state"] == "done"
|
|
await live.aclose()
|
|
assert backend._client.closed
|
|
store.close()
|