285 lines
12 KiB
Python
285 lines
12 KiB
Python
"""HEL-886 rework regressions: the manager's independent-review failures.
|
|
|
|
Two defects against the real pinned SDK (review_hel885_inflight.py):
|
|
|
|
P1 — a cancelled waiter or an expired lease treated a STILL-RUNNING worker as
|
|
dead: the finally wrote ``failed``, the next tick reclaimed the batch and
|
|
re-sent the same row while its first HTTP was in flight (2 HTTP, 2 vectors,
|
|
1 row_done entry — the late worker's result dropped).
|
|
P2 — ``add_group`` checked the ledger BEFORE proving remaining work, so a
|
|
batch whose every row was already complete could not finalize after restart
|
|
without new balance (``semantic_llm_budget_exhausted`` instead of ``done``).
|
|
|
|
These cases drive the REAL product path with barrier-blocked workers (an
|
|
in-flight HTTP call), not preseeded dead leases or fast racing tasks. The
|
|
pinned real install is covered live by tools/verify_inflight_lifecycle_hel886.py.
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import threading
|
|
import time
|
|
from contextlib import suppress
|
|
|
|
from test_mem0_backend import FakeLlm, FakeMemory, _make_ledger
|
|
from test_rework_hel885 import seed_public_two_row_turn
|
|
from test_semantic_recovery_hel882 import config_for, install_fake
|
|
|
|
from cybergril.memory.semantic import Mem0Backend, SemanticMemory
|
|
from cybergril.memory.store import Store
|
|
|
|
|
|
class BlockingLlm(FakeLlm):
|
|
"""The manager's repro shape: the extraction HTTP reached the gateway but
|
|
the answer has not come back. ``attempts`` records every wire entry (the
|
|
in-flight one included); ``calls`` only completed ones, like FakeLlm."""
|
|
|
|
def __init__(self, arrived: threading.Event, release: threading.Event) -> None:
|
|
super().__init__()
|
|
self.arrived, self.release = arrived, release
|
|
self.attempts: list[list] = []
|
|
|
|
def generate_response(self, messages=None, **kwargs):
|
|
self.attempts.append(messages)
|
|
self.arrived.set()
|
|
if not self.release.wait(10):
|
|
raise RuntimeError("test barrier timed out")
|
|
return super().generate_response(messages=messages, **kwargs)
|
|
|
|
|
|
class BlockingMemory(FakeMemory):
|
|
gate: tuple[threading.Event, threading.Event] | None = None
|
|
|
|
def __init__(self, embedder, collection):
|
|
super().__init__(embedder, collection)
|
|
self.llm = BlockingLlm(*BlockingMemory.gate)
|
|
|
|
|
|
def install_blocking(monkeypatch, arrived, release):
|
|
BlockingMemory.gate = (arrived, release)
|
|
return install_fake(monkeypatch, BlockingMemory)
|
|
|
|
|
|
def seed_user_row(store, turn_id, text):
|
|
with store.conn:
|
|
store.execute("INSERT INTO chat_message(ts,role,text,turn_id,status) VALUES"
|
|
"(1,'user',?,?,'sent')", (text, turn_id))
|
|
return store.query_one("SELECT id FROM chat_message WHERE text=?", (text,))["id"]
|
|
|
|
|
|
def batch_row(store):
|
|
return dict(store.query_one(
|
|
"SELECT state,reason,owner,entry_ids,finished_at FROM sem_batch"))
|
|
|
|
|
|
async def test_inflight_cancel_never_double_sends(tmp_path, monkeypatch):
|
|
"""Cancel the tick WHILE the row's HTTP is in flight: the worker thread
|
|
owns the batch fate, the second tick must not resend, and the late worker
|
|
result is durably recorded — one send, one vector, one row_done."""
|
|
arrived, release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, release)
|
|
ledger = tmp_path / "cancel.sqlite3"
|
|
_make_ledger(ledger)
|
|
backend = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=10))
|
|
store = Store(tmp_path / "cancel.db")
|
|
seed_user_row(store, "t-cancel", "处理中取消的合成原话")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
task = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5) # HTTP accepted, not answered
|
|
|
|
task.cancel() # cancels the WAITER only — the worker thread keeps running
|
|
with suppress(asyncio.CancelledError):
|
|
await task
|
|
after_cancel = batch_row(store)
|
|
# THE defect: the cancelled waiter used to write failed here, freeing the
|
|
# batch for reclaim while its add was still on the wire.
|
|
assert after_cancel["state"] == "pending" and after_cancel["owner"]
|
|
|
|
# ticks while the worker is live: nothing claimable, nothing re-sent
|
|
await asyncio.gather(live.tick(), live.tick())
|
|
assert len(backend._memory.llm.attempts) == 1
|
|
|
|
release.set()
|
|
await live.aclose(drain_timeout=10) # shutdown drains the live worker
|
|
final = batch_row(store)
|
|
rows = store.query("SELECT * FROM sem_row_done")
|
|
assert final["state"] == "done" and final["reason"] is None
|
|
assert len(backend._memory.llm.attempts) == 1 # still exactly one send
|
|
assert len(backend._memory.llm.calls) == 1
|
|
assert len(rows) == 1 and len(json.loads(rows[0]["entry_ids"])) == 1
|
|
assert json.loads(final["entry_ids"]) == json.loads(rows[0]["entry_ids"])
|
|
store.close()
|
|
|
|
|
|
async def test_expired_lease_with_live_worker_never_reclaims(tmp_path, monkeypatch):
|
|
"""Lease past 40s while the worker is REALLY still in the add: recover()
|
|
leaves the batch alone, concurrent ticks cannot resend the same row."""
|
|
arrived, release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, release)
|
|
ledger = tmp_path / "lease.sqlite3"
|
|
_make_ledger(ledger)
|
|
backend = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=10))
|
|
store = Store(tmp_path / "lease.db")
|
|
seed_user_row(store, "t-lease", "跨租约仍活跃的合成原话")
|
|
clock = [1000.0]
|
|
live = SemanticMemory(store, backend, clock=lambda: clock[0])
|
|
live.enqueue_uncovered()
|
|
tick_task = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5)
|
|
clock[0] += 41 # lease expired on paper — the worker is NOT dead
|
|
|
|
live.recover()
|
|
assert batch_row(store)["state"] == "pending" # never failed while live
|
|
|
|
await asyncio.gather(live.tick(), live.tick(), live.tick())
|
|
assert len(backend._memory.llm.attempts) == 1
|
|
|
|
release.set()
|
|
await live.aclose(drain_timeout=10)
|
|
with suppress(asyncio.CancelledError):
|
|
await tick_task
|
|
final = batch_row(store)
|
|
assert final["state"] == "done"
|
|
assert len(backend._memory.llm.attempts) == 1
|
|
assert len(store.query("SELECT * FROM sem_row_done")) == 1
|
|
assert len(json.loads(final["entry_ids"])) == 1
|
|
store.close()
|
|
|
|
|
|
def test_dead_owner_lease_still_recovers(tmp_path, monkeypatch):
|
|
"""Guard the other direction: a lease held by a GENUINELY dead owner (no
|
|
live worker — the crash/restart case) still recovers and retries."""
|
|
install_fake(monkeypatch)
|
|
ledger = tmp_path / "dead.sqlite3"
|
|
_make_ledger(ledger)
|
|
backend = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=50))
|
|
store = Store(tmp_path / "dead.db")
|
|
seed_public_two_row_turn(store, "t-dead", "真正死亡第一行", "真正死亡第二行")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
with store.conn: # a worker died holding the lease
|
|
store.execute("UPDATE sem_batch SET state='pending',owner='dead',lease_until=1")
|
|
live.recover()
|
|
job = batch_row(store)
|
|
assert job["state"] == "failed" and job["reason"] == "lease_expired"
|
|
# unknown-outcome honesty: the retry sends only rows without a proof;
|
|
# here that is both rows (nothing was proven complete)
|
|
calls_before = len(backend._memory.llm.calls)
|
|
asyncio.run(_drain_tick(live))
|
|
after = batch_row(store)
|
|
assert after["state"] == "done"
|
|
assert len(backend._memory.llm.calls) - calls_before == 2
|
|
store.close()
|
|
|
|
|
|
async def _drain_tick(live):
|
|
await live.tick()
|
|
await live.aclose(drain_timeout=5)
|
|
|
|
|
|
async def test_shutdown_drains_live_worker_before_close(tmp_path, monkeypatch):
|
|
"""aclose() waits for the still-running worker's durable writes BEFORE
|
|
the vector-store client closes under them."""
|
|
arrived, release = threading.Event(), threading.Event()
|
|
install_blocking(monkeypatch, arrived, release)
|
|
ledger = tmp_path / "drain.sqlite3"
|
|
_make_ledger(ledger)
|
|
backend = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=10))
|
|
store = Store(tmp_path / "drain.db")
|
|
seed_user_row(store, "t-drain", "关闭排空的合成原话")
|
|
clock = [1000.0]
|
|
live = SemanticMemory(store, backend, clock=lambda: clock[0])
|
|
live.enqueue_uncovered()
|
|
task = asyncio.create_task(live.tick())
|
|
assert await asyncio.to_thread(arrived.wait, 5)
|
|
|
|
def _finish(): # the wire answer arrives while shutdown is already waiting
|
|
clock[0] = 2000 # any durable write after this proves it postdates aclose
|
|
release.set()
|
|
|
|
timer = threading.Timer(0.2, _finish)
|
|
timer.start()
|
|
started = time.monotonic()
|
|
await live.aclose(drain_timeout=10)
|
|
assert time.monotonic() >= started + 0.15 # aclose actually waited
|
|
row = store.query_one("SELECT * FROM sem_row_done")
|
|
batch = batch_row(store)
|
|
assert row is not None and row["done_at"] >= 2000 # written after aclose began
|
|
assert batch["state"] == "done" and batch["finished_at"] >= 2000
|
|
assert backend._client.closed # closed only after the drain
|
|
with suppress(asyncio.CancelledError, Exception):
|
|
await task
|
|
timer.join()
|
|
store.close()
|
|
|
|
|
|
async def test_all_rows_done_batch_finalizes_without_new_budget(tmp_path, monkeypatch):
|
|
"""P2 (manager third example): llm_cap=1 fully spent, the single row
|
|
completed for real; a restart into the crash window 'row done persisted,
|
|
final status did not' finalizes done with ZERO new HTTP and the complete
|
|
entry_ids — no cap raise, no refund."""
|
|
install_fake(monkeypatch)
|
|
ledger = tmp_path / "alldone.sqlite3"
|
|
_make_ledger(ledger)
|
|
cfg = config_for(tmp_path, ledger, embed_cap=50, llm_cap=1)
|
|
backend = Mem0Backend(cfg)
|
|
db = tmp_path / "alldone.db"
|
|
store = Store(db)
|
|
seed_user_row(store, "t-done", "完整处理的合成原话")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
await live.tick()
|
|
original = batch_row(store)
|
|
assert original["state"] == "done" and json.loads(original["entry_ids"])
|
|
with store.conn: # crash window: final batch status never landed
|
|
store.execute("UPDATE sem_batch SET state='pending',owner='crashed',lease_until=1")
|
|
store.close()
|
|
await backend.aclose()
|
|
|
|
backend2 = Mem0Backend(cfg) # same paths, same caps — nothing new approved
|
|
store2 = Store(db)
|
|
live2 = SemanticMemory(store2, backend2)
|
|
calls_before = len(backend2._memory.llm.calls)
|
|
await live2.tick()
|
|
after = batch_row(store2)
|
|
assert after["state"] == "done" and after["reason"] is None
|
|
assert len(backend2._memory.llm.calls) == calls_before # zero new HTTP
|
|
assert json.loads(after["entry_ids"]) == json.loads(original["entry_ids"])
|
|
assert backend2.llm_guard.used() == 1 # cap unchanged, nothing refunded
|
|
store2.close()
|
|
await backend2.aclose()
|
|
|
|
|
|
async def test_partially_done_batch_keeps_the_hard_gate(tmp_path, monkeypatch):
|
|
"""The P2 fix must not buy headroom: with a row still missing and the
|
|
budget exhausted, restart keeps refusing (queued/budget) — never a fake
|
|
done, never an extra send past the cap."""
|
|
install_fake(monkeypatch)
|
|
ledger = tmp_path / "partial.sqlite3"
|
|
_make_ledger(ledger)
|
|
cfg = config_for(tmp_path, ledger, embed_cap=50, llm_cap=1)
|
|
backend = Mem0Backend(cfg)
|
|
db = tmp_path / "partial.db"
|
|
store = Store(db)
|
|
seed_public_two_row_turn(store, "t-partial", "部分完成用户行", "部分完成伴侣行")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
await live.tick() # one row really sent; the other budget-refused
|
|
first = batch_row(store)
|
|
assert first["state"] == "queued" and "budget" in (first["reason"] or "")
|
|
with store.conn: # crash window on top of the budget refusal
|
|
store.execute("UPDATE sem_batch SET state='pending',owner='crashed',lease_until=1")
|
|
store.close()
|
|
await backend.aclose()
|
|
|
|
backend2 = Mem0Backend(cfg)
|
|
store2 = Store(db)
|
|
live2 = SemanticMemory(store2, backend2)
|
|
calls_before = len(backend2._memory.llm.calls)
|
|
await live2.tick()
|
|
after = batch_row(store2)
|
|
assert after["state"] == "queued" and "budget" in (after["reason"] or "")
|
|
assert len(backend2._memory.llm.calls) == calls_before # gate held the wire
|
|
store2.close()
|
|
await backend2.aclose()
|