331 lines
16 KiB
Python
331 lines
16 KiB
Python
"""HEL-885 rework regressions: the manager's independent-review failures.
|
|
|
|
Each case first mirrors the exact reproduction from HEL-885's review (real
|
|
public user+companion queue; cross-day settlement; aged unknown reservation;
|
|
missing authority; per-row progress), then proves the fix. Fake-SDK mirrors
|
|
run here; the pinned real install is covered live by
|
|
tools/verify_semantic_recovery_hel885.py (deterministic boundary, loopback).
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import sqlite3
|
|
|
|
import pytest
|
|
from test_mem0_backend import POINT_LOG, FakeLlm, FakeMemory, _make_ledger, synthetic_authority
|
|
from test_semantic_recovery_hel882 import config_for, install_fake
|
|
|
|
from cybergril.life.model import PersistentBudget
|
|
from cybergril.memory.semantic import (
|
|
EmbedLedgerGuard,
|
|
LlmLedgerGuard,
|
|
Mem0Backend,
|
|
SemanticBudgetError,
|
|
SemanticMemory,
|
|
)
|
|
from cybergril.memory.store import Store
|
|
from cybergril.think.budget import BudgetConfig, BudgetGuard
|
|
|
|
|
|
def seed_public_two_row_turn(store, turn_id, user_text, companion_text):
|
|
"""The manager's partial-repro shape: a sent user row plus a judged+acked
|
|
(public) companion row in the same turn."""
|
|
with store.conn:
|
|
store.execute("INSERT INTO chat_message(ts,role,text,turn_id,status) VALUES"
|
|
"(1,'user',?,?,'sent')", (user_text, turn_id))
|
|
inserted = store.execute(
|
|
"INSERT INTO chat_message(ts,role,text,turn_id,status) VALUES"
|
|
"(2,'companion',?,?,'sent')", (companion_text, turn_id))
|
|
store.execute(
|
|
"INSERT INTO sharing_attempt(message_id,turn_id,epoch,token,text_hash,expires_at,"
|
|
"ack_at,candidates,state) VALUES(?,'"+turn_id+"',1,'t','h',9223372036854775806,"
|
|
"3,'[]','judged')", (inserted.lastrowid,))
|
|
return store.query_one("SELECT id FROM chat_message WHERE text=?", (user_text,))["id"], \
|
|
store.query_one("SELECT id FROM chat_message WHERE text=?", (companion_text,))["id"]
|
|
|
|
|
|
def llm_bodies(backend):
|
|
return [json.dumps(m, ensure_ascii=False) for m in backend._memory.llm.calls]
|
|
|
|
|
|
async def test_partial_batch_claims_only_the_row_actually_sent(tmp_path, monkeypatch):
|
|
"""Manager partial repro, fake mirror: llm_cap=1, two public rows — after
|
|
the first tick exactly ONE wire call happened, and the completed set is
|
|
the companion row ONLY (the HEL-882 whole-batch anchor claimed both).
|
|
Raising the cap resumes: done, and the user's original words really go
|
|
out — exactly once."""
|
|
install_fake(monkeypatch)
|
|
ledger = tmp_path / "partial.sqlite3"
|
|
_make_ledger(ledger)
|
|
backend = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=1))
|
|
store = Store(tmp_path / "partial.db")
|
|
user_id, companion_id = seed_public_two_row_turn(
|
|
store, "turn1", "合成用户原话甲", "合成已公开伴侣原话乙")
|
|
live = SemanticMemory(store, backend)
|
|
assert live.enqueue_uncovered() == 1
|
|
await live.tick()
|
|
|
|
job = dict(store.query_one("SELECT state,reason,entry_ids FROM sem_batch"))
|
|
assert job["state"] == "queued" and "budget" in (job["reason"] or "")
|
|
assert len(backend._memory.llm.calls) == 1 # one actual wire send, no retries
|
|
batch_key = store.query_one("SELECT batch_key FROM sem_batch")["batch_key"]
|
|
# THE defect: the first vector must not vouch for the unsent user row
|
|
assert backend.completed_message_ids(batch_key) == {companion_id}
|
|
points = backend._client.scroll(collection_name=backend.collection, with_payload=True,
|
|
limit=100)[0]
|
|
for point in points: # per-row anchors: each payload covers exactly one row
|
|
roles = point.payload.get("turn_roles") or []
|
|
assert len(roles) == 1
|
|
|
|
backend.llm_guard.cap = 5 # the manager raises the temporary cap
|
|
await live.tick()
|
|
after = dict(store.query_one("SELECT state,reason,entry_ids FROM sem_batch"))
|
|
assert after["state"] == "done" and json.loads(after["entry_ids"])
|
|
bodies = llm_bodies(backend)
|
|
assert sum("合成用户原话甲" in b for b in bodies) == 1 # actually sent, once
|
|
assert sum("合成已公开伴侣原话乙" in b for b in bodies) == 1 # never resent
|
|
assert backend.completed_message_ids(batch_key) == {user_id, companion_id}
|
|
store.close()
|
|
|
|
|
|
async def test_empty_extraction_row_is_done_and_never_resent(tmp_path, monkeypatch):
|
|
"""A row whose extraction legitimately returns nothing stores no vector —
|
|
it is still provably complete (sem_row_done) and never re-sent."""
|
|
install_fake(monkeypatch)
|
|
|
|
class EmptyForOneRow(FakeLlm):
|
|
def generate_response(self, messages=None, **kwargs):
|
|
user = messages[1]["content"] if messages and len(messages) > 1 else ""
|
|
if "甲" in user:
|
|
self.calls.append(messages)
|
|
return json.dumps({"memory": []}) # legitimate empty extraction
|
|
return super().generate_response(messages=messages, **kwargs)
|
|
|
|
class EmptyMemory(FakeMemory):
|
|
"""Parses the extraction like the real SDK: an empty memory list
|
|
stores NO point (the legitimate-empty path HEL-885 must prove done)."""
|
|
|
|
def __init__(self, embedder, collection):
|
|
super().__init__(embedder, collection)
|
|
self.llm = EmptyForOneRow()
|
|
|
|
def add(self, text, metadata=None, user_id=None):
|
|
if not user_id:
|
|
raise ValueError("filters must contain at least one of: user_id, agent_id, run_id")
|
|
self.embedding_model.embed(text, memory_action="search")
|
|
raw = self.llm.generate_response(messages=[
|
|
{"role": "system", "content": "extract"},
|
|
{"role": "user", "content": text},
|
|
], response_format={"type": "json_object"})
|
|
extracted = json.loads(raw).get("memory") or []
|
|
if not extracted:
|
|
return {"results": []} # nothing extracted → nothing stored
|
|
texts = [item["text"] for item in extracted]
|
|
self.embedding_model.embed_batch(texts, memory_action="add")
|
|
results = []
|
|
for one in texts:
|
|
self.seq += 1
|
|
entry_id = f"m{self.seq}"
|
|
bucket = POINT_LOG.setdefault(self.collection, {})
|
|
bucket[entry_id] = {"data": one, "user_id": user_id, **(metadata or {})}
|
|
results.append({"id": entry_id, "memory": one, "event": "ADD"})
|
|
return {"results": results}
|
|
|
|
install_fake(monkeypatch, EmptyMemory)
|
|
ledger = tmp_path / "empty.sqlite3"
|
|
_make_ledger(ledger)
|
|
backend = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=50))
|
|
store = Store(tmp_path / "empty.db")
|
|
seed_public_two_row_turn(store, "t9", "这一行提取为空甲", "这一行有提取乙")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
await live.tick()
|
|
batch = store.query_one("SELECT * FROM sem_batch")
|
|
assert batch["state"] == "done"
|
|
done = {r["message_id"] for r in store.query("SELECT message_id FROM sem_row_done")}
|
|
user_id = store.query_one("SELECT id FROM chat_message WHERE text='这一行提取为空甲'")["id"]
|
|
assert done == {user_id, store.query_one(
|
|
"SELECT id FROM chat_message WHERE text='这一行有提取乙'")["id"]}
|
|
# honest accounting: only the row that extracted something has an entry
|
|
assert len(json.loads(batch["entry_ids"])) == 1
|
|
|
|
# crash-style resume probe: batch requeued by hand, tick again — the
|
|
# empty-extraction row is skipped via sem_row_done, no second LLM send
|
|
with store.conn:
|
|
store.execute("UPDATE sem_batch SET state='queued',owner=NULL,lease_until=NULL")
|
|
calls_before = len(backend._memory.llm.calls)
|
|
await live.tick()
|
|
assert len(backend._memory.llm.calls) == calls_before # nothing re-sent
|
|
assert store.query_one("SELECT state FROM sem_batch")["state"] == "done"
|
|
store.close()
|
|
|
|
|
|
async def test_process_restart_resumes_only_the_missing_row(tmp_path, monkeypatch):
|
|
"""Crash after row 1 persisted: a NEW Store + NEW backend on the same
|
|
paths (process restart) resumes sending only the missing row."""
|
|
install_fake(monkeypatch)
|
|
ledger = tmp_path / "restart.sqlite3"
|
|
_make_ledger(ledger)
|
|
backend = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=1))
|
|
db_path = tmp_path / "restart.db"
|
|
store = Store(db_path)
|
|
seed_public_two_row_turn(store, "tr", "重启后第一行", "重启后第二行")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
await live.tick() # companion row done, user row budget-refused
|
|
batch_key = store.query_one("SELECT batch_key FROM sem_batch")["batch_key"]
|
|
store.close() # graceful close; durable state is what a crash leaves anyway
|
|
|
|
backend2 = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=50))
|
|
store2 = Store(db_path)
|
|
live2 = SemanticMemory(store2, backend2)
|
|
await live2.tick() # restart resumes the SAME batch
|
|
assert store2.query_one("SELECT state FROM sem_batch")["state"] == "done"
|
|
bodies = llm_bodies(backend2)
|
|
assert sum("重启后第一行" in b for b in bodies) == 1 # only the missing row
|
|
assert backend2.completed_message_ids(batch_key) == {
|
|
store2.query_one("SELECT id FROM chat_message WHERE text='重启后第一行'")["id"],
|
|
store2.query_one("SELECT id FROM chat_message WHERE text='重启后第二行'")["id"]}
|
|
store2.close()
|
|
|
|
|
|
async def test_concurrent_ticks_never_double_send(tmp_path, monkeypatch):
|
|
"""Two ticks racing on one batch: the BEGIN IMMEDIATE claim gives the
|
|
whole batch to one worker; every row is sent exactly once."""
|
|
install_fake(monkeypatch)
|
|
ledger = tmp_path / "race.sqlite3"
|
|
_make_ledger(ledger)
|
|
backend = Mem0Backend(config_for(tmp_path, ledger, embed_cap=50, llm_cap=50))
|
|
store = Store(tmp_path / "race.db")
|
|
seed_public_two_row_turn(store, "tc", "并发第一行", "并发第二行")
|
|
live = SemanticMemory(store, backend)
|
|
live.enqueue_uncovered()
|
|
await asyncio.gather(live.tick(), live.tick())
|
|
assert store.query_one("SELECT state FROM sem_batch")["state"] == "done"
|
|
bodies = llm_bodies(backend)
|
|
assert sum("并发第一行" in b for b in bodies) == 1
|
|
assert sum("并发第二行" in b for b in bodies) == 1
|
|
store.close()
|
|
|
|
|
|
def test_cross_day_settlement_books_on_the_reservation_day(tmp_path):
|
|
"""Manager midnight repro: day cap 100, hold 90 on day 1, actual 10
|
|
arriving on day 2 — day 1 settles to 10, day 2 stays 0 (never -80), and
|
|
a 150-token request the next day is refused."""
|
|
store = Store(tmp_path / "midnight.db")
|
|
guard = PersistentBudget(store, BudgetGuard(BudgetConfig(
|
|
daily_tokens=100, daily_requests={"dialogue": 10})))
|
|
guard._roll = lambda: None
|
|
guard.state.day = "2026-10-26"
|
|
hold = guard.acquire("dialogue_main", tokens=90)
|
|
guard.state.day = "2026-10-27"
|
|
guard.settle(hold, actual_tokens=10)
|
|
assert guard.used == 0 # new day untouched — no manufactured headroom
|
|
with pytest.raises(RuntimeError, match="budget_tokens_exhausted"):
|
|
guard.acquire("dialogue_main", tokens=150) # 150 > 100 cap → refused
|
|
rows = {r["day"]: r["used"] for r in store.query("SELECT * FROM token_usage")}
|
|
assert rows == {"2026-10-26": 10}
|
|
assert all(v >= 0 for v in rows.values())
|
|
store.close()
|
|
|
|
|
|
def test_cross_day_release_refunds_the_paying_day(tmp_path):
|
|
"""A provably-unsent hold reserved on day 1 and released on day 2 refunds
|
|
day 1 (and the one-shot totals) — day 2 never goes negative."""
|
|
store = Store(tmp_path / "release.db")
|
|
guard = PersistentBudget(store, BudgetGuard(BudgetConfig(
|
|
daily_tokens=100, daily_requests={"dialogue": 10}, total_tokens=500)))
|
|
guard._roll = lambda: None
|
|
guard.state.day = "2026-10-26"
|
|
hold = guard.acquire("dialogue_main", tokens=90)
|
|
guard.state.day = "2026-10-27"
|
|
guard.release(hold)
|
|
rows = {r["day"]: r["used"] for r in store.query("SELECT * FROM token_usage")}
|
|
assert rows == {"2026-10-26": 0}
|
|
assert guard.used == 0
|
|
assert guard.total_tokens_used() == 0
|
|
req = {r["day"]: r["requests"] for r in store.query("SELECT * FROM request_usage")}
|
|
assert req == {"2026-10-26": 0}
|
|
store.close()
|
|
|
|
|
|
def _backdate(path, table, row_id):
|
|
conn = sqlite3.connect(str(path))
|
|
with conn:
|
|
conn.execute(f"UPDATE {table} SET ts='2000-01-01T00:00:00Z' WHERE id=?", (row_id,))
|
|
conn.close()
|
|
|
|
|
|
def test_aged_unknown_llm_reservation_is_never_refunded(tmp_path):
|
|
"""Manager extra repro (llm side): a reservation that crashed before
|
|
settle is UNKNOWN — an hour (or a restart) later the cap is still spent;
|
|
no timeout quietly deletes the charging fact."""
|
|
ledger = tmp_path / "crash-llm.sqlite3"
|
|
llm = LlmLedgerGuard(ledger, cap=1)
|
|
pending = llm.reserve(1, "sent_but_crashed_before_settlement")
|
|
_backdate(ledger, "llm_ops", pending)
|
|
with pytest.raises(RuntimeError, match="semantic_llm_budget_exhausted"):
|
|
LlmLedgerGuard(ledger, cap=1).reserve(1, "second_send_after_restart")
|
|
reopened = LlmLedgerGuard(ledger, cap=1)
|
|
assert reopened.used() == 1 # the unknown consumption stayed billed
|
|
|
|
|
|
def test_aged_unknown_embed_reservation_is_never_refunded(tmp_path):
|
|
"""Same shared path, embed side (embed_ops) — unknown holds stay counted,
|
|
across process restarts; only an explicit unsent release removes one."""
|
|
ledger = tmp_path / "crash-embed.sqlite3"
|
|
_make_ledger(ledger)
|
|
guard = EmbedLedgerGuard(ledger, cap=1)
|
|
hold = guard.reserve(1, "embed_sent_then_crashed")
|
|
_backdate(ledger, "embed_ops", hold)
|
|
with pytest.raises(RuntimeError, match="semantic_embed_budget_exhausted"):
|
|
EmbedLedgerGuard(ledger, cap=1).reserve(1, "again_after_restart")
|
|
assert EmbedLedgerGuard(ledger, cap=1).used() == 1
|
|
# the explicit provably-unsent release still works — the ONLY removal path
|
|
fresh = EmbedLedgerGuard(ledger, cap=1)
|
|
fresh.release(hold)
|
|
assert fresh.used() == 0
|
|
|
|
|
|
async def test_positive_caps_without_authority_fail_closed(tmp_path, monkeypatch):
|
|
"""Manager partial repro's second finding: a fresh empty ledger + positive
|
|
caps + NO authority used to construct and send. Now every missing/empty/
|
|
malformed authority refuses construction before the SDK is touched."""
|
|
install_fake(monkeypatch)
|
|
fresh = tmp_path / "fresh-bypass.sqlite3"
|
|
conn = sqlite3.connect(str(fresh))
|
|
with conn:
|
|
conn.execute("CREATE TABLE embed_ops (id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT,"
|
|
" pid INTEGER, kind TEXT, items INTEGER, note TEXT)")
|
|
conn.execute("CREATE TABLE llm_ops (id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT,"
|
|
" pid INTEGER, kind TEXT, items INTEGER, note TEXT)")
|
|
conn.close()
|
|
|
|
def cfg(**extra):
|
|
base = config_for(tmp_path, fresh, embed_cap=0, llm_cap=0)
|
|
base.update(extra)
|
|
return base
|
|
|
|
with pytest.raises(SemanticBudgetError, match="ledger_authority_required"):
|
|
Mem0Backend(cfg(embed_cap=100, llm_cap=1)) # no authority at all
|
|
with pytest.raises(SemanticBudgetError, match="ledger_authority_required"):
|
|
Mem0Backend(cfg(embed_cap=100, llm_cap=1, authority={})) # empty block
|
|
with pytest.raises(SemanticBudgetError, match="ledger_authority_required"):
|
|
Mem0Backend(cfg(llm_cap=1, authority="see-notes")) # malformed
|
|
# a WRONG (non-matching) authority is refused by the prefix check
|
|
with pytest.raises(SemanticBudgetError):
|
|
Mem0Backend(cfg(embed_cap=100, llm_cap=1,
|
|
authority={"prefix_rows": 1, "prefix_sha256": "0" * 64, "min_used": 0}))
|
|
# zero caps stay constructible (fail-closed on WORK, not on construction)
|
|
zero = Mem0Backend(cfg(embed_cap=0, llm_cap=0))
|
|
with pytest.raises(RuntimeError, match="semantic_embed_budget_not_approved"):
|
|
await zero.add_group(
|
|
[{"message_id": 1, "turn_id": "t", "role": "user", "text": "x"}],
|
|
{"turn_ids": ["t"], "turn_roles": [], "input_sha256": "x"}, "z")
|
|
# the explicit synthetic authority path constructs (tests never relax the
|
|
# product default — they hand it a real prefix, as HEL-885 requires)
|
|
seeded = tmp_path / "seeded.sqlite3"
|
|
_make_ledger(seeded) # seeds the synthetic prefix row
|
|
okay = Mem0Backend(cfg(embed_cap=10, llm_cap=10, ledger_path=str(seeded),
|
|
authority=synthetic_authority(seeded)))
|
|
assert okay.backend == "mem0"
|