"""R3 acceptance: isolated SQLite, controlled providers, injected clocks, no upstream.""" import asyncio import json import logging import sqlite3 import subprocess from uuid import uuid4 import httpx import pytest from conftest import engineering_budget from fastapi.testclient import TestClient from test_api import CREATE, ControlledLLM, events, live_server, make_app, turn from test_r2 import ack, add_life, app_for, final_and_receipt from cybergril.config import load_settings from cybergril.contracts import LLMDelta from cybergril.life.persona import fingerprint from cybergril.life.runtime import LifeRuntime from cybergril.memory.episodic import Memories from cybergril.memory.store import Store from cybergril.think.router import build_router from cybergril.think.scenes import Tier BASE = 'c285a0fd07e9d20e4fd4984942f3ae2fb8cbbd0c' SECRET = '私密紫铜风铃独特关键词' PROTECTED = ('bazi_core', 'companion', 'persona_interp', 'persona_seed', 'persona_job', 'growth_event', 'growth_job', 'life_event', 'telling_log', 'sharing_attempt') def snapshot(store): return {table: [tuple(r) for r in store.query(f'SELECT * FROM {table} ORDER BY rowid')] for table in PROTECTED} def seed(mem, text='杭州的雨声让我安静下来', reply='普通答复', *, gist=True): s, tid = mem.store, str(uuid4()) with s.conn: for role, raw in [('user', text), ('companion', reply)]: s.execute("INSERT INTO chat_message(ts,role,text,turn_id,status) VALUES(1,?,?,?,'sent')", (role, raw, tid)) mem.record_turn(tid) if gist: for row in s.query('SELECT * FROM mem_episodic WHERE turn_id=?', (tid,)): s.set_gist(row['id'], row['raw'], {'v': .1, 'a': .2, 'd': .3}) return tid, s.query_one("SELECT id FROM mem_episodic WHERE turn_id=? AND role='user'", (tid,))[0] class GistLLM(ControlledLLM): def __init__(self): super().__init__() self.jobs = [] self.bad = False self.entered = asyncio.Event() self.release = asyncio.Event() self.block = False async def stream(self, messages, **kwargs): self.jobs.append((messages, kwargs)) self.entered.set() if self.block: await self.release.wait() if self.bad: raise OSError('controlled compression failure') yield LLMDelta(text=json.dumps({'gist': '杭州雨声里的安静时刻', 'affect': {'v': .2, 'a': -.1, 'd': .3}})) yield LLMDelta(done=True) @pytest.fixture def memory(tmp_path): with Store(tmp_path / 'memory.db') as s: p = GistLLM() router = build_router(providers={t: p for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)}, budget=engineering_budget()) clock = [1000.0] yield Memories(s, router, load_settings().models, clock=lambda: clock[0]), p, clock def test_chinese_probe_and_legacy_tokenizer(memory): mem, _, _ = memory _, mid = seed(mem) s = mem.store assert not s.search_fts('雨声') assert [r['id'] for r in mem.search('杭州的雨')[0]] == [mid] assert mem.search('杭州的雨')[1] == 'phrase' assert mem.search('雨声')[1] == 'like' assert mem.search('雨')[1] == 'like' assert not s.query("SELECT rowid FROM mem_fts_zh WHERE mem_fts_zh MATCH '雨声'") short_like = s.query("SELECT rowid FROM mem_fts_zh WHERE raw LIKE '%雨声%'") print(f'SQLite {sqlite3.sqlite_version}: trigram short LIKE rows={len(short_like)}; short MATCH=0') traced = [] s.conn.set_trace_callback(traced.append) assert mem.search('雨声')[1] == 'like' s.conn.set_trace_callback(None) assert any('FROM mem_episodic WHERE' in q and 'LIKE' in q for q in traced) assert not any('FROM mem_fts_zh' in q for q in traced) assert mem.search('想起杭州的雨声那一天')[1] == 'trigram' assert mem.search('只是杭州的')[0] == [] # a lone trigram is noise assert sqlite3.sqlite_version_info >= (3, 34) @pytest.mark.parametrize('query', ['难过"', 'a OR b', 'NEAR(a b)', 'rain*', '"unclosed', 'a-b', '雨声" OR 1=1 --', '\x00', '', '***', '"杭州的雨"']) def test_untrusted_match_syntax_is_only_data(memory, query): mem, _, _ = memory seed(mem) seed(mem, text=SECRET) assert all(SECRET not in r['raw'] for r in mem.search(query)[0]) def test_search_topk_floor_exclusion_noise_budget_and_logs(memory, caplog): mem, _, _ = memory ids, tids = [], [] for _ in range(7): tid, mid = seed(mem) ids.append(mid) tids.append(tid) s = mem.store with s.conn: s.execute('UPDATE mem_episodic SET accessibility=.04 WHERE id=?', (ids[-1],)) hits, _ = mem.search('杭州的雨', tids[-3:-1]) assert len(hits) == 4 assert {r['id'] for r in hits}.isdisjoint(ids[-3:]) with caplog.at_level(logging.INFO, logger='cybergril.memory.episodic'): block = mem.context('杭州的雨', 'new-turn', tids[-3:-1], 600) assert len(block) <= 600 logged = json.loads(caplog.records[-1].message) assert logged['turn_id'] == 'new-turn' and logged['mode'] == 'phrase' assert logged['hits'] and all({'id', 'turn_id', 'rank'} == set(h) for h in logged['hits']) assert all(s.query_one('SELECT recall_count FROM mem_episodic WHERE id=?', (h['id'],))[0] == 1 for h in logged['hits']) assert mem.context('杭州的雨', 'x', [], 1) == '' assert mem.context('完全无匹配的事情', 'x', [], 600) == '' with s.conn: s.execute("UPDATE life_control SET value='off' WHERE key='memory_search'") assert mem.context('杭州的雨', 'x', [], 600) == '' @pytest.mark.asyncio async def test_jobs_retries_recovery_dual_indexes_and_clock(memory): mem, p, clock = memory _, mid = seed(mem, gist=False) s = mem.store with s.conn: s.execute('DELETE FROM mem_job WHERE memory_id<>?', (mid,)) before = snapshot(s) p.bad = True for _ in range(4): await mem.tick() assert len(p.jobs) == 3 job = s.query_one('SELECT * FROM mem_job WHERE memory_id=?', (mid,)) assert job['state'] == 'failed' and job['attempts'] == 3 assert s.query_one('SELECT gist,affect FROM mem_episodic WHERE id=?', (mid,))[:] == (None, None) # Restart recovery must not steal a live lease, and must use the injected clock. with s.conn: s.execute("UPDATE mem_job SET state='pending',attempts=1,owner='dead',lease_until=? WHERE memory_id=?", (int(clock[0] * 1000) + 1000, mid)) mem.recover() assert s.query_one('SELECT state FROM mem_job WHERE memory_id=?', (mid,))[0] == 'pending' clock[0] += 2 mem.recover() assert s.query_one('SELECT reason FROM mem_job WHERE memory_id=?', (mid,))[0] == 'lease_expired' p.bad = False await mem.tick() assert s.query_one('SELECT state FROM mem_job WHERE memory_id=?', (mid,))[0] == 'done' assert mem.search('杭州雨声')[0] assert s.search_fts('杭州雨声里的安静时刻') assert snapshot(s) == before assert p.jobs[-1][1]['temperature'] == .4 assert p.jobs[-1][1]['max_tokens'] == 400 assert set(json.loads(p.jobs[-1][0][1].content)) == {'role', 'raw'} watermark = float(s.query_one("SELECT value FROM life_control WHERE key='last_mem_decay'")[0]) clock[0] = watermark + 3599 mem.decay() assert s.query_one('SELECT accessibility FROM mem_episodic WHERE id=?', (mid,))[0] == 1 clock[0] = watermark + 3600 mem.decay() value = s.query_one('SELECT accessibility FROM mem_episodic WHERE id=?', (mid,))[0] assert .03 < value < 1 mem.decay() assert s.query_one('SELECT accessibility FROM mem_episodic WHERE id=?', (mid,))[0] == value @pytest.mark.asyncio async def test_cancelled_worker_restart_lease_and_no_duplicate_claim(memory): mem, p, clock = memory _, mid = seed(mem, gist=False) # Only one job here to expose concurrent claim duplication clearly. with mem.store.conn: mem.store.execute('DELETE FROM mem_job WHERE memory_id<>?', (mid,)) p.block = True task = asyncio.create_task(mem.tick()) await p.entered.wait() await mem.tick() assert len(p.jobs) == 1 task.cancel() with pytest.raises(asyncio.CancelledError): await task assert mem.store.query_one('SELECT gist FROM mem_episodic WHERE id=?', (mid,))[0] is None clock[0] += 41 p.block = False await mem.tick() assert len(p.jobs) == 2 assert mem.store.query_one('SELECT state FROM mem_job')[0] == 'done' def test_success_replay_restart_and_transaction_rollback(tmp_path, monkeypatch): path = tmp_path / 'chat.db' app, _ = make_app(path) with TestClient(app) as c: c.post('/api/companion', json=CREATE) tid = str(uuid4()) assert events(turn(c, turn_id=tid))[-1]['t'] == 'chat.final' s = app.state.store saved = [tuple(r) for r in s.query('SELECT * FROM mem_episodic')] assert len(saved) == 2 and len(s.query('SELECT * FROM mem_job')) == 2 assert events(turn(c, turn_id=tid))[-1]['t'] == 'chat.final' assert [tuple(r) for r in s.query('SELECT * FROM mem_episodic')] == saved original = s.execute def fail_job(sql, params=()): if 'INSERT OR IGNORE INTO mem_job' in sql: raise OSError('disk-full') return original(sql, params) monkeypatch.setattr(s, 'execute', fail_job) failed = str(uuid4()) assert events(turn(c, turn_id=failed))[-1]['t'] == 'chat.error' assert not s.query('SELECT id FROM mem_episodic WHERE turn_id=?', (failed,)) assert not s.query("SELECT id FROM chat_message WHERE turn_id=? AND status='sent'", (failed,)) app2, p2 = make_app(path) with TestClient(app2) as c: assert events(turn(c, turn_id=tid))[-1]['t'] == 'chat.final' assert all('只返回JSON,不要Markdown' in call[0].content for call in p2.calls) assert len(app2.state.store.query('SELECT * FROM mem_episodic')) == 2 @pytest.mark.parametrize('mode', ['error', 'truncated']) def test_failed_turn_has_no_raw_or_job(tmp_path, mode): app, p = make_app(tmp_path / 'chat.db') with TestClient(app) as c: c.post('/api/companion', json=CREATE) p.mode = mode assert events(turn(c))[-1]['t'] == 'chat.error' assert not app.state.store.query('SELECT * FROM mem_episodic') assert not app.state.store.query('SELECT * FROM mem_job') @pytest.mark.asyncio @pytest.mark.parametrize('cancel', [False, True]) async def test_disconnect_and_explicit_cancel_never_create_memory(tmp_path, cancel): app, p = make_app(tmp_path / 'chat.db') with live_server(app) as url: async with httpx.AsyncClient(base_url=url) as c: await c.post('/api/companion', json=CREATE) epoch = (await c.get('/api/health')).json()['epoch'] tid = str(uuid4()) p.mode = 'slow' async with c.stream('POST', '/api/chat', json={'text': '取消我', 'turn_id': tid, 'epoch': epoch}) as r: async for line in r.aiter_lines(): if line.startswith('data:'): break if cancel: p.mode = 'normal' response = await c.post('/api/chat', json={'text': '打断这一轮', 'turn_id': str(uuid4()), 'epoch': epoch}) assert events(response)[-1]['t'] == 'chat.final' p.mode = 'normal' assert events(await c.post('/api/chat', json={'text': '新的轮次', 'turn_id': str(uuid4()), 'epoch': epoch}))[-1]['t'] == 'chat.final' assert not app.state.store.query('SELECT * FROM mem_episodic WHERE turn_id=?', (tid,)) assert len(app.state.store.query('SELECT * FROM mem_job')) == (4 if cancel else 2) @pytest.mark.parametrize('case', ['unacknowledged', 'unjudged', 'private_candidate', 'unknown']) def test_private_companion_entire_memory_hidden_at_every_stage(tmp_path, case): app, p = app_for(tmp_path) with TestClient(app) as c: c.post('/api/companion', json=CREATE) eid = add_life(app.state.store, SECRET) p.reply = SECRET _, receipt = final_and_receipt(c) s, mem = app.state.store, app.state.memories if case != 'unacknowledged': assert ack(c, receipt).status_code == 200 if case in ('unacknowledged', 'private_candidate'): c.portal.call(app.state.sharing.judge, receipt['message_id']) row = s.query_one("SELECT * FROM mem_episodic WHERE role='companion'") if case == 'unknown': with s.conn: s.execute("UPDATE mem_episodic SET source='unknown' WHERE id=?", (row['id'],)) before, birth = snapshot(s), fingerprint(s) # Compress both raw rows, including the private one: visibility cannot rely on NULL gist. for _ in range(2): c.portal.call(mem.tick) private = s.query_one('SELECT * FROM mem_episodic WHERE id=?', (row['id'],)) assert private['raw'] == SECRET and private['gist'] == SECRET assert mem.source(private) is None and mem.dto(private) is None assert not mem.search(SECRET)[0] assert mem.context(SECRET, 'next', [], 600) == '' response = c.get('/api/memories').json() assert SECRET not in json.dumps(response, ensure_ascii=False) assert str(private['id']) not in [r['id'] for r in response['memories']] for mid in (str(private['id']), '999999', 'fake', '-1', '0', '99999999999999999999999999'): r = c.get('/api/memories/' + mid) assert r.status_code == 404 and r.json() == {'detail': '这一条现在打不开。'} assert s.query_one('SELECT told FROM life_event WHERE id=?', (eid,))[0] == 0 assert snapshot(s) == before and fingerprint(s) == birth def test_memories_dto_pagination_visibility_epoch_and_source_links(tmp_path): app, _ = app_for(tmp_path) with TestClient(app) as c: c.post('/api/companion', json=CREATE) mem, s = app.state.memories, app.state.store public_ids = [seed(mem, text='回忆' + str(i))[1] for i in range(4)] _, receipt = final_and_receipt(c) assert ack(c, receipt).status_code == 200 c.portal.call(app.state.sharing.judge, receipt['message_id']) visible = add_life(s, '已公开生活', told=1) hidden = add_life(s) companion = s.query_one("SELECT * FROM mem_episodic WHERE message_id=?", (receipt['message_id'],)) with s.conn: s.set_gist(companion['id'], '已压缩的真实聊天要点', {'v': 0, 'a': 0, 'd': 0}) s.execute('INSERT INTO telling_log VALUES(?,?,1)', (visible, receipt['message_id'])) s.execute('INSERT INTO telling_log VALUES(?,?,1)', (hidden, receipt['message_id'])) pages, cursor = [], None while True: params = {'limit': 2} | ({'before': cursor} if cursor else {}) data = c.get('/api/memories', params=params).json() assert data['epoch'] == app.state.epoch pages.extend(data['memories']) cursor = data['next_before'] if cursor is None: break assert [int(r['id']) for r in pages] == [companion['id'], *reversed(public_ids)] dto = pages[0] assert dto == {'id': str(companion['id']), 'title': '已压缩的真实聊天要点', 'text': '已压缩的真实聊天要点', 'group': '我们一起的', 'source': mem.dto(s.query_one('SELECT * FROM mem_episodic WHERE id=?', (companion['id'],)))['source'], 'lifeIds': [visible], 'messageIds': [receipt['message_id']], 'ts': companion['ts']} assert c.get('/api/memories/' + dto['id']).json() == {'memory': dto, 'epoch': data['epoch']} for q in ('limit=0', 'limit=101', 'before=0', 'before=99999999999999999999999'): assert c.get('/api/memories?' + q).status_code == 422 def test_real_chat_retrieval_excludes_twenty_turns_and_preserves_birth(tmp_path, caplog): app, p = make_app(tmp_path / 'chat.db') with TestClient(app) as c: c.post('/api/companion', json=CREATE) birth = fingerprint(app.state.store) _, old = seed(app.state.memories, '杭州的雨声让我安静下来') for i in range(20): seed(app.state.memories, f'杭州的雨近轮第{i}次话题') assert app.state.memories.context('杭州的雨', 'probe', [], 600) with caplog.at_level(logging.INFO, logger='cybergril.memory.episodic'): assert events(turn(c, '杭州的雨'))[-1]['t'] == 'chat.final' context = p.calls[-1] block = next(m.content for m in context[1:] if m.role == 'system') assert '杭州的雨声让我安静下来' in block and len(block) <= 600 assert '近轮' not in block assert sum(len(m.content) for m in context) <= 4000 assert '原始' in context[0].content or '出生' in context[0].content logged = json.loads(caplog.records[-1].message) assert old in [h['id'] for h in logged['hits']] assert fingerprint(app.state.store) == birth assert c.get('/api/health').json()['capabilities']['memory'] is True with app.state.store.conn: app.state.store.execute("UPDATE life_control SET value='off' WHERE key='memory_search'") assert c.get('/api/health').json()['capabilities']['memory'] is False assert events(turn(c, '杭州的雨'))[-1]['t'] == 'chat.final' assert all(m.role != 'system' for m in p.calls[-1][1:]) assert '长期检索已关闭' in p.calls[-1][0].content def test_startup_factory_clock_precedes_recovery(tmp_path, monkeypatch): path = tmp_path / 'chat.db' with Store(path) as s: router = build_router(providers={Tier.LOW: ControlledLLM()}, budget=engineering_budget()) mem = Memories(s, router, load_settings().models) seed(mem, gist=False) with s.conn: s.execute("UPDATE mem_job SET state='pending',lease_until=200000,owner='old'") monkeypatch.setattr('cybergril.api.app.LifeRuntime', lambda *args: LifeRuntime(*args, clock=lambda: 100.0)) app, _ = make_app(path) with TestClient(app): assert app.state.memories.clock() == 100.0 assert all(r['state'] == 'pending' for r in app.state.store.query('SELECT state FROM mem_job')) app.state.life.clock = lambda: 201.0 app.state.memories.recover() assert all(r['state'] == 'failed' for r in app.state.store.query('SELECT state FROM mem_job')) def test_populated_migration_idempotence_and_old_code_rollback(tmp_path): # Execute the exact product-base Store, not a handwritten imitation of old writes. old = tmp_path / 'legacy' old.mkdir() for name in ('store.py', 'schema.sql', 'r2.sql'): (old / name).write_bytes(subprocess.check_output(['git', 'show', f'{BASE}:src/cybergril/memory/{name}'])) import importlib.util spec = importlib.util.spec_from_file_location('legacy_store', old / 'store.py') module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) path = tmp_path / 'migrate.db' with module.Store(path) as s: first = s.add_episodic(role='user', raw='迁移之前杭州的雨声', salience=.5) original = tuple(s.query_one('SELECT * FROM mem_episodic WHERE id=?', (first,))) for _ in range(2): with Store(path) as s: assert tuple(s.query_one('SELECT * FROM mem_episodic WHERE id=?', (first,)))[:len(original)] == original assert len(s.query("SELECT rowid FROM mem_fts_zh WHERE mem_fts_zh MATCH '\"杭州的雨\"'")) == 1 s.execute("INSERT INTO mem_fts_zh(mem_fts_zh,rank) VALUES('integrity-check',1)") with module.Store(path) as s: second = s.add_episodic(role='user', raw='回退期间西湖的风声', salience=.6) assert len(s.search_fts('回退期间西湖的风声')) == 1 with Store(path) as s: assert s.query("SELECT rowid FROM mem_fts_zh WHERE mem_fts_zh MATCH '\"西湖的风\"'")[0][0] == second with s.conn: s.set_gist(first, '压缩以后独特新词汇', {'v': 0, 'a': 0, 'd': 0}) assert s.search_fts('压缩以后独特新词汇') assert s.query("SELECT rowid FROM mem_fts_zh WHERE mem_fts_zh MATCH '\"独特新词\"'") for table in ('mem_fts', 'mem_fts_zh'): s.execute(f"INSERT INTO {table}({table},rank) VALUES('integrity-check',1)") def test_three_endpoint_bytes_equal_product_baseline(tmp_path): import os import sys from pathlib import Path root = Path(__file__).resolve().parents[1] old = tmp_path / 'baseline' old.mkdir() # Exact tracked source extraction only; no databases or runtime configuration. files = subprocess.check_output(['git', 'ls-tree', '-r', '--name-only', BASE, 'src/']).decode().splitlines() for name in files: target = old / name target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(subprocess.check_output(['git', 'show', f'{BASE}:{name}'])) results = [] for index, source in enumerate((old / 'src', root / 'src')): data = tmp_path / f'data-{index}' data.mkdir() env = dict(os.environ, PYTHONPATH=os.pathsep.join((str(source), str(root / 'tests')))) results.append(subprocess.check_output([sys.executable, str(root / 'tests/r3_baseline_probe.py'), str(data)], cwd=root, env=env, text=True)) assert json.loads(results[0]) == json.loads(results[1]) import hashlib for endpoint, payload in json.loads(results[0])['responses'].items(): print(f'c285a0f == current {endpoint}: sha256={hashlib.sha256(bytes.fromhex(payload)).hexdigest()}') def test_direct_record_idempotence_and_unique_role(memory): mem, _, _ = memory tid, _ = seed(mem, gist=False) before = [tuple(r) for r in mem.store.query('SELECT * FROM mem_episodic')] with mem.store.conn: mem.record_turn(tid) assert [tuple(r) for r in mem.store.query('SELECT * FROM mem_episodic')] == before assert len(mem.store.query('SELECT * FROM mem_job')) == 2 assert all(r['gist'] is None and r['affect'] is None for r in mem.store.query('SELECT * FROM mem_episodic')) with pytest.raises(sqlite3.IntegrityError), mem.store.conn: mem.store.add_episodic(role='user', raw='duplicate', salience=.5, turn_id=tid) @pytest.mark.asyncio async def test_memory_budget_is_durable_and_hard_stop_makes_no_call(tmp_path): from cybergril.life.model import PersistentBudget app, p = app_for(tmp_path) with TestClient(app) as c: c.post('/api/companion', json=CREATE) seed(app.state.memories, gist=False) guard = app.state.life.router.guard assert isinstance(guard, PersistentBudget) before = guard.used await app.state.memories.tick() assert guard.used > before assert app.state.store.query_one("SELECT used FROM token_usage WHERE scene='memory_gist'")[0] > 0 guard.record(guard.limit, 'test-hard-stop') calls = len(p.memory_calls) await app.state.memories.tick() assert len(p.memory_calls) == calls app2, p2 = app_for(tmp_path) with TestClient(app2): assert app2.state.life.router.guard.is_tight() await app2.state.memories.tick() assert not p2.memory_calls