Bug 1 (recorded call1): server-side cancel ended the voice SSE without a
terminal frame BEFORE the first delta (worker.log:108 SDK retry 13ms after
the text POST); the adapter raised a connection error, livekit-agents 1.8.8
silently retried the same turn_id, the retry re-acquired the gate, cancelled
the user's text turn and re-spoke a full ~32s reply. The HEL-907 claim that
the _retry_on_chunk_sent guard never worked is corrected: the guard only
applies after output (output_sent), verified against the installed SDK
source. Fix: explicit chat.cancelled terminal frame; a cancelled turn_id is
refused re-activation (never touches the newer active turn); the worker ends
cancelled turns on the SDK's own 499 path (no retry) and stops the cached
playout via SpeechHandle.interrupt(); a server_cancelled latch short-circuits
any re-entry with zero network. Failed-turn retries, completed-turn replay
and receipt validation are unchanged (dedicated tests).
Bug 2 (recorded row id=12): the endpoint streams implicit-open reasoning
plus a stray </think> inline in content into subtitles and TTS. Fix:
ReasoningSeparator at the OpenAI-compat adapter seam (cross-chunk safe,
reasoning_content field side-channel on LLMDelta.reasoning, never persisted)
plus a turn-level split before persist/final/receipt. Recorded stream
replays clean at every chunk size; normal zh/en content preserved
byte-for-byte; documented residual: implicit-open reasoning whose closer
lies beyond the 128-char hold streams partially (persistence stays clean).
Offline verification: .venv311 full suite 445 passed / 5 skipped (2
pre-existing test_r3 failures fail identically at 5625de6); real-SDK harness
(livekit-agents 1.8.8, agent venv) 5/5; reasoning separation 37/37; ruff
clean. Zero real requests, zero deployment, no dependency changes.
Co-authored-by: multica-agent <github@multica.ai>
379 lines
16 KiB
Python
379 lines
16 KiB
Python
"""ChatApi adapter and receipt-gate tests (HEL-877) — real adapter code against
|
|
the real unified API (fake LLM provider, ASGI/local-socket transports) plus
|
|
scripted-SSE unit cases. Zero network beyond 127.0.0.1."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import socket
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
import pytest
|
|
import uvicorn
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
|
|
|
from conftest import engineering_budget # noqa: E402
|
|
from test_r2 import R2LLM # noqa: E402
|
|
|
|
from cybergril.api.app import create_app # noqa: E402
|
|
from cybergril.life.runtime import LifeConfig # noqa: E402
|
|
from cybergril.think.router import build_router # noqa: E402
|
|
from cybergril.think.scenes import Tier # noqa: E402
|
|
from cybergril.voice.chat_api_client import ( # noqa: E402
|
|
ChatApiClient,
|
|
ChatApiConnectionError,
|
|
_sse_frames,
|
|
)
|
|
from cybergril.voice.receipt import ReceiptSender, TurnLedger, should_ack # noqa: E402
|
|
|
|
CREATE = {"name": "语音适配", "birth": "2000-10-18", "hour": "亥时", "relation": "朋友"}
|
|
|
|
|
|
@pytest.fixture
|
|
def api(tmp_path):
|
|
provider = R2LLM()
|
|
# HEL-879 integration: the mainline chat path now hard-gates every real
|
|
# send through router.guard.acquire (HEL-882) — the engineering budget is
|
|
# the same approved-caps config every mainline test uses; without it the
|
|
# correct product behavior is fail-closed at cap 0.
|
|
router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)},
|
|
budget=engineering_budget())
|
|
# file-backed: uvicorn's lifespan in the live-server tests re-opens the same
|
|
# database (":memory:" would hand it a second, empty store)
|
|
app = create_app(db_path=tmp_path / "adapter.db", router=router,
|
|
life_config=LifeConfig(enabled=True, poll_seconds=3600),
|
|
semantic_mode="stub")
|
|
from fastapi.testclient import TestClient
|
|
|
|
with TestClient(app) as client:
|
|
client.post("/api/companion", json=CREATE)
|
|
epoch = client.get("/api/health").json()["epoch"]
|
|
yield app, provider, epoch
|
|
provider.mode = ""
|
|
|
|
|
|
@pytest.fixture
|
|
def voice(api):
|
|
app, provider, epoch = api
|
|
client = ChatApiClient("http://asgi", transport=httpx.ASGITransport(app=app))
|
|
return client, provider, epoch
|
|
|
|
|
|
async def test_consecutive_multi_turn_stream_and_receipt(voice):
|
|
client, provider, epoch = voice
|
|
deltas_seen: list[str] = []
|
|
turn1 = await client.send_turn("第一句你好", on_delta=deltas_seen.append)
|
|
turn2 = await client.send_turn("第二句还在", on_delta=deltas_seen.append)
|
|
assert turn1.completed and turn2.completed
|
|
assert turn1.text == provider.reply and turn2.text == provider.reply
|
|
assert turn1.turn_id != turn2.turn_id
|
|
assert "".join(deltas_seen) == turn1.text + turn2.text
|
|
assert turn1.receipt and turn2.receipt
|
|
assert turn1.receipt["turn_id"] == turn1.turn_id
|
|
assert turn2.receipt["epoch"] == epoch
|
|
|
|
|
|
async def test_completed_turn_replay_adds_no_model_calls(voice):
|
|
client, provider, _ = voice
|
|
turn_id = str(uuid4())
|
|
first = await client.send_turn("重放测试", turn_id=turn_id)
|
|
calls = len(provider.calls)
|
|
second = await client.send_turn("重放测试", turn_id=turn_id)
|
|
assert len(provider.calls) == calls, "replay must not call the model again"
|
|
assert second.completed and second.text == first.text
|
|
assert second.receipt, "replay still carries the receipt"
|
|
|
|
|
|
async def test_chat_error_yields_phi_without_receipt(voice):
|
|
client, provider, _ = voice
|
|
provider.mode = "error"
|
|
turn = await client.send_turn("这次会失败")
|
|
assert turn.completed is False
|
|
assert turn.error_phi and "没接上话" in turn.error_phi
|
|
assert turn.receipt is None
|
|
provider.mode = ""
|
|
|
|
|
|
async def test_epoch_409_resync_bounded(voice):
|
|
client, _, epoch = voice
|
|
# server already at `epoch`; claim a stale one -> 409 -> resync once -> done
|
|
turn = await client.send_turn("旧纪元重同步", epoch=epoch + 999)
|
|
assert turn.completed and turn.epoch == epoch
|
|
|
|
|
|
async def test_epoch_409_never_loops_on_garbage(voice):
|
|
client, _, _ = voice
|
|
# a server that always answers 409 without a usable epoch must not loop
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path == "/api/health":
|
|
return httpx.Response(200, json={"epoch": 1})
|
|
return httpx.Response(409, json={"detail": "no epoch here"})
|
|
|
|
stubborn = ChatApiClient("http://x", transport=httpx.MockTransport(handler))
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
await stubborn.send_turn("永不重试")
|
|
|
|
|
|
async def test_broken_stream_raises_connection_error():
|
|
"""The SSE connection itself dies before any terminal frame."""
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
if request.url.path == "/api/health":
|
|
return httpx.Response(200, json={"epoch": 1})
|
|
return httpx.Response(
|
|
200, headers={"content-type": "text/event-stream"},
|
|
content=b'data: {"t": "chat.delta", "epoch": 1, "turn_id": "' + b"t" * 36 +
|
|
b'", "delta": "\\u4f60"}\n\n', # stream then simply ends
|
|
)
|
|
|
|
client = ChatApiClient("http://x", transport=httpx.MockTransport(handler))
|
|
with pytest.raises(ChatApiConnectionError):
|
|
await client.send_turn("断流")
|
|
|
|
|
|
async def test_voice_turn_cancels_active_text_turn_same_gate(api):
|
|
"""Voice and text contend through the server's one same-turn gate.
|
|
|
|
HEL-908: the cancelled side now ends with an explicit chat.cancelled
|
|
terminal frame — a clean end, not a connection error the caller might
|
|
mistake for a transient failure and retry (the HEL-907 defect family)."""
|
|
app, provider, _ = api
|
|
|
|
outcome: dict = {}
|
|
|
|
async def slow_text() -> None:
|
|
text_client = ChatApiClient("http://asgi", transport=httpx.ASGITransport(app=app))
|
|
outcome["turn"] = await text_client.send_turn("文字说一半")
|
|
|
|
provider.mode = "slow"
|
|
task = asyncio.create_task(slow_text())
|
|
await asyncio.sleep(1.0) # the text turn is active and hanging after its first delta
|
|
provider.mode = "" # its generator stays parked; the voice turn can complete fast
|
|
voice_client = ChatApiClient("http://asgi", transport=httpx.ASGITransport(app=app))
|
|
turn = await voice_client.send_turn("语音插进来")
|
|
assert turn.completed
|
|
provider.mode = "" # (already reset above; kept for clarity on early exits)
|
|
await asyncio.wait_for(task, timeout=10)
|
|
superseded = outcome["turn"]
|
|
assert superseded.cancelled and not superseded.completed, \
|
|
"the cancelled side ends on the explicit chat.cancelled terminal frame"
|
|
|
|
rows = app.state.store.query(
|
|
"SELECT role, text, status, turn_id FROM chat_message ORDER BY id"
|
|
)
|
|
slow = [m for m in rows if m["text"] == "文字说一半"]
|
|
assert slow and slow[0]["status"] == "cancelled", "barge-in cancels the text row"
|
|
assert not [m for m in rows if m["role"] == "companion" and m["turn_id"] == slow[0]["turn_id"]], \
|
|
"no phantom companion row for the aborted text turn"
|
|
assert any(m["role"] == "companion" and m["turn_id"] == turn.turn_id and m["status"] == "sent"
|
|
for m in rows), "the voice turn itself completes as her archived reply"
|
|
|
|
|
|
# ── SSE parser unit cases: chunk boundaries ──────────────────────────────────
|
|
|
|
|
|
class _FakeResponse:
|
|
def __init__(self, chunks: list[bytes]):
|
|
self._chunks = chunks
|
|
|
|
async def aiter_bytes(self):
|
|
for chunk in self._chunks:
|
|
yield chunk
|
|
|
|
|
|
async def collect(chunks: list[bytes]) -> list[dict]:
|
|
return [frame async for frame in _sse_frames(_FakeResponse(chunks))]
|
|
|
|
|
|
def frame(event: dict) -> str:
|
|
return "data: " + json.dumps(event, ensure_ascii=False) + "\n\n"
|
|
|
|
|
|
async def test_sse_split_inside_multibyte_and_crlf_and_multiline():
|
|
event1 = {"t": "chat.delta", "epoch": 1, "turn_id": "t1", "delta": "你也好呀"}
|
|
event2 = {"t": "chat.final", "epoch": 1, "turn_id": "t1",
|
|
"utterance": {"text": "你也好呀"}, "receipt": {"token": "x" * 48}}
|
|
blob = (frame(event1) + frame(event2)).encode()
|
|
# split inside a multibyte char, mid-frame, and across CRLF boundaries
|
|
cuts = [1, 7, 8, 9, 13, 21, len(blob) // 2, len(blob) - 3, len(blob) - 1]
|
|
chunks, prev = [], 0
|
|
for cut in cuts + [len(blob)]:
|
|
chunks.append(blob[prev:cut])
|
|
prev = cut
|
|
parsed = await collect(chunks)
|
|
assert [p["t"] for p in parsed] == ["chat.delta", "chat.final"]
|
|
assert parsed[1]["receipt"]["token"] == "x" * 48
|
|
|
|
|
|
async def test_sse_multiline_data_and_comment_lines_and_crlf_separator():
|
|
payload = (": keep-alive\r\n"
|
|
"data: {\"t\": \"chat.delta\",\r\n"
|
|
"data: \"delta\": \"两行\"}\r\n\r\n"
|
|
"data: {\"t\": \"chat.final\"}\r\n\r\n")
|
|
parsed = await collect([payload.encode("utf-8")])
|
|
assert parsed[0]["delta"] == "两行"
|
|
assert parsed[1]["t"] == "chat.final"
|
|
|
|
|
|
async def test_sse_invalid_json_frame_skipped_not_fatal():
|
|
blob = b"data: not-json\n\ndata: {\"t\": \"chat.final\"}\n\n"
|
|
parsed = await collect([blob])
|
|
assert parsed == [{"t": "chat.final"}]
|
|
|
|
|
|
async def test_sse_utf8_split_across_chunks():
|
|
text = "晚安"
|
|
first, second = text[:1], text[1:]
|
|
chunks = [b"data: {\"t\": \"chat.delta\", \"delta\": \"" + first.encode(),
|
|
second.encode() + b"\"}\n\n"]
|
|
parsed = await collect(chunks)
|
|
assert parsed[0]["delta"] == text
|
|
|
|
|
|
# ── receipt gate: playback-completion only ──────────────────────────────────
|
|
|
|
|
|
def _receipt(epoch=3):
|
|
return {"message_id": 11, "turn_id": "t" * 36, "epoch": epoch,
|
|
"token": "t" * 48, "text_hash": "h" * 64}
|
|
|
|
|
|
def test_ack_only_after_full_playout():
|
|
assert should_ack(played_to_end=True, interrupted=False, failed=False,
|
|
receipt=_receipt(), current_epoch=3)
|
|
# everything else must fail closed
|
|
assert not should_ack(played_to_end=True, interrupted=True, failed=False,
|
|
receipt=_receipt(), current_epoch=3), "barge-in never acks"
|
|
assert not should_ack(played_to_end=True, interrupted=False, failed=True,
|
|
receipt=_receipt(), current_epoch=3), "playback/generation failure never acks"
|
|
assert not should_ack(played_to_end=False, interrupted=False, failed=False,
|
|
receipt=_receipt()), "not finished playing never acks"
|
|
assert not should_ack(played_to_end=True, interrupted=False, failed=False,
|
|
receipt=None), "phi error turn has no receipt"
|
|
assert not should_ack(played_to_end=True, interrupted=False, failed=False,
|
|
receipt={"token": "x"}, current_epoch=3), "incomplete receipt never acks"
|
|
assert not should_ack(played_to_end=True, interrupted=False, failed=False,
|
|
receipt=_receipt(epoch=2), current_epoch=3), "stale epoch never acks"
|
|
|
|
|
|
def test_receipt_before_playout_not_possible_by_construction():
|
|
# should_ack requires played_to_end; a receipt arriving mid-playout has no
|
|
# path to True even if every other field is perfect.
|
|
for interrupted in (True, False):
|
|
assert not should_ack(played_to_end=False, interrupted=interrupted, failed=False,
|
|
receipt=_receipt())
|
|
|
|
|
|
async def test_ledger_binds_handle_to_its_stream():
|
|
ledger = TurnLedger()
|
|
handle = type("H", (), {"created_at": 1.0})()
|
|
stray = type("S", (), {"created_at": 0.5, "text": "old"})()
|
|
# A stream without exact attribution binds to nobody (fail-closed)...
|
|
assert ledger.note_stream(stray, None, 0.5) is False
|
|
assert ledger.unattributed == [stray]
|
|
ledger.note_handle(handle, 1.0)
|
|
stream = type("S", (), {"created_at": 1.2, "text": "new"})()
|
|
# ...and a noted handle claims ONLY the stream of its own generation.
|
|
assert ledger.note_stream(stream, handle, 1.2) is True
|
|
assert ledger.stream_for(handle) is stream, "exactly its own generation's stream"
|
|
assert ledger.stream_for(handle) is None, "binding consumed exactly once"
|
|
|
|
|
|
async def test_receipt_sender_confirms_once_and_reports_server_refusal():
|
|
seen: list[tuple[str, dict]] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen.append((request.url.path, json.loads(request.read())))
|
|
return httpx.Response(200, json={"accepted": True})
|
|
|
|
client = ChatApiClient("http://x", transport=httpx.MockTransport(handler))
|
|
sender = ReceiptSender(client)
|
|
receipt = _receipt()
|
|
assert await sender.confirm(receipt["turn_id"], receipt) is True
|
|
assert await sender.confirm(receipt["turn_id"], receipt) is True # no second POST
|
|
assert len(seen) == 1 and seen[0][0] == "/api/chat/received"
|
|
|
|
def refusing(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(404, json={"detail": "gone"})
|
|
|
|
refuser = ReceiptSender(
|
|
ChatApiClient("http://x", transport=httpx.MockTransport(refusing)))
|
|
assert await refuser.confirm(receipt["turn_id"], receipt) is False
|
|
|
|
|
|
# ── real-socket cancel path: barge-in closes the HTTP stream immediately ─────
|
|
|
|
|
|
@pytest.fixture
|
|
def live_server(api, tmp_path):
|
|
app, provider, _ = api
|
|
# HEL-879: an app's lifespan has exactly one owner — the api fixture's
|
|
# TestClient still holds its lifespan, so the live server gets its OWN app
|
|
# instance on the same file-backed database (companion row included) and
|
|
# the same provider; the old fixture entered one app's lifespan twice
|
|
# concurrently, the exact defect the app now refuses loudly.
|
|
router = build_router(providers={t: provider for t in (Tier.HIGH, Tier.STANDARD, Tier.LOW)},
|
|
budget=engineering_budget())
|
|
served = create_app(db_path=tmp_path / "adapter.db", router=router,
|
|
life_config=LifeConfig(enabled=True, poll_seconds=3600),
|
|
semantic_mode="stub")
|
|
probe = socket.socket()
|
|
probe.bind(("127.0.0.1", 0))
|
|
port = probe.getsockname()[1]
|
|
probe.close() # free the port for uvicorn to bind
|
|
config = uvicorn.Config(served, host="127.0.0.1", port=port, log_level="error")
|
|
server = uvicorn.Server(config)
|
|
thread = threading.Thread(target=server.run, daemon=True)
|
|
thread.start()
|
|
for _ in range(100):
|
|
try:
|
|
httpx.get(f"http://127.0.0.1:{port}/api/health", timeout=1)
|
|
break
|
|
except Exception:
|
|
time.sleep(0.1)
|
|
yield f"http://127.0.0.1:{port}", served, provider
|
|
server.should_exit = True
|
|
thread.join(timeout=5)
|
|
|
|
|
|
async def test_barge_in_cancels_http_stream_and_server_row(live_server):
|
|
"""Voice-turn cancellation = barge-in: the adapter's in-flight HTTP stream
|
|
closes immediately; the server marks the row cancelled without waiting for
|
|
the next ASR final (it never comes in this test)."""
|
|
url, app, provider = live_server
|
|
provider.mode = "slow" # reply hangs after first delta
|
|
client = ChatApiClient(url)
|
|
got_delta = asyncio.Event()
|
|
task = asyncio.create_task(
|
|
client.send_turn("说一半我要打断", on_delta=lambda _: got_delta.set())
|
|
)
|
|
await asyncio.wait_for(got_delta.wait(), timeout=10)
|
|
task.cancel() # the interruption
|
|
with pytest.raises(asyncio.CancelledError):
|
|
await task
|
|
# server notices the disconnect and fail-closes the row
|
|
for _ in range(50):
|
|
rows = httpx.get(f"{url}/api/messages").json()["messages"]
|
|
row = next((m for m in rows if m["text"] == "说一半我要打断"), None)
|
|
if row and row["status"] == "cancelled":
|
|
break
|
|
await asyncio.sleep(0.2)
|
|
else:
|
|
row = next((m for m in httpx.get(f"{url}/api/messages").json()["messages"]
|
|
if m["text"] == "说一半我要打断"), None)
|
|
assert row and row["status"] == "cancelled"
|
|
assert not any(m["role"] == "companion" and m["turn_id"] == row["turn_id"]
|
|
for m in rows), "no phantom companion row for the aborted turn"
|
|
# and the next turn is clean (no ghost gate)
|
|
provider.mode = ""
|
|
nxt = await client.send_turn("打断之后")
|
|
assert nxt.completed
|
|
await client.aclose()
|