Files
CyberGril/tests/test_reasoning_separation_hel908.py
T
xiaomoandmultica-agent 958c6962fb HEL-908 R1/R2 rework (HEL-909 review): whole-turn reasoning gate + cg.interrupt data channel
R1 — reasoning separation truly closed. The streaming hold leaked wholesale
on real token arrival (review A1: 'private reasoning '*20 beyond the 128-char
bound, no sentence ender; review A2: a Chinese sentence ender released the
hold) AND the adapter consumed the stray closer, so the app-level turn split
could never recover — the previous 'persistence and receipt always clean'
claim was false and is retracted. Per the engineering ruling the adapter now
has two explicit endpoint modes (GOV-006, no heuristics, no new engine):
buffer_turn (TurnReasoningGate) emits NOTHING during generation, splits the
whole turn at stream end and only then delivers the clean visible text; it
is set explicitly for all tiers in config/models.yaml (first visible text
waits for the full generation; cancelled/timed-out turns emit nothing and
end exactly as before — recorded and test-pinned, low latency NOT claimed).
stream (default, backward compatible) keeps explicit-tag/field semantics and
passes stray closers through unmodified so the turn-level split keeps its
signal. Both review strings and the recorded id=12 stream replay clean at
every chunk size through the real adapter and the real app into chat.delta,
the archive and the receipt; normal zh/en preserved byte-for-byte; unclosed
explicit thinking still fails honestly.

R2 — an authoritative interrupt when the LLM already finalized and TTS is
still playing (review evidence C: zero interrupts, old audio played out).
The page's existing hush() entry (ChatPanel untouched) now also publishes
{"v":1} on the room's own reliable data channel topic cg.interrupt
(fire-and-forget: a dropped publish never blocks or breaks the text send);
the worker subscribes data_received, validates topic/sender/payload and
calls session.interrupt(force=True) — the SDK's own programmatic interrupt,
force confirmed against the installed 1.8.8 source (non-force raises
RuntimeError on allow_interruptions=False speech). Failures log loudly; the
listener is unloaded via add_shutdown_callback; the server-cancel path in
llm_chat_api also interrupts with force and logs failures.

Verification (offline, zero real requests): real-SDK harness 8/8 including a
real SpeechHandle with a genuinely pending playout wait released by the
cg.interrupt message after a clean chat.final (evidence-C shape) plus the
full validation matrix; .venv311 full suite 480 passed / 8 skipped (2
test_r3 failures pre-exist at 5625de6); web npm ci from the existing lock,
typecheck 0 errors, lint 0 errors + 1 pre-existing warning, vitest 148/148,
build ok; ruff clean; check_requirements vs origin/main and b8ec832 with the
corrected handoff.

Co-authored-by: multica-agent <github@multica.ai>
2026-10-28 22:21:16 +08:00

309 lines
12 KiB
Python

"""HEL-908/909 reasoning separation tests: the adapter seam between raw model
output and user-visible content, against the REAL ReasoningSeparator /
TurnReasoningGate and the REAL OpenAICompatLLM adapter (fake SSE transports,
zero network).
Two explicit modes (models.yaml chooses per endpoint):
- ``buffer_turn`` (the product's tiers): the recorded implicit-open shape —
reasoning with NO ``<think>`` opener and a stray ``</think>`` closer — can
NEVER be separated safely while streaming (HEL-909 review: sentence-enders
and >128-char holds leaked on real token arrival, and the adapter consumed
the closer so the turn-level split could not recover). The gate emits
nothing during generation, splits the whole turn at the end, and only the
clean visible text is ever emitted.
- ``stream`` (default): explicit ``<think>`` blocks and the DeepSeek-style
``reasoning_content`` field; a stray closer passes through UNMODIFIED so
api/app.py's turn-level split keeps its signal.
The recorded evidence (HEL-907 companion.db row id=12 / tts-window-tail.log)
and both HEL-909 review reproductions are replayed verbatim at every chunk
size.
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
import httpx
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from cybergril.providers.llm.openai_compat import OpenAICompatLLM
from cybergril.providers.llm.reasoning import (
ReasoningSeparator,
TurnReasoningGate,
split_reasoning,
)
# companion.db row id=12 (call3, turn 3a67b253) — verbatim recorded stream.
RECORDED_RAW = (
"Considering the limited time by the user, I have to give the solution "
"based on the thinking directly now.\n</think>\n\n"
"大圣,您这定身法没点准啊,我可不是什么妖精妖怪,我是一行代码,"
"连肉都没有,怕硌着您的金箍棒。"
)
RECORDED_REASONING = (
"Considering the limited time by the user, I have to give the solution "
"based on the thinking directly now.\n</think>"
)
RECORDED_VISIBLE = (
"大圣,您这定身法没点准啊,我可不是什么妖精妖怪,我是一行代码,"
"连肉都没有,怕硌着您的金箍棒。"
)
# HEL-909 review reproductions A1/A2: the common shapes the old hold leaked.
A1_RAW = "private reasoning " * 20 + "</think>回答。"
A2_RAW = "我先想一下。内部思考</think>回答。"
def run_gate(raw: str, chunk_size: int) -> tuple[list[str], str]:
"""Feed raw through a whole-turn gate; returns (visible deltas, reasoning)."""
gate = TurnReasoningGate()
emitted: list[str] = []
for i in range(0, len(raw), chunk_size):
visible, _ = gate.feed(raw[i : i + chunk_size])
emitted.append(visible)
visible, _ = gate.finish()
emitted.append(visible)
return emitted, gate.reasoning + split_reasoning(raw)[1]
def run_separator(raw: str, chunk_size: int) -> tuple[str, str]:
separator = ReasoningSeparator()
visible_parts: list[str] = []
for i in range(0, len(raw), chunk_size):
v, _ = separator.feed(raw[i : i + chunk_size])
visible_parts.append(v)
v, _ = separator.finish()
visible_parts.append(v)
return "".join(visible_parts), separator.reasoning
# ── buffer_turn: the recorded and review shapes never leak ───────────────────
@pytest.mark.parametrize("raw", [RECORDED_RAW, A1_RAW, A2_RAW],
ids=["recorded-id12", "review-A1", "review-A2"])
@pytest.mark.parametrize("chunk", [1, 2, 3, 5, 7, 13, 1000])
def test_buffer_turn_emits_only_clean_text_at_any_chunk_size(raw, chunk):
emitted, reasoning = run_gate(raw, chunk)
assert "".join(emitted) in ("回答。", RECORDED_VISIBLE)
# nothing AT ALL is emitted before the finish — the stream cannot leak
assert emitted[:-1] == [""] * (len(emitted) - 1)
assert reasoning.endswith("</think>"), "the reasoning landed on the diagnostics side"
@pytest.mark.parametrize("chunk", [1, 2, 3, 5, 7, 13, 1000])
def test_buffer_turn_recorded_stream_zero_leak(chunk):
emitted, reasoning = run_gate(RECORDED_RAW, chunk)
assert "".join(emitted) == RECORDED_VISIBLE
assert reasoning == RECORDED_REASONING
def test_buffer_turn_unclosed_explicit_think_yields_nothing():
gate = TurnReasoningGate()
gate.feed("<think>只想不说到结尾")
visible, reasoning = gate.finish()
assert visible == ""
assert "<think>" in reasoning
def test_buffer_turn_normal_chinese_and_english_preserved():
zh = "今天水母网我背,你负责举着泡泡喊“出发”。\n\n抓完水母我们就坐在海滩上。"
en = ("Sure, listening to you. A reply that happens to be English is "
"normal content and must survive the gate untouched.")
for text in (zh, en):
gate = TurnReasoningGate()
gate.feed(text)
visible, _ = gate.finish()
assert visible == text
assert gate.reasoning == ""
def test_buffer_turn_separates_field_reasoning_too():
gate = TurnReasoningGate()
gate.feed_field("先在心里盘算")
gate.feed(RECORDED_RAW)
visible, reasoning = gate.finish()
assert visible == RECORDED_VISIBLE
assert reasoning.startswith("先在心里盘算")
# ── stream mode: explicit tags / field; stray closer passes through ─────────
@pytest.mark.parametrize("split_at", range(1, len("<think>隐藏</think>可见")))
def test_stream_explicit_tags_split_across_chunks(split_at):
text = "<think>隐藏</think>可见"
head, tail = text[:split_at], text[split_at:]
separator = ReasoningSeparator()
v1, _ = separator.feed(head)
v2, _ = separator.feed(tail)
v3, _ = separator.finish()
assert v1 + v2 + v3 == "可见"
def test_stream_explicit_block_anywhere_is_filtered():
visible, reasoning = run_separator("先说。<think>中途思考</think>后说。", 3)
assert visible == "先说。后说。"
assert "中途思考" in reasoning
def test_stream_unclosed_think_yields_no_visible_text():
separator = ReasoningSeparator()
visible, _ = separator.feed("<think>只想不说到结尾")
tail, _ = separator.finish()
assert visible + tail == ""
def test_stream_normal_chinese_and_english_passthrough():
zh = "哈哈,那走吧,派大星。今天水母网我背。"
en = "Plain English reply, kept fully. Second sentence.\nAnd a newline."
for text in (zh, en):
visible, reasoning = run_separator(text, 4)
assert visible == text
assert reasoning == ""
def test_stream_stray_closer_passes_through_untouched():
"""The HEL-909 defect was the adapter DELETING the closer; the signal must
survive so the app-level turn split can still act on it."""
visible, reasoning = run_separator(A2_RAW, 5)
assert visible == A2_RAW, "stream mode passes stray closers through"
assert reasoning == ""
# ── turn-level final split (shared definition; real persistence guard) ──────
def test_split_recorded_row_is_clean():
visible, reasoning = split_reasoning(RECORDED_RAW)
assert visible == RECORDED_VISIBLE
assert reasoning == RECORDED_REASONING
def test_split_review_shapes_are_clean():
assert split_reasoning(A1_RAW)[0] == "回答。"
assert split_reasoning(A2_RAW)[0] == "回答。"
def test_split_explicit_blocks_and_unclosed():
assert split_reasoning("<think>a</think>答") == ("答", "<think>a</think>")
assert split_reasoning("<think>a") == ("", "<think>a")
assert split_reasoning("正常回答") == ("正常回答", "")
visible, reasoning = split_reasoning("<think>a</think>一<think>b</think>二")
assert visible == "一二"
assert reasoning == "<think>a</think><think>b</think>"
# ── REAL OpenAICompatLLM adapter with fake upstream SSE ─────────────────────
def sse_response(deltas: list[dict]) -> httpx.Response:
lines = []
for d in deltas:
lines.append("data: " + json.dumps({"choices": [{"delta": d}]}, ensure_ascii=False))
lines.append("data: [DONE]")
return httpx.Response(
200, headers={"content-type": "text/event-stream"},
content=("\n\n".join(lines) + "\n\n").encode(),
)
def chunked(raw: str, size: int) -> list[dict]:
return [{"content": raw[i : i + size]} for i in range(0, len(raw), size)]
def make_provider(mode: str, pieces: list[dict]) -> OpenAICompatLLM:
def handler(request: httpx.Request) -> httpx.Response:
if request.url.path.endswith("/models"):
return httpx.Response(200, json={"data": [{"id": "m"}]})
return sse_response(pieces)
provider = OpenAICompatLLM(base_url="http://isolated/v1", model="m", reasoning=mode)
provider._client = httpx.AsyncClient(
base_url="http://isolated/v1", transport=httpx.MockTransport(handler)
)
return provider
async def collect(provider: OpenAICompatLLM) -> tuple[list[str], str, int]:
"""Returns (visible delta sequence — empty strings kept in place so the
timing of emissions is visible, reasoning, index of first non-empty)."""
text_deltas: list[str] = []
reasoning = ""
async for delta in provider.stream(
[type("M", (), {"role": "user", "content": "你好"})()]
):
text_deltas.append(delta.text)
reasoning += delta.reasoning
first = next((i for i, t in enumerate(text_deltas) if t), len(text_deltas))
return text_deltas, reasoning, first
@pytest.mark.parametrize("mode,raw", [
("buffer_turn", RECORDED_RAW),
("buffer_turn", A1_RAW),
("buffer_turn", A2_RAW),
])
@pytest.mark.parametrize("chunk", [1, 7, 60])
async def test_adapter_buffer_turn_never_leaks_review_shapes(mode, raw, chunk):
provider = make_provider(mode, chunked(raw, chunk))
try:
text_deltas, reasoning, _ = await collect(provider)
assert "".join(text_deltas) in ("回答。", RECORDED_VISIBLE)
# the whole generation ran inside the gate: the first visible delta is
# the LAST content delta (documented wait cost, not a hidden leak)
non_empty = [t for t in text_deltas if t]
assert len(non_empty) == 1
assert "private reasoning" not in "".join(text_deltas)
assert "我先想一下" not in "".join(text_deltas)
assert "Considering" not in "".join(text_deltas)
assert reasoning # the reasoning went to the diagnostics channel only
finally:
await provider.aclose()
async def test_adapter_stream_mode_filters_explicit_blocks_only():
pieces = [{"content": "先说。"}, {"content": "<th"}, {"content": "ink>隐藏"},
{"content": "</th"}, {"content": "ink>后说。"}]
provider = make_provider("stream", pieces)
try:
text_deltas, reasoning, _ = await collect(provider)
assert "".join(text_deltas) == "先说。后说。"
assert "隐藏" in reasoning
finally:
await provider.aclose()
async def test_adapter_stream_mode_routes_reasoning_field():
pieces = [{"reasoning_content": "先盘算"}, {"content": RECORDED_VISIBLE[:4]},
{"content": RECORDED_VISIBLE[4:]}]
provider = make_provider("stream", pieces)
try:
text_deltas, reasoning, _ = await collect(provider)
assert "".join(text_deltas) == RECORDED_VISIBLE
assert "先盘算" in reasoning
finally:
await provider.aclose()
async def test_adapter_buffer_turn_routes_reasoning_field():
pieces = [{"reasoning_content": "先盘算"}, *chunked(RECORDED_RAW, 13)]
provider = make_provider("buffer_turn", pieces)
try:
text_deltas, reasoning, _ = await collect(provider)
assert "".join(text_deltas) == RECORDED_VISIBLE
assert reasoning.startswith("先盘算")
finally:
await provider.aclose()
async def test_adapter_rejects_unknown_mode():
with pytest.raises(ValueError):
OpenAICompatLLM(base_url="http://x/v1", model="m", reasoning="nonsense")