Files
aigc-home/calliope-backend/tests/test_llm_json.py
T
Justo Stoll ffc1424566 Stream-internal LLMClient: chat()/chat_with_tools() consume chat_stream
Extends the 1.2.1 Agent-loop fix to every LLM caller: with completions
streamed internally, the 120 s httpx timeout bounds the gap BETWEEN chunks
(liveness) instead of total generation time, so a thinking model that
streams reasoning_content can legitimately work past 120 s while a dead
server still fails fast. chat_stream gains response_format= with an
in-stream HTTP-400 drop-ladder (response_format first, then tools); servers
that reject stream:true entirely fall back to one plain blocking POST.
A reasoning-only stream (zero content accumulated) raises ValueError so
generate_structured's json_object retry fires instead of returning blank.
2026-08-24 07:05:35 +02:00

456 lines
16 KiB
Python

"""Regression tests for LLM JSON handling (user-reported bugs).
Covers:
1. LM Studio-style servers rejecting `response_format` (HTTP 400) — the
client must retry without the field instead of failing the whole request.
2. "Extra data: line 1 column 308" — models emitting valid JSON followed by
trailing prose/chatter; extract_json must recover the object.
"""
from __future__ import annotations
import json
import httpx
import pytest
from calliope.agent.llm import LLMClient, extract_json
# ---------- extract_json ----------
def test_extract_json_plain():
assert extract_json('{"a": 1}') == {"a": 1}
def test_extract_json_fenced_block():
text = 'Here is your JSON:\n```json\n{"a": 1, "b": [2, 3]}\n```\nDone!'
assert extract_json(text) == {"a": 1, "b": [2, 3]}
def test_extract_json_trailing_prose_extra_data():
# The exact failure class the user hit: valid object + trailing text
text = '{"title": "The Long Road", "beats": []} I hope this helps!'
assert extract_json(text)["title"] == "The Long Road"
def test_extract_json_leading_prose():
text = 'Sure! Here is the storyline you asked for: {"title": "X"}'
assert extract_json(text)["title"] == "X"
def test_extract_json_two_objects_takes_first():
text = '{"first": true} {"second": true}'
assert extract_json(text) == {"first": True}
def test_extract_json_rejects_garbage():
with pytest.raises(ValueError):
extract_json("no json here at all")
def test_extract_json_empty():
with pytest.raises(ValueError):
extract_json(" ")
# ---------- LLMClient response_format fallback ----------
def _sse_body(message: dict) -> bytes:
"""Encode a full assistant message as a minimal SSE stream."""
chunks = []
if message.get("reasoning_content"):
chunks.append(
{"choices": [{"delta": {"reasoning_content": message["reasoning_content"]}}]}
)
if message.get("content"):
chunks.append({"choices": [{"delta": {"content": message["content"]}}]})
chunks.append({"choices": [{"delta": {}, "finish_reason": "stop"}]})
lines = [b"data: " + json.dumps(c).encode() for c in chunks]
lines.append(b"data: [DONE]")
return b"\n".join(lines) + b"\n"
class _FakeRouter:
"""httpx mock transport handler that 400s any request containing response_format.
Serves SSE when the request asks for stream:true, plain JSON otherwise —
chat()/chat_with_tools() stream internally, so both shapes occur.
"""
def __init__(self, content: str, reject_response_format: bool) -> None:
self.content = content
self.reject = reject_response_format
self.requests: list[dict] = []
async def __call__(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content.decode())
self.requests.append(body)
if self.reject and "response_format" in body:
return httpx.Response(
400,
json={"error": {"message": "response_format is not supported"}},
)
if body.get("stream"):
return httpx.Response(
200,
content=_sse_body({"content": self.content}),
headers={"content-type": "text/event-stream"},
)
return httpx.Response(
200,
json={"choices": [{"message": {"content": self.content}}]},
)
class _SequenceRouter(_FakeRouter):
"""Returns each reply in order instead of a fixed one."""
def __init__(self, contents: list[str], reject_response_format: bool) -> None:
super().__init__(contents[0], reject_response_format)
self.contents = contents
self.calls = 0
async def __call__(self, request: httpx.Request) -> httpx.Response:
content = self.contents[min(self.calls, len(self.contents) - 1)]
self.calls += 1
self.content = content
return await super().__call__(request)
async def test_chat_retries_without_response_format_on_400(monkeypatch):
router = _FakeRouter(content='{"ok": true}', reject_response_format=True)
transport = httpx.MockTransport(router)
client = LLMClient()
monkeypatch.setattr(client, "client", httpx.AsyncClient(transport=transport))
text = await client.chat(
[{"role": "user", "content": "hi"}],
response_format={"type": "json_object"},
)
assert text == '{"ok": true}'
assert len(router.requests) == 2
assert "response_format" in router.requests[0]
assert "response_format" not in router.requests[1]
async def test_generate_structured_no_response_format_by_default(monkeypatch):
# Default path must NOT send response_format (LM Studio compatibility)
router = _FakeRouter(content='{"a": 1}', reject_response_format=True)
transport = httpx.MockTransport(router)
client = LLMClient()
monkeypatch.setattr(client, "client", httpx.AsyncClient(transport=transport))
monkeypatch.setattr("calliope.agent.llm.LLMClient", lambda: client)
result = await generate_structured_public([{"role": "user", "content": "hi"}])
assert result == {"a": 1}
assert len(router.requests) == 1
assert "response_format" not in router.requests[0]
async def generate_structured_public(messages):
from calliope.agent.llm import generate_structured
return await generate_structured(messages)
async def test_generate_structured_recovers_via_json_mode_retry(monkeypatch):
# First reply is garbage prose; the json_object retry must rescue it.
router = _SequenceRouter(
["sorry, I cannot do that", '{"title": "Saved"}'],
reject_response_format=False,
)
transport = httpx.MockTransport(router)
client = LLMClient()
monkeypatch.setattr(client, "client", httpx.AsyncClient(transport=transport))
monkeypatch.setattr("calliope.agent.llm.LLMClient", lambda: client)
result = await generate_structured_public([{"role": "user", "content": "hi"}])
assert result == {"title": "Saved"}
assert len(router.requests) == 2
assert "response_format" not in router.requests[0]
assert router.requests[1].get("response_format") == {"type": "json_object"}
async def test_generate_structured_raises_when_both_attempts_fail(monkeypatch):
router = _SequenceRouter(
["all prose", "still prose"],
reject_response_format=False,
)
transport = httpx.MockTransport(router)
client = LLMClient()
monkeypatch.setattr(client, "client", httpx.AsyncClient(transport=transport))
monkeypatch.setattr("calliope.agent.llm.LLMClient", lambda: client)
with pytest.raises(ValueError):
await generate_structured_public([{"role": "user", "content": "hi"}])
# ---------- chat_stream: mid-stream error payloads ----------
def _sse_lines(*chunks: dict) -> list[bytes]:
out = []
for c in chunks:
out.append(b"data: " + json.dumps(c).encode())
out.append(b"data: [DONE]")
return out
class _StreamRouter:
"""Serves a scripted SSE stream."""
def __init__(self, chunks: list[dict]) -> None:
self.chunks = chunks
async def __call__(self, request: httpx.Request) -> httpx.Response:
lines = _sse_lines(*self.chunks)
body = b"\n".join(lines) + b"\n"
return httpx.Response(
200,
content=body,
headers={"content-type": "text/event-stream"},
)
async def test_chat_stream_surfaces_error_payload(monkeypatch):
"""A mid-stream {error: ...} chunk must raise, not end as a blank reply."""
router = _StreamRouter(
[
{"error": {"message": "model overloaded"}},
]
)
transport = httpx.MockTransport(router)
client = LLMClient()
monkeypatch.setattr(client, "client", httpx.AsyncClient(transport=transport))
events = []
with pytest.raises(RuntimeError, match="model overloaded"):
async for ev in client.chat_stream([{"role": "user", "content": "hi"}]):
events.append(ev)
assert events == [] # nothing yielded before the failure surfaced
async def test_chat_stream_surfaces_string_error(monkeypatch):
"""Non-dict error payloads degrade to str(), still raising."""
router = _StreamRouter([{"error": "bad gateway"}])
transport = httpx.MockTransport(router)
client = LLMClient()
monkeypatch.setattr(client, "client", httpx.AsyncClient(transport=transport))
with pytest.raises(RuntimeError, match="bad gateway"):
async for _ in client.chat_stream([{"role": "user", "content": "hi"}]):
pass
async def test_chat_stream_normal_tokens_unaffected(monkeypatch):
"""Happy path: deltas flow, done arrives, no error."""
router = _StreamRouter(
[
{"choices": [{"delta": {"content": "Hi"}}]},
{"choices": [{"delta": {"content": " there"}, "finish_reason": "stop"}]},
]
)
transport = httpx.MockTransport(router)
client = LLMClient()
monkeypatch.setattr(client, "client", httpx.AsyncClient(transport=transport))
events = [ev async for ev in client.chat_stream([{"role": "user", "content": "hi"}])]
types = [e["type"] for e in events]
assert types == ["delta", "delta", "done"]
assert events[0]["content"] == "Hi"
# ---------- reasoning-only replies (thinking models returning no `content`) ----------
# Reasoning models served via OpenAI-compatible endpoints can spend the whole
# completion in a reasoning channel (e.g. `reasoning_content`) and return a
# message with NO `content` key. That must be a retryable failure, not a
# KeyError bubbling up as an HTTP 500.
class _MessageRouter:
"""Returns full message dicts in sequence (to simulate reasoning-only replies).
Streamed requests get the message re-encoded as SSE deltas — a
reasoning-only message becomes a stream with reasoning chunks and no
content chunks, which is exactly what oMLX-style servers emit."""
def __init__(self, messages: list[dict]) -> None:
self.messages = messages
self.requests: list[dict] = []
async def __call__(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content.decode())
self.requests.append(body)
message = self.messages[min(len(self.requests) - 1, len(self.messages) - 1)]
if body.get("stream"):
return httpx.Response(
200,
content=_sse_body(message),
headers={"content-type": "text/event-stream"},
)
return httpx.Response(
200,
json={"choices": [{"message": message, "finish_reason": "length"}]},
)
async def test_chat_raises_value_error_on_missing_content(monkeypatch):
router = _MessageRouter(
[{"role": "assistant", "reasoning_content": "thinking forever..."}]
)
client = LLMClient()
monkeypatch.setattr(
client, "client", httpx.AsyncClient(transport=httpx.MockTransport(router))
)
with pytest.raises(ValueError, match="no content"):
await client.chat([{"role": "user", "content": "hi"}])
async def test_generate_structured_recovers_from_reasoning_only_reply(monkeypatch):
router = _MessageRouter(
[
{"role": "assistant", "reasoning_content": "hmm..."},
{"role": "assistant", "content": '{"title": "Saved"}'},
]
)
client = LLMClient()
monkeypatch.setattr(
client, "client", httpx.AsyncClient(transport=httpx.MockTransport(router))
)
monkeypatch.setattr("calliope.agent.llm.LLMClient", lambda: client)
result = await generate_structured_public([{"role": "user", "content": "hi"}])
assert result == {"title": "Saved"}
assert len(router.requests) == 2
assert router.requests[1].get("response_format") == {"type": "json_object"}
# ---------- streaming-internal chat: fallbacks and accumulation ----------
# chat()/chat_with_tools() now consume chat_stream, so the 120 s client
# timeout bounds the gap BETWEEN chunks (liveness), not total generation time.
# These tests pin the two fallback ladders and the accumulation contract.
class _NoStreamRouter:
"""A server that rejects stream:true outright (400), accepts blocking."""
def __init__(self, content: str) -> None:
self.content = content
self.requests: list[dict] = []
async def __call__(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content.decode())
self.requests.append(body)
if body.get("stream"):
return httpx.Response(
400, json={"error": {"message": "stream is not supported"}}
)
return httpx.Response(
200, json={"choices": [{"message": {"content": self.content}}]}
)
async def test_chat_falls_back_to_blocking_when_stream_rejected(monkeypatch):
router = _NoStreamRouter(content='{"ok": true}')
client = LLMClient()
monkeypatch.setattr(
client, "client", httpx.AsyncClient(transport=httpx.MockTransport(router))
)
text = await client.chat([{"role": "user", "content": "hi"}])
assert text == '{"ok": true}'
# one rejected stream attempt, then one blocking call
assert [b.get("stream", False) for b in router.requests] == [True, False]
async def test_chat_stream_drops_response_format_then_streams(monkeypatch):
"""The in-stream 400 ladder: response_format dropped, request retried."""
router = _FakeRouter(content='{"ok": true}', reject_response_format=True)
client = LLMClient()
monkeypatch.setattr(
client, "client", httpx.AsyncClient(transport=httpx.MockTransport(router))
)
text = await client.chat(
[{"role": "user", "content": "hi"}],
response_format={"type": "json_object"},
)
assert text == '{"ok": true}'
assert len(router.requests) == 2
assert "response_format" in router.requests[0]
assert "response_format" not in router.requests[1]
assert router.requests[1].get("stream") is True # still streaming, not blocking
class _ToolStreamRouter:
"""Streams one content delta plus one chunked tool call."""
def __init__(self) -> None:
self.requests: list[dict] = []
async def __call__(self, request: httpx.Request) -> httpx.Response:
body = json.loads(request.content.decode())
self.requests.append(body)
chunks = [
{"choices": [{"delta": {"content": "Queuing now."}}]},
{
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "call_1",
"function": {"name": "enqueue_asset", "arguments": '{"scene'},
}
]
}
}
]
},
{
"choices": [
{
"delta": {
"tool_calls": [
{"index": 0, "function": {"arguments": '_id": 3}'}}
]
},
"finish_reason": "tool_calls",
}
]
},
]
lines = [b"data: " + json.dumps(c).encode() for c in chunks]
lines.append(b"data: [DONE]")
return httpx.Response(
200,
content=b"\n".join(lines) + b"\n",
headers={"content-type": "text/event-stream"},
)
async def test_chat_with_tools_accumulates_streamed_tool_call(monkeypatch):
router = _ToolStreamRouter()
client = LLMClient()
monkeypatch.setattr(
client, "client", httpx.AsyncClient(transport=httpx.MockTransport(router))
)
msg = await client.chat_with_tools(
[{"role": "user", "content": "hi"}],
tools=[{"type": "function", "function": {"name": "enqueue_asset"}}],
)
assert msg["role"] == "assistant"
assert msg["content"] == "Queuing now."
assert len(msg["tool_calls"]) == 1
call = msg["tool_calls"][0]
assert call["function"]["name"] == "enqueue_asset"
assert json.loads(call["function"]["arguments"]) == {"scene_id": 3}