Files
aigc-home/calliope-backend/tests/test_agent_interaction.py
T
Benjiyaya f8faa72653 fix(agent): per-session event seq, parsed SSE echo, and vision attachments
Implements the intent of open PRs #48, #49, and part of #47 (all reviewed
against the working tree — none of the three bugs were fixed yet):

- #48 append_event returned cur.lastrowid (global row id) as the event's
  seq, while the INSERT allocates a per-session seq. The two only agree on
  a single-session DB, so ask_user's question_seq never matched
  latest_open_question() on a real install — card clicks never recorded an
  approval. The allocated seq is now read back from the inserted row; the
  atomic single-INSERT allocation is unchanged.

- #49 the agent.message SSE echo carried the raw agent_messages row
  (tool_args_json / tool_result_json strings, no parsed keys), while GET
  /sessions/{id} hands clients the parsed shape. _persist_message now
  returns parsed tool_args / tool_result alongside the raw keys, so the
  ask_user question card renders live instead of after a reload.

- #47 (partial) main.py carried the version as two literals that drifted
  from pyproject twice (1.3.2 reported 1.2.1; 1.4.1 reported 1.4.0). The
  version is now single-sourced in calliope.__version__, consumed by both
  FastAPI and /api/health; pyproject uses hatchling dynamic version. The
  PR's importlib.metadata approach was avoided: the venv's editable
  dist-info still says 0.1.0, which would report a third stale version.

- Vision attachments: user-attached images now project as OpenAI
  image_url content parts (data URLs read from disk), so vision-capable
  models see the actual pixels instead of a path line (derive_chat_rows /
  project_user_content, tested in test_agent_event_log).

Regression tests: per-session seq with a diverging second session, the
full card-answer flow across sessions, and the health version matching
the single-sourced metadata (tests/test_agent_interaction.py,
tests/test_projects.py).
2026-09-08 05:20:23 +08:00

347 lines
13 KiB
Python

"""Tests for the interaction plugin (ask_user) + structured approvals."""
from __future__ import annotations
import pytest
from calliope.agent.harness import log as session_log
from calliope.agent.harness import policy
from calliope.agent.harness.plugins import interaction
from calliope.agent.harness.registry import ToolContext
from calliope.agent.harness.tools import execute_tool
@pytest.fixture
def session(client):
resp = client.post("/api/agent/sessions", json={})
assert resp.status_code in (200, 201), resp.text
return resp.json()
def _ctx(session, project_id=None):
return ToolContext(session_id=session["id"], project_id=project_id)
def test_ask_user_appends_question_event(session):
result = execute_tool(
_ctx(session),
"ask_user",
{"question": "Generate images for all 3 characters?", "options": ["Yes, generate", "No, not yet"], "scope": "render"},
)
import asyncio
result = asyncio.run(result) if hasattr(result, "__await__") else result
assert result["ok"] is True
assert result["awaiting_user_input"] is True
events = session_log.read_events(session["id"])
asked = [e for e in events if e.type == session_log.QUESTION_ASKED]
assert len(asked) == 1
assert asked[0].data["scope"] == "render"
assert asked[0].data["options"] == ["Yes, generate", "No, not yet"]
def test_ask_user_validates_input(session):
import asyncio
ctx = _ctx(session)
too_few = asyncio.run(execute_tool(ctx, "ask_user", {"question": "q", "options": ["yes"]}))
assert too_few["ok"] is False
empty = asyncio.run(execute_tool(ctx, "ask_user", {"question": "", "options": ["a", "b"]}))
assert empty["ok"] is False
bad_scope = asyncio.run(
execute_tool(ctx, "ask_user", {"question": "q", "options": ["a", "b"], "scope": "bogus"})
)
assert bad_scope["ok"] is False
def test_structured_approval_grants_render(session):
import asyncio
ctx = _ctx(session)
asked = asyncio.run(
execute_tool(ctx, "ask_user", {"question": "Render?", "options": ["Yes", "No"], "scope": "render"})
)
session_log.append_event(
session["id"],
session_log.QUESTION_ANSWERED,
{"question_seq": asked["question_seq"], "answer": "Yes", "scope": "render"},
)
session_log.append_event(session["id"], session_log.USER_MESSAGE, {"content": "Yes"})
assert policy.has_structured_approval(ctx, "render") is True
assert policy.user_allows_render(ctx) is True
# scope isolation: a render approval is not a destructive approval
assert policy.has_structured_approval(ctx, "destructive_replace") is False
def test_structured_approval_negative_answer_grants_nothing(session):
import asyncio
ctx = _ctx(session)
asked = asyncio.run(
execute_tool(ctx, "ask_user", {"question": "Render?", "options": ["Yes", "No"], "scope": "render"})
)
session_log.append_event(
session["id"],
session_log.QUESTION_ANSWERED,
{"question_seq": asked["question_seq"], "answer": "No", "scope": "render"},
)
session_log.append_event(session["id"], session_log.USER_MESSAGE, {"content": "No"})
assert policy.has_structured_approval(ctx, "render") is False
assert policy.user_allows_render(ctx) is False
def test_scope_info_never_grants(session):
import asyncio
ctx = _ctx(session)
asked = asyncio.run(
execute_tool(ctx, "ask_user", {"question": "Continue?", "options": ["Yes", "No"], "scope": "info"})
)
session_log.append_event(
session["id"],
session_log.QUESTION_ANSWERED,
{"question_seq": asked["question_seq"], "answer": "Yes", "scope": "info"},
)
session_log.append_event(session["id"], session_log.USER_MESSAGE, {"content": "Yes"})
assert policy.has_structured_approval(ctx, "info") is False
assert policy.has_structured_approval(ctx, "render") is False
def test_open_question_is_visible_for_card_ui(session):
import asyncio
ctx = _ctx(session)
asked = asyncio.run(
execute_tool(ctx, "ask_user", {"question": "Which style?", "options": ["Painterly", "Anime"]})
)
open_q = interaction.latest_open_question(session["id"])
assert open_q is not None
assert open_q["seq"] == asked["question_seq"]
assert open_q["options"] == ["Painterly", "Anime"]
session_log.append_event(
session["id"],
session_log.QUESTION_ANSWERED,
{"question_seq": asked["question_seq"], "answer": "Anime", "scope": "info"},
)
assert interaction.latest_open_question(session["id"]) is None
def test_denied_tool_result_carries_reason_code(client, session, monkeypatch):
"""Task 3: guard refusals expose a stable machine token."""
import asyncio
from calliope.agent.harness import get_registry
project = client.post("/api/projects", json={"title": "Guard Codes"}).json()
ctx = ToolContext(session_id=session["id"], project_id=project["id"])
# A non-empty project + replace on a destructive tool  destructive guard
registry = get_registry()
tool = registry.get("generate_story")
assert tool is not None and tool.destructive
client.post(f"/api/projects/{project['id']}/characters", json={"name": "Someone"})
result = asyncio.run(
registry.execute(ctx, "generate_story", {"replace": True})
)
assert result["ok"] is False
assert result.get("reason_code") == "guard_destructive_replace"
def test_answer_to_message_records_structured_approval(client, session):
"""The runner records question/answered before the user/message echo, so
policy.latest_answer's adjacency rule sees a valid card answer."""
import asyncio
from calliope.agent.harness import policy
ctx = ToolContext(session_id=session["id"], project_id=None)
asked = asyncio.run(
execute_tool(ctx, "ask_user", {"question": "Render now?", "options": ["Yes", "No"], "scope": "render"})
)
# MessageCreate.answer_to flows through the API the way the card click does.
# The turn starts a real run, which needs an LLM ÔÇö the message is persisted
# regardless; the approval event is what we assert on.
resp = client.post(
f"/api/agent/sessions/{session['id']}/messages",
json={"content": "Yes", "answer_to": asked["question_seq"]},
)
# The run may fail fast without an LLM configured; the API contract only
# requires the message to be accepted (200) ÔÇö the answer event must exist.
assert resp.status_code == 200, resp.text
assert policy.has_structured_approval(ctx, "render") is True
assert policy.user_allows_render(ctx) is True
def test_card_option_sentence_grants_approval():
"""Options are full sentences ("Yes, replace with ~40 shorter scenes");
the first word decides. Exact 'yes' still grants; 'No, ' refuses."""
from calliope.agent.harness.policy import _answer_is_affirmative
assert _answer_is_affirmative("Yes, replace with ~40 shorter scenes") is True
assert _answer_is_affirmative("yes go!") is True
assert _answer_is_affirmative("Sure, do it") is True
assert _answer_is_affirmative("Go ahead") is True
assert _answer_is_affirmative("No, keep existing scenes and add more at the end") is False
assert _answer_is_affirmative("Just update durations, don't change content") is False
assert _answer_is_affirmative("no") is False
assert _answer_is_affirmative("") is False
def test_ask_user_logs_tool_result_before_turn_end(client, session):
"""The card derives from the persisted ask_user tool ROW, which needs the
tool/result event ÔÇö the loop must log + emit the result BEFORE ending the
turn, or the UI shows a stuck 'working' with no question (the reported
bug: user had to Stop/refresh to see anything)."""
import asyncio
import json
from calliope.agent.harness.loop import run_turn
class _AskStream:
def __aiter__(self):
async def gen():
yield {
"type": "tool_call",
"tool_call": {
"id": "c1",
"function": {
"name": "ask_user",
"arguments": json.dumps(
{
"question": "Regenerate the script?",
"options": ["Yes, replace", "No, keep"],
"scope": "destructive_replace",
}
),
},
},
}
return gen()
class _Client:
def chat_stream(self, messages, temperature=0.4, tools=None):
return _AskStream()
async def close(self):
return None
from calliope.agent.harness import loop as loop_mod
orig_llm = loop_mod._llm_for_role
loop_mod._llm_for_role = lambda role: _Client()
ctx = ToolContext(session_id=session["id"], project_id=None)
try:
history: list = []
asyncio.run(run_turn(ctx, history, max_iterations=5))
finally:
loop_mod._llm_for_role = orig_llm
events = session_log.read_events(session["id"])
types = [e.type for e in events]
assert session_log.TOOL_RESULT in types, types
tr_idx = max(i for i, t in enumerate(types) if t == session_log.TOOL_RESULT)
end_idx = max(i for i, t in enumerate(types) if t == session_log.TURN_END)
assert tr_idx < end_idx, "tool/result must precede turn/end"
tr = events[tr_idx]
assert tr.data["tool_name"] == "ask_user"
assert tr.data["result"]["options"] == ["Yes, replace", "No, keep"]
assert tr.data["result"]["question"] == "Regenerate the script?"
end = events[end_idx]
assert end.data["status"] == "awaiting_input", end.data
def test_append_event_returns_per_session_seq(client, session):
"""append_event must return the per-session seq it allocated, not the
table-wide row id. Once any other session has events the two diverge, and
ask_user's question_seq (taken from the returned event) must still match
what read_events / latest_open_question report for the same row."""
other = client.post("/api/agent/sessions", json={}).json()
for i in range(3):
session_log.append_event(other["id"], "assistant/message", {"content": f"noise {i}"})
ev = session_log.append_event(
session["id"],
session_log.QUESTION_ASKED,
{"question": "q?", "options": ["a", "b"], "scope": "info"},
)
events = session_log.read_events(session["id"])
stored = [e for e in events if e.type == session_log.QUESTION_ASKED]
assert len(stored) == 1
assert ev.seq == stored[0].seq
assert interaction.latest_open_question(session["id"])["seq"] == ev.seq
def test_card_answer_matches_question_when_other_sessions_exist(client, session):
"""The card click flow end to end with a second, busier session in the
table: question_seq handed to the card must still record question/answered."""
import asyncio
other = client.post("/api/agent/sessions", json={}).json()
for i in range(5):
session_log.append_event(other["id"], "assistant/message", {"content": f"noise {i}"})
ctx = ToolContext(session_id=session["id"], project_id=None)
asked = asyncio.run(
execute_tool(
ctx,
"ask_user",
{"question": "Render now?", "options": ["Yes", "No"], "scope": "render"},
)
)
resp = client.post(
f"/api/agent/sessions/{session['id']}/messages",
json={"content": "Yes", "answer_to": asked["question_seq"]},
)
assert resp.status_code == 200, resp.text
events = session_log.read_events(session["id"])
answered = [e for e in events if e.type == session_log.QUESTION_ANSWERED]
assert len(answered) == 1
assert answered[0].data["question_seq"] == asked["question_seq"]
assert policy.has_structured_approval(ctx, "render") is True
def test_live_message_echo_carries_parsed_tool_result(client, session):
"""PR #49: the agent.message SSE echo must carry parsed tool_args /
tool_result (same shape as GET rows), or the ask_user card stays hidden
until a reload."""
from calliope.agent.harness.runner import AgentRunner
runner = AgentRunner()
captured: list[dict] = []
async def fake_publish(event_type, data):
if event_type == "agent.message":
captured.append(data)
from calliope.events import bus as bus_mod
orig_publish = bus_mod.event_bus.publish
async def publish_capture(event_type, data=None):
await fake_publish(event_type, data or {})
bus_mod.event_bus.publish = publish_capture
try:
row = runner._persist_message(
session["id"],
role="tool",
content="",
tool_name="ask_user",
tool_args={"question": "Render?", "options": ["Yes", "No"]},
tool_result={"ok": True, "question_seq": 5, "options": ["Yes", "No"], "scope": "render"},
append_event=False,
)
finally:
bus_mod.event_bus.publish = orig_publish
assert row["tool_args_json"], "raw json string key still present"
assert row["tool_result_json"], "raw json string key still present"
# PR #49: parsed keys must ride along on the echoed row
assert row["tool_args"] == {"question": "Render?", "options": ["Yes", "No"]}
assert row["tool_result"]["options"] == ["Yes", "No"]
assert row["tool_result"]["scope"] == "render"