diff --git a/calliope-backend/src/calliope/agent/harness/log.py b/calliope-backend/src/calliope/agent/harness/log.py index 81eea7cb..a16872f3 100644 --- a/calliope-backend/src/calliope/agent/harness/log.py +++ b/calliope-backend/src/calliope/agent/harness/log.py @@ -624,6 +624,25 @@ def max_turn_number(session_id: int) -> int: STEERING_INJECT_HEADER = "[STEERING — user message sent while you work]" +def text_of_content(content: Any) -> str: + """The text of a projected content value (string or multimodal parts list). + + `project_user_content` returns an OpenAI parts list when a message has + usable attachments. Routing/planning code that only needs the words (the + orchestrator goal, swarm synthesis) must go through this helper — calling + string methods on the parts list directly is the multimodal-goal crash + (`'list' object has no attribute 'strip'`). + """ + if isinstance(content, list): + texts = [ + str(p.get("text") or "") + for p in content + if isinstance(p, dict) and p.get("type") == "text" + ] + return "\n".join(t for t in texts if t).strip() + return content if isinstance(content, str) else "" + + def steering_user_content(d: dict[str, Any]) -> Any: """Projected steering content with the [STEERING] header. diff --git a/calliope-backend/src/calliope/agent/harness/loop.py b/calliope-backend/src/calliope/agent/harness/loop.py index 40510b88..076de23b 100644 --- a/calliope-backend/src/calliope/agent/harness/loop.py +++ b/calliope-backend/src/calliope/agent/harness/loop.py @@ -318,7 +318,7 @@ async def run_turn( if reasoning_text: tool_msg_data["reasoning"] = reasoning_text log_append(session_log.ASSISTANT_MESSAGE, tool_msg_data) - for tc in tool_calls: + for tc_idx, tc in enumerate(tool_calls): name = tc["function"]["name"] raw_args = tc["function"].get("arguments") or "{}" try: @@ -425,6 +425,66 @@ async def run_turn( # ask_user ends the turn: the agent's question waits for the # user's answer, so the loop must not burn steps polling. if isinstance(result, dict) and result.get("awaiting_user_input"): + # Close the batch first: any REMAINING calls of this step + # never executed, and an assistant tool_calls message with + # missing results is an invalid request sequence (strict + # OpenAI-compatible servers 400 the next turn). Synthesize + # skipped receipts for the un-executed call ids. + for pending in tool_calls[tc_idx + 1:]: + skipped = { + "ok": False, + "skipped": True, + "error": ( + "Not executed — this turn paused for the user's " + "answer to an ask_user question in the same step. " + "Re-issue the call after they answer if still needed." + ), + } + pname = pending["function"]["name"] + praw_args = pending["function"].get("arguments") or "{}" + try: + pargs = json.loads(praw_args) if praw_args.strip() else None + except json.JSONDecodeError: + pargs = None + log_append( + session_log.TOOL_CALL, + { + "turn": turn_no, + "step": iteration, + "call_id": pending["id"], + "tool_name": pname, + "arguments": praw_args, + "agent_name": agent_name, + }, + ) + log_append( + session_log.TOOL_RESULT, + { + "turn": turn_no, + "step": iteration, + "call_id": pending["id"], + "tool_name": pname, + "result": skipped, + "agent_name": agent_name, + }, + ) + messages.append( + { + "role": "tool", + "tool_call_id": pending["id"], + "content": json.dumps(skipped, ensure_ascii=False), + } + ) + await emit( + { + "role": "tool", + "agent_name": agent_name, + "tool_name": pname, + "tool_args": pargs if isinstance(pargs, dict) else None, + "tool_result": skipped, + "content": "", + } + ) log_append(session_log.STEP_END, {"turn": turn_no, "step": iteration}) turn_status = "awaiting_input" raise _AwaitingInput() diff --git a/calliope-backend/src/calliope/agent/harness/orchestrator.py b/calliope-backend/src/calliope/agent/harness/orchestrator.py index e72a1e0b..daf68b1b 100644 --- a/calliope-backend/src/calliope/agent/harness/orchestrator.py +++ b/calliope-backend/src/calliope/agent/harness/orchestrator.py @@ -313,9 +313,15 @@ async def orchestrate( max_user_turns=MAX_HISTORY_USER_TURNS, max_chars=_history_char_budget(), ) - goal = next( - (m["content"] for m in reversed(derived) if m.get("role") == "user"), - "", + # Multimodal user content (image/video/document attachments) projects as + # an OpenAI parts LIST, not a string — extract the text part. Calling + # string methods on the raw value crashed every linked-session turn that + # carried an attachment ('list' object has no attribute 'strip'). + goal = session_log.text_of_content( + next( + (m["content"] for m in reversed(derived) if m.get("role") == "user"), + "", + ) ) if ctx.project_id is None: diff --git a/calliope-backend/tests/test_agent_harness.py b/calliope-backend/tests/test_agent_harness.py index 1e48027c..c6d4e236 100644 --- a/calliope-backend/tests/test_agent_harness.py +++ b/calliope-backend/tests/test_agent_harness.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import json +from pathlib import Path from typing import Any import pytest @@ -737,3 +738,253 @@ def test_sub_agent_failure_names_empty_str_exception(client): ] assert fails, "no failure message recorded" assert "ReadError" in fails[0]["content"] + + +def test_multimodal_goal_does_not_crash_orchestrate(client, tmp_path): + """A linked-session turn whose user message carries an image attachment + projects content as an OpenAI parts LIST. orchestrate must extract the + text part instead of calling string methods on the list (the crash that + failed EVERY linked-session attachment turn with + AttributeError: 'list' object has no attribute 'strip').""" + r = client.post("/api/agent/sessions", json={"title": "mm-goal"}) + sid = r.json()["id"] + pid = _make_project(client) + + import calliope.agent.harness.orchestrator as orch + from calliope.agent.harness import log as session_log + + # A readable image under assets_dir is required for the parts-list + # projection (unresolvable attachments degrade to the text appendix). + import struct + import zlib + + assets = Path(settings.assets_dir) + assets.mkdir(parents=True, exist_ok=True) + png = assets / "ref.png" + sig = b"\x89PNG\r\n\x1a\n" + + def _chunk(tag, data): + return ( + struct.pack(">I", len(data)) + tag + data + + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + ) + + ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0) + png.write_bytes( + sig + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", zlib.compress(b"\x00\xff\x00\x00")) + + _chunk(b"IEND", b"") + ) + + session_log.append_event( + sid, + session_log.USER_MESSAGE, + { + "content": "update the character to match this reference", + "attachments": [{"kind": "image", "path": str(png), "name": "ref.png"}], + }, + ) + + goal_seen: dict[str, Any] = {} + + # The attachment appendix ("[Calliope context]\nattached: …") puts newlines + # in the extracted goal, so this message is NOT trivial — the planner call + # must be stubbed too, or the test makes a real network LLM call (the + # source of its flakiness: solo runs degraded to single, full-file runs + # sometimes got a swarm plan and run_turn was never called). + class _SingleModeClient: + async def chat(self, *a, **kw): + return json.dumps({"mode": "single", "tasks": [], "note": ""}) + + async def close(self): + return None + + async def fake_run_turn(ctx, history, *, on_message=None, **kw): + goal_seen["called"] = True + return "handled the attachment" + + orig_client = orch.LLMClient + orig_run_turn = orch.run_turn + orch.LLMClient = lambda: _SingleModeClient() + orch.run_turn = fake_run_turn + try: + ctx = ToolContext(session_id=sid, project_id=pid) + # The crash fired at _is_trivial_goal BEFORE the fix — the raw parts + # list hit .strip() the moment the goal was read, planner or not. + out = asyncio.run(orchestrate(ctx, [], session_id=sid)) + finally: + orch.LLMClient = orig_client + orch.run_turn = orig_run_turn + + assert goal_seen.get("called") is True + assert out == "handled the attachment" + + +def test_text_of_content_extracts_parts_and_strings(client): + """The projection helper behind the multimodal-goal fix: strings pass + through, parts lists join their text parts, anything else is ''.""" + from calliope.agent.harness import log as session_log + + assert session_log.text_of_content("plain goal") == "plain goal" + parts = [ + {"type": "text", "text": "first"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,xxx"}}, + {"type": "text", "text": "second"}, + ] + assert session_log.text_of_content(parts) == "first\nsecond" + assert session_log.text_of_content(None) == "" + assert session_log.text_of_content(42) == "" + + +def test_swarm_goal_from_multimodal_message_plans(client, tmp_path): + """The swarm path must also survive a multimodal goal: the planner call + receives the extracted TEXT, not the raw parts list.""" + r = client.post("/api/agent/sessions", json={"title": "mm-swarm"}) + sid = r.json()["id"] + pid = _make_project(client) + + import calliope.agent.harness.orchestrator as orch + from calliope.agent.harness import log as session_log + + # A real .md document under assets_dir projects as a parts list (big text + # block in part[0]) — same list-shaped content, no binary needed. + assets = Path(settings.assets_dir) + assets.mkdir(parents=True, exist_ok=True) + doc = assets / "script-ref.md" + doc.write_text( + "# Scene notes\nFADE IN on the neon rooftop.\n\nKira waits at the rail.", + encoding="utf-8", + ) + + session_log.append_event( + sid, + session_log.USER_MESSAGE, + { + "content": "turn the attached reference into scenes and render them", + "attachments": [{"kind": "document", "path": str(doc), "name": "script-ref.md"}], + }, + ) + + seen_goals: list[str] = [] + + class _PlannerCaptureClient: + async def chat(self, messages, temperature=0.2, **kw): + seen_goals.append(messages[-1]["content"]) + return json.dumps({"mode": "single", "tasks": [], "note": ""}) + + async def close(self): + return None + + async def fake_run_turn(ctx, history, *, on_message=None, **kw): + return "single-loop handled the swarm-path goal" + + orig_client = orch.LLMClient + orig_run_turn = orch.run_turn + orch.LLMClient = lambda: _PlannerCaptureClient() + orch.run_turn = fake_run_turn + try: + ctx = ToolContext(session_id=sid, project_id=pid) + asyncio.run(orchestrate(ctx, [], session_id=sid)) + finally: + orch.LLMClient = orig_client + orch.run_turn = orig_run_turn + + assert seen_goals, "planner was never called" + assert isinstance(seen_goals[0], str) + # The extracted TEXT reached the planner: the user's words and the + # document marker — never a stringified parts list ("[{'type': + # 'text', ...}]"). + assert "turn the attached reference into scenes" in seen_goals[0] + assert "[Script document: script-ref.md]" in seen_goals[0] + assert "[{'type'" not in seen_goals[0] + + +def test_ask_user_mid_batch_closes_tool_exchange(client): + """When ask_user pauses the turn mid tool-batch, the calls AFTER it never + executed. Their call ids must still receive skipped tool results — an + assistant tool_calls message with missing results is an invalid request + sequence that strict OpenAI-compatible servers reject on the next turn.""" + r = client.post("/api/agent/sessions", json={"title": "ask-mid-batch"}) + sid = r.json()["id"] + + from calliope.agent.harness.loop import run_turn + + # Step 1: batch [ask_user, list_memories]; step 2 (never reached): text. + class _BatchStream: + def __init__(self, calls): + self._calls = calls + + def __aiter__(self): + return self._iter() + + async def _iter(self): + for c in self._calls: + yield {"type": "tool_call", "tool_call": c} + if not self._calls: + yield {"type": "delta", "content": "after answer"} + + class _Client: + def __init__(self): + self.turn = 0 + + async def close(self): + return None + + def chat_stream(self, messages, temperature=0.4, tools=None): + self.turn += 1 + if self.turn == 1: + return _BatchStream( + [ + { + "id": "call_a", + "function": { + "name": "ask_user", + "arguments": json.dumps( + { + "question": "Proceed?", + "options": ["Yes", "No"], + "scope": "info", + } + ), + }, + }, + { + "id": "call_b", + "function": {"name": "list_memories", "arguments": "{}"}, + }, + ] + ) + return _BatchStream([]) + + from calliope.agent.harness import loop as loop_mod + + orig_llm = loop_mod._llm_for_role + orig_turn_no = loop_mod._next_turn_number + loop_mod._llm_for_role = lambda role: _Client() + loop_mod._next_turn_number = lambda s: 1 + try: + ctx = ToolContext(session_id=sid, project_id=None) + history: list[dict[str, Any]] = [] + out = asyncio.run(run_turn(ctx, history, max_iterations=3)) + finally: + loop_mod._llm_for_role = orig_llm + loop_mod._next_turn_number = orig_turn_no + + # Turn ended for the user's answer. + assert out == "" # ask_user path returns empty final text + from calliope.agent.harness import log as session_log + + events = session_log.read_events(sid) + results = { + e.data["call_id"]: e.data["result"] + for e in events + if e.type == session_log.TOOL_RESULT + } + assert results.get("call_a", {}).get("awaiting_user_input") is True + skipped_b = results.get("call_b") + assert skipped_b is not None, "call_b never received a synthetic result" + assert skipped_b.get("skipped") is True + # History stays a valid request sequence: every tool_call has a result. + calls = [e.data["call_id"] for e in events if e.type == session_log.TOOL_CALL] + assert set(calls) == set(results.keys()) + tool_msgs = [m for m in history if m.get("role") == "tool"] + assert {m["tool_call_id"] for m in tool_msgs} == set(calls) diff --git a/docs/changelogs/2026-09-24-clip-workflow-runtime-budget.md b/docs/changelogs/2026-09-24-clip-workflow-runtime-budget.md index 92f2c380..1119dcba 100644 --- a/docs/changelogs/2026-09-24-clip-workflow-runtime-budget.md +++ b/docs/changelogs/2026-09-24-clip-workflow-runtime-budget.md @@ -5,7 +5,8 @@ Three reported issues fixed — [#66](https://github.com/benjiyaya/Calliope/issu [#64](https://github.com/benjiyaya/Calliope/issues/64) (scripts ignored the story's target length), and [#67](https://github.com/benjiyaya/Calliope/issues/67) (a directory path -passed as a video input crashed ComfyUI with `Errno 21`). +passed as a video input crashed ComfyUI with `Errno 21`) — plus two agent-loop +bugs found in the Sep 24 loop evaluation. ## #66 — Workflow choice is per clip @@ -53,10 +54,32 @@ opaque `ValueError: [Errno 21] Is a directory`. fails the tool call with correction guidance instead of creating a doomed job. +## Agent loop — attachments crash and dangling tool calls + +Two crashes found by the Sep 24 agent-loop evaluation, folded into this line: + +- **Attached images/videos/documents crashed every linked-session turn.** + Multimodal user content projects as an OpenAI parts *list*, but the + orchestrator read that list as the turn's goal and called `.strip()` on it + (`AttributeError: 'list' object has no attribute 'strip'`) — any + project-linked session turn with an attachment failed 100% of the time. + A new `text_of_content()` helper extracts the text part, and both the + single-loop and swarm (planner) paths now survive attachments. +- **`ask_user` mid tool-batch left dangling tool calls.** When the model + issued `ask_user` alongside other tool calls, the rest of the batch never + executed but also never received results — an assistant `tool_calls` + message with missing results is an invalid request sequence that strict + OpenAI-compatible servers reject on the next turn. The pause path now + synthesizes skipped tool results for the un-executed ids before ending + the turn. + ## Testing - `tests/test_script_chunking.py` — per-scene budget present in both prompt builders; oversized-script durations shrink toward the target. - `tests/test_video_continue.py` — directory and missing-file rejection, bare-name passthrough, existing-file upload still works. -- Full backend suite: 534 passed; `svelte-check`: 0 errors. +- `tests/test_agent_harness.py` — multimodal goal (single + swarm paths), + `text_of_content` shapes, mid-batch `ask_user` closure; planner stubbed so + the tests never make a real LLM call. +- Full backend suite: 538 passed; `svelte-check`: 0 errors. diff --git a/docs/changelogs/README.md b/docs/changelogs/README.md index 29d0ce3a..400495ea 100644 --- a/docs/changelogs/README.md +++ b/docs/changelogs/README.md @@ -2,7 +2,7 @@ Dated notes for GitHub releases. Newest first. -- [2026-09-24 clip workflow + runtime budget](./2026-09-24-clip-workflow-runtime-budget.md) — workflow per clip (not per scene); target runtime is a budget that can shrink; media inputs that are directories fail fast with a node-naming error (1.5.7) +- [2026-09-24 clip workflow + runtime budget](./2026-09-24-clip-workflow-runtime-budget.md) — workflow per clip (not per scene); target runtime is a budget that can shrink; media inputs that are directories fail fast with a node-naming error; attachment turns no longer crash the orchestrator, `ask_user` closes its tool batch (1.5.7) - [2026-09-22 project name alias patch](./2026-09-22-project-name-alias.md) — `name` accepted for a project's title on the REST API and agent tools; fixes storyline builds dead-ending on a 422 (1.5.6) - [2026-09-22 swarm render/scene fixes](./2026-09-22-swarm-render-scene-fixes.md) — video-role render tools reachable (ask_user recovery); story role gains scene tools; deep-link sessions no longer orphan; canvas auto-seeds new entities (1.5.5) - [2026-09-19 steering + Asset Library + hardening](./2026-09-19-steering-library-hardening.md) — mid-run steering; Asset Library page (filter + lazy loading); canvas delete that deletes; agent-loop hardening batch; Build Scene Brief/Cut gates (1.5.4)