fix: agent tools follow Settings Queue timeout
The hidden 600s agent_tool_timeout_sec never appeared in Settings, so tool calls still died after Queue timeout was set to wait forever.
This commit is contained in:
@@ -308,9 +308,9 @@ def register(registry: ToolRegistry) -> None:
|
||||
executor=t_wait_for_jobs,
|
||||
category="video",
|
||||
requires_project=False,
|
||||
# Owns its own wait contract (timeout_sec → queue_poll_timeout_sec):
|
||||
# exempt from the per-tool wall-clock cap, which would otherwise
|
||||
# cut a legitimate long render wait short.
|
||||
# Owns its own wait (timeout_sec → queue_poll_timeout_sec, 0 = until
|
||||
# done): exempt from the same Queue timeout wrapping every other
|
||||
# tool, so a long render wait is not cut short by a second clock.
|
||||
long_running=True,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -48,9 +48,13 @@ EXPORT_VIDEO_DENIED_NAMES = frozenset({"export_video", "exportVideo"})
|
||||
|
||||
|
||||
def _tool_timeout_sec() -> float:
|
||||
"""Per-tool wall-clock cap from Settings; 0 = disabled."""
|
||||
"""Wall-clock cap for one agent tool call: Settings → Queue → Timeout.
|
||||
|
||||
Same value as the ComfyUI job poll (``queue_poll_timeout_sec``). 0 waits
|
||||
until the tool finishes.
|
||||
"""
|
||||
try:
|
||||
t = float(settings.agent_tool_timeout_sec)
|
||||
t = float(settings.queue_poll_timeout_sec)
|
||||
except (TypeError, ValueError, AttributeError):
|
||||
return 0.0
|
||||
return max(0.0, t)
|
||||
@@ -91,8 +95,9 @@ class ToolDefinition:
|
||||
category: str = "general"
|
||||
destructive: bool = False # flagged in descriptions; pre-execute guard uses it
|
||||
requires_approval: bool = False # HITL: blocked unless the user explicitly asked
|
||||
# True → exempt from the per-tool wall-clock timeout (agent_tool_timeout_sec);
|
||||
# the tool owns its own wait contract (e.g. wait_for_jobs → queue_poll_timeout_sec).
|
||||
# True → exempt from the Queue timeout wall clock;
|
||||
# the tool owns its own wait contract (e.g. wait_for_jobs uses the same
|
||||
# queue_poll_timeout_sec, including 0 = wait until done).
|
||||
long_running: bool = False
|
||||
|
||||
|
||||
@@ -281,8 +286,8 @@ class ToolRegistry:
|
||||
try:
|
||||
result = await self._execute_with_timeout(t, ctx, args)
|
||||
except TimeoutError:
|
||||
# Per-tool wall-clock cap (0 = disabled): a hung tool must stall
|
||||
# its step, not the session. CancelledError passes through so the
|
||||
# Per-tool wall clock is Settings → Queue → Timeout. 0 waits
|
||||
# until the tool finishes. CancelledError passes through so the
|
||||
# Stop button keeps working.
|
||||
return {
|
||||
"ok": False,
|
||||
|
||||
@@ -118,17 +118,15 @@ class Settings(BaseSettings):
|
||||
|
||||
queue_concurrency: int = 1
|
||||
queue_poll_interval_sec: float = 2.0
|
||||
# How long the worker keeps polling ComfyUI /history before giving up.
|
||||
# Long video workflows routinely exceed 10 minutes; 0 = poll until the job
|
||||
# completes or is cancelled. Default 1800s (30 min).
|
||||
# How long the worker keeps polling ComfyUI /history before giving up,
|
||||
# and the wall-clock cap on one agent tool call. Long video workflows
|
||||
# routinely exceed 10 minutes; 0 = wait until the job or tool finishes.
|
||||
# Default 1800s (30 min). wait_for_jobs opts out of the tool cap and
|
||||
# applies this same value itself.
|
||||
queue_poll_timeout_sec: float = 1800.0
|
||||
queue_max_retries: int = 2
|
||||
agent_max_steps: int = 24
|
||||
agent_hardening_prompt: str = DEFAULT_AGENT_HARDENING_PROMPT
|
||||
# Wall-clock cap on ONE tool execution inside the agent loop. Guards
|
||||
# against a hung tool stalling a session indefinitely; 0 = disabled.
|
||||
# Long-by-contract tools (wait_for_jobs) opt out via long_running=True.
|
||||
agent_tool_timeout_sec: float = 600.0
|
||||
# Character budget (~4 chars/token) for the derived LLM history — the
|
||||
# 40-turn cap alone can still overflow a context window with heavy
|
||||
# multi-step turns. Oldest whole turns drop first; 0 = disabled.
|
||||
@@ -329,7 +327,6 @@ class Settings(BaseSettings):
|
||||
"queue_max_retries": self.queue_max_retries,
|
||||
"agent_max_steps": self.agent_max_steps,
|
||||
"agent_hardening_prompt": self.agent_hardening_prompt,
|
||||
"agent_tool_timeout_sec": self.agent_tool_timeout_sec,
|
||||
"agent_history_char_budget": self.agent_history_char_budget,
|
||||
"h3_rewrite_extra_body": dict(self.h3_rewrite_extra_body or {}),
|
||||
"dry_run": bool(self.dry_run),
|
||||
@@ -414,7 +411,6 @@ class Settings(BaseSettings):
|
||||
"queue_max_retries": self.queue_max_retries,
|
||||
"agent_max_steps": self.agent_max_steps,
|
||||
"agent_hardening_prompt": self.agent_hardening_prompt,
|
||||
"agent_tool_timeout_sec": self.agent_tool_timeout_sec,
|
||||
"agent_history_char_budget": self.agent_history_char_budget,
|
||||
"h3_rewrite_extra_body": dict(self.h3_rewrite_extra_body or {}),
|
||||
"dry_run": bool(self.dry_run),
|
||||
|
||||
@@ -257,7 +257,7 @@ def test_clip_tools_in_swarm_roles():
|
||||
|
||||
|
||||
def test_tool_timeout_kills_hung_tool():
|
||||
"""A tool slower than agent_tool_timeout_sec returns ok:False with the
|
||||
"""A tool slower than Settings → Queue → Timeout returns ok:False with the
|
||||
timeout marker instead of stalling the step forever."""
|
||||
import asyncio as aio
|
||||
|
||||
@@ -288,9 +288,9 @@ def test_tool_timeout_kills_hung_tool():
|
||||
ToolDefinition(name="fast", description="", parameters={}, executor=_fast, requires_project=False)
|
||||
)
|
||||
|
||||
orig = settings.agent_tool_timeout_sec
|
||||
orig = settings.queue_poll_timeout_sec
|
||||
try:
|
||||
settings.agent_tool_timeout_sec = 0.2
|
||||
settings.queue_poll_timeout_sec = 0.2
|
||||
out = aio.run(reg.execute(ToolContext(session_id=1), "hang", {}))
|
||||
assert out["ok"] is False
|
||||
assert out.get("timeout") is True
|
||||
@@ -302,13 +302,13 @@ def test_tool_timeout_kills_hung_tool():
|
||||
out_fast = aio.run(reg.execute(ToolContext(session_id=1), "fast", {}))
|
||||
assert out_fast["ok"] is True
|
||||
|
||||
# 0 disables the timeout entirely.
|
||||
settings.agent_tool_timeout_sec = 0
|
||||
# 0 waits until the tool finishes.
|
||||
settings.queue_poll_timeout_sec = 0
|
||||
from calliope.agent.harness.registry import _tool_timeout_sec
|
||||
|
||||
assert _tool_timeout_sec() == 0.0
|
||||
finally:
|
||||
settings.agent_tool_timeout_sec = orig
|
||||
settings.queue_poll_timeout_sec = orig
|
||||
|
||||
|
||||
def test_wait_for_jobs_is_long_running():
|
||||
|
||||
@@ -1000,11 +1000,11 @@ export const de: Dict = {
|
||||
'settings.agentMaxStepsField': 'Max. Schritte',
|
||||
'settings.agentMaxStepsHint': 'Obergrenze für Tool-Aufrufe pro Agentenlauf.',
|
||||
'settings.queueSection': 'Job-Warteschlange',
|
||||
'settings.queueLead': 'Parallelität und Zeitlimits für Generierungsjobs.',
|
||||
'settings.queueLead': 'Parallelität und Zeitlimits für Generierungsjobs und Agent-Tool-Aufrufe.',
|
||||
'settings.concurrencyField': 'Gleichzeitige Jobs',
|
||||
'settings.pollIntervalField': 'Abfrageintervall (ms)',
|
||||
'settings.pollTimeoutField': 'Job-Timeout (s)',
|
||||
'settings.pollTimeoutHint': 'Zeit, nach der ein Job als hängend gilt.',
|
||||
'settings.pollTimeoutHint': 'Wie lange ein ComfyUI-Job oder ein Agent-Tool laufen darf. 0 wartet bis zum Ende.',
|
||||
'settings.maxRetriesField': 'Max. Wiederholungen',
|
||||
'settings.maxError': 'Zu hoch',
|
||||
'settings.minError': 'Zu niedrig',
|
||||
|
||||
@@ -1000,11 +1000,11 @@ export const en = {
|
||||
'settings.agentMaxStepsField': 'Max steps',
|
||||
'settings.agentMaxStepsHint': 'Cap on tool calls per agent run.',
|
||||
'settings.queueSection': 'Job queue',
|
||||
'settings.queueLead': 'Concurrency and timeouts for generation jobs.',
|
||||
'settings.queueLead': 'Concurrency and timeouts for generation jobs and agent tool calls.',
|
||||
'settings.concurrencyField': 'Concurrent jobs',
|
||||
'settings.pollIntervalField': 'Poll interval (ms)',
|
||||
'settings.pollTimeoutField': 'Job timeout (s)',
|
||||
'settings.pollTimeoutHint': 'Time before a job is considered stuck.',
|
||||
'settings.pollTimeoutHint': 'How long a ComfyUI job or one agent tool call may run. 0 waits until it finishes.',
|
||||
'settings.maxRetriesField': 'Max retries',
|
||||
'settings.maxError': 'Too high',
|
||||
'settings.minError': 'Too low',
|
||||
|
||||
@@ -1000,11 +1000,11 @@ export const es: Dict = {
|
||||
'settings.agentMaxStepsField': 'Pasos máximos',
|
||||
'settings.agentMaxStepsHint': 'Límite de llamadas a herramientas por ejecución del agente.',
|
||||
'settings.queueSection': 'Cola de tareas',
|
||||
'settings.queueLead': 'Concurrencia y tiempos de espera de las tareas de generación.',
|
||||
'settings.queueLead': 'Concurrencia y tiempos de espera de las tareas de generación y las llamadas de herramientas del agente.',
|
||||
'settings.concurrencyField': 'Tareas simultáneas',
|
||||
'settings.pollIntervalField': 'Intervalo de sondeo (ms)',
|
||||
'settings.pollTimeoutField': 'Tiempo límite de tarea (s)',
|
||||
'settings.pollTimeoutHint': 'Tiempo antes de considerar que una tarea se ha bloqueado.',
|
||||
'settings.pollTimeoutHint': 'Cuánto puede durar un trabajo de ComfyUI o una llamada de herramienta del agente. 0 espera hasta que termine.',
|
||||
'settings.maxRetriesField': 'Reintentos máximos',
|
||||
'settings.maxError': 'Demasiado alto',
|
||||
'settings.minError': 'Demasiado bajo',
|
||||
|
||||
@@ -1000,11 +1000,11 @@ export const fr: Dict = {
|
||||
'settings.agentMaxStepsField': 'Étapes max.',
|
||||
'settings.agentMaxStepsHint': "Limite d'appels d'outils par exécution de l'agent.",
|
||||
'settings.queueSection': "File d'attente",
|
||||
'settings.queueLead': 'Parallélisme et délais des tâches de génération.',
|
||||
'settings.queueLead': 'Parallélisme et délais des tâches de génération et des appels d’outils de l’agent.',
|
||||
'settings.concurrencyField': 'Tâches simultanées',
|
||||
'settings.pollIntervalField': 'Intervalle de vérification (ms)',
|
||||
'settings.pollTimeoutField': 'Délai de la tâche (s)',
|
||||
'settings.pollTimeoutHint': "Délai avant qu'une tâche soit considérée comme bloquée.",
|
||||
'settings.pollTimeoutHint': 'Durée maximale d’un job ComfyUI ou d’un appel d’outil de l’agent. 0 attend la fin.',
|
||||
'settings.maxRetriesField': 'Tentatives max.',
|
||||
'settings.maxError': 'Trop élevé',
|
||||
'settings.minError': 'Trop faible',
|
||||
|
||||
@@ -1000,11 +1000,11 @@ export const ja: Dict = {
|
||||
'settings.agentMaxStepsField': '最大ステップ数',
|
||||
'settings.agentMaxStepsHint': 'エージェント実行 1 回あたりのツール呼び出しの上限。',
|
||||
'settings.queueSection': 'ジョブキュー',
|
||||
'settings.queueLead': '生成ジョブの並列数とタイムアウト。',
|
||||
'settings.queueLead': '生成ジョブとエージェントのツール呼び出しの並列数とタイムアウト。',
|
||||
'settings.concurrencyField': '同時実行ジョブ数',
|
||||
'settings.pollIntervalField': 'ポーリング間隔(ms)',
|
||||
'settings.pollTimeoutField': 'ジョブタイムアウト(秒)',
|
||||
'settings.pollTimeoutHint': 'ジョブが停止したと見なされるまでの時間。',
|
||||
'settings.pollTimeoutHint': 'ComfyUI のジョブ、またはエージェントのツール呼び出し 1 回の上限。0 は完了まで待つ。',
|
||||
'settings.maxRetriesField': '最大再試行回数',
|
||||
'settings.maxError': '大きすぎます',
|
||||
'settings.minError': '小さすぎます',
|
||||
|
||||
@@ -1000,11 +1000,11 @@ export const ko: Dict = {
|
||||
'settings.agentMaxStepsField': '최대 단계',
|
||||
'settings.agentMaxStepsHint': '에이전트 실행당 도구 호출 수 제한입니다.',
|
||||
'settings.queueSection': '작업 대기열',
|
||||
'settings.queueLead': '생성 작업의 동시 실행 수와 시간 초과를 설정합니다.',
|
||||
'settings.queueLead': '생성 작업과 에이전트 도구 호출의 동시 실행 수와 시간 초과를 설정합니다.',
|
||||
'settings.concurrencyField': '동시 작업 수',
|
||||
'settings.pollIntervalField': '폴링 간격 (ms)',
|
||||
'settings.pollTimeoutField': '작업 시간 초과 (초)',
|
||||
'settings.pollTimeoutHint': '작업이 멈춘 것으로 간주되는 시간입니다.',
|
||||
'settings.pollTimeoutHint': 'ComfyUI 작업 또는 에이전트 도구 호출 한 번의 제한 시간. 0이면 끝날 때까지 기다립니다.',
|
||||
'settings.maxRetriesField': '최대 재시도',
|
||||
'settings.maxError': '너무 큽니다',
|
||||
'settings.minError': '너무 작습니다',
|
||||
|
||||
@@ -1000,11 +1000,11 @@ export const zh: Dict = {
|
||||
'settings.agentMaxStepsField': '最大步数',
|
||||
'settings.agentMaxStepsHint': '每次智能体运行的工具调用上限。',
|
||||
'settings.queueSection': '任务队列',
|
||||
'settings.queueLead': '生成任务的并发数与超时设置。',
|
||||
'settings.queueLead': '生成任务和代理工具调用的并发数与超时设置。',
|
||||
'settings.concurrencyField': '并发任务数',
|
||||
'settings.pollIntervalField': '轮询间隔(毫秒)',
|
||||
'settings.pollTimeoutField': '任务超时(秒)',
|
||||
'settings.pollTimeoutHint': '超过此时间任务将被视为卡住。',
|
||||
'settings.pollTimeoutHint': 'ComfyUI 任务或单次代理工具调用的最长时长。0 表示一直等到结束。',
|
||||
'settings.maxRetriesField': '最大重试次数',
|
||||
'settings.maxError': '太高',
|
||||
'settings.minError': '太低',
|
||||
|
||||
Reference in New Issue
Block a user