fix: #161 — wall-clock timeout for run_turn_loop; stalled turns now abort with stop_reason='timeout'

Previously, run_turn_loop was bounded only by max_turns (turn count). If
engine.submit_message stalled — slow provider, hung network, infinite
stream — the loop blocked indefinitely with no cancellation path. Claws
calling run_turn_loop in CI or orchestration had no reliable way to
enforce a deadline; the loop would hang until OS kill or human intervention.

Fix:
- Add timeout_seconds parameter to run_turn_loop (default None = legacy unbounded).
- When set, each submit_message call runs inside a ThreadPoolExecutor and is
  bounded by the remaining wall-clock budget (total across all turns, not per-turn).
- On timeout, synthesize a TurnResult with stop_reason='timeout' carrying the
  turn's prompt and routed matches so transcripts preserve orchestration context.
- Exhausted/negative budget short-circuits before calling submit_message.
- Legacy path (timeout_seconds=None) bypasses the executor entirely — zero
  overhead for callers that don't opt in.

CLI:
- Added --timeout-seconds flag to 'turn-loop' command.
- Exit code 2 when the loop terminated on timeout (vs 0 for completed),
  so shell scripts can distinguish 'done' from 'budget exhausted'.

Tests (tests/test_run_turn_loop_timeout.py, 6 tests):
- Legacy unbounded path unchanged (timeout_seconds=None never emits 'timeout')
- Hung submit_message aborted within budget (0.3s budget, 5s mock hang → exit <1.5s)
- Budget is cumulative across turns (0.6s budget, 0.4s per turn, not per-turn)
- timeout_seconds=0 short-circuits first turn without calling submit_message
- Negative timeout treated as exhausted (guard against caller bugs)
- Timeout TurnResult carries correct prompt, matches, UsageSummary shape

Full suite: 49/49 passing, zero regression.

Blocker: none. Closes ROADMAP #161.
This commit is contained in:
YeonGyu-Kim
2026-04-22 17:23:43 +09:00
parent 6a76cc7c08
commit 3f4d46d7b4
3 changed files with 274 additions and 9 deletions

View File

@@ -65,6 +65,12 @@ def build_parser() -> argparse.ArgumentParser:
loop_parser.add_argument('--limit', type=int, default=5)
loop_parser.add_argument('--max-turns', type=int, default=3)
loop_parser.add_argument('--structured-output', action='store_true')
loop_parser.add_argument(
'--timeout-seconds',
type=float,
default=None,
help='total wall-clock budget across all turns (#161). Default: unbounded.',
)
flush_parser = subparsers.add_parser('flush-transcript', help='persist and flush a temporary session transcript')
flush_parser.add_argument('prompt')
@@ -187,11 +193,21 @@ def main(argv: list[str] | None = None) -> int:
print(PortRuntime().bootstrap_session(args.prompt, limit=args.limit).as_markdown())
return 0
if args.command == 'turn-loop':
results = PortRuntime().run_turn_loop(args.prompt, limit=args.limit, max_turns=args.max_turns, structured_output=args.structured_output)
results = PortRuntime().run_turn_loop(
args.prompt,
limit=args.limit,
max_turns=args.max_turns,
structured_output=args.structured_output,
timeout_seconds=args.timeout_seconds,
)
for idx, result in enumerate(results, start=1):
print(f'## Turn {idx}')
print(result.output)
print(f'stop_reason={result.stop_reason}')
# Exit 2 when a timeout terminated the loop so claws can distinguish
# 'ran to completion' from 'hit wall-clock budget'.
if results and results[-1].stop_reason == 'timeout':
return 2
return 0
if args.command == 'flush-transcript':
engine = QueryEnginePort.from_workspace()

View File

@@ -1,11 +1,13 @@
from __future__ import annotations
import time
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
from dataclasses import dataclass
from .commands import PORTED_COMMANDS
from .context import PortContext, build_port_context, render_context
from .history import HistoryLog
from .models import PermissionDenial, PortingModule
from .models import PermissionDenial, PortingModule, UsageSummary
from .query_engine import QueryEngineConfig, QueryEnginePort, TurnResult
from .setup import SetupReport, WorkspaceSetup, run_setup
from .system_init import build_system_init_message
@@ -151,21 +153,100 @@ class PortRuntime:
persisted_session_path=persisted_session_path,
)
def run_turn_loop(self, prompt: str, limit: int = 5, max_turns: int = 3, structured_output: bool = False) -> list[TurnResult]:
def run_turn_loop(
self,
prompt: str,
limit: int = 5,
max_turns: int = 3,
structured_output: bool = False,
timeout_seconds: float | None = None,
) -> list[TurnResult]:
"""Run a multi-turn engine loop with optional wall-clock deadline.
Args:
prompt: The initial prompt to submit.
limit: Match routing limit.
max_turns: Maximum number of turns before stopping.
structured_output: Whether to request structured output.
timeout_seconds: Total wall-clock budget across all turns. When the
budget is exhausted mid-turn, a synthetic TurnResult with
``stop_reason='timeout'`` is appended and the loop exits.
``None`` (default) preserves legacy unbounded behaviour.
Returns:
A list of TurnResult objects. The final entry's ``stop_reason``
distinguishes ``'completed'``, ``'max_turns_reached'``,
``'max_budget_reached'``, or ``'timeout'``.
#161: prior to this change a hung ``engine.submit_message`` call would
block the loop indefinitely with no cancellation path, forcing claws to
rely on external watchdogs or OS-level kills. Callers can now enforce a
deadline and receive a typed timeout signal instead.
"""
engine = QueryEnginePort.from_workspace()
engine.config = QueryEngineConfig(max_turns=max_turns, structured_output=structured_output)
matches = self.route_prompt(prompt, limit=limit)
command_names = tuple(match.name for match in matches if match.kind == 'command')
tool_names = tuple(match.name for match in matches if match.kind == 'tool')
results: list[TurnResult] = []
for turn in range(max_turns):
turn_prompt = prompt if turn == 0 else f'{prompt} [turn {turn + 1}]'
result = engine.submit_message(turn_prompt, command_names, tool_names, ())
results.append(result)
if result.stop_reason != 'completed':
break
deadline = time.monotonic() + timeout_seconds if timeout_seconds is not None else None
# ThreadPoolExecutor is reused across turns so we cancel cleanly on exit.
executor = ThreadPoolExecutor(max_workers=1) if deadline is not None else None
try:
for turn in range(max_turns):
turn_prompt = prompt if turn == 0 else f'{prompt} [turn {turn + 1}]'
if deadline is None:
# Legacy path: unbounded call, preserves existing behaviour exactly.
result = engine.submit_message(turn_prompt, command_names, tool_names, ())
else:
remaining = deadline - time.monotonic()
if remaining <= 0:
results.append(self._build_timeout_result(turn_prompt, command_names, tool_names))
break
assert executor is not None
future = executor.submit(
engine.submit_message, turn_prompt, command_names, tool_names, ()
)
try:
result = future.result(timeout=remaining)
except FuturesTimeoutError:
# Best-effort cancel; submit_message may still finish in background
# but we never read its output. The engine's own state mutation
# is owned by the engine and not our concern here.
future.cancel()
results.append(self._build_timeout_result(turn_prompt, command_names, tool_names))
break
results.append(result)
if result.stop_reason != 'completed':
break
finally:
if executor is not None:
# wait=False: don't let a hung thread block loop exit indefinitely.
# The thread will be reaped when the interpreter shuts down or when
# the engine call eventually returns.
executor.shutdown(wait=False)
return results
@staticmethod
def _build_timeout_result(
prompt: str,
command_names: tuple[str, ...],
tool_names: tuple[str, ...],
) -> TurnResult:
"""Synthesize a TurnResult representing a wall-clock timeout (#161)."""
return TurnResult(
prompt=prompt,
output='Wall-clock timeout exceeded before turn completed.',
matched_commands=command_names,
matched_tools=tool_names,
permission_denials=(),
usage=UsageSummary(),
stop_reason='timeout',
)
def _infer_permission_denials(self, matches: list[RoutedMatch]) -> list[PermissionDenial]:
denials: list[PermissionDenial] = []
for match in matches: