Prevent worker prompts from outrunning boot readiness

Add a foundational worker_boot control plane and tool surface for
reliable startup. The new registry tracks trust gates, ready-for-prompt
handshakes, prompt delivery attempts, and shell misdelivery recovery so
callers can coordinate worker boot above raw terminal transport.

Constraint: Current main has no tmux-backed worker control API to extend directly
Constraint: First slice must stay deterministic and fully testable in-process
Rejected: Wire the first implementation straight to tmux panes | would couple transport details to unfinished state semantics
Rejected: Ship parser helpers without control tools | would not enforce the ready-before-prompt contract end to end
Confidence: high
Scope-risk: moderate
Reversibility: clean
Directive: Treat WorkerObserve heuristics as a temporary transport adapter and replace them with typed runtime events before widening automation policy
Tested: cargo test -p runtime worker_boot
Tested: cargo test -p tools worker_tools
Tested: cargo check -p runtime -p tools
Not-tested: Real tmux/TTY trust prompts and live worker boot on an actual coding session
Not-tested: Full cargo clippy -p runtime -p tools --all-targets -- -D warnings (fails on pre-existing warnings outside this slice)
This commit is contained in:
Yeachan-Heo
2026-04-03 15:04:52 +00:00
parent 56ee33e057
commit f76311f9d6
3 changed files with 1090 additions and 4 deletions

View File

@@ -0,0 +1,732 @@
//! In-memory worker-boot state machine and control registry.
//!
//! This provides a foundational control plane for reliable worker startup:
//! trust-gate detection, ready-for-prompt handshakes, and prompt-misdelivery
//! detection/recovery all live above raw terminal transport.
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkerStatus {
Spawning,
TrustRequired,
ReadyForPrompt,
PromptAccepted,
Running,
Blocked,
Finished,
Failed,
}
impl std::fmt::Display for WorkerStatus {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Spawning => write!(f, "spawning"),
Self::TrustRequired => write!(f, "trust_required"),
Self::ReadyForPrompt => write!(f, "ready_for_prompt"),
Self::PromptAccepted => write!(f, "prompt_accepted"),
Self::Running => write!(f, "running"),
Self::Blocked => write!(f, "blocked"),
Self::Finished => write!(f, "finished"),
Self::Failed => write!(f, "failed"),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkerFailureKind {
TrustGate,
PromptDelivery,
Protocol,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkerFailure {
pub kind: WorkerFailureKind,
pub message: String,
pub created_at: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WorkerEventKind {
Spawning,
TrustRequired,
TrustResolved,
ReadyForPrompt,
PromptAccepted,
PromptMisdelivery,
PromptReplayArmed,
Running,
Restarted,
Finished,
Failed,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkerEvent {
pub seq: u64,
pub kind: WorkerEventKind,
pub status: WorkerStatus,
pub detail: Option<String>,
pub timestamp: u64,
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct Worker {
pub worker_id: String,
pub cwd: String,
pub status: WorkerStatus,
pub trust_auto_resolve: bool,
pub trust_gate_cleared: bool,
pub auto_recover_prompt_misdelivery: bool,
pub prompt_delivery_attempts: u32,
pub last_prompt: Option<String>,
pub replay_prompt: Option<String>,
pub last_error: Option<WorkerFailure>,
pub created_at: u64,
pub updated_at: u64,
pub events: Vec<WorkerEvent>,
}
#[derive(Debug, Clone, Default)]
pub struct WorkerRegistry {
inner: Arc<Mutex<WorkerRegistryInner>>,
}
#[derive(Debug, Default)]
struct WorkerRegistryInner {
workers: HashMap<String, Worker>,
counter: u64,
}
impl WorkerRegistry {
#[must_use]
pub fn new() -> Self {
Self::default()
}
#[must_use]
pub fn create(
&self,
cwd: &str,
trusted_roots: &[String],
auto_recover_prompt_misdelivery: bool,
) -> Worker {
let mut inner = self.inner.lock().expect("worker registry lock poisoned");
inner.counter += 1;
let ts = now_secs();
let worker_id = format!("worker_{:08x}_{}", ts, inner.counter);
let trust_auto_resolve = trusted_roots
.iter()
.any(|root| path_matches_allowlist(cwd, root));
let mut worker = Worker {
worker_id: worker_id.clone(),
cwd: cwd.to_owned(),
status: WorkerStatus::Spawning,
trust_auto_resolve,
trust_gate_cleared: false,
auto_recover_prompt_misdelivery,
prompt_delivery_attempts: 0,
last_prompt: None,
replay_prompt: None,
last_error: None,
created_at: ts,
updated_at: ts,
events: Vec::new(),
};
push_event(
&mut worker,
WorkerEventKind::Spawning,
WorkerStatus::Spawning,
Some("worker created".to_string()),
);
inner.workers.insert(worker_id, worker.clone());
worker
}
#[must_use]
pub fn get(&self, worker_id: &str) -> Option<Worker> {
let inner = self.inner.lock().expect("worker registry lock poisoned");
inner.workers.get(worker_id).cloned()
}
pub fn observe(&self, worker_id: &str, screen_text: &str) -> Result<Worker, String> {
let mut inner = self.inner.lock().expect("worker registry lock poisoned");
let worker = inner
.workers
.get_mut(worker_id)
.ok_or_else(|| format!("worker not found: {worker_id}"))?;
let lowered = screen_text.to_ascii_lowercase();
if !worker.trust_gate_cleared && detect_trust_prompt(&lowered) {
worker.status = WorkerStatus::TrustRequired;
worker.last_error = Some(WorkerFailure {
kind: WorkerFailureKind::TrustGate,
message: "worker boot blocked on trust prompt".to_string(),
created_at: now_secs(),
});
push_event(
worker,
WorkerEventKind::TrustRequired,
WorkerStatus::TrustRequired,
Some("trust prompt detected".to_string()),
);
if worker.trust_auto_resolve {
worker.trust_gate_cleared = true;
worker.last_error = None;
worker.status = WorkerStatus::Spawning;
push_event(
worker,
WorkerEventKind::TrustResolved,
WorkerStatus::Spawning,
Some("allowlisted repo auto-resolved trust prompt".to_string()),
);
} else {
return Ok(worker.clone());
}
}
if prompt_misdelivery_is_relevant(worker)
&& detect_prompt_misdelivery(&lowered, worker.last_prompt.as_deref())
{
let detail = prompt_preview(worker.last_prompt.as_deref().unwrap_or_default());
worker.last_error = Some(WorkerFailure {
kind: WorkerFailureKind::PromptDelivery,
message: format!("worker prompt landed in shell instead of coding agent: {detail}"),
created_at: now_secs(),
});
push_event(
worker,
WorkerEventKind::PromptMisdelivery,
WorkerStatus::Blocked,
Some("shell misdelivery detected".to_string()),
);
if worker.auto_recover_prompt_misdelivery {
worker.replay_prompt = worker.last_prompt.clone();
worker.status = WorkerStatus::ReadyForPrompt;
push_event(
worker,
WorkerEventKind::PromptReplayArmed,
WorkerStatus::ReadyForPrompt,
Some("prompt replay armed after shell misdelivery".to_string()),
);
} else {
worker.status = WorkerStatus::Blocked;
}
return Ok(worker.clone());
}
if detect_running_cue(&lowered)
&& matches!(
worker.status,
WorkerStatus::PromptAccepted | WorkerStatus::ReadyForPrompt
)
{
worker.status = WorkerStatus::Running;
worker.last_error = None;
push_event(
worker,
WorkerEventKind::Running,
WorkerStatus::Running,
Some("worker accepted prompt and started running".to_string()),
);
}
if detect_ready_for_prompt(screen_text, &lowered)
&& !matches!(
worker.status,
WorkerStatus::ReadyForPrompt | WorkerStatus::Running
)
{
worker.status = WorkerStatus::ReadyForPrompt;
if matches!(
worker.last_error.as_ref().map(|failure| failure.kind),
Some(WorkerFailureKind::TrustGate)
) {
worker.last_error = None;
}
push_event(
worker,
WorkerEventKind::ReadyForPrompt,
WorkerStatus::ReadyForPrompt,
Some("worker is ready for prompt delivery".to_string()),
);
}
Ok(worker.clone())
}
pub fn resolve_trust(&self, worker_id: &str) -> Result<Worker, String> {
let mut inner = self.inner.lock().expect("worker registry lock poisoned");
let worker = inner
.workers
.get_mut(worker_id)
.ok_or_else(|| format!("worker not found: {worker_id}"))?;
if worker.status != WorkerStatus::TrustRequired {
return Err(format!(
"worker {worker_id} is not waiting on trust; current status: {}",
worker.status
));
}
worker.trust_gate_cleared = true;
worker.last_error = None;
worker.status = WorkerStatus::Spawning;
push_event(
worker,
WorkerEventKind::TrustResolved,
WorkerStatus::Spawning,
Some("trust prompt resolved manually".to_string()),
);
Ok(worker.clone())
}
pub fn send_prompt(&self, worker_id: &str, prompt: Option<&str>) -> Result<Worker, String> {
let mut inner = self.inner.lock().expect("worker registry lock poisoned");
let worker = inner
.workers
.get_mut(worker_id)
.ok_or_else(|| format!("worker not found: {worker_id}"))?;
if worker.status != WorkerStatus::ReadyForPrompt {
return Err(format!(
"worker {worker_id} is not ready for prompt delivery; current status: {}",
worker.status
));
}
let next_prompt = prompt
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_owned)
.or_else(|| worker.replay_prompt.clone())
.ok_or_else(|| format!("worker {worker_id} has no prompt to send or replay"))?;
worker.prompt_delivery_attempts += 1;
worker.last_prompt = Some(next_prompt.clone());
worker.replay_prompt = None;
worker.last_error = None;
worker.status = WorkerStatus::PromptAccepted;
push_event(
worker,
WorkerEventKind::PromptAccepted,
WorkerStatus::PromptAccepted,
Some(format!(
"prompt accepted for delivery: {}",
prompt_preview(&next_prompt)
)),
);
Ok(worker.clone())
}
pub fn await_ready(&self, worker_id: &str) -> Result<WorkerReadySnapshot, String> {
let worker = self
.get(worker_id)
.ok_or_else(|| format!("worker not found: {worker_id}"))?;
Ok(WorkerReadySnapshot {
worker_id: worker.worker_id.clone(),
status: worker.status,
ready: worker.status == WorkerStatus::ReadyForPrompt,
blocked: matches!(
worker.status,
WorkerStatus::TrustRequired | WorkerStatus::Blocked
),
replay_prompt_ready: worker.replay_prompt.is_some(),
last_error: worker.last_error.clone(),
})
}
pub fn restart(&self, worker_id: &str) -> Result<Worker, String> {
let mut inner = self.inner.lock().expect("worker registry lock poisoned");
let worker = inner
.workers
.get_mut(worker_id)
.ok_or_else(|| format!("worker not found: {worker_id}"))?;
worker.status = WorkerStatus::Spawning;
worker.trust_gate_cleared = false;
worker.last_prompt = None;
worker.replay_prompt = None;
worker.last_error = None;
worker.prompt_delivery_attempts = 0;
push_event(
worker,
WorkerEventKind::Restarted,
WorkerStatus::Spawning,
Some("worker restarted".to_string()),
);
Ok(worker.clone())
}
pub fn terminate(&self, worker_id: &str) -> Result<Worker, String> {
let mut inner = self.inner.lock().expect("worker registry lock poisoned");
let worker = inner
.workers
.get_mut(worker_id)
.ok_or_else(|| format!("worker not found: {worker_id}"))?;
worker.status = WorkerStatus::Finished;
push_event(
worker,
WorkerEventKind::Finished,
WorkerStatus::Finished,
Some("worker terminated by control plane".to_string()),
);
Ok(worker.clone())
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct WorkerReadySnapshot {
pub worker_id: String,
pub status: WorkerStatus,
pub ready: bool,
pub blocked: bool,
pub replay_prompt_ready: bool,
pub last_error: Option<WorkerFailure>,
}
fn prompt_misdelivery_is_relevant(worker: &Worker) -> bool {
matches!(
worker.status,
WorkerStatus::PromptAccepted | WorkerStatus::Running
) && worker.last_prompt.is_some()
}
fn push_event(
worker: &mut Worker,
kind: WorkerEventKind,
status: WorkerStatus,
detail: Option<String>,
) {
let timestamp = now_secs();
let seq = worker.events.len() as u64 + 1;
worker.updated_at = timestamp;
worker.events.push(WorkerEvent {
seq,
kind,
status,
detail,
timestamp,
});
}
fn path_matches_allowlist(cwd: &str, trusted_root: &str) -> bool {
let cwd = normalize_path(cwd);
let trusted_root = normalize_path(trusted_root);
cwd == trusted_root || cwd.starts_with(&trusted_root)
}
fn normalize_path(path: &str) -> PathBuf {
std::fs::canonicalize(path).unwrap_or_else(|_| Path::new(path).to_path_buf())
}
fn detect_trust_prompt(lowered: &str) -> bool {
[
"do you trust the files in this folder",
"trust the files in this folder",
"trust this folder",
"allow and continue",
"yes, proceed",
]
.iter()
.any(|needle| lowered.contains(needle))
}
fn detect_ready_for_prompt(screen_text: &str, lowered: &str) -> bool {
if [
"ready for input",
"ready for your input",
"ready for prompt",
"send a message",
]
.iter()
.any(|needle| lowered.contains(needle))
{
return true;
}
let Some(last_non_empty) = screen_text
.lines()
.rev()
.find(|line| !line.trim().is_empty())
else {
return false;
};
let trimmed = last_non_empty.trim();
if is_shell_prompt(trimmed) {
return false;
}
trimmed == ">"
|| trimmed == ""
|| trimmed == ""
|| trimmed.starts_with("> ")
|| trimmed.starts_with(" ")
|| trimmed.starts_with(" ")
|| trimmed.contains("│ >")
|| trimmed.contains("")
|| trimmed.contains("")
}
fn detect_running_cue(lowered: &str) -> bool {
[
"thinking",
"working",
"running tests",
"inspecting",
"analyzing",
]
.iter()
.any(|needle| lowered.contains(needle))
}
fn is_shell_prompt(trimmed: &str) -> bool {
trimmed.ends_with('$')
|| trimmed.ends_with('%')
|| trimmed.ends_with('#')
|| trimmed.starts_with('$')
|| trimmed.starts_with('%')
|| trimmed.starts_with('#')
}
fn detect_prompt_misdelivery(lowered: &str, prompt: Option<&str>) -> bool {
let Some(prompt) = prompt else {
return false;
};
let shell_error = [
"command not found",
"syntax error near unexpected token",
"parse error near",
"no such file or directory",
"unknown command",
]
.iter()
.any(|needle| lowered.contains(needle));
if !shell_error {
return false;
}
let first_prompt_line = prompt
.lines()
.find(|line| !line.trim().is_empty())
.map(|line| line.trim().to_ascii_lowercase())
.unwrap_or_default();
first_prompt_line.is_empty() || lowered.contains(&first_prompt_line)
}
fn prompt_preview(prompt: &str) -> String {
let trimmed = prompt.trim();
if trimmed.chars().count() <= 48 {
return trimmed.to_string();
}
let preview = trimmed.chars().take(48).collect::<String>();
format!("{}", preview.trim_end())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn allowlisted_trust_prompt_auto_resolves_then_reaches_ready_state() {
let registry = WorkerRegistry::new();
let worker = registry.create(
"/tmp/worktrees/repo-a",
&["/tmp/worktrees".to_string()],
true,
);
let after_trust = registry
.observe(
&worker.worker_id,
"Do you trust the files in this folder?\n1. Yes, proceed\n2. No",
)
.expect("trust observe should succeed");
assert_eq!(after_trust.status, WorkerStatus::Spawning);
assert!(after_trust.trust_gate_cleared);
assert!(after_trust
.events
.iter()
.any(|event| event.kind == WorkerEventKind::TrustRequired));
assert!(after_trust
.events
.iter()
.any(|event| event.kind == WorkerEventKind::TrustResolved));
let ready = registry
.observe(&worker.worker_id, "Ready for your input\n>")
.expect("ready observe should succeed");
assert_eq!(ready.status, WorkerStatus::ReadyForPrompt);
assert!(ready.last_error.is_none());
}
#[test]
fn trust_prompt_blocks_non_allowlisted_worker_until_resolved() {
let registry = WorkerRegistry::new();
let worker = registry.create("/tmp/repo-b", &[], true);
let blocked = registry
.observe(
&worker.worker_id,
"Do you trust the files in this folder?\n1. Yes, proceed\n2. No",
)
.expect("trust observe should succeed");
assert_eq!(blocked.status, WorkerStatus::TrustRequired);
assert_eq!(
blocked.last_error.expect("trust error should exist").kind,
WorkerFailureKind::TrustGate
);
let send_before_resolve = registry.send_prompt(&worker.worker_id, Some("ship it"));
assert!(send_before_resolve
.expect_err("prompt delivery should be gated")
.contains("not ready for prompt delivery"));
let resolved = registry
.resolve_trust(&worker.worker_id)
.expect("manual trust resolution should succeed");
assert_eq!(resolved.status, WorkerStatus::Spawning);
assert!(resolved.trust_gate_cleared);
}
#[test]
fn ready_detection_ignores_plain_shell_prompts() {
assert!(!detect_ready_for_prompt("bellman@host %", "bellman@host %"));
assert!(!detect_ready_for_prompt("/tmp/repo $", "/tmp/repo $"));
assert!(detect_ready_for_prompt("│ >", "│ >"));
}
#[test]
fn prompt_misdelivery_is_detected_and_replay_can_be_rearmed() {
let registry = WorkerRegistry::new();
let worker = registry.create("/tmp/repo-c", &[], true);
registry
.observe(&worker.worker_id, "Ready for input\n>")
.expect("ready observe should succeed");
let accepted = registry
.send_prompt(&worker.worker_id, Some("Implement worker handshake"))
.expect("prompt send should succeed");
assert_eq!(accepted.status, WorkerStatus::PromptAccepted);
assert_eq!(accepted.prompt_delivery_attempts, 1);
let recovered = registry
.observe(
&worker.worker_id,
"% Implement worker handshake\nzsh: command not found: Implement",
)
.expect("misdelivery observe should succeed");
assert_eq!(recovered.status, WorkerStatus::ReadyForPrompt);
assert_eq!(
recovered
.last_error
.expect("misdelivery error should exist")
.kind,
WorkerFailureKind::PromptDelivery
);
assert_eq!(
recovered.replay_prompt.as_deref(),
Some("Implement worker handshake")
);
assert!(recovered
.events
.iter()
.any(|event| event.kind == WorkerEventKind::PromptMisdelivery));
assert!(recovered
.events
.iter()
.any(|event| event.kind == WorkerEventKind::PromptReplayArmed));
let replayed = registry
.send_prompt(&worker.worker_id, None)
.expect("replay send should succeed");
assert_eq!(replayed.status, WorkerStatus::PromptAccepted);
assert!(replayed.replay_prompt.is_none());
assert_eq!(replayed.prompt_delivery_attempts, 2);
}
#[test]
fn await_ready_surfaces_blocked_or_ready_worker_state() {
let registry = WorkerRegistry::new();
let worker = registry.create("/tmp/repo-d", &[], false);
let initial = registry
.await_ready(&worker.worker_id)
.expect("await should succeed");
assert!(!initial.ready);
assert!(!initial.blocked);
registry
.observe(
&worker.worker_id,
"Do you trust the files in this folder?\n1. Yes, proceed\n2. No",
)
.expect("trust observe should succeed");
let blocked = registry
.await_ready(&worker.worker_id)
.expect("await should succeed");
assert!(!blocked.ready);
assert!(blocked.blocked);
registry
.resolve_trust(&worker.worker_id)
.expect("manual trust resolution should succeed");
registry
.observe(&worker.worker_id, "Ready for your input\n>")
.expect("ready observe should succeed");
let ready = registry
.await_ready(&worker.worker_id)
.expect("await should succeed");
assert!(ready.ready);
assert!(!ready.blocked);
assert!(ready.last_error.is_none());
}
#[test]
fn restart_and_terminate_reset_or_finish_worker() {
let registry = WorkerRegistry::new();
let worker = registry.create("/tmp/repo-e", &[], true);
registry
.observe(&worker.worker_id, "Ready for input\n>")
.expect("ready observe should succeed");
registry
.send_prompt(&worker.worker_id, Some("Run tests"))
.expect("prompt send should succeed");
let restarted = registry
.restart(&worker.worker_id)
.expect("restart should succeed");
assert_eq!(restarted.status, WorkerStatus::Spawning);
assert_eq!(restarted.prompt_delivery_attempts, 0);
assert!(restarted.last_prompt.is_none());
let finished = registry
.terminate(&worker.worker_id)
.expect("terminate should succeed");
assert_eq!(finished.status, WorkerStatus::Finished);
assert!(finished
.events
.iter()
.any(|event| event.kind == WorkerEventKind::Finished));
}
}