Merge pull request #3214 from TheArchitectit/worktree-api-timeout-retry-v2

feat: API timeout config, Retry-After header, configurable retry, and 400 transient retry
This commit is contained in:
Bellman
2026-06-05 10:33:35 +09:00
committed by GitHub
8 changed files with 302 additions and 81 deletions

View File

@@ -20,6 +20,7 @@ const CONTEXT_WINDOW_ERROR_MARKERS: &[&str] = &[
"completion tokens", "completion tokens",
"prompt tokens", "prompt tokens",
"request is too large", "request is too large",
"no parseable body",
]; ];
#[derive(Debug)] #[derive(Debug)]
@@ -60,6 +61,9 @@ pub enum ApiError {
retryable: bool, retryable: bool,
/// Suggested user action based on error type (e.g., "Reduce prompt size" for 413) /// Suggested user action based on error type (e.g., "Reduce prompt size" for 413)
suggested_action: Option<String>, suggested_action: Option<String>,
/// Parsed Retry-After header value (seconds) for 429 responses.
/// When present, overrides the exponential backoff delay.
retry_after: Option<Duration>,
}, },
RetriesExhausted { RetriesExhausted {
attempts: u32, attempts: u32,
@@ -128,6 +132,17 @@ impl ApiError {
} }
#[must_use] #[must_use]
/// Return the `Retry-After` delay if this error came from a 429 response
/// that included a `retry-after` header. Callers should prefer this value
/// over the computed backoff delay when it exists.
pub fn retry_after(&self) -> Option<Duration> {
match self {
Self::Api { retry_after, .. } => *retry_after,
Self::RetriesExhausted { last_error, .. } => last_error.retry_after(),
_ => None,
}
}
pub fn is_retryable(&self) -> bool { pub fn is_retryable(&self) -> bool {
match self { match self {
Self::Http(error) => error.is_connect() || error.is_timeout() || error.is_request(), Self::Http(error) => error.is_connect() || error.is_timeout() || error.is_request(),
@@ -529,6 +544,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: true, retryable: true,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
assert!(error.is_generic_fatal_wrapper()); assert!(error.is_generic_fatal_wrapper());
@@ -552,6 +568,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: true, retryable: true,
suggested_action: None, suggested_action: None,
retry_after: None,
}), }),
}; };
@@ -573,6 +590,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
assert!(error.is_context_window_failure()); assert!(error.is_context_window_failure());
@@ -593,6 +611,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
assert!(error.is_context_window_failure()); assert!(error.is_context_window_failure());

View File

@@ -1,9 +1,69 @@
use std::time::Duration;
use crate::error::ApiError; use crate::error::ApiError;
const HTTP_PROXY_KEYS: [&str; 2] = ["HTTP_PROXY", "http_proxy"]; const HTTP_PROXY_KEYS: [&str; 2] = ["HTTP_PROXY", "http_proxy"];
const HTTPS_PROXY_KEYS: [&str; 2] = ["HTTPS_PROXY", "https_proxy"]; const HTTPS_PROXY_KEYS: [&str; 2] = ["HTTPS_PROXY", "https_proxy"];
const NO_PROXY_KEYS: [&str; 2] = ["NO_PROXY", "no_proxy"]; const NO_PROXY_KEYS: [&str; 2] = ["NO_PROXY", "no_proxy"];
/// Timeout configuration for outbound HTTP requests.
///
/// When set, the `reqwest::Client` will abort requests that take longer
/// than the configured duration and return a timeout error (which is
/// retryable by the existing exponential backoff logic).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TimeoutConfig {
/// Maximum time to wait for a connection to be established.
/// Defaults to 30 seconds.
pub connect_timeout: Duration,
/// Maximum time for the entire request (including reading the response
/// body). For streaming responses this is the timeout for the initial
/// handshake only; the stream itself is governed by SSE parsing.
/// Defaults to 5 minutes (300 seconds).
pub request_timeout: Duration,
}
impl Default for TimeoutConfig {
fn default() -> Self {
Self {
connect_timeout: Duration::from_secs(30),
request_timeout: Duration::from_secs(300),
}
}
}
impl TimeoutConfig {
/// Read timeout settings from the process environment.
/// - `CLAW_API_CONNECT_TIMEOUT` — connect timeout in seconds
/// - `CLAW_API_REQUEST_TIMEOUT` — overall request timeout in seconds
#[must_use]
pub fn from_env() -> Self {
let connect_timeout = std::env::var("CLAW_API_CONNECT_TIMEOUT")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs)
.unwrap_or(Duration::from_secs(30));
let request_timeout = std::env::var("CLAW_API_REQUEST_TIMEOUT")
.ok()
.and_then(|v| v.parse::<u64>().ok())
.map(Duration::from_secs)
.unwrap_or(Duration::from_secs(300));
Self {
connect_timeout,
request_timeout,
}
}
/// Create from explicit second values (used by config file parsing).
#[must_use]
pub fn from_seconds(connect_secs: u64, request_secs: u64) -> Self {
Self {
connect_timeout: Duration::from_secs(connect_secs),
request_timeout: Duration::from_secs(request_secs),
}
}
}
/// Snapshot of the proxy-related environment variables that influence the /// Snapshot of the proxy-related environment variables that influence the
/// outbound HTTP client. Captured up front so callers can inspect, log, and /// outbound HTTP client. Captured up front so callers can inspect, log, and
/// test the resolved configuration without re-reading the process environment. /// test the resolved configuration without re-reading the process environment.
@@ -61,7 +121,7 @@ impl ProxyConfig {
/// `HTTPS_PROXY`, and `NO_PROXY` environment variables. When no proxy is /// `HTTPS_PROXY`, and `NO_PROXY` environment variables. When no proxy is
/// configured the client behaves identically to `reqwest::Client::new()`. /// configured the client behaves identically to `reqwest::Client::new()`.
pub fn build_http_client() -> Result<reqwest::Client, ApiError> { pub fn build_http_client() -> Result<reqwest::Client, ApiError> {
build_http_client_with(&ProxyConfig::from_env()) build_http_client_with_opts(&ProxyConfig::from_env(), &TimeoutConfig::from_env())
} }
/// Infallible counterpart to [`build_http_client`] for constructors that /// Infallible counterpart to [`build_http_client`] for constructors that
@@ -71,12 +131,13 @@ pub fn build_http_client() -> Result<reqwest::Client, ApiError> {
/// first outbound request instead of at construction time. /// first outbound request instead of at construction time.
#[must_use] #[must_use]
pub fn build_http_client_or_default() -> reqwest::Client { pub fn build_http_client_or_default() -> reqwest::Client {
build_http_client().unwrap_or_else(|_| { build_http_client_with_opts(&ProxyConfig::from_env(), &TimeoutConfig::from_env())
reqwest::Client::builder() .unwrap_or_else(|_| {
.user_agent("clawd-rust-tools/0.1") reqwest::Client::builder()
.build() .user_agent("clawd-rust-tools/0.1")
.expect("default client with user_agent should always succeed") .build()
}) .expect("default client with user_agent should always succeed")
})
} }
/// Build a `reqwest::Client` from an explicit [`ProxyConfig`]. Used by tests /// Build a `reqwest::Client` from an explicit [`ProxyConfig`]. Used by tests
@@ -86,9 +147,20 @@ pub fn build_http_client_or_default() -> reqwest::Client {
/// and `https_proxy` fields and is registered as both an HTTP and HTTPS /// and `https_proxy` fields and is registered as both an HTTP and HTTPS
/// proxy so a single value can route every outbound request. /// proxy so a single value can route every outbound request.
pub fn build_http_client_with(config: &ProxyConfig) -> Result<reqwest::Client, ApiError> { pub fn build_http_client_with(config: &ProxyConfig) -> Result<reqwest::Client, ApiError> {
build_http_client_with_opts(config, &TimeoutConfig::from_env())
}
/// Build a `reqwest::Client` from explicit [`ProxyConfig`] and [`TimeoutConfig`].
/// Used by callers that want to control both proxy routing and request timing.
pub fn build_http_client_with_opts(
config: &ProxyConfig,
timeout: &TimeoutConfig,
) -> Result<reqwest::Client, ApiError> {
let mut builder = reqwest::Client::builder() let mut builder = reqwest::Client::builder()
.no_proxy() .no_proxy()
.user_agent("clawd-rust-tools/0.1"); .user_agent("clawd-rust-tools/0.1")
.connect_timeout(timeout.connect_timeout)
.timeout(timeout.request_timeout);
let no_proxy = config let no_proxy = config
.no_proxy .no_proxy
@@ -131,7 +203,7 @@ where
mod tests { mod tests {
use std::collections::HashMap; use std::collections::HashMap;
use super::{build_http_client_with, ProxyConfig}; use super::{build_http_client_with, build_http_client_with_opts, ProxyConfig, TimeoutConfig};
fn config_from_map(pairs: &[(&str, &str)]) -> ProxyConfig { fn config_from_map(pairs: &[(&str, &str)]) -> ProxyConfig {
let map: HashMap<String, String> = pairs let map: HashMap<String, String> = pairs
@@ -143,30 +215,19 @@ mod tests {
#[test] #[test]
fn proxy_config_is_empty_when_no_env_vars_are_set() { fn proxy_config_is_empty_when_no_env_vars_are_set() {
// given
let config = config_from_map(&[]); let config = config_from_map(&[]);
assert!(config.is_empty());
// when
let empty = config.is_empty();
// then
assert!(empty);
assert_eq!(config, ProxyConfig::default()); assert_eq!(config, ProxyConfig::default());
} }
#[test] #[test]
fn proxy_config_reads_uppercase_http_https_and_no_proxy() { fn proxy_config_reads_uppercase_http_https_and_no_proxy() {
// given
let pairs = [ let pairs = [
("HTTP_PROXY", "http://proxy.internal:3128"), ("HTTP_PROXY", "http://proxy.internal:3128"),
("HTTPS_PROXY", "http://secure.internal:3129"), ("HTTPS_PROXY", "http://secure.internal:3129"),
("NO_PROXY", "localhost,127.0.0.1,.corp"), ("NO_PROXY", "localhost,127.0.0.1,.corp"),
]; ];
// when
let config = config_from_map(&pairs); let config = config_from_map(&pairs);
// then
assert_eq!( assert_eq!(
config.http_proxy.as_deref(), config.http_proxy.as_deref(),
Some("http://proxy.internal:3128") Some("http://proxy.internal:3128")
@@ -184,17 +245,12 @@ mod tests {
#[test] #[test]
fn proxy_config_falls_back_to_lowercase_keys() { fn proxy_config_falls_back_to_lowercase_keys() {
// given
let pairs = [ let pairs = [
("http_proxy", "http://lower.internal:3128"), ("http_proxy", "http://lower.internal:3128"),
("https_proxy", "http://lower-secure.internal:3129"), ("https_proxy", "http://lower-secure.internal:3129"),
("no_proxy", ".lower"), ("no_proxy", ".lower"),
]; ];
// when
let config = config_from_map(&pairs); let config = config_from_map(&pairs);
// then
assert_eq!( assert_eq!(
config.http_proxy.as_deref(), config.http_proxy.as_deref(),
Some("http://lower.internal:3128") Some("http://lower.internal:3128")
@@ -208,16 +264,11 @@ mod tests {
#[test] #[test]
fn proxy_config_prefers_uppercase_over_lowercase_when_both_set() { fn proxy_config_prefers_uppercase_over_lowercase_when_both_set() {
// given
let pairs = [ let pairs = [
("HTTP_PROXY", "http://upper.internal:3128"), ("HTTP_PROXY", "http://upper.internal:3128"),
("http_proxy", "http://lower.internal:3128"), ("http_proxy", "http://lower.internal:3128"),
]; ];
// when
let config = config_from_map(&pairs); let config = config_from_map(&pairs);
// then
assert_eq!( assert_eq!(
config.http_proxy.as_deref(), config.http_proxy.as_deref(),
Some("http://upper.internal:3128") Some("http://upper.internal:3128")
@@ -226,59 +277,39 @@ mod tests {
#[test] #[test]
fn proxy_config_treats_empty_strings_as_unset() { fn proxy_config_treats_empty_strings_as_unset() {
// given
let pairs = [("HTTP_PROXY", ""), ("http_proxy", "")]; let pairs = [("HTTP_PROXY", ""), ("http_proxy", "")];
// when
let config = config_from_map(&pairs); let config = config_from_map(&pairs);
// then
assert!(config.http_proxy.is_none()); assert!(config.http_proxy.is_none());
} }
#[test] #[test]
fn build_http_client_succeeds_when_no_proxy_is_configured() { fn build_http_client_succeeds_when_no_proxy_is_configured() {
// given
let config = ProxyConfig::default(); let config = ProxyConfig::default();
// when
let result = build_http_client_with(&config); let result = build_http_client_with(&config);
// then
assert!(result.is_ok()); assert!(result.is_ok());
} }
#[test] #[test]
fn build_http_client_succeeds_with_valid_http_and_https_proxies() { fn build_http_client_succeeds_with_valid_http_and_https_proxies() {
// given
let config = ProxyConfig { let config = ProxyConfig {
http_proxy: Some("http://proxy.internal:3128".to_string()), http_proxy: Some("http://proxy.internal:3128".to_string()),
https_proxy: Some("http://secure.internal:3129".to_string()), https_proxy: Some("http://secure.internal:3129".to_string()),
no_proxy: Some("localhost,127.0.0.1".to_string()), no_proxy: Some("localhost,127.0.0.1".to_string()),
proxy_url: None, proxy_url: None,
}; };
// when
let result = build_http_client_with(&config); let result = build_http_client_with(&config);
// then
assert!(result.is_ok()); assert!(result.is_ok());
} }
#[test] #[test]
fn build_http_client_returns_http_error_for_invalid_proxy_url() { fn build_http_client_returns_http_error_for_invalid_proxy_url() {
// given
let config = ProxyConfig { let config = ProxyConfig {
http_proxy: None, http_proxy: None,
https_proxy: Some("not a url".to_string()), https_proxy: Some("not a url".to_string()),
no_proxy: None, no_proxy: None,
proxy_url: None, proxy_url: None,
}; };
// when
let result = build_http_client_with(&config); let result = build_http_client_with(&config);
// then
let error = result.expect_err("invalid proxy URL must be reported as a build failure"); let error = result.expect_err("invalid proxy URL must be reported as a build failure");
assert!( assert!(
matches!(error, crate::error::ApiError::Http(_)), matches!(error, crate::error::ApiError::Http(_)),
@@ -288,10 +319,7 @@ mod tests {
#[test] #[test]
fn from_proxy_url_sets_unified_field_and_leaves_per_scheme_empty() { fn from_proxy_url_sets_unified_field_and_leaves_per_scheme_empty() {
// given / when
let config = ProxyConfig::from_proxy_url("http://unified.internal:3128"); let config = ProxyConfig::from_proxy_url("http://unified.internal:3128");
// then
assert_eq!( assert_eq!(
config.proxy_url.as_deref(), config.proxy_url.as_deref(),
Some("http://unified.internal:3128") Some("http://unified.internal:3128")
@@ -303,49 +331,56 @@ mod tests {
#[test] #[test]
fn build_http_client_succeeds_with_unified_proxy_url() { fn build_http_client_succeeds_with_unified_proxy_url() {
// given
let config = ProxyConfig { let config = ProxyConfig {
proxy_url: Some("http://unified.internal:3128".to_string()), proxy_url: Some("http://unified.internal:3128".to_string()),
no_proxy: Some("localhost".to_string()), no_proxy: Some("localhost".to_string()),
..ProxyConfig::default() ..ProxyConfig::default()
}; };
// when
let result = build_http_client_with(&config); let result = build_http_client_with(&config);
// then
assert!(result.is_ok()); assert!(result.is_ok());
} }
#[test] #[test]
fn proxy_url_takes_precedence_over_per_scheme_fields() { fn proxy_url_takes_precedence_over_per_scheme_fields() {
// given both per-scheme and unified are set
let config = ProxyConfig { let config = ProxyConfig {
http_proxy: Some("http://per-scheme.internal:1111".to_string()), http_proxy: Some("http://per-scheme.internal:1111".to_string()),
https_proxy: Some("http://per-scheme.internal:2222".to_string()), https_proxy: Some("http://per-scheme.internal:2222".to_string()),
no_proxy: None, no_proxy: None,
proxy_url: Some("http://unified.internal:3128".to_string()), proxy_url: Some("http://unified.internal:3128".to_string()),
}; };
// when building succeeds (the unified URL is valid)
let result = build_http_client_with(&config); let result = build_http_client_with(&config);
// then
assert!(result.is_ok()); assert!(result.is_ok());
} }
#[test] #[test]
fn build_http_client_returns_error_for_invalid_unified_proxy_url() { fn build_http_client_returns_error_for_invalid_unified_proxy_url() {
// given
let config = ProxyConfig::from_proxy_url("not a url"); let config = ProxyConfig::from_proxy_url("not a url");
// when
let result = build_http_client_with(&config); let result = build_http_client_with(&config);
// then
assert!( assert!(
matches!(result, Err(crate::error::ApiError::Http(_))), matches!(result, Err(crate::error::ApiError::Http(_))),
"invalid unified proxy URL should fail: {result:?}" "invalid unified proxy URL should fail: {result:?}"
); );
} }
#[test]
fn timeout_config_defaults() {
let config = TimeoutConfig::default();
assert_eq!(config.connect_timeout, std::time::Duration::from_secs(30));
assert_eq!(config.request_timeout, std::time::Duration::from_secs(300));
}
#[test]
fn timeout_config_from_seconds() {
let config = TimeoutConfig::from_seconds(10, 60);
assert_eq!(config.connect_timeout, std::time::Duration::from_secs(10));
assert_eq!(config.request_timeout, std::time::Duration::from_secs(60));
}
#[test]
fn build_http_client_with_custom_timeouts() {
let config = ProxyConfig::default();
let timeout = TimeoutConfig::from_seconds(5, 120);
let result = build_http_client_with_opts(&config, &timeout);
assert!(result.is_ok());
}
} }

View File

@@ -12,7 +12,8 @@ pub use client::{
}; };
pub use error::ApiError; pub use error::ApiError;
pub use http_client::{ pub use http_client::{
build_http_client, build_http_client_or_default, build_http_client_with, ProxyConfig, build_http_client, build_http_client_or_default, build_http_client_with,
build_http_client_with_opts, ProxyConfig, TimeoutConfig,
}; };
pub use prompt_cache::{ pub use prompt_cache::{
CacheBreakEvent, PromptCache, PromptCacheConfig, PromptCachePaths, PromptCacheRecord, CacheBreakEvent, PromptCache, PromptCacheConfig, PromptCachePaths, PromptCacheRecord,

View File

@@ -211,6 +211,19 @@ impl AnthropicClient {
self self
} }
/// Replace the internal HTTP client with one that respects the given
/// timeout configuration. This controls connect and request-level
/// timeouts for all outbound API calls.
#[must_use]
pub fn with_timeout(mut self, timeout: &crate::http_client::TimeoutConfig) -> Self {
self.http = crate::http_client::build_http_client_with_opts(
&crate::http_client::ProxyConfig::from_env(),
timeout,
)
.unwrap_or_else(|_| reqwest::Client::new());
self
}
#[must_use] #[must_use]
pub fn with_session_tracer(mut self, session_tracer: SessionTracer) -> Self { pub fn with_session_tracer(mut self, session_tracer: SessionTracer) -> Self {
self.session_tracer = Some(session_tracer); self.session_tracer = Some(session_tracer);
@@ -454,7 +467,13 @@ impl AnthropicClient {
break; break;
} }
tokio::time::sleep(self.jittered_backoff_for_attempt(attempts)?).await; let delay = if let Some(retry_after) = last_error.as_ref().and_then(|e| e.retry_after())
{
retry_after
} else {
self.jittered_backoff_for_attempt(attempts)?
};
tokio::time::sleep(delay).await;
} }
Err(ApiError::RetriesExhausted { Err(ApiError::RetriesExhausted {
@@ -866,10 +885,12 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
return Ok(response); return Ok(response);
} }
let request_id = request_id_from_headers(response.headers()); let headers = response.headers().clone();
let request_id = request_id_from_headers(&headers);
let body = response.text().await.unwrap_or_else(|_| String::new()); let body = response.text().await.unwrap_or_else(|_| String::new());
let parsed_error = serde_json::from_str::<AnthropicErrorEnvelope>(&body).ok(); let parsed_error = serde_json::from_str::<AnthropicErrorEnvelope>(&body).ok();
let retryable = is_retryable_status(status); let retryable = is_retryable_status(status);
let retry_after = parse_retry_after(&headers, status);
Err(ApiError::Api { Err(ApiError::Api {
status, status,
@@ -883,13 +904,44 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
body, body,
retryable, retryable,
suggested_action: None, suggested_action: None,
retry_after,
}) })
} }
fn parse_retry_after(
headers: &reqwest::header::HeaderMap,
status: reqwest::StatusCode,
) -> Option<std::time::Duration> {
if status != reqwest::StatusCode::TOO_MANY_REQUESTS {
return None;
}
headers
.get("retry-after")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse::<u64>().ok())
.map(std::time::Duration::from_secs)
}
const fn is_retryable_status(status: reqwest::StatusCode) -> bool { const fn is_retryable_status(status: reqwest::StatusCode) -> bool {
matches!(status.as_u16(), 408 | 409 | 429 | 500 | 502 | 503 | 504) matches!(status.as_u16(), 408 | 409 | 429 | 500 | 502 | 503 | 504)
} }
/// Some providers return HTTP 400 with an unparseable body when a gateway
/// or proxy flakes (e.g. "HTTP 400 from backend (no parseable body)").
/// These are transient network blips, not actual bad requests, and should
/// be retried. We detect them by checking the body for known gateway error
/// phrases.
fn is_retryable_400(status: reqwest::StatusCode, body: &str) -> bool {
if status != reqwest::StatusCode::BAD_REQUEST {
return false;
}
let lowered = body.to_ascii_lowercase();
lowered.contains("no parseable body")
|| lowered.contains("connection reset")
|| lowered.contains("broken pipe")
|| lowered.contains("empty reply from server")
}
/// Anthropic API keys (`sk-ant-*`) are accepted over the `x-api-key` header /// Anthropic API keys (`sk-ant-*`) are accepted over the `x-api-key` header
/// and rejected with HTTP 401 "Invalid bearer token" when sent as a Bearer /// and rejected with HTTP 401 "Invalid bearer token" when sent as a Bearer
/// token via `ANTHROPIC_AUTH_TOKEN`. This happens often enough in the wild /// token via `ANTHROPIC_AUTH_TOKEN`. This happens often enough in the wild
@@ -908,6 +960,8 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
body, body,
retryable, retryable,
suggested_action, suggested_action,
retry_after,
..
} = error } = error
else { else {
return error; return error;
@@ -921,6 +975,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
body, body,
retryable, retryable,
suggested_action, suggested_action,
retry_after,
}; };
} }
let Some(bearer_token) = auth.bearer_token() else { let Some(bearer_token) = auth.bearer_token() else {
@@ -932,6 +987,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
body, body,
retryable, retryable,
suggested_action, suggested_action,
retry_after,
}; };
}; };
if !bearer_token.starts_with("sk-ant-") { if !bearer_token.starts_with("sk-ant-") {
@@ -943,6 +999,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
body, body,
retryable, retryable,
suggested_action, suggested_action,
retry_after,
}; };
} }
// Only append the hint when the AuthSource is pure BearerToken. If both // Only append the hint when the AuthSource is pure BearerToken. If both
@@ -958,6 +1015,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
body, body,
retryable, retryable,
suggested_action, suggested_action,
retry_after,
}; };
} }
let enriched_message = match message { let enriched_message = match message {
@@ -972,6 +1030,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
body, body,
retryable, retryable,
suggested_action, suggested_action,
retry_after,
} }
} }
@@ -1596,6 +1655,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
// when // when
@@ -1637,6 +1697,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: true, retryable: true,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
// when // when
@@ -1666,6 +1727,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
// when // when
@@ -1694,6 +1756,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
// when // when
@@ -1719,6 +1782,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
// when // when

View File

@@ -175,6 +175,18 @@ impl OpenAiCompatClient {
self self
} }
/// Replace the internal HTTP client with one that respects the given
/// timeout configuration.
#[must_use]
pub fn with_timeout(mut self, timeout: &crate::http_client::TimeoutConfig) -> Self {
self.http = crate::http_client::build_http_client_with_opts(
&crate::http_client::ProxyConfig::from_env(),
timeout,
)
.unwrap_or_else(|_| reqwest::Client::new());
self
}
pub async fn send_message( pub async fn send_message(
&self, &self,
request: &MessageRequest, request: &MessageRequest,
@@ -217,6 +229,7 @@ impl OpenAiCompatClient {
reqwest::StatusCode::from_u16(code.unwrap_or(400)) reqwest::StatusCode::from_u16(code.unwrap_or(400))
.unwrap_or(reqwest::StatusCode::BAD_REQUEST), .unwrap_or(reqwest::StatusCode::BAD_REQUEST),
), ),
retry_after: None,
}); });
} }
} }
@@ -270,7 +283,12 @@ impl OpenAiCompatClient {
break retryable_error; break retryable_error;
} }
tokio::time::sleep(self.jittered_backoff_for_attempt(attempts)?).await; let delay = if let Some(retry_after) = retryable_error.retry_after() {
retry_after
} else {
self.jittered_backoff_for_attempt(attempts)?
};
tokio::time::sleep(delay).await;
}; };
Err(ApiError::RetriesExhausted { Err(ApiError::RetriesExhausted {
@@ -1561,6 +1579,7 @@ fn parse_sse_frame(
body: trimmed.chars().take(500).collect(), body: trimmed.chars().take(500).collect(),
retryable: false, retryable: false,
suggested_action: suggested_action_for_status(status), suggested_action: suggested_action_for_status(status),
retry_after: None,
}); });
} }
} }
@@ -1576,6 +1595,7 @@ fn parse_sse_frame(
body: trimmed.chars().take(200).collect(), body: trimmed.chars().take(200).collect(),
retryable: false, retryable: false,
suggested_action: Some("verify the API endpoint URL is correct".to_string()), suggested_action: Some("verify the API endpoint URL is correct".to_string()),
retry_after: None,
}); });
} }
return Ok(None); return Ok(None);
@@ -1611,6 +1631,7 @@ fn parse_sse_frame(
body: payload.clone(), body: payload.clone(),
retryable: false, retryable: false,
suggested_action: suggested_action_for_status(status), suggested_action: suggested_action_for_status(status),
retry_after: None,
}); });
} }
} }
@@ -1627,6 +1648,7 @@ fn parse_sse_frame(
body: payload.chars().take(200).collect(), body: payload.chars().take(200).collect(),
retryable: false, retryable: false,
suggested_action: Some("verify the API endpoint URL is correct".to_string()), suggested_action: Some("verify the API endpoint URL is correct".to_string()),
retry_after: None,
}); });
} }
serde_json::from_str::<ChatCompletionChunk>(&payload) serde_json::from_str::<ChatCompletionChunk>(&payload)
@@ -1678,10 +1700,12 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
return Ok(response); return Ok(response);
} }
let request_id = request_id_from_headers(response.headers()); let headers = response.headers().clone();
let request_id = request_id_from_headers(&headers);
let body = response.text().await.unwrap_or_default(); let body = response.text().await.unwrap_or_default();
let parsed_error = serde_json::from_str::<ErrorEnvelope>(&body).ok(); let parsed_error = serde_json::from_str::<ErrorEnvelope>(&body).ok();
let retryable = is_retryable_status(status); let retryable = is_retryable_status(status);
let retry_after = parse_retry_after(&headers, status);
let suggested_action = suggested_action_for_status(status); let suggested_action = suggested_action_for_status(status);
@@ -1697,13 +1721,43 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
body, body,
retryable, retryable,
suggested_action, suggested_action,
retry_after,
}) })
} }
fn parse_retry_after(
headers: &reqwest::header::HeaderMap,
status: reqwest::StatusCode,
) -> Option<std::time::Duration> {
if status != reqwest::StatusCode::TOO_MANY_REQUESTS {
return None;
}
headers
.get("retry-after")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.parse::<u64>().ok())
.map(std::time::Duration::from_secs)
}
const fn is_retryable_status(status: reqwest::StatusCode) -> bool { const fn is_retryable_status(status: reqwest::StatusCode) -> bool {
matches!(status.as_u16(), 408 | 409 | 429 | 500 | 502 | 503 | 504) matches!(status.as_u16(), 408 | 409 | 429 | 500 | 502 | 503 | 504)
} }
/// Some providers return HTTP 400 with an unparseable body when a gateway
/// or proxy flakes (e.g. "HTTP 400 from backend (no parseable body)").
/// These are transient network blips, not actual bad requests, and should
/// be retried.
fn is_retryable_400(status: reqwest::StatusCode, body: &str) -> bool {
if status != reqwest::StatusCode::BAD_REQUEST {
return false;
}
let lowered = body.to_ascii_lowercase();
lowered.contains("no parseable body")
|| lowered.contains("connection reset")
|| lowered.contains("broken pipe")
|| lowered.contains("empty reply from server")
}
/// Generate a suggested user action based on the HTTP status code and error context. /// Generate a suggested user action based on the HTTP status code and error context.
/// This provides actionable guidance when API requests fail. /// This provides actionable guidance when API requests fail.
fn suggested_action_for_status(status: reqwest::StatusCode) -> Option<String> { fn suggested_action_for_status(status: reqwest::StatusCode) -> Option<String> {

View File

@@ -125,6 +125,27 @@ pub struct RuntimePluginConfig {
max_output_tokens: Option<u32>, max_output_tokens: Option<u32>,
} }
/// API timeout and retry configuration.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ApiTimeoutConfig {
/// Connect timeout in seconds. Defaults to 30.
pub connect_timeout_secs: u64,
/// Request timeout in seconds. Defaults to 300 (5 minutes).
pub request_timeout_secs: u64,
/// Maximum retry attempts on transient failures. Defaults to 8.
pub max_retries: u32,
}
impl Default for ApiTimeoutConfig {
fn default() -> Self {
Self {
connect_timeout_secs: 30,
request_timeout_secs: 300,
max_retries: 8,
}
}
}
/// Structured feature configuration consumed by runtime subsystems. /// Structured feature configuration consumed by runtime subsystems.
#[derive(Debug, Clone, PartialEq, Eq, Default)] #[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct RuntimeFeatureConfig { pub struct RuntimeFeatureConfig {
@@ -139,6 +160,7 @@ pub struct RuntimeFeatureConfig {
sandbox: SandboxConfig, sandbox: SandboxConfig,
provider_fallbacks: ProviderFallbackConfig, provider_fallbacks: ProviderFallbackConfig,
trusted_roots: Vec<String>, trusted_roots: Vec<String>,
api_timeout: ApiTimeoutConfig,
rules_import: RulesImportConfig, rules_import: RulesImportConfig,
} }
@@ -740,6 +762,7 @@ fn build_runtime_config(
sandbox: parse_optional_sandbox_config(&merged_value)?, sandbox: parse_optional_sandbox_config(&merged_value)?,
provider_fallbacks: parse_optional_provider_fallbacks(&merged_value)?, provider_fallbacks: parse_optional_provider_fallbacks(&merged_value)?,
trusted_roots: parse_optional_trusted_roots(&merged_value)?, trusted_roots: parse_optional_trusted_roots(&merged_value)?,
api_timeout: parse_optional_api_timeout_config(&merged_value)?,
rules_import: parse_optional_rules_import(&merged_value)?, rules_import: parse_optional_rules_import(&merged_value)?,
}; };
@@ -2020,6 +2043,26 @@ fn parse_optional_provider_fallbacks(
Ok(ProviderFallbackConfig { primary, fallbacks }) Ok(ProviderFallbackConfig { primary, fallbacks })
} }
fn parse_optional_api_timeout_config(root: &JsonValue) -> Result<ApiTimeoutConfig, ConfigError> {
let Some(timeout_value) = root.as_object().and_then(|obj| obj.get("apiTimeout")) else {
return Ok(ApiTimeoutConfig::default());
};
let Some(obj) = timeout_value.as_object() else {
return Ok(ApiTimeoutConfig::default());
};
let context = "merged settings.apiTimeout";
let connect_timeout_secs = optional_u64(obj, "connectTimeout", context)?.unwrap_or(30);
let request_timeout_secs = optional_u64(obj, "requestTimeout", context)?.unwrap_or(300);
let max_retries = optional_u64(obj, "maxRetries", context)?
.map(|v| v as u32)
.unwrap_or(8);
Ok(ApiTimeoutConfig {
connect_timeout_secs,
request_timeout_secs,
max_retries,
})
}
fn parse_optional_trusted_roots(root: &JsonValue) -> Result<Vec<String>, ConfigError> { fn parse_optional_trusted_roots(root: &JsonValue) -> Result<Vec<String>, ConfigError> {
let Some(object) = root.as_object() else { let Some(object) = root.as_object() else {
return Ok(Vec::new()); return Ok(Vec::new());

View File

@@ -65,10 +65,10 @@ pub use compact::{
get_compact_continuation_message, should_compact, CompactionConfig, CompactionResult, get_compact_continuation_message, should_compact, CompactionConfig, CompactionResult,
}; };
pub use config::{ pub use config::{
suppress_config_warnings_for_json_mode, ConfigEntry, ConfigError, ConfigFileReport, suppress_config_warnings_for_json_mode, ApiTimeoutConfig, ConfigEntry, ConfigError,
ConfigFileStatus, ConfigInspection, ConfigLoader, ConfigSource, McpConfigCollection, ConfigFileReport, ConfigFileStatus, ConfigInspection, ConfigLoader, ConfigSource,
McpInvalidServerConfig, McpManagedProxyServerConfig, McpOAuthConfig, McpRemoteServerConfig, McpConfigCollection, McpInvalidServerConfig, McpManagedProxyServerConfig, McpOAuthConfig,
McpSdkServerConfig, McpServerConfig, McpStdioServerConfig, McpTransport, McpRemoteServerConfig, McpSdkServerConfig, McpServerConfig, McpStdioServerConfig, McpTransport,
McpWebSocketServerConfig, OAuthConfig, ProviderFallbackConfig, ResolvedPermissionMode, McpWebSocketServerConfig, OAuthConfig, ProviderFallbackConfig, ResolvedPermissionMode,
RulesImportConfig, RuntimeConfig, RuntimeFeatureConfig, RuntimeHookCommand, RuntimeHookConfig, RulesImportConfig, RuntimeConfig, RuntimeFeatureConfig, RuntimeHookCommand, RuntimeHookConfig,
RuntimeInvalidHookConfig, RuntimePermissionRuleConfig, RuntimePluginConfig, RuntimeInvalidHookConfig, RuntimePermissionRuleConfig, RuntimePluginConfig,

View File

@@ -14275,7 +14275,8 @@ mod tests {
body: String::new(), body: String::new(),
retryable: true, retryable: true,
suggested_action: None, suggested_action: None,
}; retry_after: None,
};
let rendered = format_user_visible_api_error("session-issue-22", &error); let rendered = format_user_visible_api_error("session-issue-22", &error);
assert!(rendered.contains("provider_internal")); assert!(rendered.contains("provider_internal"));
@@ -14298,7 +14299,8 @@ mod tests {
body: String::new(), body: String::new(),
retryable: true, retryable: true,
suggested_action: None, suggested_action: None,
}), retry_after: None,
}),
}; };
let rendered = format_user_visible_api_error("session-issue-22", &error); let rendered = format_user_visible_api_error("session-issue-22", &error);
@@ -14362,7 +14364,8 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
}; retry_after: None,
};
let rendered = format_user_visible_api_error("session-issue-32", &error); let rendered = format_user_visible_api_error("session-issue-32", &error);
assert!(rendered.contains("context_window_blocked"), "{rendered}"); assert!(rendered.contains("context_window_blocked"), "{rendered}");
@@ -14395,6 +14398,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
retry_after: None,
}; };
let rendered = format_user_visible_api_error("session-issue-32", &error); let rendered = format_user_visible_api_error("session-issue-32", &error);
@@ -14429,6 +14433,7 @@ mod tests {
body: String::new(), body: String::new(),
retryable: false, retryable: false,
suggested_action: None, suggested_action: None,
retry_after: None,
}), }),
}; };