mirror of
https://github.com/instructkr/claw-code.git
synced 2026-06-05 22:17:10 +08:00
Merge pull request #3214 from TheArchitectit/worktree-api-timeout-retry-v2
feat: API timeout config, Retry-After header, configurable retry, and 400 transient retry
This commit is contained in:
@@ -20,6 +20,7 @@ const CONTEXT_WINDOW_ERROR_MARKERS: &[&str] = &[
|
||||
"completion tokens",
|
||||
"prompt tokens",
|
||||
"request is too large",
|
||||
"no parseable body",
|
||||
];
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -60,6 +61,9 @@ pub enum ApiError {
|
||||
retryable: bool,
|
||||
/// Suggested user action based on error type (e.g., "Reduce prompt size" for 413)
|
||||
suggested_action: Option<String>,
|
||||
/// Parsed Retry-After header value (seconds) for 429 responses.
|
||||
/// When present, overrides the exponential backoff delay.
|
||||
retry_after: Option<Duration>,
|
||||
},
|
||||
RetriesExhausted {
|
||||
attempts: u32,
|
||||
@@ -128,6 +132,17 @@ impl ApiError {
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
/// Return the `Retry-After` delay if this error came from a 429 response
|
||||
/// that included a `retry-after` header. Callers should prefer this value
|
||||
/// over the computed backoff delay when it exists.
|
||||
pub fn retry_after(&self) -> Option<Duration> {
|
||||
match self {
|
||||
Self::Api { retry_after, .. } => *retry_after,
|
||||
Self::RetriesExhausted { last_error, .. } => last_error.retry_after(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_retryable(&self) -> bool {
|
||||
match self {
|
||||
Self::Http(error) => error.is_connect() || error.is_timeout() || error.is_request(),
|
||||
@@ -529,6 +544,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: true,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
assert!(error.is_generic_fatal_wrapper());
|
||||
@@ -552,6 +568,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: true,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
}),
|
||||
};
|
||||
|
||||
@@ -573,6 +590,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
assert!(error.is_context_window_failure());
|
||||
@@ -593,6 +611,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
assert!(error.is_context_window_failure());
|
||||
|
||||
@@ -1,9 +1,69 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::error::ApiError;
|
||||
|
||||
const HTTP_PROXY_KEYS: [&str; 2] = ["HTTP_PROXY", "http_proxy"];
|
||||
const HTTPS_PROXY_KEYS: [&str; 2] = ["HTTPS_PROXY", "https_proxy"];
|
||||
const NO_PROXY_KEYS: [&str; 2] = ["NO_PROXY", "no_proxy"];
|
||||
|
||||
/// Timeout configuration for outbound HTTP requests.
|
||||
///
|
||||
/// When set, the `reqwest::Client` will abort requests that take longer
|
||||
/// than the configured duration and return a timeout error (which is
|
||||
/// retryable by the existing exponential backoff logic).
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct TimeoutConfig {
|
||||
/// Maximum time to wait for a connection to be established.
|
||||
/// Defaults to 30 seconds.
|
||||
pub connect_timeout: Duration,
|
||||
/// Maximum time for the entire request (including reading the response
|
||||
/// body). For streaming responses this is the timeout for the initial
|
||||
/// handshake only; the stream itself is governed by SSE parsing.
|
||||
/// Defaults to 5 minutes (300 seconds).
|
||||
pub request_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for TimeoutConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
connect_timeout: Duration::from_secs(30),
|
||||
request_timeout: Duration::from_secs(300),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TimeoutConfig {
|
||||
/// Read timeout settings from the process environment.
|
||||
/// - `CLAW_API_CONNECT_TIMEOUT` — connect timeout in seconds
|
||||
/// - `CLAW_API_REQUEST_TIMEOUT` — overall request timeout in seconds
|
||||
#[must_use]
|
||||
pub fn from_env() -> Self {
|
||||
let connect_timeout = std::env::var("CLAW_API_CONNECT_TIMEOUT")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_secs)
|
||||
.unwrap_or(Duration::from_secs(30));
|
||||
let request_timeout = std::env::var("CLAW_API_REQUEST_TIMEOUT")
|
||||
.ok()
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(Duration::from_secs)
|
||||
.unwrap_or(Duration::from_secs(300));
|
||||
Self {
|
||||
connect_timeout,
|
||||
request_timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create from explicit second values (used by config file parsing).
|
||||
#[must_use]
|
||||
pub fn from_seconds(connect_secs: u64, request_secs: u64) -> Self {
|
||||
Self {
|
||||
connect_timeout: Duration::from_secs(connect_secs),
|
||||
request_timeout: Duration::from_secs(request_secs),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot of the proxy-related environment variables that influence the
|
||||
/// outbound HTTP client. Captured up front so callers can inspect, log, and
|
||||
/// test the resolved configuration without re-reading the process environment.
|
||||
@@ -61,7 +121,7 @@ impl ProxyConfig {
|
||||
/// `HTTPS_PROXY`, and `NO_PROXY` environment variables. When no proxy is
|
||||
/// configured the client behaves identically to `reqwest::Client::new()`.
|
||||
pub fn build_http_client() -> Result<reqwest::Client, ApiError> {
|
||||
build_http_client_with(&ProxyConfig::from_env())
|
||||
build_http_client_with_opts(&ProxyConfig::from_env(), &TimeoutConfig::from_env())
|
||||
}
|
||||
|
||||
/// Infallible counterpart to [`build_http_client`] for constructors that
|
||||
@@ -71,12 +131,13 @@ pub fn build_http_client() -> Result<reqwest::Client, ApiError> {
|
||||
/// first outbound request instead of at construction time.
|
||||
#[must_use]
|
||||
pub fn build_http_client_or_default() -> reqwest::Client {
|
||||
build_http_client().unwrap_or_else(|_| {
|
||||
reqwest::Client::builder()
|
||||
.user_agent("clawd-rust-tools/0.1")
|
||||
.build()
|
||||
.expect("default client with user_agent should always succeed")
|
||||
})
|
||||
build_http_client_with_opts(&ProxyConfig::from_env(), &TimeoutConfig::from_env())
|
||||
.unwrap_or_else(|_| {
|
||||
reqwest::Client::builder()
|
||||
.user_agent("clawd-rust-tools/0.1")
|
||||
.build()
|
||||
.expect("default client with user_agent should always succeed")
|
||||
})
|
||||
}
|
||||
|
||||
/// Build a `reqwest::Client` from an explicit [`ProxyConfig`]. Used by tests
|
||||
@@ -86,9 +147,20 @@ pub fn build_http_client_or_default() -> reqwest::Client {
|
||||
/// and `https_proxy` fields and is registered as both an HTTP and HTTPS
|
||||
/// proxy so a single value can route every outbound request.
|
||||
pub fn build_http_client_with(config: &ProxyConfig) -> Result<reqwest::Client, ApiError> {
|
||||
build_http_client_with_opts(config, &TimeoutConfig::from_env())
|
||||
}
|
||||
|
||||
/// Build a `reqwest::Client` from explicit [`ProxyConfig`] and [`TimeoutConfig`].
|
||||
/// Used by callers that want to control both proxy routing and request timing.
|
||||
pub fn build_http_client_with_opts(
|
||||
config: &ProxyConfig,
|
||||
timeout: &TimeoutConfig,
|
||||
) -> Result<reqwest::Client, ApiError> {
|
||||
let mut builder = reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.user_agent("clawd-rust-tools/0.1");
|
||||
.user_agent("clawd-rust-tools/0.1")
|
||||
.connect_timeout(timeout.connect_timeout)
|
||||
.timeout(timeout.request_timeout);
|
||||
|
||||
let no_proxy = config
|
||||
.no_proxy
|
||||
@@ -131,7 +203,7 @@ where
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::{build_http_client_with, ProxyConfig};
|
||||
use super::{build_http_client_with, build_http_client_with_opts, ProxyConfig, TimeoutConfig};
|
||||
|
||||
fn config_from_map(pairs: &[(&str, &str)]) -> ProxyConfig {
|
||||
let map: HashMap<String, String> = pairs
|
||||
@@ -143,30 +215,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn proxy_config_is_empty_when_no_env_vars_are_set() {
|
||||
// given
|
||||
let config = config_from_map(&[]);
|
||||
|
||||
// when
|
||||
let empty = config.is_empty();
|
||||
|
||||
// then
|
||||
assert!(empty);
|
||||
assert!(config.is_empty());
|
||||
assert_eq!(config, ProxyConfig::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_config_reads_uppercase_http_https_and_no_proxy() {
|
||||
// given
|
||||
let pairs = [
|
||||
("HTTP_PROXY", "http://proxy.internal:3128"),
|
||||
("HTTPS_PROXY", "http://secure.internal:3129"),
|
||||
("NO_PROXY", "localhost,127.0.0.1,.corp"),
|
||||
];
|
||||
|
||||
// when
|
||||
let config = config_from_map(&pairs);
|
||||
|
||||
// then
|
||||
assert_eq!(
|
||||
config.http_proxy.as_deref(),
|
||||
Some("http://proxy.internal:3128")
|
||||
@@ -184,17 +245,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn proxy_config_falls_back_to_lowercase_keys() {
|
||||
// given
|
||||
let pairs = [
|
||||
("http_proxy", "http://lower.internal:3128"),
|
||||
("https_proxy", "http://lower-secure.internal:3129"),
|
||||
("no_proxy", ".lower"),
|
||||
];
|
||||
|
||||
// when
|
||||
let config = config_from_map(&pairs);
|
||||
|
||||
// then
|
||||
assert_eq!(
|
||||
config.http_proxy.as_deref(),
|
||||
Some("http://lower.internal:3128")
|
||||
@@ -208,16 +264,11 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn proxy_config_prefers_uppercase_over_lowercase_when_both_set() {
|
||||
// given
|
||||
let pairs = [
|
||||
("HTTP_PROXY", "http://upper.internal:3128"),
|
||||
("http_proxy", "http://lower.internal:3128"),
|
||||
];
|
||||
|
||||
// when
|
||||
let config = config_from_map(&pairs);
|
||||
|
||||
// then
|
||||
assert_eq!(
|
||||
config.http_proxy.as_deref(),
|
||||
Some("http://upper.internal:3128")
|
||||
@@ -226,59 +277,39 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn proxy_config_treats_empty_strings_as_unset() {
|
||||
// given
|
||||
let pairs = [("HTTP_PROXY", ""), ("http_proxy", "")];
|
||||
|
||||
// when
|
||||
let config = config_from_map(&pairs);
|
||||
|
||||
// then
|
||||
assert!(config.http_proxy.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_http_client_succeeds_when_no_proxy_is_configured() {
|
||||
// given
|
||||
let config = ProxyConfig::default();
|
||||
|
||||
// when
|
||||
let result = build_http_client_with(&config);
|
||||
|
||||
// then
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_http_client_succeeds_with_valid_http_and_https_proxies() {
|
||||
// given
|
||||
let config = ProxyConfig {
|
||||
http_proxy: Some("http://proxy.internal:3128".to_string()),
|
||||
https_proxy: Some("http://secure.internal:3129".to_string()),
|
||||
no_proxy: Some("localhost,127.0.0.1".to_string()),
|
||||
proxy_url: None,
|
||||
};
|
||||
|
||||
// when
|
||||
let result = build_http_client_with(&config);
|
||||
|
||||
// then
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_http_client_returns_http_error_for_invalid_proxy_url() {
|
||||
// given
|
||||
let config = ProxyConfig {
|
||||
http_proxy: None,
|
||||
https_proxy: Some("not a url".to_string()),
|
||||
no_proxy: None,
|
||||
proxy_url: None,
|
||||
};
|
||||
|
||||
// when
|
||||
let result = build_http_client_with(&config);
|
||||
|
||||
// then
|
||||
let error = result.expect_err("invalid proxy URL must be reported as a build failure");
|
||||
assert!(
|
||||
matches!(error, crate::error::ApiError::Http(_)),
|
||||
@@ -288,10 +319,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn from_proxy_url_sets_unified_field_and_leaves_per_scheme_empty() {
|
||||
// given / when
|
||||
let config = ProxyConfig::from_proxy_url("http://unified.internal:3128");
|
||||
|
||||
// then
|
||||
assert_eq!(
|
||||
config.proxy_url.as_deref(),
|
||||
Some("http://unified.internal:3128")
|
||||
@@ -303,49 +331,56 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn build_http_client_succeeds_with_unified_proxy_url() {
|
||||
// given
|
||||
let config = ProxyConfig {
|
||||
proxy_url: Some("http://unified.internal:3128".to_string()),
|
||||
no_proxy: Some("localhost".to_string()),
|
||||
..ProxyConfig::default()
|
||||
};
|
||||
|
||||
// when
|
||||
let result = build_http_client_with(&config);
|
||||
|
||||
// then
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_url_takes_precedence_over_per_scheme_fields() {
|
||||
// given – both per-scheme and unified are set
|
||||
let config = ProxyConfig {
|
||||
http_proxy: Some("http://per-scheme.internal:1111".to_string()),
|
||||
https_proxy: Some("http://per-scheme.internal:2222".to_string()),
|
||||
no_proxy: None,
|
||||
proxy_url: Some("http://unified.internal:3128".to_string()),
|
||||
};
|
||||
|
||||
// when – building succeeds (the unified URL is valid)
|
||||
let result = build_http_client_with(&config);
|
||||
|
||||
// then
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_http_client_returns_error_for_invalid_unified_proxy_url() {
|
||||
// given
|
||||
let config = ProxyConfig::from_proxy_url("not a url");
|
||||
|
||||
// when
|
||||
let result = build_http_client_with(&config);
|
||||
|
||||
// then
|
||||
assert!(
|
||||
matches!(result, Err(crate::error::ApiError::Http(_))),
|
||||
"invalid unified proxy URL should fail: {result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeout_config_defaults() {
|
||||
let config = TimeoutConfig::default();
|
||||
assert_eq!(config.connect_timeout, std::time::Duration::from_secs(30));
|
||||
assert_eq!(config.request_timeout, std::time::Duration::from_secs(300));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timeout_config_from_seconds() {
|
||||
let config = TimeoutConfig::from_seconds(10, 60);
|
||||
assert_eq!(config.connect_timeout, std::time::Duration::from_secs(10));
|
||||
assert_eq!(config.request_timeout, std::time::Duration::from_secs(60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_http_client_with_custom_timeouts() {
|
||||
let config = ProxyConfig::default();
|
||||
let timeout = TimeoutConfig::from_seconds(5, 120);
|
||||
let result = build_http_client_with_opts(&config, &timeout);
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,7 +12,8 @@ pub use client::{
|
||||
};
|
||||
pub use error::ApiError;
|
||||
pub use http_client::{
|
||||
build_http_client, build_http_client_or_default, build_http_client_with, ProxyConfig,
|
||||
build_http_client, build_http_client_or_default, build_http_client_with,
|
||||
build_http_client_with_opts, ProxyConfig, TimeoutConfig,
|
||||
};
|
||||
pub use prompt_cache::{
|
||||
CacheBreakEvent, PromptCache, PromptCacheConfig, PromptCachePaths, PromptCacheRecord,
|
||||
|
||||
@@ -211,6 +211,19 @@ impl AnthropicClient {
|
||||
self
|
||||
}
|
||||
|
||||
/// Replace the internal HTTP client with one that respects the given
|
||||
/// timeout configuration. This controls connect and request-level
|
||||
/// timeouts for all outbound API calls.
|
||||
#[must_use]
|
||||
pub fn with_timeout(mut self, timeout: &crate::http_client::TimeoutConfig) -> Self {
|
||||
self.http = crate::http_client::build_http_client_with_opts(
|
||||
&crate::http_client::ProxyConfig::from_env(),
|
||||
timeout,
|
||||
)
|
||||
.unwrap_or_else(|_| reqwest::Client::new());
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn with_session_tracer(mut self, session_tracer: SessionTracer) -> Self {
|
||||
self.session_tracer = Some(session_tracer);
|
||||
@@ -454,7 +467,13 @@ impl AnthropicClient {
|
||||
break;
|
||||
}
|
||||
|
||||
tokio::time::sleep(self.jittered_backoff_for_attempt(attempts)?).await;
|
||||
let delay = if let Some(retry_after) = last_error.as_ref().and_then(|e| e.retry_after())
|
||||
{
|
||||
retry_after
|
||||
} else {
|
||||
self.jittered_backoff_for_attempt(attempts)?
|
||||
};
|
||||
tokio::time::sleep(delay).await;
|
||||
}
|
||||
|
||||
Err(ApiError::RetriesExhausted {
|
||||
@@ -866,10 +885,12 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
let request_id = request_id_from_headers(response.headers());
|
||||
let headers = response.headers().clone();
|
||||
let request_id = request_id_from_headers(&headers);
|
||||
let body = response.text().await.unwrap_or_else(|_| String::new());
|
||||
let parsed_error = serde_json::from_str::<AnthropicErrorEnvelope>(&body).ok();
|
||||
let retryable = is_retryable_status(status);
|
||||
let retry_after = parse_retry_after(&headers, status);
|
||||
|
||||
Err(ApiError::Api {
|
||||
status,
|
||||
@@ -883,13 +904,44 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
|
||||
body,
|
||||
retryable,
|
||||
suggested_action: None,
|
||||
retry_after,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_retry_after(
|
||||
headers: &reqwest::header::HeaderMap,
|
||||
status: reqwest::StatusCode,
|
||||
) -> Option<std::time::Duration> {
|
||||
if status != reqwest::StatusCode::TOO_MANY_REQUESTS {
|
||||
return None;
|
||||
}
|
||||
headers
|
||||
.get("retry-after")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(std::time::Duration::from_secs)
|
||||
}
|
||||
|
||||
const fn is_retryable_status(status: reqwest::StatusCode) -> bool {
|
||||
matches!(status.as_u16(), 408 | 409 | 429 | 500 | 502 | 503 | 504)
|
||||
}
|
||||
|
||||
/// Some providers return HTTP 400 with an unparseable body when a gateway
|
||||
/// or proxy flakes (e.g. "HTTP 400 from backend (no parseable body)").
|
||||
/// These are transient network blips, not actual bad requests, and should
|
||||
/// be retried. We detect them by checking the body for known gateway error
|
||||
/// phrases.
|
||||
fn is_retryable_400(status: reqwest::StatusCode, body: &str) -> bool {
|
||||
if status != reqwest::StatusCode::BAD_REQUEST {
|
||||
return false;
|
||||
}
|
||||
let lowered = body.to_ascii_lowercase();
|
||||
lowered.contains("no parseable body")
|
||||
|| lowered.contains("connection reset")
|
||||
|| lowered.contains("broken pipe")
|
||||
|| lowered.contains("empty reply from server")
|
||||
}
|
||||
|
||||
/// Anthropic API keys (`sk-ant-*`) are accepted over the `x-api-key` header
|
||||
/// and rejected with HTTP 401 "Invalid bearer token" when sent as a Bearer
|
||||
/// token via `ANTHROPIC_AUTH_TOKEN`. This happens often enough in the wild
|
||||
@@ -908,6 +960,8 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
|
||||
body,
|
||||
retryable,
|
||||
suggested_action,
|
||||
retry_after,
|
||||
..
|
||||
} = error
|
||||
else {
|
||||
return error;
|
||||
@@ -921,6 +975,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
|
||||
body,
|
||||
retryable,
|
||||
suggested_action,
|
||||
retry_after,
|
||||
};
|
||||
}
|
||||
let Some(bearer_token) = auth.bearer_token() else {
|
||||
@@ -932,6 +987,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
|
||||
body,
|
||||
retryable,
|
||||
suggested_action,
|
||||
retry_after,
|
||||
};
|
||||
};
|
||||
if !bearer_token.starts_with("sk-ant-") {
|
||||
@@ -943,6 +999,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
|
||||
body,
|
||||
retryable,
|
||||
suggested_action,
|
||||
retry_after,
|
||||
};
|
||||
}
|
||||
// Only append the hint when the AuthSource is pure BearerToken. If both
|
||||
@@ -958,6 +1015,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
|
||||
body,
|
||||
retryable,
|
||||
suggested_action,
|
||||
retry_after,
|
||||
};
|
||||
}
|
||||
let enriched_message = match message {
|
||||
@@ -972,6 +1030,7 @@ fn enrich_bearer_auth_error(error: ApiError, auth: &AuthSource) -> ApiError {
|
||||
body,
|
||||
retryable,
|
||||
suggested_action,
|
||||
retry_after,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1596,6 +1655,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
// when
|
||||
@@ -1637,6 +1697,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: true,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
// when
|
||||
@@ -1666,6 +1727,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
// when
|
||||
@@ -1694,6 +1756,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
// when
|
||||
@@ -1719,6 +1782,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
// when
|
||||
|
||||
@@ -175,6 +175,18 @@ impl OpenAiCompatClient {
|
||||
self
|
||||
}
|
||||
|
||||
/// Replace the internal HTTP client with one that respects the given
|
||||
/// timeout configuration.
|
||||
#[must_use]
|
||||
pub fn with_timeout(mut self, timeout: &crate::http_client::TimeoutConfig) -> Self {
|
||||
self.http = crate::http_client::build_http_client_with_opts(
|
||||
&crate::http_client::ProxyConfig::from_env(),
|
||||
timeout,
|
||||
)
|
||||
.unwrap_or_else(|_| reqwest::Client::new());
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn send_message(
|
||||
&self,
|
||||
request: &MessageRequest,
|
||||
@@ -217,6 +229,7 @@ impl OpenAiCompatClient {
|
||||
reqwest::StatusCode::from_u16(code.unwrap_or(400))
|
||||
.unwrap_or(reqwest::StatusCode::BAD_REQUEST),
|
||||
),
|
||||
retry_after: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -270,7 +283,12 @@ impl OpenAiCompatClient {
|
||||
break retryable_error;
|
||||
}
|
||||
|
||||
tokio::time::sleep(self.jittered_backoff_for_attempt(attempts)?).await;
|
||||
let delay = if let Some(retry_after) = retryable_error.retry_after() {
|
||||
retry_after
|
||||
} else {
|
||||
self.jittered_backoff_for_attempt(attempts)?
|
||||
};
|
||||
tokio::time::sleep(delay).await;
|
||||
};
|
||||
|
||||
Err(ApiError::RetriesExhausted {
|
||||
@@ -1561,6 +1579,7 @@ fn parse_sse_frame(
|
||||
body: trimmed.chars().take(500).collect(),
|
||||
retryable: false,
|
||||
suggested_action: suggested_action_for_status(status),
|
||||
retry_after: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1576,6 +1595,7 @@ fn parse_sse_frame(
|
||||
body: trimmed.chars().take(200).collect(),
|
||||
retryable: false,
|
||||
suggested_action: Some("verify the API endpoint URL is correct".to_string()),
|
||||
retry_after: None,
|
||||
});
|
||||
}
|
||||
return Ok(None);
|
||||
@@ -1611,6 +1631,7 @@ fn parse_sse_frame(
|
||||
body: payload.clone(),
|
||||
retryable: false,
|
||||
suggested_action: suggested_action_for_status(status),
|
||||
retry_after: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1627,6 +1648,7 @@ fn parse_sse_frame(
|
||||
body: payload.chars().take(200).collect(),
|
||||
retryable: false,
|
||||
suggested_action: Some("verify the API endpoint URL is correct".to_string()),
|
||||
retry_after: None,
|
||||
});
|
||||
}
|
||||
serde_json::from_str::<ChatCompletionChunk>(&payload)
|
||||
@@ -1678,10 +1700,12 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
|
||||
return Ok(response);
|
||||
}
|
||||
|
||||
let request_id = request_id_from_headers(response.headers());
|
||||
let headers = response.headers().clone();
|
||||
let request_id = request_id_from_headers(&headers);
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
let parsed_error = serde_json::from_str::<ErrorEnvelope>(&body).ok();
|
||||
let retryable = is_retryable_status(status);
|
||||
let retry_after = parse_retry_after(&headers, status);
|
||||
|
||||
let suggested_action = suggested_action_for_status(status);
|
||||
|
||||
@@ -1697,13 +1721,43 @@ async fn expect_success(response: reqwest::Response) -> Result<reqwest::Response
|
||||
body,
|
||||
retryable,
|
||||
suggested_action,
|
||||
retry_after,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_retry_after(
|
||||
headers: &reqwest::header::HeaderMap,
|
||||
status: reqwest::StatusCode,
|
||||
) -> Option<std::time::Duration> {
|
||||
if status != reqwest::StatusCode::TOO_MANY_REQUESTS {
|
||||
return None;
|
||||
}
|
||||
headers
|
||||
.get("retry-after")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.map(std::time::Duration::from_secs)
|
||||
}
|
||||
|
||||
const fn is_retryable_status(status: reqwest::StatusCode) -> bool {
|
||||
matches!(status.as_u16(), 408 | 409 | 429 | 500 | 502 | 503 | 504)
|
||||
}
|
||||
|
||||
/// Some providers return HTTP 400 with an unparseable body when a gateway
|
||||
/// or proxy flakes (e.g. "HTTP 400 from backend (no parseable body)").
|
||||
/// These are transient network blips, not actual bad requests, and should
|
||||
/// be retried.
|
||||
fn is_retryable_400(status: reqwest::StatusCode, body: &str) -> bool {
|
||||
if status != reqwest::StatusCode::BAD_REQUEST {
|
||||
return false;
|
||||
}
|
||||
let lowered = body.to_ascii_lowercase();
|
||||
lowered.contains("no parseable body")
|
||||
|| lowered.contains("connection reset")
|
||||
|| lowered.contains("broken pipe")
|
||||
|| lowered.contains("empty reply from server")
|
||||
}
|
||||
|
||||
/// Generate a suggested user action based on the HTTP status code and error context.
|
||||
/// This provides actionable guidance when API requests fail.
|
||||
fn suggested_action_for_status(status: reqwest::StatusCode) -> Option<String> {
|
||||
|
||||
@@ -125,6 +125,27 @@ pub struct RuntimePluginConfig {
|
||||
max_output_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// API timeout and retry configuration.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ApiTimeoutConfig {
|
||||
/// Connect timeout in seconds. Defaults to 30.
|
||||
pub connect_timeout_secs: u64,
|
||||
/// Request timeout in seconds. Defaults to 300 (5 minutes).
|
||||
pub request_timeout_secs: u64,
|
||||
/// Maximum retry attempts on transient failures. Defaults to 8.
|
||||
pub max_retries: u32,
|
||||
}
|
||||
|
||||
impl Default for ApiTimeoutConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
connect_timeout_secs: 30,
|
||||
request_timeout_secs: 300,
|
||||
max_retries: 8,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Structured feature configuration consumed by runtime subsystems.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct RuntimeFeatureConfig {
|
||||
@@ -139,6 +160,7 @@ pub struct RuntimeFeatureConfig {
|
||||
sandbox: SandboxConfig,
|
||||
provider_fallbacks: ProviderFallbackConfig,
|
||||
trusted_roots: Vec<String>,
|
||||
api_timeout: ApiTimeoutConfig,
|
||||
rules_import: RulesImportConfig,
|
||||
}
|
||||
|
||||
@@ -740,6 +762,7 @@ fn build_runtime_config(
|
||||
sandbox: parse_optional_sandbox_config(&merged_value)?,
|
||||
provider_fallbacks: parse_optional_provider_fallbacks(&merged_value)?,
|
||||
trusted_roots: parse_optional_trusted_roots(&merged_value)?,
|
||||
api_timeout: parse_optional_api_timeout_config(&merged_value)?,
|
||||
rules_import: parse_optional_rules_import(&merged_value)?,
|
||||
};
|
||||
|
||||
@@ -2020,6 +2043,26 @@ fn parse_optional_provider_fallbacks(
|
||||
Ok(ProviderFallbackConfig { primary, fallbacks })
|
||||
}
|
||||
|
||||
fn parse_optional_api_timeout_config(root: &JsonValue) -> Result<ApiTimeoutConfig, ConfigError> {
|
||||
let Some(timeout_value) = root.as_object().and_then(|obj| obj.get("apiTimeout")) else {
|
||||
return Ok(ApiTimeoutConfig::default());
|
||||
};
|
||||
let Some(obj) = timeout_value.as_object() else {
|
||||
return Ok(ApiTimeoutConfig::default());
|
||||
};
|
||||
let context = "merged settings.apiTimeout";
|
||||
let connect_timeout_secs = optional_u64(obj, "connectTimeout", context)?.unwrap_or(30);
|
||||
let request_timeout_secs = optional_u64(obj, "requestTimeout", context)?.unwrap_or(300);
|
||||
let max_retries = optional_u64(obj, "maxRetries", context)?
|
||||
.map(|v| v as u32)
|
||||
.unwrap_or(8);
|
||||
Ok(ApiTimeoutConfig {
|
||||
connect_timeout_secs,
|
||||
request_timeout_secs,
|
||||
max_retries,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_optional_trusted_roots(root: &JsonValue) -> Result<Vec<String>, ConfigError> {
|
||||
let Some(object) = root.as_object() else {
|
||||
return Ok(Vec::new());
|
||||
|
||||
@@ -65,10 +65,10 @@ pub use compact::{
|
||||
get_compact_continuation_message, should_compact, CompactionConfig, CompactionResult,
|
||||
};
|
||||
pub use config::{
|
||||
suppress_config_warnings_for_json_mode, ConfigEntry, ConfigError, ConfigFileReport,
|
||||
ConfigFileStatus, ConfigInspection, ConfigLoader, ConfigSource, McpConfigCollection,
|
||||
McpInvalidServerConfig, McpManagedProxyServerConfig, McpOAuthConfig, McpRemoteServerConfig,
|
||||
McpSdkServerConfig, McpServerConfig, McpStdioServerConfig, McpTransport,
|
||||
suppress_config_warnings_for_json_mode, ApiTimeoutConfig, ConfigEntry, ConfigError,
|
||||
ConfigFileReport, ConfigFileStatus, ConfigInspection, ConfigLoader, ConfigSource,
|
||||
McpConfigCollection, McpInvalidServerConfig, McpManagedProxyServerConfig, McpOAuthConfig,
|
||||
McpRemoteServerConfig, McpSdkServerConfig, McpServerConfig, McpStdioServerConfig, McpTransport,
|
||||
McpWebSocketServerConfig, OAuthConfig, ProviderFallbackConfig, ResolvedPermissionMode,
|
||||
RulesImportConfig, RuntimeConfig, RuntimeFeatureConfig, RuntimeHookCommand, RuntimeHookConfig,
|
||||
RuntimeInvalidHookConfig, RuntimePermissionRuleConfig, RuntimePluginConfig,
|
||||
|
||||
@@ -14275,7 +14275,8 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: true,
|
||||
suggested_action: None,
|
||||
};
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
let rendered = format_user_visible_api_error("session-issue-22", &error);
|
||||
assert!(rendered.contains("provider_internal"));
|
||||
@@ -14298,7 +14299,8 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: true,
|
||||
suggested_action: None,
|
||||
}),
|
||||
retry_after: None,
|
||||
}),
|
||||
};
|
||||
|
||||
let rendered = format_user_visible_api_error("session-issue-22", &error);
|
||||
@@ -14362,7 +14364,8 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
};
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
let rendered = format_user_visible_api_error("session-issue-32", &error);
|
||||
assert!(rendered.contains("context_window_blocked"), "{rendered}");
|
||||
@@ -14395,6 +14398,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
};
|
||||
|
||||
let rendered = format_user_visible_api_error("session-issue-32", &error);
|
||||
@@ -14429,6 +14433,7 @@ mod tests {
|
||||
body: String::new(),
|
||||
retryable: false,
|
||||
suggested_action: None,
|
||||
retry_after: None,
|
||||
}),
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user