eeymoo/claw-code
1
0
Fork 0
mirror of https://github.com/instructkr/claw-code.git synced 2026-06-04 13:37:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: route local OpenAI-compatible models

Browse Source
This commit is contained in:
bellman
2026-06-03 23:16:46 +09:00
parent 9522674c87
commit bcc5bfde9c
7 changed files with 264 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
rust/crates/api/src/client.rs
View File

@@ -235,4 +235,22 @@ mod tests {
other => panic!("Expected ProviderClient::OpenAi for qwen-plus, got: {other:?}"),
}
}
#[test]
fn local_openai_base_url_routes_authless_ollama_models() {
let _lock = env_lock();
let _base_url = EnvVarGuard::set("OPENAI_BASE_URL", Some("http://127.0.0.1:11434/v1"));
let _openai_key = EnvVarGuard::set("OPENAI_API_KEY", None);
let _anthropic_key = EnvVarGuard::set("ANTHROPIC_API_KEY", Some("test-anthropic-key"));
let _anthropic_token = EnvVarGuard::set("ANTHROPIC_AUTH_TOKEN", None);
let client = ProviderClient::from_model("qwen2.5-coder:7b")
.expect("local model should route to OpenAI-compatible client without auth");
match client {
ProviderClient::OpenAi(openai_client) => {
assert_eq!(openai_client.base_url(), "http://127.0.0.1:11434/v1")
}
other => panic!("Expected ProviderClient::OpenAi for local model, got: {other:?}"),
}
}
}

38
rust/crates/api/src/providers/mod.rs
View File

@@ -262,6 +262,14 @@ pub fn metadata_for_model(model: &str) -> Option<ProviderMetadata> {
default_base_url: openai_compat::DEFAULT_OPENAI_BASE_URL,
});
}
if canonical.starts_with("local/") {
return Some(ProviderMetadata {
provider: ProviderKind::OpenAi,
auth_env: "OPENAI_API_KEY",
base_url_env: "OPENAI_BASE_URL",
default_base_url: openai_compat::DEFAULT_OPENAI_BASE_URL,
});
}
// Alibaba DashScope compatible-mode endpoint. Routes qwen/* and bare
// qwen-* model names (qwen-max, qwen-plus, qwen-turbo, qwen-qwq, etc.)
// to the OpenAI-compat client pointed at DashScope's /compatible-mode/v1.
@@ -337,17 +345,21 @@ pub fn provider_diagnostics_for_model(model: &str) -> ProviderDiagnostics {
}
}
fn looks_like_local_openai_model(model: &str) -> bool {
model.contains(':') || model.contains('.')
}
#[must_use]
pub fn detect_provider_kind(model: &str) -> ProviderKind {
if let Some(metadata) = metadata_for_model(model) {
let resolved_model = resolve_model_alias(model);
if let Some(metadata) = metadata_for_model(&resolved_model) {
return metadata.provider;
}
// When OPENAI_BASE_URL is set, the user explicitly configured an
// OpenAI-compatible endpoint. Prefer it over the Anthropic fallback
// even when the model name has no recognized prefix — this is the
// common case for local providers (Ollama, LM Studio, vLLM, etc.)
// where model names like "qwen2.5-coder:7b" don't match any prefix.
if std::env::var_os("OPENAI_BASE_URL").is_some() && openai_compat::has_api_key("OPENAI_API_KEY")
// When OPENAI_BASE_URL is set and the unknown model name looks like a
// local server tag (for example `llama3.2` or `qwen2.5-coder:7b`), prefer
// the OpenAI-compatible endpoint over ambient Anthropic credentials.
if std::env::var_os("OPENAI_BASE_URL").is_some()
&& looks_like_local_openai_model(&resolved_model)
{
return ProviderKind::OpenAi;
}
@@ -1042,6 +1054,18 @@ mod tests {
assert_eq!(kind2, ProviderKind::OpenAi);
}
#[test]
fn local_prefix_routes_to_openai_not_anthropic() {
let meta = super::metadata_for_model("local/Qwen/Qwen3.6-27B-FP8")
.expect("local/ prefix must resolve to OpenAI-compatible metadata");
assert_eq!(meta.provider, ProviderKind::OpenAi);
assert_eq!(meta.auth_env, "OPENAI_API_KEY");
assert_eq!(meta.base_url_env, "OPENAI_BASE_URL");
let kind = detect_provider_kind("local/Qwen/Qwen3.6-27B-FP8");
assert_eq!(kind, ProviderKind::OpenAi);
}
#[test]
fn qwen_prefix_routes_to_dashscope_not_anthropic() {
// User request from Discord #clawcode-get-help: web3g wants to use

173
rust/crates/api/src/providers/openai_compat.rs
View File

@@ -1,5 +1,6 @@
use std::borrow::Cow;
use std::collections::{BTreeMap, VecDeque};
use std::net::Ipv4Addr;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
@@ -131,13 +132,22 @@ impl OpenAiCompatClient {
}
pub fn from_env(config: OpenAiCompatConfig) -> Result<Self, ApiError> {
let Some(api_key) = read_env_non_empty(config.api_key_env)? else {
return Err(ApiError::missing_credentials(
config.provider_name,
config.credential_env_vars(),
));
let base_url = read_base_url(config);
let api_key = match read_env_non_empty(config.api_key_env)? {
Some(api_key) => api_key,
None if config.provider_name == "OpenAI"
&& is_local_openai_compatible_base_url(&base_url) =>
{
"local-dev-token".to_string()
}
None => {
return Err(ApiError::missing_credentials(
config.provider_name,
config.credential_env_vars(),
));
}
};
Ok(Self::new(api_key, config))
Ok(Self::new(api_key, config).with_base_url(base_url))
}
#[must_use]
@@ -915,14 +925,18 @@ pub fn model_requires_reasoning_content_in_history(model: &str) -> bool {
/// Strip routing prefix (e.g., "openai/gpt-4" → "gpt-4") for the wire.
/// The prefix is used only to select transport; the backend expects the
/// bare model id.
/// bare model id. Use `local/` to force OpenAI-compatible routing while
/// preserving any slashes that follow the prefix.
#[allow(dead_code)]
fn strip_routing_prefix(model: &str) -> &str {
if let Some(pos) = model.find('/') {
let prefix = &model[..pos];
// Only strip if the prefix before "/" is a known routing prefix,
// not if "/" appears in the middle of the model name for other reasons.
if matches!(prefix, "openai" | "xai" | "grok" | "qwen" | "kimi") {
if matches!(
prefix,
"openai" | "xai" | "grok" | "qwen" | "kimi" | "local"
) {
&model[pos + 1..]
} else {
model
@@ -932,6 +946,44 @@ fn strip_routing_prefix(model: &str) -> &str {
}
}
fn normalize_base_url_for_model_routing(url: &str) -> &str {
let trimmed = url.trim_end_matches('/');
trimmed
.strip_suffix("/chat/completions")
.map(|value| value.trim_end_matches('/'))
.unwrap_or(trimmed)
}
fn url_host(url: &str) -> &str {
let after_scheme = url.split_once("://").map_or(url, |(_, rest)| rest);
let authority = after_scheme.split(['/', '?', '#']).next().unwrap_or("");
let host_port = authority
.rsplit_once('@')
.map_or(authority, |(_, host_port)| host_port);
if host_port.starts_with('[') {
return host_port
.split(']')
.next()
.unwrap_or("")
.trim_start_matches('[');
}
host_port.split(':').next().unwrap_or("")
}
fn is_local_openai_compatible_base_url(url: &str) -> bool {
let host = url_host(url.trim());
if host.eq_ignore_ascii_case("localhost") || host == "::1" {
return true;
}
let Ok(address) = host.parse::<Ipv4Addr>() else {
return false;
};
let [first, second, ..] = address.octets();
matches!(first, 10 | 127)
|| first == 192 && second == 168
|| first == 172 && (16..=31).contains(&second)
}
fn wire_model_for_base_url<'a>(
model: &'a str,
config: OpenAiCompatConfig,
@@ -944,26 +996,22 @@ fn wire_model_for_base_url<'a>(
let lowered_prefix = prefix.to_ascii_lowercase();
if lowered_prefix == "openai" {
let trimmed_base_url = base_url.trim_end_matches('/');
let default_openai = DEFAULT_OPENAI_BASE_URL.trim_end_matches('/');
if matches!(
lowered_prefix.as_str(),
"xai" | "grok" | "kimi" | "gemini" | "gemma"
) {
let normalized_base_url = normalize_base_url_for_model_routing(base_url);
let default_base_url = normalize_base_url_for_model_routing(config.default_base_url);
if normalized_base_url.eq_ignore_ascii_case(default_base_url)
|| is_local_openai_compatible_base_url(base_url)
{
return Cow::Borrowed(&model[pos + 1..]);
}
if config.provider_name == "OpenAI" && trimmed_base_url != default_openai {
// Only preserve the full slug if it's NOT a model we want to strip
if !model.contains("gemini") && !model.contains("gemma") {
return Cow::Borrowed(model);
}
}
return Cow::Borrowed(&model[pos + 1..]);
return Cow::Borrowed(model);
}
if matches!(lowered_prefix.as_str(), "xai" | "grok" | "qwen" | "kimi") {
return Cow::Borrowed(&model[pos + 1..]);
}
if lowered_prefix == "local" {
return Cow::Borrowed(&model[pos + 1..]);
}
Cow::Borrowed(model)
}
@@ -1708,6 +1756,7 @@ mod tests {
ToolChoice, ToolDefinition, ToolResultContentBlock,
};
use serde_json::json;
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::sync::{Mutex, OnceLock};
@@ -2147,6 +2196,28 @@ mod tests {
));
}
#[test]
fn local_openai_base_url_does_not_require_api_key() {
let _lock = env_lock();
let original_base_url = std::env::var_os("OPENAI_BASE_URL");
let original_api_key = std::env::var_os("OPENAI_API_KEY");
std::env::set_var("OPENAI_BASE_URL", "http://127.0.0.1:11434/v1");
std::env::remove_var("OPENAI_API_KEY");
let client = OpenAiCompatClient::from_env(OpenAiCompatConfig::openai())
.expect("local OpenAI-compatible endpoint should not require an API key");
assert_eq!(client.base_url(), "http://127.0.0.1:11434/v1");
match original_base_url {
Some(value) => std::env::set_var("OPENAI_BASE_URL", value),
None => std::env::remove_var("OPENAI_BASE_URL"),
}
match original_api_key {
Some(value) => std::env::set_var("OPENAI_API_KEY", value),
None => std::env::remove_var("OPENAI_API_KEY"),
}
}
#[test]
fn endpoint_builder_accepts_base_urls_and_full_endpoints() {
assert_eq!(
@@ -2762,6 +2833,66 @@ mod tests {
}
}
#[test]
fn wire_model_strips_openai_prefix_for_default_and_local_preserves_custom_gateways() {
assert_eq!(
super::wire_model_for_base_url(
"openai/gpt-4o",
OpenAiCompatConfig::openai(),
super::DEFAULT_OPENAI_BASE_URL,
),
Cow::Borrowed("gpt-4o")
);
assert_eq!(
super::wire_model_for_base_url(
"openai/qwen2.5-coder:7b",
OpenAiCompatConfig::openai(),
"http://127.0.0.1:11434/v1",
),
Cow::Borrowed("qwen2.5-coder:7b")
);
assert_eq!(
super::wire_model_for_base_url(
"openai/llama3.2",
OpenAiCompatConfig::openai(),
"http://localhost:11434/v1/chat/completions",
),
Cow::Borrowed("llama3.2")
);
assert_eq!(
super::wire_model_for_base_url(
"openai/gpt-4.1-mini",
OpenAiCompatConfig::openai(),
"https://openrouter.ai/api/v1",
),
Cow::Borrowed("openai/gpt-4.1-mini")
);
assert_eq!(
super::wire_model_for_base_url(
"openai/gpt-4.1-mini",
OpenAiCompatConfig::openai(),
"https://not-localhost.example.com/v1",
),
Cow::Borrowed("openai/gpt-4.1-mini")
);
}
#[test]
fn local_routing_prefix_strips_only_escape_hatch() {
assert_eq!(
super::strip_routing_prefix("local/Qwen/Qwen3.6-27B-FP8"),
"Qwen/Qwen3.6-27B-FP8"
);
assert_eq!(
super::wire_model_for_base_url(
"local/Qwen/Qwen3.6-27B-FP8",
OpenAiCompatConfig::openai(),
"http://127.0.0.1:8000/v1",
),
Cow::Borrowed("Qwen/Qwen3.6-27B-FP8")
);
}
#[test]
fn check_request_body_size_allows_large_requests_for_openai() {
// Create a request that exceeds DashScope's limit but is under OpenAI's 100MB limit