mirror of
https://github.com/instructkr/claw-code.git
synced 2026-10-19 00:00:08 +08:00
fix(runtime): auto-compaction must measure the live context, not a running total
`maybe_auto_compact` gated on `usage_tracker.cumulative_usage().input_tokens`, which is wrong in two independent ways: 1. It ignores cached input. With prompt caching a warm call reports `input_tokens: 2` and puts the real context in `cache_read_input_tokens`, so the counter barely moved and auto-compaction never fired at all, however large the conversation grew. Backends without prompt caching report the whole prompt as `input_tokens`, so the bug was invisible there. 2. A sum across calls is not a context measurement, and it never decreases. Once it did cross the threshold, every subsequent iteration compacted again, permanently, even after compaction had shrunk the context. Gate on the latest call's total input instead — input + cache_creation + cache_read — which is the live context window size. It rises with the real context and falls back below the threshold once compaction has done its job, so no counter reset is needed. `CLAUDE_CODE_AUTO_COMPACT_INPUT_TOKENS` now means what it reads like: compact when the live context reaches N tokens. The default of 100_000 is unchanged and still lands at 50% of a 200k window. Adds `TokenUsage::total_input_tokens()` (saturating) and four tests: the cached-usage shape that never fired, non-latching across three turns, and two unit tests for the new helper. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013bVvRYn7CQKR6hT2j1L6Cs (cherry picked from commit aebba4e77eddf585fc5b21123967dbc4ebb6e6a1)
This commit is contained in:
1 parent
7a1391fbc1
commit
a25717aded
2 files changed
+187
-1
No files matched your search
@@ -574,7 +574,18 @@ where
|
||||
}
|
||||
|
||||
fn maybe_auto_compact(&mut self) -> Option<AutoCompactionEvent> {
|
||||
if self.usage_tracker.cumulative_usage().input_tokens
|
||||
// Gate on the LIVE context size — the most recent call's input counters — not on a
|
||||
// cumulative running total, and count cached input as the context it is.
|
||||
//
|
||||
// The previous gate (`cumulative_usage().input_tokens`) was wrong twice over:
|
||||
// 1. It ignored `cache_read_input_tokens`. With prompt caching a warm call reports
|
||||
// `input_tokens: 2`, so the counter crawled and compaction never fired however
|
||||
// large the context grew.
|
||||
// 2. Summing across calls is not a context measurement, and the sum never decreased,
|
||||
// so once it did trip, every later iteration compacted forever.
|
||||
// Reading the latest turn fixes both: it rises with the real context and falls back
|
||||
// below the threshold as soon as a compaction has done its job.
|
||||
if self.usage_tracker.current_turn_usage().total_input_tokens()
|
||||
< self.auto_compaction_input_tokens_threshold
|
||||
{
|
||||
return None;
|
||||
@@ -1701,6 +1712,133 @@ mod tests {
|
||||
assert_eq!(runtime.session().messages.len(), 2);
|
||||
}
|
||||
|
||||
/// Prompt caching reports the bulk of the context as `cache_read_input_tokens` and leaves
|
||||
/// `input_tokens` at a token or two, so a gate that reads `input_tokens` alone never fires
|
||||
/// on a cached session no matter how large the context has grown.
|
||||
#[test]
|
||||
fn auto_compaction_counts_cached_context_toward_threshold() {
|
||||
struct CachedApi;
|
||||
impl ApiClient for CachedApi {
|
||||
fn stream(
|
||||
&mut self,
|
||||
_request: ApiRequest,
|
||||
) -> Result<Vec<AssistantEvent>, RuntimeError> {
|
||||
Ok(vec![
|
||||
AssistantEvent::TextDelta("done".to_string()),
|
||||
AssistantEvent::Usage(TokenUsage {
|
||||
input_tokens: 2,
|
||||
output_tokens: 4,
|
||||
cache_creation_input_tokens: 1_804,
|
||||
cache_read_input_tokens: 112_639,
|
||||
}),
|
||||
AssistantEvent::MessageStop,
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
let mut session = Session::new();
|
||||
session.messages = vec![
|
||||
crate::session::ConversationMessage::user_text("one"),
|
||||
crate::session::ConversationMessage::assistant(vec![ContentBlock::Text {
|
||||
text: "two".to_string(),
|
||||
}]),
|
||||
crate::session::ConversationMessage::user_text("three"),
|
||||
crate::session::ConversationMessage::assistant(vec![ContentBlock::Text {
|
||||
text: "four".to_string(),
|
||||
}]),
|
||||
];
|
||||
|
||||
let mut runtime = ConversationRuntime::new(
|
||||
session,
|
||||
CachedApi,
|
||||
StaticToolExecutor::new(),
|
||||
PermissionPolicy::new(PermissionMode::DangerFullAccess),
|
||||
vec!["system".to_string()],
|
||||
)
|
||||
.with_auto_compaction_input_tokens_threshold(100_000);
|
||||
|
||||
let summary = runtime
|
||||
.run_turn("trigger", None)
|
||||
.expect("turn should succeed");
|
||||
|
||||
assert_eq!(
|
||||
summary.auto_compaction,
|
||||
Some(AutoCompactionEvent {
|
||||
removed_message_count: 2,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/// The threshold describes the LIVE context, not a monotonic running total. Once compaction
|
||||
/// has shrunk the context back down, later small turns must not keep compacting.
|
||||
#[test]
|
||||
fn auto_compaction_stops_once_the_context_shrinks_again() {
|
||||
struct ShrinkingApi {
|
||||
calls: u32,
|
||||
}
|
||||
impl ApiClient for ShrinkingApi {
|
||||
fn stream(
|
||||
&mut self,
|
||||
_request: ApiRequest,
|
||||
) -> Result<Vec<AssistantEvent>, RuntimeError> {
|
||||
let call = self.calls;
|
||||
self.calls += 1;
|
||||
// The first call is over budget; every call after it is small.
|
||||
let input_tokens = if call == 0 { 150_000 } else { 900 };
|
||||
Ok(vec![
|
||||
AssistantEvent::TextDelta("done".to_string()),
|
||||
AssistantEvent::Usage(TokenUsage {
|
||||
input_tokens,
|
||||
output_tokens: 4,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
}),
|
||||
AssistantEvent::MessageStop,
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
let mut session = Session::new();
|
||||
session.messages = vec![
|
||||
crate::session::ConversationMessage::user_text("one"),
|
||||
crate::session::ConversationMessage::assistant(vec![ContentBlock::Text {
|
||||
text: "two".to_string(),
|
||||
}]),
|
||||
crate::session::ConversationMessage::user_text("three"),
|
||||
crate::session::ConversationMessage::assistant(vec![ContentBlock::Text {
|
||||
text: "four".to_string(),
|
||||
}]),
|
||||
];
|
||||
|
||||
let mut runtime = ConversationRuntime::new(
|
||||
session,
|
||||
ShrinkingApi { calls: 0 },
|
||||
// Every turn after a compaction runs the session health probe, which calls
|
||||
// `glob_search`; without it the probe fails the turn before the assertion is reached.
|
||||
StaticToolExecutor::new().register("glob_search", |_| Ok(String::new())),
|
||||
PermissionPolicy::new(PermissionMode::DangerFullAccess),
|
||||
vec!["system".to_string()],
|
||||
)
|
||||
.with_auto_compaction_input_tokens_threshold(100_000);
|
||||
|
||||
let first = runtime.run_turn("trigger", None).expect("first turn");
|
||||
assert!(
|
||||
first.auto_compaction.is_some(),
|
||||
"the over-budget turn should compact"
|
||||
);
|
||||
|
||||
let second = runtime.run_turn("again", None).expect("second turn");
|
||||
assert_eq!(
|
||||
second.auto_compaction, None,
|
||||
"a small turn after compaction must not compact again"
|
||||
);
|
||||
let third = runtime.run_turn("and again", None).expect("third turn");
|
||||
assert_eq!(
|
||||
third.auto_compaction, None,
|
||||
"compaction must not latch on once the threshold has been crossed"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auto_compaction_threshold_defaults_and_parses_values() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -89,6 +89,19 @@ impl TokenUsage {
|
||||
+ self.cache_read_input_tokens
|
||||
}
|
||||
|
||||
/// Total tokens the provider had to read to serve this call, i.e. the live size of the
|
||||
/// context window.
|
||||
///
|
||||
/// `input_tokens` alone is NOT that number. With prompt caching a warm call reports
|
||||
/// `input_tokens: 2` and puts the real bulk in `cache_read_input_tokens`, so anything
|
||||
/// budgeting against the context window must sum all three input counters.
|
||||
#[must_use]
|
||||
pub fn total_input_tokens(self) -> u32 {
|
||||
self.input_tokens
|
||||
.saturating_add(self.cache_creation_input_tokens)
|
||||
.saturating_add(self.cache_read_input_tokens)
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn estimate_cost_usd(self) -> UsageCostEstimate {
|
||||
self.estimate_cost_usd_with_pricing(ModelPricing::default_sonnet_tier())
|
||||
@@ -243,6 +256,41 @@ mod tests {
|
||||
assert_eq!(tracker.cumulative_usage().total_tokens(), 48);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn total_input_tokens_includes_cached_context() {
|
||||
// The shape a warm prompt-cached call actually reports: input_tokens is ~nothing and
|
||||
// the real context size is in the cache counters.
|
||||
let cached = TokenUsage {
|
||||
input_tokens: 2,
|
||||
output_tokens: 1_133,
|
||||
cache_creation_input_tokens: 1_453,
|
||||
cache_read_input_tokens: 111_186,
|
||||
};
|
||||
assert_eq!(cached.total_input_tokens(), 112_641);
|
||||
// Output tokens are not part of the input budget.
|
||||
assert_ne!(cached.total_input_tokens(), cached.total_tokens());
|
||||
|
||||
// A backend without prompt caching reports the whole prompt as input_tokens.
|
||||
let uncached = TokenUsage {
|
||||
input_tokens: 29_056,
|
||||
output_tokens: 73,
|
||||
cache_creation_input_tokens: 0,
|
||||
cache_read_input_tokens: 0,
|
||||
};
|
||||
assert_eq!(uncached.total_input_tokens(), 29_056);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn total_input_tokens_saturates_instead_of_overflowing() {
|
||||
let usage = TokenUsage {
|
||||
input_tokens: u32::MAX,
|
||||
output_tokens: 0,
|
||||
cache_creation_input_tokens: u32::MAX,
|
||||
cache_read_input_tokens: u32::MAX,
|
||||
};
|
||||
assert_eq!(usage.total_input_tokens(), u32::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn computes_cost_summary_lines() {
|
||||
let usage = TokenUsage {
|
||||
|
||||
Reference in new issue
Block a user