From edf423940b29b17f1b295a524b40045963800a13 Mon Sep 17 00:00:00 2001 From: Yuanhao Li Date: Sun, 23 Aug 2026 23:28:27 +0200 Subject: [PATCH] measure: sweep the compaction headroom policy (#150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The measurement #150 asks for, and it contradicts what I wrote there. Two of the four things that matter are pure functions of the compaction code — how often a session compacts, and how much survives each time — so they sweep densely for free and deterministically, no provider in the loop. This drives the real effective_target_ratio and compact_messages rather than reimplementing them. Result at the crate-default 96K budget over 120 turns: tool-heavy (~3K/turn) Some(30): 4 compactions, 50132 mean held Some(5): 9 compactions, 71575 mean held very heavy (~8K/turn) Some(30): 10 compactions, 47764 mean held Some(5): 19 compactions, 66127 mean held Some(30) takes half to a third as many compactions — i.e. prefix-cache invalidations — for about 30% less mean retention. That is the aggressive-but-rare trade working, not a bug. I had called it "the aggression that makes compaction destructive"; the data does not support that framing. What the data does flag is the post-compaction cliff: immediately after a compaction the context drops to 8-12K of a 96K budget, against 48-66K under Some(5). Mean retention recovers, so the cliff is transient, but an agent asked something right after a compaction has very little to work with. Below ~740 tok/turn at a 26K budget (~2.7K at 96K) every policy behaves identically, so this only affects tool-heavy sessions. long_horizon takes YO_HEADROOM so the live half — does the lower compaction count actually buy cache hits — can be run per policy. That run hung twice and is not included; the offline half stands on its own. Co-Authored-By: Claude Opus 5 (1M context) --- examples/headroom_sweep.rs | 190 +++++++++++++++++++++++++++++++++++++ examples/long_horizon.rs | 9 ++ 2 files changed, 199 insertions(+) create mode 100644 examples/headroom_sweep.rs diff --git a/examples/headroom_sweep.rs b/examples/headroom_sweep.rs new file mode 100644 index 0000000..3b4855d --- /dev/null +++ b/examples/headroom_sweep.rs @@ -0,0 +1,190 @@ +//! Offline sweep of the compaction headroom policy — the measurement +//! [#150](https://github.com/yologdev/yoagent/issues/150) asks for. +//! +//! Run with: `cargo run --example headroom_sweep` +//! +//! # Why offline +//! +//! Two of the four things that matter are **pure functions of the compaction +//! code** — how often a session compacts, and how much history survives each +//! time — so they can be swept densely for free and deterministically, without +//! a provider in the loop. Only prefix-cache hit rate and briefing splice rate +//! need live runs, and those are worth spending on a shortlist rather than a +//! grid. +//! +//! This drives the real `effective_target_ratio` and `compact_messages`, not a +//! reimplementation of them. +//! +//! # The question +//! +//! `compact_headroom_turns` defaults to `Some(30)`: "leave room for 30 more +//! turns". `effective_target_ratio` computes `budget - turns * growth`, floored +//! at `MIN_HEADROOM_RATIO` (0.15). For any session growing faster than +//! `budget/30` per turn that demand exceeds the budget outright and pins the +//! ratio to the floor. +//! +//! Whether that is *wrong* is the open question. Compacting to 15% means +//! compacting rarely but destructively; compacting to 70% means often but +//! gently. Frequent compaction is the expensive one for prefix caching, since +//! each one invalidates the cached prefix past the head — so "aggressive but +//! rare" may well be the right trade, and the floor defensible. +//! +//! What this measures: for a realistic range of growth rates, how many +//! compactions a session takes and how much history it holds, under each +//! candidate policy. + +use yoagent::context::{compact_messages, total_tokens, ContextConfig, MIN_HEADROOM_RATIO}; +use yoagent::types::{AgentMessage, Content, Message}; + +/// One turn of a tool-using agent: an assistant reply plus a bulky tool result. +fn turn(i: usize, bulk_chars: usize) -> Vec { + vec![ + AgentMessage::Llm(Message::User { + content: vec![Content::Text { + text: format!("step {i}"), + }], + timestamp: i as u64, + }), + AgentMessage::Llm(Message::User { + content: vec![Content::Text { + text: format!("result {i}: {}", "x".repeat(bulk_chars)), + }], + timestamp: i as u64, + }), + ] +} + +struct Outcome { + compactions: usize, + /// Mean tokens held across the session — the retention a model actually + /// works with, not the peak. + mean_held: usize, + /// Tokens held right after each compaction, worst case. + min_after_compaction: usize, + ratio_used: f32, +} + +/// Drive `turns` turns at a fixed growth rate under one policy, compacting +/// exactly where the agent loop would. +fn simulate( + budget: usize, + headroom_turns: Option, + bulk_chars: usize, + turns: usize, +) -> Outcome { + let config = ContextConfig { + max_context_tokens: budget, + system_prompt_tokens: 0, + keep_recent: 6, + keep_first: 2, + compact_headroom_turns: headroom_turns, + ..Default::default() + }; + + let mut messages: Vec = Vec::new(); + let mut compactions = 0usize; + let mut held_samples: Vec = Vec::new(); + let mut min_after = usize::MAX; + let mut ratio_used = config.compact_target_ratio; + + // Growth measured the way the loop measures it: the mean delta per turn. + let mut last = 0usize; + let mut growth_total = 0f64; + let mut growth_samples = 0f64; + + for i in 0..turns { + messages.extend(turn(i, bulk_chars)); + let now = total_tokens(&messages); + if i > 0 { + growth_total += now.saturating_sub(last) as f64; + growth_samples += 1.0; + } + last = now; + let growth = if growth_samples > 0.0 { + growth_total / growth_samples + } else { + 0.0 + }; + + // The loop resolves the headroom policy against the budget, then hands + // the adapted config to the strategy. + let ratio = config.effective_target_ratio(growth); + ratio_used = ratio; + let effective = ContextConfig { + compact_target_ratio: ratio, + ..config.clone() + }; + + if total_tokens(&messages) > budget { + messages = compact_messages(messages, &effective); + compactions += 1; + let after = total_tokens(&messages); + min_after = min_after.min(after); + last = after; + } + held_samples.push(total_tokens(&messages)); + } + + Outcome { + compactions, + mean_held: held_samples.iter().sum::() / held_samples.len().max(1), + min_after_compaction: if min_after == usize::MAX { + 0 + } else { + min_after + }, + ratio_used, + } +} + +fn main() { + const BUDGET: usize = 96_000; // the crate default: 100K minus the 4K reserve + const TURNS: usize = 120; + + println!("\nHeadroom policy sweep — #150"); + println!( + "budget {BUDGET} tokens, {TURNS} turns, keep_first=2 keep_recent=6, \ + MIN_HEADROOM_RATIO={MIN_HEADROOM_RATIO}\n" + ); + println!( + "Growth is per-turn token growth. `ratio` is what effective_target_ratio resolved to." + ); + println!("`mean held` is the context a model actually works with; higher is better retention."); + println!("`compactions` is how often the prefix cache was invalidated; lower is better.\n"); + + // Growth rates spanning light chat through bulky tool output. The crate + // default pins the floor above ~3.2K/turn at this budget. + let growths = [ + ("light ", 800usize), + ("moderate ", 4_000), + ("tool-heavy ", 12_000), + ("very heavy ", 32_000), + ]; + let policies = [ + ("Some(30) [current]", Some(30usize)), + ("Some(10)", Some(10)), + ("Some(5)", Some(5)), + ("Some(3)", Some(3)), + ("None [flat 0.7]", None), + ]; + + for (label, bulk) in growths { + println!("--- growth: {label} (~{} tok/turn) ---", bulk / 4); + println!( + " {:<20} {:>6} {:>12} {:>12} {:>7}", + "policy", "compac", "mean held", "min after", "ratio" + ); + for (name, turns_cfg) in policies { + let o = simulate(BUDGET, turns_cfg, bulk, TURNS); + println!( + " {:<20} {:>6} {:>12} {:>12} {:>7.2}", + name, o.compactions, o.mean_held, o.min_after_compaction, o.ratio_used + ); + } + println!(); + } + + println!("Reading this: a policy is better when it holds more context (mean held) for"); + println!("fewer cache invalidations (compactions). Those pull against each other, so the"); + println!("question is whether Some(30) sits at a defensible point or an extreme one."); +} diff --git a/examples/long_horizon.rs b/examples/long_horizon.rs index 485aeee..5c94291 100644 --- a/examples/long_horizon.rs +++ b/examples/long_horizon.rs @@ -309,6 +309,15 @@ async fn main() { keep_recent: 6, keep_first: 2, tool_output_max_lines: 200, + // #150's open question: does the headroom policy's lower + // compaction count actually buy prefix-cache hits? The offline + // sweep (`examples/headroom_sweep.rs`) shows Some(30) taking half + // to a third as many compactions as Some(5) for ~30% less + // retention; only a live run can price the cache side. + compact_headroom_turns: std::env::var("YO_HEADROOM") + .ok() + .map(|v| if v == "none" { None } else { v.parse().ok() }) + .unwrap_or(Some(30)), ..Default::default() });