diff --git a/lib/llm/src/kv_router/metrics.rs b/lib/llm/src/kv_router/metrics.rs index fdcaf168b114..55e55011be6f 100644 --- a/lib/llm/src/kv_router/metrics.rs +++ b/lib/llm/src/kv_router/metrics.rs @@ -825,6 +825,8 @@ pub struct RouterRequestMetrics { pub input_sequence_tokens: prometheus::Histogram, pub output_sequence_tokens: prometheus::Histogram, pub kv_hit_rate: prometheus::Histogram, + pub kv_overlap_blocks_total: prometheus::IntCounter, + pub kv_isl_blocks_total: prometheus::IntCounter, pub kv_transfer_estimated_latency_seconds: prometheus::Histogram, pub shared_cache_hit_rate: prometheus::Histogram, pub shared_cache_beyond_blocks: prometheus::Histogram, @@ -920,6 +922,20 @@ impl RouterRequestMetrics { Some(prometheus::linear_buckets(0.0, 0.05, 21).unwrap()), ) .expect("failed to create router_kv_hit_rate"); + let kv_overlap_blocks_total = metrics + .create_intcounter( + &router_metric(frontend_service::KV_OVERLAP_BLOCKS_TOTAL), + "Sum of predicted overlap blocks at routing time; divide by router_kv_isl_blocks_total for a block-weighted hit rate comparable to vllm:prefix_cache_hits/queries", + extra_labels, + ) + .expect("failed to create router_kv_overlap_blocks_total"); + let kv_isl_blocks_total = metrics + .create_intcounter( + &router_metric(frontend_service::KV_ISL_BLOCKS_TOTAL), + "Sum of request ISL blocks at routing time; denominator for router_kv_overlap_blocks_total", + extra_labels, + ) + .expect("failed to create router_kv_isl_blocks_total"); let kv_transfer_estimated_latency_seconds = metrics .create_histogram( &router_metric(frontend_service::KV_TRANSFER_ESTIMATED_LATENCY_SECONDS), @@ -951,6 +967,8 @@ impl RouterRequestMetrics { input_sequence_tokens, output_sequence_tokens, kv_hit_rate, + kv_overlap_blocks_total, + kv_isl_blocks_total, kv_transfer_estimated_latency_seconds, shared_cache_hit_rate, shared_cache_beyond_blocks, diff --git a/lib/llm/src/kv_router/push_router.rs b/lib/llm/src/kv_router/push_router.rs index 0311f46f1f77..bf71efc5d306 100644 --- a/lib/llm/src/kv_router/push_router.rs +++ b/lib/llm/src/kv_router/push_router.rs @@ -309,6 +309,18 @@ impl KvPushRouter { if let Some(hit_rate) = tracker.kv_hit_rate() { guard.request_metrics().kv_hit_rate.observe(hit_rate); } + // Block-weighted counterpart of `kv_hit_rate`. The histogram above averages + // per-request ratios, which is NOT comparable to the engine's + // `vllm:prefix_cache_hits/queries` (a ratio of sums). On skewed ISL + // distributions the two diverge ~2x with nothing broken. + guard + .request_metrics() + .kv_overlap_blocks_total + .inc_by(selection.effective_overlap_blocks.max(0.0) as u64); + guard + .request_metrics() + .kv_isl_blocks_total + .inc_by(isl_blocks as u64); } guard .request_metrics() diff --git a/lib/runtime/src/metrics/prometheus_names.rs b/lib/runtime/src/metrics/prometheus_names.rs index 42148314deb0..9fec3b03c398 100644 --- a/lib/runtime/src/metrics/prometheus_names.rs +++ b/lib/runtime/src/metrics/prometheus_names.rs @@ -203,6 +203,12 @@ pub mod frontend_service { /// Predicted KV cache hit rate at routing time (0.0-1.0) pub const KV_HIT_RATE: &str = "kv_hit_rate"; + /// Sum of predicted overlap blocks at routing time (block-weighted numerator) + pub const KV_OVERLAP_BLOCKS_TOTAL: &str = "kv_overlap_blocks_total"; + + /// Sum of request ISL blocks at routing time (block-weighted denominator) + pub const KV_ISL_BLOCKS_TOTAL: &str = "kv_isl_blocks_total"; + /// Upper-bound estimation of KV cache transfer latency in disaggregated serving (seconds) pub const KV_TRANSFER_ESTIMATED_LATENCY_SECONDS: &str = "kv_transfer_estimated_latency_seconds";