Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@
from ..conftest import llm_models_root

MODEL = f"{llm_models_root()}/llama-models-v2/TinyLlama-1.1B-Chat-v1.0"
# Pin the host tier for all scenarios so cold-pool field coverage is deterministic.
HOST_CACHE_SIZE = 64 << 20

ALL_FIELDS = [
# Instantaneous gauges — primary (GPU) pool
Expand Down Expand Up @@ -147,10 +149,15 @@ def find_kv_entries(stats_list):
# ---------------------------------------------------------------------------
@pytest.fixture(scope="module")
def llm_instance():
"""Create a shared LLM instance for all tests in this module."""
"""Share one V2 LLM with a fixed host tier across all eight scenarios."""
llm = LLM(
model=MODEL,
kv_cache_config=KvCacheConfig(enable_block_reuse=True, iteration_stats_interval=1),
kv_cache_config=KvCacheConfig(
use_kv_cache_manager_v2=True,
enable_block_reuse=True,
iteration_stats_interval=1,
host_cache_size=HOST_CACHE_SIZE,
Comment thread
yizhang-nv marked this conversation as resolved.
),
enable_iter_perf_stats=True,
return_perf_metrics=True,
)
Expand Down Expand Up @@ -401,6 +408,11 @@ def test_field_completeness(self, llm_instance, all_collected, request):
ki = s.get("kvCacheIterationStats")
if ki:
entries_with_kv += 1
assert s.get("kvCacheIterationStatsByPoolGroup"), "missing V2 hot-pool stats"
Comment thread
yizhang-nv marked this conversation as resolved.
# The explicit host tier ensures cold-field coverage is not vacuous.
assert s.get("kvCacheIterationStatsByColdPoolGroup"), "missing V2 cold-pool stats"
if entries_with_kv == 1:
print(f" V2 cold-pool stats: {s['kvCacheIterationStatsByColdPoolGroup']}")
# V2 reports secondary gauges by cold pool group, not by window.
for ws, v in ki.items():
missing_fields = NON_SECONDARY_FIELDS - v.keys()
Expand All @@ -415,6 +427,9 @@ def test_field_completeness(self, llm_instance, all_collected, request):
f"Missing kvCacheIterationStatsByColdPoolGroup fields for group {group}: "
f"{sorted(missing_fields)}"
)
assert v["secondaryMaxNumBlocks"] > 0, (
f"Cold group {group} reports no secondary capacity: {v}"
)

print(f" Entries with kvCacheIterationStats: {entries_with_kv}/{len(all_collected)}")
assert entries_with_kv > 0, "no entries contain kvCacheIterationStats"
Expand Down Expand Up @@ -481,7 +496,12 @@ class FakeRequest:
print("Starting LLM with block_reuse + iteration_stats_interval=1")
llm = LLM(
model=MODEL,
kv_cache_config=KvCacheConfig(enable_block_reuse=True, iteration_stats_interval=1),
kv_cache_config=KvCacheConfig(
use_kv_cache_manager_v2=True,
enable_block_reuse=True,
iteration_stats_interval=1,
host_cache_size=HOST_CACHE_SIZE,
),
enable_iter_perf_stats=True,
return_perf_metrics=True,
)
Expand Down
3 changes: 3 additions & 0 deletions tests/unittest/executor/test_stats_serializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,9 @@ def test_serializer_with_v2_pool_group_stats(self):
assert cold_group["coldPoolGroupId"] == 0
assert cold_group["slotSize"] == [4 << 20]
assert cold_group["windowSizes"] == [16, 64]
assert cold_group["secondaryMaxNumBlocks"] == 8
assert cold_group["secondaryFreeNumBlocks"] == 5
assert cold_group["secondaryUsedNumBlocks"] == 3
assert cold_group["secondaryPeakFreeNumBlocks"] == 6
assert cold_group["secondaryPeakUsedNumBlocks"] == 4
assert cold_group["secondaryPeakEvictableNumBlocks"] == 2
Expand Down
8 changes: 6 additions & 2 deletions tests/unittest/metrics/test_collector.py
Original file line number Diff line number Diff line change
Expand Up @@ -833,14 +833,18 @@ def test_v2_lifecycle_and_pool_group_stats_are_aggregated(self):
},
"kvCacheIterationStatsByPoolGroup": {
"0": {
"secondaryMaxNumBlocks": 50,
"secondaryUsedNumBlocks": 20,
"iterGenAllocBlocks": 2,
"iterOnboardBytes": 4096,
"iterOffloadBytes": 2048,
"iterIntraDeviceCopyBytes": 8192,
}
},
"kvCacheIterationStatsByColdPoolGroup": {
"0": {
"secondaryMaxNumBlocks": 50,
"secondaryUsedNumBlocks": 20,
}
},
}

before_reused = _get_counter_value(collector, "kv_cache_iter_reused_blocks")
Expand Down
Loading