From 3d62778c33bb1c047a9c06b7dbcc6e280c3e8a8f Mon Sep 17 00:00:00 2001 From: Bowen Fu <5812640+BowenFu@users.noreply.github.com> Date: Fri, 14 Aug 2026 04:14:54 +0000 Subject: [PATCH 1/3] [https://nvbugs/6607481][fix] Isolate stateful KV-cache comparison Signed-off-by: Bowen Fu <5812640+BowenFu@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - tests/test_common/session_reuse_hooks.py | 4 ++++ tests/unittest/llmapi/test_session_reuse.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index c709cc7d3984..6224a1f3338e 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -292,7 +292,6 @@ full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::Tes full:RTX_PRO_6000_Blackwell_Server_Edition/accuracy/test_llm_api_pytorch.py::TestQwen3_5_4B::test_fp8 SKIP (https://nvbugs/6273850) full:sm100/unittest/bindings SKIP (Disable for Blackwell) kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill SKIP (https://nvbugs/6428002) -kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_chunked_prefill_eviction_block_reuse SKIP (https://nvbugs/6607481) kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction[cuda_graph] SKIP (https://nvbugs/6600098) kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_with_block_reuse SKIP (https://nvbugs/6462303) kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_overlap_scheduler[non_overlap] SKIP (https://nvbugs/6600098) diff --git a/tests/test_common/session_reuse_hooks.py b/tests/test_common/session_reuse_hooks.py index 7bd4899250a7..208767c5105f 100644 --- a/tests/test_common/session_reuse_hooks.py +++ b/tests/test_common/session_reuse_hooks.py @@ -22,6 +22,10 @@ "autodeploy", "auto_deploy", "/test_ad_", + # This V1/V2 comparison has hung after inheriting an older reused worker. + # Keep both executor legs isolated until their teardown resets all worker state. + "kv_cache/test_kv_cache_v2_scheduler.py::testkvcachev2llama::" + "test_chunked_prefill_eviction_block_reuse", ) diff --git a/tests/unittest/llmapi/test_session_reuse.py b/tests/unittest/llmapi/test_session_reuse.py index ef693c291c10..870a6a2482ba 100644 --- a/tests/unittest/llmapi/test_session_reuse.py +++ b/tests/unittest/llmapi/test_session_reuse.py @@ -331,6 +331,18 @@ def test_autodeploy_nodeids_are_private(): ) +def test_stateful_kv_cache_compare_nodeid_is_private(): + from test_common.session_reuse_hooks import _is_private_nodeid + + assert _is_private_nodeid( + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::" + "test_chunked_prefill_eviction_block_reuse" + ) + assert not _is_private_nodeid( + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_with_block_reuse" + ) + + def test_failed_item_fences_cached_pools(reuse_cache, monkeypatch): """After a failed item the next acquire must NOT reuse the cached pool.""" from test_common import session_reuse_hooks as hooks From 87ccd3fbbd0086b74c49adb1ce8530ba76d67f12 Mon Sep 17 00:00:00 2001 From: Bowen Fu <5812640+BowenFu@users.noreply.github.com> Date: Fri, 14 Aug 2026 12:10:58 +0000 Subject: [PATCH 2/3] [https://nvbugs/6607481][test] Annotate stateful cache regression Signed-off-by: Bowen Fu <5812640+BowenFu@users.noreply.github.com> --- tests/test_common/session_reuse_hooks.py | 4 ++-- tests/unittest/llmapi/test_session_reuse.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_common/session_reuse_hooks.py b/tests/test_common/session_reuse_hooks.py index 208767c5105f..748588d072b1 100644 --- a/tests/test_common/session_reuse_hooks.py +++ b/tests/test_common/session_reuse_hooks.py @@ -24,14 +24,14 @@ "/test_ad_", # This V1/V2 comparison has hung after inheriting an older reused worker. # Keep both executor legs isolated until their teardown resets all worker state. - "kv_cache/test_kv_cache_v2_scheduler.py::testkvcachev2llama::" + "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::" "test_chunked_prefill_eviction_block_reuse", ) def _is_private_nodeid(nodeid: str) -> bool: lowered = nodeid.lower() - return any(pat in lowered for pat in _PRIVATE_NODEID_PATTERNS) + return any(pat.lower() in lowered for pat in _PRIVATE_NODEID_PATTERNS) def pytest_configure(config): diff --git a/tests/unittest/llmapi/test_session_reuse.py b/tests/unittest/llmapi/test_session_reuse.py index 870a6a2482ba..8c8b743a0be6 100644 --- a/tests/unittest/llmapi/test_session_reuse.py +++ b/tests/unittest/llmapi/test_session_reuse.py @@ -331,7 +331,7 @@ def test_autodeploy_nodeids_are_private(): ) -def test_stateful_kv_cache_compare_nodeid_is_private(): +def test_stateful_kv_cache_compare_nodeid_is_private() -> None: from test_common.session_reuse_hooks import _is_private_nodeid assert _is_private_nodeid( From b76919570d87fc852d770e7cbea4aa728fb5708f Mon Sep 17 00:00:00 2001 From: Bowen Fu <5812640+BowenFu@users.noreply.github.com> Date: Mon, 17 Aug 2026 09:09:53 +0000 Subject: [PATCH 3/3] [https://nvbugs/6607481][test] Use private MPI marker for cache test Signed-off-by: Bowen Fu <5812640+BowenFu@users.noreply.github.com> --- .../defs/kv_cache/test_kv_cache_v2_scheduler.py | 1 + tests/test_common/session_reuse_hooks.py | 4 ---- tests/unittest/llmapi/test_session_reuse.py | 12 ------------ 3 files changed, 1 insertion(+), 16 deletions(-) diff --git a/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py b/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py index 74a03c3727b7..d37e073b1bae 100644 --- a/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py +++ b/tests/integration/defs/kv_cache/test_kv_cache_v2_scheduler.py @@ -328,6 +328,7 @@ def test_eviction_with_block_reuse(self): ) # Chunked prefill + eviction + block reuse — V2 matches V1 + @pytest.mark.private_mpi_session def test_chunked_prefill_eviction_block_reuse(self): _run_eviction_test( self.MODEL_PATH, diff --git a/tests/test_common/session_reuse_hooks.py b/tests/test_common/session_reuse_hooks.py index 748588d072b1..d92e232dabd2 100644 --- a/tests/test_common/session_reuse_hooks.py +++ b/tests/test_common/session_reuse_hooks.py @@ -22,10 +22,6 @@ "autodeploy", "auto_deploy", "/test_ad_", - # This V1/V2 comparison has hung after inheriting an older reused worker. - # Keep both executor legs isolated until their teardown resets all worker state. - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::" - "test_chunked_prefill_eviction_block_reuse", ) diff --git a/tests/unittest/llmapi/test_session_reuse.py b/tests/unittest/llmapi/test_session_reuse.py index 8c8b743a0be6..ef693c291c10 100644 --- a/tests/unittest/llmapi/test_session_reuse.py +++ b/tests/unittest/llmapi/test_session_reuse.py @@ -331,18 +331,6 @@ def test_autodeploy_nodeids_are_private(): ) -def test_stateful_kv_cache_compare_nodeid_is_private() -> None: - from test_common.session_reuse_hooks import _is_private_nodeid - - assert _is_private_nodeid( - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::" - "test_chunked_prefill_eviction_block_reuse" - ) - assert not _is_private_nodeid( - "kv_cache/test_kv_cache_v2_scheduler.py::TestKVCacheV2Llama::test_eviction_with_block_reuse" - ) - - def test_failed_item_fences_cached_pools(reuse_cache, monkeypatch): """After a failed item the next acquire must NOT reuse the cached pool.""" from test_common import session_reuse_hooks as hooks