From b56cbbef60ae6eb24666a1e21964729845ce717e Mon Sep 17 00:00:00 2001 From: Eric Tsai Date: Tue, 1 Sep 2026 03:21:19 -0700 Subject: [PATCH 1/4] [https://nvbugs/6701493][fix] Limit Gemma3 FP8 accuracy test KV cache memory fraction Signed-off-by: Eric Tsai --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 71c351ecf36b..5bc87d0b4052 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1402,7 +1402,8 @@ def test_fp8_prequantized(self): # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. kv_cache_config = KvCacheConfig(enable_block_reuse=False, enable_partial_reuse=False, - dtype="fp8") + dtype="fp8", + free_gpu_memory_fraction=0.5) # Note: This has only the LLM part quantized. Vision part is in bfloat16. prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-27b-it-fp8/" with LLM(prequantized_model_path, From 53dcb7e3c97b27f4962b48c12c09429e50f26f16 Mon Sep 17 00:00:00 2001 From: erictsai-nv Date: Wed, 2 Sep 2026 09:59:50 +0800 Subject: [PATCH 2/4] Update tests/integration/defs/accuracy/test_llm_api_pytorch.py Co-authored-by: ruodil <200874449+ruodil@users.noreply.github.com> Signed-off-by: erictsai-nv --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 5bc87d0b4052..fb2868a5abaf 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1403,6 +1403,9 @@ def test_fp8_prequantized(self): kv_cache_config = KvCacheConfig(enable_block_reuse=False, enable_partial_reuse=False, dtype="fp8", + dtype="fp8", + # Cap KV cache at 50% of free memory to leave PyTorch + # activation headroom during MMLU (nvbugs/6701493). free_gpu_memory_fraction=0.5) # Note: This has only the LLM part quantized. Vision part is in bfloat16. prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-27b-it-fp8/" From 87f5d1039e36acefc63ccb99c7db1e5682dd936a Mon Sep 17 00:00:00 2001 From: erictsai-nv Date: Wed, 2 Sep 2026 10:01:46 +0800 Subject: [PATCH 3/4] Fix duplicate dtype assignment in kv_cache_config Removed duplicate dtype assignment in kv_cache_config. Signed-off-by: erictsai-nv --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index fb2868a5abaf..360a56ac27ff 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1403,7 +1403,6 @@ def test_fp8_prequantized(self): kv_cache_config = KvCacheConfig(enable_block_reuse=False, enable_partial_reuse=False, dtype="fp8", - dtype="fp8", # Cap KV cache at 50% of free memory to leave PyTorch # activation headroom during MMLU (nvbugs/6701493). free_gpu_memory_fraction=0.5) From 2b47250fa4b49a2cc0fc8d59bc47ce8eff641d6d Mon Sep 17 00:00:00 2001 From: Eric Tsai Date: Tue, 1 Sep 2026 19:09:00 -0700 Subject: [PATCH 4/4] Fix Gemma3 FP8 test formatting Signed-off-by: Eric Tsai --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 360a56ac27ff..cefe07578497 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -1400,11 +1400,11 @@ def test_auto_dtype(self): def test_fp8_prequantized(self): # Disabling kv cache reuse as a WAR to deal with gaps in kernel support for Gemma3's non-inclusive sliding window size. + # Cap KV cache at 50% of free memory to leave PyTorch + # activation headroom during MMLU (nvbugs/6701493). kv_cache_config = KvCacheConfig(enable_block_reuse=False, enable_partial_reuse=False, dtype="fp8", - # Cap KV cache at 50% of free memory to leave PyTorch - # activation headroom during MMLU (nvbugs/6701493). free_gpu_memory_fraction=0.5) # Note: This has only the LLM part quantized. Vision part is in bfloat16. prequantized_model_path = f"{llm_models_root()}/gemma/gemma-3-27b-it-fp8/"