From 08fe068b273e4ad3fd527dc4c7f7e2e7b202ec03 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Fri, 7 Aug 2026 17:29:22 -0500 Subject: [PATCH 1/3] [None][infra] Log infra-retry classify declines instead of silent rethrow When a stage or pod/agent launch failure does not match any pattern in FailureClassifier's catalog, runKubernetesPodWithInfraRetry and the SLURM retry loop classify it as a user failure and rethrow with no log output. A misclassified infra event (for example a pod-scheduling timeout whose exception type is missing from the catalog) then leaves no trace: no [INFRA-RETRY] line to grep for, and no way to tell a deliberate retry-decline from the retry wrapper never running. Add one echo before each of the three silent rethrow sites (K8s launch loop, K8s execution loop, SLURM loop) stating the stage, the decline reason, and the exception text. Signed-off-by: Brian Nguyen --- jenkins/L0_Test.groovy | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index a3a8a6910166..9e7f9822a7df 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2401,7 +2401,13 @@ def runLLMTestlistOnSlurm(pipeline, platform, testList, config=VANILLA_CONFIG, p // ensures we only match catalog rows tagged SLURM or BOTH. def c = FailureClassifier.classify(e, InfraFailure.SLURM) if (c instanceof PipelineInterruption) throw e - if (!(c instanceof InfraFailure)) throw e // UserFailure -> don't retry + if (!(c instanceof InfraFailure)) { + // UserFailure -> don't retry, but leave a trace: a failure whose + // exception matches no catalog pattern lands here and would + // otherwise decline the retry with no log output at all. + echo "[INFRA-RETRY] ${stageName}: SLURM attempt failed with no infra pattern matched (classified as user failure); not retrying. Exception: ${e.toString()}" + throw e + } rememberAvoidedSlurmNodeLists(avoidedSlurmNodeListsByCluster, attemptPlacementContext.lastSlurmClusterName, attemptPlacementContext.lastSlurmNodeList, stageName) @@ -5068,7 +5074,14 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN } def c = FailureClassifier.classify(e, InfraFailure.K8S) if (c instanceof PipelineInterruption) throw e - if (!(c instanceof InfraFailure)) throw e // UserFailure -> don't retry + if (!(c instanceof InfraFailure)) { + // UserFailure -> don't retry, but leave a trace: a pod/agent + // launch exception missing from PATTERN_CATALOG (e.g. a + // pod-scheduling timeout) lands here and would otherwise + // vanish with no log output at all. + echo "[INFRA-RETRY] ${stageName}: pod/agent launch failed before execution with no infra pattern matched (classified as user failure); not retrying. Exception: ${e.toString()}" + throw e + } rememberAvoidedKubernetesHostNodes(avoidedKubernetesHostNodes, attemptPlacementContext.lastHostNode, stageName) @@ -5153,7 +5166,12 @@ def runKubernetesPodWithInfraRetry(Map opts = [:], pipeline, podSpec, containerN // exhausted its own budget) from being treated as K8s infra here. def c = FailureClassifier.classify(e, InfraFailure.K8S) if (c instanceof PipelineInterruption) throw e - if (!(c instanceof InfraFailure)) throw e // UserFailure -> don't retry + if (!(c instanceof InfraFailure)) { + // UserFailure -> don't retry, but leave a trace so the decline + // is diagnosable from the console (see the launch-loop twin above). + echo "[INFRA-RETRY] ${stageName}: stage failed with no infra pattern matched (classified as user failure); not retrying. Exception: ${e.toString()}" + throw e + } rememberAvoidedKubernetesHostNodes(avoidedKubernetesHostNodes, attemptPlacementContext.lastHostNode, stageName) From 3b2ff5804a6a092e6d6993eeaa7dc8f3ecdf4f27 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sun, 9 Aug 2026 09:45:44 -0700 Subject: [PATCH 2/3] test waives: skip A100X-PyTorch-1/test_unittests.py::test_unittests_v2[unittest/llmapi/test_llm_pytorch.py -m "part3"] (pre-existing failure, tracking bug pending) Signed-off-by: Brian Nguyen --- tests/integration/test_lists/waives.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 6a003bf64f0a..81f7d50ff8db 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -429,6 +429,7 @@ unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp4[False-True unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_phi3_lora_fused_modules_output_on_tp2_identical_to_tp1 SKIP (https://nvbugs/6109745) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_2gpu[1-2] SKIP (https://nvbugs/6427411) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_tp2pp2 SKIP (https://nvbugs/6427411) +unittest/llmapi/test_llm_pytorch.py -m "part3" SKIP (bug pending, tracked in PR #17432) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[None] SKIP (https://nvbugs/6162504) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[cuda_graph_config0] SKIP (https://nvbugs/6162504) unittest/llmapi/test_llm_pytorch.py::test_llm_context_only_timed_out_kv_cache_exhausted[None-UCX-1000] SKIP (https://nvbugs/6490004) From e283397a4c27b451f8a1035eb29d0da636dc5694 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sun, 9 Aug 2026 21:57:55 -0500 Subject: [PATCH 3/3] test waives: revert the part3 llm_pytorch waive (unblock full run) Signed-off-by: Brian Nguyen --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 81f7d50ff8db..6a003bf64f0a 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -429,7 +429,6 @@ unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp4[False-True unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_phi3_lora_fused_modules_output_on_tp2_identical_to_tp1 SKIP (https://nvbugs/6109745) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_2gpu[1-2] SKIP (https://nvbugs/6427411) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_tinyllama_logits_processor_tp2pp2 SKIP (https://nvbugs/6427411) -unittest/llmapi/test_llm_pytorch.py -m "part3" SKIP (bug pending, tracked in PR #17432) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[None] SKIP (https://nvbugs/6162504) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[cuda_graph_config0] SKIP (https://nvbugs/6162504) unittest/llmapi/test_llm_pytorch.py::test_llm_context_only_timed_out_kv_cache_exhausted[None-UCX-1000] SKIP (https://nvbugs/6490004)