From 683cae488d5fe4cb6a2fcecd45f5636bd9775d0c Mon Sep 17 00:00:00 2001 From: Igor Shovkun Date: Tue, 1 Sep 2026 10:36:06 -0700 Subject: [PATCH 1/7] [https://nvbugs/6655359][test] Shrink the Cosmos3-Nano T2V LPIPS gate to 9 frames, relax T2V/V2V thresholds, and unwaive The B300-cut goldens from #17780 fail deterministically on the B200 CI lane (t2v 0.1507, v2v 0.0749 vs 0.05): Cosmos3-Nano trajectories are not bit-stable across GPU steppings, and the divergence grows with trajectory length (0.020 / 0.075 / 0.151 at 1 / 9 / 189 frames, everything else held fixed). The 'bit-stable across B200/B300' provenance clause was an over-generalization from Edge-model measurements; corrected everywhere it appears. Per the VisualGen CI strategy these gates are severe-regression backstops, so: shrink the T2V KPI config from the 189-frame 720P default to 9 frames (2,760 video tokens vs 44,160; the test drops from ~9 min to ~71 s) with a fresh golden cut on the gating lane's own GPU; gate t2v at 0.20 and v2v at 0.15, both above the measured 9-frame cross-stepping floor (0.075) so a future CI GPU change does not force a re-cut, and both still firing on the historical genuine-regression class (0.45-0.61). Member-wise zip repack, 44 sibling members byte-identical. Both waives removed; a temporary pre-merge listing (marked REMOVE BEFORE MERGE) proves the gates on the B200 lane before merge. Validated on B200 (umb-b200-236, native, current main): t2v 0.000000, v2v 0.069536, 2 passed in 71 s. B300 cross-check of the retired 189-frame golden: bit-exact regeneration, 0.000000. Signed-off-by: Igor Shovkun --- .../cosmos3_edge_i2v_lpips_golden_video.json | 2 +- .../cosmos3_edge_t2i_lpips_golden.json | 2 +- .../cosmos3_edge_t2v_lpips_golden_video.json | 2 +- ...smos3_nano_fp8_blockwise_lpips_golden.json | 2 +- .../cosmos3_nano_t2i_lpips_golden.json | 2 +- .../cosmos3_nano_t2v_lpips_golden_video.json | 15 ++++++------ .../cosmos3_nano_v2v_lpips_golden_frame.json | 6 ++--- .../visual_gen_lpips_golden_media.zip | 4 ++-- .../visual_gen/test_visual_gen_cosmos3.py | 23 ++++++++++++++++--- .../visual_gen/visual_gen_test_utils.py | 16 +++++++++---- .../test_lists/test-db/l0_b200.yml | 11 ++++++++- tests/integration/test_lists/waives.txt | 2 -- 12 files changed, 60 insertions(+), 27 deletions(-) diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json index 596e180410bb..485b732664fc 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json @@ -26,6 +26,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", "sha256": "1fd9b0ab24de130f593056a32a7b8555fafbbf073b76c29a4633504870b1dad0" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json index 92ad6e4ba1f3..44b6f7182137 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json @@ -25,6 +25,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", "sha256": "3f7c9b958807356ced2de1734e301dc837fa0b095f8fed1e29da764993926046" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json index 3b053b41ad68..d1a89dc3b63b 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json @@ -25,6 +25,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", "sha256": "0ec80b5c906ae576deedf8fb48c55edd0c78203138608f12d0c439da11ab6f10" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json index 55d4cf08a551..ab628be63bca 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json @@ -34,6 +34,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", "sha256": "32c080983eb8d94d1da5d21378ea87912dadf018f00f15123a33f000640341ad" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json index c09d9cc280a0..547533918f8c 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json @@ -24,6 +24,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", "sha256": "035f3e764e6a36159071178a2d7be6ec3cabc60899736099ff89a59c037e15e1" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json index ee6014644db1..d2bb0c16d493 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json @@ -6,24 +6,25 @@ "negative_prompt": "", "height": 720, "width": 1280, - "num_frames": 189, + "num_frames": 9, "num_inference_steps": 35, "guidance_scale": 6.0, "seed": 42, "frame_rate": 24.0, + "note": "KPI config shrunk from the 189-frame 720P default (nvbugs/6655359): 9 frames = 3 latent frames = 2,760 video tokens, the smallest T2V shape exercising multi-latent-frame temporal attention; the default costs ~7 min of generation per CI run and adds no gate value.", "attention_backend": "VANILLA", "torch_compile": false, "deterministic_algorithms": false, "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.05, + "lpips_threshold": 0.2, "measured_lpips_at_creation": 0.0, - "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", + "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): media is cut on the gating lane's own GPU (self-distance ~0), and the bar sits above the measured 9-frame cross-stepping floor (0.075, B300-cut media on B200) so a future CI GPU change does not force a re-cut; severe regressions (historical reds 0.45-0.61) still fire it.", "diffusers_version": "0.39.0", "torch_version": "2.12.0+cu130", - "tensorrt_llm_version": "1.3.0rc25", - "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", - "sha256": "980849ba2f1ff1101c0dce2ac8897172212a614f23c3fb0cc6acbd970dd42976" + "tensorrt_llm_version": "1.3.0rc26", + "tensorrt_llm_commit": "5fb68830c919b8dbcc07cb1095f5e7d369509919", + "environment": "Native build, no container; NVIDIA B200 (sm100), umb-b200-236 - the GPU of the CI lane that gates this test (l0_b200.yml). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359); the threshold absorbs that.", + "sha256": "e70deaaaae9e5bcd929906b3af77476ac2a529a1cb02db1026b98952e7590d44" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json index 636b8c3b484f..176e139c0b65 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json @@ -20,13 +20,13 @@ "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.05, + "lpips_threshold": 0.15, "measured_lpips_at_creation": 0.0, - "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", + "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): the bar sits above the measured cross-stepping floor - this B300-cut media scores LPIPS 0.0749 on the B200 lane that gates it - so the gate catches severe model regressions (historical reds 0.45-0.61), not GPU/kernel drift, and survives a CI GPU change without re-cutting media. Self-regeneration on the cutting host remains 0.0.", "diffusers_version": "0.39.0", "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Portable by construction: generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), under which the trajectory measured bit-stable across torch 2.11/2.12 and B200/B300.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). The trajectory is NOT bit-stable across GPU steppings: this media measures LPIPS 0.0749 on B200 (sm100), the lane that gates it (nvbugs/6655359); the threshold sits above that floor.", "sha256": "728c9bed1c25bf7de2b727949cc8c985f0f5bf03fd78e19847f4bcc7edeb450b" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip index c298c535a41b..f3f6058b0408 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/visual_gen_lpips_golden_media.zip @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:33da5841fe5594a739240d048a0c6139c58e7e8e04d76cce1fb0cce968ead133 -size 30674088 +oid sha256:4c17c72480a5c33a0ece24ed08b9997f3bd55468d71388f69ac2973f4fb8cce3 +size 28176933 diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py index 290028266292..6466d8f63611 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py @@ -53,7 +53,13 @@ COSMOS3_LPIPS_PROMPT = "A serene mountain landscape with snow-capped peaks and a flowing river" COSMOS3_LPIPS_HEIGHT = 720 COSMOS3_LPIPS_WIDTH = 1280 -COSMOS3_LPIPS_T2V_NUM_FRAMES = 189 +# 9 frames = 3 latent frames (temporal VAE: floor((F-1)/4)+1), 2,760 video +# tokens -- the smallest T2V shape that still exercises multi-latent-frame +# temporal attention. The 720P default of 189 frames (44,160 tokens) costs ~7 +# minutes of generation per CI run and adds no gate value: a golden catches +# severe regressions, and cross-stepping drift only grows with trajectory +# length (0.020 at 1 frame -> 0.151 at 189). +COSMOS3_LPIPS_T2V_NUM_FRAMES = 9 COSMOS3_LPIPS_T2I_NUM_FRAMES = 1 # 9 frames = 3 latent frames: latents (0, 1) are pinned to the V2V reference, # latent 2 (pixel frames 5-8) is generated. Frame 8 is the golden-compared frame. @@ -64,6 +70,17 @@ COSMOS3_LPIPS_SEED = 42 COSMOS3_LPIPS_FRAME_RATE = 24.0 COSMOS3_LPIPS_THRESHOLD = 0.05 +# T2V/V2V gate at a relaxed KPI-backstop band, not at 0.05: Cosmos3-Nano +# trajectories are not bit-stable across GPU steppings (nvbugs/6655359 -- +# B300-cut media measured LPIPS 0.020/0.075/0.151 on B200 at 1/9/189 frames +# with everything else held fixed), so a tight bar just re-fires whenever the +# media host and the CI lane disagree. The bars sit above the 9-frame +# cross-stepping floor (0.075) so the gates catch severe model regressions +# (historical reds were 0.45-0.61) and survive a CI GPU change without +# re-cutting media. The V2V golden is B300-cut and scores 0.070 on the B200 +# lane; the T2V golden is cut on B200, the lane's own GPU. +COSMOS3_LPIPS_T2V_THRESHOLD = 0.20 +COSMOS3_LPIPS_V2V_THRESHOLD = 0.15 COSMOS3_I2V_4STEP_MODEL_SUBPATH = "Cosmos3-Super-Image2Video-4Step" COSMOS3_I2V_4STEP_LPIPS_PROMPT = ( "The orange sphere slowly rises while the camera pans right across the scene" @@ -363,7 +380,7 @@ def test_cosmos3_nano_t2v_lpips_against_golden(_visual_gen_deps, tmp_path): golden_path, generated_path, ) - _assert_lpips_below_threshold(score, COSMOS3_LPIPS_THRESHOLD) + _assert_lpips_below_threshold(score, COSMOS3_LPIPS_T2V_THRESHOLD) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") @@ -383,7 +400,7 @@ def test_cosmos3_nano_v2v_lpips_against_golden(_visual_gen_deps, tmp_path): golden_path, generated_path, ) - _assert_lpips_below_threshold(score, COSMOS3_LPIPS_THRESHOLD) + _assert_lpips_below_threshold(score, COSMOS3_LPIPS_V2V_THRESHOLD) @pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available") diff --git a/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py b/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py index 14bc3c16a24d..c54d5c4ce880 100644 --- a/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py +++ b/tests/integration/defs/examples/visual_gen/visual_gen_test_utils.py @@ -411,7 +411,7 @@ def _cleanup_cuda(): @contextlib.contextmanager def _lpips_pinned_fp32_matmul_precision() -> Iterator[None]: - """Pin fp32-matmul arithmetic so LPIPS goldens are portable across hosts. + """Pin fp32-matmul arithmetic so LPIPS goldens are portable across torch stacks. NGC PyTorch containers default matmul TF32 on (``float32_matmul_precision == "high"``); PyPI torch defaults it off (``"highest"``). A model with fp32 @@ -420,9 +420,17 @@ def _lpips_pinned_fp32_matmul_precision() -> Iterator[None]: ``transformer_cosmos3.py``) therefore produces a different trajectory under each default, and a golden cut under one fails under the other -- measured LPIPS-to-golden moved 0.132 -> 0.054 from this single flag. Pin "highest" - (IEEE fp32, measured bit-stable across torch 2.11/2.12 and B200/B300), and - pin cuDNN TF32 to its universal default so the second knob cannot drift. - bf16 compute -- all of the heavy kernels -- is unaffected by either knob. + (IEEE fp32, measured bit-stable across torch 2.11/2.12), and pin cuDNN TF32 + to its universal default so the second knob cannot drift. bf16 compute -- + all of the heavy kernels -- is unaffected by either knob. + + The pin's contract stops at the torch stack: it does NOT make trajectories + bit-stable across GPU steppings. Kernel selection differs between sm100 and + sm103, and the divergence compounds along the denoising trajectory -- + B300-cut Cosmos3-Nano media measured LPIPS 0.02 (1 frame) to 0.15 (189 + frames) on B200 with everything else held fixed (nvbugs/6655359). Golden + thresholds must therefore sit above the measured cross-stepping floor of + their own trajectory, or the media must be cut on the gating lane's GPU. Applied per generation path rather than from ``_lpips_deterministic_algorithms``: that helper also wraps generation for diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index a3e8504ad21f..39246d915c4f 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -15,6 +15,15 @@ l0_b200: backend: pytorch tests: # ------------- PyTorch tests --------------- + # TEMPORARY (REMOVE BEFORE MERGE): these two tests are post_merge-gated, so a + # PR pipeline never schedules them and --extra-stage cannot pull them in (the + # gate is in the test-db terms). Listed here only to prove the relaxed + # thresholds pass on the B200 CI lane before merge; the post_merge block + # below remains their permanent home. Drop this block only AFTER reading the + # B200 stage verdict (nvbugs/6655359 exists because the last such proof-run + # result landed after the listing was already dropped and went unread). + - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (10) + - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden TIMEOUT (10) - unittest/others/test_kv_cache_transceiver.py::test_cpp_nixl_sync_transfer_stress - unittest/others/test_lora_manager.py - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 @@ -435,7 +444,7 @@ l0_b200: - examples/visual_gen/test_visual_gen_qwen_image.py::test_qwenimage_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen_qwen_image.py::test_qwen_image_layered_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2i_lpips_against_golden TIMEOUT (10) - - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (15) + - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_i2v_4step_lpips_against_golden TIMEOUT (20) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_edge_t2v_lpips_against_golden TIMEOUT (20) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 0b04666d49eb..26bec61703cd 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -111,8 +111,6 @@ examples/test_deepseek_v4_pro.py::test_short_token_boundary_smoke SKIP (https:// examples/test_ray.py::test_ray_disaggregated_serving[tp2] SKIP (https://nvbugs/6632606) examples/test_ray.py::test_ray_disaggregated_serving_python[tp2] SKIP (https://nvbugs/6601574) examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6572800) -examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden SKIP (https://nvbugs/6655359) -examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden SKIP (https://nvbugs/6655359) examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux1-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux2-nvfp4] SKIP (https://nvbugs/6572800) examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6644450) From 641384f40e1ad50917633b9e20456e44e098ff08 Mon Sep 17 00:00:00 2001 From: Igor Shovkun Date: Tue, 1 Sep 2026 13:56:53 -0700 Subject: [PATCH 2/7] [https://nvbugs/6655359][test] Drop the temporary pre-merge listing Requested in review. The gates' pre-merge evidence is the local run on the lane's GPU (B200, native, main @ 5fb68830c9): t2v 0.000000 < 0.20 against the lane-cut golden, v2v 0.069536 < 0.15, 2 passed in 71 s. The post_merge block in this file remains the tests' permanent home. Signed-off-by: Igor Shovkun --- tests/integration/test_lists/test-db/l0_b200.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 39246d915c4f..fb6288fb3cd4 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -15,15 +15,6 @@ l0_b200: backend: pytorch tests: # ------------- PyTorch tests --------------- - # TEMPORARY (REMOVE BEFORE MERGE): these two tests are post_merge-gated, so a - # PR pipeline never schedules them and --extra-stage cannot pull them in (the - # gate is in the test-db terms). Listed here only to prove the relaxed - # thresholds pass on the B200 CI lane before merge; the post_merge block - # below remains their permanent home. Drop this block only AFTER reading the - # B200 stage verdict (nvbugs/6655359 exists because the last such proof-run - # result landed after the listing was already dropped and went unread). - - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (10) - - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden TIMEOUT (10) - unittest/others/test_kv_cache_transceiver.py::test_cpp_nixl_sync_transfer_stress - unittest/others/test_lora_manager.py - accuracy/test_llm_api_pytorch.py::TestLlama3_1_8B::test_nvfp4 From 49225f20b1ed419bf49a1ce466bd3b2a76c9f436 Mon Sep 17 00:00:00 2001 From: Igor Shovkun Date: Tue, 1 Sep 2026 14:49:09 -0700 Subject: [PATCH 3/7] [https://nvbugs/6655359][test] Per-gate B200 scores in sibling provenance; fix stale Edge golden-source comment Review feedback: the sibling manifests repeated the Cosmos3-Nano cross-stepping ladder without labeling it as sibling-model context and claimed an unspecified B200 pass. Record each gate's own B200 score where the CI record has one (nano_t2i 0.020043, edge_i2v 0.006800), label the Nano ladder as sibling-model data in the Edge records, and state the remaining gates' post-merge pass without inventing numbers. Also fix the stale test comment still describing the Edge goldens as diffusers-main references: they became TRT-LLM self-goldens at the fp32-pin re-baseline (the manifests were already correct); only i2v_4step remains a genuine diffusers reference. Signed-off-by: Igor Shovkun --- .../cosmos3_edge_i2v_lpips_golden_video.json | 2 +- .../visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json | 2 +- .../cosmos3_edge_t2v_lpips_golden_video.json | 2 +- .../cosmos3_nano_fp8_blockwise_lpips_golden.json | 2 +- .../visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json | 2 +- .../defs/examples/visual_gen/test_visual_gen_cosmos3.py | 9 +++++---- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json index 485b732664fc..d04bdd9817d4 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json @@ -26,6 +26,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359; measured on the sibling Cosmos3-Nano model: LPIPS 0.02-0.15 on B200 by trajectory length). This B300-cut golden measures LPIPS 0.006800 on the B200 lane that gates it (post-merge CI record), under the 0.13 threshold.", "sha256": "1fd9b0ab24de130f593056a32a7b8555fafbbf073b76c29a4633504870b1dad0" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json index 44b6f7182137..1f50e2489950 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json @@ -25,6 +25,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359; measured on the sibling Cosmos3-Nano model: LPIPS 0.02-0.15 on B200 by trajectory length). This gate passes its 0.05 threshold on the B200 post-merge lane.", "sha256": "3f7c9b958807356ced2de1734e301dc837fa0b095f8fed1e29da764993926046" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json index d1a89dc3b63b..86eb3df22c54 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json @@ -25,6 +25,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359; measured on the sibling Cosmos3-Nano model: LPIPS 0.02-0.15 on B200 by trajectory length). This gate passes its 0.1 threshold on the B200 post-merge lane.", "sha256": "0ec80b5c906ae576deedf8fb48c55edd0c78203138608f12d0c439da11ab6f10" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json index ab628be63bca..1446f72ebf80 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json @@ -34,6 +34,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02 (1 frame) to 0.15 (189 frames) on B200. This 1-frame gate passes its 0.05 threshold on the B200 post-merge lane.", "sha256": "32c080983eb8d94d1da5d21378ea87912dadf018f00f15123a33f000640341ad" } diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json index 547533918f8c..affac0b49f49 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json @@ -24,6 +24,6 @@ "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", "tensorrt_llm_commit": "a4ed9a9c13a69b3c024debd0d83e12c8c734bf95", - "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): B300-cut Cosmos3-Nano media measured LPIPS 0.02-0.15 on B200 depending on trajectory length. This gate's B200 score fits under its threshold.", + "environment": "Native build, no container; NVIDIA B300 (sm103). Generation pins float32_matmul_precision('highest') (see _lpips_pinned_fp32_matmul_precision), which makes the torch stack portable (container-vs-native on one GPU measures <=5e-3). Trajectories are NOT bit-stable across GPU steppings (nvbugs/6655359): this B300-cut golden measures LPIPS 0.020043 on the B200 lane that gates it (post-merge CI record), under the 0.05 threshold.", "sha256": "035f3e764e6a36159071178a2d7be6ec3cabc60899736099ff89a59c037e15e1" } diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py index 6466d8f63611..b24da64e8149 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py @@ -748,11 +748,12 @@ def test_cosmos3_edge_i2v_example(_visual_gen_deps, llm_root, llm_venv): assert os.path.getsize(output_path) > 0, f"Example produced an empty video at {output_path}" -# Edge LPIPS gates compare against diffusers-main reference goldens with the -# scheduler patched to the cosmos-framework native flow schedule; full +# Edge LPIPS gates compare against TRT-LLM self-goldens (originally cut from +# diffusers-main references, re-baselined as self-goldens when the fp32-matmul +# pin landed; cross-stack correctness is covered by TestDiffusersParity); full # provenance in golden/visual_gen_lpips/cosmos3_edge_*.json. The I2V gate runs -# 10 steps (cross-stack drift accumulates per step; the deployed 50-step shape -# is covered by test_cosmos3_edge_i2v_example). +# 10 steps (drift accumulates per step; the deployed 50-step shape is covered +# by test_cosmos3_edge_i2v_example). COSMOS3_EDGE_LPIPS_SEED = 42 COSMOS3_EDGE_LPIPS_FRAME_RATE = 24.0 COSMOS3_EDGE_LPIPS_NUM_FRAMES = 29 From 946c6706f64cf2b7520a43bea9b03c7a2c0bfab9 Mon Sep 17 00:00:00 2001 From: Igor Shovkun Date: Wed, 2 Sep 2026 13:50:36 -0700 Subject: [PATCH 4/7] [https://nvbugs/6655359][test] Drop the unread lpips_threshold field from the touched Cosmos3 goldens Review feedback: the field duplicated the Python constant with no reader, so the two could diverge silently and a reader could not tell which one the assertion honours. Nothing loads these manifests at runtime - every lpips_threshold in Python is a dataclass field fed from a module constant, the same pattern across the cosmos3, qwenimage, wan, glm and flux suites - so the JSON copy was dead data. Removed from the seven manifests this PR already touches; nvfp4 and i2v_4step keep theirs since this PR does not otherwise open them, and the same cleanup applies to the other models' manifests as a standalone change. Signed-off-by: Igor Shovkun --- .../visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json | 1 - .../golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json | 1 - .../visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json | 1 - .../cosmos3_nano_fp8_blockwise_lpips_golden.json | 1 - .../golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json | 1 - .../visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json | 1 - .../visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json | 1 - 7 files changed, 7 deletions(-) diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json index d04bdd9817d4..d25fc96ebe08 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_i2v_lpips_golden_video.json @@ -19,7 +19,6 @@ "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.13, "measured_lpips_at_creation": 0.0, "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", "diffusers_version": "0.39.0", diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json index 1f50e2489950..2bd362ce3201 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2i_lpips_golden.json @@ -18,7 +18,6 @@ "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.05, "measured_lpips_at_creation": 0.0, "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", "diffusers_version": "0.39.0", diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json index 86eb3df22c54..b23d0c7439af 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_edge_t2v_lpips_golden_video.json @@ -18,7 +18,6 @@ "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.1, "measured_lpips_at_creation": 0.0, "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", "diffusers_version": "0.39.0", diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json index 1446f72ebf80..8131c360a78f 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_fp8_blockwise_lpips_golden.json @@ -27,7 +27,6 @@ "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.05, "measured_lpips_at_creation": 0.0, "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", "diffusers_version": "0.39.0", diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json index affac0b49f49..f17735b95b77 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2i_lpips_golden.json @@ -17,7 +17,6 @@ "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.05, "measured_lpips_at_creation": 0.0, "threshold_rationale": "self-regeneration distance on the cutting host; threshold kept at the pre-existing gate for this test", "diffusers_version": "0.39.0", diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json index d2bb0c16d493..400c89b1d83b 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json @@ -18,7 +18,6 @@ "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.2, "measured_lpips_at_creation": 0.0, "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): media is cut on the gating lane's own GPU (self-distance ~0), and the bar sits above the measured 9-frame cross-stepping floor (0.075, B300-cut media on B200) so a future CI GPU change does not force a re-cut; severe regressions (historical reds 0.45-0.61) still fire it.", "diffusers_version": "0.39.0", diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json index 176e139c0b65..9823ff253aa9 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json @@ -20,7 +20,6 @@ "fp32_matmul_precision": "highest", "cudnn_allow_tf32": true, "lpips_net": "alex", - "lpips_threshold": 0.15, "measured_lpips_at_creation": 0.0, "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): the bar sits above the measured cross-stepping floor - this B300-cut media scores LPIPS 0.0749 on the B200 lane that gates it - so the gate catches severe model regressions (historical reds 0.45-0.61), not GPU/kernel drift, and survives a CI GPU change without re-cutting media. Self-regeneration on the cutting host remains 0.0.", "diffusers_version": "0.39.0", From 2e94dc7aaa22bf8a08de7b93c5ca0f6722144fe3 Mon Sep 17 00:00:00 2001 From: Igor Shovkun Date: Wed, 2 Sep 2026 20:45:13 -0700 Subject: [PATCH 5/7] [https://nvbugs/6655359][test] Keep the T2V post-merge timeout at 15 min Review feedback: the 71 s figure behind the tightening is a warm-cache local run on a dedicated B200 and does not cover checkpoint staging, _visual_gen_deps setup, LPIPS weight download, or a shared node. These gates are post_merge-only, so a too-tight budget would surface as a red on main with no pre-merge signal - the same class of non-model red this PR exists to remove. Keeping the existing budget costs nothing, since the generation cost is already down ~7 min; it can be tightened once post-merge builds have shown the real wall time. Signed-off-by: Igor Shovkun --- tests/integration/test_lists/test-db/l0_b200.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index fb6288fb3cd4..a3e8504ad21f 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -435,7 +435,7 @@ l0_b200: - examples/visual_gen/test_visual_gen_qwen_image.py::test_qwenimage_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen_qwen_image.py::test_qwen_image_layered_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2i_lpips_against_golden TIMEOUT (10) - - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (10) + - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_t2v_lpips_against_golden TIMEOUT (15) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_nano_v2v_lpips_against_golden TIMEOUT (10) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_i2v_4step_lpips_against_golden TIMEOUT (20) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_edge_t2v_lpips_against_golden TIMEOUT (20) From 715ae6b09335093f2204c40beadaf52733e968e4 Mon Sep 17 00:00:00 2001 From: Igor Shovkun Date: Wed, 2 Sep 2026 21:16:07 -0700 Subject: [PATCH 6/7] [https://nvbugs/6655359][test] Record the re-measured self-distance and correct the regression figure Two fixes to the same sentences. The T2V rationale said self-distance was ~0, which is indistinguishable from a value carried over from the old media; it now records that the re-cut was re-measured on its own cutting host at 0.000000. The bars were also justified against 'historical reds 0.45-0.61'. The lower figure is nvbugs/6437341 at 0.449779, whose own record labels it a flaky test alternating pass/fail on unchanged main - a stale golden, not a regression, and evidence for relaxing rather than against it. Only nvbugs/6418815 at 0.608404 was a code-caused output change, bisected to a pipeline commit. The comment and both manifests now cite that single measured figure. Signed-off-by: Igor Shovkun --- .../cosmos3_nano_t2v_lpips_golden_video.json | 2 +- .../cosmos3_nano_v2v_lpips_golden_frame.json | 2 +- .../defs/examples/visual_gen/test_visual_gen_cosmos3.py | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json index 400c89b1d83b..f94cfc57272c 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_t2v_lpips_golden_video.json @@ -19,7 +19,7 @@ "cudnn_allow_tf32": true, "lpips_net": "alex", "measured_lpips_at_creation": 0.0, - "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): media is cut on the gating lane's own GPU (self-distance ~0), and the bar sits above the measured 9-frame cross-stepping floor (0.075, B300-cut media on B200) so a future CI GPU change does not force a re-cut; severe regressions (historical reds 0.45-0.61) still fire it.", + "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): media is cut on the gating lane's own GPU and re-measured there after the cut at LPIPS 0.000000, and the bar sits above the measured 9-frame cross-stepping floor (0.075, B300-cut media on B200) so a future CI GPU change does not force a re-cut. The one code-caused regression these gates have caught measured 0.608404 (nvbugs/6418815, bisected to a pipeline change), well above this bar.", "diffusers_version": "0.39.0", "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc26", diff --git a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json index 9823ff253aa9..60c331b1dcfe 100644 --- a/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json +++ b/tests/integration/defs/examples/visual_gen/golden/visual_gen_lpips/cosmos3_nano_v2v_lpips_golden_frame.json @@ -21,7 +21,7 @@ "cudnn_allow_tf32": true, "lpips_net": "alex", "measured_lpips_at_creation": 0.0, - "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): the bar sits above the measured cross-stepping floor - this B300-cut media scores LPIPS 0.0749 on the B200 lane that gates it - so the gate catches severe model regressions (historical reds 0.45-0.61), not GPU/kernel drift, and survives a CI GPU change without re-cutting media. Self-regeneration on the cutting host remains 0.0.", + "threshold_rationale": "relaxed KPI-backstop band (nvbugs/6655359): the bar sits above the measured cross-stepping floor - this B300-cut media scores LPIPS 0.0749 on the B200 lane that gates it - so the gate catches model regressions rather than GPU/kernel drift, and survives a CI GPU change without re-cutting media. The one code-caused regression these gates have caught measured 0.608404 (nvbugs/6418815), well above this bar. Self-regeneration on the cutting host remains 0.0.", "diffusers_version": "0.39.0", "torch_version": "2.12.0+cu130", "tensorrt_llm_version": "1.3.0rc25", diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py index b24da64e8149..2c19b1be5006 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py @@ -75,10 +75,11 @@ # B300-cut media measured LPIPS 0.020/0.075/0.151 on B200 at 1/9/189 frames # with everything else held fixed), so a tight bar just re-fires whenever the # media host and the CI lane disagree. The bars sit above the 9-frame -# cross-stepping floor (0.075) so the gates catch severe model regressions -# (historical reds were 0.45-0.61) and survive a CI GPU change without -# re-cutting media. The V2V golden is B300-cut and scores 0.070 on the B200 -# lane; the T2V golden is cut on B200, the lane's own GPU. +# cross-stepping floor (0.075) so the gates catch model regressions and survive +# a CI GPU change without re-cutting media. The one code-caused regression +# these gates have caught measured 0.608404 (nvbugs/6418815), well above both +# bars. The V2V golden is B300-cut and scores 0.070 on the B200 lane; the T2V +# golden is cut on B200, the lane's own GPU. COSMOS3_LPIPS_T2V_THRESHOLD = 0.20 COSMOS3_LPIPS_V2V_THRESHOLD = 0.15 COSMOS3_I2V_4STEP_MODEL_SUBPATH = "Cosmos3-Super-Image2Video-4Step" From 7a56ec596565f180b47bc986afbab30ff3cad815 Mon Sep 17 00:00:00 2001 From: Igor Shovkun Date: Thu, 3 Sep 2026 11:06:22 -0700 Subject: [PATCH 7/7] [https://nvbugs/6655359][test] Rename COSMOS3_LPIPS_THRESHOLD to COSMOS3_LPIPS_T2I_THRESHOLD Review feedback: once T2V and V2V moved to their own constants, this one was left with a single user - the T2I gate - while its generic name still read like a shared default. Renamed to match its scope, so the three gates are named consistently, and documented why T2I keeps the tight 0.05 bar: the 1-frame shape has the smallest cross-stepping exposure in the family (0.020 measured on the B200 lane), so it needs no relaxed band. Value unchanged. Signed-off-by: Igor Shovkun --- .../defs/examples/visual_gen/test_visual_gen_cosmos3.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py index 2c19b1be5006..d4c352938c31 100644 --- a/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py +++ b/tests/integration/defs/examples/visual_gen/test_visual_gen_cosmos3.py @@ -69,7 +69,11 @@ COSMOS3_LPIPS_GUIDANCE_SCALE = 6.0 COSMOS3_LPIPS_SEED = 42 COSMOS3_LPIPS_FRAME_RATE = 24.0 -COSMOS3_LPIPS_THRESHOLD = 0.05 +# T2I stays at the original tight bar: the 1-frame shape has the smallest +# cross-stepping exposure in the family (its B300-cut golden scores 0.020 on +# the B200 lane), so 0.05 keeps ~2.5x margin over that floor without the +# relaxed band the longer trajectories need. +COSMOS3_LPIPS_T2I_THRESHOLD = 0.05 # T2V/V2V gate at a relaxed KPI-backstop band, not at 0.05: Cosmos3-Nano # trajectories are not bit-stable across GPU steppings (nvbugs/6655359 -- # B300-cut media measured LPIPS 0.020/0.075/0.151 on B200 at 1/9/189 frames @@ -419,7 +423,7 @@ def test_cosmos3_nano_t2i_lpips_against_golden(_visual_gen_deps, tmp_path): golden_path, generated_path, ) - _assert_lpips_below_threshold(score, COSMOS3_LPIPS_THRESHOLD) + _assert_lpips_below_threshold(score, COSMOS3_LPIPS_T2I_THRESHOLD) def test_cosmos3_example(_visual_gen_deps, llm_root, llm_venv):