Release 2026-09-25 (i4 358182a3)

.file_mapping.json

@@ -1,7 +1,7 @@
 {
-  "_source_commit": "461431bae40d4a8ccf27d812fc9d1757fe8d3b96-dirty",
-  "_dest_commit": "0460be81f16883aa380e716dc6f58c1189481172",
-  "_generated_at": "2026-09-23T04:47:38Z",
+  "_source_commit": "358182a3a006975c25fc079fac9ff80e704077e2-dirty",
+  "_dest_commit": "cf5d68c00d97ccd2480a2320ed652b92dec63102",
+  "_generated_at": "2026-09-25T05:52:16Z",
   "files": {
     "imaginaire/__init__.py": "cosmos_framework/__init__.py",
     "imaginaire/attention/__init__.py": "cosmos_framework/model/attention/__init__.py",
@@ -293,6 +293,8 @@
     "projects/cosmos3/cosmos3/datasets/augmentors/cropping.py": "cosmos_framework/data/generator/augmentors/cropping.py",
     "projects/cosmos3/cosmos3/datasets/augmentors/duration_fps_text_timestamps.py": "cosmos_framework/data/generator/augmentors/duration_fps_text_timestamps.py",
     "projects/cosmos3/cosmos3/datasets/augmentors/duration_fps_text_timestamps_test.py": "cosmos_framework/data/generator/augmentors/duration_fps_text_timestamps_test.py",
+    "projects/cosmos3/cosmos3/datasets/augmentors/first_caption_paragraph.py": "cosmos_framework/data/generator/augmentors/first_caption_paragraph.py",
+    "projects/cosmos3/cosmos3/datasets/augmentors/first_caption_paragraph_test.py": "cosmos_framework/data/generator/augmentors/first_caption_paragraph_test.py",
     "projects/cosmos3/cosmos3/datasets/augmentors/idle_frames_text_info.py": "cosmos_framework/data/generator/augmentors/idle_frames_text_info.py",
     "projects/cosmos3/cosmos3/datasets/augmentors/image_editing_transform.py": "cosmos_framework/data/generator/augmentors/image_editing_transform.py",
     "projects/cosmos3/cosmos3/datasets/augmentors/image_editing_transform_test.py": "cosmos_framework/data/generator/augmentors/image_editing_transform_test.py",
@@ -470,6 +472,8 @@
     "projects/cosmos3/cosmos3/models/utils/load_balancing_stats.py": "cosmos_framework/model/generator/utils/load_balancing_stats.py",
     "projects/cosmos3/cosmos3/models/utils/memory.py": "cosmos_framework/model/generator/utils/memory.py",
     "projects/cosmos3/cosmos3/models/utils/moe_utils.py": "cosmos_framework/model/generator/utils/moe_utils.py",
+    "projects/cosmos3/cosmos3/models/utils/rig_view_embedding.py": "cosmos_framework/model/generator/utils/rig_view_embedding.py",
+    "projects/cosmos3/cosmos3/models/utils/rig_view_embedding_test.py": "cosmos_framework/model/generator/utils/rig_view_embedding_test.py",
     "projects/cosmos3/cosmos3/models/utils/safetensors_loader.py": "cosmos_framework/model/generator/utils/safetensors_loader.py",
     "projects/cosmos3/cosmos3/models/utils/safetensors_loader_indexed_test.py": "cosmos_framework/model/generator/utils/safetensors_loader_indexed_test.py",
     "projects/cosmos3/cosmos3/models/utils/safetensors_loader_test.py": "cosmos_framework/model/generator/utils/safetensors_loader_test.py",
@@ -569,6 +573,7 @@
     "projects/cosmos3/cosmos3/utils/reasoner/pretrained_models_downloader_test.py": "cosmos_framework/utils/generator/reasoner/pretrained_models_downloader_test.py",
     "projects/cosmos3/cosmos3/utils/reasoner/true_packing.py": "cosmos_framework/utils/generator/reasoner/true_packing.py",
     "projects/cosmos3/cosmos3/utils/source_video_timing.py": "cosmos_framework/utils/generator/source_video_timing.py",
+    "projects/cosmos3/cosmos3/utils/spatial_patch.py": "cosmos_framework/utils/generator/spatial_patch.py",
     "projects/cosmos3/cosmos3/utils/video_frame_sampling.py": "cosmos_framework/utils/generator/video_frame_sampling.py",
     "projects/cosmos3/cosmos3/utils/video_preprocess.py": "cosmos_framework/utils/generator/video_preprocess.py",
     "projects/cosmos3/cosmos3/utils/video_source_metadata.py": "cosmos_framework/utils/generator/video_source_metadata.py",
@@ -580,6 +585,8 @@
     "projects/cosmos3/interactive/models/mot/context_parallel_test.py": "cosmos_framework/model/generator/mot/causal_context_parallel_test.py",
     "projects/cosmos3/interactive/models/mot/cosmos3_vfm_network.py": "cosmos_framework/model/generator/mot/causal_cosmos3_vfm_network.py",
     "projects/cosmos3/interactive/models/mot/flex_attention.py": "cosmos_framework/model/generator/mot/causal_flex_attention.py",
+    "projects/cosmos3/interactive/models/mot/maskless_attention.py": "cosmos_framework/model/generator/mot/maskless_attention.py",
+    "projects/cosmos3/interactive/models/mot/merge_attention.py": "cosmos_framework/model/generator/mot/merge_attention.py",
     "projects/cosmos3/interactive/models/mot/post_saturation/__init__.py": "cosmos_framework/model/generator/mot/post_saturation/__init__.py",
     "projects/cosmos3/interactive/models/mot/post_saturation/cuda_graph.py": "cosmos_framework/model/generator/mot/post_saturation/cuda_graph.py",
     "projects/cosmos3/interactive/models/mot/post_saturation/installer.py": "cosmos_framework/model/generator/mot/post_saturation/installer.py",

cosmos_framework/callbacks/every_n_draw_sample.py

@@ -45,9 +45,9 @@ class WandbAnimation(NamedTuple):
 
 
 # What a clip is previewed as in W&B: one still grid, one panel per sampled
-# frame, or an animation of every frame.
+# frame, an animation of every frame, or direct interactive point-cloud media.
 WandbClipPreview = Literal["grid", "frames", "animation"]
-WandbMedia = str | dict[str, str | WandbAnimation] | WandbAnimation
+WandbMedia = str | WandbAnimation | list[wandb.Object3D] | dict[str, "WandbMedia"]
 
 
 def resize_image(image: torch.Tensor, size: int = 1024) -> torch.Tensor:
@@ -292,10 +292,10 @@ def _decode_transfer_pixel_row(
 ) -> torch.Tensor | None:  # [V,C,F,H,W] or None
     """Decode one latent into a host-side pixel row without display conversion.
 
-    ``decode`` defaults to the model's main VAE; a joint camera + LiDAR sample passes
-    ``model.decode_lidar`` for range clips because the two streams have separate VAEs.
-    LiDAR callers pass ``decode_per_view=False``: V0 and V1 are both 1x temporal on a
-    single range view, not 4x camera-major WAN clips.
+    ``decode`` defaults to the model's main VAE; a joint camera + LiDAR sample
+    passes ``model.decode_lidar`` for those clips because the streams have
+    separate VAEs. LiDAR callers pass ``decode_per_view=False``: the rangemap
+    is 1x temporal on a single view, not 4x camera-major WAN clips.
     """
     decode = decode if decode is not None else model.decode
     if decode_per_view is None:
@@ -565,6 +565,10 @@ def _add_wandb_media(
 ) -> None:
     if media is None:
         return
+    if isinstance(media, list):
+        if media:
+            info[key_prefix] = media
+        return
     if isinstance(media, WandbAnimation):
         info[key_prefix] = wandb.Video(media.path, caption=f"{caption} | {media.num_frames} frames")
         return
@@ -747,6 +751,13 @@ class EveryNDrawSample(EveryN):
         use_negative_prompt (bool, optional): whether to use negative prompt. Defaults to False.
         fps (int, optional): frames per second when saving the video. Defaults to 16.
         wandb_log_image_size (int, optional): max side length for W&B JPEG/GIF previews. Defaults to 1024.
+        log_lidar_point_clouds (bool, optional): also log separate real/generated LiDAR point-cloud panels. Defaults to True.
+        point_cloud_max_frames (int | None, optional): preview scans per row, starting at the first generated scan.
+            None keeps all scans, including labelled conditioning inputs. Defaults to 3.
+        point_cloud_max_points (int, optional): points per cloud, at most 300000. Defaults to 300000.
+        point_cloud_color_by (str, optional): BEV height colors or grayscale intensity. Defaults to "height".
+        point_cloud_include_conditioned_frames (bool, optional): sample across the full clip, including labelled
+            conditioning inputs, when bounding the preview scan count. Defaults to False.
     """
 
     def __init__(
@@ -1005,6 +1016,9 @@ def _generate_transfer_latents(
         if plans and not getattr(plans[0], "has_lidar", False):
             generation_batch.pop("lidar", None)
             generation_batch.pop("num_lidar_items_per_sample", None)
+        if plans and not getattr(plans[0], "has_radar", False):
+            generation_batch.pop("radar", None)
+            generation_batch.pop("num_radar_items_per_sample", None)
         # Preserve an explicit caller-provided prompt. Otherwise restore the exact prompt recorded
         # by the training tokenizer before train-sample inference re-tokenizes the raw caption.
         text_system_prompt = generation_batch.pop(TEXT_SYSTEM_PROMPT_KEY, None)

cosmos_framework/configs/base/defaults/model_config.py

@@ -46,10 +46,25 @@ class DiffusionExpertConfig:
     # Enabled by default
     enable_sound_modality_embedding: bool = True
 
+    # Zero disables physical rig embeddings. When enabled, N >= 2 reserves camera IDs
+    # 0..N-2 and LiDAR ID N-1, even for camera-only inputs. N is the sensor vocabulary
+    # size for the model, not the number of sensors selected in a sample.
+    # Examples for the MADS rig (N=12, camera IDs 0..10, LiDAR ID 11):
+    # - Camera only: selecting camera 8 uses row 8; row 11 is unused.
+    # - LiDAR only: uses row 11; no vision_view_ids are needed.
+    # - Camera + LiDAR: selected cameras use their physical IDs and LiDAR uses row 11.
+    num_view_embeddings: int = 0
+
     patch_spatial: int = 2
+    # None preserves the shared camera/LiDAR/radar patch size used by existing checkpoints.
+    # An int selects square patches; (height, width) selects rectangular patches.
+    # Any is required here because OmegaConf rejects unions containing tuples.
+    lidar_patch_spatial_hw: Any = None
+    radar_patch_spatial_hw: Any = None
     max_vae_latent_side_after_patchify: int = (
         52  # Max h/w of the VAE latent after patchification; 52 -> up to ~1664px square (52*32). Was 20 (=640px).
     )
+
     # Vision/action/sound position information is always provided through
     # Qwen3VL-style 3D mRoPE attention IDs.
     enable_fps_modulation: bool = False
@@ -72,6 +87,7 @@ class DiffusionExpertConfig:
 class RectifiedFlowTrainingConfig:
     shift: Any = 5  # Training time shift. If dict, maps resolution (str) to shift value (int)
     shift_image: Any | None = None  # Image-specific shift; None inherits shift
+    shift_lidar: int | None = None  # LiDAR-only batches; None preserves the shared vision schedule
     use_dynamic_shift: bool = False  # Whether to use dynamic shifting
     train_time_image_distribution: str = "logitnormal"  # Training time distribution for images
     train_time_video_distribution: str = "logitnormal"  # Training time distribution for videos
@@ -82,6 +98,7 @@ class RectifiedFlowTrainingConfig:
     image_loss_scale: float | None = None  # If set, overrides loss_scale for images
     sound_loss_scale: float | None = None  # If set, overrides loss_scale for sound
     lidar_loss_scale: float | None = None  # If set, overrides loss_scale for lidar
+    radar_loss_scale: float | None = None  # If set, overrides loss_scale for radar
     use_discrete_rf: bool = False  # Whether to use discrete formulation of rectified flow
 
     # user: please adjust this value according to loss_scale to balance the action loss with the video loss.
@@ -110,6 +127,12 @@ class RectifiedFlowTrainingConfig:
     # False by default to preserve legacy loss magnitudes; enable for AR/DF training.
     normalize_loss_by_active: bool = False
 
+    # None preserves the legacy strategy-dependent item mean: teacher forcing
+    # excludes fully conditioned items, other strategies include them. Set
+    # False to match the bidirectional teacher's denominator, including zero-loss
+    # control items; this is independent of per-item active-token normalization.
+    exclude_fully_conditioned_items: bool | None = None
+
     # Sample-level (vs rank-level) loss averaging for the vision modality.
     #
     # By default the vision loss on each rank is a mean over that rank's samples, and
@@ -194,6 +217,20 @@ class OmniMoTModelConfig:
     seconds, which is what puts the two sensors' latents on one mRoPE time axis.
     """
 
+    radar_state_ch: int | None = None
+    """Radar VAE latent channel count, i.e. the width of the network's radar heads.
+
+    The radar VAE is as wide as the LiDAR one (128), but the two are sized from their own
+    fields because nothing ties the two sensors' widths together.
+    """
+
+    radar_fps: float | None = None
+    """Cycle rate in Hz of radar items, the counterpart of ``lidar_fps``.
+
+    Radar cycles at 20 Hz against LiDAR's 10 Hz, and its VAE does not compress time, so
+    this is what places a radar scan on the mRoPE time axis the other streams share.
+    """
+
     net: LazyDict = None
     ema: EMAConfig = EMAConfig()
 

cosmos_framework/configs/base/defaults/multiview_attention.py

@@ -184,6 +184,13 @@ class MultiviewAttentionMaskConfig:
     # ._multiview_pair_predicate.
     lidar_attends_captions: bool = True
 
+    # The same switch for the radar stream of a joint camera + radar pack, read the same way.
+    # Radar has a stronger prior claim to needing it than LiDAR does: the captions describe what
+    # the cameras see, and a BEV occupancy grid shares even less of that vocabulary than a range
+    # image does. Independent of ``lidar_attends_captions`` so a three-sensor pack can drop text
+    # for one stream and keep it for the other.
+    radar_attends_captions: bool = True
+
 
 @attrs.define(slots=False)
 class MultiviewAttentionConfig:

cosmos_framework/configs/base/defaults/replay_attention.py

@@ -12,7 +12,7 @@
     AttentionScope,
 )
 
-TeacherForcingKVImplementation = Literal["multiview_flex_kv", "singleview_threeway_kv"]
+TeacherForcingKVImplementation = Literal["multiview_flex_kv", "multiview_maskless_kv", "singleview_threeway_kv"]
 TEACHER_FORCING_KV_IMPLEMENTATIONS = get_args(TeacherForcingKVImplementation)
 
 TeacherForcingControlVisibility = Literal["global", "causal", "current"]

cosmos_framework/data/generator/augmentors/first_caption_paragraph.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Keep the opening temporal paragraph for a short clip at the caption chunk's start."""
+
+from typing import Any
+
+from cosmos_framework.data.imaginaire.webdataset.augmentors.augmentor import Augmentor
+from cosmos_framework.utils import log
+from cosmos_framework.data.generator.multiview.caption_format import first_caption_paragraph
+
+
+class FirstCaptionParagraph(Augmentor):
+    """Retain the first timestamped paragraph of a plain-text Tier-1 caption.
+
+    Timestamp boundaries also handle consecutive paragraphs separated by only
+    one newline. Untimestamped captions keep their first blank-line paragraph.
+    Run before duration/resolution metadata is appended and before CFG dropout.
+    """
+
+    def __init__(self) -> None:
+        super().__init__(input_keys=["ai_caption"], output_keys=["ai_caption"])
+
+    def __call__(self, data: dict[str, Any]) -> dict[str, Any] | None:
+        caption = data["ai_caption"]
+        if not isinstance(caption, str):
+            log.warning(
+                f"FirstCaptionParagraph: dropping non-text caption ({type(caption).__name__}); "
+                f"sample_key={data.get('__key__')}, url={data.get('__url__')}",
+                rank0_only=False,
+            )
+            return None
+        data["ai_caption"] = first_caption_paragraph(caption)
+        return data

cosmos_framework/data/generator/augmentors/first_caption_paragraph_test.py

@@ -0,0 +1,35 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Temporal-caption trimming must not retain events outside the short training window."""
+
+import pytest
+
+from cosmos_framework.data.generator.augmentors.first_caption_paragraph import FirstCaptionParagraph
+
+pytestmark = [pytest.mark.L0, pytest.mark.CPU]
+
+
+@pytest.mark.parametrize(
+    ("caption", "expected"),
+    [
+        ("[0-2s] The car approaches.\n\n[2-5s] It turns.", "[0-2s] The car approaches."),
+        ("[0-1.8s] The car approaches.\n[1.8-5s] It turns.", "[0-1.8s] The car approaches."),
+        ("[0s – 2.5s] The car approaches, [2.5s – 5s] It turns.", "[0s – 2.5s] The car approaches"),
+        ("Overview.\n\n[0-3s] The car\napproaches.\n\n[3-5s] It turns.", "[0-3s] The car\napproaches."),
+        ("  A car approaches.\n\nIt turns later.  ", "A car approaches."),
+        ("  [0-2s] A car approaches.  ", "[0-2s] A car approaches."),
+        ("", ""),
+    ],
+)
+def test_keeps_only_the_opening_caption_paragraph(caption: str, expected: str) -> None:
+    data = {"ai_caption": caption, "__key__": "clip"}
+    result = FirstCaptionParagraph()(data)
+    assert result is not None
+    assert result["ai_caption"] == expected
+    assert result["__key__"] == "clip"
+
+
+@pytest.mark.parametrize("caption", [{"scene": "not a Tier-1 paragraph"}, None, ["caption"]])
+def test_drops_nontext_captions(caption: object) -> None:
+    assert FirstCaptionParagraph()({"ai_caption": caption, "__key__": "clip"}) is None

cosmos_framework/data/generator/augmentors/pkl_to_media.py

@@ -225,7 +225,9 @@ def _bytes_to_video_frames(self, video_bytes: bytes, identifier: str = "video")
                 data=video_bytes,
                 **self.video_decoder_params,
             )
-            result["videos"] = tensor_to_pil_images(result["videos"])  # 3,T,H,W -> list of PIL images
+            result["videos"] = tensor_to_pil_images(
+                result["videos"], channels_first=True
+            )  # 3,T,H,W -> list of PIL images
             if result is not None:
                 return result
             else:

cosmos_framework/data/generator/augmentors/reasoner/bytes_to_media.py

@@ -257,7 +257,9 @@ def _bytes_to_video_frames(
                 return None
             if self.video_timestamp_mode == "source_pts":
                 validate_source_video_timing(result.get(SOURCE_VIDEO_TIMING_KEY), result["videos"].shape[1])
-            result["videos"] = tensor_to_pil_images(result["videos"])  # 3,T,H,W -> list of PIL images
+            result["videos"] = tensor_to_pil_images(
+                result["videos"], channels_first=True
+            )  # 3,T,H,W -> list of PIL images
             return result
         except Exception as e:
             if self.video_timestamp_mode == "source_pts":