From 029a4699d152d86a37ce9cefee86d47d1f2acc7a Mon Sep 17 00:00:00 2001 From: Liang Feng Date: Wed, 23 Sep 2026 04:47:56 +0000 Subject: [PATCH 1/3] Release 2026-09-23 from i4 461431bae40d4a8ccf27d812fc9d1757fe8d3b96 _source_commit: 461431bae40d4a8ccf27d812fc9d1757fe8d3b96-dirty _dest_commit: 0460be81f16883aa380e716dc6f58c1189481172 --- .file_mapping.json | 28 +- .../base/defaults/multiview_attention.py | 7 +- .../base/reasoner/defaults/augmentors.py | 4 + .../configs/base/reasoner/defaults/config.py | 14 + .../generator/action/utils/domain_utils.py | 5 + .../data/generator/augmentor_provider.py | 109 ----- .../augmentors/reasoner/bytes_to_media.py | 25 + .../augmentors/reasoner/prompt_format.py | 72 ++- .../augmentors/reasoner/prompt_format_test.py | 75 +++ .../reasoner/source_timestamps_test.py | 243 ++++++++++ .../augmentors/reasoner/timestamp.py | 41 +- .../timestamp_with_subject_tracking.py | 4 + .../timestamp_without_augment_message.py | 4 + .../reasoner/timestamp_without_end_time.py | 4 + .../augmentors/reasoner/tokenize_data.py | 101 +++- .../augmentors/reasoner/tokenize_data_test.py | 31 ++ .../generator/processors/VIDEO_TIMESTAMPS.md | 90 ++++ .../data/generator/processors/__init__.py | 7 + .../data/generator/processors/audio_utils.py | 21 +- .../data/generator/processors/base.py | 34 +- .../data/generator/processors/base_test.py | 43 ++ .../processors/cosmos3_edge_processing.py | 59 ++- .../processors/nemotron3densevl_processor.py | 20 +- .../generator/processors/qwen3vl_processor.py | 1 + .../processors/source_video_timing_test.py | 187 ++++++++ .../generator/reasoner/video_decoder_qwen.py | 74 ++- .../model/generator/mot/attention_test.py | 377 ++++++++++++--- .../generator/mot/context_parallel_test.py | 1 + .../generator/mot/cosmos3_vfm_network.py | 48 +- .../model/generator/mot/diffusion_cache.py | 447 +++++++----------- .../generator/mot/flex_attention_test.py | 5 + .../generator/mot/multiview_attention_test.py | 55 ++- .../mot/multiview_maskless_attention.py | 249 +++++++++- .../mot/multiview_maskless_attention_bench.py | 43 +- .../model/generator/omni_mot_model.py | 110 ++++- .../tokenizers/wan2pt2_vae_4x16x16.py | 88 ++-- .../utils/generator/source_video_timing.py | 133 ++++++ .../utils/generator/video_source_metadata.py | 58 +++ .../utils/one_logger/one_logger_utils.py | 3 + .../utils/one_logger/one_logger_utils_test.py | 100 ++++ 40 files changed, 2373 insertions(+), 647 deletions(-) create mode 100644 cosmos_framework/data/generator/augmentors/reasoner/prompt_format_test.py create mode 100644 cosmos_framework/data/generator/augmentors/reasoner/source_timestamps_test.py create mode 100644 cosmos_framework/data/generator/processors/VIDEO_TIMESTAMPS.md create mode 100644 cosmos_framework/data/generator/processors/base_test.py create mode 100644 cosmos_framework/data/generator/processors/source_video_timing_test.py create mode 100644 cosmos_framework/utils/generator/source_video_timing.py create mode 100644 cosmos_framework/utils/generator/video_source_metadata.py create mode 100644 cosmos_framework/utils/one_logger/one_logger_utils_test.py diff --git a/.file_mapping.json b/.file_mapping.json index 9cbaf068d..b96ddff35 100644 --- a/.file_mapping.json +++ b/.file_mapping.json @@ -1,7 +1,7 @@ { - "_source_commit": "4d9b6cfd3731fdbbca883184937d40a64d2dca52-dirty", - "_dest_commit": "c23e51f2f157ae3e51cfcd86ebfb5464850894f2", - "_generated_at": "2026-09-20T05:50:07Z", + "_source_commit": "461431bae40d4a8ccf27d812fc9d1757fe8d3b96-dirty", + "_dest_commit": "0460be81f16883aa380e716dc6f58c1189481172", + "_generated_at": "2026-09-23T04:47:38Z", "files": { "imaginaire/__init__.py": "cosmos_framework/__init__.py", "imaginaire/attention/__init__.py": "cosmos_framework/model/attention/__init__.py", @@ -169,6 +169,7 @@ "imaginaire/utils/one_logger/one_logger_global_vars.py": "cosmos_framework/utils/one_logger/one_logger_global_vars.py", "imaginaire/utils/one_logger/one_logger_override_utils.py": "cosmos_framework/utils/one_logger/one_logger_override_utils.py", "imaginaire/utils/one_logger/one_logger_utils.py": "cosmos_framework/utils/one_logger/one_logger_utils.py", + "imaginaire/utils/one_logger/one_logger_utils_test.py": "cosmos_framework/utils/one_logger/one_logger_utils_test.py", "imaginaire/utils/optim_instantiate.py": "cosmos_framework/utils/optim_instantiate.py", "imaginaire/utils/profiling.py": "cosmos_framework/utils/profiling.py", "imaginaire/utils/progress_bar.py": "cosmos_framework/utils/progress_bar.py", @@ -292,20 +293,6 @@ "projects/cosmos3/cosmos3/datasets/augmentors/cropping.py": "cosmos_framework/data/generator/augmentors/cropping.py", "projects/cosmos3/cosmos3/datasets/augmentors/duration_fps_text_timestamps.py": "cosmos_framework/data/generator/augmentors/duration_fps_text_timestamps.py", "projects/cosmos3/cosmos3/datasets/augmentors/duration_fps_text_timestamps_test.py": "cosmos_framework/data/generator/augmentors/duration_fps_text_timestamps_test.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/__init__.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/__init__.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/augmentor.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/augmentor.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/augmentor_test.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/augmentor_test.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/bench.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/bench.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/codec.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/codec.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/contact_sheet.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/contact_sheet.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/contact_sheet_test.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/contact_sheet_test.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/degrade.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/degrade.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/degrade_test.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/degrade_test.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/diffjpeg.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/diffjpeg.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/kernels.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/kernels.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/ops.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/ops.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/packing_test.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/packing_test.py", - "projects/cosmos3/cosmos3/datasets/augmentors/hr_lr_degradation/profiles.py": "cosmos_framework/data/generator/augmentors/hr_lr_degradation/profiles.py", "projects/cosmos3/cosmos3/datasets/augmentors/idle_frames_text_info.py": "cosmos_framework/data/generator/augmentors/idle_frames_text_info.py", "projects/cosmos3/cosmos3/datasets/augmentors/image_editing_transform.py": "cosmos_framework/data/generator/augmentors/image_editing_transform.py", "projects/cosmos3/cosmos3/datasets/augmentors/image_editing_transform_test.py": "cosmos_framework/data/generator/augmentors/image_editing_transform_test.py", @@ -326,7 +313,9 @@ "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/format_hot_fixes.py": "cosmos_framework/data/generator/augmentors/reasoner/format_hot_fixes.py", "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/nvlm_data_to_conversation.py": "cosmos_framework/data/generator/augmentors/reasoner/nvlm_data_to_conversation.py", "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/prompt_format.py": "cosmos_framework/data/generator/augmentors/reasoner/prompt_format.py", + "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/prompt_format_test.py": "cosmos_framework/data/generator/augmentors/reasoner/prompt_format_test.py", "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/shuffle_text_media_order.py": "cosmos_framework/data/generator/augmentors/reasoner/shuffle_text_media_order.py", + "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/source_timestamps_test.py": "cosmos_framework/data/generator/augmentors/reasoner/source_timestamps_test.py", "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/timestamp.py": "cosmos_framework/data/generator/augmentors/reasoner/timestamp.py", "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/timestamp_test.py": "cosmos_framework/data/generator/augmentors/reasoner/timestamp_test.py", "projects/cosmos3/cosmos3/datasets/augmentors/reasoner/timestamp_with_subject_tracking.py": "cosmos_framework/data/generator/augmentors/reasoner/timestamp_with_subject_tracking.py", @@ -488,9 +477,11 @@ "projects/cosmos3/cosmos3/models/utils/sr_latent_noise_test.py": "cosmos_framework/model/generator/utils/sr_latent_noise_test.py", "projects/cosmos3/cosmos3/models/vision_encoder.py": "cosmos_framework/model/generator/vision_encoder.py", "projects/cosmos3/cosmos3/models/vlm_model.py": "cosmos_framework/model/generator/vlm_model.py", + "projects/cosmos3/cosmos3/processors/VIDEO_TIMESTAMPS.md": "cosmos_framework/data/generator/processors/VIDEO_TIMESTAMPS.md", "projects/cosmos3/cosmos3/processors/__init__.py": "cosmos_framework/data/generator/processors/__init__.py", "projects/cosmos3/cosmos3/processors/audio_utils.py": "cosmos_framework/data/generator/processors/audio_utils.py", "projects/cosmos3/cosmos3/processors/base.py": "cosmos_framework/data/generator/processors/base.py", + "projects/cosmos3/cosmos3/processors/base_test.py": "cosmos_framework/data/generator/processors/base_test.py", "projects/cosmos3/cosmos3/processors/cosmos3_edge_processing.py": "cosmos_framework/data/generator/processors/cosmos3_edge_processing.py", "projects/cosmos3/cosmos3/processors/cosmos3_edge_processing_test.py": "cosmos_framework/data/generator/processors/cosmos3_edge_processing_test.py", "projects/cosmos3/cosmos3/processors/nemotron3densevl_processor.py": "cosmos_framework/data/generator/processors/nemotron3densevl_processor.py", @@ -504,6 +495,7 @@ "projects/cosmos3/cosmos3/processors/qwen3vl_nemo_chat_processor.py": "cosmos_framework/data/generator/processors/qwen3vl_nemo_chat_processor.py", "projects/cosmos3/cosmos3/processors/qwen3vl_nemo_chat_processor_test.py": "cosmos_framework/data/generator/processors/qwen3vl_nemo_chat_processor_test.py", "projects/cosmos3/cosmos3/processors/qwen3vl_processor.py": "cosmos_framework/data/generator/processors/qwen3vl_processor.py", + "projects/cosmos3/cosmos3/processors/source_video_timing_test.py": "cosmos_framework/data/generator/processors/source_video_timing_test.py", "projects/cosmos3/cosmos3/scripts/multiview_auto/multiview_collage.py": "cosmos_framework/scripts/multiview_collage.py", "projects/cosmos3/cosmos3/scripts/multiview_auto/multiview_collage_test.py": "cosmos_framework/scripts/multiview_collage_test.py", "projects/cosmos3/cosmos3/sequence_packing/__init__.py": "cosmos_framework/data/generator/sequence_packing/__init__.py", @@ -576,8 +568,10 @@ "projects/cosmos3/cosmos3/utils/reasoner/pretrained_models_downloader.py": "cosmos_framework/utils/generator/reasoner/pretrained_models_downloader.py", "projects/cosmos3/cosmos3/utils/reasoner/pretrained_models_downloader_test.py": "cosmos_framework/utils/generator/reasoner/pretrained_models_downloader_test.py", "projects/cosmos3/cosmos3/utils/reasoner/true_packing.py": "cosmos_framework/utils/generator/reasoner/true_packing.py", + "projects/cosmos3/cosmos3/utils/source_video_timing.py": "cosmos_framework/utils/generator/source_video_timing.py", "projects/cosmos3/cosmos3/utils/video_frame_sampling.py": "cosmos_framework/utils/generator/video_frame_sampling.py", "projects/cosmos3/cosmos3/utils/video_preprocess.py": "cosmos_framework/utils/generator/video_preprocess.py", + "projects/cosmos3/cosmos3/utils/video_source_metadata.py": "cosmos_framework/utils/generator/video_source_metadata.py", "projects/cosmos3/interactive/configs/defaults/flex_attention.py": "cosmos_framework/configs/base/defaults/causal_flex_attention.py", "projects/cosmos3/interactive/configs/defaults/replay_attention.py": "cosmos_framework/configs/base/defaults/replay_attention.py", "projects/cosmos3/interactive/models/attention_io_layout.py": "cosmos_framework/model/generator/attention_io_layout.py", diff --git a/cosmos_framework/configs/base/defaults/multiview_attention.py b/cosmos_framework/configs/base/defaults/multiview_attention.py index 3b3c7cb55..89b71d2fb 100644 --- a/cosmos_framework/configs/base/defaults/multiview_attention.py +++ b/cosmos_framework/configs/base/defaults/multiview_attention.py @@ -78,8 +78,11 @@ def resolve_caption_scope(access: CaptionAccess, *, per_view_captions: bool) -> # ``decomposed_temporal_window_seconds`` is set: the two streams do not share a frame # index, but they do share real capture time, which the window compares instead. # -# Read by the ``flex_*`` backends only. The ``"maskless"`` backend is its own attention pattern -# and does not take a scope -- see ``BackendPreference``. +# Read by every backend, but not the same way. A ``flex_*`` backend expresses the scope as a mask. +# The ``"maskless"`` backend expresses ``"same_view"`` and ``"decomposed"`` as partitions of the +# GEN stream and refuses ``"all_views"``, which is not a partition at all -- so there the scope +# decides whether the cross-instant pass exists rather than describing one attention two ways. See +# ``BackendPreference`` and ``models.mot.multiview_maskless_attention.MASKLESS_ATTENTION_SCOPES``. AttentionScope = Literal["all_views", "same_view", "decomposed"] # The scopes of ``AttentionScope`` at runtime, which the annotation itself is not. diff --git a/cosmos_framework/configs/base/reasoner/defaults/augmentors.py b/cosmos_framework/configs/base/reasoner/defaults/augmentors.py index 24553ad23..bd498701f 100644 --- a/cosmos_framework/configs/base/reasoner/defaults/augmentors.py +++ b/cosmos_framework/configs/base/reasoner/defaults/augmentors.py @@ -37,6 +37,7 @@ def create_data_augmentor_config() -> dict[str, Any]: max_fps_thres=60, target_fps="${data_setting.qwen_target_fps}", # type: ignore video_temporal_mode="${data_setting.qwen_video_temporal_mode}", + video_timestamp_mode="${data_setting.video_timestamp_mode}", max_video_token_length="${data_setting.qwen_max_video_token_length}", # type: ignore processor=processor, extract_audio="${model.config.sound_und}", @@ -45,6 +46,7 @@ def create_data_augmentor_config() -> dict[str, Any]: "prompt_format": L(PromptFormat)( # takes text_keys and output "conversation" input_keys=["texts"], text_chat_order="${data_setting.text_chat_order}", + strip_thinking_prob="${data_setting.strip_thinking_prob}", ), "shuffle_text_media_order": L(ShuffleTextMediaOrder)(), "format_hot_fixes": L(FormatHotFixes)(), @@ -130,6 +132,7 @@ def create_data_augmentor_config() -> dict[str, Any]: custom_system_prompt="${data_setting.custom_system_prompt}", strip_original_system_prompt="${data_setting.strip_original_system_prompt}", video_temporal_mode="${data_setting.qwen_video_temporal_mode}", + video_timestamp_mode="${data_setting.video_timestamp_mode}", text_only=False, sound_und="${model.config.sound_und}", audio_encoder_type="${model.config.sound_und_config.audio_encoder_type}", @@ -168,6 +171,7 @@ def create_data_augmentor_config() -> dict[str, Any]: processor = L(build_processor_lazy)( tokenizer_type="${model.config.policy.backbone.model_name}", + use_native_edge_processor="${data_setting.use_native_edge_processor}", credentials="${checkpoint.load_from_object_store.credentials}", bucket="${checkpoint.load_from_object_store.bucket}", ) diff --git a/cosmos_framework/configs/base/reasoner/defaults/config.py b/cosmos_framework/configs/base/reasoner/defaults/config.py index 8d7524d78..dc0bec2a1 100644 --- a/cosmos_framework/configs/base/reasoner/defaults/config.py +++ b/cosmos_framework/configs/base/reasoner/defaults/config.py @@ -17,6 +17,7 @@ class DataSetting: qwen_max_video_token_length: Maximum video token length. qwen_target_fps: Target fps for video sampling. text_chat_order: Order of text items in user messages. + strip_thinking_prob: Per-sample probability of converting thinking data into non-thinking data. custom_system_prompt: System prompt injected when a conversation has no leading system message. strip_original_system_prompt: Remove existing system messages before optional custom prompt injection. distributor_type: "with_replace" (WeightedShardlistBasic) or "no_replace" (NoReplaceShardlistBasic). @@ -30,6 +31,11 @@ class DataSetting: qwen_max_video_token_length: int = 8192 qwen_max_image_token_length: int = 8192 qwen_target_fps: float = 4.0 + use_native_edge_processor: bool = False + video_timestamp_mode: str = attrs.field( + default="qwen_index", + validator=attrs.validators.in_({"qwen_index", "legacy_fps", "source_pts"}), + ) qwen_video_temporal_mode: str = attrs.field( default="native", validator=attrs.validators.in_({"native", "framewise"}) ) @@ -38,6 +44,14 @@ class DataSetting: default="text_end", validator=attrs.validators.in_({"text_end", "text_start", "random"}), ) + strip_thinking_prob: float = attrs.field( + default=0.0, + validator=attrs.validators.and_( + attrs.validators.instance_of((int, float)), + attrs.validators.ge(0.0), + attrs.validators.le(1.0), + ), + ) custom_system_prompt: str | None = "You are a helpful assistant." strip_original_system_prompt: bool = False temporal_localization_output_format: str = attrs.field( diff --git a/cosmos_framework/data/generator/action/utils/domain_utils.py b/cosmos_framework/data/generator/action/utils/domain_utils.py index ad56547ff..4d00f36ed 100644 --- a/cosmos_framework/data/generator/action/utils/domain_utils.py +++ b/cosmos_framework/data/generator/action/utils/domain_utils.py @@ -54,6 +54,10 @@ # RoboCasa PandaOmron mobile manipulation (10/15/20D raw action per # ``use_base_action`` / ``base_encoding``); appended above the maximum. "robocasa": 30, + # embodiment_b nvidia-20260828 ingestion: a new one-shot dataset, distinct from + # "embodiment_b" (domain 9, an earlier unrelated sample drop with its own 30D + # contract). + "embodiment_b_20260828": 32, } @@ -88,6 +92,7 @@ "so101-bimanual-midtrain-conditional": 20, "geniesim3_g2a": 29, "geniesim3_g2a_joint": 16, + "embodiment_b_20260828": 50, # NOTE: ``libero`` (7/10/13 depending on ``rotation_space``), ``hand_pose`` # (variable with ``keypoint_option`` and ``rotation_format``) and ``robocasa`` # (10 arm-only, 15/20 with the mobile base, per ``use_base_action`` / diff --git a/cosmos_framework/data/generator/augmentor_provider.py b/cosmos_framework/data/generator/augmentor_provider.py index 7fc757b65..d2d6b6e4a 100644 --- a/cosmos_framework/data/generator/augmentor_provider.py +++ b/cosmos_framework/data/generator/augmentor_provider.py @@ -1543,112 +1543,3 @@ def _insert_before(augmentors: dict, anchor_keys: tuple[str, ...], new_key: str, return _insert_relative(augmentors, anchor, new_key, new_value, after=False) -def _insert_low_res_stage(augmentors: dict, add_low_res) -> dict: - """Place ``AddLowRes`` so the LR is derived from exactly the HR frame the model will see. - - - Reflection-padding path (causal VAE): LR is made from the *unpadded* frame, before ``reflection_padding``; - ``SRToTrainingFormat`` pads LR separately to half the HR bucket, so LR and HR stay aligned at the top-left. - - Crop path (non-causal / UniAE, ``crop_to_multiple``): LR is made *after* the centre crop. Making it before - would derive LR from pixels the HR no longer contains (spatial misalignment) and, when the crop changes the - size, a larger LR than the target that ``SRToTrainingFormat`` cannot pad down. - """ - if "reflection_padding" in augmentors: - return _insert_relative(augmentors, "reflection_padding", "add_low_res", add_low_res, after=False) - if "crop_to_multiple" in augmentors: - return _insert_relative(augmentors, "crop_to_multiple", "add_low_res", add_low_res, after=True) - raise KeyError("Pipeline has neither reflection_padding nor crop_to_multiple; cannot place add_low_res") - - -@augmentor_register("video_basic_augmentor_v3_json_caption_sr") -def get_video_augmentor_v3_json_caption_sr( - resolution: str, - sr_scale: float = 2.0, - sr_profiles: dict[str, float] | str = "p1_first_order", - sr_seed_salt: str = "", - sr_chunk_frames: int = 8, - sr_device: str = "cpu", - sr_jpeg_backend: str = "auto", - sr_poisson_mode: str = "auto", - sr_share_vision_temporal_positions: bool = False, - **kwargs: object, -) -> dict[str, object]: - """``video_basic_augmentor_v3_json_caption`` plus an on-the-fly HR-to-LR conditioning stream. - - Adds ``AddLowRes`` (writes ``video_lr`` at ``1/sr_scale`` of the HR frame, uint8) right before - reflection padding, and ``SRToTrainingFormat`` as the last stage, which pads LR to half the HR - bucket and packs ``video = [lr, hr]`` with per-item ``image_size`` and a two-item SequencePlan. - All other stages (caption, chunked decode, sequence plan, sound) are inherited unchanged. - """ - from cosmos_framework.data.generator.augmentors.hr_lr_degradation import augmentor as sr_augmentor - - augmentors = get_video_augmentor_v3_json_caption(resolution=resolution, **kwargs) - add_low_res = L(sr_augmentor.AddLowRes)( - input_keys=["video"], - output_keys=["video_lr"], - args={ - "scale": sr_scale, - "profiles": sr_profiles, - "seed_salt": sr_seed_salt, - "modality": "video", - "chunk_frames": sr_chunk_frames, - "device": sr_device, - "jpeg_backend": sr_jpeg_backend, - "poisson_mode": sr_poisson_mode, - }, - ) - augmentors = _insert_low_res_stage(augmentors, add_low_res) - augmentors["sr_to_training_format"] = L(sr_augmentor.SRToTrainingFormat)( - input_keys=["video", "video_lr"], - args={ - "media_key": "video", - "lr_key": "video_lr", - "scale": sr_scale, - "share_vision_temporal_positions": sr_share_vision_temporal_positions, - "dataset_name": "video_sr", - }, - ) - return augmentors - - -@augmentor_register("image_basic_augmentor_with_tokenization_sr") -def image_basic_augmentor_with_tokenization_sr( - resolution: str, - sr_scale: float = 2.0, - sr_profiles: dict[str, float] | str = "p1_first_order", - sr_seed_salt: str = "", - sr_device: str = "cpu", - sr_jpeg_backend: str = "auto", - sr_poisson_mode: str = "auto", - **kwargs: object, -) -> dict[str, object]: - """``image_basic_augmentor_with_tokenization`` plus an on-the-fly HR-to-LR conditioning image. - - ``AddLowRes`` runs before ``reflection_padding`` (on the resized uint8 image), the LR copy gets - its own ``Normalize`` so both items reach the model as float in [-1, 1], and - ``SRToTrainingFormat`` packs ``images = [lr, hr]`` with per-item ``image_size``. - """ - from cosmos_framework.data.generator.augmentors.hr_lr_degradation import augmentor as sr_augmentor - - augmentors = image_basic_augmentor_with_tokenization(resolution=resolution, **kwargs) - add_low_res = L(sr_augmentor.AddLowRes)( - input_keys=["images"], - output_keys=["images_lr"], - args={ - "scale": sr_scale, - "profiles": sr_profiles, - "seed_salt": sr_seed_salt, - "modality": "image", - "chunk_frames": 1, - "device": sr_device, - "jpeg_backend": sr_jpeg_backend, - "poisson_mode": sr_poisson_mode, - }, - ) - augmentors = _insert_before(augmentors, ("reflection_padding",), "add_low_res", add_low_res) - normalize_lr = L(normalize.Normalize)(input_keys=["images_lr"], args={"mean": 0.5, "std": 0.5}) - augmentors = _insert_before(augmentors, ("text_transform",), "normalize_lr", normalize_lr) - augmentors["sr_to_training_format"] = L(sr_augmentor.SRToTrainingFormat)( - input_keys=["images", "images_lr"], - args={"media_key": "images", "lr_key": "images_lr", "scale": sr_scale, "dataset_name": "image_sr"}, - ) - return augmentors diff --git a/cosmos_framework/data/generator/augmentors/reasoner/bytes_to_media.py b/cosmos_framework/data/generator/augmentors/reasoner/bytes_to_media.py index e0545a6b1..5d9d6fdf0 100644 --- a/cosmos_framework/data/generator/augmentors/reasoner/bytes_to_media.py +++ b/cosmos_framework/data/generator/augmentors/reasoner/bytes_to_media.py @@ -21,7 +21,14 @@ from cosmos_framework.utils import log from cosmos_framework.data.generator.reasoner.video_decoder_qwen import VideoTemporalMode, _video_decoder_qwen_func from cosmos_framework.data.generator.processors.qwen3vl_processor import Qwen3VLProcessor +from cosmos_framework.utils.generator.source_video_timing import ( + SOURCE_VIDEO_TIMING_KEY, + require_source_pts_processor, + validate_source_video_timing, + validate_video_timestamp_mode, +) from cosmos_framework.utils.generator.video_preprocess import tensor_to_pil_images +from cosmos_framework.utils.generator.video_source_metadata import VIDEO_METADATA_KEY class BytesToMedia(Augmentor): @@ -55,6 +62,7 @@ def __init__( processor: Qwen3VLProcessor = None, extract_audio: bool = False, audio_sample_rate: int = 16_000, + video_timestamp_mode: str = "qwen_index", video_temporal_mode: VideoTemporalMode = "native", ) -> None: """ @@ -74,6 +82,12 @@ def __init__( extract_audio (bool): Whether to decode the audio stream from video containers. audio_sample_rate (int): Target sample rate for decoded mono audio. """ + validate_video_timestamp_mode(video_timestamp_mode) + self.video_timestamp_mode: str = video_timestamp_mode + if video_timestamp_mode == "source_pts": + require_source_pts_processor(processor) + if extract_audio: + raise ValueError("source_pts does not support audio extraction") self.input_key = input_key self.output_key = output_key if video_temporal_mode not in ("native", "framewise"): @@ -94,6 +108,7 @@ def __init__( self.processor = processor self.extract_audio = extract_audio self.audio_sample_rate = audio_sample_rate + self.video_decoder_params["video_timestamp_mode"] = video_timestamp_mode def _is_video_key(self, name: str) -> bool: """Returns whether the media key will be decoded as video.""" @@ -236,11 +251,17 @@ def _bytes_to_video_frames( ), ) if result is None: + if self.video_timestamp_mode == "source_pts": + raise ValueError("source_pts decoder returned no frames or timing record") log.warning(f"Skipping item '{identifier}': Video decoder returned None.") return None + if self.video_timestamp_mode == "source_pts": + validate_source_video_timing(result.get(SOURCE_VIDEO_TIMING_KEY), result["videos"].shape[1]) result["videos"] = tensor_to_pil_images(result["videos"]) # 3,T,H,W -> list of PIL images return result except Exception as e: + if self.video_timestamp_mode == "source_pts": + raise ValueError(f"source_pts failed to decode and align video {identifier!r}") from e log.warning(f"Skipping item '{identifier}': Error decoding video bytes: {e}") return None @@ -303,6 +324,8 @@ def __call__(self, data_dict: Dict) -> Dict: output_data = {} if isinstance(data, dict): + if self.video_timestamp_mode == "source_pts" and any(self._is_audio_key(name) for name in data): + raise ValueError("source_pts does not support audio media") video_count = sum(1 for name, item in data.items() if isinstance(item, bytes) and self._is_video_key(name)) video_durations = self._get_video_durations(data, data_dict) if video_count > 1 else {} total_video_duration = sum(video_durations.values()) if video_durations else None @@ -352,6 +375,8 @@ def __call__(self, data_dict: Dict) -> Dict: ) if audio is not None: result["audio"] = audio + if VIDEO_METADATA_KEY in result: + result["audio_start_seconds"] = start_frame / result[VIDEO_METADATA_KEY]["fps"] output_data[name] = result elif ( diff --git a/cosmos_framework/data/generator/augmentors/reasoner/prompt_format.py b/cosmos_framework/data/generator/augmentors/reasoner/prompt_format.py index 717803705..631d01d58 100644 --- a/cosmos_framework/data/generator/augmentors/reasoner/prompt_format.py +++ b/cosmos_framework/data/generator/augmentors/reasoner/prompt_format.py @@ -4,26 +4,34 @@ """Visual-Text Transformations or Augmentations.""" import random -from typing import Dict, Literal +import re +from typing import Any, Literal from cosmos_framework.data.imaginaire.webdataset.augmentors.augmentor import Augmentor +_THINK_RE = re.compile(r".*?\s*|.*\Z", re.DOTALL) + class PromptFormat(Augmentor): def __init__( self, - input_keys: list = ["texts"], + input_keys: list[str] = ["texts"], text_chat_order: Literal["text_end", "text_start", "random"] = "text_end", + strip_thinking_prob: float = 0.0, ) -> None: """ Args: - input_keys (list): List of input keys. - text_chat_order (Literal["text_end", "text_start", "random"]): Order of text items in user messages. + input_keys: List of input keys. + text_chat_order: Order of text items in user messages. + strip_thinking_prob: Per-sample probability of dropping assistant thinking traces. """ + if not 0.0 <= strip_thinking_prob <= 1.0: + raise ValueError(f"strip_thinking_prob must be in [0, 1], got {strip_thinking_prob}") self.input_keys = input_keys self.text_chat_order = text_chat_order + self.strip_thinking_prob = strip_thinking_prob - def __call__(self, data_dict: Dict) -> Dict: + def __call__(self, data_dict: dict[str, Any]) -> dict[str, Any] | None: conversation_key = self.input_keys[0] # retrive conversations from dict @@ -56,16 +64,48 @@ def __call__(self, data_dict: Dict) -> Dict: if "reasoning_content" in message and isinstance(message["reasoning_content"], str): message["reasoning_content"] = [{"type": "text", "text": message["reasoning_content"]}] - # Merge reasoning_content into assistant message content - for message in selected_conversation: - if message.get("role") == "assistant" and message.get("reasoning_content"): - # Wrap reasoning items in ... tags - reasoning_items = message["reasoning_content"] - think_start = [{"type": "text", "text": "\n"}] - think_end = [{"type": "text", "text": "\n\n\n"}] - message["content"] = think_start + reasoning_items + think_end + message["content"] - del message["reasoning_content"] - + is_thinking_stripped = False + if random.random() < self.strip_thinking_prob: + for message in selected_conversation: + if message.get("role") != "assistant": + continue + if message.pop("reasoning_content", None): + is_thinking_stripped = True + content = message.get("content", []) + for item in content: + if not isinstance(item, dict) or item.get("type") != "text": + continue + text = item.get("text") + if not isinstance(text, str): + continue + stripped_text = _THINK_RE.sub("", text).lstrip() + if stripped_text != text: + is_thinking_stripped = True + item["text"] = stripped_text + has_text = any( + isinstance(item, dict) + and item.get("type") == "text" + and isinstance(item.get("text"), str) + and item["text"].strip() + for item in content + ) + has_media = any( + isinstance(item, dict) and item.get("type") in ("image", "video", "audio") for item in content + ) + if not has_text and not has_media: + return None + else: + # Merge reasoning_content into assistant message content + for message in selected_conversation: + if message.get("role") == "assistant" and message.get("reasoning_content"): + # Wrap reasoning items in ... tags + reasoning_items = message["reasoning_content"] + think_start = [{"type": "text", "text": "\n"}] + think_end = [{"type": "text", "text": "\n\n\n"}] + message["content"] = think_start + reasoning_items + think_end + message["content"] + del message["reasoning_content"] + + data_dict["is_thinking_stripped"] = is_thinking_stripped data_dict["conversation"] = selected_conversation del data_dict[conversation_key] @@ -75,7 +115,7 @@ def __call__(self, data_dict: Dict) -> Dict: return data_dict - def _enforce_text_chat_order(self, conversation: list) -> None: + def _enforce_text_chat_order(self, conversation: list[dict[str, Any]]) -> None: """ Reorder text content within user messages based on text_chat_order setting. NOTE (maxzhaoshuol): this does NOT work for interleaved data!!!!!! diff --git a/cosmos_framework/data/generator/augmentors/reasoner/prompt_format_test.py b/cosmos_framework/data/generator/augmentors/reasoner/prompt_format_test.py new file mode 100644 index 000000000..9e89da57c --- /dev/null +++ b/cosmos_framework/data/generator/augmentors/reasoner/prompt_format_test.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: OpenMDW-1.1 + +import pytest + +from cosmos_framework.data.generator.augmentors.reasoner.prompt_format import PromptFormat + +pytestmark = [pytest.mark.L0, pytest.mark.CPU] + + +def test_strip_thinking_removes_reasoning_and_inline_trace() -> None: + formatter = PromptFormat(strip_thinking_prob=1.0) + + result = formatter( + { + "texts": [ + {"role": "user", "content": "Question"}, + { + "role": "assistant", + "reasoning_content": "Hidden reasoning", + "content": "Inline reasoning\nFinal answer", + }, + ] + } + ) + + assert result is not None + assert result["is_thinking_stripped"] is True + assert result["conversation"][0]["content"] == [{"type": "text", "text": "Question"}] + assert result["conversation"][1]["content"] == [{"type": "text", "text": "Final answer"}] + assert "reasoning_content" not in result["conversation"][1] + + +def test_zero_probability_preserves_thinking_without_prompt_injection() -> None: + formatter = PromptFormat(strip_thinking_prob=0.0) + + result = formatter( + { + "texts": [ + {"role": "user", "content": "Question"}, + {"role": "assistant", "reasoning_content": "Reasoning", "content": "Final answer"}, + ] + } + ) + + assert result is not None + assert result["is_thinking_stripped"] is False + assert result["conversation"][0]["content"] == [{"type": "text", "text": "Question"}] + assert result["conversation"][1]["content"] == [ + {"type": "text", "text": "\n"}, + {"type": "text", "text": "Reasoning"}, + {"type": "text", "text": "\n\n\n"}, + {"type": "text", "text": "Final answer"}, + ] + + +def test_strip_thinking_drops_sample_without_assistant_supervision() -> None: + formatter = PromptFormat(strip_thinking_prob=1.0) + + result = formatter( + { + "texts": [ + {"role": "user", "content": "Question"}, + {"role": "assistant", "reasoning_content": "Only reasoning", "content": ""}, + ] + } + ) + + assert result is None + + +@pytest.mark.parametrize("probability", [-0.1, 1.1]) +def test_strip_thinking_rejects_invalid_probability(probability: float) -> None: + with pytest.raises(ValueError, match=r"must be in \[0, 1\]"): + PromptFormat(strip_thinking_prob=probability) diff --git a/cosmos_framework/data/generator/augmentors/reasoner/source_timestamps_test.py b/cosmos_framework/data/generator/augmentors/reasoner/source_timestamps_test.py new file mode 100644 index 000000000..5df1e78b1 --- /dev/null +++ b/cosmos_framework/data/generator/augmentors/reasoner/source_timestamps_test.py @@ -0,0 +1,243 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: OpenMDW-1.1 + +"""Training labels and paired cropped audio share the video source clock.""" + +from types import SimpleNamespace +from typing import Any + +import numpy as np +import pytest +from PIL import Image + +from cosmos_framework.data.generator.augmentors.reasoner.timestamp import overlay_text +from cosmos_framework.data.generator.augmentors.reasoner.tokenize_data import TokenizeData +from cosmos_framework.data.generator.augmentors.reasoner.tokenize_data_test import ( + _FakeAudioProcessor, + _FakeVLMProcessor, +) +from cosmos_framework.data.generator.processors.audio_utils import ( + AUDIO_END_TOKEN, + AUDIO_PAD_TOKEN, + AUDIO_START_TOKEN, + get_audio_segment_token_lengths, +) +from cosmos_framework.data.generator.processors.base import maybe_parse_video_content +from cosmos_framework.utils.generator.video_source_metadata import calculate_video_timestamps + +pytestmark = [pytest.mark.L1, pytest.mark.CPU] + + +@pytest.mark.parametrize("temporal_patch_size,expected", [(1, [1.0, 1.3, 1.7]), (2, [1.2, 1.2, 1.7])]) +def test_overlay_uses_temporal_patch_size_not_spatial_merge(temporal_patch_size: int, expected: list[float]) -> None: + frames = [Image.new("RGB", (64, 64))] * 3 + processor = SimpleNamespace( + name="/local/edge", temporal_patch_size=temporal_patch_size, merge_size=4, USES_SOURCE_VIDEO_TIMESTAMPS=True + ) + result, times = overlay_text( + frames, + 4.0, + processor=processor, + video_metadata={"fps": 30.0, "total_num_frames": 100, "frames_indices": [30, 40, 50]}, + ) + assert result is frames + assert times == expected + + +def test_cropped_audio_uses_same_origin_and_token_partition() -> None: + processor = _FakeVLMProcessor() + tokenize = TokenizeData( + processor=processor, + sound_und=True, + audio_processor=_FakeAudioProcessor(token_lengths=(10,), timestamp_stride=0.11), + ) + metadata = {"fps": 10.0, "total_num_frames": 100, "frames_indices": [50, 52, 56, 58]} + data = { + "__key__": "crop", + "__url__": SimpleNamespace(root="test", path="crop"), + "media": { + "video": { + "videos": [Image.new("RGB", (32, 32))] * 4, + "fps": 4.0, + "video_metadata": metadata, + "audio": np.zeros(16000, dtype=np.float32), + "audio_start_seconds": 5.0, + } + }, + "conversation": [ + {"role": "user", "content": [{"type": "video", "video": "video"}, {"type": "audio", "audio": "video"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "answer"}]}, + ], + } + assert tokenize(data) is not None + content = processor.last_conversation[0]["content"] + assert content[0]["video_metadata"] == metadata + assert ( + content[1]["text"] + == f"{AUDIO_START_TOKEN}<5.1 seconds>{AUDIO_PAD_TOKEN * 4}<5.7 seconds>{AUDIO_PAD_TOKEN * 6}{AUDIO_END_TOKEN}" + ) + + +@pytest.mark.parametrize( + "metadata,mode", + [ + (None, "qwen_index"), + ({"fps": 30.0, "total_num_frames": 100, "frames_indices": [0]}, "qwen_index"), + ({"fps": 30.0, "total_num_frames": 100, "frames_indices": [0, 1, 2, 3]}, "legacy_fps"), + ], +) +def test_tokenize_rejects_invalid_explicit_source_metadata(metadata: object, mode: str) -> None: + processor = _FakeVLMProcessor() + data = { + "__key__": "invalid", + "__url__": SimpleNamespace(root="test", path="invalid"), + "media": {"video": {"videos": [Image.new("RGB", (32, 32))] * 4, "fps": 4.0, "video_metadata": metadata}}, + "conversation": [{"role": "user", "content": [{"type": "video", "video": "video"}]}], + } + with pytest.raises(ValueError, match="video_metadata"): + TokenizeData(processor=processor, video_timestamp_mode=mode)(data) + + +def test_repeated_source_frames_partition_audio_without_losing_tokens() -> None: + assert get_audio_segment_token_lengths(5, [1.0, 1.0, 1.2], audio_token_timestamps=[1.0, 1.05, 1.1, 1.15, 1.2]) == [ + 1, + 2, + 2, + ] + with pytest.raises(ValueError, match="nondecreasing"): + get_audio_segment_token_lengths(5, [1.2, 1.0]) + + +def _multi_video_audio_sample(content_order: list[tuple[str, str]]) -> dict[str, Any]: + media = {} + for key, start, value in (("video_a", 50, 2.0), ("video_b", 200, 7.0)): + media[key] = { + "videos": [Image.new("RGB", (32, 32))] * 4, + "fps": 4.0, + "video_metadata": { + "fps": 10.0, + "total_num_frames": 300, + "frames_indices": [start + index for index in (0, 2, 6, 8)], + }, + "audio": np.full(320, value, dtype=np.float32), + "audio_start_seconds": start / 10.0, + } + return { + "__key__": "multi-video-crops", + "__url__": SimpleNamespace(root="test", path="multi-video-crops"), + "media": media, + "conversation": [ + {"role": "user", "content": [{"type": kind, kind: key} for kind, key in content_order]}, + {"role": "assistant", "content": [{"type": "text", "text": "answer"}]}, + ], + } + + +def test_multi_video_audio_uses_its_own_source_clock() -> None: + processor = _FakeVLMProcessor() + data = _multi_video_audio_sample( + [("video", "video_a"), ("video", "video_b"), ("audio", "video_a"), ("audio", "video_b")] + ) + output = TokenizeData( + processor=processor, + sound_und=True, + audio_processor=_FakeAudioProcessor(token_lengths=(10, 10), timestamp_stride=0.11), + )(data) + assert output is not None + content = processor.last_conversation[0]["content"] + for item, start in zip(content[2:], (5, 20), strict=True): + assert item["text"] == ( + f"{AUDIO_START_TOKEN}<{start}.1 seconds>{AUDIO_PAD_TOKEN * 4}" + f"<{start}.7 seconds>{AUDIO_PAD_TOKEN * 6}{AUDIO_END_TOKEN}" + ) + assert output["audio_features"][:, 0, 0].tolist() == [2.0, 7.0] + + +@pytest.mark.parametrize("audio_layout", ["separate_with_timestamps", "separate_no_timestamps", "interleaved_av"]) +def test_audio_cannot_borrow_another_video_clock_before_its_own_video(audio_layout: str) -> None: + processor = _FakeVLMProcessor() + data = _multi_video_audio_sample([("video", "video_b"), ("audio", "video_a"), ("video", "video_a")]) + output = TokenizeData( + processor=processor, + sound_und=True, + audio_layout=audio_layout, + audio_processor=_FakeAudioProcessor(token_lengths=(10,)), + )(data) + assert output is None + assert processor.last_conversation is None + + +def test_interleaved_audio_cannot_attach_to_an_unrelated_adjacent_video() -> None: + processor = _FakeVLMProcessor() + data = _multi_video_audio_sample([("video", "video_a"), ("video", "video_b"), ("audio", "video_a")]) + output = TokenizeData( + processor=processor, + sound_und=True, + audio_layout="interleaved_av", + audio_processor=_FakeAudioProcessor(token_lengths=(10,)), + )(data) + assert output is None + assert processor.last_conversation is None + + +def test_interleaved_cropped_pairs_keep_each_clock_and_audio_feature_order() -> None: + processor = _FakeVLMProcessor() + data = _multi_video_audio_sample( + [("video", "video_a"), ("audio", "video_a"), ("video", "video_b"), ("audio", "video_b")] + ) + output = TokenizeData( + processor=processor, + sound_und=True, + audio_layout="interleaved_av", + audio_processor=_FakeAudioProcessor(token_lengths=(10, 10), timestamp_stride=0.11), + )(data) + assert output is not None + tokens = processor.tokenizer.vocabulary + chunk = [tokens[key] for key in ("", "<|vision_start|>", "