diff --git a/apps/crazy_robotaxi/crazy_robotaxi/application.py b/apps/crazy_robotaxi/crazy_robotaxi/application.py index ff3d9ff0c..86b75a9f4 100644 --- a/apps/crazy_robotaxi/crazy_robotaxi/application.py +++ b/apps/crazy_robotaxi/crazy_robotaxi/application.py @@ -111,6 +111,9 @@ class ApplicationConfig: show_fps: bool """Whether the HUD displays the measured generated-video frame rate.""" + show_current_prompt: bool = False + """Whether the HUD displays the prompt currently driving generation.""" + hud_enabled: bool = True """Whether gameplay HUD overlays are visible.""" @@ -304,6 +307,7 @@ def init(self, commandline_args: Sequence[str]) -> None: else settings.diagnostics.input_trace_path ), show_fps=settings.presentation.show_fps, + show_current_prompt=settings.presentation.show_current_prompt, hud_enabled=settings.presentation.hud_enabled, show_control_hints=settings.presentation.show_control_hints, show_live_edit_buttons=settings.presentation.show_live_edit_buttons, diff --git a/apps/crazy_robotaxi/crazy_robotaxi/controls.py b/apps/crazy_robotaxi/crazy_robotaxi/controls.py index bd280f441..0f6f284de 100644 --- a/apps/crazy_robotaxi/crazy_robotaxi/controls.py +++ b/apps/crazy_robotaxi/crazy_robotaxi/controls.py @@ -110,6 +110,7 @@ "restart", "return_to_menu", "toggle_hints", + "toggle_hdmap", "style", "weather", "coins", @@ -126,6 +127,7 @@ "restart": "restart", "return_to_menu": "return_to_menu", "toggle_hints": "toggle_hints", + "toggle_hdmap": "toggle_hdmap", "cycle_style": "style", "cycle_weather": "weather", "toggle_coins": "coins", @@ -225,6 +227,11 @@ class KeyboardControls: ) """Keys that toggle gameplay control hints.""" + toggle_hdmap: BindingSlots = _bindings_field( + (_key("m"), None), "TOGGLE HD MAP VIEW" + ) + """Keys that toggle the model's HD-map conditioning view.""" + cycle_style: BindingSlots = _bindings_field( (_key("k"), None), "CYCLE STYLE", feature="style" ) @@ -277,6 +284,11 @@ class GamepadControls: ) """Controls that toggle gameplay control hints.""" + toggle_hdmap: BindingSlots = _bindings_field( + (_button(2), None), "TOGGLE HD MAP VIEW" + ) + """Controls that toggle the model's HD-map conditioning view.""" + cycle_style: BindingSlots = _bindings_field( (_button(14), None), "CYCLE STYLE", feature="style" ) @@ -329,6 +341,9 @@ class WheelControls: ) """Controls that toggle gameplay control hints.""" + toggle_hdmap: BindingSlots = _bindings_field((None, None), "TOGGLE HD MAP VIEW") + """Controls that toggle the model's HD-map conditioning view.""" + cycle_style: BindingSlots = _bindings_field( (None, None), "CYCLE STYLE", feature="style" ) diff --git a/apps/crazy_robotaxi/crazy_robotaxi/live_edit/style_ability.py b/apps/crazy_robotaxi/crazy_robotaxi/live_edit/style_ability.py index f68432bd2..8434ccb10 100644 --- a/apps/crazy_robotaxi/crazy_robotaxi/live_edit/style_ability.py +++ b/apps/crazy_robotaxi/crazy_robotaxi/live_edit/style_ability.py @@ -153,6 +153,13 @@ def active_weather_name(self) -> str: return "clear" return self._weather_config.weathers[self._active_weather].name + @property + def active_prompt(self) -> str | None: + """Return the composed prompt currently selected in the model cache.""" + if self._base_prompt is None: + return None + return self._visual_target(self._active_map_suffix).prompt + @property def skin_names(self) -> tuple[str, ...]: """Selectable skin names (empty when the style ability is off).""" diff --git a/apps/crazy_robotaxi/crazy_robotaxi/session.py b/apps/crazy_robotaxi/crazy_robotaxi/session.py index 185b02be1..ebcecd695 100644 --- a/apps/crazy_robotaxi/crazy_robotaxi/session.py +++ b/apps/crazy_robotaxi/crazy_robotaxi/session.py @@ -86,6 +86,7 @@ class ModelState: menu_video: torch.Tensor | None = None """Cached black model channel published while the menu is active.""" last_video: torch.Tensor | None = None + last_hdmap: torch.Tensor | None = None last_bev: torch.Tensor | None = None last_pose: np.ndarray | None = None last_speed_mps: float = 0.0 @@ -275,6 +276,7 @@ def reset(self) -> None: self.finished = False self.realtime_miss_count = 0 self.last_video = None + self.last_hdmap = None self.last_bev = None self.last_pose = None self.driver_input.reset() @@ -348,7 +350,9 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: simulation_timestamps_us: tuple[int, ...] | None = None cache_finalize_returned_ns: int | None = None live_edit_statuses: tuple[LiveEditHudStatus, ...] | None = None + current_prompt = "" if snapshot.session_state in active_states: + current_prompt = rollout.scene.prompt live_edit = getattr(rollout.engine, "live_edit", None) if live_edit is not None: for action in ("style", "weather", "coins", "obstacle"): @@ -419,6 +423,8 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: live_edit.style.after_v2_chunk() if live_edit is not None: live_edit_statuses = live_edit.hud_statuses() + if live_edit.style is not None: + current_prompt = live_edit.style.active_prompt or current_prompt state.blocks_generated += 1 video = generated.video_bvtchw[0, 0] expected_shape = ( @@ -432,6 +438,14 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: f"expected {expected_shape}, got {tuple(video.shape[1:])}" ) engine_step = generated.engine + hdmap = engine_step.condition.hdmap_bvtchw + expected_hdmap_shape = (1, 1, int(video.shape[0]), *expected_shape) + if tuple(hdmap.shape) != expected_hdmap_shape: + raise ValueError( + "HD-map conditioning does not match the generated video: " + f"expected {expected_hdmap_shape}, got {tuple(hdmap.shape)}" + ) + hdmap = hdmap[0, 0] game_frames = engine_step.game_frames poses = engine_step.trajectory.rig_poses_world if trace_enabled: @@ -448,13 +462,19 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: metrics["startup_prewarm_wall_ms"] = state.prewarm_wall_ms metrics["startup_prewarm_blocks"] = state.config.prewarm_blocks state.last_video = video[-1:].detach() + state.last_hdmap = hdmap[-1:].detach() state.last_bev = None if bev is None else bev[-1:].detach() state.last_pose = poses[-1].copy() state.last_speed_mps = speeds_mps[-1] else: - if state.last_video is None or state.last_pose is None: + if ( + state.last_video is None + or state.last_hdmap is None + or state.last_pose is None + ): raise RuntimeError("Terminal game state has no generated frame") video = state.last_video + hdmap = state.last_hdmap game_frames = (snapshot,) poses = state.last_pose[None, ...] speeds_mps = (state.last_speed_mps,) @@ -476,6 +496,7 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: simulation_timestamps_us=simulation_timestamps_us, cache_finalize_returned_ns=cache_finalize_returned_ns, live_edit_statuses=live_edit_statuses, + current_prompt=current_prompt, ) invoke_async( state.ui_loop, @@ -545,6 +566,12 @@ def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: output_layout=VideoTensorLayout.tchw, metrics=finalize_metrics, ), + StepResult( + step_index=step_index, + output=hdmap, + frame_count=count, + output_layout=VideoTensorLayout.tchw, + ), ] if bev is not None: results.append( @@ -603,6 +630,7 @@ def init(self) -> None: bev=self._config.renderer.bev, profile_input_latency=self._config.profile_input_latency, show_fps=self._config.show_fps, + show_current_prompt=self._config.show_current_prompt, hud_enabled=self._config.hud_enabled, live_edit=self._config.live_edit, native_dit_disabled_for_live_edit=( diff --git a/apps/crazy_robotaxi/crazy_robotaxi/settings.py b/apps/crazy_robotaxi/crazy_robotaxi/settings.py index 7c14a5b5e..2ff6ee207 100644 --- a/apps/crazy_robotaxi/crazy_robotaxi/settings.py +++ b/apps/crazy_robotaxi/crazy_robotaxi/settings.py @@ -124,6 +124,7 @@ class PresentationSettings: hud_enabled: bool = True show_fps: bool = False + show_current_prompt: bool = False show_control_hints: bool = True show_live_edit_buttons: bool = True live_edit_mapping_location: LiveEditMappingLocation = "buttons" diff --git a/apps/crazy_robotaxi/crazy_robotaxi/ui.py b/apps/crazy_robotaxi/crazy_robotaxi/ui.py index 1d23f25af..bcf9708a7 100644 --- a/apps/crazy_robotaxi/crazy_robotaxi/ui.py +++ b/apps/crazy_robotaxi/crazy_robotaxi/ui.py @@ -215,6 +215,9 @@ class TaxiHudFrame: live_edit_status: LiveEditHudStatus | None = None """Live-edit state aligned with this generated frame.""" + current_prompt: str = "" + """Model prompt aligned with this generated frame.""" + transition_timestamp_us: int | None = None """V2 input transition represented by this frame, when one was received.""" @@ -262,6 +265,9 @@ class TaxiHudState: show_fps: bool = False """Whether to display the measured generated-video frame rate.""" + show_current_prompt: bool = False + """Whether to display the frame-aligned model prompt across the HUD top.""" + hud_enabled: bool = True """Whether gameplay HUD overlays are visible.""" @@ -280,6 +286,9 @@ class TaxiHudState: live_edit_mapping_location: LiveEditMappingLocation = "buttons" """Where active live-edit mappings appear in the gameplay HUD.""" + show_hdmap: bool = False + """Whether to present the model's HD-map conditioning instead of its output.""" + settings_document: SettingsDocument | None = None """User-authored settings backing the reusable Options screen.""" @@ -556,6 +565,8 @@ def consume_input_events(self, events: UserInputEvents) -> None: self._handle_escape() if "toggle_hints" in actions: self.show_control_tooltips = not self.show_control_tooltips + if self._menu_stage == "game" and "toggle_hdmap" in actions: + self.show_hdmap = not self.show_hdmap for event in received: if isinstance(event, GamepadUserInputEvent): if event.action == "state": @@ -1123,6 +1134,8 @@ def _save_options(self) -> None: self.hud_enabled = draft.presentation.hud_enabled if ("presentation", "show_fps") not in overrides: self.show_fps = draft.presentation.show_fps + if ("presentation", "show_current_prompt") not in overrides: + self.show_current_prompt = draft.presentation.show_current_prompt if ("presentation", "show_control_hints") not in overrides: self.show_control_tooltips = draft.presentation.show_control_hints if ("presentation", "show_live_edit_buttons") not in overrides: @@ -1236,18 +1249,21 @@ def draw( if self._menu_stage == "options": self._draw_options(imgui) return - self._draw_fps_counter(imgui) if self._menu_stage == "mode": + self._draw_fps_counter(imgui) self._draw_mode_selection(imgui) return if self._menu_stage == "map": + self._draw_fps_counter(imgui) self._draw_map_selection(imgui) return if self._menu_stage == "course": + self._draw_fps_counter(imgui) self._draw_course_selection(imgui) return hud_frame = self._current if hud_frame is None: + self._draw_fps_counter(imgui) dots = "." * (1 + (ui_tick // 15) % 3) elapsed_s = max(0, int(time.monotonic() - self._loading_started_at_s)) self._draw_text_window( @@ -1259,6 +1275,16 @@ def draw( ) return snapshot = hud_frame.snapshot + active = snapshot.session_state in {"playing", "awaiting_start", "racing"} + prompt_offset = ( + self._draw_current_prompt(imgui, hud_frame.current_prompt) + if self.hud_enabled + and active + and self.show_current_prompt + and hud_frame.current_prompt + else 0.0 + ) + self._draw_fps_counter(imgui, top=14.0 + prompt_offset) if not self.hud_enabled: self._draw_terminal(imgui, snapshot) return @@ -1267,11 +1293,11 @@ def draw( "awaiting_start", "racing", }: - self._draw_race_status(imgui, snapshot) + self._draw_race_status(imgui, snapshot, top_offset=prompt_offset) self._draw_navigation_arrow( imgui, snapshot.relative_bearing_rad, - center_y=110.0, + center_y=110.0 + prompt_offset, color_rgb=(1.0, 0.18, 0.08), ) self._draw_bev_window(imgui, bev_frame, hud_frame) @@ -1279,11 +1305,11 @@ def draw( isinstance(snapshot, TaxiGameSnapshot) and snapshot.session_state == "playing" ): - self._draw_taxi_status(imgui, snapshot) + self._draw_taxi_status(imgui, snapshot, top_offset=prompt_offset) self._draw_navigation_arrow( imgui, snapshot.relative_bearing_rad, - center_y=110.0, + center_y=110.0 + prompt_offset, color_rgb=( (118.0 / 255.0, 185.0 / 255.0, 0.0) if snapshot.phase == "seeking_pickup" @@ -1291,15 +1317,76 @@ def draw( ), ) self._draw_bev_window(imgui, bev_frame, hud_frame) - if snapshot.session_state in {"playing", "awaiting_start", "racing"}: + if active: self._draw_speed(imgui, hud_frame.speed_mps) - self._draw_coin_counter(imgui, hud_frame.live_edit_status) - self._draw_live_edit_card(imgui, hud_frame.live_edit_status) + self._draw_coin_counter( + imgui, + hud_frame.live_edit_status, + top_offset=prompt_offset, + ) + self._draw_live_edit_card( + imgui, + hud_frame.live_edit_status, + top_offset=prompt_offset, + ) self._draw_control_tooltips(imgui) self._draw_terminal(imgui, snapshot) self._draw_input_diagnostic(imgui) - def _draw_taxi_status(self, imgui: Any, snapshot: TaxiGameSnapshot) -> None: + def _draw_current_prompt(self, imgui: Any, prompt: str) -> float: + """Draw the frame-aligned prompt in a plain wrapped debug window.""" + panel_width = max(1.0, float(self.width) - 28.0) + window_padding = _point_xy(imgui.get_style().window_padding) + item_spacing_y = _point_xy(imgui.get_style().item_spacing)[1] + content_width = max(1.0, panel_width - 2.0 * window_padding[0]) + frame_padding_x = _point_xy(imgui.get_style().frame_padding)[0] + wrapped, _underlying_width, _editor_height, _field_height = ( + _wrapped_editor_layout( + imgui, + prompt, + content_width + 2.0 * frame_padding_x, + ) + ) + lines = tuple(line.rstrip() for line in wrapped.splitlines()) or ("",) + font_size = float(imgui.get_font_size()) + natural_height = ( + float(imgui.get_frame_height()) + + 2.0 * window_padding[1] + + len(lines) * font_size + + max(0, len(lines) - 1) * item_spacing_y + ) + prompt_gap = 8.0 + event_top = 160.0 + event_height = _overlay_text_size(imgui, "M", 44.0)[1] + max_panel_height = float(self.height) - prompt_gap - event_top - event_height + if max_panel_height <= 0.0: + return 0.0 + panel_height = min(natural_height, max_panel_height) + if panel_height < natural_height: + fixed_height = float(imgui.get_frame_height()) + 2.0 * window_padding[1] + line_stride = font_size + item_spacing_y + visible_line_count = max( + 1, + int((panel_height - fixed_height + item_spacing_y) / line_stride), + ) + lines = lines[:visible_line_count] + lines = (*lines[:-1], "...") + self._draw_text_window( + imgui, + "Current Prompt", + position=(14.0, 14.0), + size=(panel_width, panel_height), + lines=lines, + ) + return panel_height + prompt_gap + + def _draw_taxi_status( + self, + imgui: Any, + snapshot: TaxiGameSnapshot, + *, + top_offset: float = 0.0, + ) -> None: """Draw the source game's one-line taxi status directly over the frame.""" phase = "PICKUP" if snapshot.phase == "seeking_pickup" else "DROPOFF" fare_time = ( @@ -1319,20 +1406,26 @@ def _draw_taxi_status(self, imgui: Any, snapshot: TaxiGameSnapshot) -> None: if snapshot.phase == "seeking_pickup" else (200.0 / 255.0, 150.0 / 255.0, 50.0 / 255.0) ) - self._draw_status_strip(imgui, label, color_rgb=color, top=35.0) + self._draw_status_strip(imgui, label, color_rgb=color, top=35.0 + top_offset) event = _event_label(snapshot) if event: self._draw_centered_text( imgui, event, - top=160.0, + top=160.0 + top_offset, font_size=44.0, color_rgb=color, shadow=True, font=self._gameplay_overlay_font(imgui), ) - def _draw_race_status(self, imgui: Any, snapshot: RaceGameSnapshot) -> None: + def _draw_race_status( + self, + imgui: Any, + snapshot: RaceGameSnapshot, + *, + top_offset: float = 0.0, + ) -> None: """Draw the source game's one-line race status directly over the frame.""" if snapshot.session_state == "awaiting_start": progress = "CROSS START LINE TO BEGIN" @@ -1365,7 +1458,7 @@ def _draw_race_status(self, imgui: Any, snapshot: RaceGameSnapshot) -> None: imgui, label, color_rgb=(200.0 / 255.0, 150.0 / 255.0, 50.0 / 255.0), - top=35.0, + top=35.0 + top_offset, outline=True, ) @@ -1518,7 +1611,7 @@ def _gameplay_overlay_font(self, imgui: Any) -> Any: ) return self._gameplay_font - def _draw_fps_counter(self, imgui: Any) -> None: + def _draw_fps_counter(self, imgui: Any, *, top: float = 14.0) -> None: """Draw the measured generated-video rate when the counter is enabled.""" if not self.show_fps: return @@ -1526,7 +1619,7 @@ def _draw_fps_counter(self, imgui: Any) -> None: self._draw_text_window( imgui, "Performance", - position=(float(max(14.0, self.width - width - 14.0)), 14.0), + position=(float(max(14.0, self.width - width - 14.0)), top), size=(width, 66.0), lines=(f"VIDEO FPS {self._video_fps:5.1f}",), ) @@ -1535,6 +1628,8 @@ def _draw_live_edit_card( self, imgui: Any, status: LiveEditHudStatus | None, + *, + top_offset: float = 0.0, ) -> None: """Draw frame-aligned live-edit status and action buttons.""" if status is None or not self.live_edit.any_enabled: @@ -1564,7 +1659,7 @@ def _draw_live_edit_card( ) _prepare_window( imgui, - position=(14.0, 94.0), + position=(14.0, 94.0 + top_offset), size=None, alpha=0.94, pivot=(0.0, 0.0), @@ -1613,13 +1708,15 @@ def _draw_coin_counter( self, imgui: Any, status: LiveEditHudStatus | None, + *, + top_offset: float = 0.0, ) -> None: """Draw collected coins in the upper-left while coins are available.""" if status is None or not status.coins_enabled: return _prepare_window( imgui, - position=(14.0, 14.0), + position=(14.0, 14.0 + top_offset), size=None, alpha=0.94, pivot=(0.0, 0.0), @@ -1702,6 +1799,7 @@ def display(slots: tuple[InputBinding | None, InputBinding | None]) -> str: if self.live_edit_mapping_location == "control hints" else () ), + ("TOGGLE HD MAP VIEW", display(controls.toggle_hdmap)), ) action_width = max( _point_xy(imgui.calc_text_size(action))[0] for action, _binding in entries @@ -3675,14 +3773,18 @@ def step_ui( self.state.consume_input_events(events) frames = self.presented_model_frames() video = frames[0] if frames else None - bev_frame = frames[1] if len(frames) > 1 else None + hdmap_frame = frames[1] if len(frames) > 1 else None + bev_frame = frames[2] if len(frames) > 2 else None if video is not None: self.state.select_presented_frame(video) self.state.draw_waypoints(imgui, video) self.state.draw(imgui, step_index, bev_frame=bev_frame) if video is None: return None - return self.state.composite_bev(video, bev_frame) + background = ( + hdmap_frame if self.state.show_hdmap and hdmap_frame is not None else video + ) + return self.state.composite_bev(background, bev_frame) def reset(self) -> None: """Reset UI-owned state and retained renderer resources.""" @@ -3733,6 +3835,7 @@ def build_hud_frames( simulation_timestamps_us: Sequence[int | None] | None = None, cache_finalize_returned_ns: int | None = None, live_edit_statuses: Sequence[LiveEditHudStatus | None] | None = None, + current_prompt: str = "", ) -> tuple[TaxiHudFrame, ...]: """Build immutable UI messages aligned with generated tensor frames.""" frame_count = int(video_tchw.shape[0]) @@ -3772,6 +3875,7 @@ def build_hud_frames( rig_pose_world=pose, speed_mps=float(speeds_mps[index]), live_edit_status=live_edit_statuses[index], + current_prompt=current_prompt, transition_timestamp_us=transition_timestamps_us[index], runtime_generation=runtime_generation, model_step_index=model_step_index, diff --git a/apps/crazy_robotaxi/tests/test_application.py b/apps/crazy_robotaxi/tests/test_application.py index 79603d046..e2b0f88be 100644 --- a/apps/crazy_robotaxi/tests/test_application.py +++ b/apps/crazy_robotaxi/tests/test_application.py @@ -300,6 +300,7 @@ def test_user_config_overrides_model_and_game_without_selecting_menus( presentation: show_live_edit_buttons: false live_edit_mapping_location: control hints + show_current_prompt: true runtime: prewarm_blocks: 0 """, @@ -319,6 +320,7 @@ def test_user_config_overrides_model_and_game_without_selecting_menus( assert app._config.gamepad_button_style == "PlayStation" assert not app._config.show_live_edit_buttons assert app._config.live_edit_mapping_location == "control hints" + assert app._config.show_current_prompt pipeline_config = app._pipeline_config assert pipeline_config is not None assert pipeline_config.diffusion_model.seed == 5678 @@ -530,6 +532,7 @@ def _invoke_async(self, operation) -> None: ui_loop=cast(Any, ui_loop), rollout=cast(Any, rollout), last_video=torch.zeros(1, 3, 4, 4), + last_hdmap=torch.ones(1, 3, 4, 4), last_pose=np.eye(4, dtype=np.float32), prewarm_complete=True, game_selected=True, @@ -539,7 +542,8 @@ def _invoke_async(self, operation) -> None: results = loop.step(0, UserInputEvents([])) - assert len(results) == 1 + assert len(results) == 2 + assert torch.all(results[1].read_output() == 1.0) assert not state.finished assert not loop.is_finished() assert len(ui_loop.operations) == 1 diff --git a/apps/crazy_robotaxi/tests/test_controls.py b/apps/crazy_robotaxi/tests/test_controls.py index 5bf08b332..e9387acb7 100644 --- a/apps/crazy_robotaxi/tests/test_controls.py +++ b/apps/crazy_robotaxi/tests/test_controls.py @@ -159,6 +159,8 @@ def test_gamepad_defaults_use_standard_button_indices() -> None: assert gamepad.restart == (InputBinding("button", 9), None) assert gamepad.return_to_menu == (InputBinding("button", 8), None) assert gamepad.toggle_hints == (InputBinding("button", 5), None) + assert controls.keyboard.toggle_hdmap == (InputBinding("key", "m"), None) + assert gamepad.toggle_hdmap == (InputBinding("button", 2), None) assert gamepad.cycle_style == (InputBinding("button", 14), None) assert gamepad.cycle_weather == (InputBinding("button", 15), None) assert gamepad.toggle_coins == (InputBinding("button", 12), None) @@ -169,12 +171,13 @@ def test_gamepad_defaults_use_standard_button_indices() -> None: pressed = GamepadUserInputEvent( timestamp=np.uint64(1), action="state", - pressed=tuple(index in {5, 8, 9, 12, 13, 14, 15} for index in range(16)), + pressed=tuple(index in {2, 5, 8, 9, 12, 13, 14, 15} for index in range(16)), ) assert BoundActionState(controls).apply(UserInputEvents([pressed])) == { "restart", "return_to_menu", "toggle_hints", + "toggle_hdmap", "style", "weather", "coins", diff --git a/apps/crazy_robotaxi/tests/test_live_edit_v2.py b/apps/crazy_robotaxi/tests/test_live_edit_v2.py index 18bf7248b..cf0e8d9e2 100644 --- a/apps/crazy_robotaxi/tests/test_live_edit_v2.py +++ b/apps/crazy_robotaxi/tests/test_live_edit_v2.py @@ -471,6 +471,23 @@ def test_weather_suffix_composes_without_changing_style_prompt() -> None: ) +def test_active_prompt_reports_composed_model_target() -> None: + ability = StyleAbility( + LiveEditStyleConfig( + enabled=True, + skins=(StyleSkin("comic", "Comic-book visuals."),), + ) + ) + ability._base_prompt = "A city road." + + assert ability.active_prompt == "A city road." + + ability._active_index = 0 + ability._active_map_suffix = "The taxi is driving forward." + + assert ability.active_prompt == "Comic-book visuals. The taxi is driving forward." + + def test_map_prompt_change_is_plain_and_deferred_during_guidance() -> None: ability, session, targets = _map_prompt_ability() ability._pending_map_suffix = "The taxi is driving forward." diff --git a/apps/crazy_robotaxi/tests/test_settings.py b/apps/crazy_robotaxi/tests/test_settings.py index ee76a0cb5..07e92fdc7 100644 --- a/apps/crazy_robotaxi/tests/test_settings.py +++ b/apps/crazy_robotaxi/tests/test_settings.py @@ -58,6 +58,7 @@ def test_sparse_yaml_overrides_nested_model_config( show_fps: true show_live_edit_buttons: false live_edit_mapping_location: control hints + show_current_prompt: true """, encoding="utf-8", ) @@ -70,6 +71,7 @@ def test_sparse_yaml_overrides_nested_model_config( assert document.settings.presentation.show_fps assert not document.settings.presentation.show_live_edit_buttons assert document.settings.presentation.live_edit_mapping_location == "control hints" + assert document.settings.presentation.show_current_prompt def test_launch_selections_are_not_user_yaml_settings(tmp_path: Path) -> None: diff --git a/apps/crazy_robotaxi/tests/test_ui.py b/apps/crazy_robotaxi/tests/test_ui.py index 4f153057b..d93dd0db3 100644 --- a/apps/crazy_robotaxi/tests/test_ui.py +++ b/apps/crazy_robotaxi/tests/test_ui.py @@ -266,6 +266,7 @@ def __init__(self) -> None: self.disabled_buttons: list[str] = [] self.background_draw_list = _FakeDrawList() self.window_flags: dict[str, int] = {} + self.window_positions: dict[str, tuple[float, float]] = {} self.window_sizes: dict[str, tuple[float, float]] = {} self.child_sizes: dict[str, tuple[float, float]] = {} self.child_window_flags: dict[str, int] = {} @@ -361,6 +362,7 @@ def begin(self, title: str, *, flags: int) -> bool: self.current_window = title self.windows.setdefault(title, []) self.window_flags[title] = flags + self.window_positions[title] = self.next_window_position self.window_sizes[title] = self.next_window_size return True @@ -894,6 +896,22 @@ def test_live_edit_card_is_hidden_when_map_context_has_no_visible_content() -> N assert "Live Edit" not in imgui.windows +def test_hud_frames_preserve_frame_aligned_prompt() -> None: + video = torch.zeros(2, 3, 96, 160) + + frames = build_hud_frames( + video, + (_snapshot(), _snapshot()), + np.repeat(np.eye(4, dtype=np.float32)[None], 2, axis=0), + current_prompt="A taxi driving through a city.", + ) + + assert [frame.current_prompt for frame in frames] == [ + "A taxi driving through a city.", + "A taxi driving through a city.", + ] + + def test_hud_frames_reject_misaligned_input_diagnostics() -> None: with pytest.raises(ValueError, match="Input transitions"): build_hud_frames( @@ -1014,9 +1032,81 @@ def test_fps_counter_measures_distinct_generated_video_frames( assert imgui.windows["Performance"] == ["VIDEO FPS 30.0"] +@pytest.mark.parametrize("show_current_prompt", [False, True]) +def test_current_prompt_overlay_is_configurable(show_current_prompt: bool) -> None: + video = torch.zeros(1, 3, 360, 320) + state = TaxiHudState( + 320, + 360, + _calibration(), + show_current_prompt=show_current_prompt, + ) + state._menu_stage = "game" + state.publish( + build_hud_frames( + video, + (_snapshot(),), + np.eye(4, dtype=np.float32)[None], + current_prompt=( + "A taxi driving through a wide city boulevard with buildings and trees." + ), + ) + ) + state.select_presented_frame(video[0]) + imgui = _FakeImGui() + + state.draw(imgui) + + assert ("Current Prompt" in imgui.windows) is show_current_prompt + if show_current_prompt: + assert len(imgui.windows["Current Prompt"]) > 1 + + +def test_current_prompt_overlay_preserves_room_for_gameplay_hud() -> None: + state = TaxiHudState(320, 360, _calibration(), show_current_prompt=True) + imgui = _FakeImGui() + + prompt_offset = state._draw_current_prompt(imgui, "long prompt " * 100) + + assert prompt_offset + 160.0 + 44.0 <= state.height + assert imgui.windows["Current Prompt"][-1] == "..." + + +def test_current_prompt_offsets_left_live_edit_overlays() -> None: + video = torch.zeros(1, 3, 360, 320) + live_edit = LiveEditConfig(coins=LiveEditCoinsConfig(enabled=True)) + status = LiveEditHudStatus(coins_enabled=True, coins_collected=3) + state = TaxiHudState( + 320, + 360, + _calibration(), + show_current_prompt=True, + live_edit=live_edit, + ) + state._menu_stage = "game" + state.publish( + build_hud_frames( + video, + (_snapshot(),), + np.eye(4, dtype=np.float32)[None], + live_edit_statuses=(status,), + current_prompt="A taxi driving through a wide city boulevard.", + ) + ) + state.select_presented_frame(video[0]) + imgui = _FakeImGui() + + state.draw(imgui) + + prompt_bottom = 14.0 + imgui.window_sizes["Current Prompt"][1] + assert imgui.window_positions["Coin Counter"] == (14.0, prompt_bottom + 8.0) + assert imgui.window_positions["Live Edit"] == (14.0, prompt_bottom + 88.0) + + def test_imgui_ui_loop_draws_waypoints_and_bev_in_the_ui_overlay() -> None: width, height = 160, 96 video = torch.full((1, 3, height, width), -0.5, dtype=torch.bfloat16) + hdmap = torch.full((1, 3, height, width), 0.5, dtype=torch.bfloat16) bev = torch.full((1, 4, 32, 32), 255, dtype=torch.uint8) bev[:, :3].fill_(191) hud_state = TaxiHudState(width, height, _calibration()) @@ -1033,6 +1123,7 @@ def test_imgui_ui_loop_draws_waypoints_and_bev_in_the_ui_overlay() -> None: 0, [ StepResult(0, video, 1, VideoTensorLayout.tchw), + StepResult(0, hdmap, 1, VideoTensorLayout.tchw), StepResult(0, bev, 1, VideoTensorLayout.tchw), ], ) @@ -1111,6 +1202,57 @@ def test_imgui_ui_loop_draws_waypoints_and_bev_in_the_ui_overlay() -> None: assert renderer.reset_count == 1 +def test_imgui_ui_loop_can_present_exact_hdmap_conditioning() -> None: + width, height = 160, 96 + video = torch.full((1, 3, height, width), -0.5, dtype=torch.bfloat16) + hdmap = torch.full((1, 3, height, width), 0.75, dtype=torch.bfloat16) + hud_state = TaxiHudState(width, height, _calibration(), hud_enabled=False) + hud_state.publish( + build_hud_frames( + video, + (_snapshot(),), + np.eye(4, dtype=np.float32)[None], + ) + ) + presentation = PresentationManager() + presentation.publish( + 0, + [ + StepResult(0, video, 1, VideoTensorLayout.tchw), + StepResult(0, hdmap, 1, VideoTensorLayout.tchw), + ], + ) + presentation.advance(0) + loop = CrazyRobotaxiImGuiUILoop(renderer=_Renderer(width, height)) + loop.register_session_loop_objects( + state=hud_state, + frequency=60, + shutdown_event=threading.Event(), + failure_queue=queue.Queue(), + ) + loop.register_session_ui_loop_objects( + session_desc=SessionDesc(output_layout=VideoTensorLayout.tchw), + presentation_manager=presentation, + ) + + generated = loop.step(0, UserInputEvents([])).read_output() + conditioning = loop.step( + 1, + UserInputEvents( + [ + KeyboardUserInputEvent( + timestamp=np.uint64(1), + key="m", + state=KeyboardInputState.PRESSED, + ) + ] + ), + ).read_output() + + assert torch.all(generated == -0.5) + assert torch.all(conditioning == 0.75) + + def test_bev_compositor_uses_rgba_coverage_for_black_road_pixels() -> None: state = TaxiHudState(4, 4, _calibration()) state._bev_rect = (0, 0, 4, 4) @@ -1216,7 +1358,7 @@ def test_live_hud_draws_directly_over_the_game_frame() -> None: keyboard=replace( defaults.keyboard, restart=(InputBinding("key", "p"), None), - return_to_menu=(InputBinding("key", "m"), None), + return_to_menu=(InputBinding("key", "n"), None), toggle_hints=(InputBinding("key", "j"), None), ), ), @@ -1241,7 +1383,8 @@ def test_live_hud_draws_directly_over_the_game_frame() -> None: ["FORWARD", "W / UP ARROW", "BRAKE / REVERSE", "S / DOWN ARROW"], ["STEER LEFT", "A / LEFT ARROW", "STEER RIGHT", "D / RIGHT ARROW"], ["HANDBRAKE", "SPACE", "RESTART", "P"], - ["RETURN TO MENU", "M", "HIDE CONTROLS", "J"], + ["RETURN TO MENU", "N", "HIDE CONTROLS", "J"], + ["TOGGLE HD MAP VIEW", "M"], ] controls_flags = imgui.window_flags["Controls"] assert controls_flags & imgui.WindowFlags_.always_auto_resize @@ -2133,6 +2276,32 @@ def test_h_toggles_gameplay_control_tooltips() -> None: assert state.show_control_tooltips +def test_m_toggles_hdmap_view_only_during_gameplay() -> None: + state = TaxiHudState(640, 360, _calibration()) + released = KeyboardUserInputEvent( + timestamp=np.uint64(1), + key="m", + state=KeyboardInputState.RELEASED, + ) + pressed = KeyboardUserInputEvent( + timestamp=np.uint64(2), + key="M", + state=KeyboardInputState.PRESSED, + ) + + state.consume_input_events(UserInputEvents([pressed])) + assert not state.show_hdmap + + state.consume_input_events(UserInputEvents([released])) + state._menu_stage = "game" + state.consume_input_events(UserInputEvents([pressed])) + assert state.show_hdmap + + state.consume_input_events(UserInputEvents([released])) + state.consume_input_events(UserInputEvents([pressed])) + assert not state.show_hdmap + + def test_control_tooltip_card_uses_one_pair_per_row_when_narrow() -> None: state = TaxiHudState(160, 96, _calibration()) imgui = _FakeImGui() @@ -2140,7 +2309,7 @@ def test_control_tooltip_card_uses_one_pair_per_row_when_narrow() -> None: state._draw_control_tooltips(imgui) assert imgui.table_column_counts["##gameplay-control-hints"] == 2 - assert len(imgui.tables["##gameplay-control-hints"]) == 8 + assert len(imgui.tables["##gameplay-control-hints"]) == 9 def test_connected_gamepad_replaces_keyboard_gameplay_hints() -> None: @@ -2849,6 +3018,7 @@ def test_options_save_persists_and_applies_presentation_setting( imgui.checkbox_values["##presentation.show_fps"] = True imgui.checkbox_values["##presentation.show_live_edit_buttons"] = False imgui.combo_indices["##presentation.live_edit_mapping_location"] = 1 + imgui.checkbox_values["##presentation.show_current_prompt"] = True imgui.clicked_buttons.add("SAVE") state.draw(imgui) @@ -2857,11 +3027,13 @@ def test_options_save_persists_and_applies_presentation_setting( assert state.show_fps assert not state.show_live_edit_buttons assert state.live_edit_mapping_location == "control hints" + assert state.show_current_prompt assert "show_fps: true" in document.path.read_text(encoding="utf-8") assert "show_live_edit_buttons: false" in document.path.read_text(encoding="utf-8") assert "live_edit_mapping_location: control hints" in document.path.read_text( encoding="utf-8" ) + assert "show_current_prompt: true" in document.path.read_text(encoding="utf-8") assert state._settings_notice == f"SAVED {document.path}" assert not state._settings_restart_notice options_lines = imgui.windows["Crazy Robotaxi - Options"]