From f5ec088b079c5d67b6db74106938ca87c0d87d54 Mon Sep 17 00:00:00 2001 From: devmobasa <4170275+devmobasa@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:04:22 +0200 Subject: [PATCH 1/4] feat(ocr): show what recognition is doing, and read the whole image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recognition ran on a worker with nothing on screen to say the region had even been read, and a toast afterwards that gave no sense of where it came from. A band now sweeps the rectangle while the worker runs, and a short card beside it reports the outcome. The card carries an outcome and a character count, never the recognized text. `src/ocr` keeps screen contents out of application state entirely — the worker publishes to the clipboard inside one stack frame and reports back only a count — and painting a transcript here would put exactly what that invariant exists to exclude into the overlay's own UI. `Ctrl+A` now takes the whole displayed image for recognition as well as for capture. Reading a full screen of text should not need a drag across the whole output. Measure is the only purpose left out, having nothing to submit. Details worth stating: - The hand-over deadline is fixed when recognition settles, not recomputed per tick. A deadline derived from the current elapsed time moves along with it and is only ever met on an exact sweep boundary, which a frame clock does not hit: the band would sweep forever and the card would never appear. - The overlay starts only for a request the controller accepted, and carries its id. A refused submission produces no completion, and an overlay waiting on one would sweep until something dismissed it. - Under `[ui] reduced_motion` the region takes a static tint instead of a moving band, the result shows as soon as it arrives rather than waiting out an animation that never ran, and the overlay asks for no frames — one deadline expires the card instead of pinning a repaint for its whole life. - Dismissal takes a finished card, not a running sweep: the sweep is progress feedback for work still in flight, and dropping it would discard the result about to arrive. Pointer, touch, stylus and keyboard all dismiss, the stylus before its surface-specific routing so a press on the Review bar or a toolbar counts too. Both entry points share one submit transaction, so a fix to the dragged-region path cannot leave whole-image recognition behind. --- docs/CONFIG.md | 12 + src/backend/wayland/backend/event_loop/mod.rs | 14 +- src/backend/wayland/handlers/keyboard/mod.rs | 86 ++- src/backend/wayland/handlers/pointer/press.rs | 4 + src/backend/wayland/handlers/tablet/tool.rs | 4 + src/backend/wayland/handlers/touch.rs | 3 + src/backend/wayland/state/data.rs | 3 + src/backend/wayland/state/ocr.rs | 84 ++- .../wayland/state/region_capture/delivery.rs | 12 +- .../wayland/state/region_capture/geometry.rs | 5 +- .../state/region_capture/tests/review.rs | 23 +- src/backend/wayland/state/render/mod.rs | 5 +- src/backend/wayland/state/render/ui.rs | 23 + .../wayland/state/render/ui_effect_damage.rs | 19 + src/input/state/core/base/state/init.rs | 1 + src/input/state/core/base/state/structs.rs | 3 + src/input/state/core/mod.rs | 2 +- src/input/state/core/utility/mod.rs | 1 + src/input/state/core/utility/ocr_scan.rs | 627 ++++++++++++++++++ src/input/state/mod.rs | 1 + src/ocr/controller.rs | 9 + src/ocr/mod.rs | 2 +- src/ui.rs | 4 + src/ui/ocr_scan.rs | 401 +++++++++++ 24 files changed, 1293 insertions(+), 55 deletions(-) create mode 100644 src/input/state/core/utility/ocr_scan.rs create mode 100644 src/ui/ocr_scan.rs diff --git a/docs/CONFIG.md b/docs/CONFIG.md index ac1d6ff3..177b239f 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1566,6 +1566,18 @@ does not change the active tool, the drawing history, or the board. does nothing. - On a solid whiteboard or blackboard with no visible screen capture, OCR refuses rather than reading the board. +- Ctrl+A reads the whole displayed image, so a full screen of text + does not need a drag across the whole output. +- While recognition runs, a band sweeps the region being read. When it finishes, + a short card beside the region says what happened — copied and how many + characters, no text found, or that recognition failed — and fades after a few + seconds. Any click, touch, stylus press or key dismisses the card early; a + sweep still waiting on the recognizer is left alone, so a stray keystroke + cannot discard a result that is about to arrive. The card never shows the + recognized text: Wayscriber keeps screen contents out of its own UI, and the + text goes only to the clipboard. With `[ui] reduced_motion` set, the region is + marked with a static tint instead of a moving band and the card appears as + soon as the result does. - An active OCR selection cancels if the displayed screen image changes, the zoom level or pan changes, or freeze, output, scale, or display layout state is replaced. diff --git a/src/backend/wayland/backend/event_loop/mod.rs b/src/backend/wayland/backend/event_loop/mod.rs index 1ffff8f3..a9fa8944 100644 --- a/src/backend/wayland/backend/event_loop/mod.rs +++ b/src/backend/wayland/backend/event_loop/mod.rs @@ -119,10 +119,15 @@ pub(super) fn run_event_loop( // settled fade contributes nothing. let animation_timeout = min_timeout( min_timeout( - state.ui_animation_timeout(now), - state.top_strip_fade_timeout(now), + min_timeout( + state.ui_animation_timeout(now), + state.top_strip_fade_timeout(now), + ), + state.inline_toolbar_tooltip_timeout(now), ), - state.inline_toolbar_tooltip_timeout(now), + // A still OCR card under reduced motion asks for no frames, so it + // needs one deadline to be taken away on. + state.input_state.ocr_scan_wake_after(now), ); let toolbar_handoff_timeout = state.toolbar_drag_handoff_timeout(now); let autosave_timeout = session_save::autosave_timeout(state, now); @@ -260,6 +265,9 @@ pub(super) fn run_event_loop( if !capture_active && state.ui_animation_due(std::time::Instant::now()) { state.input_state.needs_redraw = true; } + if state.input_state.ocr_scan_due(std::time::Instant::now()) { + state.input_state.needs_redraw = true; + } // When the radial paint deadline passes, this requests the redraw // that paints the menu (no-op before the deadline and after paint). diff --git a/src/backend/wayland/handlers/keyboard/mod.rs b/src/backend/wayland/handlers/keyboard/mod.rs index ece5f3e3..4fdcfffb 100644 --- a/src/backend/wayland/handlers/keyboard/mod.rs +++ b/src/backend/wayland/handlers/keyboard/mod.rs @@ -160,6 +160,9 @@ impl KeyboardHandler for WaylandState { // Any fresh key press ends the previous auto-repeat; a repeatable one // re-arms it at the end of this handler. self.clear_key_repeat(); + // A finished scan card is transient chrome: the next interaction of any + // kind takes it away rather than making the user wait it out. + self.input_state.dismiss_ocr_scan_result(); if self.input_state.region_is_engaged() { if matches!(key, Key::Space) && self.toggle_region_window_snap() { self.update_pointer_cursor(false, conn); @@ -228,7 +231,7 @@ impl KeyboardHandler for WaylandState { } return; } - if region_capture_select_all_pressed( + if region_select_all_pressed( self.input_state.region_is_active(), self.input_state.region_state().purpose(), self.input_state.modifiers.ctrl, @@ -567,14 +570,20 @@ fn screen_modal_swallows_key_release( region_selector_engaged || eyedropper_engaged } -fn region_capture_select_all_pressed( +/// `Ctrl+A` takes the whole displayed image. Recognition wants it as much as +/// capture does — reading a full screen of text should not need a drag across +/// the whole output — so Measure is the only purpose left out, having nothing +/// to submit. +fn region_select_all_pressed( region_active: bool, purpose: Option, ctrl: bool, key: Key, ) -> bool { + use crate::input::state::RegionPurposeTag; + region_active - && purpose.is_some_and(crate::input::state::RegionPurposeTag::is_capture) + && purpose.is_some_and(|purpose| purpose.is_capture() || purpose == RegionPurposeTag::Ocr) && ctrl && matches!(key, Key::Char('a' | 'A')) } @@ -666,39 +675,48 @@ mod tests { } #[test] - fn ctrl_a_selects_all_only_for_an_active_capture_picker() { + fn ctrl_a_selects_all_for_capture_and_recognition_but_not_measure() { use crate::input::state::RegionPurposeTag; - assert!(region_capture_select_all_pressed( - true, - Some(RegionPurposeTag::CaptureDeliver), - true, - Key::Char('a'), - )); - assert!(region_capture_select_all_pressed( - true, - Some(RegionPurposeTag::CaptureInteractive), - true, - Key::Char('A'), - )); - assert!(!region_capture_select_all_pressed( - false, - Some(RegionPurposeTag::CaptureDeliver), - true, - Key::Char('a'), - )); - assert!(!region_capture_select_all_pressed( - true, - Some(RegionPurposeTag::Ocr), - true, - Key::Char('a'), - )); - assert!(!region_capture_select_all_pressed( - true, - Some(RegionPurposeTag::CaptureDeliver), - false, - Key::Char('a'), - )); + for purpose in [ + RegionPurposeTag::CaptureDeliver, + RegionPurposeTag::CaptureInteractive, + RegionPurposeTag::Ocr, + ] { + assert!( + region_select_all_pressed(true, Some(purpose), true, Key::Char('a')), + "{purpose:?} can submit the whole image" + ); + assert!(region_select_all_pressed( + true, + Some(purpose), + true, + Key::Char('A') + )); + } + + assert!( + !region_select_all_pressed(true, Some(RegionPurposeTag::Measure), true, Key::Char('a')), + "measure has nothing to submit" + ); + assert!( + !region_select_all_pressed( + false, + Some(RegionPurposeTag::CaptureDeliver), + true, + Key::Char('a') + ), + "no selector open" + ); + assert!( + !region_select_all_pressed( + true, + Some(RegionPurposeTag::CaptureDeliver), + false, + Key::Char('a') + ), + "plain A is a colour, not select-all" + ); } #[test] diff --git a/src/backend/wayland/handlers/pointer/press.rs b/src/backend/wayland/handlers/pointer/press.rs index 2783d209..dd87f20e 100644 --- a/src/backend/wayland/handlers/pointer/press.rs +++ b/src/backend/wayland/handlers/pointer/press.rs @@ -34,6 +34,10 @@ impl WaylandState { .note_input_hud_mouse(&input_hud_button_label(button), self.input_state.modifiers); } + // A finished scan card is transient chrome: the next interaction of any + // kind takes it away rather than making the user wait it out. + self.input_state.dismiss_ocr_scan_result(); + let help_press_source = HelpOverlayPressSource::Pointer(button); if !self.input_state.show_help { // A new press proves any older help-owned sequence for this button diff --git a/src/backend/wayland/handlers/tablet/tool.rs b/src/backend/wayland/handlers/tablet/tool.rs index 9caabfba..8a646fad 100644 --- a/src/backend/wayland/handlers/tablet/tool.rs +++ b/src/backend/wayland/handlers/tablet/tool.rs @@ -243,6 +243,10 @@ impl Dispatch for WaylandState { if state.stylus_contact_retired { return; } + // Before any surface-specific routing: a press on the Review + // bar, a toolbar or an eyedropper never reaches the frame + // commit, and the card has to go for all of them. + state.input_state.dismiss_ocr_scan_result(); if state.input_state.region_is_active() { if state.stylus_on_toolbar { state.cancel_region_for_toolbar_interaction(); diff --git a/src/backend/wayland/handlers/touch.rs b/src/backend/wayland/handlers/touch.rs index 8174d4ff..a3105a38 100644 --- a/src/backend/wayland/handlers/touch.rs +++ b/src/backend/wayland/handlers/touch.rs @@ -194,6 +194,9 @@ impl WaylandState { surface: &wl_surface::WlSurface, position: (f64, f64), ) -> TouchTarget { + // A finished scan card is transient chrome: the next interaction of any + // kind takes it away rather than making the user wait it out. + self.input_state.dismiss_ocr_scan_result(); let target = self.classify_touch_surface(surface); let Some(screen_position) = self.touch_screen_position(surface, position, target) else { return TouchTarget::Other; diff --git a/src/backend/wayland/state/data.rs b/src/backend/wayland/state/data.rs index 72eb1439..4364da4c 100644 --- a/src/backend/wayland/state/data.rs +++ b/src/backend/wayland/state/data.rs @@ -238,6 +238,8 @@ pub struct StateData { pub(super) prev_color_picker_damage: Option, pub(super) prev_tool_preview_damage: Option, pub(super) prev_shape_measure_badge_damage: Option, + /// Union the OCR scan overlay covered last frame, so its sweep is cleared. + pub(super) prev_ocr_scan_damage: Option, /// Previous-frame strips for Measure Mode's crosshair, frame, and readout. pub(super) prev_measure_picker_damage: Vec, /// Idle-fade engine for the top-strip islands; its value is published @@ -334,6 +336,7 @@ impl StateData { prev_color_picker_damage: None, prev_tool_preview_damage: None, prev_shape_measure_badge_damage: None, + prev_ocr_scan_damage: None, prev_measure_picker_damage: Vec::new(), top_strip_fade: crate::ui::toolbar::snapshot::fade::TopStripFade::new(), shortcut_coach: super::onboarding::ShortcutCoachSession::default(), diff --git a/src/backend/wayland/state/ocr.rs b/src/backend/wayland/state/ocr.rs index 72606c14..7633ca86 100644 --- a/src/backend/wayland/state/ocr.rs +++ b/src/backend/wayland/state/ocr.rs @@ -7,6 +7,7 @@ use crate::backend::wayland::acquisition::{ScreenAcquisitionOutcome, ScreenAcquisitionOwner}; use crate::backend::wayland::zoom::ZoomWaiterOwner; +use crate::input::state::OcrScanOutcome; use crate::input::state::{ RegionInputSource, RegionPurposeTag, ScreenCaptureSource, Toast, ToastPriority, }; @@ -346,11 +347,36 @@ impl WaylandState { // The crop is taken while the capture is still held: releasing first // would leave the worker reading pixels that no longer exist. - match self.crop_ocr_selection(rect) { - Ok(pixels) => { - self.cancel_ocr(); - self.submit_ocr_request(pixels); - } + self.submit_ocr_for_rect(rect); + true + } + + /// Recognize an already-chosen rectangle — the whole image, from `Ctrl+A`. + pub(in crate::backend::wayland) fn submit_whole_image_ocr( + &mut self, + rect: crate::screen_pixels::ImagePixelRect, + ) { + self.submit_ocr_for_rect(rect); + } + + /// The one path from a chosen rectangle to a running recognition: map the + /// area the sweep will cover, crop while the source is still held, release + /// it, submit, and start the overlay. Both entry points share it so a fix + /// to one cannot leave the other behind. + fn submit_ocr_for_rect(&mut self, rect: crate::screen_pixels::ImagePixelRect) { + // Mapped before the region is released: the token is what turns the + // authoritative image rectangle into the surface pixels the sweep is + // painted over. + let scan_region = match self.data.active_screen_region { + Some(ActiveScreenRegion::Ready { source, .. }) => Some( + crate::backend::wayland::state::screen_image::screen_rect_for_image_rect( + &source, rect, + ), + ), + _ => None, + }; + let pixels = match self.crop_ocr_selection(rect) { + Ok(pixels) => pixels, Err(message) => { self.cancel_ocr(); self.input_state.push_toast( @@ -358,9 +384,20 @@ impl WaylandState { TOAST_SOURCE, Toast::warning(message), ); + return; } + }; + self.cancel_ocr(); + // The overlay starts only for a request that was actually accepted. A + // refused submission produces no completion, and an overlay waiting on + // one would sweep until something dismissed it. + let Some(id) = self.submit_ocr_request(pixels) else { + return; + }; + if let Some(region) = scan_region { + self.input_state + .begin_ocr_scan(id, region, std::time::Instant::now()); } - true } fn crop_ocr_selection( @@ -376,7 +413,12 @@ impl WaylandState { }) } - fn submit_ocr_request(&mut self, pixels: crate::screen_pixels::PackedArgb32) { + /// Hand the crop to the worker, reporting the accepted request. `None` + /// means nothing is running and no completion will arrive. + fn submit_ocr_request( + &mut self, + pixels: crate::screen_pixels::PackedArgb32, + ) -> Option { let languages = OcrLanguages::from_validated(self.config.capture.resolved_ocr_languages()); log::debug!( "OCR submitting {}x{} crop for languages {}", @@ -390,6 +432,7 @@ impl WaylandState { // the selection it publishes. self.suppress_focus_exit_for(std::time::Duration::from_millis(1500)); log::debug!("OCR request {id} started"); + Some(id) } Err(OcrSubmitError::Busy { .. }) => { self.input_state.push_toast( @@ -397,6 +440,7 @@ impl WaylandState { TOAST_SOURCE, Toast::info("OCR is already running"), ); + None } Err(error) => { log::warn!("Failed to start screen text recognition: {error}"); @@ -405,6 +449,7 @@ impl WaylandState { TOAST_SOURCE, Toast::warning("Could not start screen text recognition."), ); + None } } } @@ -423,6 +468,14 @@ impl WaylandState { }), } => { log::debug!("OCR request {id} copied {character_count} characters"); + self.input_state.settle_ocr_scan( + id, + OcrScanOutcome::Copied { + character_count, + replaced_invalid_utf8, + }, + std::time::Instant::now(), + ); let message = if replaced_invalid_utf8 { "Text copied (some characters were unreadable)" } else { @@ -435,9 +488,14 @@ impl WaylandState { ); } OcrPoll::Ready { + id, outcome: Ok(OcrSuccess::NoTextFound), - .. } => { + self.input_state.settle_ocr_scan( + id, + OcrScanOutcome::NoTextFound, + std::time::Instant::now(), + ); self.input_state.push_toast( ToastPriority::Info, TOAST_SOURCE, @@ -449,6 +507,11 @@ impl WaylandState { outcome: Err(failure), } => { log::warn!("OCR request {id} failed: {failure:?}"); + self.input_state.settle_ocr_scan( + id, + OcrScanOutcome::Failed, + std::time::Instant::now(), + ); let toast = match failure { OcrFailure::EngineMissing | OcrFailure::LanguageMissing { .. } => { Toast::warning(failure.message()) @@ -460,6 +523,11 @@ impl WaylandState { } OcrPoll::WorkerLost { id, reason } => { log::error!("OCR request {id} lost its worker: {reason}"); + self.input_state.settle_ocr_scan( + id, + OcrScanOutcome::Failed, + std::time::Instant::now(), + ); self.input_state.push_toast( ToastPriority::Critical, TOAST_SOURCE, diff --git a/src/backend/wayland/state/region_capture/delivery.rs b/src/backend/wayland/state/region_capture/delivery.rs index 1696e937..b76a75d4 100644 --- a/src/backend/wayland/state/region_capture/delivery.rs +++ b/src/backend/wayland/state/region_capture/delivery.rs @@ -204,10 +204,14 @@ impl WaylandState { }; self.clear_region_window_snap(); self.retire_region_selection_owner(self.input_state.region_state().selection_owner()); - if purpose == RegionPurposeTag::CaptureInteractive { - self.enter_region_review(rect); - } else { - self.submit_region_capture(rect); + match purpose { + RegionPurposeTag::CaptureInteractive => { + self.enter_region_review(rect); + } + RegionPurposeTag::Ocr => self.submit_whole_image_ocr(rect), + RegionPurposeTag::CaptureDeliver | RegionPurposeTag::Measure => { + self.submit_region_capture(rect); + } } } diff --git a/src/backend/wayland/state/region_capture/geometry.rs b/src/backend/wayland/state/region_capture/geometry.rs index d7a83a52..9b7b61a6 100644 --- a/src/backend/wayland/state/region_capture/geometry.rs +++ b/src/backend/wayland/state/region_capture/geometry.rs @@ -105,11 +105,14 @@ pub(super) fn selection_geometry( }) } +/// The whole displayed image, for the purposes that can submit one. Measure +/// has nothing to submit; capture and recognition both do, and reading a full +/// screen of text should not require dragging across the whole output. pub(super) fn whole_image_rect( purpose: RegionPurposeTag, bounds: (u32, u32), ) -> Option { - if !purpose.is_capture() { + if purpose == RegionPurposeTag::Measure { return None; } ImagePixelRect::whole(bounds) diff --git a/src/backend/wayland/state/region_capture/tests/review.rs b/src/backend/wayland/state/region_capture/tests/review.rs index 55a9c56d..3964a075 100644 --- a/src/backend/wayland/state/region_capture/tests/review.rs +++ b/src/backend/wayland/state/region_capture/tests/review.rs @@ -434,7 +434,7 @@ fn ocr_owner_loss_requests_its_existing_terminal_cancel_path() { } #[test] -fn whole_image_is_available_only_to_capture_purposes() { +fn whole_image_is_available_to_every_purpose_that_can_submit_one() { let capture = capture_region(); let RegionSelectionFinalize::Selected { purpose, rect } = capture .whole_image_selection() @@ -444,7 +444,26 @@ fn whole_image_is_available_only_to_capture_purposes() { }; assert_eq!(purpose, RegionPurposeTag::CaptureDeliver); assert_eq!((rect.x(), rect.y(), rect.size()), (0, 0, (100, 80))); - assert_eq!(ocr_region(1.0).whole_image_selection(), None); + + // Recognition wants the whole image as much as capture does: reading a + // full screen of text should not need a drag across the whole output. + let RegionSelectionFinalize::Selected { purpose, rect } = ocr_region(1.0) + .whole_image_selection() + .expect("ocr whole image") + else { + panic!("whole image must be a selected result") + }; + assert_eq!(purpose, RegionPurposeTag::Ocr); + assert_eq!((rect.x(), rect.y(), rect.size()), (0, 0, (100, 80))); + + // Measure has nothing to submit. + let measure = ActiveScreenRegion::Measure { + generation: 1, + bounds: (100, 80), + anchor: None, + edge: None, + }; + assert_eq!(measure.whole_image_selection(), None); } #[test] diff --git a/src/backend/wayland/state/render/mod.rs b/src/backend/wayland/state/render/mod.rs index 655796d7..b4becc32 100644 --- a/src/backend/wayland/state/render/mod.rs +++ b/src/backend/wayland/state/render/mod.rs @@ -104,6 +104,7 @@ impl WaylandState { blocked_feedback_active, text_edit_entry_active, input_hud_animating, + ocr_scan_animating, ) = record_stage!(advance_animations, { ( self.input_state.advance_click_highlights(now), @@ -112,6 +113,7 @@ impl WaylandState { self.input_state.advance_blocked_feedback(now), self.input_state.advance_text_edit_entry_feedback(now), self.input_state.advance_input_hud(now), + self.input_state.advance_ocr_scan(now), ) }); let ui_animation_active = highlight_active @@ -119,7 +121,8 @@ impl WaylandState { || ui_toast_active || blocked_feedback_active || text_edit_entry_active - || input_hud_animating; + || input_hud_animating + || ocr_scan_animating; self.update_ui_animation_tick(now, ui_animation_active); let keep_rendering = ui_animation_active && self.ui_animation_interval.is_none(); diff --git a/src/backend/wayland/state/render/ui.rs b/src/backend/wayland/state/render/ui.rs index 16680317..aa7c3f89 100644 --- a/src/backend/wayland/state/render/ui.rs +++ b/src/backend/wayland/state/render/ui.rs @@ -310,6 +310,10 @@ impl WaylandState { self.render_capture_picker(ctx, width, height); } + // The OCR scan overlay sits over the live view, above the toolbars + // it reports on but below the modal surfaces below. + self.render_ocr_scan(ctx, width, height); + // Modal overlays render last (on top of everything including toolbars) if !capture_picker { if let Some(card) = self.first_run_onboarding_card() { @@ -323,6 +327,25 @@ impl WaylandState { } } + /// The scan band while recognition runs, then the outcome card. The card + /// reports what happened, never the recognized text: keeping screen + /// contents out of application state is an invariant of `src/ocr`. + fn render_ocr_scan(&self, ctx: &cairo::Context, width: u32, height: u32) { + let Some(scan) = self.input_state.ocr_scan() else { + return; + }; + let now = std::time::Instant::now(); + if let Some((outcome, shown)) = scan.result(now) { + crate::ui::render_ocr_scan_result(ctx, scan.region(), outcome, shown, (width, height)); + } else if let Some(progress) = scan.sweep_progress(now) { + crate::ui::render_ocr_scan_sweep(ctx, scan.region(), progress); + } else if scan.is_scanning() { + // Reduced motion: the region is still marked as being read, just + // without a band travelling across it. + crate::ui::render_ocr_scan_still(ctx, scan.region()); + } + } + fn render_capture_picker(&self, ctx: &cairo::Context, width: u32, height: u32) { let purpose = self.input_state.region_state().purpose(); if !self.input_state.region_is_active() diff --git a/src/backend/wayland/state/render/ui_effect_damage.rs b/src/backend/wayland/state/render/ui_effect_damage.rs index f703e0cb..aaa14a8b 100644 --- a/src/backend/wayland/state/render/ui_effect_damage.rs +++ b/src/backend/wayland/state/render/ui_effect_damage.rs @@ -271,6 +271,25 @@ impl WaylandState { ); self.data.prev_shape_measure_badge_damage = measure_badge_rect; + // The scan overlay spans its region and, once settled, the outcome card + // beside it. Both move only when the phase changes, so the previous + // union is re-emitted to clear the sweep it leaves behind. + // Visibility, not animation: a static overlay under reduced motion + // still has to be damaged and cleared even though it asks for no + // frames of its own, so this keys off the overlay itself. + let ocr_scan_rect = self.input_state.ocr_scan().and_then(|scan| { + let outcome = scan + .result(std::time::Instant::now()) + .map(|(outcome, _)| outcome); + effect_rect( + crate::ui::ocr_scan_geometry(scan.region(), outcome, (width, height)), + width, + height, + ) + }); + push_effect_damage(&mut regions, self.data.prev_ocr_scan_damage, ocr_scan_rect); + self.data.prev_ocr_scan_damage = ocr_scan_rect; + let measure_picker_damage = if self.input_state.region_state().purpose() == Some(crate::input::state::RegionPurposeTag::Measure) { diff --git a/src/input/state/core/base/state/init.rs b/src/input/state/core/base/state/init.rs index a7f208ab..dab24347 100644 --- a/src/input/state/core/base/state/init.rs +++ b/src/input/state/core/base/state/init.rs @@ -259,6 +259,7 @@ impl InputState { show_preset_toasts: true, idle_fade: true, show_tool_preview: false, + ocr_scan: None, ui_toast: None, toast_queue: super::super::toast_queue::ToastQueue::default(), ui_toast_bounds: None, diff --git a/src/input/state/core/base/state/structs.rs b/src/input/state/core/base/state/structs.rs index 20f631a9..2a9ce7f6 100644 --- a/src/input/state/core/base/state/structs.rs +++ b/src/input/state/core/base/state/structs.rs @@ -502,6 +502,9 @@ pub struct InputState { pub idle_fade: bool, /// Whether to show the cursor tool preview bubble pub show_tool_preview: bool, + /// The scan-band overlay shown while screen text recognition runs, and the + /// outcome card that follows it. + pub(crate) ocr_scan: Option, /// Active (visible) UI toast (errors/warnings/info) pub(crate) ui_toast: Option, /// Pending toasts waiting behind the active one, plus rate-limit memory diff --git a/src/input/state/core/mod.rs b/src/input/state/core/mod.rs index 3d4ca2e0..f42b95d4 100644 --- a/src/input/state/core/mod.rs +++ b/src/input/state/core/mod.rs @@ -24,7 +24,7 @@ mod session_preflight_exact; mod status_hud; mod tool_controls; mod tour; -mod utility; +pub(crate) mod utility; mod zoom_chip; pub use base::{ diff --git a/src/input/state/core/utility/mod.rs b/src/input/state/core/utility/mod.rs index 1cd4857d..e87486fd 100644 --- a/src/input/state/core/utility/mod.rs +++ b/src/input/state/core/utility/mod.rs @@ -7,6 +7,7 @@ mod help_overlay; mod interaction; mod launcher; mod light_mode; +pub(crate) mod ocr_scan; mod pending; mod presenter_mode; mod render_profiles; diff --git a/src/input/state/core/utility/ocr_scan.rs b/src/input/state/core/utility/ocr_scan.rs new file mode 100644 index 00000000..b2482d29 --- /dev/null +++ b/src/input/state/core/utility/ocr_scan.rs @@ -0,0 +1,627 @@ +//! The scan overlay that runs while screen text recognition works. +//! +//! Recognition happens on a worker and can take a second or more, with nothing +//! on screen to say the region was even read. A band sweeps the selected +//! rectangle while the worker runs, and a short card reports the outcome when +//! it finishes. +//! +//! The card carries the outcome, never the recognized text: `src/ocr` keeps +//! recognized text out of application state entirely, and drawing it here would +//! put screen contents into the overlay's own UI. See `src/ocr/AGENTS.md`. + +use std::time::{Duration, Instant}; + +use super::super::base::InputState; +use crate::ocr::OcrRequestId; +use crate::util::Rect; + +/// One top-to-bottom pass of the scan band. +pub(crate) const SWEEP: Duration = Duration::from_millis(1200); +/// How long the outcome card stays up once the sweep lets it through. +const RESULT_LIFETIME: Duration = Duration::from_millis(4500); +/// The card fades over the last of its lifetime. +pub(crate) const RESULT_FADE: Duration = Duration::from_millis(450); + +/// What recognition produced, in the terms the card is allowed to show. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OcrScanOutcome { + Copied { + character_count: usize, + replaced_invalid_utf8: bool, + }, + NoTextFound, + Failed, +} + +impl OcrScanOutcome { + pub(crate) fn headline(self) -> &'static str { + match self { + Self::Copied { + replaced_invalid_utf8: false, + .. + } => "Copied to clipboard", + Self::Copied { + replaced_invalid_utf8: true, + .. + } => "Copied — some characters were unreadable", + Self::NoTextFound => "No text found", + Self::Failed => "Recognition failed", + } + } + + /// The detail line. Deliberately a count, not the text itself. + pub(crate) fn detail(self) -> Option { + match self { + Self::Copied { + character_count: 1, .. + } => Some("1 character".to_string()), + Self::Copied { + character_count, .. + } => Some(format!("{character_count} characters")), + Self::NoTextFound | Self::Failed => None, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OcrScanPhase { + /// The worker is running, or it has finished but the band has not reached + /// the end of its current pass. + Scanning { + /// The outcome and the instant the band may hand over, measured from + /// `started`. Fixed when recognition settles rather than recomputed per + /// tick: a deadline derived from the current elapsed time moves along + /// with it and is only ever reached on an exact sweep boundary, so the + /// band sweeps forever. + settled: Option<(OcrScanOutcome, Duration)>, + }, + Showing { + outcome: OcrScanOutcome, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct OcrScan { + /// The recognition this overlay is waiting on. A completion for anything + /// else belongs to a request whose overlay is already gone. + request: OcrRequestId, + /// The scanned rectangle in logical surface pixels. + region: Rect, + started: Instant, + phase: OcrScanPhase, +} + +impl OcrScan { + pub(crate) const fn region(self) -> Rect { + self.region + } + + /// Whether recognition is still being waited on, and so the region should + /// be marked as being read. + pub(crate) const fn is_scanning(self) -> bool { + matches!(self.phase, OcrScanPhase::Scanning { .. }) + } + + /// Progress of the band through its current pass, in `0.0..1.0`. `None` + /// under `[ui] reduced_motion`: the region is marked with a static + /// indicator instead of a moving band (WCAG 2.3.3). + pub(crate) fn sweep_progress(self, now: Instant) -> Option { + self.sweep_progress_for(now, crate::ui::anim::motion_enabled()) + } + + /// Split from the accessor so both motion settings can be exercised + /// without writing the process-wide flag, which parallel tests share. + pub(crate) fn sweep_progress_for(self, now: Instant, motion: bool) -> Option { + if !motion { + return None; + } + self.is_scanning().then(|| { + let elapsed = now.saturating_duration_since(self.started).as_secs_f64(); + let period = SWEEP.as_secs_f64(); + (elapsed % period) / period + }) + } + + /// The settled outcome and how long it has been up, once the band has + /// finished its pass. + pub(crate) fn result(self, now: Instant) -> Option<(OcrScanOutcome, Duration)> { + match self.phase { + OcrScanPhase::Showing { outcome } => { + Some((outcome, now.saturating_duration_since(self.started))) + } + OcrScanPhase::Scanning { .. } => None, + } + } +} + +impl InputState { + /// Start the sweep over `region` (logical surface pixels). + pub(crate) fn begin_ocr_scan(&mut self, request: OcrRequestId, region: Rect, now: Instant) { + self.ocr_scan = Some(OcrScan { + request, + region, + started: now, + phase: OcrScanPhase::Scanning { settled: None }, + }); + self.needs_redraw = true; + } + + /// Record what recognition produced. The card does not appear yet: the band + /// finishes the pass it is on first, and always at least one full pass, so + /// a fast recognition cannot cut the sweep off mid-screen. + pub(crate) fn settle_ocr_scan( + &mut self, + request: OcrRequestId, + outcome: OcrScanOutcome, + now: Instant, + ) { + self.settle_ocr_scan_for(request, outcome, now, crate::ui::anim::motion_enabled()); + } + + pub(crate) fn settle_ocr_scan_for( + &mut self, + request: OcrRequestId, + outcome: OcrScanOutcome, + now: Instant, + motion: bool, + ) { + let Some(scan) = self.ocr_scan.as_mut() else { + return; + }; + if scan.request != request { + return; + } + let elapsed = now.saturating_duration_since(scan.started); + if let OcrScanPhase::Scanning { settled } = &mut scan.phase + && settled.is_none() + { + *settled = Some((outcome, completed_sweep(elapsed, motion))); + self.needs_redraw = true; + } + } + + /// Take away a finished outcome card. A sweep still waiting on the worker + /// is left alone: it is progress feedback for work that is still running, + /// and dropping it on a stray keystroke would also discard the result that + /// recognition is about to report. + pub(crate) fn dismiss_ocr_scan_result(&mut self) -> bool { + let Some(scan) = self.ocr_scan else { + return false; + }; + if scan.is_scanning() { + return false; + } + self.ocr_scan = None; + self.needs_redraw = true; + true + } + + pub(crate) const fn ocr_scan(&self) -> Option { + self.ocr_scan + } + + /// When the overlay next changes on its own, for a loop that is not + /// already ticking for animation. + /// + /// `None` while something is moving — the animation tick covers that — and + /// `None` during a still scan, whose next change is the worker completing, + /// which wakes the loop itself. A still card is the one case that needs a + /// deadline: nothing else will wake the loop to expire it. + pub(crate) fn ocr_scan_wake_after(&self, now: Instant) -> Option { + self.ocr_scan_wake_after_for(now, crate::ui::anim::motion_enabled()) + } + + pub(crate) fn ocr_scan_wake_after_for(&self, now: Instant, motion: bool) -> Option { + if motion { + return None; + } + let scan = self.ocr_scan?; + let (_, shown) = scan.result(now)?; + Some(RESULT_LIFETIME.saturating_sub(shown)) + } + + /// Whether a still card has outlived its deadline and needs the frame that + /// takes it away. + pub(crate) fn ocr_scan_due(&self, now: Instant) -> bool { + self.ocr_scan_wake_after(now) == Some(Duration::ZERO) + } + + /// Advance the overlay, reporting whether it needs *continuous* frames. + /// + /// Only movement answers yes. Under `[ui] reduced_motion` the overlay is + /// static, so it is rendered and damaged like any other chrome but does not + /// pin a repaint at the animation frame rate for the length of a + /// recognition; `ocr_scan_wake_after` supplies the one deadline it needs. + pub fn advance_ocr_scan(&mut self, now: Instant) -> bool { + self.advance_ocr_scan_for(now, crate::ui::anim::motion_enabled()) + } + + pub(crate) fn advance_ocr_scan_for(&mut self, now: Instant, motion: bool) -> bool { + let Some(scan) = self.ocr_scan else { + return false; + }; + let elapsed = now.saturating_duration_since(scan.started); + let animating = motion; + match scan.phase { + OcrScanPhase::Scanning { settled: None } => animating, + OcrScanPhase::Scanning { + settled: Some((outcome, deadline)), + } => { + if elapsed < deadline { + return true; + } + self.ocr_scan = Some(OcrScan { + request: scan.request, + region: scan.region, + started: now, + phase: OcrScanPhase::Showing { outcome }, + }); + self.needs_redraw = true; + animating + } + OcrScanPhase::Showing { .. } => { + if elapsed < RESULT_LIFETIME { + return animating; + } + self.ocr_scan = None; + self.needs_redraw = true; + false + } + } + } +} + +/// The end of the pass `elapsed` falls in, never earlier than one full pass. +/// +/// Zero under `[ui] reduced_motion`: with no band sweeping there is nothing to +/// let finish, so the outcome is shown as soon as it arrives. +fn completed_sweep(elapsed: Duration, motion: bool) -> Duration { + if !motion { + return Duration::ZERO; + } + let sweep = SWEEP.as_millis().max(1); + let elapsed_ms = elapsed.as_millis(); + let passes = elapsed_ms.div_ceil(sweep).max(1); + Duration::from_millis(u64::try_from(passes * sweep).unwrap_or(u64::MAX)) +} + +/// Opacity of the outcome card `shown` into its lifetime. +pub(crate) fn result_opacity(shown: Duration) -> f64 { + result_opacity_for(shown, crate::ui::anim::motion_enabled()) +} + +/// Split from the accessor so both motion settings can be exercised without +/// writing the process-wide flag, which every parallel test shares. +/// +/// A card past its lifetime is gone either way; only the gradual fade into +/// that is animation, and reduced motion skips it by cutting straight from +/// fully opaque to nothing. +pub(crate) fn result_opacity_for(shown: Duration, motion: bool) -> f64 { + let remaining = RESULT_LIFETIME.saturating_sub(shown); + if remaining.is_zero() { + return 0.0; + } + if !motion || remaining >= RESULT_FADE { + return 1.0; + } + (remaining.as_secs_f64() / RESULT_FADE.as_secs_f64()).clamp(0.0, 1.0) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::input::state::test_support::make_test_input_state; + + fn request() -> OcrRequestId { + OcrRequestId::for_test(1) + } + + fn region() -> Rect { + Rect::new(100, 80, 240, 160).expect("a scan region") + } + + #[test] + fn the_band_sweeps_while_recognition_runs() { + let mut state = make_test_input_state(); + let start = Instant::now(); + state.begin_ocr_scan(request(), region(), start); + + let scan = state.ocr_scan().expect("a scan is up"); + assert_eq!(scan.region(), region()); + assert_eq!(scan.sweep_progress(start), Some(0.0)); + assert_eq!(scan.result(start), None, "nothing to report yet"); + + let half = start + SWEEP / 2; + let progress = scan.sweep_progress(half).expect("still scanning"); + assert!((progress - 0.5).abs() < 1e-6, "half way down: {progress}"); + + // The band restarts rather than stopping: the worker sets the pace. + let next = start + SWEEP + SWEEP / 4; + let progress = scan.sweep_progress(next).expect("still scanning"); + assert!((progress - 0.25).abs() < 1e-6, "second pass: {progress}"); + + assert!(state.advance_ocr_scan(half), "keeps asking for frames"); + } + + #[test] + fn a_fast_recognition_still_gets_a_whole_sweep_before_the_card() { + let mut state = make_test_input_state(); + let start = Instant::now(); + state.begin_ocr_scan(request(), region(), start); + state.settle_ocr_scan( + request(), + OcrScanOutcome::Copied { + character_count: 12, + replaced_invalid_utf8: false, + }, + start, + ); + + // Recognition finished almost immediately; the card must wait. + let early = start + Duration::from_millis(80); + assert!(state.advance_ocr_scan(early)); + let scan = state.ocr_scan().expect("still scanning"); + assert_eq!(scan.result(early), None, "a card here would cut the sweep"); + assert!(scan.sweep_progress(early).is_some()); + + // One full pass later it is allowed through. + let settled = start + SWEEP; + assert!(state.advance_ocr_scan(settled)); + let scan = state.ocr_scan().expect("the card is up"); + assert_eq!(scan.sweep_progress(settled), None, "the band is done"); + let (outcome, shown) = scan.result(settled).expect("an outcome"); + assert_eq!( + outcome, + OcrScanOutcome::Copied { + character_count: 12, + replaced_invalid_utf8: false, + } + ); + assert_eq!(shown, Duration::ZERO); + } + + #[test] + fn a_slow_recognition_finishes_the_pass_it_lands_in() { + let mut state = make_test_input_state(); + let start = Instant::now(); + state.begin_ocr_scan(request(), region(), start); + + // Settles a third of the way through the third pass. + let landed = start + SWEEP * 2 + SWEEP / 3; + state.settle_ocr_scan(request(), OcrScanOutcome::NoTextFound, landed); + assert!(state.advance_ocr_scan(landed)); + assert_eq!( + state.ocr_scan().and_then(|scan| scan.result(landed)), + None, + "the third pass is still running" + ); + + let completed = start + SWEEP * 3; + assert!(state.advance_ocr_scan(completed)); + assert!( + state + .ocr_scan() + .and_then(|scan| scan.result(completed)) + .is_some(), + "the card appears at the end of that pass, not the next one" + ); + } + + #[test] + fn the_band_hands_over_on_a_real_tick_not_only_on_an_exact_boundary() { + // The event loop ticks on a frame clock, so `elapsed` lands wherever it + // lands. A hand-over deadline recomputed from the current elapsed time + // moves along with it and is only ever met on an exact multiple of the + // sweep, which a real tick almost never hits — the band would sweep on + // forever. Drive it the way the loop does and require it to settle. + let mut state = make_test_input_state(); + let start = Instant::now(); + state.begin_ocr_scan(request(), region(), start); + state.settle_ocr_scan( + request(), + OcrScanOutcome::Copied { + character_count: 7, + replaced_invalid_utf8: false, + }, + start + Duration::from_millis(137), + ); + + let tick = Duration::from_millis(17); + let mut now = start; + let mut handed_over = None; + for _ in 0..500 { + now += tick; + state.advance_ocr_scan(now); + if let Some(scan) = state.ocr_scan() + && scan.result(now).is_some() + { + handed_over = Some(now.saturating_duration_since(start)); + break; + } + } + + let handed_over = handed_over.expect("the band must stop sweeping and show the card"); + assert!( + handed_over >= SWEEP, + "at least one full pass: {handed_over:?}" + ); + assert!( + handed_over < SWEEP * 2, + "and no more than the pass it settled in: {handed_over:?}" + ); + } + + #[test] + fn the_card_expires_on_its_own_and_stops_asking_for_frames() { + let mut state = make_test_input_state(); + let start = Instant::now(); + state.begin_ocr_scan(request(), region(), start); + state.settle_ocr_scan(request(), OcrScanOutcome::NoTextFound, start); + let settled = start + SWEEP; + assert!(state.advance_ocr_scan(settled)); + + assert!(state.advance_ocr_scan(settled + RESULT_LIFETIME / 2)); + assert!(state.ocr_scan().is_some()); + + assert!(!state.advance_ocr_scan(settled + RESULT_LIFETIME)); + assert!(state.ocr_scan().is_none(), "it clears itself"); + assert!(!state.advance_ocr_scan(settled + RESULT_LIFETIME)); + } + + #[test] + fn interaction_takes_the_finished_card_but_leaves_a_running_sweep() { + let mut state = make_test_input_state(); + let start = Instant::now(); + state.begin_ocr_scan(request(), region(), start); + + // Recognition is still running: the sweep is progress feedback for work + // in flight, and dropping it would also discard the result about to + // arrive. A stray keystroke must not do that. + assert!(!state.dismiss_ocr_scan_result(), "the sweep is not a card"); + assert!(state.ocr_scan().is_some(), "and it stays up"); + + state.settle_ocr_scan(request(), OcrScanOutcome::NoTextFound, start); + assert!( + !state.dismiss_ocr_scan_result(), + "still sweeping until the pass completes" + ); + + let settled = start + SWEEP; + assert!(state.advance_ocr_scan(settled)); + assert!( + state.dismiss_ocr_scan_result(), + "the card can be taken away" + ); + assert!(state.ocr_scan().is_none()); + assert!(!state.dismiss_ocr_scan_result(), "nothing left to dismiss"); + } + + #[test] + fn a_completion_for_another_request_leaves_this_sweep_alone() { + // Capacity one makes overlapping requests unlikely, but a completion + // that outlives its own overlay must not settle a newer one with a + // stale outcome. + let mut state = make_test_input_state(); + let start = Instant::now(); + state.begin_ocr_scan(request(), region(), start); + state.settle_ocr_scan(OcrRequestId::for_test(99), OcrScanOutcome::Failed, start); + + assert!(state.advance_ocr_scan(start + SWEEP * 2)); + assert_eq!( + state + .ocr_scan() + .and_then(|scan| scan.result(start + SWEEP * 2)), + None, + "the sweep is still waiting on its own request" + ); + } + + #[test] + fn settling_without_a_scan_is_inert() { + // A recognition can outlive its overlay when the user dismisses first. + let mut state = make_test_input_state(); + state.settle_ocr_scan(request(), OcrScanOutcome::Failed, Instant::now()); + assert!(state.ocr_scan().is_none()); + assert!(!state.advance_ocr_scan(Instant::now())); + } + + #[test] + fn a_still_overlay_asks_for_no_frames_and_shows_its_result_at_once() { + // Reduced motion is passed in rather than written to the process-wide + // flag, which every parallel test shares. + const STILL: bool = false; + let mut state = make_test_input_state(); + let start = Instant::now(); + state.begin_ocr_scan(request(), region(), start); + + // Nothing is moving, so nothing should pin a repaint while the worker + // runs; the completion is what wakes the loop. + assert!( + !state.advance_ocr_scan_for(start, STILL), + "no frames for a still scan" + ); + assert_eq!( + state.ocr_scan_wake_after_for(start, STILL), + None, + "and no deadline either" + ); + assert_eq!( + state + .ocr_scan() + .and_then(|scan| scan.sweep_progress_for(start, STILL)), + None, + "no band travels across the region" + ); + assert!(state.ocr_scan().is_some_and(|scan| scan.is_scanning())); + + // With no sweep to finish, the outcome is shown as soon as it arrives + // rather than waiting out an animation that never ran. + state.settle_ocr_scan_for(request(), OcrScanOutcome::NoTextFound, start, STILL); + assert!(!state.advance_ocr_scan_for(start, STILL)); + assert!( + state + .ocr_scan() + .and_then(|scan| scan.result(start)) + .is_some(), + "no pass to wait for" + ); + + // The one thing a still card needs is a deadline to be taken away on. + let wake = state + .ocr_scan_wake_after_for(start, STILL) + .expect("a deadline"); + assert!(wake > Duration::ZERO && wake <= RESULT_LIFETIME); + assert!( + state.ocr_scan_wake_after_for(start + RESULT_LIFETIME, STILL) == Some(Duration::ZERO), + "due at the end of its life" + ); + assert!(!state.advance_ocr_scan_for(start + RESULT_LIFETIME, STILL)); + assert!(state.ocr_scan().is_none(), "and it is taken away"); + } + + #[test] + fn the_card_fades_over_the_end_of_its_life_and_never_shows_text() { + // Both settings are passed in rather than written to the process-wide + // flag, which every parallel test shares. + assert_eq!(result_opacity_for(Duration::ZERO, true), 1.0); + assert_eq!(result_opacity_for(RESULT_LIFETIME - RESULT_FADE, true), 1.0); + let mid = result_opacity_for(RESULT_LIFETIME - RESULT_FADE / 2, true); + assert!( + (0.0..1.0).contains(&mid), + "part way through the fade: {mid}" + ); + + // Reduced motion cuts straight from opaque to gone: no gradual fade, + // but the card still stops being painted once its life is over. + assert_eq!( + result_opacity_for(RESULT_LIFETIME - RESULT_FADE / 2, false), + 1.0 + ); + for motion in [true, false] { + assert_eq!( + result_opacity_for(RESULT_LIFETIME, motion), + 0.0, + "an expired card paints nothing either way" + ); + } + + // The card's own words: an outcome and a count, never a transcript. + let copied = OcrScanOutcome::Copied { + character_count: 42, + replaced_invalid_utf8: false, + }; + assert_eq!(copied.headline(), "Copied to clipboard"); + assert_eq!(copied.detail().as_deref(), Some("42 characters")); + assert_eq!( + OcrScanOutcome::Copied { + character_count: 1, + replaced_invalid_utf8: false, + } + .detail() + .as_deref(), + Some("1 character") + ); + assert_eq!(OcrScanOutcome::NoTextFound.detail(), None); + assert_eq!(OcrScanOutcome::Failed.headline(), "Recognition failed"); + } +} diff --git a/src/input/state/mod.rs b/src/input/state/mod.rs index 74c90451..d32b3c7f 100644 --- a/src/input/state/mod.rs +++ b/src/input/state/mod.rs @@ -19,6 +19,7 @@ pub(crate) use core::board_picker::{ #[cfg(test)] pub(crate) use core::build_text_input_preview; pub use core::color_picker_popup::{HEX_INPUT_MAX_CHARS, color_to_hex, parse_hex_color}; +pub(crate) use core::utility::ocr_scan::{OcrScanOutcome, result_opacity}; #[allow(unused_imports)] pub use core::{ BLOCKED_ACTION_DURATION_MS, BoardPickerCursorHint, BoardPickerLayout, diff --git a/src/ocr/controller.rs b/src/ocr/controller.rs index b9bd185e..8f629808 100644 --- a/src/ocr/controller.rs +++ b/src/ocr/controller.rs @@ -20,6 +20,15 @@ use super::{ #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub(crate) struct OcrRequestId(u64); +impl OcrRequestId { + /// A request identity for tests that drive consumers of a completion + /// without running the controller that would mint one. + #[cfg(test)] + pub(crate) const fn for_test(value: u64) -> Self { + Self(value) + } +} + impl fmt::Display for OcrRequestId { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(formatter) diff --git a/src/ocr/mod.rs b/src/ocr/mod.rs index c3064bd5..7080e99b 100644 --- a/src/ocr/mod.rs +++ b/src/ocr/mod.rs @@ -14,7 +14,7 @@ use crate::screen_pixels::PackedArgb32; mod controller; mod tesseract; -pub(crate) use controller::{OcrController, OcrPoll, OcrSubmitError}; +pub(crate) use controller::{OcrController, OcrPoll, OcrRequestId, OcrSubmitError}; pub(crate) use tesseract::{TesseractRecognizer, WlCopyPublisher}; /// Turns encoded image bytes into text. The production implementation shells diff --git a/src/ui.rs b/src/ui.rs index 58dddfeb..7f360edd 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -10,6 +10,7 @@ mod eyedropper_loupe; mod help_overlay; mod input_hud; mod measure_badge; +mod ocr_scan; mod onboarding_card; mod precision_entry; mod primitives; @@ -41,6 +42,9 @@ pub use input_hud::{input_hud_geometry, render_input_hud}; pub(crate) use measure_badge::{ ShapeMeasureBadge, measure_shape_badge, shape_measure_badge_text_style, }; +pub(crate) use ocr_scan::{ + ocr_scan_geometry, render_ocr_scan_result, render_ocr_scan_still, render_ocr_scan_sweep, +}; pub use onboarding_card::{OnboardingCard, OnboardingChecklistItem, render_onboarding_card}; pub use precision_entry::render_precision_entry_popup; /// Shared measured-text trimming, also used by the standalone about dialog. diff --git a/src/ui/ocr_scan.rs b/src/ui/ocr_scan.rs new file mode 100644 index 00000000..936a01d3 --- /dev/null +++ b/src/ui/ocr_scan.rs @@ -0,0 +1,401 @@ +use crate::input::state::{OcrScanOutcome, result_opacity}; +use crate::ui::theme::{self, Rgba, overlay}; +use crate::util::Rect; + +use super::primitives::draw_rounded_rect; +use crate::ui_text::{UiTextStyle, measure_text, text_layout}; + +/// Tint held over the region for the whole sweep, so the scanned area stays +/// identifiable even at the moment the band is off its top edge. +const TINT: Rgba = theme::rgba(theme::ACCENT_RGB, 0.14); +const FRAME: Rgba = theme::rgba(theme::ACCENT_RGB, 0.95); +const FRAME_WIDTH: f64 = 1.5; +/// The band is a fraction of the region's height, bounded so it neither +/// disappears on a tall region nor swamps a short one. +const BAND_FRACTION: f64 = 0.35; +const BAND_MIN: f64 = 18.0; +const BAND_MAX: f64 = 64.0; + +const CARD_RADIUS: f64 = overlay::RADIUS_LG; +const CARD_PAD: f64 = 12.0; +const CARD_GAP: f64 = 6.0; +const CARD_MARGIN: f64 = 12.0; +const CARD_OFFSET: f64 = 14.0; +const HEADLINE_SIZE: f64 = 12.5; +const DETAIL_SIZE: f64 = 11.0; + +/// Where the outcome card sits, in logical surface pixels. +#[derive(Debug, Clone, Copy, PartialEq)] +pub(crate) struct OcrScanCard { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +fn headline_style() -> UiTextStyle<'static> { + UiTextStyle { + family: "Sans", + slant: cairo::FontSlant::Normal, + weight: cairo::FontWeight::Bold, + size: HEADLINE_SIZE, + } +} + +fn detail_style() -> UiTextStyle<'static> { + UiTextStyle { + family: "Sans", + slant: cairo::FontSlant::Normal, + weight: cairo::FontWeight::Normal, + size: DETAIL_SIZE, + } +} + +/// Card text block size, measured without a drawing context so damage +/// geometry and the drawn card cannot disagree. Both go through the shared +/// measurement cache. +fn card_text_size(outcome: OcrScanOutcome) -> Option<(f64, f64)> { + let headline = measure_text(headline_style(), outcome.headline(), None)?; + let Some(detail) = outcome.detail() else { + return Some((headline.width(), headline.height())); + }; + let detail = measure_text(detail_style(), &detail, None)?; + Some(( + headline.width().max(detail.width()), + headline.height() + CARD_GAP + detail.height(), + )) +} + +/// Place the outcome card just under the scanned region, flipping above it when +/// there is no room and clamping onto the surface either way. +pub(crate) fn ocr_scan_card( + region: Rect, + outcome: OcrScanOutcome, + screen: (u32, u32), +) -> Option { + let (text_width, text_height) = card_text_size(outcome)?; + let screen_width = f64::from(screen.0); + let screen_height = f64::from(screen.1); + let width = (text_width + CARD_PAD * 2.0).min((screen_width - CARD_MARGIN * 2.0).max(1.0)); + let height = (text_height + CARD_PAD * 2.0).min((screen_height - CARD_MARGIN * 2.0).max(1.0)); + + let bottom = f64::from(region.y.saturating_add(region.height)); + let below = bottom + CARD_OFFSET; + let preferred_y = if below + height + CARD_MARGIN <= screen_height { + below + } else { + f64::from(region.y) - CARD_OFFSET - height + }; + Some(OcrScanCard { + x: f64::from(region.x).clamp( + CARD_MARGIN, + (screen_width - width - CARD_MARGIN).max(CARD_MARGIN), + ), + y: preferred_y.clamp( + CARD_MARGIN, + (screen_height - height - CARD_MARGIN).max(CARD_MARGIN), + ), + width, + height, + }) +} + +/// Union of everything the overlay paints, for targeted damage. +pub(crate) fn ocr_scan_geometry( + region: Rect, + outcome: Option, + screen: (u32, u32), +) -> (f64, f64, f64, f64) { + let region_box = ( + f64::from(region.x) - FRAME_WIDTH, + f64::from(region.y) - FRAME_WIDTH, + f64::from(region.width) + FRAME_WIDTH * 2.0, + f64::from(region.height) + FRAME_WIDTH * 2.0, + ); + let Some(card) = outcome.and_then(|outcome| ocr_scan_card(region, outcome, screen)) else { + return region_box; + }; + let left = region_box.0.min(card.x); + let top = region_box.1.min(card.y); + let right = (region_box.0 + region_box.2).max(card.x + card.width); + let bottom = (region_box.1 + region_box.3).max(card.y + card.height); + (left, top, right - left, bottom - top) +} + +/// The region marked as being read, with no moving parts. Used under +/// `[ui] reduced_motion`, where a band sweeping the screen is exactly the kind +/// of motion the setting exists to suppress (WCAG 2.3.3). +pub(crate) fn render_ocr_scan_still(ctx: &cairo::Context, region: Rect) { + if region.width <= 0 || region.height <= 0 { + return; + } + let _ = ctx.save(); + theme::set_color(ctx, TINT); + ctx.rectangle( + f64::from(region.x), + f64::from(region.y), + f64::from(region.width), + f64::from(region.height), + ); + let _ = ctx.fill(); + let _ = ctx.restore(); + draw_frame( + ctx, + f64::from(region.x), + f64::from(region.y), + f64::from(region.width), + f64::from(region.height), + 1.0, + ); +} + +/// The sweeping band, drawn while recognition runs. `progress` walks `0.0..1.0` +/// once per pass. +pub(crate) fn render_ocr_scan_sweep(ctx: &cairo::Context, region: Rect, progress: f64) { + if region.width <= 0 || region.height <= 0 { + return; + } + let x = f64::from(region.x); + let y = f64::from(region.y); + let width = f64::from(region.width); + let height = f64::from(region.height); + + let _ = ctx.save(); + theme::set_color(ctx, TINT); + ctx.rectangle(x, y, width, height); + let _ = ctx.fill(); + + // The band enters from above the region and leaves below it, so the sweep + // covers the top and bottom edges instead of appearing to start inset. + let band = (height * BAND_FRACTION).clamp(BAND_MIN, BAND_MAX); + let band_y = y - band + progress.clamp(0.0, 1.0) * (height + band); + ctx.rectangle(x, y, width, height); + ctx.clip(); + let gradient = build_band(band_y, band); + let _ = ctx.set_source(&gradient); + ctx.rectangle(x, band_y, width, band); + let _ = ctx.fill(); + let _ = ctx.restore(); + + draw_frame(ctx, x, y, width, height, 1.0); +} + +fn build_band(top: f64, height: f64) -> cairo::LinearGradient { + let gradient = cairo::LinearGradient::new(0.0, top, 0.0, top + height); + let (r, g, b) = theme::ACCENT_RGB; + gradient.add_color_stop_rgba(0.0, r, g, b, 0.0); + gradient.add_color_stop_rgba(0.8, r, g, b, 0.5); + gradient.add_color_stop_rgba(1.0, 1.0, 1.0, 1.0, 0.9); + gradient +} + +fn draw_frame(ctx: &cairo::Context, x: f64, y: f64, width: f64, height: f64, alpha: f64) { + theme::set_color(ctx, (FRAME.0, FRAME.1, FRAME.2, FRAME.3 * alpha)); + ctx.set_line_width(FRAME_WIDTH); + ctx.rectangle(x, y, width, height); + let _ = ctx.stroke(); +} + +/// The outcome card. `shown` is how long it has been up, which drives its fade. +pub(crate) fn render_ocr_scan_result( + ctx: &cairo::Context, + region: Rect, + outcome: OcrScanOutcome, + shown: std::time::Duration, + screen: (u32, u32), +) { + let opacity = result_opacity(shown); + if opacity <= 0.0 { + return; + } + let Some(card) = ocr_scan_card(region, outcome, screen) else { + return; + }; + let _ = ctx.save(); + ctx.push_group(); + + draw_frame( + ctx, + f64::from(region.x), + f64::from(region.y), + f64::from(region.width), + f64::from(region.height), + 1.0, + ); + + theme::set_color(ctx, crate::ui::theme::popup::bg_context_menu()); + draw_rounded_rect(ctx, card.x, card.y, card.width, card.height, CARD_RADIUS); + let _ = ctx.fill(); + theme::set_color(ctx, crate::ui::theme::popup::border_context_menu()); + ctx.set_line_width(1.0); + draw_rounded_rect( + ctx, + card.x + 0.5, + card.y + 0.5, + card.width - 1.0, + card.height - 1.0, + CARD_RADIUS - 0.5, + ); + let _ = ctx.stroke(); + + let _ = ctx.save(); + ctx.rectangle(card.x, card.y, card.width, card.height); + ctx.clip(); + let headline = text_layout(ctx, headline_style(), outcome.headline(), None); + let headline_extents = headline.ink_extents(); + theme::set_color(ctx, overlay::TEXT_PRIMARY); + headline.show_at_baseline( + ctx, + card.x + CARD_PAD - headline_extents.x_bearing(), + card.y + CARD_PAD - headline_extents.y_bearing(), + ); + if let Some(detail) = outcome.detail() { + let layout = text_layout(ctx, detail_style(), &detail, None); + let extents = layout.ink_extents(); + theme::set_color(ctx, overlay::TEXT_TERTIARY); + layout.show_at_baseline( + ctx, + card.x + CARD_PAD - extents.x_bearing(), + card.y + CARD_PAD + headline_extents.height() + CARD_GAP - extents.y_bearing(), + ); + } + let _ = ctx.restore(); + + let _ = ctx.pop_group_to_source(); + let _ = ctx.paint_with_alpha(opacity); + let _ = ctx.restore(); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn region() -> Rect { + Rect::new(60, 50, 200, 140).expect("a scan region") + } + + fn copied() -> OcrScanOutcome { + OcrScanOutcome::Copied { + character_count: 128, + replaced_invalid_utf8: false, + } + } + + fn alpha_at(surface: &mut cairo::ImageSurface, x: usize, y: usize) -> u8 { + surface.flush(); + let stride = surface.stride() as usize; + let data = surface.data().expect("pixels"); + data[y * stride + x * 4 + 3] + } + + #[test] + fn the_band_covers_the_regions_edges_at_the_ends_of_its_pass() { + // The band enters from above and leaves below, so a sweep touches the + // first and last rows rather than starting inset. + let render = |progress: f64| { + let surface = cairo::ImageSurface::create(cairo::Format::ARgb32, 320, 260).unwrap(); + let ctx = cairo::Context::new(&surface).unwrap(); + render_ocr_scan_sweep(&ctx, region(), progress); + drop(ctx); + surface + }; + + let mut start = render(0.0); + assert!(alpha_at(&mut start, 160, 51) > 0, "the top row is lit"); + let mut end = render(1.0); + assert!(alpha_at(&mut end, 160, 188) > 0, "the bottom row is lit"); + + // The tint holds over the whole region throughout, so the scanned area + // stays identifiable between passes. + let mut mid = render(0.5); + assert!(alpha_at(&mut mid, 160, 60) > 0); + assert_eq!(alpha_at(&mut mid, 10, 10), 0, "nothing outside the region"); + } + + #[test] + fn the_still_indicator_marks_the_region_without_a_band() { + let surface = cairo::ImageSurface::create(cairo::Format::ARgb32, 320, 260).unwrap(); + let ctx = cairo::Context::new(&surface).unwrap(); + render_ocr_scan_still(&ctx, region()); + drop(ctx); + let mut surface = surface; + + // The tint and frame mark the area, evenly: no row is brighter than + // another, because nothing is travelling across it. + assert!(alpha_at(&mut surface, 160, 60) > 0, "the region is tinted"); + assert_eq!( + alpha_at(&mut surface, 160, 60), + alpha_at(&mut surface, 160, 180), + "no band, so every row inside reads the same" + ); + assert_eq!(alpha_at(&mut surface, 10, 10), 0, "nothing outside"); + } + + #[test] + fn the_card_sits_under_the_region_and_flips_above_it_when_it_must() { + let below = ocr_scan_card(region(), copied(), (320, 400)).expect("a card"); + assert!( + below.y >= f64::from(region().y + region().height), + "the usual place is under the scanned area" + ); + + let low = Rect::new(60, 300, 200, 90).expect("a low region"); + let flipped = ocr_scan_card(low, copied(), (320, 400)).expect("a card"); + assert!( + flipped.y + flipped.height <= f64::from(low.y), + "no room below, so it goes above" + ); + } + + #[test] + fn a_card_with_nowhere_to_go_is_still_placed_on_the_surface() { + let full = Rect::new(0, 0, 200, 200).expect("a full-surface region"); + let card = ocr_scan_card(full, copied(), (200, 200)).expect("a card"); + assert!(card.x >= 0.0 && card.y >= 0.0); + assert!(card.x + card.width <= 200.0); + assert!(card.y + card.height <= 200.0); + } + + #[test] + fn damage_covers_the_region_alone_while_scanning_and_the_card_once_settled() { + let scanning = ocr_scan_geometry(region(), None, (320, 400)); + assert!(scanning.0 <= f64::from(region().x)); + assert!(scanning.1 <= f64::from(region().y)); + assert!(scanning.0 + scanning.2 >= f64::from(region().x + region().width)); + + let settled = ocr_scan_geometry(region(), Some(copied()), (320, 400)); + let card = ocr_scan_card(region(), copied(), (320, 400)).expect("a card"); + assert!( + settled.1 + settled.3 >= card.y + card.height, + "the union has to reach the card or its pixels are never cleared" + ); + assert!(settled.3 > scanning.3, "settling grows the damaged area"); + } + + #[test] + fn the_result_card_paints_its_outcome_and_fades_to_nothing() { + let render = |shown: Duration| { + let surface = cairo::ImageSurface::create(cairo::Format::ARgb32, 320, 400).unwrap(); + let ctx = cairo::Context::new(&surface).unwrap(); + render_ocr_scan_result(&ctx, region(), copied(), shown, (320, 400)); + drop(ctx); + surface + }; + let card = ocr_scan_card(region(), copied(), (320, 400)).expect("a card"); + let probe = ( + (card.x + card.width / 2.0) as usize, + (card.y + card.height / 2.0) as usize, + ); + + let mut fresh = render(Duration::ZERO); + assert!(alpha_at(&mut fresh, probe.0, probe.1) > 0, "the card is up"); + + let mut expired = render(Duration::from_secs(60)); + assert_eq!( + alpha_at(&mut expired, probe.0, probe.1), + 0, + "an expired card paints nothing at all, whatever the motion setting" + ); + } +} From 822f600d498119362eb152df226c3088b641ee22 Mon Sep 17 00:00:00 2001 From: devmobasa <4170275+devmobasa@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:17:58 +0200 Subject: [PATCH 2/4] feat(ocr): teach the recognition selector its keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whole-image recognition is a keystroke with nothing on screen to suggest it, so `Ctrl+A` was findable only by reading the documentation. The gap was wider than one key. Recognition never went through the capture picker at all — `render_capture_picker` gates on capture and measure purposes, so OCR falls to its own overlay of a scrim and scan brackets — and every affordance the picker gained, the hint strip included, simply never reached it. The strip is now shared rather than copied, so the two selectors cannot drift apart in wording or position, and it honours the same `show_legend` setting. Its text is its own: recognition's selection policy sets `allow_square: false`, so advertising `Shift: square` would be a lie, and `Ctrl+A` reads everything rather than selecting everything. Dismissal and damage both came for free, which is the sign the seam was in the right place: `begin_selection` already clears the legend for any purpose, and `mark_region_dirty` already marks full damage for every purpose but measure, so the frame that dismisses the strip is the frame that clears it. --- docs/CONFIG.md | 4 ++- src/backend/wayland/state/ocr.rs | 11 ++++++++ src/ui.rs | 5 ++-- src/ui/region_capture_picker.rs | 43 ++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 177b239f..e1e11b21 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1567,7 +1567,9 @@ does not change the active tool, the drawing history, or the board. - On a solid whiteboard or blackboard with no visible screen capture, OCR refuses rather than reading the board. - Ctrl+A reads the whole displayed image, so a full screen of text - does not need a drag across the whole output. + does not need a drag across the whole output. The selector says so along the + top until your first drag, the same hint strip the region picker uses and + under the same `[capture.region] show_legend` setting. - While recognition runs, a band sweeps the region being read. When it finishes, a short card beside the region says what happened — copied and how many characters, no text found, or that recognition failed — and fades after a few diff --git a/src/backend/wayland/state/ocr.rs b/src/backend/wayland/state/ocr.rs index 7633ca86..ef1d9e45 100644 --- a/src/backend/wayland/state/ocr.rs +++ b/src/backend/wayland/state/ocr.rs @@ -576,6 +576,17 @@ impl WaylandState { let _ = ctx.stroke(); draw_scan_corners(ctx, x, y, w, h); } + // Whole-image recognition is a keystroke with nothing on screen to + // suggest it. The capture picker teaches its keys here; recognition + // reuses the same strip, honours the same `show_legend` setting, and + // dismisses on the first drag through the shared selector state. + if self.config.capture.region.show_legend && !self.region_picker_legend_dismissed() { + crate::ui::render_region_legend( + ctx, + (screen_width, screen_height), + crate::ui::OCR_LEGEND_TEXT, + ); + } let _ = ctx.restore(); } } diff --git a/src/ui.rs b/src/ui.rs index 7f360edd..e75e0876 100644 --- a/src/ui.rs +++ b/src/ui.rs @@ -54,8 +54,9 @@ pub use properties_panel::render_properties_panel; pub use radial_menu::render_radial_menu; pub(crate) use region_action_bar::{RegionAction, RegionActionBar}; pub(crate) use region_capture_picker::{ - RegionCaptureLoupeVisual, RegionCapturePickerVisual, RegionCaptureWindowVisual, - capture_size_text, measure_picker_damage, render_region_capture_picker, + OCR_LEGEND_TEXT, RegionCaptureLoupeVisual, RegionCapturePickerVisual, + RegionCaptureWindowVisual, capture_size_text, measure_picker_damage, + render_region_capture_picker, render_region_legend, }; pub(crate) use region_resize_handles::RegionResizeHandles; pub use status::{ diff --git a/src/ui/region_capture_picker.rs b/src/ui/region_capture_picker.rs index 15c1a22e..c16829e7 100644 --- a/src/ui/region_capture_picker.rs +++ b/src/ui/region_capture_picker.rs @@ -25,6 +25,10 @@ const LEGEND_FONT_SIZE: f64 = 12.0; const AREA_LEGEND_TEXT: &str = "Drag to select Shift: square Ctrl+A: all Esc: cancel"; const AREA_WITH_WINDOWS_LEGEND_TEXT: &str = "Drag to select Shift: square Ctrl+A: all Space: window Esc: cancel"; +/// Recognition offers no square modifier, and `Ctrl+A` reads everything rather +/// than selecting everything, so it says what it does rather than borrowing +/// the capture picker's wording. +pub(crate) const OCR_LEGEND_TEXT: &str = "Drag to read text Ctrl+A: whole screen Esc: cancel"; const WINDOW_LEGEND_TEXT: &str = "Click: select Super+Arrows: choose Enter: select Space: area Esc: cancel"; @@ -526,6 +530,12 @@ fn draw_readout_panel( let _ = ctx.restore(); } +/// The hint strip along the top of a region selector. Shared so every selector +/// teaches its keys the same way and in the same place. +pub(crate) fn render_region_legend(ctx: &cairo::Context, screen: (u32, u32), text: &str) { + draw_legend(ctx, screen, text); +} + fn draw_legend(ctx: &cairo::Context, screen: (u32, u32), text: &str) { let extents = text_extents_for( ctx, @@ -602,6 +612,39 @@ mod tests { ); } + #[test] + fn every_selector_legend_names_the_keys_that_selector_actually_has() { + // Recognition has no square modifier, and its select-all reads rather + // than selects, so it must not borrow the capture wording. + assert!(OCR_LEGEND_TEXT.contains("Ctrl+A")); + assert!( + !OCR_LEGEND_TEXT.contains("Shift"), + "recognition offers no square modifier: {OCR_LEGEND_TEXT}" + ); + for legend in [ + AREA_LEGEND_TEXT, + AREA_WITH_WINDOWS_LEGEND_TEXT, + OCR_LEGEND_TEXT, + ] { + assert!(legend.contains("Esc"), "every selector says how to leave"); + } + } + + #[test] + fn the_shared_legend_paints_across_the_top_of_any_selector() { + let mut surface = cairo::ImageSurface::create(cairo::Format::ARgb32, 800, 400).unwrap(); + let ctx = cairo::Context::new(&surface).unwrap(); + render_region_legend(&ctx, (800, 400), OCR_LEGEND_TEXT); + drop(ctx); + surface.flush(); + let stride = surface.stride() as usize; + let data = surface.data().unwrap(); + let alpha = |x: usize, y: usize| data[y * stride + x * 4 + 3]; + + assert!(alpha(400, 24) > 0, "the strip sits along the top edge"); + assert_eq!(alpha(400, 300), 0, "and nowhere else"); + } + #[test] fn pointer_panel_prefers_below_right_then_flips_and_clamps() { assert_eq!( From 75c19d2b501c324faa3e445c6330ff65b695ee7a Mon Sep 17 00:00:00 2001 From: devmobasa <4170275+devmobasa@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:22:39 +0200 Subject: [PATCH 3/4] feat(ocr): bind screen text recognition to Ctrl+Shift+X MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recognition shipped without a chord, and the stated reason — `O` is the orange quick colour — read as "we could not find a key" rather than "it should not have one". Now that it has a scan overlay, a hint strip, whole-image `Ctrl+A`, a palette entry and a toolbar button, being keyless was the last thing making it feel optional. `Ctrl+Shift+X` for e**X**tract text. Every mnemonic was taken: `Ctrl+Shift+O` captures the active window, `Ctrl+Alt+O` opens the capture folder, `R` is spoken for in both modifier sets, and `Ctrl+Shift+T` returns to transparent. This keeps recognition in the `Ctrl+Shift+` block the rest of the capture family lives in, on a letter nothing else wants. Two candidates were rejected for reasons a free-letter search does not show: `Ctrl+Shift+U` is the IBus Unicode entry sequence, which collides with text input, and `Ctrl+Alt+T` opens a terminal nearly everywhere — the overlay holds keyboard focus and would win, quietly doing something else with a chord that deep in muscle memory. Additive, so nothing existing changes: anyone who already bound the action keeps their chord, and the new default stands down through the skipped-default path. The configurator's conflict fixture used `Ctrl+Shift+X` precisely because it was free, and its claimant count is exact. It moves to a chord that is still unclaimed and now asserts that up front, so the next default to take it fails there saying why. --- README.md | 13 ++++++++----- config.example.toml | 4 ++-- .../src/models/keybindings/conflicts.rs | 16 ++++++++++++---- docs/CONFIG.md | 9 +++++---- src/config/keybindings/defaults/capture.rs | 9 ++++++--- src/config/keybindings/tests.rs | 19 +++++++++++-------- 6 files changed, 44 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 2c27e9bf..0c509e28 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ The v0.9.23+ prebuilt `wayscriber` packages require glibc 2.39 and GTK 4.12 — - Full-screen saves, active-window grabs, region capture - Copy to clipboard or save to file - Uses `grim`, `slurp`, `wl-clipboard` (installed automatically by deb/rpm/AUR packages; fallback: xdg-desktop-portal) -- Copy text from screen (OCR): drag a region of the shown desktop and get its text on the clipboard (needs `tesseract`; no default shortcut) +- Copy text from screen (OCR): Ctrl+Shift+X, then drag a region of the shown desktop — or Ctrl+A for all of it — and get its text on the clipboard (needs `tesseract`) ### Sessions and persistence - Session persistence is enabled by default for boards, undo/redo history, and tool state @@ -552,10 +552,13 @@ sudo dnf install wl-clipboard grim slurp # Fedora ### Copy text from screen (OCR) -`Copy text from screen` recognizes the text in a dragged screen region and copies -it to the clipboard. It is optional: the action has no default shortcut and its -toolbar button is hidden until you turn it on. Install Tesseract and the language -data you configure in `[capture].ocr_languages` (default `eng`): +`Copy text from screen` (Ctrl+Shift+X) recognizes the text in a +dragged screen region and copies it to the clipboard; Ctrl+A inside +the selector reads the whole screen instead. A band sweeps the region while +Tesseract runs, and a card reports the outcome — never the recognized text, +which goes only to the clipboard. Its toolbar button is hidden until you turn it +on. Install Tesseract and the language data you configure in +`[capture].ocr_languages` (default `eng`): An active OCR selection cancels if the displayed screen image changes, so the selected coordinates can never be applied to replacement freeze or zoom pixels. diff --git a/config.example.toml b/config.example.toml index 7df456b6..2899cf41 100644 --- a/config.example.toml +++ b/config.example.toml @@ -293,8 +293,8 @@ export_all_boards_pdf_file = [] open_capture_folder = ["Ctrl+Alt+O"] # Select a screen region and copy the text recognized in it (needs Tesseract). -# Unbound by default: "O" is already the orange quick color. -copy_text_from_screen = [] +# Ctrl+A inside the selector reads the whole screen instead. +copy_text_from_screen = ["Ctrl+Shift+X"] # Toggle frozen mode toggle_frozen_mode = ["Ctrl+Shift+F"] diff --git a/configurator/src/models/keybindings/conflicts.rs b/configurator/src/models/keybindings/conflicts.rs index b9eb7209..479b9f26 100644 --- a/configurator/src/models/keybindings/conflicts.rs +++ b/configurator/src/models/keybindings/conflicts.rs @@ -389,11 +389,19 @@ mod tests { #[test] fn conflict_lookup_returns_all_claimants() { + // The count below is exact, so this needs a chord no shipped default + // claims. Asserted rather than assumed: a future default taking it + // should fail here saying why, not as an off-by-one somewhere else. + let binding = Shortcut::parse("Ctrl+Shift+Q").expect("parses"); + assert!( + claimants_for(&draft(), &binding).is_empty(), + "the fixture chord must stay unbound by default" + ); + let mut draft = draft(); - draft.set(KeybindingField::ClearCanvas, "Ctrl+Shift+X".to_string()); - draft.set(KeybindingField::ToggleToolbar, "Ctrl+Shift+X".to_string()); - draft.set(KeybindingField::Undo, "ctrl+shift+x".to_string()); - let binding = Shortcut::parse("Ctrl+Shift+X").expect("parses"); + draft.set(KeybindingField::ClearCanvas, "Ctrl+Shift+Q".to_string()); + draft.set(KeybindingField::ToggleToolbar, "Ctrl+Shift+Q".to_string()); + draft.set(KeybindingField::Undo, "ctrl+shift+q".to_string()); let claimants = claimants_for(&draft, &binding); let fields: Vec<_> = claimants.iter().map(|claim| claim.field).collect(); assert!(fields.contains(&Some(KeybindingField::ClearCanvas))); diff --git a/docs/CONFIG.md b/docs/CONFIG.md index e1e11b21..431c1da0 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1554,8 +1554,9 @@ the result to the clipboard. It reads the underlying screen capture only — never your annotations, the toolbars, or any other wayscriber chrome — and it does not change the active tool, the drawing history, or the board. -- The action is `copy_text_from_screen`. It has **no default shortcut**, because - `O` is already the orange quick color; bind one in the configurator or in +- The action is `copy_text_from_screen`, bound to Ctrl+Shift+X + ("extract text"). `O` belongs to the orange quick color, so it takes a letter + the rest of the capture family had left; rebind it in the configurator or in `[keybindings.capture]`. - It is also in the command palette (search for "OCR"), and as an optional top toolbar button (`top.utility.ocr`), hidden by default like Screenshot. @@ -2081,8 +2082,8 @@ export_all_boards_pdf_file = [] open_capture_folder = ["Ctrl+Alt+O"] # Select a screen region and copy the text recognized in it (needs Tesseract). -# Unbound by default: "O" is already the orange quick color. -copy_text_from_screen = [] +# Ctrl+A inside the selector reads the whole screen instead. +copy_text_from_screen = ["Ctrl+Shift+X"] # Toggle frozen mode toggle_frozen_mode = ["Ctrl+Shift+F"] diff --git a/src/config/keybindings/defaults/capture.rs b/src/config/keybindings/defaults/capture.rs index b362a160..0e8c3e0b 100644 --- a/src/config/keybindings/defaults/capture.rs +++ b/src/config/keybindings/defaults/capture.rs @@ -74,8 +74,11 @@ pub(crate) fn default_open_capture_folder() -> Vec { vec!["Ctrl+Alt+O".to_string()] } -/// Deliberately empty: `O` is the orange quick color and no other -/// conflict-free chord is obviously right, so the user picks one. +/// e**X**tract text. `O` is the orange quick color and every other mnemonic is +/// taken — `Ctrl+Shift+O` captures the active window, `Ctrl+Alt+O` opens the +/// capture folder, `R` is spoken for in both modifier sets, and `Ctrl+Shift+T` +/// returns to transparent — so this sits with the rest of the capture family in +/// `Ctrl+Shift+` on a letter nothing else wants. pub(crate) fn default_copy_text_from_screen() -> Vec { - Vec::new() + vec!["Ctrl+Shift+X".to_string()] } diff --git a/src/config/keybindings/tests.rs b/src/config/keybindings/tests.rs index f7b45959..f29252d9 100644 --- a/src/config/keybindings/tests.rs +++ b/src/config/keybindings/tests.rs @@ -482,19 +482,22 @@ fn screen_eyedropper_defaults_to_i_and_maps_when_reconfigured() { } #[test] -fn screen_text_recognition_is_unbound_by_default_and_leaves_o_to_orange() { +fn screen_text_recognition_has_a_chord_and_still_leaves_o_to_orange() { let mut config = KeybindingsConfig::default(); - assert!(config.capture.copy_text_from_screen.is_empty()); + assert_eq!( + config.capture.copy_text_from_screen, + vec!["Ctrl+Shift+X".to_string()] + ); let default_map = config.build_action_map().unwrap(); assert_eq!( default_map.get(&Shortcut::parse("O").unwrap()), - Some(&Action::SetColorOrange) + Some(&Action::SetColorOrange), + "the quick colour keeps the letter recognition would have wanted" ); - assert!( - !default_map - .values() - .any(|action| *action == Action::CopyTextFromScreen) + assert_eq!( + default_map.get(&Shortcut::parse("Ctrl+Shift+X").unwrap()), + Some(&Action::CopyTextFromScreen) ); config.capture.copy_text_from_screen = vec!["Ctrl+Alt+T".to_string()]; @@ -833,7 +836,7 @@ const DEFAULT_BINDING_SNAPSHOT: &[(&str, &[&str])] = &[ ("open_capture_folder", &["Ctrl+Alt+O"]), // Intentionally unbound: `O` is the orange quick color, and no other // conflict-free chord is obviously right, so the user picks one. - ("copy_text_from_screen", &[]), + ("copy_text_from_screen", &["Ctrl+Shift+X"]), ("toggle_frozen_mode", &["Ctrl+Shift+F"]), ("zoom_in", &["Ctrl+Alt++", "Ctrl+Alt+="]), ("zoom_out", &["Ctrl+Alt+-", "Ctrl+Alt+_"]), From 3f29193acf5607d2277be79d93ea04c35022ad75 Mon Sep 17 00:00:00 2001 From: devmobasa <4170275+devmobasa@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:52:36 +0200 Subject: [PATCH 4/4] fix(config): correct stale unbound claims and free the fixture chord MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three leftovers from binding recognition to Ctrl+Shift+X. Two comments still said the action was unbound, and one of them argued against ever giving it a default. Both now describe the binding that exists. The configurator's conflict fixtures used Ctrl+Shift+X because nothing claimed it. One of them was more than untidy: `replace_removes_only_the_contested_binding` runs a replace that strips the contested chord from every claimant, so with a default on that chord it was silently rewriting `copy_text_from_screen` — a field the test never mentions and never asserts. It passed while exercising a polluted scenario. Every draft-based fixture moves to Ctrl+Shift+Q, which the conflict test now asserts is unclaimed so the next default to take it fails there with the reason. Chord parsing and formatting tests keep Ctrl+Shift+X: they take no draft and one of them is about that chord's canonical spelling. `render_region_legend` only forwarded to `draw_legend`, so the rename replaces the pair. --- configurator/src/app/update/shortcuts/tests.rs | 12 ++++++------ configurator/src/models/keybindings/conflicts.rs | 15 ++++++++------- configurator/src/models/keybindings/manager.rs | 10 +++++----- .../keybindings/config/types/bindings/capture.rs | 5 +++-- src/config/keybindings/tests.rs | 3 +-- src/ui/region_capture_picker.rs | 6 +----- 6 files changed, 24 insertions(+), 27 deletions(-) diff --git a/configurator/src/app/update/shortcuts/tests.rs b/configurator/src/app/update/shortcuts/tests.rs index cab17a62..2c94c391 100644 --- a/configurator/src/app/update/shortcuts/tests.rs +++ b/configurator/src/app/update/shortcuts/tests.rs @@ -672,10 +672,10 @@ fn conflict_review_queue_arms_the_next_conflict_after_replace() { app.is_loading = false; app.draft .keybindings - .set(KeybindingField::ClearCanvas, "Ctrl+Shift+X".to_string()); + .set(KeybindingField::ClearCanvas, "Ctrl+Shift+Q".to_string()); app.draft .keybindings - .set(KeybindingField::ToggleToolbar, "Ctrl+Shift+X".to_string()); + .set(KeybindingField::ToggleToolbar, "Ctrl+Shift+Q".to_string()); app.draft .keybindings .set(KeybindingField::Undo, "Ctrl+Alt+Shift+Y".to_string()); @@ -713,10 +713,10 @@ fn conflict_review_cancel_stops_the_queue() { let (mut app, _effects) = ConfiguratorApp::new_app(); app.draft .keybindings - .set(KeybindingField::ClearCanvas, "Ctrl+Shift+X".to_string()); + .set(KeybindingField::ClearCanvas, "Ctrl+Shift+Q".to_string()); app.draft .keybindings - .set(KeybindingField::ToggleToolbar, "Ctrl+Shift+X".to_string()); + .set(KeybindingField::ToggleToolbar, "Ctrl+Shift+Q".to_string()); let _ = app.handle_shortcut_conflict_review_started(); let _ = app.handle_shortcut_conflict_canceled(); assert!(!app.shortcut_conflict_review); @@ -728,10 +728,10 @@ fn jump_to_conflict_selects_the_other_claimant() { let (mut app, _effects) = ConfiguratorApp::new_app(); app.draft .keybindings - .set(KeybindingField::ClearCanvas, "Ctrl+Shift+X".to_string()); + .set(KeybindingField::ClearCanvas, "Ctrl+Shift+Q".to_string()); app.draft .keybindings - .set(KeybindingField::ToggleToolbar, "Ctrl+Shift+X".to_string()); + .set(KeybindingField::ToggleToolbar, "Ctrl+Shift+Q".to_string()); let _ = app.handle_shortcut_conflict_review_started(); let jump = app .pending_shortcut_conflict diff --git a/configurator/src/models/keybindings/conflicts.rs b/configurator/src/models/keybindings/conflicts.rs index 479b9f26..802629ea 100644 --- a/configurator/src/models/keybindings/conflicts.rs +++ b/configurator/src/models/keybindings/conflicts.rs @@ -389,9 +389,10 @@ mod tests { #[test] fn conflict_lookup_returns_all_claimants() { - // The count below is exact, so this needs a chord no shipped default - // claims. Asserted rather than assumed: a future default taking it - // should fail here saying why, not as an off-by-one somewhere else. + // This and the other generic multi-claimant fixtures use a chord no + // shipped default claims. Asserted rather than assumed: a future default + // taking `Ctrl+Shift+Q` should fail here saying why, not as an off-by-one + // somewhere else. let binding = Shortcut::parse("Ctrl+Shift+Q").expect("parses"); assert!( claimants_for(&draft(), &binding).is_empty(), @@ -439,10 +440,10 @@ mod tests { let mut draft = draft(); draft.set( KeybindingField::ToggleHelp, - "F10, F1, Ctrl+Shift+X".to_string(), + "F10, F1, Ctrl+Shift+Q".to_string(), ); - draft.set(KeybindingField::Undo, "Ctrl+Z, Ctrl+Shift+X".to_string()); - let binding = Shortcut::parse("Ctrl+Shift+X").expect("parses"); + draft.set(KeybindingField::Undo, "Ctrl+Z, Ctrl+Shift+Q".to_string()); + let binding = Shortcut::parse("Ctrl+Shift+Q").expect("parses"); let claimants = other_claimants(&draft, KeybindingField::ClearCanvas, &binding); apply_recorded_replace( &mut draft, @@ -458,7 +459,7 @@ mod tests { assert_eq!(draft.value_for(KeybindingField::Undo), Some("Ctrl+Z")); assert_eq!( draft.value_for(KeybindingField::ClearCanvas), - Some("E, Ctrl+Shift+X") + Some("E, Ctrl+Shift+Q") ); } diff --git a/configurator/src/models/keybindings/manager.rs b/configurator/src/models/keybindings/manager.rs index d8667361..c09a13b2 100644 --- a/configurator/src/models/keybindings/manager.rs +++ b/configurator/src/models/keybindings/manager.rs @@ -487,8 +487,8 @@ mod tests { #[test] fn conflict_filter_includes_every_claimant() { let (mut draft, defaults) = drafts(); - draft.set(KeybindingField::ClearCanvas, "Ctrl+Shift+X".to_string()); - draft.set(KeybindingField::ToggleToolbar, "Ctrl+Shift+X".to_string()); + draft.set(KeybindingField::ClearCanvas, "Ctrl+Shift+Q".to_string()); + draft.set(KeybindingField::ToggleToolbar, "Ctrl+Shift+Q".to_string()); let summary = ShortcutManagerSummary::from_drafts(&draft, &defaults); let visible = summary.visible_fields( ShortcutManagerFilter::Conflicts, @@ -651,10 +651,10 @@ mod tests { #[test] fn next_review_conflict_names_the_other_claimant() { let (mut draft, _defaults) = drafts(); - draft.set(KeybindingField::ClearCanvas, "Ctrl+Shift+X".to_string()); - draft.set(KeybindingField::ToggleToolbar, "Ctrl+Shift+X".to_string()); + draft.set(KeybindingField::ClearCanvas, "Ctrl+Shift+Q".to_string()); + draft.set(KeybindingField::ToggleToolbar, "Ctrl+Shift+Q".to_string()); let (field, binding, claimants) = next_review_conflict(&draft).expect("conflict"); - assert_eq!(binding.to_string(), "Ctrl+Shift+X"); + assert_eq!(binding.to_string(), "Ctrl+Shift+Q"); assert!(claimants.iter().any(|claim| claim.field == Some(field) || matches!( claim.field, diff --git a/src/config/keybindings/config/types/bindings/capture.rs b/src/config/keybindings/config/types/bindings/capture.rs index dc8a3934..972315b5 100644 --- a/src/config/keybindings/config/types/bindings/capture.rs +++ b/src/config/keybindings/config/types/bindings/capture.rs @@ -61,8 +61,9 @@ pub struct CaptureKeybindingsConfig { #[serde(default = "default_open_capture_folder")] pub open_capture_folder: Vec, - /// Screen text recognition. Unbound by default: `O` already selects the - /// orange quick color, so a default here would silently repurpose it. + /// Screen text recognition, on `Ctrl+Shift+X` — extract text. `O` selects + /// the orange quick color and every other mnemonic is taken, so it sits + /// with the rest of the capture family on a letter nothing else wants. #[serde(default = "default_copy_text_from_screen")] pub copy_text_from_screen: Vec, } diff --git a/src/config/keybindings/tests.rs b/src/config/keybindings/tests.rs index f29252d9..7fa5da0b 100644 --- a/src/config/keybindings/tests.rs +++ b/src/config/keybindings/tests.rs @@ -834,8 +834,7 @@ const DEFAULT_BINDING_SNAPSHOT: &[(&str, &[&str])] = &[ ("export_board_pdf_file", &[]), ("export_all_boards_pdf_file", &[]), ("open_capture_folder", &["Ctrl+Alt+O"]), - // Intentionally unbound: `O` is the orange quick color, and no other - // conflict-free chord is obviously right, so the user picks one. + // `O` stays the orange quick color; recognition takes the free letter. ("copy_text_from_screen", &["Ctrl+Shift+X"]), ("toggle_frozen_mode", &["Ctrl+Shift+F"]), ("zoom_in", &["Ctrl+Alt++", "Ctrl+Alt+="]), diff --git a/src/ui/region_capture_picker.rs b/src/ui/region_capture_picker.rs index c16829e7..9c57a3f6 100644 --- a/src/ui/region_capture_picker.rs +++ b/src/ui/region_capture_picker.rs @@ -286,7 +286,7 @@ pub(crate) fn render_region_capture_picker( ); } if visual.show_legend && (visual.window.active || visual.selection.is_none()) { - draw_legend( + render_region_legend( ctx, (screen_width, screen_height), picker_legend_text(visual.window), @@ -533,10 +533,6 @@ fn draw_readout_panel( /// The hint strip along the top of a region selector. Shared so every selector /// teaches its keys the same way and in the same place. pub(crate) fn render_region_legend(ctx: &cairo::Context, screen: (u32, u32), text: &str) { - draw_legend(ctx, screen, text); -} - -fn draw_legend(ctx: &cairo::Context, screen: (u32, u32), text: &str) { let extents = text_extents_for( ctx, "Sans",