From 7a4e2f501115f016ad241ab85c8390b4d549857a Mon Sep 17 00:00:00 2001 From: uniqueding Date: Tue, 4 Aug 2026 14:40:51 +0800 Subject: [PATCH 1/2] feat(input): add native X11 xdotool pointer clicks --- CHANGELOG.md | 8 ++ README.md | 3 +- src/server.rs | 229 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 239 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a89382f..1a6d447 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Native X11 coordinate clicks now use one supervised + `xdotool mousemove -- X Y click --repeat N BUTTON` command after the + absolute pointer and eligible portal paths, with `ydotool` fallback only + when `xdotool` cannot be spawned. Standard left, middle, and right buttons + are supported; extended buttons retain the existing fallback path. Set + `COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER=1` to skip xdotool. + ## [0.4.5] - 2026-08-01 ### Fixed diff --git a/README.md b/README.md index 5ea57a3..b2240f2 100644 --- a/README.md +++ b/README.md @@ -335,7 +335,7 @@ Most setups need none of these — `doctor` and the installers pick sensible def | `COMPUTER_USE_LINUX_COSMIC_HELPER` | Path to the `computer-use-linux-cosmic` helper when it isn't next to the binary or on `PATH`. | | `CU_DISABLE_ABS_POINTER` | Disable the uinput absolute pointer and click through `ydotool` instead for setups where the abs-pointer device misbehaves. | | `COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER` / `…_KEYBOARD` | Always route pointer / keyboard through the RemoteDesktop portal on Wayland, skipping auto-detection. | -| `COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER` / `…_KEYBOARD` | Always route pointer / keyboard through `ydotool`, skipping the portal and KDE clipboard paths. | +| `COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER` / `…_KEYBOARD` | Always route pointer / keyboard through `ydotool`, skipping the portal and KDE clipboard paths; pointer forcing also skips native-X11 `xdotool` coordinate clicks. | | `COMPUTER_USE_LINUX_FORCE_XDOTOOL_KEYBOARD` | Prefer `xdotool`/XTEST keyboard input when `DISPLAY` is available. `COMPUTER_USE_LINUX_FORCE_YDOTOOL_KEYBOARD=1` takes precedence. | | `COMPUTER_USE_LINUX_SCREENSHOT_BACKEND` | Force a single screenshot backend, skipping the fallback chain. Accepts `gnome-shell`, `portal`, or `gnome-screenshot`. Pin `gnome-screenshot` for background/systemd contexts where the GNOME Shell and portal DBus paths are denied. | @@ -360,6 +360,7 @@ files. - **DBus where desktops expose it** — [`zbus`](https://crates.io/crates/zbus) for portal calls (`org.freedesktop.portal.Screenshot`, `…RemoteDesktop`, `…ScreenCast`), GNOME Shell screenshots (`org.gnome.Shell.Screenshot`), the bundled GNOME extension's `dev.avifenesh.ComputerUseLinux.WindowControl` service, and temporary KWin scripting. - **MCP transport** — [`rmcp`](https://crates.io/crates/rmcp) with the `transport-io` feature; stdio framing, no network. - **Input fallback** — on X11, keyboard input prefers `xdotool`/XTEST and falls back only when xdotool cannot launch. On Wayland, when the remote-desktop portal isn't available or the host wants deterministic injection, the binary uses a compatible ydotool 1.0.3+ CLI and `ydotoold` socket, which writes to `/dev/uinput`. `install.sh` can configure `ydotoold`; the `setup` command only enables the GNOME AT-SPI bridge. +- **Native X11 coordinate clicks** — eligible native X11 sessions use one supervised `xdotool mousemove -- X Y click --repeat N BUTTON` command for left, middle, and right clicks; ydotool is used only when xdotool cannot launch, while a launched nonzero xdotool command is reported as an error without replay. `COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER=1` skips this xdotool path. - **Window registry** — `list_windows`, `focused_window`, `activate_window`, `press_key`, and `type_text` share a backend registry. It tries GNOME extension, GNOME Introspect, COSMIC helper, KWin scripting, Hyprland `hyprctl`, i3 IPC, and generic X11/EWMH in that order, skipping empty or failed backends so another compositor backend can answer. - **GNOME extension fallback** — recent GNOME builds deny `org.gnome.Shell.Introspect.GetWindows` to non-blessed clients. The bundled Shell extension exposes window data and exact activation under `dev.avifenesh.ComputerUseLinux.WindowControl`. - **COSMIC helper** — `computer-use-linux-cosmic` talks to COSMIC toplevel protocols and is resolved from `COMPUTER_USE_LINUX_COSMIC_HELPER`, next to the running binary, or from `PATH`. diff --git a/src/server.rs b/src/server.rs index 592fe4e..d6756aa 100644 --- a/src/server.rs +++ b/src/server.rs @@ -851,6 +851,41 @@ impl ComputerUseLinux { Err(_) => {} } } + if self.should_prefer_xdotool_pointer() { + if let Some(xdotool_args) = xdotool_pointer_click_args( + x, + y, + params.click_count.unwrap_or(1).clamp(1, 10), + params.button.as_deref(), + ) { + let ydotool_commands = vec![ + absolute_mousemove_args(x, y), + vec![ + "click".to_string(), + "--repeat".to_string(), + click_count.clone(), + button.clone(), + ], + ]; + let (input_guard, result) = run_cancellation_safe_input(input_guard, async move { + run_xdotool_pointer_or_fallback(Path::new("xdotool"), &xdotool_args, || async { + run_ydotool_sequence(&ydotool_commands).await + }) + .await + }) + .await; + let _input_guard = input_guard; + let used_xdotool = result + .as_ref() + .is_ok_and(|result| result.backend == KeyboardCommandBackend::Xdotool); + let mut output = + action_result("click", result.map(|result| result.outputs), received); + if output.ok && used_xdotool { + output.message = "Action sent through xdotool (X11 XTEST).".to_string(); + } + return Json(with_notes(output, off_screen_note)); + } + } let commands = vec![ absolute_mousemove_args(x, y), vec![ @@ -2346,6 +2381,17 @@ impl ComputerUseLinux { ) } + fn should_prefer_xdotool_pointer(&self) -> bool { + crate::diagnostics::hydrate_session_bus_env(); + prefer_xdotool_pointer( + env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER"), + env::var("XDG_SESSION_TYPE").ok().as_deref(), + env_var_non_empty("DISPLAY"), + env::var("WAYLAND_DISPLAY").ok().as_deref(), + xdotool_available(), + ) + } + fn is_kde_wayland_session(&self) -> bool { self.is_wayland_session() && (env_contains("XDG_CURRENT_DESKTOP", "kde") @@ -3560,6 +3606,27 @@ fn session_is_wayland(session_type: Option<&str>, wayland_display: Option<&str>) } } +fn native_x11_xdotool_pointer_session( + session_type: Option<&str>, + wayland_display: Option<&str>, +) -> bool { + session_type.is_some_and(|value| value.trim().eq_ignore_ascii_case("x11")) + && !wayland_display.is_some_and(|value| !value.trim().is_empty()) +} + +fn prefer_xdotool_pointer( + force_ydotool: bool, + session_type: Option<&str>, + display_available: bool, + wayland_display: Option<&str>, + xdotool_available: bool, +) -> bool { + !force_ydotool + && native_x11_xdotool_pointer_session(session_type, wayland_display) + && display_available + && xdotool_available +} + fn prefer_xdotool_keyboard( force_ydotool: bool, force_xdotool: bool, @@ -4018,6 +4085,61 @@ fn absolute_mousemove_args(x: i32, y: i32) -> Vec { ] } +fn xdotool_pointer_click_args( + x: i32, + y: i32, + count: u32, + button: Option<&str>, +) -> Option> { + let button = xdotool_pointer_button_code(button)?; + Some(vec![ + "mousemove".to_string(), + "--".to_string(), + x.to_string(), + y.to_string(), + "click".to_string(), + "--repeat".to_string(), + count.to_string(), + button.to_string(), + ]) +} + +fn xdotool_pointer_button_code(button: Option<&str>) -> Option<&'static str> { + match button.unwrap_or("left").to_ascii_lowercase().as_str() { + "left" => Some("1"), + "middle" => Some("2"), + "right" => Some("3"), + _ => None, + } +} + +#[derive(Debug)] +struct PointerCommandResult { + outputs: Vec, + backend: KeyboardCommandBackend, +} + +async fn run_xdotool_pointer_or_fallback( + program: &Path, + args: &[String], + fallback: F, +) -> std::result::Result +where + F: FnOnce() -> Fut, + Fut: Future, String>>, +{ + match run_xdotool(program, args).await { + XdotoolAttempt::Unavailable => fallback().await.map(|outputs| PointerCommandResult { + outputs, + backend: KeyboardCommandBackend::Ydotool, + }), + XdotoolAttempt::Finished(result) => result.map(|output| PointerCommandResult { + outputs: vec![output], + backend: KeyboardCommandBackend::Xdotool, + }), + } +} + fn wheel_mousemove_args(dx: i32, dy: i32) -> Vec { vec![ "mousemove".to_string(), @@ -5644,6 +5766,113 @@ mod tests { ); } + #[test] + fn native_x11_pointer_policy_requires_explicit_x11_without_wayland_display() { + assert!(native_x11_xdotool_pointer_session(Some("x11"), None)); + assert!(!native_x11_xdotool_pointer_session( + Some("wayland"), + Some("wayland-0") + )); + assert!(!native_x11_xdotool_pointer_session( + Some("x11"), + Some("wayland-0") + )); + } + + #[test] + fn xdotool_pointer_command_is_single_no_sync_move_and_click() { + assert_eq!( + xdotool_pointer_click_args(1550, 930, 3, Some("right")), + Some(vec![ + "mousemove".to_string(), + "--".to_string(), + "1550".to_string(), + "930".to_string(), + "click".to_string(), + "--repeat".to_string(), + "3".to_string(), + "3".to_string(), + ]) + ); + } + + #[test] + fn xdotool_pointer_policy_requires_all_pure_gating_conditions() { + let eligible = (false, Some("x11"), true, None, true); + assert!(prefer_xdotool_pointer( + eligible.0, eligible.1, eligible.2, eligible.3, eligible.4 + )); + assert!(!prefer_xdotool_pointer(true, Some("x11"), true, None, true)); + assert!(!prefer_xdotool_pointer( + false, + Some("wayland"), + true, + None, + true + )); + assert!(!prefer_xdotool_pointer( + false, + None, + true, + Some("wayland-0"), + true + )); + assert!(!prefer_xdotool_pointer( + false, + Some("x11"), + false, + None, + true + )); + assert!(!prefer_xdotool_pointer( + false, + Some("x11"), + true, + None, + false + )); + } + + #[test] + fn xdotool_pointer_supports_only_standard_buttons() { + assert!(xdotool_pointer_click_args(10, 20, 1, None).is_some()); + assert!(xdotool_pointer_click_args(10, 20, 1, Some("middle")).is_some()); + assert!(xdotool_pointer_click_args(10, 20, 1, Some("right")).is_some()); + } + + #[test] + fn extended_pointer_buttons_do_not_construct_xdotool_commands() { + for button in ["side", "extra", "forward", "back"] { + assert_eq!(xdotool_pointer_click_args(10, 20, 1, Some(button)), None); + } + } + + #[tokio::test] + async fn pointer_xdotool_spawn_failure_uses_ydotool_fallback() { + let result = run_xdotool_pointer_or_fallback( + Path::new("/definitely/missing/xdotool"), + &[], + || async { Ok::<_, String>(Vec::new()) }, + ) + .await + .expect("spawn failure should use fallback"); + + assert_eq!(result.backend, KeyboardCommandBackend::Ydotool); + } + + #[tokio::test] + async fn pointer_xdotool_nonzero_exit_does_not_use_ydotool_fallback() { + let result = run_xdotool_pointer_or_fallback( + Path::new("/bin/sh"), + &["-c".to_string(), "exit 9".to_string()], + || async { Err::, _>("fallback called".to_string()) }, + ) + .await; + + let error = result.expect_err("launched nonzero xdotool must be terminal"); + assert!(!error.contains("fallback called")); + } + #[test] fn wheel_mousemove_uses_coordinate_separator_for_negative_values() { assert_eq!( From 9aa72182d06534d03cfe4b1fa1bd809d1b3c725a Mon Sep 17 00:00:00 2001 From: Avi Fenesh Date: Tue, 4 Aug 2026 22:02:46 +0300 Subject: [PATCH 2/2] fix: preserve nested native X11 session hydration --- src/diagnostics.rs | 87 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 76 insertions(+), 11 deletions(-) diff --git a/src/diagnostics.rs b/src/diagnostics.rs index fea922a..dffe2ca 100644 --- a/src/diagnostics.rs +++ b/src/diagnostics.rs @@ -385,17 +385,48 @@ fn hydrate_desktop_env_from_systemd_user() { } fn hydrate_desktop_env_from_map(process_env: &HashMap) { - for key in DESKTOP_ENV_KEYS { - if env_var(key).is_some() { - continue; - } - if let Some(value) = process_env - .get(*key) - .filter(|value| !value.trim().is_empty()) - { - env::set_var(key, value); - } - } + let current_env = DESKTOP_ENV_KEYS + .iter() + .filter_map(|key| env_var(key).map(|value| ((*key).to_string(), value))) + .collect(); + for (key, value) in desktop_env_hydration_updates(¤t_env, process_env) { + env::set_var(key, value); + } +} + +fn desktop_env_hydration_updates( + current_env: &HashMap, + source_env: &HashMap, +) -> Vec<(&'static str, String)> { + // A nested X11 desktop can share a user manager with a Wayland host. + // Preserve its complete process-local session instead of grafting the + // host's WAYLAND_DISPLAY onto it. + let preserve_native_x11 = current_env + .get("XDG_SESSION_TYPE") + .is_some_and(|value| value.trim().eq_ignore_ascii_case("x11")) + && current_env + .get("DISPLAY") + .is_some_and(|value| !value.trim().is_empty()) + && !current_env + .get("WAYLAND_DISPLAY") + .is_some_and(|value| !value.trim().is_empty()); + + DESKTOP_ENV_KEYS + .iter() + .filter_map(|key| { + if current_env + .get(*key) + .is_some_and(|value| !value.trim().is_empty()) + || preserve_native_x11 && *key == "WAYLAND_DISPLAY" + { + return None; + } + source_env + .get(*key) + .filter(|value| !value.trim().is_empty()) + .map(|value| (*key, value.clone())) + }) + .collect() } fn desktop_process_environments() -> Vec> { @@ -1186,6 +1217,40 @@ mod tests { assert!(DESKTOP_ENV_KEYS.contains(&"XAUTHORITY")); } + #[test] + fn desktop_env_hydration_preserves_explicit_native_x11() { + let current_env = HashMap::from([ + ("DISPLAY".to_string(), ":90".to_string()), + ("XDG_SESSION_TYPE".to_string(), "x11".to_string()), + ]); + let host_env = HashMap::from([ + ("WAYLAND_DISPLAY".to_string(), "wayland-0".to_string()), + ( + "XDG_CURRENT_DESKTOP".to_string(), + "ubuntu:GNOME".to_string(), + ), + ]); + + let updates = desktop_env_hydration_updates(¤t_env, &host_env); + + assert!(!updates.iter().any(|(key, _)| *key == "WAYLAND_DISPLAY")); + assert!(updates + .iter() + .any(|(key, value)| { *key == "XDG_CURRENT_DESKTOP" && value == "ubuntu:GNOME" })); + } + + #[test] + fn desktop_env_hydration_still_imports_wayland_for_incomplete_sessions() { + let current_env = HashMap::new(); + let host_env = HashMap::from([("WAYLAND_DISPLAY".to_string(), "wayland-0".to_string())]); + + let updates = desktop_env_hydration_updates(¤t_env, &host_env); + + assert!(updates + .iter() + .any(|(key, value)| *key == "WAYLAND_DISPLAY" && value == "wayland-0")); + } + #[test] fn graphical_process_env_requires_display() { let with_display = HashMap::from([("DISPLAY".to_string(), ":0".to_string())]);