From 1415b49859204f093b3bd06d0ac84b12ea976530 Mon Sep 17 00:00:00 2001 From: caipira113 Date: Wed, 5 Aug 2026 09:33:01 +0900 Subject: [PATCH] fix(click): resolve window id when web content runs in a helper process --- src/accessibility.rs | 132 +++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 37 ++++++++++-- 2 files changed, 165 insertions(+), 4 deletions(-) diff --git a/src/accessibility.rs b/src/accessibility.rs index 040b729..0c38051 100644 --- a/src/accessibility.rs +++ b/src/accessibility.rs @@ -1743,6 +1743,138 @@ pub fn is_window_visible_at(window_id: u32, x: f64, y: f64) -> Result Option { + use std::ffi::c_void; + use objc2_core_foundation::{CFIndex, CFNumber, CFNumberType, CFString as CFS}; + use objc2_core_graphics::{CGWindowListCopyWindowInfo, CGWindowListOption}; + + unsafe extern "C" { + fn CFDictionaryGetValue(dict: *const c_void, key: *const c_void) -> *const c_void; + } + + let info = CGWindowListCopyWindowInfo( + CGWindowListOption::OptionOnScreenOnly | CGWindowListOption::ExcludeDesktopElements, + 0, + )?; + + let key_num = CFS::from_str("kCGWindowNumber"); + let key_pid = CFS::from_str("kCGWindowOwnerPID"); + let key_layer = CFS::from_str("kCGWindowLayer"); + let key_alpha = CFS::from_str("kCGWindowAlpha"); + let key_bounds = CFS::from_str("kCGWindowBounds"); + + let (fx, fy, fw, fh) = frame; + + for i in 0..info.len() { + let dict_ptr = unsafe { info.as_opaque().value_at_index(i as CFIndex) }; + if dict_ptr.is_null() { + continue; + } + let get_val = |key: &CFS| -> *const c_void { + unsafe { CFDictionaryGetValue(dict_ptr, key as *const CFS as *const c_void) } + }; + let read_i64 = |ptr: *const c_void| -> Option { + if ptr.is_null() { + return None; + } + let num = unsafe { &*(ptr as *const CFNumber) }; + let mut v: i64 = 0; + unsafe { num.value(CFNumberType(4), &mut v as *mut i64 as *mut _) }; + Some(v) + }; + + if read_i64(get_val(&key_pid)) != Some(pid as i64) { + continue; + } + if read_i64(get_val(&key_layer)).unwrap_or(0) != 0 { + continue; + } + + // Skip fully transparent overlays; Teams keeps one over its title bar. + let alpha_ptr = get_val(&key_alpha); + if !alpha_ptr.is_null() { + let num = unsafe { &*(alpha_ptr as *const CFNumber) }; + let mut alpha: f64 = 1.0; + unsafe { num.value(CFNumberType(13), &mut alpha as *mut f64 as *mut _) }; + if alpha <= 0.0 { + continue; + } + } + + let bounds_ptr = get_val(&key_bounds); + if bounds_ptr.is_null() { + continue; + } + let (bx, by, bw, bh) = read_bounds_dict(bounds_ptr); + + const TOLERANCE: f64 = 2.0; + if (bx - fx).abs() <= TOLERANCE + && (by - fy).abs() <= TOLERANCE + && (bw - fw).abs() <= TOLERANCE + && (bh - fh).abs() <= TOLERANCE + { + return read_i64(get_val(&key_num)).map(|v| v as u32); + } + } + None +} + +/// Whether `window_id` is an on-screen window owned by `pid`. +pub fn window_belongs_to_pid(window_id: u32, pid: i32) -> bool { + use std::ffi::c_void; + use objc2_core_foundation::{CFIndex, CFNumber, CFNumberType, CFString as CFS}; + use objc2_core_graphics::{CGWindowListCopyWindowInfo, CGWindowListOption}; + + unsafe extern "C" { + fn CFDictionaryGetValue(dict: *const c_void, key: *const c_void) -> *const c_void; + } + + let Some(info) = CGWindowListCopyWindowInfo( + CGWindowListOption::OptionOnScreenOnly | CGWindowListOption::ExcludeDesktopElements, + 0, + ) else { + return false; + }; + + let key_num = CFS::from_str("kCGWindowNumber"); + let key_pid = CFS::from_str("kCGWindowOwnerPID"); + + for i in 0..info.len() { + let dict_ptr = unsafe { info.as_opaque().value_at_index(i as CFIndex) }; + if dict_ptr.is_null() { + continue; + } + let get_val = |key: &CFS| -> *const c_void { + unsafe { CFDictionaryGetValue(dict_ptr, key as *const CFS as *const c_void) } + }; + let read_i64 = |ptr: *const c_void| -> Option { + if ptr.is_null() { + return None; + } + let num = unsafe { &*(ptr as *const CFNumber) }; + let mut v: i64 = 0; + unsafe { num.value(CFNumberType(4), &mut v as *mut i64 as *mut _) }; + Some(v) + }; + if read_i64(get_val(&key_num)) == Some(window_id as i64) { + return read_i64(get_val(&key_pid)) == Some(pid as i64); + } + } + false +} + fn read_bounds_dict(dict_ptr: *const std::ffi::c_void) -> (f64, f64, f64, f64) { use std::ffi::c_void; use objc2_core_foundation::{CFNumber, CFNumberType, CFString as CFS}; diff --git a/src/main.rs b/src/main.rs index de596e3..69bb17b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -929,10 +929,10 @@ fn cmd_click( if activate { eprintln!("warning: --activate is ignored for --strategy cg-pid (background by design)"); } - let wid = match node.window_id() { + let wid = match resolve_event_window_id(&node, ctx.pid) { Some(w) => w, None => { - eprintln!("debug: _AXUIElementGetWindow returned no CGWindowID for this element"); + eprintln!("debug: could not resolve a CGWindowID owned by pid {}", ctx.pid); return Err(AxError::ActionFailed( "could not find the window that owns this element".to_string(), )); @@ -986,6 +986,35 @@ fn cmd_click( } } +/// Resolve the CGWindowID to tag background events with. +/// +/// `_AXUIElementGetWindow` normally returns the right window, but for apps that +/// render web content in a helper process it returns that helper's window while +/// events are posted to the main process. `CGEventPostToPid` then fails to +/// hit-test, silently. When the reported window is not owned by the target pid, +/// fall back to matching the owning AX window's frame against the on-screen +/// window list. +fn resolve_event_window_id(node: &AXNode, pid: i32) -> Option { + let reported = node.window_id(); + if let Some(wid) = reported + && accessibility::window_belongs_to_pid(wid, pid) + { + return Some(wid); + } + + let window = find_owning_window(node)?; + let (wx, wy) = window.position()?; + let (ww, wh) = window.size()?; + let matched = accessibility::window_id_for_frame(pid, (wx, wy, ww, wh))?; + match reported { + Some(wid) => eprintln!( + "debug: window {wid} is not owned by pid {pid}; using {matched} matched by frame" + ), + None => eprintln!("debug: no CGWindowID from AX; using {matched} matched by frame"), + } + Some(matched) +} + /// Decide which click path to use when --strategy auto. /// /// Always cg-pid — background-safe, no focus steal, works on both @@ -1004,7 +1033,7 @@ fn cmd_dblclick(ctx: &ExecutionContext, locator: &str) -> Result<(), AxError> { } } - let wid = match node.window_id() { + let wid = match resolve_event_window_id(&node, ctx.pid) { Some(w) => w, None => { return Err(AxError::ActionFailed( @@ -1135,7 +1164,7 @@ fn cmd_scroll(ctx: &ExecutionContext, locator: &str, direction: &str, pixels: i3 match resolved { ScrollStrategy::Auto => unreachable!(), ScrollStrategy::CgPid => { - let wid = match node.window_id() { + let wid = match resolve_event_window_id(&node, ctx.pid) { Some(w) => w, None => { eprintln!("warning: no window ID, falling back to global scroll");