Skip to content

fix(click): resolve window id when web content runs in a helper process #4

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
caipira113 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(click): resolve window id when web content runs in a helper process #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 132 additions & 0 deletions src/accessibility.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1743,6 +1743,138 @@ pub fn is_window_visible_at(window_id: u32, x: f64, y: f64) -> Result<bool, Stri
Err(format!("window {window_id} not found on screen"))
}

/// Find the on-screen CGWindowID owned by `pid` whose bounds match `frame`.
///
/// `_AXUIElementGetWindow` can report a window that belongs to a *helper*
/// process rather than the one that will receive our events. Microsoft Teams
/// is the motivating case: web content lives in a separate "Microsoft Teams
/// WebView" process, so an element inside the page reports that process's
/// off-screen window, while the window actually on screen belongs to the main
/// application. Posting a mismatched (pid, window id) pair to
/// `CGEventPostToPid` makes hit-testing fail silently.
///
/// Matching is done on the AX frame, which is expressed in the same top-left
/// origin coordinate space as `kCGWindowBounds`.
pub fn window_id_for_frame(pid: i32, frame: (f64, f64, f64, f64)) -> Option<u32> {
use std::ffi::c_void;
use objc2_core_foundation::{CFIndex, CFNumber, CFNumberType, CFString as CFS};
use objc2_core_graphics::{CGWindowListCopyWindowInfo, CGWindowListOption};

unsafe extern "C" {
fn CFDictionaryGetValue(dict: *const c_void, key: *const c_void) -> *const c_void;
}

let info = CGWindowListCopyWindowInfo(
CGWindowListOption::OptionOnScreenOnly | CGWindowListOption::ExcludeDesktopElements,
0,
)?;

let key_num = CFS::from_str("kCGWindowNumber");
let key_pid = CFS::from_str("kCGWindowOwnerPID");
let key_layer = CFS::from_str("kCGWindowLayer");
let key_alpha = CFS::from_str("kCGWindowAlpha");
let key_bounds = CFS::from_str("kCGWindowBounds");

let (fx, fy, fw, fh) = frame;

for i in 0..info.len() {
let dict_ptr = unsafe { info.as_opaque().value_at_index(i as CFIndex) };
if dict_ptr.is_null() {
continue;
}
let get_val = |key: &CFS| -> *const c_void {
unsafe { CFDictionaryGetValue(dict_ptr, key as *const CFS as *const c_void) }
};
let read_i64 = |ptr: *const c_void| -> Option<i64> {
if ptr.is_null() {
return None;
}
let num = unsafe { &*(ptr as *const CFNumber) };
let mut v: i64 = 0;
unsafe { num.value(CFNumberType(4), &mut v as *mut i64 as *mut _) };
Some(v)
};

if read_i64(get_val(&key_pid)) != Some(pid as i64) {
continue;
}
if read_i64(get_val(&key_layer)).unwrap_or(0) != 0 {
continue;
}

// Skip fully transparent overlays; Teams keeps one over its title bar.
let alpha_ptr = get_val(&key_alpha);
if !alpha_ptr.is_null() {
let num = unsafe { &*(alpha_ptr as *const CFNumber) };
let mut alpha: f64 = 1.0;
unsafe { num.value(CFNumberType(13), &mut alpha as *mut f64 as *mut _) };
if alpha <= 0.0 {
continue;
}
}

let bounds_ptr = get_val(&key_bounds);
if bounds_ptr.is_null() {
continue;
}
let (bx, by, bw, bh) = read_bounds_dict(bounds_ptr);

const TOLERANCE: f64 = 2.0;
if (bx - fx).abs() <= TOLERANCE
&& (by - fy).abs() <= TOLERANCE
&& (bw - fw).abs() <= TOLERANCE
&& (bh - fh).abs() <= TOLERANCE
{
return read_i64(get_val(&key_num)).map(|v| v as u32);
}
}
None
}

/// Whether `window_id` is an on-screen window owned by `pid`.
pub fn window_belongs_to_pid(window_id: u32, pid: i32) -> bool {
use std::ffi::c_void;
use objc2_core_foundation::{CFIndex, CFNumber, CFNumberType, CFString as CFS};
use objc2_core_graphics::{CGWindowListCopyWindowInfo, CGWindowListOption};

unsafe extern "C" {
fn CFDictionaryGetValue(dict: *const c_void, key: *const c_void) -> *const c_void;
}

let Some(info) = CGWindowListCopyWindowInfo(
CGWindowListOption::OptionOnScreenOnly | CGWindowListOption::ExcludeDesktopElements,
0,
) else {
return false;
};

let key_num = CFS::from_str("kCGWindowNumber");
let key_pid = CFS::from_str("kCGWindowOwnerPID");

for i in 0..info.len() {
let dict_ptr = unsafe { info.as_opaque().value_at_index(i as CFIndex) };
if dict_ptr.is_null() {
continue;
}
let get_val = |key: &CFS| -> *const c_void {
unsafe { CFDictionaryGetValue(dict_ptr, key as *const CFS as *const c_void) }
};
let read_i64 = |ptr: *const c_void| -> Option<i64> {
if ptr.is_null() {
return None;
}
let num = unsafe { &*(ptr as *const CFNumber) };
let mut v: i64 = 0;
unsafe { num.value(CFNumberType(4), &mut v as *mut i64 as *mut _) };
Some(v)
};
if read_i64(get_val(&key_num)) == Some(window_id as i64) {
return read_i64(get_val(&key_pid)) == Some(pid as i64);
}
}
false
}

fn read_bounds_dict(dict_ptr: *const std::ffi::c_void) -> (f64, f64, f64, f64) {
use std::ffi::c_void;
use objc2_core_foundation::{CFNumber, CFNumberType, CFString as CFS};
Expand Down
37 changes: 33 additions & 4 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -929,10 +929,10 @@ fn cmd_click(
if activate {
eprintln!("warning: --activate is ignored for --strategy cg-pid (background by design)");
}
let wid = match node.window_id() {
let wid = match resolve_event_window_id(&node, ctx.pid) {
Some(w) => w,
None => {
eprintln!("debug: _AXUIElementGetWindow returned no CGWindowID for this element");
eprintln!("debug: could not resolve a CGWindowID owned by pid {}", ctx.pid);
return Err(AxError::ActionFailed(
"could not find the window that owns this element".to_string(),
));
Expand Down Expand Up @@ -986,6 +986,35 @@ fn cmd_click(
}
}

/// Resolve the CGWindowID to tag background events with.
///
/// `_AXUIElementGetWindow` normally returns the right window, but for apps that
/// render web content in a helper process it returns that helper's window while
/// events are posted to the main process. `CGEventPostToPid` then fails to
/// hit-test, silently. When the reported window is not owned by the target pid,
/// fall back to matching the owning AX window's frame against the on-screen
/// window list.
fn resolve_event_window_id(node: &AXNode, pid: i32) -> Option<u32> {
let reported = node.window_id();
if let Some(wid) = reported
&& accessibility::window_belongs_to_pid(wid, pid)
{
return Some(wid);
}

let window = find_owning_window(node)?;
let (wx, wy) = window.position()?;
let (ww, wh) = window.size()?;
let matched = accessibility::window_id_for_frame(pid, (wx, wy, ww, wh))?;
match reported {
Some(wid) => eprintln!(
"debug: window {wid} is not owned by pid {pid}; using {matched} matched by frame"
),
None => eprintln!("debug: no CGWindowID from AX; using {matched} matched by frame"),
}
Some(matched)
}

/// Decide which click path to use when --strategy auto.
///
/// Always cg-pid — background-safe, no focus steal, works on both
Expand All @@ -1004,7 +1033,7 @@ fn cmd_dblclick(ctx: &ExecutionContext, locator: &str) -> Result<(), AxError> {
}
}

let wid = match node.window_id() {
let wid = match resolve_event_window_id(&node, ctx.pid) {
Some(w) => w,
None => {
return Err(AxError::ActionFailed(
Expand Down Expand Up @@ -1135,7 +1164,7 @@ fn cmd_scroll(ctx: &ExecutionContext, locator: &str, direction: &str, pixels: i3
match resolved {
ScrollStrategy::Auto => unreachable!(),
ScrollStrategy::CgPid => {
let wid = match node.window_id() {
let wid = match resolve_event_window_id(&node, ctx.pid) {
Some(w) => w,
None => {
eprintln!("warning: no window ID, falling back to global scroll");
Expand Down