From c17d2a19ce3a263b15290cae9479b0af2ae8cf2b Mon Sep 17 00:00:00 2001 From: icefairy <860668820@qq.com> Date: Fri, 31 Jul 2026 16:12:59 +0800 Subject: [PATCH 1/3] feat: support remote WS host binding (--ws-host / BSK_DAEMON_WS_HOST) Allows the WebSocket server to bind to 0.0.0.0 so the browser extension can connect from another machine. Default remains 127.0.0.1 for local-only setups. --- crates/bsk-cli/src/cli/daemon.rs | 10 ++++++++++ crates/bsk-cli/src/daemon/mod.rs | 5 +++-- crates/bsk-cli/src/daemon/start.rs | 13 +++++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/crates/bsk-cli/src/cli/daemon.rs b/crates/bsk-cli/src/cli/daemon.rs index 0651873..8326036 100644 --- a/crates/bsk-cli/src/cli/daemon.rs +++ b/crates/bsk-cli/src/cli/daemon.rs @@ -31,6 +31,11 @@ pub struct StartArgs { #[arg(long, value_name = "PORT")] pub port: Option, + /// Bind the WebSocket server to a specific IP (default 127.0.0.1). + /// Use 0.0.0.0 to allow the extension to connect from other machines. + #[arg(long, value_name = "IP", default_value = "127.0.0.1")] + pub ws_host: String, + /// Run in the foreground (do not double-fork). Useful for development. #[arg(long)] pub foreground: bool, @@ -49,6 +54,11 @@ impl StartArgs { self.port.unwrap_or(DEFAULT_WS_PORT) } + /// Resolve the WS bind host, honoring the BSK_DAEMON_WS_HOST env var. + pub fn resolved_ws_host(&self) -> String { + std::env::var("BSK_DAEMON_WS_HOST").unwrap_or_else(|_| self.ws_host.clone()) + } + pub fn resolved_session_idle(&self) -> Duration { self.session_idle.unwrap_or(DEFAULT_SESSION_IDLE) } diff --git a/crates/bsk-cli/src/daemon/mod.rs b/crates/bsk-cli/src/daemon/mod.rs index c639f73..9cde7e9 100644 --- a/crates/bsk-cli/src/daemon/mod.rs +++ b/crates/bsk-cli/src/daemon/mod.rs @@ -18,7 +18,7 @@ pub mod ws; pub use start::{DaemonConfig, run_foreground}; pub use state::{DaemonHandle, DaemonState}; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; @@ -37,8 +37,9 @@ pub async fn run( ipc_socket: Option, ) -> anyhow::Result { let ws_port = config.ws_port; + let ws_host = config.ws_host; let state = Arc::new(DaemonState::new(config)); - let ws_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), ws_port); + let ws_addr = SocketAddr::new(ws_host, ws_port); let ws_handle = ws::WsServer::new(Arc::clone(&state)).bind(ws_addr).await?; let ipc_handle = match ipc_socket { Some(path) => Some(ipc::IpcServer::new(Arc::clone(&state)).bind(path).await?), diff --git a/crates/bsk-cli/src/daemon/start.rs b/crates/bsk-cli/src/daemon/start.rs index 33327a1..3bec770 100644 --- a/crates/bsk-cli/src/daemon/start.rs +++ b/crates/bsk-cli/src/daemon/start.rs @@ -11,7 +11,7 @@ //! startup, redirects stdio to `/dev/null`, calls `setsid` (Unix), and //! falls through to `run_foreground`. -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::SocketAddr; use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -38,6 +38,8 @@ pub(crate) const DAEMONIZED_ENV: &str = "BSK_DAEMONIZED"; #[derive(Debug, Clone)] pub struct DaemonConfig { pub ws_port: u16, + /// IP the WebSocket server binds to (default 127.0.0.1). + pub ws_host: std::net::IpAddr, pub session_idle: Duration, pub daemon_idle: Duration, /// Skip the Origin allow-list (tests / `--insecure-origin`). @@ -60,6 +62,7 @@ impl DaemonConfig { pub fn new(port: u16) -> Self { Self { ws_port: port, + ws_host: std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), session_idle: Duration::from_secs(60 * 5), daemon_idle: Duration::from_secs(60 * 30), allow_any_origin: false, @@ -86,8 +89,13 @@ impl DaemonConfig { impl From<&StartArgs> for DaemonConfig { fn from(args: &StartArgs) -> Self { + let ws_host: std::net::IpAddr = args + .resolved_ws_host() + .parse() + .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); Self { ws_port: args.resolved_port(), + ws_host, session_idle: args.resolved_session_idle(), daemon_idle: args.resolved_daemon_idle(), allow_any_origin: false, @@ -220,7 +228,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> { let state = Arc::new(DaemonState::new(cfg.clone())); let session_idle_task = spawn_session_idle_reaper(Arc::clone(&state)); let browser_liveness_task = spawn_browser_liveness_reaper(Arc::clone(&state)); - let ws_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), cfg.ws_port); + let ws_addr = SocketAddr::new(cfg.ws_host, cfg.ws_port); let ws_handle = ws::WsServer::new(Arc::clone(&state)) .bind(ws_addr) .await @@ -654,6 +662,7 @@ fn apply_start_args(cmd: &mut std::process::Command, args: &StartArgs) { if let Some(p) = args.port { cmd.arg("--port").arg(p.to_string()); } + cmd.arg("--ws-host").arg(args.resolved_ws_host()); if let Some(d) = args.session_idle { cmd.arg("--session-idle").arg(format_duration(d)); } From 529f25e65d2295726d748a30114834af7884f2dd Mon Sep 17 00:00:00 2001 From: icefairy <860668820@qq.com> Date: Mon, 10 Aug 2026 08:33:19 +0800 Subject: [PATCH 2/3] feat: add bsk drag command for sliders & drag-and-drop - New tool.drag RPC (DragParams/DragResult): three modes * element/ref + dx/dy delta * viewport coordinate + delta (works in cross-origin iframes, e.g. Aliyun nc_1_nocaptcha slider) * absolute path via repeatable --point - Dispatches trusted CDP mouse events (mousePressed -> interpolated mouseMoved xN -> mouseReleased) with ease-in/out + jitter so slider CAPTCHA backends can't fingerprint a linear drag - handleDrag mirrors handleClick's Agent Window overlay bypass - BSK_NO_OVERLAY build flag: automation-only browsers can hide the on-page control mask (its capture layer swallows CDP mouse events) - CLI: bsk drag [TARGET] --dx/--dy | --from-x/--from-y | --point - SKILL.md docs for the three modes --- apps/extension/src/entrypoints/background.ts | 10 +- apps/extension/src/tools/dispatcher.ts | 12 +- apps/extension/src/tools/interaction.ts | 225 +++++++++++++++++++ apps/extension/src/transport/build-info.d.ts | 8 + apps/extension/src/transport/types.ts | 39 ++++ apps/extension/vitest.config.ts | 1 + apps/extension/wxt.config.ts | 1 + crates/bsk-cli/skill/SKILL.md | 30 ++- crates/bsk-cli/src/cli/interaction.rs | 151 ++++++++++++- crates/bsk-cli/src/cli/mod.rs | 5 +- crates/bsk-cli/src/daemon/ipc.rs | 1 + crates/bsk-cli/src/main.rs | 1 + crates/bsk-protocol/src/method.rs | 4 + crates/bsk-protocol/src/tools/interaction.rs | 88 ++++++++ 14 files changed, 567 insertions(+), 9 deletions(-) diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index ce3586f..bb35f56 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -69,10 +69,18 @@ export default defineBackground(() => { generation: overlayGeneration, }; } + // Automation-only deployments (BSK_NO_OVERLAY=1 at build time) never + // show the on-page control mask: the orange stop button is useless + // in a headless browser and its capture layer swallows CDP mouse + // events (breaking drag automation). + const mode: OverlayMode = + typeof __BSK_NO_OVERLAY__ !== "undefined" && __BSK_NO_OVERLAY__ + ? "hidden" + : (controlModes.get(ctx.sessionId) ?? "control"); return { type: OVERLAY_AGENT_STATE, sessionId: ctx.sessionId, - mode: controlModes.get(ctx.sessionId) ?? "control", + mode, generation: overlayGeneration, }; } diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 8f079b3..c434bdc 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -4,6 +4,7 @@ import type { Transport } from "@/transport/transport"; import type { ClickParams, ConsoleParams, + DragParams, EvaluateParams, FillParams, GetHtmlParams, @@ -31,7 +32,7 @@ import { isRequestFrame } from "@/transport/types"; import { handleConsole } from "./console"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handlePress, handleSelect } from "./interaction"; +import { handleClick, handleDrag, handleFill, handlePress, handleSelect } from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -388,6 +389,12 @@ export class ToolDispatcher { req.params as SelectParams, this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, ); + case "tool.drag": + return handleDrag( + this.sessions, + req.params as DragParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ); case "tool.evaluate": return handleEvaluate( this.sessions, @@ -455,7 +462,7 @@ export class ToolDispatcher { default: return { code: "unknown_method", - message: `${req.method} not implemented in extension`, + message: `${req.method} not implemented in extension [v2-DRAG]`, } satisfies RpcError; } } @@ -486,6 +493,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.fill": case "tool.press": case "tool.select": + case "tool.drag": case "tool.evaluate": case "tool.observe": case "tool.request_help": diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index 87decea..fe5cd22 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -15,6 +15,8 @@ import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { ClickParams, ClickResult, + DragParams, + DragResult, FillParams, FillResult, KeyModifier, @@ -947,6 +949,229 @@ export async function handleSelect( } } +/** + * handleDrag — mouse drag via CDP trusted events. + * + * Modes: + * - ref/selector + dx/dy: press element centre, move delta, release. + * - from_x/from_y + dx/dy: raw viewport-coordinate drag (cross-origin + * iframe slider CAPTCHAs where DOM access is blocked). + * - points: explicit absolute path [[x,y], ...]; press at first, move + * through the rest, release at last. + * + * The mouseMoved path is interpolated with `steps` (default 30) and a + * small per-step jitter so the trajectory is not a perfect straight + * line — slider CAPTCHA backends fingerprint purely linear drags. + */ +export async function handleDrag( + manager: SessionManager, + params: DragParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + if (!params) { + return { code: "invalid_params", message: "drag requires params" }; + } + const ctxOrErr = lookupSession(manager, params, "drag"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "drag aborted" }; + } + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "drag"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + + // --- Resolve start point ------------------------------------------- + let fromX: number; + let fromY: number; + let usedRef: string | undefined; + let usedSelector: string | undefined; + + const hasElementTarget = !!(params.ref || params.selector); + const hasCoordTarget = !!(params.from_x !== undefined && params.from_y !== undefined); + const hasPathTarget = !!(params.points && params.points.length >= 2); + + if (hasElementTarget) { + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "drag"); + if (isRpcError(node)) return node; + try { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); + if (scrollErr) return scrollErr; + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } + const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); + if (isRpcError(centre)) return centre; + fromX = centre.x; + fromY = centre.y; + usedRef = node.usedRef; + usedSelector = node.usedSelector; + } else if (hasCoordTarget) { + fromX = params.from_x!; + fromY = params.from_y!; + } else if (hasPathTarget) { + fromX = params.points![0][0]; + fromY = params.points![0][1]; + } else { + return rpcError( + "invalid_params", + "drag_target_required", + "drag requires a ref/selector (+dx/dy), from_x/from_y (+dx/dy), or points[]", + ); + } + + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "drag aborted" }; + } + + // --- Build the point path ------------------------------------------ + const button: MouseButton = params.button ?? "left"; + const modifiers = modifiersBitfield(params.modifiers); + const steps = Math.max(1, params.steps ?? 30); + const stepDelayMs = params.step_delay_ms ?? 8; + + let path: Array<{ x: number; y: number }>; + if (hasPathTarget) { + // Absolute path: keep caller's points verbatim (they are already + // explicit), but insert a move BEFORE the first press so hover + // state activates, matching handleClick's behaviour. + path = params.points!.map((pt: number[]) => ({ x: pt[0], y: pt[1] })); + } else { + const dx = params.dx ?? 0; + const dy = params.dy ?? 0; + const toX = fromX + dx; + const toY = fromY + dy; + path = []; + for (let i = 1; i <= steps; i++) { + const t = i / steps; + // Ease-in-out for the first/last few steps, straight in the middle. + const eased = t < 0.15 ? t / 0.15 * 0.5 : t > 0.85 ? 0.5 + (t - 0.85) / 0.15 * 0.5 : 0.5 + (t - 0.5); + const x = fromX + dx * eased + jitter(0.7); + const y = fromY + dy * eased + jitter(0.7); + path.push({ x, y }); + } + } + + // --- Dispatch ------------------------------------------------------- + // The Agent Window overlay sits above the page and would swallow the + // trusted mouse events. Mirror handleClick: detect the overlay at the + // start point, temporarily hide it for the duration of the drag, and + // restore it afterwards. + let automationBypassEnabled = false; + try { + const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, fromX, fromY); + if (overlayBlocking && deps.bypassOverlay) { + await deps.bypassOverlay(target.tabId, true); + automationBypassEnabled = true; + } + } catch (err) { + console.debug("[bsk interaction] drag overlay bypass enable failed", err); + } + try { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + // Move to start (hover) then press. + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: fromX, + y: fromY, + modifiers, + }); + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "drag aborted" }; + } + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mousePressed", + x: fromX, + y: fromY, + button, + clickCount: 1, + modifiers, + }); + // Interpolated moves. + for (const pt of path) { + if (throwIfAborted(deps.signal)) { + await safeMouseRelease(deps.cdp, target.tabId, pt.x, pt.y, button, modifiers); + return { code: "cancelled", message: "drag aborted" }; + } + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: pt.x, + y: pt.y, + modifiers, + }); + if (stepDelayMs > 0) { + await delay(stepDelayMs); + } + } + // Release at final point. + const last = path[path.length - 1]; + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseReleased", + x: last.x, + y: last.y, + button, + clickCount: 1, + modifiers, + }); + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: usedRef, + used_selector: usedSelector, + from_x: fromX, + from_y: fromY, + to_x: last.x, + to_y: last.y, + steps: path.length, + }); + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } finally { + if (automationBypassEnabled && deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, false); + } catch (err) { + console.debug("[bsk interaction] drag overlay bypass disable failed", err); + } + } + } +} + +function jitter(amplitude: number): number { + return (Math.random() * 2 - 1) * amplitude; +} + +async function safeMouseRelease( + cdp: unknown, + tabId: number, + x: number, + y: number, + button: MouseButton, + modifiers: number, +): Promise { + try { + await (cdp as { send: (t: number, m: string, p: Record) => Promise }).send( + tabId, + "Input.dispatchMouseEvent", + { type: "mouseReleased", x, y, button, clickCount: 1, modifiers }, + ); + } catch { + // best effort + } +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + export const __testing__ = { DEFAULT_TIMEOUT_MS, quadCentre, diff --git a/apps/extension/src/transport/build-info.d.ts b/apps/extension/src/transport/build-info.d.ts index dc8542b..7a2ac11 100644 --- a/apps/extension/src/transport/build-info.d.ts +++ b/apps/extension/src/transport/build-info.d.ts @@ -12,3 +12,11 @@ declare const __BSK_EXT_VERSION__: string; * setting the {@code BSK_DAEMON_WS_URL} environment variable. */ declare const __BSK_DAEMON_WS_URL__: string; + +/** + * When true, the on-page "Agent 正在控制" control mask is never shown. + * Set {@code BSK_NO_OVERLAY=1} at build time for automation-only + * browsers: the orange stop button is useless there and its capture + * layer can swallow CDP mouse events. + */ +declare const __BSK_NO_OVERLAY__: boolean; diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 4bc2d61..703cb13 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -29,6 +29,7 @@ export type RpcErrorReason = | "target_not_select" | "option_not_found" | "single_select_value_count" + | "drag_target_required" | "tab_not_active" | "restricted_tab_url" | "borrow_conflict"; @@ -288,6 +289,8 @@ export interface ScreenshotParams { tab_id?: number; /** `@eN` ref from the last `tool.snapshot`. */ ref?: string; + /** When true, capture the full scrollable page instead of just the visible viewport. */ + full_page?: boolean; } export interface ScreenshotResult { @@ -456,6 +459,42 @@ export interface SelectParams { timeout_ms?: number; } +export interface DragParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + /** Start X (viewport CSS px) for coordinate drags. Requires from_y. */ + from_x?: number; + /** Start Y (viewport CSS px) for coordinate drags. Requires from_x. */ + from_y?: number; + /** Horizontal delta in CSS px. */ + dx?: number; + /** Vertical delta in CSS px. */ + dy?: number; + /** Explicit viewport points [x, y] for absolute-path drags. */ + points?: number[][]; + /** Number of interpolation steps for delta drags (default 30). */ + steps?: number; + /** Per-step delay in ms (default 8). */ + step_delay_ms?: number; + button?: MouseButton; + modifiers?: KeyModifier[]; + timeout_ms?: number; +} + +export interface DragResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + from_x: number; + from_y: number; + to_x: number; + to_y: number; + steps: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface SelectResult { tab_id: number; used_ref?: string; diff --git a/apps/extension/vitest.config.ts b/apps/extension/vitest.config.ts index 00060bf..f5df35e 100644 --- a/apps/extension/vitest.config.ts +++ b/apps/extension/vitest.config.ts @@ -22,6 +22,7 @@ export default defineConfig({ define: { __BSK_EXT_VERSION__: JSON.stringify(pkg.version), __BSK_DAEMON_WS_URL__: JSON.stringify(process.env.BSK_DAEMON_WS_URL ?? "ws://127.0.0.1:52800"), + __BSK_NO_OVERLAY__: JSON.stringify(false), }, resolve: { alias: { diff --git a/apps/extension/wxt.config.ts b/apps/extension/wxt.config.ts index 34c72b6..5ef97a1 100644 --- a/apps/extension/wxt.config.ts +++ b/apps/extension/wxt.config.ts @@ -75,6 +75,7 @@ export default defineConfig({ __BSK_DAEMON_WS_URL__: JSON.stringify( process.env.BSK_DAEMON_WS_URL ?? "ws://127.0.0.1:52800", ), + __BSK_NO_OVERLAY__: JSON.stringify(process.env.BSK_NO_OVERLAY === "1" || process.env.BSK_NO_OVERLAY === "true"), }, resolve: { alias: { diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 43d2417..18009f8 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -84,7 +84,7 @@ Start with `bsk snapshot` to understand page structure, text, controls, and elem 1. `bsk snapshot` — strict static page understanding and interaction planning 2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery 3. `bsk get-html` — when hidden DOM, metadata, or markup details are required -4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. +4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab; use `--full-page` to capture the entire scrollable page. Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. @@ -140,13 +140,13 @@ Details and flags: **`bsk --help`** | `bsk tab return ` | Return a borrowed tab to its original window | ### Observation (require `--session` unless noted) - +### Observation (require `--session` unless noted) | Command | Summary | |---------|---------| | `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | | `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | | `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | +| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element, or `--full-page` for the entire scrollable page (`--out` path optional) | ### Navigation @@ -167,6 +167,30 @@ Details and flags: **`bsk --help`** | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option values by `value` attribute. Select(SelectArgs), + /// Drag an element, a viewport coordinate, or an absolute path. + Drag(DragArgs), + /// Evaluate a JavaScript expression inside the Agent Window. Evaluate(EvaluateArgs), diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 0efb977..a685455 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -241,6 +241,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolFill | Method::ToolPress | Method::ToolSelect + | Method::ToolDrag | Method::ToolEvaluate | Method::ToolWaitForNavigation | Method::ToolRequestHelp diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index 75d912c..d88c78f 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -94,6 +94,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Fill(args) => cli::interaction::dispatch_fill(args, format), Command::Press(args) => cli::interaction::dispatch_press(args, format), Command::Select(args) => cli::interaction::dispatch_select(args, format), + Command::Drag(args) => cli::interaction::dispatch_drag(args, format), Command::Evaluate(args) => cli::evaluate::dispatch(args, format), Command::WaitForNavigation(args) => cli::waits::dispatch_wait_for_navigation(args, format), Command::WaitMs(args) => cli::waits::dispatch_wait_ms(args, format), diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index 6b712bf..8a80942 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -72,6 +72,8 @@ pub enum Method { ToolPress, #[serde(rename = "tool.select")] ToolSelect, + #[serde(rename = "tool.drag")] + ToolDrag, #[serde(rename = "tool.snapshot")] ToolSnapshot, #[serde(rename = "tool.observe")] @@ -148,6 +150,7 @@ impl Method { | Method::ToolFill | Method::ToolPress | Method::ToolSelect + | Method::ToolDrag | Method::ToolEvaluate // May navigate via optional `url` and changes Agent Window // chrome; gate behind pending-interrupt like other writes. @@ -270,6 +273,7 @@ mod tests { assert!(Method::ToolFill.is_mutating()); assert!(Method::ToolPress.is_mutating()); assert!(Method::ToolSelect.is_mutating()); + assert!(Method::ToolDrag.is_mutating()); assert!(Method::ToolEvaluate.is_mutating()); assert!(Method::ToolRecordStart.is_mutating()); } diff --git a/crates/bsk-protocol/src/tools/interaction.rs b/crates/bsk-protocol/src/tools/interaction.rs index 7837f3b..77c98f2 100644 --- a/crates/bsk-protocol/src/tools/interaction.rs +++ b/crates/bsk-protocol/src/tools/interaction.rs @@ -226,6 +226,94 @@ pub struct SelectResult { pub dialogs: Vec, } +// --------------------------------------------------------------------------- +// drag +// --------------------------------------------------------------------------- + +/// Drag parameters. Supports three targeting modes: +/// +/// 1. **Element drag** (`ref` / `selector` + `dx` / `dy`): press the +/// element's centre, move by the given pixel delta (CSS pixels), then +/// release. `steps` interpolates a human-like curved path (default 30). +/// 2. **Coordinate drag** (`from_x` / `from_y` + `dx` / `dy`): raw +/// viewport-coordinate drag, no element lookup. Useful for slider +/// CAPTCHAs inside cross-origin iframes where DOM access is blocked. +/// 3. **Absolute path** (`points`): explicit list of `[x, y]` viewport +/// points; press at the first, move through the rest, release at the +/// last. +/// +/// The drag is dispatched through CDP `Input.dispatchMouseEvent` +/// (`mousePressed` → `mouseMoved` × N → `mouseReleased`), so events are +/// trusted (native `isTrusted=true`), which is what slider CAPTCHA +/// services such as Aliyun `nc_1_nocaptcha` check for. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DragParams { + pub session_id: String, + /// Optional `@e` ref for element-targeted drags. Mutually + /// exclusive with `selector` / `from_x` / `points`. + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + /// Target tab. Defaults to the Agent Window's active tab. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + /// Start X (viewport CSS px) for coordinate drags. Requires `from_y`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_x: Option, + /// Start Y (viewport CSS px) for coordinate drags. Requires `from_x`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_y: Option, + /// Horizontal delta in CSS px (element & coordinate modes). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dx: Option, + /// Vertical delta in CSS px (element & coordinate modes). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dy: Option, + /// Explicit viewport points `[x, y]` for absolute-path drags. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub points: Option>>, + /// Number of interpolation steps for delta drags (default 30). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub steps: Option, + /// Per-step delay in ms (default 8). Slightly randomises timing to + /// look human. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub step_delay_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub button: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modifiers: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DragResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + /// Start point of the drag (viewport CSS px). + pub from_x: f64, + pub from_y: f64, + /// End point of the drag (viewport CSS px). + pub to_x: f64, + pub to_y: f64, + /// Number of mouseMoved steps actually dispatched. + pub steps: u32, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dialogs: Vec, +} + #[cfg(test)] mod tests { use super::*; From 30b13a8e0b8a1586a801f0a2bf022d4e6c0a1f80 Mon Sep 17 00:00:00 2001 From: icefairy <860668820@qq.com> Date: Mon, 10 Aug 2026 08:44:03 +0800 Subject: [PATCH 3/3] feat(drag): human-like trajectory & randomized grab point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - buildHumanDragPath: smooth bounded perpendicular random-walk (low-pass filtered, ±3px) + ease-in-out, so the mouse path wobbles like a hand instead of a perfect straight line - element drags now grab a random point in the middle 40-60% of the element box instead of the exact centroid (CAPTCHA bots press dead-centre) - 3 unit tests for trajectory shape (bounded wobble, axis independence, endpoint accuracy) --- .../src/tools/__tests__/interaction.test.ts | 45 ++++++++++ apps/extension/src/tools/interaction.ts | 85 ++++++++++++++++--- 2 files changed, 119 insertions(+), 11 deletions(-) diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index de56af4..4d970b9 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { + buildHumanDragPath, handleClick, handleFill, handlePress, @@ -982,3 +983,47 @@ describe("handleSelect", () => { expect(fake.sent.some((c) => c.method === "Runtime.callFunctionOn")).toBe(false); }); }); + +describe("buildHumanDragPath", () => { + it("starts at the origin and ends at the destination", () => { + const path = buildHumanDragPath(100, 100, 200, 100, 30); + expect(path.length).toBe(30); + expect(path[0].x).toBeGreaterThanOrEqual(99); + expect(path[0].x).toBeLessThanOrEqual(101); + expect(path[0].y).toBeGreaterThanOrEqual(99); + expect(path[0].y).toBeLessThanOrEqual(101); + expect(path[29].x).toBeGreaterThanOrEqual(199); + expect(path[29].x).toBeLessThanOrEqual(201); + expect(path[29].y).toBeGreaterThanOrEqual(99); + expect(path[29].y).toBeLessThanOrEqual(101); + }); + + it("is not a perfect straight line: has bounded perpendicular wobble", () => { + // 200 runs; every intermediate y must deviate from the exact + // line y=100 by a few px, and at least one run must wobble > 0.5px. + let maxWobble = 0; + for (let run = 0; run < 200; run++) { + const path = buildHumanDragPath(100, 100, 300, 100, 40); + for (let i = 1; i < path.length - 1; i++) { + const dev = Math.abs(path[i].y - 100); + expect(dev).toBeLessThanOrEqual(3.01); // ±3px ceiling + if (dev > maxWobble) maxWobble = dev; + } + } + expect(maxWobble).toBeGreaterThan(0.5); + }); + + it("applies perpendicular wobble to vertical drags too", () => { + // Vertical drag from (100,100) to (100,300): the wobble axis is X. + let sawDeviation = false; + for (let run = 0; run < 100; run++) { + const path = buildHumanDragPath(100, 100, 100, 300, 30); + for (let i = 1; i < path.length - 1; i++) { + const dev = Math.abs(path[i].x - 100); + expect(dev).toBeLessThanOrEqual(3.01); + if (dev > 0.5) sawDeviation = true; + } + } + expect(sawDeviation).toBe(true); + }); +}); diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index fe5cd22..92a54a3 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -31,6 +31,7 @@ import { attachDialogs, markDialogCursor } from "./dialogs"; import { backendNodeToObject, boxCentre, + nodeBoundingRect, nodeCentre, quadCentre, scrollNodeIntoView, @@ -1008,8 +1009,20 @@ export async function handleDrag( } const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); if (isRpcError(centre)) return centre; - fromX = centre.x; - fromY = centre.y; + // Pick a START POINT inside the element's box, randomly biased + // toward the middle (40–60% of width/height) instead of the exact + // centre. Humans grab a slider thumb somewhere near its middle, + // never pixel-perfectly at the centroid — CAPTCHA backends treat a + // dead-centre press as a bot tell. + const bounds = await nodeBoundingRect(deps.cdp, target.tabId, node.backendNodeId); + let startX = centre.x; + let startY = centre.y; + if (!isRpcError(bounds)) { + startX = bounds.x + bounds.width * (0.4 + Math.random() * 0.2); + startY = bounds.y + bounds.height * (0.4 + Math.random() * 0.2); + } + fromX = startX; + fromY = startY; usedRef = node.usedRef; usedSelector = node.usedSelector; } else if (hasCoordTarget) { @@ -1047,15 +1060,7 @@ export async function handleDrag( const dy = params.dy ?? 0; const toX = fromX + dx; const toY = fromY + dy; - path = []; - for (let i = 1; i <= steps; i++) { - const t = i / steps; - // Ease-in-out for the first/last few steps, straight in the middle. - const eased = t < 0.15 ? t / 0.15 * 0.5 : t > 0.85 ? 0.5 + (t - 0.85) / 0.15 * 0.5 : 0.5 + (t - 0.5); - const x = fromX + dx * eased + jitter(0.7); - const y = fromY + dy * eased + jitter(0.7); - path.push({ x, y }); - } + path = buildHumanDragPath(fromX, fromY, toX, toY, steps); } // --- Dispatch ------------------------------------------------------- @@ -1149,6 +1154,64 @@ function jitter(amplitude: number): number { return (Math.random() * 2 - 1) * amplitude; } +/** + * Build a drag trajectory that mimics a human hand: + * + * - ease-in-out along the main axis (slow start / slow stop), + * - a smooth, bounded random-walk deviation PERPENDICULAR to the + * travel direction (a few px either side), instead of a perfect + * straight line — CAPTCHA backends fingerprint both dead-straight + * and white-noise-jittered paths, + * - the perpendicular offset is low-pass filtered (running average) + * so it wobbles like a hand, not flickers like a signal. + * + * Bounds the deviation at ±3px so slider tracks that need near-linear + * motion (Aliyun `nc_1_nocaptcha`) still pass; `steps` keeps the + * per-step movement small. + */ +export function buildHumanDragPath( + fromX: number, + fromY: number, + toX: number, + toY: number, + steps: number, +): Array<{ x: number; y: number }> { + const n = Math.max(2, steps); + const dx = toX - fromX; + const dy = toY - fromY; + const dist = Math.hypot(dx, dy) || 1; + // Unit normal (perpendicular) to the travel direction. + const nx = -dy / dist; + const ny = dx / dist; + const maxDev = Math.min(3, Math.max(0.6, dist * 0.02)); // ±3px ceiling, scaled down for tiny drags + + // Random-walk perpendicular offsets, smoothed by a 3-tap average. + const raw: number[] = []; + let wander = 0; + for (let i = 0; i < n; i++) { + wander += jitter(0.9); // per-step increment, small + wander = Math.max(-maxDev, Math.min(maxDev, wander)); + raw.push(wander); + } + const offsets = raw.map((_, i) => { + const prev = raw[Math.max(0, i - 1)]; + const next = raw[Math.min(n - 1, i + 1)]; + return (prev + raw[i] + next) / 3; + }); + + const path: Array<{ x: number; y: number }> = []; + for (let i = 1; i <= n; i++) { + const t = i / n; + // Ease-in-out (smoothstep) along the travel axis. + const eased = t * t * (3 - 2 * t); + const baseX = fromX + dx * eased; + const baseY = fromY + dy * eased; + const off = offsets[i - 1]; + path.push({ x: baseX + nx * off, y: baseY + ny * off }); + } + return path; +} + async function safeMouseRelease( cdp: unknown, tabId: number,