diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index ce3586f..bb35f56 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -69,10 +69,18 @@ export default defineBackground(() => { generation: overlayGeneration, }; } + // Automation-only deployments (BSK_NO_OVERLAY=1 at build time) never + // show the on-page control mask: the orange stop button is useless + // in a headless browser and its capture layer swallows CDP mouse + // events (breaking drag automation). + const mode: OverlayMode = + typeof __BSK_NO_OVERLAY__ !== "undefined" && __BSK_NO_OVERLAY__ + ? "hidden" + : (controlModes.get(ctx.sessionId) ?? "control"); return { type: OVERLAY_AGENT_STATE, sessionId: ctx.sessionId, - mode: controlModes.get(ctx.sessionId) ?? "control", + mode, generation: overlayGeneration, }; } diff --git a/apps/extension/src/tools/__tests__/interaction.test.ts b/apps/extension/src/tools/__tests__/interaction.test.ts index 6f8eeed..26ea1eb 100644 --- a/apps/extension/src/tools/__tests__/interaction.test.ts +++ b/apps/extension/src/tools/__tests__/interaction.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import { SessionManager } from "@/session-manager/manager"; import type { CdpRunner } from "@/tools/shared"; import { + buildHumanDragPath, handleClick, handleFill, handleHover, @@ -1027,3 +1028,47 @@ describe("handleSelect", () => { expect(fake.sent.some((c) => c.method === "Runtime.callFunctionOn")).toBe(false); }); }); + +describe("buildHumanDragPath", () => { + it("starts at the origin and ends at the destination", () => { + const path = buildHumanDragPath(100, 100, 200, 100, 30); + expect(path.length).toBe(30); + expect(path[0].x).toBeGreaterThanOrEqual(99); + expect(path[0].x).toBeLessThanOrEqual(101); + expect(path[0].y).toBeGreaterThanOrEqual(99); + expect(path[0].y).toBeLessThanOrEqual(101); + expect(path[29].x).toBeGreaterThanOrEqual(199); + expect(path[29].x).toBeLessThanOrEqual(201); + expect(path[29].y).toBeGreaterThanOrEqual(99); + expect(path[29].y).toBeLessThanOrEqual(101); + }); + + it("is not a perfect straight line: has bounded perpendicular wobble", () => { + // 200 runs; every intermediate y must deviate from the exact + // line y=100 by a few px, and at least one run must wobble > 0.5px. + let maxWobble = 0; + for (let run = 0; run < 200; run++) { + const path = buildHumanDragPath(100, 100, 300, 100, 40); + for (let i = 1; i < path.length - 1; i++) { + const dev = Math.abs(path[i].y - 100); + expect(dev).toBeLessThanOrEqual(3.01); // ±3px ceiling + if (dev > maxWobble) maxWobble = dev; + } + } + expect(maxWobble).toBeGreaterThan(0.5); + }); + + it("applies perpendicular wobble to vertical drags too", () => { + // Vertical drag from (100,100) to (100,300): the wobble axis is X. + let sawDeviation = false; + for (let run = 0; run < 100; run++) { + const path = buildHumanDragPath(100, 100, 100, 300, 30); + for (let i = 1; i < path.length - 1; i++) { + const dev = Math.abs(path[i].x - 100); + expect(dev).toBeLessThanOrEqual(3.01); + if (dev > 0.5) sawDeviation = true; + } + } + expect(sawDeviation).toBe(true); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 26ec56d..bc23fdf 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -5,6 +5,7 @@ import type { ClickParams, ConsoleParams, EmulateParams, + DragParams, EvaluateParams, FillParams, GetHtmlParams, @@ -35,7 +36,7 @@ import { handleConsole } from "./console"; import { type EmulateCdpRunner, handleEmulate } from "./emulate"; import { handleEvaluate } from "./evaluate"; import { handleRequestHelp } from "./human-loop"; -import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; +import { handleClick, handleDrag, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; import { handleNavigate, handleNavigateBack, @@ -461,6 +462,12 @@ export class ToolDispatcher { this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, ), ); + case "tool.drag": + return handleDrag( + this.sessions, + req.params as DragParams, + this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined, + ); case "tool.evaluate": return handleEvaluate( this.sessions, @@ -528,7 +535,7 @@ export class ToolDispatcher { default: return { code: "unknown_method", - message: `${req.method} not implemented in extension`, + message: `${req.method} not implemented in extension [v2-DRAG]`, } satisfies RpcError; } } @@ -670,6 +677,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null { case "tool.fill": case "tool.press": case "tool.select": + case "tool.drag": case "tool.evaluate": case "tool.observe": case "tool.request_help": diff --git a/apps/extension/src/tools/interaction.ts b/apps/extension/src/tools/interaction.ts index ee66dac..3e0a8f6 100644 --- a/apps/extension/src/tools/interaction.ts +++ b/apps/extension/src/tools/interaction.ts @@ -15,6 +15,8 @@ import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { ClickParams, ClickResult, + DragParams, + DragResult, FillParams, FillResult, HoverParams, @@ -31,6 +33,7 @@ import { attachDialogs, markDialogCursor } from "./dialogs"; import { backendNodeToObject, boxCentre, + nodeBoundingRect, nodeCentre, quadCentre, scrollNodeIntoView, @@ -1070,6 +1073,291 @@ export async function handleSelect( } } +/** + * handleDrag — mouse drag via CDP trusted events. + * + * Modes: + * - ref/selector + dx/dy: press element centre, move delta, release. + * - from_x/from_y + dx/dy: raw viewport-coordinate drag (cross-origin + * iframe slider CAPTCHAs where DOM access is blocked). + * - points: explicit absolute path [[x,y], ...]; press at first, move + * through the rest, release at last. + * + * The mouseMoved path is interpolated with `steps` (default 30) and a + * small per-step jitter so the trajectory is not a perfect straight + * line — slider CAPTCHA backends fingerprint purely linear drags. + */ +export async function handleDrag( + manager: SessionManager, + params: DragParams, + deps: InteractionDeps = getDefaultDeps(), +): Promise { + if (!params) { + return { code: "invalid_params", message: "drag requires params" }; + } + const ctxOrErr = lookupSession(manager, params, "drag"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "drag aborted" }; + } + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + const denied = enforceAgentWindow(ctx, target, "drag"); + if (denied) return denied; + const dialogCursor = markDialogCursor(deps.cdp, target.tabId); + + // --- Resolve start point ------------------------------------------- + let fromX: number; + let fromY: number; + let usedRef: string | undefined; + let usedSelector: string | undefined; + + const hasElementTarget = !!(params.ref || params.selector); + const hasCoordTarget = !!(params.from_x !== undefined && params.from_y !== undefined); + const hasPathTarget = !!(params.points && params.points.length >= 2); + + if (hasElementTarget) { + const node = await resolveBackendNode(deps.cdp, ctx, target, params, "drag"); + if (isRpcError(node)) return node; + try { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId); + if (scrollErr) return scrollErr; + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } + const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId); + if (isRpcError(centre)) return centre; + // Pick a START POINT inside the element's box, randomly biased + // toward the middle (40–60% of width/height) instead of the exact + // centre. Humans grab a slider thumb somewhere near its middle, + // never pixel-perfectly at the centroid — CAPTCHA backends treat a + // dead-centre press as a bot tell. + const bounds = await nodeBoundingRect(deps.cdp, target.tabId, node.backendNodeId); + let startX = centre.x; + let startY = centre.y; + if (!isRpcError(bounds)) { + startX = bounds.x + bounds.width * (0.4 + Math.random() * 0.2); + startY = bounds.y + bounds.height * (0.4 + Math.random() * 0.2); + } + fromX = startX; + fromY = startY; + usedRef = node.usedRef; + usedSelector = node.usedSelector; + } else if (hasCoordTarget) { + fromX = params.from_x!; + fromY = params.from_y!; + } else if (hasPathTarget) { + fromX = params.points![0][0]; + fromY = params.points![0][1]; + } else { + return rpcError( + "invalid_params", + "drag_target_required", + "drag requires a ref/selector (+dx/dy), from_x/from_y (+dx/dy), or points[]", + ); + } + + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "drag aborted" }; + } + + // --- Build the point path ------------------------------------------ + const button: MouseButton = params.button ?? "left"; + const modifiers = modifiersBitfield(params.modifiers); + const steps = Math.max(1, params.steps ?? 30); + const stepDelayMs = params.step_delay_ms ?? 8; + + let path: Array<{ x: number; y: number }>; + if (hasPathTarget) { + // Absolute path: keep caller's points verbatim (they are already + // explicit), but insert a move BEFORE the first press so hover + // state activates, matching handleClick's behaviour. + path = params.points!.map((pt: number[]) => ({ x: pt[0], y: pt[1] })); + } else { + const dx = params.dx ?? 0; + const dy = params.dy ?? 0; + const toX = fromX + dx; + const toY = fromY + dy; + path = buildHumanDragPath(fromX, fromY, toX, toY, steps); + } + + // --- Dispatch ------------------------------------------------------- + // The Agent Window overlay sits above the page and would swallow the + // trusted mouse events. Mirror handleClick: detect the overlay at the + // start point, temporarily hide it for the duration of the drag, and + // restore it afterwards. + let automationBypassEnabled = false; + try { + const overlayBlocking = await checkOverlayAtPoint(deps.cdp, target.tabId, fromX, fromY); + if (overlayBlocking && deps.bypassOverlay) { + await deps.bypassOverlay(target.tabId, true); + automationBypassEnabled = true; + } + } catch (err) { + console.debug("[bsk interaction] drag overlay bypass enable failed", err); + } + try { + deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId); + // Move to start (hover) then press. + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: fromX, + y: fromY, + modifiers, + }); + if (throwIfAborted(deps.signal)) { + return { code: "cancelled", message: "drag aborted" }; + } + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mousePressed", + x: fromX, + y: fromY, + button, + clickCount: 1, + modifiers, + }); + // Interpolated moves. + for (const pt of path) { + if (throwIfAborted(deps.signal)) { + await safeMouseRelease(deps.cdp, target.tabId, pt.x, pt.y, button, modifiers); + return { code: "cancelled", message: "drag aborted" }; + } + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseMoved", + x: pt.x, + y: pt.y, + modifiers, + }); + if (stepDelayMs > 0) { + await delay(stepDelayMs); + } + } + // Release at final point. + const last = path[path.length - 1]; + await deps.cdp.send(target.tabId, "Input.dispatchMouseEvent", { + type: "mouseReleased", + x: last.x, + y: last.y, + button, + clickCount: 1, + modifiers, + }); + return attachDialogs(deps.cdp, target.tabId, dialogCursor, { + tab_id: target.tabId, + used_ref: usedRef, + used_selector: usedSelector, + from_x: fromX, + from_y: fromY, + to_x: last.x, + to_y: last.y, + steps: path.length, + }); + } catch (err) { + return { + code: "cdp_failed", + message: err instanceof Error ? err.message : String(err), + }; + } finally { + if (automationBypassEnabled && deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, false); + } catch (err) { + console.debug("[bsk interaction] drag overlay bypass disable failed", err); + } + } + } +} + +function jitter(amplitude: number): number { + return (Math.random() * 2 - 1) * amplitude; +} + +/** + * Build a drag trajectory that mimics a human hand: + * + * - ease-in-out along the main axis (slow start / slow stop), + * - a smooth, bounded random-walk deviation PERPENDICULAR to the + * travel direction (a few px either side), instead of a perfect + * straight line — CAPTCHA backends fingerprint both dead-straight + * and white-noise-jittered paths, + * - the perpendicular offset is low-pass filtered (running average) + * so it wobbles like a hand, not flickers like a signal. + * + * Bounds the deviation at ±3px so slider tracks that need near-linear + * motion (Aliyun `nc_1_nocaptcha`) still pass; `steps` keeps the + * per-step movement small. + */ +export function buildHumanDragPath( + fromX: number, + fromY: number, + toX: number, + toY: number, + steps: number, +): Array<{ x: number; y: number }> { + const n = Math.max(2, steps); + const dx = toX - fromX; + const dy = toY - fromY; + const dist = Math.hypot(dx, dy) || 1; + // Unit normal (perpendicular) to the travel direction. + const nx = -dy / dist; + const ny = dx / dist; + const maxDev = Math.min(3, Math.max(0.6, dist * 0.02)); // ±3px ceiling, scaled down for tiny drags + + // Random-walk perpendicular offsets, smoothed by a 3-tap average. + const raw: number[] = []; + let wander = 0; + for (let i = 0; i < n; i++) { + wander += jitter(0.9); // per-step increment, small + wander = Math.max(-maxDev, Math.min(maxDev, wander)); + raw.push(wander); + } + const offsets = raw.map((_, i) => { + const prev = raw[Math.max(0, i - 1)]; + const next = raw[Math.min(n - 1, i + 1)]; + return (prev + raw[i] + next) / 3; + }); + + const path: Array<{ x: number; y: number }> = []; + for (let i = 1; i <= n; i++) { + const t = i / n; + // Ease-in-out (smoothstep) along the travel axis. + const eased = t * t * (3 - 2 * t); + const baseX = fromX + dx * eased; + const baseY = fromY + dy * eased; + const off = offsets[i - 1]; + path.push({ x: baseX + nx * off, y: baseY + ny * off }); + } + return path; +} + +async function safeMouseRelease( + cdp: unknown, + tabId: number, + x: number, + y: number, + button: MouseButton, + modifiers: number, +): Promise { + try { + await (cdp as { send: (t: number, m: string, p: Record) => Promise }).send( + tabId, + "Input.dispatchMouseEvent", + { type: "mouseReleased", x, y, button, clickCount: 1, modifiers }, + ); + } catch { + // best effort + } +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + export const __testing__ = { DEFAULT_TIMEOUT_MS, quadCentre, diff --git a/apps/extension/src/transport/build-info.d.ts b/apps/extension/src/transport/build-info.d.ts index dc8542b..7a2ac11 100644 --- a/apps/extension/src/transport/build-info.d.ts +++ b/apps/extension/src/transport/build-info.d.ts @@ -12,3 +12,11 @@ declare const __BSK_EXT_VERSION__: string; * setting the {@code BSK_DAEMON_WS_URL} environment variable. */ declare const __BSK_DAEMON_WS_URL__: string; + +/** + * When true, the on-page "Agent 正在控制" control mask is never shown. + * Set {@code BSK_NO_OVERLAY=1} at build time for automation-only + * browsers: the orange stop button is useless there and its capture + * layer can swallow CDP mouse events. + */ +declare const __BSK_NO_OVERLAY__: boolean; diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 4533d7d..27a8c59 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -29,6 +29,7 @@ export type RpcErrorReason = | "target_not_select" | "option_not_found" | "single_select_value_count" + | "drag_target_required" | "tab_not_active" | "restricted_tab_url" | "borrow_conflict"; @@ -288,6 +289,8 @@ export interface ScreenshotParams { tab_id?: number; /** `@eN` ref from the last `tool.snapshot`. */ ref?: string; + /** When true, capture the full scrollable page instead of just the visible viewport. */ + full_page?: boolean; } export interface ScreenshotResult { @@ -488,6 +491,42 @@ export interface SelectParams { timeout_ms?: number; } +export interface DragParams { + session_id: string; + ref?: string; + selector?: string; + tab_id?: number; + /** Start X (viewport CSS px) for coordinate drags. Requires from_y. */ + from_x?: number; + /** Start Y (viewport CSS px) for coordinate drags. Requires from_x. */ + from_y?: number; + /** Horizontal delta in CSS px. */ + dx?: number; + /** Vertical delta in CSS px. */ + dy?: number; + /** Explicit viewport points [x, y] for absolute-path drags. */ + points?: number[][]; + /** Number of interpolation steps for delta drags (default 30). */ + steps?: number; + /** Per-step delay in ms (default 8). */ + step_delay_ms?: number; + button?: MouseButton; + modifiers?: KeyModifier[]; + timeout_ms?: number; +} + +export interface DragResult { + tab_id: number; + used_ref?: string; + used_selector?: string; + from_x: number; + from_y: number; + to_x: number; + to_y: number; + steps: number; + dialogs?: JavaScriptDialogInfo[]; +} + export interface SelectResult { tab_id: number; used_ref?: string; diff --git a/apps/extension/vitest.config.ts b/apps/extension/vitest.config.ts index 00060bf..f5df35e 100644 --- a/apps/extension/vitest.config.ts +++ b/apps/extension/vitest.config.ts @@ -22,6 +22,7 @@ export default defineConfig({ define: { __BSK_EXT_VERSION__: JSON.stringify(pkg.version), __BSK_DAEMON_WS_URL__: JSON.stringify(process.env.BSK_DAEMON_WS_URL ?? "ws://127.0.0.1:52800"), + __BSK_NO_OVERLAY__: JSON.stringify(false), }, resolve: { alias: { diff --git a/apps/extension/wxt.config.ts b/apps/extension/wxt.config.ts index 34c72b6..5ef97a1 100644 --- a/apps/extension/wxt.config.ts +++ b/apps/extension/wxt.config.ts @@ -75,6 +75,7 @@ export default defineConfig({ __BSK_DAEMON_WS_URL__: JSON.stringify( process.env.BSK_DAEMON_WS_URL ?? "ws://127.0.0.1:52800", ), + __BSK_NO_OVERLAY__: JSON.stringify(process.env.BSK_NO_OVERLAY === "1" || process.env.BSK_NO_OVERLAY === "true"), }, resolve: { alias: { diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 9a1b6a8..71bca72 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -87,7 +87,7 @@ Start with `bsk observe` to understand page structure, text, controls, element r 1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery 2. `bsk snapshot` — strict static accessibility tree fallback 3. `bsk get-html` — when hidden DOM, metadata, or markup details are required -4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab. +4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab; use `--full-page` to capture the entire scrollable page. Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page. @@ -165,13 +165,13 @@ bsk emulate --session --off | `bsk tab return ` | Return a borrowed tab to its original window | ### Observation (require `--session` unless noted) - +### Observation (require `--session` unless noted) | Command | Summary | |---------|---------| | `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs | | `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces | | `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) | -| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) | +| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element, or `--full-page` for the entire scrollable page (`--out` path optional) | ### Navigation @@ -193,6 +193,30 @@ bsk emulate --session --off | `bsk fill --value ` | Clear and type into input | | `bsk select --value ` | Set `` option values by `value` attribute. Select(SelectArgs), + /// Drag an element, a viewport coordinate, or an absolute path. + Drag(DragArgs), + /// Evaluate a JavaScript expression inside the Agent Window. Evaluate(EvaluateArgs), diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 6d0c9f3..e7381f2 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -246,6 +246,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolFill | Method::ToolPress | Method::ToolSelect + | Method::ToolDrag | Method::ToolEvaluate | Method::ToolWaitForNavigation | Method::ToolRequestHelp diff --git a/crates/bsk-cli/src/daemon/mod.rs b/crates/bsk-cli/src/daemon/mod.rs index c639f73..9cde7e9 100644 --- a/crates/bsk-cli/src/daemon/mod.rs +++ b/crates/bsk-cli/src/daemon/mod.rs @@ -18,7 +18,7 @@ pub mod ws; pub use start::{DaemonConfig, run_foreground}; pub use state::{DaemonHandle, DaemonState}; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; @@ -37,8 +37,9 @@ pub async fn run( ipc_socket: Option, ) -> anyhow::Result { let ws_port = config.ws_port; + let ws_host = config.ws_host; let state = Arc::new(DaemonState::new(config)); - let ws_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), ws_port); + let ws_addr = SocketAddr::new(ws_host, ws_port); let ws_handle = ws::WsServer::new(Arc::clone(&state)).bind(ws_addr).await?; let ipc_handle = match ipc_socket { Some(path) => Some(ipc::IpcServer::new(Arc::clone(&state)).bind(path).await?), diff --git a/crates/bsk-cli/src/daemon/start.rs b/crates/bsk-cli/src/daemon/start.rs index 33327a1..3bec770 100644 --- a/crates/bsk-cli/src/daemon/start.rs +++ b/crates/bsk-cli/src/daemon/start.rs @@ -11,7 +11,7 @@ //! startup, redirects stdio to `/dev/null`, calls `setsid` (Unix), and //! falls through to `run_foreground`. -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::SocketAddr; use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -38,6 +38,8 @@ pub(crate) const DAEMONIZED_ENV: &str = "BSK_DAEMONIZED"; #[derive(Debug, Clone)] pub struct DaemonConfig { pub ws_port: u16, + /// IP the WebSocket server binds to (default 127.0.0.1). + pub ws_host: std::net::IpAddr, pub session_idle: Duration, pub daemon_idle: Duration, /// Skip the Origin allow-list (tests / `--insecure-origin`). @@ -60,6 +62,7 @@ impl DaemonConfig { pub fn new(port: u16) -> Self { Self { ws_port: port, + ws_host: std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST), session_idle: Duration::from_secs(60 * 5), daemon_idle: Duration::from_secs(60 * 30), allow_any_origin: false, @@ -86,8 +89,13 @@ impl DaemonConfig { impl From<&StartArgs> for DaemonConfig { fn from(args: &StartArgs) -> Self { + let ws_host: std::net::IpAddr = args + .resolved_ws_host() + .parse() + .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::LOCALHOST)); Self { ws_port: args.resolved_port(), + ws_host, session_idle: args.resolved_session_idle(), daemon_idle: args.resolved_daemon_idle(), allow_any_origin: false, @@ -220,7 +228,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> { let state = Arc::new(DaemonState::new(cfg.clone())); let session_idle_task = spawn_session_idle_reaper(Arc::clone(&state)); let browser_liveness_task = spawn_browser_liveness_reaper(Arc::clone(&state)); - let ws_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), cfg.ws_port); + let ws_addr = SocketAddr::new(cfg.ws_host, cfg.ws_port); let ws_handle = ws::WsServer::new(Arc::clone(&state)) .bind(ws_addr) .await @@ -654,6 +662,7 @@ fn apply_start_args(cmd: &mut std::process::Command, args: &StartArgs) { if let Some(p) = args.port { cmd.arg("--port").arg(p.to_string()); } + cmd.arg("--ws-host").arg(args.resolved_ws_host()); if let Some(d) = args.session_idle { cmd.arg("--session-idle").arg(format_duration(d)); } diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index 3e0c2e1..41fa477 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -97,6 +97,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::Fill(args) => cli::interaction::dispatch_fill(args, format), Command::Press(args) => cli::interaction::dispatch_press(args, format), Command::Select(args) => cli::interaction::dispatch_select(args, format), + Command::Drag(args) => cli::interaction::dispatch_drag(args, format), Command::Evaluate(args) => cli::evaluate::dispatch(args, format), Command::WaitForNavigation(args) => cli::waits::dispatch_wait_for_navigation(args, format), Command::WaitMs(args) => cli::waits::dispatch_wait_ms(args, format), diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index 75f8b4d..c530bdf 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -78,6 +78,8 @@ pub enum Method { ToolPress, #[serde(rename = "tool.select")] ToolSelect, + #[serde(rename = "tool.drag")] + ToolDrag, #[serde(rename = "tool.snapshot")] ToolSnapshot, #[serde(rename = "tool.observe")] @@ -156,6 +158,7 @@ impl Method { | Method::ToolFill | Method::ToolPress | Method::ToolSelect + | Method::ToolDrag | Method::ToolEvaluate // May navigate via optional `url` and changes Agent Window // chrome; gate behind pending-interrupt like other writes. @@ -286,6 +289,7 @@ mod tests { assert!(Method::ToolFill.is_mutating()); assert!(Method::ToolPress.is_mutating()); assert!(Method::ToolSelect.is_mutating()); + assert!(Method::ToolDrag.is_mutating()); assert!(Method::ToolEvaluate.is_mutating()); assert!(Method::ToolRecordStart.is_mutating()); assert!(Method::ToolWindowResize.is_mutating()); diff --git a/crates/bsk-protocol/src/tools/interaction.rs b/crates/bsk-protocol/src/tools/interaction.rs index 2d2e808..990cc7f 100644 --- a/crates/bsk-protocol/src/tools/interaction.rs +++ b/crates/bsk-protocol/src/tools/interaction.rs @@ -272,6 +272,94 @@ pub struct SelectResult { pub dialogs: Vec, } +// --------------------------------------------------------------------------- +// drag +// --------------------------------------------------------------------------- + +/// Drag parameters. Supports three targeting modes: +/// +/// 1. **Element drag** (`ref` / `selector` + `dx` / `dy`): press the +/// element's centre, move by the given pixel delta (CSS pixels), then +/// release. `steps` interpolates a human-like curved path (default 30). +/// 2. **Coordinate drag** (`from_x` / `from_y` + `dx` / `dy`): raw +/// viewport-coordinate drag, no element lookup. Useful for slider +/// CAPTCHAs inside cross-origin iframes where DOM access is blocked. +/// 3. **Absolute path** (`points`): explicit list of `[x, y]` viewport +/// points; press at the first, move through the rest, release at the +/// last. +/// +/// The drag is dispatched through CDP `Input.dispatchMouseEvent` +/// (`mousePressed` → `mouseMoved` × N → `mouseReleased`), so events are +/// trusted (native `isTrusted=true`), which is what slider CAPTCHA +/// services such as Aliyun `nc_1_nocaptcha` check for. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DragParams { + pub session_id: String, + /// Optional `@e` ref for element-targeted drags. Mutually + /// exclusive with `selector` / `from_x` / `points`. + #[serde( + rename = "ref", + alias = "ref_", + default, + skip_serializing_if = "Option::is_none" + )] + pub ref_: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + /// Target tab. Defaults to the Agent Window's active tab. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + /// Start X (viewport CSS px) for coordinate drags. Requires `from_y`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_x: Option, + /// Start Y (viewport CSS px) for coordinate drags. Requires `from_x`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub from_y: Option, + /// Horizontal delta in CSS px (element & coordinate modes). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dx: Option, + /// Vertical delta in CSS px (element & coordinate modes). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dy: Option, + /// Explicit viewport points `[x, y]` for absolute-path drags. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub points: Option>>, + /// Number of interpolation steps for delta drags (default 30). + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub steps: Option, + /// Per-step delay in ms (default 8). Slightly randomises timing to + /// look human. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub step_delay_ms: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub button: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modifiers: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[schemars(range(min = 1))] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct DragResult { + pub tab_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_ref: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub used_selector: Option, + /// Start point of the drag (viewport CSS px). + pub from_x: f64, + pub from_y: f64, + /// End point of the drag (viewport CSS px). + pub to_x: f64, + pub to_y: f64, + /// Number of mouseMoved steps actually dispatched. + pub steps: u32, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub dialogs: Vec, +} + #[cfg(test)] mod tests { use super::*;