Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion apps/extension/src/entrypoints/background.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,18 @@ export default defineBackground(() => {
generation: overlayGeneration,
};
}
// Automation-only deployments (BSK_NO_OVERLAY=1 at build time) never
// show the on-page control mask: the orange stop button is useless
// in a headless browser and its capture layer swallows CDP mouse
// events (breaking drag automation).
const mode: OverlayMode =
typeof __BSK_NO_OVERLAY__ !== "undefined" && __BSK_NO_OVERLAY__
? "hidden"
: (controlModes.get(ctx.sessionId) ?? "control");
return {
type: OVERLAY_AGENT_STATE,
sessionId: ctx.sessionId,
mode: controlModes.get(ctx.sessionId) ?? "control",
mode,
generation: overlayGeneration,
};
}
Expand Down
45 changes: 45 additions & 0 deletions apps/extension/src/tools/__tests__/interaction.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import {
buildHumanDragPath,
handleClick,
handleFill,
handleHover,
Expand Down Expand Up @@ -1027,3 +1028,47 @@ describe("handleSelect", () => {
expect(fake.sent.some((c) => c.method === "Runtime.callFunctionOn")).toBe(false);
});
});

describe("buildHumanDragPath", () => {
it("starts at the origin and ends at the destination", () => {
const path = buildHumanDragPath(100, 100, 200, 100, 30);
expect(path.length).toBe(30);
expect(path[0].x).toBeGreaterThanOrEqual(99);
expect(path[0].x).toBeLessThanOrEqual(101);
expect(path[0].y).toBeGreaterThanOrEqual(99);
expect(path[0].y).toBeLessThanOrEqual(101);
expect(path[29].x).toBeGreaterThanOrEqual(199);
expect(path[29].x).toBeLessThanOrEqual(201);
expect(path[29].y).toBeGreaterThanOrEqual(99);
expect(path[29].y).toBeLessThanOrEqual(101);
});

it("is not a perfect straight line: has bounded perpendicular wobble", () => {
// 200 runs; every intermediate y must deviate from the exact
// line y=100 by a few px, and at least one run must wobble > 0.5px.
let maxWobble = 0;
for (let run = 0; run < 200; run++) {
const path = buildHumanDragPath(100, 100, 300, 100, 40);
for (let i = 1; i < path.length - 1; i++) {
const dev = Math.abs(path[i].y - 100);
expect(dev).toBeLessThanOrEqual(3.01); // ±3px ceiling
if (dev > maxWobble) maxWobble = dev;
}
}
expect(maxWobble).toBeGreaterThan(0.5);
});

it("applies perpendicular wobble to vertical drags too", () => {
// Vertical drag from (100,100) to (100,300): the wobble axis is X.
let sawDeviation = false;
for (let run = 0; run < 100; run++) {
const path = buildHumanDragPath(100, 100, 100, 300, 30);
for (let i = 1; i < path.length - 1; i++) {
const dev = Math.abs(path[i].x - 100);
expect(dev).toBeLessThanOrEqual(3.01);
if (dev > 0.5) sawDeviation = true;
}
}
expect(sawDeviation).toBe(true);
});
});
12 changes: 10 additions & 2 deletions apps/extension/src/tools/dispatcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type {
ClickParams,
ConsoleParams,
EmulateParams,
DragParams,
EvaluateParams,
FillParams,
GetHtmlParams,
Expand Down Expand Up @@ -35,7 +36,7 @@ import { handleConsole } from "./console";
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
import { handleClick, handleDrag, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
import {
handleNavigate,
handleNavigateBack,
Expand Down Expand Up @@ -461,6 +462,12 @@ export class ToolDispatcher {
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
);
case "tool.drag":
return handleDrag(
this.sessions,
req.params as DragParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
);
case "tool.evaluate":
return handleEvaluate(
this.sessions,
Expand Down Expand Up @@ -528,7 +535,7 @@ export class ToolDispatcher {
default:
return {
code: "unknown_method",
message: `${req.method} not implemented in extension`,
message: `${req.method} not implemented in extension [v2-DRAG]`,
} satisfies RpcError;
}
}
Expand Down Expand Up @@ -670,6 +677,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.fill":
case "tool.press":
case "tool.select":
case "tool.drag":
case "tool.evaluate":
case "tool.observe":
case "tool.request_help":
Expand Down
Loading