Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 13 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,20 @@
"description": "TypeScript SDK for Sentience AI Agent Browser Automation",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"exports": {
".": {
"types": "./dist/index.d.ts",
"default": "./dist/index.js"
},
"./browser-agent": {
"types": "./dist/browser-agent.d.ts",
"import": "./dist/esm/browser-agent.js",
"require": "./dist/browser-agent.js",
"default": "./dist/browser-agent.js"
}
},
"scripts": {
"build": "tsc",
"build": "tsc && tsc -p tsconfig.esm.json",
"test": "jest",
"lint": "eslint src --ext .ts",
"lint:fix": "eslint src --ext .ts --fix",
Expand Down
3 changes: 2 additions & 1 deletion src/agents/planner-executor/boundary-detection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,8 @@ export function isSearchLikeTypeAndSubmit(
step: { action?: string; intent?: string; input?: string; verify?: PredicateSpec[] },
element?: Pick<SnapshotElement, 'role' | 'text' | 'name' | 'ariaLabel'> | null
): boolean {
if ((step.action || '').toUpperCase() !== 'TYPE_AND_SUBMIT') {
const action = (step.action || '').toUpperCase();
if (action !== 'TYPE_AND_SUBMIT' && action !== 'TYPE') {
return false;
}

Expand Down
11 changes: 11 additions & 0 deletions src/agents/planner-executor/common-hints.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ export const COMMON_HINTS = {
roleFilter: ['button', 'link'],
priority: 10,
}),
product_card: new HeuristicHint({
intentPattern: 'product_card',
roleFilter: ['link'],
priority: 8,
attributePatterns: { href: '/dp/' },
}),
login: new HeuristicHint({
intentPattern: 'login',
textPatterns: ['log in', 'login', 'sign in', 'signin'],
Expand All @@ -31,6 +37,11 @@ export const COMMON_HINTS = {
roleFilter: ['button', 'textbox', 'searchbox', 'combobox'],
priority: 5,
}),
searchbox: new HeuristicHint({
intentPattern: 'searchbox',
roleFilter: ['searchbox'],
priority: 9,
}),
close: new HeuristicHint({
intentPattern: 'close',
textPatterns: ['close', 'dismiss', 'x', 'cancel'],
Expand Down
31 changes: 26 additions & 5 deletions src/agents/planner-executor/planner-executor-agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1434,12 +1434,16 @@ export class PlannerExecutorAgent {
const text = plannerAction.input || (parsed.args[1] as string) || '';
await runtime.type(elementId, text);

// Submit with Enter key for TYPE_AND_SUBMIT
if (plannerAction.action === 'TYPE_AND_SUBMIT') {
const elements = activeCtx.snapshot?.elements || [];
const inputElement = elements.find(element => element.id === elementId) || null;
const isSearchLike = isSearchLikeTypeAndSubmit(plannerAction, inputElement);

// Submit with Enter key for TYPE_AND_SUBMIT, plus planner TYPE actions that clearly target search.
if (
plannerAction.action === 'TYPE_AND_SUBMIT' ||
(plannerAction.action === 'TYPE' && isSearchLike)
) {
const preUrl = await runtime.getCurrentUrl();
const elements = activeCtx.snapshot?.elements || [];
const inputElement = elements.find(element => element.id === elementId) || null;
const isSearchLike = isSearchLikeTypeAndSubmit(plannerAction, inputElement);
const submitButtonId = this.findSubmitButton(elements, elementId, isSearchLike);
const hasRetryBudget = this.config.retry.executorRepairAttempts > 0;

Expand Down Expand Up @@ -1913,6 +1917,23 @@ export class PlannerExecutorAgent {
return { action: 'CLICK', args: [elementId] };
}

const matchedElement = ctx.snapshot.elements.find(element => element.id === elementId) || null;
if (
plannerAction.action === 'TYPE_AND_SUBMIT' &&
plannerAction.input &&
isSearchLikeTypeAndSubmit(plannerAction, matchedElement)
) {
return { action: 'TYPE', args: [elementId, plannerAction.input] };
}

if (
plannerAction.action === 'TYPE' &&
plannerAction.input &&
isSearchLikeTypeAndSubmit(plannerAction, matchedElement)
) {
return { action: 'TYPE', args: [elementId, plannerAction.input] };
}

return null;
}

Expand Down
44 changes: 41 additions & 3 deletions src/agents/planner-executor/predicates.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
*
* Supported predicates:
* - url_contains: Check if URL contains a substring
* - url_equals: Check if URL equals a target URL
* - url_matches: Check if URL matches a regex pattern
* - exists: Check if element with text/selector exists
* - not_exists: Check if element does not exist
Expand Down Expand Up @@ -38,11 +39,15 @@ export interface Predicate {
* Check if URL contains a substring.
*/
export function urlContains(substring: string): Predicate {
const needle = substring.trim();
return {
name: 'url_contains',
evaluate(snapshot: Snapshot): boolean {
if (!needle) {
return false;
}
const url = snapshot.url || '';
return url.toLowerCase().includes(substring.toLowerCase());
return url.toLowerCase().includes(needle.toLowerCase());
},
};
}
Expand All @@ -66,6 +71,36 @@ export function urlMatches(pattern: string): Predicate {
};
}

/**
* Check if URL equals a target URL, ignoring trailing slash differences.
*/
export function urlEquals(targetUrl: string): Predicate {
const target = normalizeUrlForEquality(targetUrl);
return {
name: 'url_equals',
evaluate(snapshot: Snapshot): boolean {
if (!target) {
return false;
}
return normalizeUrlForEquality(snapshot.url || '') === target;
},
};
}

function normalizeUrlForEquality(url: string): string {
const trimmed = url.trim();
if (!trimmed) {
return '';
}
try {
const parsed = new URL(trimmed);
parsed.hash = '';
return parsed.toString().replace(/\/$/, '').toLowerCase();
} catch {
return trimmed.replace(/\/$/, '').toLowerCase();
}
}

// ---------------------------------------------------------------------------
// Element Predicates
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -197,6 +232,9 @@ export function buildPredicate(spec: PredicateSpec): Predicate {
case 'url_contains':
return urlContains(String(args[0] || ''));

case 'url_equals':
return urlEquals(String(args[0] || ''));

case 'url_matches':
return urlMatches(String(args[0] || ''));

Expand All @@ -220,11 +258,11 @@ export function buildPredicate(spec: PredicateSpec): Predicate {
return allOf(...(args as PredicateSpec[]).map(buildPredicate));

default:
// Unknown predicate - always passes (lenient)
// Unknown predicates must fail closed so pre-step verification cannot skip real work.
return {
name: `unknown:${name}`,
evaluate(): boolean {
return true;
return false;
},
};
}
Expand Down
91 changes: 91 additions & 0 deletions src/browser-agent.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/**
* Browser-safe planner-executor entrypoint.
*
* This subpath intentionally excludes Playwright, Node filesystem tracing sinks,
* and the package root's browser automation helpers so Chrome extensions can
* bundle the planner-executor core behind their own AgentRuntime adapter.
*/

export { LLMProvider, type LLMResponse } from './llm-provider';

export {
type SnapshotEscalationConfig,
type RetryConfig,
type StepwisePlanningConfig,
type PlannerExecutorConfig,
type DeepPartial,
ConfigPreset,
getConfigPreset,
mergeConfig,
DEFAULT_CONFIG,
} from './agents/planner-executor/config';

export {
PredicateSpecSchema,
PlanStepSchema,
PlanSchema,
ReplanPatchSchema,
ActionType,
StepStatus,
type PredicateSpec,
type PlanStep,
type Plan,
type ReplanPatch,
type ActionRecord,
type StepOutcome,
type RunOutcome,
type TokenUsageTotals,
type TokenUsageSummary,
type SnapshotContext,
type ParsedAction,
type Snapshot,
type SnapshotElement,
} from './agents/planner-executor/plan-models';

export {
buildStepwisePlannerPrompt,
buildExecutorPrompt,
type StepwisePlannerResponse,
} from './agents/planner-executor/prompts';

export {
parseAction,
extractJson,
normalizePlan,
validatePlanSmoothness,
formatContext,
} from './agents/planner-executor/plan-utils';

export { TaskCategory, normalizeTaskCategory } from './agents/planner-executor/task-category';
export {
AutomationTaskSchema,
type AutomationTask,
} from './agents/planner-executor/automation-task';
export { HeuristicHint, type HeuristicHintInput } from './agents/planner-executor/heuristic-hint';
export { COMMON_HINTS, getCommonHint } from './agents/planner-executor/common-hints';
export {
ComposableHeuristics,
type ComposableHeuristicsOptions,
} from './agents/planner-executor/composable-heuristics';

export {
type Predicate,
urlContains,
urlMatches,
exists,
notExists,
elementCount,
anyOf,
allOf,
buildPredicate,
evaluatePredicates,
} from './agents/planner-executor/predicates';

export {
PlannerExecutorAgent,
type PlannerExecutorAgentOptions,
type PreActionAuthorizer,
type AuthorizationResult,
type AgentRuntime,
type IntentHeuristics,
} from './agents/planner-executor/planner-executor-agent';
50 changes: 50 additions & 0 deletions tests/agents/planner-executor/composable-heuristics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,56 @@ describe('ComposableHeuristics', () => {
).toBe(21);
});

it('matches a searchbox role even when the visible text is an implementation name', () => {
const heuristics = new ComposableHeuristics({
staticHeuristics: new StaticHeuristicsStub(null),
taskCategory: TaskCategory.SEARCH,
});

const elements = [
makeElement(31, { text: 'field-keywords', role: 'searchbox' }),
makeElement(32, { text: 'Sign in', role: 'button' }),
];

expect(
heuristics.findElementForIntent(
'searchbox',
elements,
'https://www.amazon.com/',
'Search for noise canceling earbuds'
)
).toBe(31);
});

it('matches Amazon product card intents to real product detail links', () => {
const heuristics = new ComposableHeuristics({
staticHeuristics: new StaticHeuristicsStub(null),
taskCategory: TaskCategory.SEARCH,
});

const elements = [
makeElement(41, {
text: 'Sponsored banner',
role: 'link',
href: '/gp/help/customer/display.html',
}),
makeElement(42, {
text: 'Wireless Noise Canceling Earbuds with Charging Case',
role: 'link',
href: '/dp/B0TEST1234/ref=sr_1_1',
}),
];

expect(
heuristics.findElementForIntent(
'product_card',
elements,
'https://www.amazon.com/s?k=noise+canceling+earbuds',
'Pick an earbuds product'
)
).toBe(42);
});

it('falls back to static heuristics before task-category defaults', () => {
const heuristics = new ComposableHeuristics({
staticHeuristics: new StaticHeuristicsStub(42, ['custom_fallback']),
Expand Down
17 changes: 16 additions & 1 deletion tests/agents/planner-executor/predicates.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,21 @@ describe('Predicates', () => {
expect(pred.evaluate(snapshot)).toBe(true);
});

it('should fail empty url_contains predicates instead of matching every URL', () => {
const pred = buildPredicate({ predicate: 'url_contains', args: [''] });

expect(pred.name).toBe('url_contains');
expect(pred.evaluate(createSnapshot('https://example.com/'))).toBe(false);
});

it('should build url_equals predicate without pre-verifying a different URL', () => {
const pred = buildPredicate({ predicate: 'url_equals', args: ['https://www.amazon.com/'] });

expect(pred.name).toBe('url_equals');
expect(pred.evaluate(createSnapshot('https://example.com/'))).toBe(false);
expect(pred.evaluate(createSnapshot('https://www.amazon.com/'))).toBe(true);
});

it('should build url_matches predicate', () => {
const pred = buildPredicate({ predicate: 'url_matches', args: ['/dp/.*'] });
const snapshot = createSnapshot('https://example.com/dp/123');
Expand Down Expand Up @@ -309,7 +324,7 @@ describe('Predicates', () => {
const pred = buildPredicate({ predicate: 'unknown_predicate', args: ['foo'] });

expect(pred.name).toBe('unknown:unknown_predicate');
expect(pred.evaluate(createSnapshot(''))).toBe(true); // Always passes
expect(pred.evaluate(createSnapshot(''))).toBe(false);
});
});

Expand Down
Loading
Loading