Skip to content

Commit ea85e0a

Browse files
committed
test(desktop): keep the SQL and TypeScript desktop-call classifiers in agreement
The inbox and the overdue sweep filter calls with a SQL form of isDesktopToolCall, and nothing tied the two together: a desktop tool added on one side only would silently drop out of the inbox and the sweep, or let Sim-only calls crowd them. One table of calls now runs through both against real PostgreSQL, covering every named desktop tool, read and grep under, exactly at and outside user-local, glob patterns, non-string and missing arguments, a user-localX lookalike and non-desktop tools, and the test requires them to agree on every case.
1 parent c8ef18d commit ea85e0a

1 file changed

Lines changed: 177 additions & 0 deletions

File tree

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
/**
2+
* The inbox and the overdue sweep filter calls with a SQL form of `isDesktopToolCall`, so their
3+
* limits apply only to calls the desktop runs. Nothing else ties the two together: a desktop tool
4+
* added on one side only would silently drop out of the inbox and the sweep, or let Sim-only calls
5+
* crowd them. This runs one table of calls through both against real PostgreSQL and requires them
6+
* to agree on every case.
7+
*/
8+
import { db } from '@sim/db'
9+
import {
10+
copilotAsyncToolCalls,
11+
copilotChats,
12+
copilotRuns,
13+
desktopDevices,
14+
user,
15+
workspace,
16+
} from '@sim/db/schema'
17+
import { generateId } from '@sim/utils/id'
18+
import { eq, sql } from 'drizzle-orm'
19+
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
20+
import { listDesktopInboxRows } from '@/lib/desktop/executor/repository'
21+
import { SIM_TOOL_EXECUTION_VERSION } from '@/lib/mothership/async-runs/lifecycle'
22+
import { isDesktopToolCall, NAMED_DESKTOP_TOOL_NAMES } from '@/lib/mothership/tools/desktop-tools'
23+
24+
interface ClassifierCase {
25+
label: string
26+
toolName: string
27+
args: Record<string, unknown>
28+
}
29+
30+
const CASES: ClassifierCase[] = [
31+
...NAMED_DESKTOP_TOOL_NAMES.map((toolName) => ({
32+
label: `named desktop tool ${toolName}`,
33+
toolName,
34+
args: {},
35+
})),
36+
...(['read', 'grep'] as const).flatMap((toolName) => [
37+
{ label: `${toolName} under user-local/`, toolName, args: { path: 'user-local/Project/a.md' } },
38+
{ label: `${toolName} of user-local itself`, toolName, args: { path: 'user-local' } },
39+
{
40+
label: `${toolName} of user-local/ with a trailing slash`,
41+
toolName,
42+
args: { path: 'user-local/' },
43+
},
44+
{
45+
label: `${toolName} of a sibling user-localX/`,
46+
toolName,
47+
args: { path: 'user-localX/a.md' },
48+
},
49+
{ label: `${toolName} of a LIKE-wildcard lookalike`, toolName, args: { path: 'user-local%' } },
50+
{ label: `${toolName} of Sim's own files`, toolName, args: { path: 'workspace/notes.md' } },
51+
{ label: `${toolName} with a numeric path`, toolName, args: { path: 5 } },
52+
{ label: `${toolName} with an array path`, toolName, args: { path: ['user-local/a.md'] } },
53+
{
54+
label: `${toolName} with an object path`,
55+
toolName,
56+
args: { path: { value: 'user-local/a' } },
57+
},
58+
{ label: `${toolName} with a null path`, toolName, args: { path: null } },
59+
{ label: `${toolName} with no path`, toolName, args: {} },
60+
{
61+
label: `${toolName} with the local path under pattern instead`,
62+
toolName,
63+
args: { pattern: 'user-local/**' },
64+
},
65+
]),
66+
{ label: 'glob under user-local/', toolName: 'glob', args: { pattern: 'user-local/**/*.md' } },
67+
{ label: 'glob of user-local itself', toolName: 'glob', args: { pattern: 'user-local' } },
68+
{
69+
label: 'glob of a sibling user-localX/',
70+
toolName: 'glob',
71+
args: { pattern: 'user-localX/**' },
72+
},
73+
{ label: "glob of Sim's own files", toolName: 'glob', args: { pattern: '**/*.md' } },
74+
{ label: 'glob with a numeric pattern', toolName: 'glob', args: { pattern: 7 } },
75+
{ label: 'glob with no pattern', toolName: 'glob', args: {} },
76+
{
77+
label: 'glob with the local path under path instead',
78+
toolName: 'glob',
79+
args: { path: 'user-local/Project' },
80+
},
81+
{ label: 'a non-desktop tool', toolName: 'run_workflow', args: {} },
82+
{
83+
label: 'a non-desktop tool with a local-looking path',
84+
toolName: 'edit_file',
85+
args: { path: 'user-local/a.md' },
86+
},
87+
]
88+
89+
describe('desktop call classification in SQL and TypeScript', () => {
90+
const userId = generateId()
91+
const workspaceId = generateId()
92+
const deviceId = generateId()
93+
const chatId = generateId()
94+
const runId = generateId()
95+
/** Each case's call id, to read back which ones the SQL predicate admitted. */
96+
const callIds = new Map<string, ClassifierCase>()
97+
98+
beforeAll(async () => {
99+
const now = new Date()
100+
await db.insert(user).values({
101+
id: userId,
102+
name: 'Classifier parity fixture',
103+
email: `${userId}@desktop-classifier.test`,
104+
emailVerified: true,
105+
createdAt: now,
106+
updatedAt: now,
107+
})
108+
await db.insert(workspace).values({
109+
id: workspaceId,
110+
name: 'Classifier parity fixture',
111+
ownerId: userId,
112+
billedAccountUserId: userId,
113+
})
114+
await db.insert(desktopDevices).values({
115+
id: deviceId,
116+
userId,
117+
name: 'Fixture Mac',
118+
appVersion: '0.9.0',
119+
platform: 'darwin-arm64',
120+
capabilities: { executor: 1 },
121+
})
122+
await db.insert(copilotChats).values({
123+
id: chatId,
124+
userId,
125+
workspaceId,
126+
type: 'mothership',
127+
conversationId: generateId(),
128+
})
129+
await db.insert(copilotRuns).values({
130+
id: runId,
131+
executionId: generateId(),
132+
chatId,
133+
userId,
134+
workspaceId,
135+
streamId: generateId(),
136+
toolExecutionVersion: SIM_TOOL_EXECUTION_VERSION,
137+
status: 'active',
138+
desktopDeviceId: deviceId,
139+
})
140+
for (const testCase of CASES) {
141+
const toolCallId = generateId()
142+
callIds.set(toolCallId, testCase)
143+
/** An offered, unclaimed call: the inbox lists it exactly when the SQL predicate admits it. */
144+
await db.insert(copilotAsyncToolCalls).values({
145+
runId,
146+
toolCallId,
147+
toolName: testCase.toolName,
148+
args: testCase.args,
149+
pickupDeadlineAt: sql`now() + interval '1 minute'`,
150+
})
151+
}
152+
})
153+
154+
afterAll(async () => {
155+
await db.delete(copilotChats).where(eq(copilotChats.id, chatId))
156+
await db.delete(desktopDevices).where(eq(desktopDevices.id, deviceId))
157+
await db.delete(workspace).where(eq(workspace.id, workspaceId))
158+
await db.delete(user).where(eq(user.id, userId))
159+
})
160+
161+
it('agrees on every case, including the ones the desktop runs', async () => {
162+
const admittedBySql = new Set(
163+
(await listDesktopInboxRows({ deviceId, userId })).map((row) => row.toolCallId)
164+
)
165+
const verdicts = [...callIds].map(([toolCallId, testCase]) => ({
166+
case: testCase.label,
167+
sql: admittedBySql.has(toolCallId),
168+
typescript: isDesktopToolCall(testCase.toolName, testCase.args),
169+
}))
170+
171+
expect(verdicts.filter((verdict) => verdict.sql !== verdict.typescript)).toEqual([])
172+
/** Both sides must also admit something, so agreement on "nothing" cannot pass. */
173+
expect(verdicts.filter((verdict) => verdict.typescript).length).toBeGreaterThan(
174+
NAMED_DESKTOP_TOOL_NAMES.length
175+
)
176+
})
177+
})

0 commit comments

Comments
 (0)