-
Notifications
You must be signed in to change notification settings - Fork 10
Expand file tree
/
Copy pathtool.test.ts
More file actions
285 lines (259 loc) · 10.2 KB
/
Copy pathtool.test.ts
File metadata and controls
285 lines (259 loc) · 10.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
import { describe, it, expect } from 'vitest';
import {
ToolSchema,
defineTool,
type Tool,
} from './tool.zod';
describe('ToolSchema', () => {
it('should accept minimal tool', () => {
const tool: Tool = {
name: 'list_records',
label: 'List Records',
description: 'List records from a data object',
parameters: {
type: 'object',
properties: {
objectName: { type: 'string' },
},
required: ['objectName'],
},
};
const result = ToolSchema.parse(tool);
expect(result.name).toBe('list_records');
});
it('should accept full tool', () => {
const tool = {
name: 'create_case',
label: 'Create Support Case',
description: 'Creates a new support case record',
parameters: {
type: 'object',
properties: {
subject: { type: 'string', description: 'Case subject' },
priority: { type: 'string', enum: ['low', 'medium', 'high'] },
},
required: ['subject'],
},
outputSchema: {
type: 'object',
properties: {
id: { type: 'string' },
caseNumber: { type: 'string' },
},
},
objectName: 'support_case',
};
const result = ToolSchema.parse(tool);
expect(result.name).toBe('create_case');
expect(result.objectName).toBe('support_case');
});
it('should enforce snake_case for tool name', () => {
const validNames = ['list_records', 'create_case', '_internal_tool', 'query_orders'];
validNames.forEach(name => {
expect(() => ToolSchema.parse({
name,
label: 'Test',
description: 'Test',
parameters: {},
})).not.toThrow();
});
const invalidNames = ['listRecords', 'Create-Case', '123tool'];
invalidNames.forEach(name => {
expect(() => ToolSchema.parse({
name,
label: 'Test',
description: 'Test',
parameters: {},
})).toThrow();
});
});
it('should enforce snake_case for objectName', () => {
expect(() => ToolSchema.parse({
name: 'test_tool',
label: 'Test',
description: 'Test',
parameters: {},
objectName: 'supportCase',
})).toThrow();
expect(() => ToolSchema.parse({
name: 'test_tool',
label: 'Test',
description: 'Test',
parameters: {},
objectName: 'support_case',
})).not.toThrow();
});
});
describe('defineTool', () => {
it('should return a parsed tool', () => {
const tool = defineTool({
name: 'query_records',
label: 'Query Records',
description: 'Search and filter records',
parameters: {
type: 'object',
properties: {
objectName: { type: 'string' },
filters: { type: 'object' },
},
required: ['objectName'],
},
});
expect(tool.name).toBe('query_records');
});
it('should throw on invalid tool name', () => {
expect(() => defineTool({
name: 'InvalidName',
label: 'Test',
description: 'Test',
parameters: {},
})).toThrow();
});
// ── #3715 / ADR-0033 §2 — the retired `requiresConfirmation` safety flag ──
// Removing a key from a NON-strict schema would swap one silent no-op for
// another (zod strips it wordlessly). These pin the loud rejection AND the
// prescription it must carry — the parse error is the one channel a consumer
// bumping @objectstack/spec is guaranteed to hit.
it('REJECTS the retired `requiresConfirmation` instead of silently stripping it', () => {
const authored = {
name: 'delete_everything',
label: 'Delete Everything',
description: 'Destructive',
parameters: {},
requiresConfirmation: true,
};
expect(() => ToolSchema.parse(authored)).toThrow(/requiresConfirmation/);
expect(() => defineTool(authored as never)).toThrow();
});
it('the rejection names the REAL gate, not merely the removal', () => {
let message = '';
try {
ToolSchema.parse({
name: 't', label: 'T', description: 'd', parameters: {}, requiresConfirmation: true,
});
} catch (e) {
message = String((e as Error).message);
}
// FROM → TO: the action-level flag is the only path that stops execution.
expect(message).toMatch(/ai\.requiresConfirmation/);
expect(message).toMatch(/ADR-0033/);
});
it('rejects an unrelated unknown key too (strictness is not special-cased)', () => {
expect(() => ToolSchema.parse({
name: 't', label: 'T', description: 'd', parameters: {}, notAToolField: 1,
})).toThrow(/notAToolField/);
});
// ── #3896 audit close-out — the four inert authoring keys are retired ──
// `permissions` promised an invocation gate nothing enforced; `active: false`
// read as "withdrawn" while the tool kept reaching the LLM set; `category` /
// `builtIn` were display-only. Each rejection must carry its own
// prescription, and the two dangerous ones must name the REAL mechanism.
it.each([
['permissions', ['case.create']],
['active', false],
['category', 'action'],
['builtIn', true],
])('REJECTS the retired `%s` with its prescription', (key, value) => {
let message = '';
try {
ToolSchema.parse({
name: 't', label: 'T', description: 'd', parameters: {}, [key]: value,
});
} catch (e) {
message = String((e as Error).message);
}
expect(message).toContain(key);
expect(message).toMatch(/audit close-out/);
});
it('the `permissions` rejection points at the gate the middleware actually runs', () => {
let message = '';
try {
ToolSchema.parse({
name: 't', label: 'T', description: 'd', parameters: {}, permissions: ['x'],
});
} catch (e) {
message = String((e as Error).message);
}
expect(message).toMatch(/requiredPermissions/);
expect(message).toMatch(/ADR-0066/);
});
it('the `active` rejection says how to actually withdraw a tool', () => {
let message = '';
try {
ToolSchema.parse({
name: 't', label: 'T', description: 'd', parameters: {}, active: false,
});
} catch (e) {
message = String((e as Error).message);
}
expect(message).toMatch(/skills?\/agents|skill\/agent/i);
});
// ── #6805 — the map folded into the shared `strictObject` template ────────
// `strictToolError` was a hand-written `$ZodErrorMap`, so no registry saw
// `TOOL_RETIRED_KEY_GUIDANCE` and nothing judged it (#6416's blind spot,
// which #6619's inventory recorded as closed while this one survived). The
// acceptance surface did not move — every case above is unchanged and still
// passes — so what these pin is the assembly: the two channels the template
// brings that the hand-rolled map did not have.
const rejectionMessage = (extra: Record<string, unknown>): string => {
const result = ToolSchema.safeParse({
name: 't', label: 'T', description: 'd', parameters: {}, ...extra,
});
expect(result.success).toBe(false);
return result.success ? '' : result.error.issues.map((i) => i.message).join('\n');
};
it.each([
['labl', 'label'],
['paramaters', 'parameters'],
['objectname', 'objectName'],
['outputSchem', 'outputSchema'],
])('a near-miss `%s` now gets the rename channel, not just "not a ToolSchema field"', (written, canonical) => {
// The hand-written map's fallback line named the problem and never the
// fix: every unprescribed key got "`x` is not a ToolSchema field." and
// stopped there. The template's edit-distance channel answers instead.
// This is the one reader-visible gain of the fold, so it is pinned rather
// than left as a claim in the PR body.
const message = rejectionMessage({ [written]: 'x' });
expect(message).toContain(`\`${written}\` → \`${canonical}\``);
expect(message).not.toContain('is not a ToolSchema field');
});
it('a key beyond edit distance is still named, with no misleading suggestion', () => {
// The other half of the same contract: the rename channel must stay quiet
// when it has nothing true to say. Suggesting SOMETHING for `notAToolField`
// would be ledger finding 7 — steering an author at a key they did not
// want — which is worse than the removed fallback line, not better.
const message = rejectionMessage({ notAToolField: 1 });
expect(message).toContain('`notAToolField`');
expect(message).not.toContain('Did you mean');
});
it('emission order: which key is wrong → the fix → the history, last (#5955)', () => {
// The template's ordering contract, asserted on this surface because the
// fold is what brings this surface under it. `history` sat in the middle
// until #5955 and pushed the fix past ~character 220 on the single-line
// renderers several consumers use.
const message = rejectionMessage({ requiresConfirmation: true, labl: 'x' });
const preamble = 'Unrecognized key(s) on the tool definition:';
const fix = 'action.ai.requiresConfirmation';
const history = 'the silent-strip class';
expect(message.startsWith(preamble)).toBe(true);
expect(message.indexOf(fix)).toBeGreaterThan(message.indexOf(preamble));
expect(message.indexOf(history)).toBeGreaterThan(message.indexOf(fix));
expect(message.trimEnd().endsWith(`${history}).`)).toBe(true);
// One history sentence per message, however many keys were written.
expect(message.split(history)).toHaveLength(2);
});
it('the prescriptions survive the fold byte-for-byte — the fold moved assembly, not text', () => {
// The whole value of `TOOL_RETIRED_KEY_GUIDANCE` is the text. A fold that
// quietly reworded a retirement prescription would be the defect this
// table exists to prevent, wearing the fold as cover.
expect(rejectionMessage({ permissions: ['x'] })).toContain(
'`tool.permissions` was removed in @objectstack/spec 17.0.0 (audit close-out) — it '
+ 'promised a capability gate on tool invocation that nothing ever enforced',
);
expect(rejectionMessage({ builtIn: true })).toContain(
'`tool.builtIn` was removed in @objectstack/spec 17.0.0 (audit close-out) — no '
+ 'runtime branches on it; it never affected registration, selection or execution. Delete '
+ 'the key.',
);
});
});