diff --git a/.changeset/eve-any-version-compat.md b/.changeset/eve-any-version-compat.md new file mode 100644 index 000000000..8bd1d86fe --- /dev/null +++ b/.changeset/eve-any-version-compat.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(eve): Use `any` as return type to ensure eve version compat diff --git a/e2e/scenarios/eve-instrumentation/config/typecheck.ts b/e2e/scenarios/eve-instrumentation/config/typecheck.ts new file mode 100644 index 000000000..006c00edf --- /dev/null +++ b/e2e/scenarios/eve-instrumentation/config/typecheck.ts @@ -0,0 +1,8 @@ +import { braintrustEveHook, braintrustEveInstrumentation } from "braintrust"; +import { defineState } from "eve/context"; +import { defineHook } from "eve/hooks"; +import { defineInstrumentation } from "eve/instrumentation"; + +defineHook(braintrustEveHook({ defineState })); +defineInstrumentation(braintrustEveInstrumentation({ defineState })); +defineInstrumentation(braintrustEveInstrumentation({})); diff --git a/e2e/scenarios/eve-instrumentation/package.json b/e2e/scenarios/eve-instrumentation/package.json index bc9cb1c09..e675bc8aa 100644 --- a/e2e/scenarios/eve-instrumentation/package.json +++ b/e2e/scenarios/eve-instrumentation/package.json @@ -15,6 +15,7 @@ "dependencies": { "@openrouter/ai-sdk-provider": "3.0.0", "ai": "7.0.82", + "eve": "0.52.2", "eve-v0": "npm:eve@0.22.1", "eve-v0-latest-pinned": "npm:eve@0.37.1", "eve-v0-provider": "npm:eve@0.34.0", diff --git a/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml b/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml index e102af294..5f1a3914f 100644 --- a/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml +++ b/e2e/scenarios/eve-instrumentation/pnpm-lock.yaml @@ -14,6 +14,9 @@ importers: ai: specifier: 7.0.82 version: 7.0.82(zod@4.3.6) + eve: + specifier: 0.52.2 + version: 0.52.2(ai@7.0.82(zod@4.3.6)) eve-v0: specifier: npm:eve@0.22.1 version: eve@0.22.1(ai@7.0.82(zod@4.3.6)) @@ -73,36 +76,75 @@ packages: '@oxc-project/types@0.138.0': resolution: {integrity: sha512-1a7ZKmrRTCoN1XMZ4L0PyyqrMnrNlLyPuOkdSX2MZg7IiIGRUyurNhAm73ptDOraoBcIordsIGKNPKUzy3ZmfA==} + '@oxc-project/types@0.149.0': + resolution: {integrity: sha512-Efcc+iF0j3Bf67YjEqIqWXbX5XddXoK/Mw4K1/JuXwRCZ8N16VR7iT23nlCc9XrveFVh/E5Rqs2StT0V8v9LdA==} + + '@rolldown/binding-android-arm-eabi@1.2.8': + resolution: {integrity: sha512-tN5aztYkKCte4i5SIrrz5yK/HMjEuCqCSCJa418jOV8tZ1cBY3YF2otxB1ktPxzsLA1BeTqwapK0bfjxNvHJVw==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [android] + '@rolldown/binding-android-arm64@1.1.4': resolution: {integrity: sha512-EZLpf/8y7GXkkra90ML47kzik/GMP3EMcE9bPyHmRfxLC6z9+aW5A8poCsoxjrT5GfEcNAAvWwUHjvP1pUQkfw==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [android] + '@rolldown/binding-android-arm64@1.2.8': + resolution: {integrity: sha512-dIYTWl9XprMUiQFoc55KUyk/oS8SKYH3zFl0LTR7RT0Xj4hgSVyuJcroH8JUu8RcpF8fTB6E0aOwCkZoYPcDSQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [android] + '@rolldown/binding-darwin-arm64@1.1.4': resolution: {integrity: sha512-aUi+HBvmYb7j8krl1+qJgkG8C17fO79gk3c+jPw4S8glRFc1DTija9S3EyaTSQUm5GJXYKDAsugBEhFHH2vYiQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [darwin] + '@rolldown/binding-darwin-arm64@1.2.8': + resolution: {integrity: sha512-PCSDQGXD2IyTEFrcgPyBM8jJuGmrbCMuoIOXdbEGVemruKACXoLQJrb+A45Z0L5t1RQkdfJprAYPkikbh7dzdA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [darwin] + '@rolldown/binding-darwin-x64@1.1.4': resolution: {integrity: sha512-F7hHC3gwY11+vByKPRWqwGbeXWVgKmL+pTGCinaEhdihzBV2aQ0fvZOch9cXYUOKuKKq429HeYXOqQLc7wFCEg==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [darwin] + '@rolldown/binding-darwin-x64@1.2.8': + resolution: {integrity: sha512-Uk7lRsGhPFHVX/sAUC6D5H9Ol30dFHd6iquokll2th3LpdJ3F5CzQB+7DHn0Ri2mG+U7k2zXiPHDrwZenXhwSA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [darwin] + '@rolldown/binding-freebsd-x64@1.1.4': resolution: {integrity: sha512-sI5yw+7s92SK6odiEhD5lKCBlWcpjHS5qyqpVQbZAJ0fIzEUXrmbl3DH2ybR3PZogulNJF+COLtmA8hUfvkCCQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [freebsd] + '@rolldown/binding-freebsd-x64@1.2.8': + resolution: {integrity: sha512-DjszaTEVogPqA5bYzsEeqDCQxbcp2fexQwKcRspYji2yzR68fCf+e4fx6kBSRDwX5/brZaHw/hWS9+A/+/w9sQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [freebsd] + '@rolldown/binding-linux-arm-gnueabihf@1.1.4': resolution: {integrity: sha512-mCi0OKgEieFircrtVYmQAFGszRtMnZ6fpZAXrxanXAu7lqZcsK1E1RAaZNG0uKAnxox3B1f4EyQNnoyMfN1vAA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm] os: [linux] + '@rolldown/binding-linux-arm-gnueabihf@1.2.8': + resolution: {integrity: sha512-zmwa7FTmdzB6aaEEuuls18H6Ap5JmJPSoPTuXixeJZV6tG40SyLkApQtz1g8ptZtiEKqj9OM0oNLPh1AgvE31Q==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm] + os: [linux] + '@rolldown/binding-linux-arm64-gnu@1.1.4': resolution: {integrity: sha512-B9Ial3Kv5sh0SHnB1g/QWcUQCEvCF6QKGAl4zXypYj65mVI+B4AhFBwPtSN7pDrJeIx8Z7zdy4ntx+wQABom7w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -110,6 +152,13 @@ packages: os: [linux] libc: [glibc] + '@rolldown/binding-linux-arm64-gnu@1.2.8': + resolution: {integrity: sha512-KdYQDPHwJVnbFwdTGMgxsI9SqblBlz6STGM+w1We/d5B8OWWidYH0MwkU/uA1wM5fIpO2MkOVxXrNzzuZhw9ew==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [glibc] + '@rolldown/binding-linux-arm64-musl@1.1.4': resolution: {integrity: sha512-lZVym0PuHE1KZ22gmFTC15lAkrg9iTszR617oYRB/iPY1A56ywoJzVKOJBKaot5RiikCObmur6pogpse3gRcng==} engines: {node: ^20.19.0 || >=22.12.0} @@ -117,6 +166,13 @@ packages: os: [linux] libc: [musl] + '@rolldown/binding-linux-arm64-musl@1.2.8': + resolution: {integrity: sha512-jFJTifHnNPY+yzOoNZQfSIysrVyXzEQPhPnOUjmD1bcQGHH6s7c8cViKWar8YplQImE5N9JRqMCLrM2CdxOrZA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [linux] + libc: [musl] + '@rolldown/binding-linux-ppc64-gnu@1.1.4': resolution: {integrity: sha512-t2DNiLJWNTbnEHyUzTumldML6ET4/g16467LZoDDJ3tSxGvguL5/NyC2lCsNKuyRycg9XeDQF5SSv+TNOhQEXg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -124,6 +180,13 @@ packages: os: [linux] libc: [glibc] + '@rolldown/binding-linux-ppc64-gnu@1.2.8': + resolution: {integrity: sha512-FhiOziBDWPBjbcmRzfLyIJnaP7AVMFXT7YCXPjXxj7wKU3vx24RjrCNN/zjvVa+N2vVoHJwCoUBvsrN/DG3zIA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [ppc64] + os: [linux] + libc: [glibc] + '@rolldown/binding-linux-s390x-gnu@1.1.4': resolution: {integrity: sha512-0WIRnL1Uw4BvTZRLQt+PVgo6ZKTJadlC2btP+/EOXv2f/DWbY0rEgl+y834mIVwP1FkTlWVTrGGJXf12lru7EQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -131,6 +194,13 @@ packages: os: [linux] libc: [glibc] + '@rolldown/binding-linux-s390x-gnu@1.2.8': + resolution: {integrity: sha512-WnHfADMzOV2Y55wlx1hzzQnar/wDt/VdvWSD99r18Mz9ylNieIGOkRx3UV21h7m/eJvjySYJkO26VvGNFkwsIQ==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [s390x] + os: [linux] + libc: [glibc] + '@rolldown/binding-linux-x64-gnu@1.1.4': resolution: {integrity: sha512-JWtGshGfX+oENAKonoNkqEJX+7hC8yfhi9GUyPX1VX4mdh1y5r+ZiJLR5XzAB0aoP6s/PcILsGjKq8O0mm24bw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -138,6 +208,13 @@ packages: os: [linux] libc: [glibc] + '@rolldown/binding-linux-x64-gnu@1.2.8': + resolution: {integrity: sha512-H9tRr5ibfXFVLxbPOseVewewFpl28zcEdjRDt2FTUZU7odxP0gEv1ki4/kGmcGOh78oRwZuuQllGLZ9zTJp84g==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [glibc] + '@rolldown/binding-linux-x64-musl@1.1.4': resolution: {integrity: sha512-rT6yQcxUuXs4CnbofqwHRRV0iem349rLMYpTjkgQGLjrY4ado/eDzwPZPTCgTOlF6Nkp8NEv70yLMTn6qkWxsQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -145,12 +222,25 @@ packages: os: [linux] libc: [musl] + '@rolldown/binding-linux-x64-musl@1.2.8': + resolution: {integrity: sha512-UefiqfM3D6IVNlZ8tSGs9+Ejjud2T+oxO0IHADU45Y+lyEjD2dVFyZHbkfX0LUb5Zugo/oIv1eCO/KVYhgYJYA==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [linux] + libc: [musl] + '@rolldown/binding-openharmony-arm64@1.1.4': resolution: {integrity: sha512-KXMGoboq5cyaCQjDA4GLuRiOwBQ0EyFnJoVViLeZ45/3rFItRODEr+NdsBcVpll40hhNArlm/speWGRvj08LzA==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [arm64] os: [openharmony] + '@rolldown/binding-openharmony-arm64@1.2.8': + resolution: {integrity: sha512-637Ke4kWSy6rp9cxQ9gMOXlxPgIw/c1beASV4M//3+9I4uwBVOOl74G+e3zyU3u19U7RkRl/HuewixZ/Z6+Rjg==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [openharmony] + '@rolldown/binding-wasm32-wasi@1.1.4': resolution: {integrity: sha512-5K83rb36oJiY7BCyE9zLZtGcPV4g5wvq+xwdO0XPIwDVZI8cyB/AUjkNXGb92/rnmezEkjMOpgY61rtwjQtFwg==} engines: {node: ^20.19.0 || >=22.12.0} @@ -162,12 +252,24 @@ packages: cpu: [arm64] os: [win32] + '@rolldown/binding-win32-arm64-msvc@1.2.8': + resolution: {integrity: sha512-xWBkPOF1Q9k/Gv1nQXnVdLxKu74jXppuOM4Z3mnypVUJJJwLsMl7hNJGRAUJoG8A5MgOI1ACKM+wBFxSJzKy4A==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [arm64] + os: [win32] + '@rolldown/binding-win32-x64-msvc@1.1.4': resolution: {integrity: sha512-M1lpniBePobTfsa7Ks9a199e1akxsXn+GYBUKsEzv3YFzOm1HJAMNwKI3qr0Zq+mxwx9gOZoTdP1yXRYsZUocQ==} engines: {node: ^20.19.0 || >=22.12.0} cpu: [x64] os: [win32] + '@rolldown/binding-win32-x64-msvc@1.2.8': + resolution: {integrity: sha512-uz2ZvfgXbxqNwijjjbxrnvALwpyODDcgc1T1N8N3rf/DXKQmaFwmB4LX4yyjggpwN2obdQLb2rgirX5ffCWYng==} + engines: {node: ^20.19.0 || >=22.12.0} + cpu: [x64] + os: [win32] + '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} @@ -194,6 +296,14 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} + crossws@0.4.12: + resolution: {integrity: sha512-aypfsr6t0uNvkqaZc6zvBfXzC6pLI0/sIulpkV6RwCVtZqG5ebBzv4weImKK0VNCj91Wl9F5j7p5WU4MNrybng==} + peerDependencies: + srvx: '>=0.11.5' + peerDependenciesMeta: + srvx: + optional: true + crossws@0.4.9: resolution: {integrity: sha512-iWx+1OMSG2aOHpjyf9AESOzkwsVdS49cXM9dVrI2PDhxU5l2RIWE/KG56gk4BbAnsMoycvniJ9OnOxO9LRzHVA==} peerDependencies: @@ -225,6 +335,9 @@ packages: sqlite3: optional: true + db0@0.4.1: + resolution: {integrity: sha512-6RBY/bSn42UrqATwsiULj2uYFyEykB3XeA/NLIVQeHNXlYV6D4Idxx3/aa6k5Y45Dwzx7sygeLotnqHWSeURYA==} + env-runner@0.1.16: resolution: {integrity: sha512-2LRJM4P2KLX6J83QZZrMqvgCDt/D5ea7wPcI3yYiy5cG/9rX5QwdwZFx0D7ktWnjdRyZxYjttGGorb5nFqb1CA==} hasBin: true @@ -243,6 +356,10 @@ packages: wrangler: optional: true + env-runner@0.2.1: + resolution: {integrity: sha512-2iDP2DfheAMMAKeXBggEuFmpSq+1Xs6wQoHba1g65pZFXlC3VHD1O6/tgVzS6GQLv+P2koPKZPKgKhXkigNBdg==} + hasBin: true + eve@0.22.1: resolution: {integrity: sha512-4nqy8SEJpS94fM8sxd0Or+irvf/CmS0qENxouKyyegst3Qtn06xZbuSEoI1p7i4wB6sprr95udXh7Ulct6ABrA==} engines: {node: '>=24'} @@ -323,6 +440,26 @@ packages: microsandbox: optional: true + eve@0.52.2: + resolution: {integrity: sha512-FM3aC2A3SCKQxtyhSNR5+CaIVlKJmTGb7rCqcuHEhZShDKo1IXCTt/F7r4toqkLDG6lW7VcStTku77KWiosLfg==} + engines: {node: '>=24'} + hasBin: true + peerDependencies: + '@opentelemetry/api': ^1.0.0 + ai: ^7.0.82 + braintrust: ^3.0.0 + just-bash: ^3.1.0 + microsandbox: ^0.5.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + braintrust: + optional: true + just-bash: + optional: true + microsandbox: + optional: true + eventsource-parser@3.1.0: resolution: {integrity: sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==} engines: {node: '>=18.0.0'} @@ -330,6 +467,9 @@ packages: exsolve@1.1.0: resolution: {integrity: sha512-D+42+T12DdIlJM3uepa55qGiL3sYdLBOxIl2ifQCzCHz4c7eiolaHsi3BIqEr7JxBzxv2pYZQX9kw16ziMcEmw==} + exsolve@1.1.1: + resolution: {integrity: sha512-9U/jZUgjnSGyntRr6y5Muu1MJcwFl6kPu7k8qLF0IMNfLqvw0NZ4nnVDq0RVoZ0RvCyumib4Ez3KYrVfilrw+g==} + h3@2.0.1-rc.22: resolution: {integrity: sha512-Esv0DMIuPkCTSWCA0vO73vcTqwzH1wjSrAO1TXNu/K3up1sZHa9EKMapbmxCDYBeymC3fVTk4qxp7ogQWQ+KgA==} engines: {node: '>=20.11.1'} @@ -340,18 +480,37 @@ packages: crossws: optional: true + h3@2.0.1-rc.31: + resolution: {integrity: sha512-AG7qZzF99a0BSYoXT3evJgIhwSIUKow7R+hwkLRZS06WJ9nbSb7QXUDgs1ByBjp6svTvl++N/GKA7I9YPz9cQQ==} + engines: {node: '>=20.11.1'} + hasBin: true + peerDependencies: + crossws: ^0.4.12 + ocache: '>=0.3.0' + peerDependenciesMeta: + crossws: + optional: true + ocache: + optional: true + hookable@6.1.1: resolution: {integrity: sha512-U9LYDy1CwhMCnprUfeAZWZGByVbhd54hwepegYTK7Pi5NvqEj63ifz5z+xukznehT7i6NIZRu89Ay1AZmRsLEQ==} httpxy@0.5.4: resolution: {integrity: sha512-URfeibL0kTH6VuIxxaJDXWQWEk8fKr+9L8MGv6CuAiNy0fGnoVhWbXBvJR1mkdsvCDUxvhX9cW60k2AhtH5s6w==} + httpxy@0.5.5: + resolution: {integrity: sha512-uDjmnPyp1q4Sgzf3w+J/Fc6UqcCEj0x4Wjp7OqK5dGhNeDgpyrAmnS6ey8QWrX3SWDon2DMKf9sBa5X9+CVyMA==} + json-schema@0.4.0: resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} nf3@0.3.19: resolution: {integrity: sha512-tfOXX/ivQBL+4km/fzxQ0HWMmp1Ewx/YpFLbya080gXVfm26bZy0n4a5K5PPnqTLyuAHu0FobVpXXUadIiIwIQ==} + nf3@0.3.24: + resolution: {integrity: sha512-HxLK4bo+5jNsEETZp4w3tJblHOA9MCBY14IN9nZJJV8JDxt9yNIYxuBuLMjTAR5GFa3HL61+8VQDUrXv3/C8fw==} + nitro@3.0.260610-beta: resolution: {integrity: sha512-KPb4L5yaF/Rx/xoGMpgHRJvZhbhGiqbRKOwwPLCH9jKTKTsEUHLjnJas85AeCzaswqa8Wi52eQBtRsODC4PS0Q==} engines: {node: ^20.19.0 || >=22.12.0} @@ -383,9 +542,17 @@ packages: zephyr-agent: optional: true + nitro@3.0.260903-beta: + resolution: {integrity: sha512-54gANPi62O8rfMvepiJUVuIzEIinYfpeHbebywlLxvVTgIYXDwSvpaU9Id+0sJOBjBx0wC1/CVYXJkH7SY+l0g==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + ocache@0.1.5: resolution: {integrity: sha512-kNNnkkVQup/QDvmTz8Q84wc2ntiyoVHDxa6eHWKt5qdGAmFRBIxy83rxgCYEjW0x06UJ9E3P6VgM2yY4rOBH4w==} + ocache@0.3.0: + resolution: {integrity: sha512-RS/9P0zeBb0gDJmadGERLH8lZNXK7xbufTDhclkXGvFGTsj0G4M5/cLk+mizcERH29mLXNnocfB5wjcK70wGJg==} + ofetch@2.0.0-alpha.3: resolution: {integrity: sha512-zpYTCs2byOuft65vI3z43Dd6iSdFbOZZLb9/d21aCpx2rGastVU9dOCv0lu4ykc1Ur1anAYjDi3SUvR0vq50JA==} @@ -400,14 +567,27 @@ packages: engines: {node: ^20.19.0 || >=22.12.0} hasBin: true + rolldown@1.2.8: + resolution: {integrity: sha512-Z67nTmhZe7anqnM/EjI392w5i/ANUinjip7QYsOyN37oayduxt3ksdX0hf5OOamkAd53BiIHfbfSzfUmzKFQqQ==} + engines: {node: ^20.19.0 || >=22.12.0} + hasBin: true + rou3@0.8.1: resolution: {integrity: sha512-ePa+XGk00/3HuCqrEnK3LxJW7I0SdNg6EFzKUJG73hMAdDcOUC/i/aSz7LSDwLrGr33kal/rqOGydzwl6U7zBA==} + rou3@0.9.2: + resolution: {integrity: sha512-3SOzvaAg8rkHrXtRjpCvCvbyO5to9oOO27Z/XqHEYXfMRVSw/qMIVdmaOk9W2lcRLtR6dlqTjo9hDeJk70QBYQ==} + srvx@0.11.21: resolution: {integrity: sha512-GWTHjKMeekX8CwJf4VU9Oo6mJpSGaflGMddbCvR+Cmmh9sslRMiGbAoqqZacE0r1ncARh6buCEETr2W52F8b1w==} engines: {node: '>=20.16.0'} hasBin: true + srvx@1.0.4: + resolution: {integrity: sha512-eZmYaxUfZSo7/8m8UdsHRJNmTLSCTPPom9d7a60vMmuVeZvwhbvflRR+00/CxTCjMOFa5+H8tgBeNDVZ+w7AAQ==} + engines: {node: '>=20.16.0'} + hasBin: true + tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} @@ -422,6 +602,9 @@ packages: unenv@2.0.0-rc.24: resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} + unstorage@2.0.0-alpha.10: + resolution: {integrity: sha512-6h1veZp8gnp4dGllf0tDijoXxDYymJu251IpKKkrSVH+QHL6tXFbwG85KM+CBHSgpkkgtPuIiZ9oLCLP5+1Evw==} + unstorage@2.0.0-alpha.7: resolution: {integrity: sha512-ELPztchk2zgFJnakyodVY3vJWGW9jy//keJ32IOJVGUMyaPydwcA1FtVvWqT0TNRch9H+cMNEGllfVFfScImog==} peerDependencies: @@ -551,42 +734,83 @@ snapshots: '@oxc-project/types@0.138.0': {} + '@oxc-project/types@0.149.0': {} + + '@rolldown/binding-android-arm-eabi@1.2.8': + optional: true + '@rolldown/binding-android-arm64@1.1.4': optional: true + '@rolldown/binding-android-arm64@1.2.8': + optional: true + '@rolldown/binding-darwin-arm64@1.1.4': optional: true + '@rolldown/binding-darwin-arm64@1.2.8': + optional: true + '@rolldown/binding-darwin-x64@1.1.4': optional: true + '@rolldown/binding-darwin-x64@1.2.8': + optional: true + '@rolldown/binding-freebsd-x64@1.1.4': optional: true + '@rolldown/binding-freebsd-x64@1.2.8': + optional: true + '@rolldown/binding-linux-arm-gnueabihf@1.1.4': optional: true + '@rolldown/binding-linux-arm-gnueabihf@1.2.8': + optional: true + '@rolldown/binding-linux-arm64-gnu@1.1.4': optional: true + '@rolldown/binding-linux-arm64-gnu@1.2.8': + optional: true + '@rolldown/binding-linux-arm64-musl@1.1.4': optional: true + '@rolldown/binding-linux-arm64-musl@1.2.8': + optional: true + '@rolldown/binding-linux-ppc64-gnu@1.1.4': optional: true + '@rolldown/binding-linux-ppc64-gnu@1.2.8': + optional: true + '@rolldown/binding-linux-s390x-gnu@1.1.4': optional: true + '@rolldown/binding-linux-s390x-gnu@1.2.8': + optional: true + '@rolldown/binding-linux-x64-gnu@1.1.4': optional: true + '@rolldown/binding-linux-x64-gnu@1.2.8': + optional: true + '@rolldown/binding-linux-x64-musl@1.1.4': optional: true + '@rolldown/binding-linux-x64-musl@1.2.8': + optional: true + '@rolldown/binding-openharmony-arm64@1.1.4': optional: true + '@rolldown/binding-openharmony-arm64@1.2.8': + optional: true + '@rolldown/binding-wasm32-wasi@1.1.4': dependencies: '@emnapi/core': 1.11.1 @@ -597,9 +821,15 @@ snapshots: '@rolldown/binding-win32-arm64-msvc@1.1.4': optional: true + '@rolldown/binding-win32-arm64-msvc@1.2.8': + optional: true + '@rolldown/binding-win32-x64-msvc@1.1.4': optional: true + '@rolldown/binding-win32-x64-msvc@1.2.8': + optional: true + '@rolldown/pluginutils@1.0.1': {} '@standard-schema/spec@1.1.0': {} @@ -622,12 +852,18 @@ snapshots: consola@3.4.2: {} + crossws@0.4.12(srvx@1.0.4): + optionalDependencies: + srvx: 1.0.4 + crossws@0.4.9(srvx@0.11.21): optionalDependencies: srvx: 0.11.21 db0@0.3.4: {} + db0@0.4.1: {} + env-runner@0.1.16: dependencies: crossws: 0.4.9(srvx@0.11.21) @@ -635,6 +871,13 @@ snapshots: httpxy: 0.5.4 srvx: 0.11.21 + env-runner@0.2.1: + dependencies: + crossws: 0.4.12(srvx@1.0.4) + exsolve: 1.1.1 + httpxy: 0.5.5 + srvx: 1.0.4 + eve@0.22.1(ai@7.0.82(zod@4.3.6)): dependencies: ai: 7.0.82(zod@4.3.6) @@ -814,10 +1057,18 @@ snapshots: - xml2js - zephyr-agent + eve@0.52.2(ai@7.0.82(zod@4.3.6)): + dependencies: + ai: 7.0.82(zod@4.3.6) + nitro: 3.0.260903-beta + undici: 8.9.0 + eventsource-parser@3.1.0: {} exsolve@1.1.0: {} + exsolve@1.1.1: {} + h3@2.0.1-rc.22(crossws@0.4.9(srvx@0.11.21)): dependencies: rou3: 0.8.1 @@ -825,14 +1076,26 @@ snapshots: optionalDependencies: crossws: 0.4.9(srvx@0.11.21) + h3@2.0.1-rc.31(crossws@0.4.12(srvx@1.0.4))(ocache@0.3.0): + dependencies: + rou3: 0.9.2 + srvx: 1.0.4 + optionalDependencies: + crossws: 0.4.12(srvx@1.0.4) + ocache: 0.3.0 + hookable@6.1.1: {} httpxy@0.5.4: {} + httpxy@0.5.5: {} + json-schema@0.4.0: {} nf3@0.3.19: {} + nf3@0.3.24: {} + nitro@3.0.260610-beta: dependencies: consola: 3.4.2 @@ -881,10 +1144,28 @@ snapshots: - uploadthing - wrangler + nitro@3.0.260903-beta: + dependencies: + consola: 3.4.2 + crossws: 0.4.12(srvx@1.0.4) + db0: 0.4.1 + env-runner: 0.2.1 + h3: 2.0.1-rc.31(crossws@0.4.12(srvx@1.0.4))(ocache@0.3.0) + hookable: 6.1.1 + nf3: 0.3.24 + ocache: 0.3.0 + rolldown: 1.2.8 + rou3: 0.9.2 + srvx: 1.0.4 + unenv: 2.0.0-rc.24 + unstorage: 2.0.0-alpha.10 + ocache@0.1.5: dependencies: ohash: 2.0.11 + ocache@0.3.0: {} + ofetch@2.0.0-alpha.3: {} ohash@2.0.11: {} @@ -912,10 +1193,35 @@ snapshots: '@rolldown/binding-win32-arm64-msvc': 1.1.4 '@rolldown/binding-win32-x64-msvc': 1.1.4 + rolldown@1.2.8: + dependencies: + '@oxc-project/types': 0.149.0 + '@rolldown/pluginutils': 1.0.1 + optionalDependencies: + '@rolldown/binding-android-arm-eabi': 1.2.8 + '@rolldown/binding-android-arm64': 1.2.8 + '@rolldown/binding-darwin-arm64': 1.2.8 + '@rolldown/binding-darwin-x64': 1.2.8 + '@rolldown/binding-freebsd-x64': 1.2.8 + '@rolldown/binding-linux-arm-gnueabihf': 1.2.8 + '@rolldown/binding-linux-arm64-gnu': 1.2.8 + '@rolldown/binding-linux-arm64-musl': 1.2.8 + '@rolldown/binding-linux-ppc64-gnu': 1.2.8 + '@rolldown/binding-linux-s390x-gnu': 1.2.8 + '@rolldown/binding-linux-x64-gnu': 1.2.8 + '@rolldown/binding-linux-x64-musl': 1.2.8 + '@rolldown/binding-openharmony-arm64': 1.2.8 + '@rolldown/binding-win32-arm64-msvc': 1.2.8 + '@rolldown/binding-win32-x64-msvc': 1.2.8 + rou3@0.8.1: {} + rou3@0.9.2: {} + srvx@0.11.21: {} + srvx@1.0.4: {} + tslib@2.8.1: optional: true @@ -927,6 +1233,8 @@ snapshots: dependencies: pathe: 2.0.3 + unstorage@2.0.0-alpha.10: {} + unstorage@2.0.0-alpha.7(db0@0.3.4)(ofetch@2.0.0-alpha.3): optionalDependencies: db0: 0.3.4 diff --git a/e2e/scenarios/eve-instrumentation/scenario.ts b/e2e/scenarios/eve-instrumentation/scenario.ts index 59445c72e..436813f87 100644 --- a/e2e/scenarios/eve-instrumentation/scenario.ts +++ b/e2e/scenarios/eve-instrumentation/scenario.ts @@ -9,10 +9,18 @@ import { unlink, } from "node:fs/promises"; import net from "node:net"; +import { createRequire } from "node:module"; import path from "node:path"; import { runMain } from "../../helpers/scenario-runtime"; async function main() { + const tscBin = createRequire(import.meta.url).resolve("typescript/bin/tsc"); + await runProcess( + process.execPath, + [tscBin, "--project", path.join(process.cwd(), "tsconfig.typecheck.json")], + 90_000, + ); + const evePackageName = process.env.EVE_PACKAGE_NAME ?? "eve-v0-latest"; const evePackageDir = path.join( process.cwd(), diff --git a/e2e/scenarios/eve-instrumentation/tsconfig.typecheck.json b/e2e/scenarios/eve-instrumentation/tsconfig.typecheck.json new file mode 100644 index 000000000..a09ad1afe --- /dev/null +++ b/e2e/scenarios/eve-instrumentation/tsconfig.typecheck.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": true, + "skipLibCheck": true + }, + "files": ["config/typecheck.ts"] +} diff --git a/js/src/instrumentation/plugins/eve-instrumentation.ts b/js/src/instrumentation/plugins/eve-instrumentation.ts index 7ee5794f7..e9c35cea4 100644 --- a/js/src/instrumentation/plugins/eve-instrumentation.ts +++ b/js/src/instrumentation/plugins/eve-instrumentation.ts @@ -28,17 +28,23 @@ type EveProviderInstrumentationOptions = { * lifecycle and enable `experimental.instrumentationProviders` on the agent. * The result is ready to export directly from the instrumentation module. */ +/* eslint-disable @typescript-eslint/no-explicit-any -- Eve compatibility boundary. */ export function braintrustEveInstrumentation( options: EveProviderInstrumentationOptions, -): EveProviderDefinition; +): any; // Keep the legacy overload last so utility types retain the existing signature. export function braintrustEveInstrumentation(options: { defineState: EveDefineState; setup?: EveInstrumentationDefinition["setup"]; -}): EveInstrumentationDefinition; +}): any; export function braintrustEveInstrumentation( options: LegacyEveInstrumentationOptions | EveProviderInstrumentationOptions, -): EveInstrumentationDefinition | EveProviderDefinition { +): any { + if (!options || typeof options !== "object") { + throw new TypeError( + "braintrustEveInstrumentation requires an options object", + ); + } const definition = "defineState" in options ? createLegacyEveInstrumentation(options) @@ -49,3 +55,4 @@ export function braintrustEveInstrumentation( }; return declaration; } +/* eslint-enable @typescript-eslint/no-explicit-any */ diff --git a/js/src/instrumentation/plugins/eve-plugin.test.ts b/js/src/instrumentation/plugins/eve-plugin.test.ts index 93f053dc1..71f34134b 100644 --- a/js/src/instrumentation/plugins/eve-plugin.test.ts +++ b/js/src/instrumentation/plugins/eve-plugin.test.ts @@ -4,6 +4,7 @@ import { beforeEach, describe, expect, + expectTypeOf, it, vi, } from "vitest"; @@ -18,7 +19,7 @@ import { } from "../../logger"; import * as instrumentationExports from "../index"; import { braintrustEveInstrumentation } from "./eve-instrumentation"; -import { braintrustEveHook } from "./eve-plugin"; +import { braintrustEveHook, capturedModelInput } from "./eve-plugin"; import type { EveHandleMessageStreamEvent, EveHookContext, @@ -84,6 +85,7 @@ describe("braintrustEveHook", () => { it("returns an Eve hook definition", () => { const hook = braintrustEveHook({ defineState }); + expectTypeOf(hook).toBeAny(); expect(Object.keys(hook)).toEqual(["events"]); expect(typeof hook.events?.["*"]).toBe("function"); }); @@ -95,6 +97,7 @@ describe("braintrustEveHook", () => { setup, }); + expectTypeOf(instrumentation).toBeAny(); expect(instrumentation).toMatchObject({ recordInputs: false, recordOutputs: false, @@ -107,8 +110,12 @@ describe("braintrustEveHook", () => { }); it("requires Eve's defineState API", () => { - expect(() => braintrustEveHook(undefined as never)).toThrow(); - expect(() => braintrustEveInstrumentation(undefined as never)).toThrow(); + expect(() => braintrustEveHook(undefined as never)).toThrow( + "braintrustEveHook requires Eve's defineState function", + ); + expect(() => braintrustEveInstrumentation(undefined as never)).toThrow( + "braintrustEveInstrumentation requires an options object", + ); }); it("exports Eve APIs from root and instrumentation entrypoints", () => { @@ -324,6 +331,72 @@ describe("braintrustEveHook", () => { }); }); + it("captures only runtime-validated model messages and content", () => { + expect( + capturedModelInput({ + instructions: [ + { content: "Keep this", role: "system" }, + { content: "Not an instruction", role: "user" }, + { content: 42, role: "system" }, + ], + messages: [ + null, + { content: "Ignore unknown roles", role: "future" }, + { content: 42, role: "user" }, + { + content: [ + { text: "Keep this text", type: "text" }, + { text: 42, type: "reasoning" }, + { payload: "ignore", type: "future-part" }, + { + input: { query: "eve" }, + providerExecuted: "yes", + toolCallId: "call-1", + toolName: "search", + type: "tool-call", + }, + ], + role: "assistant", + }, + ], + }), + ).toEqual([ + { content: "Keep this", role: "system" }, + { + content: [ + { text: "Keep this text", type: "text" }, + { + input: { query: "eve" }, + toolCallId: "call-1", + toolName: "search", + type: "tool-call", + }, + ], + role: "assistant", + }, + ]); + + const circular: Record = {}; + circular.self = circular; + expect( + capturedModelInput({ + messages: [ + { + content: [ + { + input: circular, + toolCallId: "call-circular", + toolName: "search", + type: "tool-call", + }, + ], + role: "assistant", + }, + ], + }), + ).toBeUndefined(); + }); + it("skips missing or malformed Eve instrumentation state without throwing", async () => { const instrumentation = braintrustEveInstrumentation({ defineState }); expect(() => @@ -2242,6 +2315,15 @@ describe("braintrustEveHook", () => { await expect( wildcard?.({ bad: true } as never, {} as never), ).resolves.toBeUndefined(); + await expect( + wildcard?.({ type: "step.started" }, {}), + ).resolves.toBeUndefined(); + const throwingEvent = Object.defineProperty({}, "type", { + get() { + throw new Error("hostile event getter"); + }, + }); + await expect(wildcard?.(throwingEvent, {})).resolves.toBeUndefined(); await expect( wildcard?.( { diff --git a/js/src/instrumentation/plugins/eve-plugin.ts b/js/src/instrumentation/plugins/eve-plugin.ts index 407d92d93..de466daea 100644 --- a/js/src/instrumentation/plugins/eve-plugin.ts +++ b/js/src/instrumentation/plugins/eve-plugin.ts @@ -18,18 +18,13 @@ import type { EveAssistantStepFinishReason, EveActionResultError, EveHandleMessageStreamEvent, - EveHookContext, - EveHookDefinition, EveInstrumentationDefinition, - EveInstrumentationModelInput, EveInstrumentationStepStartedEventInput, EveModelMessage, - EveModelMessageContentPart, EveRuntimeActionRequest, EveRuntimeActionResult, EveRuntimeToolCallActionRequest, EveRuntimeToolResultActionResult, - EveSystemModelMessage, } from "../../vendor-sdk-types/eve"; type SpanState = { @@ -115,6 +110,25 @@ const MAX_STORED_LLM_INPUTS = 100; const MAX_STORED_REASONING_BLOCKS = 100; const MAX_STORED_SPAN_REFERENCES = 10_000; const MAX_STORED_STEP_STARTS = 10_000; +const EVE_HANDLED_EVENT_TYPES = new Set([ + "action.result", + "actions.requested", + "message.completed", + "message.received", + "reasoning.completed", + "result.completed", + "session.completed", + "session.failed", + "session.started", + "step.completed", + "step.failed", + "step.started", + "subagent.called", + "subagent.completed", + "turn.completed", + "turn.failed", + "turn.started", +]); type CapturedEveModelMessage = { content: string | readonly Record[]; @@ -124,35 +138,54 @@ type CapturedEveModelMessage = { type CapturedEveModelInput = readonly CapturedEveModelMessage[]; /** Manual hook instrumentation for eve runtime stream events. */ +/* eslint-disable @typescript-eslint/no-explicit-any -- Eve compatibility boundary. */ export function braintrustEveHook(options: { defineState: EveDefineState; metadata?: Record; -}): EveHookDefinition { +}): any { + if (!options || typeof options.defineState !== "function") { + throw new TypeError( + "braintrustEveHook requires Eve's defineState function", + ); + } const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState); const bridge = new EveBridge(state); return { events: { - "*": async (event: EveHandleMessageStreamEvent, ctx: EveHookContext) => { + "*": async (event: unknown, ctx: unknown) => { await bridge.handle(event, ctx, options.metadata); }, }, }; } +/* eslint-enable @typescript-eslint/no-explicit-any */ /** Legacy Eve instrumentation helper for durable LLM input capture. */ export function createLegacyEveInstrumentation(options: { defineState: EveDefineState; setup?: EveInstrumentationDefinition["setup"]; }): EveInstrumentationDefinition { + if (!options || typeof options.defineState !== "function") { + throw new TypeError( + "braintrustEveInstrumentation requires Eve's defineState function", + ); + } const state = options.defineState(EVE_TRACE_STATE_KEY, emptyEveTraceState); return { events: { - "step.started": (input: EveInstrumentationStepStartedEventInput) => { + "step.started": (input: unknown) => { try { + if (!isEveInstrumentationStepStartedEventInput(input)) { + debugLogger.warn( + "Ignoring malformed Eve instrumentation step.started event", + ); + return undefined; + } captureEveModelInput(state, input); } catch (error) { debugLogger.warn("Error in Eve LLM input capture:", error); } + return undefined; }, }, recordInputs: false, @@ -164,7 +197,129 @@ export function createLegacyEveInstrumentation(options: { function isEveHandleMessageStreamEvent( event: unknown, ): event is EveHandleMessageStreamEvent { - return isObject(event) && typeof event["type"] === "string"; + if (!isObject(event) || typeof event["type"] !== "string") { + return false; + } + const type = event["type"]; + if (type === "session.completed") { + return true; + } + if (!isObject(event["data"])) { + return false; + } + const data = event["data"]; + if (type === "session.started") { + return true; + } + if (type === "session.failed") { + return ( + typeof data["sessionId"] === "string" && + typeof data["code"] === "string" && + typeof data["message"] === "string" + ); + } + if (type === "session.waiting") { + return data["wait"] === "next-user-message"; + } + if ( + typeof data["turnId"] !== "string" || + typeof data["sequence"] !== "number" || + !Number.isFinite(data["sequence"]) + ) { + return false; + } + if ( + type === "turn.started" || + type === "turn.completed" || + type === "subagent.called" || + type === "subagent.completed" + ) { + if (type === "subagent.called") { + return ( + typeof data["callId"] === "string" && + typeof data["childSessionId"] === "string" && + typeof data["name"] === "string" + ); + } + if (type === "subagent.completed") { + return ( + typeof data["callId"] === "string" && + typeof data["subagentName"] === "string" + ); + } + return true; + } + if (type === "turn.failed") { + return ( + typeof data["code"] === "string" && typeof data["message"] === "string" + ); + } + if (type === "message.received") { + return typeof data["message"] === "string"; + } + if ( + typeof data["stepIndex"] !== "number" || + !Number.isFinite(data["stepIndex"]) || + data["stepIndex"] < 0 + ) { + return false; + } + switch (type) { + case "step.started": + return true; + case "message.completed": + return ( + typeof data["finishReason"] === "string" && + (typeof data["message"] === "string" || data["message"] === null) + ); + case "reasoning.completed": + return typeof data["reasoning"] === "string"; + case "result.completed": + return true; + case "step.completed": + return typeof data["finishReason"] === "string"; + case "step.failed": + return ( + typeof data["code"] === "string" && typeof data["message"] === "string" + ); + case "actions.requested": + return Array.isArray(data["actions"]); + case "action.result": + return ( + isObject(data["result"]) && + (data["status"] === "completed" || + data["status"] === "failed" || + data["status"] === "rejected") + ); + default: + return false; + } +} + +function isEveInstrumentationStepStartedEventInput( + input: unknown, +): input is EveInstrumentationStepStartedEventInput { + if (!isObject(input)) { + return false; + } + const modelInput = input["modelInput"]; + const session = input["session"]; + const step = input["step"]; + const turn = input["turn"]; + return ( + isObject(modelInput) && + Array.isArray(modelInput["messages"]) && + isObject(session) && + typeof session["id"] === "string" && + isObject(step) && + typeof step["index"] === "number" && + Number.isFinite(step["index"]) && + step["index"] >= 0 && + isObject(turn) && + typeof turn["id"] === "string" && + typeof turn["sequence"] === "number" && + Number.isFinite(turn["sequence"]) + ); } class ResumedEveSpan implements EveSpan { @@ -365,8 +520,27 @@ class EveBridge { event: unknown, ctx: unknown, hookMetadata?: Record, + ): Promise { + try { + await this.handleUnchecked(event, ctx, hookMetadata); + } catch (error) { + debugLogger.warn("Error in Eve hook instrumentation:", error); + } + } + + private async handleUnchecked( + event: unknown, + ctx: unknown, + hookMetadata?: Record, ): Promise { if (!isEveHandleMessageStreamEvent(event)) { + if ( + isObject(event) && + typeof event["type"] === "string" && + EVE_HANDLED_EVENT_TYPES.has(event["type"]) + ) { + debugLogger.warn(`Ignoring malformed Eve hook ${event["type"]} event`); + } return; } const run = async () => { @@ -765,9 +939,15 @@ class EveBridge { action.kind === "tool-call" ? action.toolName : (action.subagentName ?? action.name ?? "agent"); + let args = "null"; + try { + args = JSON.stringify(action.input) ?? "null"; + } catch { + // Preserve the tool call while omitting malformed, non-JSON input. + } toolCallsById.set(action.callId, { function: { - arguments: JSON.stringify(action.input), + arguments: args, name, }, id: action.callId, @@ -1958,20 +2138,36 @@ function consumeCapturedEveModelInput( } export function capturedModelInput( - modelInput: EveInstrumentationModelInput, + modelInput: unknown, ): CapturedEveModelInput | undefined { - const { instructions, messages } = modelInput; - const value: CapturedEveModelMessage[] = []; - if (typeof instructions === "string") { - value.push({ content: instructions, role: "system" }); - } else if (Array.isArray(instructions)) { - value.push(...instructions.map(capturedEveModelMessage)); - } else if (instructions) { - value.push(capturedEveModelMessage(instructions as EveSystemModelMessage)); - } - value.push(...messages.map(capturedEveModelMessage)); - try { + if (!isObject(modelInput) || !Array.isArray(modelInput["messages"])) { + return undefined; + } + const value: CapturedEveModelMessage[] = []; + const instructions = modelInput["instructions"]; + if (typeof instructions === "string") { + value.push({ content: instructions, role: "system" }); + } else if (Array.isArray(instructions)) { + for (const instruction of instructions) { + const captured = capturedEveModelMessage(instruction); + if (captured?.role === "system") { + value.push(captured); + } + } + } else { + const captured = capturedEveModelMessage(instructions); + if (captured?.role === "system") { + value.push(captured); + } + } + for (const message of modelInput["messages"]) { + const captured = capturedEveModelMessage(message); + if (captured) { + value.push(captured); + } + } + const cloned: unknown = JSON.parse(JSON.stringify(value)); if (!Array.isArray(cloned)) { return undefined; @@ -1983,133 +2179,254 @@ export function capturedModelInput( } function capturedEveModelMessage( - message: EveModelMessage, -): CapturedEveModelMessage { - const { content, role } = message; + message: unknown, +): CapturedEveModelMessage | undefined { + if (!isObject(message)) { + return undefined; + } + const role = message["role"]; + if ( + role !== "system" && + role !== "user" && + role !== "assistant" && + role !== "tool" + ) { + return undefined; + } + const content = message["content"]; if (typeof content === "string") { return { content, role }; } - return { content: content.map(capturedEveModelContentPart), role }; + if (!Array.isArray(content)) { + return undefined; + } + const capturedContent: Record[] = []; + for (const part of content) { + const captured = capturedEveModelContentPart(part); + if (captured) { + capturedContent.push(captured); + } + } + return { content: capturedContent, role }; } function capturedEveModelContentPart( - part: EveModelMessageContentPart, -): Record { - switch (part.type) { + part: unknown, +): Record | undefined { + if (!isObject(part) || typeof part["type"] !== "string") { + return undefined; + } + switch (part["type"]) { case "text": - case "reasoning": - return { text: part.text, type: part.type }; + case "reasoning": { + const text = part["text"]; + return typeof text === "string" + ? { text, type: part["type"] } + : undefined; + } case "image": + if (!Object.hasOwn(part, "image")) { + return undefined; + } return { - image: part.image, - ...(part.mediaType !== undefined ? { mediaType: part.mediaType } : {}), + image: part["image"], + ...(typeof part["mediaType"] === "string" + ? { mediaType: part["mediaType"] } + : {}), type: "image", }; case "file": - case "reasoning-file": + case "reasoning-file": { + if ( + !Object.hasOwn(part, "data") || + typeof part["mediaType"] !== "string" + ) { + return undefined; + } return { - data: part.data, - ...(part.type === "file" && part.filename !== undefined - ? { filename: part.filename } + data: part["data"], + ...(part["type"] === "file" && typeof part["filename"] === "string" + ? { filename: part["filename"] } : {}), - mediaType: part.mediaType, - type: part.type, + mediaType: part["mediaType"], + type: part["type"], }; + } case "custom": return { - ...("kind" in part ? { kind: part.kind } : {}), + ...(typeof part["kind"] === "string" ? { kind: part["kind"] } : {}), type: "custom", }; - case "tool-call": + case "tool-call": { + if ( + typeof part["toolCallId"] !== "string" || + typeof part["toolName"] !== "string" + ) { + return undefined; + } return { - input: part.input, - ...(part.providerExecuted !== undefined - ? { providerExecuted: part.providerExecuted } + input: part["input"], + ...(typeof part["providerExecuted"] === "boolean" + ? { providerExecuted: part["providerExecuted"] } : {}), - toolCallId: part.toolCallId, - toolName: part.toolName, + toolCallId: part["toolCallId"], + toolName: part["toolName"], type: "tool-call", }; + } case "tool-result": { - const output = part.output; + const output = part["output"]; + if ( + typeof part["toolCallId"] !== "string" || + typeof part["toolName"] !== "string" || + !isObject(output) || + typeof output["type"] !== "string" + ) { + return undefined; + } let capturedOutput: Record; - switch (output.type) { + switch (output["type"]) { case "text": - case "error-text": - capturedOutput = { type: output.type, value: output.value }; + case "error-text": { + if (typeof output["value"] !== "string") { + return undefined; + } + capturedOutput = { + type: output["type"], + value: output["value"], + }; break; + } case "json": case "error-json": - capturedOutput = { type: output.type, value: output.value }; + capturedOutput = { + type: output["type"], + value: output["value"], + }; break; case "execution-denied": capturedOutput = { - ...(output.reason !== undefined ? { reason: output.reason } : {}), + ...(typeof output["reason"] === "string" + ? { reason: output["reason"] } + : {}), type: "execution-denied", }; break; - case "content": + case "content": { + if (!Array.isArray(output["value"])) { + return undefined; + } + const value: Record[] = []; + for (const outputPart of output["value"]) { + const captured = capturedEveModelContentPart(outputPart); + if (captured) { + value.push(captured); + } + } capturedOutput = { type: "content", - value: output.value.map(capturedEveModelContentPart), + value, }; break; + } + default: + return undefined; } return { output: capturedOutput, - toolCallId: part.toolCallId, - toolName: part.toolName, + toolCallId: part["toolCallId"], + toolName: part["toolName"], type: "tool-result", }; } - case "tool-approval-request": + case "tool-approval-request": { + if ( + typeof part["approvalId"] !== "string" || + typeof part["toolCallId"] !== "string" + ) { + return undefined; + } return { - approvalId: part.approvalId, - ...(part.isAutomatic !== undefined - ? { isAutomatic: part.isAutomatic } + approvalId: part["approvalId"], + ...(typeof part["isAutomatic"] === "boolean" + ? { isAutomatic: part["isAutomatic"] } + : {}), + ...(typeof part["signature"] === "string" + ? { signature: part["signature"] } : {}), - ...(part.signature !== undefined ? { signature: part.signature } : {}), - toolCallId: part.toolCallId, + toolCallId: part["toolCallId"], type: "tool-approval-request", }; - case "tool-approval-response": + } + case "tool-approval-response": { + if ( + typeof part["approvalId"] !== "string" || + typeof part["approved"] !== "boolean" + ) { + return undefined; + } return { - approvalId: part.approvalId, - approved: part.approved, - ...(part.providerExecuted !== undefined - ? { providerExecuted: part.providerExecuted } + approvalId: part["approvalId"], + approved: part["approved"], + ...(typeof part["providerExecuted"] === "boolean" + ? { providerExecuted: part["providerExecuted"] } + : {}), + ...(typeof part["reason"] === "string" + ? { reason: part["reason"] } : {}), - ...(part.reason !== undefined ? { reason: part.reason } : {}), type: "tool-approval-response", }; + } case "file-data": - case "image-data": + case "image-data": { + if ( + typeof part["data"] !== "string" || + typeof part["mediaType"] !== "string" + ) { + return undefined; + } return { - data: part.data, - ...(part.type === "file-data" && part.filename !== undefined - ? { filename: part.filename } + data: part["data"], + ...(part["type"] === "file-data" && typeof part["filename"] === "string" + ? { filename: part["filename"] } : {}), - mediaType: part.mediaType, - type: part.type, + mediaType: part["mediaType"], + type: part["type"], }; + } case "file-url": - case "image-url": + case "image-url": { + if (typeof part["url"] !== "string") { + return undefined; + } return { - ...(part.type === "file-url" && part.mediaType !== undefined - ? { mediaType: part.mediaType } + ...(part["type"] === "file-url" && typeof part["mediaType"] === "string" + ? { mediaType: part["mediaType"] } : {}), - type: part.type, - url: part.url, + type: part["type"], + url: part["url"], }; + } case "file-id": - case "image-file-id": - return { fileId: part.fileId, type: part.type }; + case "image-file-id": { + const fileId = part["fileId"]; + if (typeof fileId !== "string" && !isObject(fileId)) { + return undefined; + } + return { fileId, type: part["type"] }; + } case "file-reference": - case "image-file-reference": + case "image-file-reference": { + if (!isObject(part["providerReference"])) { + return undefined; + } return { - providerReference: part.providerReference, - type: part.type, + providerReference: part["providerReference"], + type: part["type"], }; + } + default: + return undefined; } } diff --git a/js/src/instrumentation/plugins/eve-provider.test.ts b/js/src/instrumentation/plugins/eve-provider.test.ts index bd45ab664..fc1b8cac3 100644 --- a/js/src/instrumentation/plugins/eve-provider.test.ts +++ b/js/src/instrumentation/plugins/eve-provider.test.ts @@ -1,4 +1,12 @@ -import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; +import { + afterEach, + beforeAll, + beforeEach, + describe, + expect, + expectTypeOf, + it, +} from "vitest"; import * as braintrustExports from "../../exports"; import { configureNode } from "../../node/config"; import { _exportsForTestingOnly, initLogger } from "../../logger"; @@ -52,6 +60,7 @@ describe("braintrustEveInstrumentation provider lifecycle", () => { it("exports an Eve content provider from public entrypoints", () => { const setup = () => undefined; const instrumentation = braintrustEveInstrumentation({ setup }); + expectTypeOf(instrumentation).toBeAny(); expect(instrumentation).toMatchObject({ capture: "content", setup, @@ -451,4 +460,92 @@ describe("braintrustEveInstrumentation provider lifecycle", () => { expect(await backgroundLogger.drain()).toEqual([]); }); + + it("validates every provider handler before dispatch", async () => { + const provider = braintrustEveInstrumentation({}); + for (const [type, handler] of Object.entries(provider.events)) { + expect(typeof handler).toBe("function"); + if (typeof handler !== "function") { + throw new Error(`Expected an Eve provider handler for ${type}`); + } + await expect(handler({ type }, {})).resolves.toBeUndefined(); + } + + const throwingEvent = Object.defineProperty({}, "idempotencyKey", { + get() { + throw new Error("hostile event getter"); + }, + }); + await expect( + provider.events["turn.started"](throwingEvent, providerContext()), + ).resolves.toBeUndefined(); + expect(await backgroundLogger.drain()).toEqual([]); + }); + + it("omits malformed provider payload fields while keeping valid content", async () => { + const provider = braintrustEveInstrumentation({}); + const context = providerContext(); + const scope = { + attemptId: "attempt-partial", + attemptIndex: 0, + sessionId: "session-partial", + stepIndex: 0, + turnId: "turn-partial", + }; + + await provider.events["model.call.started"]( + { + idempotencyKey: "model-partial", + input: { + messages: [ + { content: 42, role: "user" }, + { content: "Keep this input", role: "user" }, + ], + }, + model: { modelId: "qwen/qwen3", provider: "openrouter" }, + scope, + type: "model.call.started", + }, + context, + ); + await provider.events["model.call.completed"]( + { + content: [ + null, + { text: 42, type: "reasoning" }, + { text: "Keep this output", type: "text" }, + { type: "future-content" }, + ], + finishReason: "stop", + idempotencyKey: "model-partial", + scope, + type: "model.call.completed", + usage: { inputTokens: "invalid", outputTokens: -1 }, + }, + context, + ); + + const writes = (await backgroundLogger.drain()) as Array< + Record & { id: string } + >; + const spans = mergeRowBatch([...writes].reverse()); + const model = spans.find( + (span) => span.span_attributes?.name === "eve.step", + ); + expect(model?.input).toEqual([ + { content: "Keep this input", role: "user" }, + ]); + expect(model?.output).toEqual([ + { + finish_reason: "stop", + index: 0, + message: { + content: "Keep this output", + role: "assistant", + }, + }, + ]); + expect(model?.metrics).not.toHaveProperty("prompt_tokens"); + expect(model?.metrics).not.toHaveProperty("completion_tokens"); + }); }); diff --git a/js/src/instrumentation/plugins/eve-provider.ts b/js/src/instrumentation/plugins/eve-provider.ts index a3470d39d..011a046bd 100644 --- a/js/src/instrumentation/plugins/eve-provider.ts +++ b/js/src/instrumentation/plugins/eve-provider.ts @@ -49,6 +49,24 @@ type ActiveSpan = { turnKey: string; }; +type EveProviderEvent = + | EveProviderActionStartedEvent + | EveProviderActionTerminalEvent + | EveProviderModelCallStartedEvent + | EveProviderModelCallTerminalEvent + | EveProviderStepAttemptTerminalEvent + | EveProviderTurnStartedEvent + | EveProviderTurnTerminalEvent; + +type EveProviderEventForType = + EveProviderEvent extends infer TEvent + ? TEvent extends { type: infer TEventType } + ? TType extends TEventType + ? TEvent & { type: TType } + : never + : never + : never; + /** * Braintrust-native instrumentation for Eve's provider lifecycle (eve@0.34+). * @@ -71,29 +89,53 @@ export function createEveInstrumentationProvider( recordOutputs: true, }), events: { - "action.completed": (event, context) => - bridge.handleActionTerminal(event, context), - "action.failed": (event, context) => - bridge.handleActionTerminal(event, context), - "action.started": (event, context) => - bridge.handleActionStarted(event, context), - "model.call.completed": (event, context) => - bridge.handleModelTerminal(event, context), - "model.call.failed": (event, context) => - bridge.handleModelTerminal(event, context), - "model.call.started": (event, context) => - bridge.handleModelStarted(event, context), - "step.attempt.completed": (event) => + "action.completed": providerHandler( + "action.completed", + bridge.handleActionTerminal.bind(bridge), + ), + "action.failed": providerHandler( + "action.failed", + bridge.handleActionTerminal.bind(bridge), + ), + "action.started": providerHandler( + "action.started", + bridge.handleActionStarted.bind(bridge), + ), + "model.call.completed": providerHandler( + "model.call.completed", + bridge.handleModelTerminal.bind(bridge), + ), + "model.call.failed": providerHandler( + "model.call.failed", + bridge.handleModelTerminal.bind(bridge), + ), + "model.call.started": providerHandler( + "model.call.started", + bridge.handleModelStarted.bind(bridge), + ), + "step.attempt.completed": providerHandler( + "step.attempt.completed", + (event) => bridge.handleStepAttemptTerminal(event), + ), + "step.attempt.failed": providerHandler("step.attempt.failed", (event) => bridge.handleStepAttemptTerminal(event), - "step.attempt.failed": (event) => bridge.handleStepAttemptTerminal(event), - "turn.cancelled": (event, context) => - bridge.handleTurnTerminal(event, context), - "turn.completed": (event, context) => - bridge.handleTurnTerminal(event, context), - "turn.failed": (event, context) => - bridge.handleTurnTerminal(event, context), - "turn.started": (event, context) => - bridge.handleTurnStarted(event, context), + ), + "turn.cancelled": providerHandler( + "turn.cancelled", + bridge.handleTurnTerminal.bind(bridge), + ), + "turn.completed": providerHandler( + "turn.completed", + bridge.handleTurnTerminal.bind(bridge), + ), + "turn.failed": providerHandler( + "turn.failed", + bridge.handleTurnTerminal.bind(bridge), + ), + "turn.started": providerHandler( + "turn.started", + bridge.handleTurnStarted.bind(bridge), + ), }, flush, setup: options.setup, @@ -536,17 +578,28 @@ function modelOutput( type: "function"; }[] = []; for (const part of content) { - if (part.type === "text") { - text += part.text; - } else if (part.type === "reasoning" && part.text.trim().length > 0) { - reasoning.push({ content: part.text }); - } else if (part.type === "tool-call") { + if (!isObject(part)) { + continue; + } + if (part["type"] === "text" && typeof part["text"] === "string") { + text += part["text"]; + } else if ( + part["type"] === "reasoning" && + typeof part["text"] === "string" && + part["text"].trim().length > 0 + ) { + reasoning.push({ content: part["text"] }); + } else if ( + part["type"] === "tool-call" && + typeof part["toolName"] === "string" && + typeof part["callId"] === "string" + ) { toolCalls.push({ function: { - arguments: safeJsonStringify(part.input), - name: part.toolName, + arguments: safeJsonStringify(part["input"]), + name: part["toolName"], }, - id: part.callId, + id: part["callId"], type: "function", }); } @@ -566,13 +619,13 @@ function modelOutput( } function usageMetrics(usage: EveProviderUsage): Record { - const promptTokens = nonNegativeNumber(usage.inputTokens); - const completionTokens = nonNegativeNumber(usage.outputTokens); + const promptTokens = nonNegativeNumber(usage?.inputTokens); + const completionTokens = nonNegativeNumber(usage?.outputTokens); const cachedTokens = nonNegativeNumber( - usage.inputTokenDetails?.cacheReadTokens, + usage?.inputTokenDetails?.cacheReadTokens, ); const cacheCreationTokens = nonNegativeNumber( - usage.inputTokenDetails?.cacheWriteTokens, + usage?.inputTokenDetails?.cacheWriteTokens, ); return { ...(promptTokens !== undefined ? { prompt_tokens: promptTokens } : {}), @@ -632,3 +685,144 @@ function actionIdempotencyKey( ): string { return `action:${sessionId}:${turnId}:${callId}`; } + +function providerHandler( + type: TType, + handler: ( + event: EveProviderEventForType, + context: EveProviderContext, + ) => void | Promise, +): (event: unknown, context: unknown) => Promise { + return async (event, context) => { + try { + if (!isEveProviderEvent(event, type) || !isEveProviderContext(context)) { + debugLogger.warn(`Ignoring malformed Eve provider ${type} event`); + return; + } + await handler(event, context); + } catch (error) { + debugLogger.warn(`Error in Eve provider ${type}:`, error); + } + }; +} + +function isEveProviderEvent( + event: unknown, + type: TType, +): event is EveProviderEventForType { + if ( + !isObject(event) || + event["type"] !== type || + typeof event["idempotencyKey"] !== "string" || + event["idempotencyKey"].length === 0 + ) { + return false; + } + if (type === "turn.started") { + const parentLineage = event["parentLineage"]; + return ( + typeof event["rootSessionId"] === "string" && + typeof event["sessionId"] === "string" && + typeof event["turnId"] === "string" && + typeof event["sequence"] === "number" && + Number.isFinite(event["sequence"]) && + (parentLineage === undefined || isEveProviderParentLineage(parentLineage)) + ); + } + if ( + type === "turn.cancelled" || + type === "turn.completed" || + type === "turn.failed" + ) { + return ( + typeof event["sessionId"] === "string" && + typeof event["turnId"] === "string" + ); + } + if (!isEveProviderScope(event["scope"])) { + return false; + } + if (type === "model.call.started") { + const model = event["model"]; + return ( + isObject(model) && + typeof model["modelId"] === "string" && + typeof model["provider"] === "string" + ); + } + if (type === "model.call.completed") { + return ( + typeof event["finishReason"] === "string" && + isObject(event["usage"]) && + (event["content"] === undefined || Array.isArray(event["content"])) + ); + } + if (type === "model.call.failed") { + return true; + } + if (type === "action.started") { + return ( + typeof event["callId"] === "string" && + typeof event["name"] === "string" && + (event["kind"] === "load-skill" || + event["kind"] === "remote-agent-call" || + event["kind"] === "subagent-call" || + event["kind"] === "tool-call") + ); + } + if (type === "action.completed") { + const output = event["output"]; + return ( + event["outcome"] === "completed" && + isObject(output) && + (output["type"] === "result" || output["type"] === "error") + ); + } + if (type === "action.failed") { + return ( + event["outcome"] === "abandoned" || + event["outcome"] === "cancelled" || + event["outcome"] === "failed" || + event["outcome"] === "rejected" + ); + } + return type === "step.attempt.completed" || type === "step.attempt.failed"; +} + +function isEveProviderContext(value: unknown): value is EveProviderContext { + if (!isObject(value) || !isObject(value["state"])) { + return false; + } + return ( + typeof value["state"]["get"] === "function" && + typeof value["state"]["set"] === "function" + ); +} + +function isEveProviderScope( + value: unknown, +): value is EveInstrumentationAttemptScope { + return ( + isObject(value) && + typeof value["attemptId"] === "string" && + typeof value["attemptIndex"] === "number" && + Number.isFinite(value["attemptIndex"]) && + typeof value["sessionId"] === "string" && + typeof value["stepIndex"] === "number" && + Number.isFinite(value["stepIndex"]) && + typeof value["turnId"] === "string" && + (value["rootSessionId"] === undefined || + typeof value["rootSessionId"] === "string") + ); +} + +function isEveProviderParentLineage( + value: unknown, +): value is EveProviderTurnStartedEvent["parentLineage"] { + return ( + isObject(value) && + typeof value["callId"] === "string" && + typeof value["sessionId"] === "string" && + typeof value["turnId"] === "string" + ); +} diff --git a/js/src/vendor-sdk-types/eve.ts b/js/src/vendor-sdk-types/eve.ts index b8d535a88..776a45798 100644 --- a/js/src/vendor-sdk-types/eve.ts +++ b/js/src/vendor-sdk-types/eve.ts @@ -291,21 +291,6 @@ export type EveHandleMessageStreamEvent = readonly type: "session.completed"; }; -export interface EveHookDefinition { - readonly events?: { - readonly "*"?: ( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise; - readonly [eventType: string]: - | (( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise) - | undefined; - }; -} - export interface EveInstrumentationSetupContext { readonly agentName: string; } @@ -426,19 +411,6 @@ type EveToolApprovalResponse = { readonly type: "tool-approval-response"; }; -export type EveModelMessageContentPart = - | EveTextPart - | EveImagePart - | EveFilePart - | EveReasoningPart - | EveReasoningFilePart - | EveCustomPart - | EveToolCallPart - | EveToolResultPart - | EveToolApprovalRequest - | EveToolApprovalResponse - | EveToolResultContentPart; - export type EveSystemModelMessage = { readonly content: string; readonly role: "system";