diff --git a/.changeset/nameless-scorer-results.md b/.changeset/nameless-scorer-results.md new file mode 100644 index 000000000..d9f3a475a --- /dev/null +++ b/.changeset/nameless-scorer-results.md @@ -0,0 +1,5 @@ +--- +"braintrust": minor +--- + +feat: Allow nameless scorer results diff --git a/e2e/scenarios/durable-eval-webhook/scenario.ts b/e2e/scenarios/durable-eval-webhook/scenario.ts index 6a1b54899..53b861efd 100644 --- a/e2e/scenarios/durable-eval-webhook/scenario.ts +++ b/e2e/scenarios/durable-eval-webhook/scenario.ts @@ -68,7 +68,6 @@ async function main() { return items.map((item) => ({ id: item.id, score: { - name: "batch_exact", score: item.output === item.expected ? 1 : 0, metadata: { method: "batch-provider" }, }, @@ -90,7 +89,6 @@ async function main() { scores: [ function exact({ output, expected }) { return { - name: "exact", score: output === expected ? 1 : 0, metadata: { method: "shared-eval-runtime" }, }; diff --git a/js/src/framework.test.ts b/js/src/framework.test.ts index 4c86a8235..cb0940092 100644 --- a/js/src/framework.test.ts +++ b/js/src/framework.test.ts @@ -12,6 +12,8 @@ import { Eval, EvalScorer, runEvaluator, + _internalPrepareEvaluatorScore, + type OneOrMoreScores, } from "./framework"; import { _exportsForTestingOnly, @@ -2180,6 +2182,101 @@ test("scorer-only evaluator populates scores field", async () => { expect(result.results[0].classifications).toBeUndefined(); }); +test("single score objects use the same names as numeric returns", async () => { + const result = await Eval( + "test-nameless-scores", + { + data: [{ input: "hello" }], + task: (input) => input, + scores: [ + function accuracy() { + return { score: 0.8 }; + }, + async function relevance() { + return { score: 1, metadata: { reason: "relevant" } }; + }, + () => ({ score: 0 }), + () => ({ name: "explicit", score: 0.5 }), + ], + }, + { noSendLogs: true, returnResults: true }, + ); + + expect(result.results[0].scores).toEqual({ + accuracy: 0.8, + relevance: 1, + scorer_2: 0, + explicit: 0.5, + }); +}); + +describe("scorer result normalization", () => { + test.each([0, 0.8, null])( + "defaults the name and preserves fields for %s", + (score) => { + const value = Object.freeze({ score, metadata: { reason: "test" } }); + expect(_internalPrepareEvaluatorScore(value, "accuracy")).toEqual({ + results: [{ ...value, name: "accuracy" }], + output: { score }, + metadata: value.metadata, + scores: { accuracy: score }, + }); + expect(value).not.toHaveProperty("name"); + }, + ); + + test.each(["explicit", ""])("preserves an explicit name %j", (name) => { + expect( + _internalPrepareEvaluatorScore({ name, score: 1 }, "fallback").scores, + ).toEqual({ [name]: 1 }); + }); + + test("keeps named arrays and numeric returns working", () => { + expect(_internalPrepareEvaluatorScore(0.8, "accuracy").scores).toEqual({ + accuracy: 0.8, + }); + expect(_internalPrepareEvaluatorScore(null, "accuracy")).toEqual({ + results: null, + }); + expect( + _internalPrepareEvaluatorScore( + [ + { name: "accuracy", score: 0.8 }, + { name: "relevance", score: 1 }, + ], + "fallback", + ).scores, + ).toEqual({ accuracy: 0.8, relevance: 1 }); + }); + + test("requires names in arrays at typecheck and runtime", () => { + // @ts-expect-error A single-element array still requires a named score. + const unnamed: OneOrMoreScores = [{ score: 1 }]; + const mixed: OneOrMoreScores = [ + { name: "accuracy", score: 1 }, + // @ts-expect-error Every entry in a mixed array must have a name. + { score: 0.5 }, + ]; + for (const value of [unnamed, mixed]) { + expect(() => _internalPrepareEvaluatorScore(value, "fallback")).toThrow( + "each score must have a name", + ); + } + }); + + test("rejects duplicate array names", () => { + expect(() => + _internalPrepareEvaluatorScore( + [ + { name: "accuracy", score: 1 }, + { name: "accuracy", score: 0.5 }, + ], + "fallback", + ), + ).toThrow("Duplicate score name 'accuracy'"); + }); +}); + test("multiple classifiers returning the same name append items correctly", async () => { const result = await Eval( "test-classifier-append", diff --git a/js/src/framework.ts b/js/src/framework.ts index 42d4e09ff..4b4b58386 100644 --- a/js/src/framework.ts +++ b/js/src/framework.ts @@ -7,6 +7,7 @@ import { SpanTypeAttribute, spanObjectTypeV3ToTypedString, } from "../util/index"; +import type { SingleScore } from "../util/score"; import { type GitMetadataSettingsType as GitMetadataSettings, ObjectReference as ObjectReferenceSchema, @@ -177,7 +178,7 @@ export type EvalScorerArgs< trace?: Trace; }; -export type OneOrMoreScores = Score | number | null | Array; +export type OneOrMoreScores = SingleScore | number | null | Array; export type EvalScorer< Input, @@ -1045,19 +1046,29 @@ export function _internalPrepareEvaluatorScore( } { if (scoreValue === null) return { results: null }; if (Array.isArray(scoreValue)) { + const names = new Set(); for (const score of scoreValue) { if (!(typeof score === "object" && !isEmpty(score))) { throw new Error( `When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(score)}`, ); } + if (typeof score.name !== "string") { + throw new Error( + `When returning an array of scores, each score must have a name. Got: ${JSON.stringify(score)}`, + ); + } + if (names.has(score.name)) { + throw new Error(`Duplicate score name '${score.name}' in score array`); + } + names.add(score.name); } } let results: Score[]; if (Array.isArray(scoreValue)) { results = scoreValue; } else if (typeof scoreValue === "object" && !isEmpty(scoreValue)) { - results = [scoreValue]; + results = [{ ...scoreValue, name: scoreValue.name ?? name }]; } else { results = [{ name, score: scoreValue }]; } diff --git a/js/src/wrappers/shared/scorers.test.ts b/js/src/wrappers/shared/scorers.test.ts new file mode 100644 index 000000000..d38cfd405 --- /dev/null +++ b/js/src/wrappers/shared/scorers.test.ts @@ -0,0 +1,25 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { NOOP_SPAN } from "../../logger"; +import { runScorers } from "./scorers"; +import type { ScorerFunction } from "./types"; + +afterEach(() => vi.restoreAllMocks()); + +test.each([ + () => ({ score: 0.8, metadata: { reason: "test" } }), + async () => ({ score: 0.8, metadata: { reason: "test" } }), +])("test runner scorers accept nameless objects", async (scorer) => { + const log = vi.spyOn(NOOP_SPAN, "log"); + await runScorers({ + scorers: [scorer], + input: "hello", + output: "hello", + expected: "hello", + metadata: undefined, + span: NOOP_SPAN, + }); + expect(log).toHaveBeenCalledWith({ + scores: { score: 0.8 }, + metadata: { reason: "test" }, + }); +}); diff --git a/js/src/wrappers/shared/scorers.ts b/js/src/wrappers/shared/scorers.ts index 43080027f..b3c11ee45 100644 --- a/js/src/wrappers/shared/scorers.ts +++ b/js/src/wrappers/shared/scorers.ts @@ -1,6 +1,7 @@ import type { Span } from "../../logger"; import type { Score } from "../../../util/score"; import type { ScorerFunction } from "./types"; +import type { OneOrMoreScores } from "../../framework"; export async function runScorers(args: { scorers: ScorerFunction[]; @@ -60,7 +61,7 @@ function isScore(val: object): val is Score { return "name" in val && "score" in val; } -function normalizeScores(result: unknown): Score[] { +function normalizeScores(result: OneOrMoreScores): Score[] { if (result === null || result === undefined) { return []; } @@ -76,8 +77,8 @@ function normalizeScores(result: unknown): Score[] { ); } - if (typeof result === "object" && result !== null && isScore(result)) { - return [result]; + if (typeof result === "object" && result !== null && "score" in result) { + return [{ ...result, name: result.name ?? "score" }]; } return []; diff --git a/js/src/wrappers/shared/types.ts b/js/src/wrappers/shared/types.ts index ba87847a2..afc6f420e 100644 --- a/js/src/wrappers/shared/types.ts +++ b/js/src/wrappers/shared/types.ts @@ -1,4 +1,4 @@ -import type { Score } from "../../../util/score"; +import type { OneOrMoreScores } from "../../framework"; // Scorer function type export type ScorerFunction = (args: { @@ -6,7 +6,7 @@ export type ScorerFunction = (args: { expected?: unknown; input?: unknown; metadata?: Record; -}) => Score | Promise | number | null | Array; +}) => OneOrMoreScores | Promise; // Progress event types for real-time test reporting export type ProgressEvent = diff --git a/js/util/score.ts b/js/util/score.ts index 1c8a4754e..86dc03b7e 100644 --- a/js/util/score.ts +++ b/js/util/score.ts @@ -47,6 +47,9 @@ export interface Score { error?: unknown; } +// A single result can use the scorer's name; array entries must stay named. +export type SingleScore = Omit & { name?: string }; + export type ScorerArgs = { output: Output; expected?: Output;