feat: Allow nameless scorer results

.changeset/nameless-scorer-results.md

@@ -0,0 +1,5 @@
+---
+"braintrust": minor
+---
+
+feat: Allow nameless scorer results

e2e/scenarios/durable-eval-webhook/scenario.ts

@@ -68,7 +68,6 @@ async function main() {
       return items.map((item) => ({
         id: item.id,
         score: {
-          name: "batch_exact",
           score: item.output === item.expected ? 1 : 0,
           metadata: { method: "batch-provider" },
         },
@@ -90,7 +89,6 @@ async function main() {
       scores: [
         function exact({ output, expected }) {
           return {
-            name: "exact",
             score: output === expected ? 1 : 0,
             metadata: { method: "shared-eval-runtime" },
           };

js/src/framework.test.ts

@@ -12,6 +12,8 @@ import {
   Eval,
   EvalScorer,
   runEvaluator,
+  _internalPrepareEvaluatorScore,
+  type OneOrMoreScores,
 } from "./framework";
 import {
   _exportsForTestingOnly,
@@ -2180,6 +2182,101 @@ test("scorer-only evaluator populates scores field", async () => {
   expect(result.results[0].classifications).toBeUndefined();
 });
 
+test("single score objects use the same names as numeric returns", async () => {
+  const result = await Eval(
+    "test-nameless-scores",
+    {
+      data: [{ input: "hello" }],
+      task: (input) => input,
+      scores: [
+        function accuracy() {
+          return { score: 0.8 };
+        },
+        async function relevance() {
+          return { score: 1, metadata: { reason: "relevant" } };
+        },
+        () => ({ score: 0 }),
+        () => ({ name: "explicit", score: 0.5 }),
+      ],
+    },
+    { noSendLogs: true, returnResults: true },
+  );
+
+  expect(result.results[0].scores).toEqual({
+    accuracy: 0.8,
+    relevance: 1,
+    scorer_2: 0,
+    explicit: 0.5,
+  });
+});
+
+describe("scorer result normalization", () => {
+  test.each([0, 0.8, null])(
+    "defaults the name and preserves fields for %s",
+    (score) => {
+      const value = Object.freeze({ score, metadata: { reason: "test" } });
+      expect(_internalPrepareEvaluatorScore(value, "accuracy")).toEqual({
+        results: [{ ...value, name: "accuracy" }],
+        output: { score },
+        metadata: value.metadata,
+        scores: { accuracy: score },
+      });
+      expect(value).not.toHaveProperty("name");
+    },
+  );
+
+  test.each(["explicit", ""])("preserves an explicit name %j", (name) => {
+    expect(
+      _internalPrepareEvaluatorScore({ name, score: 1 }, "fallback").scores,
+    ).toEqual({ [name]: 1 });
+  });
+
+  test("keeps named arrays and numeric returns working", () => {
+    expect(_internalPrepareEvaluatorScore(0.8, "accuracy").scores).toEqual({
+      accuracy: 0.8,
+    });
+    expect(_internalPrepareEvaluatorScore(null, "accuracy")).toEqual({
+      results: null,
+    });
+    expect(
+      _internalPrepareEvaluatorScore(
+        [
+          { name: "accuracy", score: 0.8 },
+          { name: "relevance", score: 1 },
+        ],
+        "fallback",
+      ).scores,
+    ).toEqual({ accuracy: 0.8, relevance: 1 });
+  });
+
+  test("requires names in arrays at typecheck and runtime", () => {
+    // @ts-expect-error A single-element array still requires a named score.
+    const unnamed: OneOrMoreScores = [{ score: 1 }];
+    const mixed: OneOrMoreScores = [
+      { name: "accuracy", score: 1 },
+      // @ts-expect-error Every entry in a mixed array must have a name.
+      { score: 0.5 },
+    ];
+    for (const value of [unnamed, mixed]) {
+      expect(() => _internalPrepareEvaluatorScore(value, "fallback")).toThrow(
+        "each score must have a name",
+      );
+    }
+  });
+
+  test("rejects duplicate array names", () => {
+    expect(() =>
+      _internalPrepareEvaluatorScore(
+        [
+          { name: "accuracy", score: 1 },
+          { name: "accuracy", score: 0.5 },
+        ],
+        "fallback",
+      ),
+    ).toThrow("Duplicate score name 'accuracy'");
+  });
+});
+
 test("multiple classifiers returning the same name append items correctly", async () => {
   const result = await Eval(
     "test-classifier-append",

js/src/framework.ts

@@ -7,6 +7,7 @@ import {
   SpanTypeAttribute,
   spanObjectTypeV3ToTypedString,
 } from "../util/index";
+import type { SingleScore } from "../util/score";
 import {
   type GitMetadataSettingsType as GitMetadataSettings,
   ObjectReference as ObjectReferenceSchema,
@@ -177,7 +178,7 @@ export type EvalScorerArgs<
   trace?: Trace;
 };
 
-export type OneOrMoreScores = Score | number | null | Array<Score>;
+export type OneOrMoreScores = SingleScore | number | null | Array<Score>;
 
 export type EvalScorer<
   Input,
@@ -1045,19 +1046,29 @@ export function _internalPrepareEvaluatorScore(
 } {
   if (scoreValue === null) return { results: null };
   if (Array.isArray(scoreValue)) {
+    const names = new Set<string>();
     for (const score of scoreValue) {
       if (!(typeof score === "object" && !isEmpty(score))) {
         throw new Error(
           `When returning an array of scores, each score must be a non-empty object. Got: ${JSON.stringify(score)}`,
         );
       }
+      if (typeof score.name !== "string") {
+        throw new Error(
+          `When returning an array of scores, each score must have a name. Got: ${JSON.stringify(score)}`,
+        );
+      }
+      if (names.has(score.name)) {
+        throw new Error(`Duplicate score name '${score.name}' in score array`);
+      }
+      names.add(score.name);
     }
   }
   let results: Score[];
   if (Array.isArray(scoreValue)) {
     results = scoreValue;
   } else if (typeof scoreValue === "object" && !isEmpty(scoreValue)) {
-    results = [scoreValue];
+    results = [{ ...scoreValue, name: scoreValue.name ?? name }];
   } else {
     results = [{ name, score: scoreValue }];
   }

js/src/wrappers/shared/scorers.test.ts

@@ -0,0 +1,25 @@
+import { afterEach, expect, test, vi } from "vitest";
+import { NOOP_SPAN } from "../../logger";
+import { runScorers } from "./scorers";
+import type { ScorerFunction } from "./types";
+
+afterEach(() => vi.restoreAllMocks());
+
+test.each<ScorerFunction>([
+  () => ({ score: 0.8, metadata: { reason: "test" } }),
+  async () => ({ score: 0.8, metadata: { reason: "test" } }),
+])("test runner scorers accept nameless objects", async (scorer) => {
+  const log = vi.spyOn(NOOP_SPAN, "log");
+  await runScorers({
+    scorers: [scorer],
+    input: "hello",
+    output: "hello",
+    expected: "hello",
+    metadata: undefined,
+    span: NOOP_SPAN,
+  });
+  expect(log).toHaveBeenCalledWith({
+    scores: { score: 0.8 },
+    metadata: { reason: "test" },
+  });
+});

js/src/wrappers/shared/scorers.ts

@@ -1,6 +1,7 @@
 import type { Span } from "../../logger";
 import type { Score } from "../../../util/score";
 import type { ScorerFunction } from "./types";
+import type { OneOrMoreScores } from "../../framework";
 
 export async function runScorers(args: {
   scorers: ScorerFunction[];
@@ -60,7 +61,7 @@ function isScore(val: object): val is Score {
   return "name" in val && "score" in val;
 }
 
-function normalizeScores(result: unknown): Score[] {
+function normalizeScores(result: OneOrMoreScores): Score[] {
   if (result === null || result === undefined) {
     return [];
   }
@@ -76,8 +77,8 @@ function normalizeScores(result: unknown): Score[] {
     );
   }
 
-  if (typeof result === "object" && result !== null && isScore(result)) {
-    return [result];
+  if (typeof result === "object" && result !== null && "score" in result) {
+    return [{ ...result, name: result.name ?? "score" }];
   }
 
   return [];

js/src/wrappers/shared/types.ts

@@ -1,12 +1,12 @@
-import type { Score } from "../../../util/score";
+import type { OneOrMoreScores } from "../../framework";
 
 // Scorer function type
 export type ScorerFunction<Output = unknown> = (args: {
   output: Output;
   expected?: unknown;
   input?: unknown;
   metadata?: Record<string, unknown>;
-}) => Score | Promise<Score> | number | null | Array<Score>;
+}) => OneOrMoreScores | Promise<OneOrMoreScores>;
 
 // Progress event types for real-time test reporting
 export type ProgressEvent =

js/util/score.ts

@@ -47,6 +47,9 @@ export interface Score {
   error?: unknown;
 }
 
+// A single result can use the scorer's name; array entries must stay named.
+export type SingleScore = Omit<Score, "name"> & { name?: string };
+
 export type ScorerArgs<Output, Extra> = {
   output: Output;
   expected?: Output;