eval-question.ts

import type { AdaptifEvent } from "@adaptif/core";

/** Situations answer "yes"/"no"; intents answer an option id. */
export type EvalAnswer = string;

export interface RuleInput<State> {
  state: State;
  events: readonly AdaptifEvent[];
  signalNames: readonly string[];
}

interface CaseDefinition<State, Expected> {
  id: string;
  /** Cases sharing a pair produce the same signals but have different labels. */
  pair?: string;
  state: State;
  timeline: readonly AdaptifEvent[];
  /** When the question is asked, as `"m:ss"` from the start of the session. */
  askedAt: string;
  expected: Expected;
  /** Why this label, written before the model ever sees the case. */
  why: string;
}

export interface EvalCase {
  id: string;
  pair: string | null;
  state: Record<string, unknown>;
  timeline: readonly AdaptifEvent[];
  askedAt: string;
  expected: EvalAnswer;
  why: string;
}

export interface EvalQuestion {
  id: string;
  kind: "situation" | "intent";
  text: string;
  options: Readonly<Record<string, string>> | null;
  /** A question rules should handle as well as the model; a big model win here suggests biased cases. */
  isControl: boolean;
  answerByRule(input: RuleInput<Record<string, unknown>>): EvalAnswer;
  cases: readonly EvalCase[];
}

export function toSituationAnswer(isTrue: boolean): EvalAnswer {
  return isTrue ? "yes" : "no";
}

export function defineSituation<State extends Record<string, unknown>>(definition: {
  id: string;
  text: string;
  isControl?: boolean;
  rule: (input: RuleInput<State>) => boolean;
  cases: ReadonlyArray<CaseDefinition<State, boolean>>;
}): EvalQuestion {
  return {
    id: definition.id,
    kind: "situation",
    text: definition.text,
    options: null,
    isControl: definition.isControl ?? false,
    answerByRule: (input) => toSituationAnswer(definition.rule(input as RuleInput<State>)),
    cases: definition.cases.map((evalCase) => ({
      ...normalizeCase(evalCase),
      expected: toSituationAnswer(evalCase.expected),
    })),
  };
}

export function defineIntent<
  State extends Record<string, unknown>,
  const Options extends Record<string, string>,
>(definition: {
  id: string;
  text: string;
  options: Options;
  rule: (input: RuleInput<State>) => keyof Options & string;
  cases: ReadonlyArray<CaseDefinition<State, keyof Options & string>>;
}): EvalQuestion {
  return {
    id: definition.id,
    kind: "intent",
    text: definition.text,
    options: definition.options,
    isControl: false,
    answerByRule: (input) => definition.rule(input as RuleInput<State>),
    cases: definition.cases.map((evalCase) => ({
      ...normalizeCase(evalCase),
      expected: evalCase.expected,
    })),
  };
}

function normalizeCase<State extends Record<string, unknown>>(
  evalCase: CaseDefinition<State, unknown>,
): Omit<EvalCase, "expected"> {
  return {
    id: evalCase.id,
    pair: evalCase.pair ?? null,
    state: evalCase.state,
    timeline: evalCase.timeline,
    askedAt: evalCase.askedAt,
    why: evalCase.why,
  };
}

export function hasEvent(
  events: readonly AdaptifEvent[],
  matches: (event: AdaptifEvent) => boolean,
): boolean {
  return events.some(matches);
}

export function countCustomEvents(events: readonly AdaptifEvent[], name: string): number {
  return events.filter((event) => event.kind === "custom" && event.name === name).length;
}

export function containsAnyKeyword(text: string, keywords: readonly string[]): boolean {
  const lowered = text.toLowerCase();
  return keywords.some((keyword) => lowered.includes(keyword));
}