run.ts

/**
 * Model-vs-rule eval: does the real model beat the obvious rule on labeled
 * cases, or does it just mirror the detector signals? See README.md.
 *
 *   pnpm build
 *   pnpm --filter @adaptif/server eval --dry-run
 *   pnpm --filter @adaptif/server eval [--question leaving_reason] [--repeats 3] [--verbose]
 */
import { createHandler } from "@adaptif/server";
import { askEveryCase, buildConfig } from "./ask.ts";
import { parseFlags } from "./flags.ts";
import { failOnUnmatchedPairs, prepareQuestions, selectQuestions } from "./prepare.ts";
import { createProvider, loadEnvFiles } from "./providers.ts";
import {
  printCaseTables,
  printDryRun,
  printReadingGuide,
  printSummary,
  writeResults,
} from "./report.ts";
import { summarizeQuestion } from "./scoring.ts";

const ProviderTimeoutMs = 15_000;

const flags = parseFlags(process.argv.slice(2));
const questions = prepareQuestions(selectQuestions(flags.questionIds));
failOnUnmatchedPairs(questions);

if (flags.isDryRun) {
  printDryRun(questions);
} else {
  loadEnvFiles();
  const provider = createProvider(flags);
  const handler = createHandler(buildConfig(questions), {
    provider,
    providerTimeoutMs: ProviderTimeoutMs,
  });
  const callCount = questions.reduce((total, { cases }) => total + cases.length, 0) * flags.repeats;
  console.log(
    `Asking ${provider.name} ${callCount} times (${flags.repeats} per case, ${flags.concurrency} at a time)…`,
  );

  const outcomesByQuestion = await askEveryCase(handler, questions, flags);
  const summaries = questions.map(({ question }, index) =>
    summarizeQuestion(question, outcomesByQuestion[index]!),
  );

  if (flags.isVerbose) printCaseTables(questions, outcomesByQuestion);
  printSummary(summaries);
  printReadingGuide();
  writeResults({ flags, providerName: provider.name, questions, outcomesByQuestion, summaries });
}