report.ts

import { mkdirSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import type { EvalFlags } from "./flags.ts";
import type { PreparedQuestion } from "./prepare.ts";
import type { CaseOutcome, QuestionSummary } from "./scoring.ts";

export function printDryRun(questions: readonly PreparedQuestion[]): void {
  for (const { question, cases } of questions) {
    const kindLabel = `${question.kind}${question.isControl ? ", control" : ""}`;
    console.log(`\n━━ ${question.id} (${kindLabel}): ${question.text}`);
    if (question.options) console.log(`   options: ${Object.keys(question.options).join(", ")}`);
    for (const { evalCase, timeline, ruleAnswer } of cases) {
      const signals = timeline.signalNames.join(", ") || "none";
      console.log(`\n── ${evalCase.id}${evalCase.pair ? `  [pair: ${evalCase.pair}]` : ""}`);
      console.log(
        `   expected ${evalCase.expected} · rule ${markAnswer(ruleAnswer, evalCase.expected)} · signals ${signals}`,
      );
      console.log(`   why: ${evalCase.why}`);
      console.log(`   state: ${JSON.stringify(evalCase.state)}`);
      console.log(`   timeline (${timeline.compiled.tokens} tokens): ${timeline.compiled.text}`);
    }
  }
  console.log("\nAll pairs are signal-matched. Rule-only results:");
  console.table(
    questions.map(({ question, cases }) => {
      const ruleCorrect = cases.filter(
        ({ evalCase, ruleAnswer }) => ruleAnswer === evalCase.expected,
      ).length;
      return {
        question: question.id,
        cases: cases.length,
        "rule correct": `${ruleCorrect}/${cases.length}`,
      };
    }),
  );
}

export function printCaseTables(
  questions: readonly PreparedQuestion[],
  outcomesByQuestion: readonly CaseOutcome[][],
): void {
  questions.forEach(({ question }, index) => {
    console.log(`\n${question.id}: ${question.text}`);
    console.table(
      outcomesByQuestion[index]!.map((outcome) => ({
        case: outcome.caseId,
        pair: outcome.pair ?? "",
        expected: outcome.expected,
        rule: markAnswer(outcome.ruleAnswer, outcome.expected),
        model: markAnswer(outcome.modelMajority, outcome.expected),
        "model answers": outcome.modelAnswers.map((modelAnswer) => modelAnswer.shown).join(" | "),
      })),
    );
  });
}

export function printSummary(summaries: readonly QuestionSummary[]): void {
  console.log("\nSummary (model = majority of repeats)");
  console.table(
    summaries.map((summary) => ({
      question: summary.isControl ? `${summary.questionId} (control)` : summary.questionId,
      model: `${summary.modelCorrect}/${summary.caseCount}`,
      rule: `${summary.ruleCorrect}/${summary.caseCount}`,
      "only model ✓": summary.modelOnlyCorrect,
      "only rule ✓": summary.ruleOnlyCorrect,
      "sign p": summary.signTestPValue.toFixed(3),
      "pairs model": `${summary.modelPairsCorrect}/${summary.pairCount}`,
      "pairs rule": `${summary.rulePairsCorrect}/${summary.pairCount}`,
      "pairs any-signal": formatOptionalCount(summary.signalPresentPairsCorrect, summary.pairCount),
      "mirrors signals": formatOptionalCount(summary.mirrorAgreements, summary.caseCount),
      "cases w/ flips": summary.casesWithFlips,
      errors: summary.modelErrors,
    })),
  );
}

export function printReadingGuide(): void {
  console.log(`How to read this:
  • Pairs share the same detector signals but have different answers, so anything that only reads
    signals gets at most half of each pair. "pairs model" well above "pairs rule" is the main evidence.
  • "only model ✓" vs "only rule ✓" counts the cases where exactly one of them was right; with ~10
    cases per question "sign p" will rarely be small, so treat this as go/no-go, not proof.
  • "mirrors signals" near the case count means the model answers "yes" exactly when a signal fired.
  • The control (frustrated) should be close to a tie. A big model win there suggests the cases
    were written in the model's favor.`);
}

export function writeResults(results: {
  flags: EvalFlags;
  providerName: string;
  questions: readonly PreparedQuestion[];
  outcomesByQuestion: readonly CaseOutcome[][];
  summaries: readonly QuestionSummary[];
}): void {
  const outputDirectory = join(process.cwd(), ".adaptif", "eval");
  mkdirSync(outputDirectory, { recursive: true });
  const outputPath = join(outputDirectory, `${new Date().toISOString().replaceAll(":", "-")}.json`);
  const record = {
    ranAt: new Date().toISOString(),
    provider: results.providerName,
    model: results.flags.model,
    repeats: results.flags.repeats,
    summaries: results.summaries,
    questions: results.questions.map(({ question, cases }, index) => ({
      id: question.id,
      text: question.text,
      cases: cases.map((preparedCase, caseIndex) => ({
        ...results.outcomesByQuestion[index]![caseIndex],
        state: preparedCase.evalCase.state,
        timeline: preparedCase.timeline.compiled.structured,
        why: preparedCase.evalCase.why,
      })),
    })),
  };
  writeFileSync(outputPath, `${JSON.stringify(record, null, 2)}\n`);
  console.log(`\nFull results: ${outputPath}`);
}

function markAnswer(answer: string | null, expected: string): string {
  return `${answer ?? "—"} ${answer === expected ? "✓" : "✗"}`;
}

function formatOptionalCount(count: number | null, total: number): string {
  return count === null ? "—" : `${count}/${total}`;
}