From 230aaec7c5dc924ade6c9a7ca8f343a7d562a7d9 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 19:45:23 +0200 Subject: [PATCH 1/9] test: cover tutor question quality regressions --- .../question-quality-regressions.test.ts | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 tests/server/services/tutor/question-quality-regressions.test.ts diff --git a/tests/server/services/tutor/question-quality-regressions.test.ts b/tests/server/services/tutor/question-quality-regressions.test.ts new file mode 100644 index 00000000..42ffa092 --- /dev/null +++ b/tests/server/services/tutor/question-quality-regressions.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; +import { buildTutorContext, buildUserPrompt, TutorService } from "../../../../server/services/tutor/tutor-service"; +import type { LLMProvider } from "../../../../server/services/tutor/llm-provider"; +import type { TutorPlanningExtension } from "../../../../server/services/tutor/tutor-planning"; + +const revision = "a".repeat(40); + +function providerReturning(question: string): LLMProvider { + return { + listModels: vi.fn().mockResolvedValue(["pilot-model"]), + generateLearningQuestion: vi.fn().mockResolvedValue({ + model: "pilot-model", + result: { + answerRating: 5, + question, + }, + }), + }; +} + +function transitionPlanning(): TutorPlanningExtension { + return { + planInitial: vi.fn().mockResolvedValue(null), + planFollowup: vi.fn().mockResolvedValue({ + kind: "transition", + contentRevision: revision, + learningPhase: "DEEPEN", + activeTopicId: "pwm-output", + masteredTopicIds: ["pwm-output"], + strategyId: "built-in-default", + strategySource: "built-in", + }), + }; +} + +describe("Tutor question quality regressions", () => { + it("keeps the distinct question after applying transition metadata", async () => { + const repeatedQuestion = "Welchen Wert verwendet der Sketch an der betrachteten Integer-Variablen?"; + const result = await new TutorService( + providerReturning(repeatedQuestion), + transitionPlanning(), + ).generateDialogResponse( + `const int ledPin = 9; +void setup() { pinMode(ledPin, OUTPUT); } +void loop() { analogWrite(ledPin, 128); }`, + [], + repeatedQuestion, + "Der Wert ist 128.", + "key", + undefined, + 30, + ); + + expect(result.result).toMatchObject({ + answerRating: 5, + learningPhase: "DEEPEN", + strategyId: "built-in-default", + }); + expect(result.result.question).not.toBe(repeatedQuestion); + expect(result.result.question).toBe("Welches andere im Sketch sichtbare Konzept möchtest du als Nächstes mit diesem verstandenen Teilkonzept verknüpfen?"); + }); + + it("guides the provider to identify concrete variables and local code context", () => { + const code = `const int ledPin = 9; +void setup() { pinMode(ledPin, OUTPUT); } +void loop() { + for (int brightness = 0; brightness <= 255; brightness += 5) { + analogWrite(ledPin, brightness); + } +}`; + const prompt = buildUserPrompt(code, buildTutorContext(code)); + + expect(prompt).toContain("konkreten Bezeichner"); + expect(prompt).toContain("lokale Code-Stelle"); + expect(prompt).toContain("ledPin"); + expect(prompt).toContain("brightness"); + }); +}); From deebcdf2d2c8f349ee3e61fe9c7eb554a1715dd8 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 19:45:35 +0200 Subject: [PATCH 2/9] fix: preserve distinct tutor questions --- server/services/tutor/tutor-service.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/server/services/tutor/tutor-service.ts b/server/services/tutor/tutor-service.ts index 339e63da..8490e5b5 100644 --- a/server/services/tutor/tutor-service.ts +++ b/server/services/tutor/tutor-service.ts @@ -45,6 +45,7 @@ const UNSAFE_MERMAID_PATTERNS = [ ]; const TUTOR_DIFFICULTY_GUIDANCE = "Kalibriere die Frage kognitiv: 1–10 = elementare Wiedererkennung oder direkter Fakt, 11–30 = einfache Anwendung, 31–50 = Verständnis und Zusammenhang, 51–70 = Transfer oder Analyse, 71–90 = anspruchsvolle Herleitung mehrerer Konzepte, 91–100 = sehr anspruchsvolle Synthese. Die Frage muss zum aktuellen Wert passen; Difficulty ist kein Prüfungsniveau."; +const TUTOR_CONCRETE_REFERENCE_GUIDANCE = "Wenn eine Frage auf ein konkretes Sketch-Element zielt und mehrere Bezeichner oder Stellen infrage kommen, nenne den konkreten Bezeichner und genügend lokale Code-Stelle (zum Beispiel Deklaration, Schleife oder Aufruf), damit die Frage ohne Raten verständlich ist. Vermeide bei möglicher Mehrdeutigkeit unklare Formulierungen wie ‚die Integer-Variable‘, ‚dieser Wert‘ oder ‚dort‘. Frage nur nach durch den aktuellen Sketch belegten Fakten."; type TutorDialogArguments = [ code: string, @@ -83,6 +84,7 @@ export const TUTOR_SYSTEM_PROMPT = [ "Erfinde keine Hardware, Pins, Variablen, Werte oder Programmstrukturen.", "Gib keine vollständige Lösung, keinen vollständigen Ersatzcode und keine Codeänderung aus.", "Die Frage soll die eigene Analyse des Studierenden fördern und nicht die Denkarbeit ersetzen.", + TUTOR_CONCRETE_REFERENCE_GUIDANCE, "Wenn eine Nutzerantwort vorliegt, gib bei normalen inhaltlichen Antworten kurzes Feedback, responseStyle normal, eine answerRating von 1 bis 5 und danach genau eine Folgefrage.", "Für offensichtlich unsinnige, absurde oder vollständig themenfremde Antworten verwende ausschließlich den begrenzten philosophischen Fallback: responseStyle philosophical, keine answerRating, kurzer nicht-spöttischer Reflexionshinweis und genau eine Frage zurück zum aktuellen Sketch.", "Normale fachlich falsche Antworten bleiben responseStyle normal und werden bewertet.", @@ -131,6 +133,7 @@ function buildUserPrompt( "Erzeuge eine einzige Lernfrage zum folgenden aktuellen Arduino-Sketch.", `Relative didaktische Schwierigkeit für diese Frage: ${difficulty}/100 (1 = sehr leicht, 100 = sehr schwer; kein Prüfungsniveau).`, TUTOR_DIFFICULTY_GUIDANCE, + TUTOR_CONCRETE_REFERENCE_GUIDANCE, buildTutorStrategyGuidance(strategy), ...(objectivesGuidance ? [objectivesGuidance] : []), "Wenn ein Sachverhalt nicht statisch belegt ist, formuliere höchstens eine offene Reflexionsfrage statt einer Tatsachenbehauptung.", @@ -166,6 +169,7 @@ function buildDialogPrompt( "Führe den sokratischen Lerndialog zum folgenden aktuellen Arduino-Sketch fort.", `Erzeuge die Folgefrage mit relativer didaktischer Schwierigkeit ${difficulty}/100 (1 = sehr leicht, 100 = sehr schwer; kein Prüfungsniveau).`, TUTOR_DIFFICULTY_GUIDANCE, + TUTOR_CONCRETE_REFERENCE_GUIDANCE, buildTutorStrategyGuidance(strategy), ...(objectivesGuidance ? [objectivesGuidance] : []), "Bewerte die Antwort mit answerRating 1 bis 5 gemäß Verständnisrubrik, höchstens kurz, und stelle danach genau eine neue, weiterführende Frage.", @@ -630,7 +634,7 @@ export class TutorService { : validatedResult; if (validatedResult.responseStyle === "normal" && this.planningExtension) { const nextPlan = await this.planningExtension.planFollowup({ code, history: parsedHistory, currentQuestion: question, rating: validatedResult.answerRating!, difficulty, courseContent: transaction.courseContent }); - if (nextPlan) distinctResult = applyPlanningOutcome(validatedResult, nextPlan); + if (nextPlan) distinctResult = applyPlanningOutcome(distinctResult, nextPlan); } if (!distinctResult.strategyId) distinctResult = applyStrategyMetadata(distinctResult, strategy); transaction.commit(); From 1f45fdafb9a527c5d07b144b5f2b383e79fcee24 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 20:34:16 +0200 Subject: [PATCH 3/9] docs: define deterministic tutor quality foundation --- docs/ARCHITECTURE.md | 4 +- .../plans/2026-09-27-tutor-quality-stage-1.md | 95 +++++++++++++ package.json | 1 + ssot/ssot_function_definition_TutorQuality.md | 132 ++++++++++++++++++ 4 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 docs/superpowers/plans/2026-09-27-tutor-quality-stage-1.md create mode 100644 ssot/ssot_function_definition_TutorQuality.md diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index be502d47..c22d6bee 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -167,7 +167,9 @@ fixtures and authoring examples only. The normative details are in [ssot_function_definition_CourseContent.md](../ssot/ssot_function_definition_CourseContent.md), [adr/0006-unified-course-content-and-tutor-strategy.md](adr/0006-unified-course-content-and-tutor-strategy.md), [adr/0007-mastery-driven-tutor-progression.md](adr/0007-mastery-driven-tutor-progression.md), -and [ssot_function_definition_LearningQuestions.md](../ssot/ssot_function_definition_LearningQuestions.md). +[ssot_function_definition_LearningQuestions.md](../ssot/ssot_function_definition_LearningQuestions.md), +and the deterministic quality gates in +[ssot_function_definition_TutorQuality.md](../ssot/ssot_function_definition_TutorQuality.md). ### Dynamic Course Content selection diff --git a/docs/superpowers/plans/2026-09-27-tutor-quality-stage-1.md b/docs/superpowers/plans/2026-09-27-tutor-quality-stage-1.md new file mode 100644 index 00000000..63c8a030 --- /dev/null +++ b/docs/superpowers/plans/2026-09-27-tutor-quality-stage-1.md @@ -0,0 +1,95 @@ +# Tutor Quality Stage 1 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Establish deterministic Tutor quality hard gates, a reusable scenario runner, the PWM regression case, and compatible Course Content validation in UnoSim and UnoSim-Examples. + +**Architecture:** Keep runtime ownership in `TutorService`, `CurriculumTutorAdapter`, and existing Course Content schemas. Add one pure authoring validator that reuses production fact matching, one filesystem CLI adapter, and test-only scenario orchestration around the real Tutor service with only the provider faked. Pin the Examples CI consumer to the resulting UnoSim commit. + +**Tech Stack:** TypeScript, Vitest, Zod/YAML, GitHub Actions, existing Course Content loader and Tutor planner. + +--- + +Specification: `ssot/ssot_function_definition_TutorQuality.md` + +### Task 1: Establish the normative quality contract + +**Files:** +- Create: `ssot/ssot_function_definition_TutorQuality.md` +- Modify: `docs/ARCHITECTURE.md` +- Test: `scripts/check-docs.mjs` + +1. Add the deterministic learning-support proxy, trust boundaries, runtime/content invariants, scenario contract, CI contract, and explicit Stage 2 exclusions. +2. Link the SSOT from the architecture documentation. +3. Run `npm run check:docs` and commit the documentation. + +### Task 2: Build the reusable scenario runner and adversarial service cases + +**Files:** +- Create: `tests/server/services/tutor/support/tutor-quality-scenario-runner.ts` +- Create: `tests/server/services/tutor/tutor-quality-scenarios.test.ts` +- Modify: `server/services/tutor/tutor-service.ts` + +1. Write failing scenarios for provider errors before commit, invalid output, complete solutions, multiple primary questions, contradictory provider metadata, exact/near repetition, transition/blocked repair retention, and TutorPlan question authority. +2. Run the focused tests and confirm each new invariant fails for the intended reason. +3. Implement the minimal validation/metadata-boundary changes in `TutorService`. +4. Run the focused tests and existing Tutor service/planning tests; commit. + +### Task 3: Preserve TQ-REG-001 with real PWM inputs + +**Files:** +- Create: `tests/fixtures/tutor-quality/TQ-REG-001-pwm.ino` +- Create: `tests/server/services/tutor/tq-reg-001-pwm.test.ts` + +1. Write a failing scenario using the real PWM sketch and the narrowed Topic activation from the reviewed Course Content. +2. Assert that `variables-and-serial` is not selected, a strong-answer repetition is repaired, and phase/revision/state metadata remains consistent. +3. Make only runner/fixture corrections if needed; production changes require a separate failing invariant test. +4. Run the focused regression test and commit. + +### Task 4: Add deterministic Course Content quality validation + +**Files:** +- Create: `server/services/course-content/tutor-quality-schema.ts` +- Create: `server/services/course-content/tutor-quality-validator.ts` +- Create: `tests/server/services/course-content/tutor-quality-validator.test.ts` +- Create: `scripts/validate-tutor-course-content.ts` +- Create: `tests/scripts/validate-tutor-course-content.test.ts` +- Modify: `package.json` + +1. Write failing unit tests for activation cases, missing coverage, unreachable Concepts/Indicators, mastery probe/kind shortages, prerequisite reachability, DEEPEN exhaustion/kind shortages, and a valid bundle. +2. Implement a pure validator using existing facts/matcher/question applicability/deepening defaults. +3. Write a failing CLI test for local bundle loading, hashes, and quality-case parsing. +4. Implement the filesystem fetch adapter and machine-readable/nonzero CLI result. +5. Run focused tests, typecheck, and commit. + +### Task 5: Make UnoSim-Examples pass the authoring hard gate + +**Files (UnoSim-Examples worktree):** +- Create: `tutor/quality-cases.yaml` +- Modify: `tutor/topics/variables-and-serial.yaml` +- Modify: `tutor/topics/long-values.yaml` +- Modify: `tutor/manifest.yaml` + +1. Add minimal positive/negative activation cases, including PWM as a negative case for `variables-and-serial`. +2. Run the validator and observe the expected mastery/DEEPEN exhaustion failures. +3. Add the smallest structurally distinct application/transfer probes needed for reachable mastery and default DEEPEN; update hashes. +4. Re-run the pinned local validator and commit. + +### Task 6: Add explicit CI hard gates and compatible-version pinning + +**Files:** +- Modify (UnoSim): `.github/workflows/ci.yml`, `package.json` +- Create (UnoSim-Examples): `.github/workflows/tutor-quality.yml`, `.unosim-compatible-commit` + +1. Add `test:tutor-quality` as a deterministic, credential-free UnoSim PR step. +2. Pin the Examples workflow to the final UnoSim Stage 1 commit and invoke its Course Content validator against the Examples checkout. +3. Validate workflow syntax structurally and run both local commands. +4. Commit CI changes. + +### Task 7: Verify and review the complete change + +1. Run `npm run check`, `npm run check:docs`, `npm run test:tutor-quality`, focused Tutor/Course Content tests, and `npm run test:unit` in UnoSim. +2. Run the exact pinned validator against the UnoSim-Examples worktree and verify hashes/status are clean. +3. Inspect both diffs and commits for accidental changes, secrets, scope creep, or reliance on a real provider. +4. Perform a fresh self-review because parallel review agents are disabled for this task; fix verified issues and repeat affected gates. +5. Report both branch heads without pushing or merging. diff --git a/package.json b/package.json index 688eed8e..fa41cbdc 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "test:deployment": "tsx scripts/deployment/gateway-deployment.ts", "capacity:calibrate": "tsx scripts/calibrate-capacity.ts", "test:security:inputs": "TEST_BUDGET_SUITE=security-inputs TEST_BUDGET_MS=15000 LOG_LEVEL=warn vitest run --project=unit-node --project=unit-node-http tests/shared/input-limits.test.ts tests/shared/websocket-direction-schemas.test.ts tests/server/security/safe-paths.test.ts tests/server/routes/compiler.routes.test.ts tests/integration/simulation-state-sequence.test.ts --reporter=default --reporter=./scripts/test-budget-reporter.mjs", + "test:tutor-quality": "TEST_BUDGET_SUITE=tutor-quality TEST_BUDGET_MS=15000 LOG_LEVEL=warn vitest run --project=unit-node tests/server/services/tutor/tutor-quality-scenarios.test.ts tests/server/services/tutor/tq-reg-001-pwm.test.ts tests/server/services/course-content/tutor-quality-validator.test.ts tests/scripts/validate-tutor-course-content.test.ts --reporter=default --reporter=./scripts/test-budget-reporter.mjs", "test:all": "npm run test:unit && npm run test:integration && npm run test:docker", "test:watch": "LOG_LEVEL=info vitest --project=unit-client --project=unit-node --project=unit-node-http", "test:coverage": "node scripts/coverage-guard.mjs prepare && TEST_BUDGET_SUITE=unit-coverage TEST_BUDGET_WARN_MS=75000 TEST_BUDGET_FAIL_MS=90000 LOG_LEVEL=warn vitest run --coverage --project=unit-client --project=unit-node --project=unit-node-http --reporter=default --reporter=./scripts/test-budget-reporter.mjs && node scripts/coverage-guard.mjs validate", diff --git a/ssot/ssot_function_definition_TutorQuality.md b/ssot/ssot_function_definition_TutorQuality.md new file mode 100644 index 00000000..5302d6db --- /dev/null +++ b/ssot/ssot_function_definition_TutorQuality.md @@ -0,0 +1,132 @@ +# Tutor Quality – Stage 1: Deterministic Quality Foundation + +Status: normative for deterministic Tutor quality gates. + +## 1. Scope and quality model + +The Tutor's primary quality objective is learning support: each turn should help +the learner build, test, or transfer an understanding of the current sketch. +Stage 1 does not claim to measure that outcome directly. It establishes the +deterministic preconditions without which a turn cannot be reliably +learning-supportive. + +The Stage 1 proxy is **deterministic instructional integrity**: + +- the selected Topic is activated by trusted facts from the current sketch; +- Topic, Concept, Indicator, Question, Strategy, phase, revision, and session + state form one consistent application-owned plan; +- the plan can reach mastery and the configured DEEPEN criterion without an + obvious content-exhaustion path; +- a successful learner answer progresses to a distinct question or phase; +- provider output cannot replace application-owned planning metadata; +- invalid, repeated, multi-question, or complete-solution output is rejected or + repaired before state is committed. + +Passing these gates means "structurally capable of supporting learning", not +"empirically proven to improve learning". + +## 2. Trust boundaries + +The existing trust boundaries remain authoritative: + +- the Course Content schemas and loader validate repository-owned data; +- the fact extractor and Topic matcher decide fact-based applicability; +- the planner owns Topic, Concept, Indicator, Question, phase, Strategy, and + progression metadata; +- the provider owns natural-language feedback and a proposed question only; +- `TutorService` validates/repairs provider content, applies the plan, and + commits progression state only after the complete request succeeds. + +Provider-supplied planning metadata is untrusted and MUST NOT survive when it +conflicts with application-owned planning or Strategy resolution. A real +`TutorPlan` MAY replace provider question text deliberately. Transition and +blocked outcomes MUST retain an already repaired provider question because they +carry metadata but no replacement question. + +## 3. Deterministic runtime invariants + +The Stage 1 hard gate covers at least these invariants: + +1. Topic, Strategy, phase, active Topic, and content revision are mutually + consistent and application-owned. +2. Progression state is committed only after provider output and the complete + planning outcome have succeeded. +3. A content revision change starts with revision-scoped progression state; + session state never leaks between Course Content sessions. +4. Strict repetition never reuses a previously used Question ID. +5. Exact and heuristically near question repetition is repaired where the + existing bounded similarity heuristic detects it. +6. Transition and blocked metadata do not overwrite the repaired question. +7. A real `TutorPlan` does overwrite provider question text and metadata. +8. Questions and feedback containing a complete solution are rejected by the + existing bounded solution guard. +9. Provider output exposes one primary question field. Deterministically clear + multi-question output (more than one question mark) is invalid. This is a + syntax invariant, not a semantic judgment of question quality. + +## 4. Course Content authoring quality + +Schema validity is necessary but not sufficient. A Tutor-enabled repository +MUST provide a small `tutor/quality-cases.yaml` authoring manifest. Each Topic +MUST have at least one positive and one negative activation case referencing a +main sketch from the repository manifest. + +The deterministic validator MUST use the production schemas, loader, fact +extractor, matcher, question-applicability rules, and default DEEPEN criterion. +It fails on: + +- a positive case that does not activate every declared expected Topic; +- a negative case that activates any declared forbidden Topic; +- a Topic without both positive and negative coverage; +- a Concept with no applicable question in a positive case; +- a required Indicator with no applicable question; +- too few distinct applicable questions or question kinds to satisfy mastery; +- prerequisites that cannot be reached in progression order; +- too few remaining non-recall questions, or missing required question kinds, + to satisfy DEEPEN after a minimal mastery witness; +- loader failures, invalid references, inconsistent hashes, or an invalid + Course Content bundle. + +Question–Indicator–Objective semantic coherence is outside this gate. Authors +and later evaluation stages remain responsible for meaning, clarity, difficulty, +feedback quality, and actual learning effect. + +## 5. Scenario regression contract + +A Tutor Quality scenario is a small TypeScript object containing: + +- stable case ID; +- action (`initial` or `dialog`); +- sketch; +- optional Course Content snapshot and progression state; +- bounded history, current question, and learner answer; +- exactly one fake-provider outcome (content or error); +- expected result/error, metadata, provider-call count, and state-commit effect. + +The runner MUST use `TutorService` and, where Course Content is supplied, +`CurriculumTutorAdapter`. It MUST fake only the external provider. Assertions +remain in tests so failures show the violated invariant directly. + +`TQ-REG-001` permanently covers the observed PWM failure family: the PWM sketch +must not activate `variables-and-serial`; after a strong answer an exact or +heuristically near repeated question must be replaced; and result/state metadata +must remain coherent through transition or blocked planning outcomes. + +## 6. CI contract + +UnoSim pull requests run a named, fast `test:tutor-quality` hard gate with no +network or real-provider dependency. The normal unit/type/doc gates remain in +force. + +UnoSim-Examples pull requests validate the repository using a pinned compatible +UnoSim commit. The pin is explicit and reviewable; floating `main` is forbidden. +The gate loads the full bundle, verifies hashes, and runs the authoring quality +cases. No provider credential is present. + +## 7. Deferred quality layers + +Stage 1 deliberately excludes semantic learning-support evaluation, real-LLM +sampling, LLM-as-Judge, human review studies, empirical learning-progress +measurement, adaptive Strategy selection, and a broad fact-extractor expansion. +Those belong to later layers and may consume Stage 1 scenario traces and stable +case IDs without changing the deterministic contract. From 33b87cc90ac19f5fd877c146ae4dc782f10f2320 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 20:36:38 +0200 Subject: [PATCH 4/9] test: add deterministic tutor quality scenarios --- .../tutor/curriculum-tutor-adapter.ts | 6 +- server/services/tutor/tutor-service.ts | 31 +- .../support/tutor-quality-scenario-runner.ts | 101 ++++++ .../tutor/tutor-quality-scenarios.test.ts | 298 ++++++++++++++++++ 4 files changed, 430 insertions(+), 6 deletions(-) create mode 100644 tests/server/services/tutor/support/tutor-quality-scenario-runner.ts create mode 100644 tests/server/services/tutor/tutor-quality-scenarios.test.ts diff --git a/server/services/tutor/curriculum-tutor-adapter.ts b/server/services/tutor/curriculum-tutor-adapter.ts index fe4febf0..de529692 100644 --- a/server/services/tutor/curriculum-tutor-adapter.ts +++ b/server/services/tutor/curriculum-tutor-adapter.ts @@ -34,6 +34,7 @@ import { hasMetDeepeningCriteria, markExpansionTargetUsed, markTopicMastered, + resetTutorProgressionState, type DidacticPhase, type TutorProgressionState, } from "./curriculum/progression-state"; @@ -191,9 +192,8 @@ export class CurriculumTutorAdapter implements TutorPlanningExtension { const exampleContextId = snapshot.exampleId ?? requestedExampleId; const annotation = exampleContextId === undefined ? undefined : snapshot.exampleTutorAnnotation; const orderedMatches = orderTopicMatches(matches, annotation); - const state = snapshot.progressionState?.revision === snapshot.revision - ? snapshot.progressionState - : createTutorProgressionState(snapshot.revision); + const state = snapshot.progressionState ?? createTutorProgressionState(snapshot.revision); + if (state.revision !== snapshot.revision) resetTutorProgressionState(state, snapshot.revision); const learnStrategy = this.resolveSnapshotStrategy(snapshot, annotation, "LEARN"); const classifications = classifyMatches(orderedMatches, facts, history, state, difficulty, learnStrategy.strategy); const selected = selectProgressionMatch(classifications, state); diff --git a/server/services/tutor/tutor-service.ts b/server/services/tutor/tutor-service.ts index 8490e5b5..87bd7f61 100644 --- a/server/services/tutor/tutor-service.ts +++ b/server/services/tutor/tutor-service.ts @@ -446,7 +446,12 @@ function validateLearningQuestion(result: TutorContentResult, difficulty?: Tutor ...resultWithoutMermaid, ...(sanitizedMermaid ? { mermaid: sanitizedMermaid } : {}), }); - if (!parsed.success || containsCompleteSolution(parsed.data.question) || (parsed.data.feedback !== undefined && containsCompleteSolution(parsed.data.feedback))) { + if ( + !parsed.success + || (parsed.data.question.match(/\?/g)?.length ?? 0) > 1 + || containsCompleteSolution(parsed.data.question) + || (parsed.data.feedback !== undefined && containsCompleteSolution(parsed.data.feedback)) + ) { throw new TutorProviderError("invalid-response"); } if (parsed.data.responseStyle === "philosophical" && parsed.data.answerRating !== undefined) { @@ -455,6 +460,26 @@ function validateLearningQuestion(result: TutorContentResult, difficulty?: Tutor return difficulty === undefined ? parsed.data : { ...parsed.data, difficulty }; } +function stripProviderPlanningMetadata(result: TutorContentResult): TutorContentResult { + const { + topicId: _topicId, + conceptId: _conceptId, + questionId: _questionId, + indicatorId: _indicatorId, + questionKind: _questionKind, + strategyId: _strategyId, + strategySource: _strategySource, + contentRevision: _contentRevision, + learningPhase: _learningPhase, + activeTopicId: _activeTopicId, + masteredTopicIds: _masteredTopicIds, + progressionBlockedReason: _progressionBlockedReason, + extensionTargetTopicId: _extensionTargetTopicId, + ...content + } = result; + return content; +} + function applyPlanningResult(result: TutorContentResult, plan: TutorPlan): TutorContentResult { const scaffoldFeedback = plan.scaffold ? `Hinweis: ${plan.scaffold.hint}` : undefined; let feedback = result.feedback; @@ -584,7 +609,7 @@ export class TutorService { }, requestCredential, ); - const validatedResult = validateLearningQuestion(providerResult.result, difficulty); + const validatedResult = stripProviderPlanningMetadata(validateLearningQuestion(providerResult.result, difficulty)); const plannedResult = planningResult ? applyPlanningOutcome(validatedResult, planningResult) : applyStrategyMetadata(validatedResult, strategy); @@ -625,7 +650,7 @@ export class TutorService { }, requestCredential, ); - const validatedResult = validateLearningQuestion(providerResult.result, difficulty); + const validatedResult = stripProviderPlanningMetadata(validateLearningQuestion(providerResult.result, difficulty)); if (validatedResult.responseStyle === "normal" && validatedResult.answerRating === undefined) { throw new TutorProviderError("invalid-response"); } diff --git a/tests/server/services/tutor/support/tutor-quality-scenario-runner.ts b/tests/server/services/tutor/support/tutor-quality-scenario-runner.ts new file mode 100644 index 00000000..203b0504 --- /dev/null +++ b/tests/server/services/tutor/support/tutor-quality-scenario-runner.ts @@ -0,0 +1,101 @@ +import type { TutorContentResult, TutorDialogTurn, TutorDifficulty } from "@shared/tutor"; +import type { LLMProvider, LLMProviderRequest } from "../../../../../server/services/tutor/llm-provider"; +import { TutorService } from "../../../../../server/services/tutor/tutor-service"; +import { CurriculumTutorAdapter } from "../../../../../server/services/tutor/curriculum-tutor-adapter"; +import type { TutorPlanningContentContext, TutorPlanningExtension } from "../../../../../server/services/tutor/tutor-planning"; +import type { TutorProgressionState } from "../../../../../server/services/tutor/curriculum/progression-state"; + +type ProviderBehavior = + | { readonly kind: "result"; readonly result: unknown; readonly model?: string } + | { readonly kind: "error"; readonly error: Error }; + +type ScenarioAction = + | { readonly kind: "initial" } + | { + readonly kind: "dialog"; + readonly history?: readonly TutorDialogTurn[]; + readonly question: string; + readonly answer: string; + }; + +export interface TutorQualityScenario { + readonly id: string; + readonly code: string; + readonly action: ScenarioAction; + readonly provider: ProviderBehavior; + readonly difficulty?: TutorDifficulty; + readonly courseContent?: TutorPlanningContentContext; + readonly planning?: TutorPlanningExtension; +} + +export interface TutorQualityScenarioTrace { + readonly id: string; + readonly result?: TutorContentResult; + readonly error?: unknown; + readonly providerRequests: readonly LLMProviderRequest[]; + readonly stateBefore?: TutorProgressionState; + readonly stateAfter?: TutorProgressionState; +} + +export async function runTutorQualityScenario(scenario: TutorQualityScenario): Promise { + const providerRequests: LLMProviderRequest[] = []; + const provider: LLMProvider = { + async listModels() { + return ["fake-model"]; + }, + async generateLearningQuestion(request) { + providerRequests.push(request); + if (scenario.provider.kind === "error") throw scenario.provider.error; + return { + model: scenario.provider.model ?? "fake-model", + result: scenario.provider.result as TutorContentResult, + }; + }, + }; + const stateBefore = cloneState(scenario.courseContent?.progressionState); + const planning = scenario.planning ?? (scenario.courseContent ? new CurriculumTutorAdapter() : undefined); + const service = new TutorService(provider, planning); + try { + const response = scenario.action.kind === "initial" + ? await service.generateQuestion( + scenario.code, + "fake-credential", + undefined, + scenario.difficulty, + scenario.courseContent, + ) + : await service.generateDialogResponse( + scenario.code, + scenario.action.history ?? [], + scenario.action.question, + scenario.action.answer, + "fake-credential", + undefined, + scenario.difficulty, + scenario.courseContent, + ); + return { + id: scenario.id, + result: response.result, + providerRequests, + ...(stateBefore ? { stateBefore } : {}), + ...(scenario.courseContent?.progressionState + ? { stateAfter: cloneState(scenario.courseContent.progressionState) } + : {}), + }; + } catch (error) { + return { + id: scenario.id, + error, + providerRequests, + ...(stateBefore ? { stateBefore } : {}), + ...(scenario.courseContent?.progressionState + ? { stateAfter: cloneState(scenario.courseContent.progressionState) } + : {}), + }; + } +} + +function cloneState(state: TutorProgressionState | undefined): TutorProgressionState | undefined { + return state === undefined ? undefined : structuredClone(state); +} diff --git a/tests/server/services/tutor/tutor-quality-scenarios.test.ts b/tests/server/services/tutor/tutor-quality-scenarios.test.ts new file mode 100644 index 00000000..c04b3732 --- /dev/null +++ b/tests/server/services/tutor/tutor-quality-scenarios.test.ts @@ -0,0 +1,298 @@ +import { describe, expect, it, vi } from "vitest"; +import { TutorProviderError } from "../../../../server/services/tutor/llm-provider"; +import type { TutorPlan, TutorPlanningExtension } from "../../../../server/services/tutor/tutor-planning"; +import { BUILT_IN_TUTOR_STRATEGY } from "../../../../server/services/tutor/strategy/effective-tutor-strategy"; +import { createTutorProgressionState } from "../../../../server/services/tutor/curriculum/progression-state"; +import type { CurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema"; +import { runTutorQualityScenario } from "./support/tutor-quality-scenario-runner"; + +const revision = "a".repeat(40); +const nextRevision = "b".repeat(40); +const sketch = "int value = 3; void setup() { Serial.println(value); } void loop() {}"; + +function metadataOutcome(kind: "transition" | "blocked"): Awaited> { + return { + kind, + ...(kind === "blocked" ? { progressionBlockedReason: "content-exhausted" as const } : {}), + contentRevision: revision, + learningPhase: "DEEPEN", + activeTopicId: "variables-and-serial", + masteredTopicIds: ["variables-and-serial"], + strategyId: "built-in-default", + strategySource: "built-in", + }; +} + +function planningWithFollowup(kind: "transition" | "blocked"): TutorPlanningExtension { + return { + planInitial: vi.fn().mockResolvedValue(null), + planFollowup: vi.fn().mockResolvedValue(metadataOutcome(kind)), + }; +} + +function tutorPlan(): TutorPlan { + return { + topicId: "variables-and-serial", + topicTitle: "Variablen", + conceptId: "variable-values", + conceptTitle: "Variablenwerte", + objective: "Werte erklären", + questionId: "planned-question", + questionKind: "application", + indicatorId: "value-use", + indicator: "Ordnet Wert und Verwendung zu", + question: "Wie wird value im Serial.println-Aufruf des Sketches verwendet?", + misconceptions: [], + contentRevision: revision, + strategyId: BUILT_IN_TUTOR_STRATEGY.id, + strategySource: "built-in", + effectiveStrategy: BUILT_IN_TUTOR_STRATEGY, + learningPhase: "LEARN", + activeTopicId: "variables-and-serial", + masteredTopicIds: [], + }; +} + +function topic(): CurriculumTopic { + return { + schemaVersion: 1, + id: "variables-and-serial", + title: "Variablen", + locale: "de-DE", + activation: { any: [{ fact: "serial-call", values: ["print"] }] }, + concepts: [{ + id: "variable-values", + title: "Variablenwerte", + objective: "Werte erklären", + prerequisites: [], + difficulty: { entry: [1, 50], transfer: [20, 80] }, + misconceptions: [], + indicators: [{ id: "value-use", description: "Wert und Verwendung" }], + mastery: { + minimumSuccessfulProbes: 1, + successRatingAtLeast: 3, + requiredIndicators: ["value-use"], + minimumDistinctQuestionKinds: 1, + recentWeakAnswersAllowed: 0, + }, + }], + questions: [ + { + id: "first-question", + concept: "variable-values", + indicator: "value-use", + kind: "concept", + difficulty: [1, 50], + requires: [{ fact: "serial-call", values: ["print"] }], + text: "Welche Rolle hat value im Sketch?", + }, + { + id: "second-question", + concept: "variable-values", + indicator: "value-use", + kind: "application", + difficulty: [1, 50], + requires: [{ fact: "serial-call", values: ["print"] }], + text: "Wie wird value bei der seriellen Ausgabe verwendet?", + }, + ], + scaffolds: [], + progression: { + entryConcepts: ["variable-values"], + preferredOrder: ["variable-values"], + onRating: { + "1-2": "remediate", + "3": "clarify-same-indicator", + "4": "probe-missing-indicator", + "5": "evaluate-mastery-and-advance", + }, + }, + }; +} + +function courseContent(contentRevision = revision) { + return { + revision: contentRevision, + tutor: { + status: "valid" as const, + manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, + topics: [topic()], + strategies: [], + }, + progressionState: createTutorProgressionState(contentRevision), + }; +} + +describe("Tutor Quality deterministic scenarios", () => { + it.each(["transition", "blocked"] as const)("retains a repaired near-duplicate question for %s outcomes", async (kind) => { + const currentQuestion = "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"; + const trace = await runTutorQualityScenario({ + id: `TQ-ADV-repeat-${kind}`, + code: sketch, + action: { kind: "dialog", question: currentQuestion, answer: "value hat den Wert 3 und wird ausgegeben." }, + provider: { + kind: "result", + result: { + answerRating: 5, + question: "Welche Rolle hat die Variable value und ihr Wert in diesem Sketch?", + }, + }, + planning: planningWithFollowup(kind), + }); + + expect(trace.error).toBeUndefined(); + expect(trace.result).toMatchObject({ + learningPhase: "DEEPEN", + contentRevision: revision, + activeTopicId: "variables-and-serial", + }); + expect(trace.result?.question).not.toContain("Rolle hat die Variable value"); + }); + + it("lets an application-owned TutorPlan replace provider question and metadata", async () => { + const plan = tutorPlan(); + const planning: TutorPlanningExtension = { + planInitial: vi.fn().mockResolvedValue(plan), + planFollowup: vi.fn().mockResolvedValue(null), + }; + const trace = await runTutorQualityScenario({ + id: "TQ-ADV-plan-authority", + code: sketch, + action: { kind: "initial" }, + provider: { + kind: "result", + result: { + question: "Providerfrage?", + topicId: "forged-topic", + questionId: "forged-question", + learningPhase: "EXPAND", + }, + }, + planning, + }); + + expect(trace.result).toMatchObject({ + question: plan.question, + topicId: plan.topicId, + questionId: plan.questionId, + learningPhase: "LEARN", + }); + }); + + it("drops provider-owned planning metadata when there is no TutorPlan", async () => { + const trace = await runTutorQualityScenario({ + id: "TQ-ADV-untrusted-metadata", + code: sketch, + action: { kind: "initial" }, + provider: { + kind: "result", + result: { + question: "Welche Rolle hat value im Sketch?", + topicId: "forged-topic", + conceptId: "forged-concept", + questionId: "forged-question", + learningPhase: "EXPAND", + contentRevision: "f".repeat(40), + strategyId: "forged-strategy", + strategySource: "repository", + }, + }, + }); + + expect(trace.error).toBeUndefined(); + expect(trace.result).toMatchObject({ + strategyId: BUILT_IN_TUTOR_STRATEGY.id, + strategySource: "built-in", + }); + expect(trace.result).not.toHaveProperty("topicId"); + expect(trace.result).not.toHaveProperty("conceptId"); + expect(trace.result).not.toHaveProperty("questionId"); + expect(trace.result).not.toHaveProperty("learningPhase"); + expect(trace.result).not.toHaveProperty("contentRevision"); + }); + + it.each([ + ["invalid output", { feedback: "Keine Frage vorhanden." }], + ["complete solution", { question: "```cpp\nvoid setup() {}\nvoid loop() {}\n```" }], + ["multiple primary questions", { question: "Was tut setup()? Was tut loop()?" }], + ])("rejects %s before committing progression state", async (_label, result) => { + const content = courseContent(); + const trace = await runTutorQualityScenario({ + id: "TQ-ADV-invalid-provider-output", + code: sketch, + action: { kind: "initial" }, + provider: { kind: "result", result }, + courseContent: content, + }); + + expect(trace.error).toBeInstanceOf(TutorProviderError); + expect(trace.stateAfter).toEqual(trace.stateBefore); + }); + + it("does not commit state when the provider fails", async () => { + const content = courseContent(); + const trace = await runTutorQualityScenario({ + id: "TQ-ADV-provider-error", + code: sketch, + action: { kind: "initial" }, + provider: { kind: "error", error: new TutorProviderError("provider-unavailable") }, + courseContent: content, + }); + + expect(trace.error).toMatchObject({ kind: "provider-unavailable" }); + expect(trace.stateAfter).toEqual(trace.stateBefore); + }); + + it("does not reuse a forbidden Question ID in strict progression", async () => { + const content = courseContent(); + const trace = await runTutorQualityScenario({ + id: "TQ-ADV-question-id-deduplication", + code: sketch, + action: { + kind: "dialog", + history: [{ + question: "Welche Rolle hat value im Sketch?", + questionId: "first-question", + answer: "value wird ausgegeben.", + answerRating: 4, + responseStyle: "normal", + }], + question: "Welche Rolle hat value im Sketch?", + answer: "value wird ausgegeben.", + }, + provider: { kind: "result", result: { answerRating: 4, question: "Welche Rolle hat value im Sketch?" } }, + courseContent: content, + }); + + expect(trace.error).toBeUndefined(); + expect(trace.result?.questionId).toBe("second-question"); + expect(trace.result?.questionId).not.toBe("first-question"); + }); + + it("commits a reset when Course Content revision changes", async () => { + const content = courseContent(nextRevision); + content.progressionState = { + ...createTutorProgressionState(revision), + activeTopicId: "old-topic", + phase: "EXPAND", + masteredTopicIds: ["old-topic"], + retainedPhases: { "old-topic": "EXPAND" }, + }; + const trace = await runTutorQualityScenario({ + id: "TQ-ADV-revision-boundary", + code: sketch, + action: { kind: "initial" }, + provider: { kind: "result", result: { question: "Welche Rolle hat value im Sketch?" } }, + courseContent: content, + }); + + expect(trace.error).toBeUndefined(); + expect(trace.stateAfter).toMatchObject({ + revision: nextRevision, + activeTopicId: "variables-and-serial", + phase: "LEARN", + masteredTopicIds: [], + }); + expect(trace.stateAfter?.retainedPhases).toEqual({}); + }); +}); From 14eac7bb49468c2d66b23498ab5c746474e69eb4 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 20:37:54 +0200 Subject: [PATCH 5/9] test: preserve PWM tutor regression TQ-REG-001 --- .../fixtures/tutor-quality/TQ-REG-001-pwm.ino | 17 +++ .../TQ-REG-001-variables-and-serial.yaml | 45 +++++++ .../services/tutor/tq-reg-001-pwm.test.ts | 112 ++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 tests/fixtures/tutor-quality/TQ-REG-001-pwm.ino create mode 100644 tests/fixtures/tutor-quality/TQ-REG-001-variables-and-serial.yaml create mode 100644 tests/server/services/tutor/tq-reg-001-pwm.test.ts diff --git a/tests/fixtures/tutor-quality/TQ-REG-001-pwm.ino b/tests/fixtures/tutor-quality/TQ-REG-001-pwm.ino new file mode 100644 index 00000000..d444e572 --- /dev/null +++ b/tests/fixtures/tutor-quality/TQ-REG-001-pwm.ino @@ -0,0 +1,17 @@ +// Fade an LED with PWM and expose the output on the board view. +const int ledPin = 9; + +void setup() { + pinMode(ledPin, OUTPUT); +} + +void loop() { + for (int brightness = 0; brightness <= 255; brightness += 5) { + analogWrite(ledPin, brightness); + delay(20); + } + for (int brightness = 255; brightness >= 0; brightness -= 5) { + analogWrite(ledPin, brightness); + delay(20); + } +} diff --git a/tests/fixtures/tutor-quality/TQ-REG-001-variables-and-serial.yaml b/tests/fixtures/tutor-quality/TQ-REG-001-variables-and-serial.yaml new file mode 100644 index 00000000..d5a8be43 --- /dev/null +++ b/tests/fixtures/tutor-quality/TQ-REG-001-variables-and-serial.yaml @@ -0,0 +1,45 @@ +schemaVersion: 1 +id: variables-and-serial +title: Variablen, Literale und Serial-Ausgabe +locale: de-DE +activation: + any: + - fact: serial-call + values: [print] +concepts: + - id: variable-values + title: Variablenwerte + objective: Den Zusammenhang zwischen einem deklarierten Integerwert und seiner Verwendung erklären. + prerequisites: [] + difficulty: + entry: [1, 50] + transfer: [20, 80] + misconceptions: [] + indicators: + - id: relates-value-to-use + description: Ordnet einen Integerwert seiner Verwendung im Sketch zu. + mastery: + minimumSuccessfulProbes: 1 + successRatingAtLeast: 3 + requiredIndicators: [relates-value-to-use] + minimumDistinctQuestionKinds: 1 + recentWeakAnswersAllowed: 0 +questions: + - id: variable-value-recall + concept: variable-values + indicator: relates-value-to-use + kind: recall + difficulty: [1, 50] + requires: + - fact: type-used + values: [int] + text: Welche Rolle spielt der Integer-Datentyp im aktuellen Sketch? +scaffolds: [] +progression: + entryConcepts: [variable-values] + preferredOrder: [variable-values] + onRating: + 1-2: remediate + 3: clarify-same-indicator + 4: probe-missing-indicator + 5: evaluate-mastery-and-advance diff --git a/tests/server/services/tutor/tq-reg-001-pwm.test.ts b/tests/server/services/tutor/tq-reg-001-pwm.test.ts new file mode 100644 index 00000000..0052953a --- /dev/null +++ b/tests/server/services/tutor/tq-reg-001-pwm.test.ts @@ -0,0 +1,112 @@ +import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import { parse as parseYaml } from "yaml"; +import { describe, expect, it, vi } from "vitest"; +import { curriculumTopicSchema, validateCurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema"; +import { createTutorProgressionState } from "../../../../server/services/tutor/curriculum/progression-state"; +import type { TutorPlanningExtension } from "../../../../server/services/tutor/tutor-planning"; +import { runTutorQualityScenario } from "./support/tutor-quality-scenario-runner"; + +const revision = "1".repeat(40); +const pwmSketch = fixture("TQ-REG-001-pwm.ino"); +const variablesTopic = validateCurriculumTopic( + curriculumTopicSchema.parse(parseYaml(fixture("TQ-REG-001-variables-and-serial.yaml"))), +); + +function fixture(name: string): string { + return readFileSync(fileURLToPath(new URL(`../../../fixtures/tutor-quality/${name}`, import.meta.url)), "utf8"); +} + +function courseContent() { + return { + revision, + tutor: { + status: "valid" as const, + manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, + topics: [variablesTopic], + strategies: [], + }, + progressionState: createTutorProgressionState(revision), + }; +} + +describe("TQ-REG-001 PWM question quality regression", () => { + it("does not activate the narrowed variables-and-serial Topic for the PWM sketch", async () => { + const content = courseContent(); + const trace = await runTutorQualityScenario({ + id: "TQ-REG-001/activation", + code: pwmSketch, + action: { kind: "initial" }, + provider: { kind: "result", result: { question: "Wie verändert brightness den PWM-Ausgang an ledPin?" } }, + courseContent: content, + }); + + expect(trace.error).toBeUndefined(); + expect(trace.result).not.toHaveProperty("topicId", "variables-and-serial"); + expect(trace.result).not.toHaveProperty("activeTopicId", "variables-and-serial"); + expect(trace.stateAfter).toEqual(trace.stateBefore); + }); + + it("repairs repetition after a strong answer and commits coherent transition metadata", async () => { + const content = courseContent(); + const repeatedQuestion = "Welchen Wert verwendet der Sketch an der betrachteten Integer-Variablen?"; + const planning: TutorPlanningExtension = { + planInitial: vi.fn().mockResolvedValue(null), + planFollowup: vi.fn(async ({ courseContent: workingContent }) => { + const state = workingContent?.progressionState; + if (state) { + state.activeTopicId = "pwm-output"; + state.phase = "DEEPEN"; + state.masteredTopicIds.push("pwm-output"); + state.retainedPhases["pwm-output"] = "DEEPEN"; + } + return { + kind: "transition" as const, + contentRevision: revision, + learningPhase: "DEEPEN" as const, + activeTopicId: "pwm-output", + masteredTopicIds: ["pwm-output"], + strategyId: "built-in-default", + strategySource: "built-in" as const, + }; + }), + }; + const trace = await runTutorQualityScenario({ + id: "TQ-REG-001/progression", + code: pwmSketch, + action: { + kind: "dialog", + question: repeatedQuestion, + answer: "brightness läuft in Fünferschritten zwischen 0 und 255 und bestimmt den PWM-Tastgrad.", + }, + provider: { + kind: "result", + result: { + answerRating: 5, + question: "Welchen Wert hat die betrachtete Integer-Variable im Sketch?", + topicId: "variables-and-serial", + learningPhase: "LEARN", + }, + }, + courseContent: content, + planning, + }); + + expect(trace.error).toBeUndefined(); + expect(trace.result?.question).not.toContain("betrachtete Integer-Variable"); + expect(trace.result).toMatchObject({ + answerRating: 5, + contentRevision: revision, + learningPhase: "DEEPEN", + activeTopicId: "pwm-output", + masteredTopicIds: ["pwm-output"], + }); + expect(trace.stateAfter).toMatchObject({ + revision, + phase: "DEEPEN", + activeTopicId: "pwm-output", + masteredTopicIds: ["pwm-output"], + retainedPhases: { "pwm-output": "DEEPEN" }, + }); + }); +}); From fb9c375d3fbf55935dc367b4d86e09423a9941c7 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 20:42:52 +0200 Subject: [PATCH 6/9] feat: validate deterministic tutor content quality --- package.json | 3 +- scripts/validate-tutor-course-content.mjs | 20 ++ .../tutor-quality-directory-validator.ts | 89 ++++++ .../course-content/tutor-quality-schema.ts | 29 ++ .../course-content/tutor-quality-validator.ts | 293 ++++++++++++++++++ .../tutor-quality-directory-validator.test.ts | 138 +++++++++ .../tutor-quality-validator.test.ts | 163 ++++++++++ 7 files changed, 734 insertions(+), 1 deletion(-) create mode 100644 scripts/validate-tutor-course-content.mjs create mode 100644 server/services/course-content/tutor-quality-directory-validator.ts create mode 100644 server/services/course-content/tutor-quality-schema.ts create mode 100644 server/services/course-content/tutor-quality-validator.ts create mode 100644 tests/server/services/course-content/tutor-quality-directory-validator.test.ts create mode 100644 tests/server/services/course-content/tutor-quality-validator.test.ts diff --git a/package.json b/package.json index fa41cbdc..f012074a 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "test:deployment": "tsx scripts/deployment/gateway-deployment.ts", "capacity:calibrate": "tsx scripts/calibrate-capacity.ts", "test:security:inputs": "TEST_BUDGET_SUITE=security-inputs TEST_BUDGET_MS=15000 LOG_LEVEL=warn vitest run --project=unit-node --project=unit-node-http tests/shared/input-limits.test.ts tests/shared/websocket-direction-schemas.test.ts tests/server/security/safe-paths.test.ts tests/server/routes/compiler.routes.test.ts tests/integration/simulation-state-sequence.test.ts --reporter=default --reporter=./scripts/test-budget-reporter.mjs", - "test:tutor-quality": "TEST_BUDGET_SUITE=tutor-quality TEST_BUDGET_MS=15000 LOG_LEVEL=warn vitest run --project=unit-node tests/server/services/tutor/tutor-quality-scenarios.test.ts tests/server/services/tutor/tq-reg-001-pwm.test.ts tests/server/services/course-content/tutor-quality-validator.test.ts tests/scripts/validate-tutor-course-content.test.ts --reporter=default --reporter=./scripts/test-budget-reporter.mjs", + "test:tutor-quality": "TEST_BUDGET_SUITE=tutor-quality TEST_BUDGET_MS=15000 LOG_LEVEL=warn vitest run --project=unit-node tests/server/services/tutor/tutor-quality-scenarios.test.ts tests/server/services/tutor/tq-reg-001-pwm.test.ts tests/server/services/course-content/tutor-quality-validator.test.ts tests/server/services/course-content/tutor-quality-directory-validator.test.ts --reporter=default --reporter=./scripts/test-budget-reporter.mjs", + "validate:tutor-course-content": "tsx scripts/validate-tutor-course-content.mjs", "test:all": "npm run test:unit && npm run test:integration && npm run test:docker", "test:watch": "LOG_LEVEL=info vitest --project=unit-client --project=unit-node --project=unit-node-http", "test:coverage": "node scripts/coverage-guard.mjs prepare && TEST_BUDGET_SUITE=unit-coverage TEST_BUDGET_WARN_MS=75000 TEST_BUDGET_FAIL_MS=90000 LOG_LEVEL=warn vitest run --coverage --project=unit-client --project=unit-node --project=unit-node-http --reporter=default --reporter=./scripts/test-budget-reporter.mjs && node scripts/coverage-guard.mjs validate", diff --git a/scripts/validate-tutor-course-content.mjs b/scripts/validate-tutor-course-content.mjs new file mode 100644 index 00000000..2788bca8 --- /dev/null +++ b/scripts/validate-tutor-course-content.mjs @@ -0,0 +1,20 @@ +import path from "node:path"; +import { validateTutorCourseContentDirectory } from "../server/services/course-content/tutor-quality-directory-validator.ts"; + +const directory = process.argv[2]; +if (!directory) { + console.error("Usage: npm run validate:tutor-course-content -- "); + process.exitCode = 2; +} else { + const resolved = path.resolve(directory); + const issues = await validateTutorCourseContentDirectory(resolved); + if (issues.length === 0) { + console.log(`Tutor Course Content quality passed: ${resolved}`); + } else { + for (const issue of issues) { + const scope = [issue.caseId, issue.topicId, issue.conceptId, issue.indicatorId].filter(Boolean).join("/"); + console.error(`${issue.code}${scope ? ` [${scope}]` : ""}: ${issue.message}`); + } + process.exitCode = 1; + } +} diff --git a/server/services/course-content/tutor-quality-directory-validator.ts b/server/services/course-content/tutor-quality-directory-validator.ts new file mode 100644 index 00000000..4908cbe9 --- /dev/null +++ b/server/services/course-content/tutor-quality-directory-validator.ts @@ -0,0 +1,89 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { parse as parseYaml } from "yaml"; +import type { FullCommitSha, RepositorySlug } from "@shared/examples"; +import { CourseContentLoader } from "./course-content-loader"; +import { tutorQualityCasesSchema } from "./tutor-quality-schema"; +import { + validateTutorContentQuality, + type ResolvedTutorQualityCase, + type TutorContentQualityIssue, +} from "./tutor-quality-validator"; + +const LOCAL_REVISION = "0".repeat(40) as FullCommitSha; +const LOCAL_REPOSITORY = "local/course-content" as RepositorySlug; + +export async function validateTutorCourseContentDirectory(directory: string): Promise { + const root = path.resolve(directory); + let loaded: Awaited>; + try { + const loader = new CourseContentLoader({ + fetchText: async (url, maxBytes) => readBoundedLocalFile(root, url, maxBytes), + }); + loaded = await loader.load(LOCAL_REPOSITORY, LOCAL_REVISION); + } catch { + return [directoryIssue("invalid-course-content-bundle", "Course Content bundle could not be loaded")]; + } + if (loaded.tutor.status !== "valid") { + return [directoryIssue("invalid-course-content-bundle", "Course Content Tutor bundle is absent or invalid")]; + } + try { + const qualitySource = await readFile(path.join(root, "tutor/quality-cases.yaml"), "utf8"); + const parsed = tutorQualityCasesSchema.safeParse(parseYaml(qualitySource)); + if (!parsed.success) { + return [directoryIssue("invalid-quality-cases", "Tutor quality case manifest is invalid")]; + } + const resolution = resolveCases(parsed.data.cases, loaded.examples); + if (resolution.issues.length > 0) return resolution.issues; + return validateTutorContentQuality(loaded.tutor.topics, resolution.cases); + } catch { + return [directoryIssue("invalid-quality-cases", "Tutor quality case manifest could not be loaded")]; + } +} + +async function readBoundedLocalFile(root: string, url: URL, maxBytes: number): Promise { + const revisionMarker = `/${LOCAL_REVISION}/`; + const markerIndex = url.pathname.indexOf(revisionMarker); + if (markerIndex < 0) throw new Error("Invalid local Course Content URL"); + const relativePath = decodeURIComponent(url.pathname.slice(markerIndex + revisionMarker.length)); + const filePath = path.resolve(root, relativePath); + if (filePath !== root && !filePath.startsWith(`${root}${path.sep}`)) throw new Error("Path escapes Course Content root"); + const source = await readFile(filePath, "utf8"); + if (Buffer.byteLength(source, "utf8") > maxBytes) throw new Error("Course Content file exceeds size limit"); + return source; +} + +function resolveCases( + cases: readonly { id: string; example: string; expectedTopics: readonly string[]; forbiddenTopics: readonly string[] }[], + examples: Awaited>["examples"], +): { cases: ResolvedTutorQualityCase[]; issues: TutorContentQualityIssue[] } { + const resolved: ResolvedTutorQualityCase[] = []; + const issues: TutorContentQualityIssue[] = []; + for (const qualityCase of cases) { + const example = examples.find(({ id }) => id === qualityCase.example); + const main = example?.files.find(({ name }) => name === example.main); + if (!example || !main) { + issues.push({ + code: "quality-case-example-not-found", + message: `Quality case ${qualityCase.id} references missing Example ${qualityCase.example}`, + caseId: qualityCase.id, + }); + continue; + } + resolved.push({ + id: qualityCase.id, + exampleId: qualityCase.example, + code: main.content, + expectedTopics: qualityCase.expectedTopics, + forbiddenTopics: qualityCase.forbiddenTopics, + }); + } + return { cases: resolved, issues }; +} + +function directoryIssue( + code: "invalid-course-content-bundle" | "invalid-quality-cases", + message: string, +): TutorContentQualityIssue { + return { code, message }; +} diff --git a/server/services/course-content/tutor-quality-schema.ts b/server/services/course-content/tutor-quality-schema.ts new file mode 100644 index 00000000..ed44d650 --- /dev/null +++ b/server/services/course-content/tutor-quality-schema.ts @@ -0,0 +1,29 @@ +import { z } from "zod"; + +const SAFE_ID = /^[a-z][a-z0-9-]{0,63}$/; + +const tutorQualityCaseSchema = z.object({ + id: z.string().regex(SAFE_ID), + example: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/), + expectedTopics: z.array(z.string().regex(SAFE_ID)).max(64).default([]), + forbiddenTopics: z.array(z.string().regex(SAFE_ID)).max(64).default([]), +}).strict().superRefine((value, context) => { + if (value.expectedTopics.length === 0 && value.forbiddenTopics.length === 0) { + context.addIssue({ code: z.ZodIssueCode.custom, message: "A quality case must declare an expected or forbidden Topic" }); + } + const overlap = value.expectedTopics.find((topicId) => value.forbiddenTopics.includes(topicId)); + if (overlap) { + context.addIssue({ code: z.ZodIssueCode.custom, message: `Topic cannot be expected and forbidden: ${overlap}` }); + } +}); + +export const tutorQualityCasesSchema = z.object({ + schemaVersion: z.literal(1), + cases: z.array(tutorQualityCaseSchema).min(1).max(256), +}).strict().superRefine((value, context) => { + if (new Set(value.cases.map(({ id }) => id)).size !== value.cases.length) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["cases"], message: "Quality case ids must be unique" }); + } +}); + +export type TutorQualityCases = z.infer; diff --git a/server/services/course-content/tutor-quality-validator.ts b/server/services/course-content/tutor-quality-validator.ts new file mode 100644 index 00000000..091e6cac --- /dev/null +++ b/server/services/course-content/tutor-quality-validator.ts @@ -0,0 +1,293 @@ +import type { TutorDialogTurn } from "@shared/tutor"; +import type { CurriculumConcept, CurriculumQuestion, CurriculumTopic } from "../tutor/curriculum/curriculum-schema"; +import { + classifyTopic, + collectObservations, + DefaultLearningPlanner, + questionApplies, +} from "../tutor/curriculum/learning-planner"; +import { deepeningCriteria, hasMetDeepeningCriteria } from "../tutor/curriculum/progression-state"; +import { DefaultSketchFactExtractor, type SketchFacts } from "../tutor/curriculum/sketch-facts"; +import { DefaultTopicMatcher } from "../tutor/curriculum/topic-matcher"; +import { BUILT_IN_TUTOR_STRATEGY } from "../tutor/strategy/effective-tutor-strategy"; + +export interface ResolvedTutorQualityCase { + readonly id: string; + readonly exampleId: string; + readonly code: string; + readonly expectedTopics: readonly string[]; + readonly forbiddenTopics: readonly string[]; +} + +export type TutorContentQualityIssueCode = + | "expected-topic-not-activated" + | "forbidden-topic-activated" + | "missing-positive-activation-case" + | "missing-negative-activation-case" + | "unknown-topic-reference" + | "unreachable-concept" + | "unreachable-indicator" + | "unreachable-prerequisite" + | "insufficient-mastery-probes" + | "insufficient-mastery-question-kinds" + | "missing-mastery-indicator-question" + | "missing-deepening-question-kind" + | "learn-content-exhausted" + | "deepen-content-exhausted" + | "invalid-course-content-bundle" + | "invalid-quality-cases" + | "quality-case-example-not-found"; + +export interface TutorContentQualityIssue { + readonly code: TutorContentQualityIssueCode; + readonly message: string; + readonly caseId?: string; + readonly topicId?: string; + readonly conceptId?: string; + readonly indicatorId?: string; +} + +type CaseContext = { + readonly qualityCase: ResolvedTutorQualityCase; + readonly facts: SketchFacts; + readonly activatedTopicIds: ReadonlySet; +}; + +export function validateTutorContentQuality( + topics: readonly CurriculumTopic[], + qualityCases: readonly ResolvedTutorQualityCase[], +): TutorContentQualityIssue[] { + const extractor = new DefaultSketchFactExtractor(); + const matcher = new DefaultTopicMatcher(); + const topicIds = new Set(topics.map(({ id }) => id)); + const contexts = qualityCases.map((qualityCase): CaseContext => { + const facts = extractor.extract(qualityCase.code); + return { + qualityCase, + facts, + activatedTopicIds: new Set(matcher.match(topics, facts).map(({ topic }) => topic.id)), + }; + }); + const issues: TutorContentQualityIssue[] = []; + + validateActivationReferences(contexts, topicIds, issues); + for (const topic of topics) validateTopic(topic, contexts, issues); + return issues; +} + +function validateActivationReferences( + contexts: readonly CaseContext[], + topicIds: ReadonlySet, + issues: TutorContentQualityIssue[], +): void { + for (const { qualityCase, activatedTopicIds } of contexts) { + for (const topicId of [...qualityCase.expectedTopics, ...qualityCase.forbiddenTopics]) { + if (!topicIds.has(topicId)) { + issues.push(issue("unknown-topic-reference", `Quality case references unknown Topic ${topicId}`, qualityCase.id, topicId)); + } + } + for (const topicId of qualityCase.expectedTopics) { + if (topicIds.has(topicId) && !activatedTopicIds.has(topicId)) { + issues.push(issue("expected-topic-not-activated", `Expected Topic ${topicId} is not activated`, qualityCase.id, topicId)); + } + } + for (const topicId of qualityCase.forbiddenTopics) { + if (topicIds.has(topicId) && activatedTopicIds.has(topicId)) { + issues.push(issue("forbidden-topic-activated", `Forbidden Topic ${topicId} is activated`, qualityCase.id, topicId)); + } + } + } +} + +function validateTopic( + topic: CurriculumTopic, + contexts: readonly CaseContext[], + issues: TutorContentQualityIssue[], +): void { + const positive = contexts.filter(({ qualityCase }) => qualityCase.expectedTopics.includes(topic.id)); + const negative = contexts.filter(({ qualityCase }) => qualityCase.forbiddenTopics.includes(topic.id)); + if (positive.length === 0) { + issues.push(issue("missing-positive-activation-case", `Topic ${topic.id} has no positive activation case`, undefined, topic.id)); + } + if (negative.length === 0) { + issues.push(issue("missing-negative-activation-case", `Topic ${topic.id} has no negative activation case`, undefined, topic.id)); + } + + validateGlobalReachability(topic, positive, issues); + for (const context of positive) { + if (!context.activatedTopicIds.has(topic.id)) continue; + validateCaseStructure(topic, context, issues); + validateExecutablePath(topic, context, issues); + } +} + +function validateGlobalReachability( + topic: CurriculumTopic, + contexts: readonly CaseContext[], + issues: TutorContentQualityIssue[], +): void { + const applicable = topic.questions.filter((question) => contexts.some(({ facts }) => questionApplies(question, facts))); + for (const concept of topic.concepts) { + const conceptQuestions = applicable.filter((question) => question.concept === concept.id); + if (conceptQuestions.length === 0) { + issues.push(issue("unreachable-concept", `Concept ${concept.id} has no applicable question in any positive case`, undefined, topic.id, concept.id)); + } + for (const indicator of concept.indicators) { + if (!conceptQuestions.some((question) => question.indicator === indicator.id)) { + issues.push(issue("unreachable-indicator", `Indicator ${indicator.id} has no applicable question in any positive case`, undefined, topic.id, concept.id, indicator.id)); + } + } + } +} + +function validateCaseStructure( + topic: CurriculumTopic, + context: CaseContext, + issues: TutorContentQualityIssue[], +): void { + const applicable = topic.questions.filter((question) => questionApplies(question, context.facts)); + const domain = new Set(applicable.map(({ concept }) => concept)); + for (const concept of topic.concepts.filter(({ id }) => domain.has(id))) { + const questions = applicable.filter((question) => question.concept === concept.id); + validatePrerequisites(topic, concept, domain, context, issues); + validateMasteryCapacity(topic, concept, questions, context, issues); + } + const criteria = deepeningCriteria(topic); + for (const kind of criteria.requiredQuestionKinds) { + if (!applicable.some((question) => question.kind === kind)) { + issues.push(issue("missing-deepening-question-kind", `DEEPEN requires unavailable question kind ${kind}`, context.qualityCase.id, topic.id)); + } + } +} + +function validatePrerequisites( + topic: CurriculumTopic, + concept: CurriculumConcept, + domain: ReadonlySet, + context: CaseContext, + issues: TutorContentQualityIssue[], +): void { + for (const prerequisite of concept.prerequisites) { + if (!domain.has(prerequisite)) { + issues.push(issue("unreachable-prerequisite", `Concept ${concept.id} requires unprobeable Concept ${prerequisite}`, context.qualityCase.id, topic.id, concept.id)); + } + } +} + +function validateMasteryCapacity( + topic: CurriculumTopic, + concept: CurriculumConcept, + questions: readonly CurriculumQuestion[], + context: CaseContext, + issues: TutorContentQualityIssue[], +): void { + if (questions.length < concept.mastery.minimumSuccessfulProbes) { + issues.push(issue("insufficient-mastery-probes", `Concept ${concept.id} has too few distinct applicable mastery probes`, context.qualityCase.id, topic.id, concept.id)); + } + if (new Set(questions.map(({ kind }) => kind)).size < concept.mastery.minimumDistinctQuestionKinds) { + issues.push(issue("insufficient-mastery-question-kinds", `Concept ${concept.id} has too few applicable question kinds`, context.qualityCase.id, topic.id, concept.id)); + } + for (const indicator of concept.mastery.requiredIndicators) { + if (!questions.some((question) => question.indicator === indicator)) { + issues.push(issue("missing-mastery-indicator-question", `Mastery Indicator ${indicator} has no applicable question`, context.qualityCase.id, topic.id, concept.id, indicator)); + } + } +} + +function validateExecutablePath( + topic: CurriculumTopic, + context: CaseContext, + issues: TutorContentQualityIssue[], +): void { + const planner = new DefaultLearningPlanner(); + const history: TutorDialogTurn[] = []; + let masteryReached = false; + for (let step = 0; step <= topic.questions.length; step += 1) { + const observations = collectObservations(topic, history); + const classification = classifyTopic( + topic, + context.facts, + observations, + new Set(history.flatMap(({ questionId }) => questionId ? [questionId] : [])), + 30, + BUILT_IN_TUTOR_STRATEGY, + ); + if (classification.status === "mastered") { + masteryReached = true; + break; + } + const plan = planner.start(topic, "0".repeat(40), context.facts, history, 30, BUILT_IN_TUTOR_STRATEGY); + if (!plan) break; + history.push(successfulTurn(plan.brief)); + } + if (!masteryReached) { + issues.push(issue("learn-content-exhausted", "Default strict progression cannot reach Topic mastery", context.qualityCase.id, topic.id)); + return; + } + + const postMastery = [] as Array<{ + questionId: string; + conceptId: string; + indicatorId: string; + kind: CurriculumQuestion["kind"]; + rating: 5; + }>; + const criteria = deepeningCriteria(topic); + for (let step = 0; step <= topic.questions.length; step += 1) { + if (hasMetDeepeningCriteria(topic, postMastery)) return; + const plan = planner.start(topic, "0".repeat(40), context.facts, history, 30, BUILT_IN_TUTOR_STRATEGY, { + phase: "DEEPEN", + preferredQuestionKinds: criteria.requiredQuestionKinds, + existingQuestionKinds: postMastery.map(({ kind }) => kind), + }); + if (!plan) break; + history.push(successfulTurn(plan.brief)); + postMastery.push({ + questionId: plan.brief.questionId, + conceptId: plan.brief.conceptId, + indicatorId: plan.brief.indicatorId, + kind: plan.brief.questionKind, + rating: 5, + }); + } + issues.push(issue("deepen-content-exhausted", "Default strict progression cannot satisfy DEEPEN", context.qualityCase.id, topic.id)); +} + +function successfulTurn(brief: { + readonly question: string; + readonly questionId: string; + readonly topicId: string; + readonly conceptId: string; + readonly indicatorId: string; + readonly questionKind: CurriculumQuestion["kind"]; +}): TutorDialogTurn { + return { + question: brief.question, + questionId: brief.questionId, + topicId: brief.topicId, + conceptId: brief.conceptId, + indicatorId: brief.indicatorId, + questionKind: brief.questionKind, + answer: "Deterministic successful quality probe", + answerRating: 5, + responseStyle: "normal", + }; +} + +function issue( + code: TutorContentQualityIssueCode, + message: string, + caseId?: string, + topicId?: string, + conceptId?: string, + indicatorId?: string, +): TutorContentQualityIssue { + return { + code, + message, + ...(caseId ? { caseId } : {}), + ...(topicId ? { topicId } : {}), + ...(conceptId ? { conceptId } : {}), + ...(indicatorId ? { indicatorId } : {}), + }; +} diff --git a/tests/server/services/course-content/tutor-quality-directory-validator.test.ts b/tests/server/services/course-content/tutor-quality-directory-validator.test.ts new file mode 100644 index 00000000..26838727 --- /dev/null +++ b/tests/server/services/course-content/tutor-quality-directory-validator.test.ts @@ -0,0 +1,138 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { validateTutorCourseContentDirectory } from "../../../../server/services/course-content/tutor-quality-directory-validator"; + +const temporaryDirectories: string[] = []; + +afterEach(async () => { + await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); +}); + +describe("Tutor Course Content directory validator", () => { + it("loads the real bundle boundary, verifies hashes, and resolves quality cases", async () => { + const directory = await courseContentDirectory(); + + await expect(validateTutorCourseContentDirectory(directory)).resolves.toEqual([]); + }); + + it("fails closed when a declared Tutor hash is inconsistent", async () => { + const directory = await courseContentDirectory(); + await writeFile(path.join(directory, "tutor/topics/variables.yaml"), `${topicSource()}\n# changed\n`, "utf8"); + + await expect(validateTutorCourseContentDirectory(directory)).resolves.toEqual(expect.arrayContaining([ + expect.objectContaining({ code: "invalid-course-content-bundle" }), + ])); + }); + + it("fails when the quality case manifest is absent", async () => { + const directory = await courseContentDirectory(false); + + await expect(validateTutorCourseContentDirectory(directory)).resolves.toEqual(expect.arrayContaining([ + expect.objectContaining({ code: "invalid-quality-cases" }), + ])); + }); +}); + +async function courseContentDirectory(withCases = true): Promise { + const directory = await mkdtemp(path.join(tmpdir(), "unosim-tutor-quality-")); + temporaryDirectories.push(directory); + await mkdir(path.join(directory, "examples"), { recursive: true }); + await mkdir(path.join(directory, "tutor/topics"), { recursive: true }); + const serial = "int value = 3; void setup() { Serial.println(value); } void loop() {}\n"; + const pwm = "const int pin = 9; void setup() {} void loop() { analogWrite(pin, 128); }\n"; + const topic = topicSource(); + await writeFile(path.join(directory, "examples/serial.ino"), serial, "utf8"); + await writeFile(path.join(directory, "examples/pwm.ino"), pwm, "utf8"); + await writeFile(path.join(directory, "tutor/topics/variables.yaml"), topic, "utf8"); + await writeFile(path.join(directory, "manifest.json"), JSON.stringify({ + schemaVersion: 2, + examples: [ + { id: "serial", title: "Serial", category: "Test", files: [{ name: "serial.ino", path: "examples/serial.ino" }], main: "serial.ino" }, + { id: "pwm", title: "PWM", category: "Test", files: [{ name: "pwm.ino", path: "examples/pwm.ino" }], main: "pwm.ino" }, + ], + tutor: { manifest: "tutor/manifest.yaml" }, + }), "utf8"); + await writeFile(path.join(directory, "tutor/manifest.yaml"), [ + "schemaVersion: 1", + "topics:", + " - id: variables-and-serial", + " path: tutor/topics/variables.yaml", + ` sha256: ${digest(topic)}`, + "strategies: []", + "", + ].join("\n"), "utf8"); + if (withCases) { + await writeFile(path.join(directory, "tutor/quality-cases.yaml"), [ + "schemaVersion: 1", + "cases:", + " - id: serial-positive", + " example: serial", + " expectedTopics: [variables-and-serial]", + " - id: pwm-negative", + " example: pwm", + " forbiddenTopics: [variables-and-serial]", + "", + ].join("\n"), "utf8"); + } + return directory; +} + +function digest(value: string): string { + return createHash("sha256").update(value, "utf8").digest("hex"); +} + +function topicSource(): string { + return [ + "schemaVersion: 1", + "id: variables-and-serial", + "title: Variablen", + "locale: de-DE", + "activation:", + " any:", + " - fact: serial-call", + " values: [print]", + "concepts:", + " - id: values", + " title: Werte", + " objective: Werte und Ausgabe verbinden.", + " prerequisites: []", + " difficulty:", + " entry: [1, 50]", + " transfer: [20, 80]", + " misconceptions: []", + " indicators:", + " - id: value-use", + " description: Verwendung erklären.", + " mastery:", + " minimumSuccessfulProbes: 1", + " successRatingAtLeast: 3", + " requiredIndicators: [value-use]", + " minimumDistinctQuestionKinds: 1", + " recentWeakAnswersAllowed: 0", + "questions:", + ...["concept", "transfer", "prediction"].flatMap((kind, index) => [ + ` - id: question-${index + 1}`, + " concept: values", + " indicator: value-use", + ` kind: ${kind}`, + " difficulty: [1, 80]", + " requires:", + " - fact: serial-call", + " values: [print]", + ` text: Was prüft Frage ${index + 1}?`, + ]), + "scaffolds: []", + "progression:", + " entryConcepts: [values]", + " preferredOrder: [values]", + " onRating:", + " 1-2: remediate", + " 3: clarify-same-indicator", + " 4: probe-missing-indicator", + " 5: evaluate-mastery-and-advance", + "", + ].join("\n"); +} diff --git a/tests/server/services/course-content/tutor-quality-validator.test.ts b/tests/server/services/course-content/tutor-quality-validator.test.ts new file mode 100644 index 00000000..7a82b644 --- /dev/null +++ b/tests/server/services/course-content/tutor-quality-validator.test.ts @@ -0,0 +1,163 @@ +import { describe, expect, it } from "vitest"; +import type { CurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema"; +import { + validateTutorContentQuality, + type ResolvedTutorQualityCase, +} from "../../../../server/services/course-content/tutor-quality-validator"; + +const serialSketch = "int value = 3; void setup() { Serial.println(value); } void loop() {}"; +const pwmSketch = "const int pin = 9; void setup() {} void loop() { analogWrite(pin, 128); }"; + +function validTopic(): CurriculumTopic { + return { + schemaVersion: 1, + id: "variables-and-serial", + title: "Variablen", + locale: "de-DE", + activation: { any: [{ fact: "serial-call", values: ["print"] }] }, + concepts: [{ + id: "values", + title: "Werte", + objective: "Werte und Ausgabe verbinden", + prerequisites: [], + difficulty: { entry: [1, 50], transfer: [20, 80] }, + misconceptions: [], + indicators: [{ id: "value-use", description: "Verwendung erklären" }], + mastery: { + minimumSuccessfulProbes: 1, + successRatingAtLeast: 3, + requiredIndicators: ["value-use"], + minimumDistinctQuestionKinds: 1, + recentWeakAnswersAllowed: 0, + }, + }], + questions: [ + question("learn-concept", "concept"), + question("deepen-transfer", "transfer"), + question("deepen-prediction", "prediction"), + ], + scaffolds: [], + progression: { + entryConcepts: ["values"], + preferredOrder: ["values"], + onRating: { + "1-2": "remediate", + "3": "clarify-same-indicator", + "4": "probe-missing-indicator", + "5": "evaluate-mastery-and-advance", + }, + }, + }; +} + +function question(id: string, kind: "recall" | "concept" | "application" | "prediction" | "transfer") { + return { + id, + concept: "values", + indicator: "value-use", + kind, + difficulty: [1, 80] as [number, number], + requires: [{ fact: "serial-call" as const, values: ["print"] }], + text: `Frage ${id}?`, + }; +} + +function cases(): ResolvedTutorQualityCase[] { + return [ + { + id: "serial-positive", + exampleId: "serial", + code: serialSketch, + expectedTopics: ["variables-and-serial"], + forbiddenTopics: [], + }, + { + id: "pwm-negative", + exampleId: "pwm", + code: pwmSketch, + expectedTopics: [], + forbiddenTopics: ["variables-and-serial"], + }, + ]; +} + +function codes(issues: ReturnType) { + return issues.map(({ code }) => code); +} + +describe("deterministic Tutor Course Content quality", () => { + it("accepts covered activation and an executable mastery-to-DEEPEN path", () => { + expect(validateTutorContentQuality([validTopic()], cases())).toEqual([]); + }); + + it("reports positive and negative activation mismatches and missing Topic coverage", () => { + const topic = validTopic(); + const activationCases: ResolvedTutorQualityCase[] = [{ + id: "wrong-cases", + exampleId: "pwm", + code: pwmSketch, + expectedTopics: [topic.id], + forbiddenTopics: [], + }]; + + expect(codes(validateTutorContentQuality([topic], activationCases))).toEqual(expect.arrayContaining([ + "expected-topic-not-activated", + "missing-negative-activation-case", + ])); + }); + + it("reports Concepts and required Indicators with no applicable question", () => { + const topic = validTopic(); + topic.concepts.push({ + ...topic.concepts[0]!, + id: "unreachable", + title: "Unerreichbar", + indicators: [{ id: "missing-indicator", description: "Nicht prüfbar" }], + mastery: { ...topic.concepts[0]!.mastery, requiredIndicators: ["missing-indicator"] }, + }); + topic.progression.preferredOrder.push("unreachable"); + + expect(codes(validateTutorContentQuality([topic], cases()))).toEqual(expect.arrayContaining([ + "unreachable-concept", + "unreachable-indicator", + ])); + }); + + it("reports mastery probe and distinct-kind shortages", () => { + const topic = validTopic(); + topic.concepts[0]!.mastery.minimumSuccessfulProbes = 4; + topic.concepts[0]!.mastery.minimumDistinctQuestionKinds = 4; + + expect(codes(validateTutorContentQuality([topic], cases()))).toEqual(expect.arrayContaining([ + "insufficient-mastery-probes", + "insufficient-mastery-question-kinds", + "learn-content-exhausted", + ])); + }); + + it("reports an applicable Concept whose prerequisite cannot be probed", () => { + const topic = validTopic(); + topic.concepts.unshift({ + ...topic.concepts[0]!, + id: "prerequisite", + title: "Vorwissen", + indicators: [{ id: "prerequisite-indicator", description: "Vorwissen" }], + mastery: { ...topic.concepts[0]!.mastery, requiredIndicators: ["prerequisite-indicator"] }, + }); + topic.concepts[1]!.prerequisites = ["prerequisite"]; + topic.progression.entryConcepts = ["prerequisite"]; + topic.progression.preferredOrder = ["prerequisite", "values"]; + + expect(codes(validateTutorContentQuality([topic], cases()))).toContain("unreachable-prerequisite"); + }); + + it("reports a missing required DEEPEN kind and obvious post-mastery exhaustion", () => { + const topic = validTopic(); + topic.questions = [question("only-learn", "concept")]; + + expect(codes(validateTutorContentQuality([topic], cases()))).toEqual(expect.arrayContaining([ + "missing-deepening-question-kind", + "deepen-content-exhausted", + ])); + }); +}); From 2a06fe41b2b0f564f55a6ecdca159312d31bf1f3 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 20:45:15 +0200 Subject: [PATCH 7/9] fix: validate effective tutor content strategies --- .../tutor-quality-directory-validator.ts | 26 ++++++++++++++++--- .../course-content/tutor-quality-validator.ts | 16 +++++++++--- .../tutor-quality-directory-validator.test.ts | 13 ++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/server/services/course-content/tutor-quality-directory-validator.ts b/server/services/course-content/tutor-quality-directory-validator.ts index 4908cbe9..2f68fa82 100644 --- a/server/services/course-content/tutor-quality-directory-validator.ts +++ b/server/services/course-content/tutor-quality-directory-validator.ts @@ -4,6 +4,8 @@ import { parse as parseYaml } from "yaml"; import type { FullCommitSha, RepositorySlug } from "@shared/examples"; import { CourseContentLoader } from "./course-content-loader"; import { tutorQualityCasesSchema } from "./tutor-quality-schema"; +import type { LoadedCourseContentSnapshot } from "./course-content-loader"; +import { BUILT_IN_TUTOR_STRATEGY, type EffectiveTutorStrategy } from "../tutor/strategy/effective-tutor-strategy"; import { validateTutorContentQuality, type ResolvedTutorQualityCase, @@ -33,7 +35,7 @@ export async function validateTutorCourseContentDirectory(directory: string): Pr if (!parsed.success) { return [directoryIssue("invalid-quality-cases", "Tutor quality case manifest is invalid")]; } - const resolution = resolveCases(parsed.data.cases, loaded.examples); + const resolution = resolveCases(parsed.data.cases, loaded); if (resolution.issues.length > 0) return resolution.issues; return validateTutorContentQuality(loaded.tutor.topics, resolution.cases); } catch { @@ -55,12 +57,12 @@ async function readBoundedLocalFile(root: string, url: URL, maxBytes: number): P function resolveCases( cases: readonly { id: string; example: string; expectedTopics: readonly string[]; forbiddenTopics: readonly string[] }[], - examples: Awaited>["examples"], + loaded: LoadedCourseContentSnapshot, ): { cases: ResolvedTutorQualityCase[]; issues: TutorContentQualityIssue[] } { const resolved: ResolvedTutorQualityCase[] = []; const issues: TutorContentQualityIssue[] = []; for (const qualityCase of cases) { - const example = examples.find(({ id }) => id === qualityCase.example); + const example = loaded.examples.find(({ id }) => id === qualityCase.example); const main = example?.files.find(({ name }) => name === example.main); if (!example || !main) { issues.push({ @@ -76,11 +78,29 @@ function resolveCases( code: main.content, expectedTopics: qualityCase.expectedTopics, forbiddenTopics: qualityCase.forbiddenTopics, + learnStrategy: resolveCaseStrategy(loaded, example.tutorAnnotation?.strategy, "LEARN"), + deepenStrategy: resolveCaseStrategy(loaded, example.tutorAnnotation?.strategy, "DEEPEN"), }); } return { cases: resolved, issues }; } +function resolveCaseStrategy( + loaded: LoadedCourseContentSnapshot, + exampleStrategyId: string | undefined, + phase: "LEARN" | "DEEPEN", +): EffectiveTutorStrategy { + if (loaded.tutor.status !== "valid") return BUILT_IN_TUTOR_STRATEGY; + const perExample = loaded.tutor.strategies.find(({ id }) => id === exampleStrategyId); + if (perExample) return perExample; + const manifest = loaded.tutor.manifest; + const phaseStrategyId = phase === "DEEPEN" && manifest.schemaVersion === 2 + ? manifest.phaseStrategies?.deepen + : undefined; + const strategyId = phaseStrategyId ?? manifest.defaultStrategy; + return loaded.tutor.strategies.find(({ id }) => id === strategyId) ?? BUILT_IN_TUTOR_STRATEGY; +} + function directoryIssue( code: "invalid-course-content-bundle" | "invalid-quality-cases", message: string, diff --git a/server/services/course-content/tutor-quality-validator.ts b/server/services/course-content/tutor-quality-validator.ts index 091e6cac..d6bc3982 100644 --- a/server/services/course-content/tutor-quality-validator.ts +++ b/server/services/course-content/tutor-quality-validator.ts @@ -10,6 +10,7 @@ import { deepeningCriteria, hasMetDeepeningCriteria } from "../tutor/curriculum/ import { DefaultSketchFactExtractor, type SketchFacts } from "../tutor/curriculum/sketch-facts"; import { DefaultTopicMatcher } from "../tutor/curriculum/topic-matcher"; import { BUILT_IN_TUTOR_STRATEGY } from "../tutor/strategy/effective-tutor-strategy"; +import type { EffectiveTutorStrategy } from "../tutor/strategy/effective-tutor-strategy"; export interface ResolvedTutorQualityCase { readonly id: string; @@ -17,6 +18,8 @@ export interface ResolvedTutorQualityCase { readonly code: string; readonly expectedTopics: readonly string[]; readonly forbiddenTopics: readonly string[]; + readonly learnStrategy?: EffectiveTutorStrategy; + readonly deepenStrategy?: EffectiveTutorStrategy; } export type TutorContentQualityIssueCode = @@ -210,13 +213,20 @@ function validateExecutablePath( observations, new Set(history.flatMap(({ questionId }) => questionId ? [questionId] : [])), 30, - BUILT_IN_TUTOR_STRATEGY, + context.qualityCase.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY, ); if (classification.status === "mastered") { masteryReached = true; break; } - const plan = planner.start(topic, "0".repeat(40), context.facts, history, 30, BUILT_IN_TUTOR_STRATEGY); + const plan = planner.start( + topic, + "0".repeat(40), + context.facts, + history, + 30, + context.qualityCase.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY, + ); if (!plan) break; history.push(successfulTurn(plan.brief)); } @@ -235,7 +245,7 @@ function validateExecutablePath( const criteria = deepeningCriteria(topic); for (let step = 0; step <= topic.questions.length; step += 1) { if (hasMetDeepeningCriteria(topic, postMastery)) return; - const plan = planner.start(topic, "0".repeat(40), context.facts, history, 30, BUILT_IN_TUTOR_STRATEGY, { + const plan = planner.start(topic, "0".repeat(40), context.facts, history, 30, context.qualityCase.deepenStrategy ?? BUILT_IN_TUTOR_STRATEGY, { phase: "DEEPEN", preferredQuestionKinds: criteria.requiredQuestionKinds, existingQuestionKinds: postMastery.map(({ kind }) => kind), diff --git a/tests/server/services/course-content/tutor-quality-directory-validator.test.ts b/tests/server/services/course-content/tutor-quality-directory-validator.test.ts index 26838727..caaf11df 100644 --- a/tests/server/services/course-content/tutor-quality-directory-validator.test.ts +++ b/tests/server/services/course-content/tutor-quality-directory-validator.test.ts @@ -1,11 +1,14 @@ import { createHash } from "node:crypto"; +import { execFile } from "node:child_process"; import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; +import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { validateTutorCourseContentDirectory } from "../../../../server/services/course-content/tutor-quality-directory-validator"; const temporaryDirectories: string[] = []; +const executeFile = promisify(execFile); afterEach(async () => { await Promise.all(temporaryDirectories.splice(0).map((directory) => rm(directory, { recursive: true, force: true }))); @@ -34,6 +37,16 @@ describe("Tutor Course Content directory validator", () => { expect.objectContaining({ code: "invalid-quality-cases" }), ])); }); + + it("returns a nonzero CLI status when the hard gate finds issues", async () => { + const directory = await courseContentDirectory(false); + + await expect(executeFile( + path.resolve("node_modules/.bin/tsx"), + ["scripts/validate-tutor-course-content.mjs", directory], + { cwd: process.cwd() }, + )).rejects.toMatchObject({ code: 1 }); + }); }); async function courseContentDirectory(withCases = true): Promise { From 65d2bb96d4f9a503f687ec0ab9abd23ac6abe94d Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 20:45:40 +0200 Subject: [PATCH 8/9] ci: add deterministic tutor quality gate --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f3ce2786..10665a27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,8 @@ jobs: - run: npm ci - run: npm run check:node-version - run: npm run check + - name: Deterministic Tutor quality hard gate + run: npm run test:tutor-quality - run: npm run test:unit - run: npm run build From aaa52fad3d778d4ec7ab9e304a27fe9626efb270 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sun, 27 Sep 2026 20:49:25 +0200 Subject: [PATCH 9/9] test: broaden tutor quality hard gate --- .../plans/2026-09-27-tutor-quality-stage-1.md | 5 +++-- package.json | 2 +- .../tutor-quality-directory-validator.ts | 3 +-- .../course-content/tutor-quality-validator.ts | 3 +-- .../services/tutor/tutor-quality-scenarios.test.ts | 11 ++++++++--- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/plans/2026-09-27-tutor-quality-stage-1.md b/docs/superpowers/plans/2026-09-27-tutor-quality-stage-1.md index 63c8a030..8c53fd43 100644 --- a/docs/superpowers/plans/2026-09-27-tutor-quality-stage-1.md +++ b/docs/superpowers/plans/2026-09-27-tutor-quality-stage-1.md @@ -51,9 +51,10 @@ Specification: `ssot/ssot_function_definition_TutorQuality.md` **Files:** - Create: `server/services/course-content/tutor-quality-schema.ts` - Create: `server/services/course-content/tutor-quality-validator.ts` +- Create: `server/services/course-content/tutor-quality-directory-validator.ts` - Create: `tests/server/services/course-content/tutor-quality-validator.test.ts` -- Create: `scripts/validate-tutor-course-content.ts` -- Create: `tests/scripts/validate-tutor-course-content.test.ts` +- Create: `scripts/validate-tutor-course-content.mjs` +- Create: `tests/server/services/course-content/tutor-quality-directory-validator.test.ts` - Modify: `package.json` 1. Write failing unit tests for activation cases, missing coverage, unreachable Concepts/Indicators, mastery probe/kind shortages, prerequisite reachability, DEEPEN exhaustion/kind shortages, and a valid bundle. diff --git a/package.json b/package.json index f012074a..c0c85044 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "test:deployment": "tsx scripts/deployment/gateway-deployment.ts", "capacity:calibrate": "tsx scripts/calibrate-capacity.ts", "test:security:inputs": "TEST_BUDGET_SUITE=security-inputs TEST_BUDGET_MS=15000 LOG_LEVEL=warn vitest run --project=unit-node --project=unit-node-http tests/shared/input-limits.test.ts tests/shared/websocket-direction-schemas.test.ts tests/server/security/safe-paths.test.ts tests/server/routes/compiler.routes.test.ts tests/integration/simulation-state-sequence.test.ts --reporter=default --reporter=./scripts/test-budget-reporter.mjs", - "test:tutor-quality": "TEST_BUDGET_SUITE=tutor-quality TEST_BUDGET_MS=15000 LOG_LEVEL=warn vitest run --project=unit-node tests/server/services/tutor/tutor-quality-scenarios.test.ts tests/server/services/tutor/tq-reg-001-pwm.test.ts tests/server/services/course-content/tutor-quality-validator.test.ts tests/server/services/course-content/tutor-quality-directory-validator.test.ts --reporter=default --reporter=./scripts/test-budget-reporter.mjs", + "test:tutor-quality": "TEST_BUDGET_SUITE=tutor-quality TEST_BUDGET_MS=15000 LOG_LEVEL=warn vitest run --project=unit-node tests/server/services/tutor tests/server/services/course-content --reporter=default --reporter=./scripts/test-budget-reporter.mjs", "validate:tutor-course-content": "tsx scripts/validate-tutor-course-content.mjs", "test:all": "npm run test:unit && npm run test:integration && npm run test:docker", "test:watch": "LOG_LEVEL=info vitest --project=unit-client --project=unit-node --project=unit-node-http", diff --git a/server/services/course-content/tutor-quality-directory-validator.ts b/server/services/course-content/tutor-quality-directory-validator.ts index 2f68fa82..474c8a4c 100644 --- a/server/services/course-content/tutor-quality-directory-validator.ts +++ b/server/services/course-content/tutor-quality-directory-validator.ts @@ -2,9 +2,8 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { parse as parseYaml } from "yaml"; import type { FullCommitSha, RepositorySlug } from "@shared/examples"; -import { CourseContentLoader } from "./course-content-loader"; +import { CourseContentLoader, type LoadedCourseContentSnapshot } from "./course-content-loader"; import { tutorQualityCasesSchema } from "./tutor-quality-schema"; -import type { LoadedCourseContentSnapshot } from "./course-content-loader"; import { BUILT_IN_TUTOR_STRATEGY, type EffectiveTutorStrategy } from "../tutor/strategy/effective-tutor-strategy"; import { validateTutorContentQuality, diff --git a/server/services/course-content/tutor-quality-validator.ts b/server/services/course-content/tutor-quality-validator.ts index d6bc3982..711eaf49 100644 --- a/server/services/course-content/tutor-quality-validator.ts +++ b/server/services/course-content/tutor-quality-validator.ts @@ -9,8 +9,7 @@ import { import { deepeningCriteria, hasMetDeepeningCriteria } from "../tutor/curriculum/progression-state"; import { DefaultSketchFactExtractor, type SketchFacts } from "../tutor/curriculum/sketch-facts"; import { DefaultTopicMatcher } from "../tutor/curriculum/topic-matcher"; -import { BUILT_IN_TUTOR_STRATEGY } from "../tutor/strategy/effective-tutor-strategy"; -import type { EffectiveTutorStrategy } from "../tutor/strategy/effective-tutor-strategy"; +import { BUILT_IN_TUTOR_STRATEGY, type EffectiveTutorStrategy } from "../tutor/strategy/effective-tutor-strategy"; export interface ResolvedTutorQualityCase { readonly id: string; diff --git a/tests/server/services/tutor/tutor-quality-scenarios.test.ts b/tests/server/services/tutor/tutor-quality-scenarios.test.ts index c04b3732..21b99a75 100644 --- a/tests/server/services/tutor/tutor-quality-scenarios.test.ts +++ b/tests/server/services/tutor/tutor-quality-scenarios.test.ts @@ -124,7 +124,12 @@ function courseContent(contentRevision = revision) { } describe("Tutor Quality deterministic scenarios", () => { - it.each(["transition", "blocked"] as const)("retains a repaired near-duplicate question for %s outcomes", async (kind) => { + it.each([ + ["transition", "exact", "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"], + ["transition", "near", "Welche Rolle hat die Variable value und ihr Wert in diesem Sketch?"], + ["blocked", "exact", "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"], + ["blocked", "near", "Welche Rolle hat die Variable value und ihr Wert in diesem Sketch?"], + ] as const)("retains a repaired %s/%s duplicate question", async (kind, _similarity, candidate) => { const currentQuestion = "Welche Rolle spielt der Wert der Variable value im aktuellen Sketch?"; const trace = await runTutorQualityScenario({ id: `TQ-ADV-repeat-${kind}`, @@ -134,7 +139,7 @@ describe("Tutor Quality deterministic scenarios", () => { kind: "result", result: { answerRating: 5, - question: "Welche Rolle hat die Variable value und ihr Wert in diesem Sketch?", + question: candidate, }, }, planning: planningWithFollowup(kind), @@ -146,7 +151,7 @@ describe("Tutor Quality deterministic scenarios", () => { contentRevision: revision, activeTopicId: "variables-and-serial", }); - expect(trace.result?.question).not.toContain("Rolle hat die Variable value"); + expect(trace.result?.question).not.toBe(candidate); }); it("lets an application-owned TutorPlan replace provider question and metadata", async () => {