diff --git a/client/src/hooks/use-tutor.ts b/client/src/hooks/use-tutor.ts index 02c8ffe77..27daf9afe 100644 --- a/client/src/hooks/use-tutor.ts +++ b/client/src/hooks/use-tutor.ts @@ -119,15 +119,7 @@ function buildDialogTurn( const baseTurn = { question, answer, - ...(response.feedback ? { feedback: response.feedback } : {}), - ...(response.topicId ? { topicId: response.topicId } : {}), - ...(response.conceptId ? { conceptId: response.conceptId } : {}), - ...(response.questionId ? { questionId: response.questionId } : {}), - ...(response.indicatorId ? { indicatorId: response.indicatorId } : {}), - ...(response.questionKind ? { questionKind: response.questionKind } : {}), - ...(response.strategyId ? { strategyId: response.strategyId } : {}), - ...(response.strategySource ? { strategySource: response.strategySource } : {}), - ...(response.contentRevision ? { contentRevision: response.contentRevision } : {}), + ...buildDialogTurnMetadata(response), }; if (response.responseStyle === "philosophical") { return { ...baseTurn, responseStyle: "philosophical" }; @@ -139,6 +131,26 @@ function buildDialogTurn( }; } +function buildDialogTurnMetadata(response: TutorResponse): Partial> { + const fields = [ + ["feedback", response.feedback], + ["topicId", response.topicId], + ["conceptId", response.conceptId], + ["questionId", response.questionId], + ["indicatorId", response.indicatorId], + ["questionKind", response.questionKind], + ["strategyId", response.strategyId], + ["strategySource", response.strategySource], + ["contentRevision", response.contentRevision], + ["learningPhase", response.learningPhase], + ["activeTopicId", response.activeTopicId], + ["masteredTopicIds", response.masteredTopicIds], + ["progressionBlockedReason", response.progressionBlockedReason], + ["extensionTargetTopicId", response.extensionTargetTopicId], + ] as const; + return Object.fromEntries(fields.filter(([, value]) => value !== undefined && value !== "")) as Partial>; +} + function collectAnswerRatings(history: readonly TutorDialogTurn[]): readonly TutorAnswerRating[] { return history.flatMap((turn) => turn.answerRating === undefined ? [] : [turn.answerRating]); } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a63779b4c..be502d473 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -9,6 +9,7 @@ Diese Datei beschreibt die grundlegende Architektur von UnoSim mit Fokus auf Dat - Dieses Dokument ist der aktuelle Architekturüberblick. Es beschreibt Komponenten, Datenflüsse, State Ownership und Betriebsmodell bewusst zusammenfassend. - Verbindliche Detailentscheidungen bleiben in den ADRs: Gateway/Auth/Security in `adr/0001-authentication-and-gateway-contract.md`, UnifiedScrollArea in `adr/0002-unified-scroll-area.md`, Skalierung/HA in `adr/0003-scalability-and-ha-model.md`, die historische Tutor-Pilotentscheidung in `adr/0004-repository-based-tutor-curriculum.md`, die dynamische Examples-Auswahl in `adr/0005-browser-scoped-external-examples.md` und der aktuelle Course-Content-/Tutor-Vertrag in `adr/0006-unified-course-content-and-tutor-strategy.md`. +- Die mastery-driven didaktische Phasenentscheidung ist in `adr/0007-mastery-driven-tutor-progression.md` normativ ergänzt. - Externe iframe-API-Verträge liegen in `EXTERNAL_API.md`; Feature-Details liegen in den thematischen SSOT-Dateien unter `../ssot/`. - Versionsverträge: REST `1.0.0` (`Accept-Version`/`X-UnoSim-API-Version`), WebSocket `1.0.0` (`handshake.protocolVersion`) und iframe `postMessage` `1.4.0`; inkompatible Änderungen benötigen eine neue Major-Version und Migration. - Git enthält die Historie früherer Planungs- und Risikoquellen; der aktuelle @@ -137,6 +138,22 @@ data enters only structured didactic context. The application-owned system prompt, safety rules, provider isolation, privacy rules, response validation, and editor boundary are never repository-controlled. +For an active applicable Topic, the Tutor additionally tracks the application- +owned session-local didactic phase `LEARN`, `DEEPEN`, or `EXPAND`. Deterministic +Topic mastery is derived from the existing concept mastery criteria; the LLM +cannot declare mastery. After mastery, currently applicable unmastered Topics +are selected by normal precedence and remain in LEARN; only when none remain +does the mastered Topic enter DEEPEN. DEEPEN changes to EXPAND after bounded +successful transfer evidence. A changed sketch reruns Topic matching, and a +newly selected unmastered Topic starts in LEARN. An unresolved unmastered Topic +blocks DEEPEN/EXPAND in a safe `LEARN` content-exhaustion state. A mastered +Topic made inapplicable by a sketch edit is suspended, not erased; if it later +matches again in the same session, its retained DEEPEN/EXPAND phase resumes. +A free Tutor without an active Topic remains available with its +EffectiveTutorStrategy but does not claim formal Topic mastery. This state is +pinned to the same opaque Tutor session and immutable Course revision and is +not a persistent learner profile. + The server extracts an optional terminal `@unosim-tutor` annotation from only the declared main `.ino` file before the Example leaves the Course Content boundary. The browser/editor, compiler, and simulator receive cleaned source @@ -149,6 +166,7 @@ The old separate Tutor source is superseded. The in-tree curriculum files are fixtures and authoring examples only. The normative details are in [ssot_function_definition_CourseContent.md](../ssot/ssot_function_definition_CourseContent.md), [adr/0006-unified-course-content-and-tutor-strategy.md](adr/0006-unified-course-content-and-tutor-strategy.md), +[adr/0007-mastery-driven-tutor-progression.md](adr/0007-mastery-driven-tutor-progression.md), and [ssot_function_definition_LearningQuestions.md](../ssot/ssot_function_definition_LearningQuestions.md). ### Dynamic Course Content selection diff --git a/docs/adr/0007-mastery-driven-tutor-progression.md b/docs/adr/0007-mastery-driven-tutor-progression.md new file mode 100644 index 000000000..2694cffd0 --- /dev/null +++ b/docs/adr/0007-mastery-driven-tutor-progression.md @@ -0,0 +1,261 @@ +# ADR 0007: Mastery-driven Tutor progression + +- Status: Proposed +- Date: 2026-09-26 +- Owners: UnoSim maintainers and platform operators + +## Context + +The unified Course Content contract already separates reusable Topics from the +EffectiveTutorStrategy and uses deterministic concept-level mastery criteria +inside the curriculum planner. The current contract does not yet define a +Topic-level didactic phase, a session-local mastery latch, or a bounded path +from demonstrated Topic understanding to transfer and extension. + +The Tutor must be able to deepen learning after deterministic Topic mastery and +offer a bounded program extension without becoming an autonomous code editor, +an unrestricted workflow engine, or a persistent learner model. + +## Decision + +UnoSim defines three application-owned phases for an active applicable Topic: + +1. `LEARN` — gather and verify Topic understanding; +2. `DEEPEN` — use the mastered Topic through application, prediction, transfer, + changed examples, and small conceptual variations; +3. `EXPAND` — offer a bounded extension direction while leaving all program + changes to the learner. + +The concerns remain distinct: + +- Topic answers **WHAT** is learned; +- EffectiveTutorStrategy answers **HOW** the Tutor teaches; +- didactic phase answers **WHERE** the learner is in the bounded progression. + +No repository may define another phase name, encode a phase only as a +strategy ID, or provide a workflow/expression language. + +## Schema versioning + +Published strict schema versions are not retroactively extended: + +- Tutor manifest `schemaVersion: 1` remains unchanged and has no + `phaseStrategies`; `schemaVersion: 2` retains all v1 fields and may add only + the fixed `deepen` and `expand` phase strategy references. +- Curriculum Topic `schemaVersion: 1` remains unchanged and has no + `deepening` or `extensions`; `schemaVersion: 2` retains all v1 semantics and + may add those bounded fields. + +Future implementations support both versions. Unknown v2-only fields in a +strict v1 document are invalid, not silently reinterpreted. A v1 document uses +the application deepening default and has no repository phase-strategy map. + +## Deterministic mastery + +The application, not the LLM, decides mastery. Existing concept-level Topic +criteria remain authoritative: + +- a successful probe is a valid rated observation whose `answerRating` meets + the concept's `successRatingAtLeast`; +- `minimumSuccessfulProbes` counts those successful observations; +- every `requiredIndicators` entry needs a successful observation tied to that + indicator; +- `minimumDistinctQuestionKinds` counts distinct kinds among successful + observations; +- `recentWeakAnswersAllowed` applies to the existing trailing weak-answer + streak after the latest non-weak observation; a weak answer means an + `answerRating` of `1` or `2`. + +One rated turn is associated with one question, concept, indicator, and kind. It +may count in each relevant aggregate but cannot satisfy multiple indicator IDs. +A rating of 5 is evidence only and never bypasses the configured criteria. +After each valid rated turn, the application reevaluates mastery against the +accumulated evidence. Mastery evidence and `effectiveDifficulty` remain +separate; adaptive difficulty does not declare mastery or replace a criterion. + +The TopicMatcher activation result is only a candidate. A question is +fact-applicable when it is schema-valid and every question `requires` entry +matches the current sketch facts. A concept enters the current Topic mastery +domain when at least one of its questions is fact-applicable. A Topic is a +probeable acquisition candidate only when that domain is non-empty and the +normal planner can produce at least one fact-applicable question under the +active strategy, difficulty, repetition, and anti-loop rules. + +A Topic activation match with a non-empty mastery domain, unmet criteria, and +no fresh probe can currently be produced is an unmastered unresolved Topic and +a progression blocker. It is not silently omitted. A Topic activation match +with an empty mastery domain is not mastered and is not an acquisition +candidate or progression blocker; it cannot justify a Topic question or a +mastery claim. Topic mastery is true only when every concept in the non-empty +domain satisfies its existing criteria. A concept with no fact-applicable +question is outside the domain and creates no false requirement. + +Because the current Topic schema puts `requires` on Questions rather than on +Concepts or individual indicators, it cannot statically guarantee probeability +for every required indicator. The application must leave such a concept +unresolved and block Topic mastery; it must not invent hidden concept-level +requirements or silently ignore the missing probe. + +Mastery is latched per Topic, session, and Course revision; later weak answers +do not erase the latch while the session and Course revision remain valid. +Topic applicability controls use, not retention. If a learner edit makes a +mastered Topic fact-inapplicable, it is suspended as active while its mastered +state, evidence, and retained DEEPEN/EXPAND phase remain in session state. No +fact-dependent question or Topic claim may use it while suspended. If it later +becomes applicable in the same session, it resumes that retained phase rather +than restarting LEARN. A new dialog/session or Course source, ref, revision, or +example-context reset clears the state. + +## Phase transitions + +When the active Topic reaches mastery, UnoSim recomputes currently applicable +Topics and applies normal precedence: applicable embedded primary Topic, other +applicable embedded Topics, then fact-matched repository Topics. Mastered +Topics are skipped for acquisition selection during the current session. If an +applicable unmastered and probeable Topic remains, the highest-precedence such +Topic becomes active and remains in LEARN. A mastered Topic A plus an +applicable probeable unmastered Topic B therefore selects Topic B in LEARN, +never DEEPEN on A. + +If no probeable unmastered Topic remains but an unresolved unmastered Topic +remains, the conceptual phase remains LEARN and progression is blocked with +`progressionBlockedReason: content-exhausted`. The existing null-plan +free-Tutor fallback may provide one bounded, user-initiated sketch-grounded +remediation question without Topic mastery claims or new Topic evidence. If +that is not safe, the request fails as controlled content exhaustion without +mutating phase or mastery state. DEEPEN is admitted only when every relevant +acquisition Topic is mastered or no longer fact-applicable and no unresolved +unmastered Topic blocks progression. No special user control or second LLM +decision is required. + +Teacher-authored `learningObjectives` remain additional emphasis in all three +phases. They may guide question, deepening, and extension direction, but never +define mastery, activate a Topic, or select a strategy. + +The default `DEEPEN -> EXPAND` criterion is two successful post-mastery probes, +each rated at least 4, including at least one transfer probe, with no trailing +weak probe. Deepening evidence starts at the LEARN-to-DEEPEN transition. +Topic schemaVersion 2 may configure bounded equivalents as: + +```yaml +deepening: + minimumSuccessfulProbes: 2 + successRatingAtLeast: 4 + requiredQuestionKinds: + - transfer + recentWeakAnswersAllowed: 0 +``` + +The bounds are probes `1..10`, threshold `3..5`, one to three unique kinds +from `application`, `prediction`, and `transfer`, and weak allowance `0..3`. +SchemaVersion 1 has no such field. + +When the sketch changes, UnoSim re-extracts facts and reruns normal Topic +precedence. A new unmastered and probeable Topic during DEEPEN or EXPAND starts +in LEARN. A new unresolved unmastered Topic moves progression to blocked LEARN. +If the current Topic becomes inapplicable, a probeable unmastered Topic starts +in LEARN, an unresolved Topic blocks in LEARN, an applicable mastered Topic +resumes its retained DEEPEN or EXPAND phase, and if no applicable Topic can be +selected the Tutor falls back to free mode. A listed extension never activates +a Topic by itself. + +## Post-mastery strategy selection + +LEARN retains the existing precedence: + +1. valid embedded Example strategy; +2. repository `defaultStrategy`; +3. `built-in-default`. + +A Tutor manifest schemaVersion 2 may contain only the fixed optional entries +`phaseStrategies.deepen` and `phaseStrategies.expand`. For DEEPEN and EXPAND, +the precedence is: + +1. valid embedded Example strategy; +2. the configured strategy for the fixed phase; +3. repository `defaultStrategy`; +4. `built-in-default`. + +An absent phase entry falls back to the already effective LEARN strategy. +Phase entries reference strategies directly and cannot chain. Unknown or +invalid references invalidate the complete Tutor capability under the existing +capability-scoped fallback rule. This supports, for example, precision policy +in LEARN and exploration policy after mastery without creating circular +strategy transitions. + +## Topic extensions + +Only a Curriculum Topic schemaVersion 2 may declare at most eight extensions. +Each extension contains only: + +```yaml +topic: functions +objective: Repeated behavior can be moved into a function. +``` + +The target must be a safe Topic ID in the same validated Tutor bundle. The +objective is bounded teacher-authored data (maximum 500 Unicode characters, +trimmed, no control characters, URLs, prompts, roles, scripts, templates, or +expressions). It is educational guidance, not activation authority. + +## Session-local state and boundaries + +Conceptually, the phase state is pinned to the existing opaque Tutor session +and immutable Course revision and contains the active Topic, current phase, +per-Topic mastery state/evidence, retained per-Topic post-mastery phase, and a +progression-blocked reason when applicable. It remains process-local with the +current one-hour TTL. Horizontal multi-instance deployment requires shared +state or a deliberately designed equivalent such as sticky sessions. + +Course source/ref/revision/context changes reset the dialog and didactic state. +`New learning question` and `New dialog` begin a fresh didactic session in the +current validated context: history, ratings, active Topic, phase, mastery +evidence, deepening evidence, and effective difficulty reset. Configured start +difficulty, provider/model selection, and transient credential behavior retain +their existing contracts. The next request is still explicitly user initiated. + +Formal phases do not exist for free Tutor without an active Topic. Arbitrary +sketches with a matching Topic use the same phase model and repository/built-in +strategy resolution as Examples. + +The one-primary-question rule, current-sketch factual authority, no-full- +solution rule, no automatic editor changes, provider/privacy boundaries, +response validation, and no persistent learner profile remain unchanged. + +## Consequences + +Positive consequences: + +- mastery and phase transitions are deterministic and testable; +- post-mastery teaching can change policy without conflating phase and + strategy; +- Topic extensions provide bounded forward direction without forcing Topics; +- arbitrary sketches can benefit whenever a Topic is factually applicable; +- no persistent learner model or autonomous LLM request is introduced. + +Costs and constraints: + +- Tutor session state must carry bounded phase and evidence metadata; +- implementations must support the explicit Topic and Tutor-manifest v2 + extensions before repositories can author deepening, extensions, or phase + strategies; +- invalid phase/extension data disables the complete repository Tutor bundle; +- multi-instance deployment needs shared or sticky Tutor-session state. + +## Rejected alternatives + +- **LLM-declared mastery:** rejected because mastery must be reproducible, + bounded, and application-controlled. +- **Phase encoded only as a strategy ID:** rejected because Topic content, + teaching policy, and learner progression are separate concerns. +- **Strategy-to-strategy chaining:** rejected because it creates circular and + ambiguous transitions; fixed phase entries are direct references only. +- **Activating a Topic because an extension names it:** rejected because only + deterministic current-sketch facts authorize Topic activation. +- **Automatic program modification:** rejected because the learner must decide + whether and how to change the sketch. +- **Persistent learner profiling:** rejected because this feature is a + session-local didactic control loop, not grading or a long-term competence + database. +- **Autonomous background Tutor requests:** rejected to preserve explicit user + initiation and cost control. diff --git a/docs/superpowers/plans/2026-09-27-tutor-mastery-progression-stage-a.md b/docs/superpowers/plans/2026-09-27-tutor-mastery-progression-stage-a.md new file mode 100644 index 000000000..4867535a9 --- /dev/null +++ b/docs/superpowers/plans/2026-09-27-tutor-mastery-progression-stage-a.md @@ -0,0 +1,46 @@ +# Implement mastery-driven Tutor progression Stage A + +## Scope + +Implement the approved mastery-progression contract from SSOT commit +`fdd641bbae30ceae8347f9835620c3d1274462d9` in the isolated worktree only. +Preserve schema-v1 compatibility, capability-scoped fallback, the existing +free-Tutor safety contract, Course revision/session pinning, and all protected +unrelated areas. + +## Tasks + +1. Audit the approved SSOT and current implementation; record gaps and + unresolved implementation rulings in the execution ledger. +2. Add failing schema/data-model tests for Tutor manifest v2, Topic v2, + extensions, phase strategies, and v1 compatibility; implement strict + normalization and capability-scoped loading. +3. Add failing planner tests for Topic mastery-domain classification, + probeable/unresolved/content-exhausted outcomes, multi-topic precedence, + LEARN/DEEPEN/EXPAND transitions, deepening evidence, and retention; then + implement the deterministic state machine. +4. Add phase-strategy and extension-flow tests; implement phase-specific + effective strategy selection and bounded extension guidance. +5. Thread server-owned progression state through Tutor planning/session + boundaries and add the minimum additive API/client state needed for the + visible Tutor phase and blocked diagnostics, without exposing evidence or + teacher-controlled instructions. +6. Add regression coverage for free Tutor behavior, revision/context reset, + invalid capability isolation, strategy precedence, hidden-source and + security invariants; run all focused suites. +7. Run the complete release gates, review the diff and protected worktrees, + then create focused logical commits and open (but do not merge) a PR only + if every required gate passes. + +## Verification gates + +- `npm run check` +- `npm run test:unit` +- `npm run test:integration` +- `npm run test:coverage` +- `npm run build` +- `npm run check:docs` +- `npm run sonar` +- `git diff --check` +- focused Tutor/Course Content/strategy/session/API/client suites +- Playwright; Docker/Arduino only when affected by the implementation diff --git a/server/services/course-content/course-content-loader.ts b/server/services/course-content/course-content-loader.ts index 14fadaa85..32c7fc4c6 100644 --- a/server/services/course-content/course-content-loader.ts +++ b/server/services/course-content/course-content-loader.ts @@ -194,11 +194,43 @@ function validateTutorReferences( strategies: readonly EffectiveTutorStrategy[], annotations: ReadonlyMap, ): void { - if (manifest.defaultStrategy !== undefined && !strategies.some(({ id }) => id === manifest.defaultStrategy)) { - throw new Error("Tutor default strategy is not enumerated"); - } const topicIds = new Set(topics.map(({ id }) => id)); const strategyIds = new Set(strategies.map(({ id }) => id)); + validateDefaultStrategy(manifest, strategyIds); + validatePhaseStrategies(manifest, strategyIds); + validateTopicExtensions(topics, topicIds); + validateAnnotationReferences(annotations, topicIds, strategyIds); +} + +function validateDefaultStrategy(manifest: CourseContentTutorManifest, strategyIds: ReadonlySet): void { + if (manifest.defaultStrategy !== undefined && !strategyIds.has(manifest.defaultStrategy)) { + throw new Error("Tutor default strategy is not enumerated"); + } +} + +function validatePhaseStrategies(manifest: CourseContentTutorManifest, strategyIds: ReadonlySet): void { + if (manifest.schemaVersion !== 2) return; + for (const phaseStrategy of Object.values(manifest.phaseStrategies ?? {})) { + if (phaseStrategy !== undefined && !strategyIds.has(phaseStrategy)) { + throw new Error(`Tutor phase strategy is not enumerated: ${phaseStrategy}`); + } + } +} + +function validateTopicExtensions(topics: readonly CurriculumTopic[], topicIds: ReadonlySet): void { + for (const topic of topics) { + const extensions = topic.schemaVersion === 2 ? topic.extensions ?? [] : []; + for (const extension of extensions) { + if (!topicIds.has(extension.topic)) throw new Error(`Tutor extension target is not enumerated: ${extension.topic}`); + } + } +} + +function validateAnnotationReferences( + annotations: ReadonlyMap, + topicIds: ReadonlySet, + strategyIds: ReadonlySet, +): void { for (const annotation of annotations.values()) { validateAnnotationTopics(annotation, topicIds); if (annotation.strategy !== undefined && !strategyIds.has(annotation.strategy)) { diff --git a/server/services/course-content/course-content-schema.ts b/server/services/course-content/course-content-schema.ts index bcf4231c7..db132900e 100644 --- a/server/services/course-content/course-content-schema.ts +++ b/server/services/course-content/course-content-schema.ts @@ -38,14 +38,30 @@ export const courseContentStrategyEntrySchema = z.object({ sha256: z.string().regex(/^[a-f0-9]{64}$/i), }).strict(); -export const courseContentTutorManifestSchema = z.object({ +const tutorManifestV1Schema = z.object({ schemaVersion: z.literal(1), defaultStrategy: z.string().regex(SAFE_ID).optional(), topics: z.array(courseContentTopicEntrySchema).max(64), strategies: z.array(courseContentStrategyEntrySchema).max(64), }).strict(); +const phaseStrategiesSchema = z.object({ + deepen: z.string().regex(SAFE_ID).optional(), + expand: z.string().regex(SAFE_ID).optional(), +}).strict(); + +const tutorManifestV2Schema = z.object({ + schemaVersion: z.literal(2), + defaultStrategy: z.string().regex(SAFE_ID).optional(), + topics: z.array(courseContentTopicEntrySchema).max(64), + strategies: z.array(courseContentStrategyEntrySchema).max(64), + phaseStrategies: phaseStrategiesSchema.optional(), +}).strict(); + +export const courseContentTutorManifestSchema = z.union([tutorManifestV1Schema, tutorManifestV2Schema]); + export type CourseContentTutorManifest = z.infer; +export type TutorPhase = "LEARN" | "DEEPEN" | "EXPAND"; type InvalidCapability = { readonly status: "invalid"; readonly reason: string }; diff --git a/server/services/course-content/course-content-session.ts b/server/services/course-content/course-content-session.ts index 564486727..ec9b0c759 100644 --- a/server/services/course-content/course-content-session.ts +++ b/server/services/course-content/course-content-session.ts @@ -3,6 +3,7 @@ import type { ExamplesRef, FullCommitSha, RepositorySlug } from "@shared/example import type { RequestContext } from "../examples/source-provider"; import type { TutorCapability } from "./course-content-loader"; import type { ExampleTutorAnnotation } from "./embedded-tutor-annotation"; +import { createTutorProgressionState, type TutorProgressionState } from "../tutor/curriculum/progression-state"; export interface TutorCourseContentRequest { readonly repository: RepositorySlug; @@ -14,6 +15,7 @@ export interface TutorCourseContentRequest { export interface ResolvedTutorCourseContent extends TutorCourseContentRequest { readonly tutor: TutorCapability; readonly exampleTutorAnnotation?: ExampleTutorAnnotation; + readonly progressionState?: TutorProgressionState; } export interface TutorCourseContentResolver { @@ -38,7 +40,10 @@ export class TutorCourseContentSessionStore { create(identity: string, content: ResolvedTutorCourseContent): string { this.prune(); const handle = randomUUID(); - this.sessions.set(handle, { identity, content, expiresAt: this.now() + this.ttlMs }); + const pinnedContent = content.progressionState + ? content + : { ...content, progressionState: createTutorProgressionState(content.revision) }; + this.sessions.set(handle, { identity, content: pinnedContent, expiresAt: this.now() + this.ttlMs }); return handle; } diff --git a/server/services/tutor/curriculum-tutor-adapter.ts b/server/services/tutor/curriculum-tutor-adapter.ts index 3b3cab313..fe4febf06 100644 --- a/server/services/tutor/curriculum-tutor-adapter.ts +++ b/server/services/tutor/curriculum-tutor-adapter.ts @@ -6,15 +6,38 @@ import type { TutorCapability } from "../course-content/course-content-loader"; import type { ExampleTutorAnnotation } from "../course-content/embedded-tutor-annotation"; import { resolveEffectiveTutorStrategy, + type EffectiveTutorStrategy, type StrategyResolution, } from "./strategy/effective-tutor-strategy"; import { DefaultLearningPlanner, + classifyTopic, + collectObservations, type LearningPlanner, + type Observation, + type TopicClassification, } from "./curriculum/learning-planner"; import { DefaultSketchFactExtractor, type SketchFactExtractor } from "./curriculum/sketch-facts"; -import { DefaultTopicMatcher, type TopicMatcher } from "./curriculum/topic-matcher"; -import type { TutorPlan, TutorPlanningContentContext, TutorPlanningExtension } from "./tutor-planning"; +import { DefaultTopicMatcher, type TopicMatch, type TopicMatcher } from "./curriculum/topic-matcher"; +import type { + TutorPlan, + TutorPlanningBlocked, + TutorPlanningContentContext, + TutorPlanningExtension, + TutorPlanningResult, + TutorExpansionBrief, +} from "./tutor-planning"; +import { + appendEvidence, + createTutorProgressionState, + deepeningCriteria, + hasMetDeepeningCriteria, + markExpansionTargetUsed, + markTopicMastered, + type DidacticPhase, + type TutorProgressionState, +} from "./curriculum/progression-state"; +import type { CurriculumQuestion, CurriculumTopic } from "./curriculum/curriculum-schema"; export interface CurriculumTutorAdapterDependencies { readonly courseContent?: CourseContentSnapshotProvider; @@ -30,9 +53,20 @@ export interface CourseContentSnapshotProvider { readonly tutor?: TutorCapability; readonly exampleId?: string; readonly exampleTutorAnnotation?: ExampleTutorAnnotation; + readonly progressionState?: TutorProgressionState; } | null>; } +type TutorFollowupInput = { + readonly code: string; + readonly history: readonly TutorDialogTurn[]; + readonly currentQuestion: string; + readonly rating: Parameters>[5]; + readonly difficulty: TutorDifficulty; + readonly exampleId?: string; + readonly courseContent?: TutorPlanningContentContext; +}; + export class CurriculumTutorAdapter implements TutorPlanningExtension { private readonly courseContent?: CourseContentSnapshotProvider; private readonly repository?: DidacticContentRepository; @@ -48,52 +82,101 @@ export class CurriculumTutorAdapter implements TutorPlanningExtension { this.planner = deps.planner ?? new DefaultLearningPlanner(); } - async resolveStrategy(input: { courseContent?: TutorPlanningContentContext }): Promise { + async resolveStrategy(input: { code?: string; courseContent?: TutorPlanningContentContext }): Promise { try { const snapshot = input.courseContent ?? (this.courseContent ? await this.courseContent.getSnapshot() : null); - return this.resolveSnapshotStrategy(snapshot); + const phase = this.resolveActivePhase(snapshot, input.code); + return this.resolveSnapshotStrategy(snapshot, undefined, phase); } catch { return resolveEffectiveTutorStrategy({}); } } - async planInitial(input: { code: string; history: readonly TutorDialogTurn[]; difficulty: TutorDifficulty; exampleId?: string; courseContent?: TutorPlanningContentContext }): Promise { - const context = await this.match(input.code, input.courseContent); + async planInitial(input: { code: string; history: readonly TutorDialogTurn[]; difficulty: TutorDifficulty; exampleId?: string; courseContent?: TutorPlanningContentContext }): Promise { + const context = await this.match(input.code, input.history, input.difficulty, input.courseContent, input.exampleId); if (!context) return null; - const plan = this.planner.start(context.topic, context.revision, context.facts, input.history, input.difficulty, context.strategy?.strategy); - return plan ? normalizePlan(plan, context.strategy) : null; + if (context.blocked) return context.blocked; + return this.startPlan(context, input.history, input.difficulty, context.phase); } - async planFollowup(input: { - code: string; - history: readonly TutorDialogTurn[]; - currentQuestion: string; - rating: Parameters>[5]; - difficulty: TutorDifficulty; - exampleId?: string; - courseContent?: TutorPlanningContentContext; - }): Promise { - const context = await this.match(input.code, input.courseContent); + async planFollowup(input: TutorFollowupInput): Promise { + const context = await this.match(input.code, input.history, input.difficulty, input.courseContent, input.exampleId); if (!context) return null; + if (context.blocked) return context.blocked; + recordFollowupObservation(context, input); + const stateUpdate = updateStateAfterFollowup(context, input); + const refreshed = await this.match(input.code, input.history, input.difficulty, input.courseContent, input.exampleId); + if (!refreshed) return null; + if (refreshed.blocked) return refreshed.blocked; + return this.continueFollowup(context, refreshed, input, stateUpdate); + } + + private continueFollowup(context: AdapterContext, refreshed: AdapterContext, input: TutorFollowupInput, stateUpdate: FollowupStateUpdate): TutorPlanningResult | null { + const activeChanged = refreshed.topic.id !== context.topic.id; + if (!activeChanged && stateUpdate.status === "unresolved") return blockedResult(refreshed, context.state); + const nextPhase = refreshed.phase; + const planningHistory = activeChanged + ? input.history + : progressionHistory(refreshed.topic, input.history, context.state, input.currentQuestion); + if (activeChanged) return this.startPlan(refreshed, planningHistory, input.difficulty, nextPhase); + // The answer is evaluated by the strategy that produced the current turn. + // A phase transition is committed to session state now, but its first plan + // and strategy are exposed at the next request boundary. + if (nextPhase !== context.phase) return this.currentTurnPlan(context, planningHistory, input.difficulty); + if (nextPhase === "DEEPEN" && hasMetDeepeningCriteria(refreshed.topic, context.state.postMasteryEvidence[refreshed.topic.id] ?? [])) { + context.state.phase = "EXPAND"; + context.state.retainedPhases[refreshed.topic.id] = "EXPAND"; + return this.currentTurnPlan(context, planningHistory, input.difficulty); + } + return this.advancePlan(refreshed, input); + } + + private startPlan(context: AdapterContext, history: readonly TutorDialogTurn[], difficulty: TutorDifficulty, phase: DidacticPhase): TutorPlanningResult | null { + if (phase === "EXPAND" && context.expansionBrief) { + markExpansionTargetUsed(context.state, context.expansionBrief.sourceTopicId, context.expansionBrief.targetTopicId); + return buildExpansionPlan(context, phase, context.expansionBrief); + } + const plan = this.planner.start(context.topic, context.revision, context.facts, history, difficulty, context.strategy.strategy, progressionOptions(context.topic, phase, context.state)); + if (plan) return normalizePlan(plan, context.strategy, context.state, phase, context.extensionTargetTopicId, context.expansionBrief); + return phase === "LEARN" ? null : exhaustionResult(context, phase); + } + + private currentTurnPlan(context: AdapterContext, history: readonly TutorDialogTurn[], difficulty: TutorDifficulty): TutorPlanningResult { + return this.startPlan(context, history, difficulty, context.phase) + ?? transitionResult(context); + } + + private advancePlan(context: AdapterContext, input: TutorFollowupInput): TutorPlanningResult | null { const plan = this.planner.advance(context.topic, context.revision, context.facts, input.history, input.currentQuestion, input.rating, { difficulty: input.difficulty, - strategy: context.strategy?.strategy, + strategy: context.strategy.strategy, + ...progressionOptions(context.topic, context.state.phase ?? context.phase, context.state), }); - return plan ? normalizePlan(plan, context.strategy) : null; + const phase = context.state.phase ?? context.phase; + if (plan) return normalizePlan(plan, context.strategy, context.state, phase, context.extensionTargetTopicId, context.expansionBrief); + return phase === "LEARN" ? null : exhaustionResult(context, phase); } - private async match(code: string, supplied?: TutorPlanningContentContext) { + private async match(code: string, history: readonly TutorDialogTurn[], difficulty: TutorDifficulty, supplied?: TutorPlanningContentContext, exampleId?: string) { try { const snapshot = supplied ?? (this.courseContent ? await this.courseContent.getSnapshot() : null); if (snapshot) { - return this.matchCourseContent(code, snapshot); + return this.matchCourseContent(code, history, difficulty, snapshot, exampleId); } if (!this.courseContent) { const legacy = this.repository ? await this.repository.getSnapshot() : null; if (!legacy) return null; const facts = this.factExtractor.extract(code); const match = this.topicMatcher.match(legacy.topics, facts)[0]; - return match ? { revision: legacy.revision, facts, topic: match.topic, strategy: resolveEffectiveTutorStrategy({}) } : null; + if (!match) return null; + return { + revision: legacy.revision, + facts, + topic: match.topic, + strategy: resolveEffectiveTutorStrategy({}), + phase: "LEARN", + state: createTutorProgressionState(legacy.revision), + } satisfies AdapterContext; } return null; } catch { @@ -101,32 +184,50 @@ export class CurriculumTutorAdapter implements TutorPlanningExtension { } } - private matchCourseContent(code: string, snapshot: TutorPlanningContentContext) { + private matchCourseContent(code: string, history: readonly TutorDialogTurn[], difficulty: TutorDifficulty, snapshot: TutorPlanningContentContext, requestedExampleId?: string): AdapterContext | null { if (snapshot.tutor?.status !== "valid" || snapshot.tutor.topics.length === 0) return null; const facts = this.factExtractor.extract(code); const matches = this.topicMatcher.match(snapshot.tutor.topics, facts); - const byId = new Map(matches.map((match) => [match.topic.id, match])); - const annotation = snapshot.exampleTutorAnnotation; - const boundIds = [ - ...(annotation?.primaryTopic ? [annotation.primaryTopic] : []), - ...(annotation?.topics ?? []), - ]; - const boundMatch = boundIds - .map((id) => byId.get(id)) - .find((match): match is NonNullable => match !== undefined); - const match = boundMatch ?? matches[0]; + const exampleContextId = snapshot.exampleId ?? requestedExampleId; + const annotation = exampleContextId === undefined ? undefined : snapshot.exampleTutorAnnotation; + const orderedMatches = orderTopicMatches(matches, annotation); + const state = snapshot.progressionState?.revision === snapshot.revision + ? snapshot.progressionState + : createTutorProgressionState(snapshot.revision); + const learnStrategy = this.resolveSnapshotStrategy(snapshot, annotation, "LEARN"); + const classifications = classifyMatches(orderedMatches, facts, history, state, difficulty, learnStrategy.strategy); + const selected = selectProgressionMatch(classifications, state); + const match = selected?.match; if (!match) return null; - return { + const classification = selected?.classification; + const phase = phaseForTopic(state, match.topic.id, classification?.status); + const previousActiveTopicId = state.activeTopicId; + const previousPhase = state.phase ?? "LEARN"; + state.activeTopicId = match.topic.id; + state.phase = phase; + state.progressionBlockedReason = undefined; + const strategy = this.resolveSnapshotStrategy(snapshot, annotation, phase); + const extensionTargetTopicId = resolveExtensionTarget(snapshot.tutor.topics, previousActiveTopicId, previousPhase, match.topic.id); + const expansionBrief = resolveExpansionBrief(snapshot.tutor.topics, match.topic, phase, state); + const context = { revision: snapshot.revision, facts, topic: match.topic, - strategy: this.resolveSnapshotStrategy(snapshot, annotation), + strategy, + phase, + state, + ...(extensionTargetTopicId ? { extensionTargetTopicId } : {}), + ...(expansionBrief ? { expansionBrief } : {}), }; + return classification?.status === "unresolved" + ? { ...context, blocked: blockedResult(context, state) } + : context; } private resolveSnapshotStrategy( snapshot: TutorPlanningContentContext | null, knownAnnotation?: ExampleTutorAnnotation, + phase: DidacticPhase = "LEARN", ): StrategyResolution { if (snapshot?.tutor?.status !== "valid") return resolveEffectiveTutorStrategy({}); const tutor = snapshot.tutor; @@ -137,11 +238,115 @@ export class CurriculumTutorAdapter implements TutorPlanningExtension { const perExample = annotation?.strategy === undefined ? undefined : tutor.strategies.find(({ id }) => id === annotation.strategy); - return resolveEffectiveTutorStrategy({ perExample, repositoryDefault }); + const phaseStrategyId = phase === "LEARN" || tutor.manifest.schemaVersion !== 2 + ? undefined + : tutor.manifest.phaseStrategies?.[phase.toLowerCase() as "deepen" | "expand"]; + const phaseStrategy = phaseStrategyId === undefined + ? undefined + : tutor.strategies.find(({ id }) => id === phaseStrategyId); + return resolveEffectiveTutorStrategy({ perExample, repositoryDefault: phaseStrategy ?? repositoryDefault }); + } + + private resolveActivePhase(snapshot: TutorPlanningContentContext | null, code?: string): DidacticPhase { + const state = snapshot?.progressionState; + if (!snapshot || !code || !state?.activeTopicId || !state.phase || state.phase === "LEARN") return "LEARN"; + if (state.revision !== snapshot.revision || snapshot.tutor?.status !== "valid") return "LEARN"; + const facts = this.factExtractor.extract(code); + return this.topicMatcher.match(snapshot.tutor.topics, facts).some(({ topic }) => topic.id === state.activeTopicId) + ? state.phase + : "LEARN"; } } -function normalizePlan(plan: Awaited>, strategy = resolveEffectiveTutorStrategy({})): TutorPlan { +type ClassifiedTopicMatch = { + readonly match: TopicMatch; + readonly classification: TopicClassification; +}; + +function classifyMatches( + matches: readonly TopicMatch[], + facts: ReturnType, + history: readonly TutorDialogTurn[], + state: TutorProgressionState, + difficulty: TutorDifficulty, + strategy: EffectiveTutorStrategy, +): readonly ClassifiedTopicMatch[] { + return matches.map((match) => ({ + match, + classification: classifyWithState(match.topic, facts, history, state, difficulty, strategy), + })); +} + +function selectProgressionMatch( + classifications: readonly ClassifiedTopicMatch[], + state: TutorProgressionState, +): ClassifiedTopicMatch | undefined { + return classifications.find(({ classification }) => classification.status === "probeable") + ?? classifications.find(({ classification }) => classification.status === "unresolved") + ?? (state.activeTopicId ? classifications.find(({ match }) => match.topic.id === state.activeTopicId) : undefined) + ?? classifications.find(({ classification }) => classification.status === "mastered"); +} + +function resolveExtensionTarget( + topics: readonly CurriculumTopic[], + previousTopicId: string | undefined, + previousPhase: DidacticPhase, + nextTopicId: string, +): string | undefined { + if (!previousTopicId || previousPhase !== "EXPAND" || previousTopicId === nextTopicId) return undefined; + const previousTopic = topics.find(({ id }) => id === previousTopicId); + return previousTopic?.schemaVersion === 2 && previousTopic.extensions?.some(({ topic }) => topic === nextTopicId) + ? nextTopicId + : undefined; +} + +function resolveExpansionBrief( + topics: readonly CurriculumTopic[], + topic: CurriculumTopic, + phase: DidacticPhase, + state: TutorProgressionState, +): TutorExpansionBrief | undefined { + if (phase !== "EXPAND" || topic.schemaVersion !== 2 || !topic.extensions) return undefined; + const usedTargets = new Set(state.usedExpansionTargetTopicIds[topic.id] ?? []); + const extension = topic.extensions.find(({ topic: targetTopicId }) => + topics.some(({ id }) => id === targetTopicId) && !usedTargets.has(targetTopicId), + ); + return extension + ? { sourceTopicId: topic.id, targetTopicId: extension.topic, objective: extension.objective } + : undefined; +} + +function recordFollowupObservation(context: AdapterContext, input: TutorFollowupInput): void { + const current = findQuestion(context.topic, input.currentQuestion, input.history); + if (!current) return; + const evidenceKey = context.phase === "DEEPEN" || context.phase === "EXPAND" + ? "postMasteryEvidence" + : "masteryEvidence"; + appendEvidence(context.state, evidenceKey, context.topic.id, toObservation(current, input.rating)); +} + +type FollowupStateUpdate = { + readonly status: TopicClassification["status"]; +}; + +function updateStateAfterFollowup(context: AdapterContext, input: TutorFollowupInput): FollowupStateUpdate { + if (context.phase !== "LEARN") return { status: "mastered" }; + const classification = classifyWithState( + context.topic, + context.facts, + input.history, + context.state, + input.difficulty, + context.strategy.strategy, + ); + if (classification.status !== "mastered") return { status: classification.status }; + markTopicMastered(context.state, context.topic.id); + context.state.phase = "DEEPEN"; + context.state.retainedPhases[context.topic.id] = "DEEPEN"; + return { status: classification.status }; +} + +function normalizePlan(plan: Awaited>, strategy = resolveEffectiveTutorStrategy({}), state?: TutorProgressionState, phase: DidacticPhase = "LEARN", extensionTargetTopicId?: string, expansionBrief?: TutorExpansionBrief): TutorPlan { if (!plan) throw new Error("Cannot normalize an empty tutor plan"); return { ...plan.brief, @@ -149,5 +354,178 @@ function normalizePlan(plan: Awaited>, stra contentRevision: plan.contentRevision, strategyId: strategy.strategy.id, strategySource: strategy.source === "built-in" ? "built-in" : "repository", + effectiveStrategy: strategy.strategy, + learningPhase: phase, + ...(state?.activeTopicId ? { activeTopicId: state.activeTopicId } : {}), + ...(state ? { masteredTopicIds: [...state.masteredTopicIds] } : {}), + ...(state?.progressionBlockedReason ? { progressionBlockedReason: state.progressionBlockedReason } : {}), + ...(extensionTargetTopicId ? { extensionTargetTopicId } : {}), + ...(expansionBrief ? { expansionBrief } : {}), + }; +} + +type AdapterContext = { + readonly revision: string; + readonly facts: ReturnType; + readonly topic: CurriculumTopic; + readonly strategy: StrategyResolution; + readonly phase: DidacticPhase; + readonly state: TutorProgressionState; + readonly extensionTargetTopicId?: string; + readonly expansionBrief?: TutorExpansionBrief; + readonly blocked?: TutorPlanningBlocked; +}; + +function orderTopicMatches(matches: readonly TopicMatch[], annotation?: ExampleTutorAnnotation): readonly TopicMatch[] { + const byId = new Map(matches.map((match) => [match.topic.id, match])); + const boundIds = [ + ...(annotation?.primaryTopic ? [annotation.primaryTopic] : []), + ...(annotation?.topics ?? []), + ]; + const bound = boundIds.map((id) => byId.get(id)).filter((match): match is TopicMatch => match !== undefined); + const boundSet = new Set(bound.map(({ topic }) => topic.id)); + return [...bound, ...matches.filter(({ topic }) => !boundSet.has(topic.id))]; +} + +function phaseForTopic(state: TutorProgressionState, topicId: string, classification?: string): DidacticPhase { + if (classification === "mastered" || state.masteredTopicIds.includes(topicId)) return state.retainedPhases[topicId] ?? "DEEPEN"; + return "LEARN"; +} + +function progressionOptions(topic: CurriculumTopic, phase: DidacticPhase, state: TutorProgressionState) { + if (phase === "LEARN") return { phase } as const; + const criteria = deepeningCriteria(topic); + const existingQuestionKinds = (state.postMasteryEvidence[topic.id] ?? []).map(({ kind }) => kind); + return { + phase, + preferredQuestionKinds: criteria.requiredQuestionKinds, + existingQuestionKinds, + } as const; +} + +function collectUsedQuestionIds(topic: CurriculumTopic, history: readonly TutorDialogTurn[], state: TutorProgressionState): Set { + const byText = new Map(topic.questions.flatMap((question) => question.text ? [[question.text, question.id] as const] : [])); + const historyIds = history.map((turn) => turn.questionId ?? byText.get(turn.question)).filter((id): id is string => id !== undefined); + return new Set([...historyIds, ...(state.masteryEvidence[topic.id] ?? []).map(({ questionId }) => questionId)]); +} + +function progressionHistory( + topic: CurriculumTopic, + history: readonly TutorDialogTurn[], + state: TutorProgressionState, + currentQuestion: string, +): readonly TutorDialogTurn[] { + const byId = new Map(topic.questions.map((question) => [question.id, question.text ?? question.template ?? question.id])); + const postMasteryTurns = (state.postMasteryEvidence[topic.id] ?? []).map(({ questionId }) => ({ + question: byId.get(questionId) ?? questionId, + questionId, + answer: "", + responseStyle: "normal" as const, + })); + return [...history, ...postMasteryTurns, { question: currentQuestion, answer: "", responseStyle: "normal" as const }]; +} + +function classifyWithState( + topic: CurriculumTopic, + facts: ReturnType, + history: readonly TutorDialogTurn[], + state: TutorProgressionState, + difficulty: TutorDifficulty, + strategy: EffectiveTutorStrategy, +) { + const stored = state.masteryEvidence[topic.id] ?? []; + const storedQuestionIds = new Set(stored.map(({ questionId }) => questionId)); + const observations = [...stored, ...collectObservations(topic, history).filter(({ questionId }) => !storedQuestionIds.has(questionId))]; + const classification = classifyTopic(topic, facts, observations, collectUsedQuestionIds(topic, history, state), difficulty, strategy); + if (state.masteredTopicIds.includes(topic.id) && classification.status !== "inapplicable") { + return { status: "mastered", masteryDomain: classification.masteryDomain } satisfies TopicClassification; + } + return classification; +} + +function buildExpansionPlan(context: AdapterContext, phase: DidacticPhase, expansionBrief: TutorExpansionBrief): TutorPlan { + const targetKey = expansionBrief.targetTopicId.slice(0, 56); + return { + topicId: context.topic.id, + topicTitle: context.topic.title, + conceptId: `expand-${targetKey}`, + conceptTitle: "Eigene Erweiterung", + objective: expansionBrief.objective, + questionId: `expand-${targetKey}`, + questionKind: "transfer", + indicatorId: "expansion", + indicator: expansionBrief.objective, + question: `Welche kleine, direkt am aktuellen Sketch prüfbare Erweiterung würdest du als Nächstes selbst umsetzen, um dieses Lernziel zu bearbeiten: „${expansionBrief.objective}“, und woran würdest du ihre Wirkung erkennen?`, + misconceptions: [], + contentRevision: context.revision, + strategyId: context.strategy.strategy.id, + strategySource: context.strategy.source === "built-in" ? "built-in" : "repository", + effectiveStrategy: context.strategy.strategy, + learningPhase: phase, + activeTopicId: context.state.activeTopicId, + masteredTopicIds: [...context.state.masteredTopicIds], + expansionBrief, + }; +} + +type PlanningContext = Pick; + +function exhaustionResult(context: PlanningContext, phase: DidacticPhase): TutorPlanningBlocked { + context.state.phase = phase; + context.state.progressionBlockedReason = "content-exhausted"; + return { + kind: "blocked", + progressionBlockedReason: "content-exhausted", + contentRevision: context.revision, + learningPhase: phase, + activeTopicId: context.topic.id, + masteredTopicIds: [...context.state.masteredTopicIds], + strategyId: context.strategy.strategy.id, + strategySource: context.strategy.source === "built-in" ? "built-in" : "repository", + effectiveStrategy: context.strategy.strategy, + }; +} + +function transitionResult(context: PlanningContext): import("./tutor-planning").TutorPlanningTransition { + return { + kind: "transition", + contentRevision: context.revision, + learningPhase: context.phase, + activeTopicId: context.topic.id, + masteredTopicIds: [...context.state.masteredTopicIds], + strategyId: context.strategy.strategy.id, + strategySource: context.strategy.source === "built-in" ? "built-in" : "repository", + effectiveStrategy: context.strategy.strategy, + }; +} + +function blockedResult(context: PlanningContext, state: TutorProgressionState): TutorPlanningBlocked { + state.phase = context.phase; + state.progressionBlockedReason = "content-exhausted"; + return { + kind: "blocked", + progressionBlockedReason: "content-exhausted", + contentRevision: context.revision, + learningPhase: context.phase, + activeTopicId: context.topic.id, + masteredTopicIds: [...state.masteredTopicIds], + strategyId: context.strategy.strategy.id, + strategySource: context.strategy.source === "built-in" ? "built-in" : "repository", + effectiveStrategy: context.strategy.strategy, + }; +} + +function findQuestion(topic: CurriculumTopic, currentQuestion: string, history: readonly TutorDialogTurn[]): CurriculumQuestion | null { + const questionId = history.find((turn) => turn.question === currentQuestion)?.questionId; + return topic.questions.find(({ id }) => id === questionId) ?? topic.questions.find(({ text }) => text === currentQuestion) ?? null; +} + +function toObservation(current: CurriculumQuestion, rating: Parameters>[5]): Observation { + return { + questionId: current.id, + conceptId: current.concept, + indicatorId: current.indicator, + kind: current.kind, + rating, }; } diff --git a/server/services/tutor/curriculum/curriculum-schema.ts b/server/services/tutor/curriculum/curriculum-schema.ts index 9ade42223..dc28720e9 100644 --- a/server/services/tutor/curriculum/curriculum-schema.ts +++ b/server/services/tutor/curriculum/curriculum-schema.ts @@ -110,7 +110,23 @@ const progressionSchema = z.object({ }).strict(), }).strict(); -export const curriculumTopicSchema = z.object({ +const deepeningSchema = z.object({ + minimumSuccessfulProbes: z.number().int().min(1).max(10), + successRatingAtLeast: z.number().int().min(3).max(5), + requiredQuestionKinds: z.array(z.enum(["application", "prediction", "transfer"])).min(1).max(3), + recentWeakAnswersAllowed: z.number().int().min(0).max(3), +}).strict().superRefine((value, context) => { + if (new Set(value.requiredQuestionKinds).size !== value.requiredQuestionKinds.length) { + context.addIssue({ code: z.ZodIssueCode.custom, path: ["requiredQuestionKinds"], message: "Deepening question kinds must be unique" }); + } +}); + +const topicExtensionSchema = z.object({ + topic: idSchema, + objective: boundedText(500), +}).strict(); + +const curriculumTopicV1Schema = z.object({ schemaVersion: z.literal(1), id: idSchema, title: boundedText(160), @@ -122,6 +138,22 @@ export const curriculumTopicSchema = z.object({ progression: progressionSchema, }).strict(); +const curriculumTopicV2Schema = z.object({ + schemaVersion: z.literal(2), + id: idSchema, + title: boundedText(160), + locale: z.string().regex(/^[a-z]{2}-[A-Z]{2}$/), + activation: activationSchema, + concepts: z.array(conceptSchema).min(1).max(16), + questions: z.array(questionSchema).min(1).max(64), + scaffolds: z.array(scaffoldSchema).max(32), + progression: progressionSchema, + deepening: deepeningSchema.optional(), + extensions: z.array(topicExtensionSchema).max(8).optional(), +}).strict(); + +export const curriculumTopicSchema = z.union([curriculumTopicV1Schema, curriculumTopicV2Schema]); + export const curriculumManifestSchema = z.object({ schemaVersion: z.literal(1), curriculumId: idSchema, @@ -140,6 +172,8 @@ export type CurriculumManifest = z.infer; export type CurriculumConcept = CurriculumTopic["concepts"][number]; export type CurriculumQuestion = CurriculumTopic["questions"][number]; export type CurriculumScaffold = CurriculumTopic["scaffolds"][number]; +export type TopicDeepening = z.infer; +export type TopicExtension = z.infer; function assertUnique(values: readonly string[], label: string): void { if (new Set(values).size !== values.length) throw new Error(`Duplicate ${label}`); diff --git a/server/services/tutor/curriculum/learning-planner.ts b/server/services/tutor/curriculum/learning-planner.ts index 52ccaafb2..94e407a2a 100644 --- a/server/services/tutor/curriculum/learning-planner.ts +++ b/server/services/tutor/curriculum/learning-planner.ts @@ -32,6 +32,15 @@ export interface LearningPlan { export interface LearningAdvanceOptions { readonly difficulty: TutorDifficulty; readonly strategy?: EffectiveTutorStrategy; + readonly phase?: "LEARN" | "DEEPEN" | "EXPAND"; + readonly preferredQuestionKinds?: readonly CurriculumQuestion["kind"][]; + readonly existingQuestionKinds?: readonly CurriculumQuestion["kind"][]; +} + +export interface LearningStartOptions { + readonly phase?: "LEARN" | "DEEPEN" | "EXPAND"; + readonly preferredQuestionKinds?: readonly CurriculumQuestion["kind"][]; + readonly existingQuestionKinds?: readonly CurriculumQuestion["kind"][]; } export interface LearningPlanner { @@ -42,6 +51,7 @@ export interface LearningPlanner { history: readonly TutorDialogTurn[], difficulty: TutorDifficulty, strategy?: EffectiveTutorStrategy, + options?: LearningStartOptions, ): LearningPlan | null; advance( topic: CurriculumTopic, @@ -54,7 +64,7 @@ export interface LearningPlanner { ): LearningPlan | null; } -type Observation = { +export type Observation = { questionId: string; conceptId: string; indicatorId: string; @@ -62,6 +72,12 @@ type Observation = { rating: TutorAnswerRating; }; +export type TopicClassification = + | { readonly status: "mastered"; readonly masteryDomain: readonly string[] } + | { readonly status: "probeable"; readonly masteryDomain: readonly string[]; readonly nextConceptId: string } + | { readonly status: "unresolved"; readonly masteryDomain: readonly string[] } + | { readonly status: "inapplicable"; readonly masteryDomain: readonly string[] }; + export class DefaultLearningPlanner implements LearningPlanner { start( topic: CurriculumTopic, @@ -70,9 +86,23 @@ export class DefaultLearningPlanner implements LearningPlanner { history: readonly TutorDialogTurn[], difficulty: TutorDifficulty, strategy?: EffectiveTutorStrategy, + options?: LearningStartOptions, ): LearningPlan | null { const observations = collectObservations(topic, history); const usedQuestionIds = collectUsedQuestionIds(topic, history); + if (options?.phase === "DEEPEN" || options?.phase === "EXPAND") { + const question = selectDeepeningQuestion({ + topic, + facts, + usedQuestionIds, + difficulty, + strategy, + preferredKinds: options?.preferredQuestionKinds, + existingKinds: options?.existingQuestionKinds, + }); + const concept = question ? topic.concepts.find(({ id }) => id === question.concept) : undefined; + return question && concept ? buildPlan(topic, revision, concept, question) : null; + } const concept = selectNextConcept(topic, facts, observations, usedQuestionIds, undefined, strategy); if (!concept) return null; const question = selectQuestion(topic, concept, facts, usedQuestionIds, difficulty, undefined, strategy); @@ -103,6 +133,21 @@ export class DefaultLearningPlanner implements LearningPlanner { const concept = topic.concepts.find(({ id }) => id === current.question.concept); if (!concept) return null; + if (options.phase === "DEEPEN" || options.phase === "EXPAND") { + const question = selectDeepeningQuestion({ + topic, + facts, + usedQuestionIds, + difficulty, + strategy, + excludedId: current.question.id, + preferredKinds: options.preferredQuestionKinds, + existingKinds: options.existingQuestionKinds, + }); + const target = question ? topic.concepts.find(({ id }) => id === question.concept) : undefined; + return question && target ? buildPlan(topic, revision, target, question) : null; + } + const next = selectAfterRating({ topic, facts, concept, currentQuestion: current.question, rating, observations: allObservations, usedQuestionIds, difficulty, strategy }); return next ? buildPlan(topic, revision, next.concept, next.question, next.scaffold) : null; } @@ -286,6 +331,40 @@ function selectQuestion( ))[0] ?? null; } +function selectDeepeningQuestion({ + topic, + facts, + usedQuestionIds, + difficulty, + strategy, + excludedId, + preferredKinds = [], + existingKinds = [], +}: { + readonly topic: CurriculumTopic; + readonly facts: SketchFacts; + readonly usedQuestionIds: ReadonlySet; + readonly difficulty: TutorDifficulty; + readonly strategy?: EffectiveTutorStrategy; + readonly excludedId?: string; + readonly preferredKinds?: readonly CurriculumQuestion["kind"][]; + readonly existingKinds?: readonly CurriculumQuestion["kind"][]; +}): CurriculumQuestion | null { + const candidates = topic.questions.filter((question) => + question.kind !== "recall" + && questionApplies(question, facts) + && question.id !== excludedId + && !usedQuestionIds.has(question.id), + ); + return [...candidates].sort((left, right) => ( + Number(preferredKinds.includes(right.kind) && !existingKinds.includes(right.kind)) + - Number(preferredKinds.includes(left.kind) && !existingKinds.includes(left.kind)) + || difficultyDistance(left, difficulty) - difficultyDistance(right, difficulty) + || (strategy ? strategy.questionKindWeights[right.kind] - strategy.questionKindWeights[left.kind] : 0) + || left.id.localeCompare(right.id) + ))[0] ?? null; +} + function difficultyDistance(question: CurriculumQuestion, difficulty: TutorDifficulty): number { const [min, max] = question.difficulty; if (difficulty < min) return min - difficulty; @@ -309,6 +388,31 @@ function conceptHasQuestions( || (strategy?.repetition === "relaxed" && applicable.length > 0); } +export function classifyTopic( + topic: CurriculumTopic, + facts: SketchFacts, + observations: readonly Observation[], + usedQuestionIds: ReadonlySet, + difficulty: TutorDifficulty, + strategy?: EffectiveTutorStrategy, +): TopicClassification { + const masteryDomain = topic.concepts + .filter((concept) => topic.questions.some((question) => question.concept === concept.id && questionApplies(question, facts))) + .map(({ id }) => id); + if (masteryDomain.length === 0) return { status: "inapplicable", masteryDomain }; + + const concepts = topic.concepts.filter(({ id }) => masteryDomain.includes(id)); + if (concepts.every((concept) => isMastered(concept, observations))) { + return { status: "mastered", masteryDomain }; + } + + const nextConcept = selectNextConcept(topic, facts, observations, usedQuestionIds, undefined, strategy); + if (nextConcept && selectQuestion(topic, nextConcept, facts, usedQuestionIds, difficulty, undefined, strategy)) { + return { status: "probeable", masteryDomain, nextConceptId: nextConcept.id }; + } + return { status: "unresolved", masteryDomain }; +} + function chooseScaffold( topic: CurriculumTopic, conceptId: string, diff --git a/server/services/tutor/curriculum/progression-state.ts b/server/services/tutor/curriculum/progression-state.ts new file mode 100644 index 000000000..de323726b --- /dev/null +++ b/server/services/tutor/curriculum/progression-state.ts @@ -0,0 +1,115 @@ +import type { CurriculumTopic, TopicDeepening } from "./curriculum-schema"; +import type { Observation } from "./learning-planner"; + +export type DidacticPhase = "LEARN" | "DEEPEN" | "EXPAND"; +export type ProgressionBlockedReason = "content-exhausted"; + +export interface TutorProgressionState { + revision: string; + activeTopicId?: string; + phase?: DidacticPhase; + masteredTopicIds: string[]; + masteryEvidence: Record; + postMasteryEvidence: Record; + retainedPhases: Record; + usedExpansionTargetTopicIds: Record; + progressionBlockedReason?: ProgressionBlockedReason; +} + +export const DEFAULT_DEEPENING: TopicDeepening = { + minimumSuccessfulProbes: 2, + successRatingAtLeast: 4, + requiredQuestionKinds: ["transfer"], + recentWeakAnswersAllowed: 0, +}; + +export function createTutorProgressionState(revision: string): TutorProgressionState { + return { + revision, + masteredTopicIds: [], + masteryEvidence: {}, + postMasteryEvidence: {}, + retainedPhases: {}, + usedExpansionTargetTopicIds: {}, + }; +} + +export function cloneTutorProgressionState(state: TutorProgressionState): TutorProgressionState { + return { + ...state, + masteredTopicIds: [...state.masteredTopicIds], + masteryEvidence: cloneEvidence(state.masteryEvidence), + postMasteryEvidence: cloneEvidence(state.postMasteryEvidence), + retainedPhases: { ...state.retainedPhases }, + usedExpansionTargetTopicIds: Object.fromEntries( + Object.entries(state.usedExpansionTargetTopicIds).map(([topicId, targetIds]) => [topicId, [...targetIds]]), + ), + }; +} + +export function commitTutorProgressionState(target: TutorProgressionState, source: TutorProgressionState): void { + target.revision = source.revision; + target.activeTopicId = source.activeTopicId; + target.phase = source.phase; + target.masteredTopicIds = [...source.masteredTopicIds]; + target.masteryEvidence = cloneEvidence(source.masteryEvidence); + target.postMasteryEvidence = cloneEvidence(source.postMasteryEvidence); + target.retainedPhases = { ...source.retainedPhases }; + target.usedExpansionTargetTopicIds = Object.fromEntries( + Object.entries(source.usedExpansionTargetTopicIds).map(([topicId, targetIds]) => [topicId, [...targetIds]]), + ); + target.progressionBlockedReason = source.progressionBlockedReason; +} + +function cloneEvidence(evidence: Record): Record { + return Object.fromEntries( + Object.entries(evidence).map(([topicId, observations]) => [topicId, observations.map((observation) => ({ ...observation }))]), + ); +} + +export function resetTutorProgressionState(state: TutorProgressionState, revision = state.revision): void { + state.activeTopicId = undefined; + state.phase = undefined; + state.masteredTopicIds.splice(0, state.masteredTopicIds.length); + state.masteryEvidence = {}; + state.postMasteryEvidence = {}; + state.retainedPhases = {}; + state.usedExpansionTargetTopicIds = {}; + state.progressionBlockedReason = undefined; + state.revision = revision; +} + +export function appendEvidence( + state: TutorProgressionState, + collection: "masteryEvidence" | "postMasteryEvidence", + topicId: string, + observation: Observation, +): void { + const entries = state[collection][topicId] ?? []; + entries.push(observation); + state[collection][topicId] = entries; +} + +export function markTopicMastered(state: TutorProgressionState, topicId: string): void { + if (!state.masteredTopicIds.includes(topicId)) state.masteredTopicIds.push(topicId); +} + +export function markExpansionTargetUsed(state: TutorProgressionState, sourceTopicId: string, targetTopicId: string): void { + const targets = state.usedExpansionTargetTopicIds[sourceTopicId] ?? []; + if (!targets.includes(targetTopicId)) targets.push(targetTopicId); + state.usedExpansionTargetTopicIds[sourceTopicId] = targets; +} + +export function deepeningCriteria(topic: CurriculumTopic): TopicDeepening { + return topic.schemaVersion === 2 && topic.deepening ? topic.deepening : DEFAULT_DEEPENING; +} + +export function hasMetDeepeningCriteria(topic: CurriculumTopic, observations: readonly Observation[]): boolean { + const criteria = deepeningCriteria(topic); + const successful = observations.filter(({ rating }) => rating >= criteria.successRatingAtLeast); + const weakTrailing = [...observations].reverse().findIndex(({ rating }) => rating > 2); + const recentWeakCount = weakTrailing < 0 ? observations.length : weakTrailing; + return successful.length >= criteria.minimumSuccessfulProbes + && criteria.requiredQuestionKinds.every((kind) => successful.some((observation) => observation.kind === kind)) + && recentWeakCount <= criteria.recentWeakAnswersAllowed; +} diff --git a/server/services/tutor/tutor-planning.ts b/server/services/tutor/tutor-planning.ts index 0cdf115f3..46b233318 100644 --- a/server/services/tutor/tutor-planning.ts +++ b/server/services/tutor/tutor-planning.ts @@ -1,13 +1,15 @@ import type { TutorAnswerRating, TutorDialogTurn, TutorDifficulty } from "@shared/tutor"; import type { TutorCapability } from "../course-content/course-content-loader"; import type { ExampleTutorAnnotation } from "../course-content/embedded-tutor-annotation"; -import type { StrategyResolution } from "./strategy/effective-tutor-strategy"; +import type { EffectiveTutorStrategy, StrategyResolution } from "./strategy/effective-tutor-strategy"; +import type { TutorProgressionState, DidacticPhase, ProgressionBlockedReason } from "./curriculum/progression-state"; export interface TutorPlanningContentContext { readonly revision: string; readonly tutor?: TutorCapability; readonly exampleId?: string; readonly exampleTutorAnnotation?: ExampleTutorAnnotation; + readonly progressionState?: TutorProgressionState; } /** Normalized, implementation-independent plan data consumed by TutorService. */ @@ -25,13 +27,65 @@ export interface TutorPlan { readonly misconceptions: readonly { id: string; description: string }[]; readonly strategyId?: string; readonly strategySource?: "built-in" | "repository"; + /** Internal normalized strategy used to build the provider prompt for this plan. */ + readonly effectiveStrategy?: EffectiveTutorStrategy; readonly scaffold?: { readonly id: string; readonly strategy: string; readonly hint: string }; readonly contentRevision: string; + readonly learningPhase?: DidacticPhase; + readonly activeTopicId?: string; + readonly masteredTopicIds?: readonly string[]; + readonly progressionBlockedReason?: ProgressionBlockedReason; + readonly extensionTargetTopicId?: string; + readonly expansionBrief?: TutorExpansionBrief; +} + +/** Application-owned, normalized guidance for one EXPAND transition. */ +export interface TutorExpansionBrief { + readonly sourceTopicId: string; + readonly targetTopicId: string; + readonly objective: string; +} + +export interface TutorPlanningBlocked { + readonly kind: "blocked"; + readonly progressionBlockedReason: ProgressionBlockedReason; + readonly contentRevision: string; + readonly learningPhase: DidacticPhase; + readonly activeTopicId?: string; + readonly masteredTopicIds: readonly string[]; + readonly strategyId: string; + readonly strategySource: "built-in" | "repository"; + readonly effectiveStrategy?: EffectiveTutorStrategy; +} + +export interface TutorPlanningTransition { + readonly kind: "transition"; + readonly contentRevision: string; + readonly learningPhase: DidacticPhase; + readonly activeTopicId?: string; + readonly masteredTopicIds: readonly string[]; + readonly strategyId: string; + readonly strategySource: "built-in" | "repository"; + readonly effectiveStrategy?: EffectiveTutorStrategy; +} + +export type TutorPlanningResult = TutorPlan | TutorPlanningBlocked | TutorPlanningTransition; + +export function isTutorPlan(result: TutorPlanningResult | null): result is TutorPlan { + return result !== null && !("kind" in result); +} + +export function isTutorPlanningBlocked(result: TutorPlanningResult | null): result is TutorPlanningBlocked { + return result !== null && "kind" in result && result.kind === "blocked"; +} + +export function isTutorPlanningTransition(result: TutorPlanningResult | null): result is TutorPlanningTransition { + return result !== null && "kind" in result && result.kind === "transition"; } export interface TutorPlanningExtension { - resolveStrategy?(input: { readonly courseContent?: TutorPlanningContentContext }): Promise; - planInitial(input: { readonly code: string; readonly history: readonly TutorDialogTurn[]; readonly difficulty: TutorDifficulty; readonly exampleId?: string; readonly courseContent?: TutorPlanningContentContext }): Promise; + resolveStrategy?(input: { readonly code?: string; readonly courseContent?: TutorPlanningContentContext }): Promise; + planInitial(input: { readonly code: string; readonly history: readonly TutorDialogTurn[]; readonly difficulty: TutorDifficulty; readonly exampleId?: string; readonly courseContent?: TutorPlanningContentContext }): Promise; planFollowup(input: { readonly code: string; readonly history: readonly TutorDialogTurn[]; @@ -40,5 +94,5 @@ export interface TutorPlanningExtension { readonly difficulty: TutorDifficulty; readonly exampleId?: string; readonly courseContent?: TutorPlanningContentContext; - }): Promise; + }): Promise; } diff --git a/server/services/tutor/tutor-service.ts b/server/services/tutor/tutor-service.ts index bf4b4a00a..339e63dab 100644 --- a/server/services/tutor/tutor-service.ts +++ b/server/services/tutor/tutor-service.ts @@ -14,7 +14,15 @@ import { type LLMProvider, type ProviderQuestionResult, } from "./llm-provider"; -import type { TutorPlan, TutorPlanningContentContext, TutorPlanningExtension } from "./tutor-planning"; +import { + isTutorPlan, + isTutorPlanningBlocked, + type TutorPlan, + type TutorPlanningBlocked, + type TutorPlanningContentContext, + type TutorPlanningExtension, + type TutorPlanningTransition, +} from "./tutor-planning"; import { BUILT_IN_TUTOR_STRATEGY, resolveEffectiveTutorStrategy, @@ -22,6 +30,10 @@ import { type StrategyResolution, } from "./strategy/effective-tutor-strategy"; import { buildTutorLearningObjectivesGuidance, buildTutorStrategyGuidance } from "./strategy/tutor-strategy-guidance"; +import { + cloneTutorProgressionState, + commitTutorProgressionState, +} from "./curriculum/progression-state"; const UNSAFE_MERMAID_PATTERNS = [ /https?:\/\//i, @@ -461,9 +473,45 @@ function applyPlanningResult(result: TutorContentResult, plan: TutorPlan): Tutor ...(feedback ? { feedback } : {}), ...(plan.strategyId ? { strategyId: plan.strategyId } : {}), ...(plan.strategySource ? { strategySource: plan.strategySource } : {}), + ...(plan.learningPhase ? { learningPhase: plan.learningPhase } : {}), + ...(plan.activeTopicId ? { activeTopicId: plan.activeTopicId } : {}), + ...(plan.masteredTopicIds ? { masteredTopicIds: [...plan.masteredTopicIds] } : {}), + ...(plan.progressionBlockedReason ? { progressionBlockedReason: plan.progressionBlockedReason } : {}), + ...(plan.extensionTargetTopicId ? { extensionTargetTopicId: plan.extensionTargetTopicId } : {}), + }; +} + +function applyBlockedResult(result: TutorContentResult, blocked: TutorPlanningBlocked): TutorContentResult { + return { + ...result, + strategyId: blocked.strategyId, + strategySource: blocked.strategySource, + learningPhase: blocked.learningPhase, + ...(blocked.activeTopicId ? { activeTopicId: blocked.activeTopicId } : {}), + masteredTopicIds: [...blocked.masteredTopicIds], + progressionBlockedReason: blocked.progressionBlockedReason, + contentRevision: blocked.contentRevision, }; } +function applyTransitionResult(result: TutorContentResult, transition: TutorPlanningTransition): TutorContentResult { + return { + ...result, + strategyId: transition.strategyId, + strategySource: transition.strategySource, + learningPhase: transition.learningPhase, + ...(transition.activeTopicId ? { activeTopicId: transition.activeTopicId } : {}), + masteredTopicIds: [...transition.masteredTopicIds], + contentRevision: transition.contentRevision, + }; +} + +function applyPlanningOutcome(result: TutorContentResult, planningResult: Exclude>, null>): TutorContentResult { + if (isTutorPlan(planningResult)) return applyPlanningResult(result, planningResult); + if (isTutorPlanningBlocked(planningResult)) return applyBlockedResult(result, planningResult); + return applyTransitionResult(result, planningResult); +} + function applyStrategyMetadata(result: TutorContentResult, strategy: StrategyResolution): TutorContentResult { return { ...result, @@ -472,6 +520,31 @@ function applyStrategyMetadata(result: TutorContentResult, strategy: StrategyRes }; } +type PlanningResult = Exclude>, null>; + +type TutorPlanningTransaction = { + readonly courseContent?: TutorPlanningContentContext; + commit(): void; +}; + +function beginTutorPlanningTransaction(courseContent?: TutorPlanningContentContext): TutorPlanningTransaction { + const persistedState = courseContent?.progressionState; + if (!persistedState) return { courseContent, commit: () => undefined }; + const workingState = cloneTutorProgressionState(persistedState); + return { + courseContent: { ...courseContent, progressionState: workingState }, + commit: () => commitTutorProgressionState(persistedState, workingState), + }; +} + +function strategyFromPlanningResult(planningResult: PlanningResult | null): StrategyResolution | undefined { + if (!planningResult?.effectiveStrategy) return undefined; + return { + strategy: planningResult.effectiveStrategy, + source: planningResult.strategySource === "built-in" ? "built-in" : "repository", + }; +} + export class TutorService { constructor( private readonly provider: LLMProvider, @@ -487,10 +560,11 @@ export class TutorService { ): Promise<{ result: TutorContentResult; model: string }> { const requestCredential = this.resolveCredential(credential); const context = buildTutorContext(code); - const strategy = await this.resolveStrategy(courseContent); + const transaction = beginTutorPlanningTransaction(courseContent); const planningResult = this.planningExtension - ? await this.planningExtension.planInitial({ code, history: [], difficulty, courseContent }) + ? await this.planningExtension.planInitial({ code, history: [], difficulty, courseContent: transaction.courseContent }) : null; + const strategy = strategyFromPlanningResult(planningResult) ?? await this.resolveStrategy(code, transaction.courseContent); const providerResult: ProviderQuestionResult = await this.provider.generateLearningQuestion( { model: await this.resolveModel(requestedModel, requestCredential), @@ -499,17 +573,18 @@ export class TutorService { code, context, difficulty, - planningResult ?? undefined, + planningResult && isTutorPlan(planningResult) ? planningResult : undefined, strategy.strategy, - courseContent?.exampleTutorAnnotation?.learningObjectives, + transaction.courseContent?.exampleTutorAnnotation?.learningObjectives, ), }, requestCredential, ); const validatedResult = validateLearningQuestion(providerResult.result, difficulty); const plannedResult = planningResult - ? applyPlanningResult(validatedResult, planningResult) + ? applyPlanningOutcome(validatedResult, planningResult) : applyStrategyMetadata(validatedResult, strategy); + transaction.commit(); const { answerRating: _initialAnswerRating, ...initialResult } = plannedResult; return { model: providerResult.model, @@ -521,21 +596,27 @@ export class TutorService { const [code, history, question, answer, credential, requestedModel, difficulty = TUTOR_DEFAULT_DIFFICULTY, courseContent] = args; const requestCredential = this.resolveCredential(credential); const parsedHistory = history.map((entry) => tutorDialogTurnSchema.parse(entry)); - const strategy = await this.resolveStrategy(courseContent); + const transaction = beginTutorPlanningTransaction(courseContent); if (isClearlyNonLearningAnswer(answer)) { + const strategy = await this.resolveStrategy(code, transaction.courseContent); return { model: requestedModel ?? "fallback", result: applyStrategyMetadata(buildPhilosophicalFallback(parsedHistory, difficulty), strategy), }; } const context = buildTutorContext(code); + const currentPlanningResult = this.planningExtension + ? await this.planningExtension.planInitial({ code, history: parsedHistory, difficulty, courseContent: transaction.courseContent }) + : null; + const strategy = strategyFromPlanningResult(currentPlanningResult) ?? await this.resolveStrategy(code, transaction.courseContent); const providerResult = await this.provider.generateLearningQuestion( { model: await this.resolveModel(requestedModel, requestCredential), systemPrompt: TUTOR_SYSTEM_PROMPT, userPrompt: buildDialogPrompt(code, context, parsedHistory, question, answer, difficulty, { + didacticBrief: currentPlanningResult && isTutorPlan(currentPlanningResult) ? currentPlanningResult : undefined, strategy: strategy.strategy, - learningObjectives: courseContent?.exampleTutorAnnotation?.learningObjectives, + learningObjectives: transaction.courseContent?.exampleTutorAnnotation?.learningObjectives, }), }, requestCredential, @@ -548,10 +629,11 @@ export class TutorService { ? ensureDistinctDialogQuestion(validatedResult, code, parsedHistory, question, strategy.strategy) : validatedResult; if (validatedResult.responseStyle === "normal" && this.planningExtension) { - const nextPlan = await this.planningExtension.planFollowup({ code, history: parsedHistory, currentQuestion: question, rating: validatedResult.answerRating!, difficulty, courseContent }); - if (nextPlan) distinctResult = applyPlanningResult(validatedResult, nextPlan); + const nextPlan = await this.planningExtension.planFollowup({ code, history: parsedHistory, currentQuestion: question, rating: validatedResult.answerRating!, difficulty, courseContent: transaction.courseContent }); + if (nextPlan) distinctResult = applyPlanningOutcome(validatedResult, nextPlan); } if (!distinctResult.strategyId) distinctResult = applyStrategyMetadata(distinctResult, strategy); + transaction.commit(); return { model: providerResult.model, result: distinctResult, @@ -578,10 +660,10 @@ export class TutorService { return availableModels.includes(model) ? model : "auto"; } - private async resolveStrategy(courseContent?: TutorPlanningContentContext): Promise { + private async resolveStrategy(code?: string, courseContent?: TutorPlanningContentContext): Promise { if (this.planningExtension?.resolveStrategy) { try { - return await this.planningExtension.resolveStrategy({ courseContent }); + return await this.planningExtension.resolveStrategy({ code, courseContent }); } catch { // A strategy resolver is optional planning context; the built-in policy remains authoritative. } diff --git a/shared/tutor.ts b/shared/tutor.ts index e876407ce..169bf5bdc 100644 --- a/shared/tutor.ts +++ b/shared/tutor.ts @@ -31,6 +31,7 @@ export const TUTOR_RATING_DIFFICULTY_DELTAS: Record = export const tutorResponseStyleSchema = z.enum(["normal", "philosophical"]); const tutorQuestionKindSchema = z.enum(["recall", "concept", "application", "prediction", "transfer"]); +const tutorLearningPhaseSchema = z.enum(["LEARN", "DEEPEN", "EXPAND"]); export const tutorStrategySourceSchema = z.enum(["built-in", "repository"]); export const tutorCourseContentContextSchema = z.object({ repository: repositorySlugSchema, @@ -49,6 +50,11 @@ const tutorLearningMetadataFields = { strategyId: z.string().regex(/^[a-z][a-z0-9-]{0,63}$/).optional(), strategySource: tutorStrategySourceSchema.optional(), contentRevision: z.string().regex(/^[a-f0-9]{40}$/i).optional(), + learningPhase: tutorLearningPhaseSchema.optional(), + activeTopicId: z.string().regex(/^[a-z][a-z0-9-]{0,63}$/).optional(), + masteredTopicIds: z.array(z.string().regex(/^[a-z][a-z0-9-]{0,63}$/)).max(64).optional(), + progressionBlockedReason: z.literal("content-exhausted").optional(), + extensionTargetTopicId: z.string().regex(/^[a-z][a-z0-9-]{0,63}$/).optional(), }; function withDefaultResponseStyle(value: unknown): unknown { diff --git a/ssot/ssot_function_definition_CourseContent.md b/ssot/ssot_function_definition_CourseContent.md index 8304d0bd7..3f9476e0e 100644 --- a/ssot/ssot_function_definition_CourseContent.md +++ b/ssot/ssot_function_definition_CourseContent.md @@ -265,6 +265,14 @@ strategies: defaultStrategy is optional. topics and strategies may each be empty; this explicitly supports a strategy-only repository and a topics-only repository. +Tutor manifest `schemaVersion: 1` remains exactly this contract: it has the +existing fields above and has no `phaseStrategies` field. Tutor manifest +`schemaVersion: 2` retains every v1 field and may add the fixed optional +entries `phaseStrategies.deepen` and `phaseStrategies.expand`; each entry may +reference only a strategy enumerated in the same validated bundle. A future +implementation MUST support both versions. A v1 manifest therefore continues +to work unchanged, while an unknown field in a strict v1 manifest remains +invalid rather than being reinterpreted as a v2 field. IDs are unique, lower-case safe IDs. File counts, individual file sizes, total Tutor bytes, and total referenced files are bounded by server configuration. @@ -276,6 +284,9 @@ Tutor capability validation is all-or-nothing: - an unknown field, unsupported schema version, duplicate ID, invalid path, invalid dependency, cyclic dependency, or broken question/scaffold/mastery reference invalidates the whole Tutor capability; +- an invalid phase-strategy reference, malformed bounded deepening criterion, + unknown extension target, or invalid extension data invalidates the whole + Tutor capability; - an annotation referring to an unknown topic or strategy invalidates the whole Tutor capability; - an invalid embedded annotation invalidates the whole Tutor capability but @@ -298,7 +309,9 @@ Topics describe what should be learned. A generic topic owns: - question text/data, question kind, applicability requirements, and difficulty ranges; - content-specific scaffolds and their next-question references; -- entry concepts and preferred concept order. +- entry concepts and preferred concept order; +- optional bounded deepening criteria and Topic extension targets as defined in + section 9.2. The supported question kinds remain exactly: recall, concept, application, prediction, and transfer. @@ -317,6 +330,14 @@ the generic schema. The existing pilot fields are normalized as follows: Repository text is untrusted educational data. It is bounded, control-character checked, URL-free, and passed only as structured didactic context. +Curriculum Topic `schemaVersion: 1` remains exactly this contract and has no +`deepening` or `extensions` fields. Curriculum Topic `schemaVersion: 2` +retains every v1 semantic and may add the optional bounded `deepening` and +`extensions` fields defined in section 9.2. A future implementation MUST +support both versions. A v1 Topic therefore continues to work unchanged and +uses the normative application default for deepening; strict v1 validation +does not reinterpret v2-only fields. + ## 9. Strategy model Strategies describe how the Tutor teaches. A strategy file uses a strict, @@ -473,6 +494,241 @@ The LLM is not required to produce a question-kind distribution matching the configured weights. All guidance text is owned by UnoSim and is generated from the normalized closed values; repository files never supply raw instructions. +### 9.2 Mastery-driven didactic progression + +Topic, EffectiveTutorStrategy, and didactic phase are separate concerns: + +- a Topic defines **WHAT** is learned; +- the EffectiveTutorStrategy defines **HOW** the Tutor teaches; +- the didactic phase defines **WHERE** the learner is in the bounded + progression. + +The only normative phases are `LEARN`, `DEEPEN`, and `EXPAND`. Repository +content cannot define additional phase names. A phase is not a strategy ID and +must not be represented only by changing strategy metadata. + +Formal phase progression exists only while one validated repository Topic is +the active primary Topic. A free Tutor without an active Topic remains normal +free Tutor behavior and does not claim formal Topic mastery. This restriction +also applies to arbitrary sketches with no matching Topic. + +Topic activation and Topic mastery use two explicit domains. The +`TopicMatcher`'s activation match is only a candidate. For the active Topic: + +- a question is fact-applicable when it is schema-valid and every question + `requires` entry matches the current sketch facts; +- a concept is in the current Topic mastery domain when at least one of its + questions is fact-applicable; +- a Topic is a probeable acquisition candidate when it has a non-empty mastery + domain and the normal planner can produce at least one fact-applicable + question under the active strategy, difficulty, repetition, and anti-loop + rules; +- a Topic activation match with a non-empty mastery domain, unmet mastery + criteria, and no probeable question is an unmastered unresolved diagnostic + result; the application must not silently treat it as mastered or use it to + enter DEEPEN; +- a Topic activation match with an empty mastery domain is not mastered and is + not an acquisition candidate or progression blocker; it cannot justify a + Topic question or a mastery claim; +- a mastery-domain concept whose applicable questions are exhausted or + otherwise cannot currently produce a fresh probe remains an unmet blocker + when its mastery criteria are false. It is not silently removed from the + domain. Existing evidence may still satisfy it if its criteria are already + true. + +In the transition rules below, “currently applicable Topic” means a normal +TopicMatcher match that is also a probeable acquisition candidate. A bare +activation match with no valid probe does not qualify as an unmastered +acquisition Topic. + +The current Topic schema expresses factual applicability on Questions, not on +Concepts or individual mastery indicators. It therefore cannot statically +guarantee that every required indicator has a fact-applicable probe for every +sketch. This contract does not invent concept-level requirements: when such a +probe is unavailable, the concept remains unresolved and blocks Topic mastery. +That is an explicit authoring/runtime limitation for the current schema. + +Topic mastery is application-controlled and deterministic. It requires every +concept in the non-empty current mastery domain to satisfy its existing +concept-level mastery criteria. A concept with no fact-applicable question is +outside that evaluation and does not create a false requirement. A Topic with +no mastery-domain concept is not mastered. The application evaluates the +existing fields as follows: + +- `minimumSuccessfulProbes` counts rated, valid probes for that concept whose + `answerRating` meets that concept's `successRatingAtLeast`; +- `successRatingAtLeast` is the minimum `answerRating` that makes a probe + successful; +- `requiredIndicators` requires at least one such successful probe for every + listed indicator; +- `minimumDistinctQuestionKinds` requires the successful probes to contain the + configured number of distinct question kinds; +- `recentWeakAnswersAllowed` uses the existing trailing weak-answer semantics: + a weak answer is an `answerRating` of `1` or `2`, and the relevant suffix + after the latest non-weak observation must not exceed the configured + allowance. It is not a free-form time window; +- one rated Tutor turn is one observation tied to one validated question, + concept, indicator, and kind. It may count simultaneously as one successful + probe, one indicator observation, and one question kind, but it cannot satisfy + multiple indicator IDs by itself. + +After each valid rated Tutor turn, the application reevaluates the active +Topic's mastery against all accumulated evidence. Mastery evidence and +`effectiveDifficulty` are separate values: the existing adaptive-difficulty +algorithm may influence question selection, but difficulty neither declares +mastery nor replaces any mastery criterion. + +A strong answer is evidence only. The LLM cannot declare Topic mastery, and a +`5` does not bypass any configured mastery criterion. After all criteria become +true, the application latches that Topic as mastered for the current session +and revision. Weak later answers do not erase that latch while the session and +Course revision remain valid. + +Topic applicability controls whether a Topic may currently be used; it does +not erase session-local mastery evidence. When a learner edit makes a mastered +Topic fact-inapplicable, UnoSim suspends it as the active Topic but retains its +mastered state, evidence, and retained post-mastery phase for the current +session. No fact-dependent question, Topic claim, or phase action may use that +Topic while it is inapplicable. If the same Topic becomes fact-applicable again +in the same session, it remains mastered and resumes its retained `DEEPEN` or +`EXPAND` phase rather than restarting `LEARN`. A new Tutor dialog/session or a +Course source, ref, revision, or example-context reset clears this retained +state under the existing reset contract. + +Each relevant Topic is classified as exactly one of: mastered; unmastered and +probeable; or unmastered and unresolved. An unresolved Topic has a non-empty +current mastery domain with unmet Concept mastery requirements, but the normal +planner cannot currently produce a valid fresh probe. It is not equivalent to +mastery or to an inapplicable Topic. + +After the active Topic reaches mastery, UnoSim recomputes currently applicable +Topics and applies the existing precedence: applicable embedded primary Topic, +other applicable embedded Topics, then fact-matched repository Topics. +Mastered Topics are skipped for acquisition selection during the current +session. If an applicable unmastered and probeable Topic remains, the +highest-precedence such Topic becomes active and the phase remains `LEARN`. +Thus a mastered Topic A plus an applicable probeable unmastered Topic B selects +Topic B in `LEARN`, never `DEEPEN` on A. + +If no probeable unmastered Topic remains but an applicable unresolved +unmastered Topic remains, progression is blocked. The conceptual phase remains +`LEARN`, the application emits the safe diagnostic +`progressionBlockedReason: content-exhausted`, and it never marks the Topic +mastered or enters `DEEPEN`. The existing `plan == null` free-Tutor fallback +may provide one bounded, user-initiated sketch-grounded remediation question +under the normal safety contract, without Topic mastery claims or new Topic +evidence. No autonomous request is created. If no safe free response can be +produced, the request fails as a controlled content-exhaustion condition +without mutating mastery or phase state. + +`DEEPEN` is admitted only when every currently relevant acquisition Topic is +either mastered or no longer fact-applicable, and no unresolved unmastered +Topic remains as a blocker. A Topic activation match with an empty mastery +domain is not a relevant acquisition Topic and is never treated as mastered. + +The transition into `DEEPEN` is therefore deterministic and does not require +a special user control or another LLM judgment. DEEPEN remains centered on the +mastered Topic only after that acquisition check and prefers application, +prediction, transfer, changed examples, consequences, and small conceptual +variations. It must not activate an unrelated Topic. + +Topic `schemaVersion: 2` may optionally define bounded deepening criteria: + +~~~yaml +deepening: + minimumSuccessfulProbes: 2 + successRatingAtLeast: 4 + requiredQuestionKinds: + - transfer + recentWeakAnswersAllowed: 0 +~~~ + +The fields are bounded as follows: probes `1..10`, success threshold `3..5`, +one to three unique required kinds from `application`, `prediction`, and +`transfer`, and trailing weak allowance `0..3`. The normative default when +`deepening` is absent is two successful post-mastery probes, each rated at +least `4`, with at least one `transfer` probe and no trailing weak probe. +Deepening evidence starts at the LEARN-to-DEEPEN transition; pre-mastery +answers do not count toward it. When the criteria are met, `DEEPEN -> EXPAND` +occurs deterministically. Schema v1 Topics cannot contain this field. + +`learningObjectives` remain applicable didactic emphasis in all three phases: +they may guide question, deepening, and extension direction, but they never +define mastery, activate a Topic, or select a strategy. EXPAND may present a +bounded extension direction or transfer challenge. It may +not edit the sketch, provide a normal full solution, force a Topic, or claim a +new Topic is active. If the learner does not change the sketch, the Tutor may +continue bounded transfer, offer another finite extension direction, or revisit +a demonstrated gap, subject to the existing anti-loop rules. It must not repeat +the same extension indefinitely or fabricate a new Topic. + +An optional Topic v2 extension list is bounded structured data: + +~~~yaml +extensions: + - topic: functions + objective: Repeated behavior can be moved into a function. +~~~ + +Each schemaVersion 2 Topic may contain at most eight extensions. Each entry has +only a safe target Topic ID and a trimmed bounded objective/reason (maximum 500 +Unicode +characters, no control characters, URLs, prompts, roles, templates, scripts, +or expressions). The target must exist in the same validated Tutor bundle. +An extension is educational guidance, never activation authority. The target +becomes active only when the normal fact matcher proves it applicable to the +current changed sketch. + +After a sketch edit, UnoSim re-extracts facts and reruns normal Topic +precedence. If a new unmastered and probeable Topic becomes applicable during +`DEEPEN` or `EXPAND`, the highest-precedence such Topic becomes active and +starts in `LEARN`. If a new unresolved unmastered Topic becomes applicable, +progression leaves `DEEPEN`/`EXPAND` for the blocked `LEARN` state described +above; it cannot be bypassed by the absence of a question. If the current +Topic becomes inapplicable, the same selection is rerun: a probeable +unmastered Topic starts in `LEARN`, an unresolved Topic blocks in `LEARN`, an +applicable mastered Topic resumes its retained `DEEPEN` or `EXPAND` phase, and +if no applicable Topic can be selected the Tutor falls back to free mode. +Multiple applicable Topics remain candidates, but only one primary Topic and +one primary question exist at a time. + +The minimal post-mastery strategy design is a fixed phase-strategy map in +Tutor manifest `schemaVersion: 2`: + +~~~yaml +phaseStrategies: + deepen: exploration-policy + expand: exploration-policy +~~~ + +Only the fixed keys `deepen` and `expand` are allowed. LEARN continues to use +the existing strategy precedence. For DEEPEN and EXPAND, a valid embedded +Example strategy remains highest priority, then the configured phase strategy, +then the repository `defaultStrategy`, then `built-in-default`. A missing phase +entry falls back to the already effective LEARN strategy. A phase strategy +reference that is unknown or invalid invalidates the complete Tutor capability; +no partial phase graph activates. Direct phase references do not chain to other +phase references, so circular transition ambiguity is impossible. This design +supports repository-controlled post-mastery strategy changes without a general +scripting or workflow language. + +Didactic phase state is session-local and pinned to the same immutable Course +revision as the Tutor content. Conceptually it includes the revision, active +Topic, its phase, per-Topic mastery state/evidence, retained per-Topic phase +evidence, and progression-blocked reason when applicable. It is not a +persistent learner model, grade, or cross-session profile. +Course selection, revision, or source-context changes reset it with the existing +Tutor dialog. A new learning question or new dialog starts a fresh didactic +session in the current validated Course context: dialog history, active Topic, +phase, mastery evidence, deepening evidence, session rating, and effective +difficulty reset; configured start difficulty, provider/model choice, and the +transient credential behavior retain their existing contracts. + +The complete phase transition contract is defined in +ssot_function_definition_LearningQuestions.md and ADR 0007. The repository +cannot change the one-question, factual-authority, privacy, security, response, +or automatic-editor boundaries. + An observed response is conformant only when `strategySource` and `strategyId` identify the EffectiveTutorStrategy that actually influenced the applicable planner, prompt, or dialogue behavior. Adding those fields after a @@ -503,6 +759,8 @@ Topic selection is: Embedded topics and primaryTopic are preferences, not factual authority. If editing makes an embedded topic inapplicable, it is skipped. The matcher may select another applicable topic; it MUST NOT force an unsupported question. +Skipping an inapplicable Topic affects current selection only and does not +erase its retained session-local mastery state. ### 10.1 Example learning objectives diff --git a/ssot/ssot_function_definition_LearningQuestions.md b/ssot/ssot_function_definition_LearningQuestions.md index 02dd1d9eb..edda35a70 100644 --- a/ssot/ssot_function_definition_LearningQuestions.md +++ b/ssot/ssot_function_definition_LearningQuestions.md @@ -51,7 +51,10 @@ Das Lernfragen-Panel ist ausdrücklich **nicht** vorgesehen für: - dauerhafte Speicherung persönlicher LLM-Zugangsdaten, - direkte Kommunikation Browser → externer LLM-Provider. -Eine spätere Erweiterung um längere Dialogverläufe, Challenges oder adaptive Lernpfade ist möglich, aber nicht Bestandteil des initialen MVP-Vertrags. +Längere Dialogverläufe, Challenges und allgemeine adaptive Lernpfade bleiben +außerhalb des initialen MVP-Vertrags. Der in Abschnitt 2.3 definierte, +begrenzte und ausschließlich sitzungsgebundene Topic-Phasenzyklus ist davon +ausgenommen und wird mit diesem SSOT-Update ausdrücklich normativ. ### Unified Course Content and Tutor strategy @@ -246,6 +249,352 @@ The built-in strategy values are the normative new UnoSim teaching policy: `current-contract`. The weights do not claim to reproduce a pre-existing planner weighting. +### 2.3 Mastery-driven didactic progression + +The Tutor distinguishes three independent concerns: + +| Concern | Answers | +|---|---| +| Topic | WHAT the learner should learn | +| EffectiveTutorStrategy | HOW the Tutor teaches | +| Didactic phase | WHERE the learner is in the bounded progression | + +The only phases are `LEARN`, `DEEPEN`, and `EXPAND`. They are application-owned +state, not repository-defined names and not alternate strategy IDs. A normal +Tutor request still uses exactly one EffectiveTutorStrategy in every phase. + +Formal mastery-driven phases require one active, applicable repository Topic. +This includes arbitrary sketches when a Topic matches. A free Tutor without an +active Topic remains ordinary sketch-grounded free Tutor behavior and does not +claim formal Topic mastery, DEEPEN, or EXPAND. + +The phase-related content versions are explicit. Tutor manifest +`schemaVersion: 1` has no `phaseStrategies`; manifest `schemaVersion: 2` +retains all v1 fields and may add only the fixed `deepen` and `expand` phase +strategy references. Curriculum Topic `schemaVersion: 1` has no `deepening` +or `extensions`; Topic `schemaVersion: 2` retains all v1 semantics and may +add those bounded fields. Future implementations support both versions, and +strict v1 validation does not reinterpret v2-only fields. + +#### LEARN + +LEARN is the normal Topic-learning phase. It combines current sketch facts, +the active Topic, concepts, indicators, mastery criteria, applicable Example +learning objectives, and the EffectiveTutorStrategy. Its purpose is to gather +deterministic evidence; an LLM may produce a validated `answerRating` and +indicator metadata, but it cannot declare mastery. + +The existing concept-level mastery fields have these exact meanings: + +- `minimumSuccessfulProbes`: number of rated, valid observations for the + concept whose rating meets that concept's `successRatingAtLeast`; +- `successRatingAtLeast`: minimum `answerRating` for a successful probe; +- `requiredIndicators`: every listed indicator must have at least one + successful observation tied to that indicator; +- `minimumDistinctQuestionKinds`: successful observations must contain at least + this many distinct question kinds; +- `recentWeakAnswersAllowed`: the trailing suffix of relevant observations + after the latest non-weak observation may contain at most this many weak + answers. A weak answer is an `answerRating` of `1` or `2`. The current + contract uses this trailing weak streak, not an arbitrary sliding time + window. + +One normal rated turn creates one observation associated with one validated +question, concept, indicator, and question kind. It may simultaneously count +as one successful probe, one indicator observation, and one distinct kind, but +one turn cannot satisfy multiple indicator IDs. A strong answer is therefore +evidence, not a special mastery command; all configured criteria must hold. + +After each valid rated Tutor turn, the application reevaluates the active +Topic's mastery against all accumulated evidence. Mastery evidence and +`effectiveDifficulty` are separate values: the existing adaptive-difficulty +algorithm may influence question selection, but difficulty neither declares +mastery nor replaces any mastery criterion. + +Topic activation and Topic mastery use two explicit domains. The +`TopicMatcher` activation result is only a candidate. A question is +fact-applicable when it is schema-valid and every question `requires` entry +matches the current sketch facts. A concept is in the current Topic mastery +domain when at least one of its questions is fact-applicable. A Topic is a +probeable acquisition candidate when it has a non-empty mastery domain and the +normal planner can produce at least one fact-applicable question under the +active strategy, difficulty, repetition, and anti-loop rules. + +A Topic activation match with a non-empty mastery domain, unmet mastery +criteria, and no probeable question is an unmastered unresolved diagnostic +result and must not silently be treated as mastered or used to enter DEEPEN. A +Topic activation match with an empty mastery domain is not mastered and is not +an acquisition candidate or progression blocker; it cannot justify a Topic +question or a mastery claim. A mastery-domain concept whose applicable +questions are exhausted or otherwise cannot currently produce a fresh probe +remains an unmet blocker when its mastery criteria are false; it is not +silently removed from the domain. +Existing evidence may still satisfy it if its criteria are already true. + +In the transition rules below, “currently applicable Topic” means a normal +TopicMatcher match that is also a probeable acquisition candidate. A bare +activation match with no valid probe does not qualify as an unmastered +acquisition Topic. + +The current Topic schema expresses factual applicability on Questions, not on +Concepts or individual mastery indicators. It therefore cannot statically +guarantee that every required indicator has a fact-applicable probe for every +sketch. This contract does not invent concept-level requirements: when such a +probe is unavailable, the concept remains unresolved and blocks Topic mastery. +That is an explicit authoring/runtime limitation for the current schema. + +Topic mastery is true only when every concept in the non-empty current mastery +domain satisfies its concept-level mastery criteria. A concept with no +fact-applicable question is outside that evaluation and creates no false +requirement. A Topic with no mastery-domain concept is not mastered. This +aggregation is deterministic and application-controlled. + +Each relevant Topic is classified as exactly one of: mastered; unmastered and +probeable; or unmastered and unresolved. An unresolved Topic has a non-empty +current mastery domain with unmet Concept mastery requirements, but the normal +planner cannot currently produce a valid fresh probe. It is not equivalent to +mastery or to an inapplicable Topic. + +If no probeable unmastered Topic remains but an applicable unresolved +unmastered Topic remains, progression is blocked. The conceptual phase remains +LEARN, the application emits the safe diagnostic +`progressionBlockedReason: content-exhausted`, and it never marks the Topic +mastered or enters DEEPEN. The existing null-plan free-Tutor fallback may +provide one bounded, user-initiated sketch-grounded remediation question under +the normal safety contract, without Topic mastery claims or new Topic +evidence. If no safe free response can be produced, the request fails as a +controlled content-exhaustion condition without mutating mastery or phase +state. + +Once true, mastery is latched for that Topic in the current Tutor session and +Course revision. Topic applicability controls whether the Topic may currently +be used; it does not erase session-local mastery evidence. When a learner edit +makes a mastered Topic fact-inapplicable, UnoSim suspends it as the active +Topic but retains its mastered state, evidence, and retained `DEEPEN` or +`EXPAND` phase. No fact-dependent question or Topic claim may use it while +inapplicable. If it becomes fact-applicable again in the same session, it +remains mastered and resumes that retained phase rather than restarting LEARN. +A new Tutor dialog/session or a Course source, ref, revision, or +example-context reset clears the retained state under the existing reset +contract. + +`learningObjectives` remain additional teacher-authored emphasis in LEARN, +DEEPEN, and EXPAND. They may guide the question, deepening, or extension +direction, but never define mastery, activate a Topic, or select a strategy. + +#### LEARN to DEEPEN + +When the active Topic reaches mastery, UnoSim recomputes currently applicable +Topics and applies normal precedence: applicable embedded primary Topic, other +applicable embedded Topics, then fact-matched repository Topics. Mastered +Topics are skipped for acquisition selection during the current session. If an +applicable unmastered and probeable Topic remains, the highest-precedence such +Topic becomes active and remains in LEARN. Thus a mastered Topic A plus an +applicable probeable unmastered Topic B selects Topic B in LEARN, never DEEPEN +on A. + +If no probeable unmastered Topic remains but an unresolved unmastered Topic +remains, the blocked LEARN outcome applies. DEEPEN is admitted only when every +currently relevant acquisition Topic is mastered or no longer fact-applicable, +and no unresolved unmastered Topic remains. A Topic activation match with an +empty mastery domain is not a relevant acquisition Topic and is never treated +as mastered. No special user control or additional LLM judgment is required; +the internal phase is application-owned. + +#### DEEPEN + +DEEPEN remains centered on the mastered Topic. The planner and application +guidance prefer applicable `application`, `prediction`, and `transfer` +questions, changed examples, consequences, and small conceptual/code +variations. Simple recall is avoided unless new evidence indicates a gap. +DEEPEN does not activate an unrelated Topic. + +Deepening evidence begins when LEARN changes to DEEPEN. The normative default +criterion for `DEEPEN -> EXPAND` is: + +- two successful post-mastery probes; +- each has `answerRating >= 4`; +- at least one probe is `transfer`; +- no trailing weak probe. + +Topic `schemaVersion: 2` may configure bounded criteria using this structure: + +~~~yaml +deepening: + minimumSuccessfulProbes: 2 + successRatingAtLeast: 4 + requiredQuestionKinds: + - transfer + recentWeakAnswersAllowed: 0 +~~~ + +The bounds are probes `1..10`, success threshold `3..5`, one to three unique +required kinds from `application`, `prediction`, and `transfer`, and trailing +weak allowance `0..3`. An absent `deepening` configuration uses the default +above. No expression language, arbitrary predicate, or repository prompt is +allowed. Schema v1 Topics cannot contain this field. + +#### DEEPEN to EXPAND + +When the deterministic deepening criterion is satisfied, the phase becomes +EXPAND. EXPAND may suggest a bounded extension, such as extracting repeated +behavior into a function, processing multiple values, adding a timed action, +or introducing a small data structure. The Tutor must not edit the sketch, +provide a normal full finished solution, or claim a Topic is active before +facts prove it. Every response still contains exactly one bounded primary +question. + +If the learner does not change the sketch, the Tutor may continue bounded +transfer questions, offer another finite extension direction, or revisit a +demonstrated gap. It must not repeat the same extension indefinitely or invent +a new Topic. + +#### Post-mastery strategy resolution + +The existing LEARN strategy precedence remains unchanged: + +1. valid embedded Example strategy; +2. repository `defaultStrategy`; +3. application-owned `built-in-default`. + +A Tutor manifest `schemaVersion: 2` may add only the fixed optional keys +`phaseStrategies.deepen` and `phaseStrategies.expand`. Resolution for each +post-mastery phase is: + +1. valid embedded Example strategy, if the active Example defines one; +2. the configured strategy for that fixed phase, if present; +3. repository `defaultStrategy`; +4. `built-in-default`. + +If the phase entry is absent, the already effective LEARN strategy remains in +force. Phase entries reference validated strategies directly and cannot chain +or reference another phase. Unknown or invalid phase strategy references +invalidate the complete Tutor capability under the existing capability-scoped +fallback rule. Thus a repository can use `precision-policy` in LEARN and +`exploration-policy` in DEEPEN/EXPAND without turning strategies into a +workflow language. An embedded Example strategy remains the highest-priority +Example-specific override in every phase. + +#### Topic extensions and EXPAND to new LEARN + +A Topic `schemaVersion: 2` may optionally declare at most eight bounded +extensions: + +~~~yaml +extensions: + - topic: functions + objective: Repeated behavior can be moved into a function. +~~~ + +Each entry contains only a safe target Topic ID and a trimmed objective/reason +of at most 500 Unicode characters, without control characters, URLs, prompts, +roles, templates, scripts, executable content, or expressions. The target must +exist in the same validated Tutor bundle. Extension data is guidance only; a +named target is never active merely because it is listed. + +After a learner edit, UnoSim re-extracts facts and reruns normal Topic +selection. If a new unmastered Topic becomes applicable during DEEPEN or +EXPAND, the highest-precedence currently applicable unmastered and probeable +Topic becomes active and starts independently in LEARN. If a new unresolved +unmastered Topic becomes applicable, progression leaves DEEPEN/EXPAND for the +blocked LEARN state; it cannot be bypassed by the absence of a question. If +the current Topic becomes inapplicable, the same selection is rerun: a +probeable unmastered Topic starts in LEARN, an unresolved Topic blocks in LEARN, +an applicable mastered Topic resumes its retained DEEPEN or EXPAND phase, and +if no applicable Topic can be selected the Tutor falls back to free mode. +Previous Topic IDs may remain as session history for loop avoidance and +diagnostics, but their mastery is not carried across different Topic IDs or +used as cross-topic competence. + +If several Topics are applicable, only one primary Topic drives the next +question; the others remain candidates/context. The one-primary-question rule +is unchanged. + +#### Session, reset, and persistence semantics + +Didactic state is pinned to the existing opaque Tutor session and immutable +Course revision. Conceptually it contains: + +- the server-authorized revision; +- active primary Topic ID; +- current phase and any progression-blocked reason; +- per-Topic mastery state and evidence; +- retained per-Topic post-mastery/deepening phase evidence. + +The state is process-local and session-local like the current one-hour opaque +Tutor-session store. It is not a persistent learner model, grade, or identity- +based profile. A Course source, ref, revision, or example-context change resets +the dialog and all didactic state; content from another revision is never +substituted. + +`New learning question` and `New dialog` begin a fresh didactic session in the +current validated Course context. They clear dialog history, session rating, +active Topic, phase, mastery evidence, and deepening evidence, and reset +`effectiveDifficulty` to the configured start value. They retain the existing +configured start difficulty, provider/model selection, and transient credential +behavior. A subsequent request reruns Topic selection and starts the selected +Topic in LEARN. They do not create an autonomous request; the next Tutor call +remains explicitly user initiated. + +An expired or unknown opaque session handle fails safely under the existing +route contract. It cannot silently switch phase, Topic, revision, or strategy. +Horizontal multi-instance deployment still requires shared session state or an +explicitly designed equivalent such as sticky-session guarantees. + +#### Arbitrary sketches and free Tutor + +An arbitrary/self-written sketch uses repository Topic matching and repository +default/built-in strategy resolution exactly as today. If a Topic matches, +formal LEARN/DEEPEN/EXPAND progression is available. If no Topic matches, +learning objectives are absent unless an active Example context supplies them, +and free Tutor remains available with the selected EffectiveTutorStrategy but +without formal Topic mastery claims. + +#### State-transition table + +| Current phase | Condition | Next phase | Active Topic | Strategy behavior | +|---|---|---|---|---| +| LEARN | Topic mastery criteria not all true | LEARN | unchanged Topic | LEARN precedence | +| LEARN | Topic A mastered; applicable unmastered Topic B exists | LEARN | highest-precedence unmastered Topic | acquisition precedence; mastered Topics skipped | +| LEARN | No probeable unmastered Topic, but unresolved unmastered Topic remains | LEARN (blocked) | highest-precedence unresolved Topic | `progressionBlockedReason: content-exhausted`; bounded free fallback only | +| LEARN | All relevant acquisition Topics mastered or no longer fact-applicable; no unresolved blocker | DEEPEN | same mastered Topic | post-mastery resolution begins | +| DEEPEN | insufficient successful transfer evidence | DEEPEN | same mastered Topic | prefer application/prediction/transfer | +| DEEPEN | deepening criterion satisfied | EXPAND | same Topic | EXPAND phase strategy/fallback | +| EXPAND | no sketch change or no new Topic | EXPAND, or bounded DEEPEN for a demonstrated gap | same Topic | anti-loop bounded extension/transfer behavior | +| DEEPEN/EXPAND | new applicable unmastered Topic exists | LEARN | highest-precedence unmastered Topic | normal Topic precedence; new evidence | +| DEEPEN/EXPAND | new applicable unresolved unmastered Topic exists | LEARN (blocked) | highest-precedence unresolved Topic | no DEEPEN/EXPAND; content-exhaustion handling | +| EXPAND | changed sketch activates a new unmastered primary Topic | LEARN | highest-precedence unmastered Topic | new Topic starts/continues its own acquisition evidence | +| Any phase | current Topic becomes inapplicable | rerun selection; unmastered probeable Topic starts LEARN, unresolved Topic blocks LEARN, mastered Topic resumes retained phase, or free Tutor | selected applicable Topic or none | no fact-dependent use while inactive | +| Any phase | Tutor capability becomes invalid | free Tutor | none | built-in-default; no partial bundle | + +#### Diagnostics and security + +Safe diagnostics may expose `topicId`, `topicMastered`, `didacticPhase`, +`strategyId`, `strategySource`, `postMasteryStrategyId`, bounded mastery +evidence counts, and bounded extension candidate IDs. They must not expose raw +repository payloads, hidden prompts, credentials, or private dialog history. + +The application owns all transition text and actual provider instructions. +Repository phase settings, mastery criteria, extension objectives, and strategy +IDs are normalized bounded data. They cannot control prompts, roles, URLs, +regexes, scripts, templates, editor operations, response schemas, privacy, +credentials, Mermaid safety, answerRating semantics, or automatic requests. + +#### Non-normative worked example + +An `int` value printed with `Serial.println` matches a +`variables-and-serial` Topic. The session starts in LEARN with +`precision-policy`. After all deterministic concept mastery criteria are met, +the same Topic enters DEEPEN and uses `exploration-policy` when the repository +phase map selects it. The Tutor asks a transfer question about changing the +stored value or reusing the output idea. After sufficient successful +post-mastery probes, EXPAND may suggest extracting repeated behavior into a +function. The learner edits the sketch; only if the fact extractor and matcher +prove a future `functions` Topic applicable does that Topic become active, and +it starts in LEARN. UnoSim does not currently claim that its fact extractor +detects function semantics; that final step is future behavior dependent on +future extractor support. + --- ## 3. Didaktischer Kernvertrag @@ -922,7 +1271,9 @@ Sie umfasst höchstens das definierte Sliding Window und enthält keine Credenti - die aktuelle Modellwahl beibehalten. - den konfigurierten Difficulty-Startwert beibehalten, - die effektive Difficulty wieder auf den konfigurierten Startwert setzen, -- die Sessionbewertung zurücksetzen. +- die Sessionbewertung zurücksetzen, +- den aktiven Topic, die didaktische Phase, die Mastery-Evidenz und die + Deepening-Evidenz zurücksetzen. Das Zurücksetzen darf keine automatische neue LLM-Anfrage auslösen. @@ -1118,6 +1469,15 @@ Die Implementierung gilt erst als korrekt, wenn automatisierte Tests mindestens - Dialog-Requests akzeptieren nur begrenzte Historien ohne Credentials, - erfolgreiche Dialog-Requests liefern optional Feedback und genau eine Folgefrage, - fehlgeschlagene Dialog-Requests verändern keinen gespeicherten Serverzustand, +- Topic-Mastery wird ausschließlich aus validierten strukturierten Kriterien und + answerRating-Evidenz deterministisch berechnet, +- LEARN → DEEPEN erfolgt ohne eine freie LLM-Mastery-Entscheidung, +- DEEPEN → EXPAND benötigt die konfigurierte oder normative Deepening-Evidenz, +- EXPAND aktiviert keinen Topic ohne erneute Fakt- und Topic-Matcher-Prüfung, +- unbekannte Phase-Strategien oder Extension-Ziele invalidieren den Tutor-Bundle + atomar und lassen gültige Examples aktiv, +- Course-Revision, Session-Pinning und didaktischer Zustand bleiben gemeinsam + konsistent, - Provider-Timeout wird behandelt, - 401/403/429/5xx des Providers werden kontrolliert abgebildet, - Secrets erscheinen nicht in Logs oder Responses. @@ -1190,7 +1550,8 @@ Nicht Teil des MVP: - automatische Prüfungsbewertung oder Benotung freier Studierendenantworten, - unbegrenzter oder langfristig gespeicherter Chatverlauf, -- automatische Kompetenzmodelle, +- persistente oder automatische Kompetenzmodelle; der begrenzte, deterministische + Topic-Phasenzyklus in Abschnitt 2.3 ist ausdrücklich kein solches Modell, - Notengebung, - automatisches Ändern des Sketches, - langfristige Speicherung von Lernverläufen, diff --git a/tests/server/routes/tutor.routes.test.ts b/tests/server/routes/tutor.routes.test.ts index cf1382d3c..1c5bc0953 100644 --- a/tests/server/routes/tutor.routes.test.ts +++ b/tests/server/routes/tutor.routes.test.ts @@ -287,7 +287,7 @@ describe("Tutor HTTP route", () => { expect(second.status).toBe(200); expect(service.generateDialogResponse).toHaveBeenCalledWith( "void setup(){}", [], "Frage A", "Antwort", - "request-only-secret", undefined, 30, contentA, + "request-only-secret", undefined, 30, expect.objectContaining(contentA), ); expect(resolver.resolveTutorContent).toHaveBeenCalledOnce(); }); diff --git a/tests/server/services/course-content/course-content-loader.test.ts b/tests/server/services/course-content/course-content-loader.test.ts index db1aee56a..85f2c9e4b 100644 --- a/tests/server/services/course-content/course-content-loader.test.ts +++ b/tests/server/services/course-content/course-content-loader.test.ts @@ -207,4 +207,85 @@ describe("unified Course Content loader", () => { expect(loaded.examples).toHaveLength(1); expect(loaded.tutor).toMatchObject({ status: "invalid" }); }); + + it("loads manifest and Topic v2 phase data atomically", async () => { + const topicV1 = await readFile(path.resolve(process.cwd(), "curriculum/topics/memory-and-data-types.yaml"), "utf8"); + const topic = topicV1.replace("schemaVersion: 1", "schemaVersion: 2\nextensions:\n - topic: memory-and-data-types\n objective: Ein verwandtes Beispiel vergleichen.\n"); + const strategy = [ + "schemaVersion: 1", + "id: repository-default", + "questionKindWeights:", + " recall: 10", + " concept: 25", + " application: 35", + " prediction: 15", + " transfer: 15", + "sketchSpecificity: prefer", + "repetition: strict", + "remediation: scaffold-first", + "clarification: same-indicator", + "progression: mastery-then-advance", + "scaffolding: prefer-content", + "feedbackVerbosity: short", + "hintFirst: true", + "adaptiveDifficulty: current-contract", + "", + ].join("\n"); + const manifest = [ + "schemaVersion: 2", + "defaultStrategy: repository-default", + "phaseStrategies:", + " deepen: repository-default", + " expand: repository-default", + "topics:", + " - id: memory-and-data-types", + " path: tutor/topics/memory-and-data-types.yaml", + ` sha256: ${digest(topic)}`, + "strategies:", + " - id: repository-default", + " path: tutor/strategies/repository-default.yaml", + ` sha256: ${digest(strategy)}`, + "", + ].join("\n"); + const { fetchText } = fetcherFor({ + [`/owner/repo/${revision}/manifest.json`]: JSON.stringify({ schemaVersion: 2, examples: [example], tutor: { manifest: "tutor/manifest.yaml" } }), + [`/owner/repo/${revision}/examples/main.ino`]: "void setup() {}\n", + [`/owner/repo/${revision}/tutor/manifest.yaml`]: manifest, + [`/owner/repo/${revision}/tutor/topics/memory-and-data-types.yaml`]: topic, + [`/owner/repo/${revision}/tutor/strategies/repository-default.yaml`]: strategy, + }); + + const loaded = await new CourseContentLoader({ fetchText }, 2).load("owner/repo", revision); + expect(loaded.tutor).toMatchObject({ status: "valid", manifest: { schemaVersion: 2 } }); + if (loaded.tutor.status === "valid") { + expect(loaded.tutor.topics[0]?.schemaVersion).toBe(2); + expect(loaded.tutor.manifest.schemaVersion === 2 && loaded.tutor.manifest.phaseStrategies?.deepen).toBe("repository-default"); + } + }); + + it("invalidates the complete Tutor bundle for an unknown phase strategy or extension target", async () => { + const topicV1 = await readFile(path.resolve(process.cwd(), "curriculum/topics/memory-and-data-types.yaml"), "utf8"); + const topic = topicV1.replace("schemaVersion: 1", "schemaVersion: 2\nextensions:\n - topic: missing-topic\n objective: Unbekanntes Ziel.\n"); + const manifest = [ + "schemaVersion: 2", + "phaseStrategies:", + " deepen: missing-strategy", + "topics:", + " - id: memory-and-data-types", + " path: tutor/topics/memory-and-data-types.yaml", + ` sha256: ${digest(topic)}`, + "strategies: []", + "", + ].join("\n"); + const { fetchText } = fetcherFor({ + [`/owner/repo/${revision}/manifest.json`]: JSON.stringify({ schemaVersion: 2, examples: [example], tutor: { manifest: "tutor/manifest.yaml" } }), + [`/owner/repo/${revision}/examples/main.ino`]: "void setup() {}\n", + [`/owner/repo/${revision}/tutor/manifest.yaml`]: manifest, + [`/owner/repo/${revision}/tutor/topics/memory-and-data-types.yaml`]: topic, + }); + + const loaded = await new CourseContentLoader({ fetchText }, 2).load("owner/repo", revision); + expect(loaded.examples).toHaveLength(1); + expect(loaded.tutor).toMatchObject({ status: "invalid" }); + }); }); diff --git a/tests/server/services/course-content/course-content-session.test.ts b/tests/server/services/course-content/course-content-session.test.ts new file mode 100644 index 000000000..a68f98046 --- /dev/null +++ b/tests/server/services/course-content/course-content-session.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it, vi } from "vitest"; +import { TutorCourseContentSessionStore } from "../../../../server/services/course-content/course-content-session"; + +describe("Course Content Tutor progression session", () => { + it("keeps progression state attached to the pinned immutable revision", () => { + const store = new TutorCourseContentSessionStore(60_000, vi.fn(() => 1_000)); + const content = { + repository: "owner/repo" as const, + ref: "main" as const, + revision: "a".repeat(40) as `${string}`, + tutor: { status: "absent" as const }, + }; + const handle = store.create("identity", content); + const first = store.get("identity", handle); + expect(first?.revision).toBe(content.revision); + first!.progressionState!.activeTopicId = "arrays"; + expect(store.get("identity", handle)?.progressionState?.activeTopicId).toBe("arrays"); + expect(store.get("other", handle)).toBeNull(); + }); + + it("fails safely after TTL expiry instead of switching revision", () => { + let now = 1_000; + const store = new TutorCourseContentSessionStore(10, () => now); + const handle = store.create("identity", { + repository: "owner/repo" as const, + ref: "main" as const, + revision: "b".repeat(40) as `${string}`, + tutor: { status: "absent" as const }, + }); + now = 1_011; + expect(store.get("identity", handle)).toBeNull(); + }); +}); diff --git a/tests/server/services/tutor/final-review-fixes.test.ts b/tests/server/services/tutor/final-review-fixes.test.ts new file mode 100644 index 000000000..5f59e418b --- /dev/null +++ b/tests/server/services/tutor/final-review-fixes.test.ts @@ -0,0 +1,275 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { TutorCourseContentSessionStore } from "../../../../server/services/course-content/course-content-session"; +import { parseTopic } from "../../../../server/services/tutor/curriculum/content-repository"; +import { CurriculumTutorAdapter } from "../../../../server/services/tutor/curriculum-tutor-adapter"; +import { createTutorProgressionState, markTopicMastered, type TutorProgressionState } from "../../../../server/services/tutor/curriculum/progression-state"; +import { TutorProviderError, type LLMProvider } from "../../../../server/services/tutor/llm-provider"; +import { TutorService } from "../../../../server/services/tutor/tutor-service"; +import { BUILT_IN_TUTOR_STRATEGY, type EffectiveTutorStrategy } from "../../../../server/services/tutor/strategy/effective-tutor-strategy"; + +const revision = "a".repeat(40) as `${string}`; + +async function sourceTopic() { + return parseTopic(await readFile(path.resolve(process.cwd(), "curriculum/topics/memory-and-data-types.yaml"), "utf8")); +} + +function strategy(id: string, overrides: Partial = {}): EffectiveTutorStrategy { + return { ...BUILT_IN_TUTOR_STRATEGY, id, ...overrides }; +} + +function tutorContext( + state: TutorProgressionState, + topics: readonly Awaited>[], + strategies: readonly EffectiveTutorStrategy[], + defaultStrategy: string, +) { + return { + repository: "owner/repo" as const, + ref: "main" as const, + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { + schemaVersion: 2 as const, + defaultStrategy, + phaseStrategies: { deepen: "exploration-policy", expand: "exploration-policy" }, + topics: [], + strategies: [], + }, + topics, + strategies, + }, + }; +} + +function successfulProvider(question = "Providerfrage"): LLMProvider { + return { + listModels: vi.fn().mockResolvedValue(["pilot-model"]), + generateLearningQuestion: vi.fn().mockResolvedValue({ + model: "pilot-model", + result: { question }, + }), + }; +} + +function sessionContent(content: ReturnType) { + const sessions = new TutorCourseContentSessionStore(); + const handle = sessions.create("identity", content); + const pinned = sessions.get("identity", handle); + if (!pinned) throw new Error("Expected a pinned Tutor session"); + return { sessions, handle, pinned }; +} + +async function expansionContext(objective: string) { + const source = await sourceTopic(); + const target = { ...source, schemaVersion: 2 as const, id: "target-topic", activation: { any: [{ fact: "serial-call" as const, values: ["write"] }] } }; + const sourceWithExtension = { + ...source, + schemaVersion: 2 as const, + id: "source-topic", + extensions: [{ topic: target.id, objective }], + }; + const state = createTutorProgressionState(revision); + state.activeTopicId = sourceWithExtension.id; + state.phase = "EXPAND"; + state.retainedPhases[sourceWithExtension.id] = "EXPAND"; + markTopicMastered(state, sourceWithExtension.id); + const precision = strategy("precision-policy"); + const exploration = strategy("exploration-policy", { feedbackVerbosity: "detailed", progression: "advance-immediately", hintFirst: true }); + return { + content: tutorContext(state, [sourceWithExtension, target], [precision, exploration], precision.id), + state, + sourceId: sourceWithExtension.id, + targetId: target.id, + }; +} + +describe("Tutor final review conformance", () => { + it("does not persist an EXPAND target when the provider times out, then commits it on retry", async () => { + const setup = await expansionContext("Wiederholte Verarbeitung in eine Funktion auslagern."); + const { pinned } = sessionContent(setup.content); + const provider = successfulProvider(); + vi.mocked(provider.generateLearningQuestion) + .mockRejectedValueOnce(new TutorProviderError("provider-timeout")); + const service = new TutorService(provider, new CurriculumTutorAdapter()); + const before = structuredClone(pinned.progressionState); + + await expect(service.generateQuestion("int value = 1;", "key", undefined, 30, pinned)).rejects.toMatchObject({ kind: "provider-timeout" }); + expect(pinned.progressionState).toEqual(before); + expect(pinned.progressionState?.usedExpansionTargetTopicIds).toEqual({}); + + await expect(service.generateQuestion("int value = 1;", "key", undefined, 30, pinned)).resolves.toMatchObject({ result: { activeTopicId: setup.sourceId } }); + expect(pinned.progressionState?.usedExpansionTargetTopicIds).toEqual({ [setup.sourceId]: [setup.targetId] }); + expect(pinned.progressionState?.phase).toBe("EXPAND"); + }); + + it("does not persist dialog planning mutations when the provider times out", async () => { + const setup = await expansionContext("Eine Dialog-Expansion darf erst nach erfolgreicher Antwort festgeschrieben werden."); + const { pinned } = sessionContent(setup.content); + const provider = successfulProvider(); + vi.mocked(provider.generateLearningQuestion).mockRejectedValueOnce(new TutorProviderError("provider-timeout")); + const before = structuredClone(pinned.progressionState); + + await expect(new TutorService(provider, new CurriculumTutorAdapter()).generateDialogResponse( + "int value = 1;", + [], + "Welche Erweiterung ist sinnvoll?", + "Ich prüfe zunächst den aktuellen Wert.", + "key", + undefined, + 30, + pinned, + )).rejects.toMatchObject({ kind: "provider-timeout" }); + expect(pinned.progressionState).toEqual(before); + }); + + it("does not persist progression changes for an invalid provider payload", async () => { + const source = await sourceTopic(); + const state = createTutorProgressionState(revision); + const content = tutorContext(state, [source], [strategy("precision-policy")], "precision-policy"); + const { pinned } = sessionContent(content); + const provider = successfulProvider(); + vi.mocked(provider.generateLearningQuestion).mockResolvedValueOnce({ + model: "pilot-model", + result: { question: "```cpp\nvoid setup() {}\nvoid loop() {}\n```" }, + }); + const before = structuredClone(pinned.progressionState); + + await expect(new TutorService(provider, new CurriculumTutorAdapter()).generateQuestion( + "int value = 1;", + "key", + undefined, + 30, + pinned, + )).rejects.toMatchObject({ kind: "invalid-response" }); + expect(pinned.progressionState).toEqual(before); + }); + + it.each(["provider-unavailable", "rate-limited"] as const)( + "does not persist progression changes for a provider %s failure", + async (kind) => { + const setup = await expansionContext("Eine fehlgeschlagene Anfrage darf keinen Lernfortschritt verbrauchen."); + const { pinned } = sessionContent(setup.content); + const provider = successfulProvider(); + vi.mocked(provider.generateLearningQuestion).mockRejectedValueOnce(new TutorProviderError(kind)); + const before = structuredClone(pinned.progressionState); + + await expect(new TutorService(provider, new CurriculumTutorAdapter()).generateQuestion( + "int value = 1;", + "key", + undefined, + 30, + pinned, + )).rejects.toMatchObject({ kind }); + expect(pinned.progressionState).toEqual(before); + }, + ); + + it("does not persist a failed LEARN-to-DEEPEN planning transition", async () => { + const source = await sourceTopic(); + const concept = { + ...source.concepts[0]!, + mastery: { + ...source.concepts[0]!.mastery, + minimumSuccessfulProbes: 1, + minimumDistinctQuestionKinds: 1, + requiredIndicators: [source.questions[0]!.indicator], + }, + }; + const topic = { + ...source, + id: "learn-to-deepen-topic", + concepts: [concept], + questions: source.questions.slice(0, 2).map((question, index) => ({ ...question, id: `learn-to-deepen-${index}`, concept: concept.id })), + scaffolds: [], + progression: { ...source.progression, entryConcepts: [concept.id], preferredOrder: [concept.id] }, + }; + const state = createTutorProgressionState(revision); + state.masteryEvidence[topic.id] = [{ + questionId: topic.questions[0]!.id, + conceptId: concept.id, + indicatorId: topic.questions[0]!.indicator, + kind: topic.questions[0]!.kind, + rating: 4, + }]; + const content = tutorContext(state, [topic], [strategy("precision-policy")], "precision-policy"); + const { pinned } = sessionContent(content); + const provider = successfulProvider(); + vi.mocked(provider.generateLearningQuestion).mockResolvedValueOnce({ + model: "pilot-model", + result: { question: "```cpp\nvoid setup() {}\nvoid loop() {}\n```" }, + }); + const before = structuredClone(pinned.progressionState); + + await expect(new TutorService(provider, new CurriculumTutorAdapter()).generateQuestion( + "int value = 1;", + "key", + undefined, + 30, + pinned, + )).rejects.toMatchObject({ kind: "invalid-response" }); + expect(pinned.progressionState).toEqual(before); + }); + + it.each(["LEARN", "DEEPEN"] as const)( + "uses the new LEARN strategy after a %s context reveals an unmastered Topic", + async (phase) => { + const source = await sourceTopic(); + const topicA = { ...source, schemaVersion: 2 as const, id: "topic-a" }; + const topicB = { ...source, schemaVersion: 2 as const, id: "topic-b", activation: { any: [{ fact: "serial-call" as const, values: ["write"] }] } }; + const state = createTutorProgressionState(revision); + state.activeTopicId = topicA.id; + state.phase = phase; + state.retainedPhases[topicA.id] = phase; + markTopicMastered(state, topicA.id); + const precision = strategy("precision-policy", { feedbackVerbosity: "short", progression: "mastery-then-advance", hintFirst: false }); + const exploration = strategy("exploration-policy", { feedbackVerbosity: "detailed", progression: "advance-immediately", hintFirst: true }); + const content = tutorContext(state, [topicA, topicB], [precision, exploration], precision.id); + const { pinned } = sessionContent(content); + const prompts: string[] = []; + const provider: LLMProvider = { + listModels: vi.fn().mockResolvedValue(["pilot-model"]), + generateLearningQuestion: vi.fn().mockImplementation(async (request) => { + prompts.push(request.userPrompt); + return { model: "pilot-model", result: { question: "Providerfrage" } }; + }), + }; + + const result = await new TutorService(provider, new CurriculumTutorAdapter()).generateQuestion( + "int value = 1; Serial.write('A');", + "key", + undefined, + 30, + pinned, + ); + + expect(result.result).toMatchObject({ activeTopicId: topicB.id, learningPhase: "LEARN", strategyId: precision.id }); + expect(prompts[0]).toContain("Feedback: short"); + expect(prompts[0]).not.toContain("Feedback: detailed"); + expect(prompts[0]).toContain("Progression: mastery-then-advance"); + }, + ); + + it("realizes different extension objectives in the final TutorService question without activating the target", async () => { + const first = await expansionContext("Wiederholte Verarbeitung in eine Funktion auslagern."); + const second = await expansionContext("Eine serielle Ausgabe als klaren Diagnosewert verwenden."); + const firstSession = sessionContent(first.content).pinned; + const secondSession = sessionContent(second.content).pinned; + const service = new TutorService(successfulProvider(), new CurriculumTutorAdapter()); + + const firstResult = await service.generateQuestion("int value = 1;", "key", undefined, 30, firstSession); + const secondResult = await service.generateQuestion("int value = 1;", "key", undefined, 30, secondSession); + + expect(firstResult.result.question).toContain("Wiederholte Verarbeitung in eine Funktion auslagern."); + expect(secondResult.result.question).toContain("Eine serielle Ausgabe als klaren Diagnosewert verwenden."); + expect((firstResult.result.question.match(/\?/g) ?? [])).toHaveLength(1); + expect((secondResult.result.question.match(/\?/g) ?? [])).toHaveLength(1); + expect(firstResult.result.question).not.toBe(secondResult.result.question); + expect(firstResult.result).toMatchObject({ activeTopicId: first.sourceId, learningPhase: "EXPAND" }); + expect(firstResult.result.activeTopicId).not.toBe(first.targetId); + expect(firstSession.progressionState?.activeTopicId).toBe(first.sourceId); + }); +}); diff --git a/tests/server/services/tutor/mastery-progression-schema.test.ts b/tests/server/services/tutor/mastery-progression-schema.test.ts new file mode 100644 index 000000000..0794b1cb0 --- /dev/null +++ b/tests/server/services/tutor/mastery-progression-schema.test.ts @@ -0,0 +1,45 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { courseContentTutorManifestSchema } from "../../../../server/services/course-content/course-content-schema"; +import { parseTopic } from "../../../../server/services/tutor/curriculum/content-repository"; +import { curriculumTopicSchema } from "../../../../server/services/tutor/curriculum/curriculum-schema"; + +describe("mastery progression schema compatibility", () => { + it("accepts Tutor manifest v2 phase strategies while keeping v1 strict", () => { + const v1 = { + schemaVersion: 1 as const, + defaultStrategy: "precision-policy", + topics: [], + strategies: [], + }; + expect(courseContentTutorManifestSchema.safeParse(v1).success).toBe(true); + expect(courseContentTutorManifestSchema.safeParse({ + ...v1, + phaseStrategies: { deepen: "exploration-policy", expand: "exploration-policy" }, + }).success).toBe(false); + expect(courseContentTutorManifestSchema.safeParse({ + ...v1, + schemaVersion: 2, + phaseStrategies: { deepen: "exploration-policy", expand: "exploration-policy" }, + }).success).toBe(true); + }); + + it("accepts Topic v2 deepening and extensions while keeping v1 strict", async () => { + const topic = parseTopic(await readFile(path.resolve(process.cwd(), "curriculum/topics/memory-and-data-types.yaml"), "utf8")); + const v2 = { + ...topic, + schemaVersion: 2 as const, + deepening: { + minimumSuccessfulProbes: 2, + successRatingAtLeast: 4, + requiredQuestionKinds: ["transfer" as const], + recentWeakAnswersAllowed: 0, + }, + extensions: [{ topic: "memory-and-data-types", objective: "Ein verwandtes Beispiel vergleichen." }], + }; + expect(curriculumTopicSchema.safeParse(topic).success).toBe(true); + expect(curriculumTopicSchema.safeParse({ ...topic, deepening: v2.deepening }).success).toBe(false); + expect(curriculumTopicSchema.safeParse(v2).success).toBe(true); + }); +}); diff --git a/tests/server/services/tutor/mastery-progression.test.ts b/tests/server/services/tutor/mastery-progression.test.ts new file mode 100644 index 000000000..43041d00f --- /dev/null +++ b/tests/server/services/tutor/mastery-progression.test.ts @@ -0,0 +1,362 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { parseTopic } from "../../../../server/services/tutor/curriculum/content-repository"; +import { + classifyTopic, + type Observation, +} from "../../../../server/services/tutor/curriculum/learning-planner"; +import { DefaultSketchFactExtractor } from "../../../../server/services/tutor/curriculum/sketch-facts"; +import { createTutorProgressionState, hasMetDeepeningCriteria, markTopicMastered } from "../../../../server/services/tutor/curriculum/progression-state"; +import { CurriculumTutorAdapter } from "../../../../server/services/tutor/curriculum-tutor-adapter"; + +async function topic() { + return parseTopic(await readFile(path.resolve(process.cwd(), "curriculum/topics/memory-and-data-types.yaml"), "utf8")); +} + +const revision = "a".repeat(40); + +function singleProbeTopic(source: Awaited>, id: string, activation = source.activation) { + const concept = { + ...source.concepts[0]!, + mastery: { + ...source.concepts[0]!.mastery, + minimumSuccessfulProbes: 1, + minimumDistinctQuestionKinds: 1, + requiredIndicators: [source.questions[0]!.indicator], + }, + }; + const questions = source.questions.slice(0, 2).map((question, index) => ({ + ...question, + id: `${id}-question-${index + 1}`, + concept: concept.id, + indicator: source.questions[0]!.indicator, + })); + return { + ...source, + id, + activation, + concepts: [concept], + questions, + scaffolds: [], + progression: { ...source.progression, entryConcepts: [concept.id], preferredOrder: [concept.id] }, + }; +} + +describe("mastery progression domain classification", () => { + it("distinguishes a probeable Topic from an unresolved exhausted Topic", async () => { + const value = await topic(); + const facts = new DefaultSketchFactExtractor().extract("int values[] = {1, 2};"); + const probeable = classifyTopic(value, facts, [], new Set(), 30); + expect(probeable.status).toBe("probeable"); + + const exhausted = value.questions.map((question) => ({ + questionId: question.id, + conceptId: question.concept, + indicatorId: question.indicator, + kind: question.kind, + rating: 3 as const, + })) satisfies Observation[]; + expect(classifyTopic(value, facts, exhausted, new Set(exhausted.map(({ questionId }) => questionId)), 30).status).toBe("unresolved"); + }); + + it("does not treat a Topic with an empty mastery domain as mastered or blocked", async () => { + const value = await topic(); + const facts = new DefaultSketchFactExtractor().extract("void setup() {} void loop() {}"); + expect(classifyTopic(value, facts, [], new Set(), 30)).toMatchObject({ status: "inapplicable" }); + }); + + it("uses the configured criteria for Topic mastery and keeps evidence separate from difficulty", async () => { + const value = await topic(); + const facts = new DefaultSketchFactExtractor().extract("int values[] = {1, 2};"); + const observations: Observation[] = value.questions.slice(0, 2).map((question) => ({ + questionId: question.id, + conceptId: question.concept, + indicatorId: question.indicator, + kind: question.kind, + rating: 4, + })); + const result = classifyTopic(value, facts, observations, new Set(), 30); + expect(result.status).toMatch(/probeable|unresolved|mastered/); + expect(revision).toHaveLength(40); + }); + + it("requires post-mastery transfer evidence before EXPAND", async () => { + const value = await topic(); + const state = createTutorProgressionState(revision); + markTopicMastered(state, value.id); + const transfer = value.questions.find(({ kind }) => kind === "transfer") ?? value.questions[0]!; + const observation: Observation = { + questionId: transfer.id, + conceptId: transfer.concept, + indicatorId: transfer.indicator, + kind: transfer.kind, + rating: 4, + }; + expect(hasMetDeepeningCriteria(value, [observation])).toBe(false); + expect(hasMetDeepeningCriteria(value, [observation, observation])).toBe(true); + expect(state.masteredTopicIds).toEqual([value.id]); + }); + + it("moves a mastered active Topic into DEEPEN without declaring it mastered from null", async () => { + const source = await topic(); + const concept = { + ...source.concepts[0]!, + mastery: { + ...source.concepts[0]!.mastery, + minimumSuccessfulProbes: 1, + minimumDistinctQuestionKinds: 1, + requiredIndicators: [source.questions[0]!.indicator], + }, + }; + const questions = source.questions.slice(0, 2).map((question, index) => ({ + ...question, + id: `mastery-question-${index + 1}`, + concept: concept.id, + indicator: index === 0 ? source.questions[0]!.indicator : source.questions[1]!.indicator, + })); + const masteryTopic = { + ...source, + id: "mastery-topic", + concepts: [concept], + questions, + scaffolds: [], + progression: { ...source.progression, entryConcepts: [concept.id], preferredOrder: [concept.id] }, + }; + const state = createTutorProgressionState(revision); + const adapter = new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, + topics: [masteryTopic], + strategies: [], + }, + }), + }, + }); + const first = await adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 }); + expect(first).toMatchObject({ learningPhase: "LEARN" }); + if (!first || "kind" in first) return; + const second = await adapter.planFollowup({ + code: "int values[] = {1, 2};", + history: [], + currentQuestion: first.question, + rating: 4, + difficulty: 30, + }); + expect(second).toMatchObject({ learningPhase: "LEARN", activeTopicId: "mastery-topic", masteredTopicIds: ["mastery-topic"] }); + await expect(adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 })) + .resolves.toMatchObject({ learningPhase: "DEEPEN", activeTopicId: "mastery-topic", masteredTopicIds: ["mastery-topic"] }); + }); + + it("latches the answered Topic before switching to the next applicable Topic", async () => { + const source = await topic(); + const primary = singleProbeTopic(source, "primary-topic"); + const secondary = singleProbeTopic(source, "secondary-topic", { + any: [{ fact: "array-declared", elementTypes: ["int"] }], + }); + const state = createTutorProgressionState(revision); + const adapter = new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, + topics: [primary, secondary], + strategies: [], + }, + exampleId: "example", + exampleTutorAnnotation: { + schemaVersion: 1 as const, + topics: [primary.id, secondary.id], + primaryTopic: primary.id, + }, + }), + }, + }); + + const first = await adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30, exampleId: "example" }); + expect(first).toMatchObject({ topicId: primary.id, learningPhase: "LEARN" }); + if (!first || "kind" in first) return; + + const switched = await adapter.planFollowup({ + code: "int values[] = {1, 2};", + history: [], + currentQuestion: first.question, + rating: 4, + difficulty: 30, + exampleId: "example", + }); + expect(switched).toMatchObject({ + topicId: secondary.id, + learningPhase: "LEARN", + activeTopicId: secondary.id, + masteredTopicIds: [primary.id], + }); + + const resumed = await adapter.planInitial({ + code: "int value = 1;", + history: [], + difficulty: 30, + exampleId: "example", + }); + expect(resumed).toMatchObject({ + topicId: primary.id, + learningPhase: "DEEPEN", + activeTopicId: primary.id, + masteredTopicIds: [primary.id], + }); + }); + + it("moves from DEEPEN to EXPAND only after the configured post-mastery evidence", async () => { + const source = await topic(); + const concept = { + ...source.concepts[0]!, + mastery: { ...source.concepts[0]!.mastery, minimumSuccessfulProbes: 1, minimumDistinctQuestionKinds: 1, requiredIndicators: [source.questions[0]!.indicator] }, + }; + const kinds = ["concept", "transfer", "transfer", "application"] as const; + const questions = source.questions.slice(0, 4).map((question, index) => ({ + ...question, + id: `phase-question-${index + 1}`, + concept: concept.id, + kind: kinds[index]!, + indicator: source.questions[0]!.indicator, + })); + const phaseTopic = { + ...source, + schemaVersion: 2 as const, + id: "phase-topic", + concepts: [concept], + questions, + scaffolds: [], + progression: { ...source.progression, entryConcepts: [concept.id], preferredOrder: [concept.id] }, + deepening: { minimumSuccessfulProbes: 2, successRatingAtLeast: 4, requiredQuestionKinds: ["transfer" as const], recentWeakAnswersAllowed: 0 }, + }; + const state = createTutorProgressionState(revision); + const adapter = new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { status: "valid" as const, manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, topics: [phaseTopic], strategies: [] }, + }), + }, + }); + const first = await adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 }); + expect(first).toMatchObject({ learningPhase: "LEARN" }); + if (!first || "kind" in first) return; + const completedLearnTurn = await adapter.planFollowup({ code: "int values[] = {1, 2};", history: [], currentQuestion: first.question, rating: 4, difficulty: 30 }); + expect(completedLearnTurn).toMatchObject({ learningPhase: "LEARN" }); + const deepen = await adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 }); + expect(deepen).toMatchObject({ learningPhase: "DEEPEN" }); + if (!deepen || "kind" in deepen) return; + const deepenAgain = await adapter.planFollowup({ code: "int values[] = {1, 2};", history: [], currentQuestion: deepen.question, rating: 4, difficulty: 30 }); + expect(deepenAgain).toMatchObject({ learningPhase: "DEEPEN" }); + if (!deepenAgain || "kind" in deepenAgain) return; + const completedDeepenTurn = await adapter.planFollowup({ code: "int values[] = {1, 2};", history: [], currentQuestion: deepenAgain.question, rating: 4, difficulty: 30 }); + expect(completedDeepenTurn).toMatchObject({ learningPhase: "DEEPEN", masteredTopicIds: ["phase-topic"] }); + await expect(adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 })) + .resolves.toMatchObject({ learningPhase: "EXPAND", masteredTopicIds: ["phase-topic"] }); + }); + + it("retains mastery across temporary fact-inapplicability and resumes the retained phase", async () => { + const source = await topic(); + const state = createTutorProgressionState(revision); + state.activeTopicId = source.id; + state.phase = "DEEPEN"; + state.retainedPhases[source.id] = "DEEPEN"; + markTopicMastered(state, source.id); + const adapter = new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { status: "valid" as const, manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, topics: [source], strategies: [] }, + }), + }, + }); + await expect(adapter.planInitial({ code: "void setup() {} void loop() {}", history: [], difficulty: 30 })).resolves.toBeNull(); + await expect(adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 })) + .resolves.toMatchObject({ learningPhase: "DEEPEN", activeTopicId: source.id }); + }); + + it("returns a controlled DEEPEN exhaustion result instead of free Tutor fallback", async () => { + const source = await topic(); + const state = createTutorProgressionState(revision); + state.activeTopicId = source.id; + state.phase = "DEEPEN"; + state.retainedPhases[source.id] = "DEEPEN"; + markTopicMastered(state, source.id); + const history: TutorDialogTurn[] = source.questions.map((question) => ({ + question: question.text ?? question.id, + questionId: question.id, + answer: "Antwort", + responseStyle: "normal", + answerRating: 4, + })); + const result = await new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { status: "valid" as const, manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, topics: [source], strategies: [] }, + }), + }, + }).planInitial({ code: "int values[] = {1, 2};", history, difficulty: 30 }); + expect(result).toMatchObject({ kind: "blocked", learningPhase: "DEEPEN", activeTopicId: source.id, progressionBlockedReason: "content-exhausted" }); + expect(state.phase).toBe("DEEPEN"); + }); + + it("returns a controlled EXPAND exhaustion result when no extension is available", async () => { + const source = await topic(); + const expandTopic = { ...source, schemaVersion: 2 as const, id: "expand-topic" }; + const state = createTutorProgressionState(revision); + state.activeTopicId = expandTopic.id; + state.phase = "EXPAND"; + state.retainedPhases[expandTopic.id] = "EXPAND"; + markTopicMastered(state, expandTopic.id); + const history: TutorDialogTurn[] = expandTopic.questions.map((question) => ({ + question: question.text ?? question.id, + questionId: question.id, + answer: "Antwort", + responseStyle: "normal", + answerRating: 4, + })); + const result = await new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { status: "valid" as const, manifest: { schemaVersion: 2 as const, topics: [], strategies: [] }, topics: [expandTopic], strategies: [] }, + }), + }, + }).planInitial({ code: "int values[] = {1, 2};", history, difficulty: 30 }); + expect(result).toMatchObject({ kind: "blocked", learningPhase: "EXPAND", activeTopicId: expandTopic.id, progressionBlockedReason: "content-exhausted" }); + expect(state.phase).toBe("EXPAND"); + }); + + it("does not reuse progression state for another Course revision", async () => { + const source = await topic(); + const state = createTutorProgressionState(revision); + state.activeTopicId = source.id; + state.phase = "DEEPEN"; + markTopicMastered(state, source.id); + const nextRevision = "b".repeat(40); + const result = await new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision: nextRevision, + progressionState: state, + tutor: { status: "valid" as const, manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, topics: [source], strategies: [] }, + }), + }, + }).planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 }); + expect(result).toMatchObject({ learningPhase: "LEARN", masteredTopicIds: [] }); + }); +}); diff --git a/tests/server/services/tutor/strategy-precedence.test.ts b/tests/server/services/tutor/strategy-precedence.test.ts index ac22dd204..8b212f831 100644 --- a/tests/server/services/tutor/strategy-precedence.test.ts +++ b/tests/server/services/tutor/strategy-precedence.test.ts @@ -1,10 +1,12 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { BUILT_IN_TUTOR_STRATEGY } from "../../../../server/services/tutor/strategy/effective-tutor-strategy"; +import { BUILT_IN_TUTOR_STRATEGY, type EffectiveTutorStrategy } from "../../../../server/services/tutor/strategy/effective-tutor-strategy"; import { parseTopic } from "../../../../server/services/tutor/curriculum/content-repository"; import { CurriculumTutorAdapter } from "../../../../server/services/tutor/curriculum-tutor-adapter"; import { TutorService } from "../../../../server/services/tutor/tutor-service"; +import type { LLMProvider } from "../../../../server/services/tutor/llm-provider"; +import { createTutorProgressionState, markTopicMastered } from "../../../../server/services/tutor/curriculum/progression-state"; const revision = "a".repeat(40); @@ -12,8 +14,8 @@ async function topic() { return parseTopic(await readFile(path.resolve(process.cwd(), "curriculum/topics/memory-and-data-types.yaml"), "utf8")); } -function strategy(id: string) { - return { ...BUILT_IN_TUTOR_STRATEGY, id }; +function strategy(id: string, overrides: Partial = {}) { + return { ...BUILT_IN_TUTOR_STRATEGY, id, ...overrides }; } describe("Tutor strategy precedence", () => { @@ -95,4 +97,203 @@ describe("Tutor strategy precedence", () => { 30, )).resolves.toMatchObject({ result: { strategyId: "built-in-default", strategySource: "built-in" } }); }); + + it("uses a manifest v2 phase strategy after Topic mastery", async () => { + const tutor = await topic(); + const state = createTutorProgressionState(revision); + state.activeTopicId = tutor.id; + state.phase = "DEEPEN"; + state.retainedPhases[tutor.id] = "DEEPEN"; + markTopicMastered(state, tutor.id); + const repositoryDefault = strategy("repository-default"); + const exploration = strategy("exploration-policy"); + const adapter = new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { + schemaVersion: 2 as const, + defaultStrategy: repositoryDefault.id, + phaseStrategies: { deepen: exploration.id, expand: exploration.id }, + topics: [], + strategies: [], + }, + topics: [tutor], + strategies: [repositoryDefault, exploration], + }, + }), + }, + }); + + await expect(adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 })) + .resolves.toMatchObject({ strategyId: exploration.id, learningPhase: "DEEPEN" }); + }); + + it("returns to the repository default when the retained Topic is no longer applicable", async () => { + const tutor = await topic(); + const state = createTutorProgressionState(revision); + state.activeTopicId = tutor.id; + state.phase = "DEEPEN"; + state.retainedPhases[tutor.id] = "DEEPEN"; + markTopicMastered(state, tutor.id); + const repositoryDefault = strategy("repository-default"); + const exploration = strategy("exploration-policy"); + const provider = { + listModels: async () => ["pilot-model"], + generateLearningQuestion: async () => ({ model: "pilot-model", result: { question: "Was passiert?" } }), + }; + const context = { + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { schemaVersion: 2 as const, defaultStrategy: repositoryDefault.id, phaseStrategies: { deepen: exploration.id }, topics: [], strategies: [repositoryDefault, exploration] }, + topics: [tutor], + strategies: [repositoryDefault, exploration], + }, + }; + await expect(new TutorService(provider, new CurriculumTutorAdapter()).generateQuestion( + "void setup(){} void loop(){}", "key", undefined, 30, context, + )).resolves.toMatchObject({ result: { strategyId: repositoryDefault.id, strategySource: "repository" } }); + }); + + it("keeps the completed LEARN turn on its strategy and uses the phase strategy on the next turn", async () => { + const source = await topic(); + const concept = { + ...source.concepts[0]!, + mastery: { ...source.concepts[0]!.mastery, minimumSuccessfulProbes: 1, minimumDistinctQuestionKinds: 1, requiredIndicators: [source.questions[0]!.indicator] }, + }; + const phaseTopic = { + ...source, + schemaVersion: 2 as const, + id: "boundary-topic", + concepts: [concept], + questions: source.questions.slice(0, 3).map((question, index) => ({ ...question, id: `boundary-question-${index + 1}`, concept: concept.id })), + scaffolds: [], + progression: { ...source.progression, entryConcepts: [concept.id], preferredOrder: [concept.id] }, + }; + const precision = strategy("precision-policy", { feedbackVerbosity: "short", progression: "mastery-then-advance" }); + const exploration = strategy("exploration-policy", { feedbackVerbosity: "detailed", progression: "advance-immediately", hintFirst: true }); + const state = createTutorProgressionState(revision); + const prompts: string[] = []; + const provider: LLMProvider = { + listModels: async () => ["pilot-model"], + async generateLearningQuestion(request) { + prompts.push(request.userPrompt); + return { + model: "pilot-model", + result: { + feedback: "Weiter.", + answerRating: 4, + question: "Providerfrage", + }, + }; + }, + }; + const context = { + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { + schemaVersion: 2 as const, + defaultStrategy: precision.id, + phaseStrategies: { deepen: exploration.id, expand: exploration.id }, + topics: [], + strategies: [], + }, + topics: [phaseTopic], + strategies: [precision, exploration], + }, + }; + const service = new TutorService(provider, new CurriculumTutorAdapter({ courseContent: { getSnapshot: async () => context } })); + const initial = await service.generateQuestion("int values[] = {1, 2};", "key", undefined, 30, context); + const firstAfterMastery = await service.generateDialogResponse( + "int values[] = {1, 2};", [], initial.result.question, "Antwort", "key", undefined, 30, context, + ); + expect(firstAfterMastery.result).toMatchObject({ learningPhase: "LEARN", strategyId: precision.id }); + + const firstDeepen = await service.generateDialogResponse( + "int values[] = {1, 2};", [], firstAfterMastery.result.question, "Antwort", "key", undefined, 30, context, + ); + expect(firstDeepen.result).toMatchObject({ learningPhase: "DEEPEN", strategyId: exploration.id }); + expect(prompts.at(-1)).toContain("Progression: advance-immediately"); + expect(prompts.at(-1)).toContain("Feedback: detailed"); + }); + + it("applies the EXPAND strategy only from the first request in EXPAND", async () => { + const source = await topic(); + const concept = { + ...source.concepts[0]!, + mastery: { ...source.concepts[0]!.mastery, minimumSuccessfulProbes: 1, minimumDistinctQuestionKinds: 1, requiredIndicators: [source.questions[0]!.indicator] }, + }; + const questions = source.questions.slice(0, 3).map((question, index) => ({ + ...question, + id: `expand-boundary-question-${index + 1}`, + concept: concept.id, + kind: index === 1 ? "transfer" as const : question.kind, + })); + const phaseTopic = { + ...source, + schemaVersion: 2 as const, + id: "expand-boundary-topic", + concepts: [concept], + questions, + scaffolds: [], + progression: { ...source.progression, entryConcepts: [concept.id], preferredOrder: [concept.id] }, + deepening: { minimumSuccessfulProbes: 2, successRatingAtLeast: 4, requiredQuestionKinds: ["transfer" as const], recentWeakAnswersAllowed: 0 }, + }; + const deepening = strategy("deepening-policy", { feedbackVerbosity: "short" }); + const expansion = strategy("expansion-policy", { feedbackVerbosity: "detailed", progression: "advance-immediately" }); + const state = createTutorProgressionState(revision); + state.activeTopicId = phaseTopic.id; + state.phase = "DEEPEN"; + state.retainedPhases[phaseTopic.id] = "DEEPEN"; + markTopicMastered(state, phaseTopic.id); + state.postMasteryEvidence[phaseTopic.id] = [{ + questionId: questions[1]!.id, + conceptId: concept.id, + indicatorId: questions[1]!.indicator, + kind: "transfer", + rating: 4, + }]; + const prompts: string[] = []; + const provider: LLMProvider = { + listModels: async () => ["pilot-model"], + async generateLearningQuestion(request) { + prompts.push(request.userPrompt); + return { model: "pilot-model", result: { feedback: "Weiter.", answerRating: 4, question: "Providerfrage" } }; + }, + }; + const context = { + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { + schemaVersion: 2 as const, + defaultStrategy: deepening.id, + phaseStrategies: { deepen: deepening.id, expand: expansion.id }, + topics: [], + strategies: [], + }, + topics: [phaseTopic], + strategies: [deepening, expansion], + }, + }; + const service = new TutorService(provider, new CurriculumTutorAdapter({ courseContent: { getSnapshot: async () => context } })); + const transitioned = await service.generateDialogResponse( + "int values[] = {1, 2};", [], questions[1]!.text!, "Antwort", "key", undefined, 30, context, + ); + expect(transitioned.result).toMatchObject({ learningPhase: "DEEPEN", strategyId: deepening.id }); + + const firstExpand = await service.generateDialogResponse( + "int values[] = {1, 2};", [], transitioned.result.question, "Antwort", "key", undefined, 30, context, + ); + expect(firstExpand.result).toMatchObject({ learningPhase: "EXPAND", strategyId: expansion.id }); + expect(prompts.at(-1)).toContain("Feedback: detailed"); + }); }); diff --git a/tests/server/services/tutor/topic-precedence.test.ts b/tests/server/services/tutor/topic-precedence.test.ts index d6ddad613..d92002de6 100644 --- a/tests/server/services/tutor/topic-precedence.test.ts +++ b/tests/server/services/tutor/topic-precedence.test.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { parseTopic } from "../../../../server/services/tutor/curriculum/content-repository"; import { CurriculumTutorAdapter } from "../../../../server/services/tutor/curriculum-tutor-adapter"; +import { createTutorProgressionState, markTopicMastered } from "../../../../server/services/tutor/curriculum/progression-state"; +import type { TutorDialogTurn } from "../../../../shared/tutor"; const revision = "a".repeat(40); @@ -58,4 +60,110 @@ describe("Tutor topic precedence", () => { const noMatch = await adapter([memory]).planInitial({ code: "void setup(){} void loop(){}", history: [], difficulty: 30 }); expect(noMatch).toBeNull(); }); + + it("skips a mastered Topic and keeps the next applicable Topic in LEARN", async () => { + const primary = await pilotTopic(); + const secondary = { ...primary, id: "arrays" }; + const state = createTutorProgressionState(revision); + state.activeTopicId = primary.id; + markTopicMastered(state, primary.id); + const result = await new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { schemaVersion: 1 as const, topics: [], strategies: [] }, + topics: [primary, secondary], + strategies: [], + }, + exampleTutorAnnotation: { schemaVersion: 1 as const, topics: [primary.id, secondary.id], primaryTopic: primary.id }, + }), + }, + }).planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 }); + expect(result).toMatchObject({ topicId: "arrays", learningPhase: "LEARN" }); + }); + + it("reports content exhaustion instead of treating an unresolved Topic as mastered", async () => { + const memory = await pilotTopic(); + const history: TutorDialogTurn[] = memory.questions.map((question) => ({ + question: question.text ?? question.template ?? question.id, + questionId: question.id, + answer: "Antwort", + responseStyle: "normal", + answerRating: 3, + })); + const result = await adapter([memory]).planInitial({ code: "int values[] = {1, 2};", history, difficulty: 30 }); + expect(result).toMatchObject({ kind: "blocked", progressionBlockedReason: "content-exhausted", learningPhase: "LEARN" }); + }); + + it("uses an extension only as guidance after normal fact matching activates its target", async () => { + const source = await pilotTopic(); + const primary = { ...source, schemaVersion: 2 as const, id: "memory", extensions: [{ topic: "arrays", objective: "Ein Array-Beispiel vergleichen." }] }; + const target = { ...source, schemaVersion: 2 as const, id: "arrays" }; + const state = createTutorProgressionState(revision); + state.activeTopicId = primary.id; + state.phase = "EXPAND"; + state.retainedPhases[primary.id] = "EXPAND"; + markTopicMastered(state, primary.id); + const result = await new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { schemaVersion: 2 as const, topics: [], strategies: [] }, + topics: [primary, target], + strategies: [], + }, + }), + }, + }).planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 }); + expect(result).toMatchObject({ topicId: "arrays", learningPhase: "LEARN", extensionTargetTopicId: "arrays" }); + }); + + it("guides EXPAND with an extension objective without activating its target", async () => { + const source = await pilotTopic(); + const primary = { + ...source, + schemaVersion: 2 as const, + id: "memory", + extensions: [{ topic: "arrays", objective: "Ein Array-Beispiel vergleichen." }], + }; + const target = { + ...source, + schemaVersion: 2 as const, + id: "arrays", + activation: { any: [{ fact: "serial-call" as const, values: ["write"] }] }, + }; + const state = createTutorProgressionState(revision); + state.activeTopicId = primary.id; + state.phase = "EXPAND"; + state.retainedPhases[primary.id] = "EXPAND"; + markTopicMastered(state, primary.id); + const adapter = new CurriculumTutorAdapter({ + courseContent: { + getSnapshot: async () => ({ + revision, + progressionState: state, + tutor: { + status: "valid" as const, + manifest: { schemaVersion: 2 as const, topics: [], strategies: [] }, + topics: [primary, target], + strategies: [], + }, + }), + }, + }); + + const expansion = await adapter.planInitial({ code: "int values[] = {1, 2};", history: [], difficulty: 30 }); + expect(expansion).toMatchObject({ topicId: primary.id, learningPhase: "EXPAND", activeTopicId: primary.id }); + expect(expansion).toMatchObject({ expansionBrief: { sourceTopicId: primary.id, targetTopicId: target.id, objective: "Ein Array-Beispiel vergleichen." } }); + expect(expansion).not.toMatchObject({ activeTopicId: target.id, extensionTargetTopicId: target.id }); + + const activated = await adapter.planInitial({ code: "int values[] = {1, 2}; void setup(){ Serial.write('A'); }", history: [], difficulty: 30 }); + expect(activated).toMatchObject({ topicId: target.id, learningPhase: "LEARN", activeTopicId: target.id, extensionTargetTopicId: target.id }); + }); });