From b1bf8b001cb1ef26801b64f82668bf780ac043b8 Mon Sep 17 00:00:00 2001 From: Kyle Galloway Date: Fri, 31 Jul 2026 09:23:08 -0400 Subject: [PATCH] feat(course): course.yaml manifest, format spec, and a validator that catches silent losses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 0 of PLAN.md: give a course an identity and a checkable contract before Phase 1 starts extracting the drill coach against it. The manifest (course.yaml) carries what a course IS — slug, the four prompt-skinning labels, module order, assessment-marker conventions — and deliberately not where models run, so one file travels between a laptop, a GPU box, and an external provider. The slug is declared once and binds the memory expert and the knowledge collection; naming a course separately in three places is how a live deployment ended up with a 'distillery' expert reading a 'distilling' collection. The validator exists because the build fails silently. enumerateLessons drops a directory with no lesson.md or one over 1 MiB without a word; the batch readers filter oversized files out of prompt context; lesson order is a bytewise sort. Each of those makes a smaller course, not a failed run. 'course validate' reports them, and reports them as errors where content vanishes. Grounded against a real 189-lesson corpus rather than guessed, which corrected three assumptions: - the SAQ preset matches '*SAQ*', not '*_Unit_SAQ' — the narrower pattern matched 18 of 189 lesson directories and would have left 83% of the question bank unextracted - questions live under ONE heading as an ordered list, not one heading per question, so the marker schema supports both shapes - lesson numbers can RESTART per unit inside one flat module directory, which interleaves units under a bytewise sort and which zero-padding cannot fix, so that case gets its own diagnosis Also warns on SVG figures: the vision stage needs rsvg-convert, and the study guide skips SVGs entirely, so vector-only figures are missing from the EPUB. Reported per module — on a real corpus 87% of figures are SVG-only and per-lesson warnings would bury everything else. docs/course-format.md specifies the tree as a target a machine produces and a human approves, since most material does not arrive as lessons. docs/excision-checklist.md gates Phase 1's extraction from a private repo. Co-Authored-By: Claude Fable 5 --- cmd/etude/cmd_course.go | 245 +++++++++++ cmd/etude/main.go | 34 +- docs/course-format.md | 181 ++++++++ docs/course-yaml.md | 149 +++++++ docs/excision-checklist.md | 135 ++++++ internal/course/manifest.go | 403 ++++++++++++++++++ internal/course/manifest_test.go | 259 ++++++++++++ internal/course/scaffold.go | 147 +++++++ internal/course/scaffold_test.go | 130 ++++++ internal/course/templates/course.yaml.tmpl | 77 ++++ internal/course/templates/lesson.md.tmpl | 44 ++ internal/course/tree.go | 453 +++++++++++++++++++++ internal/course/tree_test.go | 324 +++++++++++++++ 13 files changed, 2578 insertions(+), 3 deletions(-) create mode 100644 cmd/etude/cmd_course.go create mode 100644 docs/course-format.md create mode 100644 docs/course-yaml.md create mode 100644 docs/excision-checklist.md create mode 100644 internal/course/manifest.go create mode 100644 internal/course/manifest_test.go create mode 100644 internal/course/scaffold.go create mode 100644 internal/course/scaffold_test.go create mode 100644 internal/course/templates/course.yaml.tmpl create mode 100644 internal/course/templates/lesson.md.tmpl create mode 100644 internal/course/tree.go create mode 100644 internal/course/tree_test.go diff --git a/cmd/etude/cmd_course.go b/cmd/etude/cmd_course.go new file mode 100644 index 0000000..a9146b6 --- /dev/null +++ b/cmd/etude/cmd_course.go @@ -0,0 +1,245 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + + "github.com/gallowaysoftware/etude/internal/course" +) + +// initCmd scaffolds a new course directory. It is the first command a +// new user runs, so it optimises for "the file explains itself": the +// generated manifest and sample lesson carry the format documentation +// inline, and every value the author must decide is a REPLACE- +// placeholder that validation rejects until it is filled in. +func initCmd() *cobra.Command { + var opts course.ScaffoldOptions + var sample bool + + cmd := &cobra.Command{ + Use: "init [dir]", + Short: "Scaffold a new course directory (course.yaml + module tree).", + Long: `init creates a course skeleton: a course.yaml manifest, one directory +per module, and (unless --no-sample) an example lesson that documents the +lesson format inline. + +The scaffolded manifest carries REPLACE- placeholders for every value you +must decide. 'etude course validate' refuses to load a manifest that still +has one, so a half-filled course fails loudly instead of producing +generically-worded lectures. + +If your material is not already split into lessons — a book, a PDF export, +a pile of documents — scaffold the course first, then use 'etude ingest' to +propose a lesson breakdown from the source material.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + opts.Dir = "." + if len(args) == 1 { + opts.Dir = args[0] + } + opts.SampleLesson = sample + + created, err := course.Scaffold(opts) + if err != nil { + return err + } + for _, p := range created { + fmt.Fprintln(cmd.OutOrStdout(), p) + } + fmt.Fprintf(cmd.OutOrStdout(), + "\nNext: fill in the REPLACE- values in %s, then run 'etude course validate %s'.\n", + filepath.Join(opts.Dir, course.FileName), opts.Dir) + return nil + }, + } + + f := cmd.Flags() + f.StringVar(&opts.Slug, "slug", "", "Course slug (lowercase letters, digits, hyphens).") + f.StringVar(&opts.Title, "title", "", "Human-readable course title.") + f.IntVar(&opts.Modules, "modules", 1, "Number of module directories to stub out.") + f.StringVar(&opts.Preset, "preset", "none", "Assessment-marker preset: none, saq.") + f.BoolVar(&sample, "sample", true, "Write an example lesson documenting the format.") + f.BoolVar(&opts.Force, "force", false, "Overwrite an existing course.yaml.") + return cmd +} + +// courseCmd is the umbrella for manifest-level operations. +func courseCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "course", + Short: "Inspect and validate a course manifest and its lesson tree.", + } + cmd.AddCommand(courseValidateCmd()) + cmd.AddCommand(courseShowCmd()) + return cmd +} + +// courseValidateCmd checks a course before a multi-hour build spends GPU +// time on it. Its real job is catching SILENT losses: the pipeline drops +// a lesson directory with no lesson.md, drops an oversized lesson.md, +// filters oversized files out of batch prompt context, and orders +// lessons lexically — none of which announce themselves at run time. +func courseValidateCmd() *cobra.Command { + var strict bool + + cmd := &cobra.Command{ + Use: "validate [dir]", + Short: "Validate a course manifest and lesson tree.", + Long: `validate checks the manifest (identity, required labels, module list, +leftover placeholders) and then walks the lesson tree looking for content +that would vanish from a build. + +The tree checks matter because the pipeline fails silently: a lesson +directory without a lesson.md, or with one over 1 MiB, is dropped from the +course without a word, and a lesson.md over 200 KiB is filtered out of +corpus-wide prompt context. A clean validate means nothing disappears. + +Exits non-zero on errors. Warnings are advisory unless --strict.`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + + m, err := course.Load(dir) + if err != nil { + return err + } + + rep := m.ScanTree() + out := cmd.OutOrStdout() + for _, f := range rep.Findings { + fmt.Fprintln(out, f) + } + + var lessons, images, assessment int + for _, mod := range rep.Modules { + fmt.Fprintf(out, "module %d (%s): %d lesson(s), %d image(s)", + mod.Num, mod.Dir, mod.Lessons, mod.Images) + if mod.Assessment > 0 { + fmt.Fprintf(out, ", %d with assessment material", mod.Assessment) + } + fmt.Fprintln(out) + lessons += mod.Lessons + images += mod.Images + assessment += mod.Assessment + } + fmt.Fprintf(out, "%s: %d module(s), %d lesson(s), %d image(s)\n", + m.Slug, len(rep.Modules), lessons, images) + + if m.HasAssessmentMaterial() && assessment == 0 { + fmt.Fprintf(out, "%s\n", course.Finding{ + Severity: course.Warning, + Message: "assessment markers are configured but no lesson matched them — the drill will have no questions to extract", + }) + rep.Findings = append(rep.Findings, course.Finding{Severity: course.Warning}) + } + + switch { + case rep.Errors() > 0: + return fmt.Errorf("%d error(s), %d warning(s)", rep.Errors(), rep.Warnings()) + case strict && rep.Warnings() > 0: + return fmt.Errorf("%d warning(s) with --strict", rep.Warnings()) + } + return nil + }, + } + cmd.Flags().BoolVar(&strict, "strict", false, "Treat warnings as errors.") + return cmd +} + +// courseShowCmd prints what the manifest resolves to. The slug bindings +// are the point: one declared slug becomes the memory expert and the +// knowledge collection, and seeing them side by side is how an author +// catches a drifted name before three services disagree. +func courseShowCmd() *cobra.Command { + return &cobra.Command{ + Use: "show [dir]", + Short: "Print the resolved manifest, including service bindings.", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dir := "." + if len(args) == 1 { + dir = args[0] + } + m, err := course.Load(dir) + if err != nil { + return err + } + out := cmd.OutOrStdout() + fmt.Fprintf(out, "slug: %s\n", m.Slug) + fmt.Fprintf(out, "title: %s\n", m.Title) + fmt.Fprintf(out, "subject: %s\n", m.Subject) + fmt.Fprintf(out, "program: %s\n", m.Program) + fmt.Fprintf(out, "persona: %s\n", m.Persona) + fmt.Fprintf(out, "assessment: %s\n", m.Assessment) + fmt.Fprintf(out, "source: %s\n", m.SourceDir()) + fmt.Fprintf(out, "memory expert: %s\n", m.MemorySlug()) + fmt.Fprintf(out, "knowledge collection: %s\n", m.KnowledgeCollection()) + if m.HasAssessmentMaterial() { + a := m.Assessments.Resolved() + fmt.Fprintf(out, "assessment markers: preset=%s lessons=%q question=%q answer=%q\n", + a.Preset, a.LessonPattern, a.QuestionHeading, a.AnswerHeading) + } else { + fmt.Fprintf(out, "assessment markers: none (questions must be generated and reviewed)\n") + } + for _, mod := range m.Modules { + fmt.Fprintf(out, "module %d: %s — %s\n", mod.Num, m.ModuleDir(mod), mod.Topic) + } + return nil + }, + } +} + +// applyCourse overlays a manifest onto the pipeline Config so a build +// driven by course.yaml and one driven by flags produce identical +// inputs. Flags win: a manifest is the course's stable identity, but an +// operator overriding one value on the command line should not have to +// edit the file. ModuleNum is deliberately untouched — it is the +// selector that chose mod in the first place. +func applyCourse(m *course.Manifest, mod course.Module) error { + if cfg.LessonRoot == "" { + cfg.LessonRoot = m.ModuleDir(mod) + } + if cfg.ModuleTopic == "" { + cfg.ModuleTopic = mod.Topic + } + if cfg.SubjectLabel == "" { + cfg.SubjectLabel = m.Subject + } + if cfg.ProgramLabel == "" { + cfg.ProgramLabel = m.Program + } + if cfg.ExpertPersona == "" { + cfg.ExpertPersona = m.Persona + } + if cfg.AssessmentLabel == "" { + cfg.AssessmentLabel = m.Assessment + } + if cfg.Voice == "" { + cfg.Voice = m.Render.Voice + } + if cfg.CoverPrompt == "" { + cfg.CoverPrompt = m.Render.CoverPrompt + } + if cfg.SearchSuffix == "" { + cfg.SearchSuffix = m.Render.SearchSuffix + } + if cfg.EPUBTitleTemplate == "" { + cfg.EPUBTitleTemplate = m.Render.EPUBTitle + } + if cfg.EPUBAuthor == "" { + cfg.EPUBAuthor = m.Render.EPUBAuthor + } + if cfg.FilenamePrefix == "" { + cfg.FilenamePrefix = m.Render.FilenamePrefix + } + if _, err := os.Stat(cfg.LessonRoot); err != nil { + return fmt.Errorf("module %d: %w", mod.Num, err) + } + return nil +} diff --git a/cmd/etude/main.go b/cmd/etude/main.go index cf9d570..708eb36 100644 --- a/cmd/etude/main.go +++ b/cmd/etude/main.go @@ -20,6 +20,7 @@ import ( "github.com/gallowaysoftware/vibe/vamp" + "github.com/gallowaysoftware/etude/internal/course" "github.com/gallowaysoftware/etude/pipeline" ) @@ -30,6 +31,12 @@ import ( // requiring an explicit --input key=value invocation. var cfg pipeline.Config +// coursePath is the optional course.yaml (or its directory) that +// supplies cfg's domain labels and the module's lesson root, so a build +// names its material once in the manifest instead of on every +// invocation. +var coursePath string + func main() { root, err := vamp.BuildRoot(func() (*vamp.Pipeline, error) { return pipeline.Build(cfg) @@ -79,6 +86,12 @@ running daemon and the right capabilities mapped. Run the // --topic) the user must supply instead. bindRunRequiredCheck(root) + // `init` and `course` own the manifest: scaffolding a course and + // checking it before a multi-hour build spends GPU time on material + // the pipeline would silently drop. + root.AddCommand(initCmd()) + root.AddCommand(courseCmd()) + // `rag` is a strict downstream of the textbook pipeline — it // consumes processed_lessons.json and produces the RAG export. // Lives alongside vamp's auto-registered `run` so the user has @@ -132,8 +145,10 @@ func bindRunFlags(root *cobra.Command) error { return err } f := run.Flags() + f.StringVar(&coursePath, "course", "", + "Course directory or course.yaml; supplies --source and the domain labels for --module-num.") f.StringVar(&cfg.LessonRoot, "source", "", - "Directory containing Lesson_*/ subdirectories (required).") + "Directory containing Lesson_*/ subdirectories (required unless --course is given).") f.IntVar(&cfg.ModuleNum, "module-num", 1, "Numeric module identifier; used in cover seeds + output filenames.") f.StringVar(&cfg.ModuleTopic, "topic", "", @@ -167,12 +182,25 @@ func bindRunRequiredCheck(root *cobra.Command) { return } run.PreRunE = func(cmd *cobra.Command, args []string) error { + if coursePath != "" { + m, err := course.Load(coursePath) + if err != nil { + return err + } + mod, ok := m.FindModule(cfg.ModuleNum) + if !ok { + return fmt.Errorf("course %s has no module %d (use --module-num)", m.Slug, cfg.ModuleNum) + } + if err := applyCourse(m, mod); err != nil { + return err + } + } var missing []string if cfg.LessonRoot == "" { - missing = append(missing, "--source") + missing = append(missing, "--source (or --course)") } if cfg.ModuleTopic == "" { - missing = append(missing, "--topic") + missing = append(missing, "--topic (or --course)") } if len(missing) > 0 { return fmt.Errorf("required flag(s) %v not set", missing) diff --git a/docs/course-format.md b/docs/course-format.md new file mode 100644 index 0000000..b28e792 --- /dev/null +++ b/docs/course-format.md @@ -0,0 +1,181 @@ +# The course format + +A course is a directory of markdown that etude compiles into lectures, a +study guide, an EPUB, retrieval chunks, and a question bank. This +document specifies that directory. + +It is written for two audiences at once, and the second one matters +more. A person may hand-author a course, but most material does not +arrive as lessons — a book has chapters, an export has files, a wiki has +pages. Turning those into lessons is a judgement call that `etude +ingest` makes with a model and a human review gate. So treat this +format as a **target that a machine produces and a human approves**, not +as a filing convention. Every rule below is one an ingest pass must +satisfy. + +Check your course against this document with: + +```bash +etude course validate +``` + +## Shape + +``` +/ + course.yaml # the manifest — see docs/course-yaml.md + Module_1/ + Lesson_01_Barley/ + lesson.md # required, exact lowercase name + images/ # optional, flat + kernel.png + Lesson_02_Barley_SAQ/ + lesson.md + Module_2/ + ... +``` + +## Rules + +MUST rules describe things that break or silently disappear. SHOULD +rules describe degradation. Both are checked by `etude course validate`, +which reports MUST violations as errors and SHOULD violations as +warnings. + +### Modules + +- **MUST** Each module listed in `course.yaml` has a directory under + `source:`, named `Module_` unless the manifest overrides `dir:`. +- **SHOULD** Every module directory present on disk is listed in the + manifest. An unlisted module never builds; nothing warns at build + time. + +### Lessons + +- **MUST** A lesson is a directory whose name begins with `Lesson_`. + This glob is the *only* thing the build enumerates. A directory named + anything else is invisible, even if it contains a perfectly good + `lesson.md` — a real scraped corpus lost nine lessons this way, + including its entire maths primer. +- **MUST** Each lesson directory contains a regular file named exactly + `lesson.md`, lowercase. A lesson directory without one is dropped from + the course with no diagnostic. +- **MUST** `lesson.md` is non-empty and at most 1 MiB. Over that, the + lesson is dropped silently. +- **SHOULD** Keep `lesson.md` under 120 KB. The lecture stage and the + study-guide stage truncate lesson text at different lengths (150 KB + and 120 KB), so past the lower cap the two stages stop seeing the same + lesson — a divergence that produces a study guide covering material + the lecture never mentions. +- **SHOULD** The lesson directory name is filesystem-safe: letters, + digits, `_`, and `-`. The name is interpolated verbatim into output + paths for per-lesson artifacts. +- **SHOULD** Zero-pad lesson numbers (`Lesson_01`, not `Lesson_1`). + Build order is a bytewise sort, under which `Lesson_10` comes before + `Lesson_2`. +- **SHOULD** Lesson numbers are unique within a module. If several units + are flattened into one module directory and each restarts at lesson 1, + sorting interleaves them and no amount of zero-padding recovers the + intended order. Give each unit its own module, or number lessons + continuously across the module. + +### Lesson content + +- **SHOULD** `lesson.md` opens with exactly one H1 that is the lesson + title, and uses H2 for sections. The pipeline treats that first + heading as the lesson's title and the study guide reuses it verbatim; + without one, a title gets invented. +- **SHOULD** No YAML frontmatter. Nothing parses it, so it reaches the + model as literal noise. +- **SHOULD** Prefer prose that explains the mechanism. Downstream + prompts are forbidden from inventing facts, so anything the lecture + should say must be present here first. +- **SHOULD** Keep equations as text — fenced blocks survive verbatim. If + equations arrive as images, their alt text is what the model reads, so + the alt text must be a faithful linearization, not "equation-3.png". + +### Figures + +- **MUST** Figures live in a flat `images/` directory inside the lesson + directory. Subdirectories are not searched. +- **MUST** Figure files use one of `.png`, `.jpg`, `.jpeg`, `.gif`, + `.webp`, `.bmp`, `.svg`. Anything else is ignored by the vision stage. +- **MUST** If any figure is an SVG, the build host has `rsvg-convert` + installed — the vision stage rasterises SVGs and fails without it. +- **SHOULD** Ship a raster copy of every SVG figure. The study guide's + figure pass skips SVGs, so a vector-only figure reaches the audio + (via its vision description) but is missing from the markdown guide + and the EPUB. On an SVG-heavy corpus this silently drops most figures + from the written deliverables. +- **Note** Figures are discovered by walking `images/`, not by following + markdown links. An unreferenced file still gets described and + attached; a dangling link is invisible. + +An `images/` directory is optional and frequently absent — roughly one +lesson in eight has no figures at all. + +### Assessment material + +Material that carries its own questions *with model answers* is the +privileged input: the drill relays those questions verbatim and grades +against the official answer, inventing neither. Corpora without them get +generated questions, which pass through human review before entering the +bank. + +Assessment lessons are ordinary `Lesson_*` directories — there is no +separate file type. `course.yaml`'s `assessment_markers` block says how +to recognise them: + +```yaml +assessment_markers: + preset: saq +``` + +The `saq` preset matches lesson directories against `*SAQ*` and expects, +inside them: + +```markdown +# Lesson 3 - Barley SAQ + +## Self-Assessment Questions + +1. \* List three objectives of malting. +2. \*\* Label the components of the diagram below. + +## Suggested Response - Q1 + +Response: + +The three objectives are ... +``` + +- The question heading opens **one ordered list** whose items are the + questions. A heading containing `{n}` means the opposite convention — + one heading per question — and both are supported. +- Each answer gets its own heading, `{n}` being the question number. + Matching is case-insensitive and tolerates an en-dash. +- Difficulty is the leading escaped-asterisk run: `\*` short, `\*\*` + progressive, `\*\*\*` long. + +Override any field to describe a different convention: + +```yaml +assessment_markers: + preset: saq + lesson_pattern: "*_Exercises" + answer_heading: "## Model Answer {n}" +``` + +## Producing this format + +`etude ingest` (planned — see [PLAN.md](../PLAN.md)) turns unstructured +source material into this shape: deterministic extraction first, then a +model proposes lesson boundaries and titles, then you accept, edit, or +reject the proposal before anything is written. Splitting a book into +lessons is the highest-leverage judgement in the whole pipeline — a bad +breakdown degrades the lectures, the study guide, and the question bank +at once — so it is a reviewed proposal rather than an automatic +conversion. + +Until then, `etude init` scaffolds the tree and a sample lesson, and +`etude course validate` tells you what a build would silently lose. diff --git a/docs/course-yaml.md b/docs/course-yaml.md new file mode 100644 index 0000000..94c0254 --- /dev/null +++ b/docs/course-yaml.md @@ -0,0 +1,149 @@ +# course.yaml + +`course.yaml` is a course's manifest: its identity, the labels that skin +every prompt, where its material lives, and how that material marks its +own questions. It sits at the root of a course directory. + +What it deliberately does **not** hold is where models run. Endpoints, +API keys, and hardware profiles are machine-scoped and live in vibe/vamp +config or `ETUDE_*` environment variables, so the same manifest travels +between a laptop, a GPU box, and an external provider unchanged. + +Scaffold one with `etude init`, check it with `etude course validate`, +and see what it resolves to with `etude course show`. + +## Example + +```yaml +version: 1 + +slug: home-llms +title: Running LLMs at Home + +subject: running language models on your own hardware +program: the Running LLMs at Home course +persona: an engineer who runs models locally +assessment: end-of-unit self-check questions + +source: . +modules: + - num: 1 + topic: how models fit in memory + - num: 2 + topic: quantization and what it costs you + +render: + voice: af_bella + +assessment_markers: + preset: saq +``` + +## Fields + +### `version` (required) + +Schema version. Currently `1`. A manifest declaring a newer version is +rejected rather than half-interpreted. + +### `slug` (required) + +The course's identity everywhere it appears: the learner-memory expert, +the knowledge-base collection, deliverable filenames. Lowercase letters, +digits, and hyphens; starts with a letter or digit; 64 characters max. + +The pattern is deliberately identical to the memory service's expert +slug rule, so one declared name binds every service without a +translation table. Declaring the name once is the point: naming a course +separately in three places is how a deployment ends up with a memory +expert called `distillery` reading a collection called `distilling`. + +### `title` (required) + +The human-readable course name. + +### `subject`, `program`, `persona`, `assessment` (required) + +The four labels that carry the entire domain identity of a course. The +pipeline itself knows nothing about your subject; these are what make +its prompts sound like they are about your material. + +| Field | What it is | Example | +|---|---|---| +| `subject` | the field of study, as a lecturer would name it | `introductory astronomy` | +| `program` | the course or certification the material belongs to | `an introductory astronomy course` | +| `persona` | the first-person identity the lecturer adopts | `an astronomy lecturer` | +| `assessment` | how learners are evaluated | `end-of-unit self-check questions` | + +Leave one generic and the lectures sound generic. That is why `etude +init` writes them as `REPLACE-` placeholders and validation refuses to +load a manifest that still carries one — a half-filled manifest +otherwise produces plausible-sounding lectures about nothing in +particular. + +### `source` + +Directory holding the module subdirectories, relative to the manifest. +Defaults to `.`. + +### `modules` (required) + +The modules in teaching order, each with: + +- `num` — the module's identifier, used in cover seeds and filenames. + Positive, unique. +- `topic` — a short prose summary, reaching cover prompts and EPUB + titles. +- `dir` — the directory name, defaulting to `Module_`. + +Order is explicit rather than derived from a directory listing because +`Module_10` sorts before `Module_2`, and because a course is a sequence, +not a set. + +### `render` + +Presentation, all optional: + +- `voice` — the narration voice (default `af_bella`). +- `cover_prompt` — the image prompt for cover art. +- `search_suffix` — appended to web-search queries during enrichment. +- `epub_title`, `epub_author` — EPUB metadata. +- `filename_prefix` — deliverable basename stem; defaults from the + module topic. + +### `assessment_markers` + +How the corpus marks its own questions and model answers. See +[course-format.md](course-format.md#assessment-material) for the +conventions and the `saq` preset. `preset: none` (the default) means the +corpus carries no extractable assessment material and questions will be +generated and reviewed instead. + +### `expert` + +Bindings to the services that make a course conversational. Both default +to `slug` and exist only as overrides for deployments that already use +different names: + +- `memory_slug` — the memory expert holding this course's learner state. +- `knowledge_collection` — the knowledge-service collection holding the + corpus. + +`etude course show` prints the resolved bindings, which is the cheap way +to catch a drifted name before three services disagree about it. + +## Using a manifest + +```bash +etude run --course . --module-num 1 +``` + +supplies the lesson root and every domain label from the manifest. +Explicit flags still win, so overriding one value for a single run does +not require editing the file. + +## Unknown fields are errors + +A misspelled key is rejected rather than ignored. A `voise:` that +quietly reverts narration to the default, discovered five hours into a +build, is exactly the failure a manifest should not have. diff --git a/docs/excision-checklist.md b/docs/excision-checklist.md new file mode 100644 index 0000000..274fba5 --- /dev/null +++ b/docs/excision-checklist.md @@ -0,0 +1,135 @@ +# Excision checklist + +etude grows by moving working code out of private repos into this public +one. That is how the drill coach arrives (Phase 1) and how the knowledge +service arrives (Phase 5). The code is worth keeping; the private +material tangled through it is not. + +This checklist gates every such move. Work through it in order and +record the verdicts in the pull request — the point is that the answer +to "did anyone check?" is a diff, not a memory. + +The rule underneath all of it: **the excision is a fresh commit of +scrubbed files, never a replay of private history.** A scrub applied +after the content lands is not a scrub; the original stays in the log +forever. + +## Gates + +**Gate 0 — Declare the excision.** In the PR description: source repo, +source commit SHA (not a branch), source prefix, destination path, and +one line on what the excision is for. Everything below is scoped to that +prefix. Anything outside it is not part of this move. + +**Gate 1 — Extract tracked files only.** + +```bash +mkdir -p /tmp/cleanroom +git -C archive / | tar -x -C /tmp/cleanroom +``` + +Never `cp -r`, `rsync`, or a working-tree copy: credential-shaped risk +lives in ignored paths (`.env`, `.claude/`, local state directories) that +a working tree still contains and `git archive` cannot emit. + +**Gate 2 — Confirm the destination history is clean.** The cleanroom has +no `.git` before scrubbing, and the first commit into etude is an +ordinary commit of scrubbed files — not a merge, graft, subtree, or +history replay. + +## Steps + +**1. Classify every path** as KEEP, REWRITE, or DELETE before editing +anything, and record the verdict per path in the PR body. A path with no +defensible KEEP is a DELETE. + +**2. Delete corpus and corpus-derived data.** Non-redistributable +content cannot be fixed by renaming. This covers verbatim extracts and +model-paraphrased derivatives alike: a cloze deck built from a +curriculum's answer keys is still that curriculum. Prove it by size — +every surviving file over 64 KB needs a reason. + +```bash +find /tmp/cleanroom -type f -size +64k -exec ls -lh {} \; | sort -k5 -h -r +du -sh /tmp/cleanroom +``` + +**3. Purge absolute home paths.** Replace with a PATH-resolved binary +name, an XDG-relative path, or a config key. Highest severity inside +prompts and skill files, where a model will execute the path verbatim on +a machine that does not have it. + +```bash +grep -rInE '(/home/[A-Za-z0-9_.-]+|/Users/[A-Za-z0-9_.-]+)(/|\b)' /tmp/cleanroom +``` + +**4. Purge private network addresses** from config, defaults, docs, and +**tests** — a fixture string is as public as a README. + +```bash +grep -rInE '(^|[^0-9.])(10\.([0-9]{1,3}\.){2}[0-9]{1,3}|192\.168\.[0-9]{1,3}\.[0-9]{1,3}|172\.(1[6-9]|2[0-9]|3[01])\.[0-9]{1,3}\.[0-9]{1,3})' /tmp/cleanroom +``` + +**5. Replace host-specific values with loud placeholders**, following +the `REPLACE-with-a-free-LAN-IP` convention already used across the +family. A `REPLACE-` value fails at startup; a plausible-looking default +silently ships someone else's topology. + +**6. Replace fleet hostnames with role names** — machine codenames +become `gpu-cell`, `utility-cell`, `front`. Keep genuinely informative +vendor-generic terms. + +**7. Convert embedded inventory to loaded config.** Keep the +`CREATE TABLE`, drop the `INSERT`. Watch for personal-life facts encoded +as schema: a `gaming_host` column, a description reading "also the +gaming PC". + +**8. Rename private-system environment variables.** Any name whose +leading token is a private business, host, or repo. The identifier is +the leak even when the value is empty — and it becomes a compatibility +promise the moment it ships. + +```bash +grep -rhoE '\b[A-Z][A-Z0-9]{2,}(_[A-Z0-9]+)+\b' /tmp/cleanroom --exclude=go.sum | sort -u +``` + +**9. De-personalize prompts, skill files, and runtime strings.** The +named learner becomes a template variable or "the learner". Search Go +and Python string literals, not just markdown — model-facing strings +returned at runtime are the usual hiding place. + +**10. Delete personal-business and personal-hobby content wholesale, +then port the mechanism.** A prompt naming a real business, its +location, and its regulatory context is not fixable by find-and-replace. +Delete it and re-derive the reusable design. + +**11. Purge the source curriculum's name from the public API surface** — +identifiers, config keys, and model-facing description strings. This is +the no-second-chance check: a curriculum name baked into a tool +description ships to every client that connects. + +**12. Run the denylist sweep**, and keep the list in the repo as a CI +tripwire so the next excision cannot regress: + +```bash +grep -rInEi 'Kyle|Galloway|pequalsnp|thegalloways' /tmp/cleanroom --exclude=go.sum +grep -rInEi '\bPEI\b|Prince Edward|craft distiller|gaming (PC|host)|the house' /tmp/cleanroom +``` + +**13. Scan for live credentials last.** A hit here means Gate 1 was +bypassed — start over rather than deleting the finding. + +```bash +grep -rInE '(sk-[A-Za-z0-9_-]{16,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|xox[baprs]-[A-Za-z0-9-]{10,})' /tmp/cleanroom +``` + +**14. Build, test, and lint the result in isolation** before it touches +etude. Public CI must never need private content: tests that exercised +the original corpus get synthetic fixtures. + +## After landing + +Point the private repo at the public package as a dependency rather than +keeping a fork. That is what keeps the excision honest — the private +consumer immediately proves the public code is complete, and there is +only one copy to maintain. diff --git a/internal/course/manifest.go b/internal/course/manifest.go new file mode 100644 index 0000000..7562f31 --- /dev/null +++ b/internal/course/manifest.go @@ -0,0 +1,403 @@ +// Package course implements the course manifest (course.yaml) and the +// lesson-tree contract it describes: loading, defaulting, validation, and +// scaffolding. +// +// The manifest answers "what is this course" — identity, the labels that +// skin every prompt, where the material lives, how the corpus marks its +// own assessment material. It deliberately does NOT answer "where do +// models run": endpoints, profiles, and API keys are machine-scoped and +// belong to vibe/vamp config or ETUDE_* environment variables, because +// the same course.yaml is meant to travel between a laptop, a GPU box, +// and an external provider unchanged. +package course + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +// FileName is the manifest's fixed basename. A course is a directory +// containing one of these. +const FileName = "course.yaml" + +// Version is the manifest schema version this build understands. A +// manifest carrying a higher version is rejected rather than +// half-interpreted. +const Version = 1 + +// slugPattern is deliberately identical to the refrain memory service's +// expert-slug rule. The course slug becomes a refrain expert directory, +// a knowledge-base collection name, and a filename stem, so it must +// satisfy the strictest consumer — and matching refrain's regex exactly +// is what lets one declared slug bind all three without a translation +// table. +var slugPattern = regexp.MustCompile(`^[a-z0-9][a-z0-9-]{0,63}$`) + +// placeholderPrefix marks a value the author must replace before the +// manifest is usable. Borrowed from vibe's profile templates: scaffolded +// files fail loudly at load time instead of producing a confusing +// downstream error (a cover prompt reading "REPLACE-with-..." would +// otherwise sail through and land in the rendered artwork). +const placeholderPrefix = "REPLACE-" + +// Manifest is the parsed course.yaml. Field order here mirrors the +// scaffolded file so a reader can follow one against the other. +type Manifest struct { + Version int `yaml:"version"` + Slug string `yaml:"slug"` + Title string `yaml:"title"` + + // Subject, Program, Persona, and Assessment are the four labels that + // skin every prompt in the pipeline. They carry the entire + // domain identity of a course — the DAG itself is subject-agnostic. + Subject string `yaml:"subject"` + Program string `yaml:"program"` + Persona string `yaml:"persona"` + Assessment string `yaml:"assessment"` + + // Source is the directory holding the module subdirectories, + // resolved relative to the manifest's own directory so a course + // moves as one tree. + Source string `yaml:"source"` + + // Modules lists the modules in teaching order. Order is explicit + // rather than derived from directory sort because "Module_10" sorts + // before "Module_2" and a course is not a set. + Modules []Module `yaml:"modules"` + + Render Render `yaml:"render"` + Assessments Assessments `yaml:"assessment_markers"` + Expert Expert `yaml:"expert"` + + // dir is the manifest's own directory, captured at load so relative + // paths resolve against the course rather than the process's cwd. + dir string `yaml:"-"` +} + +// Module is one unit of production: a directory of lessons that becomes +// one audiobook, one study guide, and one EPUB. +type Module struct { + // Num is the module's identifier, used in cover-art seeds and + // output filenames. + Num int `yaml:"num"` + // Topic is a short prose summary ("fermentation and wort + // production"). It reaches cover prompts and EPUB titles. + Topic string `yaml:"topic"` + // Dir names the module's directory under Source. Empty defaults to + // "Module_". + Dir string `yaml:"dir"` +} + +// Render holds the presentation knobs: how the course sounds, looks, and +// is named on disk. Every field is optional. +type Render struct { + Voice string `yaml:"voice"` + CoverPrompt string `yaml:"cover_prompt"` + SearchSuffix string `yaml:"search_suffix"` + EPUBTitle string `yaml:"epub_title"` + EPUBAuthor string `yaml:"epub_author"` + // FilenamePrefix overrides the deliverable basename stem. Empty + // derives it from each module's topic. + FilenamePrefix string `yaml:"filename_prefix"` +} + +// Assessments describes how the corpus marks its own questions and model +// answers, so the drill's bank extractor can find them without +// per-curriculum code. A corpus that carries no assessment material +// leaves Preset empty and drills generated questions instead. +type Assessments struct { + // Preset names a built-in marker set. "none" (or empty) means the + // corpus carries no extractable assessment material. + Preset string `yaml:"preset"` + // LessonPattern matches the lesson directories that hold assessment + // material (e.g. "*SAQ*"). Empty scans every lesson. + LessonPattern string `yaml:"lesson_pattern"` + // QuestionHeading opens the questions. Two shapes are supported and + // the "{n}" placeholder is what distinguishes them: a heading + // containing {n} marks one heading per question, while a heading + // without it opens a single markdown ordered list whose items are + // the questions. Real curricula use both. + QuestionHeading string `yaml:"question_heading"` + // AnswerHeading marks each model answer; "{n}" is the question + // number. Matching is case-insensitive and tolerates an en-dash for + // the hyphen, because real corpora are inconsistent about both. + AnswerHeading string `yaml:"answer_heading"` + // DifficultyMarker names the convention that encodes difficulty + // (e.g. "asterisks" for the * / ** / *** convention). Empty means + // difficulty is not encoded. + DifficultyMarker string `yaml:"difficulty_marker"` +} + +// Expert binds this course to the services that make it conversational. +// Both slugs default to the course slug; they exist as overrides only +// because pre-existing deployments may already have named them +// differently. +type Expert struct { + // MemorySlug is the refrain expert whose memory holds this course's + // learner state. + MemorySlug string `yaml:"memory_slug"` + // KnowledgeCollection is the knowledge-service collection holding + // this course's corpus. + KnowledgeCollection string `yaml:"knowledge_collection"` +} + +// Load reads and validates the manifest at path. A directory path is +// accepted and resolved to the course.yaml inside it. +func Load(path string) (*Manifest, error) { + resolved, err := resolveManifestPath(path) + if err != nil { + return nil, err + } + data, err := os.ReadFile(resolved) + if err != nil { + return nil, err + } + var m Manifest + // KnownFields turns a typo into an error instead of a silently + // ignored setting — a misspelled "voise:" that quietly reverts the + // narrator to the default is exactly the kind of failure a manifest + // should not have. + dec := yaml.NewDecoder(strings.NewReader(string(data))) + dec.KnownFields(true) + if err := dec.Decode(&m); err != nil { + return nil, fmt.Errorf("%s: %w", resolved, err) + } + m.dir = filepath.Dir(resolved) + if err := m.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", resolved, err) + } + return &m, nil +} + +// resolveManifestPath accepts either a course directory or a manifest +// file and returns the manifest path. +func resolveManifestPath(path string) (string, error) { + if path == "" { + path = "." + } + info, err := os.Stat(path) + if err != nil { + return "", err + } + if info.IsDir() { + return filepath.Join(path, FileName), nil + } + return path, nil +} + +// Validate checks the manifest in isolation — structure, identity, and +// placeholders — without touching the lesson tree. ScanTree covers the +// content on disk. +func (m *Manifest) Validate() error { + var errs []error + + switch { + case m.Version == 0: + errs = append(errs, errors.New("version: required (current schema is 1)")) + case m.Version > Version: + errs = append(errs, fmt.Errorf("version %d is newer than this build understands (max %d)", m.Version, Version)) + } + + if m.Slug == "" { + errs = append(errs, errors.New("slug: required")) + } else if !slugPattern.MatchString(m.Slug) { + errs = append(errs, fmt.Errorf("slug %q: must be lowercase letters, digits, and hyphens, starting with a letter or digit, at most 64 characters", m.Slug)) + } + + for field, value := range map[string]string{ + "title": m.Title, + "subject": m.Subject, + "program": m.Program, + "persona": m.Persona, + "assessment": m.Assessment, + } { + if strings.TrimSpace(value) == "" { + errs = append(errs, fmt.Errorf("%s: required", field)) + } + } + + if len(m.Modules) == 0 { + errs = append(errs, errors.New("modules: at least one module is required")) + } + seen := make(map[int]bool, len(m.Modules)) + for i, mod := range m.Modules { + if mod.Num <= 0 { + errs = append(errs, fmt.Errorf("modules[%d].num: must be a positive integer", i)) + } else if seen[mod.Num] { + errs = append(errs, fmt.Errorf("modules[%d].num: %d is used more than once", i, mod.Num)) + } + seen[mod.Num] = true + if strings.TrimSpace(mod.Topic) == "" { + errs = append(errs, fmt.Errorf("modules[%d].topic: required", i)) + } + } + + if err := m.Assessments.validate(); err != nil { + errs = append(errs, err) + } + + errs = append(errs, m.placeholderErrors()...) + + return errors.Join(errs...) +} + +// placeholderErrors reports every field still carrying a scaffolded +// REPLACE- value. Reported together so an author fixes the whole file in +// one pass rather than one error per run. +func (m *Manifest) placeholderErrors() []error { + var errs []error + check := func(field, value string) { + if strings.HasPrefix(value, placeholderPrefix) { + errs = append(errs, fmt.Errorf("%s: still holds the scaffolded placeholder %q", field, value)) + } + } + check("slug", m.Slug) + check("title", m.Title) + check("subject", m.Subject) + check("program", m.Program) + check("persona", m.Persona) + check("assessment", m.Assessment) + check("render.cover_prompt", m.Render.CoverPrompt) + for i, mod := range m.Modules { + check(fmt.Sprintf("modules[%d].topic", i), mod.Topic) + } + return errs +} + +// Presets are the built-in assessment-marker sets. A preset is a +// starting point, not a constraint: any field an author sets explicitly +// wins over the preset's value. +var Presets = map[string]Assessments{ + // saq matches curricula that ship self-assessment questions with + // model answers inside the lesson itself: one heading opening a + // numbered list of questions, then one "suggested response" heading + // per question carrying the official answer. + // + // Every value here is measured against a real 189-lesson corpus + // rather than guessed. The lesson pattern in particular is "*SAQ*" + // and not "*_Unit_SAQ": the narrower pattern matched 18 of 189 + // lesson directories and would have left 83% of the question bank + // on the floor, because per-lesson SAQs outnumber end-of-unit ones. + "saq": { + LessonPattern: "*SAQ*", + QuestionHeading: "## Self-Assessment Questions", + AnswerHeading: "## Suggested Response - Q{n}", + DifficultyMarker: "escaped-asterisks", + }, +} + +// PerQuestionHeadings reports whether QuestionHeading marks each +// question individually rather than opening a single list. +func (a Assessments) PerQuestionHeadings() bool { + return strings.Contains(a.QuestionHeading, "{n}") +} + +func (a Assessments) validate() error { + preset := strings.TrimSpace(a.Preset) + if preset == "" || preset == "none" { + return nil + } + if _, ok := Presets[preset]; !ok { + names := make([]string, 0, len(Presets)+1) + names = append(names, "none") + for k := range Presets { + names = append(names, k) + } + return fmt.Errorf("assessment_markers.preset %q: unknown (known presets: %s)", preset, strings.Join(names, ", ")) + } + return nil +} + +// Resolved returns the assessment markers with preset defaults filled in +// under any explicit overrides. +func (a Assessments) Resolved() Assessments { + base, ok := Presets[strings.TrimSpace(a.Preset)] + if !ok { + return a + } + if a.LessonPattern == "" { + a.LessonPattern = base.LessonPattern + } + if a.QuestionHeading == "" { + a.QuestionHeading = base.QuestionHeading + } + if a.AnswerHeading == "" { + a.AnswerHeading = base.AnswerHeading + } + if a.DifficultyMarker == "" { + a.DifficultyMarker = base.DifficultyMarker + } + return a +} + +// HasAssessmentMaterial reports whether the course claims extractable +// questions. The drill uses this to choose between extracting the +// corpus's own bank and generating one. +func (m *Manifest) HasAssessmentMaterial() bool { + preset := strings.TrimSpace(m.Assessments.Preset) + if preset != "" && preset != "none" { + return true + } + return m.Assessments.QuestionHeading != "" || m.Assessments.AnswerHeading != "" +} + +// Dir returns the directory containing the manifest. +func (m *Manifest) Dir() string { return m.dir } + +// SourceDir returns the absolute directory holding the module +// subdirectories. +func (m *Manifest) SourceDir() string { + src := m.Source + if src == "" { + src = "." + } + if filepath.IsAbs(src) { + return filepath.Clean(src) + } + return filepath.Join(m.dir, src) +} + +// ModuleDir returns the absolute directory for one module. +func (m *Manifest) ModuleDir(mod Module) string { + dir := mod.Dir + if dir == "" { + dir = fmt.Sprintf("Module_%d", mod.Num) + } + if filepath.IsAbs(dir) { + return filepath.Clean(dir) + } + return filepath.Join(m.SourceDir(), dir) +} + +// FindModule returns the module with the given number. +func (m *Manifest) FindModule(num int) (Module, bool) { + for _, mod := range m.Modules { + if mod.Num == num { + return mod, true + } + } + return Module{}, false +} + +// MemorySlug returns the refrain expert slug for this course. +func (m *Manifest) MemorySlug() string { + if m.Expert.MemorySlug != "" { + return m.Expert.MemorySlug + } + return m.Slug +} + +// KnowledgeCollection returns the knowledge-service collection name for +// this course. +func (m *Manifest) KnowledgeCollection() string { + if m.Expert.KnowledgeCollection != "" { + return m.Expert.KnowledgeCollection + } + return m.Slug +} diff --git a/internal/course/manifest_test.go b/internal/course/manifest_test.go new file mode 100644 index 0000000..bef0be6 --- /dev/null +++ b/internal/course/manifest_test.go @@ -0,0 +1,259 @@ +package course + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// validManifest is the smallest manifest that loads. Tests mutate a copy +// of it so each case states only what it is exercising. +const validManifest = ` +version: 1 +slug: home-llms +title: Running LLMs at Home +subject: running language models on your own hardware +program: the Running LLMs at Home course +persona: an engineer who runs models locally +assessment: end-of-unit self-check questions +source: . +modules: + - num: 1 + topic: how models fit in memory +` + +func writeCourse(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, FileName), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func TestLoad(t *testing.T) { + tests := []struct { + name string + body string + wantErr string + }{ + {name: "valid", body: validManifest}, + { + name: "missing version", + body: strings.Replace(validManifest, "version: 1\n", "", 1), + wantErr: "version: required", + }, + { + name: "future version", + body: strings.Replace(validManifest, "version: 1", "version: 99", 1), + wantErr: "newer than this build understands", + }, + { + name: "slug with uppercase", + body: strings.Replace(validManifest, "slug: home-llms", "slug: Home-LLMs", 1), + wantErr: "must be lowercase", + }, + { + name: "slug with underscore", + body: strings.Replace(validManifest, "slug: home-llms", "slug: home_llms", 1), + wantErr: "must be lowercase", + }, + { + name: "missing persona", + body: strings.Replace(validManifest, "persona: an engineer who runs models locally\n", "", 1), + wantErr: "persona: required", + }, + { + name: "no modules", + body: strings.Split(validManifest, "modules:")[0], + wantErr: "at least one module", + }, + { + name: "duplicate module numbers", + body: validManifest + ` - num: 1 + topic: a second module claiming the same number +`, + wantErr: "used more than once", + }, + { + name: "module without topic", + body: strings.Replace(validManifest, " topic: how models fit in memory\n", "", 1), + wantErr: "topic: required", + }, + { + // A typo that silently reverts a setting to its default is + // the failure mode KnownFields exists to prevent. + name: "unknown field", + body: validManifest + "voise: af_bella\n", + wantErr: "field voise not found", + }, + { + name: "unknown assessment preset", + body: validManifest + "assessment_markers:\n preset: nonesuch\n", + wantErr: "unknown (known presets", + }, + { + name: "surviving placeholder", + body: strings.Replace(validManifest, "title: Running LLMs at Home", "title: REPLACE-with the course title", 1), + wantErr: "still holds the scaffolded placeholder", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := Load(writeCourse(t, tc.body)) + switch { + case tc.wantErr == "" && err != nil: + t.Fatalf("Load() = %v, want success", err) + case tc.wantErr != "" && err == nil: + t.Fatalf("Load() = nil, want error containing %q", tc.wantErr) + case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): + t.Fatalf("Load() = %v, want error containing %q", err, tc.wantErr) + } + }) + } +} + +// TestLoadReportsEveryProblem verifies errors accumulate. An author +// fixing a scaffolded manifest should see the whole list once, not +// discover the next problem on every re-run. +func TestLoadReportsEveryProblem(t *testing.T) { + body := ` +version: 1 +slug: BAD SLUG +title: "" +subject: "" +program: "" +persona: "" +assessment: "" +modules: [] +` + _, err := Load(writeCourse(t, body)) + if err == nil { + t.Fatal("Load() = nil, want errors") + } + for _, want := range []string{"slug", "title: required", "subject: required", "persona: required", "at least one module"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %v", want, err) + } + } +} + +func TestLoadAcceptsDirectoryOrFile(t *testing.T) { + dir := writeCourse(t, validManifest) + for _, path := range []string{dir, filepath.Join(dir, FileName)} { + if _, err := Load(path); err != nil { + t.Errorf("Load(%q) = %v, want success", path, err) + } + } +} + +// TestSlugBindings pins the identity rule the whole product depends on: +// one declared slug names the memory expert and the knowledge +// collection, and an explicit override wins. The live distillery/ +// distilling mismatch is what this prevents. +func TestSlugBindings(t *testing.T) { + m, err := Load(writeCourse(t, validManifest)) + if err != nil { + t.Fatal(err) + } + if got := m.MemorySlug(); got != "home-llms" { + t.Errorf("MemorySlug() = %q, want the course slug", got) + } + if got := m.KnowledgeCollection(); got != "home-llms" { + t.Errorf("KnowledgeCollection() = %q, want the course slug", got) + } + + override := validManifest + `expert: + memory_slug: legacy-expert + knowledge_collection: legacy-collection +` + m, err = Load(writeCourse(t, override)) + if err != nil { + t.Fatal(err) + } + if got := m.MemorySlug(); got != "legacy-expert" { + t.Errorf("MemorySlug() = %q, want the override", got) + } + if got := m.KnowledgeCollection(); got != "legacy-collection" { + t.Errorf("KnowledgeCollection() = %q, want the override", got) + } +} + +func TestAssessmentPresetResolution(t *testing.T) { + body := validManifest + `assessment_markers: + preset: saq + answer_heading: "## Model Answer {n}" +` + m, err := Load(writeCourse(t, body)) + if err != nil { + t.Fatal(err) + } + got := m.Assessments.Resolved() + if got.AnswerHeading != "## Model Answer {n}" { + t.Errorf("explicit answer_heading = %q, want the override to win over the preset", got.AnswerHeading) + } + if got.QuestionHeading != Presets["saq"].QuestionHeading { + t.Errorf("question_heading = %q, want the preset value", got.QuestionHeading) + } + if !m.HasAssessmentMaterial() { + t.Error("HasAssessmentMaterial() = false, want true for a preset course") + } +} + +func TestHasAssessmentMaterial(t *testing.T) { + tests := []struct { + name string + body string + want bool + }{ + {name: "absent block", body: validManifest, want: false}, + {name: "explicit none", body: validManifest + "assessment_markers:\n preset: none\n", want: false}, + {name: "preset", body: validManifest + "assessment_markers:\n preset: saq\n", want: true}, + { + name: "custom headings without preset", + body: validManifest + "assessment_markers:\n question_heading: \"### Q{n}\"\n", + want: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m, err := Load(writeCourse(t, tc.body)) + if err != nil { + t.Fatal(err) + } + if got := m.HasAssessmentMaterial(); got != tc.want { + t.Errorf("HasAssessmentMaterial() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestModuleDirDefaultsAndOverrides(t *testing.T) { + body := validManifest + ` - num: 2 + topic: quantization + dir: unit-two +` + m, err := Load(writeCourse(t, body)) + if err != nil { + t.Fatal(err) + } + mod, ok := m.FindModule(1) + if !ok { + t.Fatal("FindModule(1) not found") + } + if got := filepath.Base(m.ModuleDir(mod)); got != "Module_1" { + t.Errorf("ModuleDir default = %q, want Module_1", got) + } + mod, ok = m.FindModule(2) + if !ok { + t.Fatal("FindModule(2) not found") + } + if got := filepath.Base(m.ModuleDir(mod)); got != "unit-two" { + t.Errorf("ModuleDir override = %q, want unit-two", got) + } + if _, ok := m.FindModule(3); ok { + t.Error("FindModule(3) = found, want missing") + } +} diff --git a/internal/course/scaffold.go b/internal/course/scaffold.go new file mode 100644 index 0000000..6dc5ede --- /dev/null +++ b/internal/course/scaffold.go @@ -0,0 +1,147 @@ +package course + +import ( + "bytes" + _ "embed" + "fmt" + "os" + "path/filepath" + "text/template" +) + +//go:embed templates/course.yaml.tmpl +var courseTemplate string + +//go:embed templates/lesson.md.tmpl +var lessonTemplate string + +// ScaffoldOptions configures a new course tree. +type ScaffoldOptions struct { + // Dir is the course directory to create or fill. + Dir string + // Slug is the course identity. Empty scaffolds a placeholder the + // author must replace. + Slug string + // Title is the human-readable course name. Empty scaffolds a + // placeholder. + Title string + // Modules is how many module directories to stub out. + Modules int + // Preset names an assessment-marker preset, or "none". + Preset string + // SampleLesson writes one example lesson per module, which doubles + // as inline documentation of the format. + SampleLesson bool + // Force allows writing into a directory that already holds a + // manifest. + Force bool +} + +// Scaffold writes a new course skeleton and returns the paths it +// created, in creation order. +func Scaffold(opts ScaffoldOptions) ([]string, error) { + if opts.Dir == "" { + return nil, fmt.Errorf("course directory is required") + } + if opts.Modules < 1 { + opts.Modules = 1 + } + if opts.Preset == "" { + opts.Preset = "none" + } + if opts.Preset != "none" { + if _, ok := Presets[opts.Preset]; !ok { + return nil, fmt.Errorf("unknown assessment preset %q", opts.Preset) + } + } + if opts.Slug != "" && !slugPattern.MatchString(opts.Slug) { + return nil, fmt.Errorf("slug %q: must be lowercase letters, digits, and hyphens, starting with a letter or digit, at most 64 characters", opts.Slug) + } + + manifestPath := filepath.Join(opts.Dir, FileName) + if _, err := os.Stat(manifestPath); err == nil && !opts.Force { + return nil, fmt.Errorf("%s already exists (use --force to overwrite)", manifestPath) + } + + if err := os.MkdirAll(opts.Dir, 0o755); err != nil { + return nil, err + } + + data := scaffoldData(opts) + manifest, err := render("course.yaml", courseTemplate, data) + if err != nil { + return nil, err + } + if err := os.WriteFile(manifestPath, manifest, 0o644); err != nil { + return nil, err + } + created := []string{manifestPath} + + for i := 1; i <= opts.Modules; i++ { + moduleDir := filepath.Join(opts.Dir, fmt.Sprintf("Module_%d", i)) + if err := os.MkdirAll(moduleDir, 0o755); err != nil { + return created, err + } + created = append(created, moduleDir) + + if !opts.SampleLesson { + continue + } + lessonDir := filepath.Join(moduleDir, "Lesson_01_Example") + if err := os.MkdirAll(filepath.Join(lessonDir, "images"), 0o755); err != nil { + return created, err + } + lesson, err := render("lesson.md", lessonTemplate, data) + if err != nil { + return created, err + } + lessonPath := filepath.Join(lessonDir, "lesson.md") + if err := os.WriteFile(lessonPath, lesson, 0o644); err != nil { + return created, err + } + created = append(created, lessonPath) + } + return created, nil +} + +// scaffoldFields is the template context for the scaffolded files. +type scaffoldFields struct { + Slug string + Title string + Modules []int + Preset string + // Placeholder is the prefix Validate rejects, exposed to the + // template so the two never drift apart. + Placeholder string +} + +func scaffoldData(opts ScaffoldOptions) scaffoldFields { + f := scaffoldFields{ + Slug: opts.Slug, + Title: opts.Title, + Preset: opts.Preset, + Placeholder: placeholderPrefix, + } + if f.Slug == "" { + f.Slug = placeholderPrefix + "course-slug" + } + if f.Title == "" { + f.Title = placeholderPrefix + "with the course title" + } + for i := 1; i <= opts.Modules; i++ { + f.Modules = append(f.Modules, i) + } + return f +} + +func render(name, tmpl string, data scaffoldFields) ([]byte, error) { + t, err := template.New(name).Parse(tmpl) + if err != nil { + return nil, err + } + var buf bytes.Buffer + if err := t.Execute(&buf, data); err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/internal/course/scaffold_test.go b/internal/course/scaffold_test.go new file mode 100644 index 0000000..de3f1c9 --- /dev/null +++ b/internal/course/scaffold_test.go @@ -0,0 +1,130 @@ +package course + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestScaffoldRoundTrip is the contract the scaffold exists to satisfy: +// a fresh course must NOT load (every placeholder is a decision the +// author owes), and must load once those decisions are made. Without the +// first half, a half-filled course produces generically-worded lectures +// that look fine until you listen to them. +func TestScaffoldRoundTrip(t *testing.T) { + dir := t.TempDir() + created, err := Scaffold(ScaffoldOptions{Dir: dir, Slug: "home-llms", Title: "Running LLMs at Home", Modules: 2, SampleLesson: true}) + if err != nil { + t.Fatal(err) + } + if len(created) == 0 { + t.Fatal("Scaffold() created nothing") + } + + if _, err := Load(dir); err == nil { + t.Fatal("Load() on a fresh scaffold = nil, want placeholder errors") + } else if !strings.Contains(err.Error(), "placeholder") { + t.Fatalf("Load() = %v, want placeholder errors", err) + } + + manifestPath := filepath.Join(dir, FileName) + body, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatal(err) + } + filled := replacePlaceholders(string(body)) + // Only values matter: the manifest's own comments mention the + // placeholder prefix by design, explaining the rule to the author. + for _, line := range strings.Split(filled, "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "#") && strings.Contains(line, placeholderPrefix) { + t.Fatalf("test helper left a placeholder value behind: %q", line) + } + } + if err := os.WriteFile(manifestPath, []byte(filled), 0o644); err != nil { + t.Fatal(err) + } + + m, err := Load(dir) + if err != nil { + t.Fatalf("Load() after filling placeholders = %v, want success", err) + } + if m.Slug != "home-llms" { + t.Errorf("slug = %q, want the scaffolded value", m.Slug) + } + if len(m.Modules) != 2 { + t.Errorf("modules = %d, want 2", len(m.Modules)) + } + + // The sample lesson must satisfy the tree contract it documents, + // otherwise `init` hands the user a course that fails validation for + // reasons they did not cause. + rep := m.ScanTree() + if rep.Errors() != 0 { + t.Errorf("scaffolded tree reported errors: %v", rep.Findings) + } + if rep.Modules[0].Lessons != 1 { + t.Errorf("sample module has %d lesson(s), want 1", rep.Modules[0].Lessons) + } +} + +// replacePlaceholders fills every REPLACE- value with plausible content, +// the way an author would. +func replacePlaceholders(body string) string { + var out []string + for _, line := range strings.Split(body, "\n") { + if idx := strings.Index(line, placeholderPrefix); idx >= 0 && !strings.HasPrefix(strings.TrimSpace(line), "#") { + out = append(out, line[:idx]+"filled in by the author") + continue + } + out = append(out, line) + } + return strings.Join(out, "\n") +} + +func TestScaffoldRefusesToClobber(t *testing.T) { + dir := t.TempDir() + if _, err := Scaffold(ScaffoldOptions{Dir: dir}); err != nil { + t.Fatal(err) + } + if _, err := Scaffold(ScaffoldOptions{Dir: dir}); err == nil { + t.Fatal("second Scaffold() = nil, want an already-exists error") + } + if _, err := Scaffold(ScaffoldOptions{Dir: dir, Force: true}); err != nil { + t.Fatalf("Scaffold(force) = %v, want success", err) + } +} + +func TestScaffoldValidatesOptions(t *testing.T) { + tests := []struct { + name string + opts ScaffoldOptions + wantErr string + }{ + {name: "bad slug", opts: ScaffoldOptions{Slug: "Not A Slug"}, wantErr: "must be lowercase"}, + {name: "unknown preset", opts: ScaffoldOptions{Preset: "nonesuch"}, wantErr: "unknown assessment preset"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + tc.opts.Dir = t.TempDir() + _, err := Scaffold(tc.opts) + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("Scaffold() = %v, want error containing %q", err, tc.wantErr) + } + }) + } +} + +func TestScaffoldPresetReachesManifest(t *testing.T) { + dir := t.TempDir() + if _, err := Scaffold(ScaffoldOptions{Dir: dir, Slug: "drilled", Title: "Drilled", Preset: "saq"}); err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(filepath.Join(dir, FileName)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), "preset: saq") { + t.Errorf("scaffolded manifest does not carry the requested preset:\n%s", body) + } +} diff --git a/internal/course/templates/course.yaml.tmpl b/internal/course/templates/course.yaml.tmpl new file mode 100644 index 0000000..10acf9c --- /dev/null +++ b/internal/course/templates/course.yaml.tmpl @@ -0,0 +1,77 @@ +# course.yaml — what this course IS. +# +# This file carries the course's identity and the labels that skin every +# prompt. It does NOT say where models run: endpoints, API keys, and +# hardware profiles are machine-scoped and live in vibe/vamp config or +# ETUDE_* environment variables, so the same course travels between a +# laptop, a GPU box, and an external provider unchanged. +# +# Every {{ .Placeholder }} value below must be replaced — the tooling +# refuses to load a manifest that still carries one. +# +# Validate as you go: etude course validate + +version: 1 + +# Identity. The slug names this course everywhere it appears: the +# learner-memory expert, the knowledge collection, deliverable +# filenames. Lowercase letters, digits, and hyphens. +slug: {{ .Slug }} +title: {{ .Title }} + +# The four labels that skin every prompt. These carry the whole domain +# identity of the course — the pipeline itself knows nothing about your +# subject. +# +# subject the field of study, as a lecturer would name it +# program the course/certification this material belongs to +# persona the first-person identity the lecturer adopts +# assessment how learners are evaluated (drives "exam pointer" language) +subject: {{ .Placeholder }}with the field of study, e.g. "introductory astronomy" +program: {{ .Placeholder }}with the course name, e.g. "an introductory astronomy course" +persona: {{ .Placeholder }}with the lecturer identity, e.g. "an astronomy lecturer" +assessment: {{ .Placeholder }}with the evaluation, e.g. "end-of-unit self-check questions" + +# Where the material lives, relative to this file. The source directory +# holds one subdirectory per module. +source: . + +# Modules in teaching order. Order is explicit because directory sort +# puts Module_10 before Module_2, and a course is not a set. +modules: +{{- range .Modules }} + - num: {{ . }} + topic: {{ $.Placeholder }}with a short summary of module {{ . }} + # dir: Module_{{ . }} # defaults to Module_ +{{- end }} + +# Presentation. Every field is optional; sensible defaults apply. +render: + voice: af_bella # Kokoro voice for the narration + # cover_prompt: ... # SDXL prompt for the cover art + # search_suffix: ... # appended to web-search queries for enrichment + # epub_title: ... + # epub_author: ... + # filename_prefix: ... # deliverable basename stem; defaults from the topic + +# How this corpus marks its own questions and model answers, if it +# carries any. Material that ships genuine assessment questions is the +# privileged input: the drill relays those verbatim and grades against +# the official answer, never inventing either. Corpora without them get +# generated questions instead, which pass through human review before +# entering the bank. +# +# Presets: none, saq +assessment_markers: + preset: {{ .Preset }} + # lesson_pattern: "*SAQ*" # which lessons hold questions + # question_heading: "## Self-Assessment Questions" + # answer_heading: "## Suggested Response - Q{n}" + # difficulty_marker: escaped-asterisks + +# Bindings to the services that make this course conversational. Both +# default to the course slug; set them only when an existing deployment +# already uses different names. +expert: + # memory_slug: {{ .Slug }} + # knowledge_collection: {{ .Slug }} diff --git a/internal/course/templates/lesson.md.tmpl b/internal/course/templates/lesson.md.tmpl new file mode 100644 index 0000000..8a59992 --- /dev/null +++ b/internal/course/templates/lesson.md.tmpl @@ -0,0 +1,44 @@ +# Example Lesson + +Replace this file with real material. It exists to show the shape a +lesson takes; delete the directory once you have your own. + +A lesson is one self-contained piece of teaching — roughly what a +lecturer would cover in one sitting. The pipeline reads the whole file +and writes lecture prose, study-guide sections, and retrieval chunks +from it, so the structure below is a guide rather than a schema: use +ordinary markdown headings and prose. + +## What a lesson should contain + +Explain the mechanism, not just the conclusion. Downstream prompts are +told never to invent facts, so anything the lecture should say has to be +here first. + +Numbers, definitions, and processes are worth stating plainly — they are +extracted into the study guide, the glossary, and the flashcard deck. + +## Figures + +Put images in an `images/` subdirectory beside this file. PNG, JPEG, +GIF, WebP, BMP, and SVG are read; anything else is ignored. A vision +model describes each figure once and folds the description into the +prose, so a diagram with no caption still reaches the audio. + +SVGs work but need `rsvg-convert` installed, and the EPUB omits them — +ship a raster copy alongside if the figure matters. + +## Equations + +Write equations in fenced code blocks so they survive verbatim: + +``` +extract yield = (litres of alcohol) / (tonnes of grain) +``` + +## If your material is not already in lessons + +Most material is not. A book, a PDF export, or a pile of documents has +chapters or files, not lessons — and a good lesson breakdown is its own +piece of work. Use `etude ingest` to propose one from unstructured +source material and review the proposal before it becomes a course. diff --git a/internal/course/tree.go b/internal/course/tree.go new file mode 100644 index 0000000..b12e5d6 --- /dev/null +++ b/internal/course/tree.go @@ -0,0 +1,453 @@ +package course + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// Lesson-tree limits enforced by the vamp template functions the +// pipeline calls. They are reproduced here because the failure mode on +// the pipeline side is SILENT: enumerateLessons drops an oversized or +// lesson.md-less directory from the course without a word, and the batch +// readers filter oversized files out of prompt context the same way. A +// course that validates clean is one where nothing disappears. +const ( + // maxLessonBytes mirrors enumerateLessons: a lesson.md above this is + // dropped from the lesson list entirely, so the lesson never reaches + // any stage. + maxLessonBytes = 1024 * 1024 + // batchReadBytes mirrors readFiles/readLessons/readFileBatch: a file + // above this is filtered out of batch prompt context, so the lesson + // is listed and rendered but its content may be missing from + // corpus-wide stages. + batchReadBytes = 200 * 1024 + // truncateBytes is the tighter of the two prompt truncation caps the + // pipeline applies per lesson (120k in the study-guide prompt, 150k + // in the lesson-processing prompt). Past it the two stages no longer + // see the same lesson, which is a quieter and more confusing failure + // than losing the lesson outright. + truncateBytes = 120 * 1000 +) + +// imageExts is the set enumerateUniqueImages and +// imageDescriptionsForLesson accept. Anything else in an images/ +// directory is ignored by the vision stage. +var imageExts = map[string]bool{ + ".png": true, ".jpg": true, ".jpeg": true, ".gif": true, + ".webp": true, ".bmp": true, ".svg": true, +} + +// Severity distinguishes "this course is broken" from "this course will +// build but something will be missing or surprising". +type Severity string + +const ( + // Error means a stage will fail or content will vanish from the + // build. + Error Severity = "error" + // Warning means the build succeeds but the result is degraded or + // the author probably did not intend it. + Warning Severity = "warning" +) + +// Finding is one diagnostic against a course tree. +type Finding struct { + Severity Severity + // Path is relative to the course directory when possible, so output + // is readable and diffable. + Path string + Message string +} + +func (f Finding) String() string { + if f.Path == "" { + return fmt.Sprintf("%s: %s", f.Severity, f.Message) + } + return fmt.Sprintf("%s: %s: %s", f.Severity, f.Path, f.Message) +} + +// Report is the result of scanning a course tree. +type Report struct { + Findings []Finding + // Modules summarises what the scan found, in manifest order. + Modules []ModuleSummary +} + +// ModuleSummary is the per-module tally a human wants to see: the point +// of validate is as much "did it find everything" as "is it valid". +type ModuleSummary struct { + Num int + Dir string + Lessons int + Images int + // Assessment counts lessons matching the assessment lesson pattern, + // i.e. the ones the drill's bank extractor will read. + Assessment int + // SVG and SVGOnly track vector figures, the latter being those with + // no raster sibling. Both are module-level because SVG-heavy corpora + // are the norm, not the exception. + SVG int + SVGOnly int +} + +// Errors reports whether the scan found anything fatal. +func (r *Report) Errors() int { return r.count(Error) } + +// Warnings reports how many advisory findings the scan produced. +func (r *Report) Warnings() int { return r.count(Warning) } + +func (r *Report) count(s Severity) int { + n := 0 + for _, f := range r.Findings { + if f.Severity == s { + n++ + } + } + return n +} + +func (r *Report) add(sev Severity, path, format string, args ...any) { + r.Findings = append(r.Findings, Finding{ + Severity: sev, + Path: path, + Message: fmt.Sprintf(format, args...), + }) +} + +// ScanTree walks the course's lesson tree and reports everything that +// would break, vanish, or surprise. It is the content half of +// validation; Manifest.Validate covers the manifest itself. +func (m *Manifest) ScanTree() *Report { + rep := &Report{} + + src := m.SourceDir() + if info, err := os.Stat(src); err != nil || !info.IsDir() { + rep.add(Error, m.rel(src), "source directory does not exist") + return rep + } + + markers := m.Assessments.Resolved() + for _, mod := range m.Modules { + rep.Modules = append(rep.Modules, m.scanModule(rep, mod, markers)) + } + + m.reportUnlistedModules(rep) + return rep +} + +// scanModule validates one module directory and returns its tally. +func (m *Manifest) scanModule(rep *Report, mod Module, markers Assessments) ModuleSummary { + dir := m.ModuleDir(mod) + sum := ModuleSummary{Num: mod.Num, Dir: m.rel(dir)} + + info, err := os.Stat(dir) + if err != nil || !info.IsDir() { + rep.add(Error, m.rel(dir), "module %d: directory does not exist", mod.Num) + return sum + } + + // The same glob the pipeline runs, so validation sees exactly what + // the build will see. + matches, err := filepath.Glob(filepath.Join(dir, "Lesson_*")) + if err != nil { + rep.add(Error, m.rel(dir), "module %d: cannot list lessons: %v", mod.Num, err) + return sum + } + sort.Strings(matches) + + var ( + names []string + skipped int + ) + for _, match := range matches { + fi, statErr := os.Stat(match) + if statErr != nil || !fi.IsDir() { + continue + } + name := filepath.Base(match) + if m.scanLesson(rep, match, name, markers, &sum) { + names = append(names, name) + } else { + skipped++ + } + } + + sum.Lessons = len(names) + if sum.Lessons == 0 { + rep.add(Error, m.rel(dir), "module %d: no usable Lesson_*/ directories (a lesson directory must contain a lesson.md)", mod.Num) + } + if skipped > 0 { + rep.add(Warning, m.rel(dir), "module %d: %d Lesson_*/ director(ies) will be skipped by the build", mod.Num, skipped) + } + if sum.SVG > 0 { + // Two distinct consequences, so say both once: vision needs a + // rasteriser on the host, and the study guide's figure pass + // skips SVGs, so vector-only figures never reach the EPUB. + msg := fmt.Sprintf("module %d: %d SVG figure(s) — the vision stage needs rsvg-convert on the host", mod.Num, sum.SVG) + if sum.SVGOnly > 0 { + msg += fmt.Sprintf(", and %d with no raster sibling will be missing from the study guide and EPUB", sum.SVGOnly) + } + rep.add(Warning, m.rel(dir), "%s", msg) + } + m.reportNonLessonDirs(rep, dir, mod) + reportLessonOrdering(rep, m.rel(dir), names) + return sum +} + +// scanLesson checks one lesson directory. It returns false when the +// pipeline would drop the lesson entirely, so the caller can count what +// disappears. +func (m *Manifest) scanLesson(rep *Report, dir, name string, markers Assessments, sum *ModuleSummary) bool { + lessonFile := filepath.Join(dir, "lesson.md") + fi, err := os.Stat(lessonFile) + switch { + case os.IsNotExist(err): + rep.add(Error, m.rel(dir), "no lesson.md — the build silently drops this directory") + return false + case err != nil: + rep.add(Error, m.rel(lessonFile), "cannot read: %v", err) + return false + case fi.IsDir(): + rep.add(Error, m.rel(lessonFile), "is a directory, not a file") + return false + } + + switch { + case fi.Size() == 0: + rep.add(Error, m.rel(lessonFile), "is empty") + case fi.Size() > maxLessonBytes: + rep.add(Error, m.rel(lessonFile), "is %s, over the %s limit — the build silently drops this lesson; split it", + humanBytes(fi.Size()), humanBytes(maxLessonBytes)) + return false + case fi.Size() > batchReadBytes: + rep.add(Warning, m.rel(lessonFile), "is %s, over the %s batch-read limit — corpus-wide stages will omit its content; consider splitting it", + humanBytes(fi.Size()), humanBytes(batchReadBytes)) + case fi.Size() > truncateBytes: + rep.add(Warning, m.rel(lessonFile), "is %s: past %s the lecture and study-guide stages truncate at different lengths and stop seeing the same lesson; consider splitting it", + humanBytes(fi.Size()), humanBytes(truncateBytes)) + } + + m.checkTitle(rep, lessonFile) + sum.Images += m.scanImages(rep, dir, sum) + + if markers.LessonPattern != "" { + if ok, _ := filepath.Match(markers.LessonPattern, name); ok { + sum.Assessment++ + } + } else if m.HasAssessmentMaterial() { + sum.Assessment++ + } + return true +} + +// checkTitle warns when a lesson has no leading H1. The pipeline treats +// that first heading as the lesson's title — the study-guide prompt is +// told to open each section with it verbatim — so a lesson without one +// gets an invented title downstream. +func (m *Manifest) checkTitle(rep *Report, lessonFile string) { + f, err := os.Open(lessonFile) + if err != nil { + return + } + defer f.Close() + + // The title is the first non-blank line by construction; reading a + // small prefix keeps validation cheap on a large corpus. + buf := make([]byte, 4096) + n, _ := f.Read(buf) + for _, line := range strings.Split(string(buf[:n]), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + if !strings.HasPrefix(line, "# ") { + rep.add(Warning, m.rel(lessonFile), "does not open with an H1 title — the study guide will invent one") + } + return + } +} + +// scanImages counts usable images and accumulates the SVG tally on the +// module summary. SVG findings are reported per module rather than per +// lesson: a corpus exported from a diagramming tool is frequently SVG +// throughout, and one warning per lesson would bury everything else. +func (m *Manifest) scanImages(rep *Report, lessonDir string, sum *ModuleSummary) int { + imgDir := filepath.Join(lessonDir, "images") + entries, err := os.ReadDir(imgDir) + if err != nil { + // No images/ directory is normal; anything else is worth saying. + if !os.IsNotExist(err) { + rep.add(Warning, m.rel(imgDir), "cannot read images directory: %v", err) + } + return 0 + } + + var usable int + stems := make(map[string]map[string]bool) + for _, ent := range entries { + if ent.IsDir() { + continue + } + name := ent.Name() + ext := strings.ToLower(filepath.Ext(name)) + if !imageExts[ext] { + rep.add(Warning, m.rel(filepath.Join(imgDir, name)), "unsupported image type %q — the vision stage ignores it", ext) + continue + } + usable++ + stem := strings.TrimSuffix(name, filepath.Ext(name)) + if stems[stem] == nil { + stems[stem] = map[string]bool{} + } + stems[stem][ext] = true + } + + for _, exts := range stems { + if !exts[".svg"] { + continue + } + sum.SVG++ + if len(exts) == 1 { + sum.SVGOnly++ + } + } + return usable +} + +// reportNonLessonDirs flags directories that hold a lesson.md but do not +// match the Lesson_* glob. Without this, a mis-named directory is +// invisible: the build simply never sees it. +func (m *Manifest) reportNonLessonDirs(rep *Report, moduleDir string, mod Module) { + entries, err := os.ReadDir(moduleDir) + if err != nil { + return + } + for _, ent := range entries { + if !ent.IsDir() || strings.HasPrefix(ent.Name(), "Lesson_") { + continue + } + if _, err := os.Stat(filepath.Join(moduleDir, ent.Name(), "lesson.md")); err == nil { + rep.add(Warning, m.rel(filepath.Join(moduleDir, ent.Name())), + "module %d: contains a lesson.md but is not named Lesson_* — the build ignores it", mod.Num) + } + } +} + +// reportUnlistedModules flags module-shaped directories the manifest +// does not list, since teaching order is explicit and an unlisted module +// simply never builds. +func (m *Manifest) reportUnlistedModules(rep *Report) { + src := m.SourceDir() + entries, err := os.ReadDir(src) + if err != nil { + return + } + listed := make(map[string]bool, len(m.Modules)) + for _, mod := range m.Modules { + listed[filepath.Base(m.ModuleDir(mod))] = true + } + for _, ent := range entries { + if !ent.IsDir() || !strings.HasPrefix(ent.Name(), "Module_") || listed[ent.Name()] { + continue + } + rep.add(Warning, m.rel(filepath.Join(src, ent.Name())), + "looks like a module but is not listed in the manifest — it will not build") + } +} + +// reportLessonOrdering warns when lexical order — what the build +// actually uses — diverges from the numeric order the author means. +// +// Two distinct faults produce that divergence and they need different +// advice. Unpadded numbers sort wrong (Lesson_10 before Lesson_2) and +// zero-padding fixes them. Numbers that REPEAT within one module mean +// several units were flattened into one directory, each restarting at +// lesson 1; sorting then interleaves the units, and no amount of padding +// recovers the intended sequence. +func reportLessonOrdering(rep *Report, dir string, names []string) { + type lesson struct { + name string + num int + } + var numbered []lesson + counts := map[int]int{} + for _, n := range names { + if num, ok := lessonNumber(n); ok { + numbered = append(numbered, lesson{name: n, num: num}) + counts[num]++ + } + } + if len(numbered) < 2 { + return + } + + var repeated int + for _, c := range counts { + if c > 1 { + repeated++ + } + } + if repeated > 0 { + rep.add(Warning, dir, + "%d lesson number(s) repeat, so several units share one flat directory and the build interleaves them — give each unit its own module, or renumber the lessons across the whole module", + repeated) + return + } + + for i := 1; i < len(numbered); i++ { + if numbered[i].num < numbered[i-1].num { + rep.add(Warning, dir, + "lesson order is lexical, so %s builds before %s — zero-pad lesson numbers (Lesson_02) to fix the order", + numbered[i-1].name, numbered[i].name) + return + } + } +} + +// lessonNumber extracts the leading number from a Lesson_... name. +func lessonNumber(name string) (int, bool) { + rest := strings.TrimPrefix(name, "Lesson_") + if rest == name { + return 0, false + } + end := 0 + for end < len(rest) && rest[end] >= '0' && rest[end] <= '9' { + end++ + } + if end == 0 { + return 0, false + } + n, err := strconv.Atoi(rest[:end]) + if err != nil { + return 0, false + } + return n, true +} + +// rel renders a path relative to the course directory for readable +// output, falling back to the absolute path. +func (m *Manifest) rel(path string) string { + if m.dir == "" { + return path + } + r, err := filepath.Rel(m.dir, path) + if err != nil || strings.HasPrefix(r, "..") { + return path + } + return r +} + +func humanBytes(n int64) string { + switch { + case n >= 1024*1024: + return fmt.Sprintf("%.1f MiB", float64(n)/(1024*1024)) + case n >= 1024: + return fmt.Sprintf("%.0f KiB", float64(n)/1024) + default: + return fmt.Sprintf("%d B", n) + } +} diff --git a/internal/course/tree_test.go b/internal/course/tree_test.go new file mode 100644 index 0000000..ce491fc --- /dev/null +++ b/internal/course/tree_test.go @@ -0,0 +1,324 @@ +package course + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// courseTree builds a course on disk. lessons maps a lesson directory +// name to its lesson.md contents; a nil body means "create the directory +// but no lesson.md". +func courseTree(t *testing.T, manifest string, lessons map[string][]byte) string { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, FileName), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + for name, body := range lessons { + lessonDir := filepath.Join(dir, "Module_1", name) + if err := os.MkdirAll(lessonDir, 0o755); err != nil { + t.Fatal(err) + } + if body == nil { + continue + } + if err := os.WriteFile(filepath.Join(lessonDir, "lesson.md"), body, 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +func scan(t *testing.T, dir string) *Report { + t.Helper() + m, err := Load(dir) + if err != nil { + t.Fatal(err) + } + return m.ScanTree() +} + +// findingsFor returns the messages at one severity, for assertions that +// care about what was said rather than how many. +func findingsFor(rep *Report, sev Severity) []string { + var out []string + for _, f := range rep.Findings { + if f.Severity == sev { + out = append(out, f.Message) + } + } + return out +} + +func assertFinding(t *testing.T, rep *Report, sev Severity, substr string) { + t.Helper() + for _, msg := range findingsFor(rep, sev) { + if strings.Contains(msg, substr) { + return + } + } + t.Errorf("no %s finding containing %q; got %v", sev, substr, findingsFor(rep, sev)) +} + +func TestScanTreeClean(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{ + "Lesson_01_Memory": []byte("# Memory\n\nHow weights fit.\n"), + "Lesson_02_Quants": []byte("# Quantization\n\nWhat q4 costs you.\n"), + }) + rep := scan(t, dir) + if rep.Errors() != 0 || rep.Warnings() != 0 { + t.Fatalf("clean course reported %d error(s), %d warning(s): %v", rep.Errors(), rep.Warnings(), rep.Findings) + } + if len(rep.Modules) != 1 || rep.Modules[0].Lessons != 2 { + t.Fatalf("summary = %+v, want 1 module with 2 lessons", rep.Modules) + } +} + +// TestScanTreeCatchesSilentDrops covers the failures that make +// validation worth running: each of these produces a smaller course with +// no complaint from the pipeline itself. +func TestScanTreeCatchesSilentDrops(t *testing.T) { + t.Run("missing lesson.md", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{ + "Lesson_01_Good": []byte("# Fine\n"), + "Lesson_02_Empty": nil, + }) + rep := scan(t, dir) + assertFinding(t, rep, Error, "no lesson.md") + if rep.Modules[0].Lessons != 1 { + t.Errorf("counted %d lesson(s), want 1 usable", rep.Modules[0].Lessons) + } + }) + + t.Run("oversized lesson is dropped", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{ + "Lesson_01_Good": []byte("# Fine\n"), + "Lesson_02_Huge": make([]byte, maxLessonBytes+1), + }) + rep := scan(t, dir) + assertFinding(t, rep, Error, "silently drops this lesson") + }) + + t.Run("batch-read limit warns", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{ + "Lesson_01_Big": make([]byte, batchReadBytes+1), + }) + rep := scan(t, dir) + assertFinding(t, rep, Warning, "batch-read limit") + if rep.Errors() != 0 { + t.Errorf("batch-read overflow should not be fatal: %v", rep.Findings) + } + }) + + t.Run("empty lesson", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_Void": {}}) + rep := scan(t, dir) + assertFinding(t, rep, Error, "is empty") + }) + + t.Run("module with no usable lessons", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_Bare": nil}) + rep := scan(t, dir) + assertFinding(t, rep, Error, "no usable Lesson_*/ directories") + }) + + t.Run("missing module directory", func(t *testing.T) { + dir := courseTree(t, validManifest, nil) + rep := scan(t, dir) + assertFinding(t, rep, Error, "directory does not exist") + }) + + t.Run("misnamed lesson directory", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_Good": []byte("# Fine\n")}) + stray := filepath.Join(dir, "Module_1", "Chapter_02") + if err := os.MkdirAll(stray, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stray, "lesson.md"), []byte("# Orphan\n"), 0o644); err != nil { + t.Fatal(err) + } + rep := scan(t, dir) + assertFinding(t, rep, Warning, "not named Lesson_*") + }) + + t.Run("unlisted module directory", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_Good": []byte("# Fine\n")}) + if err := os.MkdirAll(filepath.Join(dir, "Module_2", "Lesson_01_Ghost"), 0o755); err != nil { + t.Fatal(err) + } + rep := scan(t, dir) + assertFinding(t, rep, Warning, "not listed in the manifest") + }) +} + +// TestScanTreeLessonOrdering pins the lexical-vs-numeric trap: +// Lesson_10 sorts before Lesson_2, silently reordering a course. +func TestScanTreeLessonOrdering(t *testing.T) { + t.Run("unpadded numbers warn", func(t *testing.T) { + lessons := map[string][]byte{} + for _, n := range []string{"Lesson_1_A", "Lesson_2_B", "Lesson_10_C"} { + lessons[n] = []byte("# Content\n") + } + rep := scan(t, courseTree(t, validManifest, lessons)) + assertFinding(t, rep, Warning, "lesson order is lexical") + }) + + t.Run("zero-padded numbers are quiet", func(t *testing.T) { + lessons := map[string][]byte{} + for _, n := range []string{"Lesson_01_A", "Lesson_02_B", "Lesson_10_C"} { + lessons[n] = []byte("# Content\n") + } + rep := scan(t, courseTree(t, validManifest, lessons)) + if rep.Warnings() != 0 { + t.Errorf("padded lessons warned: %v", rep.Findings) + } + }) + + // A real scraped corpus flattened several units into one module + // directory, each restarting at lesson 1, which interleaves the + // units under a bytewise sort. Padding cannot fix that, so the + // advice has to differ. + t.Run("repeated numbers get different advice", func(t *testing.T) { + lessons := map[string][]byte{} + for _, n := range []string{"Lesson_1_-_Barley", "Lesson_2_-_Milling", "Lesson_1_-_Water", "Lesson_2_-_Yeast"} { + lessons[n] = []byte("# Content\n") + } + rep := scan(t, courseTree(t, validManifest, lessons)) + assertFinding(t, rep, Warning, "repeat") + for _, msg := range findingsFor(rep, Warning) { + if strings.Contains(msg, "zero-pad") { + t.Errorf("advised zero-padding for restarting numbers, which cannot fix it: %q", msg) + } + } + }) +} + +// TestScanTreeTitle covers the H1 contract: the pipeline treats a +// lesson's first heading as its title and the study-guide prompt is told +// to reuse it verbatim, so a lesson without one gets an invented title. +func TestScanTreeTitle(t *testing.T) { + tests := []struct { + name string + body string + wantWarn bool + }{ + {name: "leading h1", body: "# Barley\n\nProse.\n"}, + {name: "blank lines before h1", body: "\n\n# Barley\n\nProse.\n"}, + {name: "no heading", body: "Prose with no title at all.\n", wantWarn: true}, + {name: "starts at h2", body: "## Section\n\nProse.\n", wantWarn: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_A": []byte(tc.body)}) + rep := scan(t, dir) + var got bool + for _, msg := range findingsFor(rep, Warning) { + if strings.Contains(msg, "H1 title") { + got = true + } + } + if got != tc.wantWarn { + t.Errorf("H1 warning = %v, want %v (findings: %v)", got, tc.wantWarn, rep.Findings) + } + }) + } +} + +func TestScanTreeImages(t *testing.T) { + writeImages := func(t *testing.T, dir string, names ...string) { + t.Helper() + imgDir := filepath.Join(dir, "Module_1", "Lesson_01_Figures", "images") + if err := os.MkdirAll(imgDir, 0o755); err != nil { + t.Fatal(err) + } + for _, n := range names { + if err := os.WriteFile(filepath.Join(imgDir, n), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + } + + t.Run("counts supported images", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_Figures": []byte("# Figures\n")}) + writeImages(t, dir, "one.png", "two.jpg") + rep := scan(t, dir) + if rep.Modules[0].Images != 2 { + t.Errorf("counted %d image(s), want 2", rep.Modules[0].Images) + } + if rep.Warnings() != 0 { + t.Errorf("unexpected warnings: %v", rep.Findings) + } + }) + + t.Run("unsupported type warns", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_Figures": []byte("# Figures\n")}) + writeImages(t, dir, "diagram.pdf") + rep := scan(t, dir) + assertFinding(t, rep, Warning, "unsupported image type") + }) + + // SVG findings are reported once per module: a corpus exported from + // a diagramming tool can be SVG throughout, and per-lesson warnings + // would bury every other finding. + t.Run("svg reported once per module", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_Figures": []byte("# Figures\n")}) + writeImages(t, dir, "solo.svg", "paired.svg", "paired.png") + rep := scan(t, dir) + assertFinding(t, rep, Warning, "rsvg-convert") + assertFinding(t, rep, Warning, "no raster sibling") + if rep.Modules[0].SVG != 2 { + t.Errorf("SVG count = %d, want 2", rep.Modules[0].SVG) + } + if rep.Modules[0].SVGOnly != 1 { + t.Errorf("SVGOnly count = %d, want 1 (the paired SVG is not orphaned)", rep.Modules[0].SVGOnly) + } + var svgWarnings int + for _, msg := range findingsFor(rep, Warning) { + if strings.Contains(msg, "SVG figure(s)") { + svgWarnings++ + } + } + if svgWarnings != 1 { + t.Errorf("emitted %d SVG warnings, want exactly 1 per module", svgWarnings) + } + }) + + t.Run("raster-only figures are quiet", func(t *testing.T) { + dir := courseTree(t, validManifest, map[string][]byte{"Lesson_01_Figures": []byte("# Figures\n")}) + writeImages(t, dir, "fig.png") + rep := scan(t, dir) + if rep.Warnings() != 0 { + t.Errorf("unexpected warnings: %v", rep.Findings) + } + }) +} + +// TestScanTreeAssessmentMarkers verifies the drill's input is visible at +// validate time: a course claiming assessment material whose pattern +// matches nothing has no questions to extract, and that should be said +// before the drill is empty. +func TestScanTreeAssessmentMarkers(t *testing.T) { + body := validManifest + "assessment_markers:\n preset: saq\n" + + t.Run("matching lessons counted", func(t *testing.T) { + dir := courseTree(t, body, map[string][]byte{ + "Lesson_01_Content": []byte("# Content\n"), + "Lesson_02_Unit_SAQ": []byte("## Question 1\n"), + "Lesson_03_Cereals_Unit_SAQ": []byte("# Cereals SAQ\n\n## Self-Assessment Questions\n"), + }) + rep := scan(t, dir) + if rep.Modules[0].Assessment != 2 { + t.Errorf("counted %d assessment lesson(s), want 2", rep.Modules[0].Assessment) + } + }) + + t.Run("no matching lessons", func(t *testing.T) { + dir := courseTree(t, body, map[string][]byte{"Lesson_01_Content": []byte("# Content\n")}) + rep := scan(t, dir) + if rep.Modules[0].Assessment != 0 { + t.Errorf("counted %d assessment lesson(s), want 0", rep.Modules[0].Assessment) + } + }) +}