feat: add a markdown option for remark, rehype, and recma plugins

.changeset/markdown-plugins.md

@@ -0,0 +1,7 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+'@node-core/doc-kit-legacy': patch
+---
+
+Add a `markdown` option to add remark, rehype, and recma plugins to the generators processing Markdown, or configure the ones they use, such as Shiki. The `@doc-kit/core` modules the generators' pipelines replace are removed (`utils/remark.mjs`, `utils/remark-shiki.mjs`, and `utils/highlighter.mjs`), and `utils/type-annotations` moves to `plugins/type-annotations`

docs/configuration.md

@@ -123,9 +123,11 @@ Everything under the `global` key applies to every generator:
   a URL or path to parse, or a pre-parsed array. **Default:** `[]`
   (single-version output).
 - `index` {string|URL|Array} Index URL.
+- `markdown` {Object} [Markdown plugins](#markdown-plugins) for every
+  generator processing Markdown.
 
 A generator's own section (e.g., `html`, `legacy-json`) can override any of
-these for that generator alone.
+these for that generator alone, except `markdown`, which it adds to.
 
 ## Execution options
 
@@ -155,6 +157,96 @@ export default {
 };
 ```
 
+## Markdown plugins
+
+`doc-kit` processes Markdown with [unified](https://unifiedjs.com/).
+`global.markdown` adds plugins to every generator processing Markdown, and a
+generator's own `markdown` adds plugins to it alone, after the global ones.
+
+```mjs displayName="doc-kit.config.mjs"
+export default {
+  global: {
+    markdown: {
+      remarkPlugins: ['remark-math'],
+    },
+  },
+
+  // The generator rendering the site's pages
+  'jsx-ast': {
+    markdown: {
+      rehypePlugins: [
+        'rehype-katex',
+        ['./plugins/rehype-diagrams.mjs', { theme: 'neutral' }],
+      ],
+    },
+  },
+};
+```
+
+Each plugin is a module specifier (a package name, or a path relative to the
+configuration file), alone or in a `[specifier, options]` pair. The module
+default-exports the plugin, a list of plugins, or a unified preset. Markdown is
+processed in worker threads, which import the plugins themselves, so options
+must be serializable: configure a plugin taking functions in a module of your
+own.
+
+- `remarkPlugins` {Array} The global ones run on each document once the `ast`
+  generator parses it, so every output sees their changes. A generator
+  rendering Markdown, such as `jsx-ast`, runs its own on what it renders.
+- `rehypePlugins` {Array} Run on the HTML of the generators rendering
+  Markdown, such as `jsx-ast`, before code is highlighted.
+- `recmaPlugins` {Array} Run on the JavaScript `jsx-ast` compiles the pages to.
+
+A generator only takes the plugins its pipeline has a place for: `jsx-ast`
+takes all three kinds, `ast`, `metadata`, and `json` take remark plugins, and
+generators such as `html` take none. Plugins listed for a generator that
+doesn't take them are ignored with a warning.
+
+A generator runs each plugin once: listing one it already has, its own or a
+global one, configures it instead, merging the options (arrays add up, and
+objects merge).
+
+The plugins configured for `jsx-ast` don't apply to the fragments it renders
+on their own: types, parameter descriptions, and change history notes.
+
+### Code highlighting
+
+`jsx-ast` highlights code with the `@doc-kit/core/plugins/shiki/rehype.mjs`
+plugin, which runs [Shiki](https://shiki.style/) with every language it
+bundles. To configure it, list it with options in
+`jsx-ast.markdown.rehypePlugins` (with pnpm, add `@doc-kit/core` to your
+dependencies, so your configuration file can resolve it):
+
+- `langs` {Array} More languages: grammars, or modules default-exporting them.
+- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`.
+- `themes` {Object} The `light` and `dark` themes: names of themes Shiki
+  bundles, modules default-exporting themes, or themes.
+- `transformers` {Array} Modules default-exporting
+  [Shiki transformers](https://shiki.style/guide/transformers), or lists of
+  them.
+
+These modules are paths relative to the working directory, or URLs, such as
+the ones `import.meta.resolve()` gives:
+
+```mjs displayName="doc-kit.config.mjs"
+export default {
+  'jsx-ast': {
+    markdown: {
+      rehypePlugins: [
+        [
+          '@doc-kit/core/plugins/shiki/rehype.mjs',
+          {
+            langAlias: { conf: 'ini' },
+            themes: { light: 'github-light', dark: 'github-dark' },
+            transformers: [import.meta.resolve('./shiki-transformers.mjs')],
+          },
+        ],
+      ],
+    },
+  },
+};
+```
+
 ## Configuration Merging
 
 Configurations are merged in the following order (higher sources take

docs/creating-generators.md

@@ -468,6 +468,64 @@ Use a dependent when a generator transforms an intermediate representation
 format of its own. See the [`section-pages`](./generators/section-pages.md) generator for
 a worked example.
 
+## Markdown pipelines
+
+A generator processing Markdown declares its [unified](https://unifiedjs.com/)
+pipeline as `markdown`, listing plugins as the
+[`markdown` option](./configuration.md#markdown-plugins) does, with paths
+relative to its module. The rehype list starts with `remark-rehype`, and the
+recma list with `rehype-recma`. Each list takes the configured plugins in place
+of its `'...'`, and a list without one takes none. As every thread running the
+generator imports it, options may be anything, functions included.
+
+```mjs displayName="index.mjs"
+export default {
+  name: 'my-generator',
+
+  dependsOn: '@doc-kit/core/metadata',
+
+  markdown: {
+    remarkPlugins: ['remark-parse', 'remark-gfm', '...'],
+    rehypePlugins: [
+      ['remark-rehype', { allowDangerousHtml: true }],
+      '...',
+      ['rehype-stringify', { allowDangerousHtml: true }],
+    ],
+  },
+
+  generate,
+};
+```
+
+`getProcessor(name)` gives the generator's processor, on each thread it runs
+on:
+
+```mjs displayName="generate.mjs"
+import { getProcessor } from '@doc-kit/core/utils/markdown/processor.mjs';
+
+export async function generate(input) {
+  const processor = getProcessor('my-generator');
+
+  return Promise.all(
+    input.map(async ({ content }) =>
+      processor.stringify(await processor.run(content))
+    )
+  );
+}
+```
+
+`getProcessor(name, { mdx: true })` returns an MDX processor (plugins can
+check `this.data('mdx')`), and `getProcessor(name, { configured: false })` one
+without the configured plugins. A generator rendering Markdown (with rehype or
+recma plugins) only takes its own remark plugins, as the global ones already
+ran when `ast` parsed the Markdown.
+
+A plugin needing asynchronous setup can export an async `load(options)`
+returning the plugin, instead of a default export. The Shiki plugin does, and
+`getHighlighter(name)` (from `@doc-kit/core/plugins/shiki/highlighter.mjs`)
+returns the highlighter of a generator's Shiki plugin, to highlight code as its
+pipeline does.
+
 ## File Output
 
 ### Writing Output Files

packages/core/package.json

@@ -39,6 +39,7 @@
       "./src/parsers/*",
       "./src/parsers/*.d.ts"
     ],
+    "#plugins/*": "./src/plugins/*",
     "#utils/*": "./src/utils/*"
   },
   "files": [
@@ -60,7 +61,6 @@
     "dedent": "^1.7.2",
     "github-slugger": "^2.0.0",
     "glob-parent": "^6.0.2",
-    "hastscript": "^9.0.1",
     "mdast-util-slice-markdown": "^2.0.1",
     "mdast-util-to-string": "^4.0.0",
     "piscina": "^5.3.2",

packages/core/src/__tests__/generators.test.mjs

@@ -150,6 +150,17 @@ mock.module('../threading/parallel.mjs', {
   },
 });
 
+// The Markdown pipelines loaded, with the generators that had run by then
+const loadedPipelines = [];
+
+mock.module('../utils/markdown/plugins.mjs', {
+  exports: {
+    loadMarkdownPlugins: async ({ name }, markdown) => {
+      loadedPipelines.push({ name, markdown, ran: Object.keys(runs) });
+    },
+  },
+});
+
 const createGenerator = (await import('../generators.mjs')).default;
 
 describe('createGenerator orchestration', () => {
@@ -215,4 +226,23 @@ describe('createGenerator orchestration', () => {
       { all: { d: [{ meta: 1 }, { spliced: true }] } },
     ]);
   });
+
+  it('loads the Markdown pipeline of each generator as it starts', async () => {
+    const { runGenerators } = createGenerator();
+    const markdown = { remarkPlugins: ['file:///plugin.mjs'] };
+
+    loadedPipelines.length = 0;
+
+    await runGenerators({
+      target: ['gen-a'],
+      threads: 1,
+      'gen-a': { markdown },
+    });
+
+    assert.deepStrictEqual(loadedPipelines, [
+      { name: 'ast', markdown: undefined, ran: [] },
+      { name: 'metadata', markdown: undefined, ran: ['ast'] },
+      { name: 'gen-a', markdown, ran: ['ast', 'metadata'] },
+    ]);
+  });
 });

packages/core/src/generators.mjs

@@ -9,6 +9,7 @@ import { resolvePipeline } from './generators/pipeline.mjs';
 import logger from './logger/index.mjs';
 import createWorkerPool from './threading/index.mjs';
 import createParallelWorker from './threading/parallel.mjs';
+import { loadMarkdownPlugins } from './utils/markdown/plugins.mjs';
 import { isAsyncIterable } from './utils/misc.mjs';
 
 const generatorsLogger = logger.child('generators');
@@ -78,6 +79,9 @@ const createGenerator = () => {
 
         generatorsLogger.debug(`Starting "${name}"`);
 
+        // Load its Markdown pipeline, for what it processes on this thread
+        await loadMarkdownPlugins(generator, configuration[name]?.markdown);
+
         // Create parallel worker for streaming generators
         const worker = hasParallelProcessor
           ? createParallelWorker(specifier, generator, pool, configuration)

packages/core/src/generators/ast/__tests__/generate.test.mjs

@@ -3,10 +3,19 @@ import { mkdtemp, writeFile, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join, sep } from 'node:path';
 import { after, before, describe, it } from 'node:test';
+import { pathToFileURL } from 'node:url';
+
+import { loadGenerator } from '#generators/loader.mjs';
+import { loadMarkdownPlugins } from '#utils/markdown/plugins.mjs';
 
 import { STABILITY_INDEX_URL } from '../constants.mjs';
 import { processChunk } from '../generate.mjs';
 
+const ast = await loadGenerator(import.meta.resolve('../index.mjs'));
+
+// Files are parsed with the pipeline of `ast`
+await loadMarkdownPlugins(ast);
+
 let dir;
 
 const toPosixPath = value => value.split(sep).join('/');
@@ -157,6 +166,42 @@ describe('processChunk', () => {
     });
   });
 
+  describe('remark plugins', () => {
+    before(async () => {
+      // Records the file it runs on, and the headings of its document
+      await writeFile(
+        join(dir, 'recorder.mjs'),
+        `export default () => (tree, file) => {
+          const headings = tree.children.filter(node => node.type === 'heading');
+
+          tree.data = { recorded: file.path + ':' + headings.length };
+        };`
+      );
+
+      await loadMarkdownPlugins(ast, {
+        remarkPlugins: [pathToFileURL(join(dir, 'recorder.mjs')).href],
+      });
+    });
+
+    after(() => loadMarkdownPlugins(ast));
+
+    it('run on each whole document, knowing its file', async () => {
+      const tuple = await file('plugged.md', '# A\n\n## B\n\n## C\n');
+
+      const { tree } = await process(tuple);
+
+      assert.strictEqual(tree.data.recorded, `${tuple[0]}:3`);
+    });
+
+    it('run on MDX documents too', async () => {
+      const tuple = await file('plugged.mdx', '# A\n\n<B />\n');
+
+      const { tree } = await process(tuple);
+
+      assert.strictEqual(tree.data.recorded, `${tuple[0]}:1`);
+    });
+  });
+
   describe('chunking', () => {
     it('only processes the requested indices', async () => {
       const a = await file('a.md', '# A\n');

packages/core/src/generators/ast/generate.mjs

@@ -9,8 +9,8 @@ import { parse as parseYaml } from 'yaml';
 
 import getConfig from '#utils/configuration/index.mjs';
 import { withExt } from '#utils/file.mjs';
+import { getProcessor } from '#utils/markdown/processor.mjs';
 import { QUERIES } from '#utils/queries/index.mjs';
-import { getRemark as remark, getRemarkMdx } from '#utils/remark.mjs';
 
 import { STABILITY_INDEX_URL } from './constants.mjs';
 
@@ -67,6 +67,8 @@ export async function processChunk(inputSlice, itemIndices) {
     // The path is the relative path minus the extension
     const relativePath = sep + withExt(relative(parent, path));
 
+    const processor = getProcessor('ast', { mdx });
+
     let tree;
 
     if (mdx) {
@@ -79,7 +81,7 @@ export async function processChunk(inputSlice, itemIndices) {
         ''
       );
 
-      tree = getRemarkMdx().parse(source);
+      tree = processor.parse(source);
 
       if (frontmatter) {
         tree.children.unshift({
@@ -93,9 +95,11 @@ export async function processChunk(inputSlice, itemIndices) {
         (_, yaml) => `<!-- YAML\n${yaml}\n-->`
       );
 
-      tree = remark().parse(value);
+      tree = processor.parse(value);
     }
 
+    tree = await processor.run(tree, { path });
+
     results.push({ tree, path: relativePath, mdx });
   }
 

packages/core/src/generators/ast/index.mjs

@@ -15,6 +15,15 @@ export default {
 
   hasParallelProcessor: true,
 
+  markdown: {
+    remarkPlugins: [
+      'remark-parse',
+      '#plugins/type-annotations/remark.mjs',
+      'remark-gfm',
+      '...',
+    ],
+  },
+
   generate,
   processChunk,
 };