perf: reduce build memory and time (PoC)

.changeset/build-memory.md

@@ -0,0 +1,8 @@
+---
+'@doc-kit/cli': minor
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+'@node-core/doc-kit-legacy': patch
+---
+
+Build with about a third less memory, in less than half the time: Shiki registers each language once code in it is highlighted, highlighted code reaches pages as static markup rather than a tree per token, the default Vite bundler runs in a child process (`createChildProcess`), `all.html` is rendered by the worker pool and no longer minified, and each worker's heap is limited to just under 2GB by default (the new `workerHeapSize` option, or `--worker-heap-size`)

docs/cli.md

@@ -39,4 +39,6 @@ Runs the generators and writes their output. Requires a `target` and an
 - `--type-map <url>` {string} Type map URL or path (custom type-name → URL
   links).
 - `-p, --threads <n>` {number} Worker threads to use (minimum 1).
+- `--worker-heap-size <mb>` {number} Heap size limit of each worker thread, in
+  MB (minimum 1).
 - `--chunk-size <n>` {number} Items per worker thread (minimum 1).

docs/configuration.md

@@ -135,6 +135,10 @@ Top-level, alongside `target` and `global`:
 
 - `threads` {number} Worker threads used for generation. Defaults to your
   CPU count.
+- `workerHeapSize` {number} Heap size limit of each worker thread (V8's old
+  space), in MB. Defaults to V8's own limit, at most `2047`: from 2GB up, V8
+  lets a heap grow to four times its live data before collecting it, and only
+  to about twice below that.
 - `chunkSize` {number} Items processed per worker thread. **Default:** `10`.
 
 ## Generator options
@@ -196,6 +200,7 @@ own.
 - `rehypePlugins` {Array} Run on the HTML of the generators rendering
   Markdown, such as `jsx-ast`, before code is highlighted.
 - `recmaPlugins` {Array} Run on the JavaScript `jsx-ast` compiles the pages to.
+  Highlighted code is in it as the markup it renders to, rather than as JSX.
 
 A generator only takes the plugins its pipeline has a place for: `jsx-ast`
 takes all three kinds, `ast`, `metadata`, and `json` take remark plugins, and
@@ -261,17 +266,18 @@ precedence):
 
 CLI options map to configuration properties:
 
-| CLI Option             | Config Property    | Example                   |
-| ---------------------- | ------------------ | ------------------------- |
-| `--input <path>`       | `global.input`     | `--input src/`            |
-| `--output <path>`      | `global.output`    | `--output dist/`          |
-| `--ignore <pattern>`   | `global.ignore[]`  | `--ignore test/`          |
-| `--minify`             | `global.minify`    | `--minify`                |
-| `--git-ref <ref>`      | `global.ref`       | `--git-ref v20.0.0`       |
-| `--version <version>`  | `global.version`   | `--version 20.0.0`        |
-| `--changelog <url>`    | `global.changelog` | `--changelog https://...` |
-| `--index <url>`        | `global.index`     | `--index file://...`      |
-| `--type-map <map>`     | `metadata.typeMap` | `--type-map file://...`   |
-| `--target <generator>` | `target`           | `--target json`           |
-| `--threads <n>`        | `threads`          | `--threads 4`             |
-| `--chunk-size <n>`     | `chunkSize`        | `--chunk-size 10`         |
+| CLI Option                | Config Property    | Example                   |
+| ------------------------- | ------------------ | ------------------------- |
+| `--input <path>`          | `global.input`     | `--input src/`            |
+| `--output <path>`         | `global.output`    | `--output dist/`          |
+| `--ignore <pattern>`      | `global.ignore[]`  | `--ignore test/`          |
+| `--minify`                | `global.minify`    | `--minify`                |
+| `--git-ref <ref>`         | `global.ref`       | `--git-ref v20.0.0`       |
+| `--version <version>`     | `global.version`   | `--version 20.0.0`        |
+| `--changelog <url>`       | `global.changelog` | `--changelog https://...` |
+| `--index <url>`           | `global.index`     | `--index file://...`      |
+| `--type-map <map>`        | `metadata.typeMap` | `--type-map file://...`   |
+| `--target <generator>`    | `target`           | `--target json`           |
+| `--threads <n>`           | `threads`          | `--threads 4`             |
+| `--worker-heap-size <mb>` | `workerHeapSize`   | `--worker-heap-size 1024` |
+| `--chunk-size <n>`        | `chunkSize`        | `--chunk-size 10`         |

packages/cli/bin/commands/generate.mjs

@@ -18,6 +18,7 @@ const { runGenerators } = createGenerator();
  * @property {string[]} ignore
  * @property {string} output
  * @property {number} threads
+ * @property {number} workerHeapSize
  * @property {number} chunkSize
  * @property {string} version
  * @property {string} changelog
@@ -53,6 +54,12 @@ export default new Command('generate')
       'Number of threads to use (minimum: 1)'
     )
   )
+  .addOption(
+    new Option(
+      '--worker-heap-size <mb>',
+      'Heap size limit of each worker thread, in MB (minimum: 1)'
+    )
+  )
   .addOption(
     new Option(
       '--chunk-size <number>',

packages/core/package.json

@@ -57,10 +57,12 @@
     "@swc/html-wasm": "^1.16.2",
     "@swc/wasm": "^1.15.46",
     "acorn": "^8.17.0",
+    "birpc": "^4.2.0",
     "cosmiconfig": "^9.0.2",
     "dedent": "^1.7.2",
     "github-slugger": "^2.0.0",
     "glob-parent": "^6.0.2",
+    "hast-util-to-string": "^3.0.1",
     "mdast-util-slice-markdown": "^2.0.1",
     "mdast-util-to-string": "^4.0.0",
     "piscina": "^5.3.2",
@@ -84,8 +86,7 @@
   },
   "devDependencies": {
     "ajv": "^8.20.0",
-    "hast-util-to-html": "^9.0.5",
-    "hast-util-to-string": "^3.0.1"
+    "hast-util-to-html": "^9.0.5"
   },
   "peerDependencies": {
     "@doc-kit/generator-react": "workspace:>=0.1.0",

packages/core/src/generators.mjs

@@ -107,7 +107,7 @@ const createGenerator = () => {
    * @returns {Promise<unknown[]>} Results of all requested generators
    */
   const runGenerators = async configuration => {
-    const { target, threads } = configuration;
+    const { target, threads, workerHeapSize } = configuration;
 
     // Resolve shorthand names and load the full dependency closure up front,
     // so scheduling below is fully synchronous.
@@ -132,7 +132,7 @@ const createGenerator = () => {
     cache.populateConsumerCounts(targets, specifier => inputOf.get(specifier));
 
     // Create worker pool
-    pool = createWorkerPool(threads);
+    pool = createWorkerPool(threads, workerHeapSize);
 
     // Schedule all generators
     for (const specifier of targets) {

packages/core/src/generators/metadata/generate.mjs

@@ -3,15 +3,17 @@
 import getConfig from '#utils/configuration/index.mjs';
 import { loadFromURL } from '#utils/loaders.mjs';
 
-import { parseApiDoc } from './utils/parse.mjs';
-
 /**
  * Process a chunk of API doc files in a worker thread.
  * Called by chunk-worker.mjs for parallel processing.
  *
  * @type {import('./types').Generator['processChunk']}
  */
 export async function processChunk(fullInput, itemIndices, typeMap) {
+  // Loaded on first use rather than with the generator, so the main thread,
+  // which never parses, does not load the TypeScript parser (~40MB of WASM)
+  const { parseApiDoc } = await import('./utils/parse.mjs');
+
   const results = [];
 
   for (const idx of itemIndices) {

packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs

@@ -86,6 +86,57 @@ describe('createHighlighter', () => {
     );
   });
 
+  it('registers a bundled language once code in it is highlighted', async () => {
+    const highlighter = await createHighlighter({
+      langAlias: { py: 'python' },
+    });
+    const loaded = () => highlighter.shiki.getLoadedLanguages();
+
+    assert.ok(!loaded().includes('python'));
+    assert.ok(!loaded().includes('javascript'));
+
+    // By its name, an alias of its own, or one of the options
+    assert.equal(highlighter.resolveLanguage('py'), 'py');
+    assert.equal(highlighter.resolveLanguage('mjs'), 'mjs');
+
+    assert.ok(loaded().includes('python'));
+    assert.ok(loaded().includes('javascript'));
+    assert.ok(loaded().includes('cjs'));
+
+    assert.equal(highlighter.resolveLanguage(undefined), 'text');
+    assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext');
+  });
+
+  it('lists the bundled languages without registering them', async () => {
+    const highlighter = await createHighlighter({ langs: [grammar] });
+
+    assert.deepStrictEqual(
+      highlighter.langs.find(({ name }) => name === 'rust'),
+      { name: 'rust', displayName: 'Rust', aliases: ['rs'] }
+    );
+    assert.equal(highlighter.langs.at(-1), grammar);
+    assert.ok(!highlighter.shiki.getLoadedLanguages().includes('rust'));
+  });
+
+  it('registers the bundled languages a language embeds', async () => {
+    const highlighter = await createHighlighter({
+      langs: [
+        {
+          ...grammar,
+          name: 'oxcscript',
+          scopeName: 'source.oxcscript',
+          embeddedLangs: ['javascript'],
+          patterns: [{ include: 'source.js' }],
+        },
+      ],
+    });
+
+    assert.match(
+      highlighter.highlightToHtml('const on = 1', 'oxcscript'),
+      /--shiki-dark/
+    );
+  });
+
   it('gives the same highlighter for the same options', async () => {
     const highlighter = await createHighlighter({ langs: [grammar] });
 

packages/core/src/plugins/shiki/__tests__/rehype.test.mjs

@@ -6,28 +6,47 @@ import { toHtml } from 'hast-util-to-html';
 import { createHighlighter } from '../highlighter.mjs';
 import { load } from '../rehype.mjs';
 
+/**
+ * A code block as `remark-rehype` makes it.
+ *
+ * @param {string} code
+ * @param {string} [language]
+ * @param {string} [meta]
+ */
+const codeBlock = (code, language, meta) => ({
+  type: 'element',
+  tagName: 'pre',
+  properties: {},
+  children: [
+    {
+      type: 'element',
+      tagName: 'code',
+      properties: { className: language ? [`language-${language}`] : [] },
+      children: [{ type: 'text', value: code }],
+      data: { meta },
+    },
+  ],
+});
+
+/**
+ * Highlights a tree of the given nodes with the default plugin.
+ *
+ * @param {Array<import('hast').ElementContent>} children
+ */
+const highlight = async children => {
+  const shiki = await load();
+  const tree = { type: 'root', children };
+
+  shiki()(tree);
+
+  return tree;
+};
+
 describe('load', () => {
   it('gives the plugin highlighting code blocks, with its highlighter', async () => {
     const shiki = await load({ langAlias: { conf: 'ini' } });
 
-    const tree = {
-      type: 'root',
-      children: [
-        {
-          type: 'element',
-          tagName: 'pre',
-          properties: {},
-          children: [
-            {
-              type: 'element',
-              tagName: 'code',
-              properties: { className: ['language-conf'] },
-              children: [{ type: 'text', value: '[section]' }],
-            },
-          ],
-        },
-      ],
-    };
+    const tree = { type: 'root', children: [codeBlock('[section]', 'conf')] };
 
     shiki()(tree);
 
@@ -37,4 +56,42 @@ describe('load', () => {
       await createHighlighter({ langAlias: { conf: 'ini' } })
     );
   });
+
+  it('adds the language and layout classes to a highlighted block', async () => {
+    const short = await highlight([codeBlock('a();\n', 'js')]);
+    const long = await highlight([codeBlock('1\n2\n3\n4\n5', 'js')]);
+
+    assert.match(
+      short.children[0].properties.class,
+      /^shiki .* language-js no-line-numbers no-footer$/
+    );
+    assert.match(long.children[0].properties.class, / language-js$/);
+  });
+
+  it('leaves a code block without a language as it is', async () => {
+    const tree = await highlight([codeBlock('plain\n')]);
+
+    assert.deepStrictEqual(tree.children, [codeBlock('plain\n')]);
+  });
+
+  it('groups adjacent code blocks into tabs', async () => {
+    const {
+      children: [tabs],
+    } = await highlight([
+      codeBlock('a();\n', 'mjs', 'displayName="ESM"'),
+      { type: 'text', value: '\n' },
+      codeBlock('b();\n', 'cjs', 'displayName="CJS" active="true"'),
+    ]);
+
+    assert.equal(tabs.tagName, 'CodeTabs');
+    assert.deepStrictEqual(tabs.properties, {
+      languages: 'mjs|cjs',
+      displayNames: 'ESM|CJS',
+      defaultTab: '1',
+    });
+    assert.deepStrictEqual(
+      tabs.children.map(({ properties }) => properties.class.split(' ').at(-3)),
+      ['language-mjs', 'language-cjs']
+    );
+  });
 });