perf: reduce build memory and time

.changeset/build-memory.md

@@ -0,0 +1,7 @@
+---
+'@doc-kit/core': minor
+'@doc-kit/generator-react': minor
+'@node-core/doc-kit-legacy': minor
+---
+
+Builds take about half the memory and a third of the time (measured on the Node.js API docs): Shiki registers each language once code in it is highlighted, highlighted code reaches pages as static markup rather than a tree per token, `all.html` is rendered by the worker pool, output is no longer minified by default (serve it compressed instead), idle workers end after 250ms, and each worker's heap is limited to 512MB, which Node.js's `--max-old-space-size` raises.

docs/configuration.md

@@ -109,7 +109,7 @@ Everything under the `global` key applies to every generator:
   logos, and templated text. Defaults to the `name` in your `package.json`.
 - `version` {string|SemVer} Documentation version. **Default:**
   `process.version`.
-- `minify` {boolean} Whether to minify output. **Default:** `true`.
+- `minify` {boolean} Whether to minify output. **Default:** `false`.
 - `repository` {string} GitHub repository in `owner/repo` format; without
   one, repository UI is omitted.
 - `ref` {string} Git reference (branch, tag, or commit SHA). **Default:**
@@ -134,7 +134,8 @@ these for that generator alone, except `markdown`, which it adds to.
 Top-level, alongside `target` and `global`:
 
 - `threads` {number} Worker threads used for generation. Defaults to your
-  CPU count.
+  CPU count. Each worker's heap is limited to 512MB; to raise the limit, run
+  Node.js with `--max-old-space-size` (directly or through `NODE_OPTIONS`).
 - `chunkSize` {number} Items processed per worker thread. **Default:** `10`.
 
 ## Generator options

docs/creating-generators.md

@@ -524,7 +524,9 @@ A plugin needing asynchronous setup can export an async `load(options)`
 returning the plugin, instead of a default export. The Shiki plugin does, and
 `getHighlighter(name)` (from `@doc-kit/core/plugins/shiki/highlighter.mjs`)
 returns the highlighter of a generator's Shiki plugin, to highlight code as its
-pipeline does.
+pipeline does. It registers Shiki's bundled languages as code in them is
+highlighted, so before using its `shiki` instance directly, pass the language
+through its `resolveLanguage`.
 
 ## File Output
 

docs/publishing.md

@@ -56,6 +56,22 @@ costs a request per asset each time a new tab opens the site. On Vercel:
 Within a visit, moving between pages loads no assets at all: the site
 navigates client-side, swapping in the next page's content.
 
+## Compress the pages
+
+doc-kit doesn't minify its output by default: compressed (gzip or brotli), as
+hosts serve it, it is far smaller than minifying would make it.
+
+Cloudflare, Vercel, Netlify, and GitHub Pages compress on their own, and so
+does `npx serve out`. If you serve the output yourself, turn compression on:
+
+- **nginx:** `gzip on;` compresses HTML; add the other types with
+  `gzip_types text/css application/javascript application/json;`.
+- **Caddy:** `encode zstd gzip`.
+- **Apache:** with `mod_deflate`,
+  `AddOutputFilterByType DEFLATE text/html text/css application/javascript application/json`.
+- **Amazon S3:** serve the bucket through CloudFront, with **Compress objects
+  automatically** on.
+
 ## Tell doc-kit its public URL
 
 Set `baseURL` to where the site will live. Generators that emit absolute

packages/core/package.json

@@ -52,8 +52,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@actions/core": "^3.0.0",
-    "@node-core/rehype-shiki": "^1.5.0",
+    "@node-core/rehype-shiki": "^1.5.1",
     "@swc/html-wasm": "^1.16.2",
     "@swc/wasm": "^1.15.46",
     "acorn": "^8.17.0",

packages/core/src/generators/json-all/README.md

@@ -23,7 +23,7 @@ generator's schema for the documents.
 
 - `output` {string} The directory where `all.json` will be written.
 - `minify` {boolean} Whether to minify the output. Inherited from `global`.
-  **Default:** `true`.
+  **Default:** `false`.
 - `index` {Array} The `{ api }` objects defining the document order. Inherited
   from `global`.
 - `schemaURL` {string} Where the bundle's schema is published.

packages/core/src/generators/json-simple/README.md

@@ -6,4 +6,4 @@ The `json-simple` generator creates a simplified JSON version of the API documen
 
 - `output` {string} The directory where `api-docs.json` will be written.
 - `minify` {boolean} Whether to minify the output JSON. Inherited from
-  `global`. **Default:** `true`.
+  `global`. **Default:** `false`.

packages/core/src/generators/json/README.md

@@ -18,7 +18,7 @@ Files keep the input's directory layout: `doc/api/some/fs.md` becomes `out/some/
 
 - `output` {string} The directory to write the documents to.
 - `minify` {boolean} Whether to minify the output. Inherited from `global`.
-  **Default:** `true`.
+  **Default:** `false`.
 - `repository` {string} The `owner/name` repository source links resolve
   against. Inherited from `global`; without one, `sourceLink.url` is `null`.
 - `schemaURL` {string} Where the schema is published. `{schemaVersion}` is

packages/core/src/generators/metadata/generate.mjs

@@ -3,15 +3,17 @@
 import getConfig from '#utils/configuration/index.mjs';
 import { loadFromURL } from '#utils/loaders.mjs';
 
-import { parseApiDoc } from './utils/parse.mjs';
-
 /**
  * Process a chunk of API doc files in a worker thread.
  * Called by chunk-worker.mjs for parallel processing.
  *
  * @type {import('./types').Generator['processChunk']}
  */
 export async function processChunk(fullInput, itemIndices, typeMap) {
+  // Loaded on first use rather than with the generator, so the main thread,
+  // which never parses, does not load the TypeScript parser (~40MB of WASM)
+  const { parseApiDoc } = await import('./utils/parse.mjs');
+
   const results = [];
 
   for (const idx of itemIndices) {

packages/core/src/logger/__tests__/transports/github.test.mjs

@@ -148,6 +148,43 @@ describe('github', () => {
     );
   });
 
+  it('should escape messages and files', t => {
+    process.env.FORCE_COLOR = '1';
+
+    t.mock.timers.enable({ apis: ['Date'] });
+
+    const fn = t.mock.method(process.stdout, 'write');
+
+    // noop
+    fn.mock.mockImplementation(() => {});
+
+    github({
+      level: LogLevel.info,
+      message: '100%: a, b\r\nc',
+      metadata: {
+        file: {
+          path: 'a,b:c%.md',
+          position: {
+            start: { line: 1 },
+            end: { line: 2 },
+          },
+        },
+      },
+      timestamp: Date.now(),
+    });
+
+    const callsArgs = process.stdout.write.mock.calls.map(
+      call => call.arguments[0]
+    );
+
+    strictEqual(process.stdout.write.mock.callCount(), 1);
+    // The message keeps its `,` and `:`, which only separate properties
+    strictEqual(
+      callsArgs[0].trim(),
+      '::notice file=a%2Cb%3Ac%25.md,line=1,endLine=2::[00:00:00.000] \x1B[32mINFO\x1B[39m: 100%25: a, b%0D%0Ac'
+    );
+  });
+
   it('should print child logger name', t => {
     process.env.FORCE_COLOR = '1';
 

packages/core/src/logger/constants.mjs

@@ -32,3 +32,38 @@ export const levelToColorMap = {
   [LogLevel.error]: 'magenta',
   [LogLevel.fatal]: 'red',
 };
+
+/**
+ * Maps log level numbers to the GitHub Actions workflow commands reporting
+ * them.
+ */
+export const levelToGitHubCommand = {
+  [LogLevel.debug]: 'debug',
+  [LogLevel.info]: 'notice',
+  [LogLevel.warn]: 'warning',
+  [LogLevel.error]: 'error',
+  [LogLevel.fatal]: 'error',
+};
+
+/**
+ * Maps the characters a workflow command's message can't hold to their
+ * escapes. A command is a line, `::command key=value,key=value::message`,
+ * so its message, which runs to the end of the line, can't break it, nor hold
+ * the `%` an escape starts with.
+ */
+export const commandDataEscapes = {
+  '%': '%25',
+  '\r': '%0D',
+  '\n': '%0A',
+};
+
+/**
+ * Maps the characters a workflow command's property can't hold to their
+ * escapes: those its message can't, and the `,` separating properties and
+ * the `:` of the `::` ending them.
+ */
+export const commandPropertyEscapes = {
+  ...commandDataEscapes,
+  ':': '%3A',
+  ',': '%2C',
+};

packages/core/src/logger/transports/github.mjs

@@ -1,23 +1,35 @@
 'use strict';
 
-import { debug, notice, warning, error } from '@actions/core';
+import { EOL } from 'node:os';
 
-import { LogLevel } from '#logger/constants.mjs';
+import {
+  commandDataEscapes,
+  commandPropertyEscapes,
+  levelToGitHubCommand,
+} from '#logger/constants.mjs';
 import { prettifyLevel } from '#logger/utils/colors.mjs';
 import { prettifyTimestamp } from '#logger/utils/time.mjs';
 
-const actions = {
-  [LogLevel.debug]: debug,
-  [LogLevel.info]: notice,
-  [LogLevel.warn]: warning,
-  [LogLevel.error]: error,
-  [LogLevel.fatal]: error,
-};
+/**
+ * Escapes a value of a workflow command: its message, with
+ * `commandDataEscapes`, or one of its properties, with
+ * `commandPropertyEscapes`.
+ *
+ * @param {string | number} value - The value
+ * @param {Record<string, string>} escapes - The escapes of the characters it can't hold
+ * @returns {string}
+ */
+const escapeCommandValue = (value, escapes) =>
+  String(value).replace(
+    /[%\r\n:,]/g,
+    character => escapes[character] ?? character
+  );
 
 /**
- * Logs messages to GitHub Actions with formatted output and file info with
- * appropriate gh actions based on level.
+ * Logs messages to GitHub Actions as workflow commands, with the file and
+ * lines they're about, for them to annotate.
  *
+ * @see https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands
  * @param {import('../types').TransportContext} context
  * @returns {void}
  */
@@ -30,13 +42,34 @@ const github = ({ level, message, timestamp, metadata = {}, module }) => {
 
   const logMessage = `[${time}] ${prettyLevel}${module ? ` (${module})` : ''}: ${message}`;
 
-  const logFn = actions[level] ?? notice;
+  const command = levelToGitHubCommand[level] ?? 'notice';
+
+  const data = escapeCommandValue(logMessage, commandDataEscapes);
 
-  logFn(logMessage, {
+  // Debug messages take no properties
+  if (command === 'debug') {
+    process.stdout.write(`::debug::${data}${EOL}`);
+
+    return;
+  }
+
+  const properties = {
     file: file?.path,
-    startLine: file?.position?.start.line,
+    line: file?.position?.start.line,
     endLine: file?.position?.end.line,
-  });
+  };
+
+  const annotations = [];
+
+  for (const [key, value] of Object.entries(properties)) {
+    if (value) {
+      annotations.push(
+        `${key}=${escapeCommandValue(value, commandPropertyEscapes)}`
+      );
+    }
+  }
+
+  process.stdout.write(`::${command} ${annotations.join(',')}::${data}${EOL}`);
 };
 
 export default github;

packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs

@@ -86,6 +86,73 @@ describe('createHighlighter', () => {
     );
   });
 
+  it('registers a bundled language once code in it is highlighted', async () => {
+    const highlighter = await createHighlighter({
+      langAlias: { py: 'python' },
+    });
+    const loaded = () => highlighter.shiki.getLoadedLanguages();
+
+    assert.ok(!loaded().includes('python'));
+    assert.ok(!loaded().includes('javascript'));
+
+    // By its name, an alias of its own, or one of the options
+    assert.equal(highlighter.resolveLanguage('py'), 'py');
+    assert.equal(highlighter.resolveLanguage('mjs'), 'mjs');
+
+    assert.ok(loaded().includes('python'));
+    assert.ok(loaded().includes('javascript'));
+    assert.ok(loaded().includes('cjs'));
+
+    assert.equal(highlighter.resolveLanguage(undefined), 'text');
+    assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext');
+  });
+
+  it('lists the bundled languages without registering them', async () => {
+    const highlighter = await createHighlighter({ langs: [grammar] });
+
+    assert.deepStrictEqual(
+      highlighter.langs.find(({ name }) => name === 'rust'),
+      { name: 'rust', displayName: 'Rust', aliases: ['rs'] }
+    );
+    assert.equal(highlighter.langs.at(-1), grammar);
+    assert.ok(!highlighter.shiki.getLoadedLanguages().includes('rust'));
+  });
+
+  it('registers the bundled languages a language embeds', async () => {
+    const highlighter = await createHighlighter({
+      langs: [
+        {
+          ...grammar,
+          name: 'oxcscript',
+          scopeName: 'source.oxcscript',
+          embeddedLangs: ['javascript'],
+          patterns: [{ include: 'source.js' }],
+        },
+      ],
+    });
+
+    assert.match(
+      highlighter.highlightToHtml('const on = 1', 'oxcscript'),
+      /--shiki-dark/
+    );
+  });
+
+  it('registers the bundled languages a language embeds lazily', async () => {
+    const highlighter = await createHighlighter({
+      langs: [
+        {
+          ...grammar,
+          name: 'oxcdoc',
+          scopeName: 'source.oxcdoc',
+          embeddedLangsLazy: ['rust'],
+          patterns: [{ include: 'source.rust' }],
+        },
+      ],
+    });
+
+    assert.ok(highlighter.shiki.getLoadedLanguages().includes('rust'));
+  });
+
   it('gives the same highlighter for the same options', async () => {
     const highlighter = await createHighlighter({ langs: [grammar] });