perf: highlight with fewer grammars by default

.changeset/highlight-fewer-grammars.md

@@ -0,0 +1,5 @@
+---
+'@doc-kit/core': minor
+---
+
+Code is highlighted with fewer grammars by default, as each one takes its own memory on every thread: JavaScript (`js`, `mjs`, `cjs`, `javascript`) with TypeScript's grammar, which already highlights types, and C++ (`cpp`, `c++`) with C's. The Shiki plugin's `langAlias` option takes precedence over these defaults, so `{ cpp: 'cpp', 'c++': 'cpp' }` brings back C++'s own grammar, and aliases now keep working once a grammar with the same aliases is registered. A highlighter's `resolveLanguage` now returns the language code is highlighted as, as its Shiki instance names it (`typescript` for `mjs`): highlight with what it returns. Transformers see that language too.

docs/configuration.md

@@ -219,12 +219,20 @@ bundles. To configure it, list it with options in
 dependencies, so your configuration file can resolve it):
 
 - `langs` {Array} More languages: grammars, or modules default-exporting them.
-- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`.
+- `langAlias` {Object} Aliases of languages, such as `{ conf: 'ini' }`. An
+  alias leads to the language it names, or on to the one that language is an
+  alias of. By default, JavaScript (`javascript`, `js`, `mjs`, `cjs`) is
+  highlighted with TypeScript's grammar, and C++ (`cpp`, `c++`) with C's:
+  every grammar takes its own memory on each thread, and those highlight that
+  code too. Aliases take precedence over these defaults:
+  `{ cpp: 'cpp', 'c++': 'cpp' }` highlights C++ with its own grammar, and a
+  grammar in `langs` for one of these languages needs an alias to it.
 - `themes` {Object} The `light` and `dark` themes: names of themes Shiki
   bundles, modules default-exporting themes, or themes.
 - `transformers` {Array} Modules default-exporting
   [Shiki transformers](https://shiki.style/guide/transformers), or lists of
-  them.
+  them. They see the language code is highlighted as (`options.lang`), such as
+  `typescript` for `mjs`.
 
 These modules are paths relative to the working directory, or URLs, such as
 the ones `import.meta.resolve()` gives:

docs/creating-generators.md

@@ -526,7 +526,9 @@ returning the plugin, instead of a default export. The Shiki plugin does, and
 returns the highlighter of a generator's Shiki plugin, to highlight code as its
 pipeline does. It registers Shiki's bundled languages as code in them is
 highlighted, so before using its `shiki` instance directly, pass the language
-through its `resolveLanguage`.
+through its `resolveLanguage`, and highlight with the language it returns,
+which may not be the one the code names (`mjs` gives `typescript`, and an
+unknown language `text`).
 
 ## File Output
 

packages/core/src/plugins/shiki/__tests__/highlighter.test.mjs

@@ -92,21 +92,106 @@ describe('createHighlighter', () => {
     });
     const loaded = () => highlighter.shiki.getLoadedLanguages();
 
+    // Rust stands for any bundled language nothing aliases: JavaScript's
+    // aliases now lead to TypeScript
     assert.ok(!loaded().includes('python'));
-    assert.ok(!loaded().includes('javascript'));
+    assert.ok(!loaded().includes('rust'));
 
     // By its name, an alias of its own, or one of the options
-    assert.equal(highlighter.resolveLanguage('py'), 'py');
-    assert.equal(highlighter.resolveLanguage('mjs'), 'mjs');
+    assert.equal(highlighter.resolveLanguage('py'), 'python');
+    assert.equal(highlighter.resolveLanguage('rs'), 'rs');
 
     assert.ok(loaded().includes('python'));
-    assert.ok(loaded().includes('javascript'));
-    assert.ok(loaded().includes('cjs'));
+    assert.ok(loaded().includes('rust'));
 
     assert.equal(highlighter.resolveLanguage(undefined), 'text');
     assert.equal(highlighter.resolveLanguage('plaintext'), 'plaintext');
   });
 
+  it('highlights JavaScript as TypeScript, and C++ as C', async () => {
+    const highlighter = await createHighlighter();
+    const loaded = () => highlighter.shiki.getLoadedLanguages();
+
+    for (const language of ['javascript', 'JS', 'mjs', 'cjs']) {
+      assert.equal(highlighter.resolveLanguage(language), 'typescript');
+    }
+
+    assert.equal(highlighter.resolveLanguage('cpp'), 'c');
+    assert.equal(highlighter.resolveLanguage('c++'), 'c');
+    assert.ok(!loaded().includes('cpp'));
+
+    // Even once a grammar that has them as its own aliases is registered
+    assert.equal(highlighter.resolveLanguage('coffee'), 'coffee');
+    assert.ok(loaded().includes('javascript'));
+    assert.equal(highlighter.resolveLanguage('js'), 'typescript');
+
+    // Not the names every object inherits
+    assert.equal(highlighter.resolveLanguage('constructor'), 'text');
+  });
+
+  it('takes aliases over the default ones', async () => {
+    const highlighter = await createHighlighter({
+      langAlias: { cpp: 'cpp', js: 'javascript' },
+    });
+
+    assert.equal(highlighter.resolveLanguage('cpp'), 'cpp');
+    assert.equal(highlighter.resolveLanguage('js'), 'javascript');
+    assert.equal(highlighter.resolveLanguage('mjs'), 'typescript');
+    assert.ok(highlighter.shiki.getLoadedLanguages().includes('cpp'));
+  });
+
+  it('follows configured aliases, and finds languages named in any case', async () => {
+    const highlighter = await createHighlighter({
+      langs: [{ ...grammar, name: 'OxcConf', scopeName: 'source.oxcconf2' }],
+      langAlias: {
+        a: 'b',
+        b: 'python',
+        loop: 'pool',
+        pool: 'loop',
+        Up: 'DOWN',
+        down: 'up',
+        oc: 'OxcConf',
+        conf: 'INI',
+        MyConf: 'ini',
+      },
+    });
+
+    assert.equal(highlighter.resolveLanguage('a'), 'python');
+    assert.equal(highlighter.resolveLanguage('loop'), 'text');
+    assert.equal(highlighter.resolveLanguage('UP'), 'text');
+    assert.equal(highlighter.resolveLanguage('Python'), 'python');
+    assert.equal(highlighter.resolveLanguage('oc'), 'OxcConf');
+    assert.equal(highlighter.resolveLanguage('conf'), 'ini');
+    assert.equal(highlighter.resolveLanguage('myconf'), 'ini');
+    assert.equal(highlighter.resolveLanguage('MYCONF'), 'ini');
+    assert.equal(
+      highlighter.highlightToHtml('[section]', 'conf'),
+      highlighter.highlightToHtml('[section]', 'ini')
+    );
+  });
+
+  it('highlights with a language of `langs` an alias leads to', async () => {
+    const highlighter = await createHighlighter({
+      langs: [
+        {
+          ...grammar,
+          name: 'javascript',
+          aliases: ['js'],
+          scopeName: 'source.oxc-js',
+        },
+      ],
+      langAlias: { js: 'javascript' },
+    });
+
+    assert.equal(highlighter.resolveLanguage('js'), 'javascript');
+    assert.match(
+      highlighter.highlightToHtml('minify on', 'js'),
+      /--shiki-dark/
+    );
+    assert.ok(!highlighter.shiki.getLoadedLanguages().includes('typescript'));
+    assert.equal(highlighter.resolveLanguage('mjs'), 'typescript');
+  });
+
   it('lists the bundled languages without registering them', async () => {
     const highlighter = await createHighlighter({ langs: [grammar] });
 

packages/core/src/plugins/shiki/constants.mjs

@@ -0,0 +1,17 @@
+'use strict';
+
+// What code in an unknown language is highlighted as
+export const FALLBACK_LANGUAGE = 'text';
+
+// What code in these languages is highlighted as, unless configured otherwise.
+// Every grammar takes its own memory on each thread, and time to register, and
+// TypeScript's highlights JavaScript (types are highlighted as TypeScript
+// anyway), as C's does most of C++
+export const DEFAULT_ALIASES = new Map([
+  ['javascript', 'typescript'],
+  ['js', 'typescript'],
+  ['mjs', 'typescript'],
+  ['cjs', 'typescript'],
+  ['cpp', 'c'],
+  ['c++', 'c'],
+]);

packages/core/src/plugins/shiki/highlighter.mjs

@@ -12,6 +12,8 @@ import { importFromURL } from '#utils/loaders.mjs';
 import { getMarkdownPlugins } from '#utils/markdown/plugins.mjs';
 import { lazy } from '#utils/misc.mjs';
 
+import { DEFAULT_ALIASES, FALLBACK_LANGUAGE } from './constants.mjs';
+
 const require = createRequire(import.meta.url);
 
 /**
@@ -25,7 +27,7 @@ const require = createRequire(import.meta.url);
  *
  * @typedef {Object} SyntaxHighlighter
  * @property {import('shiki').HighlighterCore} shiki - The Shiki instance
- * @property {(languageId?: string) => string} resolveLanguage - Resolves a language, falling back to plain text for unknown ones
+ * @property {(languageId?: string) => string} resolveLanguage - Resolves the language code is highlighted as, falling back to plain text for unknown ones
  * @property {(code: string, lang: string, meta?: Record<string, unknown>) => string} highlightToHtml - Highlights code, returning the inner HTML of its `<code>` element
  * @property {(code: string, lang: string, meta?: Record<string, unknown>) => ReturnType<import('shiki').HighlighterCore['codeToHast']>} highlightToHast - Highlights code, returning a HAST tree
  * @property {Array<Language>} langs - The languages it highlights
@@ -64,9 +66,6 @@ let engine;
 // The highlighters of the options given, by their JSON
 const highlighters = new Map();
 
-// What code in an unknown language is highlighted as
-const FALLBACK_LANGUAGE = 'text';
-
 // The languages Shiki bundles. Registering a grammar takes time and memory,
 // and slows down every highlight after it, so each one is only imported and
 // registered once code in it is highlighted (see `resolveLanguage`).
@@ -220,10 +219,14 @@ const importHighlighter = async ({
     engine: regexEngine,
     // The bundled languages are registered as code uses them
     langs: withEmbeddedLanguages(importedLangs),
-    // A copy, as Shiki adds the aliases of the languages it registers to it
-    langAlias: { ...langAlias },
   };
 
+  // Resolved here, as Shiki replaces an alias with a grammar's own as it
+  // registers. Keyed in lowercase, as they're matched in any case
+  const aliases = new Map(
+    Object.entries(langAlias).map(([from, to]) => [from.toLowerCase(), to])
+  );
+
   // Without themes of its own, the highlighter has a default light and dark one
   if (themes) {
     const [light, dark] = await Promise.all([
@@ -256,8 +259,9 @@ const importHighlighter = async ({
   });
 
   /**
-   * Resolves a language, falling back to plain text for unknown ones. A
-   * bundled language is registered the first time it's resolved.
+   * Resolves the language code is highlighted as, through the aliases, falling
+   * back to plain text for unknown ones. A bundled language is registered the
+   * first time it's resolved.
    *
    * @param {string} [languageId]
    * @returns {string}
@@ -267,20 +271,41 @@ const importHighlighter = async ({
       return FALLBACK_LANGUAGE;
     }
 
+    const followed = new Set();
+
+    let language = languageId;
+    let key = language.toLowerCase();
+
+    // Each configured alias once, so circular ones end
+    while (aliases.has(key) && !followed.has(key)) {
+      followed.add(key);
+      language = aliases.get(key);
+      key = language.toLowerCase();
+    }
+
+    if (followed.size === 0) {
+      language = DEFAULT_ALIASES.get(language.toLowerCase()) ?? language;
+    }
+
     const instance = load().shiki;
-    const name = instance.resolveLangAlias(languageId.toLowerCase());
+    const loaded = instance.getLoadedLanguages();
 
-    if (isSpecialLang(name) || instance.getLoadedLanguages().includes(name)) {
-      return languageId;
+    // Shiki has bundled languages' names and aliases in lowercase
+    for (const name of [language, language.toLowerCase()]) {
+      if (isSpecialLang(name) || loaded.includes(name)) {
+        return name;
+      }
     }
 
+    const name = language.toLowerCase();
+
     if (!BUNDLED_NAMES.has(name)) {
       return FALLBACK_LANGUAGE;
     }
 
     instance.loadLanguageSync(importBundledLanguage(name));
 
-    return languageId;
+    return name;
   };
 
   /**