diff --git a/README.md b/README.md index 2ab3cc5..124920b 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,13 @@ Do you miss an important language or alias? Contributions are welcome! | `operator` | `===`, `=>`, `&&` | | `tag` | `
`, `` | | `attribute` | `class=`, `href=` | +| `punctuation` | `{`, `;`, `>` | | `property` | `color:`, `margin:` | | `selector` | `.container`, `#app` | | `builtin` | `console`, `Promise`, `Math` | +| `type` | `string`, `Record` (TypeScript) | +| `doctype` | `` | +| `entity` | `&`, `©` | | `variable` | `$HOME`, `${name}` | | `command` | `npm`, `git`, `ssh` | | `flag` | `--port`, `-v` | diff --git a/package-lock.json b/package-lock.json index e319b2a..2186d3b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "syntaxp", - "version": "2.4.0", + "version": "2.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "syntaxp", - "version": "2.4.0", + "version": "2.4.1", "license": "MIT", "devDependencies": { "esbuild": "^0.28.2" diff --git a/package.json b/package.json index 7d4fb46..679d3af 100644 --- a/package.json +++ b/package.json @@ -13,5 +13,5 @@ "pretest": "npm run build", "test": "node --test" }, - "version": "2.4.0" + "version": "2.4.1" } \ No newline at end of file diff --git a/syntaxp.css b/syntaxp.css index bdab0aa..4ba0711 100644 --- a/syntaxp.css +++ b/syntaxp.css @@ -29,7 +29,9 @@ --s5p-operator: #06708a; + /* #c22068 */ --s5p-builtin: #c22068; + --s5p-type: #c22068; --s5p-variable: #1f2937; @@ -71,7 +73,9 @@ --s5p-operator: #89dceb; + /* #f5c2e7 */ --s5p-builtin: #f5c2e7; + --s5p-type: #f5c2e7; --s5p-variable: #cdd6f4; @@ -132,6 +136,10 @@ color: var(--s5p-builtin); } +::highlight(type) { + color: var(--s5p-type); +} + ::highlight(doctype) { color: var(--s5p-doctype); } diff --git a/syntaxp.js b/syntaxp.js index af299c2..ef46197 100644 --- a/syntaxp.js +++ b/syntaxp.js @@ -61,9 +61,6 @@ // likely to be wrong as right. Pages in that position still need an // explicit `data-theme`. function detectPageTheme() { - if (typeof getComputedStyle !== 'function' || !document.documentElement) { - return null; - } const colorScheme = getComputedStyle(document.documentElement).colorScheme; return (colorScheme === 'light' || colorScheme === 'dark') ? colorScheme : null; } @@ -86,7 +83,9 @@ document.head.appendChild(style); } - // Token patterns per language + // Token patterns per language; a lookbehind that opens a pattern gets a + // lookahead for the token’s first character in front of it, so it doesn’t + // rescan whitespace at every position (quadratic in older V8) const languages = { html: [ @@ -105,23 +104,32 @@ { type: 'string', regex: /`[^`]+`/g }, { type: 'attribute', regex: /\*\*[^*]+\*\*|__[^_]+__/g }, { type: 'entity', regex: /(? !context.beforeBrace[end] }, { type: 'function', regex: /(?:rgba?|hsla?|linear-gradient|radial-gradient|calc|var|env|clamp|min|max|url)\s*(?=\()/g }, { type: 'punctuation', regex: /[{}();:,]/g } ], @@ -131,9 +139,9 @@ { type: 'string', regex: /`[\s\S]*?`|"[^"]*"|'[^']*'/g }, { type: 'keyword', regex: /\b(const|let|var|function|return|if|else|for|while|do|switch|case|break|continue|new|delete|typeof|instanceof|in|of|class|extends|super|this|import|from|export|default|async|await|try|catch|finally|throw|yield|static|get|set)\b/g }, { type: 'number', regex: /\b\d+(\.\d+)?([eE][+-]?\d+)?\b/g }, - { type: 'function', regex: /\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\s*\()/g }, { type: 'builtin', regex: /\b(console|document|window|Math|Array|Object|String|Number|Boolean|Promise|Map|Set|RegExp|Error|JSON|Proxy|Reflect|Symbol|WeakMap|WeakSet|Intl|URL|URLSearchParams|FormData|Headers|Request|Response|fetch|setTimeout|setInterval|clearTimeout|clearInterval|parseInt|parseFloat|isNaN|isFinite|encodeURI|decodeURI|encodeURIComponent|decodeURIComponent|alert|confirm|prompt|requestAnimationFrame|cancelAnimationFrame|structuredClone)\b/g }, - { type: 'operator', regex: /[=!<>]=?=?|&&|\|\||[+\-*/%]=?|\?\?|\.\.\.|=>/g }, + { type: 'function', regex: /\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\s*\()/g }, + { type: 'operator', regex: /=>|[=!<>]=?=?|&&|\|\||[+\-*/%]=?|\?\?|\.\.\./g }, { type: 'punctuation', regex: /[{}[\]();:,.]/g } ], @@ -149,18 +157,18 @@ // example this tokenizer was asked to score during language detection { type: 'function', regex: /\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\s*\(|<)/g }, { type: 'type', regex: /\b(string|number|boolean|bigint|symbol|object|Array|Promise|Record|Partial|Required|Pick|Omit|Exclude|Extract|NonNullable|ReturnType|Parameters|ConstructorParameters|InstanceType)\b/g }, - { type: 'operator', regex: /[=!<>]=?=?|&&|\|\||[+\-*/%]=?|\?\?|\.\.\.|=>/g }, + { type: 'operator', regex: /=>|[=!<>]=?=?|&&|\|\||[+\-*/%]=?|\?\?|\.\.\./g }, { type: 'punctuation', regex: /[{}[\]();:,.]/g } ], python: [ { type: 'comment', regex: /#.*$/gm }, { type: 'string', regex: /(?:[rRbBfFuU]{1,2})?("""[\s\S]*?"""|'''[\s\S]*?''')|(?:[rRbBfFuU]{1,2})?"(?:[^"\\]|\\.)*"|(?:[rRbBfFuU]{1,2})?'(?:[^'\\]|\\.)*'/g }, - { type: 'keyword', regex: /^\s*@[a-zA-Z_][a-zA-Z0-9_.]*|\b(?:False|None|True|and|as|assert|async|await|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield)\b/gm }, + { type: 'keyword', regex: /^[ \t]*@[a-zA-Z_][a-zA-Z0-9_.]*|\b(?:False|None|True|and|as|assert|async|await|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield)\b/gm }, { type: 'builtin', regex: /\b(?:print|len|range|str|int|float|bool|list|dict|set|tuple|type|isinstance|super|self|enumerate|zip|map|filter|sorted|reversed|open|input|format|repr|iter|next|abs|min|max|sum|any|all|Exception|ValueError|TypeError|KeyError|IndexError|StopIteration)\b/g }, { type: 'number', regex: /\b\d+(\.\d+)?([eE][+-]?\d+)?[jJ]?\b/g }, { type: 'function', regex: /\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, - { type: 'operator', regex: /<<=|>>=|<<|>>|[=!<>]=?|\*\*=?|\/\/=?|[+\-%]=?|[*/&|^]=?|~|->|:=/g }, + { type: 'operator', regex: /->|<<=|>>=|<<|>>|[=!<>]=?|\*\*=?|\/\/=?|[+\-%]=?|[*/&|^]=?|~|:=/g }, { type: 'punctuation', regex: /[{}[\]();,:.]/g } ], @@ -171,7 +179,7 @@ { type: 'variable', regex: /\$[a-zA-Z_][a-zA-Z0-9_]*/g }, { type: 'number', regex: /\b\d+(\.\d+)?\b/g }, { type: 'function', regex: /\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, - { type: 'operator', regex: /[=!<>]=?=?|&&|\|\||[+\-*/%.]=?|->|=>|::|\?\?/g }, + { type: 'operator', regex: /->|=>|[=!<>]=?=?|&&|\|\||[+\-*/%.]=?|::|\?\?/g }, { type: 'punctuation', regex: /[{}[\]();,]/g } ], @@ -181,7 +189,7 @@ { type: 'variable', regex: /\$[{(]?[a-zA-Z_][a-zA-Z0-9_]*[})]?/g }, { type: 'flag', regex: /(?<=^|\s)--?[a-zA-Z-]+/g }, { type: 'path', regex: /(?:\.{0,2}\/)[^\s;|&<>"]+/g }, - { type: 'command', regex: /^(?:sudo|cd|ls|cat|grep|find|mkdir|rm|cp|mv|chmod|chown|ssh|scp|rsync|npm|npx|node|git|docker|docker-compose|curl|wget|tar|zip|unzip|make|cmake|brew|apt|yum|dnf|pacman|kill|ps|top|htop|df|du|mount|umount|echo|export|source|alias|unalias|which|whereis|man|tail|head|less|more|sort|uniq|wc|awk|sed|cut|tr|tee|xargs|chmod|chown|ln|touch|date|cal|env|history|clear|exit)\b/gm }, + { type: 'command', regex: /^(?:sudo|cd|ls|cat|grep|find|mkdir|rm|cp|mv|chmod|chown|ssh|scp|rsync|npm|npx|node|git|docker|docker-compose|curl|wget|tar|zip|unzip|make|cmake|brew|apt|yum|dnf|pacman|kill|ps|top|htop|df|du|mount|umount|echo|export|source|alias|unalias|which|whereis|man|tail|head|less|more|sort|uniq|wc|awk|sed|cut|tr|tee|xargs|ln|touch|date|cal|env|history|clear|exit)\b/gm }, { type: 'operator', regex: /[|;&><]/g } ], @@ -195,11 +203,11 @@ yaml: [ { type: 'comment', regex: /#.*$/gm }, - { type: 'property', regex: /^\s*(?:-\s*)?[a-zA-Z_][\w.-]*(?=\s*:(?:\s|$))/gm }, + { type: 'property', regex: /^[ \t]*(?:-[ \t]*)?[a-zA-Z_][\w.-]*(?=\s*:(?:\s|$))/gm }, { type: 'string', regex: /"[^"]*"|'[^']*'/g }, { type: 'keyword', regex: /\b(?:true|false|null)\b/g }, { type: 'number', regex: /-?\b\d+(\.\d+)?\b/g }, - { type: 'punctuation', regex: /^---$|^\.\.\.$|[:,[\]{}]|^\s*-(?=\s)/gm } + { type: 'punctuation', regex: /^---$|^\.\.\.$|[:,[\]{}]|^[ \t]*-(?=\s)/gm } ], sql: [ @@ -225,7 +233,7 @@ // after `|`—no fixed filter list is kept here since templates commonly // add their own—while a called macro/function still needs the usual // name-before-`(` pattern - { type: 'function', regex: /(?<=\|\s*)[a-zA-Z_][a-zA-Z0-9_]*|\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, + { type: 'function', regex: /(?=[a-zA-Z_])(?<=\|\s*)[a-zA-Z_][a-zA-Z0-9_]*|\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, // `=` is included alongside the comparison/arithmetic operators above // for assignment (`{% set x = 1 %}`)—`={2,3}` is tried first in the // alternation, so a real `==`/`===` still matches whole, never as two @@ -259,13 +267,13 @@ { type: 'comment', regex: /#.*$/gm }, { type: 'string', regex: /"[^"]*"|'[^']*'/g }, { type: 'tag', regex: /<\/?[A-Za-z][A-Za-z0-9]*/g }, - { type: 'variable', regex: /[$%]\{[^}]*\}|\$\d+|%\d+/g }, - { type: 'flag', regex: /\[[^\]\n]*\]/g }, + { type: 'variable', regex: /[$%]\{[^{}\n]*\}|\$\d+|%\d+/g }, + { type: 'flag', regex: /\[[^[\]\n]*\]/g }, // Whatever identifier opens a line is the directive: Apache ships // hundreds of them and each module adds its own, so any list here would // leave real configuration half-styled. Detection stays conservative // regardless—see `APACHE_DIRECTIVES` - { type: 'keyword', regex: /(?<=^[ \t]*)[A-Za-z][A-Za-z0-9_]*(?=[ \t]|$)/gm }, + { type: 'keyword', regex: /(?=[A-Za-z])(?<=^[ \t]*)[A-Za-z][A-Za-z0-9_]*(?=[ \t]|$)/gm }, // Only as a standalone value, so an `on`/`off` inside a path or // pattern (e.g., `/errors/on.html`) stays plain { type: 'builtin', regex: /(?<=[ \t])(?:on|off)(?=[ \t]|$)/gim }, @@ -290,25 +298,50 @@ languages.njk = languages.nunjucks; languages.patch = languages.diff; + // Per position, whether the next `;`/`{`/`}` is a `{` and whether an `@` + // came after the previous one, so `accept` checks take constant time + // instead of rescanning the source per match (quadratic on long inputs) + function getStatementContext(source) { + const beforeBrace = new Uint8Array(source.length + 1); + const afterAt = new Uint8Array(source.length + 1); + for (let i = source.length - 1, next = 0; i >= 0; i--) { + const char = source[i]; + next = char === '{' ? 1 : (char === ';' || char === '}') ? 0 : next; + beforeBrace[i] = next; + } + for (let i = 0, at = 0; i < source.length; i++) { + afterAt[i] = at; + const char = source[i]; + at = char === '@' ? 1 : (char === ';' || char === '{' || char === '}') ? 0 : at; + } + return { beforeBrace, afterAt }; + } + + function isSelectorContext(context, start, end) { + return context.beforeBrace[end] === 1 && context.afterAt[start] === 0; + } + // Tokenize source code and return flat token list function tokenize(source, lang) { const patterns = languages[lang]; - if (!patterns) { - return []; - } + let context; // Collect all matches with positions const matches = []; - for (const { type, regex } of patterns) { + for (const { type, regex, accept } of patterns) { let match; while ((match = regex.exec(source)) !== null) { - matches.push({ - type, - start: match.index, - end: match.index + match[0].length - }); + const start = match.index; + const end = start + match[0].length; + if (accept) { + context = context || getStatementContext(source); + if (!accept(context, start, end)) { + continue; + } + } + matches.push({ type, start, end }); } } @@ -388,7 +421,7 @@ function isStrongCssSelector(token, source) { const text = source.slice(token.start, token.end); if (text[0] === ':') { - return CSS_PSEUDO.has(text.slice(1).toLowerCase()); + return CSS_PSEUDO.has(text.replace(/^::?/, '').toLowerCase()); } if (text[0] === '.' || text[0] === '#') { // `)`/`]` also disqualify: A chained call/index like `.slice().sort()` @@ -492,9 +525,6 @@ function hasStrongSignal(tokens, source, lang) { const spec = STRONG_SIGNAL[lang]; - if (!spec) { - return true; - } if (spec.custom) { return spec.custom(tokens, source); } @@ -505,7 +535,10 @@ if (spec.words && spec.words[t.type]) { // `tag` matches include the leading `<`/` code:not([class*="language-"])').forEach((element) => { - const textNode = element.firstChild; - if (!textNode || textNode.nodeType !== Node.TEXT_NODE) { - return; - } - const lang = detectLanguage(textNode.textContent); - if (lang) { - collectRanges(element, rangesByType, lang); - } + collectRanges(element, rangesByType, true); }); } + // Ranges are added one by one, since spreading them as arguments + // overflows the call stack on very large blocks for (const [type, ranges] of rangesByType) { - CSS.highlights.set(type, new Highlight(...ranges)); + const highlight = new Highlight(); + ranges.forEach((range) => highlight.add(range)); + CSS.highlights.set(type, highlight); } } diff --git a/test/autodetect.test.mjs b/test/autodetect.test.mjs index 1c6b09e..6b2ce80 100644 --- a/test/autodetect.test.mjs +++ b/test/autodetect.test.mjs @@ -222,3 +222,16 @@ function assertHasOne(tokens, type, text) { const matches = tokens.filter((t) => t.type === type && t.text === text); assert.equal(matches.length, 1, `expected exactly one ${type} token ${JSON.stringify(text)}, got ${matches.length}`); } + +test('`detectLanguage` does not mistake a JS condition with index access (`list[i].active) {`) for a CSS type selector', () => { + const { window } = runSyntaxp(jsSource); + for (const source of ['} else if (list[i].active) {\n go()\n}', 'if (rows[n].cells[m].hidden) { show() }']) { + assert.notEqual(window.syntaxp.detectLanguage(source), 'css'); + } +}); + +test('`detectLanguage` recognizes type selectors with attribute and functional pseudo-class compounds', () => { + const { window } = runSyntaxp(jsSource); + assert.equal(window.syntaxp.detectLanguage('input[type="text"] {\n border: 0;\n}'), 'css'); + assert.equal(window.syntaxp.detectLanguage('li:nth-child(2n) {\n color: red;\n}'), 'css'); +}); diff --git a/test/helpers/dom.mjs b/test/helpers/dom.mjs index bbf459d..58b367c 100644 --- a/test/helpers/dom.mjs +++ b/test/helpers/dom.mjs @@ -18,7 +18,6 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre const styleElements = []; const highlightsByType = new Map(); - const listeners = {}; const sandbox = { CSS: { @@ -31,7 +30,7 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre } } }, - getComputedStyle(element) { + getComputedStyle() { return { colorScheme: colorScheme || '' }; }, document: { @@ -60,8 +59,8 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre styleElements.push(el); } }, - createElement(tag) { - return { tagName: tag, textContent: '', nonce: '' }; + createElement() { + return { textContent: '', nonce: '' }; }, querySelectorAll(selector) { if (selector === 'code[class*="language-"]') { @@ -72,9 +71,7 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre } return []; }, - addEventListener(name, fn) { - listeners[name] = fn; - } + addEventListener() {} }, window: {}, Node: { TEXT_NODE: 3 }, @@ -87,6 +84,9 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre constructor(...ranges) { this.ranges = ranges; } + add(range) { + this.ranges.push(range); + } }, console }; @@ -94,7 +94,7 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre vm.createContext(sandbox); vm.runInContext(jsSource, sandbox); - return { styleElements, highlightsByType, window: sandbox.window, listeners }; + return { styleElements, highlightsByType, window: sandbox.window }; } // Flattens the `{type -> StaticRange[]}` map from one `runSyntaxp` call diff --git a/test/languages.test.mjs b/test/languages.test.mjs index 484c199..2877e0d 100644 --- a/test/languages.test.mjs +++ b/test/languages.test.mjs @@ -33,24 +33,57 @@ test('`xml` is an alias for `html`', () => { }); test('`css`', () => { - // Note: A leading space is part of the `property` match (consistent with - // this tokenizer's handling of leading whitespace elsewhere, e.g., YAML/ - // Markdown—harmless for rendering since whitespace carries no color), and - // a `#rrggbb` color value is matched by the `selector` pattern (same shape - // as an ID selector)—a pre-existing quirk of this grammar, not something - // introduced here const tokens = tokenize('language-css', '@media (min-width: 1px) { .a { color: #fff; } }'); assertHasToken(tokens, 'keyword', '@media'); assertHasToken(tokens, 'selector', '.a'); - assertHasToken(tokens, 'property', ' color'); + assertHasToken(tokens, 'property', 'color'); +}); + +test('`css` excludes leading whitespace from a property', () => { + const text = '.a {\n color: red;\n margin: 0;\n}'; + const properties = tokenize('language-css', text).filter((t) => t.type === 'property'); + assert.deepEqual(properties.map((t) => t.text), ['color', 'margin']); +}); + +test('`css` does not tag values as selectors', () => { + const tokens = tokenize('language-css', '.a { color:red; background: #fff url(a.png) }'); + const selectors = tokens.filter((t) => t.type === 'selector').map((t) => t.text); + assert.deepEqual(selectors, ['.a']); +}); + +test('`css` tags compound and nested selectors, pseudo-classes included', () => { + const tokens = tokenize('language-css', 'a:hover, .b:not(.c) { color: red; }\np.e::before { content: none; }\n.d {\n &:focus { color: blue }\n}'); + for (const selector of [':hover', '.b', ':not', '.c', 'p', '.e', '::before', '.d', ':focus']) { + assertHasToken(tokens, 'selector', selector); + } + assert.ok(!tokens.some((t) => t.type === 'property' && t.text === 'a'), 'tagged `a` in `a:hover` as a property'); +}); + +test('`css` does not tag at-rule conditions as selectors', () => { + const tokens = tokenize('language-css', '@media (hover:hover) { a { color: red; } }'); + assert.deepEqual(tokens.filter((t) => t.type === 'selector').map((t) => t.text), ['a']); +}); + +test('`css` tokenizes long inputs in linear time', () => { + const inputs = [ + Array.from({ length: 20000 }, (_, i) => `.s${i}`).join(', ') + ' { color: red; }', + 'word.Next '.repeat(20000), + 'a' + ':b'.repeat(40) + ' x', + 'a' + '.b'.repeat(20000) + ]; + for (const text of inputs) { + const start = performance.now(); + tokenize('language-css', text); + assert.ok(performance.now() - start < 1000, `took too long for ${JSON.stringify(text.slice(0, 20))}…`); + } +}); + +test('`css` tags a percentage, unit included, as a number', () => { + const tokens = tokenize('language-css', 'a { width: 50%; }'); + assertHasToken(tokens, 'number', '50%'); }); test('`javascript`', () => { - // `console.log(…)` rather than a bare builtin call like `fetch(…)`: When a - // builtin is called directly, `function`’s pattern (listed first) and - // `builtin`’s pattern match the identical span, and `function` wins by - // pattern order—so `builtin` only surfaces for non-called references like - // `console` here const tokens = tokenize('language-javascript', 'const x = console.log(1); // c'); assertHasToken(tokens, 'keyword', 'const'); assertHasToken(tokens, 'builtin', 'console'); @@ -58,6 +91,19 @@ test('`javascript`', () => { assertHasToken(tokens, 'comment', '// c'); }); +test('`javascript` tags a called builtin as `builtin`, not `function`', () => { + const tokens = tokenize('language-javascript', 'fetch(url); parseInt(x); f(1);'); + assertHasToken(tokens, 'builtin', 'fetch'); + assertHasToken(tokens, 'builtin', 'parseInt'); + assertHasToken(tokens, 'function', 'f'); +}); + +test('`javascript` and `typescript` tag `=>` as a single operator', () => { + for (const className of ['language-javascript', 'language-typescript']) { + assertHasToken(tokenize(className, 'const f = (a) => a;'), 'operator', '=>'); + } +}); + test('`js` is an alias for `javascript`', () => { assert.ok(sameHighlighting(jsSource, 'const x = 1; // c', 'language-javascript', 'language-js')); }); @@ -127,6 +173,12 @@ test('`php`', () => { assertHasToken(tokens, 'comment', '// c'); }); +test('`php` tags `->` and `=>` as single operators', () => { + const tokens = tokenize('language-php', '$a->b(); $x = [1 => 2];'); + assertHasToken(tokens, 'operator', '->'); + assertHasToken(tokens, 'operator', '=>'); +}); + test('`python`', () => { const tokens = tokenize('language-python', "def f():\n return fr'a\\d' # c"); assertHasToken(tokens, 'keyword', 'def'); @@ -140,6 +192,11 @@ test('`python` multi-letter and prefixed triple-quoted string prefixes', () => { assertHasToken(tokens, 'string', 'f"""b"""'); }); +test('`python` tags the return annotation arrow `->` as a single operator', () => { + const tokens = tokenize('language-python', 'def f(a) -> int:\n return a'); + assertHasToken(tokens, 'operator', '->'); +}); + test('`py` is an alias for `python`', () => { assert.ok(sameHighlighting(jsSource, "def f():\n return 1", 'language-python', 'language-py')); }); @@ -334,4 +391,37 @@ test('`apacheconf` tags mod_expires periods as numbers, but not digits inside id assert.ok(!tokens.some((t) => t.type === 'number' && t.text === '8')); // The `404` of `/404.html` is part of a filename assert.equal(tokens.filter((t) => t.type === 'number' && t.text === '404').length, 1); -}); \ No newline at end of file +}); +test('Every token type has a `::highlight()` rule and a custom property in both palettes', () => { + const css = readFileSync(`${dirRoot}/syntaxp.css`, 'utf8'); + const [light, dark] = css.split('@media'); + const types = new Set([...jsSource.matchAll(/\btype: '(\w+)'/g)].map((match) => match[1])); + for (const type of types) { + assert.match(css, new RegExp(`::highlight\\(${type}\\)\\s*\\{\\s*color: var\\(--s5p-${type}\\);`), `no rule for ${type}`); + assert.match(light, new RegExp(`--s5p-${type}:`), `no light color for ${type}`); + assert.match(dark, new RegExp(`--s5p-${type}:`), `no dark color for ${type}`); + } +}); + +test('Every language tokenizes adversarial input in linear time', () => { + const languages = ['html', 'markdown', 'css', 'javascript', 'typescript', 'python', 'php', 'shell', 'json', 'yaml', 'sql', 'nunjucks', 'diff', 'http', 'apacheconf']; + for (const unit of ['\n', ' \n', ' ', '[', '](', '{%']) { + const text = unit.repeat(80000 / unit.length); + for (const language of languages) { + const start = performance.now(); + tokenize(`language-${language}`, text); + assert.ok(performance.now() - start < 1000, `${language} took too long for ${JSON.stringify(unit)}`); + } + } +}); + +test('Highlights a block with more ranges than fit in a call’s arguments', () => { + assert.doesNotThrow(() => tokenize('language-css', '{'.repeat(300000))); +}); + +test('Ignores a `language-*` class naming an inherited object property', () => { + const codeSamples = ['constructor', '__proto__', 'toString'].map((name) => ({ className: `language-${name}`, text: 'x' })); + codeSamples.push({ className: 'language-js', text: 'const a = 1;' }); + const { highlightsByType } = runSyntaxp(jsSource, { codeSamples }); + assert.ok(highlightsByType.has('keyword')); +});