From 83c5f3537da2ca30dc4ed7ae083f36d905954a05 Mon Sep 17 00:00:00 2001 From: Jens Oliver Meiert Date: Sat, 31 Oct 2026 12:43:54 +0100 Subject: [PATCH 1/5] feat: refine syntax highlighting rules and add new tests Enhanced tokenization logic for improved accuracy across multiple languages, including refined CSS selector handling, property detection, and JavaScript operator tagging. Added comprehensive test cases to validate new syntax grammar rules and edge cases. Incremented version to 2.4.1 for patch release. (This commit message was AI-generated.) --- README.md | 4 +++ package-lock.json | 4 +-- package.json | 2 +- syntaxp.css | 8 +++++ syntaxp.js | 67 ++++++++++++++++------------------ test/helpers/dom.mjs | 13 +++---- test/languages.test.mjs | 79 ++++++++++++++++++++++++++++++++++------- 7 files changed, 117 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 2ab3cc5..124920b 100644 --- a/README.md +++ b/README.md @@ -116,9 +116,13 @@ Do you miss an important language or alias? Contributions are welcome! | `operator` | `===`, `=>`, `&&` | | `tag` | `
`, `` | | `attribute` | `class=`, `href=` | +| `punctuation` | `{`, `;`, `>` | | `property` | `color:`, `margin:` | | `selector` | `.container`, `#app` | | `builtin` | `console`, `Promise`, `Math` | +| `type` | `string`, `Record` (TypeScript) | +| `doctype` | `` | +| `entity` | `&`, `©` | | `variable` | `$HOME`, `${name}` | | `command` | `npm`, `git`, `ssh` | | `flag` | `--port`, `-v` | diff --git a/package-lock.json b/package-lock.json index e319b2a..2186d3b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "syntaxp", - "version": "2.4.0", + "version": "2.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "syntaxp", - "version": "2.4.0", + "version": "2.4.1", "license": "MIT", "devDependencies": { "esbuild": "^0.28.2" diff --git a/package.json b/package.json index 7d4fb46..679d3af 100644 --- a/package.json +++ b/package.json @@ -13,5 +13,5 @@ "pretest": "npm run build", "test": "node --test" }, - "version": "2.4.0" + "version": "2.4.1" } \ No newline at end of file diff --git a/syntaxp.css b/syntaxp.css index bdab0aa..4ba0711 100644 --- a/syntaxp.css +++ b/syntaxp.css @@ -29,7 +29,9 @@ --s5p-operator: #06708a; + /* #c22068 */ --s5p-builtin: #c22068; + --s5p-type: #c22068; --s5p-variable: #1f2937; @@ -71,7 +73,9 @@ --s5p-operator: #89dceb; + /* #f5c2e7 */ --s5p-builtin: #f5c2e7; + --s5p-type: #f5c2e7; --s5p-variable: #cdd6f4; @@ -132,6 +136,10 @@ color: var(--s5p-builtin); } +::highlight(type) { + color: var(--s5p-type); +} + ::highlight(doctype) { color: var(--s5p-doctype); } diff --git a/syntaxp.js b/syntaxp.js index af299c2..bf3a163 100644 --- a/syntaxp.js +++ b/syntaxp.js @@ -61,9 +61,6 @@ // likely to be wrong as right. Pages in that position still need an // explicit `data-theme`. function detectPageTheme() { - if (typeof getComputedStyle !== 'function' || !document.documentElement) { - return null; - } const colorScheme = getComputedStyle(document.documentElement).colorScheme; return (colorScheme === 'light' || colorScheme === 'dark') ? colorScheme : null; } @@ -113,15 +110,22 @@ css: [ { type: 'comment', regex: /\/\*[\s\S]*?\*\//g }, { type: 'string', regex: /"[^"]*"|'[^']*'/g }, - { type: 'number', regex: /\b\d+(\.\d+)?(px|em|rem|%|vh|vw|s|ms|deg|fr)?\b/g }, + // `%` takes no trailing `\b`, as that would require a word character to follow + { type: 'number', regex: /\b\d+(?:\.\d+)?(?:(?:px|em|rem|vh|vw|s|ms|deg|fr)\b|%|\b)/g }, { type: 'keyword', regex: /@(media|import|keyframes|font-face|supports|charset|namespace|layer|property)\b/g }, - { type: 'selector', regex: /[.#:][a-zA-Z_-][a-zA-Z0-9_-]*/g }, + // Selectors are followed by a `{` before any `;`/`}`, unlike values + // (`color: #fff;`, `url(a.png)`), and never sit in an at-rule’s prelude + // (`@media (hover:hover)`) + { type: 'selector', regex: /(?]=?=?|&&|\|\||[+\-*/%]=?|\?\?|\.\.\.|=>/g }, + { type: 'function', regex: /\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\s*\()/g }, + { type: 'operator', regex: /=>|[=!<>]=?=?|&&|\|\||[+\-*/%]=?|\?\?|\.\.\./g }, { type: 'punctuation', regex: /[{}[\]();:,.]/g } ], @@ -149,7 +153,7 @@ // example this tokenizer was asked to score during language detection { type: 'function', regex: /\b[a-zA-Z_$][a-zA-Z0-9_$]*(?=\s*\(|<)/g }, { type: 'type', regex: /\b(string|number|boolean|bigint|symbol|object|Array|Promise|Record|Partial|Required|Pick|Omit|Exclude|Extract|NonNullable|ReturnType|Parameters|ConstructorParameters|InstanceType)\b/g }, - { type: 'operator', regex: /[=!<>]=?=?|&&|\|\||[+\-*/%]=?|\?\?|\.\.\.|=>/g }, + { type: 'operator', regex: /=>|[=!<>]=?=?|&&|\|\||[+\-*/%]=?|\?\?|\.\.\./g }, { type: 'punctuation', regex: /[{}[\]();:,.]/g } ], @@ -160,7 +164,7 @@ { type: 'builtin', regex: /\b(?:print|len|range|str|int|float|bool|list|dict|set|tuple|type|isinstance|super|self|enumerate|zip|map|filter|sorted|reversed|open|input|format|repr|iter|next|abs|min|max|sum|any|all|Exception|ValueError|TypeError|KeyError|IndexError|StopIteration)\b/g }, { type: 'number', regex: /\b\d+(\.\d+)?([eE][+-]?\d+)?[jJ]?\b/g }, { type: 'function', regex: /\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, - { type: 'operator', regex: /<<=|>>=|<<|>>|[=!<>]=?|\*\*=?|\/\/=?|[+\-%]=?|[*/&|^]=?|~|->|:=/g }, + { type: 'operator', regex: /->|<<=|>>=|<<|>>|[=!<>]=?|\*\*=?|\/\/=?|[+\-%]=?|[*/&|^]=?|~|:=/g }, { type: 'punctuation', regex: /[{}[\]();,:.]/g } ], @@ -171,7 +175,7 @@ { type: 'variable', regex: /\$[a-zA-Z_][a-zA-Z0-9_]*/g }, { type: 'number', regex: /\b\d+(\.\d+)?\b/g }, { type: 'function', regex: /\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, - { type: 'operator', regex: /[=!<>]=?=?|&&|\|\||[+\-*/%.]=?|->|=>|::|\?\?/g }, + { type: 'operator', regex: /->|=>|[=!<>]=?=?|&&|\|\||[+\-*/%.]=?|::|\?\?/g }, { type: 'punctuation', regex: /[{}[\]();,]/g } ], @@ -181,7 +185,7 @@ { type: 'variable', regex: /\$[{(]?[a-zA-Z_][a-zA-Z0-9_]*[})]?/g }, { type: 'flag', regex: /(?<=^|\s)--?[a-zA-Z-]+/g }, { type: 'path', regex: /(?:\.{0,2}\/)[^\s;|&<>"]+/g }, - { type: 'command', regex: /^(?:sudo|cd|ls|cat|grep|find|mkdir|rm|cp|mv|chmod|chown|ssh|scp|rsync|npm|npx|node|git|docker|docker-compose|curl|wget|tar|zip|unzip|make|cmake|brew|apt|yum|dnf|pacman|kill|ps|top|htop|df|du|mount|umount|echo|export|source|alias|unalias|which|whereis|man|tail|head|less|more|sort|uniq|wc|awk|sed|cut|tr|tee|xargs|chmod|chown|ln|touch|date|cal|env|history|clear|exit)\b/gm }, + { type: 'command', regex: /^(?:sudo|cd|ls|cat|grep|find|mkdir|rm|cp|mv|chmod|chown|ssh|scp|rsync|npm|npx|node|git|docker|docker-compose|curl|wget|tar|zip|unzip|make|cmake|brew|apt|yum|dnf|pacman|kill|ps|top|htop|df|du|mount|umount|echo|export|source|alias|unalias|which|whereis|man|tail|head|less|more|sort|uniq|wc|awk|sed|cut|tr|tee|xargs|ln|touch|date|cal|env|history|clear|exit)\b/gm }, { type: 'operator', regex: /[|;&><]/g } ], @@ -293,9 +297,6 @@ // Tokenize source code and return flat token list function tokenize(source, lang) { const patterns = languages[lang]; - if (!patterns) { - return []; - } // Collect all matches with positions const matches = []; @@ -388,7 +389,7 @@ function isStrongCssSelector(token, source) { const text = source.slice(token.start, token.end); if (text[0] === ':') { - return CSS_PSEUDO.has(text.slice(1).toLowerCase()); + return CSS_PSEUDO.has(text.replace(/^::?/, '').toLowerCase()); } if (text[0] === '.' || text[0] === '#') { // `)`/`]` also disqualify: A chained call/index like `.slice().sort()` @@ -492,9 +493,6 @@ function hasStrongSignal(tokens, source, lang) { const spec = STRONG_SIGNAL[lang]; - if (!spec) { - return true; - } if (spec.custom) { return spec.custom(tokens, source); } @@ -505,7 +503,10 @@ if (spec.words && spec.words[t.type]) { // `tag` matches include the leading `<`/` code:not([class*="language-"])').forEach((element) => { - const textNode = element.firstChild; - if (!textNode || textNode.nodeType !== Node.TEXT_NODE) { - return; - } - const lang = detectLanguage(textNode.textContent); - if (lang) { - collectRanges(element, rangesByType, lang); - } + collectRanges(element, rangesByType, true); }); } diff --git a/test/helpers/dom.mjs b/test/helpers/dom.mjs index bbf459d..027ae47 100644 --- a/test/helpers/dom.mjs +++ b/test/helpers/dom.mjs @@ -18,7 +18,6 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre const styleElements = []; const highlightsByType = new Map(); - const listeners = {}; const sandbox = { CSS: { @@ -31,7 +30,7 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre } } }, - getComputedStyle(element) { + getComputedStyle() { return { colorScheme: colorScheme || '' }; }, document: { @@ -60,8 +59,8 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre styleElements.push(el); } }, - createElement(tag) { - return { tagName: tag, textContent: '', nonce: '' }; + createElement() { + return { textContent: '', nonce: '' }; }, querySelectorAll(selector) { if (selector === 'code[class*="language-"]') { @@ -72,9 +71,7 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre } return []; }, - addEventListener(name, fn) { - listeners[name] = fn; - } + addEventListener() {} }, window: {}, Node: { TEXT_NODE: 3 }, @@ -94,7 +91,7 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre vm.createContext(sandbox); vm.runInContext(jsSource, sandbox); - return { styleElements, highlightsByType, window: sandbox.window, listeners }; + return { styleElements, highlightsByType, window: sandbox.window }; } // Flattens the `{type -> StaticRange[]}` map from one `runSyntaxp` call diff --git a/test/languages.test.mjs b/test/languages.test.mjs index 484c199..274c508 100644 --- a/test/languages.test.mjs +++ b/test/languages.test.mjs @@ -33,24 +33,43 @@ test('`xml` is an alias for `html`', () => { }); test('`css`', () => { - // Note: A leading space is part of the `property` match (consistent with - // this tokenizer's handling of leading whitespace elsewhere, e.g., YAML/ - // Markdown—harmless for rendering since whitespace carries no color), and - // a `#rrggbb` color value is matched by the `selector` pattern (same shape - // as an ID selector)—a pre-existing quirk of this grammar, not something - // introduced here const tokens = tokenize('language-css', '@media (min-width: 1px) { .a { color: #fff; } }'); assertHasToken(tokens, 'keyword', '@media'); assertHasToken(tokens, 'selector', '.a'); - assertHasToken(tokens, 'property', ' color'); + assertHasToken(tokens, 'property', 'color'); +}); + +test('`css` excludes leading whitespace from a property', () => { + const text = '.a {\n color: red;\n margin: 0;\n}'; + const properties = tokenize('language-css', text).filter((t) => t.type === 'property'); + assert.deepEqual(properties.map((t) => t.text), ['color', 'margin']); +}); + +test('`css` does not tag values as selectors', () => { + const tokens = tokenize('language-css', '.a { color:red; background: #fff url(a.png) }'); + const selectors = tokens.filter((t) => t.type === 'selector').map((t) => t.text); + assert.deepEqual(selectors, ['.a']); +}); + +test('`css` tags compound and nested selectors, pseudo-classes included', () => { + const tokens = tokenize('language-css', 'a:hover, .b:not(.c) { color: red; }\np.e::before { content: none; }\n.d {\n &:focus { color: blue }\n}'); + for (const selector of [':hover', '.b', ':not', '.c', 'p', '.e', '::before', '.d', ':focus']) { + assertHasToken(tokens, 'selector', selector); + } + assert.ok(!tokens.some((t) => t.type === 'property' && t.text === 'a'), 'tagged `a` in `a:hover` as a property'); +}); + +test('`css` does not tag at-rule conditions as selectors', () => { + const tokens = tokenize('language-css', '@media (hover:hover) { a { color: red; } }'); + assert.deepEqual(tokens.filter((t) => t.type === 'selector').map((t) => t.text), ['a']); +}); + +test('`css` tags a percentage, unit included, as a number', () => { + const tokens = tokenize('language-css', 'a { width: 50%; }'); + assertHasToken(tokens, 'number', '50%'); }); test('`javascript`', () => { - // `console.log(…)` rather than a bare builtin call like `fetch(…)`: When a - // builtin is called directly, `function`’s pattern (listed first) and - // `builtin`’s pattern match the identical span, and `function` wins by - // pattern order—so `builtin` only surfaces for non-called references like - // `console` here const tokens = tokenize('language-javascript', 'const x = console.log(1); // c'); assertHasToken(tokens, 'keyword', 'const'); assertHasToken(tokens, 'builtin', 'console'); @@ -58,6 +77,19 @@ test('`javascript`', () => { assertHasToken(tokens, 'comment', '// c'); }); +test('`javascript` tags a called builtin as `builtin`, not `function`', () => { + const tokens = tokenize('language-javascript', 'fetch(url); parseInt(x); f(1);'); + assertHasToken(tokens, 'builtin', 'fetch'); + assertHasToken(tokens, 'builtin', 'parseInt'); + assertHasToken(tokens, 'function', 'f'); +}); + +test('`javascript` and `typescript` tag `=>` as a single operator', () => { + for (const className of ['language-javascript', 'language-typescript']) { + assertHasToken(tokenize(className, 'const f = (a) => a;'), 'operator', '=>'); + } +}); + test('`js` is an alias for `javascript`', () => { assert.ok(sameHighlighting(jsSource, 'const x = 1; // c', 'language-javascript', 'language-js')); }); @@ -127,6 +159,12 @@ test('`php`', () => { assertHasToken(tokens, 'comment', '// c'); }); +test('`php` tags `->` and `=>` as single operators', () => { + const tokens = tokenize('language-php', '$a->b(); $x = [1 => 2];'); + assertHasToken(tokens, 'operator', '->'); + assertHasToken(tokens, 'operator', '=>'); +}); + test('`python`', () => { const tokens = tokenize('language-python', "def f():\n return fr'a\\d' # c"); assertHasToken(tokens, 'keyword', 'def'); @@ -140,6 +178,11 @@ test('`python` multi-letter and prefixed triple-quoted string prefixes', () => { assertHasToken(tokens, 'string', 'f"""b"""'); }); +test('`python` tags the return annotation arrow `->` as a single operator', () => { + const tokens = tokenize('language-python', 'def f(a) -> int:\n return a'); + assertHasToken(tokens, 'operator', '->'); +}); + test('`py` is an alias for `python`', () => { assert.ok(sameHighlighting(jsSource, "def f():\n return 1", 'language-python', 'language-py')); }); @@ -334,4 +377,14 @@ test('`apacheconf` tags mod_expires periods as numbers, but not digits inside id assert.ok(!tokens.some((t) => t.type === 'number' && t.text === '8')); // The `404` of `/404.html` is part of a filename assert.equal(tokens.filter((t) => t.type === 'number' && t.text === '404').length, 1); -}); \ No newline at end of file +}); +test('Every token type has a `::highlight()` rule and a custom property in both palettes', () => { + const css = readFileSync(`${dirRoot}/syntaxp.css`, 'utf8'); + const [light, dark] = css.split('@media'); + const types = new Set([...jsSource.matchAll(/\btype: '(\w+)'/g)].map((match) => match[1])); + for (const type of types) { + assert.match(css, new RegExp(`::highlight\\(${type}\\)\\s*\\{\\s*color: var\\(--s5p-${type}\\);`), `no rule for ${type}`); + assert.match(light, new RegExp(`--s5p-${type}:`), `no light color for ${type}`); + assert.match(dark, new RegExp(`--s5p-${type}:`), `no dark color for ${type}`); + } +}); From 2a73dcfdfd3327807f964baf1772c2e7e2660c64 Mon Sep 17 00:00:00 2001 From: Jens Oliver Meiert Date: Sat, 31 Oct 2026 14:07:08 +0100 Subject: [PATCH 2/5] feat: optimize tokenization and improve performance tests Refined tokenization logic for CSS selectors, properties, and punctuation, introducing linear-time validation and additional context-aware rules. Enhanced performance by capping identifier lengths and adding tests for long and adversarial inputs across multiple languages. Improved handling of large highlight ranges to prevent stack overflows. (This commit message was AI-generated.) --- syntaxp.js | 73 ++++++++++++++++++++++++++++++----------- test/helpers/dom.mjs | 3 ++ test/languages.test.mjs | 30 +++++++++++++++++ 3 files changed, 86 insertions(+), 20 deletions(-) diff --git a/syntaxp.js b/syntaxp.js index bf3a163..fedccda 100644 --- a/syntaxp.js +++ b/syntaxp.js @@ -102,9 +102,9 @@ { type: 'string', regex: /`[^`]+`/g }, { type: 'attribute', regex: /\*\*[^*]+\*\*|__[^_]+__/g }, { type: 'entity', regex: /(? !context.beforeBrace[end] }, { type: 'function', regex: /(?:rgba?|hsla?|linear-gradient|radial-gradient|calc|var|env|clamp|min|max|url)\s*(?=\()/g }, { type: 'punctuation', regex: /[{}();:,]/g } ], @@ -160,7 +161,7 @@ python: [ { type: 'comment', regex: /#.*$/gm }, { type: 'string', regex: /(?:[rRbBfFuU]{1,2})?("""[\s\S]*?"""|'''[\s\S]*?''')|(?:[rRbBfFuU]{1,2})?"(?:[^"\\]|\\.)*"|(?:[rRbBfFuU]{1,2})?'(?:[^'\\]|\\.)*'/g }, - { type: 'keyword', regex: /^\s*@[a-zA-Z_][a-zA-Z0-9_.]*|\b(?:False|None|True|and|as|assert|async|await|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield)\b/gm }, + { type: 'keyword', regex: /^[ \t]*@[a-zA-Z_][a-zA-Z0-9_.]*|\b(?:False|None|True|and|as|assert|async|await|break|class|continue|def|del|elif|else|except|finally|for|from|global|if|import|in|is|lambda|nonlocal|not|or|pass|raise|return|try|while|with|yield)\b/gm }, { type: 'builtin', regex: /\b(?:print|len|range|str|int|float|bool|list|dict|set|tuple|type|isinstance|super|self|enumerate|zip|map|filter|sorted|reversed|open|input|format|repr|iter|next|abs|min|max|sum|any|all|Exception|ValueError|TypeError|KeyError|IndexError|StopIteration)\b/g }, { type: 'number', regex: /\b\d+(\.\d+)?([eE][+-]?\d+)?[jJ]?\b/g }, { type: 'function', regex: /\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, @@ -199,11 +200,11 @@ yaml: [ { type: 'comment', regex: /#.*$/gm }, - { type: 'property', regex: /^\s*(?:-\s*)?[a-zA-Z_][\w.-]*(?=\s*:(?:\s|$))/gm }, + { type: 'property', regex: /^[ \t]*(?:-[ \t]*)?[a-zA-Z_][\w.-]*(?=\s*:(?:\s|$))/gm }, { type: 'string', regex: /"[^"]*"|'[^']*'/g }, { type: 'keyword', regex: /\b(?:true|false|null)\b/g }, { type: 'number', regex: /-?\b\d+(\.\d+)?\b/g }, - { type: 'punctuation', regex: /^---$|^\.\.\.$|[:,[\]{}]|^\s*-(?=\s)/gm } + { type: 'punctuation', regex: /^---$|^\.\.\.$|[:,[\]{}]|^[ \t]*-(?=\s)/gm } ], sql: [ @@ -263,8 +264,8 @@ { type: 'comment', regex: /#.*$/gm }, { type: 'string', regex: /"[^"]*"|'[^']*'/g }, { type: 'tag', regex: /<\/?[A-Za-z][A-Za-z0-9]*/g }, - { type: 'variable', regex: /[$%]\{[^}]*\}|\$\d+|%\d+/g }, - { type: 'flag', regex: /\[[^\]\n]*\]/g }, + { type: 'variable', regex: /[$%]\{[^{}\n]*\}|\$\d+|%\d+/g }, + { type: 'flag', regex: /\[[^[\]\n]*\]/g }, // Whatever identifier opens a line is the directive: Apache ships // hundreds of them and each module adds its own, so any list here would // leave real configuration half-styled. Detection stays conservative @@ -294,22 +295,50 @@ languages.njk = languages.nunjucks; languages.patch = languages.diff; + // Per position, whether the next `;`/`{`/`}` is a `{` and whether an `@` + // came after the previous one, so `accept` checks take constant time + // instead of rescanning the source per match (quadratic on long inputs) + function getStatementContext(source) { + const beforeBrace = new Uint8Array(source.length + 1); + const afterAt = new Uint8Array(source.length + 1); + for (let i = source.length - 1, next = 0; i >= 0; i--) { + const char = source[i]; + next = char === '{' ? 1 : (char === ';' || char === '}') ? 0 : next; + beforeBrace[i] = next; + } + for (let i = 0, at = 0; i < source.length; i++) { + afterAt[i] = at; + const char = source[i]; + at = char === '@' ? 1 : (char === ';' || char === '{' || char === '}') ? 0 : at; + } + return { beforeBrace, afterAt }; + } + + function isSelectorContext(context, start, end) { + return context.beforeBrace[end] === 1 && context.afterAt[start] === 0; + } + // Tokenize source code and return flat token list function tokenize(source, lang) { const patterns = languages[lang]; + let context; // Collect all matches with positions const matches = []; - for (const { type, regex } of patterns) { + for (const { type, regex, accept } of patterns) { let match; while ((match = regex.exec(source)) !== null) { - matches.push({ - type, - start: match.index, - end: match.index + match[0].length - }); + const start = match.index; + const end = start + match[0].length; + if (accept) { + context = context || getStatementContext(source); + if (!accept(context, start, end)) { + continue; + } + } + matches.push({ type, start, end }); } } @@ -623,8 +652,12 @@ }); } + // Ranges are added one by one, since spreading them as arguments + // overflows the call stack on very large blocks for (const [type, ranges] of rangesByType) { - CSS.highlights.set(type, new Highlight(...ranges)); + const highlight = new Highlight(); + ranges.forEach((range) => highlight.add(range)); + CSS.highlights.set(type, highlight); } } diff --git a/test/helpers/dom.mjs b/test/helpers/dom.mjs index 027ae47..58b367c 100644 --- a/test/helpers/dom.mjs +++ b/test/helpers/dom.mjs @@ -84,6 +84,9 @@ export function runSyntaxp(jsSource, { codeSamples = [], autoSamples = [], curre constructor(...ranges) { this.ranges = ranges; } + add(range) { + this.ranges.push(range); + } }, console }; diff --git a/test/languages.test.mjs b/test/languages.test.mjs index 274c508..64d4e7b 100644 --- a/test/languages.test.mjs +++ b/test/languages.test.mjs @@ -64,6 +64,20 @@ test('`css` does not tag at-rule conditions as selectors', () => { assert.deepEqual(tokens.filter((t) => t.type === 'selector').map((t) => t.text), ['a']); }); +test('`css` tokenizes long inputs in linear time', () => { + const inputs = [ + Array.from({ length: 20000 }, (_, i) => `.s${i}`).join(', ') + ' { color: red; }', + 'word.Next '.repeat(20000), + 'a' + ':b'.repeat(40) + ' x', + 'a' + '.b'.repeat(20000) + ]; + for (const text of inputs) { + const start = performance.now(); + tokenize('language-css', text); + assert.ok(performance.now() - start < 1000, `took too long for ${JSON.stringify(text.slice(0, 20))}…`); + } +}); + test('`css` tags a percentage, unit included, as a number', () => { const tokens = tokenize('language-css', 'a { width: 50%; }'); assertHasToken(tokens, 'number', '50%'); @@ -388,3 +402,19 @@ test('Every token type has a `::highlight()` rule and a custom property in both assert.match(dark, new RegExp(`--s5p-${type}:`), `no dark color for ${type}`); } }); + +test('Every language tokenizes adversarial input in linear time', () => { + const languages = ['html', 'markdown', 'css', 'javascript', 'typescript', 'python', 'php', 'shell', 'json', 'yaml', 'sql', 'nunjucks', 'diff', 'http', 'apacheconf']; + for (const unit of ['\n', ' \n', ' ', '[', '](', '{%']) { + const text = unit.repeat(80000 / unit.length); + for (const language of languages) { + const start = performance.now(); + tokenize(`language-${language}`, text); + assert.ok(performance.now() - start < 1000, `${language} took too long for ${JSON.stringify(unit)}`); + } + } +}); + +test('Highlights a block with more ranges than fit in a call’s arguments', () => { + assert.doesNotThrow(() => tokenize('language-css', '{'.repeat(300000))); +}); From dd63c47ec90bd2bf95eff479b9516ce76bd47efd Mon Sep 17 00:00:00 2001 From: Jens Oliver Meiert Date: Sat, 31 Oct 2026 14:26:45 +0100 Subject: [PATCH 3/5] feat: improve regex efficiency in syntax highlighting Optimized regex patterns with lookahead/lookbehind combinations to reduce redundant scanning and improve performance in older V8 engines. Updated tokenization rules for functions and keywords to handle edge cases more efficiently. (This commit message was AI-generated.) --- syntaxp.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/syntaxp.js b/syntaxp.js index fedccda..167adff 100644 --- a/syntaxp.js +++ b/syntaxp.js @@ -83,7 +83,9 @@ document.head.appendChild(style); } - // Token patterns per language + // Token patterns per language; a lookbehind that opens a pattern gets a + // lookahead for the token’s first character in front of it, so it doesn’t + // rescan whitespace at every position (quadratic in older V8) const languages = { html: [ @@ -230,7 +232,7 @@ // after `|`—no fixed filter list is kept here since templates commonly // add their own—while a called macro/function still needs the usual // name-before-`(` pattern - { type: 'function', regex: /(?<=\|\s*)[a-zA-Z_][a-zA-Z0-9_]*|\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, + { type: 'function', regex: /(?=[a-zA-Z_])(?<=\|\s*)[a-zA-Z_][a-zA-Z0-9_]*|\b[a-zA-Z_][a-zA-Z0-9_]*(?=\s*\()/g }, // `=` is included alongside the comparison/arithmetic operators above // for assignment (`{% set x = 1 %}`)—`={2,3}` is tried first in the // alternation, so a real `==`/`===` still matches whole, never as two @@ -270,7 +272,7 @@ // hundreds of them and each module adds its own, so any list here would // leave real configuration half-styled. Detection stays conservative // regardless—see `APACHE_DIRECTIVES` - { type: 'keyword', regex: /(?<=^[ \t]*)[A-Za-z][A-Za-z0-9_]*(?=[ \t]|$)/gm }, + { type: 'keyword', regex: /(?=[A-Za-z])(?<=^[ \t]*)[A-Za-z][A-Za-z0-9_]*(?=[ \t]|$)/gm }, // Only as a standalone value, so an `on`/`off` inside a path or // pattern (e.g., `/errors/on.html`) stays plain { type: 'builtin', regex: /(?<=[ \t])(?:on|off)(?=[ \t]|$)/gim }, From 91a2527fadd3055359124c7c358eeba73670c04d Mon Sep 17 00:00:00 2001 From: Jens Oliver Meiert Date: Sat, 31 Oct 2026 20:02:15 +0100 Subject: [PATCH 4/5] feat: prevent conflicts with inherited object properties in class names Improved language detection by ensuring `language-*` classes are validated against own properties of the `languages` object. Added tests to handle cases where inherited object properties like `constructor` and `toString` are encountered. (This commit message was AI-generated.) --- syntaxp.js | 2 +- test/languages.test.mjs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/syntaxp.js b/syntaxp.js index 167adff..caef80c 100644 --- a/syntaxp.js +++ b/syntaxp.js @@ -614,7 +614,7 @@ const source = textNode.textContent; const lang = detect ? detectLanguage(source) : (element.className.match(/language-(\w+)/) || [])[1]; - if (!lang || !languages[lang]) { + if (!lang || !Object.hasOwn(languages, lang)) { return; } diff --git a/test/languages.test.mjs b/test/languages.test.mjs index 64d4e7b..2877e0d 100644 --- a/test/languages.test.mjs +++ b/test/languages.test.mjs @@ -418,3 +418,10 @@ test('Every language tokenizes adversarial input in linear time', () => { test('Highlights a block with more ranges than fit in a call’s arguments', () => { assert.doesNotThrow(() => tokenize('language-css', '{'.repeat(300000))); }); + +test('Ignores a `language-*` class naming an inherited object property', () => { + const codeSamples = ['constructor', '__proto__', 'toString'].map((name) => ({ className: `language-${name}`, text: 'x' })); + codeSamples.push({ className: 'language-js', text: 'const a = 1;' }); + const { highlightsByType } = runSyntaxp(jsSource, { codeSamples }); + assert.ok(highlightsByType.has('keyword')); +}); From c533ca68b8d47ac0a2aaa6a3b81fb9a88221817a Mon Sep 17 00:00:00 2001 From: Jens Oliver Meiert Date: Sat, 31 Oct 2026 20:58:45 +0100 Subject: [PATCH 5/5] feat: enhance `detectLanguage` logic for JS and CSS differentiation Improved `detectLanguage` handling to prevent misclassification of JavaScript conditions with index access as CSS type selectors. Updated regex for type selectors to ensure balanced parentheses in compound selectors and added tests to validate functional pseudo-classes and attribute selectors. (This commit message was AI-generated.) --- syntaxp.js | 9 +++++---- test/autodetect.test.mjs | 13 +++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/syntaxp.js b/syntaxp.js index caef80c..ef46197 100644 --- a/syntaxp.js +++ b/syntaxp.js @@ -122,10 +122,11 @@ // A bare type selector (`p`, `h1`, `body`…) has no punctuation prefix // to key off, unlike `.class`/`#id`/`:pseudo` above, so it’s only // trusted directly before a rule’s `{`, optionally via a compound - // (`a:hover {`, `p.note {`), not in looser shapes JS shares - // (`foo(a, b) {`, `key: {`); the compound’s length is capped - // to keep long unbroken runs (minified code) linear - { type: 'selector', regex: /(? !context.beforeBrace[end] }, diff --git a/test/autodetect.test.mjs b/test/autodetect.test.mjs index 1c6b09e..6b2ce80 100644 --- a/test/autodetect.test.mjs +++ b/test/autodetect.test.mjs @@ -222,3 +222,16 @@ function assertHasOne(tokens, type, text) { const matches = tokens.filter((t) => t.type === type && t.text === text); assert.equal(matches.length, 1, `expected exactly one ${type} token ${JSON.stringify(text)}, got ${matches.length}`); } + +test('`detectLanguage` does not mistake a JS condition with index access (`list[i].active) {`) for a CSS type selector', () => { + const { window } = runSyntaxp(jsSource); + for (const source of ['} else if (list[i].active) {\n go()\n}', 'if (rows[n].cells[m].hidden) { show() }']) { + assert.notEqual(window.syntaxp.detectLanguage(source), 'css'); + } +}); + +test('`detectLanguage` recognizes type selectors with attribute and functional pseudo-class compounds', () => { + const { window } = runSyntaxp(jsSource); + assert.equal(window.syntaxp.detectLanguage('input[type="text"] {\n border: 0;\n}'), 'css'); + assert.equal(window.syntaxp.detectLanguage('li:nth-child(2n) {\n color: red;\n}'), 'css'); +});