From e317277f98570fefec1bccbf42109486f0f520ef Mon Sep 17 00:00:00 2001 From: Max Rice Date: Tue, 15 Sep 2026 20:18:37 -0400 Subject: [PATCH 1/2] Scan in linear time instead of quadratic The scanner addresses text by character offset. Resolving a character offset against a UTF-8 string means walking it from the start, so calling mb_substr() once per character made scanning cost time proportional to the square of the document length. Both the per-character reads in StringHelper::charCodeAt() and the per-token reads in StringHelper::substring() paid it. Split the document into characters once in the constructor and index that array instead. charCodeAt() takes an ord() fast path for single-byte characters, which covers essentially all of a JSON document's structure. Memory cost is roughly 46x the source size, paid once per scanner. The text property is no longer read now that every access goes through the character array, so it is gone. Measured over a document shaped like a Shopify theme locale file, running parse + modify + applyEdits: 8 KB 55 ms -> 9.3 ms 33 KB 720 ms -> 22.4 ms 90 KB 5,307 ms -> 61.4 ms 183 KB 21,384 ms -> 128.0 ms Scaling is linear after the change: twice the input costs 2.1x the time. The new complexity tests assert a ratio between two input sizes rather than a wall-clock budget, so they carry the same meaning on any machine. Quadratic scanning takes ~16x as long for 4x the input and linear scanning takes ~4x, which leaves a wide margin around the threshold of 8. The second test guards against a fix that speeds up ASCII by falling back to byte offsets while leaving multibyte documents quadratic. --- src/Scanner/Scanner.php | 122 +++++++++++++------ tests/Unit/Scanner/ScannerComplexityTest.php | 109 +++++++++++++++++ 2 files changed, 196 insertions(+), 35 deletions(-) create mode 100644 tests/Unit/Scanner/ScannerComplexityTest.php diff --git a/src/Scanner/Scanner.php b/src/Scanner/Scanner.php index 3f10a9f..00416ef 100644 --- a/src/Scanner/Scanner.php +++ b/src/Scanner/Scanner.php @@ -13,7 +13,19 @@ */ final class Scanner implements JsonScanner { - private string $text; + /** + * The document split into individual characters, indexed by character offset. + * + * The scanner addresses text by character offset, not byte offset. Resolving + * such an offset against a UTF-8 string means walking it from the start, so + * doing that once per character makes scanning quadratic in the length of the + * document. Splitting once up front turns every subsequent access into an + * array lookup, at a memory cost of roughly 46x the source size. + * + * @var list + */ + private array $chars; + private int $len; private int $pos = 0; private string $value = ''; @@ -40,12 +52,52 @@ private function __construct( string $text, private readonly bool $ignoreTrivia ) { - $this->text = $text; - $this->len = StringHelper::length($text); + $this->chars = mb_str_split($text, 1, 'UTF-8'); + $this->len = count($this->chars); $this->token = SyntaxKind::Unknown; $this->scanError = ScanError::None; } + /** + * Get the Unicode code point at a character offset, or 0 when out of bounds. + */ + private function charCodeAt(int $pos): int + { + $char = $this->chars[$pos] ?? ''; + + if ($char === '') { + return 0; + } + + // A single-byte UTF-8 character is its own code point, which covers + // essentially all of a JSON document's structure and keys. + if (strlen($char) === 1) { + return ord($char); + } + + $code = mb_ord($char, 'UTF-8'); + + return $code !== false ? $code : 0; + } + + /** + * Get the text between two character offsets, or to the end when $end is null. + */ + private function substring(int $start, ?int $end = null): string + { + if ($end === null) { + return implode('', array_slice($this->chars, $start)); + } + + $length = $end - $start; + + if ($length < 0) { + return ''; + } + + return implode('', array_slice($this->chars, $start, $length)); + } + public function setPosition(int $pos): void { $this->pos = $pos; @@ -106,7 +158,7 @@ private function scanHexDigits(int $count, bool $exact = false): int $value = 0; while ($digits < $count || !$exact) { - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); if ($ch >= CC::DIGIT_0 && $ch <= CC::DIGIT_9) { $value = $value * 16 + $ch - CC::DIGIT_0; @@ -133,42 +185,42 @@ private function scanNumber(): string { $start = $this->pos; - if (StringHelper::charCodeAt($this->text, $this->pos) === CC::DIGIT_0) { + if ($this->charCodeAt($this->pos) === CC::DIGIT_0) { $this->pos++; } else { $this->pos++; - while ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + while ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; } } - if ($this->pos < $this->len && StringHelper::charCodeAt($this->text, $this->pos) === CC::DOT) { + if ($this->pos < $this->len && $this->charCodeAt($this->pos) === CC::DOT) { $this->pos++; - if ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + if ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; - while ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + while ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; } } else { $this->scanError = ScanError::UnexpectedEndOfNumber; - return StringHelper::substring($this->text, $start, $this->pos); + return $this->substring($start, $this->pos); } } $end = $this->pos; if ($this->pos < $this->len) { - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); if ($ch === CC::UPPER_E || $ch === CC::LOWER_E) { $this->pos++; if ($this->pos < $this->len) { - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); if ($ch === CC::PLUS || $ch === CC::MINUS) { $this->pos++; } } - if ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + if ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; - while ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + while ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; } $end = $this->pos; @@ -178,7 +230,7 @@ private function scanNumber(): string } } - return StringHelper::substring($this->text, $start, $end); + return $this->substring($start, $end); } private function scanString(): string @@ -188,21 +240,21 @@ private function scanString(): string while (true) { if ($this->pos >= $this->len) { - $result .= StringHelper::substring($this->text, $start, $this->pos); + $result .= $this->substring($start, $this->pos); $this->scanError = ScanError::UnexpectedEndOfString; break; } - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); if ($ch === CC::DOUBLE_QUOTE) { - $result .= StringHelper::substring($this->text, $start, $this->pos); + $result .= $this->substring($start, $this->pos); $this->pos++; break; } if ($ch === CC::BACKSLASH) { - $result .= StringHelper::substring($this->text, $start, $this->pos); + $result .= $this->substring($start, $this->pos); $this->pos++; if ($this->pos >= $this->len) { @@ -210,7 +262,7 @@ private function scanString(): string break; } - $ch2 = StringHelper::charCodeAt($this->text, $this->pos++); + $ch2 = $this->charCodeAt($this->pos++); switch ($ch2) { case CC::DOUBLE_QUOTE: @@ -255,7 +307,7 @@ private function scanString(): string if ($ch >= 0 && $ch <= 0x1f) { if ($this->isLineBreak($ch)) { - $result .= StringHelper::substring($this->text, $start, $this->pos); + $result .= $this->substring($start, $this->pos); $this->scanError = ScanError::UnexpectedEndOfString; break; } else { @@ -285,14 +337,14 @@ private function scanNext(): SyntaxKind return $this->token = SyntaxKind::EOF; } - $code = StringHelper::charCodeAt($this->text, $this->pos); + $code = $this->charCodeAt($this->pos); // trivia: whitespace if ($this->isWhiteSpace($code)) { do { $this->pos++; $this->value .= StringHelper::fromCharCode($code); - $code = StringHelper::charCodeAt($this->text, $this->pos); + $code = $this->charCodeAt($this->pos); } while ($this->isWhiteSpace($code)); return $this->token = SyntaxKind::Trivia; @@ -302,7 +354,7 @@ private function scanNext(): SyntaxKind if ($this->isLineBreak($code)) { $this->pos++; $this->value .= StringHelper::fromCharCode($code); - if ($code === CC::CARRIAGE_RETURN && StringHelper::charCodeAt($this->text, $this->pos) === CC::LINE_FEED) { + if ($code === CC::CARRIAGE_RETURN && $this->charCodeAt($this->pos) === CC::LINE_FEED) { $this->pos++; $this->value .= "\n"; } @@ -342,31 +394,31 @@ private function scanNext(): SyntaxKind case CC::SLASH: $start = $this->pos; // Single-line comment - if (StringHelper::charCodeAt($this->text, $this->pos + 1) === CC::SLASH) { + if ($this->charCodeAt($this->pos + 1) === CC::SLASH) { $this->pos += 2; while ($this->pos < $this->len) { - if ($this->isLineBreak(StringHelper::charCodeAt($this->text, $this->pos))) { + if ($this->isLineBreak($this->charCodeAt($this->pos))) { break; } $this->pos++; } - $this->value = StringHelper::substring($this->text, $start, $this->pos); + $this->value = $this->substring($start, $this->pos); return $this->token = SyntaxKind::LineCommentTrivia; } // Multi-line comment - if (StringHelper::charCodeAt($this->text, $this->pos + 1) === CC::ASTERISK) { + if ($this->charCodeAt($this->pos + 1) === CC::ASTERISK) { $this->pos += 2; $safeLength = $this->len - 1; // For lookahead $commentClosed = false; while ($this->pos < $safeLength) { - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); - if ($ch === CC::ASTERISK && StringHelper::charCodeAt($this->text, $this->pos + 1) === CC::SLASH) { + if ($ch === CC::ASTERISK && $this->charCodeAt($this->pos + 1) === CC::SLASH) { $this->pos += 2; $commentClosed = true; break; @@ -375,7 +427,7 @@ private function scanNext(): SyntaxKind $this->pos++; if ($this->isLineBreak($ch)) { - if ($ch === CC::CARRIAGE_RETURN && StringHelper::charCodeAt($this->text, $this->pos) === CC::LINE_FEED) { + if ($ch === CC::CARRIAGE_RETURN && $this->charCodeAt($this->pos) === CC::LINE_FEED) { $this->pos++; } $this->lineNumber++; @@ -388,7 +440,7 @@ private function scanNext(): SyntaxKind $this->scanError = ScanError::UnexpectedEndOfComment; } - $this->value = StringHelper::substring($this->text, $start, $this->pos); + $this->value = $this->substring($start, $this->pos); return $this->token = SyntaxKind::BlockCommentTrivia; } @@ -401,7 +453,7 @@ private function scanNext(): SyntaxKind case CC::MINUS: $this->value .= StringHelper::fromCharCode($code); $this->pos++; - if ($this->pos === $this->len || !$this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + if ($this->pos === $this->len || !$this->isDigit($this->charCodeAt($this->pos))) { return $this->token = SyntaxKind::Unknown; } // found a minus, followed by a number so @@ -425,11 +477,11 @@ private function scanNext(): SyntaxKind // is a literal? Read the full word. while ($this->pos < $this->len && $this->isUnknownContentCharacter($code)) { $this->pos++; - $code = StringHelper::charCodeAt($this->text, $this->pos); + $code = $this->charCodeAt($this->pos); } if ($this->tokenOffset !== $this->pos) { - $this->value = StringHelper::substring($this->text, $this->tokenOffset, $this->pos); + $this->value = $this->substring($this->tokenOffset, $this->pos); // keywords: true, false, null switch ($this->value) { case 'true': diff --git a/tests/Unit/Scanner/ScannerComplexityTest.php b/tests/Unit/Scanner/ScannerComplexityTest.php new file mode 100644 index 0000000..dbea27a --- /dev/null +++ b/tests/Unit/Scanner/ScannerComplexityTest.php @@ -0,0 +1,109 @@ +scan() !== SyntaxKind::EOF) { + // Tokenizing for its own sake; the tokens are not the point here. + } + $elapsed = (hrtime(true) - $started) / 1e6; + + $best = min($best, $elapsed); + } + + return $best; +} + +describe('Scanner complexity', function () { + test('scanning 4x the input costs about 4x the time, not 16x', function () { + $small = localeShapedDocument(12_000); + $large = localeShapedDocument(48_000); + + // Guard the premise: the ratio below is only meaningful if the inputs + // really do differ by ~4x. + $sizeRatio = mb_strlen($large, 'UTF-8') / mb_strlen($small, 'UTF-8'); + expect($sizeRatio)->toBeGreaterThan(3.5)->toBeLessThan(4.5); + + $timeRatio = fastestScanMs($large) / fastestScanMs($small); + + expect($timeRatio)->toBeLessThan( + 8.0, + sprintf( + 'Scanning scaled %.1fx for %.1fx the input, which is quadratic, not linear.', + $timeRatio, + $sizeRatio, + ), + ); + }); + + test('multibyte input scales no worse than ASCII', function () { + // Every character access walks the string when it is indexed by + // character offset, so a document full of 3-byte characters must not + // be dramatically worse than an ASCII one of the same length. + $ascii = localeShapedDocument(24_000); + $cjk = localeShapedDocument(24_000, '日本語のテキストです'); + + $penalty = fastestScanMs($cjk) / fastestScanMs($ascii); + + expect($penalty)->toBeLessThan( + 4.0, + sprintf('Multibyte input scanned %.1fx slower than ASCII of the same length.', $penalty), + ); + }); +}); From 6335f2641d7ca42113794ea2046309e73d8a8289 Mon Sep 17 00:00:00 2001 From: Max Rice Date: Tue, 15 Sep 2026 20:24:40 -0400 Subject: [PATCH 2/2] Cover the scanner's multibyte handling directly The scanner reports offsets and lengths in characters rather than bytes, and the editor depends on that agreement: when the two disagreed, an edit after a multibyte character landed at the wrong position and mangled the output. The editor tests cover that end to end, but the scanner itself had no multibyte tests at all, so a change to how it addresses characters failed a long way from its cause. These assert offsets, token values, lengths, line and column numbers, and the token stream itself across 2-, 3- and 4-byte characters, Greek, Thai, Cyrillic, CJK and emoji. They pass against the scanner both before and after the switch to indexing a character array, which is the point: they describe behavior that was already correct rather than behavior the change introduced. --- tests/Unit/Scanner/ScannerMultibyteTest.php | 112 ++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 tests/Unit/Scanner/ScannerMultibyteTest.php diff --git a/tests/Unit/Scanner/ScannerMultibyteTest.php b/tests/Unit/Scanner/ScannerMultibyteTest.php new file mode 100644 index 0000000..74d87c5 --- /dev/null +++ b/tests/Unit/Scanner/ScannerMultibyteTest.php @@ -0,0 +1,112 @@ + ['abc', 7], + '2-byte (é)' => ['éée', 7], + '3-byte (日)' => ['日本語', 7], + '4-byte (🎉)' => ['🎉🎉🎉', 7], + 'mixed widths' => ['a é 日 🎉', 11], + ]; + + foreach ($cases as $label => [$key, $expectedOffset]) { + $scanner = Scanner::create('{"'.$key.'":"x"}'); + $scanner->scan(); // { + $scanner->scan(); // the key + $scanner->scan(); // : + $scanner->scan(); // the value + + expect($scanner->getToken())->toBe(SyntaxKind::StringLiteral, $label); + expect($scanner->getTokenValue())->toBe('x', $label); + expect($scanner->getTokenOffset())->toBe($expectedOffset, $label); + } + }); + + test('returns multibyte string values intact', function () { + $values = [ + 'greek' => 'Καλάθι αγορών', + 'cyrillic' => 'Корзина покупок', + 'thai' => 'ตะกร้าสินค้า', + 'japanese' => 'カートに追加する', + 'korean' => '장바구니에 추가', + 'chinese' => '加入購物車', + 'emoji' => 'Sold out 🎉🛒', + 'accented' => 'Panier — livraison incluse', + ]; + + foreach ($values as $label => $value) { + $scanner = Scanner::create('"'.$value.'"'); + + expect($scanner->scan())->toBe(SyntaxKind::StringLiteral, $label); + expect($scanner->getTokenValue())->toBe($value, $label); + expect($scanner->getTokenLength())->toBe(mb_strlen($value, 'UTF-8') + 2, $label); + } + }); + + test('counts lines and columns in characters across multibyte content', function () { + $json = "{\n \"日本語\": \"値\",\n \"target\": 1\n}"; + + $scanner = Scanner::create($json, ignoreTrivia: true); + while ($scanner->scan() !== SyntaxKind::EOF) { + if ($scanner->getTokenValue() === 'target') { + break; + } + } + + expect($scanner->getTokenStartLine())->toBe(2); + // Two spaces of indent, so the quote opening "target" is character 2. + expect($scanner->getTokenStartCharacter())->toBe(2); + }); + + test('scans a multibyte document to the same tokens as its ascii twin', function () { + // Same structure, same character count, different byte widths: the token + // stream must not notice the difference. + $ascii = '{"aaa":"bbb","ccc":[1,2,3]}'; + $wide = '{"日本語":"한국어","中文字":[1,2,3]}'; + + $kindsOf = function (string $json): array { + $scanner = Scanner::create($json); + $kinds = []; + while (($kind = $scanner->scan()) !== SyntaxKind::EOF) { + $kinds[] = $kind; + } + + return $kinds; + }; + + expect($kindsOf($wide))->toBe($kindsOf($ascii)); + }); + + test('handles a multibyte character split across a comment boundary', function () { + $json = "{\n // 日本語のコメント 🎉\n \"key\": \"値\"\n}"; + + $scanner = Scanner::create($json, ignoreTrivia: true); + $scanner->scan(); // { + + expect($scanner->scan())->toBe(SyntaxKind::StringLiteral); + expect($scanner->getTokenValue())->toBe('key'); + }); +});