diff --git a/src/Scanner/Scanner.php b/src/Scanner/Scanner.php index 3f10a9f..00416ef 100644 --- a/src/Scanner/Scanner.php +++ b/src/Scanner/Scanner.php @@ -13,7 +13,19 @@ */ final class Scanner implements JsonScanner { - private string $text; + /** + * The document split into individual characters, indexed by character offset. + * + * The scanner addresses text by character offset, not byte offset. Resolving + * such an offset against a UTF-8 string means walking it from the start, so + * doing that once per character makes scanning quadratic in the length of the + * document. Splitting once up front turns every subsequent access into an + * array lookup, at a memory cost of roughly 46x the source size. + * + * @var list + */ + private array $chars; + private int $len; private int $pos = 0; private string $value = ''; @@ -40,12 +52,52 @@ private function __construct( string $text, private readonly bool $ignoreTrivia ) { - $this->text = $text; - $this->len = StringHelper::length($text); + $this->chars = mb_str_split($text, 1, 'UTF-8'); + $this->len = count($this->chars); $this->token = SyntaxKind::Unknown; $this->scanError = ScanError::None; } + /** + * Get the Unicode code point at a character offset, or 0 when out of bounds. + */ + private function charCodeAt(int $pos): int + { + $char = $this->chars[$pos] ?? ''; + + if ($char === '') { + return 0; + } + + // A single-byte UTF-8 character is its own code point, which covers + // essentially all of a JSON document's structure and keys. + if (strlen($char) === 1) { + return ord($char); + } + + $code = mb_ord($char, 'UTF-8'); + + return $code !== false ? $code : 0; + } + + /** + * Get the text between two character offsets, or to the end when $end is null. + */ + private function substring(int $start, ?int $end = null): string + { + if ($end === null) { + return implode('', array_slice($this->chars, $start)); + } + + $length = $end - $start; + + if ($length < 0) { + return ''; + } + + return implode('', array_slice($this->chars, $start, $length)); + } + public function setPosition(int $pos): void { $this->pos = $pos; @@ -106,7 +158,7 @@ private function scanHexDigits(int $count, bool $exact = false): int $value = 0; while ($digits < $count || !$exact) { - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); if ($ch >= CC::DIGIT_0 && $ch <= CC::DIGIT_9) { $value = $value * 16 + $ch - CC::DIGIT_0; @@ -133,42 +185,42 @@ private function scanNumber(): string { $start = $this->pos; - if (StringHelper::charCodeAt($this->text, $this->pos) === CC::DIGIT_0) { + if ($this->charCodeAt($this->pos) === CC::DIGIT_0) { $this->pos++; } else { $this->pos++; - while ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + while ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; } } - if ($this->pos < $this->len && StringHelper::charCodeAt($this->text, $this->pos) === CC::DOT) { + if ($this->pos < $this->len && $this->charCodeAt($this->pos) === CC::DOT) { $this->pos++; - if ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + if ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; - while ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + while ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; } } else { $this->scanError = ScanError::UnexpectedEndOfNumber; - return StringHelper::substring($this->text, $start, $this->pos); + return $this->substring($start, $this->pos); } } $end = $this->pos; if ($this->pos < $this->len) { - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); if ($ch === CC::UPPER_E || $ch === CC::LOWER_E) { $this->pos++; if ($this->pos < $this->len) { - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); if ($ch === CC::PLUS || $ch === CC::MINUS) { $this->pos++; } } - if ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + if ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; - while ($this->pos < $this->len && $this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + while ($this->pos < $this->len && $this->isDigit($this->charCodeAt($this->pos))) { $this->pos++; } $end = $this->pos; @@ -178,7 +230,7 @@ private function scanNumber(): string } } - return StringHelper::substring($this->text, $start, $end); + return $this->substring($start, $end); } private function scanString(): string @@ -188,21 +240,21 @@ private function scanString(): string while (true) { if ($this->pos >= $this->len) { - $result .= StringHelper::substring($this->text, $start, $this->pos); + $result .= $this->substring($start, $this->pos); $this->scanError = ScanError::UnexpectedEndOfString; break; } - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); if ($ch === CC::DOUBLE_QUOTE) { - $result .= StringHelper::substring($this->text, $start, $this->pos); + $result .= $this->substring($start, $this->pos); $this->pos++; break; } if ($ch === CC::BACKSLASH) { - $result .= StringHelper::substring($this->text, $start, $this->pos); + $result .= $this->substring($start, $this->pos); $this->pos++; if ($this->pos >= $this->len) { @@ -210,7 +262,7 @@ private function scanString(): string break; } - $ch2 = StringHelper::charCodeAt($this->text, $this->pos++); + $ch2 = $this->charCodeAt($this->pos++); switch ($ch2) { case CC::DOUBLE_QUOTE: @@ -255,7 +307,7 @@ private function scanString(): string if ($ch >= 0 && $ch <= 0x1f) { if ($this->isLineBreak($ch)) { - $result .= StringHelper::substring($this->text, $start, $this->pos); + $result .= $this->substring($start, $this->pos); $this->scanError = ScanError::UnexpectedEndOfString; break; } else { @@ -285,14 +337,14 @@ private function scanNext(): SyntaxKind return $this->token = SyntaxKind::EOF; } - $code = StringHelper::charCodeAt($this->text, $this->pos); + $code = $this->charCodeAt($this->pos); // trivia: whitespace if ($this->isWhiteSpace($code)) { do { $this->pos++; $this->value .= StringHelper::fromCharCode($code); - $code = StringHelper::charCodeAt($this->text, $this->pos); + $code = $this->charCodeAt($this->pos); } while ($this->isWhiteSpace($code)); return $this->token = SyntaxKind::Trivia; @@ -302,7 +354,7 @@ private function scanNext(): SyntaxKind if ($this->isLineBreak($code)) { $this->pos++; $this->value .= StringHelper::fromCharCode($code); - if ($code === CC::CARRIAGE_RETURN && StringHelper::charCodeAt($this->text, $this->pos) === CC::LINE_FEED) { + if ($code === CC::CARRIAGE_RETURN && $this->charCodeAt($this->pos) === CC::LINE_FEED) { $this->pos++; $this->value .= "\n"; } @@ -342,31 +394,31 @@ private function scanNext(): SyntaxKind case CC::SLASH: $start = $this->pos; // Single-line comment - if (StringHelper::charCodeAt($this->text, $this->pos + 1) === CC::SLASH) { + if ($this->charCodeAt($this->pos + 1) === CC::SLASH) { $this->pos += 2; while ($this->pos < $this->len) { - if ($this->isLineBreak(StringHelper::charCodeAt($this->text, $this->pos))) { + if ($this->isLineBreak($this->charCodeAt($this->pos))) { break; } $this->pos++; } - $this->value = StringHelper::substring($this->text, $start, $this->pos); + $this->value = $this->substring($start, $this->pos); return $this->token = SyntaxKind::LineCommentTrivia; } // Multi-line comment - if (StringHelper::charCodeAt($this->text, $this->pos + 1) === CC::ASTERISK) { + if ($this->charCodeAt($this->pos + 1) === CC::ASTERISK) { $this->pos += 2; $safeLength = $this->len - 1; // For lookahead $commentClosed = false; while ($this->pos < $safeLength) { - $ch = StringHelper::charCodeAt($this->text, $this->pos); + $ch = $this->charCodeAt($this->pos); - if ($ch === CC::ASTERISK && StringHelper::charCodeAt($this->text, $this->pos + 1) === CC::SLASH) { + if ($ch === CC::ASTERISK && $this->charCodeAt($this->pos + 1) === CC::SLASH) { $this->pos += 2; $commentClosed = true; break; @@ -375,7 +427,7 @@ private function scanNext(): SyntaxKind $this->pos++; if ($this->isLineBreak($ch)) { - if ($ch === CC::CARRIAGE_RETURN && StringHelper::charCodeAt($this->text, $this->pos) === CC::LINE_FEED) { + if ($ch === CC::CARRIAGE_RETURN && $this->charCodeAt($this->pos) === CC::LINE_FEED) { $this->pos++; } $this->lineNumber++; @@ -388,7 +440,7 @@ private function scanNext(): SyntaxKind $this->scanError = ScanError::UnexpectedEndOfComment; } - $this->value = StringHelper::substring($this->text, $start, $this->pos); + $this->value = $this->substring($start, $this->pos); return $this->token = SyntaxKind::BlockCommentTrivia; } @@ -401,7 +453,7 @@ private function scanNext(): SyntaxKind case CC::MINUS: $this->value .= StringHelper::fromCharCode($code); $this->pos++; - if ($this->pos === $this->len || !$this->isDigit(StringHelper::charCodeAt($this->text, $this->pos))) { + if ($this->pos === $this->len || !$this->isDigit($this->charCodeAt($this->pos))) { return $this->token = SyntaxKind::Unknown; } // found a minus, followed by a number so @@ -425,11 +477,11 @@ private function scanNext(): SyntaxKind // is a literal? Read the full word. while ($this->pos < $this->len && $this->isUnknownContentCharacter($code)) { $this->pos++; - $code = StringHelper::charCodeAt($this->text, $this->pos); + $code = $this->charCodeAt($this->pos); } if ($this->tokenOffset !== $this->pos) { - $this->value = StringHelper::substring($this->text, $this->tokenOffset, $this->pos); + $this->value = $this->substring($this->tokenOffset, $this->pos); // keywords: true, false, null switch ($this->value) { case 'true': diff --git a/tests/Unit/Scanner/ScannerComplexityTest.php b/tests/Unit/Scanner/ScannerComplexityTest.php new file mode 100644 index 0000000..dbea27a --- /dev/null +++ b/tests/Unit/Scanner/ScannerComplexityTest.php @@ -0,0 +1,109 @@ +scan() !== SyntaxKind::EOF) { + // Tokenizing for its own sake; the tokens are not the point here. + } + $elapsed = (hrtime(true) - $started) / 1e6; + + $best = min($best, $elapsed); + } + + return $best; +} + +describe('Scanner complexity', function () { + test('scanning 4x the input costs about 4x the time, not 16x', function () { + $small = localeShapedDocument(12_000); + $large = localeShapedDocument(48_000); + + // Guard the premise: the ratio below is only meaningful if the inputs + // really do differ by ~4x. + $sizeRatio = mb_strlen($large, 'UTF-8') / mb_strlen($small, 'UTF-8'); + expect($sizeRatio)->toBeGreaterThan(3.5)->toBeLessThan(4.5); + + $timeRatio = fastestScanMs($large) / fastestScanMs($small); + + expect($timeRatio)->toBeLessThan( + 8.0, + sprintf( + 'Scanning scaled %.1fx for %.1fx the input, which is quadratic, not linear.', + $timeRatio, + $sizeRatio, + ), + ); + }); + + test('multibyte input scales no worse than ASCII', function () { + // Every character access walks the string when it is indexed by + // character offset, so a document full of 3-byte characters must not + // be dramatically worse than an ASCII one of the same length. + $ascii = localeShapedDocument(24_000); + $cjk = localeShapedDocument(24_000, '日本語のテキストです'); + + $penalty = fastestScanMs($cjk) / fastestScanMs($ascii); + + expect($penalty)->toBeLessThan( + 4.0, + sprintf('Multibyte input scanned %.1fx slower than ASCII of the same length.', $penalty), + ); + }); +}); diff --git a/tests/Unit/Scanner/ScannerMultibyteTest.php b/tests/Unit/Scanner/ScannerMultibyteTest.php new file mode 100644 index 0000000..74d87c5 --- /dev/null +++ b/tests/Unit/Scanner/ScannerMultibyteTest.php @@ -0,0 +1,112 @@ + ['abc', 7], + '2-byte (é)' => ['éée', 7], + '3-byte (日)' => ['日本語', 7], + '4-byte (🎉)' => ['🎉🎉🎉', 7], + 'mixed widths' => ['a é 日 🎉', 11], + ]; + + foreach ($cases as $label => [$key, $expectedOffset]) { + $scanner = Scanner::create('{"'.$key.'":"x"}'); + $scanner->scan(); // { + $scanner->scan(); // the key + $scanner->scan(); // : + $scanner->scan(); // the value + + expect($scanner->getToken())->toBe(SyntaxKind::StringLiteral, $label); + expect($scanner->getTokenValue())->toBe('x', $label); + expect($scanner->getTokenOffset())->toBe($expectedOffset, $label); + } + }); + + test('returns multibyte string values intact', function () { + $values = [ + 'greek' => 'Καλάθι αγορών', + 'cyrillic' => 'Корзина покупок', + 'thai' => 'ตะกร้าสินค้า', + 'japanese' => 'カートに追加する', + 'korean' => '장바구니에 추가', + 'chinese' => '加入購物車', + 'emoji' => 'Sold out 🎉🛒', + 'accented' => 'Panier — livraison incluse', + ]; + + foreach ($values as $label => $value) { + $scanner = Scanner::create('"'.$value.'"'); + + expect($scanner->scan())->toBe(SyntaxKind::StringLiteral, $label); + expect($scanner->getTokenValue())->toBe($value, $label); + expect($scanner->getTokenLength())->toBe(mb_strlen($value, 'UTF-8') + 2, $label); + } + }); + + test('counts lines and columns in characters across multibyte content', function () { + $json = "{\n \"日本語\": \"値\",\n \"target\": 1\n}"; + + $scanner = Scanner::create($json, ignoreTrivia: true); + while ($scanner->scan() !== SyntaxKind::EOF) { + if ($scanner->getTokenValue() === 'target') { + break; + } + } + + expect($scanner->getTokenStartLine())->toBe(2); + // Two spaces of indent, so the quote opening "target" is character 2. + expect($scanner->getTokenStartCharacter())->toBe(2); + }); + + test('scans a multibyte document to the same tokens as its ascii twin', function () { + // Same structure, same character count, different byte widths: the token + // stream must not notice the difference. + $ascii = '{"aaa":"bbb","ccc":[1,2,3]}'; + $wide = '{"日本語":"한국어","中文字":[1,2,3]}'; + + $kindsOf = function (string $json): array { + $scanner = Scanner::create($json); + $kinds = []; + while (($kind = $scanner->scan()) !== SyntaxKind::EOF) { + $kinds[] = $kind; + } + + return $kinds; + }; + + expect($kindsOf($wide))->toBe($kindsOf($ascii)); + }); + + test('handles a multibyte character split across a comment boundary', function () { + $json = "{\n // 日本語のコメント 🎉\n \"key\": \"値\"\n}"; + + $scanner = Scanner::create($json, ignoreTrivia: true); + $scanner->scan(); // { + + expect($scanner->scan())->toBe(SyntaxKind::StringLiteral); + expect($scanner->getTokenValue())->toBe('key'); + }); +});