diff --git a/tatsu/peg/basic.py b/tatsu/peg/basic.py index 22135aec..52a18024 100644 --- a/tatsu/peg/basic.py +++ b/tatsu/peg/basic.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: BSD-4-Clause from __future__ import annotations +import re from functools import cached_property from typing import Any @@ -14,6 +15,61 @@ EOF_SYM = '$' EOL_SYM = '⏎' +# Words that the `literal` rule in the constant syntax evaluates to +# non-string values (booleans, None, and the JSON names). +_LITERAL_KEYWORDS = frozenset( + {'True', 'False', 'None', 'true', 'false', 'null'}, +) + +# Bodies that the '`' =literal '`' alternative of the `constant` rule would +# consume whole and evaluate, so writing them bare would not read back as +# the original string (see grammar/tatsu.ebnf). +_LITERAL_LOOKALIKES = ( + re.compile(r'[-+]?\d+'), # int + re.compile(r'[-+]?(?:\d+\.\d*|\d*\.\d+)(?:[Ee][-+]?\d+)?'), # float + re.compile(r'-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?'), # number + re.compile(r'\s+[_\w][_\w\d]*'), # word, which drops leading whitespace +) + + +def _bare_safe(literal: str) -> bool: + """Whether ``literal`` written bare between backticks reads back as itself.""" + if '`' in literal: + return False # /`(.*?)`/ would stop at the first embedded backtick + if literal.splitlines() not in ([], [literal]): + return False # pretty() output is re-indented line by line + if literal in _LITERAL_KEYWORDS: + return False + if literal[:1] in '\'"' or literal[:2] in {"r'", 'r"'}: + return False # could be consumed as a (raw) string literal + return not any(lookalike.fullmatch(literal) for lookalike in _LITERAL_LOOKALIKES) + + +def constant_source(literal: Any) -> str: + """Render a constant literal as grammar source that parses back to it.""" + if not isinstance(literal, str): + return f'`{literal}`' # int/float/bool/None read back via `literal` + if _bare_safe(literal): + return f'`{literal}`' + if "'" not in literal or '"' not in literal: + # repr() picks a quote character not present in the string, so the + # result contains no escaped quotes, on which the single-line string + # patterns stop short + return f'`{literal!r}`' + # Both quote characters present: use a multiline string with every quote + # escaped so the body cannot contain the closing delimiter. Line breaks + # and tabs are escaped because multiline strings are trimmed of common + # indentation when read back, and whitespace at either end is escaped + # because it is stripped. + escaped = literal.replace('\\', '\\\\').replace("'", "\\'") + for char in '\n\r\t\v\f\x1c\x1d\x1e\x85\u2028\u2029': + escaped = escaped.replace(char, f'\\u{ord(char):04x}') + if escaped[:1].isspace(): + escaped = f'\\u{ord(escaped[0]):04x}' + escaped[1:] + if escaped[-1:].isspace(): + escaped = escaped[:-1] + f'\\u{ord(escaped[-1]):04x}' + return f"`'''{escaped}'''`" + @nodedataclass class Dot(Leaf): @@ -132,7 +188,7 @@ def _first(self, k: int, f: dict[str, ffset]) -> ffset: def _pretty(self, lean=False): _ = lean - return f'`{self.literal!s}`' + return constant_source(self.literal) @cached_property def _nullable(self) -> bool: diff --git a/tests/grammar/pretty_test.py b/tests/grammar/pretty_test.py index 07e7657d..dbbb356a 100644 --- a/tests/grammar/pretty_test.py +++ b/tests/grammar/pretty_test.py @@ -4,8 +4,10 @@ import unittest +import pytest + from tatsu.api import compile -from tatsu.util import trim +from tatsu.util import asjson, trim class PrettyTests(unittest.TestCase): @@ -60,3 +62,86 @@ def test_slashed_pattern(self): self.assertEqual('abc/123', ast) print(model.pretty()) self.assertEqual(trim(grammar), model.pretty()) + + +def assert_pretty_roundtrip(grammar): + """The output of pretty() must compile back to an identical model.""" + model = compile(grammar) + pretty = model.pretty() + reparsed = compile(pretty) # pretty output must still be valid grammar + assert asjson(reparsed) == asjson(model) + assert reparsed.pretty() == pretty # pretty() is a fixpoint + return pretty + + +CONSTANT_ROUNDTRIP_GRAMMARS = [ + # bare constants that were already stable + 'start = `plain` $ ;', + 'start = `two words` $ ;', + 'start = ` padded out ` $ ;', + "start = `r'raw'` $ ;", + # non-string literal constants + 'start = `42` $ ;', + 'start = `-42` $ ;', + 'start = `4.5` $ ;', + 'start = `True` $ ;', + 'start = `False` $ ;', + 'start = `None` $ ;', + # backticks and line breaks in the value used to break the reparse + 'start = ```a`b``` $ ;', + 'start = ```a\nb``` $ ;', + "start = `'a\\nb'` $ ;", + "start = `'a\\rb'` $ ;", + "start = `'a\\tb'` $ ;", + "start = `'ab`'` $ ;", + "start = `'a```b'` $ ;", + # string values that used to be re-read as other literal types + "start = `'42'` $ ;", + "start = `'4.5'` $ ;", + "start = `'1e5'` $ ;", + "start = `'-7'` $ ;", + "start = `'True'` $ ;", + "start = `'None'` $ ;", + "start = `'true'` $ ;", + "start = `'null'` $ ;", + "start = `' x'` $ ;", + "start = `'0x1f'` $ ;", + # values holding both kinds of quotes need the multiline string form + 'start = `\'\'\'it\\\'s "quoted"\'\'\'` $ ;', + 'start = `\'\'\'it\\\'s "q"\\nnext\'\'\'` $ ;', + 'start = \'x\' `\'\'\'it\\\'s "q"\\nnext\'\'\'` \'y\' $ ;', + 'start = `\'\'\'a\\\'\\\'\\\'b "q" `x\'\'\'` $ ;', + 'start = `\'\'\'back\\\\slash \\\'n\\\' "q" `t\'\'\'` $ ;', + # a value that itself looks like a quoted string + "start = `'\\x27abc\\x27'` $ ;", + # both quotes with whitespace at the ends, which trim() would eat + 'start = `\'\'\'\\u0020it\\\'s "q"\\u0020\'\'\'` $ ;', + # both quotes with an indented continuation line + 'start = `\'\'\'it\\\'s "q"\\n indented\'\'\'` $ ;', + # both quotes, a backtick, and whitespace at the ends + 'start = `\'\'\'\\u0020it\\\'s "q" `t\\u0020\'\'\'` $ ;', + # alerts share the constant rendering + 'start = ^`warning` $ ;', + 'start = ^^```w`x``` $ ;', + 'start = ^```a\nb``` $ ;', +] + + +@pytest.mark.parametrize('grammar', CONSTANT_ROUNDTRIP_GRAMMARS) +def test_constant_pretty_roundtrip(grammar): + assert_pretty_roundtrip(grammar) + + +def test_constant_pretty_value_survives(): + grammar = "start = c:`'it is\\n42'` $ ;" + model = compile(grammar) + reparsed = compile(model.pretty()) + assert model.parse('') == reparsed.parse('') + assert reparsed.parse('')['c'] == 'it is\n42' + + +def test_constant_pretty_stays_bare_for_plain_literals(): + assert '`plain`' in compile('start = `plain` $ ;').pretty() + assert '`two words`' in compile('start = `two words` $ ;').pretty() + assert '`42`' in compile('start = `42` $ ;').pretty() + assert '`True`' in compile('start = `True` $ ;').pretty()