Fix pretty() of constants so the output recompiles to the same model

tatsu/peg/basic.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: BSD-4-Clause
 from __future__ import annotations
 
+import re
 from functools import cached_property
 from typing import Any
 
@@ -14,6 +15,61 @@
 EOF_SYM = '$'
 EOL_SYM = '⏎'
 
+# Words that the `literal` rule in the constant syntax evaluates to
+# non-string values (booleans, None, and the JSON names).
+_LITERAL_KEYWORDS = frozenset(
+    {'True', 'False', 'None', 'true', 'false', 'null'},
+)
+
+# Bodies that the '`' =literal '`' alternative of the `constant` rule would
+# consume whole and evaluate, so writing them bare would not read back as
+# the original string (see grammar/tatsu.ebnf).
+_LITERAL_LOOKALIKES = (
+    re.compile(r'[-+]?\d+'),  # int
+    re.compile(r'[-+]?(?:\d+\.\d*|\d*\.\d+)(?:[Ee][-+]?\d+)?'),  # float
+    re.compile(r'-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?'),  # number
+    re.compile(r'\s+[_\w][_\w\d]*'),  # word, which drops leading whitespace
+)
+
+
+def _bare_safe(literal: str) -> bool:
+    """Whether ``literal`` written bare between backticks reads back as itself."""
+    if '`' in literal:
+        return False  # /`(.*?)`/ would stop at the first embedded backtick
+    if literal.splitlines() not in ([], [literal]):
+        return False  # pretty() output is re-indented line by line
+    if literal in _LITERAL_KEYWORDS:
+        return False
+    if literal[:1] in '\'"' or literal[:2] in {"r'", 'r"'}:
+        return False  # could be consumed as a (raw) string literal
+    return not any(lookalike.fullmatch(literal) for lookalike in _LITERAL_LOOKALIKES)
+
+
+def constant_source(literal: Any) -> str:
+    """Render a constant literal as grammar source that parses back to it."""
+    if not isinstance(literal, str):
+        return f'`{literal}`'  # int/float/bool/None read back via `literal`
+    if _bare_safe(literal):
+        return f'`{literal}`'
+    if "'" not in literal or '"' not in literal:
+        # repr() picks a quote character not present in the string, so the
+        # result contains no escaped quotes, on which the single-line string
+        # patterns stop short
+        return f'`{literal!r}`'
+    # Both quote characters present: use a multiline string with every quote
+    # escaped so the body cannot contain the closing delimiter.  Line breaks
+    # and tabs are escaped because multiline strings are trimmed of common
+    # indentation when read back, and whitespace at either end is escaped
+    # because it is stripped.
+    escaped = literal.replace('\\', '\\\\').replace("'", "\\'")
+    for char in '\n\r\t\v\f\x1c\x1d\x1e\x85\u2028\u2029':
+        escaped = escaped.replace(char, f'\\u{ord(char):04x}')
+    if escaped[:1].isspace():
+        escaped = f'\\u{ord(escaped[0]):04x}' + escaped[1:]
+    if escaped[-1:].isspace():
+        escaped = escaped[:-1] + f'\\u{ord(escaped[-1]):04x}'
+    return f"`'''{escaped}'''`"
+
 
 @nodedataclass
 class Dot(Leaf):
@@ -132,7 +188,7 @@ def _first(self, k: int, f: dict[str, ffset]) -> ffset:
 
     def _pretty(self, lean=False):
         _ = lean
-        return f'`{self.literal!s}`'
+        return constant_source(self.literal)
 
     @cached_property
     def _nullable(self) -> bool:

tests/grammar/pretty_test.py

@@ -4,8 +4,10 @@
 
 import unittest
 
+import pytest
+
 from tatsu.api import compile
-from tatsu.util import trim
+from tatsu.util import asjson, trim
 
 
 class PrettyTests(unittest.TestCase):
@@ -60,3 +62,86 @@ def test_slashed_pattern(self):
         self.assertEqual('abc/123', ast)
         print(model.pretty())
         self.assertEqual(trim(grammar), model.pretty())
+
+
+def assert_pretty_roundtrip(grammar):
+    """The output of pretty() must compile back to an identical model."""
+    model = compile(grammar)
+    pretty = model.pretty()
+    reparsed = compile(pretty)  # pretty output must still be valid grammar
+    assert asjson(reparsed) == asjson(model)
+    assert reparsed.pretty() == pretty  # pretty() is a fixpoint
+    return pretty
+
+
+CONSTANT_ROUNDTRIP_GRAMMARS = [
+    # bare constants that were already stable
+    'start = `plain` $ ;',
+    'start = `two words` $ ;',
+    'start = ` padded out ` $ ;',
+    "start = `r'raw'` $ ;",
+    # non-string literal constants
+    'start = `42` $ ;',
+    'start = `-42` $ ;',
+    'start = `4.5` $ ;',
+    'start = `True` $ ;',
+    'start = `False` $ ;',
+    'start = `None` $ ;',
+    # backticks and line breaks in the value used to break the reparse
+    'start = ```a`b``` $ ;',
+    'start = ```a\nb``` $ ;',
+    "start = `'a\\nb'` $ ;",
+    "start = `'a\\rb'` $ ;",
+    "start = `'a\\tb'` $ ;",
+    "start = `'ab`'` $ ;",
+    "start = `'a```b'` $ ;",
+    # string values that used to be re-read as other literal types
+    "start = `'42'` $ ;",
+    "start = `'4.5'` $ ;",
+    "start = `'1e5'` $ ;",
+    "start = `'-7'` $ ;",
+    "start = `'True'` $ ;",
+    "start = `'None'` $ ;",
+    "start = `'true'` $ ;",
+    "start = `'null'` $ ;",
+    "start = `' x'` $ ;",
+    "start = `'0x1f'` $ ;",
+    # values holding both kinds of quotes need the multiline string form
+    'start = `\'\'\'it\\\'s "quoted"\'\'\'` $ ;',
+    'start = `\'\'\'it\\\'s "q"\\nnext\'\'\'` $ ;',
+    'start = \'x\' `\'\'\'it\\\'s "q"\\nnext\'\'\'` \'y\' $ ;',
+    'start = `\'\'\'a\\\'\\\'\\\'b "q" `x\'\'\'` $ ;',
+    'start = `\'\'\'back\\\\slash \\\'n\\\' "q" `t\'\'\'` $ ;',
+    # a value that itself looks like a quoted string
+    "start = `'\\x27abc\\x27'` $ ;",
+    # both quotes with whitespace at the ends, which trim() would eat
+    'start = `\'\'\'\\u0020it\\\'s "q"\\u0020\'\'\'` $ ;',
+    # both quotes with an indented continuation line
+    'start = `\'\'\'it\\\'s "q"\\n  indented\'\'\'` $ ;',
+    # both quotes, a backtick, and whitespace at the ends
+    'start = `\'\'\'\\u0020it\\\'s "q" `t\\u0020\'\'\'` $ ;',
+    # alerts share the constant rendering
+    'start = ^`warning` $ ;',
+    'start = ^^```w`x``` $ ;',
+    'start = ^```a\nb``` $ ;',
+]
+
+
+@pytest.mark.parametrize('grammar', CONSTANT_ROUNDTRIP_GRAMMARS)
+def test_constant_pretty_roundtrip(grammar):
+    assert_pretty_roundtrip(grammar)
+
+
+def test_constant_pretty_value_survives():
+    grammar = "start = c:`'it is\\n42'` $ ;"
+    model = compile(grammar)
+    reparsed = compile(model.pretty())
+    assert model.parse('') == reparsed.parse('')
+    assert reparsed.parse('')['c'] == 'it is\n42'
+
+
+def test_constant_pretty_stays_bare_for_plain_literals():
+    assert '`plain`' in compile('start = `plain` $ ;').pretty()
+    assert '`two words`' in compile('start = `two words` $ ;').pretty()
+    assert '`42`' in compile('start = `42` $ ;').pretty()
+    assert '`True`' in compile('start = `True` $ ;').pretty()