From 785362709e756067df9bea2755b25d296916377f Mon Sep 17 00:00:00 2001 From: Vincent Gao Date: Wed, 5 Aug 2026 06:41:40 +0200 Subject: [PATCH] Fix pretty() of constants so the output recompiles to the same model Constant._pretty() wrote every literal bare between single backticks. Values holding a backtick or a line break produced grammars that fail to recompile, and values that read back through the 'literal' rule were silently reinterpreted ('42' -> 42, 'True' -> True, ' x' -> 'x'). Alert inherits the rendering. Pick a source form that reads back to the same value: bare when safe, repr() when a single quote kind suffices, and an escaped '''...''' string when the value holds both kinds (multiline strings are trimmed and stripped when read back, so line breaks, tabs, and end whitespace are escaped as well). --- tatsu/peg/basic.py | 58 +++++++++++++++++++++++- tests/grammar/pretty_test.py | 87 +++++++++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/tatsu/peg/basic.py b/tatsu/peg/basic.py index 22135aec..52a18024 100644 --- a/tatsu/peg/basic.py +++ b/tatsu/peg/basic.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: BSD-4-Clause from __future__ import annotations +import re from functools import cached_property from typing import Any @@ -14,6 +15,61 @@ EOF_SYM = '$' EOL_SYM = '⏎' +# Words that the `literal` rule in the constant syntax evaluates to +# non-string values (booleans, None, and the JSON names). +_LITERAL_KEYWORDS = frozenset( + {'True', 'False', 'None', 'true', 'false', 'null'}, +) + +# Bodies that the '`' =literal '`' alternative of the `constant` rule would +# consume whole and evaluate, so writing them bare would not read back as +# the original string (see grammar/tatsu.ebnf). +_LITERAL_LOOKALIKES = ( + re.compile(r'[-+]?\d+'), # int + re.compile(r'[-+]?(?:\d+\.\d*|\d*\.\d+)(?:[Ee][-+]?\d+)?'), # float + re.compile(r'-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?'), # number + re.compile(r'\s+[_\w][_\w\d]*'), # word, which drops leading whitespace +) + + +def _bare_safe(literal: str) -> bool: + """Whether ``literal`` written bare between backticks reads back as itself.""" + if '`' in literal: + return False # /`(.*?)`/ would stop at the first embedded backtick + if literal.splitlines() not in ([], [literal]): + return False # pretty() output is re-indented line by line + if literal in _LITERAL_KEYWORDS: + return False + if literal[:1] in '\'"' or literal[:2] in {"r'", 'r"'}: + return False # could be consumed as a (raw) string literal + return not any(lookalike.fullmatch(literal) for lookalike in _LITERAL_LOOKALIKES) + + +def constant_source(literal: Any) -> str: + """Render a constant literal as grammar source that parses back to it.""" + if not isinstance(literal, str): + return f'`{literal}`' # int/float/bool/None read back via `literal` + if _bare_safe(literal): + return f'`{literal}`' + if "'" not in literal or '"' not in literal: + # repr() picks a quote character not present in the string, so the + # result contains no escaped quotes, on which the single-line string + # patterns stop short + return f'`{literal!r}`' + # Both quote characters present: use a multiline string with every quote + # escaped so the body cannot contain the closing delimiter. Line breaks + # and tabs are escaped because multiline strings are trimmed of common + # indentation when read back, and whitespace at either end is escaped + # because it is stripped. + escaped = literal.replace('\\', '\\\\').replace("'", "\\'") + for char in '\n\r\t\v\f\x1c\x1d\x1e\x85\u2028\u2029': + escaped = escaped.replace(char, f'\\u{ord(char):04x}') + if escaped[:1].isspace(): + escaped = f'\\u{ord(escaped[0]):04x}' + escaped[1:] + if escaped[-1:].isspace(): + escaped = escaped[:-1] + f'\\u{ord(escaped[-1]):04x}' + return f"`'''{escaped}'''`" + @nodedataclass class Dot(Leaf): @@ -132,7 +188,7 @@ def _first(self, k: int, f: dict[str, ffset]) -> ffset: def _pretty(self, lean=False): _ = lean - return f'`{self.literal!s}`' + return constant_source(self.literal) @cached_property def _nullable(self) -> bool: diff --git a/tests/grammar/pretty_test.py b/tests/grammar/pretty_test.py index 07e7657d..dbbb356a 100644 --- a/tests/grammar/pretty_test.py +++ b/tests/grammar/pretty_test.py @@ -4,8 +4,10 @@ import unittest +import pytest + from tatsu.api import compile -from tatsu.util import trim +from tatsu.util import asjson, trim class PrettyTests(unittest.TestCase): @@ -60,3 +62,86 @@ def test_slashed_pattern(self): self.assertEqual('abc/123', ast) print(model.pretty()) self.assertEqual(trim(grammar), model.pretty()) + + +def assert_pretty_roundtrip(grammar): + """The output of pretty() must compile back to an identical model.""" + model = compile(grammar) + pretty = model.pretty() + reparsed = compile(pretty) # pretty output must still be valid grammar + assert asjson(reparsed) == asjson(model) + assert reparsed.pretty() == pretty # pretty() is a fixpoint + return pretty + + +CONSTANT_ROUNDTRIP_GRAMMARS = [ + # bare constants that were already stable + 'start = `plain` $ ;', + 'start = `two words` $ ;', + 'start = ` padded out ` $ ;', + "start = `r'raw'` $ ;", + # non-string literal constants + 'start = `42` $ ;', + 'start = `-42` $ ;', + 'start = `4.5` $ ;', + 'start = `True` $ ;', + 'start = `False` $ ;', + 'start = `None` $ ;', + # backticks and line breaks in the value used to break the reparse + 'start = ```a`b``` $ ;', + 'start = ```a\nb``` $ ;', + "start = `'a\\nb'` $ ;", + "start = `'a\\rb'` $ ;", + "start = `'a\\tb'` $ ;", + "start = `'ab`'` $ ;", + "start = `'a```b'` $ ;", + # string values that used to be re-read as other literal types + "start = `'42'` $ ;", + "start = `'4.5'` $ ;", + "start = `'1e5'` $ ;", + "start = `'-7'` $ ;", + "start = `'True'` $ ;", + "start = `'None'` $ ;", + "start = `'true'` $ ;", + "start = `'null'` $ ;", + "start = `' x'` $ ;", + "start = `'0x1f'` $ ;", + # values holding both kinds of quotes need the multiline string form + 'start = `\'\'\'it\\\'s "quoted"\'\'\'` $ ;', + 'start = `\'\'\'it\\\'s "q"\\nnext\'\'\'` $ ;', + 'start = \'x\' `\'\'\'it\\\'s "q"\\nnext\'\'\'` \'y\' $ ;', + 'start = `\'\'\'a\\\'\\\'\\\'b "q" `x\'\'\'` $ ;', + 'start = `\'\'\'back\\\\slash \\\'n\\\' "q" `t\'\'\'` $ ;', + # a value that itself looks like a quoted string + "start = `'\\x27abc\\x27'` $ ;", + # both quotes with whitespace at the ends, which trim() would eat + 'start = `\'\'\'\\u0020it\\\'s "q"\\u0020\'\'\'` $ ;', + # both quotes with an indented continuation line + 'start = `\'\'\'it\\\'s "q"\\n indented\'\'\'` $ ;', + # both quotes, a backtick, and whitespace at the ends + 'start = `\'\'\'\\u0020it\\\'s "q" `t\\u0020\'\'\'` $ ;', + # alerts share the constant rendering + 'start = ^`warning` $ ;', + 'start = ^^```w`x``` $ ;', + 'start = ^```a\nb``` $ ;', +] + + +@pytest.mark.parametrize('grammar', CONSTANT_ROUNDTRIP_GRAMMARS) +def test_constant_pretty_roundtrip(grammar): + assert_pretty_roundtrip(grammar) + + +def test_constant_pretty_value_survives(): + grammar = "start = c:`'it is\\n42'` $ ;" + model = compile(grammar) + reparsed = compile(model.pretty()) + assert model.parse('') == reparsed.parse('') + assert reparsed.parse('')['c'] == 'it is\n42' + + +def test_constant_pretty_stays_bare_for_plain_literals(): + assert '`plain`' in compile('start = `plain` $ ;').pretty() + assert '`two words`' in compile('start = `two words` $ ;').pretty() + assert '`42`' in compile('start = `42` $ ;').pretty() + assert '`True`' in compile('start = `True` $ ;').pretty()