From 2199d5f02205b6abf2386e914aae12a65e51393f Mon Sep 17 00:00:00 2001 From: git-hulk Date: Thu, 10 Sep 2026 17:21:09 +0800 Subject: [PATCH] Fix quoted tokens and comment handling Quoted identifiers stop at escaped delimiters, dollar strings become identifiers, and block comments end at the first nested closing marker. Hash comments are also rejected. Preserve identifier spelling, normalize complete dollar strings into the existing escaped StringLiteral representation, and recognize nested block and supported hash comments. Add lexer, invalid-input, error-position, and golden regressions. Verify both formatting modes against ClickHouse 26.7.1.1315 and pass make test, make lint, and the CLI build. --- parser/error_test.go | 2 + parser/lexer.go | 82 +++++++- parser/lexer_test.go | 87 +++++++++ parser/parser_test.go | 9 + .../format/beautify/lexical_fidelity.sql | 21 +++ .../basic/format/lexical_fidelity.sql | 12 ++ parser/testdata/basic/lexical_fidelity.sql | 5 + .../output/lexical_fidelity.sql.golden.json | 177 ++++++++++++++++++ 8 files changed, 387 insertions(+), 8 deletions(-) create mode 100644 parser/testdata/basic/format/beautify/lexical_fidelity.sql create mode 100644 parser/testdata/basic/format/lexical_fidelity.sql create mode 100644 parser/testdata/basic/lexical_fidelity.sql create mode 100644 parser/testdata/basic/output/lexical_fidelity.sql.golden.json diff --git a/parser/error_test.go b/parser/error_test.go index b926f7d..a04b8a2 100644 --- a/parser/error_test.go +++ b/parser/error_test.go @@ -55,6 +55,8 @@ func TestParseError_LexicalFailure(t *testing.T) { msg string }{ {"/*", "unclosed multi-line comment"}, + {"/* outer /* inner */", "unclosed multi-line comment"}, + {"$$unclosed", "invalid dollar-quoted string"}, {"'unclosed", "invalid string"}, {"`unclosed", "unclosed quoted identifier"}, {"1e+", "exponent part should contain at least one digit"}, diff --git a/parser/lexer.go b/parser/lexer.go index c8a861e..21bf2ea 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -215,12 +215,25 @@ func (l *Lexer) consumeIdent(_ Pos) error { i++ } } else { - for l.peekOk(i) && (quoteType == BackTicks && l.peekN(i) != '`' || - quoteType == DoubleQuote && l.peekN(i) != '"') { + quote := l.input[l.offset-1] + for l.peekOk(i) { + if l.peekN(i) == '\\' { + i++ + if l.peekOk(i) { + i++ + } + continue + } + if l.peekN(i) == quote { + if l.peekOk(i+1) && l.peekN(i+1) == quote { + i += 2 + continue + } + break + } i++ } - if !l.peekOk(i) || (quoteType == BackTicks && l.peekN(i) != '`') || - (quoteType == DoubleQuote && l.peekN(i) != '"') { + if !l.peekOk(i) { return fmt.Errorf("unclosed quoted identifier: %s", l.slice(0, i)) } } @@ -246,7 +259,7 @@ func (l *Lexer) consumeIdent(_ Pos) error { func (l *Lexer) consumeSingleLineComment() { l.skipN(2) i := 0 - for l.peekOk(i) && l.peekN(i) != '\r' && l.peekN(i) != '\n' { + for l.peekOk(i) && l.peekN(i) != '\n' { i++ } if l.peekOk(i) { @@ -260,10 +273,21 @@ func (l *Lexer) consumeMultiLineComment() error { pos := Pos(l.offset) l.skipN(2) i := 0 + depth := 1 for l.peekOk(i) { + if l.peekOk(i+1) && l.peekN(i) == '/' && l.peekN(i+1) == '*' { + depth++ + i += 2 + continue + } if l.peekOk(i+1) && l.peekN(i) == '*' && l.peekN(i+1) == '/' { - l.skipN(i + 2) - return nil + depth-- + i += 2 + if depth == 0 { + l.skipN(i) + return nil + } + continue } i++ } @@ -308,6 +332,40 @@ func (l *Lexer) consumeString() error { return nil } +func (l *Lexer) consumeDollarQuotedString() error { + i := 1 + for l.peekOk(i) && (IsIdentStart(l.peekN(i)) || IsDigit(l.peekN(i))) { + i++ + } + if l.peekOk(i) && l.peekN(i) == '$' { + delimiter := l.slice(0, i+1) + start := l.offset + len(delimiter) + if end := strings.Index(l.input[start:], delimiter); end >= 0 { + // StringLiteral stores the escaped interior of a single-quoted + // string. Heredoc contents are literal, so escape them once here. + literal := strings.ReplaceAll(l.input[start:start+end], "\\", "\\\\") + literal = strings.ReplaceAll(literal, "'", "\\'") + literal = strings.ReplaceAll(literal, "\n", "\\n") + literal = strings.ReplaceAll(literal, "\r", "\\r") + l.currentToken = &Token{ + Kind: TokenKindString, + String: literal, + Pos: Pos(start), + End: Pos(start + end), + } + l.offset = start + end + len(delimiter) + return nil + } + } + + // ClickHouse treats an unmatched named delimiter as a bare identifier. + // A standalone dollar or an unmatched $$ cannot start an identifier. + if !l.peekOk(1) || (!IsIdentStart(l.peekN(1)) && !IsDigit(l.peekN(1))) { + return errors.New("invalid dollar-quoted string") + } + return l.consumeIdent(Pos(l.offset)) +} + func (l *Lexer) skipComments() error { for !l.isEOF() { l.skipSpace() @@ -315,6 +373,12 @@ func (l *Lexer) skipComments() error { return nil } switch l.peekN(0) { + case '#': + if l.peekOk(1) && (l.peekN(1) == ' ' || l.peekN(1) == '!') { + l.consumeSingleLineComment() + continue + } + return nil case '-': if l.peekOk(1) && l.peekN(1) == '-' { l.consumeSingleLineComment() @@ -417,7 +481,9 @@ func (l *Lexer) consumeToken() (err error) { } case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': return l.consumeNumber() - case '`', '$', '"': + case '$': + return l.consumeDollarQuotedString() + case '`', '"': return l.consumeIdent(Pos(l.offset)) case '\'': return l.consumeString() diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 61011be..1de899e 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -23,11 +23,20 @@ func TestConsumeComment(t *testing.T) { "/* hello world */ /* hello world */\n", "/* hello world */ /* hello world */\r\n", "/* hello world */ /* hello world */\r", + "/* outer /* inner */ outer */", + "/* outer /* middle /* inner */ middle */ outer */", + "# hello world", + "# ", + "#!", + "#!/usr/bin/clickhouse\n", + "# comment\rstill comment", } for _, c := range comments { lexer := NewLexer(c) err := lexer.consumeToken() require.NoError(t, err) + require.Nil(t, lexer.currentToken) + require.Equal(t, len(c), lexer.offset) } } @@ -69,6 +78,7 @@ func TestConsumeUnterminatedComment(t *testing.T) { "/* unterminated", "/* unterminated *", "SELECT 1 /* unterminated", + "/* outer /* inner */", } for _, c := range inputs { c := c @@ -372,3 +382,80 @@ func TestNegativeHexLiteral(t *testing.T) { require.NoError(t, err) require.Len(t, stmts, 1) } + +func TestConsumeQuotedIdent(t *testing.T) { + for _, quote := range []byte{'`', '"'} { + for _, content := range []string{ + "a" + string(quote) + string(quote) + "b", + "a\\" + string(quote) + "b", + "a\\\\", + "中文", + } { + input := string(quote) + content + string(quote) + t.Run(input, func(t *testing.T) { + lexer := NewLexer(input + ",") + require.NoError(t, lexer.consumeToken()) + require.Equal(t, TokenKindIdent, lexer.currentToken.Kind) + require.Equal(t, content, lexer.currentToken.String) + require.Equal(t, Pos(1), lexer.currentToken.Pos) + require.Equal(t, Pos(len(input)-1), lexer.currentToken.End) + require.NoError(t, lexer.consumeToken()) + require.Equal(t, TokenKindComma, lexer.currentToken.Kind) + + stmts, err := NewParser("SELECT " + input).ParseStmts() + require.NoError(t, err) + require.Equal(t, "SELECT "+input, Format(stmts[0])) + }) + } + } +} + +func TestConsumeDollarQuotedString(t *testing.T) { + for _, tc := range []struct { + input, content string + pos Pos + }{ + {"$$hello$$", "hello", 2}, + {"$$$$", "", 2}, + {"$tag$it's\\n$tag$", "it\\'s\\\\n", 5}, + {"$1_$a$$b$1_$", "a$$b", 4}, + {"$tag$one$TAG$two$tag$", "one$TAG$two", 5}, + {"$$中文\n/* # */$$", "中文\\n/* # */", 2}, + {"$$a\r\nb$$", "a\\r\\nb", 2}, + {"$$'\\$$", "\\'\\\\", 2}, + } { + t.Run(tc.input, func(t *testing.T) { + lexer := NewLexer(tc.input + ",") + require.NoError(t, lexer.consumeToken()) + require.Equal(t, TokenKindString, lexer.currentToken.Kind) + require.Equal(t, tc.content, lexer.currentToken.String) + require.Equal(t, tc.pos, lexer.currentToken.Pos) + require.Equal(t, Pos(len(tc.input))-tc.pos, lexer.currentToken.End) + require.NoError(t, lexer.consumeToken()) + require.Equal(t, TokenKindComma, lexer.currentToken.Kind) + + stmts, err := NewParser("SELECT " + tc.input).ParseStmts() + require.NoError(t, err) + for _, beautify := range []bool{false, true} { + formatter := NewFormatter() + if beautify { + formatter.WithBeautify() + } + formatter.WriteExpr(stmts[0]) + formatted := formatter.String() + reparsed, err := NewParser(formatted).ParseStmts() + require.NoError(t, err) + require.Equal(t, "SELECT '"+tc.content+"'", Format(reparsed[0])) + } + }) + } + + // Without a complete matching delimiter, a named tag can be an identifier. + for _, input := range []string{"$name", "$tag$unclosed", "$tag$x$other$", "name$dollar"} { + lexer := NewLexer(input) + require.NoError(t, lexer.consumeToken()) + require.Equal(t, TokenKindIdent, lexer.currentToken.Kind) + require.Equal(t, input, lexer.currentToken.String) + require.True(t, lexer.isEOF()) + } +} diff --git a/parser/parser_test.go b/parser/parser_test.go index 17ab8e9..934dbce 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -192,6 +192,15 @@ func TestParser_InvalidSyntax(t *testing.T) { "SELECT", "SELECT 1; SELECT", "SELECT 1 /*", + "SELECT 1 /* outer /* inner */", + "SELECT 1 #word", + "SELECT 1 #\tcomment", + "SELECT 1 #", + "SELECT 1 #\n+2", + "SELECT $$unclosed", + "SELECT $", + `SELECT "a\"`, + "SELECT `a\\`", "SELECT 1 +> 2", "SELECT * FROM", // WITH FILL error cases diff --git a/parser/testdata/basic/format/beautify/lexical_fidelity.sql b/parser/testdata/basic/format/beautify/lexical_fidelity.sql new file mode 100644 index 0000000..1954029 --- /dev/null +++ b/parser/testdata/basic/format/beautify/lexical_fidelity.sql @@ -0,0 +1,21 @@ +-- Origin SQL: +#!/usr/bin/clickhouse +SELECT `a``b`, `a\`b`, "a""b", "a\"b"; +SELECT $$hello$$, $tag$it's\n$tag$, $1_$a$$b$1_$, $$$$; +SELECT /* outer /* inner */ outer */ 1 # comment ++ 2; + + +-- Beautify SQL: +SELECT + `a``b`, + `a\`b`, + "a""b", + "a\"b"; +SELECT + 'hello', + 'it\'s\\n', + 'a$$b', + ''; +SELECT + 1 + 2; diff --git a/parser/testdata/basic/format/lexical_fidelity.sql b/parser/testdata/basic/format/lexical_fidelity.sql new file mode 100644 index 0000000..a544761 --- /dev/null +++ b/parser/testdata/basic/format/lexical_fidelity.sql @@ -0,0 +1,12 @@ +-- Origin SQL: +#!/usr/bin/clickhouse +SELECT `a``b`, `a\`b`, "a""b", "a\"b"; +SELECT $$hello$$, $tag$it's\n$tag$, $1_$a$$b$1_$, $$$$; +SELECT /* outer /* inner */ outer */ 1 # comment ++ 2; + + +-- Format SQL: +SELECT `a``b`, `a\`b`, "a""b", "a\"b"; +SELECT 'hello', 'it\'s\\n', 'a$$b', ''; +SELECT 1 + 2; diff --git a/parser/testdata/basic/lexical_fidelity.sql b/parser/testdata/basic/lexical_fidelity.sql new file mode 100644 index 0000000..1dfe298 --- /dev/null +++ b/parser/testdata/basic/lexical_fidelity.sql @@ -0,0 +1,5 @@ +#!/usr/bin/clickhouse +SELECT `a``b`, `a\`b`, "a""b", "a\"b"; +SELECT $$hello$$, $tag$it's\n$tag$, $1_$a$$b$1_$, $$$$; +SELECT /* outer /* inner */ outer */ 1 # comment ++ 2; diff --git a/parser/testdata/basic/output/lexical_fidelity.sql.golden.json b/parser/testdata/basic/output/lexical_fidelity.sql.golden.json new file mode 100644 index 0000000..6a6caae --- /dev/null +++ b/parser/testdata/basic/output/lexical_fidelity.sql.golden.json @@ -0,0 +1,177 @@ +[ + { + "SelectPos": 22, + "StatementEnd": 58, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "Name": "a``b", + "QuoteType": 3, + "NamePos": 30, + "NameEnd": 34 + }, + "Modifiers": [], + "Alias": null + }, + { + "Expr": { + "Name": "a\\`b", + "QuoteType": 3, + "NamePos": 38, + "NameEnd": 42 + }, + "Modifiers": [], + "Alias": null + }, + { + "Expr": { + "Name": "a\"\"b", + "QuoteType": 2, + "NamePos": 46, + "NameEnd": 50 + }, + "Modifiers": [], + "Alias": null + }, + { + "Expr": { + "Name": "a\\\"b", + "QuoteType": 2, + "NamePos": 54, + "NameEnd": 58 + }, + "Modifiers": [], + "Alias": null + } + ], + "From": null, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + }, + { + "SelectPos": 61, + "StatementEnd": 113, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "LiteralPos": 70, + "LiteralEnd": 75, + "Literal": "hello" + }, + "Modifiers": [], + "Alias": null + }, + { + "Expr": { + "LiteralPos": 84, + "LiteralEnd": 90, + "Literal": "it\\'s\\\\n" + }, + "Modifiers": [], + "Alias": null + }, + { + "Expr": { + "LiteralPos": 101, + "LiteralEnd": 105, + "Literal": "a$$b" + }, + "Modifiers": [], + "Alias": null + }, + { + "Expr": { + "LiteralPos": 113, + "LiteralEnd": 113, + "Literal": "" + }, + "Modifiers": [], + "Alias": null + } + ], + "From": null, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + }, + { + "SelectPos": 117, + "StatementEnd": 169, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "LeftExpr": { + "NumPos": 154, + "NumEnd": 155, + "Literal": "1", + "Base": 10 + }, + "Operation": "+", + "RightExpr": { + "NumPos": 168, + "NumEnd": 169, + "Literal": "2", + "Base": 10 + }, + "HasGlobal": false, + "HasNot": false + }, + "Modifiers": [], + "Alias": null + } + ], + "From": null, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + } +] \ No newline at end of file