From 8fa10be261fffc16bf99efef6f1f958f7b558d88 Mon Sep 17 00:00:00 2001 From: git-hulk Date: Thu, 10 Sep 2026 10:48:03 +0800 Subject: [PATCH 1/6] Reject incomplete SQL and preserve lexical errors ## Problem ParseStmts checks the byte offset before consuming the final token, and SELECT permits an empty projection at EOF. Ignored token errors and failed lookahead can also discard lexical failures or replace them with a later grammar error. The shared sign branch incorrectly recognizes +> as ->. ## Reproduction ```go parser.NewParser("SELECT").ParseStmts() // succeeds with no statements parser.NewParser("SELECT 1; SELECT").ParseStmts() // loses the final SELECT parser.NewParser("SELECT 1 /*").ParseStmts() // accepts an unclosed comment parser.NewParser("SELECT 1 +> 2").ParseStmts() // formats +> as -> ``` ## Fix Retain the first lexical error and its byte position outside cursor checkpoints, restore lookahead on every outcome, and return the original failure from ParseStmts. Test token exhaustion, require a SELECT expression, and recognize an arrow only when it starts with a minus. Public APIs and existing golden output are unchanged. ## Test Add regressions for final-token boundaries, lexical failures across lookahead/backtracking, original error locations, and cursor restoration. The focused tests failed before the fix. Add AST and formatting fixtures for valid arrow syntax and a final statement without a semicolon. Validate the reproductions with ClickHouse 26.7.1.1315. A separate probe also rejects malformed suffixes in 684 variations of the existing SQL fixtures. `make`, `make test`, and `make lint` pass with Go 1.21.13 and the CI-pinned golangci-lint v1.53.3. Local macOS verification uses the external linker and an ad-hoc signature for the temporary linter binary. --- parser/error_test.go | 31 ++++ parser/lexer.go | 27 +++- parser/lexer_test.go | 23 +++ parser/parser_column.go | 13 +- parser/parser_common.go | 5 + parser/parser_query.go | 8 +- parser/parser_table.go | 4 +- parser/parser_test.go | 27 ++++ .../format/beautify/statement_boundaries.sql | 9 ++ .../basic/format/statement_boundaries.sql | 7 + .../statement_boundaries.sql.golden.json | 151 ++++++++++++++++++ .../testdata/basic/statement_boundaries.sql | 2 + 12 files changed, 290 insertions(+), 17 deletions(-) create mode 100644 parser/testdata/basic/format/beautify/statement_boundaries.sql create mode 100644 parser/testdata/basic/format/statement_boundaries.sql create mode 100644 parser/testdata/basic/output/statement_boundaries.sql.golden.json create mode 100644 parser/testdata/basic/statement_boundaries.sql diff --git a/parser/error_test.go b/parser/error_test.go index 9d185cc..a0ee782 100644 --- a/parser/error_test.go +++ b/parser/error_test.go @@ -47,3 +47,34 @@ func TestParseError_ExpectedTokenKind(t *testing.T) { require.Equal(t, []TokenKind{TokenKindRParen}, pe.Expected) require.True(t, strings.HasPrefix(pe.Error(), "line ")) } + +func TestParseError_LexicalFailure(t *testing.T) { + for _, prefix := range []string{"SELECT 1 ", "SELECT case ", "SELECT interval + ", "SELECT 1;\n", "SELECT 1 /* closed */\n"} { + for _, suffix := range []struct { + sql string + msg string + }{ + {"/*", "unclosed multi-line comment"}, + {"'unclosed", "invalid string"}, + {"`unclosed", "unclosed quoted identifier"}, + {"1e+", "exponent part should contain at least one digit"}, + {"中文", "unexpected character"}, + } { + sql := prefix + suffix.sql + t.Run(sql, func(t *testing.T) { + stmts, err := NewParser(sql).ParseStmts() + require.Error(t, err) + require.Nil(t, stmts) + + var pe *ParseError + require.ErrorAs(t, err, &pe) + require.Contains(t, pe.Msg, suffix.msg) + require.Equal(t, Pos(len(prefix)), pe.Pos) + if strings.HasSuffix(prefix, "\n") { + require.Equal(t, 2, pe.Line) + require.Equal(t, 1, pe.Column) + } + }) + } + } +} diff --git a/parser/lexer.go b/parser/lexer.go index 96df452..fcb8b33 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -83,6 +83,12 @@ type Lexer struct { lexerState input string + + // Lexical failures are fatal for this input, even when discovered during + // lookahead. Keep them outside lexerState so restoring a cursor cannot + // discard the error or its original position. + err error + errPos Pos } func NewLexer(buf string) *Lexer { @@ -308,6 +314,7 @@ func (l *Lexer) skipComments() error { if !l.peekOk(0) { return nil } + l.errPos = Pos(l.offset) switch l.peekN(0) { case '-': if l.peekOk(1) && l.peekN(1) == '-' { @@ -335,13 +342,11 @@ func (l *Lexer) skipComments() error { func (l *Lexer) peekToken() (*Token, error) { savedState := l.saveState() + defer l.restoreState(savedState) if err := l.consumeToken(); err != nil { return nil, err } - token := l.currentToken - - l.restoreState(savedState) - return token, nil + return l.currentToken, nil } func (l *Lexer) hasPrecedenceToken(last *Token) bool { @@ -356,10 +361,19 @@ func (l *Lexer) hasPrecedenceToken(last *Token) bool { last.Kind == TokenKindRBracket) } -func (l *Lexer) consumeToken() error { +func (l *Lexer) consumeToken() (err error) { // replace the current token; keep the previous one to disambiguate unary +/- prevToken := l.currentToken l.currentToken = nil + if l.err != nil { + return l.err + } + defer func() { + if err != nil { + l.err = err + } + }() + if err := l.skipComments(); err != nil { return err } @@ -367,6 +381,7 @@ func (l *Lexer) consumeToken() error { if l.isEOF() { return nil } + l.errPos = Pos(l.offset) switch l.peekN(0) { case '>', '<', '!', '=', '|': if l.peekN(0) == '|' && l.peekOk(1) && l.peekN(1) == '|' || // || @@ -387,7 +402,7 @@ func (l *Lexer) consumeToken() error { // hasPrecedenceToken is used to distinguish between unary and binary operators if !l.hasPrecedenceToken(prevToken) && l.peekOk(1) && IsDigit(l.peekN(1)) { return l.consumeNumber() - } else if l.peekOk(1) && l.peekN(1) == '>' { + } else if l.peekN(0) == '-' && l.peekOk(1) && l.peekN(1) == '>' { l.currentToken = &Token{ String: l.slice(0, 2), Kind: TokenKindArrow, diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 8cff73f..08f996e 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -32,6 +32,29 @@ func TestConsumeComment(t *testing.T) { } +func TestPeekToken_RestoresState(t *testing.T) { + for _, suffix := range []string{"", " + 1", " /*", " 'unclosed", " `unclosed", " 1e+"} { + t.Run(suffix, func(t *testing.T) { + lexer := NewLexer("x" + suffix) + require.NoError(t, lexer.consumeToken()) + before := lexer.saveState() + + token, peekErr := lexer.peekToken() + require.Equal(t, before, lexer.saveState()) + err := lexer.consumeToken() + if peekErr != nil { + require.Equal(t, peekErr, err) + require.Nil(t, lexer.currentToken) + // Further advancement must not turn a lexical failure into EOF. + require.Equal(t, err, lexer.consumeToken()) + } else { + require.NoError(t, err) + require.Equal(t, token, lexer.currentToken) + } + }) + } +} + // TestConsumeUnterminatedComment guards against an infinite loop (a DoS hang) // when a block comment is never closed. consumeMultiLineComment previously // looped on isEOF() while only advancing a local index, so l.offset never diff --git a/parser/parser_column.go b/parser/parser_column.go index 30de285..21ec057 100644 --- a/parser/parser_column.go +++ b/parser/parser_column.go @@ -488,7 +488,10 @@ func (p *Parser) peekIsClauseStarterKeyword() bool { // peekIsEndOfStatement reports whether the next token is EOF or `;`. func (p *Parser) peekIsEndOfStatement() bool { next, err := p.lexer.peekToken() - if err != nil || next == nil { + if err != nil { + return false + } + if next == nil { return true } return next.Kind == ";" @@ -593,10 +596,10 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen // (e.g. `WHERE interval > 1`), and no fixed lookahead separates the // two readings: `INTERVAL a + b DAY` only reveals the operator use at // the unit, four tokens out. Try the operator reading first and fall - // back to the identifier when it fails; the lexer state is the only - // parse state, so the restore is total and the outcome at a position - // never changes. That determinism makes memoizing failures sound, and - // retrying each position at most once is what keeps repeated interval + // back to the identifier when it fails. Restoring the lexer cursor + // retries the syntax; lexical failures remain fatal for the input. + // The outcome at a position never changes, so memoizing failures is sound. + // Retrying each position at most once is what keeps repeated interval // columns (`SELECT interval + interval + ...`) from backtracking // exponentially: a failed attempt reparses its whole suffix, retrying // every later INTERVAL inside it. diff --git a/parser/parser_common.go b/parser/parser_common.go index 7532601..141b1e7 100644 --- a/parser/parser_common.go +++ b/parser/parser_common.go @@ -459,6 +459,11 @@ func (p *Parser) parseFormat(pos Pos) (*FormatClause, error) { // captured position and expected-token information; the long tail of // fmt.Errorf sites is wrapped here with the current position. func (p *Parser) wrapError(err error) error { + // A grammar error can be a consequence of an ignored lexical failure. + // Report the original failure, including when lookahead restored the cursor. + if p.lexer.err != nil { + err = &ParseError{Pos: p.lexer.errPos, Msg: p.lexer.err.Error()} + } if err == nil { return nil } diff --git a/parser/parser_query.go b/parser/parser_query.go index 128d008..acce0da 100644 --- a/parser/parser_query.go +++ b/parser/parser_query.go @@ -1172,11 +1172,11 @@ func (p *Parser) parseSelectStmt(pos Pos) (*SelectQuery, error) { // nolint: fun if err != nil { return nil, err } - - statementEnd := pos - if len(selectItems) > 0 { - statementEnd = selectItems[len(selectItems)-1].End() + if len(selectItems) == 0 { + return nil, fmt.Errorf("expected SELECT expression") } + + statementEnd := selectItems[len(selectItems)-1].End() from, err := p.tryParseFromClause(p.Pos()) if err != nil { return nil, err diff --git a/parser/parser_table.go b/parser/parser_table.go index 72156a9..bb4de5c 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -1674,14 +1674,14 @@ func (p *Parser) ParseStmts() ([]Expr, error) { if err := p.lexer.consumeToken(); err != nil { return nil, p.wrapError(err) } - if p.lexer.isEOF() { + if p.current() == nil { break } if p.matchTokenKind(";") { continue } stmt, err := p.parseStmt(p.Pos()) - if err != nil { + if err != nil || p.lexer.err != nil { return nil, p.wrapError(err) } stmts = append(stmts, stmt) diff --git a/parser/parser_test.go b/parser/parser_test.go index 07f64e2..17ab8e9 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -189,6 +189,10 @@ func validFormatSQL(t *testing.T, sql string) { func TestParser_InvalidSyntax(t *testing.T) { invalidSQLs := []string{ + "SELECT", + "SELECT 1; SELECT", + "SELECT 1 /*", + "SELECT 1 +> 2", "SELECT * FROM", // WITH FILL error cases "SELECT n FROM t ORDER BY n WITH", // WITH without FILL @@ -292,6 +296,29 @@ func TestParser_InvalidSyntax(t *testing.T) { } } +func TestParser_FinalTokenConsumption(t *testing.T) { + for _, suffix := range []string{"", " ", ";", " -- trailing comment", " /* trailing comment */"} { + t.Run(suffix, func(t *testing.T) { + for _, sql := range []string{"SELECT", "SELECT 1; SELECT"} { + stmts, err := NewParser(sql + suffix).ParseStmts() + require.Error(t, err, "SQL: %s", sql+suffix) + require.Nil(t, stmts) + } + + stmts, err := NewParser("SELECT 1; SELECT 2" + suffix).ParseStmts() + require.NoError(t, err) + require.Len(t, stmts, 2) + require.Equal(t, "SELECT 2", Format(stmts[1])) + }) + } + + for _, sql := range []string{"", " ", ";;", "-- comment", "/* comment */"} { + stmts, err := NewParser(sql).ParseStmts() + require.NoError(t, err) + require.Empty(t, stmts) + } +} + func TestParser_ParenthesizedSetOperationOperands(t *testing.T) { // A parenthesized operand becomes a group node, so the operator after // ')' binds to the whole group instead of leaking into its chain. diff --git a/parser/testdata/basic/format/beautify/statement_boundaries.sql b/parser/testdata/basic/format/beautify/statement_boundaries.sql new file mode 100644 index 0000000..034b6de --- /dev/null +++ b/parser/testdata/basic/format/beautify/statement_boundaries.sql @@ -0,0 +1,9 @@ +-- Origin SQL: +SELECT arrayMap(x -> x + 1, [1, 2]); +SELECT 2 + +-- Beautify SQL: +SELECT + arrayMap(x -> x + 1, [1, 2]); +SELECT + 2; diff --git a/parser/testdata/basic/format/statement_boundaries.sql b/parser/testdata/basic/format/statement_boundaries.sql new file mode 100644 index 0000000..bae0a81 --- /dev/null +++ b/parser/testdata/basic/format/statement_boundaries.sql @@ -0,0 +1,7 @@ +-- Origin SQL: +SELECT arrayMap(x -> x + 1, [1, 2]); +SELECT 2 + +-- Format SQL: +SELECT arrayMap(x -> x + 1, [1, 2]); +SELECT 2; diff --git a/parser/testdata/basic/output/statement_boundaries.sql.golden.json b/parser/testdata/basic/output/statement_boundaries.sql.golden.json new file mode 100644 index 0000000..32112ff --- /dev/null +++ b/parser/testdata/basic/output/statement_boundaries.sql.golden.json @@ -0,0 +1,151 @@ +[ + { + "SelectPos": 0, + "StatementEnd": 34, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "Name": { + "Name": "arrayMap", + "QuoteType": 1, + "NamePos": 7, + "NameEnd": 15 + }, + "Params": { + "LeftParenPos": 15, + "RightParenPos": 34, + "Items": { + "ListPos": 16, + "ListEnd": 33, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "LeftExpr": { + "Name": "x", + "QuoteType": 1, + "NamePos": 16, + "NameEnd": 17 + }, + "Operation": "-\u003e", + "RightExpr": { + "LeftExpr": { + "Name": "x", + "QuoteType": 1, + "NamePos": 21, + "NameEnd": 22 + }, + "Operation": "+", + "RightExpr": { + "NumPos": 25, + "NumEnd": 26, + "Literal": "1", + "Base": 10 + }, + "HasGlobal": false, + "HasNot": false + }, + "HasGlobal": false, + "HasNot": false + }, + "Alias": null + }, + { + "Expr": { + "LeftBracketPos": 28, + "RightBracketPos": 33, + "Items": { + "ListPos": 29, + "ListEnd": 33, + "HasDistinct": false, + "Items": [ + { + "Expr": { + "NumPos": 29, + "NumEnd": 30, + "Literal": "1", + "Base": 10 + }, + "Alias": null + }, + { + "Expr": { + "NumPos": 32, + "NumEnd": 33, + "Literal": "2", + "Base": 10 + }, + "Alias": null + } + ] + } + }, + "Alias": null + } + ] + }, + "ColumnArgList": null + } + }, + "Modifiers": [], + "Alias": null + } + ], + "From": null, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + }, + { + "SelectPos": 37, + "StatementEnd": 45, + "With": null, + "Top": null, + "HasDistinct": false, + "DistinctOn": null, + "SelectItems": [ + { + "Expr": { + "NumPos": 44, + "NumEnd": 45, + "Literal": "2", + "Base": 10 + }, + "Modifiers": [], + "Alias": null + } + ], + "From": null, + "Window": null, + "Prewhere": null, + "Where": null, + "GroupBy": null, + "WithTotal": false, + "Having": null, + "OrderBy": null, + "LimitBy": null, + "Limit": null, + "Settings": null, + "Format": null, + "UnionAll": null, + "UnionDistinct": null, + "Except": null, + "Intersect": null + } +] \ No newline at end of file diff --git a/parser/testdata/basic/statement_boundaries.sql b/parser/testdata/basic/statement_boundaries.sql new file mode 100644 index 0000000..c0d0ccc --- /dev/null +++ b/parser/testdata/basic/statement_boundaries.sql @@ -0,0 +1,2 @@ +SELECT arrayMap(x -> x + 1, [1, 2]); +SELECT 2 \ No newline at end of file From 88255b353f9133b0c80ff46d1efada48baa16e7a Mon Sep 17 00:00:00 2001 From: git-hulk Date: Thu, 10 Sep 2026 10:52:32 +0800 Subject: [PATCH 2/6] Keep lexical errors and their positions together Replace the separate lexer error and position fields with a private lexerError wrapper. Capture comment positions at the error source and preserve already-positioned failures when token consumption returns. Extend the lookahead regression to inspect the returned error position. Build, lint, and the full race/compatibility suite pass with Go 1.21.13. --- parser/error.go | 10 ++++++++++ parser/lexer.go | 15 +++++++++------ parser/lexer_test.go | 3 +++ parser/parser_common.go | 2 +- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/parser/error.go b/parser/error.go index ac73a2b..5e6ee18 100644 --- a/parser/error.go +++ b/parser/error.go @@ -5,6 +5,16 @@ import ( "strings" ) +// lexerError keeps a lexical failure attached to its original byte position. +type lexerError struct { + pos Pos + err error +} + +func (e *lexerError) Error() string { return e.err.Error() } + +func (e *lexerError) Unwrap() error { return e.err } + // ParseError is a structured parse error. It carries the byte offset and the // 1-based line/column where parsing stopped, the offending token, and (when // known) the tokens the grammar expected at that point. Callers such as diff --git a/parser/lexer.go b/parser/lexer.go index fcb8b33..c8a861e 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -87,8 +87,7 @@ type Lexer struct { // Lexical failures are fatal for this input, even when discovered during // lookahead. Keep them outside lexerState so restoring a cursor cannot // discard the error or its original position. - err error - errPos Pos + err *lexerError } func NewLexer(buf string) *Lexer { @@ -258,6 +257,7 @@ func (l *Lexer) consumeSingleLineComment() { } func (l *Lexer) consumeMultiLineComment() error { + pos := Pos(l.offset) l.skipN(2) i := 0 for l.peekOk(i) { @@ -268,7 +268,7 @@ func (l *Lexer) consumeMultiLineComment() error { i++ } l.skipN(i) - return errors.New("unclosed multi-line comment") + return &lexerError{pos: pos, err: errors.New("unclosed multi-line comment")} } func (l *Lexer) consumeString() error { @@ -314,7 +314,6 @@ func (l *Lexer) skipComments() error { if !l.peekOk(0) { return nil } - l.errPos = Pos(l.offset) switch l.peekN(0) { case '-': if l.peekOk(1) && l.peekN(1) == '-' { @@ -368,9 +367,13 @@ func (l *Lexer) consumeToken() (err error) { if l.err != nil { return l.err } + pos := Pos(l.offset) defer func() { if err != nil { - l.err = err + if !errors.As(err, &l.err) { + l.err = &lexerError{pos: pos, err: err} + } + err = l.err } }() @@ -381,7 +384,7 @@ func (l *Lexer) consumeToken() (err error) { if l.isEOF() { return nil } - l.errPos = Pos(l.offset) + pos = Pos(l.offset) switch l.peekN(0) { case '>', '<', '!', '=', '|': if l.peekN(0) == '|' && l.peekOk(1) && l.peekN(1) == '|' || // || diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 08f996e..61011be 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -43,6 +43,9 @@ func TestPeekToken_RestoresState(t *testing.T) { require.Equal(t, before, lexer.saveState()) err := lexer.consumeToken() if peekErr != nil { + var le *lexerError + require.ErrorAs(t, peekErr, &le) + require.Equal(t, Pos(2), le.pos) require.Equal(t, peekErr, err) require.Nil(t, lexer.currentToken) // Further advancement must not turn a lexical failure into EOF. diff --git a/parser/parser_common.go b/parser/parser_common.go index 141b1e7..b4694d0 100644 --- a/parser/parser_common.go +++ b/parser/parser_common.go @@ -462,7 +462,7 @@ func (p *Parser) wrapError(err error) error { // A grammar error can be a consequence of an ignored lexical failure. // Report the original failure, including when lookahead restored the cursor. if p.lexer.err != nil { - err = &ParseError{Pos: p.lexer.errPos, Msg: p.lexer.err.Error()} + err = &ParseError{Pos: p.lexer.err.pos, Msg: p.lexer.err.Error()} } if err == nil { return nil From 9a201874849dc755c8b5977a6bb6cafbefd6167e Mon Sep 17 00:00:00 2001 From: git-hulk Date: Thu, 10 Sep 2026 11:39:06 +0800 Subject: [PATCH 3/6] Return retained lexer errors through token consumption Remove the duplicate lexer error check after parseStmt. The next loop iteration always calls consumeToken before checking EOF, so retained lexical failures already return through that path. wrapError preserves the lexical cause when the grammar returns a secondary error first. --- parser/parser_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/parser_table.go b/parser/parser_table.go index bb4de5c..38c866f 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -1681,7 +1681,7 @@ func (p *Parser) ParseStmts() ([]Expr, error) { continue } stmt, err := p.parseStmt(p.Pos()) - if err != nil || p.lexer.err != nil { + if err != nil { return nil, p.wrapError(err) } stmts = append(stmts, stmt) From 3dd5cb8a1afa5f44bfe8f10a5bb957e1aa3f1bcf Mon Sep 17 00:00:00 2001 From: git-hulk Date: Thu, 10 Sep 2026 11:56:41 +0800 Subject: [PATCH 4/6] Propagate errors from optional token consumption Return (*Token, error) from tryConsumeTokenKind and check the error in expectTokenKind and every optional-token caller. Preserve non-matching and EOF behavior while returning lexical failures at consumption. Add regressions for direct error propagation, original error positions, non-matching tokens, and final-token consumption. make test, make lint, and make pass with Go 1.21.13 and the CI-pinned linter. --- parser/error_test.go | 58 ++++++++++++++++++++ parser/parse_system.go | 106 ++++++++++++++++++++++++++++++----- parser/parser_alter.go | 22 +++++++- parser/parser_column.go | 88 ++++++++++++++++++++++------- parser/parser_common.go | 30 +++++++--- parser/parser_query.go | 43 ++++++++++++--- parser/parser_table.go | 119 +++++++++++++++++++++++++++++++++------- 7 files changed, 394 insertions(+), 72 deletions(-) diff --git a/parser/error_test.go b/parser/error_test.go index a0ee782..b926f7d 100644 --- a/parser/error_test.go +++ b/parser/error_test.go @@ -78,3 +78,61 @@ func TestParseError_LexicalFailure(t *testing.T) { } } } + +func TestParser_TokenConsumptionError(t *testing.T) { + for _, tc := range []struct { + name string + sql string + parse func(*Parser) error + }{ + {"required token", ") /*", func(p *Parser) error { return p.expectTokenKind(TokenKindRParen) }}, + {"optional token", ". /*", func(p *Parser) error { + _, err := p.tryParseDotIdent(p.Pos()) + return err + }}, + {"list separator", "a, /*", func(p *Parser) error { + _, err := p.parseUserNames() + return err + }}, + } { + t.Run(tc.name, func(t *testing.T) { + p := NewParser(tc.sql) + require.NoError(t, p.lexer.consumeToken()) + err := tc.parse(p) + var lexicalErr *lexerError + require.ErrorAs(t, err, &lexicalErr) + require.Equal(t, Pos(strings.Index(tc.sql, "/*")), lexicalErr.pos) + require.EqualError(t, lexicalErr, "unclosed multi-line comment") + }) + } +} + +func TestParser_TryConsumeTokenKind(t *testing.T) { + for _, tc := range []struct { + name string + sql string + kind TokenKind + matched bool + next string + }{ + {"mismatch", "a /*", TokenKindComma, false, "a"}, + {"advance", "a b", TokenKindIdent, true, "b"}, + {"last token", "a", TokenKindIdent, true, ""}, + {"empty input", "", TokenKindIdent, false, ""}, + } { + t.Run(tc.name, func(t *testing.T) { + p := NewParser(tc.sql) + require.NoError(t, p.lexer.consumeToken()) + current := p.current() + token, err := p.tryConsumeTokenKind(tc.kind) + require.NoError(t, err) + if tc.matched { + require.Same(t, current, token) + } else { + require.Nil(t, token) + require.Equal(t, current, p.current()) + } + require.Equal(t, tc.next, p.currentTokenString()) + }) + } +} diff --git a/parser/parse_system.go b/parser/parse_system.go index f75f091..beb500b 100644 --- a/parser/parse_system.go +++ b/parser/parse_system.go @@ -389,7 +389,9 @@ func (p *Parser) parseRoleName(_ Pos) (*RoleName, error) { return nil, err } var scope *StringLiteral - if p.tryConsumeTokenKind(TokenKindAtSign) != nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindAtSign); consumeErr != nil { + return nil, consumeErr + } else if token != nil { scope, err = p.parseString(p.Pos()) if err != nil { return nil, err @@ -449,7 +451,9 @@ func (p *Parser) parseRoleSetting(_ Pos) (*RoleSetting, error) { p.matchTokenKind(TokenKindFloat), p.matchTokenKind(TokenKindString): var op TokenKind - if token := p.tryConsumeTokenKind(TokenKindSingleEQ); token != nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindSingleEQ); consumeErr != nil { + return nil, consumeErr + } else if token != nil { op = token.Kind } value, err := p.parseLiteral(p.Pos()) @@ -487,7 +491,9 @@ func (p *Parser) parseRoleSettings(_ Pos) ([]*RoleSetting, error) { return nil, err } settings = append(settings, setting) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -525,7 +531,14 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { return nil, err } roleNames = append(roleNames, roleName) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } roleName, err := p.parseRoleName(p.Pos()) if err != nil { return nil, err @@ -677,7 +690,14 @@ func (p *Parser) parseDefaultRoleClause(pos Pos) (*DefaultRoleClause, error) { } roles = append(roles, role) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } role, err := p.parseRoleName(p.Pos()) if err != nil { return nil, err @@ -712,7 +732,14 @@ func (p *Parser) parseGranteesClause(pos Pos) (*GranteesClause, error) { } granteeList = append(granteeList, grantee) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } grantee, err := p.parseRoleName(p.Pos()) if err != nil { return nil, err @@ -733,7 +760,14 @@ func (p *Parser) parseGranteesClause(pos Pos) (*GranteesClause, error) { } exceptList = append(exceptList, except) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } except, err := p.parseRoleName(p.Pos()) if err != nil { return nil, err @@ -777,7 +811,14 @@ func (p *Parser) parseUserNames() ([]*RoleName, error) { } userNames = append(userNames, userName) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } userName, err := p.parseRoleName(p.Pos()) if err != nil { return nil, err @@ -795,7 +836,14 @@ func (p *Parser) parseHostClauses() ([]*HostClause, error) { } hosts = append(hosts, host) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } host, err := p.parseHostClause(p.Pos()) if err != nil { return nil, err @@ -954,7 +1002,14 @@ func (p *Parser) parserDropUserOrRole(pos Pos) (*DropUserOrRole, error) { return nil, err } names = append(names, name) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } name, err := p.parseRoleName(p.Pos()) if err != nil { return nil, err @@ -1278,7 +1333,14 @@ func (p *Parser) parsePrivilegeRoles(_ Pos) ([]*Ident, error) { return nil, err } roles = append(roles, role) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } role, err := p.parseIdent() if err != nil { return nil, err @@ -1322,7 +1384,9 @@ func (p *Parser) parseGrantSource(_ Pos) (*TableIdentifier, error) { return nil, err } - if p.tryConsumeTokenKind(TokenKindDot) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindDot); consumeErr != nil { + return nil, consumeErr + } else if token == nil { return &TableIdentifier{ Table: ident, }, nil @@ -1356,7 +1420,14 @@ func (p *Parser) parseGrantPrivilegeStmt(pos Pos) (*GrantPrivilegeStmt, error) { return nil, err } privileges = append(privileges, privilege) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } privilege, err := p.parsePrivilegeClause(p.Pos()) if err != nil { return nil, err @@ -1418,7 +1489,14 @@ func (p *Parser) parseAlterRole(pos Pos) (*AlterRole, error) { return nil, err } roleRenamePairs = append(roleRenamePairs, roleRenamePair) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } roleRenamePair, err := p.parseRoleRenamePair(p.Pos()) if err != nil { return nil, err diff --git a/parser/parser_alter.go b/parser/parser_alter.go index 2637dff..7c84c93 100644 --- a/parser/parser_alter.go +++ b/parser/parser_alter.go @@ -67,7 +67,9 @@ func (p *Parser) parseAlterTable(pos Pos) (*AlterTable, error) { return nil, err } alterTable.AlterExprs = append(alterTable.AlterExprs, alter) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -868,7 +870,14 @@ func (p *Parser) parseAlterTableReset(pos Pos) (AlterTableClause, error) { } settings = append(settings, setting) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } setting, err = p.parseIdent() if err != nil { return nil, err @@ -922,7 +931,14 @@ func (p *Parser) parseAlterTableUpdate(pos Pos) (AlterTableClause, error) { assignments = append(assignments, assignment) // Parse additional comma-separated assignments - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } assignment, err = p.parseUpdateAssignment(p.Pos()) if err != nil { return nil, err diff --git a/parser/parser_column.go b/parser/parser_column.go index 21ec057..16257aa 100644 --- a/parser/parser_column.go +++ b/parser/parser_column.go @@ -188,7 +188,11 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { case p.matchTokenKind(TokenKindDot): _ = p.lexer.consumeToken() operation := TokenKindDot - hasTypeQualifier := p.tryConsumeTokenKind(TokenKindColon) != nil + qualifier, consumeErr := p.tryConsumeTokenKind(TokenKindColon) + if consumeErr != nil { + return nil, consumeErr + } + hasTypeQualifier := qualifier != nil if hasTypeQualifier { // Dynamic JSON subcolumns can pin their result type with // `.:Type`, for example `json.path.:`Array(JSON)``. @@ -381,7 +385,9 @@ func (p *Parser) parseColumnExtractExpr(pos Pos) (*ExtractExpr, error) { parameters = append(parameters, expr) } - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -766,7 +772,9 @@ func (p *Parser) parseColumnExprListWithTerm(term TokenKind, pos Pos) (*ColumnEx break } columnList = append(columnList, columnExpr) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -788,7 +796,9 @@ func (p *Parser) parseSelectItems() ([]*SelectItem, error) { break } selectItems = append(selectItems, selectItem) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } if p.isSelectItemTerminatorKeyword() { @@ -857,7 +867,9 @@ func (p *Parser) parseColumnArgList(pos Pos) (*ColumnArgList, error) { return nil, err } items = append(items, item) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1014,7 +1026,9 @@ func (p *Parser) parseKeywordArgFunctionParams(pos Pos, form keywordArgForm) (*P } items = append(items, &ColumnExpr{Expr: item, Alias: alias}) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } @@ -1112,7 +1126,9 @@ func (p *Parser) parseMapLiteral(pos Pos) (*MapLiteral, error) { Key: *key, Value: value, }) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1314,7 +1330,9 @@ func (p *Parser) parseColumnType(_ Pos) (ColumnType, error) { } func (p *Parser) parseColumnTypeArgs(ident *Ident) (ColumnType, error) { // nolint:funlen - if lParen := p.tryConsumeTokenKind(TokenKindLParen); lParen != nil { + if lParen, consumeErr := p.tryConsumeTokenKind(TokenKindLParen); consumeErr != nil { + return nil, consumeErr + } else if lParen != nil { switch { case p.matchTokenKind(TokenKindIdent): switch { @@ -1372,7 +1390,9 @@ func (p *Parser) parseComplexType(name *Ident, pos Pos) (*ComplexType, error) { return nil, err } subTypes = append(subTypes, subExpr) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1403,7 +1423,9 @@ func (p *Parser) parseEnumType(name *Ident, pos Pos) (*EnumType, error) { break } enumType.Values = append(enumType.Values, *enumValue) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1423,7 +1445,14 @@ func (p *Parser) parseColumnTypeWithParams(name *Ident, pos Pos) (*TypeWithParam return nil, err } params = append(params, param) - for !p.lexer.isEOF() && p.tryConsumeTokenKind(TokenKindComma) != nil { + for !p.lexer.isEOF() { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } size, err := p.parseLiteral(p.Pos()) if err != nil { return nil, err @@ -1451,7 +1480,14 @@ func (p *Parser) parseJSONPath() (*JSONPath, error) { } idents = append(idents, ident) - for !p.lexer.isEOF() && p.tryConsumeTokenKind(TokenKindDot) != nil { + for !p.lexer.isEOF() { + token, consumeErr := p.tryConsumeTokenKind(TokenKindDot) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } ident, err := p.parseAnyKeyword() if err != nil { return nil, err @@ -1525,7 +1561,9 @@ func (p *Parser) parseJSONOption() (*JSONOption, error) { if err != nil { return nil, err } - if p.tryConsumeTokenKind(TokenKindSingleEQ) != nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindSingleEQ); consumeErr != nil { + return nil, consumeErr + } else if token != nil { // This is a max_dynamic_* option; only valid when path is a single ident of that name // Reconstruct handling similar to parseJSONMaxDynamicOptions but we already consumed ident and '=' // Determine which option based on the first ident name @@ -1573,7 +1611,9 @@ func (p *Parser) parseJSONType(name *Ident, pos Pos) (*JSONType, error) { return nil, err } options = append(options, option) - if p.tryConsumeTokenKind(",") == nil { + if token, consumeErr := p.tryConsumeTokenKind(","); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1650,7 +1690,9 @@ func (p *Parser) parseNestedTypeFieldsWithNames(columnName *Ident) ([]Expr, erro ColumnEnd: columnType.End(), }) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { return columns, nil } @@ -1664,7 +1706,9 @@ func (p *Parser) parseNestedTypeFieldsWithNames(columnName *Ident) ([]Expr, erro } columns = append(columns, column) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1682,7 +1726,9 @@ func (p *Parser) parseNestedTypeFieldsWithoutNames(columnType *Ident) ([]Expr, e columns = append(columns, column) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { return columns, nil } @@ -1696,7 +1742,9 @@ func (p *Parser) parseNestedTypeFieldsWithoutNames(columnType *Ident) ([]Expr, e } columns = append(columns, column) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1842,7 +1890,9 @@ func (p *Parser) parseColumnStar(pos Pos) (*Ident, error) { } func (p *Parser) tryParseCompressionLevel(pos Pos) (*NumberLiteral, error) { - if p.tryConsumeTokenKind(TokenKindLParen) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindLParen); consumeErr != nil { + return nil, consumeErr + } else if token == nil { return nil, nil // nolint } diff --git a/parser/parser_common.go b/parser/parser_common.go index b4694d0..1b72da1 100644 --- a/parser/parser_common.go +++ b/parser/parser_common.go @@ -98,7 +98,11 @@ func (p *Parser) matchTokenKind(kinds ...TokenKind) bool { // expectTokenKind consumes the current token if it is the given kind. func (p *Parser) expectTokenKind(kind TokenKind) error { - if curToken := p.tryConsumeTokenKind(kind); curToken != nil { + curToken, err := p.tryConsumeTokenKind(kind) + if err != nil { + return err + } + if curToken != nil { return nil } return &ParseError{ @@ -108,13 +112,17 @@ func (p *Parser) expectTokenKind(kind TokenKind) error { } } -func (p *Parser) tryConsumeTokenKind(kind TokenKind) *Token { +// tryConsumeTokenKind returns nil, nil on a mismatch and propagates errors +// encountered while advancing past a matching token. +func (p *Parser) tryConsumeTokenKind(kind TokenKind) (*Token, error) { if p.matchTokenKind(kind) { curToken := p.current() - _ = p.lexer.consumeToken() - return curToken + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } + return curToken, nil } - return nil + return nil, nil } func (p *Parser) matchKeyword(keyword string) bool { @@ -241,7 +249,9 @@ func (p *Parser) parseIdentOrString() (*Ident, error) { } func (p *Parser) tryParseDotIdent(_ Pos) (*Ident, error) { - if p.tryConsumeTokenKind(TokenKindDot) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindDot); consumeErr != nil { + return nil, consumeErr + } else if token == nil { return nil, nil // nolint } // After a dot the token can only be a member name, so even reserved @@ -250,7 +260,9 @@ func (p *Parser) tryParseDotIdent(_ Pos) (*Ident, error) { } func (p *Parser) tryParseDotIdentOrString(_ Pos) (*Ident, error) { - if p.tryConsumeTokenKind(TokenKindDot) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindDot); consumeErr != nil { + return nil, consumeErr + } else if token == nil { return nil, nil // nolint } // After a dot the token can only be a member name, so even reserved @@ -491,7 +503,9 @@ func (p *Parser) parseRatioExpr(pos Pos) (*RatioExpr, error) { } var denominator *NumberLiteral - if p.tryConsumeTokenKind(TokenKindDiv) != nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindDiv); consumeErr != nil { + return nil, consumeErr + } else if token != nil { // the denominator starts at its own token, not at the numerator denominator, err = p.parseNumber(p.Pos()) if err != nil { diff --git a/parser/parser_query.go b/parser/parser_query.go index acce0da..4df0c8a 100644 --- a/parser/parser_query.go +++ b/parser/parser_query.go @@ -25,7 +25,14 @@ func (p *Parser) parseWithClause(pos Pos) (*WithClause, error) { return nil, err } ctes := []*CTEStmt{cteExpr} - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } cteExpr, err := p.parseCTEStmt(p.Pos()) if err != nil { return nil, err @@ -151,7 +158,11 @@ func (p *Parser) tryParseJoinConstraints(pos Pos) (Expr, error) { On: columnExprList, }, nil case p.tryConsumeKeywords(KeywordUsing): - hasParen := p.tryConsumeTokenKind(TokenKindLParen) != nil + lparen, consumeErr := p.tryConsumeTokenKind(TokenKindLParen) + if consumeErr != nil { + return nil, consumeErr + } + hasParen := lparen != nil columnExprList, err := p.parseColumnExprListWithLParen(p.Pos()) if err != nil { return nil, err @@ -335,8 +346,12 @@ func (p *Parser) peekJoinAfterLocality() bool { func (p *Parser) parseJoinRightExpr(pos Pos) (expr Expr, err error) { var rightExpr Expr var modifiers []string + comma, err := p.tryConsumeTokenKind(TokenKindComma) + if err != nil { + return nil, err + } switch { - case p.tryConsumeTokenKind(TokenKindComma) != nil: + case comma != nil: return p.parseJoinExpr(p.Pos()) default: // GLOBAL/LOCAL only says how the right-hand table is distributed, so @@ -641,7 +656,9 @@ func (p *Parser) parseLimitClause(pos Pos) (*LimitClause, error) { if p.tryConsumeKeywords(KeywordOffset) { offset, err = p.parseExpr(p.Pos()) - } else if p.tryConsumeTokenKind(TokenKindComma) != nil { + } else if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token != nil { offset = limit limit, err = p.parseExpr(p.Pos()) } @@ -988,7 +1005,9 @@ func (p *Parser) parseWindowClause(pos Pos) (*WindowClause, error) { Expr: condition, }) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1030,7 +1049,11 @@ func (p *Parser) parseHavingClause(pos Pos) (*HavingClause, error) { func (p *Parser) parseSubQuery(_ Pos) (*SubQuery, error) { - hasParen := p.tryConsumeTokenKind(TokenKindLParen) != nil + lparen, consumeErr := p.tryConsumeTokenKind(TokenKindLParen) + if consumeErr != nil { + return nil, consumeErr + } + hasParen := lparen != nil selectQuery, err := p.parseSelectQuery(p.Pos()) if err != nil { @@ -1055,7 +1078,9 @@ func (p *Parser) parseSelectQuery(_ Pos) (*SelectQuery, error) { var selectStmt *SelectQuery var err error - if lparen := p.tryConsumeTokenKind(TokenKindLParen); lparen != nil { + if lparen, consumeErr := p.tryConsumeTokenKind(TokenKindLParen); consumeErr != nil { + return nil, consumeErr + } else if lparen != nil { inner, err := p.parseSelectQuery(p.Pos()) if err != nil { return nil, err @@ -1317,7 +1342,9 @@ func (p *Parser) parseCTEStmt(pos Pos) (*CTEStmt, error) { if err := p.expectKeyword(KeywordAs); err != nil { return nil, err } - if p.tryConsumeTokenKind(TokenKindLParen) != nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindLParen); consumeErr != nil { + return nil, consumeErr + } else if token != nil { selectQuery, err := p.parseSelectQuery(p.Pos()) if err != nil { return nil, err diff --git a/parser/parser_table.go b/parser/parser_table.go index 38c866f..6d962c5 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -245,7 +245,9 @@ func (p *Parser) parseCreateNamedCollection(pos Pos) (*CreateNamedCollection, er params = append(params, param) // Check if there's another parameter - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -663,7 +665,9 @@ func (p *Parser) parseTableColumns() ([]Expr, error) { } columns = append(columns, column) } - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -880,7 +884,9 @@ func (p *Parser) parseTableArgList(pos Pos) (*TableArgListExpr, error) { return nil, err } args = append(args, arg) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1011,7 +1017,12 @@ func (p *Parser) parseOrderByClause(pos Pos) (*OrderByClause, error) { } items = append(items, expr) - if p.lexer.isEOF() || p.tryConsumeTokenKind(TokenKindComma) == nil { + if p.lexer.isEOF() { + break + } + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1142,7 +1153,9 @@ func (p *Parser) parseInterpolateClause(interpolatePos Pos) (*InterpolateClause, ListEnd: interpolatePos + Pos(len("INTERPOLATE")), } - if p.tryConsumeTokenKind(TokenKindLParen) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindLParen); consumeErr != nil { + return nil, consumeErr + } else if token == nil { // INTERPOLATE without columns is valid return interpolate, nil } @@ -1166,12 +1179,17 @@ func (p *Parser) parseInterpolateClause(interpolatePos Pos) (*InterpolateClause, items = append(items, item) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } - rparen := p.tryConsumeTokenKind(TokenKindRParen) + rparen, err := p.tryConsumeTokenKind(TokenKindRParen) + if err != nil { + return nil, err + } if rparen == nil { return nil, fmt.Errorf("expected ')' after INTERPOLATE column list") } @@ -1208,7 +1226,14 @@ func (p *Parser) parseTTLClause(pos Pos, allowMultiValues bool) ([]*TTLExpr, err return nil, err } items = append(items, expr) - for allowMultiValues && !p.lexer.isEOF() && p.tryConsumeTokenKind(TokenKindComma) != nil { + for allowMultiValues && !p.lexer.isEOF() { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } expr, err = p.parseTTLExpr(pos) if err != nil { return nil, err @@ -1314,7 +1339,9 @@ func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { } keys.Items = append(keys.Items, key) keys.ListEnd = key.End() - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1339,7 +1366,9 @@ func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { // none follows so parseTTLClause can treat the comma as a rule // separator. savedState := p.lexer.saveState() - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } set, err := p.parseTTLPolicySet(p.Pos()) @@ -1442,7 +1471,14 @@ func (p *Parser) parseSettingsList(pos Pos) ([]*SettingExpr, error) { return nil, err } items = append(items, expr) - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } expr, err = p.parseSettingsExpr(p.Pos()) if err != nil { return nil, err @@ -1532,7 +1568,9 @@ func (p *Parser) parseEngineExpr(pos Pos) (*EngineExpr, error) { if err := p.expectKeyword(KeywordEngine); err != nil { return nil, err } - _ = p.tryConsumeTokenKind(TokenKindSingleEQ) + if _, err := p.tryConsumeTokenKind(TokenKindSingleEQ); err != nil { + return nil, err + } engineExpr := &EngineExpr{EnginePos: pos} var engineEnd Pos @@ -1941,14 +1979,23 @@ func (p *Parser) parseColumnNamesExpr(pos Pos) (*ColumnNamesExpr, error) { } var columnNames []NestedIdentifier - for !p.lexer.isEOF() && p.tryConsumeTokenKind(TokenKindRParen) == nil { + for !p.lexer.isEOF() { + token, consumeErr := p.tryConsumeTokenKind(TokenKindRParen) + if consumeErr != nil { + return nil, consumeErr + } + if token != nil { + break + } name, err := p.ParseNestedIdentifier(p.Pos()) if err != nil { return nil, err } columnNames = append(columnNames, *name) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -1999,7 +2046,14 @@ func (p *Parser) parseAssignmentValues(pos Pos) (*AssignmentValues, error) { var value Expr var err error values := make([]Expr, 0) - for !p.lexer.isEOF() && p.tryConsumeTokenKind(TokenKindRParen) == nil { + for !p.lexer.isEOF() { + token, consumeErr := p.tryConsumeTokenKind(TokenKindRParen) + if consumeErr != nil { + return nil, consumeErr + } + if token != nil { + break + } switch { case p.matchTokenKind(TokenKindLParen): value, err = p.parseAssignmentValues(p.Pos()) @@ -2013,7 +2067,9 @@ func (p *Parser) parseAssignmentValues(pos Pos) (*AssignmentValues, error) { return nil, err } values = append(values, value) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -2073,7 +2129,9 @@ func (p *Parser) parseInsertStmt(pos Pos) (*InsertStmt, error) { return nil, err } values = append(values, value) - if p.tryConsumeTokenKind(TokenKindComma) == nil { + if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { + return nil, consumeErr + } else if token == nil { break } } @@ -2112,7 +2170,14 @@ func (p *Parser) parseRenameStmt(pos Pos) (*RenameStmt, error) { return nil, err } tablePairList := []*TargetPair{targetPair} - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } tablePair, err := p.parseTargetPair(p.Pos()) if err != nil { return nil, err @@ -2219,7 +2284,14 @@ func (p *Parser) parseDictionarySchemaClause(pos Pos) (*DictionarySchemaClause, schema.Attributes = append(schema.Attributes, attr) // Parse additional attributes - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } attr, err := p.parseDictionaryAttribute(p.Pos()) if err != nil { return nil, err @@ -2712,7 +2784,14 @@ func (p *Parser) parseDictionarySettingsClause(pos Pos) (*SettingsClause, error) items = append(items, expr) // Parse additional settings - for p.tryConsumeTokenKind(TokenKindComma) != nil { + for { + token, consumeErr := p.tryConsumeTokenKind(TokenKindComma) + if consumeErr != nil { + return nil, consumeErr + } + if token == nil { + break + } expr, err := p.parseSettingsExpr(p.Pos()) if err != nil { return nil, err From 03b5f95d38ce736c165d204d7a26d91a9b800ecd Mon Sep 17 00:00:00 2001 From: git-hulk Date: Thu, 10 Sep 2026 12:29:25 +0800 Subject: [PATCH 5/6] Return lexer errors without retaining them Remove Lexer.err and propagate consumption and lookahead failures through parser helpers. Keep error positions in returned lexerError values and retry only grammar errors during backtracking. Preserve keyword mismatch rollback and normal EOF. Cover direct keyword, lookahead, and backtracking errors, plus scanning valid input after a failed scan is rewound. make test, make lint, and make pass with Go 1.21.13 and the CI-pinned linter; 684 malformed fixture suffixes are rejected. --- parser/error_test.go | 56 +++++++ parser/lexer.go | 14 +- parser/lexer_test.go | 6 +- parser/parse_system.go | 354 +++++++++++++++++++++++++++++++--------- parser/parser_alter.go | 68 ++++++-- parser/parser_column.go | 346 ++++++++++++++++++++++++++------------- parser/parser_common.go | 74 ++++++--- parser/parser_drop.go | 26 ++- parser/parser_query.go | 284 ++++++++++++++++++++++++-------- parser/parser_table.go | 343 +++++++++++++++++++++++++++++--------- parser/parser_view.go | 66 ++++++-- 11 files changed, 1214 insertions(+), 423 deletions(-) diff --git a/parser/error_test.go b/parser/error_test.go index b926f7d..45f43a6 100644 --- a/parser/error_test.go +++ b/parser/error_test.go @@ -94,6 +94,35 @@ func TestParser_TokenConsumptionError(t *testing.T) { _, err := p.parseUserNames() return err }}, + {"required keyword", "SELECT /*", func(p *Parser) error { return p.expectKeyword(KeywordSelect) }}, + {"optional keywords", "WITH FILL /*", func(p *Parser) error { + _, err := p.tryConsumeKeywords(KeywordWith, KeywordFill) + return err + }}, + {"identifier", "a /*", func(p *Parser) error { + _, err := p.tryParseIdent() + return err + }}, + {"keyword lookahead", "case /*", func(p *Parser) error { + _, err := p.keywordIsSelectItemIdentifier() + return err + }}, + {"join lookahead", "GLOBAL LEFT /*", func(p *Parser) error { + _, err := p.peekJoinAfterLocality() + return err + }}, + {"interval backtracking", "interval 1 + /*", func(p *Parser) error { + _, err := p.parseColumnExpr(p.Pos()) + return err + }}, + {"TTL backtracking", "GROUP BY ALL + /*", func(p *Parser) error { + _, err := p.parseTTLPolicyGroupBy(p.Pos()) + return err + }}, + {"TTL assignment backtracking", "GROUP BY a SET x = 1, y = /*", func(p *Parser) error { + _, err := p.parseTTLPolicyGroupBy(p.Pos()) + return err + }}, } { t.Run(tc.name, func(t *testing.T) { p := NewParser(tc.sql) @@ -107,6 +136,33 @@ func TestParser_TokenConsumptionError(t *testing.T) { } } +func TestParser_TryConsumeKeywords(t *testing.T) { + for _, tc := range []struct { + sql string + matched bool + next string + }{ + {"WITH FILL", true, ""}, + {"WITH FILL tail", true, "tail"}, + {"SELECT 1", false, "SELECT"}, + {"WITH foo", false, "WITH"}, + {"WITH", false, "WITH"}, + } { + t.Run(tc.sql, func(t *testing.T) { + p := NewParser(tc.sql) + require.NoError(t, p.lexer.consumeToken()) + before := p.lexer.saveState() + matched, err := p.tryConsumeKeywords(KeywordWith, KeywordFill) + require.NoError(t, err) + require.Equal(t, tc.matched, matched) + require.Equal(t, tc.next, p.currentTokenString()) + if !matched { + require.Equal(t, before, p.lexer.saveState()) + } + }) + } +} + func TestParser_TryConsumeTokenKind(t *testing.T) { for _, tc := range []struct { name string diff --git a/parser/lexer.go b/parser/lexer.go index c8a861e..89e6942 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -83,11 +83,6 @@ type Lexer struct { lexerState input string - - // Lexical failures are fatal for this input, even when discovered during - // lookahead. Keep them outside lexerState so restoring a cursor cannot - // discard the error or its original position. - err *lexerError } func NewLexer(buf string) *Lexer { @@ -364,16 +359,13 @@ func (l *Lexer) consumeToken() (err error) { // replace the current token; keep the previous one to disambiguate unary +/- prevToken := l.currentToken l.currentToken = nil - if l.err != nil { - return l.err - } pos := Pos(l.offset) defer func() { if err != nil { - if !errors.As(err, &l.err) { - l.err = &lexerError{pos: pos, err: err} + var lexicalErr *lexerError + if !errors.As(err, &lexicalErr) { + err = &lexerError{pos: pos, err: err} } - err = l.err } }() diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 61011be..d79b847 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -48,8 +48,10 @@ func TestPeekToken_RestoresState(t *testing.T) { require.Equal(t, Pos(2), le.pos) require.Equal(t, peekErr, err) require.Nil(t, lexer.currentToken) - // Further advancement must not turn a lexical failure into EOF. - require.Equal(t, err, lexer.consumeToken()) + // Rewinding to valid input must not retain the failed scan's error. + lexer.restoreState(lexerState{}) + require.NoError(t, lexer.consumeToken()) + require.Equal(t, "x", lexer.currentToken.String) } else { require.NoError(t, err) require.Equal(t, token, lexer.currentToken) diff --git a/parser/parse_system.go b/parser/parse_system.go index beb500b..ab64673 100644 --- a/parser/parse_system.go +++ b/parser/parse_system.go @@ -41,13 +41,18 @@ func (p *Parser) parseSystemFlushExpr(pos Pos) (*SystemFlushExpr, error) { switch { case p.matchKeyword(KeywordLogs): curToken := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &SystemFlushExpr{ FlushPos: pos, StatementEnd: curToken.End, Logs: true, }, nil - case p.tryConsumeKeywords(KeywordDistributed): + case p.matchKeyword(KeywordDistributed): + if err := p.expectKeyword(KeywordDistributed); err != nil { + return nil, err + } distributed, err := p.parseTableIdentifier(p.Pos()) if err != nil { return nil, err @@ -75,13 +80,20 @@ func (p *Parser) parseSystemReloadExpr(pos Pos) (*SystemReloadExpr, error) { case p.matchKeyword(KeywordDictionaries): typ = KeywordDictionaries statementEnd = p.current().End - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } case p.matchKeyword(KeywordDictionary): typ = KeywordDictionary statementEnd = p.current().End hasDictionaryName = true - _ = p.lexer.consumeToken() - case p.tryConsumeKeywords(KeywordEmbedded): + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } + case p.matchKeyword(KeywordEmbedded): + if err := p.expectKeyword(KeywordEmbedded); err != nil { + return nil, err + } typ = "EMBEDDED DICTIONARIES" statementEnd = p.current().End if err := p.expectKeyword(KeywordDictionaries); err != nil { @@ -150,11 +162,16 @@ func (p *Parser) parseSystemCtrlExpr(pos Pos) (*SystemCtrlExpr, error) { return nil, fmt.Errorf("expected START|STOP") } command := strings.ToUpper(p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } var typ string switch { - case p.tryConsumeKeywords(KeywordDistributed): + case p.matchKeyword(KeywordDistributed): + if err := p.expectKeyword(KeywordDistributed); err != nil { + return nil, err + } switch { case p.matchKeyword(KeywordSends): typ = "DISTRIBUTED SENDS" @@ -181,7 +198,10 @@ func (p *Parser) parseSystemCtrlExpr(pos Pos) (*SystemCtrlExpr, error) { Type: typ, Cluster: cluster, }, nil - case p.tryConsumeKeywords(KeywordReplicated): + case p.matchKeyword(KeywordReplicated): + if err := p.expectKeyword(KeywordReplicated); err != nil { + return nil, err + } curToken := p.current() if err := p.expectKeyword(KeywordSends); err != nil { return nil, err @@ -209,7 +229,9 @@ func (p *Parser) parseSystemDropExpr(pos Pos) (*SystemDropExpr, error) { p.matchKeyword(KeywordFileSystem), p.matchKeyword(KeywordQuery): prefixToken := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } curToken := p.current() if err := p.expectKeyword(KeywordCache); err != nil { return nil, err @@ -220,7 +242,9 @@ func (p *Parser) parseSystemDropExpr(pos Pos) (*SystemDropExpr, error) { Type: prefixToken.String + " CACHE", }, nil case p.matchKeyword(KeywordCompiled): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordExpression); err != nil { return nil, err } @@ -249,7 +273,9 @@ func (p *Parser) parseDeduplicateClause(pos Pos) (*DeduplicateClause, error) { if err := p.expectKeyword(KeywordDeduplicate); err != nil { return nil, err } - if !p.tryConsumeKeywords(KeywordBy) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordBy); consumeErr != nil { + return nil, consumeErr + } else if !matched { return &DeduplicateClause{ DeduplicatePos: pos, }, nil @@ -260,7 +286,9 @@ func (p *Parser) parseDeduplicateClause(pos Pos) (*DeduplicateClause, error) { return nil, err } var except *ColumnExprList - if p.tryConsumeKeywords(KeywordExcept) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordExcept); consumeErr != nil { + return nil, consumeErr + } else if matched { except, err = p.parseColumnExprList(p.Pos()) if err != nil { return nil, err @@ -305,7 +333,9 @@ func (p *Parser) parseOptimizeStmt(pos Pos) (*OptimizeStmt, error) { hasFinal := false lastPos := p.Pos() - if p.tryConsumeKeywords(KeywordFinal) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordFinal); consumeErr != nil { + return nil, consumeErr + } else if matched { hasFinal = true statementEnd = lastPos } @@ -425,7 +455,9 @@ func (p *Parser) parseRoleName(_ Pos) (*RoleName, error) { } func (p *Parser) tryParseRoleSettings(pos Pos) ([]*RoleSetting, error) { - if !p.tryConsumeKeywords(KeywordSettings) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordSettings); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil } return p.parseRoleSettings(pos) @@ -509,7 +541,9 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { orReplace := false switch { case p.matchKeyword(KeywordIf): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordNot); err != nil { return nil, err } @@ -518,7 +552,9 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { } ifNotExists = true case p.matchKeyword(KeywordOr): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordReplace); err != nil { return nil, err } @@ -548,7 +584,9 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { statementEnd := roleNames[len(roleNames)-1].End() var accessStorageType *Ident - if p.tryConsumeKeywords(KeywordIn) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordIn); consumeErr != nil { + return nil, consumeErr + } else if matched { accessStorageType, err = p.parseIdent() if err != nil { return nil, err @@ -578,7 +616,9 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { func (p *Parser) parseAuthenticationClause(pos Pos) (*AuthenticationClause, error) { auth := &AuthenticationClause{AuthPos: pos} - if p.tryConsumeKeywords(KeywordNot) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordNot); consumeErr != nil { + return nil, consumeErr + } else if matched { if err := p.expectKeyword(KeywordIdentified); err != nil { return nil, err } @@ -592,9 +632,13 @@ func (p *Parser) parseAuthenticationClause(pos Pos) (*AuthenticationClause, erro } auth.AuthEnd = p.current().End - if p.tryConsumeKeywords(KeywordWith) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordWith); consumeErr != nil { + return nil, consumeErr + } else if matched { if p.matchKeyword(KeywordLdap) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordServer); err != nil { return nil, err } @@ -605,10 +649,14 @@ func (p *Parser) parseAuthenticationClause(pos Pos) (*AuthenticationClause, erro auth.LdapServer = server auth.AuthEnd = server.End() } else if p.matchKeyword(KeywordKerberos) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } auth.IsKerberos = true auth.AuthEnd = p.current().End - if p.tryConsumeKeywords(KeywordRealm) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordRealm); consumeErr != nil { + return nil, consumeErr + } else if matched { realm, err := p.parseString(p.Pos()) if err != nil { return nil, err @@ -619,11 +667,15 @@ func (p *Parser) parseAuthenticationClause(pos Pos) (*AuthenticationClause, erro } else if p.matchTokenKind(TokenKindIdent) { // Auth types like no_password, plaintext_password, etc. authType := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } auth.AuthType = authType auth.AuthEnd = p.current().End - if p.tryConsumeKeywords(KeywordBy) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordBy); consumeErr != nil { + return nil, consumeErr + } else if matched { value, err := p.parseString(p.Pos()) if err != nil { return nil, err @@ -647,12 +699,16 @@ func (p *Parser) parseHostClause(pos Pos) (*HostClause, error) { switch { case p.matchOneOfKeywords(KeywordLocal, KeywordAny, KeywordNone): hostType := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } host.HostType = hostType host.HostEnd = p.current().End case p.matchOneOfKeywords(KeywordName, KeywordRegexp, KeywordIp, KeywordLike): hostType := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } host.HostType = hostType value, err := p.parseString(p.Pos()) if err != nil { @@ -677,7 +733,9 @@ func (p *Parser) parseDefaultRoleClause(pos Pos) (*DefaultRoleClause, error) { defaultRole := &DefaultRoleClause{DefaultPos: pos} - if p.tryConsumeKeywords(KeywordNone) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordNone); consumeErr != nil { + return nil, consumeErr + } else if matched { defaultRole.None = true defaultRole.DefaultEnd = p.current().End return defaultRole, nil @@ -717,10 +775,14 @@ func (p *Parser) parseGranteesClause(pos Pos) (*GranteesClause, error) { grantees := &GranteesClause{GranteesPos: pos} - if p.tryConsumeKeywords(KeywordAny) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAny); consumeErr != nil { + return nil, consumeErr + } else if matched { grantees.Any = true grantees.GranteesEnd = p.current().End - } else if p.tryConsumeKeywords(KeywordNone) { + } else if matched, consumeErr := p.tryConsumeKeywords(KeywordNone); consumeErr != nil { + return nil, consumeErr + } else if matched { grantees.None = true grantees.GranteesEnd = p.current().End } else { @@ -752,7 +814,9 @@ func (p *Parser) parseGranteesClause(pos Pos) (*GranteesClause, error) { } // Check for EXCEPT clause - if p.tryConsumeKeywords(KeywordExcept) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordExcept); consumeErr != nil { + return nil, consumeErr + } else if matched { exceptList := make([]*RoleName, 0) except, err := p.parseRoleName(p.Pos()) if err != nil { @@ -785,7 +849,9 @@ func (p *Parser) parseGranteesClause(pos Pos) (*GranteesClause, error) { func (p *Parser) parseCreateUserModifiers(createUser *CreateUser) error { switch { case p.matchKeyword(KeywordIf): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return err + } if err := p.expectKeyword(KeywordNot); err != nil { return err } @@ -794,7 +860,9 @@ func (p *Parser) parseCreateUserModifiers(createUser *CreateUser) error { } createUser.IfNotExists = true case p.matchKeyword(KeywordOr): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return err + } if err := p.expectKeyword(KeywordReplace); err != nil { return err } @@ -868,9 +936,15 @@ func (p *Parser) parseDefaultClause(createUser *CreateUser) (bool, error) { createUser.StatementEnd = defaultRole.End() return true, nil } else if nextToken.String == KeywordDatabase { - _ = p.lexer.consumeToken() // consume DEFAULT - _ = p.lexer.consumeToken() // consume DATABASE - if p.tryConsumeKeywords(KeywordNone) { + if err := p.lexer.consumeToken(); err != nil { // consume DEFAULT + return false, err + } + if err := p.lexer.consumeToken(); err != nil { // consume DATABASE + return false, err + } + if matched, consumeErr := p.tryConsumeKeywords(KeywordNone); consumeErr != nil { + return false, consumeErr + } else if matched { createUser.DefaultDbNone = true createUser.StatementEnd = p.current().End } else { @@ -899,7 +973,9 @@ func (p *Parser) parseOptionalClauses(createUser *CreateUser) error { createUser.StatementEnd = auth.End() case p.matchKeyword(KeywordValid): - _ = p.lexer.consumeToken() // consume VALID keyword + if err := p.lexer.consumeToken(); err != nil { // consume VALID keyword + return err + } if err := p.expectKeyword(KeywordUntil); err != nil { return err } @@ -936,7 +1012,9 @@ func (p *Parser) parseOptionalClauses(createUser *CreateUser) error { createUser.StatementEnd = grantees.End() case p.matchKeyword(KeywordSettings): - _ = p.lexer.consumeToken() // consume SETTINGS keyword + if err := p.lexer.consumeToken(); err != nil { // consume SETTINGS keyword + return err + } settings, err := p.parseRoleSettings(p.Pos()) if err != nil { return err @@ -986,7 +1064,9 @@ func (p *Parser) parserDropUserOrRole(pos Pos) (*DropUserOrRole, error) { switch { case p.matchOneOfKeywords(KeywordUser, KeywordRole): target = p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } default: return nil, fmt.Errorf("expected USER|ROLE") } @@ -1027,7 +1107,9 @@ func (p *Parser) parserDropUserOrRole(pos Pos) (*DropUserOrRole, error) { } var from *Ident - if p.tryConsumeKeywords(KeywordFrom) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordFrom); consumeErr != nil { + return nil, consumeErr + } else if matched { from, err = p.parseIdent() if err != nil { return nil, err @@ -1052,7 +1134,9 @@ func (p *Parser) parserDropUserOrRole(pos Pos) (*DropUserOrRole, error) { func (p *Parser) parsePrivilegeSelectOrInsert(pos Pos) (*PrivilegeClause, error) { keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } var err error var params *ParamExprList @@ -1072,52 +1156,88 @@ func (p *Parser) parsePrivilegeSelectOrInsert(pos Pos) (*PrivilegeClause, error) func (p *Parser) parsePrivilegeAlter(pos Pos) (*PrivilegeClause, error) { keywords := []string{KeywordAlter} switch { - case p.tryConsumeKeywords(KeywordIndex): + case p.matchKeyword(KeywordIndex): + if err := p.expectKeyword(KeywordIndex); err != nil { + return nil, err + } keywords = append(keywords, KeywordIndex) case p.matchOneOfKeywords(KeywordUpdate, KeywordDelete, KeywordUser, KeywordRole, KeywordQuota): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) case p.matchOneOfKeywords(KeywordAdd, KeywordDrop, KeywordModify, KeywordClear, KeywordComment, KeywordRename, KeywordMaterialized): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) switch { - case p.tryConsumeKeywords(KeywordColumn): + case p.matchKeyword(KeywordColumn): + if err := p.expectKeyword(KeywordColumn); err != nil { + return nil, err + } keywords = append(keywords, KeywordColumn) - case p.tryConsumeKeywords(KeywordIndex): + case p.matchKeyword(KeywordIndex): + if err := p.expectKeyword(KeywordIndex); err != nil { + return nil, err + } keywords = append(keywords, KeywordIndex) keywords = append(keywords, KeywordConstraint) - case p.tryConsumeKeywords(KeywordTtl): + case p.matchKeyword(KeywordTtl): + if err := p.expectKeyword(KeywordTtl); err != nil { + return nil, err + } keywords = append(keywords, KeywordTtl) default: return nil, fmt.Errorf("expected COLUMN|INDEX") } - case p.tryConsumeKeywords(KeywordOrder): + case p.matchKeyword(KeywordOrder): + if err := p.expectKeyword(KeywordOrder); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordBy); err != nil { return nil, err } keywords = append(keywords, KeywordOrder, KeywordBy) - case p.tryConsumeKeywords(KeywordSample): + case p.matchKeyword(KeywordSample): + if err := p.expectKeyword(KeywordSample); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordBy); err != nil { return nil, err } keywords = append(keywords, KeywordSample, KeywordBy) - case p.tryConsumeKeywords(KeywordSettings): + case p.matchKeyword(KeywordSettings): + if err := p.expectKeyword(KeywordSettings); err != nil { + return nil, err + } keywords = append(keywords, KeywordSettings) - case p.tryConsumeKeywords(KeywordView): + case p.matchKeyword(KeywordView): + if err := p.expectKeyword(KeywordView); err != nil { + return nil, err + } keywords = append(keywords, KeywordView) switch { - case p.tryConsumeKeywords(KeywordModify): + case p.matchKeyword(KeywordModify): + if err := p.expectKeyword(KeywordModify); err != nil { + return nil, err + } keywords = append(keywords, KeywordModify) - case p.tryConsumeKeywords(KeywordRefresh): + case p.matchKeyword(KeywordRefresh): + if err := p.expectKeyword(KeywordRefresh); err != nil { + return nil, err + } keywords = append(keywords, KeywordRefresh) default: return nil, fmt.Errorf("expected MODIFY|REFRESH") } case p.matchOneOfKeywords(KeywordMove, KeywordFreeze): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) if err := p.expectKeyword(KeywordPartition); err != nil { return nil, err @@ -1137,14 +1257,22 @@ func (p *Parser) parsePrivilegeCreate(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordDatabase, KeywordDictionary, KeywordTable, KeywordFunction, KeywordView, KeywordUser, KeywordRole, KeywordQuota): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) - case p.tryConsumeKeywords(KeywordTemporary): + case p.matchKeyword(KeywordTemporary): + if err := p.expectKeyword(KeywordTemporary); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordTable); err != nil { return nil, err } keywords = append(keywords, KeywordTemporary, KeywordTable) - case p.tryConsumeKeywords(KeywordRows): + case p.matchKeyword(KeywordRows): + if err := p.expectKeyword(KeywordRows); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordPolicy); err != nil { return nil, err } @@ -1163,7 +1291,9 @@ func (p *Parser) parsePrivilegeDrop(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordDatabase, KeywordDictionary, KeywordUser, KeywordRole, KeywordQuota, KeywordTable, KeywordFunction, KeywordView): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) default: return nil, fmt.Errorf("expected DATABASE|DICTIONARY|TABLE|FUNCTION|VIEW") @@ -1179,7 +1309,9 @@ func (p *Parser) parsePrivilegeShow(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordDatabases, KeywordDictionaries, KeywordTables, KeywordColumns): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) default: return nil, fmt.Errorf("expected DATABASES|DICTIONARIES|TABLES|COLUMNS") @@ -1195,16 +1327,26 @@ func (p *Parser) parsePrivilegeSystem(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordShutdown, KeywordMerges, KeywordFetches, KeywordSends, KeywordMoves, KeywordCluster): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) - case p.tryConsumeKeywords(KeywordDrop): + case p.matchKeyword(KeywordDrop): + if err := p.expectKeyword(KeywordDrop); err != nil { + return nil, err + } keywords = append(keywords, KeywordDrop) switch { - case p.tryConsumeKeywords(KeywordCache): + case p.matchKeyword(KeywordCache): + if err := p.expectKeyword(KeywordCache); err != nil { + return nil, err + } keywords = append(keywords, KeywordCache) case p.matchOneOfKeywords(KeywordMark, KeywordDNS, KeywordUncompressed): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) if err := p.expectKeyword(KeywordCache); err != nil { return nil, err @@ -1213,27 +1355,40 @@ func (p *Parser) parsePrivilegeSystem(pos Pos) (*PrivilegeClause, error) { default: return nil, fmt.Errorf("expected CACHE|MARK|DNS|UNCOMPRESSED") } - case p.tryConsumeKeywords(KeywordReload): + case p.matchKeyword(KeywordReload): + if err := p.expectKeyword(KeywordReload); err != nil { + return nil, err + } keywords = append(keywords, KeywordReload) switch { case p.matchOneOfKeywords(KeywordDictionary, KeywordFunction, KeywordFunctions, KeywordConfig): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) default: return nil, fmt.Errorf("expected DICTIONARY|FUNCTION|FUNCTIONS|CONFIG") } - case p.tryConsumeKeywords(KeywordFlush): + case p.matchKeyword(KeywordFlush): + if err := p.expectKeyword(KeywordFlush); err != nil { + return nil, err + } keywords = append(keywords, KeywordFlush) switch { case p.matchOneOfKeywords(KeywordLogs, KeywordDistributed): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) default: return nil, fmt.Errorf("expected LOGS|DISTRIBUTED") } - case p.tryConsumeKeywords(KeywordTtl): + case p.matchKeyword(KeywordTtl): + if err := p.expectKeyword(KeywordTtl); err != nil { + return nil, err + } keywords = append(keywords, KeywordTtl) if err := p.expectKeyword(KeywordMerges); err != nil { return nil, err @@ -1241,13 +1396,18 @@ func (p *Parser) parsePrivilegeSystem(pos Pos) (*PrivilegeClause, error) { keywords = append(keywords, KeywordMerges) case p.matchOneOfKeywords(KeywordSync, KeywordRestart): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } keywords = append(keywords, keyword) if err := p.expectKeyword(KeywordReplica); err != nil { return nil, err } keywords = append(keywords, KeywordReplica) - case p.tryConsumeKeywords(KeywordReplication): + case p.matchKeyword(KeywordReplication): + if err := p.expectKeyword(KeywordReplication); err != nil { + return nil, err + } keywords = append(keywords, KeywordReplication) if err := p.expectKeyword(KeywordQueues); err != nil { return nil, err @@ -1265,7 +1425,9 @@ func (p *Parser) parsePrivilegeSystem(pos Pos) (*PrivilegeClause, error) { func (p *Parser) parsePrivilegeClause(pos Pos) (*PrivilegeClause, error) { if p.matchTokenKind(TokenKindIdent) { if p.current().String == "dictGet" { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &PrivilegeClause{ PrivilegePos: pos, Keywords: []string{"dictGet"}, @@ -1275,21 +1437,38 @@ func (p *Parser) parsePrivilegeClause(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordSelect, KeywordInsert): return p.parsePrivilegeSelectOrInsert(pos) - case p.tryConsumeKeywords(KeywordAlter): + case p.matchKeyword(KeywordAlter): + if err := p.expectKeyword(KeywordAlter); err != nil { + return nil, err + } return p.parsePrivilegeAlter(pos) - case p.tryConsumeKeywords(KeywordCreate): + case p.matchKeyword(KeywordCreate): + if err := p.expectKeyword(KeywordCreate); err != nil { + return nil, err + } return p.parsePrivilegeCreate(pos) - case p.tryConsumeKeywords(KeywordDrop): + case p.matchKeyword(KeywordDrop): + if err := p.expectKeyword(KeywordDrop); err != nil { + return nil, err + } return p.parsePrivilegeDrop(pos) - case p.tryConsumeKeywords(KeywordShow): + case p.matchKeyword(KeywordShow): + if err := p.expectKeyword(KeywordShow); err != nil { + return nil, err + } return p.parsePrivilegeShow(pos) case p.matchKeyword(KeywordAll), p.matchTokenKind(KeywordNone): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &PrivilegeClause{ PrivilegePos: pos, Keywords: []string{KeywordAll}, }, nil - case p.tryConsumeKeywords(KeywordKill): + case p.matchKeyword(KeywordKill): + if err := p.expectKeyword(KeywordKill); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordQuery); err != nil { return nil, err } @@ -1297,9 +1476,15 @@ func (p *Parser) parsePrivilegeClause(pos Pos) (*PrivilegeClause, error) { PrivilegePos: pos, Keywords: []string{KeywordKill, KeywordQuery}, }, nil - case p.tryConsumeKeywords(KeywordSystem): + case p.matchKeyword(KeywordSystem): + if err := p.expectKeyword(KeywordSystem); err != nil { + return nil, err + } return p.parsePrivilegeSystem(pos) - case p.tryConsumeKeywords(KeywordAdmin): + case p.matchKeyword(KeywordAdmin): + if err := p.expectKeyword(KeywordAdmin); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordOption); err != nil { return nil, err } @@ -1309,12 +1494,17 @@ func (p *Parser) parsePrivilegeClause(pos Pos) (*PrivilegeClause, error) { }, nil case p.matchOneOfKeywords(KeywordOptimize, KeywordTruncate): keyword := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &PrivilegeClause{ PrivilegePos: pos, Keywords: []string{keyword}, }, nil - case p.tryConsumeKeywords(KeywordRole): + case p.matchKeyword(KeywordRole): + if err := p.expectKeyword(KeywordRole); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordAdmin); err != nil { return nil, err } @@ -1531,7 +1721,9 @@ func (p *Parser) parseRoleRenamePair(_ Pos) (*RoleRenamePair, error) { RoleName: roleName, StatementEnd: roleName.End(), } - if p.tryConsumeKeywords(KeywordRename) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordRename); consumeErr != nil { + return nil, consumeErr + } else if matched { if err := p.expectKeyword(KeywordTo); err != nil { return nil, err } diff --git a/parser/parser_alter.go b/parser/parser_alter.go index 7c84c93..6046c76 100644 --- a/parser/parser_alter.go +++ b/parser/parser_alter.go @@ -38,7 +38,9 @@ func (p *Parser) parseAlterTable(pos Pos) (*AlterTable, error) { // like the sibling branches, the clause position is the keyword // itself, so capture it before consuming DETACH detachPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } alter, err = p.parseAlterTableDetachPartition(detachPos) case p.matchKeyword(KeywordFreeze): alter, err = p.parseAlterTableFreezePartition(p.Pos()) @@ -180,7 +182,9 @@ func (p *Parser) parseAlterTableAddIndex(pos Pos) (*AlterTableAddIndex, error) { } func (p *Parser) tryParseProjectionOrderBy(pos Pos) (*ProjectionOrderByClause, error) { - if !p.tryConsumeKeywords(KeywordOrder) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordOrder); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } if err := p.expectKeyword(KeywordBy); err != nil { @@ -379,13 +383,17 @@ func (p *Parser) parsePartitionClause(pos Pos) (*PartitionClause, error) { partition := &PartitionClause{ PartitionPos: pos, } - if p.tryConsumeKeywords(KeywordId) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordId); consumeErr != nil { + return nil, consumeErr + } else if matched { id, err := p.parseString(p.Pos()) if err != nil { return nil, err } partition.ID = id - } else if p.tryConsumeKeywords(KeywordAll) { + } else if matched, consumeErr := p.tryConsumeKeywords(KeywordAll); consumeErr != nil { + return nil, consumeErr + } else if matched { partition.All = true } else { expr, err := p.parseExpr(p.Pos()) @@ -410,7 +418,9 @@ func (p *Parser) parseAlterTableAttachPartition(pos Pos) (AlterTableClause, erro } alterTable.Partition = partition // FROM [db.]table? - if p.tryConsumeKeywords(KeywordFrom) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordFrom); consumeErr != nil { + return nil, consumeErr + } else if matched { tableIdentifier, err := p.parseTableIdentifier(p.Pos()) if err != nil { return nil, err @@ -432,7 +442,9 @@ func (p *Parser) parseAlterTableDropClause(pos Pos) (AlterTableClause, error) { default: return nil, fmt.Errorf("expected token: COLUMN|INDEX|PROJECTION, but got %s", p.currentTokenKind()) } - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } ifExists, err := p.tryParseIfExists() if err != nil { @@ -466,7 +478,9 @@ func (p *Parser) parseAlterTableDropClause(pos Pos) (AlterTableClause, error) { } func (p *Parser) tryParseAfterClause() (*NestedIdentifier, error) { - if !p.tryConsumeKeywords(KeywordAfter) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAfter); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } @@ -477,7 +491,9 @@ func (p *Parser) tryParseAfterClause() (*NestedIdentifier, error) { func (p *Parser) parseAlterTableDropPartition(pos Pos) (AlterTableClause, error) { var hasDetached bool if p.matchKeyword(KeywordDetached) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } hasDetached = true } partitionPos := p.Pos() @@ -561,7 +577,9 @@ func (p *Parser) parseAlterTableClearClause(pos Pos) (AlterTableClause, error) { default: return nil, fmt.Errorf("expected keyword: COLUMN|INDEX|PROJECTION, but got %q", p.currentTokenKind()) } - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } ifExists, err := p.tryParseIfExists() if err != nil { @@ -575,7 +593,9 @@ func (p *Parser) parseAlterTableClearClause(pos Pos) (AlterTableClause, error) { statementEnd := name.End() var partition *PartitionClause - if p.tryConsumeKeywords(KeywordIn) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordIn); consumeErr != nil { + return nil, consumeErr + } else if matched { partition, err = p.tryParsePartitionClause(p.Pos()) if err != nil { return nil, err @@ -659,7 +679,9 @@ func (p *Parser) parseAlterTableModify(pos Pos) (AlterTableClause, error) { return p.parseAlterTableModifyColumn(pos) case p.matchKeyword(KeywordTtl): ttlPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } items, err := p.parseTTLClause(ttlPos, true) if err != nil { return nil, err @@ -679,7 +701,9 @@ func (p *Parser) parseAlterTableModify(pos Pos) (AlterTableClause, error) { TTL: ttlClause, }, nil case p.matchKeyword(KeywordQuery): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } selectQuery, err := p.parseSelectQuery(pos) if err != nil { return nil, err @@ -690,7 +714,9 @@ func (p *Parser) parseAlterTableModify(pos Pos) (AlterTableClause, error) { SelectExpr: selectQuery, }, nil case p.matchKeyword(KeywordOrder): - _ = p.lexer.consumeToken() // consume "ORDER" + if err := p.lexer.consumeToken(); err != nil { // consume "ORDER" + return nil, err + } if err := p.expectKeyword(KeywordBy); err != nil { return nil, err } @@ -704,7 +730,9 @@ func (p *Parser) parseAlterTableModify(pos Pos) (AlterTableClause, error) { OrderBy: orderBy, }, nil case p.matchKeyword(KeywordSetting): - _ = p.lexer.consumeToken() // consume "SETTING" + if err := p.lexer.consumeToken(); err != nil { // consume "SETTING" + return nil, err + } settings, err := p.parseSettingsList(p.Pos()) if err != nil { return nil, err @@ -816,7 +844,9 @@ func (p *Parser) parseAlterTableMaterialize(pos Pos) (AlterTableClause, error) { default: return nil, fmt.Errorf("expected keyword: INDEX|PROJECTION, but got %q", p.currentTokenKind()) } - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } ifExists, err := p.tryParseIfExists() if err != nil { @@ -828,7 +858,9 @@ func (p *Parser) parseAlterTableMaterialize(pos Pos) (AlterTableClause, error) { } statementEnd := name.End() var partition *PartitionClause - if p.tryConsumeKeywords(KeywordIn) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordIn); consumeErr != nil { + return nil, consumeErr + } else if matched { partition, err = p.tryParsePartitionClause(p.Pos()) if err != nil { return nil, err @@ -947,7 +979,9 @@ func (p *Parser) parseAlterTableUpdate(pos Pos) (AlterTableClause, error) { } var inPartition *PartitionClause - if p.tryConsumeKeywords(KeywordIn) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordIn); consumeErr != nil { + return nil, consumeErr + } else if matched { inPartition, err = p.parsePartitionClause(p.Pos()) if err != nil { return nil, err diff --git a/parser/parser_column.go b/parser/parser_column.go index 16257aa..5a49cd1 100644 --- a/parser/parser_column.go +++ b/parser/parser_column.go @@ -1,6 +1,7 @@ package parser import ( + "errors" "fmt" "strings" ) @@ -30,66 +31,77 @@ const ( ) func (p *Parser) tryParseColumnComment(pos Pos) (*StringLiteral, error) { - if !p.tryConsumeKeywords(KeywordComment) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordComment); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } return p.parseString(pos) } -func (p *Parser) getNextPrecedence() int { +func (p *Parser) getNextPrecedence() (int, error) { switch { case p.matchKeyword(KeywordOr): - return PrecedenceOr + return PrecedenceOr, nil case p.matchKeyword(KeywordAnd): - return PrecedenceAnd + return PrecedenceAnd, nil case p.matchKeyword(KeywordIs): - return PrecedenceIs + return PrecedenceIs, nil case p.matchKeyword(KeywordNot): // Infix NOT only begins NOT IN/LIKE/ILIKE/BETWEEN, so it binds with // the precedence of the operator it negates; `a = b NOT IN (1)` must // group the same way `a = b IN (1)` does. - switch { - case p.peekKeyword(KeywordIn): - return precedenceIn - case p.peekKeyword(KeywordLike), p.peekKeyword(KeywordIlike), p.peekKeyword(KeywordBetween): - return PrecedenceBetweenLike + next, err := p.lexer.peekToken() + if err != nil { + return 0, err + } + if next == nil || next.Kind != TokenKindKeyword { + return PrecedenceNot, nil + } + switch next.ToString() { + case KeywordIn: + return precedenceIn, nil + case KeywordLike, KeywordIlike, KeywordBetween: + return PrecedenceBetweenLike, nil default: - return PrecedenceNot + return PrecedenceNot, nil } case p.matchTokenKind(TokenKindDot): - return PrecedenceDot + return PrecedenceDot, nil case p.matchTokenKind(TokenKindDash): - return PrecedenceDoubleColon + return PrecedenceDoubleColon, nil case p.matchTokenKind(TokenKindSingleEQ), p.matchTokenKind(TokenKindLT), p.matchTokenKind(TokenKindLE), p.matchTokenKind(TokenKindGE), p.matchTokenKind(TokenKindGT), p.matchTokenKind(TokenKindDoubleEQ), p.matchTokenKind(TokenKindNE), p.matchTokenKind("<>"): - return PrecedenceCompare + return PrecedenceCompare, nil case p.matchTokenKind(TokenKindConcat): - return PrecedenceConcat + return PrecedenceConcat, nil case p.matchTokenKind(TokenKindPlus), p.matchTokenKind(TokenKindMinus): - return PrecedenceAddSub + return PrecedenceAddSub, nil case p.matchTokenKind(TokenKindMul), p.matchTokenKind(TokenKindDiv), p.matchTokenKind(TokenKindMod): - return PrecedenceMulDivMod + return PrecedenceMulDivMod, nil case p.matchTokenKind(TokenKindArrow): - return PrecedenceArrow + return PrecedenceArrow, nil case p.matchTokenKind(TokenKindLParen), p.matchTokenKind(TokenKindLBracket): - return PrecedenceBracket + return PrecedenceBracket, nil case p.matchKeyword(KeywordBetween), p.matchKeyword(KeywordLike), p.matchKeyword(KeywordIlike), p.matchKeyword(KeywordRegexp): - return PrecedenceBetweenLike + return PrecedenceBetweenLike, nil case p.matchKeyword(KeywordIn): - return precedenceIn + return precedenceIn, nil case p.matchKeyword(KeywordGlobal): // GLOBAL is also a join locality: in `ON a = b GLOBAL LEFT JOIN c` it // belongs to the FROM clause, so the expression has to end here. - if p.peekJoinAfterLocality() { - return PrecedenceUnknown + if join, err := p.peekJoinAfterLocality(); err != nil { + return 0, err + } else if join { + return PrecedenceUnknown, nil } - return precedenceIn + return precedenceIn, nil case p.matchTokenKind(TokenKindQuestionMark): - return PrecedenceQuery + return PrecedenceQuery, nil default: - return PrecedenceUnknown + return PrecedenceUnknown, nil } } @@ -105,7 +117,9 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { p.matchKeyword(KeywordAnd), p.matchKeyword(KeywordOr), p.matchTokenKind(TokenKindDoubleEQ): op := p.current().ToString() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } rightExpr, err := p.parseSubExpr(p.Pos(), precedence) if err != nil { return nil, err @@ -116,7 +130,9 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { RightExpr: rightExpr, }, nil case p.matchTokenKind(TokenKindArrow): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } // Lambdas are right-associative: `x -> y -> body` is `x -> (y -> body)`, // so the body is parsed one level below the arrow's own precedence. rightExpr, err := p.parseSubExpr(p.Pos(), precedence-1) @@ -129,7 +145,9 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { RightExpr: rightExpr, }, nil case p.matchTokenKind(TokenKindDash): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if p.matchTokenKind(TokenKindIdent) && p.current().String == "Tuple" { name, err := p.parseIdent() @@ -163,8 +181,13 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { case p.matchKeyword(KeywordBetween): return p.parseBetweenClause(expr, false) case p.matchKeyword(KeywordGlobal): - _ = p.lexer.consumeToken() - hasNot := p.tryConsumeKeywords(KeywordNot) + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } + hasNot, consumeErr := p.tryConsumeKeywords(KeywordNot) + if consumeErr != nil { + return nil, consumeErr + } if p.expectKeyword(KeywordIn) != nil { return nil, fmt.Errorf("expected IN after GLOBAL, got %s", p.currentTokenString()) } @@ -186,7 +209,9 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { HasNot: hasNot, }, nil case p.matchTokenKind(TokenKindDot): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } operation := TokenKindDot qualifier, consumeErr := p.tryConsumeTokenKind(TokenKindColon) if consumeErr != nil { @@ -217,7 +242,9 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { Index: rightExpr, }, nil case p.matchKeyword(KeywordNot): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if p.matchKeyword(KeywordBetween) { return p.parseBetweenClause(expr, true) } @@ -229,7 +256,9 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { return nil, fmt.Errorf("expected IN, LIKE, ILIKE or BETWEEN after NOT, got %s", p.currentTokenKind()) } op := p.current().ToString() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } rightExpr, err := p.parseSubExpr(p.Pos(), precedence) if err != nil { return nil, err @@ -252,8 +281,13 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { return p.parseTernaryExpr(expr) case p.matchKeyword(KeywordIs): isPos := p.Pos() - _ = p.lexer.consumeToken() - isNotNull := p.tryConsumeKeywords(KeywordNot) + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } + isNotNull, consumeErr := p.tryConsumeKeywords(KeywordNot) + if consumeErr != nil { + return nil, consumeErr + } // the expression ends at the NULL keyword; capture its end before // expectKeyword consumes it nullEnd := p.End() @@ -302,12 +336,14 @@ func (p *Parser) parseSubExpr(pos Pos, precedence int) (Expr, error) { // and `SELECT a GLOBAL` silently read the operator as an implicit alias. func (p *Parser) parseInfixLoop(expr Expr, precedence int) (Expr, error) { for !p.lexer.isEOF() || p.current() != nil { - nextPrecedence := p.getNextPrecedence() + nextPrecedence, err := p.getNextPrecedence() + if err != nil { + return nil, err + } if nextPrecedence <= precedence { return expr, nil } // parse binary operation - var err error expr, err = p.parseInfix(expr, nextPrecedence) if err != nil { return nil, err @@ -414,10 +450,14 @@ func (p *Parser) parseUnaryExpr(pos Pos) (Expr, error) { switch { case p.matchTokenKind(TokenKindPlus), p.matchTokenKind(TokenKindMinus): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } expr, err = p.parseColumnExpr(p.Pos()) case p.matchKeyword(KeywordNot): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } // Prefix NOT binds looser than comparisons: `NOT a = b` negates the // whole comparison, so the operand is parsed at NOT's own precedence // instead of stopping at the primary expression. @@ -437,26 +477,36 @@ func (p *Parser) parseUnaryExpr(pos Pos) (Expr, error) { } -func (p *Parser) peekTokenKind(kind TokenKind) bool { - if p.lexer.isEOF() { - return false - } +func (p *Parser) peekTokenKind(kinds ...TokenKind) (bool, error) { token, err := p.lexer.peekToken() - if err != nil || token == nil { - return false + if err != nil { + return false, err + } + if token == nil { + return false, nil } - return token.Kind == kind + for _, kind := range kinds { + if token.Kind == kind { + return true, nil + } + } + return false, nil } -func (p *Parser) peekKeyword(keyword string) bool { - if p.lexer.isEOF() { - return false - } +func (p *Parser) peekKeyword(keywords ...string) (bool, error) { token, err := p.lexer.peekToken() - if err != nil || token == nil { - return false + if err != nil { + return false, err } - return token.Kind == TokenKindKeyword && strings.EqualFold(token.String, keyword) + if token == nil || token.Kind != TokenKindKeyword { + return false, nil + } + for _, keyword := range keywords { + if strings.EqualFold(token.String, keyword) { + return true, nil + } + } + return false, nil } // clauseStarterKeywords lists the keywords that begin a clause following the @@ -482,25 +532,17 @@ func (p *Parser) matchClauseStarterKeyword() bool { // peekIsClauseStarterKeyword reports whether the next token is one of the // clause-starter keywords. -func (p *Parser) peekIsClauseStarterKeyword() bool { - for _, kw := range clauseStarterKeywords { - if p.peekKeyword(kw) { - return true - } - } - return false +func (p *Parser) peekIsClauseStarterKeyword() (bool, error) { + return p.peekKeyword(clauseStarterKeywords...) } // peekIsEndOfStatement reports whether the next token is EOF or `;`. -func (p *Parser) peekIsEndOfStatement() bool { +func (p *Parser) peekIsEndOfStatement() (bool, error) { next, err := p.lexer.peekToken() if err != nil { - return false + return false, err } - if next == nil { - return true - } - return next.Kind == ";" + return next == nil || next.Kind == ";", nil } var expressionContinuationTokenKinds = []TokenKind{ @@ -516,18 +558,11 @@ var expressionContinuationKeywords = []string{ KeywordBetween, KeywordIs, KeywordThen, KeywordElse, KeywordEnd, } -func (p *Parser) peekIsExpressionContinuation() bool { - for _, kind := range expressionContinuationTokenKinds { - if p.peekTokenKind(kind) { - return true - } +func (p *Parser) peekIsExpressionContinuation() (bool, error) { + if matched, err := p.peekTokenKind(expressionContinuationTokenKinds...); err != nil || matched { + return matched, err } - for _, keyword := range expressionContinuationKeywords { - if p.peekKeyword(keyword) { - return true - } - } - return false + return p.peekKeyword(expressionContinuationKeywords...) } // keywordIsSelectItemIdentifier reports whether the current keyword token is @@ -551,24 +586,31 @@ func (p *Parser) peekIsExpressionContinuation() bool { // trailing clause-starter keyword like `FROM` at EOF must still be treated // as a terminator, not a no-AS alias). parseColumnExpr applies the eos // disambiguator inline. -func (p *Parser) keywordIsSelectItemIdentifier() bool { +func (p *Parser) keywordIsSelectItemIdentifier() (bool, error) { if !p.matchTokenKind(TokenKindKeyword) { - return false + return false, nil } - return p.peekTokenKind(TokenKindComma) || - p.peekKeyword(KeywordAs) || - p.peekIsClauseStarterKeyword() || - p.peekIsExpressionContinuation() + if matched, err := p.peekTokenKind(TokenKindComma); err != nil || matched { + return matched, err + } + if matched, err := p.peekKeyword(KeywordAs); err != nil || matched { + return matched, err + } + if matched, err := p.peekIsClauseStarterKeyword(); err != nil || matched { + return matched, err + } + return p.peekIsExpressionContinuation() } // isSelectItemTerminatorKeyword checks whether the current token is a keyword // that begins a clause following the SELECT item list. When true, we should // not treat the keyword itself as a bare alias. -func (p *Parser) isSelectItemTerminatorKeyword() bool { - if p.keywordIsSelectItemIdentifier() { - return false +func (p *Parser) isSelectItemTerminatorKeyword() (bool, error) { + identifier, err := p.keywordIsSelectItemIdentifier() + if err != nil { + return false, err } - return p.matchClauseStarterKeyword() + return !identifier && p.matchClauseStarterKeyword(), nil } func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen @@ -576,9 +618,12 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen // qualified column reference, even when it is otherwise reserved (for // example, `kill.item_id`). INTERVAL must reach its dedicated parser // first because a dot can also start its numeric operand. - if !p.matchKeyword(KeywordInterval) && - p.matchTokenKind(TokenKindKeyword) && p.peekTokenKind(TokenKindDot) { - return p.parseIdentOrFunction(pos) + if !p.matchKeyword(KeywordInterval) && p.matchTokenKind(TokenKindKeyword) { + if dot, err := p.peekTokenKind(TokenKindDot); err != nil { + return nil, err + } else if dot { + return p.parseIdentOrFunction(pos) + } } // Parse the keyword as an identifier if it is followed by `,`, `AS`, @@ -591,11 +636,29 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen // is only valid in expression position, so it's applied inline here // rather than in keywordIsSelectItemIdentifier (which is shared with the // terminator/alias check). - if p.keywordIsSelectItemIdentifier() || - (p.matchTokenKind(TokenKindKeyword) && - (p.peekIsEndOfStatement() || p.peekIsExpressionContinuation())) { + identifier, err := p.keywordIsSelectItemIdentifier() + if err != nil { + return nil, err + } + if identifier { return p.parseAnyKeyword() } + if p.matchTokenKind(TokenKindKeyword) { + if end, err := p.peekIsEndOfStatement(); err != nil { + return nil, err + } else if end { + return p.parseAnyKeyword() + } + } + + keywordFunction := false + if p.matchTokenKind(TokenKindKeyword) { + keywordFunction, err = p.peekTokenKind(TokenKindLParen) + if err != nil { + return nil, err + } + } + switch { case p.matchKeyword(KeywordInterval): // ClickHouse also accepts an unquoted `interval` as a column name @@ -617,6 +680,10 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen savedState := p.lexer.saveState() interval, err := p.parseInterval(true) if err != nil { + var lexicalErr *lexerError + if errors.As(err, &lexicalErr) { + return nil, err + } if p.failedIntervalOffsets == nil { p.failedIntervalOffsets = make(map[Pos]struct{}) } @@ -646,7 +713,7 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen return p.parseColumnExtractExpr(pos) case p.matchTokenKind(TokenKindIdent): return p.parseIdentOrFunction(pos) - case p.matchTokenKind(TokenKindKeyword) && p.peekTokenKind(TokenKindLParen): + case keywordFunction: // Reserved operator keywords stay callable as ordinary functions when // followed by '(': and(a, b), or(a, b), in(x, set), like(s, pat), ... // Keywords with dedicated syntax (CAST, CASE, EXTRACT, INTERVAL, ...) @@ -658,7 +725,9 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen p.matchTokenKind(TokenKindFloat): // number literal return p.parseNumber(pos) case p.matchTokenKind(TokenKindLParen): - if peek, _ := p.lexer.peekToken(); peek != nil { + if peek, err := p.lexer.peekToken(); err != nil { + return nil, err + } else if peek != nil { if peek.Kind == TokenKindKeyword && strings.EqualFold(peek.String, KeywordSelect) { return p.parseSubQuery(pos) } @@ -673,7 +742,9 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen // to determine if it is a map literal or a query param. // Treat both identifiers and keywords as identifier-like for placeholders. // parseIdent accepts keywords-as-ident, so this is safe. - if p.peekTokenKind(TokenKindIdent) || p.peekTokenKind(TokenKindKeyword) { + if matched, err := p.peekTokenKind(TokenKindIdent, TokenKindKeyword); err != nil { + return nil, err + } else if matched { return p.parseQueryParam(p.Pos()) } return p.parseMapLiteral(p.Pos()) @@ -681,7 +752,9 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen return p.parseNumber(p.Pos()) case p.matchTokenKind(TokenKindQuestionMark): // Placeholder `?` - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &PlaceHolder{ PlaceholderPos: pos, PlaceHolderEnd: pos, @@ -712,7 +785,9 @@ func (p *Parser) parseColumnCastExpr(pos Pos) (Expr, error) { // CAST(x, T) and CAST(x AS T) are equivalent case p.matchKeyword(KeywordAs), p.matchTokenKind(","): separator = p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } default: return nil, fmt.Errorf("expected AS or , but got %s", p.currentTokenKind()) } @@ -754,11 +829,15 @@ func (p *Parser) parseColumnExprList(pos Pos) (*ColumnExprList, error) { } func (p *Parser) parseColumnExprListWithTerm(term TokenKind, pos Pos) (*ColumnExprList, error) { + var err error columnExprList := &ColumnExprList{ ListPos: pos, ListEnd: pos, } - columnExprList.HasDistinct = p.tryConsumeKeywords(KeywordDistinct) + columnExprList.HasDistinct, err = p.tryConsumeKeywords(KeywordDistinct) + if err != nil { + return nil, err + } columnList := make([]Expr, 0) for !p.lexer.isEOF() || p.current() != nil { if term != "" && p.matchTokenKind(term) { @@ -801,7 +880,9 @@ func (p *Parser) parseSelectItems() ([]*SelectItem, error) { } else if token == nil { break } - if p.isSelectItemTerminatorKeyword() { + if terminator, err := p.isSelectItemTerminatorKeyword(); err != nil { + return nil, err + } else if terminator { break } } @@ -858,7 +939,10 @@ func (p *Parser) parseColumnArgList(pos Pos) (*ColumnArgList, error) { if err := p.expectTokenKind(TokenKindLParen); err != nil { return nil, err } - distinct := p.tryConsumeKeywords(KeywordDistinct) + distinct, consumeErr := p.tryConsumeKeywords(KeywordDistinct) + if consumeErr != nil { + return nil, consumeErr + } var items []Expr for !p.lexer.isEOF() && !p.matchTokenKind(TokenKindRParen) { @@ -971,7 +1055,9 @@ func (p *Parser) parseKeywordArgModifier(form keywordArgForm, slot int) (Expr, b return expr, false, err } // Only a modifier when an expression follows it; a bare `both` is an identifier. - if p.peekTokenKind(TokenKindComma) || p.peekTokenKind(TokenKindRParen) { + if matched, err := p.peekTokenKind(TokenKindComma, TokenKindRParen); err != nil { + return nil, false, err + } else if matched { expr, err := p.parseExpr(p.Pos()) return expr, false, err } @@ -1000,7 +1086,10 @@ func (p *Parser) parseKeywordArgFunctionParams(pos Pos, form keywordArgForm) (*P } itemsPos := p.Pos() - hasDistinct := p.tryConsumeKeywords(KeywordDistinct) + hasDistinct, consumeErr := p.tryConsumeKeywords(KeywordDistinct) + if consumeErr != nil { + return nil, consumeErr + } items := make([]Expr, 0) // A comma advances the argument slot just as a separator keyword does, so the @@ -1019,7 +1108,9 @@ func (p *Parser) parseKeywordArgFunctionParams(pos Pos, form keywordArgForm) (*P slot += consumed var alias *Ident - if p.tryConsumeKeywords(KeywordAs) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { + return nil, consumeErr + } else if matched { if alias, err = p.parseAnyKeyword(); err != nil { return nil, err } @@ -1199,7 +1290,9 @@ func (p *Parser) parseColumnsExpr(pos Pos) (*ColumnExpr, error) { } var alias *Ident - if p.tryConsumeKeywords(KeywordAs) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { + return nil, consumeErr + } else if matched { // after AS the token can only be an alias name, reserved keyword or not alias, err = p.parseAnyKeyword() if err != nil { @@ -1233,14 +1326,22 @@ func (p *Parser) parseSelectItem() (*SelectItem, error) { var alias *Ident switch { - case p.tryConsumeKeywords(KeywordAs): + case p.matchKeyword(KeywordAs): + if err := p.expectKeyword(KeywordAs); err != nil { + return nil, err + } // `SELECT 1 AS ` works for any keyword, reserved or not: // after AS the token can only be an alias name. alias, err = p.parseAnyKeyword() if err != nil { return nil, err } - case p.matchTokenKind(TokenKindIdent) && !p.isSelectItemTerminatorKeyword(): + case p.matchTokenKind(TokenKindIdent): + if terminator, err := p.isSelectItemTerminatorKeyword(); err != nil { + return nil, err + } else if terminator { + break + } // A bare alias can be a normal identifier or non-reserved keyword; a // reserved keyword here starts the next clause (e.g. `SELECT a FROM ...`). alias, err = p.parseIdent() @@ -1278,7 +1379,9 @@ func (p *Parser) parseColumnCaseExpr(pos Pos) (*CaseExpr, error) { whenClauses := make([]*WhenClause, 0) for p.matchKeyword(KeywordWhen) { whenPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } whenCondition, err := p.parseExpr(p.Pos()) if err != nil { return nil, err @@ -1304,7 +1407,9 @@ func (p *Parser) parseColumnCaseExpr(pos Pos) (*CaseExpr, error) { // ELSE expr elsePos := p.Pos() - if p.tryConsumeKeywords(KeywordElse) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordElse); consumeErr != nil { + return nil, consumeErr + } else if matched { elseExpr, err := p.parseExpr(p.Pos()) if err != nil { return nil, err @@ -1348,7 +1453,9 @@ func (p *Parser) parseColumnTypeArgs(ident *Ident) (ColumnType, error) { // noli return p.parseComplexType(ident, p.Pos()) } case p.matchTokenKind(TokenKindString): - if peekToken, err := p.lexer.peekToken(); err == nil && peekToken.Kind == TokenKindSingleEQ { + if peekToken, err := p.lexer.peekToken(); err != nil { + return nil, err + } else if peekToken != nil && peekToken.Kind == TokenKindSingleEQ { // enum values return p.parseEnumType(ident, p.Pos()) } @@ -1359,7 +1466,9 @@ func (p *Parser) parseColumnTypeArgs(ident *Ident) (ColumnType, error) { // noli return p.parseColumnTypeWithParams(ident, p.Pos()) case p.matchTokenKind(TokenKindRParen): rightParenPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &TypeWithParams{ Name: ident, LeftParenPos: lParen.Pos, @@ -1529,8 +1638,13 @@ func (p *Parser) parseJSONMaxDynamicOptions(pos Pos) (*JSONOption, error) { func (p *Parser) parseJSONOption() (*JSONOption, error) { switch { - case p.tryConsumeKeywords(KeywordSkip): - if p.tryConsumeKeywords(KeywordRegexp) { + case p.matchKeyword(KeywordSkip): + if err := p.expectKeyword(KeywordSkip); err != nil { + return nil, err + } + if matched, consumeErr := p.tryConsumeKeywords(KeywordRegexp); consumeErr != nil { + return nil, consumeErr + } else if matched { regex, err := p.parseString(p.Pos()) if err != nil { return nil, err @@ -1774,7 +1888,9 @@ func (p *Parser) parseNestedTypeFieldWithName() (Expr, error) { } func (p *Parser) tryParseCompressionCodecs(pos Pos) (*CompressionCodec, error) { - if !p.tryConsumeKeywords(KeywordCodec) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordCodec); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } diff --git a/parser/parser_common.go b/parser/parser_common.go index 1b72da1..b9bc018 100644 --- a/parser/parser_common.go +++ b/parser/parser_common.go @@ -146,34 +146,37 @@ func (p *Parser) expectKeyword(keyword string) error { Keyword: keyword, } } - _ = p.lexer.consumeToken() - return nil + return p.lexer.consumeToken() } -func (p *Parser) tryConsumeKeywords(keywords ...string) bool { +func (p *Parser) tryConsumeKeywords(keywords ...string) (bool, error) { savedState := p.lexer.saveState() for _, keyword := range keywords { if !p.matchKeyword(keyword) { p.lexer.restoreState(savedState) - return false + return false, nil + } + if err := p.lexer.consumeToken(); err != nil { + return false, err } - _ = p.lexer.consumeToken() } - return true + return true, nil } -func (p *Parser) tryParseIdent() *Ident { +func (p *Parser) tryParseIdent() (*Ident, error) { if p.currentTokenKind() != TokenKindIdent { - return nil + return nil, nil } curToken := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &Ident{ NamePos: curToken.Pos, NameEnd: curToken.End, Name: curToken.String, QuoteType: curToken.QuoteType, - } + }, nil } // parseAnyKeyword parses the current token as an identifier, accepting @@ -190,7 +193,9 @@ func (p *Parser) parseAnyKeyword() (*Ident, error) { Expected: []TokenKind{TokenKindIdent}, } } - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &Ident{ NamePos: last.Pos, NameEnd: last.End, @@ -219,7 +224,9 @@ func (p *Parser) parseIdentOrStar() (*Ident, error) { return p.parseIdent() case p.matchTokenKind("*"): curToken := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &Ident{ NamePos: curToken.Pos, NameEnd: curToken.End, @@ -236,7 +243,9 @@ func (p *Parser) parseIdentOrString() (*Ident, error) { return p.parseIdent() case p.matchTokenKind(TokenKindString): curToken := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } return &Ident{ NamePos: curToken.Pos, NameEnd: curToken.End, @@ -295,14 +304,18 @@ func (p *Parser) tryParseUUID() (*UUID, error) { } func (p *Parser) tryParseComment() (*StringLiteral, error) { - if !p.tryConsumeKeywords(KeywordComment) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordComment); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil } return p.parseString(p.Pos()) } func (p *Parser) tryParseIfExists() (bool, error) { - if !p.tryConsumeKeywords(KeywordIf) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordIf); consumeErr != nil { + return false, consumeErr + } else if !matched { return false, nil } @@ -313,7 +326,9 @@ func (p *Parser) tryParseIfExists() (bool, error) { } func (p *Parser) tryParseIfNotExists() (bool, error) { - if !p.tryConsumeKeywords(KeywordIf) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordIf); consumeErr != nil { + return false, consumeErr + } else if !matched { return false, nil } @@ -327,15 +342,19 @@ func (p *Parser) tryParseIfNotExists() (bool, error) { return true, nil } -func (p *Parser) tryParseNull(pos Pos) *NullLiteral { - if !p.tryConsumeKeywords(KeywordNull) { - return nil +func (p *Parser) tryParseNull(pos Pos) (*NullLiteral, error) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordNull); consumeErr != nil { + return nil, consumeErr + } else if !matched { + return nil, nil } - return &NullLiteral{NullPos: pos} + return &NullLiteral{NullPos: pos}, nil } func (p *Parser) tryParseNotNull(pos Pos) (*NotNullLiteral, error) { - if !p.tryConsumeKeywords(KeywordNot) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordNot); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } notNull := &NotNullLiteral{NotPos: pos} @@ -369,7 +388,9 @@ func (p *Parser) parseNumber(pos Pos) (*NumberLiteral, error) { case p.matchTokenKind(TokenKindFloat): err = p.expectTokenKind(TokenKindFloat) case p.matchTokenKind(TokenKindDot): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } curToken = p.current() if err := p.expectTokenKind(TokenKindInt); err != nil { return nil, err @@ -471,14 +492,13 @@ func (p *Parser) parseFormat(pos Pos) (*FormatClause, error) { // captured position and expected-token information; the long tail of // fmt.Errorf sites is wrapped here with the current position. func (p *Parser) wrapError(err error) error { - // A grammar error can be a consequence of an ignored lexical failure. - // Report the original failure, including when lookahead restored the cursor. - if p.lexer.err != nil { - err = &ParseError{Pos: p.lexer.err.pos, Msg: p.lexer.err.Error()} - } if err == nil { return nil } + var lexicalErr *lexerError + if errors.As(err, &lexicalErr) { + err = &ParseError{Pos: lexicalErr.pos, Msg: lexicalErr.Error()} + } var pe *ParseError if !errors.As(err, &pe) { diff --git a/parser/parser_drop.go b/parser/parser_drop.go index dc06329..a5ca752 100644 --- a/parser/parser_drop.go +++ b/parser/parser_drop.go @@ -44,15 +44,25 @@ func (p *Parser) parseDropDatabase(pos Pos) (*DropDatabase, error) { } func (p *Parser) parseDropStmt(pos Pos) (*DropStmt, error) { + var err error var isTemporary bool dropTarget := KeywordTable switch { - case p.tryConsumeKeywords(KeywordDictionary): + case p.matchKeyword(KeywordDictionary): + if err := p.expectKeyword(KeywordDictionary); err != nil { + return nil, err + } dropTarget = KeywordDictionary - case p.tryConsumeKeywords(KeywordView): + case p.matchKeyword(KeywordView): + if err := p.expectKeyword(KeywordView); err != nil { + return nil, err + } dropTarget = KeywordView default: - isTemporary = p.tryConsumeKeywords(KeywordTemporary) + isTemporary, err = p.tryConsumeKeywords(KeywordTemporary) + if err != nil { + return nil, err + } if err := p.expectKeyword(KeywordTable); err != nil { return nil, err } @@ -92,9 +102,15 @@ func (p *Parser) parseDropStmt(pos Pos) (*DropStmt, error) { func (p *Parser) tryParseModifier() (string, error) { switch { - case p.tryConsumeKeywords(KeywordSync): + case p.matchKeyword(KeywordSync): + if err := p.expectKeyword(KeywordSync); err != nil { + return "", err + } return "SYNC", nil - case p.tryConsumeKeywords(KeywordNo): + case p.matchKeyword(KeywordNo): + if err := p.expectKeyword(KeywordNo); err != nil { + return "", err + } if err := p.expectKeyword(KeywordDelay); err != nil { return "", err } diff --git a/parser/parser_query.go b/parser/parser_query.go index 4df0c8a..105e263 100644 --- a/parser/parser_query.go +++ b/parser/parser_query.go @@ -66,7 +66,9 @@ func (p *Parser) parseTopClause(pos Pos) (*TopClause, error) { topEnd := number.End() withTies := false - if p.tryConsumeKeywords(KeywordWith) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordWith); consumeErr != nil { + return nil, consumeErr + } else if matched { topEnd = p.End() if err := p.expectKeyword(KeywordTies); err != nil { return nil, err @@ -104,7 +106,9 @@ func (p *Parser) parseDistinctOn(pos Pos) (*DistinctOn, error) { idents := []*NestedIdentifier{col} for p.matchTokenKind(TokenKindComma) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } col, err = p.ParseNestedIdentifier(p.Pos()) if err != nil { @@ -148,7 +152,10 @@ func (p *Parser) parseFromClause(pos Pos) (*FromClause, error) { func (p *Parser) tryParseJoinConstraints(pos Pos) (Expr, error) { switch { - case p.tryConsumeKeywords(KeywordOn): + case p.matchKeyword(KeywordOn): + if err := p.expectKeyword(KeywordOn); err != nil { + return nil, err + } columnExprList, err := p.parseColumnExprList(p.Pos()) if err != nil { return nil, err @@ -157,7 +164,10 @@ func (p *Parser) tryParseJoinConstraints(pos Pos) (Expr, error) { OnPos: pos, On: columnExprList, }, nil - case p.tryConsumeKeywords(KeywordUsing): + case p.matchKeyword(KeywordUsing): + if err := p.expectKeyword(KeywordUsing); err != nil { + return nil, err + } lparen, consumeErr := p.tryConsumeTokenKind(TokenKindLParen) if consumeErr != nil { return nil, consumeErr @@ -180,82 +190,121 @@ func (p *Parser) tryParseJoinConstraints(pos Pos) (Expr, error) { return nil, nil } -func (p *Parser) parseJoinType(_ Pos) []string { +func (p *Parser) parseJoinType(_ Pos) ([]string, error) { //nolint:funlen var modifiers []string switch { - case p.tryConsumeKeywords(KeywordCross): // cross join + case p.matchKeyword(KeywordCross): // cross join + if err := p.expectKeyword(KeywordCross); err != nil { + return nil, err + } modifiers = append(modifiers, KeywordCross) case p.matchKeyword(KeywordAny), p.matchKeyword(KeywordAll): modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if p.matchKeyword(KeywordFull) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } if p.matchKeyword(KeywordLeft) || p.matchKeyword(KeywordRight) || p.matchKeyword(KeywordInner) || p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } case p.matchKeyword(KeywordSemi), p.matchKeyword(KeywordAsof): modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if p.matchKeyword(KeywordLeft) || p.matchKeyword(KeywordRight) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } if p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } case p.matchKeyword(KeywordInner): modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAsof) || p.matchKeyword(KeywordArray) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } case p.matchKeyword(KeywordLeft): modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } if p.matchKeyword(KeywordSemi) || p.matchKeyword(KeywordAnti) || p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAsof) || p.matchKeyword(KeywordArray) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } case p.matchKeyword(KeywordRight): modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } if p.matchKeyword(KeywordSemi) || p.matchKeyword(KeywordAnti) || p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAsof) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } case p.matchKeyword(KeywordFull): modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } if p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAny) { modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } case p.matchKeyword(KeywordArray): modifiers = append(modifiers, p.current().String) - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } - return modifiers + return modifiers, nil } func (p *Parser) parseJoinTableExpr(_ Pos) (Expr, error) { @@ -270,7 +319,9 @@ func (p *Parser) parseJoinTableExpr(_ Pos) (Expr, error) { hasFinal := p.matchKeyword(KeywordFinal) if hasFinal { statementEnd = p.End() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } sampleRatio, err := p.tryParseSampleClause(p.Pos()) @@ -297,14 +348,19 @@ func (p *Parser) parseJoinTableExpr(_ Pos) (Expr, error) { // leaving the keyword for the caller to reject. func (p *Parser) parseJoinModifiers(pos Pos) ([]string, error) { if !p.matchOneOfKeywords(KeywordGlobal, KeywordLocal) { - return p.parseJoinType(pos), nil + return p.parseJoinType(pos) } savedState := p.lexer.saveState() locality := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } - joinType := p.parseJoinType(p.Pos()) + joinType, err := p.parseJoinType(p.Pos()) + if err != nil { + return nil, err + } if len(joinType) == 0 && !p.matchKeyword(KeywordJoin) { p.lexer.restoreState(savedState) return nil, nil @@ -329,18 +385,22 @@ func (p *Parser) parseJoinModifiers(pos Pos) ([]string, error) { // peekJoinAfterLocality reports whether the current GLOBAL/LOCAL keyword is // followed by a join operator, leaving the lexer where it found it. Expression // parsing uses it to tell a locality apart from the GLOBAL IN operator. -func (p *Parser) peekJoinAfterLocality() bool { +func (p *Parser) peekJoinAfterLocality() (bool, error) { savedState := p.lexer.saveState() defer p.lexer.restoreState(savedState) modifiers, err := p.parseJoinModifiers(p.Pos()) if err != nil { + var lexicalErr *lexerError + if errors.As(err, &lexicalErr) { + return false, err + } // A malformed locality join such as GLOBAL ARRAY JOIN still belongs // to the FROM clause, which reports the error. - return true + return true, nil } - return modifiers != nil && p.matchKeyword(KeywordJoin) + return modifiers != nil && p.matchKeyword(KeywordJoin), nil } func (p *Parser) parseJoinRightExpr(pos Pos) (expr Expr, err error) { @@ -365,7 +425,9 @@ func (p *Parser) parseJoinRightExpr(pos Pos) (expr Expr, err error) { if len(modifiers) != 0 && !p.matchKeyword(KeywordJoin) { return nil, fmt.Errorf("expected JOIN, got %s", p.currentTokenKind()) } - if !p.tryConsumeKeywords(KeywordJoin) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordJoin); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil } @@ -469,7 +531,9 @@ func (p *Parser) parseTableExpr(pos Pos) (*TableExpr, error) { } tableEnd := expr.End() - if p.tryConsumeKeywords(KeywordAs) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { + return nil, consumeErr + } else if matched { // After AS the token can only be an alias name, so even reserved // keywords are accepted (e.g. `FROM t AS from`). alias, err := p.parseAnyKeyword() @@ -496,7 +560,9 @@ func (p *Parser) parseTableExpr(pos Pos) (*TableExpr, error) { } isFinalExist := false - if p.tryConsumeKeywords(KeywordFinal) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordFinal); consumeErr != nil { + return nil, consumeErr + } else if matched { switch expr.(type) { case *TableFunctionExpr: return nil, errors.New("table function doesn't support FINAL") @@ -578,18 +644,26 @@ func (p *Parser) parseGroupByClause(pos Pos) (*GroupByClause, error) { var err error var groupByEnd Pos aggregateType := "" + groupingSets, consumeErr := p.tryConsumeKeywords(KeywordGrouping, KeywordSets) + if consumeErr != nil { + return nil, consumeErr + } switch { + case groupingSets: + aggregateType = "GROUPING SETS" + expr, err = p.parseFunctionParams(p.Pos()) case p.matchKeyword(KeywordCube) || p.matchKeyword(KeywordRollup): aggregateType = p.current().String - _ = p.lexer.consumeToken() - expr, err = p.parseFunctionParams(p.Pos()) - case p.tryConsumeKeywords(KeywordGrouping, KeywordSets): - aggregateType = "GROUPING SETS" + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } expr, err = p.parseFunctionParams(p.Pos()) case p.matchKeyword(KeywordAll): // GROUP BY ALL has no expression list; the clause ends at ALL itself groupByEnd = p.End() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } aggregateType = "ALL" default: expr, err = p.parseColumnExprListWithLParen(p.Pos()) @@ -608,16 +682,32 @@ func (p *Parser) parseGroupByClause(pos Pos) (*GroupByClause, error) { } // parse WITH CUBE, ROLLUP, TOTALS - for p.tryConsumeKeywords(KeywordWith) { + for { + matched, consumeErr := p.tryConsumeKeywords(KeywordWith) + if consumeErr != nil { + return nil, consumeErr + } + if !matched { + break + } // the clause now extends to the CUBE/ROLLUP/TOTALS token; capture its // end before it is consumed keywordEnd := p.End() switch { - case p.tryConsumeKeywords(KeywordCube): + case p.matchKeyword(KeywordCube): + if err := p.expectKeyword(KeywordCube); err != nil { + return nil, err + } groupBy.WithCube = true - case p.tryConsumeKeywords(KeywordRollup): + case p.matchKeyword(KeywordRollup): + if err := p.expectKeyword(KeywordRollup); err != nil { + return nil, err + } groupBy.WithRollup = true - case p.tryConsumeKeywords(KeywordTotals): + case p.matchKeyword(KeywordTotals): + if err := p.expectKeyword(KeywordTotals); err != nil { + return nil, err + } groupBy.WithTotals = true default: return nil, fmt.Errorf("expected CUBE, ROLLUP or TOTALS, got %s", p.currentTokenKind()) @@ -648,13 +738,17 @@ func (p *Parser) parseLimitClause(pos Pos) (*LimitClause, error) { var limit Expr var offset Expr var err error - if p.tryConsumeKeywords(KeywordLimit) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordLimit); consumeErr != nil { + return nil, consumeErr + } else if matched { limit, err = p.parseExpr(p.Pos()) if err != nil { return nil, err } - if p.tryConsumeKeywords(KeywordOffset) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordOffset); consumeErr != nil { + return nil, consumeErr + } else if matched { offset, err = p.parseExpr(p.Pos()) } else if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { return nil, consumeErr @@ -662,7 +756,9 @@ func (p *Parser) parseLimitClause(pos Pos) (*LimitClause, error) { offset = limit limit, err = p.parseExpr(p.Pos()) } - } else if p.tryConsumeKeywords(KeywordOffset) { + } else if matched, consumeErr := p.tryConsumeKeywords(KeywordOffset); consumeErr != nil { + return nil, consumeErr + } else if matched { offset, err = p.parseExpr(p.Pos()) } @@ -720,7 +816,9 @@ func (p *Parser) parseLimitByClause(pos Pos) (Expr, error) { } var by *ColumnExprList - if !p.tryConsumeKeywords(KeywordBy) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordBy); consumeErr != nil { + return nil, consumeErr + } else if !matched { return limit, nil } if by, err = p.parseColumnExprListWithLParen(p.Pos()); err != nil { @@ -743,13 +841,17 @@ func (p *Parser) parseWindowFrameClause(pos Pos) (*WindowFrameClause, error) { var windowFrameType string if p.matchKeyword(KeywordRows) || p.matchKeyword(KeywordRange) { windowFrameType = p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } else { return nil, fmt.Errorf("expected ROWS or RANGE for window frame") } var expr Expr - if p.tryConsumeKeywords(KeywordBetween) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordBetween); consumeErr != nil { + return nil, consumeErr + } else if matched { left, err := p.parseFrameExtent() if err != nil { return nil, err @@ -803,7 +905,9 @@ func (p *Parser) parseFrameExtent() (Expr, error) { func (p *Parser) parseFrameCurrentRow() (Expr, error) { currentPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordRow); err != nil { return nil, err } @@ -816,7 +920,9 @@ func (p *Parser) parseFrameCurrentRow() (Expr, error) { func (p *Parser) parseFrameUnbounded() (Expr, error) { unboundedPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } direction, err := p.parseFrameDirection() if err != nil { @@ -883,7 +989,9 @@ func (p *Parser) parseFrameDirection() (string, error) { switch { case p.matchKeyword(KeywordPreceding), p.matchKeyword(KeywordFollowing): direction := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return "", err + } return direction, nil default: return "", fmt.Errorf("expected PRECEDING or FOLLOWING, got %s", p.currentTokenKind()) @@ -896,7 +1004,9 @@ func (p *Parser) parseFrameDirectionWithEnd() (string, Pos, error) { } endPos := p.End() direction := p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return "", 0, err + } return direction, endPos, nil } @@ -912,7 +1022,9 @@ func (p *Parser) parseWindowCondition(pos Pos) (*WindowExpr, error) { return nil, err } var windowName *Ident - if p.canParseWindowNameInParens() { + if name, err := p.canParseWindowNameInParens(); err != nil { + return nil, err + } else if name { // canParseWindowNameInParens already disambiguated keyword tokens // (e.g. `OVER (order)` vs `OVER (ORDER BY ...)`). var err error @@ -947,12 +1059,12 @@ func (p *Parser) parseWindowCondition(pos Pos) (*WindowExpr, error) { }, nil } -func (p *Parser) canParseWindowNameInParens() bool { +func (p *Parser) canParseWindowNameInParens() (bool, error) { if !p.matchTokenKind(TokenKindIdent, TokenKindKeyword) { - return false + return false, nil } if !p.matchTokenKind(TokenKindKeyword) { - return true + return true, nil } savedState := p.lexer.saveState() @@ -960,18 +1072,22 @@ func (p *Parser) canParseWindowNameInParens() bool { switch { case p.matchKeyword(KeywordPartition), p.matchKeyword(KeywordOrder): - _ = p.lexer.consumeToken() - return !p.matchKeyword(KeywordBy) + if err := p.lexer.consumeToken(); err != nil { + return false, err + } + return !p.matchKeyword(KeywordBy), nil case p.matchKeyword(KeywordRows), p.matchKeyword(KeywordRange): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return false, err + } return !p.matchKeyword(KeywordBetween) && !p.matchKeyword(KeywordCurrent) && !p.matchKeyword(KeywordUnbounded) && !p.matchTokenKind(TokenKindInt) && !p.matchTokenKind(TokenKindLBrace) && - !p.matchKeyword(KeywordInterval) + !p.matchKeyword(KeywordInterval), nil default: - return true + return true, nil } } @@ -1140,15 +1256,24 @@ func (p *Parser) parseSelectQuery(_ Pos) (*SelectQuery, error) { // parseSelectQuery, so at most one operator is bound per call. func (p *Parser) parseSetOperation(selectStmt *SelectQuery) error { switch { - case p.tryConsumeKeywords(KeywordUnion): + case p.matchKeyword(KeywordUnion): + if err := p.expectKeyword(KeywordUnion); err != nil { + return err + } switch { - case p.tryConsumeKeywords(KeywordAll): + case p.matchKeyword(KeywordAll): + if err := p.expectKeyword(KeywordAll); err != nil { + return err + } unionAllExpr, err := p.parseSelectQuery(p.Pos()) if err != nil { return err } selectStmt.UnionAll = unionAllExpr - case p.tryConsumeKeywords(KeywordDistinct): + case p.matchKeyword(KeywordDistinct): + if err := p.expectKeyword(KeywordDistinct); err != nil { + return err + } unionDistinctExpr, err := p.parseSelectQuery(p.Pos()) if err != nil { return err @@ -1157,13 +1282,19 @@ func (p *Parser) parseSetOperation(selectStmt *SelectQuery) error { default: return fmt.Errorf("expected ALL or DISTINCT, got %s", p.currentTokenKind()) } - case p.tryConsumeKeywords(KeywordExcept): + case p.matchKeyword(KeywordExcept): + if err := p.expectKeyword(KeywordExcept); err != nil { + return err + } exceptExpr, err := p.parseSelectQuery(p.Pos()) if err != nil { return err } selectStmt.Except = exceptExpr - case p.tryConsumeKeywords(KeywordIntersect): + case p.matchKeyword(KeywordIntersect): + if err := p.expectKeyword(KeywordIntersect); err != nil { + return err + } intersectExpr, err := p.parseSelectQuery(p.Pos()) if err != nil { return err @@ -1183,7 +1314,10 @@ func (p *Parser) parseSelectStmt(pos Pos) (*SelectQuery, error) { // nolint: fun return nil, err } // DISTINCT? - hasDistinct := p.tryConsumeKeywords(KeywordDistinct) + hasDistinct, consumeErr := p.tryConsumeKeywords(KeywordDistinct) + if consumeErr != nil { + return nil, consumeErr + } distinctOn, err := p.tryParseDistinctOn(p.Pos()) if err != nil { return nil, err @@ -1232,7 +1366,9 @@ func (p *Parser) parseSelectStmt(pos Pos) (*SelectQuery, error) { // nolint: fun statementEnd = groupBy.End() } withTotal := false - if p.tryConsumeKeywords(KeywordWith) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordWith); consumeErr != nil { + return nil, consumeErr + } else if matched { // the statement now ends at the TOTALS token; capture its end before // expectKeyword consumes it totalsEnd := p.End() @@ -1389,7 +1525,9 @@ func (p *Parser) parseSampleClause(pos Pos) (*SampleClause, error) { var offset *RatioExpr if p.matchKeyword(KeywordOffset) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } offset, err = p.parseRatioExpr(p.Pos()) if err != nil { return nil, err @@ -1415,7 +1553,9 @@ func (p *Parser) parseExplainStmt(pos Pos) (*ExplainStmt, error) { p.matchKeyword(KeywordEstimate), p.matchKeyword(KeywordAst): explainType = p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } default: return nil, fmt.Errorf("expected SYNTAX, PIPELINE, ESTIMATE or AST, got %s", p.currentTokenKind()) } diff --git a/parser/parser_table.go b/parser/parser_table.go index 6d962c5..0530826 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -11,8 +11,13 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { case p.matchKeyword(KeywordCreate), p.matchKeyword(KeywordAttach): isAttach := p.matchKeyword(KeywordAttach) - _ = p.lexer.consumeToken() - orReplace := p.tryConsumeKeywords(KeywordOr, KeywordReplace) + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } + orReplace, consumeErr := p.tryConsumeKeywords(KeywordOr, KeywordReplace) + if consumeErr != nil { + return nil, consumeErr + } if orReplace { // MATERIALIZED VIEW accepts OR REPLACE only under CREATE; // ClickHouse rejects an ATTACH OR REPLACE combination. @@ -51,7 +56,9 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { p.currentTokenKind()) } case p.matchKeyword(KeywordAlter): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } switch { case p.matchKeyword(KeywordRole): return p.parseAlterRole(pos) @@ -62,7 +69,9 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { } case p.matchKeyword(KeywordDrop), p.matchKeyword(KeywordDetach): - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } switch { case p.matchKeyword(KeywordDatabase): return p.parseDropDatabase(pos) @@ -306,12 +315,16 @@ func (p *Parser) parseNamedCollectionParam(pos Pos) (*NamedCollectionParam, erro } // Parse optional [NOT] OVERRIDABLE clause - if p.tryConsumeKeywords(KeywordNot) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordNot); consumeErr != nil { + return nil, consumeErr + } else if matched { param.NotOverridable = true if err := p.expectKeyword(KeywordOverridable); err != nil { return nil, err } - } else if p.tryConsumeKeywords(KeywordOverridable) { + } else if matched, consumeErr := p.tryConsumeKeywords(KeywordOverridable); consumeErr != nil { + return nil, consumeErr + } else if matched { param.Overridable = true } @@ -319,15 +332,18 @@ func (p *Parser) parseNamedCollectionParam(pos Pos) (*NamedCollectionParam, erro } func (p *Parser) parseCreateTable(pos Pos, orReplace bool) (*CreateTable, error) { + var err error createTable := &CreateTable{CreatePos: pos, OrReplace: orReplace} - createTable.HasTemporary = p.tryConsumeKeywords(KeywordTemporary) + createTable.HasTemporary, err = p.tryConsumeKeywords(KeywordTemporary) + if err != nil { + return nil, err + } if err := p.expectKeyword(KeywordTable); err != nil { return nil, err } // parse IF NOT EXISTS clause if exists - var err error createTable.IfNotExists, err = p.tryParseIfNotExists() if err != nil { return nil, err @@ -367,7 +383,9 @@ func (p *Parser) parseCreateTable(pos Pos, orReplace bool) (*CreateTable, error) createTable.StatementEnd = engineExpr.End() } - if p.tryConsumeKeywords(KeywordAs) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { + return nil, consumeErr + } else if matched { // After AS, we can have: SELECT/WITH (with or without parens), or table_function(...) // Check if it's a SELECT/WITH query (explicitly check keywords/paren before ident) if p.matchKeyword(KeywordSelect) || p.matchKeyword(KeywordWith) || p.matchTokenKind(TokenKindLParen) { @@ -414,8 +432,14 @@ func (p *Parser) parseCreateTable(pos Pos, orReplace bool) (*CreateTable, error) func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { var ident *Ident var err error - if p.matchTokenKind(TokenKindKeyword) && - (p.peekTokenKind(TokenKindLParen) || p.peekTokenKind(TokenKindDot)) { + keywordName := false + if p.matchTokenKind(TokenKindKeyword) { + keywordName, err = p.peekTokenKind(TokenKindLParen, TokenKindDot) + if err != nil { + return nil, err + } + } + if keywordName { // Reserved keywords remain valid when context proves they are function // names or the first field of a qualified name. ident, err = p.parseAnyKeyword() @@ -451,7 +475,9 @@ func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { } overPos := p.Pos() - if p.tryConsumeKeywords(KeywordOver) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordOver); consumeErr != nil { + return nil, consumeErr + } else if matched { var overExpr Expr switch { case p.matchTokenKind(TokenKindIdent, TokenKindKeyword): @@ -478,8 +504,15 @@ func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { }, nil } return funcExpr, nil - case p.matchTokenKind(TokenKindDot) && !p.peekTokenKind(TokenKindColon): - _ = p.lexer.consumeToken() + case p.matchTokenKind(TokenKindDot): + if qualifier, err := p.peekTokenKind(TokenKindColon); err != nil { + return nil, err + } else if qualifier { + return ident, nil + } + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } switch { case p.matchTokenKind(TokenKindIdent, TokenKindKeyword): fields := []*Ident{ident} @@ -491,10 +524,17 @@ func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { return nil, err } fields = append(fields, child) - if !p.matchTokenKind(TokenKindDot) || p.peekTokenKind(TokenKindColon) { + if !p.matchTokenKind(TokenKindDot) { break } - _ = p.lexer.consumeToken() + if qualifier, err := p.peekTokenKind(TokenKindColon); err != nil { + return nil, err + } else if qualifier { + break + } + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } return &Path{Fields: fields}, nil case p.matchTokenKind("*"): @@ -565,11 +605,21 @@ func (p *Parser) parseTableSchemaClause(pos Pos) (*TableSchemaClause, error) { SchemaEnd: rightParenPos, Columns: columns, }, nil - case p.matchKeyword(KeywordAs) && !p.peekKeyword(KeywordSelect) && !p.peekKeyword(KeywordWith) && !p.peekTokenKind(TokenKindLParen): + case p.matchKeyword(KeywordAs): + next, err := p.lexer.peekToken() + if err != nil { + return nil, err + } + if next != nil && (next.Kind == TokenKindLParen || + next.Kind == TokenKindKeyword && (strings.EqualFold(next.String, KeywordSelect) || strings.EqualFold(next.String, KeywordWith))) { + return nil, nil + } // Handle AS only if followed by identifier (not SELECT/WITH/LPAREN) // This handles: AS ident, AS ident.ident, AS ident(...) // CREATE TABLE will handle: AS SELECT, AS WITH, AS (SELECT ...) - p.tryConsumeKeywords(KeywordAs) + if _, err := p.tryConsumeKeywords(KeywordAs); err != nil { + return nil, err + } ident, err := p.parseIdent() if err != nil { @@ -624,7 +674,9 @@ func (p *Parser) parseTableColumns() ([]Expr, error) { switch { case p.matchKeyword(KeywordIndex): indexPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } index, err := p.parseTableIndex(indexPos) if err != nil { return nil, err @@ -632,7 +684,9 @@ func (p *Parser) parseTableColumns() ([]Expr, error) { columns = append(columns, index) case p.matchKeyword(KeywordConstraint): constraintPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } ident, err := p.parseIdent() if err != nil { return nil, err @@ -702,7 +756,10 @@ func (p *Parser) parseTableColumnExpr(pos Pos) (*ColumnDef, error) { columnEnd = columnType.End() } - nullable := p.tryParseNull(p.Pos()) + nullable, err := p.tryParseNull(p.Pos()) + if err != nil { + return nil, err + } if nullable != nil { columnEnd = nullable.End() } @@ -716,13 +773,22 @@ func (p *Parser) parseTableColumnExpr(pos Pos) (*ColumnDef, error) { var valueExpr Expr switch { - case p.tryConsumeKeywords(KeywordDefault): + case p.matchKeyword(KeywordDefault): + if err := p.expectKeyword(KeywordDefault); err != nil { + return nil, err + } column.DefaultExpr, err = p.parseExpr(p.Pos()) valueExpr = column.DefaultExpr - case p.tryConsumeKeywords(KeywordMaterialized): + case p.matchKeyword(KeywordMaterialized): + if err := p.expectKeyword(KeywordMaterialized); err != nil { + return nil, err + } column.MaterializedExpr, err = p.parseExpr(p.Pos()) valueExpr = column.MaterializedExpr - case p.tryConsumeKeywords(KeywordAlias): + case p.matchKeyword(KeywordAlias): + if err := p.expectKeyword(KeywordAlias); err != nil { + return nil, err + } column.AliasExpr, err = p.parseExpr(p.Pos()) valueExpr = column.AliasExpr } @@ -814,7 +880,9 @@ func (p *Parser) parseTableArgPrimaryExpr(pos Pos) (Expr, error) { // a leading '(' opens a subquery only when SELECT or WITH follows, // e.g. remote('127.0.0.1', (SELECT 1)); anything else is a // parenthesized expression, e.g. numbers((1 + 1)) - if p.peekKeyword(KeywordSelect) || p.peekKeyword(KeywordWith) { + if query, err := p.peekKeyword(KeywordSelect, KeywordWith); err != nil { + return nil, err + } else if query { return p.parseSubQuery(p.Pos()) } @@ -845,7 +913,10 @@ func (p *Parser) parseTableArgList(pos Pos) (*TableArgListExpr, error) { // Last token is an identifier, peek at the next token nextToken, peekErr := p.lexer.peekToken() - if peekErr == nil && nextToken != nil && nextToken.Kind == TokenKindSingleEQ { + if peekErr != nil { + return nil, peekErr + } + if nextToken != nil && nextToken.Kind == TokenKindSingleEQ { isNamedParam = true } } @@ -904,7 +975,9 @@ func (p *Parser) parseTableArgList(pos Pos) (*TableArgListExpr, error) { } func (p *Parser) tryParseClusterClause(pos Pos) (*ClusterClause, error) { - if !p.tryConsumeKeywords(KeywordOn) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordOn); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } if err := p.expectKeyword(KeywordCluster); err != nil { @@ -931,7 +1004,9 @@ func (p *Parser) tryParseClusterClause(pos Pos) (*ClusterClause, error) { } func (p *Parser) tryParsePartitionByClause(pos Pos) (*PartitionByClause, error) { - if !p.tryConsumeKeywords(KeywordPartition) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordPartition); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } @@ -951,7 +1026,9 @@ func (p *Parser) tryParsePartitionByClause(pos Pos) (*PartitionByClause, error) } func (p *Parser) tryParsePrimaryKeyClause(pos Pos) (*PrimaryKeyClause, error) { - if !p.tryConsumeKeywords(KeywordPrimary) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordPrimary); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } @@ -971,7 +1048,9 @@ func (p *Parser) tryParsePrimaryKeyClause(pos Pos) (*PrimaryKeyClause, error) { } func (p *Parser) tryParseOrderByClause(pos Pos) (*OrderByClause, error) { - if !p.tryConsumeKeywords(KeywordOrder) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordOrder); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } @@ -985,7 +1064,9 @@ func (p *Parser) tryParseOrderByClause(pos Pos) (*OrderByClause, error) { // the sorting key. Unlike the ORDER BY of a query it holds a single expression: // a sorting key over several columns is written as a tuple. func (p *Parser) tryParseTableOrderByClause(pos Pos) (*OrderByClause, error) { - if !p.tryConsumeKeywords(KeywordOrder) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordOrder); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } @@ -1034,7 +1115,9 @@ func (p *Parser) parseOrderByClause(pos Pos) (*OrderByClause, error) { // Parse optional INTERPOLATE clause if p.matchKeyword(KeywordInterpolate) { interpolatePos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } interpolate, err := p.parseInterpolateClause(interpolatePos) if err != nil { return nil, err @@ -1056,14 +1139,18 @@ func (p *Parser) parseOrderExpr(pos Pos) (*OrderExpr, error) { var alias *Ident if p.matchKeyword(KeywordAs) { // It should be a subquery instead of an order by alias if the `AS` is followed by `SELECT` keyword. - if nextToken, err := p.lexer.peekToken(); err == nil && nextToken.ToString() == KeywordSelect { + if nextToken, err := p.lexer.peekToken(); err != nil { + return nil, err + } else if nextToken != nil && nextToken.ToString() == KeywordSelect { return &OrderExpr{ OrderPos: pos, Expr: columnExpr, }, nil } // consume the `AS` keyword - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } // after AS the token can only be an alias name, reserved keyword or not alias, err = p.parseAnyKeyword() if err != nil { @@ -1080,15 +1167,21 @@ func (p *Parser) parseOrderExpr(pos Pos) (*OrderExpr, error) { switch { case p.matchKeyword(KeywordAsc), p.matchKeyword(KeywordAscending): direction = OrderDirectionAsc - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } case p.matchKeyword(KeywordDesc), p.matchKeyword(KeywordDescending): direction = OrderDirectionDesc - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } // Parse optional WITH FILL clause var fill *Fill - if p.tryConsumeKeywords(KeywordWith, KeywordFill) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordWith, KeywordFill); consumeErr != nil { + return nil, consumeErr + } else if matched { fillPos := p.Pos() fill, err = p.parseFillClause(fillPos) if err != nil { @@ -1109,7 +1202,9 @@ func (p *Parser) parseFillClause(fillPos Pos) (*Fill, error) { fill := &Fill{FillPos: fillPos} // Parse optional FROM clause - if p.tryConsumeKeywords(KeywordFrom) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordFrom); consumeErr != nil { + return nil, consumeErr + } else if matched { fromExpr, err := p.parseExpr(fillPos) if err != nil { return nil, err @@ -1118,7 +1213,9 @@ func (p *Parser) parseFillClause(fillPos Pos) (*Fill, error) { } // Parse optional TO clause - if p.tryConsumeKeywords(KeywordTo) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordTo); consumeErr != nil { + return nil, consumeErr + } else if matched { toExpr, err := p.parseExpr(fillPos) if err != nil { return nil, err @@ -1127,7 +1224,9 @@ func (p *Parser) parseFillClause(fillPos Pos) (*Fill, error) { } // Parse optional STEP clause - if p.tryConsumeKeywords(KeywordStep) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordStep); consumeErr != nil { + return nil, consumeErr + } else if matched { stepExpr, err := p.parseExpr(fillPos) if err != nil { return nil, err @@ -1136,7 +1235,9 @@ func (p *Parser) parseFillClause(fillPos Pos) (*Fill, error) { } // Parse optional STALENESS clause - if p.tryConsumeKeywords(KeywordStaleness) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordStaleness); consumeErr != nil { + return nil, consumeErr + } else if matched { stalenessExpr, err := p.parseExpr(fillPos) if err != nil { return nil, err @@ -1169,7 +1270,9 @@ func (p *Parser) parseInterpolateClause(interpolatePos Pos) (*InterpolateClause, item := &InterpolateItem{Column: column} - if p.tryConsumeKeywords(KeywordAs) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { + return nil, consumeErr + } else if matched { expr, err := p.parseExpr(interpolatePos) if err != nil { return nil, err @@ -1201,7 +1304,9 @@ func (p *Parser) parseInterpolateClause(interpolatePos Pos) (*InterpolateClause, } func (p *Parser) tryParseTTLClause(pos Pos, allowMultiValues bool) (*TTLClause, error) { - if !p.tryConsumeKeywords(KeywordTtl) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordTtl); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } ttlExprList := &TTLClause{TTLPos: pos, ListEnd: pos} @@ -1247,14 +1352,21 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { var rule *TTLPolicyRule var where *WhereClause switch { - case p.tryConsumeKeywords(KeywordTo): - if p.tryConsumeKeywords(KeywordDisk) { + case p.matchKeyword(KeywordTo): + if err := p.expectKeyword(KeywordTo); err != nil { + return nil, err + } + if matched, consumeErr := p.tryConsumeKeywords(KeywordDisk); consumeErr != nil { + return nil, consumeErr + } else if matched { value, err := p.parseString(p.Pos()) if err != nil { return nil, err } rule = &TTLPolicyRule{RulePos: pos, ToDisk: value} - } else if p.tryConsumeKeywords(KeywordVolume) { + } else if matched, consumeErr := p.tryConsumeKeywords(KeywordVolume); consumeErr != nil { + return nil, consumeErr + } else if matched { value, err := p.parseString(p.Pos()) if err != nil { return nil, err @@ -1266,7 +1378,9 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { case p.matchKeyword(KeywordDelete), p.matchKeyword(KeywordRecompress): isDelete := p.matchKeyword(KeywordDelete) token := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } action := &TTLPolicyRuleAction{ ActionPos: token.Pos, ActionEnd: token.End, @@ -1328,6 +1442,10 @@ func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { savedState := p.lexer.saveState() key, err = p.parseExpr(p.Pos()) if err != nil { + var lexicalErr *lexerError + if errors.As(err, &lexicalErr) { + return nil, err + } p.lexer.restoreState(savedState) key, err = p.parseAnyKeyword() } @@ -1353,7 +1471,9 @@ func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { Expr: keys, }, } - if p.tryConsumeKeywords(KeywordSet) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordSet); consumeErr != nil { + return nil, consumeErr + } else if matched { set, err := p.parseTTLPolicySet(p.Pos()) if err != nil { return nil, err @@ -1373,6 +1493,10 @@ func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { } set, err := p.parseTTLPolicySet(p.Pos()) if err != nil { + var lexicalErr *lexerError + if errors.As(err, &lexicalErr) { + return nil, err + } p.lexer.restoreState(savedState) break } @@ -1423,7 +1547,9 @@ func (p *Parser) parseTTLExpr(pos Pos) (*TTLExpr, error) { } func (p *Parser) tryParseSampleByClause(pos Pos) (*SampleByClause, error) { - if !p.tryConsumeKeywords(KeywordSample) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordSample); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } @@ -1443,7 +1569,9 @@ func (p *Parser) tryParseSampleByClause(pos Pos) (*SampleByClause, error) { } func (p *Parser) tryParseSettingsClause(pos Pos) (*SettingsClause, error) { - if !p.tryConsumeKeywords(KeywordSettings) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordSettings); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } return p.parseSettingsClause(pos) @@ -1521,7 +1649,9 @@ func (p *Parser) parseSettingsExpr(pos Pos) (*SettingExpr, error) { case p.matchKeyword(KeywordTrue), p.matchKeyword(KeywordFalse): // Handle TRUE/FALSE keywords as boolean literals curToken := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } expr = &BoolLiteral{ LiteralPos: curToken.Pos, LiteralEnd: curToken.End, @@ -1757,7 +1887,9 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { case p.matchKeyword(KeywordCreate): // SHOW CREATE TABLE table_name showType = "CREATE" - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordTable); err != nil { return nil, err @@ -1773,12 +1905,16 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { case p.matchKeyword(KeywordDatabases): // SHOW DATABASES [optional clauses] showType = "DATABASES" - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } case p.matchKeyword(KeywordTables): // SHOW TABLES showType = "TABLES" - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } default: return nil, fmt.Errorf("expected CREATE, DATABASES, or TABLES after SHOW, got %q", p.currentTokenString()) @@ -1795,7 +1931,9 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { // Parse [[NOT] LIKE | ILIKE ''] if p.matchKeyword(KeywordNot) { stmt.NotLike = true - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } if p.matchKeyword(KeywordLike) || p.matchKeyword(KeywordIlike) { @@ -1804,7 +1942,9 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { } else { stmt.LikeType = "ILIKE" } - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } // Parse pattern expression pattern, err := p.parseExpr(p.Pos()) @@ -1816,7 +1956,9 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { // Parse [LIMIT ] if p.matchKeyword(KeywordLimit) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } limit, err := p.parseExpr(p.Pos()) if err != nil { return nil, err @@ -1826,7 +1968,9 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { // Parse [INTO OUTFILE filename] if p.matchKeyword(KeywordInto) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } if err := p.expectKeyword(KeywordOutfile); err != nil { return nil, err } @@ -1841,7 +1985,9 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { // Parse [FORMAT format] if p.matchKeyword(KeywordFormat) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } // Format can be an identifier or a string if p.matchTokenKind(TokenKindString) { @@ -1853,7 +1999,9 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { } else if p.matchTokenKind(TokenKindIdent) { // Handle format as identifier (like JSON, CSV, etc.) token := p.current() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } stmt.Format = &StringLiteral{ LiteralPos: token.Pos, LiteralEnd: token.End, @@ -1876,12 +2024,16 @@ func (p *Parser) parseDescribeStmt(pos Pos) (*DescribeStmt, error) { if !p.matchKeyword(KeywordDesc) && !p.matchKeyword(KeywordDescribe) { return nil, fmt.Errorf("expected DESC or DESCRIBE") } - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } // TABLE keyword is optional after DESC/DESCRIBE var describeType string if p.matchKeyword(KeywordTable) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } describeType = "TABLE" } @@ -1904,7 +2056,10 @@ func (p *Parser) parseTruncateTable(pos Pos) (*TruncateTable, error) { return nil, err } - isTemporary := p.tryConsumeKeywords(KeywordTemporary) + isTemporary, consumeErr := p.tryConsumeKeywords(KeywordTemporary) + if consumeErr != nil { + return nil, consumeErr + } if err := p.expectKeyword(KeywordTable); err != nil { return nil, err @@ -2086,6 +2241,7 @@ func (p *Parser) parseAssignmentValues(pos Pos) (*AssignmentValues, error) { } func (p *Parser) parseInsertStmt(pos Pos) (*InsertStmt, error) { + var err error if err := p.expectKeyword(KeywordInsert); err != nil { return nil, err } @@ -2094,11 +2250,15 @@ func (p *Parser) parseInsertStmt(pos Pos) (*InsertStmt, error) { } insertExpr := &InsertStmt{InsertPos: pos} - insertExpr.HasTableKeyword = p.tryConsumeKeywords(KeywordTable) + insertExpr.HasTableKeyword, err = p.tryConsumeKeywords(KeywordTable) + if err != nil { + return nil, err + } var table Expr - var err error - if p.tryConsumeKeywords(KeywordFunction) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordFunction); consumeErr != nil { + return nil, consumeErr + } else if matched { table, err = p.parseFunctionExpr(p.Pos()) } else { table, err = p.parseTableIdentifier(p.Pos()) @@ -2121,7 +2281,9 @@ func (p *Parser) parseInsertStmt(pos Pos) (*InsertStmt, error) { insertExpr.Format, err = p.parseFormat(p.Pos()) case p.matchKeyword(KeywordValues): // consume VALUES keyword - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } values := make([]*AssignmentValues, 0) for !p.lexer.isEOF() { value, err := p.parseAssignmentValues(p.Pos()) @@ -2155,9 +2317,15 @@ func (p *Parser) parseRenameStmt(pos Pos) (*RenameStmt, error) { renameTarget := KeywordTable switch { - case p.tryConsumeKeywords(KeywordDictionary): + case p.matchKeyword(KeywordDictionary): + if err := p.expectKeyword(KeywordDictionary); err != nil { + return nil, err + } renameTarget = KeywordDictionary - case p.tryConsumeKeywords(KeywordDatabase): + case p.matchKeyword(KeywordDatabase): + if err := p.expectKeyword(KeywordDatabase); err != nil { + return nil, err + } renameTarget = KeywordDatabase default: if err := p.expectKeyword(KeywordTable); err != nil { @@ -2332,7 +2500,10 @@ func (p *Parser) parseDictionaryAttribute(pos Pos) (*DictionaryAttribute, error) // properties (HIERARCHICAL, ...) end at the keyword itself keywordEnd := p.End() switch { - case p.tryConsumeKeywords(KeywordDefault): + case p.matchKeyword(KeywordDefault): + if err := p.expectKeyword(KeywordDefault); err != nil { + return nil, err + } if attr.Default != nil { return nil, fmt.Errorf("duplicate DEFAULT clause") } @@ -2342,7 +2513,10 @@ func (p *Parser) parseDictionaryAttribute(pos Pos) (*DictionaryAttribute, error) } attr.Default = literal attr.AttrEnd = literal.End() - case p.tryConsumeKeywords(KeywordExpression): + case p.matchKeyword(KeywordExpression): + if err := p.expectKeyword(KeywordExpression); err != nil { + return nil, err + } if attr.Expression != nil { return nil, fmt.Errorf("duplicate EXPRESSION clause") } @@ -2352,19 +2526,28 @@ func (p *Parser) parseDictionaryAttribute(pos Pos) (*DictionaryAttribute, error) } attr.Expression = expr attr.AttrEnd = expr.End() - case p.tryConsumeKeywords(KeywordHierarchical): + case p.matchKeyword(KeywordHierarchical): + if err := p.expectKeyword(KeywordHierarchical); err != nil { + return nil, err + } if attr.Hierarchical { return nil, fmt.Errorf("duplicate HIERARCHICAL clause") } attr.Hierarchical = true attr.AttrEnd = keywordEnd - case p.tryConsumeKeywords(KeywordInjective): + case p.matchKeyword(KeywordInjective): + if err := p.expectKeyword(KeywordInjective); err != nil { + return nil, err + } if attr.Injective { return nil, fmt.Errorf("duplicate INJECTIVE clause") } attr.Injective = true attr.AttrEnd = keywordEnd - case p.tryConsumeKeywords(KeywordIs_object_id): + case p.matchKeyword(KeywordIs_object_id): + if err := p.expectKeyword(KeywordIs_object_id); err != nil { + return nil, err + } if attr.IsObjectId { return nil, fmt.Errorf("duplicate IS_OBJECT_ID clause") } @@ -2536,7 +2719,9 @@ func (p *Parser) parseDictionaryArgExpr(pos Pos) (*DictionaryArgExpr, error) { // HEADERS(HEADER(NAME 'API-KEY' VALUE 'key')). if p.matchTokenKind(TokenKindLParen) { lParenPos := p.Pos() - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } args, err := p.parseDictionaryArgExprs() if err != nil { @@ -2579,7 +2764,9 @@ func (p *Parser) parseDictionaryArgExpr(pos Pos) (*DictionaryArgExpr, error) { } // Check if it's followed by optional parentheses if p.matchTokenKind(TokenKindLParen) { - _ = p.lexer.consumeToken() // consume ( + if err := p.lexer.consumeToken(); err != nil { // consume ( + return nil, err + } if err := p.expectTokenKind(TokenKindRParen); err != nil { return nil, err } @@ -2616,7 +2803,9 @@ func (p *Parser) parseDictionaryLifetimeClause(pos Pos) (*DictionaryLifetimeClau // Check for MIN/MAX form if p.matchKeyword(KeywordMin) || p.matchKeyword(KeywordMax) { isMinFirst := p.matchKeyword(KeywordMin) - _ = p.lexer.consumeToken() // consume MIN or MAX + if err := p.lexer.consumeToken(); err != nil { // consume MIN or MAX + return nil, err + } first, err := p.parseNumber(p.Pos()) if err != nil { diff --git a/parser/parser_view.go b/parser/parser_view.go index effbb9c..8677b51 100644 --- a/parser/parser_view.go +++ b/parser/parser_view.go @@ -53,14 +53,18 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } createMaterializedView.Refresh = refreshExpr - if p.tryConsumeKeywords(KeywordRandomize, KeywordFor) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordRandomize, KeywordFor); consumeErr != nil { + return nil, consumeErr + } else if matched { randomizeFor, err := p.parseInterval(false) if err != nil { return nil, err } createMaterializedView.RandomizeFor = randomizeFor } - if p.tryConsumeKeywords(KeywordDepends, KeywordOn) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordDepends, KeywordOn); consumeErr != nil { + return nil, consumeErr + } else if matched { dependsOnTables := make([]*TableIdentifier, 0) table, err := p.parseTableIdentifier(p.Pos()) if err != nil { @@ -68,7 +72,9 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } dependsOnTables = append(dependsOnTables, table) for p.matchTokenKind(TokenKindComma) { - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } table, err := p.parseTableIdentifier(p.Pos()) if err != nil { return nil, err @@ -82,7 +88,10 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa return nil, err } createMaterializedView.Settings = settings - createMaterializedView.HasAppend = p.tryConsumeKeywords(KeywordAppend) + createMaterializedView.HasAppend, err = p.tryConsumeKeywords(KeywordAppend) + if err != nil { + return nil, err + } switch { case p.matchKeyword(KeywordTo): @@ -125,10 +134,15 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa default: return nil, fmt.Errorf("unexpected token: %q, expected TO or ENGINE", p.currentTokenKind()) } - createMaterializedView.HasEmpty = p.tryConsumeKeywords(KeywordEmpty) + createMaterializedView.HasEmpty, err = p.tryConsumeKeywords(KeywordEmpty) + if err != nil { + return nil, err + } // Parse DEFINER clause - if p.tryConsumeKeywords(KeywordDefiner) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordDefiner); consumeErr != nil { + return nil, consumeErr + } else if matched { if err := p.expectTokenKind(TokenKindSingleEQ); err != nil { return nil, err } @@ -140,16 +154,22 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } // Parse SQL SECURITY clause - if p.tryConsumeKeywords(KeywordSQL, KeywordSecurity) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordSQL, KeywordSecurity); consumeErr != nil { + return nil, consumeErr + } else if matched { if !p.matchOneOfKeywords(KeywordDefiner, KeywordNone) { return nil, fmt.Errorf("expected DEFINER or NONE after SQL SECURITY, got %q", p.currentTokenKind()) } createMaterializedView.SQLSecurity = p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } } // Check for POPULATE before AS SELECT - only valid with ENGINE and no Destination - if p.tryConsumeKeywords(KeywordPopulate) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordPopulate); consumeErr != nil { + return nil, consumeErr + } else if matched { if createMaterializedView.Destination != nil { return nil, fmt.Errorf("POPULATE is only allowed when using ENGINE, not with TO clause") } @@ -169,7 +189,9 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } createMaterializedView.Comment = comment - if p.tryConsumeKeywords(KeywordAs) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { + return nil, consumeErr + } else if matched { subQuery, err := p.parseSubQuery(p.Pos()) if err != nil { return nil, err @@ -190,7 +212,9 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } func (p *Parser) tryParseRefreshExpr(pos Pos) (*RefreshExpr, error) { - if !p.tryConsumeKeywords(KeywordRefresh) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordRefresh); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } @@ -200,7 +224,9 @@ func (p *Parser) tryParseRefreshExpr(pos Pos) (*RefreshExpr, error) { return nil, fmt.Errorf("expected EVERY or AFTER, but got %q", p.currentTokenKind()) } refreshExpr.Frequency = p.current().String - _ = p.lexer.consumeToken() + if err := p.lexer.consumeToken(); err != nil { + return nil, err + } interval, err := p.parseInterval(false) if err != nil { @@ -209,7 +235,9 @@ func (p *Parser) tryParseRefreshExpr(pos Pos) (*RefreshExpr, error) { refreshExpr.Interval = interval // [OFFSET interval] - if p.tryConsumeKeywords(KeywordOffset) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordOffset); consumeErr != nil { + return nil, consumeErr + } else if matched { offset, err := p.parseInterval(false) if err != nil { return nil, err @@ -266,7 +294,9 @@ func (p *Parser) parseCreateView(pos Pos, orReplace bool) (*CreateView, error) { } createView.Comment = comment - if p.tryConsumeKeywords(KeywordAs) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { + return nil, consumeErr + } else if matched { subQuery, err := p.parseSubQuery(p.Pos()) if err != nil { return nil, err @@ -339,7 +369,9 @@ func (p *Parser) parseCreateLiveView(pos Pos) (*CreateLiveView, error) { createLiveView.TableSchema = tableSchema } - if p.tryConsumeKeywords(KeywordAs) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { + return nil, consumeErr + } else if matched { subQuery, err := p.parseSubQuery(p.Pos()) if err != nil { return nil, err @@ -352,7 +384,9 @@ func (p *Parser) parseCreateLiveView(pos Pos) (*CreateLiveView, error) { } func (p *Parser) tryParseWithTimeout(pos Pos) (*WithTimeoutClause, error) { - if !p.tryConsumeKeywords(KeywordWith) { + if matched, consumeErr := p.tryConsumeKeywords(KeywordWith); consumeErr != nil { + return nil, consumeErr + } else if !matched { return nil, nil // nolint } if err := p.expectKeyword(KeywordTimeout); err != nil { From bf4d56253384ec2c61fce8a7977523c3af09255e Mon Sep 17 00:00:00 2001 From: git-hulk Date: Thu, 10 Sep 2026 13:01:14 +0800 Subject: [PATCH 6/6] Revert "Return lexer errors without retaining them" This reverts commit 03b5f95d38ce736c165d204d7a26d91a9b800ecd. --- parser/error_test.go | 56 ------- parser/lexer.go | 14 +- parser/lexer_test.go | 6 +- parser/parse_system.go | 354 +++++++++------------------------------- parser/parser_alter.go | 68 ++------ parser/parser_column.go | 346 +++++++++++++-------------------------- parser/parser_common.go | 74 +++------ parser/parser_drop.go | 26 +-- parser/parser_query.go | 284 ++++++++------------------------ parser/parser_table.go | 343 +++++++++----------------------------- parser/parser_view.go | 66 ++------ 11 files changed, 423 insertions(+), 1214 deletions(-) diff --git a/parser/error_test.go b/parser/error_test.go index 45f43a6..b926f7d 100644 --- a/parser/error_test.go +++ b/parser/error_test.go @@ -94,35 +94,6 @@ func TestParser_TokenConsumptionError(t *testing.T) { _, err := p.parseUserNames() return err }}, - {"required keyword", "SELECT /*", func(p *Parser) error { return p.expectKeyword(KeywordSelect) }}, - {"optional keywords", "WITH FILL /*", func(p *Parser) error { - _, err := p.tryConsumeKeywords(KeywordWith, KeywordFill) - return err - }}, - {"identifier", "a /*", func(p *Parser) error { - _, err := p.tryParseIdent() - return err - }}, - {"keyword lookahead", "case /*", func(p *Parser) error { - _, err := p.keywordIsSelectItemIdentifier() - return err - }}, - {"join lookahead", "GLOBAL LEFT /*", func(p *Parser) error { - _, err := p.peekJoinAfterLocality() - return err - }}, - {"interval backtracking", "interval 1 + /*", func(p *Parser) error { - _, err := p.parseColumnExpr(p.Pos()) - return err - }}, - {"TTL backtracking", "GROUP BY ALL + /*", func(p *Parser) error { - _, err := p.parseTTLPolicyGroupBy(p.Pos()) - return err - }}, - {"TTL assignment backtracking", "GROUP BY a SET x = 1, y = /*", func(p *Parser) error { - _, err := p.parseTTLPolicyGroupBy(p.Pos()) - return err - }}, } { t.Run(tc.name, func(t *testing.T) { p := NewParser(tc.sql) @@ -136,33 +107,6 @@ func TestParser_TokenConsumptionError(t *testing.T) { } } -func TestParser_TryConsumeKeywords(t *testing.T) { - for _, tc := range []struct { - sql string - matched bool - next string - }{ - {"WITH FILL", true, ""}, - {"WITH FILL tail", true, "tail"}, - {"SELECT 1", false, "SELECT"}, - {"WITH foo", false, "WITH"}, - {"WITH", false, "WITH"}, - } { - t.Run(tc.sql, func(t *testing.T) { - p := NewParser(tc.sql) - require.NoError(t, p.lexer.consumeToken()) - before := p.lexer.saveState() - matched, err := p.tryConsumeKeywords(KeywordWith, KeywordFill) - require.NoError(t, err) - require.Equal(t, tc.matched, matched) - require.Equal(t, tc.next, p.currentTokenString()) - if !matched { - require.Equal(t, before, p.lexer.saveState()) - } - }) - } -} - func TestParser_TryConsumeTokenKind(t *testing.T) { for _, tc := range []struct { name string diff --git a/parser/lexer.go b/parser/lexer.go index 89e6942..c8a861e 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -83,6 +83,11 @@ type Lexer struct { lexerState input string + + // Lexical failures are fatal for this input, even when discovered during + // lookahead. Keep them outside lexerState so restoring a cursor cannot + // discard the error or its original position. + err *lexerError } func NewLexer(buf string) *Lexer { @@ -359,13 +364,16 @@ func (l *Lexer) consumeToken() (err error) { // replace the current token; keep the previous one to disambiguate unary +/- prevToken := l.currentToken l.currentToken = nil + if l.err != nil { + return l.err + } pos := Pos(l.offset) defer func() { if err != nil { - var lexicalErr *lexerError - if !errors.As(err, &lexicalErr) { - err = &lexerError{pos: pos, err: err} + if !errors.As(err, &l.err) { + l.err = &lexerError{pos: pos, err: err} } + err = l.err } }() diff --git a/parser/lexer_test.go b/parser/lexer_test.go index d79b847..61011be 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -48,10 +48,8 @@ func TestPeekToken_RestoresState(t *testing.T) { require.Equal(t, Pos(2), le.pos) require.Equal(t, peekErr, err) require.Nil(t, lexer.currentToken) - // Rewinding to valid input must not retain the failed scan's error. - lexer.restoreState(lexerState{}) - require.NoError(t, lexer.consumeToken()) - require.Equal(t, "x", lexer.currentToken.String) + // Further advancement must not turn a lexical failure into EOF. + require.Equal(t, err, lexer.consumeToken()) } else { require.NoError(t, err) require.Equal(t, token, lexer.currentToken) diff --git a/parser/parse_system.go b/parser/parse_system.go index ab64673..beb500b 100644 --- a/parser/parse_system.go +++ b/parser/parse_system.go @@ -41,18 +41,13 @@ func (p *Parser) parseSystemFlushExpr(pos Pos) (*SystemFlushExpr, error) { switch { case p.matchKeyword(KeywordLogs): curToken := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &SystemFlushExpr{ FlushPos: pos, StatementEnd: curToken.End, Logs: true, }, nil - case p.matchKeyword(KeywordDistributed): - if err := p.expectKeyword(KeywordDistributed); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDistributed): distributed, err := p.parseTableIdentifier(p.Pos()) if err != nil { return nil, err @@ -80,20 +75,13 @@ func (p *Parser) parseSystemReloadExpr(pos Pos) (*SystemReloadExpr, error) { case p.matchKeyword(KeywordDictionaries): typ = KeywordDictionaries statementEnd = p.current().End - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() case p.matchKeyword(KeywordDictionary): typ = KeywordDictionary statementEnd = p.current().End hasDictionaryName = true - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } - case p.matchKeyword(KeywordEmbedded): - if err := p.expectKeyword(KeywordEmbedded); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() + case p.tryConsumeKeywords(KeywordEmbedded): typ = "EMBEDDED DICTIONARIES" statementEnd = p.current().End if err := p.expectKeyword(KeywordDictionaries); err != nil { @@ -162,16 +150,11 @@ func (p *Parser) parseSystemCtrlExpr(pos Pos) (*SystemCtrlExpr, error) { return nil, fmt.Errorf("expected START|STOP") } command := strings.ToUpper(p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() var typ string switch { - case p.matchKeyword(KeywordDistributed): - if err := p.expectKeyword(KeywordDistributed); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDistributed): switch { case p.matchKeyword(KeywordSends): typ = "DISTRIBUTED SENDS" @@ -198,10 +181,7 @@ func (p *Parser) parseSystemCtrlExpr(pos Pos) (*SystemCtrlExpr, error) { Type: typ, Cluster: cluster, }, nil - case p.matchKeyword(KeywordReplicated): - if err := p.expectKeyword(KeywordReplicated); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordReplicated): curToken := p.current() if err := p.expectKeyword(KeywordSends); err != nil { return nil, err @@ -229,9 +209,7 @@ func (p *Parser) parseSystemDropExpr(pos Pos) (*SystemDropExpr, error) { p.matchKeyword(KeywordFileSystem), p.matchKeyword(KeywordQuery): prefixToken := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() curToken := p.current() if err := p.expectKeyword(KeywordCache); err != nil { return nil, err @@ -242,9 +220,7 @@ func (p *Parser) parseSystemDropExpr(pos Pos) (*SystemDropExpr, error) { Type: prefixToken.String + " CACHE", }, nil case p.matchKeyword(KeywordCompiled): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordExpression); err != nil { return nil, err } @@ -273,9 +249,7 @@ func (p *Parser) parseDeduplicateClause(pos Pos) (*DeduplicateClause, error) { if err := p.expectKeyword(KeywordDeduplicate); err != nil { return nil, err } - if matched, consumeErr := p.tryConsumeKeywords(KeywordBy); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordBy) { return &DeduplicateClause{ DeduplicatePos: pos, }, nil @@ -286,9 +260,7 @@ func (p *Parser) parseDeduplicateClause(pos Pos) (*DeduplicateClause, error) { return nil, err } var except *ColumnExprList - if matched, consumeErr := p.tryConsumeKeywords(KeywordExcept); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordExcept) { except, err = p.parseColumnExprList(p.Pos()) if err != nil { return nil, err @@ -333,9 +305,7 @@ func (p *Parser) parseOptimizeStmt(pos Pos) (*OptimizeStmt, error) { hasFinal := false lastPos := p.Pos() - if matched, consumeErr := p.tryConsumeKeywords(KeywordFinal); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordFinal) { hasFinal = true statementEnd = lastPos } @@ -455,9 +425,7 @@ func (p *Parser) parseRoleName(_ Pos) (*RoleName, error) { } func (p *Parser) tryParseRoleSettings(pos Pos) ([]*RoleSetting, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordSettings); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordSettings) { return nil, nil } return p.parseRoleSettings(pos) @@ -541,9 +509,7 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { orReplace := false switch { case p.matchKeyword(KeywordIf): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordNot); err != nil { return nil, err } @@ -552,9 +518,7 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { } ifNotExists = true case p.matchKeyword(KeywordOr): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordReplace); err != nil { return nil, err } @@ -584,9 +548,7 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { statementEnd := roleNames[len(roleNames)-1].End() var accessStorageType *Ident - if matched, consumeErr := p.tryConsumeKeywords(KeywordIn); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordIn) { accessStorageType, err = p.parseIdent() if err != nil { return nil, err @@ -616,9 +578,7 @@ func (p *Parser) parseCreateRole(pos Pos) (*CreateRole, error) { func (p *Parser) parseAuthenticationClause(pos Pos) (*AuthenticationClause, error) { auth := &AuthenticationClause{AuthPos: pos} - if matched, consumeErr := p.tryConsumeKeywords(KeywordNot); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordNot) { if err := p.expectKeyword(KeywordIdentified); err != nil { return nil, err } @@ -632,13 +592,9 @@ func (p *Parser) parseAuthenticationClause(pos Pos) (*AuthenticationClause, erro } auth.AuthEnd = p.current().End - if matched, consumeErr := p.tryConsumeKeywords(KeywordWith); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordWith) { if p.matchKeyword(KeywordLdap) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordServer); err != nil { return nil, err } @@ -649,14 +605,10 @@ func (p *Parser) parseAuthenticationClause(pos Pos) (*AuthenticationClause, erro auth.LdapServer = server auth.AuthEnd = server.End() } else if p.matchKeyword(KeywordKerberos) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() auth.IsKerberos = true auth.AuthEnd = p.current().End - if matched, consumeErr := p.tryConsumeKeywords(KeywordRealm); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordRealm) { realm, err := p.parseString(p.Pos()) if err != nil { return nil, err @@ -667,15 +619,11 @@ func (p *Parser) parseAuthenticationClause(pos Pos) (*AuthenticationClause, erro } else if p.matchTokenKind(TokenKindIdent) { // Auth types like no_password, plaintext_password, etc. authType := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() auth.AuthType = authType auth.AuthEnd = p.current().End - if matched, consumeErr := p.tryConsumeKeywords(KeywordBy); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordBy) { value, err := p.parseString(p.Pos()) if err != nil { return nil, err @@ -699,16 +647,12 @@ func (p *Parser) parseHostClause(pos Pos) (*HostClause, error) { switch { case p.matchOneOfKeywords(KeywordLocal, KeywordAny, KeywordNone): hostType := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() host.HostType = hostType host.HostEnd = p.current().End case p.matchOneOfKeywords(KeywordName, KeywordRegexp, KeywordIp, KeywordLike): hostType := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() host.HostType = hostType value, err := p.parseString(p.Pos()) if err != nil { @@ -733,9 +677,7 @@ func (p *Parser) parseDefaultRoleClause(pos Pos) (*DefaultRoleClause, error) { defaultRole := &DefaultRoleClause{DefaultPos: pos} - if matched, consumeErr := p.tryConsumeKeywords(KeywordNone); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordNone) { defaultRole.None = true defaultRole.DefaultEnd = p.current().End return defaultRole, nil @@ -775,14 +717,10 @@ func (p *Parser) parseGranteesClause(pos Pos) (*GranteesClause, error) { grantees := &GranteesClause{GranteesPos: pos} - if matched, consumeErr := p.tryConsumeKeywords(KeywordAny); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAny) { grantees.Any = true grantees.GranteesEnd = p.current().End - } else if matched, consumeErr := p.tryConsumeKeywords(KeywordNone); consumeErr != nil { - return nil, consumeErr - } else if matched { + } else if p.tryConsumeKeywords(KeywordNone) { grantees.None = true grantees.GranteesEnd = p.current().End } else { @@ -814,9 +752,7 @@ func (p *Parser) parseGranteesClause(pos Pos) (*GranteesClause, error) { } // Check for EXCEPT clause - if matched, consumeErr := p.tryConsumeKeywords(KeywordExcept); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordExcept) { exceptList := make([]*RoleName, 0) except, err := p.parseRoleName(p.Pos()) if err != nil { @@ -849,9 +785,7 @@ func (p *Parser) parseGranteesClause(pos Pos) (*GranteesClause, error) { func (p *Parser) parseCreateUserModifiers(createUser *CreateUser) error { switch { case p.matchKeyword(KeywordIf): - if err := p.lexer.consumeToken(); err != nil { - return err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordNot); err != nil { return err } @@ -860,9 +794,7 @@ func (p *Parser) parseCreateUserModifiers(createUser *CreateUser) error { } createUser.IfNotExists = true case p.matchKeyword(KeywordOr): - if err := p.lexer.consumeToken(); err != nil { - return err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordReplace); err != nil { return err } @@ -936,15 +868,9 @@ func (p *Parser) parseDefaultClause(createUser *CreateUser) (bool, error) { createUser.StatementEnd = defaultRole.End() return true, nil } else if nextToken.String == KeywordDatabase { - if err := p.lexer.consumeToken(); err != nil { // consume DEFAULT - return false, err - } - if err := p.lexer.consumeToken(); err != nil { // consume DATABASE - return false, err - } - if matched, consumeErr := p.tryConsumeKeywords(KeywordNone); consumeErr != nil { - return false, consumeErr - } else if matched { + _ = p.lexer.consumeToken() // consume DEFAULT + _ = p.lexer.consumeToken() // consume DATABASE + if p.tryConsumeKeywords(KeywordNone) { createUser.DefaultDbNone = true createUser.StatementEnd = p.current().End } else { @@ -973,9 +899,7 @@ func (p *Parser) parseOptionalClauses(createUser *CreateUser) error { createUser.StatementEnd = auth.End() case p.matchKeyword(KeywordValid): - if err := p.lexer.consumeToken(); err != nil { // consume VALID keyword - return err - } + _ = p.lexer.consumeToken() // consume VALID keyword if err := p.expectKeyword(KeywordUntil); err != nil { return err } @@ -1012,9 +936,7 @@ func (p *Parser) parseOptionalClauses(createUser *CreateUser) error { createUser.StatementEnd = grantees.End() case p.matchKeyword(KeywordSettings): - if err := p.lexer.consumeToken(); err != nil { // consume SETTINGS keyword - return err - } + _ = p.lexer.consumeToken() // consume SETTINGS keyword settings, err := p.parseRoleSettings(p.Pos()) if err != nil { return err @@ -1064,9 +986,7 @@ func (p *Parser) parserDropUserOrRole(pos Pos) (*DropUserOrRole, error) { switch { case p.matchOneOfKeywords(KeywordUser, KeywordRole): target = p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() default: return nil, fmt.Errorf("expected USER|ROLE") } @@ -1107,9 +1027,7 @@ func (p *Parser) parserDropUserOrRole(pos Pos) (*DropUserOrRole, error) { } var from *Ident - if matched, consumeErr := p.tryConsumeKeywords(KeywordFrom); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordFrom) { from, err = p.parseIdent() if err != nil { return nil, err @@ -1134,9 +1052,7 @@ func (p *Parser) parserDropUserOrRole(pos Pos) (*DropUserOrRole, error) { func (p *Parser) parsePrivilegeSelectOrInsert(pos Pos) (*PrivilegeClause, error) { keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() var err error var params *ParamExprList @@ -1156,88 +1072,52 @@ func (p *Parser) parsePrivilegeSelectOrInsert(pos Pos) (*PrivilegeClause, error) func (p *Parser) parsePrivilegeAlter(pos Pos) (*PrivilegeClause, error) { keywords := []string{KeywordAlter} switch { - case p.matchKeyword(KeywordIndex): - if err := p.expectKeyword(KeywordIndex); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordIndex): keywords = append(keywords, KeywordIndex) case p.matchOneOfKeywords(KeywordUpdate, KeywordDelete, KeywordUser, KeywordRole, KeywordQuota): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) case p.matchOneOfKeywords(KeywordAdd, KeywordDrop, KeywordModify, KeywordClear, KeywordComment, KeywordRename, KeywordMaterialized): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) switch { - case p.matchKeyword(KeywordColumn): - if err := p.expectKeyword(KeywordColumn); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordColumn): keywords = append(keywords, KeywordColumn) - case p.matchKeyword(KeywordIndex): - if err := p.expectKeyword(KeywordIndex); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordIndex): keywords = append(keywords, KeywordIndex) keywords = append(keywords, KeywordConstraint) - case p.matchKeyword(KeywordTtl): - if err := p.expectKeyword(KeywordTtl); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordTtl): keywords = append(keywords, KeywordTtl) default: return nil, fmt.Errorf("expected COLUMN|INDEX") } - case p.matchKeyword(KeywordOrder): - if err := p.expectKeyword(KeywordOrder); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordOrder): if err := p.expectKeyword(KeywordBy); err != nil { return nil, err } keywords = append(keywords, KeywordOrder, KeywordBy) - case p.matchKeyword(KeywordSample): - if err := p.expectKeyword(KeywordSample); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordSample): if err := p.expectKeyword(KeywordBy); err != nil { return nil, err } keywords = append(keywords, KeywordSample, KeywordBy) - case p.matchKeyword(KeywordSettings): - if err := p.expectKeyword(KeywordSettings); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordSettings): keywords = append(keywords, KeywordSettings) - case p.matchKeyword(KeywordView): - if err := p.expectKeyword(KeywordView); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordView): keywords = append(keywords, KeywordView) switch { - case p.matchKeyword(KeywordModify): - if err := p.expectKeyword(KeywordModify); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordModify): keywords = append(keywords, KeywordModify) - case p.matchKeyword(KeywordRefresh): - if err := p.expectKeyword(KeywordRefresh); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordRefresh): keywords = append(keywords, KeywordRefresh) default: return nil, fmt.Errorf("expected MODIFY|REFRESH") } case p.matchOneOfKeywords(KeywordMove, KeywordFreeze): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) if err := p.expectKeyword(KeywordPartition); err != nil { return nil, err @@ -1257,22 +1137,14 @@ func (p *Parser) parsePrivilegeCreate(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordDatabase, KeywordDictionary, KeywordTable, KeywordFunction, KeywordView, KeywordUser, KeywordRole, KeywordQuota): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) - case p.matchKeyword(KeywordTemporary): - if err := p.expectKeyword(KeywordTemporary); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordTemporary): if err := p.expectKeyword(KeywordTable); err != nil { return nil, err } keywords = append(keywords, KeywordTemporary, KeywordTable) - case p.matchKeyword(KeywordRows): - if err := p.expectKeyword(KeywordRows); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordRows): if err := p.expectKeyword(KeywordPolicy); err != nil { return nil, err } @@ -1291,9 +1163,7 @@ func (p *Parser) parsePrivilegeDrop(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordDatabase, KeywordDictionary, KeywordUser, KeywordRole, KeywordQuota, KeywordTable, KeywordFunction, KeywordView): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) default: return nil, fmt.Errorf("expected DATABASE|DICTIONARY|TABLE|FUNCTION|VIEW") @@ -1309,9 +1179,7 @@ func (p *Parser) parsePrivilegeShow(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordDatabases, KeywordDictionaries, KeywordTables, KeywordColumns): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) default: return nil, fmt.Errorf("expected DATABASES|DICTIONARIES|TABLES|COLUMNS") @@ -1327,26 +1195,16 @@ func (p *Parser) parsePrivilegeSystem(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordShutdown, KeywordMerges, KeywordFetches, KeywordSends, KeywordMoves, KeywordCluster): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) - case p.matchKeyword(KeywordDrop): - if err := p.expectKeyword(KeywordDrop); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDrop): keywords = append(keywords, KeywordDrop) switch { - case p.matchKeyword(KeywordCache): - if err := p.expectKeyword(KeywordCache); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordCache): keywords = append(keywords, KeywordCache) case p.matchOneOfKeywords(KeywordMark, KeywordDNS, KeywordUncompressed): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) if err := p.expectKeyword(KeywordCache); err != nil { return nil, err @@ -1355,40 +1213,27 @@ func (p *Parser) parsePrivilegeSystem(pos Pos) (*PrivilegeClause, error) { default: return nil, fmt.Errorf("expected CACHE|MARK|DNS|UNCOMPRESSED") } - case p.matchKeyword(KeywordReload): - if err := p.expectKeyword(KeywordReload); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordReload): keywords = append(keywords, KeywordReload) switch { case p.matchOneOfKeywords(KeywordDictionary, KeywordFunction, KeywordFunctions, KeywordConfig): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) default: return nil, fmt.Errorf("expected DICTIONARY|FUNCTION|FUNCTIONS|CONFIG") } - case p.matchKeyword(KeywordFlush): - if err := p.expectKeyword(KeywordFlush); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordFlush): keywords = append(keywords, KeywordFlush) switch { case p.matchOneOfKeywords(KeywordLogs, KeywordDistributed): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) default: return nil, fmt.Errorf("expected LOGS|DISTRIBUTED") } - case p.matchKeyword(KeywordTtl): - if err := p.expectKeyword(KeywordTtl); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordTtl): keywords = append(keywords, KeywordTtl) if err := p.expectKeyword(KeywordMerges); err != nil { return nil, err @@ -1396,18 +1241,13 @@ func (p *Parser) parsePrivilegeSystem(pos Pos) (*PrivilegeClause, error) { keywords = append(keywords, KeywordMerges) case p.matchOneOfKeywords(KeywordSync, KeywordRestart): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() keywords = append(keywords, keyword) if err := p.expectKeyword(KeywordReplica); err != nil { return nil, err } keywords = append(keywords, KeywordReplica) - case p.matchKeyword(KeywordReplication): - if err := p.expectKeyword(KeywordReplication); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordReplication): keywords = append(keywords, KeywordReplication) if err := p.expectKeyword(KeywordQueues); err != nil { return nil, err @@ -1425,9 +1265,7 @@ func (p *Parser) parsePrivilegeSystem(pos Pos) (*PrivilegeClause, error) { func (p *Parser) parsePrivilegeClause(pos Pos) (*PrivilegeClause, error) { if p.matchTokenKind(TokenKindIdent) { if p.current().String == "dictGet" { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &PrivilegeClause{ PrivilegePos: pos, Keywords: []string{"dictGet"}, @@ -1437,38 +1275,21 @@ func (p *Parser) parsePrivilegeClause(pos Pos) (*PrivilegeClause, error) { switch { case p.matchOneOfKeywords(KeywordSelect, KeywordInsert): return p.parsePrivilegeSelectOrInsert(pos) - case p.matchKeyword(KeywordAlter): - if err := p.expectKeyword(KeywordAlter); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordAlter): return p.parsePrivilegeAlter(pos) - case p.matchKeyword(KeywordCreate): - if err := p.expectKeyword(KeywordCreate); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordCreate): return p.parsePrivilegeCreate(pos) - case p.matchKeyword(KeywordDrop): - if err := p.expectKeyword(KeywordDrop); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDrop): return p.parsePrivilegeDrop(pos) - case p.matchKeyword(KeywordShow): - if err := p.expectKeyword(KeywordShow); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordShow): return p.parsePrivilegeShow(pos) case p.matchKeyword(KeywordAll), p.matchTokenKind(KeywordNone): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &PrivilegeClause{ PrivilegePos: pos, Keywords: []string{KeywordAll}, }, nil - case p.matchKeyword(KeywordKill): - if err := p.expectKeyword(KeywordKill); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordKill): if err := p.expectKeyword(KeywordQuery); err != nil { return nil, err } @@ -1476,15 +1297,9 @@ func (p *Parser) parsePrivilegeClause(pos Pos) (*PrivilegeClause, error) { PrivilegePos: pos, Keywords: []string{KeywordKill, KeywordQuery}, }, nil - case p.matchKeyword(KeywordSystem): - if err := p.expectKeyword(KeywordSystem); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordSystem): return p.parsePrivilegeSystem(pos) - case p.matchKeyword(KeywordAdmin): - if err := p.expectKeyword(KeywordAdmin); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordAdmin): if err := p.expectKeyword(KeywordOption); err != nil { return nil, err } @@ -1494,17 +1309,12 @@ func (p *Parser) parsePrivilegeClause(pos Pos) (*PrivilegeClause, error) { }, nil case p.matchOneOfKeywords(KeywordOptimize, KeywordTruncate): keyword := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &PrivilegeClause{ PrivilegePos: pos, Keywords: []string{keyword}, }, nil - case p.matchKeyword(KeywordRole): - if err := p.expectKeyword(KeywordRole); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordRole): if err := p.expectKeyword(KeywordAdmin); err != nil { return nil, err } @@ -1721,9 +1531,7 @@ func (p *Parser) parseRoleRenamePair(_ Pos) (*RoleRenamePair, error) { RoleName: roleName, StatementEnd: roleName.End(), } - if matched, consumeErr := p.tryConsumeKeywords(KeywordRename); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordRename) { if err := p.expectKeyword(KeywordTo); err != nil { return nil, err } diff --git a/parser/parser_alter.go b/parser/parser_alter.go index 6046c76..7c84c93 100644 --- a/parser/parser_alter.go +++ b/parser/parser_alter.go @@ -38,9 +38,7 @@ func (p *Parser) parseAlterTable(pos Pos) (*AlterTable, error) { // like the sibling branches, the clause position is the keyword // itself, so capture it before consuming DETACH detachPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() alter, err = p.parseAlterTableDetachPartition(detachPos) case p.matchKeyword(KeywordFreeze): alter, err = p.parseAlterTableFreezePartition(p.Pos()) @@ -182,9 +180,7 @@ func (p *Parser) parseAlterTableAddIndex(pos Pos) (*AlterTableAddIndex, error) { } func (p *Parser) tryParseProjectionOrderBy(pos Pos) (*ProjectionOrderByClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordOrder); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordOrder) { return nil, nil // nolint } if err := p.expectKeyword(KeywordBy); err != nil { @@ -383,17 +379,13 @@ func (p *Parser) parsePartitionClause(pos Pos) (*PartitionClause, error) { partition := &PartitionClause{ PartitionPos: pos, } - if matched, consumeErr := p.tryConsumeKeywords(KeywordId); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordId) { id, err := p.parseString(p.Pos()) if err != nil { return nil, err } partition.ID = id - } else if matched, consumeErr := p.tryConsumeKeywords(KeywordAll); consumeErr != nil { - return nil, consumeErr - } else if matched { + } else if p.tryConsumeKeywords(KeywordAll) { partition.All = true } else { expr, err := p.parseExpr(p.Pos()) @@ -418,9 +410,7 @@ func (p *Parser) parseAlterTableAttachPartition(pos Pos) (AlterTableClause, erro } alterTable.Partition = partition // FROM [db.]table? - if matched, consumeErr := p.tryConsumeKeywords(KeywordFrom); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordFrom) { tableIdentifier, err := p.parseTableIdentifier(p.Pos()) if err != nil { return nil, err @@ -442,9 +432,7 @@ func (p *Parser) parseAlterTableDropClause(pos Pos) (AlterTableClause, error) { default: return nil, fmt.Errorf("expected token: COLUMN|INDEX|PROJECTION, but got %s", p.currentTokenKind()) } - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() ifExists, err := p.tryParseIfExists() if err != nil { @@ -478,9 +466,7 @@ func (p *Parser) parseAlterTableDropClause(pos Pos) (AlterTableClause, error) { } func (p *Parser) tryParseAfterClause() (*NestedIdentifier, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordAfter); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordAfter) { return nil, nil // nolint } @@ -491,9 +477,7 @@ func (p *Parser) tryParseAfterClause() (*NestedIdentifier, error) { func (p *Parser) parseAlterTableDropPartition(pos Pos) (AlterTableClause, error) { var hasDetached bool if p.matchKeyword(KeywordDetached) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() hasDetached = true } partitionPos := p.Pos() @@ -577,9 +561,7 @@ func (p *Parser) parseAlterTableClearClause(pos Pos) (AlterTableClause, error) { default: return nil, fmt.Errorf("expected keyword: COLUMN|INDEX|PROJECTION, but got %q", p.currentTokenKind()) } - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() ifExists, err := p.tryParseIfExists() if err != nil { @@ -593,9 +575,7 @@ func (p *Parser) parseAlterTableClearClause(pos Pos) (AlterTableClause, error) { statementEnd := name.End() var partition *PartitionClause - if matched, consumeErr := p.tryConsumeKeywords(KeywordIn); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordIn) { partition, err = p.tryParsePartitionClause(p.Pos()) if err != nil { return nil, err @@ -679,9 +659,7 @@ func (p *Parser) parseAlterTableModify(pos Pos) (AlterTableClause, error) { return p.parseAlterTableModifyColumn(pos) case p.matchKeyword(KeywordTtl): ttlPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() items, err := p.parseTTLClause(ttlPos, true) if err != nil { return nil, err @@ -701,9 +679,7 @@ func (p *Parser) parseAlterTableModify(pos Pos) (AlterTableClause, error) { TTL: ttlClause, }, nil case p.matchKeyword(KeywordQuery): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() selectQuery, err := p.parseSelectQuery(pos) if err != nil { return nil, err @@ -714,9 +690,7 @@ func (p *Parser) parseAlterTableModify(pos Pos) (AlterTableClause, error) { SelectExpr: selectQuery, }, nil case p.matchKeyword(KeywordOrder): - if err := p.lexer.consumeToken(); err != nil { // consume "ORDER" - return nil, err - } + _ = p.lexer.consumeToken() // consume "ORDER" if err := p.expectKeyword(KeywordBy); err != nil { return nil, err } @@ -730,9 +704,7 @@ func (p *Parser) parseAlterTableModify(pos Pos) (AlterTableClause, error) { OrderBy: orderBy, }, nil case p.matchKeyword(KeywordSetting): - if err := p.lexer.consumeToken(); err != nil { // consume "SETTING" - return nil, err - } + _ = p.lexer.consumeToken() // consume "SETTING" settings, err := p.parseSettingsList(p.Pos()) if err != nil { return nil, err @@ -844,9 +816,7 @@ func (p *Parser) parseAlterTableMaterialize(pos Pos) (AlterTableClause, error) { default: return nil, fmt.Errorf("expected keyword: INDEX|PROJECTION, but got %q", p.currentTokenKind()) } - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() ifExists, err := p.tryParseIfExists() if err != nil { @@ -858,9 +828,7 @@ func (p *Parser) parseAlterTableMaterialize(pos Pos) (AlterTableClause, error) { } statementEnd := name.End() var partition *PartitionClause - if matched, consumeErr := p.tryConsumeKeywords(KeywordIn); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordIn) { partition, err = p.tryParsePartitionClause(p.Pos()) if err != nil { return nil, err @@ -979,9 +947,7 @@ func (p *Parser) parseAlterTableUpdate(pos Pos) (AlterTableClause, error) { } var inPartition *PartitionClause - if matched, consumeErr := p.tryConsumeKeywords(KeywordIn); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordIn) { inPartition, err = p.parsePartitionClause(p.Pos()) if err != nil { return nil, err diff --git a/parser/parser_column.go b/parser/parser_column.go index 5a49cd1..16257aa 100644 --- a/parser/parser_column.go +++ b/parser/parser_column.go @@ -1,7 +1,6 @@ package parser import ( - "errors" "fmt" "strings" ) @@ -31,77 +30,66 @@ const ( ) func (p *Parser) tryParseColumnComment(pos Pos) (*StringLiteral, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordComment); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordComment) { return nil, nil // nolint } return p.parseString(pos) } -func (p *Parser) getNextPrecedence() (int, error) { +func (p *Parser) getNextPrecedence() int { switch { case p.matchKeyword(KeywordOr): - return PrecedenceOr, nil + return PrecedenceOr case p.matchKeyword(KeywordAnd): - return PrecedenceAnd, nil + return PrecedenceAnd case p.matchKeyword(KeywordIs): - return PrecedenceIs, nil + return PrecedenceIs case p.matchKeyword(KeywordNot): // Infix NOT only begins NOT IN/LIKE/ILIKE/BETWEEN, so it binds with // the precedence of the operator it negates; `a = b NOT IN (1)` must // group the same way `a = b IN (1)` does. - next, err := p.lexer.peekToken() - if err != nil { - return 0, err - } - if next == nil || next.Kind != TokenKindKeyword { - return PrecedenceNot, nil - } - switch next.ToString() { - case KeywordIn: - return precedenceIn, nil - case KeywordLike, KeywordIlike, KeywordBetween: - return PrecedenceBetweenLike, nil + switch { + case p.peekKeyword(KeywordIn): + return precedenceIn + case p.peekKeyword(KeywordLike), p.peekKeyword(KeywordIlike), p.peekKeyword(KeywordBetween): + return PrecedenceBetweenLike default: - return PrecedenceNot, nil + return PrecedenceNot } case p.matchTokenKind(TokenKindDot): - return PrecedenceDot, nil + return PrecedenceDot case p.matchTokenKind(TokenKindDash): - return PrecedenceDoubleColon, nil + return PrecedenceDoubleColon case p.matchTokenKind(TokenKindSingleEQ), p.matchTokenKind(TokenKindLT), p.matchTokenKind(TokenKindLE), p.matchTokenKind(TokenKindGE), p.matchTokenKind(TokenKindGT), p.matchTokenKind(TokenKindDoubleEQ), p.matchTokenKind(TokenKindNE), p.matchTokenKind("<>"): - return PrecedenceCompare, nil + return PrecedenceCompare case p.matchTokenKind(TokenKindConcat): - return PrecedenceConcat, nil + return PrecedenceConcat case p.matchTokenKind(TokenKindPlus), p.matchTokenKind(TokenKindMinus): - return PrecedenceAddSub, nil + return PrecedenceAddSub case p.matchTokenKind(TokenKindMul), p.matchTokenKind(TokenKindDiv), p.matchTokenKind(TokenKindMod): - return PrecedenceMulDivMod, nil + return PrecedenceMulDivMod case p.matchTokenKind(TokenKindArrow): - return PrecedenceArrow, nil + return PrecedenceArrow case p.matchTokenKind(TokenKindLParen), p.matchTokenKind(TokenKindLBracket): - return PrecedenceBracket, nil + return PrecedenceBracket case p.matchKeyword(KeywordBetween), p.matchKeyword(KeywordLike), p.matchKeyword(KeywordIlike), p.matchKeyword(KeywordRegexp): - return PrecedenceBetweenLike, nil + return PrecedenceBetweenLike case p.matchKeyword(KeywordIn): - return precedenceIn, nil + return precedenceIn case p.matchKeyword(KeywordGlobal): // GLOBAL is also a join locality: in `ON a = b GLOBAL LEFT JOIN c` it // belongs to the FROM clause, so the expression has to end here. - if join, err := p.peekJoinAfterLocality(); err != nil { - return 0, err - } else if join { - return PrecedenceUnknown, nil + if p.peekJoinAfterLocality() { + return PrecedenceUnknown } - return precedenceIn, nil + return precedenceIn case p.matchTokenKind(TokenKindQuestionMark): - return PrecedenceQuery, nil + return PrecedenceQuery default: - return PrecedenceUnknown, nil + return PrecedenceUnknown } } @@ -117,9 +105,7 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { p.matchKeyword(KeywordAnd), p.matchKeyword(KeywordOr), p.matchTokenKind(TokenKindDoubleEQ): op := p.current().ToString() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() rightExpr, err := p.parseSubExpr(p.Pos(), precedence) if err != nil { return nil, err @@ -130,9 +116,7 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { RightExpr: rightExpr, }, nil case p.matchTokenKind(TokenKindArrow): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() // Lambdas are right-associative: `x -> y -> body` is `x -> (y -> body)`, // so the body is parsed one level below the arrow's own precedence. rightExpr, err := p.parseSubExpr(p.Pos(), precedence-1) @@ -145,9 +129,7 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { RightExpr: rightExpr, }, nil case p.matchTokenKind(TokenKindDash): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if p.matchTokenKind(TokenKindIdent) && p.current().String == "Tuple" { name, err := p.parseIdent() @@ -181,13 +163,8 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { case p.matchKeyword(KeywordBetween): return p.parseBetweenClause(expr, false) case p.matchKeyword(KeywordGlobal): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } - hasNot, consumeErr := p.tryConsumeKeywords(KeywordNot) - if consumeErr != nil { - return nil, consumeErr - } + _ = p.lexer.consumeToken() + hasNot := p.tryConsumeKeywords(KeywordNot) if p.expectKeyword(KeywordIn) != nil { return nil, fmt.Errorf("expected IN after GLOBAL, got %s", p.currentTokenString()) } @@ -209,9 +186,7 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { HasNot: hasNot, }, nil case p.matchTokenKind(TokenKindDot): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() operation := TokenKindDot qualifier, consumeErr := p.tryConsumeTokenKind(TokenKindColon) if consumeErr != nil { @@ -242,9 +217,7 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { Index: rightExpr, }, nil case p.matchKeyword(KeywordNot): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if p.matchKeyword(KeywordBetween) { return p.parseBetweenClause(expr, true) } @@ -256,9 +229,7 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { return nil, fmt.Errorf("expected IN, LIKE, ILIKE or BETWEEN after NOT, got %s", p.currentTokenKind()) } op := p.current().ToString() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() rightExpr, err := p.parseSubExpr(p.Pos(), precedence) if err != nil { return nil, err @@ -281,13 +252,8 @@ func (p *Parser) parseInfix(expr Expr, precedence int) (Expr, error) { return p.parseTernaryExpr(expr) case p.matchKeyword(KeywordIs): isPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } - isNotNull, consumeErr := p.tryConsumeKeywords(KeywordNot) - if consumeErr != nil { - return nil, consumeErr - } + _ = p.lexer.consumeToken() + isNotNull := p.tryConsumeKeywords(KeywordNot) // the expression ends at the NULL keyword; capture its end before // expectKeyword consumes it nullEnd := p.End() @@ -336,14 +302,12 @@ func (p *Parser) parseSubExpr(pos Pos, precedence int) (Expr, error) { // and `SELECT a GLOBAL` silently read the operator as an implicit alias. func (p *Parser) parseInfixLoop(expr Expr, precedence int) (Expr, error) { for !p.lexer.isEOF() || p.current() != nil { - nextPrecedence, err := p.getNextPrecedence() - if err != nil { - return nil, err - } + nextPrecedence := p.getNextPrecedence() if nextPrecedence <= precedence { return expr, nil } // parse binary operation + var err error expr, err = p.parseInfix(expr, nextPrecedence) if err != nil { return nil, err @@ -450,14 +414,10 @@ func (p *Parser) parseUnaryExpr(pos Pos) (Expr, error) { switch { case p.matchTokenKind(TokenKindPlus), p.matchTokenKind(TokenKindMinus): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() expr, err = p.parseColumnExpr(p.Pos()) case p.matchKeyword(KeywordNot): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() // Prefix NOT binds looser than comparisons: `NOT a = b` negates the // whole comparison, so the operand is parsed at NOT's own precedence // instead of stopping at the primary expression. @@ -477,36 +437,26 @@ func (p *Parser) parseUnaryExpr(pos Pos) (Expr, error) { } -func (p *Parser) peekTokenKind(kinds ...TokenKind) (bool, error) { - token, err := p.lexer.peekToken() - if err != nil { - return false, err - } - if token == nil { - return false, nil +func (p *Parser) peekTokenKind(kind TokenKind) bool { + if p.lexer.isEOF() { + return false } - for _, kind := range kinds { - if token.Kind == kind { - return true, nil - } + token, err := p.lexer.peekToken() + if err != nil || token == nil { + return false } - return false, nil + return token.Kind == kind } -func (p *Parser) peekKeyword(keywords ...string) (bool, error) { - token, err := p.lexer.peekToken() - if err != nil { - return false, err +func (p *Parser) peekKeyword(keyword string) bool { + if p.lexer.isEOF() { + return false } - if token == nil || token.Kind != TokenKindKeyword { - return false, nil - } - for _, keyword := range keywords { - if strings.EqualFold(token.String, keyword) { - return true, nil - } + token, err := p.lexer.peekToken() + if err != nil || token == nil { + return false } - return false, nil + return token.Kind == TokenKindKeyword && strings.EqualFold(token.String, keyword) } // clauseStarterKeywords lists the keywords that begin a clause following the @@ -532,17 +482,25 @@ func (p *Parser) matchClauseStarterKeyword() bool { // peekIsClauseStarterKeyword reports whether the next token is one of the // clause-starter keywords. -func (p *Parser) peekIsClauseStarterKeyword() (bool, error) { - return p.peekKeyword(clauseStarterKeywords...) +func (p *Parser) peekIsClauseStarterKeyword() bool { + for _, kw := range clauseStarterKeywords { + if p.peekKeyword(kw) { + return true + } + } + return false } // peekIsEndOfStatement reports whether the next token is EOF or `;`. -func (p *Parser) peekIsEndOfStatement() (bool, error) { +func (p *Parser) peekIsEndOfStatement() bool { next, err := p.lexer.peekToken() if err != nil { - return false, err + return false } - return next == nil || next.Kind == ";", nil + if next == nil { + return true + } + return next.Kind == ";" } var expressionContinuationTokenKinds = []TokenKind{ @@ -558,11 +516,18 @@ var expressionContinuationKeywords = []string{ KeywordBetween, KeywordIs, KeywordThen, KeywordElse, KeywordEnd, } -func (p *Parser) peekIsExpressionContinuation() (bool, error) { - if matched, err := p.peekTokenKind(expressionContinuationTokenKinds...); err != nil || matched { - return matched, err +func (p *Parser) peekIsExpressionContinuation() bool { + for _, kind := range expressionContinuationTokenKinds { + if p.peekTokenKind(kind) { + return true + } } - return p.peekKeyword(expressionContinuationKeywords...) + for _, keyword := range expressionContinuationKeywords { + if p.peekKeyword(keyword) { + return true + } + } + return false } // keywordIsSelectItemIdentifier reports whether the current keyword token is @@ -586,31 +551,24 @@ func (p *Parser) peekIsExpressionContinuation() (bool, error) { // trailing clause-starter keyword like `FROM` at EOF must still be treated // as a terminator, not a no-AS alias). parseColumnExpr applies the eos // disambiguator inline. -func (p *Parser) keywordIsSelectItemIdentifier() (bool, error) { +func (p *Parser) keywordIsSelectItemIdentifier() bool { if !p.matchTokenKind(TokenKindKeyword) { - return false, nil + return false } - if matched, err := p.peekTokenKind(TokenKindComma); err != nil || matched { - return matched, err - } - if matched, err := p.peekKeyword(KeywordAs); err != nil || matched { - return matched, err - } - if matched, err := p.peekIsClauseStarterKeyword(); err != nil || matched { - return matched, err - } - return p.peekIsExpressionContinuation() + return p.peekTokenKind(TokenKindComma) || + p.peekKeyword(KeywordAs) || + p.peekIsClauseStarterKeyword() || + p.peekIsExpressionContinuation() } // isSelectItemTerminatorKeyword checks whether the current token is a keyword // that begins a clause following the SELECT item list. When true, we should // not treat the keyword itself as a bare alias. -func (p *Parser) isSelectItemTerminatorKeyword() (bool, error) { - identifier, err := p.keywordIsSelectItemIdentifier() - if err != nil { - return false, err +func (p *Parser) isSelectItemTerminatorKeyword() bool { + if p.keywordIsSelectItemIdentifier() { + return false } - return !identifier && p.matchClauseStarterKeyword(), nil + return p.matchClauseStarterKeyword() } func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen @@ -618,12 +576,9 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen // qualified column reference, even when it is otherwise reserved (for // example, `kill.item_id`). INTERVAL must reach its dedicated parser // first because a dot can also start its numeric operand. - if !p.matchKeyword(KeywordInterval) && p.matchTokenKind(TokenKindKeyword) { - if dot, err := p.peekTokenKind(TokenKindDot); err != nil { - return nil, err - } else if dot { - return p.parseIdentOrFunction(pos) - } + if !p.matchKeyword(KeywordInterval) && + p.matchTokenKind(TokenKindKeyword) && p.peekTokenKind(TokenKindDot) { + return p.parseIdentOrFunction(pos) } // Parse the keyword as an identifier if it is followed by `,`, `AS`, @@ -636,29 +591,11 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen // is only valid in expression position, so it's applied inline here // rather than in keywordIsSelectItemIdentifier (which is shared with the // terminator/alias check). - identifier, err := p.keywordIsSelectItemIdentifier() - if err != nil { - return nil, err - } - if identifier { + if p.keywordIsSelectItemIdentifier() || + (p.matchTokenKind(TokenKindKeyword) && + (p.peekIsEndOfStatement() || p.peekIsExpressionContinuation())) { return p.parseAnyKeyword() } - if p.matchTokenKind(TokenKindKeyword) { - if end, err := p.peekIsEndOfStatement(); err != nil { - return nil, err - } else if end { - return p.parseAnyKeyword() - } - } - - keywordFunction := false - if p.matchTokenKind(TokenKindKeyword) { - keywordFunction, err = p.peekTokenKind(TokenKindLParen) - if err != nil { - return nil, err - } - } - switch { case p.matchKeyword(KeywordInterval): // ClickHouse also accepts an unquoted `interval` as a column name @@ -680,10 +617,6 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen savedState := p.lexer.saveState() interval, err := p.parseInterval(true) if err != nil { - var lexicalErr *lexerError - if errors.As(err, &lexicalErr) { - return nil, err - } if p.failedIntervalOffsets == nil { p.failedIntervalOffsets = make(map[Pos]struct{}) } @@ -713,7 +646,7 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen return p.parseColumnExtractExpr(pos) case p.matchTokenKind(TokenKindIdent): return p.parseIdentOrFunction(pos) - case keywordFunction: + case p.matchTokenKind(TokenKindKeyword) && p.peekTokenKind(TokenKindLParen): // Reserved operator keywords stay callable as ordinary functions when // followed by '(': and(a, b), or(a, b), in(x, set), like(s, pat), ... // Keywords with dedicated syntax (CAST, CASE, EXTRACT, INTERVAL, ...) @@ -725,9 +658,7 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen p.matchTokenKind(TokenKindFloat): // number literal return p.parseNumber(pos) case p.matchTokenKind(TokenKindLParen): - if peek, err := p.lexer.peekToken(); err != nil { - return nil, err - } else if peek != nil { + if peek, _ := p.lexer.peekToken(); peek != nil { if peek.Kind == TokenKindKeyword && strings.EqualFold(peek.String, KeywordSelect) { return p.parseSubQuery(pos) } @@ -742,9 +673,7 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen // to determine if it is a map literal or a query param. // Treat both identifiers and keywords as identifier-like for placeholders. // parseIdent accepts keywords-as-ident, so this is safe. - if matched, err := p.peekTokenKind(TokenKindIdent, TokenKindKeyword); err != nil { - return nil, err - } else if matched { + if p.peekTokenKind(TokenKindIdent) || p.peekTokenKind(TokenKindKeyword) { return p.parseQueryParam(p.Pos()) } return p.parseMapLiteral(p.Pos()) @@ -752,9 +681,7 @@ func (p *Parser) parseColumnExpr(pos Pos) (Expr, error) { //nolint:funlen return p.parseNumber(p.Pos()) case p.matchTokenKind(TokenKindQuestionMark): // Placeholder `?` - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &PlaceHolder{ PlaceholderPos: pos, PlaceHolderEnd: pos, @@ -785,9 +712,7 @@ func (p *Parser) parseColumnCastExpr(pos Pos) (Expr, error) { // CAST(x, T) and CAST(x AS T) are equivalent case p.matchKeyword(KeywordAs), p.matchTokenKind(","): separator = p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() default: return nil, fmt.Errorf("expected AS or , but got %s", p.currentTokenKind()) } @@ -829,15 +754,11 @@ func (p *Parser) parseColumnExprList(pos Pos) (*ColumnExprList, error) { } func (p *Parser) parseColumnExprListWithTerm(term TokenKind, pos Pos) (*ColumnExprList, error) { - var err error columnExprList := &ColumnExprList{ ListPos: pos, ListEnd: pos, } - columnExprList.HasDistinct, err = p.tryConsumeKeywords(KeywordDistinct) - if err != nil { - return nil, err - } + columnExprList.HasDistinct = p.tryConsumeKeywords(KeywordDistinct) columnList := make([]Expr, 0) for !p.lexer.isEOF() || p.current() != nil { if term != "" && p.matchTokenKind(term) { @@ -880,9 +801,7 @@ func (p *Parser) parseSelectItems() ([]*SelectItem, error) { } else if token == nil { break } - if terminator, err := p.isSelectItemTerminatorKeyword(); err != nil { - return nil, err - } else if terminator { + if p.isSelectItemTerminatorKeyword() { break } } @@ -939,10 +858,7 @@ func (p *Parser) parseColumnArgList(pos Pos) (*ColumnArgList, error) { if err := p.expectTokenKind(TokenKindLParen); err != nil { return nil, err } - distinct, consumeErr := p.tryConsumeKeywords(KeywordDistinct) - if consumeErr != nil { - return nil, consumeErr - } + distinct := p.tryConsumeKeywords(KeywordDistinct) var items []Expr for !p.lexer.isEOF() && !p.matchTokenKind(TokenKindRParen) { @@ -1055,9 +971,7 @@ func (p *Parser) parseKeywordArgModifier(form keywordArgForm, slot int) (Expr, b return expr, false, err } // Only a modifier when an expression follows it; a bare `both` is an identifier. - if matched, err := p.peekTokenKind(TokenKindComma, TokenKindRParen); err != nil { - return nil, false, err - } else if matched { + if p.peekTokenKind(TokenKindComma) || p.peekTokenKind(TokenKindRParen) { expr, err := p.parseExpr(p.Pos()) return expr, false, err } @@ -1086,10 +1000,7 @@ func (p *Parser) parseKeywordArgFunctionParams(pos Pos, form keywordArgForm) (*P } itemsPos := p.Pos() - hasDistinct, consumeErr := p.tryConsumeKeywords(KeywordDistinct) - if consumeErr != nil { - return nil, consumeErr - } + hasDistinct := p.tryConsumeKeywords(KeywordDistinct) items := make([]Expr, 0) // A comma advances the argument slot just as a separator keyword does, so the @@ -1108,9 +1019,7 @@ func (p *Parser) parseKeywordArgFunctionParams(pos Pos, form keywordArgForm) (*P slot += consumed var alias *Ident - if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAs) { if alias, err = p.parseAnyKeyword(); err != nil { return nil, err } @@ -1290,9 +1199,7 @@ func (p *Parser) parseColumnsExpr(pos Pos) (*ColumnExpr, error) { } var alias *Ident - if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAs) { // after AS the token can only be an alias name, reserved keyword or not alias, err = p.parseAnyKeyword() if err != nil { @@ -1326,22 +1233,14 @@ func (p *Parser) parseSelectItem() (*SelectItem, error) { var alias *Ident switch { - case p.matchKeyword(KeywordAs): - if err := p.expectKeyword(KeywordAs); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordAs): // `SELECT 1 AS ` works for any keyword, reserved or not: // after AS the token can only be an alias name. alias, err = p.parseAnyKeyword() if err != nil { return nil, err } - case p.matchTokenKind(TokenKindIdent): - if terminator, err := p.isSelectItemTerminatorKeyword(); err != nil { - return nil, err - } else if terminator { - break - } + case p.matchTokenKind(TokenKindIdent) && !p.isSelectItemTerminatorKeyword(): // A bare alias can be a normal identifier or non-reserved keyword; a // reserved keyword here starts the next clause (e.g. `SELECT a FROM ...`). alias, err = p.parseIdent() @@ -1379,9 +1278,7 @@ func (p *Parser) parseColumnCaseExpr(pos Pos) (*CaseExpr, error) { whenClauses := make([]*WhenClause, 0) for p.matchKeyword(KeywordWhen) { whenPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() whenCondition, err := p.parseExpr(p.Pos()) if err != nil { return nil, err @@ -1407,9 +1304,7 @@ func (p *Parser) parseColumnCaseExpr(pos Pos) (*CaseExpr, error) { // ELSE expr elsePos := p.Pos() - if matched, consumeErr := p.tryConsumeKeywords(KeywordElse); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordElse) { elseExpr, err := p.parseExpr(p.Pos()) if err != nil { return nil, err @@ -1453,9 +1348,7 @@ func (p *Parser) parseColumnTypeArgs(ident *Ident) (ColumnType, error) { // noli return p.parseComplexType(ident, p.Pos()) } case p.matchTokenKind(TokenKindString): - if peekToken, err := p.lexer.peekToken(); err != nil { - return nil, err - } else if peekToken != nil && peekToken.Kind == TokenKindSingleEQ { + if peekToken, err := p.lexer.peekToken(); err == nil && peekToken.Kind == TokenKindSingleEQ { // enum values return p.parseEnumType(ident, p.Pos()) } @@ -1466,9 +1359,7 @@ func (p *Parser) parseColumnTypeArgs(ident *Ident) (ColumnType, error) { // noli return p.parseColumnTypeWithParams(ident, p.Pos()) case p.matchTokenKind(TokenKindRParen): rightParenPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &TypeWithParams{ Name: ident, LeftParenPos: lParen.Pos, @@ -1638,13 +1529,8 @@ func (p *Parser) parseJSONMaxDynamicOptions(pos Pos) (*JSONOption, error) { func (p *Parser) parseJSONOption() (*JSONOption, error) { switch { - case p.matchKeyword(KeywordSkip): - if err := p.expectKeyword(KeywordSkip); err != nil { - return nil, err - } - if matched, consumeErr := p.tryConsumeKeywords(KeywordRegexp); consumeErr != nil { - return nil, consumeErr - } else if matched { + case p.tryConsumeKeywords(KeywordSkip): + if p.tryConsumeKeywords(KeywordRegexp) { regex, err := p.parseString(p.Pos()) if err != nil { return nil, err @@ -1888,9 +1774,7 @@ func (p *Parser) parseNestedTypeFieldWithName() (Expr, error) { } func (p *Parser) tryParseCompressionCodecs(pos Pos) (*CompressionCodec, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordCodec); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordCodec) { return nil, nil // nolint } diff --git a/parser/parser_common.go b/parser/parser_common.go index b9bc018..1b72da1 100644 --- a/parser/parser_common.go +++ b/parser/parser_common.go @@ -146,37 +146,34 @@ func (p *Parser) expectKeyword(keyword string) error { Keyword: keyword, } } - return p.lexer.consumeToken() + _ = p.lexer.consumeToken() + return nil } -func (p *Parser) tryConsumeKeywords(keywords ...string) (bool, error) { +func (p *Parser) tryConsumeKeywords(keywords ...string) bool { savedState := p.lexer.saveState() for _, keyword := range keywords { if !p.matchKeyword(keyword) { p.lexer.restoreState(savedState) - return false, nil - } - if err := p.lexer.consumeToken(); err != nil { - return false, err + return false } + _ = p.lexer.consumeToken() } - return true, nil + return true } -func (p *Parser) tryParseIdent() (*Ident, error) { +func (p *Parser) tryParseIdent() *Ident { if p.currentTokenKind() != TokenKindIdent { - return nil, nil + return nil } curToken := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &Ident{ NamePos: curToken.Pos, NameEnd: curToken.End, Name: curToken.String, QuoteType: curToken.QuoteType, - }, nil + } } // parseAnyKeyword parses the current token as an identifier, accepting @@ -193,9 +190,7 @@ func (p *Parser) parseAnyKeyword() (*Ident, error) { Expected: []TokenKind{TokenKindIdent}, } } - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &Ident{ NamePos: last.Pos, NameEnd: last.End, @@ -224,9 +219,7 @@ func (p *Parser) parseIdentOrStar() (*Ident, error) { return p.parseIdent() case p.matchTokenKind("*"): curToken := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &Ident{ NamePos: curToken.Pos, NameEnd: curToken.End, @@ -243,9 +236,7 @@ func (p *Parser) parseIdentOrString() (*Ident, error) { return p.parseIdent() case p.matchTokenKind(TokenKindString): curToken := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() return &Ident{ NamePos: curToken.Pos, NameEnd: curToken.End, @@ -304,18 +295,14 @@ func (p *Parser) tryParseUUID() (*UUID, error) { } func (p *Parser) tryParseComment() (*StringLiteral, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordComment); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordComment) { return nil, nil } return p.parseString(p.Pos()) } func (p *Parser) tryParseIfExists() (bool, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordIf); consumeErr != nil { - return false, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordIf) { return false, nil } @@ -326,9 +313,7 @@ func (p *Parser) tryParseIfExists() (bool, error) { } func (p *Parser) tryParseIfNotExists() (bool, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordIf); consumeErr != nil { - return false, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordIf) { return false, nil } @@ -342,19 +327,15 @@ func (p *Parser) tryParseIfNotExists() (bool, error) { return true, nil } -func (p *Parser) tryParseNull(pos Pos) (*NullLiteral, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordNull); consumeErr != nil { - return nil, consumeErr - } else if !matched { - return nil, nil +func (p *Parser) tryParseNull(pos Pos) *NullLiteral { + if !p.tryConsumeKeywords(KeywordNull) { + return nil } - return &NullLiteral{NullPos: pos}, nil + return &NullLiteral{NullPos: pos} } func (p *Parser) tryParseNotNull(pos Pos) (*NotNullLiteral, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordNot); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordNot) { return nil, nil // nolint } notNull := &NotNullLiteral{NotPos: pos} @@ -388,9 +369,7 @@ func (p *Parser) parseNumber(pos Pos) (*NumberLiteral, error) { case p.matchTokenKind(TokenKindFloat): err = p.expectTokenKind(TokenKindFloat) case p.matchTokenKind(TokenKindDot): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() curToken = p.current() if err := p.expectTokenKind(TokenKindInt); err != nil { return nil, err @@ -492,13 +471,14 @@ func (p *Parser) parseFormat(pos Pos) (*FormatClause, error) { // captured position and expected-token information; the long tail of // fmt.Errorf sites is wrapped here with the current position. func (p *Parser) wrapError(err error) error { + // A grammar error can be a consequence of an ignored lexical failure. + // Report the original failure, including when lookahead restored the cursor. + if p.lexer.err != nil { + err = &ParseError{Pos: p.lexer.err.pos, Msg: p.lexer.err.Error()} + } if err == nil { return nil } - var lexicalErr *lexerError - if errors.As(err, &lexicalErr) { - err = &ParseError{Pos: lexicalErr.pos, Msg: lexicalErr.Error()} - } var pe *ParseError if !errors.As(err, &pe) { diff --git a/parser/parser_drop.go b/parser/parser_drop.go index a5ca752..dc06329 100644 --- a/parser/parser_drop.go +++ b/parser/parser_drop.go @@ -44,25 +44,15 @@ func (p *Parser) parseDropDatabase(pos Pos) (*DropDatabase, error) { } func (p *Parser) parseDropStmt(pos Pos) (*DropStmt, error) { - var err error var isTemporary bool dropTarget := KeywordTable switch { - case p.matchKeyword(KeywordDictionary): - if err := p.expectKeyword(KeywordDictionary); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDictionary): dropTarget = KeywordDictionary - case p.matchKeyword(KeywordView): - if err := p.expectKeyword(KeywordView); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordView): dropTarget = KeywordView default: - isTemporary, err = p.tryConsumeKeywords(KeywordTemporary) - if err != nil { - return nil, err - } + isTemporary = p.tryConsumeKeywords(KeywordTemporary) if err := p.expectKeyword(KeywordTable); err != nil { return nil, err } @@ -102,15 +92,9 @@ func (p *Parser) parseDropStmt(pos Pos) (*DropStmt, error) { func (p *Parser) tryParseModifier() (string, error) { switch { - case p.matchKeyword(KeywordSync): - if err := p.expectKeyword(KeywordSync); err != nil { - return "", err - } + case p.tryConsumeKeywords(KeywordSync): return "SYNC", nil - case p.matchKeyword(KeywordNo): - if err := p.expectKeyword(KeywordNo); err != nil { - return "", err - } + case p.tryConsumeKeywords(KeywordNo): if err := p.expectKeyword(KeywordDelay); err != nil { return "", err } diff --git a/parser/parser_query.go b/parser/parser_query.go index 105e263..4df0c8a 100644 --- a/parser/parser_query.go +++ b/parser/parser_query.go @@ -66,9 +66,7 @@ func (p *Parser) parseTopClause(pos Pos) (*TopClause, error) { topEnd := number.End() withTies := false - if matched, consumeErr := p.tryConsumeKeywords(KeywordWith); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordWith) { topEnd = p.End() if err := p.expectKeyword(KeywordTies); err != nil { return nil, err @@ -106,9 +104,7 @@ func (p *Parser) parseDistinctOn(pos Pos) (*DistinctOn, error) { idents := []*NestedIdentifier{col} for p.matchTokenKind(TokenKindComma) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() col, err = p.ParseNestedIdentifier(p.Pos()) if err != nil { @@ -152,10 +148,7 @@ func (p *Parser) parseFromClause(pos Pos) (*FromClause, error) { func (p *Parser) tryParseJoinConstraints(pos Pos) (Expr, error) { switch { - case p.matchKeyword(KeywordOn): - if err := p.expectKeyword(KeywordOn); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordOn): columnExprList, err := p.parseColumnExprList(p.Pos()) if err != nil { return nil, err @@ -164,10 +157,7 @@ func (p *Parser) tryParseJoinConstraints(pos Pos) (Expr, error) { OnPos: pos, On: columnExprList, }, nil - case p.matchKeyword(KeywordUsing): - if err := p.expectKeyword(KeywordUsing); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordUsing): lparen, consumeErr := p.tryConsumeTokenKind(TokenKindLParen) if consumeErr != nil { return nil, consumeErr @@ -190,121 +180,82 @@ func (p *Parser) tryParseJoinConstraints(pos Pos) (Expr, error) { return nil, nil } -func (p *Parser) parseJoinType(_ Pos) ([]string, error) { //nolint:funlen +func (p *Parser) parseJoinType(_ Pos) []string { var modifiers []string switch { - case p.matchKeyword(KeywordCross): // cross join - if err := p.expectKeyword(KeywordCross); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordCross): // cross join modifiers = append(modifiers, KeywordCross) case p.matchKeyword(KeywordAny), p.matchKeyword(KeywordAll): modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if p.matchKeyword(KeywordFull) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } if p.matchKeyword(KeywordLeft) || p.matchKeyword(KeywordRight) || p.matchKeyword(KeywordInner) || p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } case p.matchKeyword(KeywordSemi), p.matchKeyword(KeywordAsof): modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if p.matchKeyword(KeywordLeft) || p.matchKeyword(KeywordRight) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } if p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } case p.matchKeyword(KeywordInner): modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAsof) || p.matchKeyword(KeywordArray) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } case p.matchKeyword(KeywordLeft): modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } if p.matchKeyword(KeywordSemi) || p.matchKeyword(KeywordAnti) || p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAsof) || p.matchKeyword(KeywordArray) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } case p.matchKeyword(KeywordRight): modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } if p.matchKeyword(KeywordSemi) || p.matchKeyword(KeywordAnti) || p.matchKeyword(KeywordAny) || p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAsof) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } case p.matchKeyword(KeywordFull): modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if p.matchKeyword(KeywordOuter) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } if p.matchKeyword(KeywordAll) || p.matchKeyword(KeywordAny) { modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } case p.matchKeyword(KeywordArray): modifiers = append(modifiers, p.current().String) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } - return modifiers, nil + return modifiers } func (p *Parser) parseJoinTableExpr(_ Pos) (Expr, error) { @@ -319,9 +270,7 @@ func (p *Parser) parseJoinTableExpr(_ Pos) (Expr, error) { hasFinal := p.matchKeyword(KeywordFinal) if hasFinal { statementEnd = p.End() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } sampleRatio, err := p.tryParseSampleClause(p.Pos()) @@ -348,19 +297,14 @@ func (p *Parser) parseJoinTableExpr(_ Pos) (Expr, error) { // leaving the keyword for the caller to reject. func (p *Parser) parseJoinModifiers(pos Pos) ([]string, error) { if !p.matchOneOfKeywords(KeywordGlobal, KeywordLocal) { - return p.parseJoinType(pos) + return p.parseJoinType(pos), nil } savedState := p.lexer.saveState() locality := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() - joinType, err := p.parseJoinType(p.Pos()) - if err != nil { - return nil, err - } + joinType := p.parseJoinType(p.Pos()) if len(joinType) == 0 && !p.matchKeyword(KeywordJoin) { p.lexer.restoreState(savedState) return nil, nil @@ -385,22 +329,18 @@ func (p *Parser) parseJoinModifiers(pos Pos) ([]string, error) { // peekJoinAfterLocality reports whether the current GLOBAL/LOCAL keyword is // followed by a join operator, leaving the lexer where it found it. Expression // parsing uses it to tell a locality apart from the GLOBAL IN operator. -func (p *Parser) peekJoinAfterLocality() (bool, error) { +func (p *Parser) peekJoinAfterLocality() bool { savedState := p.lexer.saveState() defer p.lexer.restoreState(savedState) modifiers, err := p.parseJoinModifiers(p.Pos()) if err != nil { - var lexicalErr *lexerError - if errors.As(err, &lexicalErr) { - return false, err - } // A malformed locality join such as GLOBAL ARRAY JOIN still belongs // to the FROM clause, which reports the error. - return true, nil + return true } - return modifiers != nil && p.matchKeyword(KeywordJoin), nil + return modifiers != nil && p.matchKeyword(KeywordJoin) } func (p *Parser) parseJoinRightExpr(pos Pos) (expr Expr, err error) { @@ -425,9 +365,7 @@ func (p *Parser) parseJoinRightExpr(pos Pos) (expr Expr, err error) { if len(modifiers) != 0 && !p.matchKeyword(KeywordJoin) { return nil, fmt.Errorf("expected JOIN, got %s", p.currentTokenKind()) } - if matched, consumeErr := p.tryConsumeKeywords(KeywordJoin); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordJoin) { return nil, nil } @@ -531,9 +469,7 @@ func (p *Parser) parseTableExpr(pos Pos) (*TableExpr, error) { } tableEnd := expr.End() - if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAs) { // After AS the token can only be an alias name, so even reserved // keywords are accepted (e.g. `FROM t AS from`). alias, err := p.parseAnyKeyword() @@ -560,9 +496,7 @@ func (p *Parser) parseTableExpr(pos Pos) (*TableExpr, error) { } isFinalExist := false - if matched, consumeErr := p.tryConsumeKeywords(KeywordFinal); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordFinal) { switch expr.(type) { case *TableFunctionExpr: return nil, errors.New("table function doesn't support FINAL") @@ -644,26 +578,18 @@ func (p *Parser) parseGroupByClause(pos Pos) (*GroupByClause, error) { var err error var groupByEnd Pos aggregateType := "" - groupingSets, consumeErr := p.tryConsumeKeywords(KeywordGrouping, KeywordSets) - if consumeErr != nil { - return nil, consumeErr - } switch { - case groupingSets: - aggregateType = "GROUPING SETS" - expr, err = p.parseFunctionParams(p.Pos()) case p.matchKeyword(KeywordCube) || p.matchKeyword(KeywordRollup): aggregateType = p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() + expr, err = p.parseFunctionParams(p.Pos()) + case p.tryConsumeKeywords(KeywordGrouping, KeywordSets): + aggregateType = "GROUPING SETS" expr, err = p.parseFunctionParams(p.Pos()) case p.matchKeyword(KeywordAll): // GROUP BY ALL has no expression list; the clause ends at ALL itself groupByEnd = p.End() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() aggregateType = "ALL" default: expr, err = p.parseColumnExprListWithLParen(p.Pos()) @@ -682,32 +608,16 @@ func (p *Parser) parseGroupByClause(pos Pos) (*GroupByClause, error) { } // parse WITH CUBE, ROLLUP, TOTALS - for { - matched, consumeErr := p.tryConsumeKeywords(KeywordWith) - if consumeErr != nil { - return nil, consumeErr - } - if !matched { - break - } + for p.tryConsumeKeywords(KeywordWith) { // the clause now extends to the CUBE/ROLLUP/TOTALS token; capture its // end before it is consumed keywordEnd := p.End() switch { - case p.matchKeyword(KeywordCube): - if err := p.expectKeyword(KeywordCube); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordCube): groupBy.WithCube = true - case p.matchKeyword(KeywordRollup): - if err := p.expectKeyword(KeywordRollup); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordRollup): groupBy.WithRollup = true - case p.matchKeyword(KeywordTotals): - if err := p.expectKeyword(KeywordTotals); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordTotals): groupBy.WithTotals = true default: return nil, fmt.Errorf("expected CUBE, ROLLUP or TOTALS, got %s", p.currentTokenKind()) @@ -738,17 +648,13 @@ func (p *Parser) parseLimitClause(pos Pos) (*LimitClause, error) { var limit Expr var offset Expr var err error - if matched, consumeErr := p.tryConsumeKeywords(KeywordLimit); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordLimit) { limit, err = p.parseExpr(p.Pos()) if err != nil { return nil, err } - if matched, consumeErr := p.tryConsumeKeywords(KeywordOffset); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordOffset) { offset, err = p.parseExpr(p.Pos()) } else if token, consumeErr := p.tryConsumeTokenKind(TokenKindComma); consumeErr != nil { return nil, consumeErr @@ -756,9 +662,7 @@ func (p *Parser) parseLimitClause(pos Pos) (*LimitClause, error) { offset = limit limit, err = p.parseExpr(p.Pos()) } - } else if matched, consumeErr := p.tryConsumeKeywords(KeywordOffset); consumeErr != nil { - return nil, consumeErr - } else if matched { + } else if p.tryConsumeKeywords(KeywordOffset) { offset, err = p.parseExpr(p.Pos()) } @@ -816,9 +720,7 @@ func (p *Parser) parseLimitByClause(pos Pos) (Expr, error) { } var by *ColumnExprList - if matched, consumeErr := p.tryConsumeKeywords(KeywordBy); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordBy) { return limit, nil } if by, err = p.parseColumnExprListWithLParen(p.Pos()); err != nil { @@ -841,17 +743,13 @@ func (p *Parser) parseWindowFrameClause(pos Pos) (*WindowFrameClause, error) { var windowFrameType string if p.matchKeyword(KeywordRows) || p.matchKeyword(KeywordRange) { windowFrameType = p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } else { return nil, fmt.Errorf("expected ROWS or RANGE for window frame") } var expr Expr - if matched, consumeErr := p.tryConsumeKeywords(KeywordBetween); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordBetween) { left, err := p.parseFrameExtent() if err != nil { return nil, err @@ -905,9 +803,7 @@ func (p *Parser) parseFrameExtent() (Expr, error) { func (p *Parser) parseFrameCurrentRow() (Expr, error) { currentPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordRow); err != nil { return nil, err } @@ -920,9 +816,7 @@ func (p *Parser) parseFrameCurrentRow() (Expr, error) { func (p *Parser) parseFrameUnbounded() (Expr, error) { unboundedPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() direction, err := p.parseFrameDirection() if err != nil { @@ -989,9 +883,7 @@ func (p *Parser) parseFrameDirection() (string, error) { switch { case p.matchKeyword(KeywordPreceding), p.matchKeyword(KeywordFollowing): direction := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return "", err - } + _ = p.lexer.consumeToken() return direction, nil default: return "", fmt.Errorf("expected PRECEDING or FOLLOWING, got %s", p.currentTokenKind()) @@ -1004,9 +896,7 @@ func (p *Parser) parseFrameDirectionWithEnd() (string, Pos, error) { } endPos := p.End() direction := p.current().String - if err := p.lexer.consumeToken(); err != nil { - return "", 0, err - } + _ = p.lexer.consumeToken() return direction, endPos, nil } @@ -1022,9 +912,7 @@ func (p *Parser) parseWindowCondition(pos Pos) (*WindowExpr, error) { return nil, err } var windowName *Ident - if name, err := p.canParseWindowNameInParens(); err != nil { - return nil, err - } else if name { + if p.canParseWindowNameInParens() { // canParseWindowNameInParens already disambiguated keyword tokens // (e.g. `OVER (order)` vs `OVER (ORDER BY ...)`). var err error @@ -1059,12 +947,12 @@ func (p *Parser) parseWindowCondition(pos Pos) (*WindowExpr, error) { }, nil } -func (p *Parser) canParseWindowNameInParens() (bool, error) { +func (p *Parser) canParseWindowNameInParens() bool { if !p.matchTokenKind(TokenKindIdent, TokenKindKeyword) { - return false, nil + return false } if !p.matchTokenKind(TokenKindKeyword) { - return true, nil + return true } savedState := p.lexer.saveState() @@ -1072,22 +960,18 @@ func (p *Parser) canParseWindowNameInParens() (bool, error) { switch { case p.matchKeyword(KeywordPartition), p.matchKeyword(KeywordOrder): - if err := p.lexer.consumeToken(); err != nil { - return false, err - } - return !p.matchKeyword(KeywordBy), nil + _ = p.lexer.consumeToken() + return !p.matchKeyword(KeywordBy) case p.matchKeyword(KeywordRows), p.matchKeyword(KeywordRange): - if err := p.lexer.consumeToken(); err != nil { - return false, err - } + _ = p.lexer.consumeToken() return !p.matchKeyword(KeywordBetween) && !p.matchKeyword(KeywordCurrent) && !p.matchKeyword(KeywordUnbounded) && !p.matchTokenKind(TokenKindInt) && !p.matchTokenKind(TokenKindLBrace) && - !p.matchKeyword(KeywordInterval), nil + !p.matchKeyword(KeywordInterval) default: - return true, nil + return true } } @@ -1256,24 +1140,15 @@ func (p *Parser) parseSelectQuery(_ Pos) (*SelectQuery, error) { // parseSelectQuery, so at most one operator is bound per call. func (p *Parser) parseSetOperation(selectStmt *SelectQuery) error { switch { - case p.matchKeyword(KeywordUnion): - if err := p.expectKeyword(KeywordUnion); err != nil { - return err - } + case p.tryConsumeKeywords(KeywordUnion): switch { - case p.matchKeyword(KeywordAll): - if err := p.expectKeyword(KeywordAll); err != nil { - return err - } + case p.tryConsumeKeywords(KeywordAll): unionAllExpr, err := p.parseSelectQuery(p.Pos()) if err != nil { return err } selectStmt.UnionAll = unionAllExpr - case p.matchKeyword(KeywordDistinct): - if err := p.expectKeyword(KeywordDistinct); err != nil { - return err - } + case p.tryConsumeKeywords(KeywordDistinct): unionDistinctExpr, err := p.parseSelectQuery(p.Pos()) if err != nil { return err @@ -1282,19 +1157,13 @@ func (p *Parser) parseSetOperation(selectStmt *SelectQuery) error { default: return fmt.Errorf("expected ALL or DISTINCT, got %s", p.currentTokenKind()) } - case p.matchKeyword(KeywordExcept): - if err := p.expectKeyword(KeywordExcept); err != nil { - return err - } + case p.tryConsumeKeywords(KeywordExcept): exceptExpr, err := p.parseSelectQuery(p.Pos()) if err != nil { return err } selectStmt.Except = exceptExpr - case p.matchKeyword(KeywordIntersect): - if err := p.expectKeyword(KeywordIntersect); err != nil { - return err - } + case p.tryConsumeKeywords(KeywordIntersect): intersectExpr, err := p.parseSelectQuery(p.Pos()) if err != nil { return err @@ -1314,10 +1183,7 @@ func (p *Parser) parseSelectStmt(pos Pos) (*SelectQuery, error) { // nolint: fun return nil, err } // DISTINCT? - hasDistinct, consumeErr := p.tryConsumeKeywords(KeywordDistinct) - if consumeErr != nil { - return nil, consumeErr - } + hasDistinct := p.tryConsumeKeywords(KeywordDistinct) distinctOn, err := p.tryParseDistinctOn(p.Pos()) if err != nil { return nil, err @@ -1366,9 +1232,7 @@ func (p *Parser) parseSelectStmt(pos Pos) (*SelectQuery, error) { // nolint: fun statementEnd = groupBy.End() } withTotal := false - if matched, consumeErr := p.tryConsumeKeywords(KeywordWith); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordWith) { // the statement now ends at the TOTALS token; capture its end before // expectKeyword consumes it totalsEnd := p.End() @@ -1525,9 +1389,7 @@ func (p *Parser) parseSampleClause(pos Pos) (*SampleClause, error) { var offset *RatioExpr if p.matchKeyword(KeywordOffset) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() offset, err = p.parseRatioExpr(p.Pos()) if err != nil { return nil, err @@ -1553,9 +1415,7 @@ func (p *Parser) parseExplainStmt(pos Pos) (*ExplainStmt, error) { p.matchKeyword(KeywordEstimate), p.matchKeyword(KeywordAst): explainType = p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() default: return nil, fmt.Errorf("expected SYNTAX, PIPELINE, ESTIMATE or AST, got %s", p.currentTokenKind()) } diff --git a/parser/parser_table.go b/parser/parser_table.go index 0530826..6d962c5 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -11,13 +11,8 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { case p.matchKeyword(KeywordCreate), p.matchKeyword(KeywordAttach): isAttach := p.matchKeyword(KeywordAttach) - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } - orReplace, consumeErr := p.tryConsumeKeywords(KeywordOr, KeywordReplace) - if consumeErr != nil { - return nil, consumeErr - } + _ = p.lexer.consumeToken() + orReplace := p.tryConsumeKeywords(KeywordOr, KeywordReplace) if orReplace { // MATERIALIZED VIEW accepts OR REPLACE only under CREATE; // ClickHouse rejects an ATTACH OR REPLACE combination. @@ -56,9 +51,7 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { p.currentTokenKind()) } case p.matchKeyword(KeywordAlter): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() switch { case p.matchKeyword(KeywordRole): return p.parseAlterRole(pos) @@ -69,9 +62,7 @@ func (p *Parser) parseDDL(pos Pos) (DDL, error) { } case p.matchKeyword(KeywordDrop), p.matchKeyword(KeywordDetach): - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() switch { case p.matchKeyword(KeywordDatabase): return p.parseDropDatabase(pos) @@ -315,16 +306,12 @@ func (p *Parser) parseNamedCollectionParam(pos Pos) (*NamedCollectionParam, erro } // Parse optional [NOT] OVERRIDABLE clause - if matched, consumeErr := p.tryConsumeKeywords(KeywordNot); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordNot) { param.NotOverridable = true if err := p.expectKeyword(KeywordOverridable); err != nil { return nil, err } - } else if matched, consumeErr := p.tryConsumeKeywords(KeywordOverridable); consumeErr != nil { - return nil, consumeErr - } else if matched { + } else if p.tryConsumeKeywords(KeywordOverridable) { param.Overridable = true } @@ -332,18 +319,15 @@ func (p *Parser) parseNamedCollectionParam(pos Pos) (*NamedCollectionParam, erro } func (p *Parser) parseCreateTable(pos Pos, orReplace bool) (*CreateTable, error) { - var err error createTable := &CreateTable{CreatePos: pos, OrReplace: orReplace} - createTable.HasTemporary, err = p.tryConsumeKeywords(KeywordTemporary) - if err != nil { - return nil, err - } + createTable.HasTemporary = p.tryConsumeKeywords(KeywordTemporary) if err := p.expectKeyword(KeywordTable); err != nil { return nil, err } // parse IF NOT EXISTS clause if exists + var err error createTable.IfNotExists, err = p.tryParseIfNotExists() if err != nil { return nil, err @@ -383,9 +367,7 @@ func (p *Parser) parseCreateTable(pos Pos, orReplace bool) (*CreateTable, error) createTable.StatementEnd = engineExpr.End() } - if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAs) { // After AS, we can have: SELECT/WITH (with or without parens), or table_function(...) // Check if it's a SELECT/WITH query (explicitly check keywords/paren before ident) if p.matchKeyword(KeywordSelect) || p.matchKeyword(KeywordWith) || p.matchTokenKind(TokenKindLParen) { @@ -432,14 +414,8 @@ func (p *Parser) parseCreateTable(pos Pos, orReplace bool) (*CreateTable, error) func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { var ident *Ident var err error - keywordName := false - if p.matchTokenKind(TokenKindKeyword) { - keywordName, err = p.peekTokenKind(TokenKindLParen, TokenKindDot) - if err != nil { - return nil, err - } - } - if keywordName { + if p.matchTokenKind(TokenKindKeyword) && + (p.peekTokenKind(TokenKindLParen) || p.peekTokenKind(TokenKindDot)) { // Reserved keywords remain valid when context proves they are function // names or the first field of a qualified name. ident, err = p.parseAnyKeyword() @@ -475,9 +451,7 @@ func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { } overPos := p.Pos() - if matched, consumeErr := p.tryConsumeKeywords(KeywordOver); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordOver) { var overExpr Expr switch { case p.matchTokenKind(TokenKindIdent, TokenKindKeyword): @@ -504,15 +478,8 @@ func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { }, nil } return funcExpr, nil - case p.matchTokenKind(TokenKindDot): - if qualifier, err := p.peekTokenKind(TokenKindColon); err != nil { - return nil, err - } else if qualifier { - return ident, nil - } - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + case p.matchTokenKind(TokenKindDot) && !p.peekTokenKind(TokenKindColon): + _ = p.lexer.consumeToken() switch { case p.matchTokenKind(TokenKindIdent, TokenKindKeyword): fields := []*Ident{ident} @@ -524,17 +491,10 @@ func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { return nil, err } fields = append(fields, child) - if !p.matchTokenKind(TokenKindDot) { + if !p.matchTokenKind(TokenKindDot) || p.peekTokenKind(TokenKindColon) { break } - if qualifier, err := p.peekTokenKind(TokenKindColon); err != nil { - return nil, err - } else if qualifier { - break - } - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } return &Path{Fields: fields}, nil case p.matchTokenKind("*"): @@ -605,21 +565,11 @@ func (p *Parser) parseTableSchemaClause(pos Pos) (*TableSchemaClause, error) { SchemaEnd: rightParenPos, Columns: columns, }, nil - case p.matchKeyword(KeywordAs): - next, err := p.lexer.peekToken() - if err != nil { - return nil, err - } - if next != nil && (next.Kind == TokenKindLParen || - next.Kind == TokenKindKeyword && (strings.EqualFold(next.String, KeywordSelect) || strings.EqualFold(next.String, KeywordWith))) { - return nil, nil - } + case p.matchKeyword(KeywordAs) && !p.peekKeyword(KeywordSelect) && !p.peekKeyword(KeywordWith) && !p.peekTokenKind(TokenKindLParen): // Handle AS only if followed by identifier (not SELECT/WITH/LPAREN) // This handles: AS ident, AS ident.ident, AS ident(...) // CREATE TABLE will handle: AS SELECT, AS WITH, AS (SELECT ...) - if _, err := p.tryConsumeKeywords(KeywordAs); err != nil { - return nil, err - } + p.tryConsumeKeywords(KeywordAs) ident, err := p.parseIdent() if err != nil { @@ -674,9 +624,7 @@ func (p *Parser) parseTableColumns() ([]Expr, error) { switch { case p.matchKeyword(KeywordIndex): indexPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() index, err := p.parseTableIndex(indexPos) if err != nil { return nil, err @@ -684,9 +632,7 @@ func (p *Parser) parseTableColumns() ([]Expr, error) { columns = append(columns, index) case p.matchKeyword(KeywordConstraint): constraintPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() ident, err := p.parseIdent() if err != nil { return nil, err @@ -756,10 +702,7 @@ func (p *Parser) parseTableColumnExpr(pos Pos) (*ColumnDef, error) { columnEnd = columnType.End() } - nullable, err := p.tryParseNull(p.Pos()) - if err != nil { - return nil, err - } + nullable := p.tryParseNull(p.Pos()) if nullable != nil { columnEnd = nullable.End() } @@ -773,22 +716,13 @@ func (p *Parser) parseTableColumnExpr(pos Pos) (*ColumnDef, error) { var valueExpr Expr switch { - case p.matchKeyword(KeywordDefault): - if err := p.expectKeyword(KeywordDefault); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDefault): column.DefaultExpr, err = p.parseExpr(p.Pos()) valueExpr = column.DefaultExpr - case p.matchKeyword(KeywordMaterialized): - if err := p.expectKeyword(KeywordMaterialized); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordMaterialized): column.MaterializedExpr, err = p.parseExpr(p.Pos()) valueExpr = column.MaterializedExpr - case p.matchKeyword(KeywordAlias): - if err := p.expectKeyword(KeywordAlias); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordAlias): column.AliasExpr, err = p.parseExpr(p.Pos()) valueExpr = column.AliasExpr } @@ -880,9 +814,7 @@ func (p *Parser) parseTableArgPrimaryExpr(pos Pos) (Expr, error) { // a leading '(' opens a subquery only when SELECT or WITH follows, // e.g. remote('127.0.0.1', (SELECT 1)); anything else is a // parenthesized expression, e.g. numbers((1 + 1)) - if query, err := p.peekKeyword(KeywordSelect, KeywordWith); err != nil { - return nil, err - } else if query { + if p.peekKeyword(KeywordSelect) || p.peekKeyword(KeywordWith) { return p.parseSubQuery(p.Pos()) } @@ -913,10 +845,7 @@ func (p *Parser) parseTableArgList(pos Pos) (*TableArgListExpr, error) { // Last token is an identifier, peek at the next token nextToken, peekErr := p.lexer.peekToken() - if peekErr != nil { - return nil, peekErr - } - if nextToken != nil && nextToken.Kind == TokenKindSingleEQ { + if peekErr == nil && nextToken != nil && nextToken.Kind == TokenKindSingleEQ { isNamedParam = true } } @@ -975,9 +904,7 @@ func (p *Parser) parseTableArgList(pos Pos) (*TableArgListExpr, error) { } func (p *Parser) tryParseClusterClause(pos Pos) (*ClusterClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordOn); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordOn) { return nil, nil // nolint } if err := p.expectKeyword(KeywordCluster); err != nil { @@ -1004,9 +931,7 @@ func (p *Parser) tryParseClusterClause(pos Pos) (*ClusterClause, error) { } func (p *Parser) tryParsePartitionByClause(pos Pos) (*PartitionByClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordPartition); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordPartition) { return nil, nil // nolint } @@ -1026,9 +951,7 @@ func (p *Parser) tryParsePartitionByClause(pos Pos) (*PartitionByClause, error) } func (p *Parser) tryParsePrimaryKeyClause(pos Pos) (*PrimaryKeyClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordPrimary); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordPrimary) { return nil, nil // nolint } @@ -1048,9 +971,7 @@ func (p *Parser) tryParsePrimaryKeyClause(pos Pos) (*PrimaryKeyClause, error) { } func (p *Parser) tryParseOrderByClause(pos Pos) (*OrderByClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordOrder); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordOrder) { return nil, nil // nolint } @@ -1064,9 +985,7 @@ func (p *Parser) tryParseOrderByClause(pos Pos) (*OrderByClause, error) { // the sorting key. Unlike the ORDER BY of a query it holds a single expression: // a sorting key over several columns is written as a tuple. func (p *Parser) tryParseTableOrderByClause(pos Pos) (*OrderByClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordOrder); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordOrder) { return nil, nil // nolint } @@ -1115,9 +1034,7 @@ func (p *Parser) parseOrderByClause(pos Pos) (*OrderByClause, error) { // Parse optional INTERPOLATE clause if p.matchKeyword(KeywordInterpolate) { interpolatePos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() interpolate, err := p.parseInterpolateClause(interpolatePos) if err != nil { return nil, err @@ -1139,18 +1056,14 @@ func (p *Parser) parseOrderExpr(pos Pos) (*OrderExpr, error) { var alias *Ident if p.matchKeyword(KeywordAs) { // It should be a subquery instead of an order by alias if the `AS` is followed by `SELECT` keyword. - if nextToken, err := p.lexer.peekToken(); err != nil { - return nil, err - } else if nextToken != nil && nextToken.ToString() == KeywordSelect { + if nextToken, err := p.lexer.peekToken(); err == nil && nextToken.ToString() == KeywordSelect { return &OrderExpr{ OrderPos: pos, Expr: columnExpr, }, nil } // consume the `AS` keyword - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() // after AS the token can only be an alias name, reserved keyword or not alias, err = p.parseAnyKeyword() if err != nil { @@ -1167,21 +1080,15 @@ func (p *Parser) parseOrderExpr(pos Pos) (*OrderExpr, error) { switch { case p.matchKeyword(KeywordAsc), p.matchKeyword(KeywordAscending): direction = OrderDirectionAsc - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() case p.matchKeyword(KeywordDesc), p.matchKeyword(KeywordDescending): direction = OrderDirectionDesc - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } // Parse optional WITH FILL clause var fill *Fill - if matched, consumeErr := p.tryConsumeKeywords(KeywordWith, KeywordFill); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordWith, KeywordFill) { fillPos := p.Pos() fill, err = p.parseFillClause(fillPos) if err != nil { @@ -1202,9 +1109,7 @@ func (p *Parser) parseFillClause(fillPos Pos) (*Fill, error) { fill := &Fill{FillPos: fillPos} // Parse optional FROM clause - if matched, consumeErr := p.tryConsumeKeywords(KeywordFrom); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordFrom) { fromExpr, err := p.parseExpr(fillPos) if err != nil { return nil, err @@ -1213,9 +1118,7 @@ func (p *Parser) parseFillClause(fillPos Pos) (*Fill, error) { } // Parse optional TO clause - if matched, consumeErr := p.tryConsumeKeywords(KeywordTo); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordTo) { toExpr, err := p.parseExpr(fillPos) if err != nil { return nil, err @@ -1224,9 +1127,7 @@ func (p *Parser) parseFillClause(fillPos Pos) (*Fill, error) { } // Parse optional STEP clause - if matched, consumeErr := p.tryConsumeKeywords(KeywordStep); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordStep) { stepExpr, err := p.parseExpr(fillPos) if err != nil { return nil, err @@ -1235,9 +1136,7 @@ func (p *Parser) parseFillClause(fillPos Pos) (*Fill, error) { } // Parse optional STALENESS clause - if matched, consumeErr := p.tryConsumeKeywords(KeywordStaleness); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordStaleness) { stalenessExpr, err := p.parseExpr(fillPos) if err != nil { return nil, err @@ -1270,9 +1169,7 @@ func (p *Parser) parseInterpolateClause(interpolatePos Pos) (*InterpolateClause, item := &InterpolateItem{Column: column} - if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAs) { expr, err := p.parseExpr(interpolatePos) if err != nil { return nil, err @@ -1304,9 +1201,7 @@ func (p *Parser) parseInterpolateClause(interpolatePos Pos) (*InterpolateClause, } func (p *Parser) tryParseTTLClause(pos Pos, allowMultiValues bool) (*TTLClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordTtl); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordTtl) { return nil, nil // nolint } ttlExprList := &TTLClause{TTLPos: pos, ListEnd: pos} @@ -1352,21 +1247,14 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { var rule *TTLPolicyRule var where *WhereClause switch { - case p.matchKeyword(KeywordTo): - if err := p.expectKeyword(KeywordTo); err != nil { - return nil, err - } - if matched, consumeErr := p.tryConsumeKeywords(KeywordDisk); consumeErr != nil { - return nil, consumeErr - } else if matched { + case p.tryConsumeKeywords(KeywordTo): + if p.tryConsumeKeywords(KeywordDisk) { value, err := p.parseString(p.Pos()) if err != nil { return nil, err } rule = &TTLPolicyRule{RulePos: pos, ToDisk: value} - } else if matched, consumeErr := p.tryConsumeKeywords(KeywordVolume); consumeErr != nil { - return nil, consumeErr - } else if matched { + } else if p.tryConsumeKeywords(KeywordVolume) { value, err := p.parseString(p.Pos()) if err != nil { return nil, err @@ -1378,9 +1266,7 @@ func (p *Parser) tryParseTTLPolicy(pos Pos) (*TTLPolicy, error) { case p.matchKeyword(KeywordDelete), p.matchKeyword(KeywordRecompress): isDelete := p.matchKeyword(KeywordDelete) token := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() action := &TTLPolicyRuleAction{ ActionPos: token.Pos, ActionEnd: token.End, @@ -1442,10 +1328,6 @@ func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { savedState := p.lexer.saveState() key, err = p.parseExpr(p.Pos()) if err != nil { - var lexicalErr *lexerError - if errors.As(err, &lexicalErr) { - return nil, err - } p.lexer.restoreState(savedState) key, err = p.parseAnyKeyword() } @@ -1471,9 +1353,7 @@ func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { Expr: keys, }, } - if matched, consumeErr := p.tryConsumeKeywords(KeywordSet); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordSet) { set, err := p.parseTTLPolicySet(p.Pos()) if err != nil { return nil, err @@ -1493,10 +1373,6 @@ func (p *Parser) parseTTLPolicyGroupBy(pos Pos) (*TTLPolicyRule, error) { } set, err := p.parseTTLPolicySet(p.Pos()) if err != nil { - var lexicalErr *lexerError - if errors.As(err, &lexicalErr) { - return nil, err - } p.lexer.restoreState(savedState) break } @@ -1547,9 +1423,7 @@ func (p *Parser) parseTTLExpr(pos Pos) (*TTLExpr, error) { } func (p *Parser) tryParseSampleByClause(pos Pos) (*SampleByClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordSample); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordSample) { return nil, nil // nolint } @@ -1569,9 +1443,7 @@ func (p *Parser) tryParseSampleByClause(pos Pos) (*SampleByClause, error) { } func (p *Parser) tryParseSettingsClause(pos Pos) (*SettingsClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordSettings); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordSettings) { return nil, nil // nolint } return p.parseSettingsClause(pos) @@ -1649,9 +1521,7 @@ func (p *Parser) parseSettingsExpr(pos Pos) (*SettingExpr, error) { case p.matchKeyword(KeywordTrue), p.matchKeyword(KeywordFalse): // Handle TRUE/FALSE keywords as boolean literals curToken := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() expr = &BoolLiteral{ LiteralPos: curToken.Pos, LiteralEnd: curToken.End, @@ -1887,9 +1757,7 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { case p.matchKeyword(KeywordCreate): // SHOW CREATE TABLE table_name showType = "CREATE" - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordTable); err != nil { return nil, err @@ -1905,16 +1773,12 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { case p.matchKeyword(KeywordDatabases): // SHOW DATABASES [optional clauses] showType = "DATABASES" - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() case p.matchKeyword(KeywordTables): // SHOW TABLES showType = "TABLES" - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() default: return nil, fmt.Errorf("expected CREATE, DATABASES, or TABLES after SHOW, got %q", p.currentTokenString()) @@ -1931,9 +1795,7 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { // Parse [[NOT] LIKE | ILIKE ''] if p.matchKeyword(KeywordNot) { stmt.NotLike = true - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } if p.matchKeyword(KeywordLike) || p.matchKeyword(KeywordIlike) { @@ -1942,9 +1804,7 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { } else { stmt.LikeType = "ILIKE" } - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() // Parse pattern expression pattern, err := p.parseExpr(p.Pos()) @@ -1956,9 +1816,7 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { // Parse [LIMIT ] if p.matchKeyword(KeywordLimit) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() limit, err := p.parseExpr(p.Pos()) if err != nil { return nil, err @@ -1968,9 +1826,7 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { // Parse [INTO OUTFILE filename] if p.matchKeyword(KeywordInto) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() if err := p.expectKeyword(KeywordOutfile); err != nil { return nil, err } @@ -1985,9 +1841,7 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { // Parse [FORMAT format] if p.matchKeyword(KeywordFormat) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() // Format can be an identifier or a string if p.matchTokenKind(TokenKindString) { @@ -1999,9 +1853,7 @@ func (p *Parser) parseShowStmt(pos Pos) (*ShowStmt, error) { } else if p.matchTokenKind(TokenKindIdent) { // Handle format as identifier (like JSON, CSV, etc.) token := p.current() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() stmt.Format = &StringLiteral{ LiteralPos: token.Pos, LiteralEnd: token.End, @@ -2024,16 +1876,12 @@ func (p *Parser) parseDescribeStmt(pos Pos) (*DescribeStmt, error) { if !p.matchKeyword(KeywordDesc) && !p.matchKeyword(KeywordDescribe) { return nil, fmt.Errorf("expected DESC or DESCRIBE") } - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() // TABLE keyword is optional after DESC/DESCRIBE var describeType string if p.matchKeyword(KeywordTable) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() describeType = "TABLE" } @@ -2056,10 +1904,7 @@ func (p *Parser) parseTruncateTable(pos Pos) (*TruncateTable, error) { return nil, err } - isTemporary, consumeErr := p.tryConsumeKeywords(KeywordTemporary) - if consumeErr != nil { - return nil, consumeErr - } + isTemporary := p.tryConsumeKeywords(KeywordTemporary) if err := p.expectKeyword(KeywordTable); err != nil { return nil, err @@ -2241,7 +2086,6 @@ func (p *Parser) parseAssignmentValues(pos Pos) (*AssignmentValues, error) { } func (p *Parser) parseInsertStmt(pos Pos) (*InsertStmt, error) { - var err error if err := p.expectKeyword(KeywordInsert); err != nil { return nil, err } @@ -2250,15 +2094,11 @@ func (p *Parser) parseInsertStmt(pos Pos) (*InsertStmt, error) { } insertExpr := &InsertStmt{InsertPos: pos} - insertExpr.HasTableKeyword, err = p.tryConsumeKeywords(KeywordTable) - if err != nil { - return nil, err - } + insertExpr.HasTableKeyword = p.tryConsumeKeywords(KeywordTable) var table Expr - if matched, consumeErr := p.tryConsumeKeywords(KeywordFunction); consumeErr != nil { - return nil, consumeErr - } else if matched { + var err error + if p.tryConsumeKeywords(KeywordFunction) { table, err = p.parseFunctionExpr(p.Pos()) } else { table, err = p.parseTableIdentifier(p.Pos()) @@ -2281,9 +2121,7 @@ func (p *Parser) parseInsertStmt(pos Pos) (*InsertStmt, error) { insertExpr.Format, err = p.parseFormat(p.Pos()) case p.matchKeyword(KeywordValues): // consume VALUES keyword - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() values := make([]*AssignmentValues, 0) for !p.lexer.isEOF() { value, err := p.parseAssignmentValues(p.Pos()) @@ -2317,15 +2155,9 @@ func (p *Parser) parseRenameStmt(pos Pos) (*RenameStmt, error) { renameTarget := KeywordTable switch { - case p.matchKeyword(KeywordDictionary): - if err := p.expectKeyword(KeywordDictionary); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDictionary): renameTarget = KeywordDictionary - case p.matchKeyword(KeywordDatabase): - if err := p.expectKeyword(KeywordDatabase); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDatabase): renameTarget = KeywordDatabase default: if err := p.expectKeyword(KeywordTable); err != nil { @@ -2500,10 +2332,7 @@ func (p *Parser) parseDictionaryAttribute(pos Pos) (*DictionaryAttribute, error) // properties (HIERARCHICAL, ...) end at the keyword itself keywordEnd := p.End() switch { - case p.matchKeyword(KeywordDefault): - if err := p.expectKeyword(KeywordDefault); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordDefault): if attr.Default != nil { return nil, fmt.Errorf("duplicate DEFAULT clause") } @@ -2513,10 +2342,7 @@ func (p *Parser) parseDictionaryAttribute(pos Pos) (*DictionaryAttribute, error) } attr.Default = literal attr.AttrEnd = literal.End() - case p.matchKeyword(KeywordExpression): - if err := p.expectKeyword(KeywordExpression); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordExpression): if attr.Expression != nil { return nil, fmt.Errorf("duplicate EXPRESSION clause") } @@ -2526,28 +2352,19 @@ func (p *Parser) parseDictionaryAttribute(pos Pos) (*DictionaryAttribute, error) } attr.Expression = expr attr.AttrEnd = expr.End() - case p.matchKeyword(KeywordHierarchical): - if err := p.expectKeyword(KeywordHierarchical); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordHierarchical): if attr.Hierarchical { return nil, fmt.Errorf("duplicate HIERARCHICAL clause") } attr.Hierarchical = true attr.AttrEnd = keywordEnd - case p.matchKeyword(KeywordInjective): - if err := p.expectKeyword(KeywordInjective); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordInjective): if attr.Injective { return nil, fmt.Errorf("duplicate INJECTIVE clause") } attr.Injective = true attr.AttrEnd = keywordEnd - case p.matchKeyword(KeywordIs_object_id): - if err := p.expectKeyword(KeywordIs_object_id); err != nil { - return nil, err - } + case p.tryConsumeKeywords(KeywordIs_object_id): if attr.IsObjectId { return nil, fmt.Errorf("duplicate IS_OBJECT_ID clause") } @@ -2719,9 +2536,7 @@ func (p *Parser) parseDictionaryArgExpr(pos Pos) (*DictionaryArgExpr, error) { // HEADERS(HEADER(NAME 'API-KEY' VALUE 'key')). if p.matchTokenKind(TokenKindLParen) { lParenPos := p.Pos() - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() args, err := p.parseDictionaryArgExprs() if err != nil { @@ -2764,9 +2579,7 @@ func (p *Parser) parseDictionaryArgExpr(pos Pos) (*DictionaryArgExpr, error) { } // Check if it's followed by optional parentheses if p.matchTokenKind(TokenKindLParen) { - if err := p.lexer.consumeToken(); err != nil { // consume ( - return nil, err - } + _ = p.lexer.consumeToken() // consume ( if err := p.expectTokenKind(TokenKindRParen); err != nil { return nil, err } @@ -2803,9 +2616,7 @@ func (p *Parser) parseDictionaryLifetimeClause(pos Pos) (*DictionaryLifetimeClau // Check for MIN/MAX form if p.matchKeyword(KeywordMin) || p.matchKeyword(KeywordMax) { isMinFirst := p.matchKeyword(KeywordMin) - if err := p.lexer.consumeToken(); err != nil { // consume MIN or MAX - return nil, err - } + _ = p.lexer.consumeToken() // consume MIN or MAX first, err := p.parseNumber(p.Pos()) if err != nil { diff --git a/parser/parser_view.go b/parser/parser_view.go index 8677b51..effbb9c 100644 --- a/parser/parser_view.go +++ b/parser/parser_view.go @@ -53,18 +53,14 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } createMaterializedView.Refresh = refreshExpr - if matched, consumeErr := p.tryConsumeKeywords(KeywordRandomize, KeywordFor); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordRandomize, KeywordFor) { randomizeFor, err := p.parseInterval(false) if err != nil { return nil, err } createMaterializedView.RandomizeFor = randomizeFor } - if matched, consumeErr := p.tryConsumeKeywords(KeywordDepends, KeywordOn); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordDepends, KeywordOn) { dependsOnTables := make([]*TableIdentifier, 0) table, err := p.parseTableIdentifier(p.Pos()) if err != nil { @@ -72,9 +68,7 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } dependsOnTables = append(dependsOnTables, table) for p.matchTokenKind(TokenKindComma) { - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() table, err := p.parseTableIdentifier(p.Pos()) if err != nil { return nil, err @@ -88,10 +82,7 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa return nil, err } createMaterializedView.Settings = settings - createMaterializedView.HasAppend, err = p.tryConsumeKeywords(KeywordAppend) - if err != nil { - return nil, err - } + createMaterializedView.HasAppend = p.tryConsumeKeywords(KeywordAppend) switch { case p.matchKeyword(KeywordTo): @@ -134,15 +125,10 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa default: return nil, fmt.Errorf("unexpected token: %q, expected TO or ENGINE", p.currentTokenKind()) } - createMaterializedView.HasEmpty, err = p.tryConsumeKeywords(KeywordEmpty) - if err != nil { - return nil, err - } + createMaterializedView.HasEmpty = p.tryConsumeKeywords(KeywordEmpty) // Parse DEFINER clause - if matched, consumeErr := p.tryConsumeKeywords(KeywordDefiner); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordDefiner) { if err := p.expectTokenKind(TokenKindSingleEQ); err != nil { return nil, err } @@ -154,22 +140,16 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } // Parse SQL SECURITY clause - if matched, consumeErr := p.tryConsumeKeywords(KeywordSQL, KeywordSecurity); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordSQL, KeywordSecurity) { if !p.matchOneOfKeywords(KeywordDefiner, KeywordNone) { return nil, fmt.Errorf("expected DEFINER or NONE after SQL SECURITY, got %q", p.currentTokenKind()) } createMaterializedView.SQLSecurity = p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() } // Check for POPULATE before AS SELECT - only valid with ENGINE and no Destination - if matched, consumeErr := p.tryConsumeKeywords(KeywordPopulate); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordPopulate) { if createMaterializedView.Destination != nil { return nil, fmt.Errorf("POPULATE is only allowed when using ENGINE, not with TO clause") } @@ -189,9 +169,7 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } createMaterializedView.Comment = comment - if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAs) { subQuery, err := p.parseSubQuery(p.Pos()) if err != nil { return nil, err @@ -212,9 +190,7 @@ func (p *Parser) parseCreateMaterializedView(pos Pos, orReplace bool) (*CreateMa } func (p *Parser) tryParseRefreshExpr(pos Pos) (*RefreshExpr, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordRefresh); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordRefresh) { return nil, nil // nolint } @@ -224,9 +200,7 @@ func (p *Parser) tryParseRefreshExpr(pos Pos) (*RefreshExpr, error) { return nil, fmt.Errorf("expected EVERY or AFTER, but got %q", p.currentTokenKind()) } refreshExpr.Frequency = p.current().String - if err := p.lexer.consumeToken(); err != nil { - return nil, err - } + _ = p.lexer.consumeToken() interval, err := p.parseInterval(false) if err != nil { @@ -235,9 +209,7 @@ func (p *Parser) tryParseRefreshExpr(pos Pos) (*RefreshExpr, error) { refreshExpr.Interval = interval // [OFFSET interval] - if matched, consumeErr := p.tryConsumeKeywords(KeywordOffset); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordOffset) { offset, err := p.parseInterval(false) if err != nil { return nil, err @@ -294,9 +266,7 @@ func (p *Parser) parseCreateView(pos Pos, orReplace bool) (*CreateView, error) { } createView.Comment = comment - if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAs) { subQuery, err := p.parseSubQuery(p.Pos()) if err != nil { return nil, err @@ -369,9 +339,7 @@ func (p *Parser) parseCreateLiveView(pos Pos) (*CreateLiveView, error) { createLiveView.TableSchema = tableSchema } - if matched, consumeErr := p.tryConsumeKeywords(KeywordAs); consumeErr != nil { - return nil, consumeErr - } else if matched { + if p.tryConsumeKeywords(KeywordAs) { subQuery, err := p.parseSubQuery(p.Pos()) if err != nil { return nil, err @@ -384,9 +352,7 @@ func (p *Parser) parseCreateLiveView(pos Pos) (*CreateLiveView, error) { } func (p *Parser) tryParseWithTimeout(pos Pos) (*WithTimeoutClause, error) { - if matched, consumeErr := p.tryConsumeKeywords(KeywordWith); consumeErr != nil { - return nil, consumeErr - } else if !matched { + if !p.tryConsumeKeywords(KeywordWith) { return nil, nil // nolint } if err := p.expectKeyword(KeywordTimeout); err != nil {