Reject incomplete SQL and preserve lexical errors

parser/error.go

@@ -5,6 +5,16 @@ import (
 	"strings"
 )
 
+// lexerError keeps a lexical failure attached to its original byte position.
+type lexerError struct {
+	pos Pos
+	err error
+}
+
+func (e *lexerError) Error() string { return e.err.Error() }
+
+func (e *lexerError) Unwrap() error { return e.err }
+
 // ParseError is a structured parse error. It carries the byte offset and the
 // 1-based line/column where parsing stopped, the offending token, and (when
 // known) the tokens the grammar expected at that point. Callers such as

parser/error_test.go

@@ -47,3 +47,92 @@ func TestParseError_ExpectedTokenKind(t *testing.T) {
 	require.Equal(t, []TokenKind{TokenKindRParen}, pe.Expected)
 	require.True(t, strings.HasPrefix(pe.Error(), "line "))
 }
+
+func TestParseError_LexicalFailure(t *testing.T) {
+	for _, prefix := range []string{"SELECT 1 ", "SELECT case ", "SELECT interval + ", "SELECT 1;\n", "SELECT 1 /* closed */\n"} {
+		for _, suffix := range []struct {
+			sql string
+			msg string
+		}{
+			{"/*", "unclosed multi-line comment"},
+			{"'unclosed", "invalid string"},
+			{"`unclosed", "unclosed quoted identifier"},
+			{"1e+", "exponent part should contain at least one digit"},
+			{"中文", "unexpected character"},
+		} {
+			sql := prefix + suffix.sql
+			t.Run(sql, func(t *testing.T) {
+				stmts, err := NewParser(sql).ParseStmts()
+				require.Error(t, err)
+				require.Nil(t, stmts)
+
+				var pe *ParseError
+				require.ErrorAs(t, err, &pe)
+				require.Contains(t, pe.Msg, suffix.msg)
+				require.Equal(t, Pos(len(prefix)), pe.Pos)
+				if strings.HasSuffix(prefix, "\n") {
+					require.Equal(t, 2, pe.Line)
+					require.Equal(t, 1, pe.Column)
+				}
+			})
+		}
+	}
+}
+
+func TestParser_TokenConsumptionError(t *testing.T) {
+	for _, tc := range []struct {
+		name  string
+		sql   string
+		parse func(*Parser) error
+	}{
+		{"required token", ") /*", func(p *Parser) error { return p.expectTokenKind(TokenKindRParen) }},
+		{"optional token", ". /*", func(p *Parser) error {
+			_, err := p.tryParseDotIdent(p.Pos())
+			return err
+		}},
+		{"list separator", "a, /*", func(p *Parser) error {
+			_, err := p.parseUserNames()
+			return err
+		}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			p := NewParser(tc.sql)
+			require.NoError(t, p.lexer.consumeToken())
+			err := tc.parse(p)
+			var lexicalErr *lexerError
+			require.ErrorAs(t, err, &lexicalErr)
+			require.Equal(t, Pos(strings.Index(tc.sql, "/*")), lexicalErr.pos)
+			require.EqualError(t, lexicalErr, "unclosed multi-line comment")
+		})
+	}
+}
+
+func TestParser_TryConsumeTokenKind(t *testing.T) {
+	for _, tc := range []struct {
+		name    string
+		sql     string
+		kind    TokenKind
+		matched bool
+		next    string
+	}{
+		{"mismatch", "a /*", TokenKindComma, false, "a"},
+		{"advance", "a b", TokenKindIdent, true, "b"},
+		{"last token", "a", TokenKindIdent, true, "<EOF>"},
+		{"empty input", "", TokenKindIdent, false, "<EOF>"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			p := NewParser(tc.sql)
+			require.NoError(t, p.lexer.consumeToken())
+			current := p.current()
+			token, err := p.tryConsumeTokenKind(tc.kind)
+			require.NoError(t, err)
+			if tc.matched {
+				require.Same(t, current, token)
+			} else {
+				require.Nil(t, token)
+				require.Equal(t, current, p.current())
+			}
+			require.Equal(t, tc.next, p.currentTokenString())
+		})
+	}
+}

parser/lexer.go

@@ -83,6 +83,11 @@ type Lexer struct {
 	lexerState
 
 	input string
+
+	// Lexical failures are fatal for this input, even when discovered during
+	// lookahead. Keep them outside lexerState so restoring a cursor cannot
+	// discard the error or its original position.
+	err *lexerError
 }
 
 func NewLexer(buf string) *Lexer {
@@ -252,6 +257,7 @@ func (l *Lexer) consumeSingleLineComment() {
 }
 
 func (l *Lexer) consumeMultiLineComment() error {
+	pos := Pos(l.offset)
 	l.skipN(2)
 	i := 0
 	for l.peekOk(i) {
@@ -262,7 +268,7 @@ func (l *Lexer) consumeMultiLineComment() error {
 		i++
 	}
 	l.skipN(i)
-	return errors.New("unclosed multi-line comment")
+	return &lexerError{pos: pos, err: errors.New("unclosed multi-line comment")}
 }
 
 func (l *Lexer) consumeString() error {
@@ -335,13 +341,11 @@ func (l *Lexer) skipComments() error {
 
 func (l *Lexer) peekToken() (*Token, error) {
 	savedState := l.saveState()
+	defer l.restoreState(savedState)
 	if err := l.consumeToken(); err != nil {
 		return nil, err
 	}
-	token := l.currentToken
-
-	l.restoreState(savedState)
-	return token, nil
+	return l.currentToken, nil
 }
 
 func (l *Lexer) hasPrecedenceToken(last *Token) bool {
@@ -356,17 +360,31 @@ func (l *Lexer) hasPrecedenceToken(last *Token) bool {
 		last.Kind == TokenKindRBracket)
 }
 
-func (l *Lexer) consumeToken() error {
+func (l *Lexer) consumeToken() (err error) {
 	// replace the current token; keep the previous one to disambiguate unary +/-
 	prevToken := l.currentToken
 	l.currentToken = nil
+	if l.err != nil {
+		return l.err
+	}
+	pos := Pos(l.offset)
+	defer func() {
+		if err != nil {
+			if !errors.As(err, &l.err) {
+				l.err = &lexerError{pos: pos, err: err}
+			}
+			err = l.err
+		}
+	}()
+
 	if err := l.skipComments(); err != nil {
 		return err
 	}
 	l.skipSpace()
 	if l.isEOF() {
 		return nil
 	}
+	pos = Pos(l.offset)
 	switch l.peekN(0) {
 	case '>', '<', '!', '=', '|':
 		if l.peekN(0) == '|' && l.peekOk(1) && l.peekN(1) == '|' || // ||
@@ -387,7 +405,7 @@ func (l *Lexer) consumeToken() error {
 		// hasPrecedenceToken is used to distinguish between unary and binary operators
 		if !l.hasPrecedenceToken(prevToken) && l.peekOk(1) && IsDigit(l.peekN(1)) {
 			return l.consumeNumber()
-		} else if l.peekOk(1) && l.peekN(1) == '>' {
+		} else if l.peekN(0) == '-' && l.peekOk(1) && l.peekN(1) == '>' {
 			l.currentToken = &Token{
 				String: l.slice(0, 2),
 				Kind:   TokenKindArrow,

parser/lexer_test.go

@@ -32,6 +32,32 @@ func TestConsumeComment(t *testing.T) {
 
 }
 
+func TestPeekToken_RestoresState(t *testing.T) {
+	for _, suffix := range []string{"", " + 1", " /*", " 'unclosed", " `unclosed", " 1e+"} {
+		t.Run(suffix, func(t *testing.T) {
+			lexer := NewLexer("x" + suffix)
+			require.NoError(t, lexer.consumeToken())
+			before := lexer.saveState()
+
+			token, peekErr := lexer.peekToken()
+			require.Equal(t, before, lexer.saveState())
+			err := lexer.consumeToken()
+			if peekErr != nil {
+				var le *lexerError
+				require.ErrorAs(t, peekErr, &le)
+				require.Equal(t, Pos(2), le.pos)
+				require.Equal(t, peekErr, err)
+				require.Nil(t, lexer.currentToken)
+				// Further advancement must not turn a lexical failure into EOF.
+				require.Equal(t, err, lexer.consumeToken())
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, token, lexer.currentToken)
+			}
+		})
+	}
+}
+
 // TestConsumeUnterminatedComment guards against an infinite loop (a DoS hang)
 // when a block comment is never closed. consumeMultiLineComment previously
 // looped on isEOF() while only advancing a local index, so l.offset never