diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 1798341..7c73dfb 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -8,50 +8,111 @@ Results ``` $ go test -bench=. -benchmem ./parser -goos: linux -goarch: amd64 +goos: darwin +goarch: arm64 pkg: github.com/AfterShip/clickhouse-sql-parser/parser -cpu: Intel(R) Xeon(R) CPU E5-2697 v3 @ 2.60GHz -BenchmarkParseSQLFiles/access_tuple_with_dot.sql-28 23294 58467 ns/op 13448 B/op 293 allocs/op -BenchmarkParseSQLFiles/query_with_expr_compare.sql-28 43560 25704 ns/op 6240 B/op 132 allocs/op -BenchmarkParseSQLFiles/select_cast.sql-28 75055 16518 ns/op 4648 B/op 92 allocs/op -BenchmarkParseSQLFiles/select_column_alias_string.sql-28 499798 2785 ns/op 704 B/op 13 allocs/op -BenchmarkParseSQLFiles/select_expr.sql-28 488187 2448 ns/op 696 B/op 12 allocs/op -BenchmarkParseSQLFiles/select_item_with_modifiers.sql-28 54124 23305 ns/op 6232 B/op 136 allocs/op -BenchmarkParseSQLFiles/select_order_by_timestamp.sql-28 232302 5809 ns/op 1368 B/op 28 allocs/op -BenchmarkParseSQLFiles/select_simple.sql-28 30602 50022 ns/op 9920 B/op 216 allocs/op -BenchmarkParseSQLFiles/select_simple_field_alias.sql-28 178126 6316 ns/op 1712 B/op 39 allocs/op -BenchmarkParseSQLFiles/select_simple_with_bracket.sql-28 71902 16172 ns/op 3824 B/op 85 allocs/op -BenchmarkParseSQLFiles/select_simple_with_cte_with_column_aliases.sql-28 67050 19916 ns/op 4648 B/op 101 allocs/op -BenchmarkParseSQLFiles/select_simple_with_group_by_with_cube_totals.sql-28 107047 10348 ns/op 2768 B/op 58 allocs/op -BenchmarkParseSQLFiles/select_simple_with_is_not_null.sql-28 55285 21957 ns/op 5224 B/op 111 allocs/op -BenchmarkParseSQLFiles/select_simple_with_is_null.sql-28 66648 22412 ns/op 4728 B/op 102 allocs/op -BenchmarkParseSQLFiles/select_simple_with_top_clause.sql-28 269910 4166 ns/op 1088 B/op 22 allocs/op -BenchmarkParseSQLFiles/select_simple_with_with_clause.sql-28 58494 18417 ns/op 5144 B/op 109 allocs/op -BenchmarkParseSQLFiles/select_table_alias_without_keyword.sql-28 121261 10011 ns/op 2896 B/op 65 allocs/op -BenchmarkParseSQLFiles/select_table_function_with_query.sql-28 98017 14929 ns/op 4168 B/op 81 allocs/op -BenchmarkParseSQLFiles/select_when_condition.sql-28 220394 5457 ns/op 1304 B/op 28 allocs/op -BenchmarkParseSQLFiles/select_with_distinct.sql-28 172948 6531 ns/op 1560 B/op 33 allocs/op -BenchmarkParseSQLFiles/select_with_join_only.sql-28 286346 5594 ns/op 1520 B/op 32 allocs/op -BenchmarkParseSQLFiles/select_with_left_join.sql-28 88200 13627 ns/op 3880 B/op 75 allocs/op -BenchmarkParseSQLFiles/select_with_literal_table_name.sql-28 241094 5099 ns/op 1304 B/op 27 allocs/op -BenchmarkParseSQLFiles/select_with_multi_join.sql-28 44700 31964 ns/op 8240 B/op 188 allocs/op -BenchmarkParseSQLFiles/select_with_multi_line_comment.sql-28 363499 4460 ns/op 824 B/op 18 allocs/op -BenchmarkParseSQLFiles/select_with_multi_union.sql-28 146233 7827 ns/op 2176 B/op 36 allocs/op -BenchmarkParseSQLFiles/select_with_number_field.sql-28 129945 8746 ns/op 2352 B/op 51 allocs/op -BenchmarkParseSQLFiles/select_with_query_parameter.sql-28 33850 36346 ns/op 9936 B/op 209 allocs/op -BenchmarkParseSQLFiles/select_with_string_expr.sql-28 142882 7530 ns/op 1880 B/op 34 allocs/op -BenchmarkParseSQLFiles/select_with_union_distinct.sql-28 147031 9601 ns/op 2352 B/op 47 allocs/op -BenchmarkParseSQLFiles/select_with_variable.sql-28 179158 6669 ns/op 1880 B/op 36 allocs/op -BenchmarkParseSQLFiles/select_with_window_function.sql-28 54925 31320 ns/op 6720 B/op 136 allocs/op -BenchmarkParseSQLFiles/select_with_placeholder.sql-28 196771 5145 ns/op 1272 B/op 26 allocs/op -BenchmarkParseSQLFiles/set_simple.sql-28 172419 7062 ns/op 2480 B/op 49 allocs/op -BenchmarkParseComplexQueries/testdata/query/select_with_multi_join.sql-28 39056 36897 ns/op 8240 B/op 188 allocs/op -BenchmarkParseComplexQueries/testdata/query/select_with_window_function.sql-28 47629 29916 ns/op 6720 B/op 136 allocs/op -BenchmarkParseComplexQueries/testdata/query/select_simple_with_with_clause.sql-28 69210 19731 ns/op 5144 B/op 109 allocs/op -BenchmarkParseComplexQueries/testdata/query/select_with_left_join.sql-28 74576 15338 ns/op 3880 B/op 75 allocs/op -BenchmarkParseComplexQueries/testdata/benchdata/posthog_huge_0.sql-28 235 6231253 ns/op 1236189 B/op 26696 allocs/op -BenchmarkParseComplexQueries/testdata/benchdata/posthog_huge_1.sql-28 279 4438280 ns/op 1043374 B/op 22717 allocs/op +cpu: Apple M5 +BenchmarkParseSQLFiles/access_tuple_with_dot.sql-10 41946 32091 ns/op 14157 B/op 160 allocs/op +BenchmarkParseSQLFiles/create_window_view.sql-10 151291 7624 ns/op 7272 B/op 50 allocs/op +BenchmarkParseSQLFiles/query_with_expr_compare.sql-10 135202 8740 ns/op 7160 B/op 73 allocs/op +BenchmarkParseSQLFiles/select_case_multiple_when.sql-10 191094 6216 ns/op 4912 B/op 34 allocs/op +BenchmarkParseSQLFiles/select_case_when_exists.sql-10 127548 10734 ns/op 5600 B/op 44 allocs/op +BenchmarkParseSQLFiles/select_case_when_regexp.sql-10 287572 3867 ns/op 2816 B/op 27 allocs/op +BenchmarkParseSQLFiles/select_cast.sql-10 91058 11445 ns/op 8856 B/op 66 allocs/op +BenchmarkParseSQLFiles/select_clause_keyword_as_column.sql-10 493710 2411 ns/op 2272 B/op 26 allocs/op +BenchmarkParseSQLFiles/select_clause_keyword_as_only_column.sql-10 1479050 868.7 ns/op 696 B/op 8 allocs/op +BenchmarkParseSQLFiles/select_column_alias_string.sql-10 462427 2358 ns/op 1984 B/op 19 allocs/op +BenchmarkParseSQLFiles/select_concat_expr.sql-10 336728 5401 ns/op 3256 B/op 29 allocs/op +BenchmarkParseSQLFiles/select_dynamic_subcolumn_type_hint.sql-10 257604 3905 ns/op 1920 B/op 20 allocs/op +BenchmarkParseSQLFiles/select_end_as_column_name.sql-10 148753 8255 ns/op 4304 B/op 44 allocs/op +BenchmarkParseSQLFiles/select_expr.sql-10 1000000 1002 ns/op 808 B/op 10 allocs/op +BenchmarkParseSQLFiles/select_expr_keyword_as_column.sql-10 479106 3358 ns/op 2048 B/op 24 allocs/op +BenchmarkParseSQLFiles/select_extract_with_regex.sql-10 71292 17824 ns/op 11936 B/op 110 allocs/op +BenchmarkParseSQLFiles/select_function_keyword_args.sql-10 20080 51650 ns/op 38464 B/op 406 allocs/op +BenchmarkParseSQLFiles/select_intersect.sql-10 155968 7772 ns/op 6784 B/op 67 allocs/op +BenchmarkParseSQLFiles/select_interval_as_column_name.sql-10 22311 49430 ns/op 35352 B/op 352 allocs/op +BenchmarkParseSQLFiles/select_item_with_modifiers.sql-10 165195 6981 ns/op 6416 B/op 80 allocs/op +BenchmarkParseSQLFiles/select_json_type.sql-10 112285 11913 ns/op 10272 B/op 88 allocs/op +BenchmarkParseSQLFiles/select_keyword_alias_no_as.sql-10 788174 1348 ns/op 1104 B/op 14 allocs/op +BenchmarkParseSQLFiles/select_keyword_as_alias.sql-10 585783 1827 ns/op 1880 B/op 18 allocs/op +BenchmarkParseSQLFiles/select_keyword_as_column.sql-10 403886 2995 ns/op 2912 B/op 28 allocs/op +BenchmarkParseSQLFiles/select_keyword_as_last_column.sql-10 860019 1448 ns/op 1304 B/op 17 allocs/op +BenchmarkParseSQLFiles/select_keyword_as_only_column.sql-10 1491062 848.1 ns/op 696 B/op 8 allocs/op +BenchmarkParseSQLFiles/select_keyword_as_only_column_semicolon.sql-10 1290060 880.4 ns/op 696 B/op 8 allocs/op +BenchmarkParseSQLFiles/select_keyword_operand_arithmetic.sql-10 304675 3783 ns/op 2992 B/op 31 allocs/op +BenchmarkParseSQLFiles/select_keyword_operand_as_function_argument.sql-10 229147 5205 ns/op 3608 B/op 47 allocs/op +BenchmarkParseSQLFiles/select_keyword_operand_before_operator_keyword.sql-10 238518 5192 ns/op 3504 B/op 35 allocs/op +BenchmarkParseSQLFiles/select_keyword_operand_cast_and_subscript.sql-10 261912 5058 ns/op 3488 B/op 47 allocs/op +BenchmarkParseSQLFiles/select_keyword_operand_in_where.sql-10 575012 1749 ns/op 1464 B/op 17 allocs/op +BenchmarkParseSQLFiles/select_keyword_operand_multiple_items.sql-10 623074 2015 ns/op 1680 B/op 20 allocs/op +BenchmarkParseSQLFiles/select_order_by_timestamp.sql-10 634240 1907 ns/op 1712 B/op 17 allocs/op +BenchmarkParseSQLFiles/select_order_by_with_fill_basic.sql-10 191694 5874 ns/op 5160 B/op 52 allocs/op +BenchmarkParseSQLFiles/select_order_by_with_fill_from_to.sql-10 194827 6163 ns/op 5704 B/op 55 allocs/op +BenchmarkParseSQLFiles/select_order_by_with_fill_interpolate.sql-10 148472 8786 ns/op 7384 B/op 69 allocs/op +BenchmarkParseSQLFiles/select_order_by_with_fill_interpolate_no_columns.sql-10 170904 6282 ns/op 6024 B/op 57 allocs/op +BenchmarkParseSQLFiles/select_order_by_with_fill_staleness.sql-10 238216 5637 ns/op 4376 B/op 44 allocs/op +BenchmarkParseSQLFiles/select_order_by_with_fill_step.sql-10 166125 7995 ns/op 5472 B/op 51 allocs/op +BenchmarkParseSQLFiles/select_regexp.sql-10 447778 2446 ns/op 1688 B/op 20 allocs/op +BenchmarkParseSQLFiles/select_reserved_keyword_qualifier.sql-10 779988 1700 ns/op 1920 B/op 20 allocs/op +BenchmarkParseSQLFiles/select_signed_number_after_bracket.sql-10 173463 6093 ns/op 5640 B/op 59 allocs/op +BenchmarkParseSQLFiles/select_simple.sql-10 101438 12097 ns/op 10504 B/op 113 allocs/op +BenchmarkParseSQLFiles/select_simple_field_alias.sql-10 611782 2030 ns/op 1888 B/op 21 allocs/op +BenchmarkParseSQLFiles/select_simple_with_bracket.sql-10 276570 4635 ns/op 4080 B/op 55 allocs/op +BenchmarkParseSQLFiles/select_simple_with_cte_with_column_aliases.sql-10 193498 6424 ns/op 4976 B/op 54 allocs/op +BenchmarkParseSQLFiles/select_simple_with_group_by_with_cube_totals.sql-10 271335 4109 ns/op 3056 B/op 33 allocs/op +BenchmarkParseSQLFiles/select_simple_with_is_not_null.sql-10 163068 8633 ns/op 5616 B/op 57 allocs/op +BenchmarkParseSQLFiles/select_simple_with_is_null.sql-10 124342 8121 ns/op 4672 B/op 53 allocs/op +BenchmarkParseSQLFiles/select_simple_with_limit.sql-10 279214 4178 ns/op 2904 B/op 24 allocs/op +BenchmarkParseSQLFiles/select_simple_with_top_clause.sql-10 678796 1555 ns/op 1424 B/op 15 allocs/op +BenchmarkParseSQLFiles/select_simple_with_with_clause.sql-10 195356 6100 ns/op 5496 B/op 67 allocs/op +BenchmarkParseSQLFiles/select_table_alias_without_keyword.sql-10 343621 3535 ns/op 3048 B/op 44 allocs/op +BenchmarkParseSQLFiles/select_table_function_arg_exprs.sql-10 44941 27705 ns/op 26056 B/op 263 allocs/op +BenchmarkParseSQLFiles/select_table_function_with_query.sql-10 247592 4908 ns/op 4264 B/op 46 allocs/op +BenchmarkParseSQLFiles/select_trailing_comma_before_from.sql-10 672033 1888 ns/op 1528 B/op 19 allocs/op +BenchmarkParseSQLFiles/select_trailing_comma_before_from_keyword_table.sql-10 545592 2195 ns/op 1512 B/op 20 allocs/op +BenchmarkParseSQLFiles/select_when_condition.sql-10 343081 3903 ns/op 1616 B/op 13 allocs/op +BenchmarkParseSQLFiles/select_window_comprehensive.sql-10 13759 73222 ns/op 64640 B/op 542 allocs/op +BenchmarkParseSQLFiles/select_window_cte.sql-10 66794 17567 ns/op 16336 B/op 142 allocs/op +BenchmarkParseSQLFiles/select_window_keyword_name_in_parens.sql-10 305048 4016 ns/op 3624 B/op 38 allocs/op +BenchmarkParseSQLFiles/select_window_named_in_parens.sql-10 278389 3977 ns/op 3480 B/op 39 allocs/op +BenchmarkParseSQLFiles/select_window_named_reference_extensions.sql-10 125362 10458 ns/op 7848 B/op 68 allocs/op +BenchmarkParseSQLFiles/select_window_params.sql-10 78658 14711 ns/op 15088 B/op 118 allocs/op +BenchmarkParseSQLFiles/select_with_distinct.sql-10 552853 2407 ns/op 1856 B/op 23 allocs/op +BenchmarkParseSQLFiles/select_with_distinct_keyword.sql-10 905398 1306 ns/op 1280 B/op 13 allocs/op +BenchmarkParseSQLFiles/select_with_distinct_on_dotted_columns.sql-10 339792 4032 ns/op 3576 B/op 44 allocs/op +BenchmarkParseSQLFiles/select_with_distinct_on_keyword.sql-10 549628 1900 ns/op 1752 B/op 20 allocs/op +BenchmarkParseSQLFiles/select_with_global_join_locality.sql-10 19771 58056 ns/op 53776 B/op 619 allocs/op +BenchmarkParseSQLFiles/select_with_group_by.sql-10 137679 8616 ns/op 8288 B/op 79 allocs/op +BenchmarkParseSQLFiles/select_with_join_only.sql-10 602020 2157 ns/op 1752 B/op 25 allocs/op +BenchmarkParseSQLFiles/select_with_keyword_in_group_by.sql-10 193964 7432 ns/op 5432 B/op 58 allocs/op +BenchmarkParseSQLFiles/select_with_keyword_placeholder.sql-10 385830 2951 ns/op 2488 B/op 25 allocs/op +BenchmarkParseSQLFiles/select_with_left_join.sql-10 249094 5832 ns/op 4928 B/op 43 allocs/op +BenchmarkParseSQLFiles/select_with_literal_table_name.sql-10 644481 2233 ns/op 1824 B/op 16 allocs/op +BenchmarkParseSQLFiles/select_with_multi_array_and_inner_join.sql-10 102230 11334 ns/op 10184 B/op 130 allocs/op +BenchmarkParseSQLFiles/select_with_multi_array_join.sql-10 163647 6441 ns/op 5288 B/op 66 allocs/op +BenchmarkParseSQLFiles/select_with_multi_except.sql-10 235374 5238 ns/op 4632 B/op 50 allocs/op +BenchmarkParseSQLFiles/select_with_multi_join.sql-10 115492 10876 ns/op 9216 B/op 94 allocs/op +BenchmarkParseSQLFiles/select_with_multi_line_comment.sql-10 839001 1636 ns/op 1680 B/op 13 allocs/op +BenchmarkParseSQLFiles/select_with_multi_union.sql-10 485973 2470 ns/op 2424 B/op 19 allocs/op +BenchmarkParseSQLFiles/select_with_multi_union_distinct.sql-10 506588 2576 ns/op 2552 B/op 19 allocs/op +BenchmarkParseSQLFiles/select_with_number_field.sql-10 380826 3483 ns/op 2632 B/op 33 allocs/op +BenchmarkParseSQLFiles/select_with_parenthesized_union.sql-10 41083 28368 ns/op 28928 B/op 191 allocs/op +BenchmarkParseSQLFiles/select_with_placeholder.sql-10 701680 1788 ns/op 1496 B/op 18 allocs/op +BenchmarkParseSQLFiles/select_with_query_parameter.sql-10 115622 10473 ns/op 10368 B/op 99 allocs/op +BenchmarkParseSQLFiles/select_with_settings_additional_table_filters.sql-10 69403 18035 ns/op 15392 B/op 152 allocs/op +BenchmarkParseSQLFiles/select_with_single_quote_table.sql-10 1000000 1383 ns/op 1008 B/op 13 allocs/op +BenchmarkParseSQLFiles/select_with_string_expr.sql-10 412788 2446 ns/op 2320 B/op 23 allocs/op +BenchmarkParseSQLFiles/select_with_union_distinct.sql-10 356832 3263 ns/op 3408 B/op 26 allocs/op +BenchmarkParseSQLFiles/select_with_variable.sql-10 563797 2305 ns/op 2176 B/op 23 allocs/op +BenchmarkParseSQLFiles/select_with_window_function.sql-10 91530 13406 ns/op 11280 B/op 92 allocs/op +BenchmarkParseSQLFiles/select_without_from_where.sql-10 391298 3189 ns/op 2928 B/op 28 allocs/op +BenchmarkParseSQLFiles/set_simple.sql-10 495375 2762 ns/op 3800 B/op 25 allocs/op +BenchmarkParseComplexQueries/testdata/query/select_with_multi_join.sql-10 114853 10731 ns/op 9216 B/op 94 allocs/op +BenchmarkParseComplexQueries/testdata/query/select_with_window_function.sql-10 86808 13809 ns/op 11280 B/op 92 allocs/op +BenchmarkParseComplexQueries/testdata/query/select_simple_with_with_clause.sql-10 192193 6218 ns/op 5496 B/op 67 allocs/op +BenchmarkParseComplexQueries/testdata/query/select_with_left_join.sql-10 246328 4892 ns/op 4928 B/op 43 allocs/op +BenchmarkParseComplexQueries/testdata/benchdata/posthog_huge_0.sql-10 660 1938482 ns/op 1313672 B/op 14554 allocs/op +BenchmarkParseComplexQueries/testdata/benchdata/posthog_huge_1.sql-10 759 1579115 ns/op 1106177 B/op 12559 allocs/op PASS -ok github.com/AfterShip/clickhouse-sql-parser/parser 66.547s +ok github.com/AfterShip/clickhouse-sql-parser/parser 139.761s ``` diff --git a/parser/keyword.go b/parser/keyword.go index e3457df..13fa627 100644 --- a/parser/keyword.go +++ b/parser/keyword.go @@ -590,3 +590,17 @@ var keywords = NewSet( KeywordSQL, KeywordSecurity, ) + +// lookupKeyword returns the canonical upper-case spelling of s when it is a +// keyword, letting callers reuse the interned constant instead of allocating. +func lookupKeyword(s string) (string, bool) { + return lookupFold(canonicalKeywords, s) +} + +var canonicalKeywords = func() map[string]string { + m := make(map[string]string, len(keywords.m)) + for keyword := range keywords.m { + m[keyword] = keyword + } + return m +}() diff --git a/parser/lexer.go b/parser/lexer.go index 96df452..d00d213 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -69,6 +69,9 @@ type Token struct { func (t *Token) ToString() string { if t.Kind == TokenKindKeyword { + if keyword, ok := lookupKeyword(t.String); ok { + return keyword + } return strings.ToUpper(t.String) } return t.String @@ -83,6 +86,47 @@ type Lexer struct { lexerState input string + + // peekFrom/peekTo cache the state transition performed by the last + // successful peekToken, so repeated peeks from the same state and the + // consumeToken that follows them do not lex the same token again. + peekFrom lexerState + peekTo lexerState + peekValid bool + + // tokens is the slab the next tokens are carved from; see newToken. + tokens []Token +} + +// Bounds on how many tokens one slab allocation holds; see tokenChunkSize. +const ( + minTokenChunk = 4 + maxTokenChunk = 64 +) + +// tokenChunkSize sizes the next slab from the input still to lex, at roughly +// one token per four bytes, so a tiny statement does not pay for a full chunk. +func (l *Lexer) tokenChunkSize() int { + return max(minTokenChunk, min((len(l.input)-l.offset)/4, maxTokenChunk)) +} + +// newToken carves a token from the slab. The AST never retains tokens, so a +// slab removes a malloc per token at the cost of one chunk of slack. +func (l *Lexer) newToken(kind TokenKind, str string, pos, end Pos) *Token { + if len(l.tokens) == 0 { + l.tokens = make([]Token, l.tokenChunkSize()) + } + token := &l.tokens[0] + l.tokens = l.tokens[1:] + token.Kind, token.String, token.Pos, token.End = kind, str, pos, end + return token +} + +// emitOperator makes the n bytes at the current offset the current token and +// advances past them. +func (l *Lexer) emitOperator(kind TokenKind, n int) { + l.currentToken = l.newToken(kind, l.slice(0, n), Pos(l.offset), Pos(l.offset+n)) + l.skipN(n) } func NewLexer(buf string) *Lexer { @@ -114,7 +158,7 @@ func (l *Lexer) peekOk(n int) bool { } func (l *Lexer) isKeyword(ident string) bool { - return keywords.Contains(ident) + return containsFold(keywords, ident) } func (l *Lexer) consumeNumber() error { @@ -178,19 +222,13 @@ func (l *Lexer) consumeNumber() error { if (l.peekOk(i) && IsIdentPart(l.peekN(i))) || !hasNumberPart { return errors.New("invalid number") } - l.currentToken = &Token{ - Kind: tokenKind, - String: l.slice(0, i), - Pos: Pos(l.offset), - End: Pos(l.offset + i), - Base: base, - } + l.currentToken = l.newToken(tokenKind, l.slice(0, i), Pos(l.offset), Pos(l.offset+i)) + l.currentToken.Base = base l.skipN(i) return nil } func (l *Lexer) consumeIdent(_ Pos) error { - token := &Token{} quoteType := Unquoted if l.peekOk(0) && (l.peekN(0) == '`' || l.peekN(0) == '"') { if l.peekOk(0) && l.peekN(0) == '`' { @@ -220,16 +258,12 @@ func (l *Lexer) consumeIdent(_ Pos) error { } } slice := l.slice(0, i) - if quoteType == Unquoted && l.isKeyword(strings.ToUpper(slice)) { - token.Kind = TokenKindKeyword - } else { - token.Kind = TokenKindIdent + kind := TokenKindIdent + if quoteType == Unquoted && l.isKeyword(slice) { + kind = TokenKindKeyword } - token.Pos = Pos(l.offset) - token.End = Pos(l.offset + i) - token.String = slice - token.QuoteType = quoteType - l.currentToken = token + l.currentToken = l.newToken(kind, slice, Pos(l.offset), Pos(l.offset+i)) + l.currentToken.QuoteType = quoteType l.skipN(i) if quoteType != Unquoted { @@ -292,12 +326,7 @@ func (l *Lexer) consumeString() error { if !l.peekOk(i) || l.peekN(i) != endChar { return errors.New("invalid string") } - l.currentToken = &Token{ - Kind: TokenKindString, - String: l.slice(1, i), - Pos: Pos(l.offset + 1), - End: Pos(l.offset + i), - } + l.currentToken = l.newToken(TokenKindString, l.slice(1, i), Pos(l.offset+1), Pos(l.offset+i)) l.skipN(i + 1) return nil } @@ -334,11 +363,15 @@ func (l *Lexer) skipComments() error { } func (l *Lexer) peekToken() (*Token, error) { + if l.peekValid && l.peekFrom == l.lexerState { + return l.peekTo.currentToken, nil + } savedState := l.saveState() if err := l.consumeToken(); err != nil { return nil, err } token := l.currentToken + l.peekFrom, l.peekTo, l.peekValid = savedState, l.lexerState, true l.restoreState(savedState) return token, nil @@ -357,6 +390,10 @@ func (l *Lexer) hasPrecedenceToken(last *Token) bool { } func (l *Lexer) consumeToken() error { + if l.peekValid && l.peekFrom == l.lexerState { + l.lexerState = l.peekTo + return nil + } // replace the current token; keep the previous one to disambiguate unary +/- prevToken := l.currentToken l.currentToken = nil @@ -373,13 +410,7 @@ func (l *Lexer) consumeToken() error { l.peekN(0) == '<' && l.peekOk(1) && l.peekN(1) == '>' || // <> l.peekN(0) == '=' && l.peekOk(1) && l.peekN(1) == '=' || // == l.peekN(0) != '|' && l.peekOk(1) && l.peekN(1) == '=' { // |= - l.currentToken = &Token{ - String: l.slice(0, 2), - Kind: TokenKind(l.slice(0, 2)), - Pos: Pos(l.offset), - End: Pos(l.offset + 2), - } - l.skipN(2) + l.emitOperator(TokenKind(l.slice(0, 2)), 2) return nil } @@ -388,13 +419,7 @@ func (l *Lexer) consumeToken() error { if !l.hasPrecedenceToken(prevToken) && l.peekOk(1) && IsDigit(l.peekN(1)) { return l.consumeNumber() } else if l.peekOk(1) && l.peekN(1) == '>' { - l.currentToken = &Token{ - String: l.slice(0, 2), - Kind: TokenKindArrow, - Pos: Pos(l.offset), - End: Pos(l.offset + 2), - } - l.skipN(2) + l.emitOperator(TokenKindArrow, 2) return nil } case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': @@ -405,23 +430,11 @@ func (l *Lexer) consumeToken() error { return l.consumeString() case ':': if l.peekOk(1) && l.peekN(1) == ':' { - l.currentToken = &Token{ - String: l.slice(0, 2), - Kind: TokenKindDash, - Pos: Pos(l.offset), - End: Pos(l.offset + 2), - } - l.skipN(2) + l.emitOperator(TokenKindDash, 2) return nil } case '.': - l.currentToken = &Token{ - String: l.slice(0, 1), - Kind: TokenKindDot, - Pos: Pos(l.offset), - End: Pos(l.offset + 1), - } - l.skipN(1) + l.emitOperator(TokenKindDot, 1) return nil } @@ -437,13 +450,7 @@ func (l *Lexer) consumeToken() error { return fmt.Errorf("unexpected character %q", r) } - token := &Token{} - token.Pos = Pos(l.offset) - token.End = Pos(l.offset + 1) - token.String = l.input[l.offset : l.offset+1] - token.Kind = TokenKind(token.String) - l.skipN(1) - l.currentToken = token + l.emitOperator(TokenKind(l.slice(0, 1)), 1) return nil } @@ -453,6 +460,14 @@ func (l *Lexer) isEOF() bool { func (l *Lexer) skipSpace() { for !l.isEOF() { + if c := l.input[l.offset]; c < utf8.RuneSelf { + // the ASCII subset of unicode.IsSpace + if c != ' ' && c != '\t' && c != '\n' && c != '\v' && c != '\f' && c != '\r' { + break + } + l.offset++ + continue + } r, size := utf8.DecodeRuneInString(l.input[l.offset:]) if !unicode.IsSpace(r) { break diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 8cff73f..2c29762 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -346,3 +346,33 @@ func TestNegativeHexLiteral(t *testing.T) { require.NoError(t, err) require.Len(t, stmts, 1) } + +// The peek cache is keyed on the lexer state, including the token before the +// peeked one, because `-` lexes differently after an operand (binary minus) +// than after an operator (a signed number). Backtracking to the same state +// must reuse the cached result, and any other state must lex afresh. +func TestPeekTokenCacheFollowsLexerState(t *testing.T) { + lexer := NewLexer("a -1") + require.NoError(t, lexer.consumeToken()) // a + + peeked, err := lexer.peekToken() + require.NoError(t, err) + require.Equal(t, TokenKindMinus, peeked.Kind) + + savedState := lexer.saveState() + require.NoError(t, lexer.consumeToken()) + require.Same(t, peeked, lexer.currentToken) + require.NoError(t, lexer.consumeToken()) + require.Equal(t, "1", lexer.currentToken.String) + + lexer.restoreState(savedState) + require.NoError(t, lexer.consumeToken()) + require.Same(t, peeked, lexer.currentToken) + + // after a non-operand the same bytes lex as a signed number + lexer.restoreState(lexerState{offset: savedState.offset}) + peeked, err = lexer.peekToken() + require.NoError(t, err) + require.Equal(t, TokenKindInt, peeked.Kind) + require.Equal(t, "-1", peeked.String) +} diff --git a/parser/parser_column.go b/parser/parser_column.go index 30de285..eac9337 100644 --- a/parser/parser_column.go +++ b/parser/parser_column.go @@ -36,15 +36,51 @@ func (p *Parser) tryParseColumnComment(pos Pos) (*StringLiteral, error) { return p.parseString(pos) } +// getNextPrecedence maps the current token to the binding power of the +// binary operator it starts. Token kinds are mutually exclusive, so a single +// switch on the kind replaces a chain of per-operator match calls; this runs +// after every primary expression and is hot. func (p *Parser) getNextPrecedence() int { - switch { - case p.matchKeyword(KeywordOr): + token := p.current() + if token == nil { + return PrecedenceUnknown + } + switch token.Kind { + case TokenKindKeyword: + return p.getKeywordPrecedence(token.ToString()) + case TokenKindDot: + return PrecedenceDot + case TokenKindDash: + return PrecedenceDoubleColon + case TokenKindSingleEQ, TokenKindLT, TokenKindLE, TokenKindGE, TokenKindGT, + TokenKindDoubleEQ, TokenKindNE, "<>": + return PrecedenceCompare + case TokenKindConcat: + return PrecedenceConcat + case TokenKindPlus, TokenKindMinus: + return PrecedenceAddSub + case TokenKindMul, TokenKindDiv, TokenKindMod: + return PrecedenceMulDivMod + case TokenKindArrow: + return PrecedenceArrow + case TokenKindLParen, TokenKindLBracket: + return PrecedenceBracket + case TokenKindQuestionMark: + return PrecedenceQuery + default: + return PrecedenceUnknown + } +} + +func (p *Parser) getKeywordPrecedence(keyword string) int { + switch keyword { + case KeywordOr: return PrecedenceOr - case p.matchKeyword(KeywordAnd): + case KeywordAnd: return PrecedenceAnd - case p.matchKeyword(KeywordIs): + case KeywordIs: return PrecedenceIs - case p.matchKeyword(KeywordNot): + case KeywordNot: // Infix NOT only begins NOT IN/LIKE/ILIKE/BETWEEN, so it binds with // the precedence of the operator it negates; `a = b NOT IN (1)` must // group the same way `a = b IN (1)` does. @@ -56,38 +92,17 @@ func (p *Parser) getNextPrecedence() int { default: return PrecedenceNot } - case p.matchTokenKind(TokenKindDot): - return PrecedenceDot - case p.matchTokenKind(TokenKindDash): - return PrecedenceDoubleColon - case p.matchTokenKind(TokenKindSingleEQ), p.matchTokenKind(TokenKindLT), p.matchTokenKind(TokenKindLE), - p.matchTokenKind(TokenKindGE), p.matchTokenKind(TokenKindGT), p.matchTokenKind(TokenKindDoubleEQ), - p.matchTokenKind(TokenKindNE), p.matchTokenKind("<>"): - return PrecedenceCompare - case p.matchTokenKind(TokenKindConcat): - return PrecedenceConcat - case p.matchTokenKind(TokenKindPlus), p.matchTokenKind(TokenKindMinus): - return PrecedenceAddSub - case p.matchTokenKind(TokenKindMul), p.matchTokenKind(TokenKindDiv), p.matchTokenKind(TokenKindMod): - return PrecedenceMulDivMod - case p.matchTokenKind(TokenKindArrow): - return PrecedenceArrow - case p.matchTokenKind(TokenKindLParen), p.matchTokenKind(TokenKindLBracket): - return PrecedenceBracket - case p.matchKeyword(KeywordBetween), p.matchKeyword(KeywordLike), p.matchKeyword(KeywordIlike), p.matchKeyword(KeywordRegexp): + case KeywordBetween, KeywordLike, KeywordIlike, KeywordRegexp: return PrecedenceBetweenLike - case p.matchKeyword(KeywordIn): + case KeywordIn: return precedenceIn - case p.matchKeyword(KeywordGlobal): + case KeywordGlobal: // GLOBAL is also a join locality: in `ON a = b GLOBAL LEFT JOIN c` it // belongs to the FROM clause, so the expression has to end here. if p.peekJoinAfterLocality() { return PrecedenceUnknown } - return precedenceIn - case p.matchTokenKind(TokenKindQuestionMark): - return PrecedenceQuery default: return PrecedenceUnknown } @@ -368,7 +383,7 @@ func (p *Parser) parseColumnExtractExpr(pos Pos) (*ExtractExpr, error) { var param Expr if ident, ok := expr.(*Ident); ok { - if intervalUnits.Contains(strings.ToUpper(ident.Name)) && p.matchKeyword(KeywordFrom) { + if p.matchKeyword(KeywordFrom) && containsFold(intervalUnits, ident.Name) { param, err = p.parseExtractFrom(ident) if err != nil { return nil, err @@ -477,8 +492,15 @@ func (p *Parser) matchClauseStarterKeyword() bool { // peekIsClauseStarterKeyword reports whether the next token is one of the // clause-starter keywords. func (p *Parser) peekIsClauseStarterKeyword() bool { + if p.lexer.isEOF() { + return false + } + token, err := p.lexer.peekToken() + if err != nil || token == nil || token.Kind != TokenKindKeyword { + return false + } for _, kw := range clauseStarterKeywords { - if p.peekKeyword(kw) { + if strings.EqualFold(token.String, kw) { return true } } @@ -812,7 +834,7 @@ func (p *Parser) parseInterval(requireKeyword bool) (*IntervalExpr, error) { if err != nil { return nil, err } - if !intervalUnits.Contains(strings.ToUpper(unit.Name)) { + if !containsFold(intervalUnits, unit.Name) { return nil, fmt.Errorf("unknown interval type: <%q>", unit.Name) } return &IntervalExpr{ diff --git a/parser/parser_common.go b/parser/parser_common.go index 7532601..68f0f4e 100644 --- a/parser/parser_common.go +++ b/parser/parser_common.go @@ -90,7 +90,7 @@ func (p *Parser) matchTokenKind(kinds ...TokenKind) bool { } for _, kind := range kinds { if kind == TokenKindIdent { - return !reservedKeywords.Contains(strings.ToUpper(p.current().String)) + return !containsFold(reservedKeywords, p.current().String) } } return false @@ -118,7 +118,8 @@ func (p *Parser) tryConsumeTokenKind(kind TokenKind) *Token { } func (p *Parser) matchKeyword(keyword string) bool { - return p.matchTokenKind(TokenKindKeyword) && strings.EqualFold(p.current().String, keyword) + token := p.current() + return token != nil && token.Kind == TokenKindKeyword && strings.EqualFold(token.String, keyword) } func (p *Parser) matchOneOfKeywords(keywords ...string) bool { diff --git a/parser/parser_table.go b/parser/parser_table.go index 72156a9..4e90da4 100644 --- a/parser/parser_table.go +++ b/parser/parser_table.go @@ -3,7 +3,6 @@ package parser import ( "errors" "fmt" - "strings" ) func (p *Parser) parseDDL(pos Pos) (DDL, error) { @@ -435,7 +434,7 @@ func (p *Parser) parseIdentOrFunction(_ Pos) (Expr, error) { }, nil case p.matchTokenKind(TokenKindLParen): var params *ParamExprList - if form, ok := keywordArgFunctions[strings.ToUpper(ident.Name)]; ok { + if form, ok := lookupFold(keywordArgFunctions, ident.Name); ok { params, err = p.parseKeywordArgFunctionParams(p.Pos(), form) } else { params, err = p.parseFunctionParams(p.Pos()) diff --git a/parser/set.go b/parser/set.go index f40166a..84081b7 100644 --- a/parser/set.go +++ b/parser/set.go @@ -1,5 +1,10 @@ package parser +import ( + "strings" + "unicode/utf8" +) + type Set[T comparable] struct { m map[T]struct{} } @@ -32,3 +37,46 @@ func (s *Set[T]) Members() []T { } return members } + +// maxFoldLen bounds the stack buffer used by lookupFold and containsFold; +// longer inputs fall back to strings.ToUpper. +const maxFoldLen = 64 + +// upperASCII writes the upper-cased form of s into buf and reports whether it +// could do so without allocating. It refuses non-ASCII input so callers can +// fall back to strings.ToUpper and keep identical semantics. +func upperASCII(s string, buf *[maxFoldLen]byte) bool { + if len(s) > len(buf) { + return false + } + for i := 0; i < len(s); i++ { + c := s[i] + if c >= utf8.RuneSelf { + return false + } + if 'a' <= c && c <= 'z' { + c -= 'a' - 'A' + } + buf[i] = c + } + return true +} + +// lookupFold indexes m by the upper-cased form of s. Indexing with +// string(buf[:n]) lets the compiler skip the string allocation that +// strings.ToUpper would need for every lower-case identifier. +func lookupFold[V any](m map[string]V, s string) (V, bool) { + var buf [maxFoldLen]byte + if !upperASCII(s, &buf) { + v, ok := m[strings.ToUpper(s)] + return v, ok + } + v, ok := m[string(buf[:len(s)])] + return v, ok +} + +// containsFold reports whether the upper-cased form of s is a member of set. +func containsFold(set *Set[string], s string) bool { + _, ok := lookupFold(set.m, s) + return ok +}