diff --git a/spec/text_search_spec.cr b/spec/text_search_spec.cr new file mode 100644 index 00000000..2505e27c --- /dev/null +++ b/spec/text_search_spec.cr @@ -0,0 +1,68 @@ +require "./helper" + +module PlaceOS::Api + describe Utils::TextSearch do + describe ".tsquery" do + it "treats nil, blank and wildcard input as match-all" do + Utils::TextSearch.tsquery(nil).should be_nil + Utils::TextSearch.tsquery("").should be_nil + Utils::TextSearch.tsquery(" ").should be_nil + Utils::TextSearch.tsquery("*").should be_nil + Utils::TextSearch.tsquery("& ! | ( ) ~ \" \\").should be_nil + end + + it "prefix-matches a single token" do + Utils::TextSearch.tsquery("sydney").should eq "sydney:*" + end + + it "OR-joins tokens, prefixing only the final one (Elasticsearch parity)" do + # the old simple_query_string OR-ed terms and Neuroplastic appended + # `*` to the query's last token only + Utils::TextSearch.tsquery("sydney room").should eq "sydney | room:*" + Utils::TextSearch.tsquery("main boardroom 4").should eq "main | boardroom | 4:*" + end + + it "degrades ES field syntax into plain terms (Backoffice zone tag filter)" do + Utils::TextSearch.tsquery("tags:(+level AND +building)").should eq "level | building:*" + end + + it "neutralises ES operators, quotes and boolean words" do + # `garbage:` reads as a field prefix and is stripped with it + Utils::TextSearch.tsquery(%(name:(+weird AND "syntax) | garbage:* ~)).should eq "weird | syntax:*" + Utils::TextSearch.tsquery("name^2 boost").should eq "name | 2 | boost:*" + end + + it "keeps email addresses as a single quoted lexeme" do + # OR-splitting an address would match every user on the domain; the + # whole-address lexeme preserves the old field-scoped precision + Utils::TextSearch.tsquery("adele@example.onmicrosoft.com").should eq "'adele@example.onmicrosoft.com':*" + Utils::TextSearch.tsquery("meeting adele@example.com notes").should eq "meeting | 'adele@example.com' | notes:*" + end + + it "cannot confuse literal input with the email handling (Greptile P2 on #446)" do + # the single-pass tokenizer has no placeholder namespace to collide with + Utils::TextSearch.tsquery("adele@example.com placeosemailtoken0x") + .should eq "'adele@example.com' | placeosemailtoken0x:*" + # an email immediately followed by a colon is not mistaken for a field prefix + Utils::TextSearch.tsquery("adele@example.com: hello") + .should eq "'adele@example.com' | hello:*" + end + + it "splits hyphenated identifiers" do + Utils::TextSearch.tsquery("sys-abc123").should eq "sys | abc123:*" + end + + it "keeps unicode terms without folding" do + Utils::TextSearch.tsquery("café").should eq "café:*" + end + + it "caps token count and input length without erroring" do + many = (1..40).join(' ') { |i| "tok#{i}" } + query = Utils::TextSearch.tsquery(many).not_nil! + query.split(" | ").size.should eq Utils::TextSearch::MAX_TOKENS + + Utils::TextSearch.tsquery("x" * 20_000).not_nil!.size.should be <= Utils::TextSearch::MAX_QUERY_CHARS + 2 + end + end + end +end diff --git a/src/placeos-rest-api/utilities/text-search.cr b/src/placeos-rest-api/utilities/text-search.cr index 0e9508bc..ee0ab849 100644 --- a/src/placeos-rest-api/utilities/text-search.cr +++ b/src/placeos-rest-api/utilities/text-search.cr @@ -1,18 +1,19 @@ module PlaceOS::Api # PPT-2644: translates the free-form `q` search param into a PostgreSQL # tsquery matched against the generated `search_vector` columns - # (see placeos-models migration 20260806100500000). + # (see placeos-models migration 20260810100500000). # # Guarantees: # - never raises, and the output can never produce a tsquery syntax error: - # emitted tokens contain only letters/digits joined with `:*` and `&` + # emitted tokens contain only letters/digits joined with `:*` and `|` # - Elasticsearch-era query syntax that clients still send (field prefixes # like `tags:(+level AND +building)`, boolean operators, quotes, wildcards) # degrades gracefully into plain terms instead of erroring - # - every token is a prefix match (parity with the trailing `*` the old - # Neuroplastic layer appended to every query); tokens are ANDed, which is - # equal-or-stricter than the old OR and matches what autocomplete UIs - # expect (they intersect results client-side) + # - token semantics mirror the old Elasticsearch simple_query_string + # exactly: tokens are OR-joined whole-word matches, with the FINAL token + # a prefix match (Neuroplastic appended `*` to the query's last token) — + # so a record matches when ANY term matches, and the term being typed + # still autocompletes module Utils::TextSearch extend self @@ -22,6 +23,22 @@ module PlaceOS::Api MAX_QUERY_CHARS = 512 MAX_TOKENS = 16 + # Email addresses are kept as a single (quoted) lexeme rather than being + # split: the search vectors index every address both whole and tokenized, + # and whole-address matching is what preserves the precise user-lookup + # behavior clients had via the old field-scoped email phrase match — + # OR-splitting an address would match everyone on the same domain. + # The character class cannot match `'` or `\`, so the quoted lexeme can + # never break out of the tsquery syntax. + EMAIL = /[\p{L}\p{N}._%+-]+@[\p{L}\p{N}.-]+\.[\p{L}]{2,}/ + + # One ordered pass over the input: capture whole email addresses, consume + # `field:` prefixes (Backoffice's zone tag filter sends ES syntax like + # `tags:(+level AND +building)`) without emitting them, and collect plain + # word tokens. A single scan means no intermediate placeholder text, so no + # user-typed input can collide with the email handling. + TOKENIZER = /(#{EMAIL.source})|[\w.]+\s*:|([\p{L}\p{N}]+)/ + # Builds the argument for `to_tsquery('simple', ?)` from user input, or # returns `nil` when the input imposes no text filter (nil / blank / "*" / # nothing searchable) — ES treated those as match-all. @@ -29,17 +46,19 @@ module PlaceOS::Api return nil if q.nil? q = q[0, MAX_QUERY_CHARS] if q.size > MAX_QUERY_CHARS - # drop `field:` prefixes (Backoffice's zone tag filter sends ES syntax - # like `tags:(+level AND +building)`) - text = q.gsub(/[\w.]+\s*:/, ' ') - - tokens = text - .split(/[^\p{L}\p{N}]+/, remove_empty: true) - .reject { |token| OPERATOR_WORDS.includes?(token.downcase) } - .first(MAX_TOKENS) + tokens = [] of String + q.scan(TOKENIZER) do |match| + break if tokens.size >= MAX_TOKENS + if address = match[1]? + tokens << "'#{address}'" + elsif word = match[2]? + tokens << word unless OPERATOR_WORDS.includes?(word.downcase) + end + end return nil if tokens.empty? - tokens.join(" & ") { |token| "#{token}:*" } + last = tokens.size - 1 + tokens.map_with_index { |token, i| i == last ? "#{token}:*" : token }.join(" | ") end end end