feat(search): match Elasticsearch token semantics exactly (PPT-2644)

spec/text_search_spec.cr

@@ -0,0 +1,68 @@
+require "./helper"
+
+module PlaceOS::Api
+  describe Utils::TextSearch do
+    describe ".tsquery" do
+      it "treats nil, blank and wildcard input as match-all" do
+        Utils::TextSearch.tsquery(nil).should be_nil
+        Utils::TextSearch.tsquery("").should be_nil
+        Utils::TextSearch.tsquery("   ").should be_nil
+        Utils::TextSearch.tsquery("*").should be_nil
+        Utils::TextSearch.tsquery("& ! | ( ) ~ \" \\").should be_nil
+      end
+
+      it "prefix-matches a single token" do
+        Utils::TextSearch.tsquery("sydney").should eq "sydney:*"
+      end
+
+      it "OR-joins tokens, prefixing only the final one (Elasticsearch parity)" do
+        # the old simple_query_string OR-ed terms and Neuroplastic appended
+        # `*` to the query's last token only
+        Utils::TextSearch.tsquery("sydney room").should eq "sydney | room:*"
+        Utils::TextSearch.tsquery("main boardroom 4").should eq "main | boardroom | 4:*"
+      end
+
+      it "degrades ES field syntax into plain terms (Backoffice zone tag filter)" do
+        Utils::TextSearch.tsquery("tags:(+level AND +building)").should eq "level | building:*"
+      end
+
+      it "neutralises ES operators, quotes and boolean words" do
+        # `garbage:` reads as a field prefix and is stripped with it
+        Utils::TextSearch.tsquery(%(name:(+weird AND "syntax) | garbage:* ~)).should eq "weird | syntax:*"
+        Utils::TextSearch.tsquery("name^2 boost").should eq "name | 2 | boost:*"
+      end
+
+      it "keeps email addresses as a single quoted lexeme" do
+        # OR-splitting an address would match every user on the domain; the
+        # whole-address lexeme preserves the old field-scoped precision
+        Utils::TextSearch.tsquery("adele@example.onmicrosoft.com").should eq "'adele@example.onmicrosoft.com':*"
+        Utils::TextSearch.tsquery("meeting adele@example.com notes").should eq "meeting | 'adele@example.com' | notes:*"
+      end
+
+      it "cannot confuse literal input with the email handling (Greptile P2 on #446)" do
+        # the single-pass tokenizer has no placeholder namespace to collide with
+        Utils::TextSearch.tsquery("adele@example.com placeosemailtoken0x")
+          .should eq "'adele@example.com' | placeosemailtoken0x:*"
+        # an email immediately followed by a colon is not mistaken for a field prefix
+        Utils::TextSearch.tsquery("adele@example.com: hello")
+          .should eq "'adele@example.com' | hello:*"
+      end
+
+      it "splits hyphenated identifiers" do
+        Utils::TextSearch.tsquery("sys-abc123").should eq "sys | abc123:*"
+      end
+
+      it "keeps unicode terms without folding" do
+        Utils::TextSearch.tsquery("café").should eq "café:*"
+      end
+
+      it "caps token count and input length without erroring" do
+        many = (1..40).join(' ') { |i| "tok#{i}" }
+        query = Utils::TextSearch.tsquery(many).not_nil!
+        query.split(" | ").size.should eq Utils::TextSearch::MAX_TOKENS
+
+        Utils::TextSearch.tsquery("x" * 20_000).not_nil!.size.should be <= Utils::TextSearch::MAX_QUERY_CHARS + 2
+      end
+    end
+  end
+end

src/placeos-rest-api/utilities/text-search.cr

@@ -1,18 +1,19 @@
 module PlaceOS::Api
   # PPT-2644: translates the free-form `q` search param into a PostgreSQL
   # tsquery matched against the generated `search_vector` columns
-  # (see placeos-models migration 20260806100500000).
+  # (see placeos-models migration 20260810100500000).
   #
   # Guarantees:
   # - never raises, and the output can never produce a tsquery syntax error:
-  #   emitted tokens contain only letters/digits joined with `:*` and `&`
+  #   emitted tokens contain only letters/digits joined with `:*` and `|`
   # - Elasticsearch-era query syntax that clients still send (field prefixes
   #   like `tags:(+level AND +building)`, boolean operators, quotes, wildcards)
   #   degrades gracefully into plain terms instead of erroring
-  # - every token is a prefix match (parity with the trailing `*` the old
-  #   Neuroplastic layer appended to every query); tokens are ANDed, which is
-  #   equal-or-stricter than the old OR and matches what autocomplete UIs
-  #   expect (they intersect results client-side)
+  # - token semantics mirror the old Elasticsearch simple_query_string
+  #   exactly: tokens are OR-joined whole-word matches, with the FINAL token
+  #   a prefix match (Neuroplastic appended `*` to the query's last token) —
+  #   so a record matches when ANY term matches, and the term being typed
+  #   still autocompletes
   module Utils::TextSearch
     extend self
 
@@ -22,24 +23,42 @@ module PlaceOS::Api
     MAX_QUERY_CHARS = 512
     MAX_TOKENS      =  16
 
+    # Email addresses are kept as a single (quoted) lexeme rather than being
+    # split: the search vectors index every address both whole and tokenized,
+    # and whole-address matching is what preserves the precise user-lookup
+    # behavior clients had via the old field-scoped email phrase match —
+    # OR-splitting an address would match everyone on the same domain.
+    # The character class cannot match `'` or `\`, so the quoted lexeme can
+    # never break out of the tsquery syntax.
+    EMAIL = /[\p{L}\p{N}._%+-]+@[\p{L}\p{N}.-]+\.[\p{L}]{2,}/
+
+    # One ordered pass over the input: capture whole email addresses, consume
+    # `field:` prefixes (Backoffice's zone tag filter sends ES syntax like
+    # `tags:(+level AND +building)`) without emitting them, and collect plain
+    # word tokens. A single scan means no intermediate placeholder text, so no
+    # user-typed input can collide with the email handling.
+    TOKENIZER = /(#{EMAIL.source})|[\w.]+\s*:|([\p{L}\p{N}]+)/
+
     # Builds the argument for `to_tsquery('simple', ?)` from user input, or
     # returns `nil` when the input imposes no text filter (nil / blank / "*" /
     # nothing searchable) — ES treated those as match-all.
     def tsquery(q : String?) : String?
       return nil if q.nil?
       q = q[0, MAX_QUERY_CHARS] if q.size > MAX_QUERY_CHARS
 
-      # drop `field:` prefixes (Backoffice's zone tag filter sends ES syntax
-      # like `tags:(+level AND +building)`)
-      text = q.gsub(/[\w.]+\s*:/, ' ')
-
-      tokens = text
-        .split(/[^\p{L}\p{N}]+/, remove_empty: true)
-        .reject { |token| OPERATOR_WORDS.includes?(token.downcase) }
-        .first(MAX_TOKENS)
+      tokens = [] of String
+      q.scan(TOKENIZER) do |match|
+        break if tokens.size >= MAX_TOKENS
+        if address = match[1]?
+          tokens << "'#{address}'"
+        elsif word = match[2]?
+          tokens << word unless OPERATOR_WORDS.includes?(word.downcase)
+        end
+      end
 
       return nil if tokens.empty?
-      tokens.join(" & ") { |token| "#{token}:*" }
+      last = tokens.size - 1
+      tokens.map_with_index { |token, i| i == last ? "#{token}:*" : token }.join(" | ")
     end
   end
 end