From be7b73d8ad134137e2e38c3da712b3e95559366c Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Sat, 17 Oct 2026 07:59:11 -0400 Subject: [PATCH] Add independent post-search highlighting without positional storage Map sampled query term matches back to original grapheme ranges, return plain and escaped HTML snippets, and expose an optional document helper without changing search result types. No database reads or index migration are needed. Implements #11 originally requested by @luizkowalski. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .rubocop_todo.yml | 14 +-- CHANGELOG.md | 1 + README.md | 49 +++++++++ lib/mongoid/full_text_search.rb | 23 +++++ lib/mongoid/full_text_search/highlighter.rb | 104 ++++++++++++++++++++ lib/mongoid/full_text_search/ngrams.rb | 4 + spec/mongoid/highlighting_spec.rb | 66 +++++++++++++ spec/unit/highlighter_spec.rb | 90 +++++++++++++++++ 8 files changed, 344 insertions(+), 7 deletions(-) create mode 100644 lib/mongoid/full_text_search/highlighter.rb create mode 100644 spec/mongoid/highlighting_spec.rb create mode 100644 spec/unit/highlighter_spec.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 8453e50..13e79ff 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2026-10-17 11:40:34 UTC using RuboCop version 1.92.0. +# on 2026-10-17 12:00:05 UTC using RuboCop version 1.92.0. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -44,23 +44,23 @@ Lint/UselessAssignment: Exclude: - 'spec/mongoid/full_text_search_spec.rb' -# Offense count: 10 +# Offense count: 14 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes. Metrics/AbcSize: Max: 98 -# Offense count: 17 +# Offense count: 19 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. # AllowedMethods: refine Metrics/BlockLength: Max: 644 -# Offense count: 8 +# Offense count: 12 # Configuration parameters: AllowedMethods, AllowedPatterns. Metrics/CyclomaticComplexity: Max: 21 -# Offense count: 10 +# Offense count: 14 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. Metrics/MethodLength: Max: 53 @@ -70,7 +70,7 @@ Metrics/MethodLength: Metrics/ModuleLength: Max: 200 -# Offense count: 7 +# Offense count: 11 # Configuration parameters: AllowedMethods, AllowedPatterns. Metrics/PerceivedComplexity: Max: 23 @@ -154,7 +154,7 @@ Style/SymbolProc: Exclude: - 'spec/models/filtered_artwork.rb' -# Offense count: 27 +# Offense count: 30 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings. # URISchemes: http, https diff --git a/CHANGELOG.md b/CHANGELOG.md index 3df7ef7..9e78ae5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### 0.9.0 (Next) +* [#67](https://github.com/mongoid/mongoid_fulltext/pull/67): Add `fulltext_search_with_highlights` with complete highlighted fields, bold default markup and custom rendering blocks without positional index storage - [@dblock](https://github.com/dblock). * [#68](https://github.com/mongoid/mongoid_fulltext/pull/68): Cover candidate-budget boundaries and combine coverage from actual legacy and modern drivers - [@dblock](https://github.com/dblock). * [#61](https://github.com/mongoid/mongoid_fulltext/pull/61): Support weighted fields with configurable maximum or summed field-score aggregation - [@dblock](https://github.com/dblock). * [#60](https://github.com/mongoid/mongoid_fulltext/pull/60): Extract ngram calculation and index-definition planning for maintainability without changing model APIs or search behavior - [@tomasc](https://github.com/tomasc), [@dblock](https://github.com/dblock). diff --git a/README.md b/README.md index 5f6bb6f..67cf534 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,55 @@ Weights multiply stored document-term scores only; query-term scoring and candid The rough score thresholds above describe unweighted search, not weighted or summed scores. Higher weights or additional matching fields can outweigh a more exact match, and candidate limits still make search approximate. No per-field metadata or additional records are stored. +### Highlighting + +Search and get complete highlighted field values, without storing positional metadata or rebuilding indexes: + +```ruby +Artwork.fulltext_search_with_highlights('ghost', fields: [:title]).each do |result| + result.document # the original artwork + result.score # its search score + result.highlights[:title] # "A ghost story" +end +``` + +`fulltext_search_with_highlights` returns an array of results exposing `document`, `score`, and `highlights`. Search ordering, criteria, filters and `max_results` work as in `fulltext_search`, whose return values are unchanged. `fields` selects indexed fields to display; by default all fields in the selected index are included. Unknown fields raise `ArgumentError`. Pass `index: 'index_name'` for models with multiple indexes. + +`highlights` is a hash keyed by field name. Scalar values become complete HTML-escaped strings with `` and `` around matches; array values become arrays preserving their order and nil elements. Unmatched values are escaped and returned unchanged otherwise, so a title is still available when another field matched. Nil scalar values remain nil. Text is never truncated or split into snippets. Documents are not modified. Computed readers and the default `to_s` are supported. Localized fields use the current `I18n.locale` and Mongoid's configured fallbacks. + +Pass a block to render each merged match with custom markup: + +```ruby +Artwork.fulltext_search_with_highlights('ghost', fields: [:title]) { |match| "#{match}" } +``` + +The callback receives the HTML-escaped original match and must return a string. Overlapping and adjacent matches are merged before rendering. Surrounding text is escaped automatically. Returned markup is inserted verbatim, so use trusted markup and escape any additional untrusted content you introduce. Output is for HTML element content, not attributes, scripts or URLs. The same callback is supported by `Highlighter.highlight`: + +```ruby +Mongoid::FullTextSearch::Highlighter.highlight(text, query, config) { |match| "#{match}" } +``` + +For an already-loaded document, `artwork.fulltext_highlights('ghost', fields: [:title])` returns the field hash directly, without searching. + +The independent module can also be loaded without Mongoid or a database: + +```ruby +require 'mongoid/full_text_search/highlighter' + +config = { + alphabet: /[a-z]/, word_separators: { ' ' => true }, + ngram_width: 3, max_ngrams_to_search: 6, remove_accents: true, + apply_prefix_scoring_to_all_words: true, index_full_words: true, + index_short_prefixes: false, stop_words: {} +} +Mongoid::FullTextSearch::Highlighter.highlight('A ghost story', 'ghost', config) +# => 'A ghost story' +``` + +When using the module with a model, pass that index's compiled configuration, such as `Artwork.mongoid_fulltext_config.fetch(index_name)`. It uses the same bounded query ngrams and normalization as search and considers only terms that would be indexed from that text. Accents, decomposed characters, and discarded punctuation map back to the original text rather than normalized display text. + +Highlighting scans only the text you pass to it; it does not consult the database, change search return types, or explain which occurrence contributed to a stored score. A stale index can return a document with no current highlight. Partial ngram matches can produce partial-word highlights, and query sampling can leave portions unmarked. Keep application authorization and field-display policy separate; avoid highlighting internal or sensitive fields merely because they are indexed. For individual public fields, call the module directly. + If you don't specify a field to index, the default is the result of `to_s` called on the object. The following definition will index the first and last name of an artist: diff --git a/lib/mongoid/full_text_search.rb b/lib/mongoid/full_text_search.rb index 7643f1b..9ad53bd 100644 --- a/lib/mongoid/full_text_search.rb +++ b/lib/mongoid/full_text_search.rb @@ -11,16 +11,19 @@ require 'mongoid/full_text_search/ngrams' require 'mongoid/full_text_search/index_definition' require 'mongoid/full_text_search/field_scores' +require 'mongoid/full_text_search/highlighter' module Mongoid::FullTextSearch extend ActiveSupport::Concern included do cattr_accessor :mongoid_fulltext_config + extend HighlightedSearch end class UnspecifiedIndexError < StandardError; end class UnknownFilterQueryOperator < StandardError; end + HighlightedResult = Struct.new(:document, :highlights, :score) module ClassMethods def fulltext_search_in(*args) @@ -273,6 +276,26 @@ def format_query_filter(operator, key, value) end end + def fulltext_highlights(query, options = {}, &block) + index = options.fetch(:index) do + raise UnspecifiedIndexError, 'Specify :index for a model with multiple full-text indexes' if mongoid_fulltext_config.length > 1 + + mongoid_fulltext_config.keys.first + end + config = mongoid_fulltext_config.fetch(index) + fields = Array(options.fetch(:fields, config[:ngram_fields])).map(&:to_sym) + raise ArgumentError, ':fields must contain indexed field names' unless (fields - config[:ngram_fields].map(&:to_sym)).empty? + + fields.each_with_object({}) do |field, highlights| + value = send(field) + highlights[field] = if value.is_a?(Array) + value.map { |text| Highlighter.highlight(text, query, config, &block) } + else + Highlighter.highlight(value, query, config, &block) + end + end + end + def update_ngram_index self.class.each_fulltext_index do |index_name, fulltext_config, locale| if condition = fulltext_config[:update_if] diff --git a/lib/mongoid/full_text_search/highlighter.rb b/lib/mongoid/full_text_search/highlighter.rb new file mode 100644 index 0000000..cf84f72 --- /dev/null +++ b/lib/mongoid/full_text_search/highlighter.rb @@ -0,0 +1,104 @@ +require 'cgi' +require_relative 'ngrams' + +module Mongoid + module FullTextSearch + # Adds display values to ranked search results without modifying documents. + module HighlightedSearch + def fulltext_search_with_highlights(query_string, options = {}, &block) + search_options = options.dup + fields = search_options.delete(:fields) + index = search_options.fetch(:index) do + raise UnspecifiedIndexError, 'Specify :index for a model with multiple full-text indexes' if mongoid_fulltext_config.length > 1 + + mongoid_fulltext_config.keys.first + end + indexed_fields = mongoid_fulltext_config.fetch(index)[:ngram_fields] + fields = (fields.nil? ? indexed_fields : Array(fields)).map(&:to_sym) + raise ArgumentError, ':fields must contain indexed field names' unless (fields - indexed_fields.map(&:to_sym)).empty? + + search_options[:return_scores] = true + fulltext_search(query_string, search_options).map do |document, score| + highlights = document.fulltext_highlights(query_string, index: index, fields: fields, &block) + HighlightedResult.new(document, highlights, score) + end + end + end + + # Highlights current text without storing positional index metadata. + module Highlighter + module_function + + def highlight(text, query, config, &block) + return nil if text.nil? + + terms = Ngrams.calculate(query, config).keys.reject(&:empty?) + return CGI.escapeHTML(text) if terms.empty? + + normalized, positions = normalized_positions(text, config) + indexed_terms = Ngrams.calculate(text, config, false) + ranges = matching_ranges(normalized, terms & indexed_terms.keys, config).map do |start, finish| + [positions[start][0], positions[finish - 1][1]] + end + merged = merge_ranges(ranges) + render(text, merged, &block) + end + + def normalized_positions(text, config) + positions = [] + offset = 0 + UnicodeUtils.each_grapheme(text) do |grapheme| + part = Ngrams.normalized_text(grapheme, config) + part.length.times { positions << [offset, offset + grapheme.length] } + offset += grapheme.length + end + # Whole-string case conversion can be contextual; preserve its actual characters. + whole = Ngrams.normalized_text(text, config) + raise ArgumentError, 'Normalization changed character counts across grapheme boundaries' unless whole.length == positions.length + + [whole, positions] + end + + def matching_ranges(text, terms, config) + separators = config[:word_separators] + terms.each_with_object([]) do |term, ranges| + offset = 0 + while (start = text.index(term, offset)) + finish = start + term.length + word_start = start.zero? || separators.key?(text[start - 1]) + word_end = finish == text.length || separators.key?(text[finish]) + window = term.length == config[:ngram_width] + prefix = config[:index_short_prefixes] && term.length == config[:ngram_width] - 1 && word_start + word = config[:index_full_words] && word_start && word_end + ranges << [start, finish] if window || prefix || word + offset = start + 1 + end + end + end + + def merge_ranges(ranges) + ranges.sort.each_with_object([]) do |range, merged| + if !merged.empty? && range[0] <= merged.last[1] + merged.last[1] = [merged.last[1], range[1]].max + else + merged << range.dup + end + end + end + + def render(text, ranges) + html = '' + cursor = 0 + ranges.each do |first, last| + html << CGI.escapeHTML(text[cursor...first]) + match = CGI.escapeHTML(text[first...last]) + html << (block_given? ? yield(match) : "#{match}") + cursor = last + end + html << CGI.escapeHTML(text[cursor..-1]) + end + + private_class_method :normalized_positions, :matching_ranges, :merge_ranges, :render + end + end +end diff --git a/lib/mongoid/full_text_search/ngrams.rb b/lib/mongoid/full_text_search/ngrams.rb index 749ccd8..6032e84 100644 --- a/lib/mongoid/full_text_search/ngrams.rb +++ b/lib/mongoid/full_text_search/ngrams.rb @@ -31,6 +31,10 @@ def calculate(str, config, bound_number_returned = true) Hash[ngram_array.group_by { |h| h[:ngram] }.map { |key, values| [key, values.map { |v| v[:score] }.sum] }] end + def normalized_text(str, config) + normalize(str, config) + end + def normalize(str, config) if config[:remove_accents] if defined?(UnicodeUtils) diff --git a/spec/mongoid/highlighting_spec.rb b/spec/mongoid/highlighting_spec.rb new file mode 100644 index 0000000..7cf444e --- /dev/null +++ b/spec/mongoid/highlighting_spec.rb @@ -0,0 +1,66 @@ +require 'spec_helper' + +describe 'Search with highlights' do + it 'returns ranked documents, scores and complete highlighted fields without changing the index' do + first = TaggedArtwork.create!(title: 'Ghost story', tags: [nil, 'ghost', 'painting']) + TaggedArtwork.create!(title: 'Other & painting', tags: ['ghost']) + index = TaggedArtwork.mongoid_fulltext_config.keys.first + collection = first.collection.database[index] + before = collection.find.to_a + expected = TaggedArtwork.fulltext_search('ghost', return_scores: true) + options = { fields: %i[title tags] } + results = TaggedArtwork.fulltext_search_with_highlights('ghost', options) + + expect(results.map { |result| [result.document, result.score] }).to eq(expected) + expect(results.find { |result| result.document == first }.highlights) + .to eq(title: 'Ghost story', tags: [nil, 'ghost', 'painting']) + expect(results.find { |result| result.document.title == 'Other & painting' }.highlights[:title]).to eq('Other & painting') + expect(options).to eq(fields: %i[title tags]) + expect(first.title).to eq('Ghost story') + expect(collection.find.to_a).to eq(before) + end + + it 'accepts a renderer and restricts display fields' do + TaggedArtwork.create!(title: 'Ghost story', tags: ['ghost']) + result = TaggedArtwork.fulltext_search_with_highlights('ghost', fields: ['title']) { |match| "#{match}" }.first + expect(result.highlights).to eq(title: 'Ghost story') + end + + it 'preserves search limits and criteria' do + selected = BasicArtwork.create!(title: 'ghost selected') + BasicArtwork.create!(title: 'ghost other') + results = BasicArtwork.where(id: selected.id).fulltext_search_with_highlights('ghost', max_results: 1) + expect(results.map(&:document)).to eq([selected]) + expect(BasicArtwork.fulltext_search_with_highlights('unmatched')).to eq([]) + end + + it 'uses the current translation and computed readers' do + original_locales = I18n.available_locales + begin + I18n.available_locales = %i[en cs] + artwork = LocalizedArtwork.create!(title_translations: { 'en' => 'Flowers', 'cs' => 'Kvetiny' }) + I18n.with_locale(:cs) do + result = LocalizedArtwork.fulltext_search_with_highlights('kvetiny').first + expect(result.document).to eq(artwork) + expect(result.highlights[:title]).to eq('Kvetiny') + expect(artwork.fulltext_highlights('cesky')[:description]).to eq('Cesky') + expect(I18n.locale).to eq(:cs) + end + ensure + I18n.available_locales = original_locales + end + end + + it 'requires an index when ambiguous and rejects unknown indexes or fields' do + expect { MultiExternalArtwork.fulltext_search_with_highlights('ghost') }.to raise_error(Mongoid::FullTextSearch::UnspecifiedIndexError) + expect { BasicArtwork.fulltext_search_with_highlights('ghost', index: 'missing') }.to raise_error(KeyError) + expect { BasicArtwork.fulltext_search_with_highlights('ghost', fields: [:missing]) }.to raise_error(ArgumentError) + end + + it 'uses the selected index' do + artwork = MultiExternalArtwork.create!(title: 'ghost', artist: 'ghost', year: 'ghost') + result = MultiExternalArtwork.fulltext_search_with_highlights('ghost', index: 'mongoid_fulltext.titles').first + expect(result.document).to eq(artwork) + expect(result.highlights).to eq(title: 'ghost') + end +end diff --git a/spec/unit/highlighter_spec.rb b/spec/unit/highlighter_spec.rb new file mode 100644 index 0000000..d9085d3 --- /dev/null +++ b/spec/unit/highlighter_spec.rb @@ -0,0 +1,90 @@ +require 'rspec' +require 'mongoid/full_text_search/highlighter' + +describe Mongoid::FullTextSearch::Highlighter do + let(:config) do + { + alphabet: Hash[('a'..'z').map { |character| [character, true] }], + word_separators: { ' ' => true, '-' => true }, + ngram_width: 3, max_ngrams_to_search: 6, remove_accents: true, + apply_prefix_scoring_to_all_words: true, index_full_words: true, + index_short_prefixes: false, stop_words: { 'the' => true } + } + end + + it 'highlights complete text and escapes source HTML' do + expect(described_class.highlight('A GHOST & ', 'ghost', config)) + .to eq('A GHOST & <script>alert(1)</script>') + end + + it 'preserves composed and decomposed graphemes' do + %w[Café Café].each do |text| + expect(described_class.highlight(text, 'cafe', config)).to eq("#{text}") + end + end + + it 'includes discarded punctuation inside a merged match' do + expect(described_class.highlight('g.h.o.s.t', 'ghost', config)).to eq('g.h.o.s.t') + end + + it 'supports Unicode alphabets' do + options = config.merge(alphabet: /[\p{L}\p{M}\p{N}]/) + expect(described_class.highlight("\u4e2d\u6587 Caf\u00e9", 'cafe', options)).to eq("\u4e2d\u6587 Caf\u00e9") + end + + it 'highlights all distant matches without truncating text' do + text = "ghost#{' ' * 200}ghost" + expect(described_class.highlight(text, 'ghost', config)).to eq("ghost#{' ' * 200}ghost") + end + + it 'passes escaped merged matches to a custom renderer' do + matches = [] + html = described_class.highlight('gost & ghost', 'ghost', config) do |match| + matches << match + "#{match}" + end + expect(matches).to eq(['g<h>ost', 'ghost']) + expect(html).to eq('g<h>ost & ghost') + end + + it 'preserves nil and escapes unmatched or empty-query text without calling the renderer' do + expect(described_class.highlight(nil, 'ghost', config)).to be_nil + [nil, '', '!!!', 'ghost'].each do |query| + expect(described_class.highlight('art & painting', query, config) { raise 'unexpected callback' }).to eq('art & painting') + end + expect(described_class.highlight('', 'ghost', config)).to eq('') + end + + it 'propagates renderer errors' do + expect do + described_class.highlight('ghost', 'ghost', config) { raise 'renderer failed' } + end.to raise_error(RuntimeError, 'renderer failed') + end + + it 'respects bounded query samples' do + options = config.merge(max_ngrams_to_search: 2, index_full_words: false) + expect(described_class.highlight('def', 'abcdefghijk', options)).to eq('def') + expect(described_class.highlight('abc and ijk', 'abcdefghijk', options)).to eq('abc and ijk') + end + + it 'respects full-word and short-prefix boundaries' do + expect(described_class.highlight('bob bobcat', 'bob', config.merge(ngram_width: 6))).to eq('bob bobcat') + options = config.merge(ngram_width: 4, index_full_words: false, index_short_prefixes: true) + expect(described_class.highlight('abcat cater', 'cat', options)).to eq('abcat cater') + end + + it 'respects stop words while retaining sliding-window matches' do + expect(described_class.highlight('the cat', 'the', config.merge(ngram_width: 4))).to eq('the cat') + expect(described_class.highlight('the cat', 'the', config)).to eq('the cat') + end + + it 'does not mutate inputs' do + text = 'A ghost'.freeze + query = 'ghost'.freeze + original = Marshal.dump(config) + described_class.highlight(text, query, config.freeze) + expect(text).to eq('A ghost') + expect(query).to eq('ghost') + expect(Marshal.dump(config)).to eq(original) + end +end