From 07a6ee1c62e188cb4557fd006ac0de6648c51ce7 Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Fri, 16 Oct 2026 22:48:54 -0400 Subject: [PATCH] Bound query ngram samples and include both endpoints Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .rubocop_todo.yml | 8 ++--- CHANGELOG.md | 1 + README.md | 2 +- UPGRADING.md | 8 +++++ lib/mongoid/full_text_search.rb | 19 ++++++----- spec/mongoid/ngram_sampling_spec.rb | 50 +++++++++++++++++++++++++++++ 6 files changed, 73 insertions(+), 15 deletions(-) create mode 100644 spec/mongoid/ngram_sampling_spec.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 2f8b484..d325b57 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2026-10-17 03:29:20 UTC using RuboCop version 1.92.0. +# on 2026-10-17 03:36:52 UTC using RuboCop version 1.92.0. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -49,7 +49,7 @@ Lint/UselessAssignment: Metrics/AbcSize: Max: 111 -# Offense count: 10 +# Offense count: 13 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. # AllowedMethods: refine Metrics/BlockLength: @@ -58,7 +58,7 @@ Metrics/BlockLength: # Offense count: 5 # Configuration parameters: AllowedMethods, AllowedPatterns. Metrics/CyclomaticComplexity: - Max: 34 + Max: 35 # Offense count: 5 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. @@ -153,7 +153,7 @@ Style/SymbolProc: Exclude: - 'spec/models/filtered_artwork.rb' -# Offense count: 21 +# Offense count: 22 # This cop supports safe autocorrection (--autocorrect). # Configuration parameters: AllowHeredoc, AllowURI, AllowQualifiedName, URISchemes, AllowRBSInlineAnnotation, AllowCopDirectives, AllowedPatterns, SplitStrings. # URISchemes: http, https diff --git a/CHANGELOG.md b/CHANGELOG.md index c2ae7fa..97da880 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ * [#56](https://github.com/mongoid/mongoid_fulltext/pull/56): Support Unicode-aware regular expressions for alphabets without enumerating characters - [@dblock](https://github.com/dblock). * [#54](https://github.com/mongoid/mongoid_fulltext/pull/54): Insert ngrams in a batch per document index and continue past empty indexes - [@dblock](https://github.com/dblock). * [#53](https://github.com/mongoid/mongoid_fulltext/pull/53): Support indexing array fields containing strings - [@dblock](https://github.com/dblock). +* [#55](https://github.com/mongoid/mongoid_fulltext/pull/55): Correct query ngram sampling to respect the configured limit and cover both ends of the query - [@dblock](https://github.com/dblock). * [#51](https://github.com/mongoid/mongoid_fulltext/pull/51): Modernize CI through Ruby 4.0, Mongoid 9, and MongoDB 8 with real coverage, RuboCop 1.92, and fork-safe Danger workflows - [@dblock](https://github.com/dblock). * [#45](https://github.com/mongoid/mongoid_fulltext/pull/45): Allow for class level ngram updates to not timeout - [@roykolak](https://github.com/roykolak). * Your contribution here. diff --git a/README.md b/README.md index 34db79a..f77cd04 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,7 @@ Additional indexing/query options can be used as parameters to `fulltext_search_ * `stop_words`: a hash of words to avoid indexing as full words. Used only if `index_full_words` is set to `true`. Defaults to a hash containing a list of common English stop words. * `apply_prefix_scoring_to_all_words`: score n-grams at beginning of words higher, default is `true`. -* `max_ngrams_to_search`: maximum number of ngrams to query at any given time, default is `6`. +* `max_ngrams_to_search`: positive integer limiting sampled sliding-window ngrams, default is `6`. Samples span the query including the first and last windows when at least two are selected; a limit of one selects the first window. Full-word and short-prefix bonuses are added separately and may increase the total number of queried terms. * `max_candidate_set_size`: maximum number of candidate ngrams to examine for a given query. Defaults to 1000. If you're seeing poor results, you can try increasing this value to consider more ngrams per query (changing this parameter does not require a re-index.) The amount of time diff --git a/UPGRADING.md b/UPGRADING.md index ead629d..b2a9ec4 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -2,6 +2,14 @@ ## 0.9.0 +### Query ngram sampling + +Query sampling now respects `max_ngrams_to_search` and spans both ends of the query when selecting at least two sliding-window ngrams. For example, selecting three 3-character ngrams from `abcdefghijk` previously produced `abc`, `def`, and `ghi`; it now produces `abc`, `efg`, and `ijk`. A limit of one selects the first window. Full-word and short-prefix terms are added separately and can increase the total number of queried terms. + +The same query can return different matches or a different result order because scores are calculated from a different selection of query fragments. Review any application expectations that depend on exact matches, rankings, or score thresholds. No index rebuild is needed for this sampling change: document indexes already store every ngram, and only query-time selection changes. + +### Localized fields + Indexes that explicitly include a Mongoid localized field now store ngrams for all configured locales in the original index collection, distinguished by a string `locale` field. Previously only the active translation was indexed, without locale metadata. Searches now filter by locale, and the compound search index begins with `locale`, `ngram`, and `score`. Nonlocalized indexes are unchanged. Only models whose full-text index explicitly includes a localized field need this migration. Existing nonlocalized indexes require no changes. diff --git a/lib/mongoid/full_text_search.rb b/lib/mongoid/full_text_search.rb index 1643cc2..e51c7cb 100644 --- a/lib/mongoid/full_text_search.rb +++ b/lib/mongoid/full_text_search.rb @@ -58,6 +58,7 @@ def fulltext_search_in(*args) config.update(options) raise ArgumentError, ':alphabet must be a String or Regexp' unless config[:alphabet].is_a?(String) || config[:alphabet].is_a?(Regexp) + raise ArgumentError, ':max_ngrams_to_search must be a positive integer' unless config[:max_ngrams_to_search].is_a?(Integer) && config[:max_ngrams_to_search] > 0 args = [:to_s] if args.empty? config[:ngram_fields] = args @@ -258,22 +259,20 @@ def all_ngrams(str, config, bound_number_returned = true) (alphabet.is_a?(Regexp) ? alphabet =~ ch : alphabet[ch]) || config[:word_separators][ch] end.join('') - # Figure out how many ngrams to extract from the string. If we can't afford to extract all ngrams, - # step over the string in evenly spaced strides to extract ngrams. For example, to extract 3 3-letter - # ngrams from 'abcdefghijk', we'd want to extract 'abc', 'efg', and 'ijk'. - step_size = if bound_number_returned - [((filtered_str.length - config[:ngram_width]).to_f / config[:max_ngrams_to_search]).ceil, 1].max - else - 1 - end + # Sample across the full string, including both endpoints when taking multiple ngrams. + # Three 3-letter ngrams from 'abcdefghijk' are 'abc', 'efg', and 'ijk'. + last_start = filtered_str.length - config[:ngram_width] + ngram_count = last_start + 1 + ngram_count = [ngram_count, config[:max_ngrams_to_search]].min if bound_number_returned # Create an array of records of the form {:ngram => x, :score => y} for all ngrams that occur in the - # input string using the step size that we just computed. Let score(x,y) be the score of string x + # input string using the sample positions. Let score(x,y) be the score of string x # compared with string y - assigning scores to ngrams with the square root-based scoring function # below and multiplying scores of matching ngrams together yields a score function that has the # property that score(x,y) > score(x,z) for any string z containing y and score(x,y) > score(x,z) # for any string z contained in y. - ngram_array = (0..filtered_str.length - config[:ngram_width]).step(step_size).map do |i| + ngram_array = (0...ngram_count).map do |sample| + i = ngram_count == 1 ? 0 : (sample * last_start.to_f / (ngram_count - 1)).round score = if i == 0 || (config[:apply_prefix_scoring_to_all_words] && \ config[:word_separators].key?(filtered_str[i - 1].chr)) Math.sqrt(1 + 1.0 / filtered_str.length) diff --git a/spec/mongoid/ngram_sampling_spec.rb b/spec/mongoid/ngram_sampling_spec.rb new file mode 100644 index 0000000..3965dd6 --- /dev/null +++ b/spec/mongoid/ngram_sampling_spec.rb @@ -0,0 +1,50 @@ +require 'spec_helper' + +describe 'Bounded ngram sampling' do + let(:config) do + BasicArtwork.mongoid_fulltext_config.values.first.merge( + max_ngrams_to_search: 3, index_full_words: false, index_short_prefixes: false + ) + end + + it 'selects the documented evenly spaced sample including the last ngram' do + expect(BasicArtwork.all_ngrams('abcdefghijk', config).keys).to eq(%w[abc efg ijk]) + expect(BasicArtwork.all_ngrams('abcdefghij', config).keys).to eq(%w[abc efg hij]) + end + + it 'never exceeds the configured number of sliding-window ngrams' do + (1..12).each do |limit| + (0..26).each do |length| + text = 'abcdefghijklmnopqrstuvwxyz'[0, length] + ngrams = BasicArtwork.all_ngrams(text, config.merge(max_ngrams_to_search: limit)) + expected_count = [[length - config[:ngram_width] + 1, 0].max, limit].min + expect(ngrams.length).to eq(expected_count) + next unless expected_count > 1 + + expect(ngrams.keys.first).to eq(text[0, 3]) + expect(ngrams.keys.last).to eq(text[-3, 3]) + end + end + end + + it 'selects the prefix when only one ngram is requested' do + expect(BasicArtwork.all_ngrams('abcdefghijk', config.merge(max_ngrams_to_search: 1)).keys).to eq(['abc']) + end + + it 'still indexes every sliding window when the bound is disabled' do + expect(BasicArtwork.all_ngrams('abcdefghijk', config, false).keys).to eq(%w[abc bcd cde def efg fgh ghi hij ijk]) + end + + it 'preserves full-word and short-prefix bonuses in addition to sampled ngrams' do + options = config.merge(index_full_words: true, index_short_prefixes: true) + expect(BasicArtwork.all_ngrams('abcdefghijk', options).keys).to eq(%w[abc efg ijk ab abcdefghijk]) + end + + it 'rejects a nonpositive or noninteger search bound' do + [0, -1, 1.5, nil].each do |limit| + expect do + BasicArtwork.fulltext_search_in :title, max_ngrams_to_search: limit + end.to raise_error(ArgumentError, ':max_ngrams_to_search must be a positive integer') + end + end +end