From 02fadba23e9d702c4e1e000211b41d3abf4afa8d Mon Sep 17 00:00:00 2001 From: "Daniel (dB.) Doubrovkine" Date: Fri, 16 Oct 2026 22:50:14 -0400 Subject: [PATCH] Document and cover configured non-Latin alphabets Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .rubocop_todo.yml | 12 +++---- CHANGELOG.md | 1 + README.md | 23 +++++++++++-- lib/mongoid/full_text_search.rb | 9 ++++-- spec/models/chinese_artwork.rb | 7 ++++ spec/models/multilingual_artwork.rb | 7 ++++ spec/mongoid/non_latin_spec.rb | 50 +++++++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 10 deletions(-) create mode 100644 spec/models/chinese_artwork.rb create mode 100644 spec/models/multilingual_artwork.rb create mode 100644 spec/mongoid/non_latin_spec.rb diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index fe707a5..2f8b484 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -1,6 +1,6 @@ # This configuration was generated by # `rubocop --auto-gen-config` -# on 2026-10-17 03:25:55 UTC using RuboCop version 1.92.0. +# on 2026-10-17 03:29:20 UTC using RuboCop version 1.92.0. # The point is for the user to remove these configuration records # one by one as the offenses are removed from the code base. # Note that changes in the inspected code, or installation of new @@ -47,9 +47,9 @@ Lint/UselessAssignment: # Offense count: 5 # Configuration parameters: AllowedMethods, AllowedPatterns, CountRepeatedAttributes. Metrics/AbcSize: - Max: 109 + Max: 111 -# Offense count: 9 +# Offense count: 10 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. # AllowedMethods: refine Metrics/BlockLength: @@ -58,7 +58,7 @@ Metrics/BlockLength: # Offense count: 5 # Configuration parameters: AllowedMethods, AllowedPatterns. Metrics/CyclomaticComplexity: - Max: 33 + Max: 34 # Offense count: 5 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns. @@ -68,12 +68,12 @@ Metrics/MethodLength: # Offense count: 1 # Configuration parameters: CountComments, CountAsOne. Metrics/ModuleLength: - Max: 258 + Max: 262 # Offense count: 5 # Configuration parameters: AllowedMethods, AllowedPatterns. Metrics/PerceivedComplexity: - Max: 36 + Max: 37 # Offense count: 1 Naming/ConstantName: diff --git a/CHANGELOG.md b/CHANGELOG.md index c9bd0a3..42e71a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * [#52](https://github.com/mongoid/mongoid_fulltext/pull/52): Fix full-text index creation before any documents exist - [@dblock](https://github.com/dblock). * [#58](https://github.com/mongoid/mongoid_fulltext/pull/58): Support localized fields with locale-tagged ngrams in one index collection; existing localized indexes must be rebuilt - [@tomasc](https://github.com/tomasc), [@dblock](https://github.com/dblock). * [#57](https://github.com/mongoid/mongoid_fulltext/pull/57): Respect criteria and single-collection inheritance, skip dangling references, and apply result limits after filtering - [@tomasc](https://github.com/tomasc), [@dblock](https://github.com/dblock). +* [#56](https://github.com/mongoid/mongoid_fulltext/pull/56): Support Unicode-aware regular expressions for alphabets without enumerating characters - [@dblock](https://github.com/dblock). * [#51](https://github.com/mongoid/mongoid_fulltext/pull/51): Modernize CI through Ruby 4.0, Mongoid 9, and MongoDB 8 with real coverage, RuboCop 1.92, and fork-safe Danger workflows - [@dblock](https://github.com/dblock). * [#45](https://github.com/mongoid/mongoid_fulltext/pull/45): Allow for class level ngram updates to not timeout - [@roykolak](https://github.com/roykolak). * Your contribution here. diff --git a/README.md b/README.md index ec7f523..4da7a07 100644 --- a/README.md +++ b/README.md @@ -266,9 +266,28 @@ Searches can be prefixed with Mongoid criteria, for example `Artwork.where(publi Single-collection inheritance is respected: a root-class search includes its descendants, while a subclass search excludes its parent and siblings. An unscoped search of an index shared by unrelated models still returns all those models. Adding criteria restricts the results to the calling model and its descendants. Existing index records already contain the class name, so no reindex is required. +The default `alphabet` only includes ASCII letters, digits, and spaces. Use a Unicode-aware regular expression to support whole scripts without enumerating their characters. For example, `/\p{Han}/` accepts Han characters, including those used in Chinese: + +```ruby +class ChineseArtwork + include Mongoid::Document + include Mongoid::FullTextSearch + + field :title, type: String + fulltext_search_in :title, alphabet: /\p{Han}/, ngram_width: 2 +end + +artwork = ChineseArtwork.create!(title: "\u5367\u864e\u85cf\u9f99") +ChineseArtwork.fulltext_search("\u85cf\u9f99") # => [artwork] +``` + +For multilingual text, use `alphabet: /[\p{L}\p{M}\p{N}]/` to accept Unicode letters, combining marks, and numbers, or `/[\p{Latin}\p{Cyrillic}]/` to restrict letters to those scripts. Existing string allowlists remain supported and the ASCII default is unchanged. + +The expression is tested against each character after accent normalization and lowercasing, not against whole words. Configured word separators are retained independently of the alphabet; other unmatched characters are discarded. `remove_accents: false` preserves combining marks and accents when needed. This is character-ngram matching, not language-aware word segmentation. A shorter `ngram_width` helps with short search terms. Changing the alphabet or ngram width requires rebuilding existing indexes with `ChineseArtwork.update_ngram_index`. Configuring the alphabet is separate from indexing Mongoid localized fields. + Additional indexing/query options can be used as parameters to `fulltext_search_in`. -* `alphabet`: letters to index, default is `abcdefghijklmnopqrstuvwxyz0123456789 `. +* `alphabet`: a string allowlist or a regular expression matched against each character, default is `abcdefghijklmnopqrstuvwxyz0123456789 `. * `word_separators`: word separators, default is the space character. * `ngram_width`: ngram width, default is `3`. * `index_full_words`: index full words, which improves exact matches, default is `true`. @@ -293,7 +312,7 @@ Additional indexing/query options can be used as parameters to `fulltext_search_ * When set to any other type of object, the document's index will not be updated. * `reindex_immediately`: whether models will be reindexed automatically upon saves or updates. Defaults to true. When set to false, the class-level `update_ngram_index` method can be called to perform reindexing. -If you work with Cyrillic texts, use this option: `:alphabet = 'abcdefghijklmnopqrstuvwxyz0123456789абвгдежзиклмнопрстуфхцчшщъыьэюя'`. +For Cyrillic and Latin text with numbers, use `alphabet: /[\p{Latin}\p{Cyrillic}\p{N}]/`. Array filters ------------- diff --git a/lib/mongoid/full_text_search.rb b/lib/mongoid/full_text_search.rb index 4d75496..9f49cd1 100644 --- a/lib/mongoid/full_text_search.rb +++ b/lib/mongoid/full_text_search.rb @@ -57,9 +57,11 @@ def fulltext_search_in(*args) config.update(options) + raise ArgumentError, ':alphabet must be a String or Regexp' unless config[:alphabet].is_a?(String) || config[:alphabet].is_a?(Regexp) + args = [:to_s] if args.empty? config[:ngram_fields] = args - config[:alphabet] = Hash[config[:alphabet].split('').map { |ch| [ch, ch] }] + config[:alphabet] = Hash[config[:alphabet].split('').map { |ch| [ch, ch] }] if config[:alphabet].is_a?(String) config[:word_separators] = Hash[config[:word_separators].split('').map { |ch| [ch, ch] }] mongoid_fulltext_config[index_name] = config @@ -251,7 +253,10 @@ def all_ngrams(str, config, bound_number_returned = true) end # Remove any characters that aren't in the alphabet and aren't word separators - filtered_str = str.mb_chars.downcase.to_s.split('').find_all { |ch| config[:alphabet][ch] || config[:word_separators][ch] }.join('') + alphabet = config[:alphabet] + filtered_str = str.mb_chars.downcase.to_s.split('').find_all do |ch| + (alphabet.is_a?(Regexp) ? alphabet =~ ch : alphabet[ch]) || config[:word_separators][ch] + end.join('') # Figure out how many ngrams to extract from the string. If we can't afford to extract all ngrams, # step over the string in evenly spaced strides to extract ngrams. For example, to extract 3 3-letter diff --git a/spec/models/chinese_artwork.rb b/spec/models/chinese_artwork.rb new file mode 100644 index 0000000..06fb1ff --- /dev/null +++ b/spec/models/chinese_artwork.rb @@ -0,0 +1,7 @@ +class ChineseArtwork + include Mongoid::Document + include Mongoid::FullTextSearch + + field :title, type: String + fulltext_search_in :title, alphabet: /\p{Han}/, ngram_width: 2 +end diff --git a/spec/models/multilingual_artwork.rb b/spec/models/multilingual_artwork.rb new file mode 100644 index 0000000..6472af9 --- /dev/null +++ b/spec/models/multilingual_artwork.rb @@ -0,0 +1,7 @@ +class MultilingualArtwork + include Mongoid::Document + include Mongoid::FullTextSearch + + field :title, type: String + fulltext_search_in :title, alphabet: /[\p{L}\p{M}\p{N}]/, ngram_width: 2, remove_accents: false +end diff --git a/spec/mongoid/non_latin_spec.rb b/spec/mongoid/non_latin_spec.rb new file mode 100644 index 0000000..7f7498c --- /dev/null +++ b/spec/mongoid/non_latin_spec.rb @@ -0,0 +1,50 @@ +require 'spec_helper' + +describe 'Non-Latin alphabets' do + it 'searches Chinese characters included in the configured alphabet' do + artwork = ChineseArtwork.create!(title: "\u5367\u864e\u85cf\u9f99") + expect(ChineseArtwork.fulltext_search("\u85cf\u9f99")).to eq([artwork]) + expect(ChineseArtwork.fulltext_search("\u5367\u864e")).to eq([artwork]) + end + + it 'does not index characters outside the configured alphabet' do + artwork = BasicArtwork.create!(title: "\u5367\u864e\u85cf\u9f99") + expect(BasicArtwork.fulltext_search(artwork.title)).to eq([]) + end + + it 'supports Han characters outside the example title without enumerating them' do + artwork = ChineseArtwork.create!(title: "\u4e2d\u6587\u641c\u7d22") + expect(ChineseArtwork.fulltext_search("\u641c\u7d22")).to eq([artwork]) + expect(ChineseArtwork.fulltext_search("\u4e2d\u6587")).to eq([artwork]) + expect(ChineseArtwork.fulltext_search('search')).to eq([]) + end + + it 'supports Unicode letters, combining marks, and numbers across scripts' do + artwork = MultilingualArtwork.create!(title: "Flowers \u0428\u0418\u0428\u041a\u0418\u041d \u4e2d\u6587 123") + %w[flowers шишкин 中文 123].each do |query| + expect(MultilingualArtwork.fulltext_search(query)).to eq([artwork]) + end + config = MultilingualArtwork.mongoid_fulltext_config.values.first.merge(index_full_words: false) + expect(MultilingualArtwork.all_ngrams("e\u0301", config).keys).to eq(["e\u0301"]) + end + + it 'retains word separators independently of the regular expression' do + config = ChineseArtwork.mongoid_fulltext_config.values.first.merge(index_full_words: false) + expect(ChineseArtwork.all_ngrams("\u4e2d-\u6587", config).keys).to eq(["\u4e2d-", "-\u6587"]) + expect(ChineseArtwork.all_ngrams("\u4e2d!\u6587", config).keys).to eq(["\u4e2d\u6587"]) + end + + it 'preserves explicit string allowlists' do + expect(RussianArtwork.mongoid_fulltext_config.values.first[:alphabet]).to be_a(Hash) + artwork = RussianArtwork.create!(title: "\u0428\u0438\u0448\u043a\u0438\u043d") + expect(RussianArtwork.fulltext_search("\u0448\u0438\u0448\u043a\u0438\u043d")).to eq([artwork]) + end + + it 'rejects unsupported alphabet types at declaration time' do + [nil, [], 123].each do |alphabet| + expect do + ChineseArtwork.fulltext_search_in :title, alphabet: alphabet + end.to raise_error(ArgumentError, ':alphabet must be a String or Regexp') + end + end +end