diff --git a/src/plugin/interface/ob_plugin_ftparser_intf.h b/src/plugin/interface/ob_plugin_ftparser_intf.h index c1eafdb8d7..97ab2a6ac7 100644 --- a/src/plugin/interface/ob_plugin_ftparser_intf.h +++ b/src/plugin/interface/ob_plugin_ftparser_intf.h @@ -101,6 +101,8 @@ class ObFTParserParam final : public ObFTParserParamExport public: ObFTParserParam() : ObFTParserParamExport(), + metadata_alloc_(nullptr), + scratch_alloc_(nullptr), ngram_token_size_(NGRAM_TOKEN_SIZE), min_ngram_size_(NGRAM_TOKEN_SIZE), max_ngram_size_(NGRAM_TOKEN_SIZE) @@ -112,13 +114,18 @@ class ObFTParserParam final : public ObFTParserParamExport { ObFTParserParamExport::reset(); allocator_ = nullptr; + metadata_alloc_ = nullptr; + scratch_alloc_ = nullptr; ngram_token_size_ = NGRAM_TOKEN_SIZE; } - INHERIT_TO_STRING_KV("base", ObFTParserParamExport, KP_(allocator), K_(ngram_token_size)); + INHERIT_TO_STRING_KV("base", ObFTParserParamExport, KP_(allocator), + KP_(metadata_alloc), KP_(scratch_alloc), K_(ngram_token_size)); public: common::ObIAllocator *allocator_ = nullptr; + common::ObIAllocator *metadata_alloc_; + common::ObIAllocator *scratch_alloc_; // ik parser params ObFTIKParam ik_param_; diff --git a/src/share/text_analysis/ob_token_stream.cpp b/src/share/text_analysis/ob_token_stream.cpp index 5fbb91076f..fd077e95bf 100644 --- a/src/share/text_analysis/ob_token_stream.cpp +++ b/src/share/text_analysis/ob_token_stream.cpp @@ -310,7 +310,14 @@ int ObBasicEnglishNormalizer::get_next(ObDatum &next_token, int64_t &token_freq) } else { ObString norm_alnum_token(norm_token_len, norm_token_ptr); ObString norm_lower_token; - if (OB_FAIL(ObCharset::tolower(cs_, norm_alnum_token, norm_lower_token, norm_allocator_))) { + bool is_lower_ascii = true; + for (int64_t i = 0; is_lower_ascii && i < norm_alnum_token.length(); ++i) { + const unsigned char ch = static_cast(norm_alnum_token.ptr()[i]); + is_lower_ascii = ch < 0x80 && !(ch >= 'A' && ch <= 'Z'); + } + if (is_lower_ascii) { + next_token.set_string(norm_alnum_token); + } else if (OB_FAIL(ObCharset::tolower(cs_, norm_alnum_token, norm_lower_token, norm_allocator_))) { LOG_WARN("norm token to lower case failed", K(ret), K_(cs), K(norm_alnum_token)); } else { next_token.set_string(norm_lower_token); diff --git a/src/sql/das/iter/ob_das_iter_utils.cpp b/src/sql/das/iter/ob_das_iter_utils.cpp index aca0e39732..7238366604 100644 --- a/src/sql/das/iter/ob_das_iter_utils.cpp +++ b/src/sql/das/iter/ob_das_iter_utils.cpp @@ -1661,15 +1661,19 @@ int ObDASIterUtils::create_text_retrieval_sub_tree( if (ir_scan_ctdef->has_pushdown_topk() && !has_duplicate_boolean_tokens) { merge_iter_param.topk_mode_ = 1; merge_iter_param.daat_mode_ = 1; - } else if (merge_iter_param.query_tokens_.count() > OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT - || (!is_func_lookup && !ir_scan_ctdef->need_proj_relevance_score())) { + } else if (merge_iter_param.query_tokens_.count() > OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT) { if (BOOLEAN_MODE == ir_scan_ctdef->mode_flag_) { ret = OB_NOT_SUPPORTED; LOG_WARN("boolean mode with too many tokens not supported", K(ret), K(merge_iter_param.query_tokens_.count())); LOG_USER_ERROR(OB_NOT_SUPPORTED, "Boolean mode with more than 256 tokens is"); } merge_iter_param.taat_mode_ = 1; + } else if (!is_func_lookup + && !ir_scan_ctdef->need_proj_relevance_score() + && BOOLEAN_MODE == ir_scan_ctdef->mode_flag_) { + merge_iter_param.taat_mode_ = 1; } else { + // Predicate-only natural-language MATCH can stream sorted posting lists. merge_iter_param.daat_mode_ = 1; } if (OB_SUCC(ret) && ir_scan_ctdef->has_pushdown_topk() && has_duplicate_boolean_tokens) { @@ -4426,4 +4430,3 @@ int ObDASIterUtils::create_vec_ivf_lookup_tree(ObTableScanParam &scan_param, } // namespace sql } // namespace oceanbase - diff --git a/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp b/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp index f5148c4e5f..2ff3280603 100644 --- a/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp +++ b/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp @@ -59,6 +59,17 @@ ObDASTRMergeIter::ObDASTRMergeIter() { } +bool ObDASTRMergeIter::skip_relevance_for_predicate() const +{ + return OB_NOT_NULL(ir_ctdef_) + && OB_NOT_NULL(ir_rtdef_) + && !topk_mode_ + && NATURAL_LANGUAGE_MODE == ir_ctdef_->mode_flag_ + && ir_rtdef_->minimum_should_match_ <= 1 + && OB_ISNULL(ir_ctdef_->relevance_proj_col_) + && OB_ISNULL(ir_ctdef_->match_filter_); +} + int ObDASTRMergeIter::inner_init(ObDASIterParam ¶m) { int ret = OB_SUCCESS; @@ -116,7 +127,8 @@ int ObDASTRMergeIter::init_das_iter_scan_params() { int ret = OB_SUCCESS; const int64_t dim_iter_cnt = taat_mode_ ? 1 : query_tokens_.count(); - if (!ir_ctdef_->need_estimate_total_doc_cnt() && 0 != dim_iter_cnt) { + const bool need_relevance = !skip_relevance_for_predicate(); + if (need_relevance && !ir_ctdef_->need_estimate_total_doc_cnt() && 0 != dim_iter_cnt) { void *buf = nullptr; if (OB_ISNULL(buf = myself_allocator_.alloc(sizeof(ObTableScanParam)))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -145,7 +157,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params() LOG_WARN("failed to prepare allocate inv scan params array", K(ret)); } - if (OB_FAIL(ret) || !ir_ctdef_->need_inv_idx_agg()) { + if (OB_FAIL(ret) || !need_relevance || !ir_ctdef_->need_inv_idx_agg()) { } else if (FALSE_IT(inv_agg_params_.set_allocator(&myself_allocator_))) { } else if (OB_FAIL(inv_agg_params_.init(dim_iter_cnt))) { LOG_WARN("failed to init inv agg params array", K(ret)); @@ -154,7 +166,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params() } if (OB_FAIL(ret)) { - } else if (!ir_ctdef_->need_fwd_idx_agg()) { + } else if (!need_relevance || !ir_ctdef_->need_fwd_idx_agg()) { } else if (OB_ISNULL(ir_rtdef_->eval_ctx_) || OB_UNLIKELY(ir_rtdef_->eval_ctx_->max_batch_size_ >= 1)) { ret = OB_NOT_IMPLEMENT; LOG_ERROR("fwd idx agg with max batch size not implemented", K(ret)); @@ -186,7 +198,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params() } } - if (OB_SUCC(ret) && ir_ctdef_->need_inv_idx_agg()) { + if (OB_SUCC(ret) && need_relevance && ir_ctdef_->need_inv_idx_agg()) { void *agg_buf = nullptr; if (OB_ISNULL(agg_buf = myself_allocator_.alloc(sizeof(ObTableScanParam) * dim_iter_cnt))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -210,7 +222,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params() } } - if (OB_SUCC(ret) && ir_ctdef_->need_fwd_idx_agg()) { + if (OB_SUCC(ret) && need_relevance && ir_ctdef_->need_fwd_idx_agg()) { void *fwd_buf = nullptr; if (OB_ISNULL(fwd_buf = myself_allocator_.alloc(sizeof(ObTableScanParam) * dim_iter_cnt))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -432,10 +444,11 @@ int ObDASTRMergeIter::init_dim_iter_param(ObTextRetrievalScanIterParam &iter_par ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null pointer", K(ret), KP_(ir_ctdef), KP_(ir_rtdef)); } else { + const bool skip_relevance = skip_relevance_for_predicate(); const int64_t dim_iter_cnt = taat_mode_ ? 1 : query_tokens_.count(); iter_param.inv_idx_scan_param_ = inv_scan_params_[idx]; iter_param.inv_idx_scan_iter_ = static_cast(children_[idx]); - if (ir_ctdef_->need_inv_idx_agg()) { + if (!skip_relevance && ir_ctdef_->need_inv_idx_agg()) { iter_param.inv_idx_agg_param_ = inv_agg_params_[idx]; iter_param.inv_idx_agg_iter_ = static_cast(children_[idx + dim_iter_cnt]); if (OB_UNLIKELY(1 != ir_ctdef_->get_inv_idx_agg_ctdef() @@ -447,7 +460,7 @@ int ObDASTRMergeIter::init_dim_iter_param(ObTextRetrievalScanIterParam &iter_par ->pd_expr_spec_.pd_storage_aggregate_output_.at(0); } } - if (OB_SUCC(ret) && ir_ctdef_->need_fwd_idx_agg()) { + if (OB_SUCC(ret) && !skip_relevance && ir_ctdef_->need_fwd_idx_agg()) { iter_param.fwd_idx_scan_param_ = fwd_scan_params_[idx]; iter_param.fwd_idx_agg_iter_ = static_cast(children_[idx + dim_iter_cnt * 2]); if (OB_UNLIKELY(1 != ir_ctdef_->get_fwd_idx_agg_ctdef() @@ -462,7 +475,7 @@ int ObDASTRMergeIter::init_dim_iter_param(ObTextRetrievalScanIterParam &iter_par iter_param.allocator_ = &myself_allocator_; iter_param.mem_context_ = mem_context_; iter_param.eval_ctx_ = ir_rtdef_->eval_ctx_; - iter_param.relevance_expr_ = ir_ctdef_->relevance_expr_; + iter_param.relevance_expr_ = skip_relevance ? nullptr : ir_ctdef_->relevance_expr_; iter_param.inv_scan_doc_length_col_ = ir_ctdef_->inv_scan_doc_length_col_; iter_param.inv_scan_domain_id_col_ = ir_ctdef_->inv_scan_domain_id_col_; iter_param.inv_idx_agg_cache_mode_ = function_lookup_mode_ && !taat_mode_; @@ -481,7 +494,8 @@ int ObDASTRMergeIter::create_sparse_retrieval_iter() sr_iter_param_.relevance_proj_expr_ = ir_ctdef_->relevance_proj_col_; sr_iter_param_.filter_expr_ = ir_ctdef_->match_filter_; sr_iter_param_.topk_limit_ = topk_limit_; - if (OB_NOT_NULL(ir_ctdef_->field_boost_expr_)) { + sr_iter_param_.skip_relevance_for_predicate_ = skip_relevance_for_predicate(); + if (sr_iter_param_.need_calc_relevance() && OB_NOT_NULL(ir_ctdef_->field_boost_expr_)) { ObDatum *boost_datum = nullptr; if (OB_FAIL(ir_ctdef_->field_boost_expr_->eval(*ir_rtdef_->eval_ctx_, boost_datum))) { LOG_WARN("failed to eval field boost expr", K(ret)); @@ -592,15 +606,22 @@ int ObDASTRMergeIter::init_daat_iter_param(ObTextDaaTParam &iter_param) iter_param.allocator_ = &myself_allocator_; iter_param.mode_flag_ = ir_ctdef_->mode_flag_; iter_param.function_lookup_mode_ = function_lookup_mode_; - iter_param.bm25_param_est_ctx_.total_doc_cnt_expr_ - = ir_ctdef_->get_doc_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0); - iter_param.bm25_param_est_ctx_.estimated_total_doc_cnt_ = ir_ctdef_->estimated_total_doc_cnt_; - iter_param.bm25_param_est_ctx_.need_est_avg_doc_token_cnt_ = ir_ctdef_->need_avg_doc_len_est(); - iter_param.bm25_param_est_ctx_.can_est_by_sum_skip_index_ = ir_ctdef_->avg_doc_len_est_spec_.can_est_by_sum_skip_index_; - iter_param.bm25_param_est_ctx_.avg_doc_token_cnt_expr_ = ir_ctdef_->avg_doc_token_cnt_expr_; - iter_param.bm25_param_est_ctx_.doc_length_est_param_ = &doc_length_est_param_; if (query_tokens_.count() == 0) { // do nothing + } else if (!sr_iter_param_.need_calc_relevance()) { + // Predicate-only MATCH only needs document ids. + } else if (FALSE_IT(iter_param.bm25_param_est_ctx_.total_doc_cnt_expr_ + = ir_ctdef_->get_doc_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0))) { + } else if (FALSE_IT(iter_param.bm25_param_est_ctx_.estimated_total_doc_cnt_ + = ir_ctdef_->estimated_total_doc_cnt_)) { + } else if (FALSE_IT(iter_param.bm25_param_est_ctx_.need_est_avg_doc_token_cnt_ + = ir_ctdef_->need_avg_doc_len_est())) { + } else if (FALSE_IT(iter_param.bm25_param_est_ctx_.can_est_by_sum_skip_index_ + = ir_ctdef_->avg_doc_len_est_spec_.can_est_by_sum_skip_index_)) { + } else if (FALSE_IT(iter_param.bm25_param_est_ctx_.avg_doc_token_cnt_expr_ + = ir_ctdef_->avg_doc_token_cnt_expr_)) { + } else if (FALSE_IT(iter_param.bm25_param_est_ctx_.doc_length_est_param_ + = &doc_length_est_param_)) { } else if (OB_ISNULL(iter_param.bm25_param_est_ctx_.total_doc_cnt_expr_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null total doc cnt expr", K(ret)); @@ -614,7 +635,7 @@ int ObDASTRMergeIter::init_daat_iter_param(ObTextDaaTParam &iter_param) iter_param.bm25_param_est_ctx_.total_doc_cnt_iter_ = static_cast(children_[children_cnt_ - 1]); } } - if (OB_SUCC(ret)) { + if (OB_SUCC(ret) && sr_iter_param_.need_calc_relevance()) { if (BOOLEAN_MODE == ir_ctdef_->mode_flag_) { ObSRDaaTBooleanRelevanceCollector *boolean_relevance_collector = nullptr; if (OB_ISNULL(boolean_relevance_collector = OB_NEWx(ObSRDaaTBooleanRelevanceCollector, &myself_allocator_))) { diff --git a/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h b/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h index f819a96a72..8e9ee4a3db 100644 --- a/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h +++ b/src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h @@ -148,6 +148,7 @@ class ObDASTRMergeIter : public ObDASIter int init_topk_limit(); int init_block_max_iter_param(); int init_doc_length_est_param(); + bool skip_relevance_for_predicate() const; private: static const int64_t FWD_IDX_ROWKEY_COL_CNT = 2; static const int64_t INV_IDX_ROWKEY_COL_CNT = 2; diff --git a/src/sql/das/ob_das_domain_utils.cpp b/src/sql/das/ob_das_domain_utils.cpp index 5d416a9a7b..0cbeb95034 100644 --- a/src/sql/das/ob_das_domain_utils.cpp +++ b/src/sql/das/ob_das_domain_utils.cpp @@ -42,7 +42,12 @@ ObExprOperatorType ObFTIndexRowCache::FTS_INDEX_EXPR_TYPE[] = {T_FUN_SYS_WORD_SE ObExprOperatorType ObFTIndexRowCache::FTS_DOC_WORD_EXPR_TYPE[] = {T_FUN_SYS_DOC_ID, T_FUN_SYS_WORD_SEGMENT, T_FUN_SYS_WORD_COUNT, T_FUN_SYS_DOC_LENGTH}; ObFTIndexRowCache::ObFTIndexRowCache() - : rows_(), + : merge_memctx_(nullptr), + row_allocator_(common::ObMemAttr("FTIndexRow")), + datum_row_(), + word_map_(), + words_(), + counts_(), row_idx_(0), is_fts_index_aux_(true), helper_(), @@ -68,7 +73,13 @@ int ObFTIndexRowCache::init( LOG_WARN("init fulltext dml iterator twice", K(ret), K(is_inited_)); } else if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(merge_memctx_, param))) { LOG_WARN("failed to create merge memctx", K(ret)); - } else if (OB_FAIL(helper_.init(&(merge_memctx_->get_arena_allocator()), parser_name, parser_properties))) { + } else if (OB_FAIL(datum_row_.init(row_allocator_, 4))) { + LOG_WARN("failed to initialize reusable full-text row", K(ret)); + } else if (OB_FAIL(word_map_.create(FT_WORD_MAP_BUCKET_COUNT, + common::ObMemAttr("FTWordMap")))) { + LOG_WARN("failed to create reusable full-text word map", K(ret)); + } else if (OB_FAIL(helper_.init(&(merge_memctx_->get_arena_allocator()), parser_name, parser_properties, + true /* enable_parser_reuse */))) { LOG_WARN("fail to init full-text parser helper", K(ret)); } else { row_idx_ = 0; @@ -86,22 +97,38 @@ int ObFTIndexRowCache::segment(const common::ObObjMeta &ft_obj_meta, const ObString &fulltext) { int ret = OB_SUCCESS; + int64_t doc_length = 0; if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObFTIndexRowCache hasn't be initialized", K(ret), K(is_inited_)); } else if (FALSE_IT(reuse())) { - } else if (OB_FAIL(ObDASDomainUtils::generate_fulltext_word_rows(merge_memctx_->get_arena_allocator(), - &helper_, - ft_obj_meta, - doc_id_datum, - fulltext, - is_fts_index_aux_, - rows_))) { - LOG_WARN("fail to generate fulltext word rows", K(ret), K(helper_), K(is_fts_index_aux_)); + } else if (fulltext.empty()) { + ret = OB_ITER_END; + } else if (OB_FAIL(helper_.segment(ft_obj_meta, + fulltext.ptr(), + fulltext.length(), + doc_length, + word_map_))) { + LOG_WARN("fail to segment fulltext", K(ret), K(helper_), K(is_fts_index_aux_)); + } else if (word_map_.size() <= 0) { + ret = OB_ITER_END; } else { - row_idx_ = 0; + for (ObFTWordMap::const_iterator iter = word_map_.begin(); + OB_SUCC(ret) && iter != word_map_.end(); + ++iter) { + if (OB_FAIL(words_.push_back(iter->first))) { + LOG_WARN("failed to cache full-text word", K(ret)); + } else if (OB_FAIL(counts_.push_back(iter->second))) { + LOG_WARN("failed to cache full-text word count", K(ret)); + } + } + if (OB_SUCC(ret)) { + const int64_t doc_id_idx = is_fts_index_aux_ ? 1 : 0; + datum_row_.storage_datums_[doc_id_idx].shallow_copy_from_datum(doc_id_datum); + datum_row_.storage_datums_[3].set_uint(doc_length); + } } - LOG_TRACE("word segment", K(ret), K(row_idx_), K(rows_.count()), K(doc_id_datum), K(fulltext)); + LOG_TRACE("word segment", K(ret), K(row_idx_), K(words_.count()), K(doc_id_datum), K(fulltext)); return ret; } @@ -111,19 +138,26 @@ int ObFTIndexRowCache::get_next_row(blocksstable::ObDatumRow *&row) if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("ObFTIndexRowCache hasn't be initialized", K(ret), K(is_inited_)); - } else if (row_idx_ >= rows_.count()) { + } else if (row_idx_ >= words_.count()) { ret = OB_ITER_END; } else { - row = (rows_[row_idx_]); + const int64_t word_idx = is_fts_index_aux_ ? 0 : 1; + datum_row_.storage_datums_[word_idx].set_datum(words_.at(row_idx_).get_word()); + datum_row_.storage_datums_[2].set_uint(counts_.at(row_idx_)); + row = &datum_row_; ++row_idx_; } - LOG_TRACE("get next row", K(ret), KPC(row), K(row_idx_), K(rows_.count())); + LOG_TRACE("get next row", K(ret), KPC(row), K(row_idx_), K(words_.count())); return ret; } void ObFTIndexRowCache::reset() { - rows_.reset(); + words_.reset(); + counts_.reset(); + word_map_.destroy(); + datum_row_.reset(); + row_allocator_.reset(); row_idx_ = 0; is_fts_index_aux_ = true; helper_.reset(); @@ -136,7 +170,9 @@ void ObFTIndexRowCache::reset() void ObFTIndexRowCache::reuse() { - rows_.reuse(); + words_.reuse(); + counts_.reuse(); + word_map_.reuse(); row_idx_ = 0; if (OB_NOT_NULL(merge_memctx_)) { merge_memctx_->reset_remain_one_page(); @@ -947,7 +983,8 @@ int ObFTDMLIterator::rewind() // This is the same as the parser name of the previous index. // nothing to do, just skip. } else if (FALSE_IT(ft_parse_helper_.reset())) { - } else if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_str, parser_property_str))) { + } else if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_str, parser_property_str, + true /* enable_parser_reuse */))) { LOG_WARN("fail to init fulltext parse helper", K(ret), K(parser_str), K(parser_property_str)); } } else if (ObDomainDMLMode::DOMAIN_DML_MODE_FT_SCAN == mode_) { @@ -981,7 +1018,8 @@ int ObFTDMLIterator::init( } else { switch (mode_) { case ObDomainDMLMode::DOMAIN_DML_MODE_DEFAULT: { - if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_name, parser_properties))) { + if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_name, parser_properties, + true /* enable_parser_reuse */))) { LOG_WARN("fail to init fulltext parse helper", K(ret), K(parser_name), K(parser_properties)); } break; @@ -1043,7 +1081,8 @@ int ObFTDMLIterator::change_domain_dml_mode(const ObDomainDMLMode &mode) // This is the same as the parser name of the previous index. // nothing to do, just skip. } else if (FALSE_IT(ft_parse_helper_.reset())) { - } else if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_str, parser_property_str))) { + } else if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_str, parser_property_str, + true /* enable_parser_reuse */))) { LOG_WARN("fail to init fulltext parse helper", K(ret), K(parser_str), K(parser_property_str)); } break; diff --git a/src/sql/das/ob_das_domain_utils.h b/src/sql/das/ob_das_domain_utils.h index c1295c3651..bc8c25eb5a 100644 --- a/src/sql/das/ob_das_domain_utils.h +++ b/src/sql/das/ob_das_domain_utils.h @@ -18,6 +18,7 @@ #define OCEANBASE_DAS_DOMAIN_UTILS_H #include "lib/allocator/page_arena.h" +#include "lib/container/ob_se_array.h" #include "lib/hash/ob_hashset.h" #include "common/datum/ob_datum.h" #include "sql/das/ob_das_dml_ctx_define.h" @@ -48,10 +49,16 @@ class ObFTIndexRowCache final int get_next_row(blocksstable::ObDatumRow *&row); void reset(); void reuse(); - TO_STRING_KV(K_(row_idx), K_(is_fts_index_aux), K_(helper), K_(is_inited), K_(rows)); + TO_STRING_KV(K_(row_idx), K_(is_fts_index_aux), K_(helper), K_(is_inited), + K_(words), K_(counts)); private: + static constexpr int64_t FT_WORD_MAP_BUCKET_COUNT = 128; lib::MemoryContext merge_memctx_; - ObDomainIndexRow rows_; + common::ObArenaAllocator row_allocator_; + blocksstable::ObDatumRow datum_row_; + ObFTWordMap word_map_; + common::ObSEArray words_; + common::ObSEArray counts_; uint64_t row_idx_; bool is_fts_index_aux_; storage::ObFTParseHelper helper_; diff --git a/src/sql/engine/expr/ob_expr_bm25.cpp b/src/sql/engine/expr/ob_expr_bm25.cpp index f89e99b57e..f24ca973ba 100644 --- a/src/sql/engine/expr/ob_expr_bm25.cpp +++ b/src/sql/engine/expr/ob_expr_bm25.cpp @@ -95,9 +95,6 @@ int ObExprBM25::eval_bm25_relevance_expr(const ObExpr &expr, ObEvalCtx &ctx, ObD const double doc_weight = doc_token_weight(related_token_cnt, norm_len); const double relevance = token_weight * doc_weight; res_datum.set_double(relevance); - LOG_DEBUG("show bm25 parameters for current document", - K(token_doc_cnt), K(total_doc_cnt), K(related_token_cnt), K(doc_token_cnt), K(avg_doc_token_cnt), - K(norm_len), K(token_weight), K(doc_weight), K(relevance)); } } else { ObDatum *token_doc_cnt_datum = nullptr; @@ -132,9 +129,6 @@ int ObExprBM25::eval_bm25_relevance_expr(const ObExpr &expr, ObEvalCtx &ctx, ObD const double doc_weight = doc_token_weight(related_token_cnt, norm_len); const double relevance = token_weight * doc_weight; res_datum.set_double(relevance); - LOG_DEBUG("show bm25 parameters for current document", - K(token_doc_cnt), K(total_doc_cnt), K(related_token_cnt), K(doc_token_cnt), K(avg_doc_token_cnt), - K(norm_len), K(token_weight), K(doc_weight), K(relevance)); } } return ret; @@ -171,22 +165,28 @@ int ObExprBM25::eval_batch_bm25_relevance_expr(const ObExpr &expr, ObEvalCtx &ct const double avg_doc_token_cnt = avg_doc_token_cnt_datum.at(0)->get_double(); ObDatum *res_datum = expr.locate_batch_datums(ctx); ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); + ObDatum *doc_token_cnt_datums = doc_token_cnt_datum.datums_; + ObDatum *related_token_cnt_datums = related_token_cnt_datum.datums_; + const uint64_t doc_token_cnt_mask = doc_token_cnt_datum.mask_; + const uint64_t related_token_cnt_mask = related_token_cnt_datum.mask_; for(int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { - if (OB_UNLIKELY(doc_token_cnt_datum.at(i)->is_null() || related_token_cnt_datum.at(i)->is_null())) { + ObDatum *doc_token_cnt = doc_token_cnt_datums + (doc_token_cnt_mask & i); + ObDatum *related_token_cnt = related_token_cnt_datums + (related_token_cnt_mask & i); + const uint64_t row_bit = 1ULL << (i & 63); + const int64_t word_idx = i >> 6; + if (OB_UNLIKELY(doc_token_cnt->is_null() || related_token_cnt->is_null())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null datum", K(ret), KPC(doc_token_cnt_datum.at(i)), KPC(related_token_cnt_datum.at(i))); - } else if (!skip.contain(i) && !eval_flags.at(i)) { - const int64_t related_token_cnt = related_token_cnt_datum.at(i)->get_uint(); - const uint64_t doc_token_cnt = doc_token_cnt_datum.at(i)->get_uint(); - const double norm_len = doc_token_cnt / avg_doc_token_cnt; - const double doc_weight = doc_token_weight(related_token_cnt, norm_len); + LOG_WARN("unexpected null datum", K(ret), KPC(doc_token_cnt), KPC(related_token_cnt)); + } else if (0 == (skip.data_[word_idx] & row_bit) + && 0 == (eval_flags.data_[word_idx] & row_bit)) { + const int64_t related_token_freq = related_token_cnt->get_uint(); + const uint64_t doc_token_length = doc_token_cnt->get_uint(); + const double norm_len = doc_token_length / avg_doc_token_cnt; + const double doc_weight = doc_token_weight(related_token_freq, norm_len); const double relevance = token_weight * doc_weight; res_datum[i].set_double(relevance); - eval_flags.set(i); - LOG_DEBUG("show bm25 parameters for current document", - K(token_doc_cnt), K(total_doc_cnt), K(related_token_cnt), K(doc_token_cnt), K(avg_doc_token_cnt), - K(norm_len), K(token_weight), K(doc_weight), K(relevance)); + eval_flags.data_[word_idx] |= row_bit; } } } @@ -219,22 +219,28 @@ int ObExprBM25::eval_batch_bm25_relevance_expr(const ObExpr &expr, ObEvalCtx &ct const double avg_doc_token_cnt = avg_doc_token_cnt_datum.at(0)->get_double(); ObDatum *res_datum = expr.locate_batch_datums(ctx); ObBitVector &eval_flags = expr.get_evaluated_flags(ctx); + ObDatum *doc_length_datums = doc_length_datum.datums_; + ObDatum *related_token_cnt_datums = related_token_cnt_datum.datums_; + const uint64_t doc_length_mask = doc_length_datum.mask_; + const uint64_t related_token_cnt_mask = related_token_cnt_datum.mask_; for(int64_t i = 0; OB_SUCC(ret) && i < size; ++i) { - if (OB_UNLIKELY(doc_length_datum.at(i)->is_null() || related_token_cnt_datum.at(i)->is_null())) { + ObDatum *doc_length = doc_length_datums + (doc_length_mask & i); + ObDatum *related_token_cnt = related_token_cnt_datums + (related_token_cnt_mask & i); + const uint64_t row_bit = 1ULL << (i & 63); + const int64_t word_idx = i >> 6; + if (OB_UNLIKELY(doc_length->is_null() || related_token_cnt->is_null())) { ret = OB_ERR_UNEXPECTED; - LOG_WARN("unexpected null datum", K(ret), KPC(doc_length_datum.at(i)), KPC(related_token_cnt_datum.at(i))); - } else if (!skip.contain(i) && !eval_flags.at(i)) { - const int64_t related_token_cnt = related_token_cnt_datum.at(i)->get_uint(); - const uint64_t doc_token_cnt = doc_length_datum.at(i)->get_uint(); - const double norm_len = doc_token_cnt / avg_doc_token_cnt; - const double doc_weight = doc_token_weight(related_token_cnt, norm_len); + LOG_WARN("unexpected null datum", K(ret), KPC(doc_length), KPC(related_token_cnt)); + } else if (0 == (skip.data_[word_idx] & row_bit) + && 0 == (eval_flags.data_[word_idx] & row_bit)) { + const int64_t related_token_freq = related_token_cnt->get_uint(); + const uint64_t doc_token_length = doc_length->get_uint(); + const double norm_len = doc_token_length / avg_doc_token_cnt; + const double doc_weight = doc_token_weight(related_token_freq, norm_len); const double relevance = token_weight * doc_weight; res_datum[i].set_double(relevance); - eval_flags.set(i); - LOG_DEBUG("show bm25 parameters for current document", - K(token_doc_cnt), K(total_doc_cnt), K(related_token_cnt), K(doc_token_cnt), K(avg_doc_token_cnt), - K(norm_len), K(token_weight), K(doc_weight), K(relevance)); + eval_flags.data_[word_idx] |= row_bit; } } } @@ -242,12 +248,6 @@ int ObExprBM25::eval_batch_bm25_relevance_expr(const ObExpr &expr, ObEvalCtx &ct return ret; } -double ObExprBM25::doc_token_weight(const int64_t token_freq, const double norm_len) -{ - const double tf = static_cast(token_freq); - return tf / (tf + p_k1 * (1.0 - p_b + p_b * norm_len)); -} - double ObExprBM25::query_token_weight(const int64_t doc_freq, const int64_t doc_cnt) { const double df = static_cast(doc_freq); diff --git a/src/sql/engine/expr/ob_expr_bm25.h b/src/sql/engine/expr/ob_expr_bm25.h index dafdbc741d..5151a624ed 100644 --- a/src/sql/engine/expr/ob_expr_bm25.h +++ b/src/sql/engine/expr/ob_expr_bm25.h @@ -66,7 +66,11 @@ class ObExprBM25 : public ObFuncExprOperator const double score = query_token_weight(doc_freq, doc_cnt) * doc_token_weight(token_freq, norm_len); return score; } - static double doc_token_weight(const int64_t token_freq, const double norm_len); + static OB_INLINE double doc_token_weight(const int64_t token_freq, const double norm_len) + { + const double tf = static_cast(token_freq); + return tf / (tf + p_k1 * (1.0 - p_b + p_b * norm_len)); + } static double query_token_weight(const int64_t doc_freq, const int64_t doc_cnt); static bool use_new_version(const ObExpr &expr) { diff --git a/src/sql/engine/expr/ob_expr_tokenize.cpp b/src/sql/engine/expr/ob_expr_tokenize.cpp index b913f7d054..84f06dfc70 100644 --- a/src/sql/engine/expr/ob_expr_tokenize.cpp +++ b/src/sql/engine/expr/ob_expr_tokenize.cpp @@ -21,6 +21,8 @@ #include "lib/charset/ob_charset.h" #include "common/json_type/ob_json_base.h" #include "common/json_type/ob_json_tree.h" +#include "lib/hash/ob_hashutils.h" +#include "lib/lock/ob_spin_lock.h" #include "lib/ob_errno.h" #include "lib/oblog/ob_log_module.h" #include "lib/string/ob_string.h" @@ -29,6 +31,7 @@ #include "plugin/sys/ob_plugin_helper.h" #include "sql/resolver/ddl/ob_fts_index_builder_util.h" #include "share/ob_json_access_utils.h" +#include "storage/fts/dict/ob_ft_dict_hub.h" #include "storage/fts/dict/ob_gen_dic_loader.h" #include "storage/fts/ob_fts_parser_property.h" #include "storage/fts/ob_fts_plugin_helper.h" @@ -40,6 +43,211 @@ namespace oceanbase { namespace sql { +namespace +{ + +struct ObTokenizeMemoKey +{ + ObTokenizeMemoKey() + : fulltext_(), + parser_name_(), + collation_(CS_TYPE_INVALID), + dict_epoch_(0), + hash_(0) + {} + + void finish_hash() + { + uint64_t value = common::murmurhash(fulltext_.ptr(), fulltext_.length(), 0); + value = common::murmurhash(parser_name_.ptr(), parser_name_.length(), value); + value = common::murmurhash(&collation_, sizeof(collation_), value); + value = common::murmurhash(&dict_epoch_, sizeof(dict_epoch_), value); + hash_ = value; + } + + ObString fulltext_; + ObString parser_name_; + ObCollationType collation_; + uint64_t dict_epoch_; + uint64_t hash_; +}; + +class ObTokenizeMemo final +{ +public: + static constexpr int64_t SLOT_COUNT = 16; + static constexpr int64_t MAX_FULLTEXT_LENGTH = 4096; + static constexpr int64_t MAX_PARSER_NAME_LENGTH = 128; + static constexpr int64_t MAX_RESULT_LENGTH = 32768; + + static ObTokenizeMemo &instance() + { + static ObTokenizeMemo memo; + return memo; + } + + bool can_store(const ObTokenizeMemoKey &key) const + { + return key.fulltext_.length() <= MAX_FULLTEXT_LENGTH + && key.parser_name_.length() <= MAX_PARSER_NAME_LENGTH; + } + + int find(const ObTokenizeMemoKey &key, + const ObExpr &expr, + ObEvalCtx &ctx, + ObDatum &result, + bool &found) + { + int ret = OB_SUCCESS; + found = false; + Slot &slot = slots_[key.hash_ & (SLOT_COUNT - 1)]; + common::ObSpinLockGuard guard(slot.lock_); + if (slot.matches(key)) { + char *buffer = nullptr; + if (slot.result_length_ > 0 + && OB_ISNULL(buffer = expr.get_str_res_mem(ctx, slot.result_length_))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate tokenize memo result", K(ret), K(slot.result_length_)); + } else { + if (slot.result_length_ > 0) { + MEMCPY(buffer, slot.result_, slot.result_length_); + } + result.set_string(buffer, slot.result_length_); + found = true; + } + } + return ret; + } + + void store(const ObTokenizeMemoKey &key, const ObString &result) + { + if (can_store(key) && result.length() <= MAX_RESULT_LENGTH) { + Slot &slot = slots_[key.hash_ & (SLOT_COUNT - 1)]; + common::ObSpinLockGuard guard(slot.lock_); + slot.assign(key, result); + } + } + +private: + struct Slot + { + Slot() + : lock_(), + hash_(0), + collation_(CS_TYPE_INVALID), + dict_epoch_(0), + fulltext_length_(0), + parser_name_length_(0), + result_length_(0), + valid_(false) + {} + + bool matches(const ObTokenizeMemoKey &key) const + { + return valid_ + && hash_ == key.hash_ + && collation_ == key.collation_ + && dict_epoch_ == key.dict_epoch_ + && fulltext_length_ == key.fulltext_.length() + && parser_name_length_ == key.parser_name_.length() + && (0 == fulltext_length_ + || 0 == MEMCMP(fulltext_, key.fulltext_.ptr(), fulltext_length_)) + && (0 == parser_name_length_ + || 0 == MEMCMP(parser_name_, key.parser_name_.ptr(), parser_name_length_)); + } + + void assign(const ObTokenizeMemoKey &key, const ObString &result) + { + hash_ = key.hash_; + collation_ = key.collation_; + dict_epoch_ = key.dict_epoch_; + fulltext_length_ = static_cast(key.fulltext_.length()); + parser_name_length_ = static_cast(key.parser_name_.length()); + result_length_ = static_cast(result.length()); + if (fulltext_length_ > 0) { + MEMCPY(fulltext_, key.fulltext_.ptr(), fulltext_length_); + } + if (parser_name_length_ > 0) { + MEMCPY(parser_name_, key.parser_name_.ptr(), parser_name_length_); + } + if (result_length_ > 0) { + MEMCPY(result_, result.ptr(), result_length_); + } + valid_ = true; + } + + common::ObSpinLock lock_; + char fulltext_[MAX_FULLTEXT_LENGTH]; + char parser_name_[MAX_PARSER_NAME_LENGTH]; + char result_[MAX_RESULT_LENGTH]; + uint64_t hash_; + ObCollationType collation_; + uint64_t dict_epoch_; + int32_t fulltext_length_; + int32_t parser_name_length_; + int32_t result_length_; + bool valid_; + }; + + ObTokenizeMemo() : slots_() {} + ~ObTokenizeMemo() = default; + + Slot slots_[SLOT_COUNT]; + DISALLOW_COPY_AND_ASSIGN(ObTokenizeMemo); +}; + +bool is_memoizable_parser(const ObString &parser_name) +{ + return 0 == parser_name.case_compare("space") + || 0 == parser_name.case_compare("ngram") + || 0 == parser_name.case_compare("beng") + || 0 == parser_name.case_compare("ik") + || 0 == parser_name.case_compare("ngram2"); +} + +int prepare_tokenize_memo_key(const ObExpr &expr, + ObEvalCtx &ctx, + ObTokenizeMemoKey &key, + bool &memoizable) +{ + int ret = OB_SUCCESS; + ObDatum *fulltext_datum = nullptr; + ObDatum *parser_datum = nullptr; + memoizable = false; + if (expr.arg_cnt_ > 2 || expr.args_[0]->obj_meta_.has_lob_header()) { + // Parser properties and LOB locators retain the general evaluation path. + } else if (OB_FAIL(expr.args_[0]->eval(ctx, fulltext_datum))) { + LOG_WARN("fail to evaluate tokenize input for memo key", K(ret)); + } else if (OB_ISNULL(fulltext_datum) || fulltext_datum->is_null()) { + // Keep NULL handling on the general path. + } else if (expr.arg_cnt_ >= 2 && OB_FAIL(expr.args_[1]->eval(ctx, parser_datum))) { + LOG_WARN("fail to evaluate tokenize parser for memo key", K(ret)); + } else { + key.fulltext_ = fulltext_datum->get_string(); + key.parser_name_ = expr.arg_cnt_ < 2 || OB_ISNULL(parser_datum) || parser_datum->is_null() + ? ObString::make_string(OB_DEFAULT_FULLTEXT_PARSER_NAME) + : parser_datum->get_string().trim(); + key.collation_ = expr.args_[0]->obj_meta_.get_collation_type(); + if (!is_memoizable_parser(key.parser_name_)) { + // A user parser can carry state outside of the built-in dictionary hub. + } else if (0 == key.parser_name_.case_compare("ik")) { + storage::ObFTDictHub *dict_hub = nullptr; + const int tmp_ret = storage::ObFTParsePluginData::instance().get_dict_hub(dict_hub); + if (OB_SUCCESS == tmp_ret && OB_NOT_NULL(dict_hub)) { + key.dict_epoch_ = dict_hub->get_generation_epoch(); + key.finish_hash(); + memoizable = ObTokenizeMemo::instance().can_store(key); + } + } else { + key.finish_hash(); + memoizable = ObTokenizeMemo::instance().can_store(key); + } + } + return ret; +} + +} // namespace + ObExprTokenize::ObExprTokenize(common::ObIAllocator &alloc) : ObStringExprOperator(alloc, T_FUN_TOKENIZE, @@ -60,11 +268,22 @@ int ObExprTokenize::eval_tokenize(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &e ObIJsonBase *json_result = nullptr; TokenizeParam param; + ObTokenizeMemoKey memo_key; + bool memoizable = false; + bool memo_hit = false; // check param num, which is checked in ObExprOperator::calc_result_typeN. if (OB_UNLIKELY(expr.arg_cnt_ < 1 || expr.arg_cnt_ > 3)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("Args count invalid.", K(ret), K(expr.arg_cnt_)); + } else if (OB_FAIL(prepare_tokenize_memo_key(expr, ctx, memo_key, memoizable))) { + LOG_WARN("fail to prepare tokenize memo key", K(ret)); + } else if (memoizable + && OB_FAIL(ObTokenizeMemo::instance().find( + memo_key, expr, ctx, expr_datum, memo_hit))) { + LOG_WARN("fail to find tokenize memo", K(ret)); + } else if (memo_hit) { + // The cached value is the packed binary JSON datum. } else if (OB_FAIL(parse_param(expr, ctx, temp_allocator, param))) { LOG_WARN("Fail to parse param", K(ret)); } else if (OB_FAIL(tokenize_fulltext(param, param.output_mode_, temp_allocator, json_result))) { @@ -75,6 +294,8 @@ int ObExprTokenize::eval_tokenize(const ObExpr &expr, ObEvalCtx &ctx, ObDatum &e json_result, expr_datum))) { LOG_WARN("fail to pack json result", K(ret)); + } else if (memoizable && !expr_datum.is_null()) { + ObTokenizeMemo::instance().store(memo_key, expr_datum.get_string()); } return ret; diff --git a/src/sql/engine/table/ob_table_scan_op.cpp b/src/sql/engine/table/ob_table_scan_op.cpp index e143fc3869..a16523bde2 100644 --- a/src/sql/engine/table/ob_table_scan_op.cpp +++ b/src/sql/engine/table/ob_table_scan_op.cpp @@ -763,6 +763,8 @@ ObTableScanOp::ObTableScanOp(ObExecContext &exec_ctx, const ObOpSpec &spec, ObOp in_rescan_(false), domain_index_(), fts_index_(), + fts_expr_cache_{}, + fts_lob_allocator_("FTSLobBuffer"), output_ (nullptr), fold_iter_(nullptr), iter_tree_(nullptr), @@ -1678,6 +1680,8 @@ int ObTableScanOp::inner_open() } else if (MY_SPEC.is_fts_ddl_ && OB_FAIL(fts_index_.init(MY_SPEC.is_fts_index_aux_, MY_SPEC.parser_name_, MY_SPEC.parser_properties_))) { LOG_WARN("fail to init fts index cache", K(ret)); + } else if (MY_SPEC.is_fts_ddl_ && OB_FAIL(init_fts_expr_cache())) { + LOG_WARN("fail to initialize fulltext expression cache", K(ret)); } else { if (MY_SPEC.report_col_checksum_) { if (PHY_TABLE_SCAN == MY_SPEC.get_type()) { @@ -4041,8 +4045,10 @@ int ObTableScanOp::fetch_next_fts_index_rows() int ret = OB_SUCCESS; bool has_segment_word = false; while (OB_SUCC(ret) && !has_segment_word) { - ObExpr *ft_expr = nullptr; - ObExpr *doc_id_expr = nullptr; + const int64_t word_idx = MY_SPEC.is_fts_index_aux_ ? 0 : 1; + const int64_t doc_id_idx = MY_SPEC.is_fts_index_aux_ ? 1 : 0; + ObExpr *ft_expr = fts_expr_cache_[word_idx]; + ObExpr *doc_id_expr = fts_expr_cache_[doc_id_idx]; ObDatum *ft_datum = nullptr; ObDatum *doc_id_datum = nullptr; @@ -4050,10 +4056,6 @@ int ObTableScanOp::fetch_next_fts_index_rows() if (OB_ITER_END != ret) { LOG_WARN("fail to get next row implement", K(ret)); } - } else if (OB_FAIL(get_output_fts_col_expr_by_type(T_FUN_SYS_DOC_ID, doc_id_expr))) { - LOG_WARN("fail to get doc id column expr from output", K(ret)); - } else if (OB_FAIL(get_output_fts_col_expr_by_type(T_FUN_SYS_WORD_SEGMENT, ft_expr))) { - LOG_WARN("fail to get word segment column expr from output", K(ret)); } else if (OB_ISNULL(ft_expr) || OB_ISNULL(doc_id_expr)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpeted error, ft or doc id expr is nullptr", K(ret), KP(ft_expr), KP(doc_id_expr)); @@ -4066,8 +4068,8 @@ int ObTableScanOp::fetch_next_fts_index_rows() LOG_WARN("unexpeted error, ft or doc id datum is nullptr", K(ret), KP(ft_datum), KP(doc_id_datum)); } else { ObString ft = ft_datum->get_string(); - ObArenaAllocator tmp_allocator(ObModIds::OB_LOB_ACCESS_BUFFER, OB_MALLOC_NORMAL_BLOCK_SIZE); - if (OB_FAIL(ObTextStringHelper::read_real_string_data(tmp_allocator, + fts_lob_allocator_.reset_remain_one_page(); + if (OB_FAIL(ObTextStringHelper::read_real_string_data(fts_lob_allocator_, *ft_datum, ft_expr->datum_meta_, ft_expr->obj_meta_.has_lob_header(), @@ -4096,9 +4098,10 @@ int ObTableScanOp::fill_generated_fts_cols(blocksstable::ObDatumRow *row) LOG_WARN("invalid argument, row is nullptr", K(ret), KP(row)); } else { for (int64_t i = 0; OB_SUCC(ret) && i < share::ObFtsIndexBuilderUtil::OB_FTS_INDEX_OR_DOC_WORD_TABLE_COL_CNT; ++i) { - ObExpr *expr = nullptr; - if (OB_FAIL(get_output_fts_col_expr_by_type(expr_types[i], expr))) { - LOG_WARN("fail to get fts column expr", K(ret), K(i), K(expr_types[i])); + ObExpr *expr = fts_expr_cache_[i]; + if (OB_ISNULL(expr)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected null fulltext expression cache entry", K(ret), K(i), K(expr_types[i])); } else { ObDatum &datum = expr->locate_datum_for_write(eval_ctx_); ObEvalInfo &eval_info = expr->get_eval_info(eval_ctx_); @@ -4114,6 +4117,23 @@ int ObTableScanOp::fill_generated_fts_cols(blocksstable::ObDatumRow *row) return ret; } +int ObTableScanOp::init_fts_expr_cache() +{ + int ret = OB_SUCCESS; + const ObExprOperatorType *expr_types = MY_SPEC.is_fts_index_aux_ + ? ObFTIndexRowCache::FTS_INDEX_EXPR_TYPE + : ObFTIndexRowCache::FTS_DOC_WORD_EXPR_TYPE; + for (int64_t i = 0; + OB_SUCC(ret) && i < share::ObFtsIndexBuilderUtil::OB_FTS_INDEX_OR_DOC_WORD_TABLE_COL_CNT; + ++i) { + fts_expr_cache_[i] = nullptr; + if (OB_FAIL(get_output_fts_col_expr_by_type(expr_types[i], fts_expr_cache_[i]))) { + LOG_WARN("fail to cache fulltext output expression", K(ret), K(i), K(expr_types[i])); + } + } + return ret; +} + int ObTableScanOp::get_output_fts_col_expr_by_type( const ObExprOperatorType &type, ObExpr *&expr) diff --git a/src/sql/engine/table/ob_table_scan_op.h b/src/sql/engine/table/ob_table_scan_op.h index d3d698a18b..9861a5a210 100644 --- a/src/sql/engine/table/ob_table_scan_op.h +++ b/src/sql/engine/table/ob_table_scan_op.h @@ -724,6 +724,7 @@ class ObTableScanOp : public ObOperator int inner_get_next_fts_index_row(); int fetch_next_fts_index_rows(); int fill_generated_fts_cols(ObDatumRow *row); + int init_fts_expr_cache(); int get_output_fts_col_expr_by_type(const ObExprOperatorType &type, ObExpr *&expr); bool is_resume_point_saved(); protected: @@ -754,6 +755,8 @@ class ObTableScanOp : public ObOperator bool in_rescan_; ObDomainIndexCache domain_index_; ObFTIndexRowCache fts_index_; + ObExpr *fts_expr_cache_[share::ObFtsIndexBuilderUtil::OB_FTS_INDEX_OR_DOC_WORD_TABLE_COL_CNT]; + common::ObArenaAllocator fts_lob_allocator_; // output_ is used to output data, TSC operator directly invokes output_::get_next_row(s), // it points to fold_iter_ in group rescan and iter_tree_ in normal scan. diff --git a/src/sql/optimizer/ob_log_plan.cpp b/src/sql/optimizer/ob_log_plan.cpp index f745e9fac9..d2333a55cb 100644 --- a/src/sql/optimizer/ob_log_plan.cpp +++ b/src/sql/optimizer/ob_log_plan.cpp @@ -14650,7 +14650,7 @@ int ObLogPlan::prepare_text_retrieval_scan(const ObIArray &scan_mat } else { ObTextRetrievalInfo &tr_info = table_scan->get_text_retrieval_info(); tr_info.match_expr_ = match_against; - tr_info.pushdown_match_filter_ = match_pred; + tr_info.pushdown_match_filter_ = tr_info.need_calc_relevance_ ? match_pred : nullptr; // in new version, doc_id_idx_tid_ is invalid. table_scan->set_doc_id_index_table_id(tr_info.doc_id_idx_tid_); if (table_scan->is_vec_adaptive_scan() || table_scan->is_vec_idx_scan_post_filter()) { @@ -14895,18 +14895,22 @@ int ObLogPlan::prepare_text_retrieval_info(const uint64_t ref_table_id, } } if (OB_SUCC(ret)) { - /* - if (OB_FAIL(ObTransformUtils::check_need_calc_match_score(get_optimizer_context().get_exec_ctx(), - get_stmt(), - match_against, - need_calc_relevance, - constraints))) { + if (BOOLEAN_MODE == match_against->get_mode_flag()) { + // Boolean syntax can encode required, excluded and grouped tokens. + need_calc_relevance = true; + } else if (OB_FAIL(ObTransformUtils::check_need_calc_match_score( + get_optimizer_context().get_exec_ctx(), + get_stmt(), + match_against, + need_calc_relevance, + constraints))) { LOG_WARN("failed to check need calc relevance", K(ret)); } else if (!need_calc_relevance && OB_FAIL(append_array_no_dup(get_optimizer_context().get_query_ctx()->all_expr_constraints_, constraints))) { LOG_WARN("failed to append array no dup", K(ret)); } - */ + } + if (OB_SUCC(ret)) { tr_info.match_expr_ = match_against; tr_info.inv_idx_tid_ = inv_idx_tid; tr_info.fwd_idx_tid_ = fwd_idx_tid; diff --git a/src/sql/rewrite/ob_transform_utils.cpp b/src/sql/rewrite/ob_transform_utils.cpp index c089a6be80..2dc434ddc6 100644 --- a/src/sql/rewrite/ob_transform_utils.cpp +++ b/src/sql/rewrite/ob_transform_utils.cpp @@ -14787,23 +14787,14 @@ int ObTransformUtils::inner_check_need_calc_match_score(ObExecContext *exec_ctx, need_calc = true; need_check_child = false; } else if (expr->get_expr_type() == T_OP_GT || expr->get_expr_type() == T_OP_LT) { - ObRawExpr *gt_param = NULL; - ObRawExpr *lt_param = NULL; - bool is_param_zero = false; if (expr->get_param_count() != 2 || OB_ISNULL(expr->get_param_expr(0)) || OB_ISNULL(expr->get_param_expr(1))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret)); - } else if (OB_FALSE_IT(gt_param = expr->get_expr_type() == T_OP_GT ? expr->get_param_expr(0) : - expr->get_param_expr(1))) { - } else if (OB_FALSE_IT(lt_param = expr->get_expr_type() == T_OP_GT ? expr->get_param_expr(1) : - expr->get_param_expr(0))) { - } else if (gt_param != match_expr) { - /* do nothing */ - } else if (OB_FAIL(check_expr_eq_zero(exec_ctx, lt_param, is_param_zero, constraints))) { - LOG_WARN("failed to check param eq zero", K(ret)); - } else if (is_param_zero) { - need_calc = false; + } else if (expr->get_param_expr(0) == match_expr + || expr->get_param_expr(1) == match_expr) { + // A comparison consumes the numeric MATCH value even when the constant is zero. + need_calc = true; need_check_child = false; } } else if (expr->get_expr_type() == T_OP_BOOL) { @@ -14829,6 +14820,65 @@ int ObTransformUtils::inner_check_need_calc_match_score(ObExecContext *exec_ctx, return ret; } +int ObTransformUtils::check_need_calc_match_score(ObExecContext *exec_ctx, + const ObDMLStmt *stmt, + ObRawExpr *match_expr, + bool &need_calc, + ObIArray &constraints) +{ + int ret = OB_SUCCESS; + ObSEArray relation_exprs; + need_calc = false; + if (OB_ISNULL(exec_ctx) || OB_ISNULL(stmt) || OB_ISNULL(match_expr)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid argument", K(ret), KP(exec_ctx), KP(stmt), KP(match_expr)); + } else if (OB_FAIL(stmt->get_relation_exprs(relation_exprs))) { + LOG_WARN("failed to collect relation expressions", K(ret)); + } + for (int64_t i = 0; OB_SUCC(ret) && !need_calc && i < relation_exprs.count(); ++i) { + ObRawExpr *root_expr = relation_exprs.at(i); + ObSEArray match_exprs; + bool references_target = false; + bool used_as_condition = false; + bool root_needs_score = false; + if (OB_FAIL(ObRawExprUtils::extract_match_exprs(root_expr, match_exprs))) { + LOG_WARN("failed to extract match expressions", K(ret), K(i)); + } + for (int64_t j = 0; OB_SUCC(ret) && !references_target && j < match_exprs.count(); ++j) { + references_target = match_exprs.at(j) == match_expr; + } + if (OB_FAIL(ret) || !references_target) { + // This relation expression does not use the target MATCH expression. + } else { + if (ObOptimizerUtil::find_item(stmt->get_condition_exprs(), root_expr)) { + used_as_condition = true; + } else if (stmt->is_select_stmt() + && ObOptimizerUtil::find_item( + static_cast(stmt)->get_having_exprs(), + root_expr)) { + used_as_condition = true; + } else if (OB_FAIL(check_expr_used_as_condition( + stmt, root_expr, match_expr, used_as_condition))) { + LOG_WARN("failed to classify match expression usage", K(ret), K(i)); + } + if (OB_FAIL(ret)) { + } else if (!used_as_condition) { + need_calc = true; + } else if (OB_FAIL(inner_check_need_calc_match_score( + exec_ctx, + root_expr, + match_expr, + root_needs_score, + constraints))) { + LOG_WARN("failed to inspect match score usage", K(ret), K(i)); + } else if (root_needs_score) { + need_calc = true; + } + } + } + return ret; +} + int ObTransformUtils::check_expr_eq_zero(ObExecContext *ctx, ObRawExpr *expr, bool &eq_zero, @@ -14876,7 +14926,7 @@ int ObTransformUtils::check_expr_eq_zero(ObExecContext *ctx, return ret; } -int ObTransformUtils::check_expr_used_as_condition(ObDMLStmt *stmt, +int ObTransformUtils::check_expr_used_as_condition(const ObDMLStmt *stmt, ObRawExpr *root_expr, ObRawExpr *expr, bool &used_as_condition) @@ -14894,7 +14944,7 @@ int ObTransformUtils::check_expr_used_as_condition(ObDMLStmt *stmt, } else if (ObOptimizerUtil::find_item(stmt->get_condition_exprs(), cur_expr)) { parent_as_condition = true; } else if (stmt->is_select_stmt() && ObOptimizerUtil::find_item( - static_cast(stmt)->get_having_exprs(), cur_expr)) { + static_cast(stmt)->get_having_exprs(), cur_expr)) { parent_as_condition = true; } if (OB_SUCC(ret)) { diff --git a/src/sql/rewrite/ob_transform_utils.h b/src/sql/rewrite/ob_transform_utils.h index c5adbb3e55..e46e13aa26 100644 --- a/src/sql/rewrite/ob_transform_utils.h +++ b/src/sql/rewrite/ob_transform_utils.h @@ -1885,6 +1885,11 @@ class ObTransformUtils static int check_contain_lost_deterministic_expr(const ObIArray &exprs, bool &is_contain); // check whether the score calculated by match expr is actually utilized + static int check_need_calc_match_score(ObExecContext *exec_ctx, + const ObDMLStmt *stmt, + ObRawExpr *match_expr, + bool &need_calc, + ObIArray &constraints); static int check_expr_eq_zero(ObExecContext *ctx, ObRawExpr *expr, bool &eq_zero, @@ -1893,7 +1898,7 @@ class ObTransformUtils const ObIArray &raw_having_exprs, const ObIArray &group_clause_exprs, ObIArray &having_exprs_for_deduce); - static int check_expr_used_as_condition(ObDMLStmt *stmt, + static int check_expr_used_as_condition(const ObDMLStmt *stmt, ObRawExpr *root_expr, ObRawExpr *expr, bool &used_as_condition); diff --git a/src/storage/fts/dict/ob_ft_dict_hub.cpp b/src/storage/fts/dict/ob_ft_dict_hub.cpp index b8122a0ff0..f7dd0a629c 100644 --- a/src/storage/fts/dict/ob_ft_dict_hub.cpp +++ b/src/storage/fts/dict/ob_ft_dict_hub.cpp @@ -32,6 +32,7 @@ int ObFTDictHub::init() { static constexpr int K_MAX_DICT_BUCKET = 128; // for now, only built-in dicts. int ret = OB_SUCCESS; + ATOMIC_STORE(&generation_epoch_, 0); if (OB_FAIL(dict_map_.create(K_MAX_DICT_BUCKET, "dict_map"))) { LOG_WARN("init dict map failed", K(ret)); } else if (OB_FAIL(rw_dict_lock_.init(K_MAX_DICT_BUCKET))) { @@ -83,6 +84,8 @@ int ObFTDictHub::build_cache(const ObFTDictDesc &desc, ObFTCacheRangeContainer & } else if (FALSE_IT(info.range_count_ = container.get_handles().size())) { } else if (OB_FAIL(put_dict_info(key, info))) { LOG_WARN("Failed to put dict info", K(ret)); + } else { + ATOMIC_INC(&generation_epoch_); } } } diff --git a/src/storage/fts/dict/ob_ft_dict_hub.h b/src/storage/fts/dict/ob_ft_dict_hub.h index d8de9d1727..5896ef7b9e 100644 --- a/src/storage/fts/dict/ob_ft_dict_hub.h +++ b/src/storage/fts/dict/ob_ft_dict_hub.h @@ -17,6 +17,7 @@ #ifndef _OCEANBASE_STORAGE_FTS_DICT_OB_FT_DICT_HUB_H_ #define _OCEANBASE_STORAGE_FTS_DICT_OB_FT_DICT_HUB_H_ +#include "lib/atomic/ob_atomic.h" #include "lib/charset/ob_charset.h" #include "lib/lock/ob_bucket_lock.h" #include "storage/fts/dict/ob_ft_dict_def.h" @@ -96,7 +97,7 @@ class ObFTCacheRangeContainer; class ObFTDictHub { public: - ObFTDictHub() : is_inited_(false), dict_map_(), rw_dict_lock_() {} + ObFTDictHub() : is_inited_(false), generation_epoch_(0), dict_map_(), rw_dict_lock_() {} ~ObFTDictHub() {} int init(); @@ -107,6 +108,11 @@ class ObFTDictHub int load_cache(const ObFTDictDesc &desc, ObFTCacheRangeContainer &container); + OB_INLINE uint64_t get_generation_epoch() const + { + return ATOMIC_LOAD(&generation_epoch_); + } + private: int get_dict_info(const ObFTDictInfoKey &key, ObFTDictInfo &info); @@ -115,6 +121,7 @@ class ObFTDictHub private: bool is_inited_; + uint64_t generation_epoch_; // holds info of dict hash::ObHashMap dict_map_; ObBucketLock rw_dict_lock_; diff --git a/src/storage/fts/ik/ob_fast_list.h b/src/storage/fts/ik/ob_fast_list.h new file mode 100644 index 0000000000..140077af0c --- /dev/null +++ b/src/storage/fts/ik/ob_fast_list.h @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _OCEANBASE_STORAGE_FTS_OB_FAST_LIST_H_ +#define _OCEANBASE_STORAGE_FTS_OB_FAST_LIST_H_ + +#include "storage/fts/ik/ob_fast_segment_array.h" + +namespace oceanbase +{ +namespace storage +{ + +// POD-like list whose nodes are allocated and reclaimed in blocks. +template +class ObFastList +{ + struct Node + { + Node() : next(nullptr), prev(nullptr), value() {} + explicit Node(const T &val) : next(nullptr), prev(nullptr), value(val) {} + Node *next; + Node *prev; + T value; + }; + + struct NodeHolder + { + operator Node *() { return reinterpret_cast(this); } + operator const Node *() const { return reinterpret_cast(this); } + Node *next; + Node *prev; + }; + +public: + class iterator + { + public: + iterator() : node_(nullptr) {} + explicit iterator(Node *node) : node_(node) {} + T &operator*() const { return node_->value; } + T *operator->() const { return &node_->value; } + iterator &operator++() { node_ = nullptr == node_ ? nullptr : node_->next; return *this; } + iterator operator++(int) { iterator tmp(*this); ++(*this); return tmp; } + iterator &operator--() { node_ = nullptr == node_ ? nullptr : node_->prev; return *this; } + iterator operator--(int) { iterator tmp(*this); --(*this); return tmp; } + bool operator==(const iterator &other) const { return node_ == other.node_; } + bool operator!=(const iterator &other) const { return node_ != other.node_; } + + private: + friend class ObFastList; + Node *node_; + }; + + class const_iterator + { + public: + const_iterator() : node_(nullptr) {} + explicit const_iterator(const Node *node) : node_(const_cast(node)) {} + const_iterator(const iterator &iter) : node_(iter.node_) {} + const T &operator*() const { return node_->value; } + const T *operator->() const { return &node_->value; } + const_iterator &operator++() { node_ = nullptr == node_ ? nullptr : node_->next; return *this; } + const_iterator operator++(int) { const_iterator tmp(*this); ++(*this); return tmp; } + const_iterator &operator--() { node_ = nullptr == node_ ? nullptr : node_->prev; return *this; } + const_iterator operator--(int) { const_iterator tmp(*this); --(*this); return tmp; } + bool operator==(const const_iterator &other) const { return node_ == other.node_; } + bool operator!=(const const_iterator &other) const { return node_ != other.node_; } + + private: + friend class ObFastList; + Node *node_; + }; + +public: + explicit ObFastList(ObIAllocator &allocator) : pool_(allocator), size_(0) + { + root_.next = root_; + root_.prev = root_; + } + + ~ObFastList() { reset(); } + + void reset() + { + root_.next = root_; + root_.prev = root_; + size_ = 0; + pool_.reset(); + } + + void reuse() + { + root_.next = root_; + root_.prev = root_; + size_ = 0; + pool_.reuse(); + } + + bool empty() const { return 0 == size_; } + int64_t size() const { return size_; } + T &get_first() { return root_.next->value; } + const T &get_first() const { return root_.next->value; } + T &get_last() { return root_.prev->value; } + const T &get_last() const { return root_.prev->value; } + iterator begin() { return iterator(root_.next); } + iterator end() { return iterator(root_); } + const_iterator begin() const { return const_iterator(root_.next); } + const_iterator end() const { return const_iterator(root_); } + iterator last() { return iterator(root_.prev); } + const_iterator last() const { return const_iterator(root_.prev); } + int push_front(const T &value) { return insert_before_(root_.next, value); } + int push_back(const T &value) { return insert_before_(root_, value); } + int insert(iterator pos, const T &value) { return insert_before_(pos.node_, value); } + int insert(const_iterator pos, const T &value) { return insert_before_(pos.node_, value); } + + int pop_front() + { + int ret = OB_SUCCESS; + if (empty()) { + ret = OB_ENTRY_NOT_EXIST; + } else { + remove_node_(root_.next); + } + return ret; + } + + int pop_back() + { + int ret = OB_SUCCESS; + if (empty()) { + ret = OB_ENTRY_NOT_EXIST; + } else { + remove_node_(root_.prev); + } + return ret; + } + +private: + int insert_before_(Node *pos, const T &value) + { + int ret = OB_SUCCESS; + Node *node = nullptr; + Node dummy(value); + if (OB_FAIL(pool_.push_back(dummy))) { + } else { + node = &pool_.at(pool_.count() - 1); + Node *anchor = nullptr == pos ? root_ : pos; + node->next = anchor; + node->prev = anchor->prev; + anchor->prev->next = node; + anchor->prev = node; + ++size_; + } + return ret; + } + + void remove_node_(Node *node) + { + if (nullptr != node && node != root_) { + node->prev->next = node->next; + node->next->prev = node->prev; + node->next = nullptr; + node->prev = nullptr; + --size_; + } + } + +private: + ObFastSegmentArray pool_; + NodeHolder root_; + int64_t size_; +}; + +} // namespace storage +} // namespace oceanbase + +#endif // _OCEANBASE_STORAGE_FTS_OB_FAST_LIST_H_ diff --git a/src/storage/fts/ik/ob_fast_segment_array.h b/src/storage/fts/ik/ob_fast_segment_array.h new file mode 100644 index 0000000000..f6f7fded74 --- /dev/null +++ b/src/storage/fts/ik/ob_fast_segment_array.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _OCEANBASE_STORAGE_FTS_OB_FAST_SEGMENT_ARRAY_H_ +#define _OCEANBASE_STORAGE_FTS_OB_FAST_SEGMENT_ARRAY_H_ + +#define USING_LOG_PREFIX STORAGE_FTS + +#include "lib/allocator/ob_allocator.h" +#include "lib/ob_errno.h" +#include "lib/oblog/ob_log_module.h" + +namespace oceanbase +{ +namespace storage +{ + +template +class ObFastSegmentArray +{ + static_assert(block_capacity > 2 && block_capacity <= (1 << 30), + "block capacity must be greater than 2 and at most (1 << 30)"); + +public: + static constexpr int64_t BLOCK_POINTER_ARRAY_CAPACITY = 64; + static constexpr int64_t BLOCK_CAPACITY_POWER = + 64 - __builtin_clzll(static_cast(block_capacity) - 1); + static constexpr int64_t REAL_BLOCK_CAPACITY = static_cast(1) << BLOCK_CAPACITY_POWER; + static constexpr int64_t BLOCK_LOCATOR = REAL_BLOCK_CAPACITY - 1; + + explicit ObFastSegmentArray( + ObIAllocator &allocator, + int64_t init_block_arr_cap = BLOCK_POINTER_ARRAY_CAPACITY) + : allocator_(allocator), + block_arr_(nullptr), + block_arr_cap_(0), + block_count_(0), + size_(0), + init_block_arr_cap_(init_block_arr_cap > 0 + ? init_block_arr_cap + : BLOCK_POINTER_ARRAY_CAPACITY) + {} + + ~ObFastSegmentArray() { reset(); } + + int push_back(const T &value) + { + int ret = OB_SUCCESS; + const int64_t block_idx = size_ >> BLOCK_CAPACITY_POWER; + if (OB_FAIL(ensure_block_(block_idx))) { + } else { + const int64_t inner_idx = size_ & BLOCK_LOCATOR; + block_arr_[block_idx][inner_idx] = value; + ++size_; + } + return ret; + } + + const T &at(const int64_t idx) const + { + const int64_t block_idx = idx >> BLOCK_CAPACITY_POWER; + const int64_t inner_idx = idx & BLOCK_LOCATOR; + return block_arr_[block_idx][inner_idx]; + } + + T &at(const int64_t idx) + { + return const_cast(static_cast(*this).at(idx)); + } + + int64_t count() const { return size_; } + bool empty() const { return 0 == size_; } + void reuse() { size_ = 0; } + + void reset() + { + for (int64_t i = 0; i < block_count_; ++i) { + if (nullptr != block_arr_[i]) { + allocator_.free(block_arr_[i]); + block_arr_[i] = nullptr; + } + } + if (nullptr != block_arr_) { + allocator_.free(block_arr_); + } + block_arr_ = nullptr; + block_arr_cap_ = 0; + block_count_ = 0; + size_ = 0; + } + +private: + int ensure_block_(const int64_t block_idx) + { + int ret = OB_SUCCESS; + if (block_idx >= block_count_) { + if (block_idx >= block_arr_cap_ && OB_FAIL(expand_block_array_(block_idx + 1))) { + LOG_WARN("failed to expand block array", K(ret), K(block_idx)); + } + if (OB_SUCC(ret)) { + void *buf = allocator_.alloc(REAL_BLOCK_CAPACITY * static_cast(sizeof(T))); + if (OB_ISNULL(buf)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate segment block", K(ret), K(block_idx)); + } else { + block_arr_[block_idx] = static_cast(buf); + ++block_count_; + } + } + } + return ret; + } + + int expand_block_array_(const int64_t need_cap) + { + int ret = OB_SUCCESS; + int64_t next_cap = block_arr_cap_ > 0 ? block_arr_cap_ : init_block_arr_cap_; + while (next_cap < need_cap) { + next_cap <<= 1; + } + const int64_t bytes = next_cap * static_cast(sizeof(T *)); + T **next_arr = static_cast(allocator_.alloc(bytes)); + if (OB_ISNULL(next_arr)) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate block pointer array", K(ret), K(next_cap)); + } else { + MEMSET(next_arr, 0, bytes); + if (nullptr != block_arr_) { + MEMCPY(next_arr, block_arr_, block_count_ * static_cast(sizeof(T *))); + allocator_.free(block_arr_); + } + block_arr_ = next_arr; + block_arr_cap_ = next_cap; + } + return ret; + } + +private: + ObIAllocator &allocator_; + T **block_arr_; + int64_t block_arr_cap_; + int64_t block_count_; + int64_t size_; + const int64_t init_block_arr_cap_; + + DISALLOW_COPY_AND_ASSIGN(ObFastSegmentArray); +}; + +} // namespace storage +} // namespace oceanbase + +#endif // _OCEANBASE_STORAGE_FTS_OB_FAST_SEGMENT_ARRAY_H_ diff --git a/src/storage/fts/ik/ob_ik_arbitrator.cpp b/src/storage/fts/ik/ob_ik_arbitrator.cpp index 600b407317..bed0d97dba 100644 --- a/src/storage/fts/ik/ob_ik_arbitrator.cpp +++ b/src/storage/fts/ik/ob_ik_arbitrator.cpp @@ -36,11 +36,10 @@ int ObIKArbitrator::process(TokenizeContext &ctx) { int ret = OB_SUCCESS; - ObList &tokens = ctx.token_list().tokens(); + ObFastList &tokens = ctx.token_list().tokens(); ObIKTokenChain *chain_need_arbitrate = nullptr; bool use_smart = ctx.is_smart(); - if (OB_FAIL(prepare(ctx))) { - } else if (OB_ISNULL(chain_need_arbitrate = OB_NEWx(ObIKTokenChain, &alloc_, alloc_))) { + if (OB_ISNULL(chain_need_arbitrate = OB_NEWx(ObIKTokenChain, &alloc_, alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("alloc memory failed"); } else { @@ -52,7 +51,7 @@ int ObIKArbitrator::process(TokenizeContext &ctx) } else if (OB_FAIL(chain_need_arbitrate->add_token_if_conflict(token, is_add))) { LOG_WARN("add token if conflict failed", K(ret)); } else if (!is_add) { - ObFTSortList::CellIter iter = chain_need_arbitrate->list().tokens().begin(); + ObFTLightSortList::CellIter iter = chain_need_arbitrate->list().tokens().begin(); ObIKTokenChain *judge_result = nullptr; if (chain_need_arbitrate->list().tokens().size() == 1 || !use_smart) { if (OB_FAIL(add_chain(chain_need_arbitrate))) { @@ -121,7 +120,8 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) int64_t char_len = 0; ObIKTokenChain *chain = nullptr; - for (int64_t current = 0; OB_SUCC(ret) && current < ctx.fulltext_len();) { + for (int64_t current = ctx.get_buffer_start_cursor(); + OB_SUCC(ret) && current < ctx.get_buffer_end_cursor();) { ObFTCharUtil::CharType type; // maybe not so good to keep single, check it later if (OB_FAIL(ctx.first_valid_char( @@ -147,7 +147,7 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) bool is_ignore = false; if (ObFTCharUtil::CharType::CHINESE == type) { token.type_ = ObIKTokenType::IK_CHINESE_TOKEN; - if (OB_FAIL(ctx.result_list().push_back(token))) { + if (OB_FAIL(ctx.get_results().push_back(token))) { LOG_WARN("Failed to output result to ctx", K(ret)); } } else if (ObFTCharUtil::CharType::OTHER_CJK == type) { @@ -157,7 +157,7 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) is_ignore))) { LOG_WARN("Failed to check ignore", K(ret)); } else if (!is_ignore && !FALSE_IT(token.type_ = ObIKTokenType::IK_OTHER_CJK_TOKEN) - && OB_FAIL(ctx.result_list().push_back(token))) { + && OB_FAIL(ctx.get_results().push_back(token))) { LOG_WARN("Failed to add token to ctx result", K(ret)); } else { // ignore @@ -188,7 +188,7 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) bool is_ignore = false; if (ObFTCharUtil::CharType::CHINESE == type) { token.type_ = ObIKTokenType::IK_CHINESE_TOKEN; - ctx.result_list().push_back(token); + ctx.get_results().push_back(token); } else if (ObFTCharUtil::CharType::OTHER_CJK == type) { if (OB_FAIL(ObFTCharUtil::is_ignore_single_cjk(ctx.charset_type(), ctx.fulltext() + current, @@ -197,7 +197,7 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) LOG_WARN("Failed to check ignore", K(ret)); } else if (!is_ignore) { token.type_ = ObIKTokenType::IK_OTHER_CJK_TOKEN; - ctx.result_list().push_back(token); + ctx.get_results().push_back(token); } else { } } @@ -207,7 +207,7 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) } if (OB_FAIL(ret)) { - } else if (OB_FAIL(ctx.result_list().push_back(token))) { + } else if (OB_FAIL(ctx.get_results().push_back(token))) { LOG_WARN("Failed to add token to ctx result", K(ret)); } else // output the token @@ -226,14 +226,14 @@ int ObIKArbitrator::output_result(TokenizeContext &ctx) int ObIKArbitrator::optimize(TokenizeContext &ctx, ObIKTokenChain *chain, - ObFTSortList::CellIter iter, + ObFTLightSortList::CellIter iter, int64_t fulltext_len, ObIKTokenChain *&best) { int ret = OB_SUCCESS; ObIKTokenChain *option = nullptr; - ObList conflict_stack(alloc_); + ObList conflict_stack(alloc_); if (OB_ISNULL(option = OB_NEWx(ObIKTokenChain, &alloc_, alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; @@ -247,7 +247,7 @@ int ObIKArbitrator::optimize(TokenizeContext &ctx, LOG_WARN("Copy best option failed", K(ret)); } else { while (OB_SUCC(ret) && !conflict_stack.empty()) { - ObFTSortList::CellIter iter = conflict_stack.get_last(); + ObFTLightSortList::CellIter iter = conflict_stack.get_last(); conflict_stack.pop_back(); if (OB_FAIL(remove_conflict(*iter, option))) { LOG_WARN("Failed to remove conflict", K(ret)); @@ -279,10 +279,10 @@ int ObIKArbitrator::optimize(TokenizeContext &ctx, } int ObIKArbitrator::try_add_next_words(ObIKTokenChain *chain, - ObFTSortList::CellIter iter, + ObFTLightSortList::CellIter iter, ObIKTokenChain *option, bool need_conflict, - ObList &conflict_stack) + ObList &conflict_stack) { int ret = OB_SUCCESS; if (OB_ISNULL(chain) || OB_ISNULL(option)) { @@ -317,13 +317,10 @@ int ObIKArbitrator::remove_conflict(const ObIKToken &token, ObIKTokenChain *opti return ret; } -int ObIKArbitrator::prepare(TokenizeContext &ctx) +int ObIKArbitrator::prepare() { int ret = OB_SUCCESS; - - int cal_bucket_num = MAX(ctx.fulltext_len() / 100, 10); - cal_bucket_num = MIN(cal_bucket_num, 100); - if (OB_FAIL(chains_.create(cal_bucket_num, ObMemAttr("IK ARBITRATE")))) { + if (OB_FAIL(chains_.create(IK_HANDLE_SIZE_LIMIT * 2L, ObMemAttr("IK ARBITRATE")))) { LOG_WARN("create chain map failed", K(ret)); } return ret; @@ -350,5 +347,11 @@ ObIKArbitrator::~ObIKArbitrator() chains_.destroy(); alloc_.reset(); } + +void ObIKArbitrator::reuse() +{ + chains_.reuse(); + alloc_.reset_remain_one_page(); +} } // namespace storage } // namespace oceanbase diff --git a/src/storage/fts/ik/ob_ik_arbitrator.h b/src/storage/fts/ik/ob_ik_arbitrator.h index 40837f39f0..f21d90075e 100644 --- a/src/storage/fts/ik/ob_ik_arbitrator.h +++ b/src/storage/fts/ik/ob_ik_arbitrator.h @@ -38,22 +38,24 @@ class ObIKArbitrator int output_result(TokenizeContext &ctx); -private: - int prepare(TokenizeContext &ctx); + int prepare(); + + void reuse(); +private: int add_chain(ObIKTokenChain *chain); int optimize(TokenizeContext &ctx, ObIKTokenChain *option, - ObFTSortList::CellIter iter, + ObFTLightSortList::CellIter iter, int64_t fulltext_len, ObIKTokenChain *&best); int try_add_next_words(ObIKTokenChain *chain, - ObFTSortList::CellIter iter, + ObFTLightSortList::CellIter iter, ObIKTokenChain *option, bool need_conflict, - ObList &conflict_stack); + ObList &conflict_stack); int remove_conflict(const ObIKToken &token, ObIKTokenChain *option); diff --git a/src/storage/fts/ik/ob_ik_cjk_processor.h b/src/storage/fts/ik/ob_ik_cjk_processor.h index c1df9da3a4..d3ae73f93a 100644 --- a/src/storage/fts/ik/ob_ik_cjk_processor.h +++ b/src/storage/fts/ik/ob_ik_cjk_processor.h @@ -35,6 +35,8 @@ class ObIKCJKProcessor : public ObIIKProcessor } ~ObIKCJKProcessor() override { hits_.reset(); } + void reuse() override { hits_.reset(); } + public: int do_process(TokenizeContext &ctx, const char *ch, diff --git a/src/storage/fts/ik/ob_ik_letter_processor.cpp b/src/storage/fts/ik/ob_ik_letter_processor.cpp index 8d8526d1b4..de75bdd7a2 100644 --- a/src/storage/fts/ik/ob_ik_letter_processor.cpp +++ b/src/storage/fts/ik/ob_ik_letter_processor.cpp @@ -29,6 +29,13 @@ namespace oceanbase { namespace storage { +void ObIKLetterProcessor::reuse() +{ + reset_english_state(); + reset_arabic_state(); + reset_mix_state(); +} + ObIKLetterProcessor::ObIKLetterProcessor() {} int ObIKLetterProcessor::do_process(TokenizeContext &ctx, diff --git a/src/storage/fts/ik/ob_ik_letter_processor.h b/src/storage/fts/ik/ob_ik_letter_processor.h index 144d18487d..6eea7b08b9 100644 --- a/src/storage/fts/ik/ob_ik_letter_processor.h +++ b/src/storage/fts/ik/ob_ik_letter_processor.h @@ -29,6 +29,8 @@ class ObIKLetterProcessor : public ObIIKProcessor ObIKLetterProcessor(); ~ObIKLetterProcessor() override {} + void reuse() override; + int do_process(TokenizeContext &ctx, const char *ch, const uint8_t char_len, diff --git a/src/storage/fts/ik/ob_ik_processor.cpp b/src/storage/fts/ik/ob_ik_processor.cpp index 525ce5fa4e..2eb466032a 100644 --- a/src/storage/fts/ik/ob_ik_processor.cpp +++ b/src/storage/fts/ik/ob_ik_processor.cpp @@ -52,7 +52,7 @@ TokenizeContext::TokenizeContext(ObCollationType coll_type, : coll_type_(coll_type), charset_info_(nullptr), charset_handler_(nullptr), charset_type_(CHARSET_INVALID), fulltext_(fulltext), fulltext_len_(fulltext_len), cursor_(0), next_char_len_(0), handle_size_(0), is_smart_(is_smart), token_list_(allocator), - result_list_(allocator) + results_(allocator), result_idx_(0), buffer_start_cursor_(0) { } @@ -81,11 +81,33 @@ int TokenizeContext::init() return ret; } +int TokenizeContext::reuse_context(const char *fulltext, const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_ISNULL(fulltext) || OB_UNLIKELY(fulltext_len <= 0)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid fulltext", K(ret), KP(fulltext), K(fulltext_len)); + } else { + reset_resource(); + fulltext_ = fulltext; + fulltext_len_ = fulltext_len; + cursor_ = 0; + buffer_start_cursor_ = 0; + next_char_len_ = 0; + next_char_type_ = ObFTCharUtil::CharType::USELESS; + if (OB_FAIL(prepare_next_char())) { + LOG_WARN("failed to prepare first char", K(ret)); + } + } + return ret; +} + int TokenizeContext::reset_resource() { handle_size_ = 0; - result_list_.reset(); - token_list_.reset(); + results_.reuse(); + result_idx_ = 0; + token_list_.reuse(); return OB_SUCCESS; } @@ -200,6 +222,8 @@ bool TokenizeContext::iter_end() const { return cursor_ >= fulltext_len_; } bool TokenizeContext::is_smart() const { return is_smart_; } +bool TokenizeContext::is_results_exhaust() const { return result_idx_ >= results_.count(); } + int TokenizeContext::add_token(const char *fulltext, int64_t offset, int64_t length, @@ -222,7 +246,7 @@ int TokenizeContext::add_token(const char *fulltext, TokenizeContext::~TokenizeContext() { token_list_.reset(); - result_list_.reset(); + results_.reset(); } int TokenizeContext::get_next_token(const char *&word, @@ -231,10 +255,9 @@ int TokenizeContext::get_next_token(const char *&word, int64_t &char_cnt) { int ret = OB_SUCCESS; - if (!result_list_.empty()) { - ObIKToken &token = result_list_.get_first(); - result_list_.pop_front(); - if (!result_list_.empty()) { + if (result_idx_ < results_.count()) { + ObIKToken &token = results_.at(result_idx_++); + if (result_idx_ < results_.count()) { if (OB_FAIL(compound(token))) { LOG_WARN("Failed to compound", K(ret)); } else { @@ -256,11 +279,11 @@ int TokenizeContext::get_next_token(const char *&word, int TokenizeContext::compound(ObIKToken &token) { int ret = OB_SUCCESS; - ObList &list = result_list_; + ObFastSegmentArray &list = results_; if (is_smart_) { - if (!list.empty()) { + if (result_idx_ < list.count()) { if (ObIKTokenType::IK_ARABIC_TOKEN == token.type_) { - ObIKToken &next = list.get_first(); + ObIKToken &next = list.at(result_idx_); bool append = false; if (ObIKTokenType::IK_CNNUM_TOKEN == next.type_) { @@ -283,12 +306,12 @@ int TokenizeContext::compound(ObIKToken &token) // pass } if (append) { - list.pop_front(); + ++result_idx_; } } // There may be another round of append - if (OB_SUCC(ret)) { - ObIKToken next = list.get_first(); + if (OB_SUCC(ret) && result_idx_ < list.count()) { + ObIKToken &next = list.at(result_idx_); bool append = false; if (ObIKTokenType::IK_COUNT_TOKEN == next.type_) { if (token.offset_ + token.length_ == next.offset_) { @@ -298,7 +321,7 @@ int TokenizeContext::compound(ObIKToken &token) } } if (append) { - list.pop_front(); + ++result_idx_; } } } diff --git a/src/storage/fts/ik/ob_ik_processor.h b/src/storage/fts/ik/ob_ik_processor.h index 2a2ea95998..b9f1754826 100644 --- a/src/storage/fts/ik/ob_ik_processor.h +++ b/src/storage/fts/ik/ob_ik_processor.h @@ -22,6 +22,7 @@ #include "lib/utility/ob_macro_utils.h" #include "storage/fts/ik/ob_ik_char_util.h" #include "storage/fts/ik/ob_ik_token.h" +#include "storage/fts/ik/ob_fast_segment_array.h" namespace oceanbase { @@ -39,6 +40,7 @@ class TokenizeContext ~TokenizeContext(); int init(); + int reuse_context(const char *fulltext, const int64_t fulltext_len); int reset_resource(); int get_next_token(const char *&word, int64_t &word_len, int64_t &offset, int64_t &char_cnt); @@ -65,6 +67,7 @@ class TokenizeContext bool is_last() const; bool iter_end() const; bool is_smart() const; + bool is_results_exhaust() const; int add_chain(ObIKTokenChain *chain); int add_token(const char *fulltext, @@ -73,12 +76,16 @@ class TokenizeContext int64_t char_cnt, ObIKTokenType type); - ObFTSortList &token_list() { return token_list_; } + ObFTFastSortList &token_list() { return token_list_; } - ObList &result_list() { return result_list_; } + ObFastSegmentArray &get_results() { return results_; } int32_t handle_size() const { return handle_size_; } + void calc_buffer_start_cursor() { buffer_start_cursor_ = cursor_; } + int64_t get_buffer_start_cursor() const { return buffer_start_cursor_; } + int64_t get_buffer_end_cursor() const { return cursor_; } + private: int prepare_next_char(); @@ -96,8 +103,10 @@ class TokenizeContext uint32_t handle_size_; bool is_smart_; - ObFTSortList token_list_; - ObList result_list_; + ObFTFastSortList token_list_; + ObFastSegmentArray results_; + int64_t result_idx_; + int64_t buffer_start_cursor_; private: DISALLOW_COPY_AND_ASSIGN(TokenizeContext); @@ -110,6 +119,8 @@ class ObIIKProcessor virtual ~ObIIKProcessor() {} + virtual void reuse() {} + int process(TokenizeContext &ctx, const char *ch, const uint8_t char_len, diff --git a/src/storage/fts/ik/ob_ik_quantifier_processor.h b/src/storage/fts/ik/ob_ik_quantifier_processor.h index 570ddc2ecf..10514ad95a 100644 --- a/src/storage/fts/ik/ob_ik_quantifier_processor.h +++ b/src/storage/fts/ik/ob_ik_quantifier_processor.h @@ -36,6 +36,12 @@ class ObIKQuantifierProcessor : public ObIIKProcessor } ~ObIKQuantifierProcessor() override { count_hits_.reset(); } + void reuse() override + { + count_hits_.reset(); + reset(); + } + int do_process(TokenizeContext &ctx, const char *ch, const uint8_t char_len, diff --git a/src/storage/fts/ik/ob_ik_surrogate_processor.h b/src/storage/fts/ik/ob_ik_surrogate_processor.h index 2beeb9e95a..913d6fd0d0 100644 --- a/src/storage/fts/ik/ob_ik_surrogate_processor.h +++ b/src/storage/fts/ik/ob_ik_surrogate_processor.h @@ -30,6 +30,8 @@ class ObIKSurrogateProcessor : public ObIIKProcessor ObIKSurrogateProcessor() {} ~ObIKSurrogateProcessor() override {} + void reuse() override { reset(); } + int do_process(TokenizeContext &ctx, const char *ch, const uint8_t char_len, diff --git a/src/storage/fts/ik/ob_ik_token.cpp b/src/storage/fts/ik/ob_ik_token.cpp index 6cdc4eb821..d7ede7eca6 100644 --- a/src/storage/fts/ik/ob_ik_token.cpp +++ b/src/storage/fts/ik/ob_ik_token.cpp @@ -25,7 +25,8 @@ namespace oceanbase { namespace storage { -int ObFTSortList::add_token(const ObIKToken &token) +template +int ObFTSortList::add_token(const ObIKToken &token) { int ret = OB_SUCCESS; @@ -44,7 +45,8 @@ int ObFTSortList::add_token(const ObIKToken &token) LOG_WARN("fail to push back token", K(ret)); } } else { - for (ObFTSortList::CellIter iter = tokens_.last(); OB_SUCC(ret) && iter != tokens_.end(); + for (typename ObFTSortList::CellIter iter = tokens_.last(); + OB_SUCC(ret) && iter != tokens_.end(); --iter) { if (token < *iter) { continue; @@ -63,6 +65,18 @@ int ObFTSortList::add_token(const ObIKToken &token) return ret; } +template +int64_t ObFTSortList::max() +{ + return tokens_.empty() ? 0 : tokens_.get_last().offset_ + tokens_.get_last().length_; +} + +template +int64_t ObFTSortList::min() +{ + return tokens_.empty() ? 0 : tokens_.get_first().offset_; +} + int ObIKTokenChain::add_token_if_conflict(const ObIKToken &token, bool &added) { int ret = OB_SUCCESS; @@ -165,7 +179,7 @@ int ObIKTokenChain::copy(ObIKTokenChain *other) min_offset_ = other->min_offset_; max_offset_ = other->max_offset_; payload_ = other->payload_; - ObFTSortList::CellIter iter = other->list().tokens().begin(); + ObFTLightSortList::CellIter iter = other->list().tokens().begin(); for (; OB_SUCC(ret) && iter != other->list().tokens().end(); ++iter) { bool added = false; @@ -218,19 +232,5 @@ int ObIKTokenChain::pop_back(ObIKToken &token) return ret; } -int64_t ObFTSortList::max() -{ - if (tokens_.empty()) { - return 0; - } - return tokens_.get_last().offset_ + tokens_.get_last().length_; -} -int64_t ObFTSortList::min() -{ - if (tokens_.empty()) { - return 0; - } - return tokens_.get_first().offset_; -} } // namespace storage } // namespace oceanbase diff --git a/src/storage/fts/ik/ob_ik_token.h b/src/storage/fts/ik/ob_ik_token.h index 3788bccd87..c5372658c5 100644 --- a/src/storage/fts/ik/ob_ik_token.h +++ b/src/storage/fts/ik/ob_ik_token.h @@ -19,10 +19,13 @@ #include "lib/allocator/ob_allocator.h" #include "lib/list/ob_list.h" +#include "storage/fts/ik/ob_fast_list.h" namespace oceanbase { namespace storage { +static constexpr uint32_t IK_HANDLE_SIZE_LIMIT = 256; + enum class ObIKTokenType : int8_t { IK_CHINESE_TOKEN = 0, @@ -71,6 +74,7 @@ struct ObIKToken } }; +template class ObFTSortList { public: @@ -83,21 +87,26 @@ class ObFTSortList void reset() { tokens_.reset(); } + void reuse() { tokens_.reuse(); } + int64_t min(); int64_t max(); - ObList &tokens() { return tokens_; } - const ObList &tokens() const { return tokens_; } + ListType &tokens() { return tokens_; } + const ListType &tokens() const { return tokens_; } public: - typedef ObList::iterator CellIter; - typedef ObList::const_iterator ConstCellIter; + typedef typename ListType::iterator CellIter; + typedef typename ListType::const_iterator ConstCellIter; private: - ObList tokens_; + ListType tokens_; }; +typedef ObFTSortList> ObFTLightSortList; +typedef ObFTSortList> ObFTFastSortList; + class ObIKTokenChain { public: @@ -113,7 +122,7 @@ class ObIKTokenChain bool check_conflict(const ObIKToken &token); - ObFTSortList &list() { return list_; } + ObFTLightSortList &list() { return list_; } bool better_than(const ObIKTokenChain &other) const; @@ -135,7 +144,7 @@ class ObIKTokenChain int min_offset_ = -1; int max_offset_ = -1; int payload_ = -1; - ObFTSortList list_; + ObFTLightSortList list_; }; } // namespace storage diff --git a/src/storage/fts/ob_beng_ft_parser.cpp b/src/storage/fts/ob_beng_ft_parser.cpp index 5acb7ff780..c3ff909702 100644 --- a/src/storage/fts/ob_beng_ft_parser.cpp +++ b/src/storage/fts/ob_beng_ft_parser.cpp @@ -44,6 +44,8 @@ int ObBEngFTParser::get_next_token( if (OB_UNLIKELY(!is_inited_)) { ret = OB_NOT_INIT; LOG_WARN("beng ft parser isn't initialized", K(ret), K(is_inited_)); + } else if (ascii_fast_path_) { + ret = get_next_ascii_token(word, word_len, char_len, word_freq); } else if (OB_ISNULL(token_stream_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("token stream is nullptr", K(ret), KP(token_stream_)); @@ -54,7 +56,7 @@ int ObBEngFTParser::get_next_token( } else if (OB_ISNULL(token.ptr_) || OB_UNLIKELY(0 >= token.len_ || 0 >= token_freq)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), KP(token.ptr_), K(token.len_), K(token_freq)); - } else if (OB_ISNULL(buf = static_cast(allocator_.alloc(token.len_)))) { + } else if (OB_ISNULL(buf = static_cast(scratch_alloc_.alloc(token.len_)))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("fail to allocate word memory", K(ret), K(token.len_)); } else { @@ -68,6 +70,79 @@ int ObBEngFTParser::get_next_token( return ret; } +bool ObBEngFTParser::is_ascii_alnum(const unsigned char ch) +{ + return (ch >= '0' && ch <= '9') + || (ch >= 'A' && ch <= 'Z') + || (ch >= 'a' && ch <= 'z'); +} + +bool ObBEngFTParser::is_ascii_document(const char *text, const int64_t text_len) +{ + bool is_ascii = OB_NOT_NULL(text) && text_len > 0; + for (int64_t i = 0; is_ascii && i < text_len; ++i) { + is_ascii = 0 == (static_cast(text[i]) & 0x80); + } + return is_ascii; +} + +int ObBEngFTParser::get_next_ascii_token( + const char *&word, + int64_t &word_len, + int64_t &char_len, + int64_t &word_freq) +{ + int ret = OB_SUCCESS; + const char *text = doc_.ptr_; + const int64_t text_len = doc_.len_; + while (ascii_pos_ < text_len + && !is_ascii_alnum(static_cast(text[ascii_pos_]))) { + ++ascii_pos_; + } + if (ascii_pos_ >= text_len) { + ret = OB_ITER_END; + } else { + const int64_t token_start = ascii_pos_; + while (ascii_pos_ < text_len + && is_ascii_alnum(static_cast(text[ascii_pos_]))) { + ++ascii_pos_; + } + word = text + token_start; + word_len = ascii_pos_ - token_start; + char_len = word_len; + word_freq = 1; + } + return ret; +} + +int ObBEngFTParser::reuse_parser(const char *fulltext, const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("basic english ft parser hasn't been initialized", K(ret)); + } else if (OB_UNLIKELY(nullptr == fulltext || 0 >= fulltext_len)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid arguments", K(ret), KP(fulltext), K(fulltext_len)); + } else if (OB_UNLIKELY(UINT32_MAX < fulltext_len)) { + ret = OB_NOT_SUPPORTED; + LOG_WARN("too large document, english analyzer hasn't be supported", K(ret), K(fulltext_len)); + } else { + doc_.set_string(fulltext, fulltext_len); + ascii_pos_ = 0; + ascii_fast_path_ = is_ascii_document(fulltext, fulltext_len); + if (ascii_fast_path_) { + token_stream_ = nullptr; + } else if (OB_FAIL(segment(doc_, token_stream_))) { + LOG_WARN("fail to segment fulltext", K(ret)); + } else if (OB_ISNULL(token_stream_)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("token stream is null", K(ret)); + } + } + return ret; +} + int ObBEngFTParser::init(ObFTParserParam *param) { int ret = OB_SUCCESS; @@ -85,7 +160,13 @@ int ObBEngFTParser::init(ObFTParserParam *param) analysis_ctx_.cs_ = param->cs_; analysis_ctx_.filter_stopword_ = false; analysis_ctx_.need_grouping_ = false; - if (OB_FAIL(english_analyzer_.init(analysis_ctx_, *param->allocator_))) { + ascii_pos_ = 0; + ascii_fast_path_ = is_ascii_document(param->fulltext_, param->ft_length_); + common::ObIAllocator *metadata_alloc = OB_NOT_NULL(param->metadata_alloc_) + ? param->metadata_alloc_ : param->allocator_; + if (ascii_fast_path_) { + is_inited_ = true; + } else if (OB_FAIL(english_analyzer_.init(analysis_ctx_, *metadata_alloc))) { LOG_WARN("fail to init english analyzer", K(ret), KPC(param), K(analysis_ctx_)); } else if (OB_FAIL(segment(doc_, token_stream_))) { LOG_WARN("fail to segment fulltext by parser", K(ret), KP(param->fulltext_), K(param->ft_length_)); @@ -126,6 +207,8 @@ void ObBEngFTParser::reset() english_analyzer_.reset(); doc_.reset(); token_stream_ = nullptr; + ascii_pos_ = 0; + ascii_fast_path_ = false; is_inited_ = false; } @@ -158,17 +241,28 @@ int ObBasicEnglishFTParserDesc::segment( } else if (OB_ISNULL(param) || OB_ISNULL(param->fulltext_) || OB_UNLIKELY(!param->is_valid())) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid argument", K(ret), KPC(param)); - } else if (OB_ISNULL(parser = OB_NEWx(ObBEngFTParser, param->allocator_, *(param->allocator_)))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate basic english ft parser", K(ret)); - } else if (OB_FAIL(parser->init(param))) { - LOG_WARN("fail to init basic english parser", K(ret), KPC(param)); } else { - iter = parser; - } - - if (OB_FAIL(ret)) { - OB_DELETEx(ObBEngFTParser, param->allocator_, parser); + common::ObIAllocator *metadata_alloc = OB_NOT_NULL(param->metadata_alloc_) + ? param->metadata_alloc_ : param->allocator_; + common::ObIAllocator *scratch_alloc = OB_NOT_NULL(param->scratch_alloc_) + ? param->scratch_alloc_ : param->allocator_; + if (OB_ISNULL(metadata_alloc) || OB_ISNULL(scratch_alloc)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid allocator", K(ret), KP(metadata_alloc), KP(scratch_alloc)); + } else if (OB_ISNULL(parser = OB_NEWx(ObBEngFTParser, + metadata_alloc, + *metadata_alloc, + *scratch_alloc))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate basic english ft parser", K(ret)); + } else if (OB_FAIL(parser->init(param))) { + LOG_WARN("fail to init basic english parser", K(ret), KPC(param)); + } else { + iter = parser; + } + if (OB_FAIL(ret) && OB_NOT_NULL(metadata_alloc)) { + OB_DELETEx(ObBEngFTParser, metadata_alloc, parser); + } } return ret; @@ -180,9 +274,12 @@ void ObBasicEnglishFTParserDesc::free_token_iter( { if (OB_NOT_NULL(iter)) { abort_unless(nullptr != param); - abort_unless(nullptr != param->allocator_); + common::ObIAllocator *metadata_alloc = OB_NOT_NULL(param->metadata_alloc_) + ? param->metadata_alloc_ : param->allocator_; + abort_unless(nullptr != metadata_alloc); iter->~ObITokenIterator(); - param->allocator_->free(iter); + metadata_alloc->free(iter); + iter = nullptr; } } @@ -192,6 +289,7 @@ int ObBasicEnglishFTParserDesc::get_add_word_flag(ObAddWordFlag &flag) const flag.set_min_max_word(); flag.set_stop_word(); flag.set_groupby_word(); + flag.set_casedown(); return ret; } diff --git a/src/storage/fts/ob_beng_ft_parser.h b/src/storage/fts/ob_beng_ft_parser.h index fea7ed7ce3..fffc4e80f2 100644 --- a/src/storage/fts/ob_beng_ft_parser.h +++ b/src/storage/fts/ob_beng_ft_parser.h @@ -20,30 +20,35 @@ #include "lib/utility/ob_macro_utils.h" #include "lib/utility/ob_print_utils.h" #include "share/text_analysis/ob_text_analyzer.h" -#include "plugin/interface/ob_plugin_ftparser_intf.h" +#include "storage/fts/ob_i_ft_parser.h" namespace oceanbase { namespace storage { -class ObBEngFTParser final : public plugin::ObITokenIterator +class ObBEngFTParser final : public ObIFTParser { public: static const int64_t FT_MIN_WORD_LEN = 3; static const int64_t FT_MAX_WORD_LEN = 84; public: - explicit ObBEngFTParser(common::ObIAllocator &allocator) - : allocator_(allocator), + ObBEngFTParser(common::ObIAllocator &metadata_alloc, + common::ObIAllocator &scratch_alloc) + : metadata_alloc_(metadata_alloc), + scratch_alloc_(scratch_alloc), analysis_ctx_(), english_analyzer_(), doc_(), token_stream_(nullptr), + ascii_pos_(0), + ascii_fast_path_(false), is_inited_(false) {} ~ObBEngFTParser() { reset(); } int init(plugin::ObFTParserParam *param); + int reuse_parser(const char *fulltext, const int64_t fulltext_len) override; void reset(); virtual int get_next_token( const char *&word, @@ -56,12 +61,22 @@ class ObBEngFTParser final : public plugin::ObITokenIterator int segment( const common::ObDatum &doc, share::ObITokenStream *&token_stream); + int get_next_ascii_token( + const char *&word, + int64_t &word_len, + int64_t &char_len, + int64_t &word_freq); + static bool is_ascii_document(const char *text, const int64_t text_len); + static bool is_ascii_alnum(const unsigned char ch); private: - common::ObIAllocator &allocator_; + common::ObIAllocator &metadata_alloc_; + common::ObIAllocator &scratch_alloc_; share::ObTextAnalysisCtx analysis_ctx_; share::ObEnglishTextAnalyzer english_analyzer_; common::ObDatum doc_; share::ObITokenStream *token_stream_; + int64_t ascii_pos_; + bool ascii_fast_path_; bool is_inited_; DISALLOW_COPY_AND_ASSIGN(ObBEngFTParser); diff --git a/src/storage/fts/ob_fts_plugin_helper.cpp b/src/storage/fts/ob_fts_plugin_helper.cpp index 6f59ece2a9..31c073fbf4 100644 --- a/src/storage/fts/ob_fts_plugin_helper.cpp +++ b/src/storage/fts/ob_fts_plugin_helper.cpp @@ -26,6 +26,7 @@ #include "share/ob_force_print_log.h" #include "storage/fts/dict/ob_ft_dict_hub.h" #include "storage/fts/ob_fts_parser_property.h" +#include "storage/fts/ob_i_ft_parser.h" #include "storage/fts/ob_fts_stop_word.h" using namespace oceanbase::plugin; @@ -237,16 +238,27 @@ int ObFTParseHelper::segment( const char *ft, const int64_t ft_len, common::ObIAllocator &allocator, - ObAddWord &add_word) + ObAddWord &add_word) const { int ret = OB_SUCCESS; if (OB_UNLIKELY(parser_version < 0 || nullptr == parser_desc || nullptr == cs || nullptr == ft || 0 >= ft_len)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(parser_version), KP(parser_desc), KP(cs), K(ft), K(ft_len)); } else { - ObFTParserParam param; - ObITokenIterator *iter = nullptr; + ObFTParserParam local_param; + const bool reuse_parser = can_reuse_parser(); + ObFTParserParam ¶m = reuse_parser ? parser_param_ : local_param; + ObITokenIterator *iter = reuse_parser ? cached_iter_ : nullptr; + if (reuse_parser && OB_NOT_NULL(iter) && cached_charset_ != cs) { + destroy_cached_parser(); + iter = nullptr; + } + if (reuse_parser) { + parser_scratch_alloc_.reset_remain_one_page(); + } param.allocator_ = &allocator; + param.metadata_alloc_ = reuse_parser ? &parser_metadata_alloc_ : &allocator; + param.scratch_alloc_ = reuse_parser ? &parser_scratch_alloc_ : &allocator; param.cs_ = cs; param.fulltext_ = ft; param.ft_length_ = ft_len; @@ -258,12 +270,23 @@ int ObFTParseHelper::segment( param.min_ngram_size_ = property.min_ngram_token_size_; param.max_ngram_size_ = property.max_ngram_token_size_; - if (OB_FAIL(parser_desc->segment(¶m, iter))) { + if (reuse_parser && OB_NOT_NULL(iter)) { + if (OB_FAIL(static_cast(iter)->reuse_parser(ft, ft_len))) { + LOG_WARN("fail to reuse parser", K(ret), K(parser_name_), KP(iter)); + } + } else if (OB_FAIL(parser_desc->segment(¶m, iter))) { LOG_WARN("fail to segment", K(ret), K(param)); + } else if (reuse_parser) { + cached_iter_ = iter; + cached_charset_ = cs; } else if (OB_ISNULL(iter)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected error, token iterator is nullptr", K(ret), KP(iter)); - } else { + } + if (OB_SUCC(ret) && OB_ISNULL(iter)) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("unexpected error, token iterator is nullptr", K(ret), KP(iter)); + } else if (OB_SUCC(ret)) { const char *word = nullptr; int64_t word_len = 0; int64_t char_cnt = 0; @@ -281,9 +304,11 @@ int ObFTParseHelper::segment( ret = OB_SUCCESS; } } - if (OB_NOT_NULL(iter)) { + if (!reuse_parser && OB_NOT_NULL(iter)) { parser_desc->free_token_iter(¶m, iter); iter = nullptr; + } else if (reuse_parser && OB_SUCCESS != ret) { + destroy_cached_parser(); } } return ret; @@ -296,6 +321,12 @@ ObFTParseHelper::ObFTParseHelper() parser_name_(), add_word_flag_(), parser_property_(), + parser_metadata_alloc_(), + parser_scratch_alloc_(), + parser_param_(), + cached_iter_(nullptr), + cached_charset_(nullptr), + enable_parser_reuse_(false), is_inited_(false) { } @@ -308,7 +339,8 @@ ObFTParseHelper::~ObFTParseHelper() int ObFTParseHelper::init( common::ObIAllocator *allocator, const common::ObString &plugin_name, - const common::ObString &plugin_properties) + const common::ObString &plugin_properties, + const bool enable_parser_reuse) { int ret = OB_SUCCESS; if (OB_UNLIKELY(is_inited_)) { @@ -335,6 +367,9 @@ int ObFTParseHelper::init( LOG_WARN("fail to set add word flag", K(ret), K(parser_name_)); } else { allocator_ = allocator; + parser_metadata_alloc_.set_attr(common::ObMemAttr("FTParserMeta")); + parser_scratch_alloc_.set_attr(common::ObMemAttr("FTParserScratch")); + enable_parser_reuse_ = enable_parser_reuse; is_inited_ = true; LOG_TRACE("succeed to init ft parser helper", K(ret), K(plugin_name), K(plugin_properties), KPC(this)); } @@ -346,13 +381,33 @@ int ObFTParseHelper::init( void ObFTParseHelper::reset() { + destroy_cached_parser(); parser_desc_ = nullptr; plugin_param_ = nullptr; allocator_ = nullptr; add_word_flag_.clear(); + parser_param_.reset(); + parser_scratch_alloc_.reset(); + parser_metadata_alloc_.reset(); + cached_charset_ = nullptr; + enable_parser_reuse_ = false; is_inited_ = false; } +bool ObFTParseHelper::can_reuse_parser() const +{ + return enable_parser_reuse_ && parser_name_.is_ik(); +} + +void ObFTParseHelper::destroy_cached_parser() const +{ + if (OB_NOT_NULL(cached_iter_) && OB_NOT_NULL(parser_desc_)) { + parser_desc_->free_token_iter(&parser_param_, cached_iter_); + } + cached_iter_ = nullptr; + cached_charset_ = nullptr; +} + int ObFTParseHelper::segment( const ObObjMeta &meta, const char *fulltext, diff --git a/src/storage/fts/ob_fts_plugin_helper.h b/src/storage/fts/ob_fts_plugin_helper.h index 84fefd75bf..0cd5fbf62f 100644 --- a/src/storage/fts/ob_fts_plugin_helper.h +++ b/src/storage/fts/ob_fts_plugin_helper.h @@ -18,6 +18,7 @@ #define OB_FTS_PLUGIN_HELPER_H_ #include "lib/allocator/ob_fifo_allocator.h" +#include "lib/allocator/page_arena.h" #include "lib/charset/ob_charset.h" #include "lib/string/ob_string.h" #include "object/ob_object.h" @@ -36,6 +37,7 @@ namespace plugin { class ObIFTParserDesc; class ObPluginParam; +class ObITokenIterator; } namespace storage @@ -162,7 +164,8 @@ class ObFTParseHelper final int init( common::ObIAllocator *allocator, const common::ObString &plugin_name, - const common::ObString &plugin_properties); + const common::ObString &plugin_properties, + const bool enable_parser_reuse = false); /** * Split document into multiple words * @@ -209,7 +212,7 @@ class ObFTParseHelper final TO_STRING_KV(KP_(allocator), K_(parser_name), KP_(parser_desc), K_(is_inited)); private: - static int segment( + int segment( const ObFTParserProperty &property, const int64_t parser_version, const plugin::ObIFTParserDesc *parser_desc, @@ -218,7 +221,9 @@ class ObFTParseHelper final const char *fulltext, const int64_t fulltext_len, common::ObIAllocator &allocator, - ObAddWord &add_word); + ObAddWord &add_word) const; + bool can_reuse_parser() const; + void destroy_cached_parser() const; int set_add_word_flag(const plugin::ObIFTParserDesc &ftparser_desc); private: common::ObIAllocator *allocator_; @@ -227,6 +232,12 @@ class ObFTParseHelper final ObFTParser parser_name_; ObAddWordFlag add_word_flag_; ObFTParserProperty parser_property_; + mutable common::ObArenaAllocator parser_metadata_alloc_; + mutable common::ObArenaAllocator parser_scratch_alloc_; + mutable plugin::ObFTParserParam parser_param_; + mutable plugin::ObITokenIterator *cached_iter_; + mutable const ObCharsetInfo *cached_charset_; + bool enable_parser_reuse_; bool is_inited_; private: diff --git a/src/storage/fts/ob_fts_stop_word.cpp b/src/storage/fts/ob_fts_stop_word.cpp index a8344c7f19..6b6802fe32 100644 --- a/src/storage/fts/ob_fts_stop_word.cpp +++ b/src/storage/fts/ob_fts_stop_word.cpp @@ -30,6 +30,25 @@ namespace storage { using common::ObArenaAllocator; +namespace +{ + +class ObWordFrequencyUpdater +{ +public: + explicit ObWordFrequencyUpdater(const int64_t increment) : increment_(increment) {} + + void operator()(common::hash::HashMapPair &entry) const + { + entry.second += increment_; + } + +private: + int64_t increment_; +}; + +} // namespace + //////////////////////////////////////////////////////////////////////////////// // class ObStopWordChecker ObStopWordChecker::~ObStopWordChecker() @@ -196,16 +215,26 @@ int ObAddWord::casedown_word(const ObFTWord &src, ObFTWord &dst) ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid src ft word", K(ret), K(src)); } else if (flag_.casedown()) { - ObString dst_str; - if (OB_FAIL(ObCharset::tolower( - word_meta_.get_collation_type(), - src.get_word().get_string(), - dst_str, - allocator_))) { - LOG_WARN("fail to tolower", K(ret), K(src), K(word_meta_)); + const ObString &src_str = src.get_word().get_string(); + bool is_lower_ascii = true; + for (int64_t i = 0; is_lower_ascii && i < src_str.length(); ++i) { + const unsigned char ch = static_cast(src_str.ptr()[i]); + is_lower_ascii = ch < 0x80 && !(ch >= 'A' && ch <= 'Z'); + } + if (is_lower_ascii) { + dst = src; } else { - ObFTWord tmp(dst_str.length(), dst_str.ptr(), word_meta_); - dst = tmp; + ObString dst_str; + if (OB_FAIL(ObCharset::tolower( + word_meta_.get_collation_type(), + src_str, + dst_str, + allocator_))) { + LOG_WARN("fail to tolower", K(ret), K(src), K(word_meta_)); + } else { + ObFTWord tmp(dst_str.length(), dst_str.ptr(), word_meta_); + dst = tmp; + } } } else { dst = src; @@ -252,7 +281,6 @@ int ObAddWord::check_stopword(const ObFTWord &ft_word, bool &is_stopword) int ObAddWord::groupby_word(const ObFTWord &word, const int64_t word_freq) { int ret = OB_SUCCESS; - int64_t word_count = 0; if (OB_UNLIKELY(word.empty() || word_freq <= 0)) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid arguments", K(ret), K(word), K(word_freq)); @@ -260,16 +288,10 @@ int ObAddWord::groupby_word(const ObFTWord &word, const int64_t word_freq) if (OB_FAIL(word_map_.set_refactored(word, 1/*word count*/))) { LOG_WARN("fail to set fulltext word and count", K(ret), K(word)); } - } else if (OB_FAIL(word_map_.get_refactored(word, word_count)) && OB_HASH_NOT_EXIST != ret) { - LOG_WARN("fail to get fulltext word", K(ret), K(word)); } else { - if (OB_HASH_NOT_EXIST == ret) { - word_count = 1; - } else { - word_count += word_freq; - } - if (OB_FAIL(word_map_.set_refactored(word, word_count, 1/*overwrite*/))) { - LOG_WARN("fail to set fulltext word and count", K(ret), K(word), K(word_count)); + ObWordFrequencyUpdater updater(word_freq); + if (OB_FAIL(word_map_.set_or_update(word, 1 /* first occurrence */, updater))) { + LOG_WARN("fail to insert or update fulltext word", K(ret), K(word), K(word_freq)); } } return ret; diff --git a/src/storage/fts/ob_i_ft_parser.h b/src/storage/fts/ob_i_ft_parser.h new file mode 100644 index 0000000000..96dcf58383 --- /dev/null +++ b/src/storage/fts/ob_i_ft_parser.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2025 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OB_I_FT_PARSER_H_ +#define OB_I_FT_PARSER_H_ + +#include "plugin/interface/ob_plugin_ftparser_intf.h" + +namespace oceanbase +{ +namespace storage +{ + +class ObIFTParser : public plugin::ObITokenIterator +{ +public: + ObIFTParser() = default; + virtual ~ObIFTParser() = default; + virtual int reuse_parser(const char *fulltext, const int64_t fulltext_len) = 0; +}; + +} // namespace storage +} // namespace oceanbase + +#endif // OB_I_FT_PARSER_H_ diff --git a/src/storage/fts/ob_ik_ft_parser.cpp b/src/storage/fts/ob_ik_ft_parser.cpp index b468544f05..f839da0f3b 100644 --- a/src/storage/fts/ob_ik_ft_parser.cpp +++ b/src/storage/fts/ob_ik_ft_parser.cpp @@ -64,6 +64,8 @@ int ObIKFTParser::init(const ObFTParserParam ¶m) LOG_WARN("Failed to init ctx", K(ret)); } else if (OB_FAIL(init_segmenter(param))) { LOG_WARN("Failed to init segmenters", K(ret)); + } else if (OB_FAIL(arbitrator_.prepare())) { + LOG_WARN("Failed to prepare arbitrator", K(ret)); } if (OB_FAIL(ret)) { @@ -122,11 +124,33 @@ int ObIKFTParser::get_next_token(const char *&word, return ret; } +int ObIKFTParser::reuse_parser(const char *fulltext, const int64_t fulltext_len) +{ + int ret = OB_SUCCESS; + if (OB_UNLIKELY(!is_inited_)) { + ret = OB_NOT_INIT; + LOG_WARN("ik ft parser has not been initialized", K(ret)); + } else if (OB_UNLIKELY(nullptr == fulltext || 0 >= fulltext_len)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("invalid fulltext", K(ret), KP(fulltext), K(fulltext_len)); + } else { + for (ObList::iterator iter = segmenters_.begin(); + iter != segmenters_.end(); ++iter) { + (*iter)->reuse(); + } + if (OB_FAIL(ctx_->reuse_context(fulltext, fulltext_len))) { + LOG_WARN("failed to reuse tokenize context", K(ret)); + } + scratch_alloc_.reset_remain_one_page(); + } + return ret; +} + int ObIKFTParser::produce() { int ret = OB_SUCCESS; // Loop until end or has data to output - while (OB_SUCC(ret) && ctx_->result_list().empty() && !ctx_->iter_end()) { + while (OB_SUCC(ret) && ctx_->is_results_exhaust() && !ctx_->iter_end()) { if (OB_FAIL(process_next_batch())) { if (OB_ITER_END == ret) { // ok @@ -169,6 +193,7 @@ int ObIKFTParser::process_next_batch() if (ctx_->iter_end()) { ret = OB_ITER_END; } else { + ctx_->calc_buffer_start_cursor(); while (OB_SUCC(ret) && !do_seg && !ctx_->iter_end()) { const char *ch; uint8_t char_len = 0; @@ -181,7 +206,8 @@ int ObIKFTParser::process_next_batch() LOG_WARN("Failed to process one char", K(ret)); } else { // 1. check segmention - if (ctx_->handle_size() > SEGMENT_LIMIT && type == ObFTCharUtil::CharType::USELESS) { + if (ctx_->handle_size() >= IK_HANDLE_SIZE_LIMIT + && type == ObFTCharUtil::CharType::USELESS) { do_seg = true; } @@ -196,11 +222,12 @@ int ObIKFTParser::process_next_batch() } if (OB_SUCC(ret) || OB_ITER_END == ret) { - ObIKArbitrator arb; - if (OB_FAIL(arb.process(*ctx_))) { + if (OB_FAIL(arbitrator_.process(*ctx_))) { LOG_WARN("Failed to process arbitrator", K(ret)); - } else if (OB_FAIL(arb.output_result(*ctx_))) { + } else if (OB_FAIL(arbitrator_.output_result(*ctx_))) { LOG_WARN("Failed to make result list"); + } else { + arbitrator_.reuse(); } } else { // Already logged. @@ -235,17 +262,23 @@ int ObIKFTParserDesc::segment(ObFTParserParam *param, ObITokenIterator *&iter) c LOG_WARN("invalid argument", K(ret), KPC(param)); } else if (OB_FAIL(ObFTParsePluginData::instance().get_dict_hub(hub))) { LOG_WARN("Failed to get dict hub.", K(ret)); - } else if (OB_ISNULL(parser = OB_NEWx(ObIKFTParser, param->allocator_, *(param->allocator_), hub))) { - ret = OB_ALLOCATE_MEMORY_FAILED; - LOG_WARN("fail to allocate ik ft parser", K(ret)); - } else if (OB_FAIL(parser->init(*param))) { - LOG_WARN("fail to init ik parser", K(ret), KPC(param)); } else { - iter = parser; - } - - if (OB_FAIL(ret)) { - OB_DELETEx(ObIKFTParser, param->allocator_, parser); + common::ObIAllocator *metadata_alloc = OB_NOT_NULL(param->metadata_alloc_) + ? param->metadata_alloc_ : param->allocator_; + if (OB_ISNULL(metadata_alloc)) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("metadata allocator is null", K(ret)); + } else if (OB_ISNULL(parser = OB_NEWx(ObIKFTParser, metadata_alloc, *metadata_alloc, hub))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("fail to allocate ik ft parser", K(ret)); + } else if (OB_FAIL(parser->init(*param))) { + LOG_WARN("fail to init ik parser", K(ret), KPC(param)); + } else { + iter = parser; + } + if (OB_FAIL(ret) && OB_NOT_NULL(metadata_alloc)) { + OB_DELETEx(ObIKFTParser, metadata_alloc, parser); + } } return ret; @@ -254,8 +287,14 @@ int ObIKFTParserDesc::segment(ObFTParserParam *param, ObITokenIterator *&iter) c void ObIKFTParserDesc::free_token_iter(ObFTParserParam *param, ObITokenIterator *&iter) const { - iter->~ObITokenIterator(); - param->allocator_->free(iter); + if (OB_NOT_NULL(iter)) { + common::ObIAllocator *metadata_alloc = OB_NOT_NULL(param->metadata_alloc_) + ? param->metadata_alloc_ : param->allocator_; + abort_unless(nullptr != metadata_alloc); + iter->~ObITokenIterator(); + metadata_alloc->free(iter); + iter = nullptr; + } } @@ -341,9 +380,9 @@ int ObIKFTParser::init_ctx(const ObFTParserParam ¶m) ret = OB_INVALID_ARGUMENT; LOG_WARN("Illegal collation type", K(ret)); } else if (OB_ISNULL(ctx_ = OB_NEWx(TokenizeContext, - &allocator_, + &metadata_alloc_, coll_type_, - allocator_, + metadata_alloc_, param.fulltext_, param.ft_length_, param.ik_param_.mode_ == ObFTIKParam::Mode::SMART))) { @@ -353,7 +392,7 @@ int ObIKFTParser::init_ctx(const ObFTParserParam ¶m) LOG_WARN("Failed to init ctx", K(ret)); } if (OB_FAIL(ret)) { - OB_DELETEx(TokenizeContext, &allocator_, ctx_); + OB_DELETEx(TokenizeContext, &metadata_alloc_, ctx_); } return ret; } @@ -366,22 +405,22 @@ int ObIKFTParser::init_segmenter(const ObFTParserParam ¶m) ObIKQuantifierProcessor *cnqsg = nullptr; ObIKCJKProcessor *cjksg = nullptr; ObIKSurrogateProcessor *surrogate_seg = nullptr; - if (OB_ISNULL(letter_seg = OB_NEWx(ObIKLetterProcessor, &allocator_))) { + if (OB_ISNULL(letter_seg = OB_NEWx(ObIKLetterProcessor, &metadata_alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc letter segmenter", K(ret)); } else if (OB_ISNULL(dict_quan_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Dict quan is null.", K(ret)); - } else if (OB_ISNULL(cnqsg = OB_NEWx(ObIKQuantifierProcessor, &allocator_, *dict_quan_, allocator_))) { + } else if (OB_ISNULL(cnqsg = OB_NEWx(ObIKQuantifierProcessor, &metadata_alloc_, *dict_quan_, scratch_alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc cn quantifier segmenter", K(ret)); } else if (OB_ISNULL(dict_main_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Dict main is null.", K(ret)); - } else if (OB_ISNULL(cjksg = OB_NEWx(ObIKCJKProcessor, &allocator_, *dict_main_, allocator_))) { + } else if (OB_ISNULL(cjksg = OB_NEWx(ObIKCJKProcessor, &metadata_alloc_, *dict_main_, scratch_alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc cjk segmenter", K(ret)); - } else if (OB_ISNULL(surrogate_seg = OB_NEWx(ObIKSurrogateProcessor, &allocator_))) { + } else if (OB_ISNULL(surrogate_seg = OB_NEWx(ObIKSurrogateProcessor, &metadata_alloc_))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc surrogate segmenter", K(ret)); } else if (OB_FAIL(segmenters_.push_back(letter_seg))) { @@ -400,10 +439,10 @@ int ObIKFTParser::init_segmenter(const ObFTParserParam ¶m) // push back by order, quantifier is before cjk if (OB_FAIL(ret)) { - OB_DELETEx(ObIKLetterProcessor, &allocator_, letter_seg); - OB_DELETEx(ObIKQuantifierProcessor, &allocator_, cnqsg); - OB_DELETEx(ObIKCJKProcessor, &allocator_, cjksg); - OB_DELETEx(ObIKSurrogateProcessor, &allocator_, surrogate_seg); + OB_DELETEx(ObIKLetterProcessor, &metadata_alloc_, letter_seg); + OB_DELETEx(ObIKQuantifierProcessor, &metadata_alloc_, cnqsg); + OB_DELETEx(ObIKCJKProcessor, &metadata_alloc_, cjksg); + OB_DELETEx(ObIKSurrogateProcessor, &metadata_alloc_, surrogate_seg); } return ret; } @@ -412,13 +451,14 @@ void ObIKFTParser::reset() { if (!OB_ISNULL(ctx_)) { ctx_->~TokenizeContext(); - allocator_.free(ctx_); + metadata_alloc_.free(ctx_); + ctx_ = nullptr; } for (ObIIKProcessor *segmenter : segmenters_) { if (!OB_ISNULL(segmenter)) { segmenter->~ObIIKProcessor(); - allocator_.free(segmenter); + metadata_alloc_.free(segmenter); } } segmenters_.clear(); @@ -429,17 +469,22 @@ void ObIKFTParser::reset() if (!OB_ISNULL(dict_main_)) { dict_main_->~ObIFTDict(); - allocator_.free(dict_main_); + metadata_alloc_.free(dict_main_); + dict_main_ = nullptr; } if (!OB_ISNULL(dict_quan_)) { dict_quan_->~ObIFTDict(); - allocator_.free(dict_quan_); + metadata_alloc_.free(dict_quan_); + dict_quan_ = nullptr; } if (!OB_ISNULL(dict_stop_)) { dict_stop_->~ObIFTDict(); - allocator_.free(dict_stop_); + metadata_alloc_.free(dict_stop_); + dict_stop_ = nullptr; } + scratch_alloc_.reset(); + is_inited_ = false; } @@ -452,14 +497,14 @@ int ObIKFTParser::build_dict_from_cache(const ObFTDictDesc &desc, if (OB_ISNULL(hub_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("Hub is null", K(ret)); - } else if (OB_ISNULL(dict = OB_NEWx(ObFTRangeDict, &allocator_, allocator_, &container, desc))) { + } else if (OB_ISNULL(dict = OB_NEWx(ObFTRangeDict, &metadata_alloc_, metadata_alloc_, &container, desc))) { ret = OB_ALLOCATE_MEMORY_FAILED; LOG_WARN("Failed to alloc dict", K(ret)); } else if (OB_FAIL(dict->init())) { LOG_WARN("Failed to init dict", K(ret)); } if (OB_FAIL(ret)) { - OB_DELETEx(ObIFTDict, &allocator_, dict); + OB_DELETEx(ObIFTDict, &metadata_alloc_, dict); } return ret; } diff --git a/src/storage/fts/ob_ik_ft_parser.h b/src/storage/fts/ob_ik_ft_parser.h index d25901f528..f34473fd06 100644 --- a/src/storage/fts/ob_ik_ft_parser.h +++ b/src/storage/fts/ob_ik_ft_parser.h @@ -18,11 +18,13 @@ #define _OCEANBASE_STORAGE_FTS_OB_IK_FT_PARSER_H_ #include "lib/allocator/ob_allocator.h" +#include "lib/allocator/page_arena.h" #include "storage/fts/dict/ob_ft_cache_container.h" #include "storage/fts/dict/ob_ft_dict.h" #include "storage/fts/dict/ob_ft_dict_def.h" +#include "storage/fts/ik/ob_ik_arbitrator.h" #include "storage/fts/ik/ob_ik_processor.h" -#include "plugin/interface/ob_plugin_ftparser_intf.h" +#include "storage/fts/ob_i_ft_parser.h" #include namespace oceanbase @@ -31,28 +33,32 @@ namespace storage { class ObFTDictHub; -class ObIKFTParser final : public plugin::ObITokenIterator +class ObIKFTParser final : public ObIFTParser { public: - ObIKFTParser(ObIAllocator &allocator, ObFTDictHub *hub) - : allocator_(allocator), + ObIKFTParser(ObIAllocator &metadata_alloc, ObFTDictHub *hub) + : metadata_alloc_(metadata_alloc), is_inited_(false), coll_type_(ObCollationType::CS_TYPE_INVALID), ctx_(nullptr), hub_(hub), - segmenters_(allocator_), - cache_main_(allocator), - cache_quan_(allocator), - cache_stop_(allocator), + segmenters_(metadata_alloc_), + cache_main_(metadata_alloc), + cache_quan_(metadata_alloc), + cache_stop_(metadata_alloc), dict_main_(nullptr), dict_quan_(nullptr), - dict_stop_(nullptr) + dict_stop_(nullptr), + scratch_alloc_(), + arbitrator_() { + scratch_alloc_.set_attr(common::ObMemAttr("IKParserScratch")); } virtual ~ObIKFTParser() { reset(); } int init(const plugin::ObFTParserParam ¶m); + int reuse_parser(const char *fulltext, const int64_t fulltext_len) override; int get_next_token(const char *&word, int64_t &word_len, @@ -89,8 +95,7 @@ class ObIKFTParser final : public plugin::ObITokenIterator ObIFTDict *&dict); private: - static constexpr int SEGMENT_LIMIT = 1000; - ObIAllocator &allocator_; + ObIAllocator &metadata_alloc_; bool is_inited_; ObCollationType coll_type_; @@ -106,6 +111,8 @@ class ObIKFTParser final : public plugin::ObITokenIterator ObIFTDict *dict_main_; ObIFTDict *dict_quan_; ObIFTDict *dict_stop_; + ObArenaAllocator scratch_alloc_; + ObIKArbitrator arbitrator_; DISABLE_COPY_ASSIGN(ObIKFTParser); }; diff --git a/src/storage/retrieval/ob_i_sparse_retrieval_iter.h b/src/storage/retrieval/ob_i_sparse_retrieval_iter.h index 34cdc1cf0b..757004162e 100644 --- a/src/storage/retrieval/ob_i_sparse_retrieval_iter.h +++ b/src/storage/retrieval/ob_i_sparse_retrieval_iter.h @@ -108,15 +108,17 @@ struct ObSparseRetrievalMergeParam filter_expr_(nullptr), topk_limit_(0), field_boost_(1.0), - max_batch_size_(1) + max_batch_size_(1), + skip_relevance_for_predicate_(false) {} ~ObSparseRetrievalMergeParam() {} bool need_project_relevance() const { return relevance_proj_expr_ != nullptr; } + bool need_calc_relevance() const { return !skip_relevance_for_predicate_; } bool need_filter() const { return filter_expr_ != nullptr; } bool need_pushdown_topk() const { return topk_limit_ > 0; } TO_STRING_KV(KPC_(dim_weights), KPC(limit_param_), KP_(eval_ctx), KP_(id_proj_expr), KP_(relevance_expr), KP_(relevance_proj_expr), KP_(filter_expr), - K_(topk_limit), K_(max_batch_size)); + K_(topk_limit), K_(max_batch_size), K_(skip_relevance_for_predicate)); const ObIArray *dim_weights_; // score weight for each dimension const common::ObLimitParam *limit_param_; sql::ObEvalCtx *eval_ctx_; @@ -127,6 +129,7 @@ struct ObSparseRetrievalMergeParam int64_t topk_limit_; double field_boost_; int64_t max_batch_size_; + bool skip_relevance_for_predicate_; }; class ObISparseRetrievalMergeIter diff --git a/src/storage/retrieval/ob_sparse_bmw_iter.cpp b/src/storage/retrieval/ob_sparse_bmw_iter.cpp index 743fc9c744..d883710266 100644 --- a/src/storage/retrieval/ob_sparse_bmw_iter.cpp +++ b/src/storage/retrieval/ob_sparse_bmw_iter.cpp @@ -42,7 +42,7 @@ int ObSRBMWIterImpl::init( ObSRDaaTRelevanceCollector &relevance_collector) { int ret = OB_SUCCESS; - if (OB_FAIL(ObSRDaaTIterImpl::init(iter_param, dim_iters, iter_allocator, relevance_collector))) { + if (OB_FAIL(ObSRDaaTIterImpl::init(iter_param, dim_iters, iter_allocator, &relevance_collector))) { LOG_WARN("failed to init ObSRDaaTIterImpl", K(ret)); } else if (OB_UNLIKELY(iter_param.topk_limit_ < 0)) { ret = OB_INVALID_ARGUMENT; @@ -142,7 +142,6 @@ int ObSRBMWIterImpl::top_k_search() int64_t pivot_iter_idx = 0; while (OB_SUCC(ret) && BMWStatus::FINISHED != status_) { - LOG_DEBUG("[Sparse Retrieval] top k search status", K_(status)); switch (status_) { case BMWStatus::FIND_NEXT_PIVOT: { if (OB_FAIL(next_pivot(pivot_iter_idx))) { @@ -381,8 +380,6 @@ int ObSRBMWIterImpl::next_pivot_range(int64_t &skip_range_cnt) // try to generate next candidate range bool is_candidate_range = false; while (OB_SUCC(ret) && !is_candidate_range) { - LOG_DEBUG("[Sparse Retrieval] BMW next pivot range", K(ret), K(is_candidate_range), - KPC(max_evaluated_id), KPC(min_unevaluated_id)); if (OB_FAIL(fill_merge_heap_with_shallow_dims(max_evaluated_id, last_border_inclusive))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { LOG_WARN("failed to fill merge heap with shallow dims", K(ret)); @@ -482,8 +479,6 @@ int ObSRBMWIterImpl::evaluate_pivot_range(const int64_t pivot_iter_idx, bool &is is_candidate = block_max_score > get_top_k_threshold(); } - LOG_DEBUG("[Sparse Retrieval] BMW evaluate pivot range", K(ret), - K(pivot_iter_idx), KPC(pivot_id), K(block_max_score), K(is_candidate), K(get_top_k_threshold())); return ret; } diff --git a/src/storage/retrieval/ob_sparse_daat_iter.cpp b/src/storage/retrieval/ob_sparse_daat_iter.cpp index 598f9064eb..6c73d55b5d 100644 --- a/src/storage/retrieval/ob_sparse_daat_iter.cpp +++ b/src/storage/retrieval/ob_sparse_daat_iter.cpp @@ -25,6 +25,7 @@ namespace storage ObSRMergeCmp::ObSRMergeCmp() : cmp_func_(nullptr), + fast_cmp_mode_(FastCmpMode::GENERIC), iter_ids_(nullptr), is_inited_(false) { @@ -44,6 +45,12 @@ int ObSRMergeCmp::init(ObDatumMeta id_meta, const ObFixedArray right_id ? 1 : 0); } else { - cmp_ret = tmp_ret; + const ObString left_id = left.get_string(); + const ObString right_id = right.get_string(); + const int32_t common_len = MIN(left_id.length(), right_id.length()); + cmp_ret = common_len > 0 ? MEMCMP(left_id.ptr(), right_id.ptr(), common_len) : 0; + if (0 == cmp_ret) { + cmp_ret = left_id.length() < right_id.length() + ? -1 + : (left_id.length() > right_id.length() ? 1 : 0); + } } } return ret; @@ -75,6 +101,7 @@ ObSRDaaTIterImpl::ObSRDaaTIterImpl() iter_allocator_(nullptr), iter_param_(nullptr), dim_iters_(nullptr), + dim_iter_cache_(nullptr), merge_cmp_(), merge_heap_(nullptr), relevance_collector_(nullptr), @@ -91,7 +118,7 @@ int ObSRDaaTIterImpl::init( ObSparseRetrievalMergeParam &iter_param, ObIArray &dim_iters, ObIAllocator &iter_allocator, - ObSRDaaTRelevanceCollector &relevance_collector) + ObSRDaaTRelevanceCollector *relevance_collector) { int ret = OB_SUCCESS; if (IS_INIT) { @@ -104,7 +131,7 @@ int ObSRDaaTIterImpl::init( iter_allocator_ = &iter_allocator; iter_param_ = &iter_param; dim_iters_ = &dim_iters; - relevance_collector_ = &relevance_collector; + relevance_collector_ = relevance_collector; const int64_t max_batch_size = OB_MAX(iter_param_->eval_ctx_->max_batch_size_, iter_param_->max_batch_size_); if (iter_param_->id_proj_expr_->datum_meta_.type_ == common::ObUInt64Type) { set_datum_func_ = ObISparseRetrievalMergeIter::set_datum_int; @@ -112,6 +139,10 @@ int ObSRDaaTIterImpl::init( set_datum_func_ = ObISparseRetrievalMergeIter::set_datum_shallow; } if (OB_UNLIKELY(dim_iters.count() == 0)) { + } else if (OB_ISNULL(dim_iter_cache_ = static_cast( + iter_allocator_->alloc(sizeof(ObISRDaaTDimIter *) * dim_iters.count())))) { + ret = OB_ALLOCATE_MEMORY_FAILED; + LOG_WARN("failed to allocate dimension iterator cache", K(ret), K(dim_iters.count())); } else if (OB_NOT_NULL(iter_param_->dim_weights_) && dim_iters.count() != iter_param_->dim_weights_->count()) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected dim iters count", K(ret), K(dim_iters_->count()), KP(iter_param_->dim_weights_)); @@ -130,10 +161,13 @@ int ObSRDaaTIterImpl::init( LOG_WARN("failed to init buffered domain ids array", K(ret)); } else if (OB_FAIL(buffered_domain_ids_.prepare_allocate(max_batch_size))) { LOG_WARN("failed to prepare allocate buffered domain ids array", K(ret)); - } else if (FALSE_IT(buffered_relevances_.set_allocator(iter_allocator_))) { - } else if (OB_FAIL(buffered_relevances_.init(max_batch_size))) { + } else if (iter_param_->need_project_relevance() + && FALSE_IT(buffered_relevances_.set_allocator(iter_allocator_))) { + } else if (iter_param_->need_project_relevance() + && OB_FAIL(buffered_relevances_.init(max_batch_size))) { LOG_WARN("failed to init buffered relevances array", K(ret)); - } else if (OB_FAIL(buffered_relevances_.prepare_allocate(max_batch_size))) { + } else if (iter_param_->need_project_relevance() + && OB_FAIL(buffered_relevances_.prepare_allocate(max_batch_size))) { LOG_WARN("failed to prepare allocate buffered relevances array", K(ret)); } else if (OB_FAIL(merge_cmp_.init(iter_param_->id_proj_expr_->datum_meta_, &iter_domain_ids_))) { LOG_WARN("failed to init loser tree comparator", K(ret)); @@ -141,6 +175,7 @@ int ObSRDaaTIterImpl::init( LOG_WARN("failed to init merge heap", K(ret)); } else { for (int64_t i = 0; i < dim_iters.count(); ++i) { + dim_iter_cache_[i] = dim_iters.at(i); next_round_iter_idxes_[i] = i; } } @@ -170,6 +205,7 @@ void ObSRDaaTIterImpl::reset() buffered_relevances_.reset(); next_round_iter_idxes_.reset(); next_round_cnt_ = 0; + dim_iter_cache_ = nullptr; input_row_cnt_ = 0; output_row_cnt_ = 0; is_inited_ = false; @@ -199,7 +235,7 @@ void ObSRDaaTIterImpl::reuse(const bool switch_tablet) int ret = OB_SUCCESS; score = 0.0; for (int64_t i = 0; OB_SUCC(ret) && i < dim_iters_->count(); ++i) { - ObISRDaaTDimIter *dim_iter = dim_iters_->at(i); + ObISRDaaTDimIter *dim_iter = dim_iter_cache_[i]; double dim_score = 0.0; if (OB_ISNULL(dim_iter) || OB_ISNULL(iter_param_->dim_weights_)) { ret = OB_ERR_UNEXPECTED; @@ -305,7 +341,7 @@ int ObSRDaaTIterImpl::fill_merge_heap() for (int64_t i = 0; OB_SUCC(ret) && i < next_round_cnt_; ++i) { const int64_t iter_idx = next_round_iter_idxes_[i]; ObISRDaaTDimIter *dim_iter = nullptr; - if (OB_ISNULL(dim_iter = dim_iters_->at(iter_idx))) { + if (OB_ISNULL(dim_iter = dim_iter_cache_[iter_idx])) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null dimension iter", K(ret), K(iter_idx), KPC_(iter_param)); } else if (OB_FAIL(dim_iter->get_next_row())) { @@ -314,16 +350,19 @@ int ObSRDaaTIterImpl::fill_merge_heap() } else { ret = OB_SUCCESS; } - } else if (OB_FAIL(dim_iter->get_curr_score(item.relevance_))) { + } else if (iter_param_->need_calc_relevance() + && OB_FAIL(dim_iter->get_curr_score(item.relevance_))) { LOG_WARN("fail to get current score", K(ret)); - } else if (OB_NOT_NULL(iter_param_->dim_weights_) && FALSE_IT(item.relevance_ = item.relevance_ * iter_param_->field_boost_ * iter_param_->dim_weights_->at(iter_idx))) { + } else if (iter_param_->need_calc_relevance() + && OB_NOT_NULL(iter_param_->dim_weights_) + && FALSE_IT(item.relevance_ = item.relevance_ + * iter_param_->field_boost_ + * iter_param_->dim_weights_->at(iter_idx))) { } else if (OB_FAIL(dim_iter->get_curr_id(iter_domain_ids_[iter_idx]))) { LOG_WARN("fail to get current doc id", K(ret)); } else if (FALSE_IT(item.iter_idx_ = iter_idx)) { } else if (OB_FAIL(merge_heap_->push(item))) { LOG_WARN("fail to push item to merge heap", K(ret), K(item)); - } else { - LOG_DEBUG("push item to merge heap", K(ret), K(i), K(iter_idx), K(next_round_cnt_), K(item), K(iter_domain_ids_[iter_idx])); } } @@ -354,7 +393,9 @@ int ObSRDaaTIterImpl::collect_dims_by_id(const ObDatum *&id_datum, double &relev } if (OB_FAIL(merge_heap_->top(top_item))) { LOG_WARN("failed to get top item from merge heap", K(ret)); - } else if (OB_FAIL(relevance_collector_->collect_one_dim(top_item->iter_idx_, top_item->relevance_))) { + } else if (iter_param_->need_calc_relevance() + && OB_FAIL(relevance_collector_->collect_one_dim( + top_item->iter_idx_, top_item->relevance_))) { LOG_WARN("failed to collect one dimension", K(ret)); } else if (FALSE_IT(iter_idx = top_item->iter_idx_)) { } else if (OB_FAIL(merge_heap_->pop())) { @@ -366,11 +407,13 @@ int ObSRDaaTIterImpl::collect_dims_by_id(const ObDatum *&id_datum, double &relev if (OB_SUCC(ret)) { id_datum = iter_domain_ids_[iter_idx]; - LOG_DEBUG("collect one dim", KPC(id_datum)); if (OB_ISNULL(id_datum)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null id datum", K(ret)); - } else if (OB_FAIL(relevance_collector_->get_result(relevance, got_valid_id))) { + } else if (!iter_param_->need_calc_relevance() + && FALSE_IT(got_valid_id = true)) { + } else if (iter_param_->need_calc_relevance() + && OB_FAIL(relevance_collector_->get_result(relevance, got_valid_id))) { LOG_WARN("failed to get result", K(ret)); } else if (got_valid_id && OB_FAIL(process_collected_row(*id_datum, relevance))) { LOG_WARN("failed to process collected row", K(ret)); @@ -435,12 +478,16 @@ int ObSRDaaTIterImpl::cache_result(int64_t &count, const ObDatum &id_datum, cons if (limit_param->is_valid() && input_row_cnt_ <= offset) { // TODO: Maybe we should not process offset logic here // don't need to project - } else if (OB_UNLIKELY(count >= buffered_domain_ids_.count() || count >= buffered_relevances_.count())) { + } else if (OB_UNLIKELY(count >= buffered_domain_ids_.count() + || (iter_param_->need_project_relevance() + && count >= buffered_relevances_.count()))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("invalid buffered idx", K(ret), K(count), K(buffered_domain_ids_.count()), K(buffered_relevances_.count())); } else { buffered_domain_ids_[count].from_datum(id_datum); - buffered_relevances_[count] = relevance; + if (iter_param_->need_project_relevance()) { + buffered_relevances_[count] = relevance; + } ++count; ++output_row_cnt_; if (limit_param->is_valid() && output_row_cnt_ >= limit) { @@ -485,9 +532,11 @@ int ObSRDaaTIterImpl::project_results(const int64_t count) ObDatum &id_proj_datum = id_proj_expr->locate_datum_for_write(*eval_ctx); set_datum_func_(id_proj_datum, buffered_domain_ids_[0]); id_proj_expr->set_evaluated_projected(*eval_ctx); - ObDatum &relevance_proj_datum = relevance_proj_expr->locate_datum_for_write(*eval_ctx); - relevance_proj_datum.set_double(buffered_relevances_[0]); - relevance_proj_expr->set_evaluated_projected(*eval_ctx); + if (iter_param_->need_project_relevance()) { + ObDatum &relevance_proj_datum = relevance_proj_expr->locate_datum_for_write(*eval_ctx); + relevance_proj_datum.set_double(buffered_relevances_[0]); + relevance_proj_expr->set_evaluated_projected(*eval_ctx); + } } return ret; } diff --git a/src/storage/retrieval/ob_sparse_daat_iter.h b/src/storage/retrieval/ob_sparse_daat_iter.h index 256226262a..8f61922ef6 100644 --- a/src/storage/retrieval/ob_sparse_daat_iter.h +++ b/src/storage/retrieval/ob_sparse_daat_iter.h @@ -40,6 +40,13 @@ struct ObSRMergeItem struct ObSRMergeCmp { + enum class FastCmpMode : uint8_t + { + GENERIC, + UINT64, + BINARY_STRING + }; + ObSRMergeCmp(); virtual ~ObSRMergeCmp() {} @@ -54,6 +61,7 @@ struct ObSRMergeCmp } private: common::ObDatumCmpFuncType cmp_func_; + FastCmpMode fast_cmp_mode_; // TODO: if memory lifetime of docid datum is guaranteed by dim_iters, we can use pointer to datum directly // and avoid deep copy into merge heap here const ObFixedArray *iter_ids_; @@ -76,7 +84,7 @@ class ObSRDaaTIterImpl : public ObISparseRetrievalMergeIter ObSparseRetrievalMergeParam &iter_param, ObIArray &dim_iters, ObIAllocator &iter_allocator, - ObSRDaaTRelevanceCollector &relevance_collector); + ObSRDaaTRelevanceCollector *relevance_collector); virtual void reuse(const bool switch_tablet = false) override; virtual void reset() override; virtual int get_query_max_score(double &score) override; @@ -97,6 +105,7 @@ class ObSRDaaTIterImpl : public ObISparseRetrievalMergeIter ObIAllocator *iter_allocator_; ObSparseRetrievalMergeParam *iter_param_; ObIArray *dim_iters_; + ObISRDaaTDimIter **dim_iter_cache_; ObSRMergeCmp merge_cmp_; ObSRMergeHeap *merge_heap_; ObSRDaaTRelevanceCollector *relevance_collector_; diff --git a/src/storage/retrieval/ob_spiv_daat_iter.cpp b/src/storage/retrieval/ob_spiv_daat_iter.cpp index ebe4807e8a..8a5242d290 100644 --- a/src/storage/retrieval/ob_spiv_daat_iter.cpp +++ b/src/storage/retrieval/ob_spiv_daat_iter.cpp @@ -137,7 +137,7 @@ int ObSPIVDaaTNaiveIter::inner_init(const ObSPIVDaaTParam ¶m) if (OB_ISNULL(param.base_param_) || OB_ISNULL(param.dim_iters_) || OB_ISNULL(param.allocator_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("unexpected null base param", K(ret)); - } else if (OB_FAIL(ObSRDaaTIterImpl::init(*param.base_param_, *param.dim_iters_, *param.allocator_, *param.relevance_collector_))) { + } else if (OB_FAIL(ObSRDaaTIterImpl::init(*param.base_param_, *param.dim_iters_, *param.allocator_, param.relevance_collector_))) { LOG_WARN("failed to init spiv daat iter", K(ret)); } else { is_pre_filter_ = param.is_pre_filter_; diff --git a/src/storage/retrieval/ob_text_daat_iter.cpp b/src/storage/retrieval/ob_text_daat_iter.cpp index f029dba0f2..816ca62982 100644 --- a/src/storage/retrieval/ob_text_daat_iter.cpp +++ b/src/storage/retrieval/ob_text_daat_iter.cpp @@ -26,14 +26,16 @@ int ObTextDaaTIter::init(const ObTextDaaTParam ¶m) { int ret = OB_SUCCESS; if (OB_ISNULL(param.base_param_) || OB_ISNULL(param.dim_iters_) || OB_ISNULL(param.allocator_) - || OB_ISNULL(param.relevance_collector_)) { + || (param.base_param_->need_calc_relevance() && OB_ISNULL(param.relevance_collector_))) { ret = OB_INVALID_ARGUMENT; LOG_WARN("unexpected null pointer in param", K(ret), KP_(param.base_param), KP_(param.dim_iters), KP_(param.allocator), KP_(param.relevance_collector)); } else if (OB_FAIL(ObSRDaaTIterImpl::init(*param.base_param_, *param.dim_iters_, - *param.allocator_, *param.relevance_collector_))) { + *param.allocator_, param.relevance_collector_))) { LOG_WARN("failed to init sr daat iter", K(ret)); - } else if (OB_FAIL(bm25_param_estimator_.init(param.bm25_param_est_ctx_))) { + } else if (param.base_param_->need_calc_relevance() + && !param.dim_iters_->empty() + && OB_FAIL(bm25_param_estimator_.init(param.bm25_param_est_ctx_))) { LOG_WARN("failed to init bm25 param estimator", K(ret)); } else { mode_flag_ = param.mode_flag_; @@ -74,7 +76,7 @@ int ObTextDaaTIter::pre_process() int ret = OB_SUCCESS; if (dim_iters_->count() == 0) { ret = OB_ITER_END; - } else if (iter_param_->need_project_relevance()) { + } else if (iter_param_->need_calc_relevance()) { if (OB_FAIL(bm25_param_estimator_.do_estimation(*iter_param_->eval_ctx_))) { LOG_WARN("failed to do bm25 param estimation", K(ret)); } diff --git a/src/storage/retrieval/ob_text_retrieval_token_iter.cpp b/src/storage/retrieval/ob_text_retrieval_token_iter.cpp index dc0bd35784..50b78ca0f5 100644 --- a/src/storage/retrieval/ob_text_retrieval_token_iter.cpp +++ b/src/storage/retrieval/ob_text_retrieval_token_iter.cpp @@ -111,7 +111,8 @@ void ObTextRetrievalTokenIter::reset() void ObTextRetrievalTokenIter::reuse() { - if (inv_idx_agg_cache_mode_ && !inv_idx_agg_param_->need_switch_param_) { + if (inv_idx_agg_cache_mode_ && OB_NOT_NULL(inv_idx_agg_param_) + && !inv_idx_agg_param_->need_switch_param_) { // do nothing } else { token_doc_cnt_calculated_ = false; @@ -287,7 +288,6 @@ int ObTextRetrievalTokenIter::do_token_cnt_agg(const ObDocIdExt &doc_id) } else { token_count = fwd_idx_agg_expr_->locate_expr_datum(*eval_ctx_).get_int(); } - LOG_DEBUG("retrieval iterator get token cnt for doc", K(ret), K(doc_id), K(token_count)); } return ret; } @@ -370,7 +370,9 @@ int ObTextRetrievalTokenIter::get_next_row() if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("retrieval token iterator not inited", K(ret)); - } else if (!token_doc_cnt_calculated_ && OB_FAIL(estimate_token_doc_cnt())) { + } else if (need_calc_relevance() + && !token_doc_cnt_calculated_ + && OB_FAIL(estimate_token_doc_cnt())) { LOG_WARN("failed to estimate token doc cnt", K(ret)); } else if (OB_FAIL(inv_idx_scan_iter_->get_next_row())) { if (OB_UNLIKELY(OB_ITER_END != ret)) { @@ -455,7 +457,9 @@ int ObTextRetrievalTokenIter::get_next_batch(const int64_t capacity, int64_t &co if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("retrieval token iterator not inited", K(ret)); - } else if (!token_doc_cnt_calculated_ && OB_FAIL(estimate_token_doc_cnt())) { + } else if (need_calc_relevance() + && !token_doc_cnt_calculated_ + && OB_FAIL(estimate_token_doc_cnt())) { LOG_WARN("failed to estimate token doc cnt", K(ret)); } else if (OB_FAIL(inv_idx_scan_iter_->get_next_rows(count, OB_MIN(max_batch_size_, capacity)))) { if (OB_UNLIKELY(OB_ITER_END != ret)) { @@ -717,10 +721,13 @@ int ObTextRetrievalDaaTTokenIter::init(const ObTextRetrievalScanIterParam &iter_ if (OB_ISNULL(cmp_func_)) { ret = OB_ERR_UNEXPECTED; LOG_WARN("failed to init IRIterLoserTreeCmp", K(ret)); - } else if (FALSE_IT(relevance_.set_allocator(allocator_))) { - } else if (OB_FAIL(relevance_.init(max_batch_size_))) { + } else if (OB_NOT_NULL(relevance_expr_) + && FALSE_IT(relevance_.set_allocator(allocator_))) { + } else if (OB_NOT_NULL(relevance_expr_) + && OB_FAIL(relevance_.init(max_batch_size_))) { LOG_WARN("failed to init next batch iter idxes array", K(ret)); - } else if (OB_FAIL(relevance_.prepare_allocate(max_batch_size_))) { + } else if (OB_NOT_NULL(relevance_expr_) + && OB_FAIL(relevance_.prepare_allocate(max_batch_size_))) { LOG_WARN("failed to prepare allocate next batch iter idxes array", K(ret)); } else if (FALSE_IT(doc_id_.set_allocator(allocator_))) { } else if (OB_FAIL(doc_id_.init(max_batch_size_))) { @@ -817,10 +824,8 @@ int ObTextRetrievalDaaTTokenIter::save_docids() cur_idx_ = 0; const ObDatumVector &doc_id_datum = inv_scan_domain_id_col_->locate_expr_datumvector(*eval_ctx_); for (int64_t i = 0; OB_SUCC(ret) && i < count_; ++i) { - if (OB_LIKELY(!token_iter_->get_skip()->at(i))) { - if (OB_FAIL(doc_id_[i].from_datum(*doc_id_datum.at(i)))) { - LOG_WARN("failed to get doc id", K(ret)); - }; + if (OB_FAIL(doc_id_[i].from_datum(*doc_id_datum.at(i)))) { + LOG_WARN("failed to get doc id", K(ret)); } } } @@ -1013,7 +1018,6 @@ int ObTextRetrievalBlockMaxIter::advance_to(const ObDatum &id_datum) int ObTextRetrievalBlockMaxIter::advance_shallow(const ObDatum &id_datum, const bool inclusive) { int ret = OB_SUCCESS; - LOG_DEBUG("[Sparse Retrieval] advance shallow", K(ret), K(id_datum), K(inclusive)); if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("not initialized", K(ret)); @@ -1125,8 +1129,6 @@ int ObTextRetrievalBlockMaxIter::calc_dim_max_score( LOG_WARN("unexpected nullptr to max score tuple", K(ret), KP_(max_score_tuple)); } else { dim_max_score_ = std::max(dim_max_score_, max_score_tuple->max_score_); - LOG_DEBUG("[Text Retrieval] calc dim max score", K(ret), K(dim_max_score_), K(max_score_tuple->max_score_), - KPC(max_score_tuple->max_domain_id_), KPC(max_score_tuple->min_domain_id_)); } }