Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion src/plugin/interface/ob_plugin_ftparser_intf.h
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ class ObFTParserParam final : public ObFTParserParamExport
public:
ObFTParserParam()
: ObFTParserParamExport(),
metadata_alloc_(nullptr),
scratch_alloc_(nullptr),
ngram_token_size_(NGRAM_TOKEN_SIZE),
min_ngram_size_(NGRAM_TOKEN_SIZE),
max_ngram_size_(NGRAM_TOKEN_SIZE)
Expand All @@ -112,13 +114,18 @@ class ObFTParserParam final : public ObFTParserParamExport
{
ObFTParserParamExport::reset();
allocator_ = nullptr;
metadata_alloc_ = nullptr;
scratch_alloc_ = nullptr;
ngram_token_size_ = NGRAM_TOKEN_SIZE;
}

INHERIT_TO_STRING_KV("base", ObFTParserParamExport, KP_(allocator), K_(ngram_token_size));
INHERIT_TO_STRING_KV("base", ObFTParserParamExport, KP_(allocator),
KP_(metadata_alloc), KP_(scratch_alloc), K_(ngram_token_size));

public:
common::ObIAllocator *allocator_ = nullptr;
common::ObIAllocator *metadata_alloc_;
common::ObIAllocator *scratch_alloc_;

// ik parser params
ObFTIKParam ik_param_;
Expand Down
9 changes: 8 additions & 1 deletion src/share/text_analysis/ob_token_stream.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,14 @@ int ObBasicEnglishNormalizer::get_next(ObDatum &next_token, int64_t &token_freq)
} else {
ObString norm_alnum_token(norm_token_len, norm_token_ptr);
ObString norm_lower_token;
if (OB_FAIL(ObCharset::tolower(cs_, norm_alnum_token, norm_lower_token, norm_allocator_))) {
bool is_lower_ascii = true;
for (int64_t i = 0; is_lower_ascii && i < norm_alnum_token.length(); ++i) {
const unsigned char ch = static_cast<unsigned char>(norm_alnum_token.ptr()[i]);
is_lower_ascii = ch < 0x80 && !(ch >= 'A' && ch <= 'Z');
}
if (is_lower_ascii) {
next_token.set_string(norm_alnum_token);
} else if (OB_FAIL(ObCharset::tolower(cs_, norm_alnum_token, norm_lower_token, norm_allocator_))) {
LOG_WARN("norm token to lower case failed", K(ret), K_(cs), K(norm_alnum_token));
} else {
next_token.set_string(norm_lower_token);
Expand Down
9 changes: 6 additions & 3 deletions src/sql/das/iter/ob_das_iter_utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1661,15 +1661,19 @@ int ObDASIterUtils::create_text_retrieval_sub_tree(
if (ir_scan_ctdef->has_pushdown_topk() && !has_duplicate_boolean_tokens) {
merge_iter_param.topk_mode_ = 1;
merge_iter_param.daat_mode_ = 1;
} else if (merge_iter_param.query_tokens_.count() > OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT
|| (!is_func_lookup && !ir_scan_ctdef->need_proj_relevance_score())) {
} else if (merge_iter_param.query_tokens_.count() > OB_MAX_TEXT_RETRIEVAL_TOKEN_CNT) {
if (BOOLEAN_MODE == ir_scan_ctdef->mode_flag_) {
ret = OB_NOT_SUPPORTED;
LOG_WARN("boolean mode with too many tokens not supported", K(ret), K(merge_iter_param.query_tokens_.count()));
LOG_USER_ERROR(OB_NOT_SUPPORTED, "Boolean mode with more than 256 tokens is");
}
merge_iter_param.taat_mode_ = 1;
} else if (!is_func_lookup
&& !ir_scan_ctdef->need_proj_relevance_score()
&& BOOLEAN_MODE == ir_scan_ctdef->mode_flag_) {
merge_iter_param.taat_mode_ = 1;
} else {
// Predicate-only natural-language MATCH can stream sorted posting lists.
merge_iter_param.daat_mode_ = 1;
}
if (OB_SUCC(ret) && ir_scan_ctdef->has_pushdown_topk() && has_duplicate_boolean_tokens) {
Expand Down Expand Up @@ -4426,4 +4430,3 @@ int ObDASIterUtils::create_vec_ivf_lookup_tree(ObTableScanParam &scan_param,

} // namespace sql
} // namespace oceanbase

55 changes: 38 additions & 17 deletions src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,17 @@ ObDASTRMergeIter::ObDASTRMergeIter()
{
}

bool ObDASTRMergeIter::skip_relevance_for_predicate() const
{
return OB_NOT_NULL(ir_ctdef_)
&& OB_NOT_NULL(ir_rtdef_)
&& !topk_mode_
&& NATURAL_LANGUAGE_MODE == ir_ctdef_->mode_flag_
&& ir_rtdef_->minimum_should_match_ <= 1
&& OB_ISNULL(ir_ctdef_->relevance_proj_col_)
&& OB_ISNULL(ir_ctdef_->match_filter_);
}

int ObDASTRMergeIter::inner_init(ObDASIterParam &param)
{
int ret = OB_SUCCESS;
Expand Down Expand Up @@ -116,7 +127,8 @@ int ObDASTRMergeIter::init_das_iter_scan_params()
{
int ret = OB_SUCCESS;
const int64_t dim_iter_cnt = taat_mode_ ? 1 : query_tokens_.count();
if (!ir_ctdef_->need_estimate_total_doc_cnt() && 0 != dim_iter_cnt) {
const bool need_relevance = !skip_relevance_for_predicate();
if (need_relevance && !ir_ctdef_->need_estimate_total_doc_cnt() && 0 != dim_iter_cnt) {
void *buf = nullptr;
if (OB_ISNULL(buf = myself_allocator_.alloc(sizeof(ObTableScanParam)))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
Expand Down Expand Up @@ -145,7 +157,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params()
LOG_WARN("failed to prepare allocate inv scan params array", K(ret));
}

if (OB_FAIL(ret) || !ir_ctdef_->need_inv_idx_agg()) {
if (OB_FAIL(ret) || !need_relevance || !ir_ctdef_->need_inv_idx_agg()) {
} else if (FALSE_IT(inv_agg_params_.set_allocator(&myself_allocator_))) {
} else if (OB_FAIL(inv_agg_params_.init(dim_iter_cnt))) {
LOG_WARN("failed to init inv agg params array", K(ret));
Expand All @@ -154,7 +166,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params()
}

if (OB_FAIL(ret)) {
} else if (!ir_ctdef_->need_fwd_idx_agg()) {
} else if (!need_relevance || !ir_ctdef_->need_fwd_idx_agg()) {
} else if (OB_ISNULL(ir_rtdef_->eval_ctx_) || OB_UNLIKELY(ir_rtdef_->eval_ctx_->max_batch_size_ >= 1)) {
ret = OB_NOT_IMPLEMENT;
LOG_ERROR("fwd idx agg with max batch size not implemented", K(ret));
Expand Down Expand Up @@ -186,7 +198,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params()
}
}

if (OB_SUCC(ret) && ir_ctdef_->need_inv_idx_agg()) {
if (OB_SUCC(ret) && need_relevance && ir_ctdef_->need_inv_idx_agg()) {
void *agg_buf = nullptr;
if (OB_ISNULL(agg_buf = myself_allocator_.alloc(sizeof(ObTableScanParam) * dim_iter_cnt))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
Expand All @@ -210,7 +222,7 @@ int ObDASTRMergeIter::init_das_iter_scan_params()
}
}

if (OB_SUCC(ret) && ir_ctdef_->need_fwd_idx_agg()) {
if (OB_SUCC(ret) && need_relevance && ir_ctdef_->need_fwd_idx_agg()) {
void *fwd_buf = nullptr;
if (OB_ISNULL(fwd_buf = myself_allocator_.alloc(sizeof(ObTableScanParam) * dim_iter_cnt))) {
ret = OB_ALLOCATE_MEMORY_FAILED;
Expand Down Expand Up @@ -432,10 +444,11 @@ int ObDASTRMergeIter::init_dim_iter_param(ObTextRetrievalScanIterParam &iter_par
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null pointer", K(ret), KP_(ir_ctdef), KP_(ir_rtdef));
} else {
const bool skip_relevance = skip_relevance_for_predicate();
const int64_t dim_iter_cnt = taat_mode_ ? 1 : query_tokens_.count();
iter_param.inv_idx_scan_param_ = inv_scan_params_[idx];
iter_param.inv_idx_scan_iter_ = static_cast<ObDASScanIter*>(children_[idx]);
if (ir_ctdef_->need_inv_idx_agg()) {
if (!skip_relevance && ir_ctdef_->need_inv_idx_agg()) {
iter_param.inv_idx_agg_param_ = inv_agg_params_[idx];
iter_param.inv_idx_agg_iter_ = static_cast<ObDASScanIter*>(children_[idx + dim_iter_cnt]);
if (OB_UNLIKELY(1 != ir_ctdef_->get_inv_idx_agg_ctdef()
Expand All @@ -447,7 +460,7 @@ int ObDASTRMergeIter::init_dim_iter_param(ObTextRetrievalScanIterParam &iter_par
->pd_expr_spec_.pd_storage_aggregate_output_.at(0);
}
}
if (OB_SUCC(ret) && ir_ctdef_->need_fwd_idx_agg()) {
if (OB_SUCC(ret) && !skip_relevance && ir_ctdef_->need_fwd_idx_agg()) {
iter_param.fwd_idx_scan_param_ = fwd_scan_params_[idx];
iter_param.fwd_idx_agg_iter_ = static_cast<ObDASScanIter*>(children_[idx + dim_iter_cnt * 2]);
if (OB_UNLIKELY(1 != ir_ctdef_->get_fwd_idx_agg_ctdef()
Expand All @@ -462,7 +475,7 @@ int ObDASTRMergeIter::init_dim_iter_param(ObTextRetrievalScanIterParam &iter_par
iter_param.allocator_ = &myself_allocator_;
iter_param.mem_context_ = mem_context_;
iter_param.eval_ctx_ = ir_rtdef_->eval_ctx_;
iter_param.relevance_expr_ = ir_ctdef_->relevance_expr_;
iter_param.relevance_expr_ = skip_relevance ? nullptr : ir_ctdef_->relevance_expr_;
iter_param.inv_scan_doc_length_col_ = ir_ctdef_->inv_scan_doc_length_col_;
iter_param.inv_scan_domain_id_col_ = ir_ctdef_->inv_scan_domain_id_col_;
iter_param.inv_idx_agg_cache_mode_ = function_lookup_mode_ && !taat_mode_;
Expand All @@ -481,7 +494,8 @@ int ObDASTRMergeIter::create_sparse_retrieval_iter()
sr_iter_param_.relevance_proj_expr_ = ir_ctdef_->relevance_proj_col_;
sr_iter_param_.filter_expr_ = ir_ctdef_->match_filter_;
sr_iter_param_.topk_limit_ = topk_limit_;
if (OB_NOT_NULL(ir_ctdef_->field_boost_expr_)) {
sr_iter_param_.skip_relevance_for_predicate_ = skip_relevance_for_predicate();
if (sr_iter_param_.need_calc_relevance() && OB_NOT_NULL(ir_ctdef_->field_boost_expr_)) {
ObDatum *boost_datum = nullptr;
if (OB_FAIL(ir_ctdef_->field_boost_expr_->eval(*ir_rtdef_->eval_ctx_, boost_datum))) {
LOG_WARN("failed to eval field boost expr", K(ret));
Expand Down Expand Up @@ -592,15 +606,22 @@ int ObDASTRMergeIter::init_daat_iter_param(ObTextDaaTParam &iter_param)
iter_param.allocator_ = &myself_allocator_;
iter_param.mode_flag_ = ir_ctdef_->mode_flag_;
iter_param.function_lookup_mode_ = function_lookup_mode_;
iter_param.bm25_param_est_ctx_.total_doc_cnt_expr_
= ir_ctdef_->get_doc_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0);
iter_param.bm25_param_est_ctx_.estimated_total_doc_cnt_ = ir_ctdef_->estimated_total_doc_cnt_;
iter_param.bm25_param_est_ctx_.need_est_avg_doc_token_cnt_ = ir_ctdef_->need_avg_doc_len_est();
iter_param.bm25_param_est_ctx_.can_est_by_sum_skip_index_ = ir_ctdef_->avg_doc_len_est_spec_.can_est_by_sum_skip_index_;
iter_param.bm25_param_est_ctx_.avg_doc_token_cnt_expr_ = ir_ctdef_->avg_doc_token_cnt_expr_;
iter_param.bm25_param_est_ctx_.doc_length_est_param_ = &doc_length_est_param_;
if (query_tokens_.count() == 0) {
// do nothing
} else if (!sr_iter_param_.need_calc_relevance()) {
// Predicate-only MATCH only needs document ids.
} else if (FALSE_IT(iter_param.bm25_param_est_ctx_.total_doc_cnt_expr_
= ir_ctdef_->get_doc_agg_ctdef()->pd_expr_spec_.pd_storage_aggregate_output_.at(0))) {
} else if (FALSE_IT(iter_param.bm25_param_est_ctx_.estimated_total_doc_cnt_
= ir_ctdef_->estimated_total_doc_cnt_)) {
} else if (FALSE_IT(iter_param.bm25_param_est_ctx_.need_est_avg_doc_token_cnt_
= ir_ctdef_->need_avg_doc_len_est())) {
} else if (FALSE_IT(iter_param.bm25_param_est_ctx_.can_est_by_sum_skip_index_
= ir_ctdef_->avg_doc_len_est_spec_.can_est_by_sum_skip_index_)) {
} else if (FALSE_IT(iter_param.bm25_param_est_ctx_.avg_doc_token_cnt_expr_
= ir_ctdef_->avg_doc_token_cnt_expr_)) {
} else if (FALSE_IT(iter_param.bm25_param_est_ctx_.doc_length_est_param_
= &doc_length_est_param_)) {
} else if (OB_ISNULL(iter_param.bm25_param_est_ctx_.total_doc_cnt_expr_)) {
ret = OB_ERR_UNEXPECTED;
LOG_WARN("unexpected null total doc cnt expr", K(ret));
Expand All @@ -614,7 +635,7 @@ int ObDASTRMergeIter::init_daat_iter_param(ObTextDaaTParam &iter_param)
iter_param.bm25_param_est_ctx_.total_doc_cnt_iter_ = static_cast<sql::ObDASScanIter*>(children_[children_cnt_ - 1]);
}
}
if (OB_SUCC(ret)) {
if (OB_SUCC(ret) && sr_iter_param_.need_calc_relevance()) {
if (BOOLEAN_MODE == ir_ctdef_->mode_flag_) {
ObSRDaaTBooleanRelevanceCollector *boolean_relevance_collector = nullptr;
if (OB_ISNULL(boolean_relevance_collector = OB_NEWx(ObSRDaaTBooleanRelevanceCollector, &myself_allocator_))) {
Expand Down
1 change: 1 addition & 0 deletions src/sql/das/iter/sparse_retrieval/ob_das_tr_merge_iter.h
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,7 @@ class ObDASTRMergeIter : public ObDASIter
int init_topk_limit();
int init_block_max_iter_param();
int init_doc_length_est_param();
bool skip_relevance_for_predicate() const;
private:
static const int64_t FWD_IDX_ROWKEY_COL_CNT = 2;
static const int64_t INV_IDX_ROWKEY_COL_CNT = 2;
Expand Down
79 changes: 59 additions & 20 deletions src/sql/das/ob_das_domain_utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,12 @@ ObExprOperatorType ObFTIndexRowCache::FTS_INDEX_EXPR_TYPE[] = {T_FUN_SYS_WORD_SE
ObExprOperatorType ObFTIndexRowCache::FTS_DOC_WORD_EXPR_TYPE[] = {T_FUN_SYS_DOC_ID, T_FUN_SYS_WORD_SEGMENT, T_FUN_SYS_WORD_COUNT, T_FUN_SYS_DOC_LENGTH};

ObFTIndexRowCache::ObFTIndexRowCache()
: rows_(),
: merge_memctx_(nullptr),
row_allocator_(common::ObMemAttr("FTIndexRow")),
datum_row_(),
word_map_(),
words_(),
counts_(),
row_idx_(0),
is_fts_index_aux_(true),
helper_(),
Expand All @@ -68,7 +73,13 @@ int ObFTIndexRowCache::init(
LOG_WARN("init fulltext dml iterator twice", K(ret), K(is_inited_));
} else if (OB_FAIL(CURRENT_CONTEXT->CREATE_CONTEXT(merge_memctx_, param))) {
LOG_WARN("failed to create merge memctx", K(ret));
} else if (OB_FAIL(helper_.init(&(merge_memctx_->get_arena_allocator()), parser_name, parser_properties))) {
} else if (OB_FAIL(datum_row_.init(row_allocator_, 4))) {
LOG_WARN("failed to initialize reusable full-text row", K(ret));
} else if (OB_FAIL(word_map_.create(FT_WORD_MAP_BUCKET_COUNT,
common::ObMemAttr("FTWordMap")))) {
LOG_WARN("failed to create reusable full-text word map", K(ret));
} else if (OB_FAIL(helper_.init(&(merge_memctx_->get_arena_allocator()), parser_name, parser_properties,
true /* enable_parser_reuse */))) {
LOG_WARN("fail to init full-text parser helper", K(ret));
} else {
row_idx_ = 0;
Expand All @@ -86,22 +97,38 @@ int ObFTIndexRowCache::segment(const common::ObObjMeta &ft_obj_meta,
const ObString &fulltext)
{
int ret = OB_SUCCESS;
int64_t doc_length = 0;
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObFTIndexRowCache hasn't be initialized", K(ret), K(is_inited_));
} else if (FALSE_IT(reuse())) {
} else if (OB_FAIL(ObDASDomainUtils::generate_fulltext_word_rows(merge_memctx_->get_arena_allocator(),
&helper_,
ft_obj_meta,
doc_id_datum,
fulltext,
is_fts_index_aux_,
rows_))) {
LOG_WARN("fail to generate fulltext word rows", K(ret), K(helper_), K(is_fts_index_aux_));
} else if (fulltext.empty()) {
ret = OB_ITER_END;
} else if (OB_FAIL(helper_.segment(ft_obj_meta,
fulltext.ptr(),
fulltext.length(),
doc_length,
word_map_))) {
LOG_WARN("fail to segment fulltext", K(ret), K(helper_), K(is_fts_index_aux_));
} else if (word_map_.size() <= 0) {
ret = OB_ITER_END;
} else {
row_idx_ = 0;
for (ObFTWordMap::const_iterator iter = word_map_.begin();
OB_SUCC(ret) && iter != word_map_.end();
++iter) {
if (OB_FAIL(words_.push_back(iter->first))) {
LOG_WARN("failed to cache full-text word", K(ret));
} else if (OB_FAIL(counts_.push_back(iter->second))) {
LOG_WARN("failed to cache full-text word count", K(ret));
}
}
if (OB_SUCC(ret)) {
const int64_t doc_id_idx = is_fts_index_aux_ ? 1 : 0;
datum_row_.storage_datums_[doc_id_idx].shallow_copy_from_datum(doc_id_datum);
datum_row_.storage_datums_[3].set_uint(doc_length);
}
}
LOG_TRACE("word segment", K(ret), K(row_idx_), K(rows_.count()), K(doc_id_datum), K(fulltext));
LOG_TRACE("word segment", K(ret), K(row_idx_), K(words_.count()), K(doc_id_datum), K(fulltext));
return ret;
}

Expand All @@ -111,19 +138,26 @@ int ObFTIndexRowCache::get_next_row(blocksstable::ObDatumRow *&row)
if (OB_UNLIKELY(!is_inited_)) {
ret = OB_NOT_INIT;
LOG_WARN("ObFTIndexRowCache hasn't be initialized", K(ret), K(is_inited_));
} else if (row_idx_ >= rows_.count()) {
} else if (row_idx_ >= words_.count()) {
ret = OB_ITER_END;
} else {
row = (rows_[row_idx_]);
const int64_t word_idx = is_fts_index_aux_ ? 0 : 1;
datum_row_.storage_datums_[word_idx].set_datum(words_.at(row_idx_).get_word());
datum_row_.storage_datums_[2].set_uint(counts_.at(row_idx_));
row = &datum_row_;
++row_idx_;
}
LOG_TRACE("get next row", K(ret), KPC(row), K(row_idx_), K(rows_.count()));
LOG_TRACE("get next row", K(ret), KPC(row), K(row_idx_), K(words_.count()));
return ret;
}

void ObFTIndexRowCache::reset()
{
rows_.reset();
words_.reset();
counts_.reset();
word_map_.destroy();
datum_row_.reset();
row_allocator_.reset();
row_idx_ = 0;
is_fts_index_aux_ = true;
helper_.reset();
Expand All @@ -136,7 +170,9 @@ void ObFTIndexRowCache::reset()

void ObFTIndexRowCache::reuse()
{
rows_.reuse();
words_.reuse();
counts_.reuse();
word_map_.reuse();
row_idx_ = 0;
if (OB_NOT_NULL(merge_memctx_)) {
merge_memctx_->reset_remain_one_page();
Expand Down Expand Up @@ -947,7 +983,8 @@ int ObFTDMLIterator::rewind()
// This is the same as the parser name of the previous index.
// nothing to do, just skip.
} else if (FALSE_IT(ft_parse_helper_.reset())) {
} else if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_str, parser_property_str))) {
} else if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_str, parser_property_str,
true /* enable_parser_reuse */))) {
LOG_WARN("fail to init fulltext parse helper", K(ret), K(parser_str), K(parser_property_str));
}
} else if (ObDomainDMLMode::DOMAIN_DML_MODE_FT_SCAN == mode_) {
Expand Down Expand Up @@ -981,7 +1018,8 @@ int ObFTDMLIterator::init(
} else {
switch (mode_) {
case ObDomainDMLMode::DOMAIN_DML_MODE_DEFAULT: {
if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_name, parser_properties))) {
if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_name, parser_properties,
true /* enable_parser_reuse */))) {
LOG_WARN("fail to init fulltext parse helper", K(ret), K(parser_name), K(parser_properties));
}
break;
Expand Down Expand Up @@ -1043,7 +1081,8 @@ int ObFTDMLIterator::change_domain_dml_mode(const ObDomainDMLMode &mode)
// This is the same as the parser name of the previous index.
// nothing to do, just skip.
} else if (FALSE_IT(ft_parse_helper_.reset())) {
} else if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_str, parser_property_str))) {
} else if (OB_FAIL(ft_parse_helper_.init(&allocator_, parser_str, parser_property_str,
true /* enable_parser_reuse */))) {
LOG_WARN("fail to init fulltext parse helper", K(ret), K(parser_str), K(parser_property_str));
}
break;
Expand Down
Loading