Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions fts/src/catalog/fts_index_catalog_entry.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,12 +72,27 @@ std::string FTSIndexAuxInfo::toCypher(const catalog::IndexCatalogEntry& indexEnt
i == propertyIDs.size() - 1 ? "" : ", ");
}

auto stopWords = stopWordsIsTable(config, indexToCypherInfo.context) ?
config.stopWordsSource :
getStopWordsName(indexToCypherInfo.exportFileInfo);
cypher += std::format("CALL CREATE_FTS_INDEX('{}', '{}', [{}], stemmer := '{}', "
"stopWords := '{}');",
tableName, indexEntry.getIndexName(), std::move(propertyStr), config.stemmer,
stopWordsIsTable(config, indexToCypherInfo.context) ?
config.stopWordsSource :
getStopWordsName(indexToCypherInfo.exportFileInfo));
"stopWords := '{}'",
tableName, indexEntry.getIndexName(), std::move(propertyStr),
FTSUtils::escapeCypherString(config.stemmer), FTSUtils::escapeCypherString(stopWords));
// Options left at their defaults are omitted, so exports of default indexes do not change.
if (config.ignorePattern != IgnorePattern::DEFAULT_VALUE) {
cypher += std::format(", ignore_pattern := '{}'",
FTSUtils::escapeCypherString(config.ignorePattern));
}
if (config.tokenizer != Tokenizer::DEFAULT_VALUE) {
cypher +=
std::format(", tokenizer := '{}'", FTSUtils::escapeCypherString(config.tokenizer));
}
if (config.jiebaDictDir != TokenizerInfo{}.jiebaDictDir) {
cypher += std::format(", jieba_dict_dir := '{}'",
FTSUtils::escapeCypherString(config.jiebaDictDir));
}
cypher += ");";
return cypher;
}

Expand Down
22 changes: 6 additions & 16 deletions fts/src/function/create_fts_index.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -125,25 +125,15 @@ static std::string createStopWordsTable([[maybe_unused]] const ClientContext& co
case StopWordsSource::FILE: {
query +=
std::format("CREATE NODE TABLE `{}` (sw STRING, PRIMARY KEY(sw));", info.tableName);
query += std::format("COPY `{}` FROM '{}';", info.tableName, info.stopWords);
query += std::format("COPY `{}` FROM '{}';", info.tableName,
FTSUtils::escapeCypherString(info.stopWords));
} break;
default:
UNREACHABLE_CODE;
}
return query;
}

static std::string formatStrInCypher(const std::string& input) {
std::string result;
for (char c : input) {
if (c == '\\' || c == '\'') {
result += '\\';
}
result += c;
}
return result;
}

std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& bindData) {
auto ftsBindData = bindData.constPtrCast<CreateFTSBindData>();
auto tableID = ftsBindData->tableID;
Expand All @@ -161,7 +151,7 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData&
query += std::format(R"(CREATE MACRO `{}`(query) AS
TOKENIZE(lower(regexp_replace(CAST(query as STRING), '{}', ' ', 'g')), '{}', '{}');)",
FTSUtils::getTokenizeMacroName(tableID, indexName),
formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern),
FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.ignorePattern),
ftsBindData->createFTSConfig.tokenizerInfo.tokenizer,
ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir);
}
Expand Down Expand Up @@ -236,13 +226,13 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData&
std::string params;
params += std::format("stemmer := '{}', ", ftsBindData->createFTSConfig.stemmer);
params += std::format("stopWords := '{}', ",
ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords);
FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords));
params += std::format("ignore_pattern := '{}', ",
formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern));
FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.ignorePattern));
params +=
std::format("tokenizer := '{}', ", ftsBindData->createFTSConfig.tokenizerInfo.tokenizer);
params += std::format("jieba_dict_dir := '{}'",
formatStrInCypher(ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir));
FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir));
query += std::format("CALL _CREATE_FTS_INDEX('{}', '{}', {}, {});", tableName, indexName,
properties, params);
query += std::format("RETURN 'Index {} has been created.' as result;", ftsBindData->indexName);
Expand Down
3 changes: 3 additions & 0 deletions fts/src/include/utils/fts_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ struct FTSUtils {

static bool hasWildcardPattern(const std::string& term);

// Escapes backslashes and single quotes for a single-quoted Cypher string literal.
static std::string escapeCypherString(const std::string& str);

static std::vector<std::string> stemTerms(std::vector<std::string> terms,
const FTSConfig& config, storage::MemoryManager* mm, storage::NodeTable* stopwordsTable,
transaction::Transaction* tx, bool isConjunctive, bool isQuery);
Expand Down
11 changes: 11 additions & 0 deletions fts/src/utils/fts_utils.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,17 @@ StopWordsChecker::StopWordsChecker(MemoryManager* mm, NodeTable* stopwordsTable,
}
}

std::string FTSUtils::escapeCypherString(const std::string& str) {
std::string result;
for (char c : str) {
if (c == '\\' || c == '\'') {
result += '\\';
}
result += c;
}
return result;
}

bool FTSUtils::hasWildcardPattern(const std::string& term) {
for (auto& c : term) {
if (c == '*' || c == '?') {
Expand Down
52 changes: 52 additions & 0 deletions fts/test/test_files/export_options.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
-DATASET CSV empty-db

--

# EXPORT DATABASE keeps ignore_pattern and tokenizer, so the imported index normalizes queries and
# new rows the way the original did.
-CASE ExportKeepsOptions
-SKIP_IN_MEM
-LOAD_DYNAMIC_EXTENSION fts
-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING);
---- ok
-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'}), (:Doc {id: 2, body: '我来到北京清华大学'});
---- ok
-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'digits', ['body'], ignore_pattern := '[!]+', stemmer := 'porter');
---- ok
-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'zh', ['body'], tokenizer := 'jieba');
---- ok
-STATEMENT EXPORT DATABASE '${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options'
---- ok
-IMPORT_DATABASE "${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options"
-STATEMENT IMPORT DATABASE '${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options';
---- ok
-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'digits', '12345') RETURN node.id;
---- 1
1
-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'zh', '北京清华') RETURN node.id;
---- 1
2
-STATEMENT CREATE (:Doc {id: 3, body: '清华大学很好'});
---- ok
-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'zh', '清华') RETURN node.id ORDER BY node.id;
---- 2
2
3

# A stopwords file path is quoted correctly, even when it contains a quote.
-CASE StopWordsPathWithQuote
-SKIP_IN_MEM
-LOAD_DYNAMIC_EXTENSION fts
-STATEMENT COPY (RETURN 'order') TO '${LBUG_EXPORT_DB_DIRECTORY}_it\'s_stopwords.csv' (header=false);
---- ok
-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING);
---- ok
-STATEMENT CREATE (:Doc {id: 1, body: 'order shipped'});
---- ok
-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := '${LBUG_EXPORT_DB_DIRECTORY}_it\'s_stopwords.csv');
---- ok
-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'order') RETURN node.id;
---- 0
-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'shipped') RETURN node.id;
---- 1
1