diff --git a/fts/src/catalog/fts_index_catalog_entry.cpp b/fts/src/catalog/fts_index_catalog_entry.cpp index 5ed5e360..3c1c4340 100644 --- a/fts/src/catalog/fts_index_catalog_entry.cpp +++ b/fts/src/catalog/fts_index_catalog_entry.cpp @@ -72,12 +72,27 @@ std::string FTSIndexAuxInfo::toCypher(const catalog::IndexCatalogEntry& indexEnt i == propertyIDs.size() - 1 ? "" : ", "); } + auto stopWords = stopWordsIsTable(config, indexToCypherInfo.context) ? + config.stopWordsSource : + getStopWordsName(indexToCypherInfo.exportFileInfo); cypher += std::format("CALL CREATE_FTS_INDEX('{}', '{}', [{}], stemmer := '{}', " - "stopWords := '{}');", - tableName, indexEntry.getIndexName(), std::move(propertyStr), config.stemmer, - stopWordsIsTable(config, indexToCypherInfo.context) ? - config.stopWordsSource : - getStopWordsName(indexToCypherInfo.exportFileInfo)); + "stopWords := '{}'", + tableName, indexEntry.getIndexName(), std::move(propertyStr), + FTSUtils::escapeCypherString(config.stemmer), FTSUtils::escapeCypherString(stopWords)); + // Options left at their defaults are omitted, so exports of default indexes do not change. + if (config.ignorePattern != IgnorePattern::DEFAULT_VALUE) { + cypher += std::format(", ignore_pattern := '{}'", + FTSUtils::escapeCypherString(config.ignorePattern)); + } + if (config.tokenizer != Tokenizer::DEFAULT_VALUE) { + cypher += + std::format(", tokenizer := '{}'", FTSUtils::escapeCypherString(config.tokenizer)); + } + if (config.jiebaDictDir != TokenizerInfo{}.jiebaDictDir) { + cypher += std::format(", jieba_dict_dir := '{}'", + FTSUtils::escapeCypherString(config.jiebaDictDir)); + } + cypher += ");"; return cypher; } diff --git a/fts/src/function/create_fts_index.cpp b/fts/src/function/create_fts_index.cpp index e3a1cc91..b5c9962c 100644 --- a/fts/src/function/create_fts_index.cpp +++ b/fts/src/function/create_fts_index.cpp @@ -125,7 +125,8 @@ static std::string createStopWordsTable([[maybe_unused]] const ClientContext& co case StopWordsSource::FILE: { query += std::format("CREATE NODE TABLE `{}` (sw STRING, PRIMARY KEY(sw));", info.tableName); - query += std::format("COPY `{}` FROM '{}';", info.tableName, info.stopWords); + query += std::format("COPY `{}` FROM '{}';", info.tableName, + FTSUtils::escapeCypherString(info.stopWords)); } break; default: UNREACHABLE_CODE; @@ -133,17 +134,6 @@ static std::string createStopWordsTable([[maybe_unused]] const ClientContext& co return query; } -static std::string formatStrInCypher(const std::string& input) { - std::string result; - for (char c : input) { - if (c == '\\' || c == '\'') { - result += '\\'; - } - result += c; - } - return result; -} - std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& bindData) { auto ftsBindData = bindData.constPtrCast(); auto tableID = ftsBindData->tableID; @@ -161,7 +151,7 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& query += std::format(R"(CREATE MACRO `{}`(query) AS TOKENIZE(lower(regexp_replace(CAST(query as STRING), '{}', ' ', 'g')), '{}', '{}');)", FTSUtils::getTokenizeMacroName(tableID, indexName), - formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern), + FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.ignorePattern), ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir); } @@ -236,13 +226,13 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& std::string params; params += std::format("stemmer := '{}', ", ftsBindData->createFTSConfig.stemmer); params += std::format("stopWords := '{}', ", - ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords); + FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords)); params += std::format("ignore_pattern := '{}', ", - formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern)); + FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.ignorePattern)); params += std::format("tokenizer := '{}', ", ftsBindData->createFTSConfig.tokenizerInfo.tokenizer); params += std::format("jieba_dict_dir := '{}'", - formatStrInCypher(ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir)); + FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir)); query += std::format("CALL _CREATE_FTS_INDEX('{}', '{}', {}, {});", tableName, indexName, properties, params); query += std::format("RETURN 'Index {} has been created.' as result;", ftsBindData->indexName); diff --git a/fts/src/include/utils/fts_utils.h b/fts/src/include/utils/fts_utils.h index bacb88ff..ec96cf61 100644 --- a/fts/src/include/utils/fts_utils.h +++ b/fts/src/include/utils/fts_utils.h @@ -26,6 +26,9 @@ struct FTSUtils { static bool hasWildcardPattern(const std::string& term); + // Escapes backslashes and single quotes for a single-quoted Cypher string literal. + static std::string escapeCypherString(const std::string& str); + static std::vector stemTerms(std::vector terms, const FTSConfig& config, storage::MemoryManager* mm, storage::NodeTable* stopwordsTable, transaction::Transaction* tx, bool isConjunctive, bool isQuery); diff --git a/fts/src/utils/fts_utils.cpp b/fts/src/utils/fts_utils.cpp index 93e6e5ab..ad6027fb 100644 --- a/fts/src/utils/fts_utils.cpp +++ b/fts/src/utils/fts_utils.cpp @@ -83,6 +83,17 @@ StopWordsChecker::StopWordsChecker(MemoryManager* mm, NodeTable* stopwordsTable, } } +std::string FTSUtils::escapeCypherString(const std::string& str) { + std::string result; + for (char c : str) { + if (c == '\\' || c == '\'') { + result += '\\'; + } + result += c; + } + return result; +} + bool FTSUtils::hasWildcardPattern(const std::string& term) { for (auto& c : term) { if (c == '*' || c == '?') { diff --git a/fts/test/test_files/export_options.test b/fts/test/test_files/export_options.test new file mode 100644 index 00000000..2ec30d4d --- /dev/null +++ b/fts/test/test_files/export_options.test @@ -0,0 +1,52 @@ +-DATASET CSV empty-db + +-- + +# EXPORT DATABASE keeps ignore_pattern and tokenizer, so the imported index normalizes queries and +# new rows the way the original did. +-CASE ExportKeepsOptions +-SKIP_IN_MEM +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'}), (:Doc {id: 2, body: '我来到北京清华大学'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'digits', ['body'], ignore_pattern := '[!]+', stemmer := 'porter'); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'zh', ['body'], tokenizer := 'jieba'); +---- ok +-STATEMENT EXPORT DATABASE '${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options' +---- ok +-IMPORT_DATABASE "${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options" +-STATEMENT IMPORT DATABASE '${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options'; +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'digits', '12345') RETURN node.id; +---- 1 +1 +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'zh', '北京清华') RETURN node.id; +---- 1 +2 +-STATEMENT CREATE (:Doc {id: 3, body: '清华大学很好'}); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'zh', '清华') RETURN node.id ORDER BY node.id; +---- 2 +2 +3 + +# A stopwords file path is quoted correctly, even when it contains a quote. +-CASE StopWordsPathWithQuote +-SKIP_IN_MEM +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT COPY (RETURN 'order') TO '${LBUG_EXPORT_DB_DIRECTORY}_it\'s_stopwords.csv' (header=false); +---- ok +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'order shipped'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := '${LBUG_EXPORT_DB_DIRECTORY}_it\'s_stopwords.csv'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'order') RETURN node.id; +---- 0 +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'shipped') RETURN node.id; +---- 1 +1