From 2985e38374ee2fbae6a4c0e0e4538838275e5af1 Mon Sep 17 00:00:00 2001 From: Darek Date: Wed, 14 Oct 2026 06:33:00 +0200 Subject: [PATCH] fix(fts): export ignore_pattern, tokenizer and jieba_dict_dir EXPORT DATABASE wrote each FTS index to index.cypher with only stemmer and stopWords, so IMPORT DATABASE created an index with the default ignore_pattern and tokenizer: queries and rows inserted after the import were normalized differently from the rows built from the exported macro. toCypher now writes the options that differ from their defaults, so exports of default indexes do not change, and escapes every option value. The escaping helper moves to FTSUtils and now also covers the stopwords path in CREATE_FTS_INDEX, which broke on a path containing a quote. Co-Authored-By: Claude Opus 5.5 --- fts/src/catalog/fts_index_catalog_entry.cpp | 25 ++++++++-- fts/src/function/create_fts_index.cpp | 22 +++------ fts/src/include/utils/fts_utils.h | 3 ++ fts/src/utils/fts_utils.cpp | 11 +++++ fts/test/test_files/export_options.test | 52 +++++++++++++++++++++ 5 files changed, 92 insertions(+), 21 deletions(-) create mode 100644 fts/test/test_files/export_options.test diff --git a/fts/src/catalog/fts_index_catalog_entry.cpp b/fts/src/catalog/fts_index_catalog_entry.cpp index 5ed5e360..3c1c4340 100644 --- a/fts/src/catalog/fts_index_catalog_entry.cpp +++ b/fts/src/catalog/fts_index_catalog_entry.cpp @@ -72,12 +72,27 @@ std::string FTSIndexAuxInfo::toCypher(const catalog::IndexCatalogEntry& indexEnt i == propertyIDs.size() - 1 ? "" : ", "); } + auto stopWords = stopWordsIsTable(config, indexToCypherInfo.context) ? + config.stopWordsSource : + getStopWordsName(indexToCypherInfo.exportFileInfo); cypher += std::format("CALL CREATE_FTS_INDEX('{}', '{}', [{}], stemmer := '{}', " - "stopWords := '{}');", - tableName, indexEntry.getIndexName(), std::move(propertyStr), config.stemmer, - stopWordsIsTable(config, indexToCypherInfo.context) ? - config.stopWordsSource : - getStopWordsName(indexToCypherInfo.exportFileInfo)); + "stopWords := '{}'", + tableName, indexEntry.getIndexName(), std::move(propertyStr), + FTSUtils::escapeCypherString(config.stemmer), FTSUtils::escapeCypherString(stopWords)); + // Options left at their defaults are omitted, so exports of default indexes do not change. + if (config.ignorePattern != IgnorePattern::DEFAULT_VALUE) { + cypher += std::format(", ignore_pattern := '{}'", + FTSUtils::escapeCypherString(config.ignorePattern)); + } + if (config.tokenizer != Tokenizer::DEFAULT_VALUE) { + cypher += + std::format(", tokenizer := '{}'", FTSUtils::escapeCypherString(config.tokenizer)); + } + if (config.jiebaDictDir != TokenizerInfo{}.jiebaDictDir) { + cypher += std::format(", jieba_dict_dir := '{}'", + FTSUtils::escapeCypherString(config.jiebaDictDir)); + } + cypher += ");"; return cypher; } diff --git a/fts/src/function/create_fts_index.cpp b/fts/src/function/create_fts_index.cpp index e3a1cc91..b5c9962c 100644 --- a/fts/src/function/create_fts_index.cpp +++ b/fts/src/function/create_fts_index.cpp @@ -125,7 +125,8 @@ static std::string createStopWordsTable([[maybe_unused]] const ClientContext& co case StopWordsSource::FILE: { query += std::format("CREATE NODE TABLE `{}` (sw STRING, PRIMARY KEY(sw));", info.tableName); - query += std::format("COPY `{}` FROM '{}';", info.tableName, info.stopWords); + query += std::format("COPY `{}` FROM '{}';", info.tableName, + FTSUtils::escapeCypherString(info.stopWords)); } break; default: UNREACHABLE_CODE; @@ -133,17 +134,6 @@ static std::string createStopWordsTable([[maybe_unused]] const ClientContext& co return query; } -static std::string formatStrInCypher(const std::string& input) { - std::string result; - for (char c : input) { - if (c == '\\' || c == '\'') { - result += '\\'; - } - result += c; - } - return result; -} - std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& bindData) { auto ftsBindData = bindData.constPtrCast(); auto tableID = ftsBindData->tableID; @@ -161,7 +151,7 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& query += std::format(R"(CREATE MACRO `{}`(query) AS TOKENIZE(lower(regexp_replace(CAST(query as STRING), '{}', ' ', 'g')), '{}', '{}');)", FTSUtils::getTokenizeMacroName(tableID, indexName), - formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern), + FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.ignorePattern), ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir); } @@ -236,13 +226,13 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& std::string params; params += std::format("stemmer := '{}', ", ftsBindData->createFTSConfig.stemmer); params += std::format("stopWords := '{}', ", - ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords); + FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.stopWordsTableInfo.stopWords)); params += std::format("ignore_pattern := '{}', ", - formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern)); + FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.ignorePattern)); params += std::format("tokenizer := '{}', ", ftsBindData->createFTSConfig.tokenizerInfo.tokenizer); params += std::format("jieba_dict_dir := '{}'", - formatStrInCypher(ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir)); + FTSUtils::escapeCypherString(ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir)); query += std::format("CALL _CREATE_FTS_INDEX('{}', '{}', {}, {});", tableName, indexName, properties, params); query += std::format("RETURN 'Index {} has been created.' as result;", ftsBindData->indexName); diff --git a/fts/src/include/utils/fts_utils.h b/fts/src/include/utils/fts_utils.h index bacb88ff..ec96cf61 100644 --- a/fts/src/include/utils/fts_utils.h +++ b/fts/src/include/utils/fts_utils.h @@ -26,6 +26,9 @@ struct FTSUtils { static bool hasWildcardPattern(const std::string& term); + // Escapes backslashes and single quotes for a single-quoted Cypher string literal. + static std::string escapeCypherString(const std::string& str); + static std::vector stemTerms(std::vector terms, const FTSConfig& config, storage::MemoryManager* mm, storage::NodeTable* stopwordsTable, transaction::Transaction* tx, bool isConjunctive, bool isQuery); diff --git a/fts/src/utils/fts_utils.cpp b/fts/src/utils/fts_utils.cpp index 93e6e5ab..ad6027fb 100644 --- a/fts/src/utils/fts_utils.cpp +++ b/fts/src/utils/fts_utils.cpp @@ -83,6 +83,17 @@ StopWordsChecker::StopWordsChecker(MemoryManager* mm, NodeTable* stopwordsTable, } } +std::string FTSUtils::escapeCypherString(const std::string& str) { + std::string result; + for (char c : str) { + if (c == '\\' || c == '\'') { + result += '\\'; + } + result += c; + } + return result; +} + bool FTSUtils::hasWildcardPattern(const std::string& term) { for (auto& c : term) { if (c == '*' || c == '?') { diff --git a/fts/test/test_files/export_options.test b/fts/test/test_files/export_options.test new file mode 100644 index 00000000..2ec30d4d --- /dev/null +++ b/fts/test/test_files/export_options.test @@ -0,0 +1,52 @@ +-DATASET CSV empty-db + +-- + +# EXPORT DATABASE keeps ignore_pattern and tokenizer, so the imported index normalizes queries and +# new rows the way the original did. +-CASE ExportKeepsOptions +-SKIP_IN_MEM +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'}), (:Doc {id: 2, body: '我来到北京清华大学'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'digits', ['body'], ignore_pattern := '[!]+', stemmer := 'porter'); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'zh', ['body'], tokenizer := 'jieba'); +---- ok +-STATEMENT EXPORT DATABASE '${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options' +---- ok +-IMPORT_DATABASE "${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options" +-STATEMENT IMPORT DATABASE '${LBUG_EXPORT_DB_DIRECTORY}_fts/export_options'; +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'digits', '12345') RETURN node.id; +---- 1 +1 +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'zh', '北京清华') RETURN node.id; +---- 1 +2 +-STATEMENT CREATE (:Doc {id: 3, body: '清华大学很好'}); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'zh', '清华') RETURN node.id ORDER BY node.id; +---- 2 +2 +3 + +# A stopwords file path is quoted correctly, even when it contains a quote. +-CASE StopWordsPathWithQuote +-SKIP_IN_MEM +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT COPY (RETURN 'order') TO '${LBUG_EXPORT_DB_DIRECTORY}_it\'s_stopwords.csv' (header=false); +---- ok +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'order shipped'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := '${LBUG_EXPORT_DB_DIRECTORY}_it\'s_stopwords.csv'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'order') RETURN node.id; +---- 0 +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'shipped') RETURN node.id; +---- 1 +1