From 4e3e5a9834ca0098067a7ea582bba937f49b8baa Mon Sep 17 00:00:00 2001 From: Darek Date: Wed, 14 Oct 2026 18:25:32 +0200 Subject: [PATCH 1/2] fix(fts): drop the tokenize macro with the index DROP_FTS_INDEX left the index's tokenize macro behind, so an index created again under the same name reused it: rows built by the CREATE were tokenized with the old ignore_pattern or tokenizer while rows inserted later used the new one. DROP MACRO's WAL replay is fixed (LadybugDB/ladybug#1165), so DROP_FTS_INDEX now drops the macro and reports "Index ... has been dropped.", and CREATE_FTS_INDEX replaces a tokenize macro that already exists, from IMPORT DATABASE, an interrupted CREATE, or a drop by an earlier version. The macro is dropped only when it exists, because DROP MACRO IF EXISTS throws for a missing macro. Co-Authored-By: Claude Opus 5.5 --- fts/src/function/create_fts_index.cpp | 21 +++---- fts/src/function/drop_fts_index.cpp | 7 +-- fts/src/include/utils/fts_utils.h | 5 ++ fts/src/utils/fts_utils.cpp | 10 +++ fts/test/test_files/drop_recreate.test | 85 ++++++++++++++++++++++++++ fts/test/test_files/multi_index.test | 4 -- 6 files changed, 113 insertions(+), 19 deletions(-) create mode 100644 fts/test/test_files/drop_recreate.test diff --git a/fts/src/function/create_fts_index.cpp b/fts/src/function/create_fts_index.cpp index 17508413..7c4369b2 100644 --- a/fts/src/function/create_fts_index.cpp +++ b/fts/src/function/create_fts_index.cpp @@ -266,18 +266,17 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& context.setUseInternalCatalogEntry(prevUseInternalCatalogEntry); throw; } - // Create the tokenize macro unless one already exists (a residue macro is reused; - // dropping it is avoided: macro-drop WAL replay is currently broken engine-side). - if (!catalog::Catalog::Get(context)->containsMacro(transaction::Transaction::Get(context), - FTSUtils::getTokenizeMacroName(tableID, indexName))) { - // TOKENIZE(text, tokenizer, extra_param) - query += std::format(R"(CREATE MACRO `{}`(query) AS + // An existing tokenize macro comes from IMPORT DATABASE, an interrupted CREATE, or an index of + // the same name dropped by an earlier version; it is replaced so the bulk build tokenizes with + // this index's options. + query += FTSUtils::dropTokenizeMacroQuery(context, tableID, indexName); + // TOKENIZE(text, tokenizer, extra_param) + query += std::format(R"(CREATE MACRO `{}`(query) AS TOKENIZE(lower(regexp_replace(CAST(query as STRING), '{}', ' ', 'g')), '{}', '{}');)", - FTSUtils::getTokenizeMacroName(tableID, indexName), - formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern), - ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, - ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir); - } + FTSUtils::getTokenizeMacroName(tableID, indexName), + formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern), + ftsBindData->createFTSConfig.tokenizerInfo.tokenizer, + ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir); // Create the stop words table if not exists, or the user is not using the default english // one. diff --git a/fts/src/function/drop_fts_index.cpp b/fts/src/function/drop_fts_index.cpp index d3733b63..a86ddeb9 100644 --- a/fts/src/function/drop_fts_index.cpp +++ b/fts/src/function/drop_fts_index.cpp @@ -56,10 +56,9 @@ std::string dropFTSIndexQuery(ClientContext& context, const TableFuncBindData& b if (stopWordsTableName != FTSUtils::getDefaultStopWordsTableName()) { query += std::format("DROP TABLE IF EXISTS `{}`;", stopWordsTableName); } - // NOTE: the per-index tokenize macro is intentionally left behind. Dropping it - // here (or recreating it in CREATE) would emit macro-drop WAL records whose - // replay is currently broken engine-side (getScalarMacroCatalogEntry looks in the - // wrong catalog set), breaking un-checkpointed reopen. Revisit once fixed. + query += + FTSUtils::dropTokenizeMacroQuery(context, ftsBindData->tableID, ftsBindData->indexName); + query += std::format("RETURN 'Index {} has been dropped.' as result;", ftsBindData->indexName); return query; } diff --git a/fts/src/include/utils/fts_utils.h b/fts/src/include/utils/fts_utils.h index bacb88ff..89d21ec1 100644 --- a/fts/src/include/utils/fts_utils.h +++ b/fts/src/include/utils/fts_utils.h @@ -69,6 +69,11 @@ struct FTSUtils { static std::vector tokenizeString(std::string& str, const FTSConfig& tokenizer); + // DROP MACRO IF EXISTS throws when the macro is missing, so the statement is only emitted for + // an existing macro. + static std::string dropTokenizeMacroQuery(const main::ClientContext& context, + common::table_id_t tableID, const std::string& indexName); + // Returns a process-wide cached jieba instance for the given dictionary directory. // The first call for a directory loads the dictionaries (~0.5s); subsequent calls // reuse the instance. The returned instance is const and safe for concurrent use by diff --git a/fts/src/utils/fts_utils.cpp b/fts/src/utils/fts_utils.cpp index 83baa282..98fd3dc7 100644 --- a/fts/src/utils/fts_utils.cpp +++ b/fts/src/utils/fts_utils.cpp @@ -7,6 +7,7 @@ #include #include +#include "catalog/catalog.h" #include "common/exception/binder.h" #include "common/string_utils.h" #include "cppjieba/Jieba.hpp" @@ -203,5 +204,14 @@ std::vector FTSUtils::tokenizeString(std::string& str, const FTSCon return terms; } +std::string FTSUtils::dropTokenizeMacroQuery(const main::ClientContext& context, + common::table_id_t tableID, const std::string& indexName) { + auto macroName = getTokenizeMacroName(tableID, indexName); + if (!Catalog::Get(context)->containsMacro(Transaction::Get(context), macroName)) { + return ""; + } + return std::format("DROP MACRO `{}`;", macroName); +} + } // namespace fts_extension } // namespace lbug diff --git a/fts/test/test_files/drop_recreate.test b/fts/test/test_files/drop_recreate.test new file mode 100644 index 00000000..018eb148 --- /dev/null +++ b/fts/test/test_files/drop_recreate.test @@ -0,0 +1,85 @@ +-DATASET CSV empty-db + +-- + +# DROP_FTS_INDEX removes the index's tokenize macro and stopwords copy, so an index created again +# under the same name uses its own options. +-CASE RecreateWithNewOptions +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body']); +---- ok +-STATEMENT CALL DROP_FTS_INDEX('Doc', 'idx'); +---- 1 +Index idx has been dropped. +# The default pattern strips digits; the new one keeps them for existing and new rows alike. +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], ignore_pattern := '[!]+'); +---- ok +-STATEMENT CREATE (:Doc {id: 2, body: 'order 12345'}); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', '12345') RETURN node.id ORDER BY node.id; +---- 2 +1 +2 + +-CASE RecreateWithStopWordsTable +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE Stop(sw STRING PRIMARY KEY); +---- ok +-STATEMENT CREATE (:Stop {sw: 'order'}); +---- ok +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'order shipped'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := 'Stop'); +---- ok +-STATEMENT CALL DROP_FTS_INDEX('Doc', 'idx'); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := 'Stop'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'order') RETURN node.id; +---- 0 +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'shipped') RETURN node.id; +---- 1 +1 + +-CASE RecreateWithStopWordsFile +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'toronto weather'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := '${LBUG_ROOT_DIRECTORY}/dataset/fts-small/city_stopwords.csv'); +---- ok +-STATEMENT CALL DROP_FTS_INDEX('Doc', 'idx'); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := '${LBUG_ROOT_DIRECTORY}/dataset/fts-small/city_stopwords.csv'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'toronto') RETURN node.id; +---- 0 +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'weather') RETURN node.id; +---- 1 +1 + +# Earlier versions of DROP_FTS_INDEX left the tokenize macro behind; CREATE_FTS_INDEX replaces it. +-CASE LeftoverMacroIsReplaced +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'}); +---- ok +# The macro name embeds the table ID. +-STATEMENT CALL SHOW_TABLES() WHERE name = 'Doc' RETURN id; +---- 1 +1 +-STATEMENT CREATE MACRO `1_idx_tokenize`(query) AS TOKENIZE(lower(regexp_replace(CAST(query as STRING), '[0-9]+', ' ', 'g')), 'simple', ''); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], ignore_pattern := '[!]+'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', '12345') RETURN node.id; +---- 1 +1 diff --git a/fts/test/test_files/multi_index.test b/fts/test/test_files/multi_index.test index bfe327f0..e57ee66f 100644 --- a/fts/test/test_files/multi_index.test +++ b/fts/test/test_files/multi_index.test @@ -34,10 +34,6 @@ Exported database successfully. ---- ok -STATEMENT DROP TABLE INVALID_STOPWORDS_COL_TYPE ---- ok --STATEMENT DROP MACRO `0_AUTHORIDX_TOKENIZE` ----- ok --STATEMENT DROP MACRO `0_NAMEIDX_TOKENIZE` ----- ok -STATEMENT IMPORT DATABASE "${LBUG_EXPORT_DB_DIRECTORY}/fts-db" ---- 1 Imported database successfully. From 53a30543b968ea61d20df076f1967122598dc4dc Mon Sep 17 00:00:00 2001 From: Darek Date: Wed, 14 Oct 2026 15:00:31 +0200 Subject: [PATCH 2/2] test(fts): recreate an index after its drop is replayed from the WAL DROP_FTS_INDEX now also drops the tokenize macro and the stopwords copy. The new case drops an index with a stopwords table, reopens without a checkpoint so the drop is replayed, then recreates the index with a new ignore_pattern and finds the row it now indexes. Co-Authored-By: Claude Opus 5.5 --- fts/test/test_files/drop_recreate.test | 28 ++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/fts/test/test_files/drop_recreate.test b/fts/test/test_files/drop_recreate.test index 018eb148..490c8a5c 100644 --- a/fts/test/test_files/drop_recreate.test +++ b/fts/test/test_files/drop_recreate.test @@ -83,3 +83,31 @@ Index idx has been dropped. -STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', '12345') RETURN node.id; ---- 1 1 + +# A drop replayed from the WAL removes the macro and stopwords copy too. +-CASE RecreateAfterDropReplay +-SKIP_IN_MEM +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CREATE NODE TABLE Stop(sw STRING PRIMARY KEY); +---- ok +-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING); +---- ok +-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'}); +---- ok +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := 'Stop'); +---- ok +-RELOADDB +-STATEMENT CALL force_checkpoint_on_close=false; +---- ok +-STATEMENT CALL auto_checkpoint=false; +---- ok +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CALL DROP_FTS_INDEX('Doc', 'idx'); +---- ok +-RELOADDB +-LOAD_DYNAMIC_EXTENSION fts +-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], ignore_pattern := '[!]+', stopwords := 'Stop'); +---- ok +-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', '12345') RETURN node.id; +---- 1 +1