fix(fts): drop the tokenize macro with the index

fts/src/function/create_fts_index.cpp

@@ -266,18 +266,17 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData&
         context.setUseInternalCatalogEntry(prevUseInternalCatalogEntry);
         throw;
     }
-    // Create the tokenize macro unless one already exists (a residue macro is reused;
-    // dropping it is avoided: macro-drop WAL replay is currently broken engine-side).
-    if (!catalog::Catalog::Get(context)->containsMacro(transaction::Transaction::Get(context),
-            FTSUtils::getTokenizeMacroName(tableID, indexName))) {
-        // TOKENIZE(text, tokenizer, extra_param)
-        query += std::format(R"(CREATE MACRO `{}`(query) AS
+    // An existing tokenize macro comes from IMPORT DATABASE, an interrupted CREATE, or an index of
+    // the same name dropped by an earlier version; it is replaced so the bulk build tokenizes with
+    // this index's options.
+    query += FTSUtils::dropTokenizeMacroQuery(context, tableID, indexName);
+    // TOKENIZE(text, tokenizer, extra_param)
+    query += std::format(R"(CREATE MACRO `{}`(query) AS
                             TOKENIZE(lower(regexp_replace(CAST(query as STRING), '{}', ' ', 'g')), '{}', '{}');)",
-            FTSUtils::getTokenizeMacroName(tableID, indexName),
-            formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern),
-            ftsBindData->createFTSConfig.tokenizerInfo.tokenizer,
-            ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir);
-    }
+        FTSUtils::getTokenizeMacroName(tableID, indexName),
+        formatStrInCypher(ftsBindData->createFTSConfig.ignorePattern),
+        ftsBindData->createFTSConfig.tokenizerInfo.tokenizer,
+        ftsBindData->createFTSConfig.tokenizerInfo.jiebaDictDir);
 
     // Create the stop words table if not exists, or the user is not using the default english
     // one.

fts/src/function/drop_fts_index.cpp

@@ -56,10 +56,9 @@ std::string dropFTSIndexQuery(ClientContext& context, const TableFuncBindData& b
     if (stopWordsTableName != FTSUtils::getDefaultStopWordsTableName()) {
         query += std::format("DROP TABLE IF EXISTS `{}`;", stopWordsTableName);
     }
-    // NOTE: the per-index tokenize macro is intentionally left behind. Dropping it
-    // here (or recreating it in CREATE) would emit macro-drop WAL records whose
-    // replay is currently broken engine-side (getScalarMacroCatalogEntry looks in the
-    // wrong catalog set), breaking un-checkpointed reopen. Revisit once fixed.
+    query +=
+        FTSUtils::dropTokenizeMacroQuery(context, ftsBindData->tableID, ftsBindData->indexName);
+    query += std::format("RETURN 'Index {} has been dropped.' as result;", ftsBindData->indexName);
     return query;
 }
 

fts/src/include/utils/fts_utils.h

@@ -69,6 +69,11 @@ struct FTSUtils {
 
     static std::vector<std::string> tokenizeString(std::string& str, const FTSConfig& tokenizer);
 
+    // DROP MACRO IF EXISTS throws when the macro is missing, so the statement is only emitted for
+    // an existing macro.
+    static std::string dropTokenizeMacroQuery(const main::ClientContext& context,
+        common::table_id_t tableID, const std::string& indexName);
+
     // Returns a process-wide cached jieba instance for the given dictionary directory.
     // The first call for a directory loads the dictionaries (~0.5s); subsequent calls
     // reuse the instance. The returned instance is const and safe for concurrent use by

fts/src/utils/fts_utils.cpp

@@ -7,6 +7,7 @@
 #include <optional>
 #include <unordered_map>
 
+#include "catalog/catalog.h"
 #include "common/exception/binder.h"
 #include "common/string_utils.h"
 #include "cppjieba/Jieba.hpp"
@@ -203,5 +204,14 @@ std::vector<std::string> FTSUtils::tokenizeString(std::string& str, const FTSCon
     return terms;
 }
 
+std::string FTSUtils::dropTokenizeMacroQuery(const main::ClientContext& context,
+    common::table_id_t tableID, const std::string& indexName) {
+    auto macroName = getTokenizeMacroName(tableID, indexName);
+    if (!Catalog::Get(context)->containsMacro(Transaction::Get(context), macroName)) {
+        return "";
+    }
+    return std::format("DROP MACRO `{}`;", macroName);
+}
+
 } // namespace fts_extension
 } // namespace lbug

fts/test/test_files/drop_recreate.test

@@ -0,0 +1,113 @@
+-DATASET CSV empty-db
+
+--
+
+# DROP_FTS_INDEX removes the index's tokenize macro and stopwords copy, so an index created again
+# under the same name uses its own options.
+-CASE RecreateWithNewOptions
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING);
+---- ok
+-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'});
+---- ok
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body']);
+---- ok
+-STATEMENT CALL DROP_FTS_INDEX('Doc', 'idx');
+---- 1
+Index idx has been dropped.
+# The default pattern strips digits; the new one keeps them for existing and new rows alike.
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], ignore_pattern := '[!]+');
+---- ok
+-STATEMENT CREATE (:Doc {id: 2, body: 'order 12345'});
+---- ok
+-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', '12345') RETURN node.id ORDER BY node.id;
+---- 2
+1
+2
+
+-CASE RecreateWithStopWordsTable
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CREATE NODE TABLE Stop(sw STRING PRIMARY KEY);
+---- ok
+-STATEMENT CREATE (:Stop {sw: 'order'});
+---- ok
+-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING);
+---- ok
+-STATEMENT CREATE (:Doc {id: 1, body: 'order shipped'});
+---- ok
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := 'Stop');
+---- ok
+-STATEMENT CALL DROP_FTS_INDEX('Doc', 'idx');
+---- ok
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := 'Stop');
+---- ok
+-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'order') RETURN node.id;
+---- 0
+-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'shipped') RETURN node.id;
+---- 1
+1
+
+-CASE RecreateWithStopWordsFile
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING);
+---- ok
+-STATEMENT CREATE (:Doc {id: 1, body: 'toronto weather'});
+---- ok
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := '${LBUG_ROOT_DIRECTORY}/dataset/fts-small/city_stopwords.csv');
+---- ok
+-STATEMENT CALL DROP_FTS_INDEX('Doc', 'idx');
+---- ok
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := '${LBUG_ROOT_DIRECTORY}/dataset/fts-small/city_stopwords.csv');
+---- ok
+-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'toronto') RETURN node.id;
+---- 0
+-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', 'weather') RETURN node.id;
+---- 1
+1
+
+# Earlier versions of DROP_FTS_INDEX left the tokenize macro behind; CREATE_FTS_INDEX replaces it.
+-CASE LeftoverMacroIsReplaced
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING);
+---- ok
+-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'});
+---- ok
+# The macro name embeds the table ID.
+-STATEMENT CALL SHOW_TABLES() WHERE name = 'Doc' RETURN id;
+---- 1
+1
+-STATEMENT CREATE MACRO `1_idx_tokenize`(query) AS TOKENIZE(lower(regexp_replace(CAST(query as STRING), '[0-9]+', ' ', 'g')), 'simple', '');
+---- ok
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], ignore_pattern := '[!]+');
+---- ok
+-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', '12345') RETURN node.id;
+---- 1
+1
+
+# A drop replayed from the WAL removes the macro and stopwords copy too.
+-CASE RecreateAfterDropReplay
+-SKIP_IN_MEM
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CREATE NODE TABLE Stop(sw STRING PRIMARY KEY);
+---- ok
+-STATEMENT CREATE NODE TABLE Doc(id INT64 PRIMARY KEY, body STRING);
+---- ok
+-STATEMENT CREATE (:Doc {id: 1, body: 'order 12345'});
+---- ok
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], stopwords := 'Stop');
+---- ok
+-RELOADDB
+-STATEMENT CALL force_checkpoint_on_close=false;
+---- ok
+-STATEMENT CALL auto_checkpoint=false;
+---- ok
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CALL DROP_FTS_INDEX('Doc', 'idx');
+---- ok
+-RELOADDB
+-LOAD_DYNAMIC_EXTENSION fts
+-STATEMENT CALL CREATE_FTS_INDEX('Doc', 'idx', ['body'], ignore_pattern := '[!]+', stopwords := 'Stop');
+---- ok
+-STATEMENT CALL QUERY_FTS_INDEX('Doc', 'idx', '12345') RETURN node.id;
+---- 1
+1

fts/test/test_files/multi_index.test

@@ -34,10 +34,6 @@ Exported database successfully.
 ---- ok
 -STATEMENT DROP TABLE INVALID_STOPWORDS_COL_TYPE
 ---- ok
--STATEMENT DROP MACRO `0_AUTHORIDX_TOKENIZE`
----- ok
--STATEMENT DROP MACRO `0_NAMEIDX_TOKENIZE`
----- ok
 -STATEMENT IMPORT DATABASE "${LBUG_EXPORT_DB_DIRECTORY}/fts-db"
 ---- 1
 Imported database successfully.