fix(fts): tolerate interrupted CREATE/DROP residue (fixes #77)

fts/src/function/create_fts_index.cpp

@@ -1,8 +1,13 @@
 #include "function/create_fts_index.h"
 
+#include <functional>
+
 #include "binder/expression/literal_expression.h"
+#include "catalog/catalog_entry/node_table_catalog_entry.h"
+#include "catalog/catalog_entry/rel_group_catalog_entry.h"
 #include "catalog/fts_index_catalog_entry.h"
 #include "common/exception/binder.h"
+#include "common/string_utils.h"
 #include "common/types/value/nested.h"
 #include "function/fts_bind_data.h"
 #include "function/fts_config.h"
@@ -63,26 +68,123 @@ static std::vector<property_id_t> bindProperties(const catalog::NodeTableCatalog
     return result;
 }
 
-static void validateInternalTableNotExist(const std::string& tableName,
-    const catalog::Catalog& catalog, const transaction::Transaction* transaction) {
-    if (catalog.containsTable(transaction, tableName)) {
-        throw BinderException{
-            std::format("Table: {} already exists. Please drop or rename the table before "
-                        "creating a full text search index.",
-                tableName)};
+// Drops internal-only residue left behind by an interrupted CREATE/DROP so a failed
+// CREATE can simply be retried in place. The catalog index entry is written last
+// during CREATE, so an internal table present without an index entry is residue
+// (bind already validated the index itself doesn't exist) — with two safeguards:
+// (1) tables visible in the regular catalog are genuine user tables and still raise;
+// (2) remaining candidates are dropped only if their schema exactly matches the FTS
+// internal schema, so a misfiled user table can never be dropped silently. (A
+// useInternalCatalogEntry leak on failed statements — reset at the end of
+// ClientContext::query but not on its exception path — can misfile later user DDL
+// into the internal set; see the save/restore in createFTSIndexQuery.)
+static bool hasExactProperties(const catalog::TableCatalogEntry& entry,
+    const std::vector<std::pair<std::string, common::LogicalTypeID>>& expected) {
+    if (entry.getNumProperties() != expected.size()) {
+        return false;
+    }
+    for (auto& prop : entry.getProperties()) {
+        auto it = std::find_if(expected.begin(), expected.end(), [&](auto& e) {
+            return common::StringUtils::caseInsensitiveEquals(e.first, prop.getName());
+        });
+        if (it == expected.end() || prop.getType().getLogicalTypeID() != it->second) {
+            return false;
+        }
+    }
+    return true;
+}
+
+static bool isResidueNodeTable(catalog::TableCatalogEntry* entry, const std::string& pkName,
+    const std::vector<std::pair<std::string, common::LogicalTypeID>>& expectedProps) {
+    if (entry->getTableType() != common::TableType::NODE) {
+        return false;
+    }
+    auto& nodeEntry = entry->constCast<catalog::NodeTableCatalogEntry>();
+    return common::StringUtils::caseInsensitiveEquals(nodeEntry.getPrimaryKeyName(), pkName) &&
+           hasExactProperties(*entry, expectedProps);
+}
+
+static bool isResidueAppearsInTable(catalog::Catalog* catalog,
+    const transaction::Transaction* transaction, catalog::TableCatalogEntry* entry,
+    const std::string& termsTableName, const std::string& docsTableName) {
+    if (entry->getTableType() != common::TableType::REL) {
+        return false;
+    }
+    auto& relGroup = entry->constCast<catalog::RelGroupCatalogEntry>();
+    auto& infos = relGroup.getRelEntryInfos();
+    if (infos.size() != 1) {
+        return false;
+    }
+    // When both endpoint tables exist, require exact endpoint match; when an endpoint
+    // is missing the rel table is orphaned and dropping it is still the safe repair.
+    if (catalog->containsTable(transaction, termsTableName) &&
+        catalog->containsTable(transaction, docsTableName)) {
+        auto termsID = catalog->getTableCatalogEntry(transaction, termsTableName)->getTableID();
+        auto docsID = catalog->getTableCatalogEntry(transaction, docsTableName)->getTableID();
+        if (infos[0].nodePair.srcTableID != termsID || infos[0].nodePair.dstTableID != docsID) {
+            return false;
+        }
     }
+    return hasExactProperties(*entry, {{"tf", common::LogicalTypeID::UINT64}});
 }
 
-static void validateInternalTablesNotExist(table_id_t tableID, const std::string& indexName,
-    const catalog::Catalog& catalog, const transaction::Transaction* transaction) {
-    validateInternalTableNotExist(FTSUtils::getDocsTableName(tableID, indexName), catalog,
-        transaction);
-    validateInternalTableNotExist(FTSUtils::getAppearsInTableName(tableID, indexName), catalog,
-        transaction);
-    validateInternalTableNotExist(FTSUtils::getAppearsInfoTableName(tableID, indexName), catalog,
-        transaction);
-    validateInternalTableNotExist(FTSUtils::getTermsTableName(tableID, indexName), catalog,
-        transaction);
+static std::string dropResidueStatements(ClientContext& context, table_id_t tableID,
+    const std::string& indexName, const StopWordsTableInfo& stopWordsInfo) {
+    auto catalog = catalog::Catalog::Get(context);
+    auto transaction = transaction::Transaction::Get(context);
+    auto docsTableName = FTSUtils::getDocsTableName(tableID, indexName);
+    auto termsTableName = FTSUtils::getTermsTableName(tableID, indexName);
+    auto appearsInfoTableName = FTSUtils::getAppearsInfoTableName(tableID, indexName);
+    auto appearsInTableName = FTSUtils::getAppearsInTableName(tableID, indexName);
+    std::string query;
+    // Throws if tableName is a genuine user table; emits a guarded DROP if it is
+    // schema-confirmed residue; emits nothing if the name is free. isResidue must
+    // exactly mirror the CREATE statements below.
+    auto maybeDropTable = [&](const std::string& tableName,
+                              const std::function<bool(catalog::TableCatalogEntry*)>& isResidue) {
+        if (catalog->containsTable(transaction, tableName, false /* useInternal */)) {
+            throw BinderException{
+                std::format("Table: {} already exists. Please drop or rename the table before "
+                            "creating a full text search index.",
+                    tableName)};
+        }
+        if (!catalog->containsTable(transaction, tableName)) {
+            return;
+        }
+        auto entry = catalog->getTableCatalogEntry(transaction, tableName);
+        if (!isResidue(entry)) {
+            throw BinderException{
+                std::format("Table: {} already exists. Please drop or rename the table before "
+                            "creating a full text search index.",
+                    tableName)};
+        }
+        query += std::format("DROP TABLE IF EXISTS `{}`;", tableName);
+    };
+    // Drop the rel table before the node tables it references.
+    maybeDropTable(appearsInTableName, [&](catalog::TableCatalogEntry* entry) {
+        return isResidueAppearsInTable(catalog, transaction, entry, termsTableName, docsTableName);
+    });
+    maybeDropTable(appearsInfoTableName, [](catalog::TableCatalogEntry* entry) {
+        return isResidueNodeTable(entry, "ID",
+            {{"ID", common::LogicalTypeID::SERIAL}, {"term", common::LogicalTypeID::STRING},
+                {"docID", common::LogicalTypeID::INT64}});
+    });
+    maybeDropTable(docsTableName, [](catalog::TableCatalogEntry* entry) {
+        return isResidueNodeTable(entry, "docID",
+            {{"docID", common::LogicalTypeID::INT64}, {"len", common::LogicalTypeID::UINT64}});
+    });
+    maybeDropTable(termsTableName, [](catalog::TableCatalogEntry* entry) {
+        return isResidueNodeTable(entry, "term",
+            {{"term", common::LogicalTypeID::STRING}, {"df", common::LogicalTypeID::UINT64}});
+    });
+    // The shared default stopwords table is idempotent (IF NOT EXISTS + MERGE) and
+    // must never be dropped; only per-index copies are residue candidates.
+    if (stopWordsInfo.source != StopWordsSource::DEFAULT) {
+        maybeDropTable(stopWordsInfo.tableName, [](catalog::TableCatalogEntry* entry) {
+            return isResidueNodeTable(entry, "sw", {{"sw", common::LogicalTypeID::STRING}});
+        });
+    }
+    return query;
 }
 
 static std::unique_ptr<TableFuncBindData> bindFunc(ClientContext* context,
@@ -148,13 +250,24 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData&
     auto ftsBindData = bindData.constPtrCast<CreateFTSBindData>();
     auto tableID = ftsBindData->tableID;
     auto indexName = ftsBindData->indexName;
-    validateInternalTablesNotExist(ftsBindData->tableID, ftsBindData->indexName,
-        *catalog::Catalog::Get(context), transaction::Transaction::Get(context));
     context.setUseInternalCatalogEntry(true /* useInternalCatalogEntry */);
     // TODO(Ziyi): Copy statement can't be wrapped in manual transaction, so we can't wrap all
     // statements in a single transaction there.
-    // Create the tokenize macro.
     std::string query = "";
+    // Self-heal schema-confirmed residue from an interrupted CREATE/DROP before rebuilding.
+    // Save/restore the internal-catalog flag: dropResidueStatements throws on user-table
+    // collisions, and ClientContext only resets the flag on the success path, so
+    // without this a failed CREATE would misfile later user DDL into the internal set.
+    auto prevUseInternalCatalogEntry = context.useInternalCatalogEntry();
+    try {
+        query += dropResidueStatements(context, tableID, indexName,
+            ftsBindData->createFTSConfig.stopWordsTableInfo);
+    } catch (...) {
+        context.setUseInternalCatalogEntry(prevUseInternalCatalogEntry);
+        throw;
+    }
+    // Create the tokenize macro unless one already exists (a residue macro is reused;
+    // dropping it is avoided: macro-drop WAL replay is currently broken engine-side).
     if (!catalog::Catalog::Get(context)->containsMacro(transaction::Transaction::Get(context),
             FTSUtils::getTokenizeMacroName(tableID, indexName))) {
         // TOKENIZE(text, tokenizer, extra_param)

fts/src/function/drop_fts_index.cpp

@@ -1,6 +1,7 @@
 #include "function/drop_fts_index.h"
 
 #include "catalog/catalog.h"
+#include "catalog/fts_index_catalog_entry.h"
 #include "function/fts_bind_data.h"
 #include "function/fts_index_utils.h"
 #include "function/table/bind_data.h"
@@ -32,14 +33,33 @@ static std::unique_ptr<TableFuncBindData> bindFunc(ClientContext* context,
 std::string dropFTSIndexQuery(ClientContext& context, const TableFuncBindData& bindData) {
     context.setUseInternalCatalogEntry(true /* useInternalCatalogEntry */);
     auto ftsBindData = bindData.constPtrCast<FTSBindData>();
+    auto catalog = catalog::Catalog::Get(context);
+    auto transaction = transaction::Transaction::Get(context);
     auto query = std::format("CALL _DROP_FTS_INDEX('{}', '{}');", ftsBindData->tableName,
         ftsBindData->indexName);
-    query += std::format("DROP TABLE `{}`;",
+    // Drop the rel table before the node tables it references. IF EXISTS keeps a
+    // previously-interrupted DROP retryable: _DROP_FTS_INDEX above removes the catalog
+    // index entry first, so a retry can no longer rely on the entry being present.
+    query += std::format("DROP TABLE IF EXISTS `{}`;",
         FTSUtils::getAppearsInTableName(ftsBindData->tableID, ftsBindData->indexName));
-    query += std::format("DROP TABLE `{}`;",
+    // Transient build table; residue if CREATE was interrupted before its final DROP.
+    query += std::format("DROP TABLE IF EXISTS `{}`;",
+        FTSUtils::getAppearsInfoTableName(ftsBindData->tableID, ftsBindData->indexName));
+    query += std::format("DROP TABLE IF EXISTS `{}`;",
         FTSUtils::getDocsTableName(ftsBindData->tableID, ftsBindData->indexName));
-    query += std::format("DROP TABLE `{}`;",
+    query += std::format("DROP TABLE IF EXISTS `{}`;",
         FTSUtils::getTermsTableName(ftsBindData->tableID, ftsBindData->indexName));
+    // Per-index stopwords copy (the shared default table is never dropped).
+    auto indexEntry = catalog->getIndex(transaction, ftsBindData->tableID, ftsBindData->indexName);
+    auto stopWordsTableName =
+        indexEntry->getAuxInfo().cast<FTSIndexAuxInfo>().config.stopWordsTableName;
+    if (stopWordsTableName != FTSUtils::getDefaultStopWordsTableName()) {
+        query += std::format("DROP TABLE IF EXISTS `{}`;", stopWordsTableName);
+    }
+    // NOTE: the per-index tokenize macro is intentionally left behind. Dropping it
+    // here (or recreating it in CREATE) would emit macro-drop WAL records whose
+    // replay is currently broken engine-side (getScalarMacroCatalogEntry looks in the
+    // wrong catalog set), breaking un-checkpointed reopen. Revisit once fixed.
     return query;
 }
 

fts/src/main/fts_extension.cpp

@@ -10,18 +10,39 @@
 #include "index/fts_index.h"
 #include "main/client_context.h"
 #include "storage/storage_manager.h"
+#include "utils/fts_utils.h"
 
 namespace lbug {
 namespace fts_extension {
 
 using namespace extension;
 
+// An interrupted CREATE/DROP can leave the catalog index entry present while some of
+// its internal tables are missing. Loading such an index throws and would fail the
+// whole database open, so detect the torn state up front and skip the index instead.
+// The database still opens; repair with DROP_FTS_INDEX + CREATE_FTS_INDEX (a plain
+// re-CREATE also works: CREATE self-heals internal-only residue).
+static bool ftsInternalTablesComplete(main::ClientContext* context, common::table_id_t tableID,
+    const std::string& indexName, const std::string& stopWordsTableName) {
+    auto catalog = catalog::Catalog::Get(*context);
+    auto transaction = transaction::Transaction::Get(*context);
+    return catalog->containsTable(transaction, stopWordsTableName) &&
+           catalog->containsTable(transaction, FTSUtils::getDocsTableName(tableID, indexName)) &&
+           catalog->containsTable(transaction, FTSUtils::getTermsTableName(tableID, indexName)) &&
+           catalog->containsTable(transaction, FTSUtils::getAppearsInTableName(tableID, indexName));
+}
+
 static void initFTSEntries(main::ClientContext* context, catalog::Catalog& catalog) {
     auto storageManager = storage::StorageManager::Get(*context);
     for (auto& indexEntry : catalog.getIndexEntries(transaction::Transaction::Get(*context))) {
         if (indexEntry->getIndexType() == FTSIndexCatalogEntry::TYPE_NAME &&
             !indexEntry->isLoaded()) {
             indexEntry->setAuxInfo(FTSIndexAuxInfo::deserialize(indexEntry->getAuxBufferReader()));
+            auto& auxConfig = indexEntry->getAuxInfo().cast<FTSIndexAuxInfo>().config;
+            if (!ftsInternalTablesComplete(context, indexEntry->getTableID(),
+                    indexEntry->getIndexName(), auxConfig.stopWordsTableName)) {
+                continue;
+            }
             auto& nodeTable =
                 storageManager->getTable(indexEntry->getTableID())->cast<storage::NodeTable>();
             auto optionalIndex = nodeTable.getIndexHolder(indexEntry->getIndexName());