diff --git a/fts/src/function/create_fts_index.cpp b/fts/src/function/create_fts_index.cpp index e3a1cc91..b4e9f976 100644 --- a/fts/src/function/create_fts_index.cpp +++ b/fts/src/function/create_fts_index.cpp @@ -1,8 +1,13 @@ #include "function/create_fts_index.h" +#include + #include "binder/expression/literal_expression.h" +#include "catalog/catalog_entry/node_table_catalog_entry.h" +#include "catalog/catalog_entry/rel_group_catalog_entry.h" #include "catalog/fts_index_catalog_entry.h" #include "common/exception/binder.h" +#include "common/string_utils.h" #include "common/types/value/nested.h" #include "function/fts_bind_data.h" #include "function/fts_config.h" @@ -63,26 +68,123 @@ static std::vector bindProperties(const catalog::NodeTableCatalog return result; } -static void validateInternalTableNotExist(const std::string& tableName, - const catalog::Catalog& catalog, const transaction::Transaction* transaction) { - if (catalog.containsTable(transaction, tableName)) { - throw BinderException{ - std::format("Table: {} already exists. Please drop or rename the table before " - "creating a full text search index.", - tableName)}; +// Drops internal-only residue left behind by an interrupted CREATE/DROP so a failed +// CREATE can simply be retried in place. The catalog index entry is written last +// during CREATE, so an internal table present without an index entry is residue +// (bind already validated the index itself doesn't exist) — with two safeguards: +// (1) tables visible in the regular catalog are genuine user tables and still raise; +// (2) remaining candidates are dropped only if their schema exactly matches the FTS +// internal schema, so a misfiled user table can never be dropped silently. (A +// useInternalCatalogEntry leak on failed statements — reset at the end of +// ClientContext::query but not on its exception path — can misfile later user DDL +// into the internal set; see the save/restore in createFTSIndexQuery.) +static bool hasExactProperties(const catalog::TableCatalogEntry& entry, + const std::vector>& expected) { + if (entry.getNumProperties() != expected.size()) { + return false; + } + for (auto& prop : entry.getProperties()) { + auto it = std::find_if(expected.begin(), expected.end(), [&](auto& e) { + return common::StringUtils::caseInsensitiveEquals(e.first, prop.getName()); + }); + if (it == expected.end() || prop.getType().getLogicalTypeID() != it->second) { + return false; + } + } + return true; +} + +static bool isResidueNodeTable(catalog::TableCatalogEntry* entry, const std::string& pkName, + const std::vector>& expectedProps) { + if (entry->getTableType() != common::TableType::NODE) { + return false; + } + auto& nodeEntry = entry->constCast(); + return common::StringUtils::caseInsensitiveEquals(nodeEntry.getPrimaryKeyName(), pkName) && + hasExactProperties(*entry, expectedProps); +} + +static bool isResidueAppearsInTable(catalog::Catalog* catalog, + const transaction::Transaction* transaction, catalog::TableCatalogEntry* entry, + const std::string& termsTableName, const std::string& docsTableName) { + if (entry->getTableType() != common::TableType::REL) { + return false; + } + auto& relGroup = entry->constCast(); + auto& infos = relGroup.getRelEntryInfos(); + if (infos.size() != 1) { + return false; + } + // When both endpoint tables exist, require exact endpoint match; when an endpoint + // is missing the rel table is orphaned and dropping it is still the safe repair. + if (catalog->containsTable(transaction, termsTableName) && + catalog->containsTable(transaction, docsTableName)) { + auto termsID = catalog->getTableCatalogEntry(transaction, termsTableName)->getTableID(); + auto docsID = catalog->getTableCatalogEntry(transaction, docsTableName)->getTableID(); + if (infos[0].nodePair.srcTableID != termsID || infos[0].nodePair.dstTableID != docsID) { + return false; + } } + return hasExactProperties(*entry, {{"tf", common::LogicalTypeID::UINT64}}); } -static void validateInternalTablesNotExist(table_id_t tableID, const std::string& indexName, - const catalog::Catalog& catalog, const transaction::Transaction* transaction) { - validateInternalTableNotExist(FTSUtils::getDocsTableName(tableID, indexName), catalog, - transaction); - validateInternalTableNotExist(FTSUtils::getAppearsInTableName(tableID, indexName), catalog, - transaction); - validateInternalTableNotExist(FTSUtils::getAppearsInfoTableName(tableID, indexName), catalog, - transaction); - validateInternalTableNotExist(FTSUtils::getTermsTableName(tableID, indexName), catalog, - transaction); +static std::string dropResidueStatements(ClientContext& context, table_id_t tableID, + const std::string& indexName, const StopWordsTableInfo& stopWordsInfo) { + auto catalog = catalog::Catalog::Get(context); + auto transaction = transaction::Transaction::Get(context); + auto docsTableName = FTSUtils::getDocsTableName(tableID, indexName); + auto termsTableName = FTSUtils::getTermsTableName(tableID, indexName); + auto appearsInfoTableName = FTSUtils::getAppearsInfoTableName(tableID, indexName); + auto appearsInTableName = FTSUtils::getAppearsInTableName(tableID, indexName); + std::string query; + // Throws if tableName is a genuine user table; emits a guarded DROP if it is + // schema-confirmed residue; emits nothing if the name is free. isResidue must + // exactly mirror the CREATE statements below. + auto maybeDropTable = [&](const std::string& tableName, + const std::function& isResidue) { + if (catalog->containsTable(transaction, tableName, false /* useInternal */)) { + throw BinderException{ + std::format("Table: {} already exists. Please drop or rename the table before " + "creating a full text search index.", + tableName)}; + } + if (!catalog->containsTable(transaction, tableName)) { + return; + } + auto entry = catalog->getTableCatalogEntry(transaction, tableName); + if (!isResidue(entry)) { + throw BinderException{ + std::format("Table: {} already exists. Please drop or rename the table before " + "creating a full text search index.", + tableName)}; + } + query += std::format("DROP TABLE IF EXISTS `{}`;", tableName); + }; + // Drop the rel table before the node tables it references. + maybeDropTable(appearsInTableName, [&](catalog::TableCatalogEntry* entry) { + return isResidueAppearsInTable(catalog, transaction, entry, termsTableName, docsTableName); + }); + maybeDropTable(appearsInfoTableName, [](catalog::TableCatalogEntry* entry) { + return isResidueNodeTable(entry, "ID", + {{"ID", common::LogicalTypeID::SERIAL}, {"term", common::LogicalTypeID::STRING}, + {"docID", common::LogicalTypeID::INT64}}); + }); + maybeDropTable(docsTableName, [](catalog::TableCatalogEntry* entry) { + return isResidueNodeTable(entry, "docID", + {{"docID", common::LogicalTypeID::INT64}, {"len", common::LogicalTypeID::UINT64}}); + }); + maybeDropTable(termsTableName, [](catalog::TableCatalogEntry* entry) { + return isResidueNodeTable(entry, "term", + {{"term", common::LogicalTypeID::STRING}, {"df", common::LogicalTypeID::UINT64}}); + }); + // The shared default stopwords table is idempotent (IF NOT EXISTS + MERGE) and + // must never be dropped; only per-index copies are residue candidates. + if (stopWordsInfo.source != StopWordsSource::DEFAULT) { + maybeDropTable(stopWordsInfo.tableName, [](catalog::TableCatalogEntry* entry) { + return isResidueNodeTable(entry, "sw", {{"sw", common::LogicalTypeID::STRING}}); + }); + } + return query; } static std::unique_ptr bindFunc(ClientContext* context, @@ -148,13 +250,24 @@ std::string createFTSIndexQuery(ClientContext& context, const TableFuncBindData& auto ftsBindData = bindData.constPtrCast(); auto tableID = ftsBindData->tableID; auto indexName = ftsBindData->indexName; - validateInternalTablesNotExist(ftsBindData->tableID, ftsBindData->indexName, - *catalog::Catalog::Get(context), transaction::Transaction::Get(context)); context.setUseInternalCatalogEntry(true /* useInternalCatalogEntry */); // TODO(Ziyi): Copy statement can't be wrapped in manual transaction, so we can't wrap all // statements in a single transaction there. - // Create the tokenize macro. std::string query = ""; + // Self-heal schema-confirmed residue from an interrupted CREATE/DROP before rebuilding. + // Save/restore the internal-catalog flag: dropResidueStatements throws on user-table + // collisions, and ClientContext only resets the flag on the success path, so + // without this a failed CREATE would misfile later user DDL into the internal set. + auto prevUseInternalCatalogEntry = context.useInternalCatalogEntry(); + try { + query += dropResidueStatements(context, tableID, indexName, + ftsBindData->createFTSConfig.stopWordsTableInfo); + } catch (...) { + context.setUseInternalCatalogEntry(prevUseInternalCatalogEntry); + throw; + } + // Create the tokenize macro unless one already exists (a residue macro is reused; + // dropping it is avoided: macro-drop WAL replay is currently broken engine-side). if (!catalog::Catalog::Get(context)->containsMacro(transaction::Transaction::Get(context), FTSUtils::getTokenizeMacroName(tableID, indexName))) { // TOKENIZE(text, tokenizer, extra_param) diff --git a/fts/src/function/drop_fts_index.cpp b/fts/src/function/drop_fts_index.cpp index fadf414f..d3733b63 100644 --- a/fts/src/function/drop_fts_index.cpp +++ b/fts/src/function/drop_fts_index.cpp @@ -1,6 +1,7 @@ #include "function/drop_fts_index.h" #include "catalog/catalog.h" +#include "catalog/fts_index_catalog_entry.h" #include "function/fts_bind_data.h" #include "function/fts_index_utils.h" #include "function/table/bind_data.h" @@ -32,14 +33,33 @@ static std::unique_ptr bindFunc(ClientContext* context, std::string dropFTSIndexQuery(ClientContext& context, const TableFuncBindData& bindData) { context.setUseInternalCatalogEntry(true /* useInternalCatalogEntry */); auto ftsBindData = bindData.constPtrCast(); + auto catalog = catalog::Catalog::Get(context); + auto transaction = transaction::Transaction::Get(context); auto query = std::format("CALL _DROP_FTS_INDEX('{}', '{}');", ftsBindData->tableName, ftsBindData->indexName); - query += std::format("DROP TABLE `{}`;", + // Drop the rel table before the node tables it references. IF EXISTS keeps a + // previously-interrupted DROP retryable: _DROP_FTS_INDEX above removes the catalog + // index entry first, so a retry can no longer rely on the entry being present. + query += std::format("DROP TABLE IF EXISTS `{}`;", FTSUtils::getAppearsInTableName(ftsBindData->tableID, ftsBindData->indexName)); - query += std::format("DROP TABLE `{}`;", + // Transient build table; residue if CREATE was interrupted before its final DROP. + query += std::format("DROP TABLE IF EXISTS `{}`;", + FTSUtils::getAppearsInfoTableName(ftsBindData->tableID, ftsBindData->indexName)); + query += std::format("DROP TABLE IF EXISTS `{}`;", FTSUtils::getDocsTableName(ftsBindData->tableID, ftsBindData->indexName)); - query += std::format("DROP TABLE `{}`;", + query += std::format("DROP TABLE IF EXISTS `{}`;", FTSUtils::getTermsTableName(ftsBindData->tableID, ftsBindData->indexName)); + // Per-index stopwords copy (the shared default table is never dropped). + auto indexEntry = catalog->getIndex(transaction, ftsBindData->tableID, ftsBindData->indexName); + auto stopWordsTableName = + indexEntry->getAuxInfo().cast().config.stopWordsTableName; + if (stopWordsTableName != FTSUtils::getDefaultStopWordsTableName()) { + query += std::format("DROP TABLE IF EXISTS `{}`;", stopWordsTableName); + } + // NOTE: the per-index tokenize macro is intentionally left behind. Dropping it + // here (or recreating it in CREATE) would emit macro-drop WAL records whose + // replay is currently broken engine-side (getScalarMacroCatalogEntry looks in the + // wrong catalog set), breaking un-checkpointed reopen. Revisit once fixed. return query; } diff --git a/fts/src/main/fts_extension.cpp b/fts/src/main/fts_extension.cpp index 2d3ce598..fe5f4f2c 100644 --- a/fts/src/main/fts_extension.cpp +++ b/fts/src/main/fts_extension.cpp @@ -10,18 +10,39 @@ #include "index/fts_index.h" #include "main/client_context.h" #include "storage/storage_manager.h" +#include "utils/fts_utils.h" namespace lbug { namespace fts_extension { using namespace extension; +// An interrupted CREATE/DROP can leave the catalog index entry present while some of +// its internal tables are missing. Loading such an index throws and would fail the +// whole database open, so detect the torn state up front and skip the index instead. +// The database still opens; repair with DROP_FTS_INDEX + CREATE_FTS_INDEX (a plain +// re-CREATE also works: CREATE self-heals internal-only residue). +static bool ftsInternalTablesComplete(main::ClientContext* context, common::table_id_t tableID, + const std::string& indexName, const std::string& stopWordsTableName) { + auto catalog = catalog::Catalog::Get(*context); + auto transaction = transaction::Transaction::Get(*context); + return catalog->containsTable(transaction, stopWordsTableName) && + catalog->containsTable(transaction, FTSUtils::getDocsTableName(tableID, indexName)) && + catalog->containsTable(transaction, FTSUtils::getTermsTableName(tableID, indexName)) && + catalog->containsTable(transaction, FTSUtils::getAppearsInTableName(tableID, indexName)); +} + static void initFTSEntries(main::ClientContext* context, catalog::Catalog& catalog) { auto storageManager = storage::StorageManager::Get(*context); for (auto& indexEntry : catalog.getIndexEntries(transaction::Transaction::Get(*context))) { if (indexEntry->getIndexType() == FTSIndexCatalogEntry::TYPE_NAME && !indexEntry->isLoaded()) { indexEntry->setAuxInfo(FTSIndexAuxInfo::deserialize(indexEntry->getAuxBufferReader())); + auto& auxConfig = indexEntry->getAuxInfo().cast().config; + if (!ftsInternalTablesComplete(context, indexEntry->getTableID(), + indexEntry->getIndexName(), auxConfig.stopWordsTableName)) { + continue; + } auto& nodeTable = storageManager->getTable(indexEntry->getTableID())->cast(); auto optionalIndex = nodeTable.getIndexHolder(indexEntry->getIndexName());