diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1f5515b8f..06993ae04 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -213,11 +213,15 @@ jobs: exit 1 fi # test_rocksdb_cloud_sst_reader exercises the rocksdb-cloud storage - # provider end-to-end against RustFS. The DBCloud::Open-based cases - # currently crash inside librocksdb-cloud (ABI/vtable mismatch) and are - # tracked separately, so they are excluded here. + # provider end-to-end against RustFS; test_cdc_scan_cloud_state, + # test_cdc_retention_floor_cloud_state and + # test_cdc_ddl_history_cloud_state the CDC log scan, the CDC + # retention floor and the CDC DDL history on the cloud log state. + # The DBCloud::Open-based cases currently crash inside + # librocksdb-cloud (ABI/vtable mismatch) and are tracked separately, + # so they are excluded here. cd eloq_log_service/bld_cloud - ./log_server_rocksdb_cloud_tests "test_rocksdb_cloud_sst_reader" \ + ./log_server_rocksdb_cloud_tests "test_rocksdb_cloud_sst_reader,test_cdc_scan_cloud_state,test_cdc_retention_floor_cloud_state,test_cdc_ddl_history_cloud_state" \ --region us-east-1 \ --aws_access_key_id "$AWS_ACCESS_KEY_ID" \ --aws_secret_key "$AWS_SECRET_ACCESS_KEY" diff --git a/build_eloq_log_service.cmake b/build_eloq_log_service.cmake index e83e7a0da..321e3a5a9 100644 --- a/build_eloq_log_service.cmake +++ b/build_eloq_log_service.cmake @@ -281,6 +281,8 @@ ADD_LIBRARY(logservice ${LOG_SOURCE_DIR}/src/log_state_rocksdb_impl.cpp ${LOG_SOURCE_DIR}/src/log_state_rocksdb_cloud_impl.cpp ${LOG_SOURCE_DIR}/src/log_state_memory_impl.cpp + ${LOG_SOURCE_DIR}/src/cdc_long_poll.cpp + ${LOG_SOURCE_DIR}/src/cdc_scan.cpp ${LOG_SOURCE_DIR}/src/fault_inject.cpp ${CMAKE_CURRENT_LIST_DIR}/third_party/ini/INIReader.cpp ${CMAKE_CURRENT_LIST_DIR}/third_party/ini/ini.c @@ -288,6 +290,9 @@ ADD_LIBRARY(logservice ) target_include_directories(logservice PUBLIC ${LOG_INCLUDE_DIR}) +# The log groups' metrics use eloq_metrics' header-only Meter against the +# registry the process passes in, so the library itself is not linked. +target_include_directories(logservice PUBLIC ${ELOQ_METRICS_INCLUDE_DIR}) if (WITH_LOG_STATE MATCHES "ROCKSDB|ROCKSDB_CLOUD_S3|ROCKSDB_CLOUD_GCS") target_include_directories(logservice PUBLIC ${ROCKSDB_INCLUDE_PATH}) endif() diff --git a/build_tx_service.cmake b/build_tx_service.cmake index bfe10b4a4..604b73cc7 100644 --- a/build_tx_service.cmake +++ b/build_tx_service.cmake @@ -200,6 +200,8 @@ SET(ELOQ_SOURCES ${TX_SERVICE_SOURCE_DIR}/src/fault/fault_inject.cpp ${TX_SERVICE_SOURCE_DIR}/src/dead_lock_check.cpp ${TX_SERVICE_SOURCE_DIR}/src/tx_index_operation.cpp + ${TX_SERVICE_SOURCE_DIR}/src/tx_cluster_gate_operation.cpp + ${TX_SERVICE_SOURCE_DIR}/src/tx_upsert_table_operation.cpp ${TX_SERVICE_SOURCE_DIR}/src/sk_generator.cpp ${TX_SERVICE_SOURCE_DIR}/src/data_sync_task.cpp ${TX_SERVICE_SOURCE_DIR}/src/store/snapshot_manager.cpp diff --git a/core/src/data_substrate.cpp b/core/src/data_substrate.cpp index cb66253ed..026dc832c 100644 --- a/core/src/data_substrate.cpp +++ b/core/src/data_substrate.cpp @@ -20,6 +20,7 @@ * */ +#include #include #include @@ -33,6 +34,9 @@ // clang-format on #include "data_substrate.h" #include "log_server.h" +#if (WITH_LOG_SERVICE) +#include "log_service_metrics.h" +#endif #include "sequences/sequences.h" #include "store/data_store_handler.h" #include "util.h" @@ -87,6 +91,35 @@ const auto NUM_VCPU = std::thread::hardware_concurrency(); // Static mutex for protecting initialization state static std::mutex instance_mutex_; +namespace +{ +#ifdef ELOQ_MODULE_ENABLED +/** + * @brief Starts brpc's bthread workers if none runs yet, by running one + * bthread to completion. + * + * With ELOQ_MODULE_ENABLED, EloqStore's shards run on the bthread workers as + * a brpc module, so a request to EloqStore completes only once workers exist. + * Attaching a data shard in cloud mode is such a request, made while the + * storage handler starts. A serving node has started the workers by then, its + * brpc servers and clients having run bthreads; a bootstrap may have started + * none, and would wait on the attach forever. + * @return true if the workers run. + */ +bool EnsureBthreadWorkers() +{ + bthread_t tid; + if (bthread_start_background( + &tid, nullptr, [](void *) -> void * { return nullptr; }, nullptr) != + 0) + { + return false; + } + return bthread_join(tid, nullptr) == 0; +} +#endif +} // namespace + DataSubstrate &DataSubstrate::Instance() { static DataSubstrate instance_; @@ -180,6 +213,25 @@ bool DataSubstrate::Start() LOG(ERROR) << "Failed to initialize metrics"; return false; } +#if (WITH_LOG_SERVICE) + // The log service started in phase 1, before the registry existed. + if (metrics::enable_metrics && metrics::enable_log_service_metrics && + instance.log_server_ != nullptr) + { + instance.log_server_->EnableMetrics(instance.metrics_registry_.get(), + instance.tx_service_common_labels_); + } +#endif + +#ifdef ELOQ_MODULE_ENABLED + // The storage handler's first EloqStore requests need running bthread + // workers (EnsureBthreadWorkers). + if (!EnsureBthreadWorkers()) + { + LOG(ERROR) << "Failed to start the bthread workers"; + return false; + } +#endif // Phase 3: Initialize storage handler if (instance.GetCoreConfig().enable_data_store && @@ -817,6 +869,18 @@ bool DataSubstrate::LoadCoreAndNetworkConfig(const INIReader &config_reader) : config_reader.GetBoolean( "local", "enable_mvcc", FLAGS_enable_mvcc); + // The shard cleaner's release threshold is a tx service gflag that the + // cleaner reads directly; the config file sets it unless the command line + // already did. + const char *field_release = "shard_clean_release_kb"; + if (CheckCommandLineFlagIsDefault(field_release) && + config_reader.HasValue("local", field_release)) + { + GFLAGS_NAMESPACE::SetCommandLineOption( + field_release, + config_reader.Get("local", field_release, "").c_str()); + } + // Load network configuration if (!LoadNetworkConfig(core_config_.bootstrap, config_reader, diff --git a/core/src/log_init.cpp b/core/src/log_init.cpp index 161982764..9d7747d88 100644 --- a/core/src/log_init.cpp +++ b/core/src/log_init.cpp @@ -115,6 +115,13 @@ DEFINE_string(notify_checkpointer_threshold_size, "When the replay log size reaches this threshold the txlog " "server sends a checkpoint request to tx_service."); +DEFINE_uint64(txlog_cdc_retention_seconds, + 0, + "How far behind the checkpoint the log state keeps the data log " + "for change data capture: no purge removes a record within this " + "window. Set it above the worst-case CDC consumer lag. 0 keeps " + "no window, so every checkpointed record may be purged."); + bool DataSubstrate::InitializeLogService(const INIReader &config_reader) { std::string txlog_service = @@ -284,6 +291,14 @@ bool DataSubstrate::InitializeLogService(const INIReader &config_reader) "notify_checkpointer_threshold_size", FLAGS_notify_checkpointer_threshold_size)); + [[maybe_unused]] uint64_t txlog_cdc_retention_seconds = + !CheckCommandLineFlagIsDefault("txlog_cdc_retention_seconds") + ? FLAGS_txlog_cdc_retention_seconds + : config_reader.GetInteger( + "local", + "txlog_cdc_retention_seconds", + FLAGS_txlog_cdc_retention_seconds); + #if defined(LOG_STATE_TYPE_RKDB_ALL) std::string txlog_rocksdb_storage_path = !CheckCommandLineFlagIsDefault("txlog_rocksdb_storage_path") @@ -534,7 +549,8 @@ bool DataSubstrate::InitializeLogService(const INIReader &config_reader) logserver_snapshot_interval, enable_txlog_request_checkpoint, check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size_val); + notify_checkpointer_threshold_size_val, + txlog_cdc_retention_seconds); } #else size_t txlog_rocksdb_sst_files_size_limit_val = @@ -585,7 +601,8 @@ bool DataSubstrate::InitializeLogService(const INIReader &config_reader) logserver_snapshot_interval, enable_txlog_request_checkpoint, check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size_val); + notify_checkpointer_threshold_size_val, + txlog_cdc_retention_seconds); } #endif #endif diff --git a/core/src/tx_service_init.cpp b/core/src/tx_service_init.cpp index 7e2bdc653..bd7a6da5c 100644 --- a/core/src/tx_service_init.cpp +++ b/core/src/tx_service_init.cpp @@ -7,7 +7,18 @@ #include "eloq_log_wrapper.h" #include "sequences/sequences.h" #include "tx_service.h" -DEFINE_int32(checkpointer_interval, 10, "Checkpointer interval in seconds"); +DEFINE_int32(checkpointer_interval, + 10, + "Checkpointer interval in seconds; values below 5 are raised to " + "5, negative values are rejected"); + +/** + * @brief The shortest checkpointer interval, in seconds. + * + * A shorter configured interval is raised to this. A zero interval would run + * checkpoint rounds back to back. + */ +static constexpr int64_t kMinCheckpointerIntervalSec = 5; DEFINE_int32(checkpointer_min_ckpt_request_interval, 5, "Minimum checkpoint request interval in seconds, to avoid too " @@ -42,6 +53,11 @@ DEFINE_string(hm_bin_path, DEFINE_uint32(deadlock_check_interval_seconds, 10, "Deadlock check interval in seconds"); +DEFINE_int32(ddl_kv_retry_interval_seconds, + 30, + "Pause in seconds before a schema op runs a failed data store " + "step again after its point of no return; 0 is raised to 1, " + "negative values are rejected"); DEFINE_bool(realtime_sampling, true, "Whether enable realtime sampling. If disable it, user may need " @@ -65,12 +81,28 @@ DEFINE_uint64(dirty_memory_size_threshold_mb, bool DataSubstrate::InitializeTxService(const INIReader &config_reader) { - uint64_t checkpointer_interval = + // Read signed: a negative value stored as unsigned becomes an interval of + // about 136 years, which silently stops the periodic checkpoints. + int64_t checkpointer_interval = !CheckCommandLineFlagIsDefault("checkpointer_interval") ? FLAGS_checkpointer_interval : config_reader.GetInteger("local", "checkpointer_interval", FLAGS_checkpointer_interval); + if (checkpointer_interval < 0) + { + LOG(ERROR) << "Invalid `checkpointer_interval` " + << checkpointer_interval << ": it must not be negative"; + return false; + } + if (checkpointer_interval < kMinCheckpointerIntervalSec) + { + LOG(WARNING) << "`checkpointer_interval` " << checkpointer_interval + << " is below the minimum of " + << kMinCheckpointerIntervalSec << " seconds; using " + << kMinCheckpointerIntervalSec; + checkpointer_interval = kMinCheckpointerIntervalSec; + } uint64_t checkpointer_delay_seconds = !CheckCommandLineFlagIsDefault("checkpointer_delay_seconds") @@ -144,6 +176,31 @@ bool DataSubstrate::InitializeTxService(const INIReader &config_reader) FLAGS_deadlock_check_interval_seconds); txservice::DeadLockCheck::SetTimeInterval(deadlock_check_interval_seconds); + // Read signed, as checkpointer_interval is: a negative value stored as + // unsigned would stall a retrying schema op for decades. + int64_t ddl_kv_retry_interval_seconds = + !CheckCommandLineFlagIsDefault("ddl_kv_retry_interval_seconds") + ? FLAGS_ddl_kv_retry_interval_seconds + : config_reader.GetInteger("local", + "ddl_kv_retry_interval_seconds", + FLAGS_ddl_kv_retry_interval_seconds); + if (ddl_kv_retry_interval_seconds < 0) + { + LOG(ERROR) << "Invalid `ddl_kv_retry_interval_seconds` " + << ddl_kv_retry_interval_seconds + << ": it must not be negative"; + return false; + } + if (ddl_kv_retry_interval_seconds == 0) + { + // Retries need a pause between them. + LOG(WARNING) << "`ddl_kv_retry_interval_seconds` 0 is below the " + "minimum of 1 second; using 1"; + ddl_kv_retry_interval_seconds = 1; + } + txservice::SchemaOp::SetKvRetryInterval( + std::chrono::seconds(ddl_kv_retry_interval_seconds)); + bool realtime_sampling = !CheckCommandLineFlagIsDefault("realtime_sampling") ? FLAGS_realtime_sampling diff --git a/docs/01-architecture-overview.md b/docs/01-architecture-overview.md index 6b75c355c..ea428ab18 100644 --- a/docs/01-architecture-overview.md +++ b/docs/01-architecture-overview.md @@ -18,7 +18,7 @@ The proprietary host manager participates through RPC-visible topology and leade ## Lifecycle and data flow -`DataSubstrate::Init` loads process and cluster configuration. API layers then call `RegisterEngine` with their catalog factory and prebuilt tables. `Start` initializes log, metrics and storage dependencies before constructing and starting the transaction service. Shutdown reverses ownership so transaction work cannot outlive its persistence dependencies. +`DataSubstrate::Init` loads process and cluster configuration. API layers then call `RegisterEngine` with their catalog factory and prebuilt tables. `Start` initializes log, metrics and storage dependencies before constructing and starting the transaction service; an in-process log server starts before the metrics registry exists and is handed it afterwards (`LogServer::EnableMetrics`). Shutdown reverses ownership so transaction work cannot outlive its persistence dependencies. A foreground request follows this path: diff --git a/docs/03-concurrency-control.md b/docs/03-concurrency-control.md index 933dc8a07..02c380b4e 100644 --- a/docs/03-concurrency-control.md +++ b/docs/03-concurrency-control.md @@ -95,13 +95,29 @@ entry before the read resumes. Snapshot reads may select an archived version, with durable version retrieval delegated to the store boundary. Writes are buffered by the transaction until commit. Commit-time acquisition -takes the required write locks and returns version/timestamp bounds. +takes the required write locks and returns version/timestamp bounds: each grant +reports the granting shard's clock above the reads it has served +(`last_vali_ts`), and the commit timestamp is chosen above every bound. A lock +taken on every core of every NG (`AcquireAllCc`, used by schema, range, bucket +and cluster gate operations) reports the later of the shard's last read +timestamp and its clock, so a schema operation's timestamps lie above its +catalog write lock's grant. A catalog reader publishes its commit timestamp on +release, including EloqKV's fast catalog read (`ReleaseCatalogRead`), so a later +schema operation commits above it. Post-read requests validate optimistic reads and release read ownership; post-write requests install the committed value or discard the pending value, then release the write lock and wake waiters. Object-command maps use the same lock lifecycle but apply and commit engine commands against shard-owned objects. +A snapshot-isolation transaction starts at the later of its node's timestamp +base and the wall clock, and a newer wall clock advances the base: commits take +timestamps above their own node's base, so a start taken from another node's +base alone could miss writes that node had acknowledged. Across hosts, a +read-only transaction's start can trail another host's commits by their clock +skew, which is accepted; a write transaction spanning nodes is ordered by the +read timestamps every participant records. + Scanners are batched cursors over memory and, when needed, persistent data. They retain the intents or locks needed to keep continuation points valid and transfer semantic reads to the transaction read set until final validation or @@ -124,6 +140,34 @@ handler, while WAL replay reconstructs it during recovery. Those durability boundaries are described in [07-durability-and-recovery.md](07-durability-and-recovery.md). +A request that finds the shard heap at its limit parks on the shard's wait +list; the park is counted, and the shard's single cleaner is scheduled if it +is idle. A cleaner that is already sweeping or deferred is left alone. A pass +starts at the LRU head and evicts in bounded batches while allocated memory is +above the eviction target (5% of the limit free). It releases parked requests +in batches of 20 whenever the heap is below the hard limit. Once a request has +found the heap at its limit since the pass started or last released, which the +park count reveals whoever parked, the room handed out is used up: the next +release waits until the pass has reclaimed `shard_clean_release_kb` more (a tx +service gflag, default 256 KB, zero meaning any amount, also a `[local]` key of +the data substrate config), so the waiters follow the pace of eviction rather +than a share of the +heap. At the tail no further batch can add to that, so whatever the pass +reclaimed since its last release goes out before the pass ends; an execution +that starts at the tail reclaims nothing and releases nothing more, so the pass +ends. A +pass that reaches the tail with requests still parked aborts only requests +that explicitly allow OOM failure, and only when the heap is full or the pass +freed nothing; asks for a checkpoint when it reclaimed less than +`shard_clean_release_kb`, since eviction is then not keeping up and a flush is +what frees the dirty pages left; and defers. The timer thread retries a +deferred cleaner after 100 ms, doubling the delay up to 1.6 s while passes stay +under that bar; the backoff clears when a pass ends with nothing parked, so +the next episode of pressure starts from 100 ms again. A checkpoint flush or a +committed +range split runs a deferred cleaner at once and marks a running pass to run +once more; nothing rewinds a pass in progress. + ## Deadlock and orphan-lock coordination Each shard maintains a registry from transactions to the entry locks and @@ -131,6 +175,22 @@ intents they hold. The registry is part of the correctness model: checkpointing uses it to derive active-transaction bounds, failover recovery uses it to find orphaned ownership, and deadlock detection uses it to build the wait-for graph. +For each transaction the registry records its latest write-lock grant and +whether it holds a write lock that keeps the checkpoint below that grant. Write +locks on data tables and catalog entries count; range, bucket and +cluster-configuration write locks, intents and read locks do not, since split +and migration operations hold them for long stretches and nothing needs them +ordered against the checkpoint. The decision is made per lock when it is +registered, as one transaction's locks span many tables. A downgrade stops the +lock from counting unless another counting lock of the transaction remains in +the shard; the commit-stage upgrade counts again from the upgrade time. + +DML chooses its commit timestamp after its last write lock, so the latest grant +is also the tightest lower bound on its commit. Schema operations rely on the +exclusive section instead: index operations choose their commit timestamp at +prepare under the write lock, and create, drop and truncate choose it again +under their commit-stage write lock, above every DML that lock drained. + The deadlock service gathers local shard wait relationships and corresponding relationships from other node-group leaders, detects cycles, and asks the owner shard to abort a victim. Victim abort and lock release re-enter the normal @@ -151,6 +211,8 @@ directly. preserves their location and lifetime. - The transaction-to-lock registry must remain synchronized with logical lock ownership; stale registry state can pin checkpoints or misdirect recovery. +- While a transaction holds a write lock on a data table or catalog entry, the + checkpoint stays below its latest write-lock grant. - Eviction cannot remove dirty or transactionally owned entries. ## Source map @@ -167,4 +229,7 @@ directly. | Typed maps own pages, entries, versions, and object-command specializations | `tx_service/include/cc/cc_entry.h`; `tx_service/include/cc/template_cc_map.h`; `tx_service/include/cc/object_cc_map.h` | | Scanners retain CC ownership across batched progress | `tx_service/include/cc/ccm_scanner.h`; `tx_service/include/cc/cc_request.h`; `tx_service/src/tx_execution.cpp` | | Shard-local eviction admits only unowned persistent entries | `tx_service/include/cc/cc_entry.h`; `tx_service/include/cc/cc_page_clean_guard.h`; `tx_service/src/cc/cc_shard.cpp` | +| The lock registry records each transaction's latest write-lock grant and whether it bounds the checkpoint | `tx_service/include/cc/cc_shard.h`; `tx_service/src/cc/cc_shard.cpp`; `tx_service/src/cc/cc_map.cpp`; `tx_service/tests/CkptLockRegistry-Test.cpp` | +| Lock grants report timestamp bounds, and catalog readers publish their commit timestamps | `tx_service/include/cc/template_cc_map.h`; `tx_service/src/cc/local_cc_handler.cpp`; `tx_service/src/tx_upsert_table_operation.cpp` | +| Snapshot transactions start at the later of the timestamp base and the wall clock | `tx_service/src/cc/cc_shard.cpp`; `tx_service/src/cc/local_cc_handler.cpp` | | Lock ownership feeds deadlock detection and recovery bookkeeping | `tx_service/include/cc/cc_shard.h`; `tx_service/include/dead_lock_check.h`; `tx_service/src/dead_lock_check.cpp` | diff --git a/docs/04-transaction-execution.md b/docs/04-transaction-execution.md index 59ee0651b..e9b16d95d 100644 --- a/docs/04-transaction-execution.md +++ b/docs/04-transaction-execution.md @@ -104,6 +104,25 @@ cluster migration are composite operations built on the same stack and result model, but their subsystem-specific protocols live in the corresponding architecture documents rather than here. +A table schema operation (`UpsertTableOp`) takes the cluster configuration +read lock first, which the cluster gate can refuse +([06](06-distribution-and-clustering.md)), then catalog write intents, the +prepare log, the dirty schema, the commit-stage write locks, the commit log, +the store steps, the committed schema and the clean log; an index operation +(`UpsertTableIndexOp`) orders its steps around its build +([08](08-range-and-bucket-management.md)). Each has a point of no return: the +commit log for create, drop and truncate, the prepare log for index operations. +A store failure before it rolls the operation back at once; one after it is +retried until it succeeds, `ddl_kv_retry_interval_seconds` apart, holding the +operation's lock. Create, drop and truncate choose their commit timestamp +again under their commit-stage write lock (`AssignCommitStageTs`), above every +DML that lock drained. A cluster gate operation (`ClusterGateOp`, always on log +group 0) reads the gate record for write, takes write intents on every NG with +optimistic control, then write locks, which wait for admitted schema +operations, logs, and installs; any failure before its log releases everything +and starts over after a short random back-off, and after its log it rolls +forward. + Abort uses the same post-processing ownership map: every successfully acquired read, intent, write lock, catalog lock, and placement lock is released through CC requests. A recovering coordinator may start with an already determined @@ -154,5 +173,7 @@ contract. | The commit pipeline acquires placement and write ownership, sets a timestamp, validates, logs, and post-processes | `tx_service/src/tx_execution.cpp`; `tx_service/include/tx_operation.h`; `tx_service/src/tx_operation.cpp` | | WAL failures distinguish abort from an indeterminate outcome | `tx_service/src/tx_execution.cpp` (`Process`/`PostProcess(WriteToLogOp&)`) | | Object-command recovery images outlive operation completion and are released at terminal reset | `tx_service/include/tx_operation_result.h`; `tx_service/include/tx_execution.h`; `tx_service/src/tx_execution.cpp` | +| Schema operations retry store steps after their point of no return and re-choose their commit timestamp under the commit-stage lock | `tx_service/src/tx_upsert_table_operation.cpp`; `tx_service/src/tx_index_operation.cpp`; `tx_service/include/tx_operation.h` | +| Cluster gate operations drain admitted schema operations and roll forward after their log | `tx_service/src/tx_cluster_gate_operation.cpp`; `tx_service/include/tx_operation.h` | | Lock modes derive from isolation level and CC protocol | `tx_service/include/cc_protocol.h`; `tx_service/src/tx_execution.cpp` | | Transaction and cross-node commit behavior has integration coverage | `tx_service/tests/TxConsistency-Test.cpp`; `tx_service/tests/ClusterCrossNg-Test.cpp`; `tx_service/tests/TestNodeSmoke-Test.cpp` | diff --git a/docs/05-data-model-and-catalog.md b/docs/05-data-model-and-catalog.md index 769abf771..5627fb7d3 100644 --- a/docs/05-data-model-and-catalog.md +++ b/docs/05-data-model-and-catalog.md @@ -87,9 +87,22 @@ entry rather than constructing unrelated schema objects. A schema change has current and dirty states. Preparation installs a newer dirty schema while the current schema remains readable; commit promotes the -dirty schema, and rejection discards it. Version comparisons make replay and -repeated delivery idempotent. DDL uses replicated catalog locks so that schema -visibility and data-path use cannot cross inconsistently. +dirty schema at the operation's commit timestamp, and rejection discards it. An +operation rolled back after its prepare log commits too: it keeps the current +schema, whose version becomes its commit timestamp. Version comparisons make +replay and repeated delivery idempotent. DDL uses replicated catalog locks so +that schema visibility and data-path use cannot cross inconsistently. A catalog +read enters the transaction's read set at the catalog entry's commit timestamp, +the one the last DDL chose under its write lock, so every transaction that read +a schema commits above that DDL. + +The schema record's version, the last DDL's commit timestamp, is distinct from +the base and index tables' own versions: a table's CC map carries its table's +version, requests carry it, and replay skips a log record below it. An engine +builds a DDL's image before the DDL has a timestamp, so the tx service resolves +every table's version in the images once the prepare stage has chosen it and +before anything is logged (`SchemaOp::ResolveSchemaImages`); no logged, stored +or replicated image leaves a table's version unresolved. Data-path requests carry the schema version under which their key/record was interpreted. Typed CC maps reject nonzero versions that do not match the map's @@ -131,6 +144,8 @@ format authority. - Object-command replay preserves base-version and commit-version ordering. - A prepared schema does not become current until the catalog transaction commits; replaying preparation or completion must be idempotent. +- Every image that is logged, stored or replicated records each table's real + version. ## Source map @@ -144,6 +159,7 @@ format authority. | Table identity carries name, role, and engine and selects partitioning behavior | `tx_service/include/type.h`; `tx_service/include/constants.h` | | `CatalogFactory` constructs engine-specific schemas, maps, keys, records, scanners, and statistics | `tx_service/include/catalog_factory.h`; `tx_service/include/eloq_basic_catalog_factory.h`; `tx_service/src/eloq_basic_catalog_factory.cpp` | | `LocalCcShards` owns current/dirty catalog entries shared by shard catalog records | `tx_service/include/catalog_key_record.h`; `tx_service/include/cc/local_cc_shards.h`; `tx_service/src/cc/local_cc_shards.cpp`; `tx_service/include/cc/catalog_cc_map.h` | +| Schema images carry resolved table versions, and a rolled-back operation keeps the current schema at its commit timestamp | `tx_service/include/tx_operation.h`; `tx_service/src/tx_operation.cpp`; `tx_service/include/catalog_key_record.h`; `tx_service/include/cc/catalog_cc_map.h` | | Typed maps reject requests interpreted under a different schema version | `tx_service/include/cc/cc_request.h`; `tx_service/include/cc/template_cc_map.h`; `tx_service/src/tx_execution.cpp` | | Sequences use an internal transactional table | `tx_service/include/sequences/sequences.h`; `tx_service/src/sequences/sequences.cpp` | | Statistics are schema-bound, assigned across cores/node groups, exchanged, and persisted | `tx_service/include/statistics.h`; `tx_service/include/table_statistics.h`; `tx_service/src/cc/local_cc_shards.cpp` | diff --git a/docs/06-distribution-and-clustering.md b/docs/06-distribution-and-clustering.md index 5e08169c7..38fc7edfc 100644 --- a/docs/06-distribution-and-clustering.md +++ b/docs/06-distribution-and-clustering.md @@ -61,10 +61,27 @@ generation before accepting a response. Leader-location caches are advisory; senders must handle a stale destination, refresh leadership, and retry through the owning operation's retry policy. +Every leader update carries its term, and the term alone orders them. A node +learns a leader from the host manager's `NotifyNewLeaderStart`, from a log +write's reply, which carries the log service's view of each NG's leader and +term, and from a lookup after a "not leader" error or at start-up. The lookup +asks the host manager, whose answer names the node only, then asks that node +for its term (`FetchNodeInfo`); a node that does not lead the NG answers none, +and the lookup changes nothing. The cache applies an update only above its +term, the leader and the term together. Every update it applies goes to the +store handler (`DataStoreHandler::OnLeaderChanged`), also when the leader is +the same node, so a store that binds data shards to NGs keeps each shard's +version at the leader's term and refuses an older move +([09](09-store-handler.md)). + ## Leadership lifecycle and term fencing Election does not immediately make a node serviceable. `OnLeaderStart` first -places the NG in candidate-leader state and starts recovery. The leader term is +places the NG in candidate-leader state and starts recovery, ending with the +NG's only replay request to the log groups +([07](07-durability-and-recovery.md)). Once it succeeds, the host manager sends +every node the NG, its new leader and the term (`NotifyNewLeaderStart`, best +effort). The leader term is published only after every log group has completed replay, so normal CC requests cannot observe a partially recovered in-memory replica. A synchronized standby may retain its warm state and replay only the missing tail; a cold @@ -98,6 +115,24 @@ Adding election peers without changing bucket ownership does not require data migration. Changes that do move ownership must not expose the new owner before the synchronized data is durable and routable. +## Cluster gate + +The cluster configuration record carries an optional cluster gate, which +EloqDoc's change data capture pauses while a consumer copies the data. Schema +operations hold the cluster configuration read lock from their first step +through their last write to all NGs, so a scale waits for them and every such +step reaches the same NGs. While the gate is paused, a schema operation is +refused at that first step with `CLUSTER_GATED`, before writing any log; one +past it, or one resumed by recovery, is never refused. A pause or resume is a +transaction on log group 0 that takes the record's write lock on every NG, +which is granted only once every admitted schema operation has released its +read lock, so the pause's commit timestamp bounds every schema operation +committed before it. Its update sets only the gate in place and never changes +`Sharder`'s topology, and a retried update whose transaction no longer holds +the lock stores nothing. Replay reinstalls the gate on every recovering NG +([07](07-durability-and-recovery.md)), and with metrics on each NG exports +`cluster_gate_paused`. + ## Standby replication boundary Candidate data-bearing members subscribe to the leader's per-shard committed @@ -117,6 +152,9 @@ candidate becomes a complete in-memory replica. updates at an older version are idempotent no-ops or are rejected. - Remote responses are valid only for the transaction generation and leader term that issued the request. +- A leader update is applied only above the cached term for its NG. +- While the cluster gate is paused no new schema operation is admitted, and a + pause commits only after every admitted one has released its read lock. - Every successful node-group data pin must be released; otherwise leader stepdown cannot safely reclaim the NG's state. - The host manager implementation is outside this documentation boundary. Its @@ -133,5 +171,7 @@ candidate becomes a complete in-memory replica. | Local and remote requests converge on the CC-request/shard execution model | `tx_service/include/remote/remote_cc_handler.h`; `tx_service/include/remote/remote_cc_request.h`; `tx_service/src/remote/cc_stream_sender.cpp`; `tx_service/src/remote/cc_stream_receiver.cpp` | | Cluster configuration and bucket ownership are transactional metadata | `tx_service/include/cc/cluster_config_cc_map.h`; `tx_service/include/cluster_config_record.h`; `tx_service/include/cc/range_bucket_cc_map.h`; `tx_service/src/cc/local_cc_shards.cpp` | | Scaling and data migration are resumable transaction operations | `tx_service/include/tx_operation.h`; `tx_service/src/tx_operation.cpp`; `tx_service/tx-log-protos/log.proto` | +| Leader updates carry terms; lookups ask the named node for its term | `tx_service/src/sharder.cpp`; `tx_service/src/remote/cc_node_service.cpp`; `tx_service/src/tx_operation.cpp` | +| The cluster gate admits schema operations at their first step and is paused, resumed and recovered by its own transaction | `tx_service/include/cluster_config_record.h`; `tx_service/include/cc/cluster_config_cc_map.h`; `tx_service/src/tx_cluster_gate_operation.cpp`; `tx_service/src/tx_upsert_table_operation.cpp`; `tx_service/src/tx_index_operation.cpp`; `tx_service/tests/ClusterGate-Test.cpp` | | Cross-NG request routing is exercised by the cluster harness | `tx_service/tests/ClusterCrossNg-Test.cpp`; `tx_service/tests/cluster/` | | Standby replication has a distinct sequencing and snapshot protocol | `tx_service/include/standby.h`; `tx_service/src/standby.cpp`; `tx_service/tests/StandbyForward-Test.cpp` | diff --git a/docs/07-durability-and-recovery.md b/docs/07-durability-and-recovery.md index e3e31a7be..e2d35e210 100644 --- a/docs/07-durability-and-recovery.md +++ b/docs/07-durability-and-recovery.md @@ -23,7 +23,14 @@ Ordinary data transactions write their effects to one selected log group. Schema changes, range splits, cluster scaling, and bucket migration use staged records because their effects span metadata and data lifecycles. Those stages are recovery state, not a second execution history: replay uses them to decide -which incomplete operation must be resumed or completed. +which incomplete operation must be resumed or completed. The cluster gate, +which pauses schema operations while a change data capture consumer copies the +data, writes one unstaged record per change, kept in log group 0's slot. + +A transaction's record holds a blob per NG it writes. A key double-written +while its range splits or its bucket migrates goes into the blob of every NG it +is written to, once per NG: until that NG checkpoints the key, its blob is all +that NG's recovery has of it. An uncertain log result is not treated as an abort. The coordinator preserves write locks until the result can be established from the log service, because @@ -41,9 +48,16 @@ store copy. ## Checkpointing -`Checkpointer` runs per node and advances each locally led NG independently. -It first computes a safe timestamp that does not pass active transactions or -outstanding write locks. Data-sync tasks then scan dirty entries on their owner +`Checkpointer` runs per node and advances each locally led NG independently, +every `checkpointer_interval` (at least 5 s) or on request. It first computes a +safe timestamp that does not pass active transactions or outstanding write +locks: below each transaction's latest write-lock grant on a data table or +catalog entry (range, bucket and cluster-config write locks, intents and read +locks do not count). A DML transaction picks its commit timestamp after its +last write lock, and a schema operation commits above its catalog write lock, +so no transaction commits at or below a checkpoint timestamp once it is taken. +That is what makes the timestamp an NG reports to the log service a watermark +for change data capture. Data-sync tasks then scan dirty entries on their owner shards, materialize base rows and archives, and flush them through `DataStoreHandler`. @@ -92,10 +106,18 @@ the warm cache and start replay after its last consistent timestamp; otherwise it reconstructs state from the durable store plus the complete applicable log history. +The node asks each log group for replay once, at its leader start, with one +bounded attempt, and asks again only a group that has neither finished the +term's replay nor opened a replay stream a few seconds after the last request +(`RecoveryService`'s watchdog); a stream that stalls is closed by its idle +timeout, which asks that group again. A group asked twice ships twice, the +second shipping replacing the first. + Replay orders dependencies before ordinary data. Cluster and bucket metadata, -catalog operations, and range-split state are restored before data records are -allowed to complete. Barriers across log-group streams prevent a data record -from being interpreted against an obsolete schema or placement model. +the cluster gate, catalog operations, and range-split state are restored before +data records are allowed to complete. Barriers across log-group streams prevent +a data record from being interpreted against an obsolete schema or placement +model. Each replay request is fenced by the candidate term. Data replay is idempotent: records already represented by an equal or newer committed version are skipped, @@ -103,6 +125,19 @@ and records outside the recovered NG's current ownership are ignored. In-flight schema, split, scale, and migration operations are resumed from their durable stage through recovery transaction requests. +A recovered schema operation's coordinator NG restores a write intent on the +catalog entry, so data replay of the table proceeds, and the operation resumes +after the log record it was recovered from, at the timestamp that record +carries. A participant restores the lock the coordinator may rely on: after a +prepare-stage record the write lock, unless a range split it coordinates holds +the table's catalog read lock, and for an index build only after its last +prepare-data record; after a commit-stage record the committed schema and no +lock ([range-split-schema-op-recovery.md](range-split-schema-op-recovery.md)). A +node whose catalog entry is write-locked this way finishes start-up only once +the operation commits. The recovered cluster gate is set in place, and the +gate's coordinator re-sends its install with no lock, releasing the write lock +wherever the gate transaction still holds it. + The node publishes its serving leader term only after every log group reports replay complete. A broken replay stream is retried; partial progress never authorizes normal traffic. @@ -115,7 +150,10 @@ that is inconclusive, `RecoveryService` asks the log service whether the transaction is committed under the recorded coordinator and participant terms. An uncommitted transaction can be cleared. A committed transaction is replayed -onto entries that still retain its write lock and then released. An unreachable +onto entries that still retain its write lock and then released. A transaction +found as a multi-stage operation or as the cluster gate transaction is also +committed, but nothing is shipped: its metadata locks stay until its +coordinator's recovery releases them. An unreachable log service leaves the lock in place for a later retry. This favors correctness over availability when the commit decision is unknown. @@ -123,14 +161,21 @@ over availability when the commit decision is unknown. When MVCC is enabled, an update archives the prior committed version before installing the new one. Checkpoint data sync persists both base values and -historical versions. Snapshot reads first consult the in-memory version chain -and fetch the necessary base or archive version from the store when the chain no -longer covers the requested timestamp. - -The oldest active snapshot timestamp bounds archive reclamation and delays -checkpoint choices that would otherwise discard a still-visible version. -Consequently, active-transaction timestamp collection is part of the durability -and garbage-collection model rather than only an observability feature. +historical versions: per entry, the newest version at or below the round's +checkpoint timestamp goes to the base table and older ones to the archives +table. Snapshot reads first consult the in-memory version chain and fetch the +necessary base or archive version from the store when the chain no longer +covers the requested timestamp. A snapshot transaction starts at the later of +the shard's timestamp base and its wall clock, and advances the base. + +The oldest active snapshot transaction's start alone bounds the reclamation of +in-memory versions (`CcEntry::KickOutArchiveRecords`). The checkpoint delay +(`checkpointer_delay_seconds`, 5 s by default) is a cache residency choice: +while a snapshot transaction that started earlier runs, it lowers a round's +checkpoint timestamp by at most the delay, so recently updated entries stay +dirty and therefore resident, and a snapshot read finds their versions in +memory. Active-transaction timestamp collection is therefore part of the +garbage-collection model rather than only an observability feature. ## Snapshots and standby bootstrap @@ -154,7 +199,7 @@ accepted-request event. ## Cross-cutting invariants - A checkpoint timestamp never passes an active write whose outcome is not - represented durably. + represented durably, and no transaction commits at or below it afterwards. - Log truncation advances only after every included store write and required persistence barrier succeeds. - A candidate term fences replay; a serving leader term is published only after @@ -181,4 +226,8 @@ accepted-request event. | Replay restores metadata dependencies and resumes incomplete durable operations | `tx_service/include/cc/catalog_cc_map.h`; `tx_service/include/cc/range_cc_map.h`; `tx_service/include/cc/cluster_config_cc_map.h`; `tx_service/include/cc/range_bucket_cc_map.h` | | MVCC versions are archived, fetched, and reclaimed against active snapshots | `tx_service/include/cc/cc_entry.h`; `tx_service/include/tx_start_ts_collector.h`; `tx_service/include/store/data_store_handler.h` | | Standby and backup snapshots are coordinated above the storage interface | `tx_service/include/store/snapshot_manager.h`; `tx_service/src/store/snapshot_manager.cpp`; `tx_service/include/store/data_store_handler.h` | +| The checkpoint counts each transaction's latest data or catalog write-lock grant | `tx_service/include/cc/cc_shard.h`; `tx_service/src/cc/cc_shard.cpp`; `tx_service/tests/CkptLockRegistry-Test.cpp` | +| Forwarded keys are logged to every NG they are written to | `tx_service/src/tx_execution.cpp` | +| Replay is requested once per log group and again only for a silent group | `tx_service/src/sharder.cpp`; `tx_service/src/fault/log_replay_service.cpp`; `tx_service/tx-log-protos/log_agent.cpp` | +| Recovered schema operations restore intents or write locks by role, and the gate is reinstalled | `tx_service/include/cc/catalog_cc_map.h`; `tx_service/include/cc/cluster_config_cc_map.h`; `tx_service/src/tx_execution.cpp`; `docs/range-split-schema-op-recovery.md` | | Checkpoint publication, split ownership, state and MVCC behavior have focused unit coverage | `tx_service/tests/CheckpointFlush-Test.cpp`; `tx_service/tests/RangeSplitCheckpoint-Test.cpp`; `tx_service/tests/CheckpointMetricsState-Test.cpp`; `tx_service/tests/CcEntry-Test.cpp`; `tx_service/tests/StartTsCollector-Test.cpp` | diff --git a/docs/08-range-and-bucket-management.md b/docs/08-range-and-bucket-management.md index 40d30f599..1c7dc821b 100644 --- a/docs/08-range-and-bucket-management.md +++ b/docs/08-range-and-bucket-management.md @@ -62,7 +62,9 @@ A range split is a logged metadata-and-data transaction. At a high level it: 5. removes data that is no longer visible through the old range. The publish point follows durable data synchronization, so readers cannot be -routed to a range whose base data is incomplete. WAL records preserve enough +routed to a range whose base data is incomplete. A split that fails before its +commit log has installed nothing: it rolls back at commit timestamp 0 and only +releases its locks. WAL records preserve enough stage and range metadata for recovery to rebuild pending definitions and resume the operation after failover. Cleanup is idempotent because the committed range layout, not the former process state, determines which keys remain visible. @@ -97,6 +99,22 @@ upload capacity, or sleeping for retry. Encoded index entries and the resume key therefore require independent ownership; releasing the source batch must not invalidate them. +An index build is a schema operation whose prepare log is its point of no +return. Under the prepare-stage write lock it creates each new index's kv table +(`DataStoreHandler::CreateKvTable`, one attempt) and writes the prepare log; a +failure of either drops the tables and fails the operation with no log. Recovery +returns an NG to the state right after the prepare log, where the dirty schema +is installed and DML writes the new index, so the kv table must exist in every +state recovery can reach. The installed dirty schema puts the prepare +timestamp on the catalog entry, so every DML that sees it commits above that +timestamp and its index writes survive replay. Data-store steps after the +prepare log are retried until they succeed, `ddl_kv_retry_interval_seconds` +apart. The kv catalog receives the new schema once, after the commit log and +under the commit-stage write lock, so no later schema operation's catalog +write can be overwritten by it. A build that finds a constraint violation +commits as rolled back at its commit-stage timestamp, drops the index's kv +tables and keeps the current schema. + Index expression details and batch encoding belong to the index implementation, not to the range architecture. @@ -112,6 +130,8 @@ not to the range architecture. synchronization succeeds. - Secondary-index generation releases consumed scan batches on their source shard before reset, backpressure, or retry. +- A new index's kv table exists in every state that recovery of its build can + reach. - Split and migration replay is versioned and idempotent; delayed work from an older ownership version must not become visible. @@ -125,4 +145,5 @@ not to the range architecture. | `StoreRange` and `StoreSlice` own cached-range, pin, load, and slice-spec lifecycles | `tx_service/include/cc/range_slice.h`; `tx_service/src/cc/range_slice.cpp`; `tx_service/include/cc/range_slice_type.h` | | Slice loading and range checkpointing cross the store-handler boundary | `tx_service/include/store/data_store_handler.h`; `tx_service/include/data_sync_task.h`; `tx_service/src/cc/local_cc_shards.cpp` | | Range split and bucket migration are staged, recoverable transaction operations | `tx_service/include/tx_operation.h`; `tx_service/src/tx_operation.cpp`; `tx_service/tx-log-protos/log.proto` | +| An index build creates its kv tables before its prepare log and writes the kv catalog once, after its commit log | `tx_service/include/tx_index_operation.h`; `tx_service/src/tx_index_operation.cpp`; `tx_service/include/store/data_store_handler.h`; `tx_service/include/cc/template_cc_map.h`; `tx_service/include/cc/catalog_cc_map.h` | | Secondary-index build dispatches range work, releases consumed source batches, and uploads derived entries by ownership | `tx_service/include/tx_index_operation.h`; `tx_service/src/tx_index_operation.cpp`; `tx_service/include/sk_generator.h`; `tx_service/src/sk_generator.cpp`; `tx_service/include/cc/local_cc_shards.h` | diff --git a/docs/09-store-handler.md b/docs/09-store-handler.md index f6940c014..815d3b738 100644 --- a/docs/09-store-handler.md +++ b/docs/09-store-handler.md @@ -17,7 +17,9 @@ The interface groups several durable responsibilities: - checkpoint writes and an optional explicit persistence barrier; - point reads, bucket reads, and ordered slice scans used on cache misses; -- catalog, database, range, and statistics metadata; +- catalog, database, range, and statistics metadata, including a table's kv + table without its catalog row (`CreateKvTable`, the counterpart of + `DropKvTable`, which an index build calls before its prepare log); - MVCC archive writes and historical-version reads; - leader/standby lifecycle hooks and storage snapshots; and - backend capability queries used to select shared-storage, copy, and cache @@ -45,7 +47,18 @@ Startup selects one `DataStoreHandler` implementation at build time: The EloqStore engine is a separate submodule with its own architecture documentation. At this layer, only its `DataStore` adapter and the -`DataStoreHandler`/EloqDSS integration contract are authoritative. +`DataStoreHandler`/EloqDSS integration contract are authoritative. A node runs +one EloqStore, owned by the factory and started with the first data store, and +that store hosts every data shard the node leads: opening a data shard +attaches it at its term (`EloqStore::AttachShard`) and closing it detaches it, +while the other shards keep serving, so the node has one buffer pool, fd +budget and cache quota however many node groups it leads. RocksDB backends +give each data shard a database of its own, but EloqStore keeps every shard's +partitions side by side, so each request names its data shard and the +storage-wide sweeps (cloud prewarm, archives for backups and standby +snapshots, branches, a global reopen) consult a filter for the partitions of +that shard's buckets, set before every attach (`SetPartitionBuckets`); +ordinary reads and writes do not. EloqDSS can run as a separate brpc service or be colocated with the tx service. The client resolves a KV partition to a DSS shard and current owner. A colocated @@ -62,9 +75,27 @@ ownership of the closure until the operation reaches terminal completion. `DataStoreService` gates requests by per-shard ownership and lifecycle state. Reads require an open readable shard; writes require the current writable -owner. When a client reaches a former owner, the response carries newer -sharding information so the client can refresh its versioned routing view and -retry within the operation's bounded retry policy. +owner. A request that finds its shard `Starting`, opening or closing as its +node's leadership changes, waits on the shard's waiting list and runs again +once the shard leaves `Starting`. When a client reaches a former owner, the +response carries newer sharding information so the client can refresh its +versioned routing view and retry within the operation's bounded retry policy. + +After a failed RPC the client sends the request again at once only if that can +reach the shard: the shard is now local, or the channel to its current owner is +healthy (`CanRetryShardNow`). brpc keeps a failed socket down until its health +check reconnects, so an immediate retry to the same address would only fail +again; the operation fails instead and its caller's retry policy applies. + +With data shards bound to NGs (EloqDSS colocated with the tx service, shard *g* +for NG *g*), the node hosting a shard is the NG's leader. Every node reaches +other NGs' shards directly for shared metadata and checkpoint flushes, so each +node points a shard at its NG's leader whenever the tx service learns a newer +leader term (`DataStoreHandler::OnLeaderChanged`), even for the same node. The +shard's version in this mode is that term: every node derives the same number +for the same leadership, so a redirect's version compares with this node's. A +move whose term is below the shard's version changes nothing, checked against +the routing slot it replaces, so moves and redirects may arrive in any order. For shared-storage backends, DSS shard migration transfers serving ownership rather than copying the underlying dataset: the source stops accepting writes, @@ -134,6 +165,8 @@ term writable again. does not reject reads. - Local EloqDSS bypass and remote RPC implement the same request semantics. - Shard ownership and topology versions fence writes to a moved DSS shard. +- With data shards bound to NGs, a shard's routing version never falls below + the newest leader term its node has learned. - The partitioning-kind flag is part of routing correctness, not a storage optimization hint. @@ -146,6 +179,9 @@ term writable again. | EloqDSS client routing supports local service bypass and remote RPC completion | `store_handler/data_store_service_client.h`; `store_handler/data_store_service_client.cpp`; `store_handler/data_store_service_client_closure.h`; `store_handler/data_store_service_client_closure.cpp` | | Read closures retain values through callbacks and release unconsumed buffers before pool reuse | `store_handler/data_store_service_client_closure.h`; `store_handler/eloq_data_store_service/object_pool.h`; `tx_service/tests/ReadClosureLifetime-Test.cpp` | | EloqDSS owns versioned data shards behind a pluggable `DataStore` interface | `store_handler/eloq_data_store_service/data_store_service.h`; `store_handler/eloq_data_store_service/data_store_service.cpp`; `store_handler/eloq_data_store_service/data_store.h`; `store_handler/eloq_data_store_service/ds_request.proto` | +| Requests for a `Starting` shard wait on its waiting list | `store_handler/eloq_data_store_service/data_store_service.h`; `store_handler/eloq_data_store_service/data_store_service.cpp` | +| The client retries at once only when the shard is reachable, and binds data shards to NG leaders by term | `store_handler/data_store_service_client.h`; `store_handler/data_store_service_client.cpp`; `store_handler/data_store_service_client_closure.h`; `store_handler/eloq_data_store_service/data_store_service_config.cpp` | +| One EloqStore hosts every data shard the node leads, and sweeps only the partitions of the shard a request names | `store_handler/eloq_data_store_service/eloq_store_data_store_factory.h`; `store_handler/eloq_data_store_service/eloq_store_data_store.h`; `store_handler/eloq_data_store_service/eloq_store_data_store.cpp`; `store_handler/eloq_data_store_service/data_store.h` | | Embedded RocksDB implements the same handler contract without DSS transport | `store_handler/rocksdb_handler.h`; `store_handler/rocksdb_handler.cpp` | | Checkpoint data sync depends on `PutAll` and the optional persistence barrier | `tx_service/src/cc/local_cc_shards.cpp`; `tx_service/include/data_sync_task.h`; `tx_service/src/data_sync_task.cpp`; `tx_service/src/checkpointer.cpp` | | Cache-miss requests are coalesced, retained across retry, and safely recycled or destroyed | `tx_service/include/cc/cc_req_misc.h`; `tx_service/src/cc/cc_req_misc.cpp`; `tx_service/include/cc/cc_shard.h`; `tx_service/src/cc/cc_shard.cpp` | diff --git a/docs/10-log-service.md b/docs/10-log-service.md index e3c583c4d..1bcd24b63 100644 --- a/docs/10-log-service.md +++ b/docs/10-log-service.md @@ -57,9 +57,14 @@ leader from making a transaction durable after its authority has ended. Data records contain the serialized transaction effects needed by tx-side replay. Multi-stage catalog, range-split, cluster-scale, and migration records retain the durable operation state needed to resolve work that was interrupted -between metadata and data publication. The exact protobuf fields are a wire -contract and remain defined in `log.proto`; architecture depends on their -semantic categories, not a duplicated field inventory. +between metadata and data publication. A cluster gate record (the gate that +pauses schema operations, [06](06-distribution-and-clustering.md)) carries the +terms of the NGs that granted its lock, so the log service refuses it if one of +them failed over in between. Every gate transaction writes to log group 0, whose +one cluster gate slot keeps the newest gate and its version; the slot is never +purged, because a pause lasts until a resume replaces it. The exact protobuf +fields are a wire contract and remain defined in `log.proto`; architecture +depends on their semantic categories, not a duplicated field inventory. ## Checkpoint and retention @@ -78,13 +83,29 @@ Log-service pressure may request an earlier tx-side checkpoint; it does not independently choose a higher safe truncation timestamp. The checkpointer remains responsible for proving that store persistence completed. +The change data capture (CDC) retention floor keeps data records in the live +store for `txlog_cdc_retention_seconds` behind the checkpoint: every backend's +purge spares a file holding a record at or above `min_ckpt_ts` minus that +window (`LogState::CdcRetentionFloor`), and the DDL history is trimmed at the +same floor. The default, 0, leaves retention exactly as above. The cloud +backend's archive, which nothing reads, is outside the window. + ## Replay and transaction recovery `ReplayLog` records the candidate CC leader and term, selects the requested records from `LogState`, and opens a shipping stream to the tx node's recovery service. Separate shipping work per log group allows a node to receive replay in parallel. The tx side orders metadata before ordinary data and publishes its -leader term only after all groups finish. +leader term only after all groups finish. The replay list carries the cluster +gate slot after cluster-scale and before schema metadata. + +The tx node sends each log group one bounded replay request per attempt +(`LogAgent::ReplayLogOnce`) and asks again only a group that opened no replay +stream, since a lost request, a group without a leader and a lost reply look +the same to it. A request at a term no lower than the NG's current one replaces +the NG's shipping: the earlier agent stops at its next message, closes its +stream without the finish message, and the replacing request waits until it +has stopped. `RecoverTx` resolves locks left on participant NGs when a coordinator disappears. The log service compares the recorded terms and transaction record @@ -104,6 +125,8 @@ work runs on a dedicated pthread. Captured state remains owned until the worker finishes cleanup, before braft completion permits another snapshot or node shutdown to finish. Braft retains responsibility for snapshot finalization. +Raft snapshots also carry the cluster gate slot and the DDL history (below). + An application snapshot is publishable only after its complete file set is written and registered; local RocksDB metadata and directory entries are synced before publication. Failed application attempts leave the previous snapshot @@ -111,6 +134,32 @@ intact and allow the node to keep serving and retry. They use a non-fatal braft status; braft's own storage or publication failures may still disable the replica until restart or recreation. +## Change data capture scan + +`CdcScan` is a read-only, leader-only RPC for EloqDoc change streams. It +returns a page of the log group's committed data records and DDL records +merged by commit timestamp, data first within one timestamp, only those at or +below the group's arrival-complete timestamp: the lowest checkpoint timestamp +the CC NGs have reported (`min_ckpt_ts`), which the reply also carries. No +transaction commits at or below that timestamp later, so no later page holds a +record below it. A `continuation` resumes both streams exactly after the page's +last record, and a page is cut near `max_bytes` but is never empty while a +record remains. A follower answers `NotLeader` with the leader's address, and a +cloud-backend leader whose database is not yet in sync answers with its own. + +The DDL history holds one record per EloqDoc schema operation, positioned at +its prepare-log timestamp for index operations and at its commit-log timestamp +otherwise, plus one at the commit-log timestamp for an index build rolled back +after its prepare log; it lives in the replicated log state and its Raft +snapshot. An idle +scan with a wait parks on the leader until the arrival-complete timestamp +advances, its deadline (capped by `cdc_scan_max_park_ms`), the caller's +cancellation, or the leader's stop, whichever comes first; past +`cdc_scan_max_parked` parked requests a scan answers at once (`CdcLongPoll`). + +With metrics enabled, every log group exports `cdc_arrival_complete_ts` on each +replica and `cdc_scan_parked_requests` on the leader, labelled by `lg_id`. + ## Cross-cutting invariants - Mutating log-service state is ordered by the log group's braft state machine. @@ -121,6 +170,10 @@ replica until restart or recreation. contract. - Log-group leader caches are advisory; clients must refresh and retry after a `NotLeader` response. +- `CdcScan` returns no record above the arrival-complete timestamp, so no + record it has not yet returned can appear at or below a timestamp it has + passed. +- The cluster gate slot is never purged; every recovering NG replays it. ## Source map @@ -133,4 +186,10 @@ replica until restart or recreation. | WAL, replay, checkpoint, recovery, and staged-operation wire contracts are defined in protobuf | `tx_service/tx-log-protos/log.proto` | | Replay records are selected and shipped back to tx recovery services | `eloq_log_service/include/log_shipping_agent.h`; `eloq_log_service/src/log_instance.cpp`; `tx_service/src/fault/log_replay_service.cpp` | | Startup selects in-process or external log-service deployment | `core/src/log_init.cpp`; `build_eloq_log_service.cmake`; `CMakeLists.txt` | +| `CdcScan` merges data and DDL records up to the arrival-complete timestamp, with continuations | `eloq_log_service/include/cdc_scan.h`; `eloq_log_service/src/cdc_scan.cpp`; `eloq_log_service/src/log_instance.cpp`; `eloq_log_service/include/log_service.h` | +| Idle scans park until the checkpoint advances, a deadline, cancellation or leader stop | `eloq_log_service/include/cdc_long_poll.h`; `eloq_log_service/src/cdc_long_poll.cpp` | +| The DDL history, the CDC retention floor and the cluster gate slot live in the log state and its snapshot | `eloq_log_service/include/log_state.h`; `eloq_log_service/src/log_state_rocksdb_impl.cpp`; `eloq_log_service/src/log_state_rocksdb_cloud_impl.cpp` | +| Per-log-group CDC metrics | `eloq_log_service/include/log_service_metrics.h` | +| Bounded replay requests and shipping replacement | `tx_service/tx-log-protos/log_agent.cpp`; `eloq_log_service/include/log_shipping_agent.h`; `tx_service/src/fault/log_replay_service.cpp` | +| CDC scan, long poll, gate slot and schema-op state have focused tests | `eloq_log_service/test/CdcScan-Test.cpp`; `eloq_log_service/test/CdcLongPoll-Test.cpp`; `eloq_log_service/test/LogStateClusterGate-Test.cpp`; `eloq_log_service/test/LogStateSchemaOp-Test.cpp`; `tx_service/tests/LogAgentCdc-Test.cpp` | | Replicated writes, replay and snapshot recovery have focused service tests | `eloq_log_service/test/write_log_test.cpp`; `eloq_log_service/test/async_write_log_test.cpp`; `eloq_log_service/test/rocksdb_test.cpp`; `eloq_log_service/test/log_server_rocksdb_cloud_tests.cpp`; `eloq_log_service/test/log_snapshot_test.cpp`; `eloq_log_service/test/log_snapshot_stream_test.cpp` | diff --git a/docs/README.md b/docs/README.md index 58dd8313f..72ff3a2e1 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,6 +20,8 @@ Read the overview first, then the focused documents for the subsystem in scope. | [10 — Log service](10-log-service.md) | Replicated WAL service and the transaction node's log client | `eloq_log_service/`, `tx_service/tx-log-protos/` | | [Standby replication protocol](standby_replication_protocol.md) | Primary-to-standby change stream, snapshot barrier and replay invariants | `tx_service/include/standby.h`, `tx_service/src/standby.cpp` | +Design records keep their own status and are not current-architecture authority: [range-split-schema-op-recovery.md](range-split-schema-op-recovery.md) records which catalog lock a recovered schema operation restores, by role, beside a range split on the same table, and the tests behind it; its conclusions are reconciled into [07](07-durability-and-recovery.md). + `tx_service/raft_host_manager/` is proprietary. Current documentation may describe only the RPC-visible boundary used by the open components. The EloqStore submodule is likewise documented here only at the `DataStoreHandler`/EloqDSS integration boundary. ## Architecture authoring standard diff --git a/docs/range-split-schema-op-recovery.md b/docs/range-split-schema-op-recovery.md new file mode 100644 index 000000000..625984dc8 --- /dev/null +++ b/docs/range-split-schema-op-recovery.md @@ -0,0 +1,223 @@ +# The catalog lock a recovered schema op restores, and range splits + +**Status: resolved 2026-10-09**; section 6.2 resolved 2026-10-15. Found while reviewing +schema-op recovery in the EloqDoc CDC project (PR 7, 2026-09-29). Section 6.1 records a data loss +in range splits that the tests found, fixed 2026-10-10. Section 7 records the target design this +replaces. + +Key files: `tx_service/include/cc/catalog_cc_map.h` (`Execute(ReplayLogCc&)`, the schema record's +lock), `tx_service/src/fault/log_replay_service.cpp` (barrier #1, the split set), +`tx_service/include/cc/range_cc_map.h` (`Execute(ReplayLogCc&)`, a split's recovery), +`tx_service/src/cc/local_cc_shards.cpp` (the data sync's catalog read lock, `SplitFlushRange`), +`tx_service/src/tx_operation.cpp` (`AcquireAllOp::Forward`, the liveness check of a blocked +request). + +## 1. Coordinator and participant + +Every node group holds a copy of a table's catalog entry, and a schema op locks the copy on every +node group. When a node group recovers, its replay of the op's log record restores a lock on its +copy, and what it must restore differs by role: + +- **The coordinator's node group** (the node group of the op's tx, + `(txn >> 32) >> 10 == ng_id`, and this node is its candidate leader) also recovers the op itself + (`CreateSchemaRecoveryTx`). The recovered op redoes every step after the log record it was + recovered from, including the write locks it takes before its commit log, and that step drains + the DML admitted in the meantime. So the coordinator only needs to keep a newer DDL off the table + until then: a write intent. +- **A participant** recovers while the coordinator lives on elsewhere, and cannot tell how far the + coordinator got. The coordinator may already hold this node group's write lock, the last step + before its commit log, and it does not ask for it again. So a participant restores the lock the + coordinator may rely on. After a prepare-stage record, that is the write lock, the state right + before the commit log: without it, DML admitted on the recovered node group could commit above + the op's commit timestamp under the old schema, breaking the barrier the commit log draws. After a + commit-stage record, it restores the committed schema and no lock. The coordinator's later post + writes find no lock of the tx there and finish at once. + +| Record | Coordinator's node group | Participant | +|---|---|---| +| `PrepareSchema` | write intent; the op resumes after its prepare log | write lock, with the two exceptions in section 2 | +| `PrepareData` (add index) | write intent; the op resumes after its prepare data log | after the build's last record, the write lock, with the split exception of section 2; before it, a write intent (section 6.2) | +| `CommitSchema` | write intent; the op resumes after its commit log and releases the participants | the committed schema, no lock | + +The restored lock's holder carries term 0: after the prepare log the lock and the op are recovered +whatever fails, so no term marks the holder as failed. + +## 2. When a participant can restore only an intent + +A participant restores a write intent instead of the write lock exactly when the coordinator cannot +hold this node group's write lock: + +1. **An add index's `PrepareSchema` record.** Between its prepare log and its first prepare data log + the op holds only an intent: its write lock spans the prepare log, and the step right after + (`PrepareCommit`, `downgrade_all_lock_to_intent_op_`) turns it into an intent. Replay rebuilds + the state after that step (the dirty schema at the op's timestamp), and the downgrade, when it + comes, finds an intent and changes nothing. +2. **A split of the table that this node group coordinates is in flight.** The data sync that runs + a split reads the table's catalog with a local read lock (`read_local`), on its own node group + only. It takes the lock before the split's first log, runs the split inside the same tx + (`SplitFlushRange`, `split_txm` is the data sync's tx) through its clean log, and commits only + after it. So while the split's record is in the log state, the read lock is held, and on + recovery the split's recovery tx (`CreateSplitRangeRecoveryTx`) takes it again before replay + finishes. A read lock excludes every write lock, so the coordinator holds no write lock on this + node group, neither a prepare-stage one nor its commit-phase one. Restoring the write lock would + both overstate the state and stall recovery: replay finishes only once the recovered split holds + its catalog read lock, which the write lock would hold up until the op's `PostCommit`. + +A split coordinated by another node group locks nothing on this one and proves nothing about it. +Barrier #1 therefore collects, from every log group's first replay message, only the splits whose +tx belongs to the recovering node group, under the condition `range_cc_map.h` uses to recover a +split there. + +This holds for every writer of a prepare-stage schema record: + +| Writer | Write lock across its prepare record | Write lock again before its commit log | +|---|---|---| +| `UpsertTableOp` (create, drop, truncate, alter) | no, intents | yes, at the commit-stage timestamp | +| `UpsertTableIndexOp` (add, drop index) | yes, until `PrepareCommit` downgrades it | yes (drop index acquires; add index after its last prepare data log) | +| A DML that updates a catalog (`CatalogAcquireAllOp`, `FlushUpdateTableOp`; EloqDoc's insert that makes an index multikey) | yes, across the data log that carries the record, until `PrepareCommit` downgrades it | no, intents until its commit log | + +Whatever the writer, a split's read lock on a node group means the op is outside every write-lock +section there. Create and truncate cannot have a split in flight: a new table has no ranges to +split, and a truncate runs only on EloqKV tables, which are hash partitioned. + +## 3. The coordinator's lost request + +When a participant recovers with a split of its own in flight, the coordinator's write lock +request to it was waiting behind the split's read lock, and it is lost with the old leader. The +coordinator finds this and sends the request again: + +1. Under 2PL a participant that cannot grant a lock at once acknowledges the request, with the + blocked entry's address and its term. Once the request stays unfinished past the tx's timeout + (`TransactionExecution::IsTimeOut`, 10 s), `AcquireAllOp::Forward` sends + `BlockAcquireAllCcReqCheck` to the node group. The new leader compares the request's term with + its own and answers `ErrorTerm`, which fails the request with `NG_TERM_CHANGED`. A request no + longer queued is answered `REQUEST_LOST`; a check unanswered for 10 s, or a request never + acknowledged, fails the same way. A request still waiting legitimately is answered `BlockQueue` + and keeps waiting. +2. The failed step is retried while the coordinator leads: `UpsertTableOp::AcquireWriteLocks`, and + the index op's commit-stage `acquire_all_lock_op_`, both because a schema op past its prepare + log can only roll forward. +3. The new request reaches the participant after its recovery has finished, behind the recovered + split's read lock and the read locks of the DML admitted since. The grant waits for them, and + its timestamps put the commit timestamp above them: the barrier holds. + +The same mechanism serves a participant that restored the write lock: the request sent again finds +the tx's own lock. + +## 4. What this replaced + +Before, a `PrepareSchema` record restored a write lock on every node group, the coordinator's +included, and an intent on any node group when the table had a split in flight anywhere in the +cluster (barrier #1 collected every split record of every log group). Two consequences: + +- **A hole on participants.** With a split in flight on node group M, the coordinator's write lock + can be granted on another node group N while its request waits on M. If N then recovered, it + restored only an intent, although the coordinator held, and relied on, N's write lock. DML + admitted on N could commit above the op's commit timestamp under the old schema. For EloqDoc's + change data capture that is a write event after its collection's drop or alter. +- **A stall on the coordinator's node group.** Its write lock held up the recovery of a split on the + same table there until the recovered op's `PostCommit`, possibly for as long as a failing data + store step kept retrying. + +## 5. Tests + +- `cmake/tests/ddl_recovery.py` (viii) in EloqDoc: an ALTER killed after its prepare log, on a single + node (the coordinator's node group), is rolled forward, and the table's documents written before + the crash are replayed. +- `cmake/tests/multinode/ddl_split_recovery.py` in EloqDoc, on a cluster of three node groups run as + separate processes (`cluster.py`; a build with fault points and the EloqStore data store service, + first run with RocksDB). A checkpoint splits the loaded collection's range on node group X and a fault point + (`split_flush_range_install_dirty_continue`) holds the split after its prepare log; an ALTER from + Y waits for X's write lock; Z is the third node group: + - (i) X recovers: replay finishes, the coordinator's lost request fails and is sent again, and the + ALTER commits once the recovered split finishes; + - (ii) Z, which granted its write lock, recovers: it restores the write lock, so it serves nothing + until the ALTER commits (its start-up reads every collection's catalog, EloqDoc's + `KVCatalog::init`). Built without this change, Z restored an intent and served at once at the + old catalog version: the test fails there; + - (iii) Y recovers: the recovered ALTER takes its write locks again, Y's own included, and commits + once the split is released; + - (iv) X coordinates the ALTER itself and recovers: the recovered split and the ALTER both finish. + + A node whose catalog entry a recovered schema op write-locks finishes its start-up only once the + op commits, as reads of the table through it would wait without the crash. + + Controls without a schema op test the split itself, every document checked after each: (c0) a + split; (c1) a split held and released; (c2) a split held while the new range's owner-to-be + restarts; (c3) a split, then a restart of the new range's owner at once; (c4) the same with a + clean restart of the whole cluster; (c5) the same after 60 s of checkpoints; (c6), (c7), (c8) 1,000 + documents of the new range written during a held split, 250, 1 and 10 per transaction, then a + restart of the new range's owner at once. + + The controls exposed a separate, older defect, section 6.1: (c2), (c3), (c4), (c6) and (c8) lost + documents, and (ii)'s document check failed whenever Z owned the new range. With its fix, all of + (c0)–(c8) and (i)–(iv) pass. + +## 6. Related findings + +### 6.1 Forwarded writes logged once per transaction (fixed 2026-10-10) + +A split's new range lost records when its owner restarted before checkpointing them: during the +split (c2), right after it (c3), or in a clean restart of the whole cluster (c4). Schema ops play no +part. The records lost were all writes committed during the split, after it installed its dirty +range; the old data reaches the new owner safely. + +- **Old data** (committed at or below the split's data sync timestamp) is flushed by the split's + own data sync into the new range's partition, on the new owner's data store shard, before the + split's commit log. The records the old owner then uploads to the new owner are installed as + already persisted. +- **Writes during the split** are double written: once the dirty range is installed, a write to + the new range goes to the old owner and to the new owner (`WriteSetEntry::forward_addr_`), and + the tx's data log record carries the key in both node groups' blobs. The split's data sync does + not cover them: until the new owner checkpoints them, its blob in the log is their only durable + copy, which its recovery replays. + +`TransactionExecution::FillDataLogRequest` put a forwarded key into the new owner's blob only if +that node group had no entry yet (`if (!inserted) continue;`, commit a64a31d, #454, 2026-03-16), so +it logged the tx's first such key per node group and dropped the rest. A key forwarded to several +cores of one node group needs logging there only once; the check applied that to the whole +transaction instead of to the key. In a 320 MB +load in batches of 500, the split started during the load, and every document of the new range +written after its dirty range install was lost except the first of each batch. Single-key +transactions lost nothing (c7). 1,000 documents in transactions of 250 kept 4 (c6), and in +transactions of 10 kept 100 (c8). Waiting 60 s of checkpoints before the restart lost nothing (c5). + +The fix logs every forwarded key into the new owner's blob, once per node group, skipping only the +key's own node group and a node group it is already logged to. A bucket migration's double writes +take the same path (`LockWriteRangeBucketsOp::Advance` sets `forward_addr_` for a hash-partitioned +key whose bucket migrates and for a range whose bucket migrates), so their transactions were logged +the same way; no test here runs a migration. Object commands (`CmdSetEntry::forward_entry_`) were +always logged per entry. With the fix, every control and scenario of section 5 passes on the +EloqStore build, and so do `ddl_recovery.py` and `cluster_gate.py` on a single node. + +### 6.2 Add index's prepare data records (resolved 2026-10-15) + +Add index writes a `PrepareData` record after each batch of its build, under intents, and after +the last one takes its commit-stage write locks. The record carries the build's last finished key, +and the last record's is positive infinity (`last_key_type = PosInfKey`). The log state keeps the +newest record's key (a `PrepareData` record for an op already at that stage replaces the key), so a +recovering node group replays it. + +A participant therefore restores: + +- **After the last record** (`PosInfKey`): the write lock. The build is over, and the coordinator, + about to take or already holding its commit-stage write lock, does not ask for it again. The + split exception of section 2 still applies: with a split that this node group coordinates in + flight, a write intent, since that split's catalog read lock excludes the coordinator's write + lock here. +- **Before it**: a write intent, since the build still runs under intents. A write lock then would + block DML on the node group until the index commits, and could block the build's own reads. + +The coordinator's node group restores a write intent in both cases: the recovered op resumes from +the record and, at the last one, takes the commit-stage write locks itself. No new record is +needed; the last key already marks the end of the build. + +## 7. The target design this replaces + +The earlier version of this document kept the write lock on every node group and proposed letting a +recovered split finish its part of recovery before taking its catalog read lock +(`CreateSplitRangeRecoveryTx` calling `SetFinish()` first), so that the split resumes after the +schema op and gives up when the op was a drop. With the roles of section 1 that is not needed: the +coordinator restores an intent, and a participant whose own split is in flight restores one +because the coordinator cannot hold its write lock, so no recovered split waits on a recovered +schema op. diff --git a/eloq_log_service/CMakeLists.txt b/eloq_log_service/CMakeLists.txt index 9a391622f..8172e8ec1 100644 --- a/eloq_log_service/CMakeLists.txt +++ b/eloq_log_service/CMakeLists.txt @@ -326,12 +326,17 @@ add_library(logservice STATIC ${LOG_SOURCE_DIR}/src/log_state_rocksdb_impl.cpp ${LOG_SOURCE_DIR}/src/log_state_rocksdb_cloud_impl.cpp ${LOG_SOURCE_DIR}/src/log_state_memory_impl.cpp + ${LOG_SOURCE_DIR}/src/cdc_long_poll.cpp + ${LOG_SOURCE_DIR}/src/cdc_scan.cpp ${LOG_SOURCE_DIR}/src/fault_inject.cpp ${CMAKE_CURRENT_LIST_DIR}/../third_party/ini/INIReader.cpp ${CMAKE_CURRENT_LIST_DIR}/../third_party/ini/ini.c) target_compile_definitions(logservice PUBLIC LOG_SHIPPING_THREADS_NUM=${LOG_SHIPPING_THREADS_NUM}) list(APPEND LOG_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/../third_party/ini) +# The log groups' metrics use eloq_metrics' header-only Meter against the +# registry the process passes in, so the library itself is not linked. +list(APPEND LOG_INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/../eloq_metrics/include) target_include_directories(logservice PUBLIC ${LOG_INCLUDE_DIR}) target_link_libraries(logservice PUBLIC ${LOG_LIB}) @@ -366,8 +371,12 @@ if (TEST_LOG_SERVICE) if (WITH_LOG_STATE MATCHES "ROCKSDB|ROCKSDB_CLOUD_S3|ROCKSDB_CLOUD_GCS") add_executable(rocksdb_test ${LOG_SOURCE_DIR}/test/rocksdb_test.cpp) target_link_libraries(rocksdb_test PUBLIC logservice) + find_package(Catch2 CONFIG REQUIRED) + # Also built by the data_substrate test suite (tx_service/tests). + add_executable(CdcScan-Test ${LOG_SOURCE_DIR}/test/CdcScan-Test.cpp) + target_link_libraries(CdcScan-Test PUBLIC logservice Catch2::Catch2WithMain) + set_property(TARGET CdcScan-Test PROPERTY CXX_STANDARD 20) if (WITH_LOG_STATE MATCHES "ROCKSDB_CLOUD_S3|ROCKSDB_CLOUD_GCS") - find_package(Catch2 CONFIG REQUIRED) add_executable(log_server_rocksdb_cloud_tests ${LOG_SOURCE_DIR}/test/log_server_rocksdb_cloud_tests.cpp) target_link_libraries(log_server_rocksdb_cloud_tests PUBLIC logservice Catch2::Catch2) add_executable(rocksdb_cloud_delete_range_test ${LOG_SOURCE_DIR}/test/rocksdb_cloud_delete_range_test.cpp) diff --git a/eloq_log_service/include/cdc_long_poll.h b/eloq_log_service/include/cdc_long_poll.h new file mode 100644 index 000000000..fe9106788 --- /dev/null +++ b/eloq_log_service/include/cdc_long_poll.h @@ -0,0 +1,187 @@ +/** + * @file + * The CdcScan long poll (docs/cdc-change-stream.md in the EloqDoc repo, §4.2 + * "Long poll", §5.8 "Logger side"): the scans a log group holds until + * something may have become emittable. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace txlog +{ +/** + * @brief The parked long-poll CdcScan requests of one log group. + * + * A scan that finds nothing emittable and may wait (`wait_ms > 0`) is parked + * here instead of being answered. Four sources complete it, on different + * threads, and the first one wins: an advance of the group's + * arrival_complete_ts (the only change that can make a record emittable), the + * request's deadline, the client's cancellation, and the node's step-down or + * shutdown. Every source takes the request's `completed` flag with an + * exchange, and only the winner runs its completion, so the brpc `done` + * closure runs exactly once: twice is undefined behavior, never leaks the + * call until the client's timeout. + * + * Completions run on bthreads of their own, never on the thread of the source: + * the raft apply thread and the timer thread must not do a scan's work. The + * list is guarded by a plain mutex, held only to insert and remove entries; + * nothing waits on a condition variable (see the threading gotcha in the + * data_substrate CLAUDE.md). + * + * Use: Prepare() a request, register its cancellation (Cancel) with the call, + * then Park() it. The cancellation is registered first because it may fire at + * once; a parked request could complete and end the call before the + * registration, which would then use a finished call. + */ +class CdcLongPoll +{ +public: + /** @brief Why a parked request completes. */ + enum class Wake + { + kAdvance, ///< arrival_complete_ts advanced: scan again. + kDeadline, ///< The request's wait elapsed: scan again. + kCancel, ///< The client cancelled the call: any answer will do. + kStop, ///< The node stopped leading or shuts down: NotLeader. + }; + + /** + * @brief Answers a parked request, running the call's done closure. + * Called exactly once, on a bthread. + */ + using Completion = std::function; + + /** @brief One request, prepared or parked. */ + struct Request; + + CdcLongPoll() = default; + CdcLongPoll(const CdcLongPoll &) = delete; + CdcLongPoll &operator=(const CdcLongPoll &) = delete; + + /** + * @brief Completes every request with kStop, from now on also those parked + * later, and waits until no completion or deadline timer is running. + */ + ~CdcLongPoll(); + + /** + * @brief Makes a request that can be completed but is not parked yet. + * + * @param completion How to answer it. + * @return The request, to pass to Cancel() and Park(). + */ + std::shared_ptr Prepare(Completion completion); + + /** + * @brief Parks a prepared request and starts its deadline. A request + * already completed (cancelled) is left alone; one parked after the poll + * was stopped is completed with kStop. + * + * @param request The request. + * @param wait_ms How long it may wait; its deadline completes it with + * kDeadline. + */ + void Park(const std::shared_ptr &request, uint32_t wait_ms); + + /** + * @brief Completes one request, unless another source already has. + * + * @param request The request, prepared or parked. + * @param why The reason passed to its completion. + */ + void Complete(const std::shared_ptr &request, Wake why); + + /** + * @brief Completes every parked request. + * + * @param why The reason passed to their completions. + */ + void CompleteAll(Wake why); + + /** + * @brief Completes a request with kCancel, unless another source already + * has. The target of the call's NotifyOnCancel closure, which brpc runs + * once the call is cancelled or has ended, whichever comes first; after + * the end it finds the request completed and touches nothing else. + * + * @param request The request; the closure keeps it alive. + */ + static void Cancel(std::shared_ptr request); + + /** @return The number of parked requests. */ + size_t Size() const; + + /** + * @brief Called with the number of parked requests each time it changes, + * under the poll's lock, so it must be quick and must not call back into + * the poll. + */ + using SizeObserver = std::function; + + /** + * @brief Sets the observer of the parked count, replacing any earlier one, + * and reports the current count to it. + * + * @param observer The observer; empty to stop observing. + */ + void SetSizeObserver(SizeObserver observer); + +private: + /** @brief A deadline timer's argument; keeps its request alive. */ + struct TimerArg + { + CdcLongPoll *poll; + std::shared_ptr request; + }; + + /** @brief A completion's bthread argument. */ + struct Task + { + CdcLongPoll *poll; + std::shared_ptr request; + Wake why; + }; + + /** + * @brief The bthread_timer_add callback of a request's deadline. + * + * @param arg The TimerArg, which it deletes. + */ + static void OnDeadline(void *arg); + + /** + * @brief The bthread that runs a completion. + * + * @param arg The Task, which it deletes. + * @return nullptr. + */ + static void *RunCompletion(void *arg); + + /** + * @brief Completes a request whose `completed` flag the caller took: + * unparks it, deletes its timer and starts its completion. + * + * @param request The request. + * @param why The reason passed to its completion. + */ + void Finish(const std::shared_ptr &request, Wake why); + + mutable std::mutex mutex_; ///< Guards parked_, stopped_ and each + ///< request's list and timer fields. + std::list> parked_; ///< The parked requests. + bool stopped_{false}; ///< Set by the destructor: parks complete at once. + SizeObserver size_observer_; ///< See SetSizeObserver(); under mutex_. + /** + * @brief Prepared requests whose completion has not returned, plus + * deadline timers that may still run; the destructor waits for zero. + */ + std::atomic pending_{0}; +}; +} // namespace txlog diff --git a/eloq_log_service/include/cdc_scan.h b/eloq_log_service/include/cdc_scan.h new file mode 100644 index 000000000..2c276a030 --- /dev/null +++ b/eloq_log_service/include/cdc_scan.h @@ -0,0 +1,310 @@ +/** + * @file + * The CDC log scan behind the CdcScan RPC. + * + * CdcScan reads a log group's two timestamp-ordered streams, the committed + * data records and the DDL history, as one sequence merged by timestamp, and + * returns it in pages. Its contract is in log.proto next to CdcScanRequest; + * the pieces below implement it. + * + * Merge order: by timestamp; at one timestamp every data record precedes + * every DDL record. The tie rule carries no meaning of its own, but it must be + * fixed, because it is what lets each stream resume from its own last key: a + * page is a prefix of the merged sequence, so if it ends inside a timestamp + * the stream that comes second at that timestamp has returned none of it and + * the other has returned all of it, or the page ends in the first stream. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "log.pb.h" + +namespace rocksdb +{ +class DB; +} + +namespace txlog +{ +/** + * @brief Where each stream of a scan resumes. + * + * A position is a storage key read as "resume strictly after this key", in + * the bytewise order the stores use: + * - empty: the stream has returned nothing and starts at the beginning of + * complete history; + * - eight bytes, the big-endian timestamp `ts`: the stream has returned + * nothing since a scan started at `from_ts = ts`. Every storage key starts + * with its big-endian timestamp and is longer than eight bytes, so each key + * at `ts` sorts after this one and each key below `ts` before it: "after" + * means "at the first key whose timestamp is >= ts"; + * - anything longer: the last storage key the stream returned. + */ +struct CdcScanPosition +{ + std::string data; ///< Position in the data stream. + std::string ddl; ///< Position in the DDL stream. +}; + +/** + * @brief Encodes a position as the `continuation` bytes. + * + * The layout is `u16 len | data position | u16 len | ddl position` with + * big-endian lengths, shared with the EloqDoc runtime, which keeps + * continuations as scan positions (log.proto). + * + * @param position The position to encode. + * @return The continuation bytes. + */ +std::string EncodeCdcContinuation(const CdcScanPosition &position); + +/** + * @brief Decodes continuation bytes; the inverse of EncodeCdcContinuation. + * + * @param bytes The continuation a client passed back. + * @param[out] position Set to the decoded position on success. + * @param[out] error Set to the reason on failure. + * @return False for bytes that are not a continuation this service could + * have issued. + */ +bool DecodeCdcContinuation(std::string_view bytes, + CdcScanPosition *position, + std::string *error); + +/** + * @brief The eight-byte position that starts a stream at a timestamp. + * + * @param ts The timestamp to start at (see CdcScanPosition). + * @return The big-endian `ts`. + */ +std::string CdcTimestampPosition(uint64_t ts); + +/** + * @brief A DDL record's position in the DDL stream: its storage key. + * + * DDLs on one table hold its catalog write lock in turn, so they have distinct + * timestamps; records at one timestamp belong to different transactions, and + * the transaction number orders them stably. The position begins with the + * big-endian timestamp and is longer than eight bytes, as the continuation's + * timestamp form requires, and the runtime rebuilds it from a record's ddl_ts + * and txn_number. + * + * @param ddl_ts The record's stream timestamp. + * @param txn The schema op's transaction number. + * @return be64 ddl_ts ‖ be64 txn. + */ +std::string CdcDdlPosition(uint64_t ddl_ts, uint64_t txn); + +/** + * @brief Where a scan starts. + * + * The request's continuation, both streams at `from_ts`, or, with neither, + * the beginning of complete history. + * + * @param request The scan request. + * @param[out] start Set to the starting position on success. + * @param[out] error Set to the reason on failure. + * @return False for a malformed continuation. + */ +bool ParseCdcScanStart(const CdcScanRequest &request, + CdcScanPosition *start, + std::string *error); + +/** + * @brief The page size to use for a request. + * + * @param requested The request's `max_bytes`; 0 means the default. + * @return The size, reduced to the service's ceiling so that one page always + * fits a brpc message comfortably. + */ +size_t CdcScanPageBytes(uint32_t requested); + +/** + * @brief One of the two streams a scan walks, positioned by CdcScanPosition. + */ +class CdcStreamCursor +{ +public: + virtual ~CdcStreamCursor() = default; + + /** + * @brief Positions the cursor at the first record strictly after `after` + * whose timestamp is at most `upper_ts`. + * + * @param after The position to resume after (see CdcScanPosition). + * @param upper_ts The highest timestamp the cursor yields, ever. + */ + virtual void Seek(const std::string &after, uint64_t upper_ts) = 0; + + /** + * @return False once the stream is exhausted through the upper bound, or + * has failed. + */ + virtual bool Valid() const = 0; + + /** @return False if reading the store failed; ErrorText() says why. */ + virtual bool Ok() const = 0; + + /** @return Why reading the store failed; empty while Ok(). */ + virtual std::string ErrorText() const = 0; + + /** @return The current record's timestamp. Only while Valid(). */ + virtual uint64_t Timestamp() const = 0; + + /** + * @return The current record's storage key, valid until Next(). Only + * while Valid(). + */ + virtual std::string_view Key() const = 0; + + /** + * @return An estimate of what the current record adds to the response, + * for the page cut. Only while Valid(). + */ + virtual size_t ByteSize() const = 0; + + /** + * @brief Appends the current record to a response. Only while Valid(). + * + * @param response The page being filled. + */ + virtual void AppendTo(CdcScanResponse *response) const = 0; + + /** @brief Moves to the next record. Only while Valid(). */ + virtual void Next() = 0; +}; + +/** + * @brief The DDL stream: a log state's in-memory DDL history. + * + * Seek() has the source copy, under the log state's lock, every record + * strictly after the position and at or below the upper bound; the cursor then + * walks the copy. DDLs are rare, so the copy is small, and the history can + * change under the scan without affecting it. + */ +class CdcDdlHistoryCursor final : public CdcStreamCursor +{ +public: + /** + * @brief Copies the history's records strictly after `after` whose + * ddl_ts is at most `upper_ts`, in position order. + */ + using Source = std::function *records)>; + + /** + * @param source Where Seek() copies the records from. + */ + explicit CdcDdlHistoryCursor(Source source) : source_(std::move(source)) + { + } + + void Seek(const std::string &after, uint64_t upper_ts) override; + bool Valid() const override + { + return pos_ < records_.size(); + } + bool Ok() const override + { + return true; + } + std::string ErrorText() const override + { + return {}; + } + uint64_t Timestamp() const override + { + return records_[pos_].ddl_ts(); + } + std::string_view Key() const override + { + return key_; + } + size_t ByteSize() const override + { + return records_[pos_].ByteSizeLong(); + } + void AppendTo(CdcScanResponse *response) const override; + void Next() override; + +private: + /** @brief Sets key_ to the current record's position, if there is one. */ + void LoadKey(); + + Source source_; ///< The history's copy function. + std::vector records_; ///< The copy Seek() took. + size_t pos_{0}; ///< The current record's index. + std::string key_; ///< The current record's position. +}; + +/** + * @brief Fills a response with the next page of the merged sequence. + * + * The page is a prefix of the merged sequence starting after `start`, holds + * only records at or below `arrival_complete_ts`, and is cut once it reaches + * `page_bytes`, but always holds at least one record when one is available. + * Appends to `data` and `ddl` and sets `has_more`, `continuation` and + * `arrival_complete_ts`; `retention_floor_ts` stays 0, since the history + * boundary is not tracked yet, and `status` is the caller's. + * + * @param data The data stream. + * @param ddl The DDL stream. + * @param start Where both streams resume. + * @param arrival_complete_ts The group's min_ckpt_ts_. It must have been read + * before either cursor opened its view of the store: a DDL record is + * appended above the bound current at its append, so one appended + * after the scan's view was taken lies above a bound read before it, + * and the page cannot miss it. + * @param page_bytes The soft cap on the page size. + * @param[out] response The page. + * @param[out] error Set to the reason if a stream failed. + * @return False if a stream failed. + */ +bool RunCdcScan(CdcStreamCursor &data, + CdcStreamCursor &ddl, + const CdcScanPosition &start, + uint64_t arrival_complete_ts, + size_t page_bytes, + CdcScanResponse *response, + std::string *error); + +/** @brief How a log state answered a scan; see LogState::CdcScan. */ +enum struct CdcScanStatus +{ + Ok, ///< The page is filled. + NotReady, ///< No consistent view right now; retry shortly. + NotLeader, ///< Not the leader, or leadership changed mid-scan. + Unsupported, ///< This log state keeps no scannable history. + InvalidArgument, ///< The request is malformed. + StorageError, ///< Reading the store failed. +}; + +/** + * @brief Runs a whole scan against a RocksDB-backed log state. + * + * The page's `status` is left to the caller. + * + * @param db The log state's data stream: an open database that outlives the + * call. + * @param ddl The log state's DDL stream, positioned by the scan. + * @param arrival_complete_ts Subject to the read-order rule of RunCdcScan. + * @param request The scan request. + * @param[out] response The page. + * @param[out] error Set to the reason on failure. + * @return Ok, InvalidArgument for a malformed continuation, or StorageError. + */ +CdcScanStatus CdcScanRocksDB(rocksdb::DB *db, + CdcStreamCursor &ddl, + uint64_t arrival_complete_ts, + const CdcScanRequest &request, + CdcScanResponse *response, + std::string *error); +} // namespace txlog diff --git a/eloq_log_service/include/log_instance.h b/eloq_log_service/include/log_instance.h index f7957bae5..f530e4205 100644 --- a/eloq_log_service/include/log_instance.h +++ b/eloq_log_service/include/log_instance.h @@ -24,9 +24,12 @@ #include #include +#include "cdc_long_poll.h" #include "log.pb.h" #include "log_shipping_agent.h" #include "log_state.h" +#include "meter.h" +#include "metrics.h" #if defined(LOG_STATE_TYPE_RKDB_CLOUD) #include "rocksdb_cloud_config.h" @@ -115,7 +118,8 @@ class LogInstance : public braft::StateMachine uint32_t snapshot_interval, bool enable_request_checkpoint, uint32_t check_replay_log_size_interval_sec, - uint64_t notify_checkpointer_threshold_size) + uint64_t notify_checkpointer_threshold_size, + uint64_t cdc_retention_seconds) : log_group_id_(log_group_id), node_id_(node_id), node_(NULL), @@ -189,6 +193,7 @@ class LogInstance : public braft::StateMachine #else log_state_ = std::make_unique(); #endif + log_state_->SetCdcRetentionSeconds(cdc_retention_seconds); } ~LogInstance() override @@ -327,6 +332,44 @@ class LogInstance : public braft::StateMachine LogResponse *response, google::protobuf::Closure *done); + /** + * @brief Serves one page of the CDC log scan (cdc_scan.h) from this + * replica, holding it first if the request asks to wait. + * + * A page with nothing emittable whose request has a wait_ms is parked + * (the long poll, cdc_long_poll.h) and served again once the group's + * arrival_complete_ts advances or the wait elapses; it is answered with + * NotLeader if this node stops leading meanwhile, and with the empty page + * if the client cancels. With cdc_scan_max_parked scans already parked, + * the empty page is answered at once. + * + * @param cntl The RPC controller; failures that no retry fixes fail the + * RPC through it. + * @param request The scan request. + * @param[out] response The page, or the NotLeader answer. + * @param done Run when the response is complete, possibly after this + * returns. + */ + void CdcScan(brpc::Controller *cntl, + const CdcScanRequest *request, + CdcScanResponse *response, + google::protobuf::Closure *done); + + /** + * @brief Starts exporting the group's change data capture metrics: + * cdc_arrival_complete_ts and cdc_scan_parked_requests + * (log_service_metrics.h), labelled with the node's labels and lg_id. + * + * Called once, after the group has started serving: the metrics registry + * is made after the log service. The meter is complete before it is + * published, so a thread that sees it can use it. + * + * @param registry The process's metrics registry; outlives this instance. + * @param node_labels The node's common labels. + */ + void EnableMetrics(metrics::MetricsRegistry *registry, + const metrics::CommonLabels &node_labels); + void TransferLeader(const TransferRequest *request, TransferResponse *response, google::protobuf::Closure *done); @@ -503,6 +546,7 @@ class LogInstance : public braft::StateMachine void on_shutdown() override { + cdc_long_poll_.CompleteAll(CdcLongPoll::Wake::kStop); } void on_error(const ::braft::Error &e) override @@ -566,6 +610,9 @@ class LogInstance : public braft::StateMachine int read_snapshot_status = log_state_->ReadSnapshot(snapshot_path, files); LOG(INFO) << "finish loading snapshot"; + arrival_complete_ts_.store(log_state_->MinCkptTs(), + std::memory_order_release); + ReportArrivalComplete(); return read_snapshot_status; } @@ -588,6 +635,32 @@ class LogInstance : public braft::StateMachine void LogGroupLeaderUpdate(); + /** + * @return The address of the group's leader as this node knows it, + * "host:port", or empty while no leader is known. + */ + std::string LeaderAddress() const; + + /** + * @brief Serves one page of the CDC log scan at once. + * + * Only the leader serves. A follower, or a leader that lost its term + * while the log state built the page, answers NotLeader with the + * leader's address; a leader whose log state cannot serve a consistent + * view yet answers NotLeader with its own, so the caller backs off and + * retries here. + * + * @param cntl The RPC controller; failures that no retry fixes fail the + * RPC through it. + * @param request The scan request. + * @param[out] response The page, or the NotLeader answer. + * @return True if the page holds no record and none is left below its + * bound: nothing was emittable, and the scan may wait. + */ + bool ServeCdcScan(brpc::Controller *cntl, + const CdcScanRequest *request, + CdcScanResponse *response); + /** * If this node is the group's latest leader. * Only available when all nodes in the raft group set @@ -603,6 +676,39 @@ class LogInstance : public braft::StateMachine void ProposeTaskAndWait(braft::Task &task, const LogRequest *request, LogResponse *response); + + /** + * @brief Sets cdc_arrival_complete_ts to arrival_complete_ts_, if the + * meter is published. Under arrival_metric_mutex_, so that + * EnableMetrics()'s first report and the apply thread's later ones + * cannot land out of order and leave the gauge behind the state. + */ + void ReportArrivalComplete(); + + /** + * @brief The group's arrival_complete_ts as the apply thread last left + * it (LogState::MinCkptTs()), for EnableMetrics(), which runs on another + * thread and cannot read the log state. + */ + std::atomic arrival_complete_ts_{0}; + /** @brief Serializes ReportArrivalComplete(). */ + std::mutex arrival_metric_mutex_; + + /** @brief Owns the meter EnableMetrics() makes; kept until destruction. */ + std::unique_ptr meter_; + /** + * @brief The meter once it is complete, else nullptr; what other threads + * read. Declared before cdc_long_poll_, whose size observer uses it, so it + * outlives the poll. + */ + std::atomic published_meter_{nullptr}; + + /** + * @brief The parked long-poll CdcScan requests. Declared last so that it + * is destroyed first: its destructor waits for the completions, which + * serve scans from the members above. + */ + CdcLongPoll cdc_long_poll_; }; /** diff --git a/eloq_log_service/include/log_server.h b/eloq_log_service/include/log_server.h index c10f257ef..5605d4724 100644 --- a/eloq_log_service/include/log_server.h +++ b/eloq_log_service/include/log_server.h @@ -60,7 +60,8 @@ class LogServer uint32_t snapshot_interval = 600, bool enable_request_checkpoint = false, uint32_t check_replay_log_size_interval_sec = 10, - uint64_t notify_checkpointer_threshold_size = 128 * 1024 * 1024); + uint64_t notify_checkpointer_threshold_size = 128 * 1024 * 1024, + uint64_t cdc_retention_seconds = 0); ~LogServer() { @@ -79,6 +80,20 @@ class LogServer int Start(bool enable_brpc_builtin_services = true); + /** + * @brief Starts exporting the log groups' change data capture metrics. The + * metrics registry is made after the log server, so this is called once + * both exist. + * + * @param registry The process's metrics registry; outlives the server. + * @param node_labels The node's common labels. + */ + void EnableMetrics(metrics::MetricsRegistry *registry, + const metrics::CommonLabels &node_labels) + { + raft_log_service_.EnableMetrics(registry, node_labels); + } + const std::string &GetStoragePath() const { return storage_path_; diff --git a/eloq_log_service/include/log_service.h b/eloq_log_service/include/log_service.h index fddd343f3..8e1fec7a1 100644 --- a/eloq_log_service/include/log_service.h +++ b/eloq_log_service/include/log_service.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -82,7 +83,8 @@ class LogServiceImpl : public LogService uint32_t snapshot_interval, bool enable_request_checkpoint, uint32_t check_replay_log_size_interval_sec, - uint64_t notify_checkpointer_threshold_size) + uint64_t notify_checkpointer_threshold_size, + uint64_t cdc_retention_seconds = 0) { std::pair, uint32_t> @@ -152,7 +154,8 @@ class LogServiceImpl : public LogService snapshot_interval, enable_request_checkpoint, check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size); + notify_checkpointer_threshold_size, + cdc_retention_seconds); } } } @@ -268,6 +271,8 @@ class LogServiceImpl : public LogService std::string notify_checkpointer_threshold_size_str = config_reader.GetString( "local", "notify_checkpointer_threshold_size", "1GB"); + uint64_t cdc_retention_seconds = + config_reader.GetInteger("local", "txlog_cdc_retention_seconds", 0); #if defined(LOG_STATE_TYPE_RKDB_ALL) std::string rocksdb_storage_path = @@ -342,6 +347,7 @@ class LogServiceImpl : public LogService << check_replay_log_size_interval_sec << ", notify_checkpointer_threashold_size = " << notify_checkpointer_threshold_size_str + << ", txlog_cdc_retention_seconds = " << cdc_retention_seconds #if defined(LOG_STATE_TYPE_RKDB_S3) << ", aws_access_key_id = " << aws_access_key_id << ", aws_secret_key = " @@ -492,6 +498,9 @@ class LogServiceImpl : public LogService #endif + // Under replicas_mutex_, against EnableMetrics' walk of the map; the + // replicas get their metrics if they were enabled before the attach. + std::unique_lock replicas_lk(replicas_mutex_); AddLogReplicas(node_id, ip_list, port_list, @@ -520,7 +529,10 @@ class LogServiceImpl : public LogService snapshot_interval, enable_request_checkpoint, check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size); + notify_checkpointer_threshold_size, + cdc_retention_seconds); + EnableReplicaMetricsLocked(); + replicas_lk.unlock(); assert(service_status_ == Status::DETACHED); @@ -529,7 +541,10 @@ class LogServiceImpl : public LogService if (err != 0) { LOG(ERROR) << "Attach log service: fail to start raft node"; - log_replicas_.clear(); + { + std::lock_guard lk(replicas_mutex_); + log_replicas_.clear(); + } service_status_.store(Status::DETACHED, std::memory_order_release); response->set_err_code( ::txlog::AttachLogErrorCode::INSTANCE_INIT_ERROR); @@ -590,7 +605,10 @@ class LogServiceImpl : public LogService instance.shutdown(); instance.join(); } - log_replicas_.clear(); + { + std::lock_guard lk(replicas_mutex_); + log_replicas_.clear(); + } service_status_.store(Status::DETACHED, std::memory_order_release); LOG(INFO) << "Detach log service: service is detached"; @@ -644,6 +662,37 @@ class LogServiceImpl : public LogService request, response, done); } + /** + * @brief Finds the raft log instance of the requested log group and + * serves one page of the CDC log scan from it. + * + * @param controller The RPC controller. + * @param request The scan request. + * @param[out] response The page, or NotLeader when this node holds no + * replica of the group. + * @param done Run when the response is complete. + */ + void CdcScan(::google::protobuf::RpcController *controller, + const ::txlog::CdcScanRequest *request, + ::txlog::CdcScanResponse *response, + ::google::protobuf::Closure *done) override + { + auto logger_it = log_replicas_.find(request->log_group_id()); + if (logger_it == log_replicas_.end()) + { + // No replica of the group here, which includes a detached + // service. The caller finds the leader through its own cache. + brpc::ClosureGuard done_guard(done); + response->set_status(CdcScanResponse::NotLeader); + return; + } + + logger_it->second.CdcScan(static_cast(controller), + request, + response, + done); + } + /* * Find the right raft log instance based on log group id and update * node term. @@ -939,9 +988,55 @@ class LogServiceImpl : public LogService } } + /** + * @brief Starts exporting every log group's change data capture metrics + * (LogInstance::EnableMetrics), for the groups present now and for those + * a later AttachLog adds. + * + * @param registry The process's metrics registry. + * @param node_labels The node's common labels. + */ + void EnableMetrics(metrics::MetricsRegistry *registry, + const metrics::CommonLabels &node_labels) + { + std::lock_guard lk(replicas_mutex_); + metrics_registry_ = registry; + metrics_labels_ = node_labels; + EnableReplicaMetricsLocked(); + } + private: + /** + * @brief Enables the metrics of every replica that has none, once + * EnableMetrics() has been called; LogInstance::EnableMetrics ignores a + * replica already enabled. Called with replicas_mutex_ held. + */ + void EnableReplicaMetricsLocked() + { + if (metrics_registry_ == nullptr) + { + return; + } + for (auto &logger_pair : log_replicas_) + { + logger_pair.second.EnableMetrics(metrics_registry_, + metrics_labels_); + } + } + std::atomic service_status_{Status::ATTACHED}; + /** + * @brief Guards log_replicas_ insertions and clears (AttachLog, DetachLog) + * against EnableMetrics' walk of it, and metrics_registry_ and + * metrics_labels_. + */ + std::mutex replicas_mutex_; + /** @brief Set by EnableMetrics(); nullptr while metrics are off. */ + metrics::MetricsRegistry *metrics_registry_{nullptr}; + /** @brief The node's common labels EnableMetrics() was given. */ + metrics::CommonLabels metrics_labels_; + // raft log nodes (no matter leader or followers) which belong to this // raft service. std::unordered_map log_replicas_; diff --git a/eloq_log_service/include/log_service_metrics.h b/eloq_log_service/include/log_service_metrics.h index dc71cc3d7..c5d2cbf99 100644 --- a/eloq_log_service/include/log_service_metrics.h +++ b/eloq_log_service/include/log_service_metrics.h @@ -2,7 +2,24 @@ #include +#include "metrics.h" + namespace metrics { inline std::atomic_bool enable_log_service_metrics{false}; + +/** + * The log group's arrival_complete_ts, in microseconds since the epoch: every + * record at or below it has arrived, so change data capture can emit up to it + * (EloqDoc docs/cdc-change-stream.md §4.2). Set on each replica when it + * advances; how far it trails the wall clock is the change streams' lag floor. + */ +inline const Name NAME_CDC_ARRIVAL_COMPLETE_TS{"cdc_arrival_complete_ts"}; + +/** + * The CdcScan requests parked on the log group's leader, waiting for + * arrival_complete_ts to advance or for their wait to end; at most + * cdc_scan_max_parked. + */ +inline const Name NAME_CDC_SCAN_PARKED_REQUESTS{"cdc_scan_parked_requests"}; } // namespace metrics diff --git a/eloq_log_service/include/log_shipping_agent.h b/eloq_log_service/include/log_shipping_agent.h index 4706313e9..1986a5a58 100644 --- a/eloq_log_service/include/log_shipping_agent.h +++ b/eloq_log_service/include/log_shipping_agent.h @@ -136,12 +136,20 @@ class LogShippingAgent finish_msg->set_last_ckpt_ts(last_ckpt_ts_); int eagain = 0; - if (SendMessage(replay_msg, iobuf_, true, eagain) != 0) + // Not on a new stream: the finish message vouches for + // the records sent before it on this one, which a + // broken stream may have lost. Without it the cc node + // asks for the replay again and gets all of it. + if (SendMessage(replay_msg, iobuf_, false, eagain) != 0) { // interrupted or stream invalid - LOG(ERROR) << "log group " << log_group_id_ - << " faild to send finish message to cc" - << " node group " << cc_node_group_id_; + LOG(ERROR) + << "log group " << log_group_id_ + << " faild to send finish message to cc" + << " node group " << cc_node_group_id_ + << " term " << cc_node_group_term_ + << (IsTerminating() ? ": the agent was replaced" + : ""); brpc::StreamClose(stream_id_); shipping_agent_status_.store( Status::Terminated, std::memory_order_release); @@ -246,6 +254,17 @@ class LogShippingAgent return cc_node_group_term_; } + /** + * @brief Whether Terminate() was called (or the agent stopped on its own): + * the agent ships nothing more, skips the finish message and closes its + * stream. + */ + [[nodiscard]] bool IsTerminating() const + { + return shipping_agent_status_.load(std::memory_order_acquire) != + Status::Active; + } + private: int ConnectStream() { @@ -348,6 +367,14 @@ class LogShippingAgent scale_msg->set_cluster_scale_op_blob(item->log_message_); scale_msg->set_txn(item->tx_number_); } + else if (item->item_type_ == LogItemType::ClusterGateLog) + { + ReplayClusterGateMsg *gate_msg = + replay_msg.mutable_cluster_gate_msg(); + gate_msg->set_commit_ts(item->timestamp_); + gate_msg->set_cluster_gate_blob(item->log_message_); + gate_msg->set_txn(item->tx_number_); + } else if (item->item_type_ == LogItemType::SchemaLog) { ReplaySchemaMsg *schema_msg = replay_msg.add_schema_op_msgs(); @@ -441,6 +468,14 @@ class LogShippingAgent scale_msg->set_cluster_scale_op_blob(item.log_message_); scale_msg->set_txn(item.tx_number_); } + else if (item.item_type_ == LogItemType::ClusterGateLog) + { + ReplayClusterGateMsg *gate_msg = + replay_msg.mutable_cluster_gate_msg(); + gate_msg->set_commit_ts(item.timestamp_); + gate_msg->set_cluster_gate_blob(item.log_message_); + gate_msg->set_txn(item.tx_number_); + } else if (item.item_type_ == LogItemType::SchemaLog) { ReplaySchemaMsg *schema_msg = replay_msg.add_schema_op_msgs(); @@ -489,6 +524,19 @@ class LogShippingAgent for (iterator->SeekToFirst(i); iterator->Valid(i); iterator->Next(i)) { + if (IsTerminating()) + { + // Replaced by another agent for this cc node + // group. SendMessage already writes nothing more; + // this stops the reading too: the loop sends only + // when a batch of this cc node group's records + // fills, so it may read far between sends, and + // the replacing request waits for this agent to + // stop in the log group's apply loop. + data_log_send_err.store(ECANCELED, + std::memory_order_relaxed); + return; + } cnt++; const Item &item = iterator->GetItem(i); if (item.item_type_ == LogItemType::DataLog) @@ -569,6 +617,15 @@ class LogShippingAgent bool reconnect_if_fail, int &eagain) { + if (IsTerminating()) + { + // Replaced by another agent for this cc node group, which ships + // from the start: nothing more goes out on this stream. Every + // message passes here (the replay's batches, its DDL and finish + // messages, lock-recovery records), so this alone keeps a replaced + // agent silent. + return ECANCELED; + } buf.clear(); butil::IOBufAsZeroCopyOutputStream wrapper(&buf); msg.SerializeToZeroCopyStream(&wrapper); diff --git a/eloq_log_service/include/log_state.h b/eloq_log_service/include/log_state.h index e55ec855f..8cc99d728 100644 --- a/eloq_log_service/include/log_state.h +++ b/eloq_log_service/include/log_state.h @@ -4,16 +4,20 @@ #include #include +#include #include #include #include +#include #include +#include #include #include #include #include #include +#include "cdc_scan.h" #include "log.pb.h" namespace txlog @@ -23,7 +27,8 @@ enum struct LogItemType DataLog, SchemaLog, SplitRangeLog, - ClusterScaleLog + ClusterScaleLog, + ClusterGateLog ///< The log group's cluster gate slot, while it has one. }; struct Item @@ -114,6 +119,27 @@ class LogState virtual std::pair SearchTxDataLog( uint64_t tx_number, uint32_t cc_ng_id, uint64_t lower_bound_ts = 0) = 0; + /** + * @brief Serves one page of the CDC log scan (cdc_scan.h). + * + * Called from the RPC thread, concurrently with on_apply; only the leader + * calls it. Every implementation must read MinCkptTs() for + * `arrival_complete_ts` before it opens its view of the stores. The + * default is for states that keep no scannable history. + * + * @param request The scan request. + * @param[out] response The page; its `status` is left to the caller. + * @param[out] error Set to the reason when the result is not Ok. + * @return How the scan went; see CdcScanStatus. + */ + virtual CdcScanStatus CdcScan(const CdcScanRequest & /*request*/, + CdcScanResponse * /*response*/, + std::string *error) + { + *error = "this log state keeps no history a CDC scan could read"; + return CdcScanStatus::Unsupported; + } + /** * Stores cc node group's latest state. * @@ -200,6 +226,17 @@ class LogState LOG(INFO) << "duplicate prepare log detected, txn: " << tx_no << ", ignore"; } + else + { + // An index op holds the catalog write lock across its prepare + // log, so its CDC record goes in here, at the timestamp that + // log carries: the op's catalog version. + const uint32_t op_type = CdcDdlOpType(schema_op); + if (op_type == kCdcAddIndex || op_type == kCdcDropIndex) + { + AppendCdcDdl(commit_ts, commit_ts, tx_no, schema_op); + } + } } else { @@ -210,11 +247,11 @@ class LogState CatalogOp &catalog_op = catalog_it->second; if (new_stage > catalog_op.SchemasOpStage()) { - // For the schema operation that need to deal with the data, - // such as ADD INDEX. if (new_stage == SchemaOpMessage_Stage:: SchemaOpMessage_Stage_PrepareData) { + // For the schema operation that need to deal with the + // data, such as ADD INDEX. SchemaOpMessage &schema_op_msg = *catalog_op.GetSchemaOpMsg( schema_op.table_name_str(), @@ -225,26 +262,49 @@ class LogState schema_op.last_key_value()); schema_op_msg.set_new_catalog_blob( schema_op.new_catalog_blob()); + schema_op_msg.set_stage(new_stage); } - - // Encounter flush error after write prepare log. Need to - // set commit_ts_ to 0(previously set by prepare_log) - if (commit_ts == 0 && - new_stage == SchemaOpMessage_Stage:: - SchemaOpMessage_Stage_CommitSchema) + else if (new_stage == + SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_CommitSchema) { - catalog_op.SetCommitTs(0); + // The op's catalog version, the timestamp its prepare + // log carried, before the commit log's replaces it. + const uint64_t catalog_ts = catalog_op.CommitTs(); + // The commit log's timestamp replaces the prepare + // log's: 0 for an op that failed after its prepare + // log, else the commit timestamp, which CREATE, DROP + // and TRUNCATE TABLE re-assign at the commit stage. An + // op recovered from its commit log then resumes with + // the timestamp it committed at. + catalog_op.SetCommitTs(commit_ts); + SchemaOpMessage &schema_op_msg = + *catalog_op.GetSchemaOpMsg( + schema_op.table_name_str(), + schema_op.table_type()); + // A rolled-back add index commits at a real timestamp + // too; recovery reads the mark instead. + if (schema_op.rolled_back()) + { + schema_op_msg.set_rolled_back(true); + } + schema_op_msg.set_stage(new_stage); + // The commit log itself carries neither the engine + // nor the op type; the prepare log's message does. + RecordCdcDdlCommit( + tx_no, catalog_ts, commit_ts, schema_op_msg); } - - // Schema logs at CleanSchema stage will be kept in LogState - // for some time instead of be erased immediately. This is - // to filter those retried stale WriteLogRequest of previous - // stage (prepare log) yet come to log service after clean - // log finished. The schema log will be erased when all node - // group's ckpt_ts are one hour greater than its commit ts. - if (new_stage == SchemaOpMessage_Stage:: - SchemaOpMessage_Stage_CleanSchema) + else if (new_stage == SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_CleanSchema) { + // Schema logs at CleanSchema stage will be kept in + // LogState for some time instead of be erased + // immediately. This is to filter those retried stale + // WriteLogRequest of previous stage (prepare log) yet + // come to log service after clean log finished. The + // schema log will be erased when all node group's + // ckpt_ts are one hour greater than its commit ts. + // // For pure DDL transactions, CatalogOp contains only // one catalog image. For logical alter table DDL inside // DML transactions, CatalogOp might contain multiple @@ -254,13 +314,6 @@ class LogState catalog_op.Clear(schema_op.table_name_str(), schema_op.table_type()); } - else - { - catalog_op - .GetSchemaOpMsg(schema_op.table_name_str(), - schema_op.table_type()) - ->set_stage(new_stage); - } } else if (new_stage == catalog_op.SchemasOpStage()) { @@ -957,10 +1010,19 @@ class LogState snapshot_tx_catalog_ops_.clear(); snapshot_tx_split_range_ops_.clear(); snapshot_tx_cluster_scale_ops_.clear(); + snapshot_cdc_ddl_history_.clear(); + snapshot_cluster_gate_.reset(); } virtual void Clear() { + // RecoverTx and CDC scans read this state from RPC threads, and a + // scan begun while this node led can outlast its step-down. + std::unique_lock lk(log_state_mutex_); + cdc_ddl_history_.clear(); + snapshot_cdc_ddl_history_.clear(); + cluster_gate_.reset(); + snapshot_cluster_gate_.reset(); cc_ng_info_.clear(); snapshot_cc_ng_info_.clear(); tx_catalog_ops_.clear(); @@ -1076,6 +1138,85 @@ class LogState return min_ckpt_ts_; } + /** + * @brief Sets how far behind the checkpoint the data log is kept for CDC + * consumers (docs/cdc-change-stream.md in the EloqDoc repo, §4.5). + * + * @param seconds The retention window; 0 keeps no window, which is how + * the purges behaved before it existed. + */ + void SetCdcRetentionSeconds(uint64_t seconds) + { + cdc_retention_us_.store(seconds * 1000 * 1000, + std::memory_order_relaxed); + } + + /** + * @brief The CDC retention floor for a checkpoint. + * + * A purge may remove a file only if every key in it lies below the + * floor, so every record within the retention window behind the + * checkpoint stays readable. + * + * @param ckpt_ts The checkpoint timestamp, in microseconds like every + * commit timestamp. + * @return `ckpt_ts` less the retention window, or 0 if the window reaches + * past it. + */ + uint64_t CdcRetentionFloor(uint64_t ckpt_ts) const + { + const uint64_t retention_us = + cdc_retention_us_.load(std::memory_order_relaxed); + return ckpt_ts > retention_us ? ckpt_ts - retention_us : 0; + } + + /** + * @brief Drops the CDC DDL history's records below a data log purge's + * floor. + * + * Called by the purges of the data log (the RocksDB state's SST purge, the + * cloud state's move to the archive) whenever one runs, with the floor it + * purges by, so that DDL records leave with the data they are scanned + * with. Trimming at every checkpoint instead would drop each record as + * soon as a scan could return it unless a retention window is set. Runs + * on the purge's own thread. + * + * @param floor The purge's floor: records whose ddl_ts is below it go. + */ + void TrimCdcDdlHistory(uint64_t floor) + { + std::unique_lock lk(log_state_mutex_); + cdc_ddl_history_.erase( + cdc_ddl_history_.begin(), + cdc_ddl_history_.lower_bound(CdcTimestampPosition(floor))); + } + + /** + * @brief Copies part of the CDC DDL history, the source of a scan's DDL + * stream (CdcDdlHistoryCursor). + * + * Called from the RPC thread, concurrently with on_apply. + * + * @param after A DDL-stream position: empty, a timestamp + * (CdcTimestampPosition) or a record's position (CdcDdlPosition). + * @param upper_ts The highest ddl_ts to copy. + * @param[out] records Appended with every record positioned strictly + * after `after` whose ddl_ts is at most `upper_ts`, in position + * order. + */ + void CopyCdcDdlHistory(const std::string &after, + uint64_t upper_ts, + std::vector *records) const + { + std::shared_lock s_lk(log_state_mutex_); + for (auto it = cdc_ddl_history_.upper_bound(after); + it != cdc_ddl_history_.end() && it->second.ddl_ts() <= upper_ts; + ++it) + { + records->push_back(it->second); + } + } + uint64_t LastCkptTimestamp(uint32_t cc_ng_id) { // this func is also called in RecoverTx rpc thread, which might be @@ -1113,6 +1254,98 @@ class LogState return 0; }; +protected: + /** + * @brief The snapshot file of the CDC DDL history, in the states that + * serve CDC scans. + */ + static constexpr const char *kCdcDdlHistoryFile = "cdc_ddl_history"; + + /** + * @brief The snapshot file of the cluster gate slot, in the states that + * serve CDC scans. + */ + static constexpr const char *kClusterGateFile = "cluster_gate"; + + /** + * @brief A scan's DDL stream over this state's CDC DDL history. + * + * @return A cursor that copies from the history when it is positioned. + */ + CdcDdlHistoryCursor CdcDdlStream() const + { + return CdcDdlHistoryCursor( + [this](const std::string &after, + uint64_t upper_ts, + std::vector *records) + { CopyCdcDdlHistory(after, upper_ts, records); }); + } + +public: + /** + * @brief Applies a cluster gate log entry (EloqDoc + * docs/cdc-change-stream.md §6.2): the slot takes the entry's value if + * the entry is newer than the value it holds. + * + * The slot is one value overwritten in place, not a per-operation record, + * and is never purged: only a resume clears a pause. Every gate + * transaction is pinned to log group 0, so only that group's slot is ever + * set, and its successive entries carry increasing commit timestamps. + * + * @param txn The gate transaction. + * @param commit_ts The entry's commit timestamp, the value's version. + * @param msg The gate's new value. + */ + void UpdateClusterGate(uint64_t txn, + uint64_t commit_ts, + const ClusterGateLogMessage &msg) + { + // on_apply writes; RecoverTx reads from an RPC thread. + std::unique_lock lk(log_state_mutex_); + if (cluster_gate_.has_value() && cluster_gate_->version >= commit_ts) + { + return; + } + cluster_gate_ = ClusterGateSlot{msg, commit_ts, txn}; + } + + /** + * @brief Whether a gate transaction's log entry was applied, for a write + * log retried after its result went unknown and for orphan lock recovery + * (RecoverTx), which must not clear a logged gate tx's lock. + * + * @param txn The gate transaction. + * @return true if the slot holds that transaction's value. + */ + bool SearchTxClusterGate(uint64_t txn) const + { + std::shared_lock s_lk(log_state_mutex_); + return cluster_gate_.has_value() && cluster_gate_->txn == txn; + } + + /** + * @brief The cluster gate slot, for tests and for the gate's replay. + * + * @param[out] msg The gate's value, when the slot holds one. + * @param[out] version Its commit timestamp. + * @param[out] txn The transaction that set it. + * @return false if the slot is empty. + */ + bool GetClusterGate(ClusterGateLogMessage *msg, + uint64_t *version, + uint64_t *txn) const + { + std::shared_lock s_lk(log_state_mutex_); + if (!cluster_gate_.has_value()) + { + return false; + } + *msg = cluster_gate_->value; + *version = cluster_gate_->version; + *txn = cluster_gate_->txn; + return true; + } + protected: void MakeCopyOfNgInfoAndCatalogOps() { @@ -1120,6 +1353,185 @@ class LogState snapshot_tx_catalog_ops_ = tx_catalog_ops_; snapshot_tx_split_range_ops_ = tx_split_range_ops_; snapshot_tx_cluster_scale_ops_ = tx_cluster_scale_ops_; + // A purge thread may trim the history meanwhile (TrimCdcDdlHistory). + std::shared_lock s_lk(log_state_mutex_); + // Under the lock for the reason GetClusterGateList() gives. + snapshot_cluster_gate_ = cluster_gate_; + snapshot_cdc_ddl_history_.clear(); + snapshot_cdc_ddl_history_.reserve(cdc_ddl_history_.size()); + for (const auto &[position, record] : cdc_ddl_history_) + { + snapshot_cdc_ddl_history_.push_back(record); + } + } + + /** + * @brief Writes the snapshot's copy of the CDC DDL history. + * + * The history has a file of its own in the states that serve CDC scans, + * so the file the memory state shares with them keeps its format. Layout: + * a u32 record count, then per record a u32 length and the serialized + * CdcDdlRecord, which holds everything its position is rebuilt from. + * + * @param os The snapshot file. + */ + void WriteSnapshotCdcDdlHistoryTo(std::ofstream &os) + { + uint32_t cnt = snapshot_cdc_ddl_history_.size(); + os.write(reinterpret_cast(&cnt), sizeof(uint32_t)); + LOG(INFO) << "write CDC DDL history size: " << cnt; + std::string buf; + for (const CdcDdlRecord &record : snapshot_cdc_ddl_history_) + { + buf.clear(); + record.SerializeToString(&buf); + uint32_t length = buf.size(); + os.write(reinterpret_cast(&length), sizeof(uint32_t)); + os.write(buf.data(), length); + } + } + + /** + * @brief Loads a CDC DDL history that WriteSnapshotCdcDdlHistoryTo() + * wrote, replacing the current one. + * + * @param is The snapshot file. + * @return False if the file is truncated, a record's length runs past the + * file's end, or a record does not parse; the history is then + * left as it was. + */ + bool LoadCdcDdlHistoryFrom(std::ifstream &is) + { + // The file's end bounds every record length read from it, so a + // corrupt length fails the load instead of sizing a huge buffer. + const std::streampos begin = is.tellg(); + is.seekg(0, std::ios::end); + const std::streampos end = is.tellg(); + is.seekg(begin); + + std::map history; + uint32_t cnt; + is.read(reinterpret_cast(&cnt), sizeof(uint32_t)); + if (is.fail()) + { + LOG(ERROR) << "snapshot read failed: could not read the CDC DDL " + "history count"; + return false; + } + std::string buf; + for (uint32_t i = 0; i < cnt; i++) + { + uint32_t length; + is.read(reinterpret_cast(&length), sizeof(uint32_t)); + if (is.fail() || length > end - is.tellg()) + { + LOG(ERROR) << "snapshot read failed: CDC DDL record " << i + << " of " << cnt << " is truncated or corrupt"; + return false; + } + buf.resize(length); + is.read(buf.data(), length); + CdcDdlRecord record; + if (is.fail() || !record.ParseFromString(buf)) + { + LOG(ERROR) << "snapshot read failed: CDC DDL record " << i + << " of " << cnt << " is truncated or corrupt"; + return false; + } + std::string position = + CdcDdlPosition(record.ddl_ts(), record.txn_number()); + history.emplace(std::move(position), std::move(record)); + } + LOG(INFO) << "read CDC DDL history size: " << cnt; + std::unique_lock lk(log_state_mutex_); + cdc_ddl_history_ = std::move(history); + return true; + } + + /** + * @brief Writes the snapshot's copy of the cluster gate slot. + * + * The slot has a file of its own, as the CDC DDL history does, so the + * file the memory state shares keeps its format. Layout: a u8 that is 1 + * when the slot holds a value, then the u64 version, the u64 txn, a u32 + * length and the serialized ClusterGateLogMessage. + * + * @param os The snapshot file. + */ + void WriteSnapshotClusterGateTo(std::ofstream &os) + { + uint8_t present = snapshot_cluster_gate_.has_value() ? 1 : 0; + os.write(reinterpret_cast(&present), sizeof(uint8_t)); + if (!present) + { + return; + } + std::string buf; + snapshot_cluster_gate_->value.SerializeToString(&buf); + uint32_t length = buf.size(); + os.write(reinterpret_cast(&snapshot_cluster_gate_->version), + sizeof(uint64_t)); + os.write(reinterpret_cast(&snapshot_cluster_gate_->txn), + sizeof(uint64_t)); + os.write(reinterpret_cast(&length), sizeof(uint32_t)); + os.write(buf.data(), length); + LOG(INFO) << "write cluster gate, version: " + << snapshot_cluster_gate_->version + << ", paused: " << snapshot_cluster_gate_->value.paused(); + } + + /** + * @brief Loads a cluster gate slot that WriteSnapshotClusterGateTo() + * wrote, replacing the current one. + * + * @param is The snapshot file. + * @return False if the file is truncated, the value's length runs past + * the file's end, or the value does not parse; the slot is then + * left as it was. + */ + bool LoadClusterGateFrom(std::ifstream &is) + { + const std::streampos begin = is.tellg(); + is.seekg(0, std::ios::end); + const std::streampos end = is.tellg(); + is.seekg(begin); + + uint8_t present; + is.read(reinterpret_cast(&present), sizeof(uint8_t)); + if (is.fail() || present > 1) + { + LOG(ERROR) << "snapshot read failed: corrupt cluster gate file"; + return false; + } + std::optional slot; + if (present) + { + ClusterGateSlot read; + uint32_t length; + is.read(reinterpret_cast(&read.version), sizeof(uint64_t)); + is.read(reinterpret_cast(&read.txn), sizeof(uint64_t)); + is.read(reinterpret_cast(&length), sizeof(uint32_t)); + if (is.fail() || length > end - is.tellg()) + { + LOG(ERROR) << "snapshot read failed: the cluster gate is " + "truncated or corrupt"; + return false; + } + std::string buf(length, '\0'); + is.read(buf.data(), length); + if (is.fail() || !read.value.ParseFromString(buf)) + { + LOG(ERROR) << "snapshot read failed: the cluster gate is " + "truncated or corrupt"; + return false; + } + slot = std::move(read); + LOG(INFO) << "read cluster gate, version: " << slot->version + << ", paused: " << slot->value.paused(); + } + std::unique_lock lk(log_state_mutex_); + cluster_gate_ = std::move(slot); + return true; } void WriteSnapshotNgInfoAndCatalogOpsTo(std::ofstream &os) @@ -1536,6 +1948,32 @@ class LogState assert(res.size() <= 1); } + /** + * @brief Adds the cluster gate slot to a replay list, as one item, when + * the slot holds a value: a recovering node group reinstalls the gate + * from it, however old the value is. + * + * @param[out] res The replay list. + */ + void GetClusterGateList(std::vector &res) + { + // The list is built while the ReplayLog entry applies, on the thread + // that writes the slot, so no write interleaves today. The lock keeps + // the read safe if a list is ever built elsewhere: a torn copy of the + // protobuf would crash, not ship a stale item. + std::shared_lock s_lk(log_state_mutex_); + if (!cluster_gate_.has_value()) + { + return; + } + std::string gate_str; + cluster_gate_->value.AppendToString(&gate_str); + res.emplace_back(std::make_shared(cluster_gate_->txn, + cluster_gate_->version, + std::move(gate_str), + LogItemType::ClusterGateLog)); + } + bool UpdateMinCkptTsOfAllNodeGroups() { uint64_t min_ts = UINT64_MAX; @@ -1617,12 +2055,179 @@ class LogState } } + /** + * @name CDC DDL history + * txservice::OperationType values of the schema ops the history records; + * the log service does not include the tx service's headers. + * @{ + */ + static constexpr uint32_t kCdcCreateTable = 5; ///< CreateTable. + static constexpr uint32_t kCdcDropTable = 6; ///< DropTable. + static constexpr uint32_t kCdcAddIndex = 8; ///< AddIndex. + static constexpr uint32_t kCdcDropIndex = 9; ///< DropIndex. + /** @} */ + + /** + * @brief Which kind of CDC DDL record a schema op's message makes, if + * any. + * + * Only EloqDoc's CREATE and DROP TABLE and its index ops are recorded. + * Index ops arrive as table ops too, with their index names in + * alter_table_info_blob. Not recorded: ALTER (OperationType::Update, how + * collMod reaches the tx service), TRUNCATE, other engines, and the + * logical alters logged inside DML. + * + * @param msg The schema op's message as the prepare log carried it. + * @return The op type (kCdcCreateTable, kCdcDropTable, kCdcAddIndex or + * kCdcDropIndex), or 0 if the op is not recorded. + */ + static uint32_t CdcDdlOpType(const SchemaOpMessage &msg) + { + if (msg.table_engine() != TableEngine::EloqDoc || !msg.has_table_op()) + { + return 0; + } + const uint32_t op_type = msg.table_op().op_type(); + return op_type == kCdcCreateTable || op_type == kCdcDropTable || + op_type == kCdcAddIndex || op_type == kCdcDropIndex + ? op_type + : 0; + } + + /** + * @brief Adds a record to the CDC DDL history. Caller holds the unique + * lock. + * + * The record must be appended above the log group's checkpoint + * (docs/cdc-change-stream.md in the EloqDoc repo, §4.4): a scan may + * already have passed any lower timestamp. A replayed log entry appends + * the same record at the same position again, which leaves the history + * as it was. + * + * @param ddl_ts The record's stream timestamp. + * @param catalog_ts The op's catalog version, the timestamp its prepare + * log carried. + * @param txn The schema op's transaction number. + * @param msg The schema op's message. + */ + void AppendCdcDdl(uint64_t ddl_ts, + uint64_t catalog_ts, + uint64_t txn, + const SchemaOpMessage &msg) + { + if (ddl_ts <= min_ckpt_ts_) + { + LOG(WARNING) << "CDC DDL record of txn " << txn << " at " << ddl_ts + << " is not above the checkpoint " << min_ckpt_ts_ + << "; a CDC scan may already have passed it"; + } + CdcDdlRecord record; + record.set_ddl_ts(ddl_ts); + record.set_catalog_ts(catalog_ts); + record.set_txn_number(txn); + *record.mutable_schema_op() = msg; + cdc_ddl_history_.try_emplace(CdcDdlPosition(ddl_ts, txn), + std::move(record)); + } + + /** + * @brief Updates the CDC DDL history for a schema op's commit log. + * Caller holds the unique lock. + * + * CREATE and DROP TABLE take the catalog write lock only before their + * commit log, so their record goes in here, at the commit log's + * timestamp; rolled back (at timestamp 0) they make none. An index op's + * record went in at its prepare log: a rollback at timestamp 0 takes it + * out again, and one at a timestamp above 0, the add index whose build + * broke a constraint, is a record of its own there, carrying the + * rolled-back mark, from which the runtime makes the dropIndexes event. + * + * @param txn The schema op's transaction number. + * @param catalog_ts The op's catalog version, the timestamp its prepare + * log carried. + * @param commit_ts The commit log's timestamp. + * @param msg The op's message as the log state holds it after the commit + * log, with its rolled-back mark. + */ + void RecordCdcDdlCommit(uint64_t txn, + uint64_t catalog_ts, + uint64_t commit_ts, + const SchemaOpMessage &msg) + { + const uint32_t op_type = CdcDdlOpType(msg); + if (op_type == kCdcCreateTable || op_type == kCdcDropTable) + { + if (commit_ts != 0 && !msg.rolled_back()) + { + AppendCdcDdl(commit_ts, catalog_ts, txn, msg); + } + } + else if (op_type == kCdcAddIndex || op_type == kCdcDropIndex) + { + if (commit_ts == 0) + { + cdc_ddl_history_.erase(CdcDdlPosition(catalog_ts, txn)); + } + else if (msg.rolled_back()) + { + AppendCdcDdl(commit_ts, catalog_ts, txn, msg); + } + } + } + + /** + * @brief The CDC DDL history (docs/cdc-change-stream.md in the EloqDoc + * repo, §4.3): one record per recorded schema op, two for a rolled-back + * add index, keyed by position (CdcDdlPosition), so that key order is + * (ddl_ts, txn) order. + * + * on_apply and the purges write it under the unique lock; scans copy + * from it under the shared lock. Kept in memory and saved in the raft + * snapshot by the states that serve CDC scans, and trimmed by the purges + * of the data log, by their floor (TrimCdcDdlHistory). The memory state + * purges nothing, and keeps its history too. + */ + std::map cdc_ddl_history_; + /** + * @brief The history's records as of the snapshot being saved, in + * position order. A plain copy: the snapshot only writes them out, and + * loading rebuilds each record's position. + */ + std::vector snapshot_cdc_ddl_history_; + + /** + * @brief The cluster gate slot's value, as UpdateClusterGate keeps it. + */ + struct ClusterGateSlot + { + ClusterGateLogMessage value; ///< The gate's latest value. + uint64_t version{0}; ///< Commit timestamp of its entry. + uint64_t txn{0}; ///< The gate transaction that set it. + }; + + /** + * @brief The cluster gate slot (EloqDoc docs/cdc-change-stream.md + * §6.2): set by cluster gate log entries, saved in the raft snapshot by + * the states that serve CDC scans, replayed to recovering node groups, + * and never purged. Empty until the first gate entry, and in every log + * group but group 0. + */ + std::optional cluster_gate_; + /** @brief The slot as of the snapshot being saved. */ + std::optional snapshot_cluster_gate_; + std::unordered_map cc_ng_info_; std::unordered_map snapshot_cc_ng_info_; // to erase finished schema ops and split range ops after one hour uint64_t min_ckpt_ts_{}; + /** + * @brief The CDC retention window in microseconds. Set once at start and + * read by the purge threads, hence atomic. + */ + std::atomic cdc_retention_us_{0}; + struct CatalogOp { CatalogOp(const SchemaOpMessage &schema_op, uint64_t commit_ts) diff --git a/eloq_log_service/include/log_state_rocksdb_impl.h b/eloq_log_service/include/log_state_rocksdb_impl.h index a52a7916a..8a614115c 100644 --- a/eloq_log_service/include/log_state_rocksdb_impl.h +++ b/eloq_log_service/include/log_state_rocksdb_impl.h @@ -322,6 +322,10 @@ class LogStateRocksDBImpl : public LogState uint32_t ng_id, uint64_t lower_bound_ts = 0) override; + CdcScanStatus CdcScan(const CdcScanRequest &request, + CdcScanResponse *response, + std::string *error) override; + void BeginSnapshot() override; /** @@ -361,6 +365,17 @@ class LogStateRocksDBImpl : public LogState private: void StopRocksDB(); + /** + * @brief Stops new CDC scans from reading db_ and waits until the scans + * already reading it have finished, so that db_ can be closed. + * + * Polls with bthread_usleep rather than blocking on a lock: it runs on the + * braft FSM bthread (snapshot load) and must not stall a worker. The wait + * is bounded by one page read per scan in flight. Start() lets scans in + * again once db_ is reopened. + */ + void StopCdcScans(); + static void PrintKey(rocksdb::Slice key); /** @@ -377,6 +392,16 @@ class LogStateRocksDBImpl : public LogState * looking for the log record committed by a specific tx. */ rocksdb::DB *db_; + /** + * @brief Whether CDC scans may read db_: set once Start() has opened it, + * cleared by StopCdcScans() before db_ is closed. + */ + std::atomic cdc_scans_allowed_{false}; + /** + * @brief CDC scans that have registered to read db_ and not yet finished. + * StopCdcScans() waits for it to reach zero. + */ + std::atomic cdc_scans_in_flight_{0}; std::string rocksdb_storage_path_; rocksdb::WriteOptions write_option_; rocksdb::WriteBatch write_batch_; @@ -674,12 +699,16 @@ class LogStateRocksDBCloudImpl : public LogState uint32_t ng_id, uint64_t lower_bound_ts = 0) override; + CdcScanStatus CdcScan(const CdcScanRequest &request, + CdcScanResponse *response, + std::string *error) override; + void BeginSnapshot() override; void CleanSnapshotState() override; static void MoveFilesToArchive( const std::shared_ptr &dbc_move_log, - uint64_t min_ckpt_ts, + uint64_t purge_ts, const RocksDBCloudConfig &cloud_config); static void PurgeArchiveFiles( diff --git a/eloq_log_service/src/cdc_long_poll.cpp b/eloq_log_service/src/cdc_long_poll.cpp new file mode 100644 index 000000000..bac5e9087 --- /dev/null +++ b/eloq_log_service/src/cdc_long_poll.cpp @@ -0,0 +1,214 @@ +/** + * @file + * The CdcScan long poll; see cdc_long_poll.h. + */ +#include "cdc_long_poll.h" + +#include +#include +#include +#include + +#include +#include + +namespace txlog +{ +/** + * @brief One request's state. `completion` and `completed` are set once; the + * list position and the timer fields are guarded by the poll's mutex. + */ +struct CdcLongPoll::Request +{ + /** + * @brief The poll it was prepared by, which outlives it until its + * completion has returned. + */ + CdcLongPoll *poll{nullptr}; + Completion completion; ///< Runs once, when the request ends. + std::atomic completed{false}; ///< Taken by the source that wins. + bool parked{false}; ///< In the poll's list. + std::list>::iterator position; ///< Its entry. + bool has_timer{false}; ///< A deadline timer is set. + bthread_timer_t timer{}; ///< The deadline timer. + TimerArg *timer_arg{nullptr}; ///< The timer's argument. +}; + +CdcLongPoll::~CdcLongPoll() +{ + { + std::lock_guard lk(mutex_); + stopped_ = true; + } + CompleteAll(Wake::kStop); + // Completions and timer callbacks use this object; a prepared request is + // parked, and so completed, by its caller right after its preparation. + while (pending_.load(std::memory_order_acquire) != 0) + { + bthread_usleep(1000); + } +} + +std::shared_ptr CdcLongPoll::Prepare( + Completion completion) +{ + auto request = std::make_shared(); + request->poll = this; + request->completion = std::move(completion); + pending_.fetch_add(1, std::memory_order_acq_rel); + return request; +} + +void CdcLongPoll::Park(const std::shared_ptr &request, + uint32_t wait_ms) +{ + { + std::lock_guard lk(mutex_); + if (request->completed.load(std::memory_order_acquire)) + { + return; + } + if (!stopped_) + { + request->position = parked_.insert(parked_.end(), request); + request->parked = true; + if (size_observer_) + { + size_observer_(parked_.size()); + } + auto *arg = new TimerArg{this, request}; + // The callback may run at once, on the timer thread; it waits for + // this lock before it touches the request. + pending_.fetch_add(1, std::memory_order_acq_rel); + if (bthread_timer_add(&request->timer, + butil::milliseconds_from_now(wait_ms), + &CdcLongPoll::OnDeadline, + arg) == 0) + { + request->has_timer = true; + request->timer_arg = arg; + } + else + { + // Without a timer the request still ends at the next advance + // of arrival_complete_ts or at the step-down. + LOG(WARNING) << "CdcScan long poll: no deadline timer for a " + "parked scan"; + pending_.fetch_sub(1, std::memory_order_acq_rel); + delete arg; + } + return; + } + } + Complete(request, Wake::kStop); +} + +void CdcLongPoll::Complete(const std::shared_ptr &request, Wake why) +{ + if (request->completed.exchange(true, std::memory_order_acq_rel)) + { + return; + } + Finish(request, why); +} + +void CdcLongPoll::CompleteAll(Wake why) +{ + std::vector> woken; + { + std::lock_guard lk(mutex_); + woken.assign(parked_.begin(), parked_.end()); + } + for (const std::shared_ptr &request : woken) + { + Complete(request, why); + } +} + +void CdcLongPoll::Cancel(std::shared_ptr request) +{ + // Once the call has ended the request is completed and its poll may be + // gone, so only the winner of the flag touches the poll; the poll waits + // for every uncompleted request before it is destroyed. + if (request->completed.exchange(true, std::memory_order_acq_rel)) + { + return; + } + request->poll->Finish(request, Wake::kCancel); +} + +size_t CdcLongPoll::Size() const +{ + std::lock_guard lk(mutex_); + return parked_.size(); +} + +void CdcLongPoll::SetSizeObserver(SizeObserver observer) +{ + std::lock_guard lk(mutex_); + size_observer_ = std::move(observer); + if (size_observer_) + { + size_observer_(parked_.size()); + } +} + +void CdcLongPoll::OnDeadline(void *arg) +{ + auto *timer_arg = static_cast(arg); + CdcLongPoll *poll = timer_arg->poll; + poll->Complete(timer_arg->request, Wake::kDeadline); + delete timer_arg; + poll->pending_.fetch_sub(1, std::memory_order_acq_rel); +} + +void *CdcLongPoll::RunCompletion(void *arg) +{ + auto *task = static_cast(arg); + CdcLongPoll *poll = task->poll; + task->request->completion(task->why); + task->request->completion = nullptr; + delete task; + poll->pending_.fetch_sub(1, std::memory_order_acq_rel); + return nullptr; +} + +void CdcLongPoll::Finish(const std::shared_ptr &request, Wake why) +{ + TimerArg *timer_arg = nullptr; + bthread_timer_t timer{}; + { + std::lock_guard lk(mutex_); + if (request->parked) + { + parked_.erase(request->position); + request->parked = false; + if (size_observer_) + { + size_observer_(parked_.size()); + } + } + if (request->has_timer) + { + request->has_timer = false; + timer = request->timer; + timer_arg = request->timer_arg; + } + } + // A timer deleted before it ran never runs, and its argument is freed + // here; a running or finished one frees its own. + if (timer_arg != nullptr && bthread_timer_del(timer) == 0) + { + delete timer_arg; + pending_.fetch_sub(1, std::memory_order_acq_rel); + } + + bthread_t tid; + auto *task = new Task{this, request, why}; + if (bthread_start_background( + &tid, nullptr, &CdcLongPoll::RunCompletion, task) != 0) + { + RunCompletion(task); + } +} +} // namespace txlog diff --git a/eloq_log_service/src/cdc_scan.cpp b/eloq_log_service/src/cdc_scan.cpp new file mode 100644 index 000000000..69552a5f5 --- /dev/null +++ b/eloq_log_service/src/cdc_scan.cpp @@ -0,0 +1,451 @@ +#include "cdc_scan.h" + +#include +#include +#include + +#include "log_utils.h" + +#if defined(LOG_STATE_TYPE_RKDB_ALL) +#include +#include +#include +#endif + +namespace txlog +{ +namespace +{ +/** @brief Page size when the request asks for none (`max_bytes == 0`). */ +constexpr size_t kDefaultCdcScanPageBytes = 4 * 1024 * 1024; +/** + * @brief Largest page size a request may ask for. A page exceeds it only when + * one record alone does. + */ +constexpr size_t kMaxCdcScanPageBytes = 32 * 1024 * 1024; + +/** @brief What a data record adds to the response besides its blob. */ +constexpr size_t kCdcDataRecordOverhead = 32; + +/** + * @brief Length of a data-stream storage key: + * `bswap64(ts) | bswap32(ng) | bswap64(txn)`. + */ +constexpr size_t kDataKeySize = 20; +/** @brief Length of a timestamp position (see CdcScanPosition). */ +constexpr size_t kTimestampSize = sizeof(uint64_t); + +/** + * @brief Appends a big-endian u16. + * + * @param[out] out The buffer to append to. + * @param value The value; only its low 16 bits are written. + */ +void AppendU16(std::string *out, size_t value) +{ + out->push_back(static_cast((value >> 8) & 0xff)); + out->push_back(static_cast(value & 0xff)); +} + +/** + * @brief Reads one `u16 len | bytes` field off the front of a buffer. + * + * @param[in,out] in The buffer; the field is removed from its front. + * @param[out] out Set to the field's bytes. + * @return False if the buffer ends inside the field. + */ +bool ReadLengthPrefixed(std::string_view *in, std::string *out) +{ + if (in->size() < 2) + { + return false; + } + const size_t len = + (static_cast(static_cast((*in)[0])) << 8) | + static_cast((*in)[1]); + in->remove_prefix(2); + if (in->size() < len) + { + return false; + } + out->assign(in->data(), len); + in->remove_prefix(len); + return true; +} +} // namespace + +std::string EncodeCdcContinuation(const CdcScanPosition &position) +{ + std::string out; + out.reserve(4 + position.data.size() + position.ddl.size()); + AppendU16(&out, position.data.size()); + out.append(position.data); + AppendU16(&out, position.ddl.size()); + out.append(position.ddl); + return out; +} + +bool DecodeCdcContinuation(std::string_view bytes, + CdcScanPosition *position, + std::string *error) +{ + CdcScanPosition decoded; + if (!ReadLengthPrefixed(&bytes, &decoded.data) || + !ReadLengthPrefixed(&bytes, &decoded.ddl)) + { + *error = "continuation is truncated"; + return false; + } + if (!bytes.empty()) + { + *error = "continuation has trailing bytes"; + return false; + } + // A data position is empty, a timestamp, or a real data key; a DDL + // position is empty, a timestamp, or a record's position (CdcDdlPosition), + // which is always longer than its timestamp prefix. + const size_t data_len = decoded.data.size(); + if (data_len != 0 && data_len != kTimestampSize && data_len != kDataKeySize) + { + *error = "continuation holds a data-stream key of " + + std::to_string(data_len) + " bytes"; + return false; + } + const size_t ddl_len = decoded.ddl.size(); + if (ddl_len != 0 && ddl_len < kTimestampSize) + { + *error = "continuation holds a DDL-stream key of " + + std::to_string(ddl_len) + " bytes"; + return false; + } + *position = std::move(decoded); + return true; +} + +std::string CdcTimestampPosition(uint64_t ts) +{ + const uint64_t be = __builtin_bswap64(ts); + return std::string(reinterpret_cast(&be), sizeof(be)); +} + +std::string CdcDdlPosition(uint64_t ddl_ts, uint64_t txn) +{ + std::string position = CdcTimestampPosition(ddl_ts); + position.append(CdcTimestampPosition(txn)); + return position; +} + +void CdcDdlHistoryCursor::Seek(const std::string &after, uint64_t upper_ts) +{ + records_.clear(); + pos_ = 0; + source_(after, upper_ts, &records_); + LoadKey(); +} + +void CdcDdlHistoryCursor::AppendTo(CdcScanResponse *response) const +{ + *response->add_ddl() = records_[pos_]; +} + +void CdcDdlHistoryCursor::Next() +{ + ++pos_; + LoadKey(); +} + +void CdcDdlHistoryCursor::LoadKey() +{ + if (Valid()) + { + key_ = CdcDdlPosition(records_[pos_].ddl_ts(), + records_[pos_].txn_number()); + } +} + +bool ParseCdcScanStart(const CdcScanRequest &request, + CdcScanPosition *start, + std::string *error) +{ + switch (request.start_case()) + { + case CdcScanRequest::kContinuation: + return DecodeCdcContinuation(request.continuation(), start, error); + case CdcScanRequest::kFromTs: + { + // Timestamp 0 precedes every record, which is the empty position. + const uint64_t from_ts = request.from_ts(); + std::string position = + from_ts == 0 ? std::string() : CdcTimestampPosition(from_ts); + start->data = position; + start->ddl = std::move(position); + return true; + } + case CdcScanRequest::START_NOT_SET: + default: + // The beginning of complete history. With the history boundary not + // tracked yet, that is the beginning of each store. + start->data.clear(); + start->ddl.clear(); + return true; + } +} + +size_t CdcScanPageBytes(uint32_t requested) +{ + if (requested == 0) + { + return kDefaultCdcScanPageBytes; + } + return std::min(requested, kMaxCdcScanPageBytes); +} + +bool RunCdcScan(CdcStreamCursor &data, + CdcStreamCursor &ddl, + const CdcScanPosition &start, + uint64_t arrival_complete_ts, + size_t page_bytes, + CdcScanResponse *response, + std::string *error) +{ + response->set_arrival_complete_ts(arrival_complete_ts); + response->set_retention_floor_ts(0); + + data.Seek(start.data, arrival_complete_ts); + ddl.Seek(start.ddl, arrival_complete_ts); + + // A stream that returns nothing keeps its starting position, which is + // exact: every record it holds at or below the bound lies after it. + CdcScanPosition position = start; + size_t bytes = 0; + bool has_more = false; + bool page_empty = true; + while (data.Ok() && ddl.Ok()) + { + const bool data_valid = data.Valid(); + const bool ddl_valid = ddl.Valid(); + if (!data_valid && !ddl_valid) + { + // Both streams are exhausted through the bound. + break; + } + // Data first at one timestamp; see the merge order in cdc_scan.h. + const bool take_data = + data_valid && (!ddl_valid || data.Timestamp() <= ddl.Timestamp()); + CdcStreamCursor &next = take_data ? data : ddl; + + const size_t size = next.ByteSize(); + if (!page_empty && bytes + size > page_bytes) + { + // `next` is a record at or below the bound beyond this page. + has_more = true; + break; + } + next.AppendTo(response); + bytes += size; + page_empty = false; + std::string &stream_position = take_data ? position.data : position.ddl; + stream_position.assign(next.Key().data(), next.Key().size()); + next.Next(); + } + + if (!data.Ok()) + { + *error = "data stream: " + data.ErrorText(); + return false; + } + if (!ddl.Ok()) + { + *error = "DDL stream: " + ddl.ErrorText(); + return false; + } + + response->set_has_more(has_more); + response->set_continuation(EncodeCdcContinuation(position)); + return true; +} + +#if defined(LOG_STATE_TYPE_RKDB_ALL) +namespace +{ +/** + * @param p Eight bytes holding a big-endian timestamp. + * @return The timestamp. + */ +uint64_t ReadBigEndianTimestamp(const char *p) +{ + uint64_t be; + std::memcpy(&be, p, sizeof(be)); + return __builtin_bswap64(be); +} + +/** + * The data stream of a RocksDB-backed log state: the default column family, + * keyed `bswap64(commit_ts) | bswap32(ng) | bswap64(txn)` with the per-ng + * blob as the value. The iterator it opens at Seek is one consistent view of + * the database. + */ +class RocksDBDataCursor final : public CdcStreamCursor +{ +public: + explicit RocksDBDataCursor(rocksdb::DB *db) : db_(db) + { + } + + void Seek(const std::string &after, uint64_t upper_ts) override + { + rocksdb::ReadOptions options; + // The bounds must outlive the iterator, hence the members. + if (upper_ts != UINT64_MAX) + { + // Keys at or below upper_ts sort before the timestamp key of + // upper_ts + 1, and keys at upper_ts + 1 after it. + upper_key_ = CdcTimestampPosition(upper_ts + 1); + upper_bound_ = rocksdb::Slice(upper_key_); + options.iterate_upper_bound = &upper_bound_; + } + if (!after.empty()) + { + lower_key_ = after; + lower_bound_ = rocksdb::Slice(lower_key_); + options.iterate_lower_bound = &lower_bound_; + } + iter_.reset(db_->NewIterator(options)); + if (after.empty()) + { + iter_->SeekToFirst(); + } + else + { + iter_->Seek(lower_bound_); + // A real key position was returned already; resume after it. + if (iter_->Valid() && iter_->key() == lower_bound_) + { + iter_->Next(); + } + } + Load(); + } + + bool Valid() const override + { + return error_.empty() && iter_ != nullptr && iter_->Valid(); + } + + bool Ok() const override + { + return error_.empty() && (iter_ == nullptr || iter_->status().ok()); + } + + std::string ErrorText() const override + { + if (!error_.empty()) + { + return error_; + } + return iter_ == nullptr ? std::string() : iter_->status().ToString(); + } + + uint64_t Timestamp() const override + { + return ts_; + } + + std::string_view Key() const override + { + const rocksdb::Slice key = iter_->key(); + return std::string_view(key.data(), key.size()); + } + + size_t ByteSize() const override + { + return iter_->value().size() + kCdcDataRecordOverhead; + } + + void AppendTo(CdcScanResponse *response) const override + { + CdcDataRecord *record = response->add_data(); + record->set_commit_ts(ts_); + record->set_cc_ng_id(ng_); + record->set_txn_number(txn_); + const rocksdb::Slice value = iter_->value(); + record->set_blob(value.data(), value.size()); + } + + void Next() override + { + iter_->Next(); + Load(); + } + +private: + /** + * @brief Decodes the key under the iterator, if any, into ts_, ng_ and + * txn_; a key of the wrong length sets error_. + */ + void Load() + { + if (!iter_->Valid()) + { + return; + } + const rocksdb::Slice key = iter_->key(); + if (key.size() != kDataKeySize) + { + // Nothing else is stored in this column family, so this is + // corruption; stopping keeps it from being skipped silently. + error_ = "data-stream key of " + std::to_string(key.size()) + + " bytes, expected " + std::to_string(kDataKeySize); + return; + } + const char *p = key.data(); + ts_ = ReadBigEndianTimestamp(p); + uint32_t ng_be; + std::memcpy(&ng_be, p + kTimestampSize, sizeof(ng_be)); + ng_ = __builtin_bswap32(ng_be); + uint64_t txn_be; + std::memcpy( + &txn_be, p + kTimestampSize + sizeof(ng_be), sizeof(txn_be)); + txn_ = __builtin_bswap64(txn_be); + } + + rocksdb::DB *db_; ///< Outlives the cursor. + std::string upper_key_; ///< Backs upper_bound_. + std::string lower_key_; ///< Backs lower_bound_. + rocksdb::Slice upper_bound_; ///< Must outlive iter_. + rocksdb::Slice lower_bound_; ///< Must outlive iter_. + std::unique_ptr iter_; ///< Opened by Seek(). + std::string error_; ///< Set by Load() on a bad key. + uint64_t ts_{0}; ///< Current record's commit ts. + uint32_t ng_{0}; ///< Current record's node group. + uint64_t txn_{0}; ///< Current record's txn number. +}; +} // namespace + +CdcScanStatus CdcScanRocksDB(rocksdb::DB *db, + CdcStreamCursor &ddl, + uint64_t arrival_complete_ts, + const CdcScanRequest &request, + CdcScanResponse *response, + std::string *error) +{ + CdcScanPosition start; + if (!ParseCdcScanStart(request, &start, error)) + { + return CdcScanStatus::InvalidArgument; + } + RocksDBDataCursor data(db); + if (!RunCdcScan(data, + ddl, + start, + arrival_complete_ts, + CdcScanPageBytes(request.max_bytes()), + response, + error)) + { + return CdcScanStatus::StorageError; + } + return CdcScanStatus::Ok; +} +#endif +} // namespace txlog diff --git a/eloq_log_service/src/launch_sv.cpp b/eloq_log_service/src/launch_sv.cpp index fcc2acbc7..b48b2f19f 100644 --- a/eloq_log_service/src/launch_sv.cpp +++ b/eloq_log_service/src/launch_sv.cpp @@ -62,6 +62,12 @@ DEFINE_string(notify_checkpointer_threshold_size, "When the size of non-checkpoint txlogs reache this threshold, " "the log_service sends a checkpoint request to tx_service."); +DEFINE_uint64(txlog_cdc_retention_seconds, + 0, + "How far behind the checkpoint the log state keeps the data log " + "for change data capture: no purge removes a record within this " + "window. 0 keeps no window."); + DEFINE_bool(enable_brpc_builtin_services, true, "Enable showing brpc builtin services through http."); @@ -255,7 +261,8 @@ void launch(const std::string &tt_conf, FLAGS_snapshot_interval, FLAGS_enable_request_checkpoint, FLAGS_check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size); + notify_checkpointer_threshold_size, + FLAGS_txlog_cdc_retention_seconds); int start_status = server.Start(FLAGS_enable_brpc_builtin_services); if (start_status != 0) @@ -417,6 +424,12 @@ int main(int argc, char *argv[]) : config_reader.GetString("local", "notify_checkpointer_threshold_size", FLAGS_notify_checkpointer_threshold_size); + FLAGS_txlog_cdc_retention_seconds = + !CheckCommandLineFlagIsDefault("txlog_cdc_retention_seconds") + ? FLAGS_txlog_cdc_retention_seconds + : config_reader.GetInteger("local", + "txlog_cdc_retention_seconds", + FLAGS_txlog_cdc_retention_seconds); FLAGS_enable_brpc_builtin_services = !CheckCommandLineFlagIsDefault("enable_brpc_builtin_services") ? FLAGS_enable_brpc_builtin_services diff --git a/eloq_log_service/src/log_instance.cpp b/eloq_log_service/src/log_instance.cpp index 22f6e3739..4b07a69ac 100644 --- a/eloq_log_service/src/log_instance.cpp +++ b/eloq_log_service/src/log_instance.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -12,6 +13,15 @@ #include "closure.h" #include "log_service.h" +#include "log_service_metrics.h" + +DEFINE_uint32(cdc_scan_max_parked, + 1024, + "Most CdcScan long polls a log group holds at once; beyond it a " + "scan with nothing to return is answered at once"); +DEFINE_uint32(cdc_scan_max_park_ms, + 60000, + "Longest a CdcScan long poll is held, whatever its wait_ms asks"); namespace txlog { @@ -279,6 +289,18 @@ void LogInstance::RecoverTx(const RecoverTxRequest *request, log_state_->SearchTxDataMigrationOp(request->lock_tx_number()); } + if (!has_multi_stage_log) + { + // A gate tx that logged holds the cluster config write lock until + // its coordinator's replay installs the gate and releases it. + // Clearing the lock on a node group that has not installed the gate + // yet would let that group admit DDL during a pause. The slot names + // only the latest gate tx, which is enough: a later one could not + // have logged while this tx's lock was still held anywhere. + has_multi_stage_log = + log_state_->SearchTxClusterGate(request->lock_tx_number()); + } + if (record == nullptr && !has_multi_stage_log) { // No log record found for the specified tx. The tx is deemed aborted. @@ -349,6 +371,171 @@ void LogInstance::CheckMigrationIsFinished( response->set_finished(all_finished); } +void LogInstance::EnableMetrics(metrics::MetricsRegistry *registry, + const metrics::CommonLabels &node_labels) +{ + if (meter_ != nullptr) + { + return; + } + metrics::CommonLabels labels = node_labels; + labels["lg_id"] = std::to_string(log_group_id_); + auto meter = std::make_unique(registry, labels); + meter->Register(metrics::NAME_CDC_ARRIVAL_COMPLETE_TS, + metrics::Type::Gauge); + meter->Register(metrics::NAME_CDC_SCAN_PARKED_REQUESTS, + metrics::Type::Gauge); + metrics::Meter *published = meter.get(); + meter_ = std::move(meter); + published_meter_.store(published, std::memory_order_release); + // The group has been serving since before metrics existed: report where + // arrival_complete_ts already is, rather than 0 until its next advance. + ReportArrivalComplete(); + cdc_long_poll_.SetSizeObserver( + [published](size_t parked) + { + published->Collect(metrics::NAME_CDC_SCAN_PARKED_REQUESTS, + static_cast(parked)); + }); +} + +void LogInstance::ReportArrivalComplete() +{ + std::lock_guard lk(arrival_metric_mutex_); + if (metrics::Meter *meter = + published_meter_.load(std::memory_order_acquire)) + { + meter->Collect(metrics::NAME_CDC_ARRIVAL_COMPLETE_TS, + static_cast(arrival_complete_ts_.load( + std::memory_order_acquire))); + } +} + +void LogInstance::CdcScan(brpc::Controller *cntl, + const CdcScanRequest *request, + CdcScanResponse *response, + google::protobuf::Closure *done) +{ + brpc::ClosureGuard done_guard(done); + if (!ServeCdcScan(cntl, request, response) || request->wait_ms() == 0 || + cdc_long_poll_.Size() >= FLAGS_cdc_scan_max_parked) + { + return; + } + + // Nothing is emittable, and the caller may wait: hold the scan until + // something may have become emittable (cdc_long_poll.h). The completion + // runs once, on a bthread, and ends the call. + const uint64_t bound = response->arrival_complete_ts(); + std::shared_ptr parked = cdc_long_poll_.Prepare( + [this, cntl, request, response, done](CdcLongPoll::Wake why) + { + brpc::ClosureGuard done_guard(done); + switch (why) + { + case CdcLongPoll::Wake::kAdvance: + case CdcLongPoll::Wake::kDeadline: + response->Clear(); + ServeCdcScan(cntl, request, response); + break; + case CdcLongPoll::Wake::kStop: + response->Clear(); + response->set_status(CdcScanResponse::NotLeader); + response->set_redirect(LeaderAddress()); + break; + case CdcLongPoll::Wake::kCancel: + // No one reads the answer; the empty page will do. + break; + } + }); + done_guard.release(); + // Before the park: once parked, the scan may complete and end the call at + // any moment, and the call must not be used after that. + cntl->NotifyOnCancel(brpc::NewCallback(&CdcLongPoll::Cancel, parked)); + // Capped: the cancellation fires only when the connection breaks, not + // when the client gives up on the call, so an abandoned scan would hold + // its slot until the checkpoint next advances, however long it stalls. + cdc_long_poll_.Park( + parked, std::min(request->wait_ms(), FLAGS_cdc_scan_max_park_ms)); + + // A checkpoint applied, or a step-down, since the page was built found + // nothing parked to complete. Both update their state before they + // complete the parked scans, and the park took the same mutex after + // them, so these reads see their updates. + if (term_if_is_lg_leader_.load(butil::memory_order_acquire) < 0) + { + cdc_long_poll_.Complete(parked, CdcLongPoll::Wake::kStop); + } + else if (log_state_->MinCkptTs() > bound) + { + cdc_long_poll_.Complete(parked, CdcLongPoll::Wake::kAdvance); + } +} + +bool LogInstance::ServeCdcScan(brpc::Controller *cntl, + const CdcScanRequest *request, + CdcScanResponse *response) +{ + // A read needs no raft round: a replica that still believes it leads + // but has been deposed serves only what it applied, and its own + // min_ckpt_ts_ bounds that view, so the page stays exact, merely older. + if (term_if_is_lg_leader_.load(butil::memory_order_acquire) < 0) + { + response->set_status(CdcScanResponse::NotLeader); + response->set_redirect(LeaderAddress()); + return false; + } + + std::string error; + switch (log_state_->CdcScan(*request, response, &error)) + { + case CdcScanStatus::Ok: + response->set_status(CdcScanResponse::Success); + return response->data_size() == 0 && response->ddl_size() == 0 && + !response->has_more(); + case CdcScanStatus::NotReady: + LOG(INFO) << "CdcScan on log group " << log_group_id_ + << " is not ready: " << error; + response->Clear(); + response->set_status(CdcScanResponse::NotLeader); + response->set_redirect(ip_ + ":" + std::to_string(port_)); + break; + case CdcScanStatus::NotLeader: + // Leadership moved while the log state served the scan. + response->Clear(); + response->set_status(CdcScanResponse::NotLeader); + response->set_redirect(LeaderAddress()); + break; + case CdcScanStatus::Unsupported: + cntl->SetFailed(brpc::ENOMETHOD, "%s", error.c_str()); + break; + case CdcScanStatus::InvalidArgument: + cntl->SetFailed(brpc::EREQUEST, "%s", error.c_str()); + break; + case CdcScanStatus::StorageError: + LOG(ERROR) << "CdcScan on log group " << log_group_id_ + << " failed reading the log state: " << error; + cntl->SetFailed(brpc::EINTERNAL, "%s", error.c_str()); + break; + } + return false; +} + +std::string LogInstance::LeaderAddress() const +{ + const braft::PeerId leader = node_->leader_id(); + if (leader.is_empty()) + { + return {}; + } + if (leader.type_ == braft::PeerId::Type::HostName) + { + return leader.hostname_addr.hostname + ":" + + std::to_string(leader.hostname_addr.port); + } + return butil::endpoint2str(leader.addr).c_str(); +} + void LogInstance::CheckClusterScaleStatus(const LogRequest *request, LogResponse *response, google::protobuf::Closure *done) @@ -405,6 +592,10 @@ bool LogInstance::BatchAddLogItems( } } log_state_->FlushLogItemBatch(); + // Acknowledge only after the flush. CdcScan relies on it: a transaction + // keeps its write locks until its log write is acknowledged, so no + // checkpoint can cover a record before the record is readable here, and + // min_ckpt_ts_ never runs ahead of the stored data log. for (size_t i = 0; i < num; ++i) { LogStateAddLogItemTask task = write_log_tasks[i]; @@ -696,6 +887,13 @@ LogInstance::ExecuteResult LogInstance::execute(const LogRequest *request, committed = (found && flushed_stage >= stage); break; } + case LogContentMessage::ContentCase::kClusterGateLog: + { + // A gate entry is applied once: the slot holds its + // transaction's value until a newer entry replaces it. + committed = log_state_->SearchTxClusterGate(tx_number); + break; + } default: break; } @@ -775,6 +973,15 @@ LogInstance::ExecuteResult LogInstance::execute(const LogRequest *request, req.txn_number(), timestamp, migration_tx_log_message); break; } + case LogContentMessage::ContentCase::kClusterGateLog: + { + // Updates the gate slot, like UpdateCkptTs updates state; the + // entry itself is not retained. + log_state_->UpdateClusterGate(req.txn_number(), + timestamp, + log_content.cluster_gate_log()); + break; + } default: break; } @@ -817,7 +1024,9 @@ LogInstance::ExecuteResult LogInstance::execute(const LogRequest *request, log_content.content_case() == LogContentMessage::ContentCase::kClusterScaleLog || log_content.content_case() == - LogContentMessage::ContentCase::kMigrationLog) + LogContentMessage::ContentCase::kMigrationLog || + log_content.content_case() == + LogContentMessage::ContentCase::kClusterGateLog) { // Pass all node group latest leader info to the multi-stage // txn coordinator. So that the coordinator can send @@ -909,11 +1118,27 @@ LogInstance::ExecuteResult LogInstance::execute(const LogRequest *request, { std::unique_ptr &curr_ship_agent = it->second; + // The old agent stops at its next message and closes its + // stream, so the two overlap only briefly. + const auto terminate_start = + std::chrono::steady_clock::now(); curr_ship_agent->Terminate(); while (!curr_ship_agent->IsTerminated()) { bthread_usleep(1000); } + LOG(INFO) + << "Replaced the log shipping agent of cc node group " + << cc_ng_id << " term " + << curr_ship_agent->NodeGroupTerm() << " with one for " + << "term " << cc_ng_term + << "; the old agent stopped in " + << std::chrono::duration_cast< + std::chrono::milliseconds>( + std::chrono::steady_clock::now() - + terminate_start) + .count() + << " ms"; } // The new log shipping agent spawns a background @@ -978,7 +1203,18 @@ LogInstance::ExecuteResult LogInstance::execute(const LogRequest *request, // log_state_->DeleteLogItems(node_id, timestamp); + const uint64_t arrival_complete_ts = log_state_->MinCkptTs(); log_state_->UpdateCkptTs(node_group, timestamp); + // Only this advance can make a record emittable, so it completes + // the parked long polls, which then scan again. + const uint64_t advanced_to = log_state_->MinCkptTs(); + if (advanced_to > arrival_complete_ts) + { + cdc_long_poll_.CompleteAll(CdcLongPoll::Wake::kAdvance); + arrival_complete_ts_.store(advanced_to, + std::memory_order_release); + ReportArrivalComplete(); + } } if (response) { @@ -1366,6 +1602,9 @@ void LogInstance::on_leader_stop(const butil::Status &status) std::unique_lock lk(log_replay_workers_mutex_); term_if_is_lg_leader_.store(-1, butil::memory_order_release); + // After the term is cleared: a scan parked later sees it and completes + // itself. + cdc_long_poll_.CompleteAll(CdcLongPoll::Wake::kStop); // clear log_replay_workers_, as new leader will reship on_leader_start log_replay_workers_.clear(); diff --git a/eloq_log_service/src/log_server.cpp b/eloq_log_service/src/log_server.cpp index 694676819..f25c6e31a 100644 --- a/eloq_log_service/src/log_server.cpp +++ b/eloq_log_service/src/log_server.cpp @@ -68,7 +68,8 @@ LogServer::LogServer(uint32_t node_id, uint32_t snapshot_interval, bool enable_request_checkpoint, uint32_t check_replay_log_size_interval_sec, - uint64_t notify_checkpointer_threshold_size) + uint64_t notify_checkpointer_threshold_size, + uint64_t cdc_retention_seconds) : port_(port), storage_path_(storage_path) #if defined(LOG_STATE_TYPE_RKDB_ALL) @@ -109,7 +110,8 @@ LogServer::LogServer(uint32_t node_id, snapshot_interval, enable_request_checkpoint, check_replay_log_size_interval_sec, - notify_checkpointer_threshold_size); + notify_checkpointer_threshold_size, + cdc_retention_seconds); } int LogServer::Start(bool enable_brpc_builtin_services) diff --git a/eloq_log_service/src/log_state_memory_impl.cpp b/eloq_log_service/src/log_state_memory_impl.cpp index 5e4743b10..d24669eae 100644 --- a/eloq_log_service/src/log_state_memory_impl.cpp +++ b/eloq_log_service/src/log_state_memory_impl.cpp @@ -42,6 +42,9 @@ LogStateMemoryImpl::GetLogReplayList(uint32_t node_group_id, size_t rs_size = ddl_list.size() - scale_size - schema_size; LOG(INFO) << "split_range_op_list size: " << rs_size; + // After the cluster scale list, which asserts it holds at most one item. + GetClusterGateList(ddl_list); + std::vector data_log_list; auto shard = cc_ng_state_.find(node_group_id); if (shard != cc_ng_state_.end()) diff --git a/eloq_log_service/src/log_state_rocksdb_cloud_impl.cpp b/eloq_log_service/src/log_state_rocksdb_cloud_impl.cpp index 431eec71a..ad10da0a9 100644 --- a/eloq_log_service/src/log_state_rocksdb_cloud_impl.cpp +++ b/eloq_log_service/src/log_state_rocksdb_cloud_impl.cpp @@ -379,10 +379,17 @@ bool LogStateRocksDBCloudImpl::CheckOrWaitForMemDBInSync( return true; } -// Stage 1: Move files from active DB to archive based on min_ckpt_ts +/** + * @brief Stage 1 of the purge: moves files from the active DB to the archive. + * + * @param dbc_move_log The active DB; skipped while it is not open. + * @param purge_ts Only files whose keys all lie below it move: min_ckpt_ts + * less the CDC retention window. + * @param cloud_config Where the active DB and the archive live. + */ void LogStateRocksDBCloudImpl::MoveFilesToArchive( const std::shared_ptr &dbc_move_log, - uint64_t min_ckpt_ts, + uint64_t purge_ts, const RocksDBCloudConfig &cloud_config) { if (!dbc_move_log->IsOpened()) @@ -407,12 +414,11 @@ void LogStateRocksDBCloudImpl::MoveFilesToArchive( return; } - // Use min_ckpt_ts as the purge threshold (files with keys before - // min_ckpt_ts are safe to move) + // Files whose keys all lie before purge_ts are safe to move std::array start_key{}; std::array end_key{}; Serialize(start_key, 0, 0, 0); - Serialize(end_key, min_ckpt_ts, 0, 0); + Serialize(end_key, purge_ts, 0, 0); rocksdb::Slice lower_bound = rocksdb::Slice(start_key.data(), start_key.size()); rocksdb::Slice upper_bound = rocksdb::Slice(end_key.data(), end_key.size()); @@ -802,10 +808,19 @@ LogStateRocksDBCloudImpl::LogStateRocksDBCloudImpl( dbc_move_log = dbc_; } - uint64_t min_ckpt_ts = MinCkptTs(); + // Nothing at or above the CDC retention floor leaves the live DB: + // CDC scans read only the live DB, never the archive. + uint64_t purge_ts = CdcRetentionFloor(MinCkptTs()); LogStateRocksDBCloudImpl::MoveFilesToArchive( - dbc_move_log, min_ckpt_ts, cloud_config_); + dbc_move_log, purge_ts, cloud_config_); + // The CDC DDL history goes with the data, by the same floor, + // whenever the mover could move files. + if (dbc_move_log != nullptr && dbc_move_log->IsOpened() && + !cloud_config_.archive_object_path_.empty()) + { + TrimCdcDdlHistory(purge_ts); + } }); }; @@ -1024,6 +1039,9 @@ LogStateRocksDBCloudImpl::GetLogReplayList(uint32_t ng_id, size_t rs_size = ddl_list.size() - scale_size - schema_size; LOG(INFO) << "split_range_op_list size: " << rs_size; + // After the cluster scale list, which asserts it holds at most one item. + GetClusterGateList(ddl_list); + result = std::make_unique(rocksdb_scan_threads_, std::move(ddl_list), dbc_copy->GetDBPtr(), @@ -1115,6 +1133,83 @@ std::pair LogStateRocksDBCloudImpl::SearchTxDataLog( return std::make_pair(true, ptr); } +namespace +{ +/** @brief How many times a scan re-checks mem_db_in_sync_ before NotReady. */ +constexpr int kCdcScanInSyncRetries = 10; +/** @brief The pause between those checks, in microseconds. */ +constexpr uint64_t kCdcScanInSyncRetryUs = 50 * 1000; +} // namespace + +CdcScanStatus LogStateRocksDBCloudImpl::CdcScan(const CdcScanRequest &request, + CdcScanResponse *response, + std::string *error) +{ + // Every leadership term opens its own DB container, and mem_db_in_sync_ + // speaks only for the current term's. So the term is read first and read + // again once the page is built: if it moved, the in-sync flag seen and + // the container captured may belong to different terms (a container + // already open but not yet holding the records buffered while it opened), + // and the page is discarded. With the term unchanged, the container is + // the in-sync one of that term; the copy keeps it open to the end. + const int64_t term = term_if_is_lg_leader_.load(std::memory_order_acquire); + if (term < 0) + { + *error = "this node is not the leader"; + return CdcScanStatus::NotLeader; + } + + // A new leader opens the cloud DB asynchronously and buffers writes in + // memory meanwhile; only once that buffer has been dumped into the DB + // does the DB alone hold the whole log. Wait a little for that, as + // replay does, rather than read a partial view. + for (int retry = 0; !mem_db_in_sync_.load(std::memory_order_acquire); + ++retry) + { + if (term_if_is_lg_leader_.load(std::memory_order_acquire) != term) + { + *error = "leadership changed while RocksDB Cloud was opening"; + return CdcScanStatus::NotLeader; + } + if (retry == kCdcScanInSyncRetries) + { + *error = "RocksDB Cloud is still being opened on the new leader"; + return CdcScanStatus::NotReady; + } + bthread_usleep(kCdcScanInSyncRetryUs); + } + + std::shared_ptr dbc_copy; + { + std::lock_guard lk(dbc_mutex_); + dbc_copy = dbc_; + } + if (!dbc_copy || !dbc_copy->IsOpened()) + { + // Only a leadership change since the in-sync check leaves no open + // container; the check below would discard the page anyway. + *error = "RocksDB Cloud is not open"; + return CdcScanStatus::NotReady; + } + + // The bound before the view of the store (RunCdcScan). + const uint64_t arrival_complete_ts = MinCkptTs(); + CdcDdlHistoryCursor ddl = CdcDdlStream(); + const CdcScanStatus status = CdcScanRocksDB(dbc_copy->GetDBPtr(), + ddl, + arrival_complete_ts, + request, + response, + error); + if (term_if_is_lg_leader_.load(std::memory_order_acquire) != term) + { + response->Clear(); + *error = "leadership changed during the scan"; + return CdcScanStatus::NotLeader; + } + return status; +} + void LogStateRocksDBCloudImpl::BeginSnapshot() { LogState::MakeCopyOfNgInfoAndCatalogOps(); @@ -1155,6 +1250,26 @@ int LogStateRocksDBCloudImpl::ReadSnapshot( LoadNgInfoAndCatalogOpsFrom(is); } + // The CDC DDL history, absent from snapshots taken before it existed. + { + auto file_path = snapshot_path + "/" + kCdcDdlHistoryFile; + std::ifstream is(file_path.c_str(), std::ios::binary); + if (is.good() && !LoadCdcDdlHistoryFrom(is)) + { + return -1; + } + } + + // The cluster gate slot, likewise absent from older snapshots. + { + auto file_path = snapshot_path + "/" + kClusterGateFile; + std::ifstream is(file_path.c_str(), std::ios::binary); + if (is.good() && !LoadClusterGateFrom(is)) + { + return -1; + } + } + // load in mem data log { auto file_path = snapshot_path + "/in_mem_data_log"; @@ -1221,6 +1336,35 @@ std::vector LogStateRocksDBCloudImpl::WriteSnapshot( } res.emplace_back("nginfo_and_catalog"); + { + // Checked like the other files: a snapshot published with a + // truncated history would lose DDL records on the follower. + auto path = snapshot_path + "/" + kCdcDdlHistoryFile; + std::ofstream os(path.c_str(), std::ios::binary); + WriteSnapshotCdcDdlHistoryTo(os); + os.close(); + if (!os) + { + LOG(ERROR) << "Failed to write snapshot CDC DDL history: " + << path; + return {}; + } + } + res.emplace_back(kCdcDdlHistoryFile); + + { + auto path = snapshot_path + "/" + kClusterGateFile; + std::ofstream os(path.c_str(), std::ios::binary); + WriteSnapshotClusterGateTo(os); + os.close(); + if (!os) + { + LOG(ERROR) << "Failed to write snapshot cluster gate: " << path; + return {}; + } + } + res.emplace_back(kClusterGateFile); + { auto path = snapshot_path + "/in_mem_data_log"; std::ofstream os(path.c_str()); diff --git a/eloq_log_service/src/log_state_rocksdb_impl.cpp b/eloq_log_service/src/log_state_rocksdb_impl.cpp index e12b36487..459126967 100644 --- a/eloq_log_service/src/log_state_rocksdb_impl.cpp +++ b/eloq_log_service/src/log_state_rocksdb_impl.cpp @@ -1,6 +1,7 @@ #include "log_state_rocksdb_impl.h" #if defined(LOG_STATE_TYPE_RKDB) +#include #include #include #include @@ -122,6 +123,9 @@ LogStateRocksDBImpl::GetLogReplayList(uint32_t ng_id, uint64_t start_timestamp) size_t rs_size = ddl_list.size() - scale_size - schema_size; LOG(INFO) << "split_range_op_list size: " << rs_size; + // After the cluster scale list, which asserts it holds at most one item. + GetClusterGateList(ddl_list); + std::unique_ptr result = std::make_unique(rocksdb_scan_threads_, std::move(ddl_list), @@ -200,6 +204,46 @@ std::pair LogStateRocksDBImpl::SearchTxDataLog( return std::make_pair(true, ptr); } +CdcScanStatus LogStateRocksDBImpl::CdcScan(const CdcScanRequest &request, + CdcScanResponse *response, + std::string *error) +{ + // The bound first, then the view of the store (RunCdcScan). + const uint64_t arrival_complete_ts = MinCkptTs(); + + // Register before checking whether scans are allowed. StopCdcScans() + // clears the flag before it reads the count, and all four accesses are + // sequentially consistent, so either this scan sees the flag cleared or + // StopCdcScans() sees this scan and waits for it: db_ is never closed + // under a scan. + cdc_scans_in_flight_.fetch_add(1, std::memory_order_seq_cst); + struct Deregister + { + std::atomic &in_flight; + ~Deregister() + { + in_flight.fetch_sub(1, std::memory_order_seq_cst); + } + } deregister{cdc_scans_in_flight_}; + if (!cdc_scans_allowed_.load(std::memory_order_seq_cst)) + { + *error = "the log state is being closed or reopened"; + return CdcScanStatus::NotReady; + } + CdcDdlHistoryCursor ddl = CdcDdlStream(); + return CdcScanRocksDB( + db_, ddl, arrival_complete_ts, request, response, error); +} + +void LogStateRocksDBImpl::StopCdcScans() +{ + cdc_scans_allowed_.store(false, std::memory_order_seq_cst); + while (cdc_scans_in_flight_.load(std::memory_order_seq_cst) != 0) + { + bthread_usleep(100); + } +} + void LogStateRocksDBImpl::BeginSnapshot() { LogState::MakeCopyOfNgInfoAndCatalogOps(); @@ -227,6 +271,27 @@ int LogStateRocksDBImpl::ReadSnapshot(const std::string &snapshot_path, LoadNgInfoAndCatalogOpsFrom(is); continue; } + if (file == kCdcDdlHistoryFile) + { + // A snapshot taken before the history existed has no such file, + // and the history stays empty. + std::ifstream is(path.c_str(), std::ios::binary); + if (!LoadCdcDdlHistoryFrom(is)) + { + return -1; + } + continue; + } + if (file == kClusterGateFile) + { + // Likewise for the cluster gate slot, which stays empty. + std::ifstream is(path.c_str(), std::ios::binary); + if (!LoadClusterGateFrom(is)) + { + return -1; + } + continue; + } // rocksdb checkpoint file // create hard links to checkpoint file in rocksdb storage path @@ -280,6 +345,32 @@ std::vector LogStateRocksDBImpl::WriteSnapshot( } res.emplace_back("data"); } + { + // Checked and synced like 'data': a snapshot published with a + // truncated history would lose DDL records on the follower. + auto path = snapshot_path + "/" + kCdcDdlHistoryFile; + std::ofstream os(path.c_str(), std::ios::binary); + WriteSnapshotCdcDdlHistoryTo(os); + os.close(); + if (!os || !SyncSnapshotPath(path)) + { + LOG(ERROR) << "Failed to write snapshot CDC DDL history: " << path; + return {}; + } + res.emplace_back(kCdcDdlHistoryFile); + } + { + auto path = snapshot_path + "/" + kClusterGateFile; + std::ofstream os(path.c_str(), std::ios::binary); + WriteSnapshotClusterGateTo(os); + os.close(); + if (!os || !SyncSnapshotPath(path)) + { + LOG(ERROR) << "Failed to write snapshot cluster gate: " << path; + return {}; + } + res.emplace_back(kClusterGateFile); + } // do rocksdb checkpoint, add checkpoint files to snapshot rocksdb::Checkpoint *checkpoint_raw = nullptr; @@ -416,6 +507,8 @@ int LogStateRocksDBImpl::Start() LOG(ERROR) << status.ToString(); return -1; } + // db_ is open; CDC scans may read it (StopCdcScans). + cdc_scans_allowed_.store(true, std::memory_order_seq_cst); write_option_.disableWAL = true; LOG(INFO) << "local rocksdb started"; stop_purge_thread_.store(false, std::memory_order_release); @@ -437,6 +530,7 @@ void LogStateRocksDBImpl::StopRocksDB() purge_thread_.join(); } + StopCdcScans(); if (db_ != nullptr) { db_->Close(); @@ -477,6 +571,12 @@ void LogStateRocksDBImpl::CloseAndClearRocksDB() purge_thread_.join(); } + // A CDC scan that began while this node led may still be reading db_, + // even though on_snapshot_load runs only on a follower: the node may have + // stepped down since. Wait for such scans before closing; none can start + // until Start() reopens db_. + StopCdcScans(); + // first close db_ if (db_ != nullptr) { @@ -619,8 +719,16 @@ void LogStateRocksDBImpl::PurgingSstFiles() // prepare the range to delete // all log entries before the min_last_ckpt_ts belongs to all cc - // node group could be deleted - min_last_ckpt_ts -= 1; + // node group could be deleted, except those CDC consumers may + // still need: nothing at or above the retention floor goes. + min_last_ckpt_ts = std::min(min_last_ckpt_ts - 1, + CdcRetentionFloor(min_last_ckpt_ts)); + if (min_last_ckpt_ts == 0) + { + continue; + } + // The CDC DDL history goes with the data, by the same floor. + TrimCdcDdlHistory(min_last_ckpt_ts); std::array start_key{}; std::array end_key{}; Serialize(start_key, 0, 0, 0); diff --git a/eloq_log_service/test/CdcLongPoll-Test.cpp b/eloq_log_service/test/CdcLongPoll-Test.cpp new file mode 100644 index 000000000..346a89a20 --- /dev/null +++ b/eloq_log_service/test/CdcLongPoll-Test.cpp @@ -0,0 +1,273 @@ +/** + * @file + * Tests for the CdcScan long poll (cdc_long_poll.h): every parked request is + * completed exactly once, by whichever source comes first, on a bthread of its + * own, and the poll outlives every completion. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "cdc_long_poll.h" + +// butil's logging, pulled in through bthread, defines a one-argument CHECK +// that would replace Catch's. Catch comes last so that its definition is the +// one in force. +#undef CHECK +#include + +using namespace txlog; +using Wake = CdcLongPoll::Wake; + +namespace +{ +/** @brief What the completions of one request saw. */ +struct Outcome +{ + std::atomic runs{0}; ///< How often it completed. + std::atomic why{-1}; ///< The last reason, as an int. + std::atomic on_bthread{false}; ///< It ran on a bthread. +}; + +/** + * @param outcome Where the completion records itself. + * @return A completion that records its runs, reason and thread. + */ +CdcLongPoll::Completion Recorder(Outcome &outcome) +{ + return [&outcome](Wake why) + { + outcome.why.store(static_cast(why)); + outcome.on_bthread.store(bthread_self() != 0); + outcome.runs.fetch_add(1); + }; +} + +/** + * @brief Polls until a condition holds or five seconds pass. + * + * @param done The condition. + * @return Whether it held. + */ +template +bool WaitFor(Pred done) +{ + for (int i = 0; i < 5000 && !done(); ++i) + { + usleep(1000); + } + return done(); +} +} // namespace + +TEST_CASE("A parked scan completes once, with the first wake", + "[cdc_long_poll]") +{ + CdcLongPoll poll; + Outcome outcome; + auto request = poll.Prepare(Recorder(outcome)); + poll.Park(request, 60'000); + CHECK(poll.Size() == 1); + + poll.CompleteAll(Wake::kAdvance); + REQUIRE(WaitFor([&] { return outcome.runs.load() == 1; })); + CHECK(outcome.why.load() == static_cast(Wake::kAdvance)); + CHECK(outcome.on_bthread.load()); + CHECK(poll.Size() == 0); + + // Later sources find it completed. + poll.CompleteAll(Wake::kStop); + poll.Complete(request, Wake::kStop); + CdcLongPoll::Cancel(request); + usleep(20 * 1000); + CHECK(outcome.runs.load() == 1); +} + +TEST_CASE("A parked scan's deadline completes it", "[cdc_long_poll]") +{ + CdcLongPoll poll; + Outcome outcome; + const auto start = std::chrono::steady_clock::now(); + auto request = poll.Prepare(Recorder(outcome)); + poll.Park(request, 50); + REQUIRE(WaitFor([&] { return outcome.runs.load() == 1; })); + CHECK(std::chrono::steady_clock::now() - start >= + std::chrono::milliseconds(45)); + CHECK(outcome.why.load() == static_cast(Wake::kDeadline)); + CHECK(poll.Size() == 0); +} + +TEST_CASE("A cancellation completes a scan, parked or not", "[cdc_long_poll]") +{ + CdcLongPoll poll; + + // Cancelled before the park, as when the call was cancelled already when + // it registered its cancellation: the park leaves it alone. + Outcome early; + auto first = poll.Prepare(Recorder(early)); + CdcLongPoll::Cancel(first); + poll.Park(first, 60'000); + REQUIRE(WaitFor([&] { return early.runs.load() == 1; })); + CHECK(early.why.load() == static_cast(Wake::kCancel)); + CHECK(poll.Size() == 0); + + Outcome parked; + auto second = poll.Prepare(Recorder(parked)); + poll.Park(second, 60'000); + CdcLongPoll::Cancel(second); + REQUIRE(WaitFor([&] { return parked.runs.load() == 1; })); + CHECK(parked.why.load() == static_cast(Wake::kCancel)); + CHECK(poll.Size() == 0); +} + +TEST_CASE("Racing sources complete every scan exactly once", "[cdc_long_poll]") +{ + CdcLongPoll poll; + constexpr int kRequests = 300; + std::vector outcomes(kRequests); + std::vector> requests; + for (int i = 0; i < kRequests; ++i) + { + requests.push_back(poll.Prepare(Recorder(outcomes[i]))); + // Some deadlines fire while the other sources race. + poll.Park(requests.back(), i % 3); + } + std::thread advance([&] { poll.CompleteAll(Wake::kAdvance); }); + std::thread cancel( + [&] + { + for (const auto &request : requests) + { + CdcLongPoll::Cancel(request); + } + }); + std::thread stop( + [&] + { + for (auto it = requests.rbegin(); it != requests.rend(); ++it) + { + poll.Complete(*it, Wake::kStop); + } + }); + advance.join(); + cancel.join(); + stop.join(); + + REQUIRE(WaitFor( + [&] + { + for (const Outcome &o : outcomes) + { + if (o.runs.load() == 0) + { + return false; + } + } + return true; + })); + usleep(20 * 1000); + int more_than_once = 0; + for (const Outcome &o : outcomes) + { + more_than_once += o.runs.load() > 1; + } + CHECK(more_than_once == 0); + CHECK(poll.Size() == 0); +} + +TEST_CASE("The size observer sees every change of the parked count", + "[cdc_long_poll]") +{ + CdcLongPoll poll; + std::vector seen; // Written under the poll's lock. + std::mutex seen_mutex; + auto snapshot = [&] + { + std::lock_guard lk(seen_mutex); + return seen; + }; + poll.SetSizeObserver( + [&](size_t parked) + { + std::lock_guard lk(seen_mutex); + seen.push_back(parked); + }); + // Setting the observer reports the count at once. + CHECK(snapshot() == std::vector{0}); + + Outcome first, second, cancelled; + auto a = poll.Prepare(Recorder(first)); + auto b = poll.Prepare(Recorder(second)); + auto c = poll.Prepare(Recorder(cancelled)); + poll.Park(a, 60'000); + poll.Park(b, 60'000); + poll.Park(c, 60'000); + CHECK(snapshot() == std::vector{0, 1, 2, 3}); + + CdcLongPoll::Cancel(c); + REQUIRE(WaitFor([&] { return cancelled.runs.load() == 1; })); + CHECK(snapshot().back() == 2); + + poll.CompleteAll(Wake::kAdvance); + REQUIRE(WaitFor( + [&] { return first.runs.load() == 1 && second.runs.load() == 1; })); + CHECK(snapshot() == std::vector{0, 1, 2, 3, 2, 1, 0}); + + // A request completed before it is parked never counts. + Outcome early; + auto d = poll.Prepare(Recorder(early)); + CdcLongPoll::Cancel(d); + poll.Park(d, 60'000); + REQUIRE(WaitFor([&] { return early.runs.load() == 1; })); + CHECK(snapshot().size() == 7); + + // Clearing the observer stops the reports. + poll.SetSizeObserver(nullptr); + Outcome late; + auto e = poll.Prepare(Recorder(late)); + poll.Park(e, 60'000); + CHECK(snapshot().size() == 7); + // Completed here, while 'late' is alive; the poll outlives it. + poll.CompleteAll(Wake::kStop); + REQUIRE(WaitFor([&] { return late.runs.load() == 1; })); +} + +TEST_CASE("The poll completes its scans with kStop and waits for them", + "[cdc_long_poll]") +{ + constexpr int kRequests = 3; + std::vector outcomes(kRequests); + std::atomic finished{0}; + { + CdcLongPoll poll; + for (int i = 0; i < kRequests; ++i) + { + Outcome &outcome = outcomes[i]; + poll.Park(poll.Prepare( + [&outcome, &finished](Wake why) + { + // A completion still running holds the poll up. + bthread_usleep(50 * 1000); + outcome.why.store(static_cast(why)); + outcome.runs.fetch_add(1); + finished.fetch_add(1); + }), + 60'000); + } + CHECK(poll.Size() == kRequests); + } + CHECK(finished.load() == kRequests); + for (const Outcome &o : outcomes) + { + CHECK(o.runs.load() == 1); + CHECK(o.why.load() == static_cast(Wake::kStop)); + } +} diff --git a/eloq_log_service/test/CdcScan-Test.cpp b/eloq_log_service/test/CdcScan-Test.cpp new file mode 100644 index 000000000..d281bf526 --- /dev/null +++ b/eloq_log_service/test/CdcScan-Test.cpp @@ -0,0 +1,1093 @@ +/** + * @file + * Tests for the CDC log scan (cdc_scan.h): the continuation codec, the page + * merge over the data and DDL streams, the scan served by the RocksDB log + * state over its data log and its DDL history, and the CDC retention floor in + * its purge. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cdc_scan.h" +#include "log.pb.h" + +#if defined(LOG_STATE_TYPE_RKDB) +#include "log_state_rocksdb_impl.h" +#endif + +// butil's logging, which the log state pulls in, defines a one-argument CHECK +// that would replace Catch's; nothing here uses butil's. Catch comes last so +// that its definition is the one in force. +#undef CHECK +#include + +using namespace txlog; + +namespace +{ +/** + * @brief A storage key laid out the way both stores lay theirs out. + * + * The big-endian timestamp first, then what tells records at one timestamp + * apart, padded to the 20 bytes of a data key so that the same keys serve + * both streams. + * + * @param ts The record's timestamp. + * @param suffix What tells it apart from others at `ts`. + * @return The key. + */ +std::string KeyAt(uint64_t ts, const std::string &suffix) +{ + std::string key = CdcTimestampPosition(ts) + suffix; + key.resize(std::max(key.size(), 20), '\0'); + return key; +} + +struct FakeRecord +{ + uint64_t ts; + std::string key; + size_t size; +}; + +FakeRecord Rec(uint64_t ts, const std::string &suffix, size_t size = 10) +{ + return {ts, KeyAt(ts, suffix), size}; +} + +/** + * A stream over records held in memory, in key order, as a store would hold + * them. It appends data records with the key as the blob and DDL records with + * the key as the table name, so a test can tell every returned record apart. + */ +class VectorCursor final : public CdcStreamCursor +{ +public: + VectorCursor(std::vector records, bool is_ddl) + : records_(std::move(records)), is_ddl_(is_ddl) + { + std::sort(records_.begin(), + records_.end(), + [](const FakeRecord &a, const FakeRecord &b) + { return a.key < b.key; }); + } + + void Seek(const std::string &after, uint64_t upper_ts) override + { + upper_ts_ = upper_ts; + idx_ = 0; + while (idx_ < records_.size() && !(records_[idx_].key > after)) + { + ++idx_; + } + } + bool Valid() const override + { + return idx_ < records_.size() && records_[idx_].ts <= upper_ts_; + } + bool Ok() const override + { + return true; + } + std::string ErrorText() const override + { + return {}; + } + uint64_t Timestamp() const override + { + return records_[idx_].ts; + } + std::string_view Key() const override + { + return records_[idx_].key; + } + size_t ByteSize() const override + { + return records_[idx_].size; + } + void AppendTo(CdcScanResponse *response) const override + { + const FakeRecord &r = records_[idx_]; + if (is_ddl_) + { + CdcDdlRecord *ddl = response->add_ddl(); + ddl->set_ddl_ts(r.ts); + ddl->mutable_schema_op()->set_table_name_str(r.key); + } + else + { + CdcDataRecord *data = response->add_data(); + data->set_commit_ts(r.ts); + data->set_blob(r.key); + } + } + void Next() override + { + ++idx_; + } + +private: + std::vector records_; + bool is_ddl_; + uint64_t upper_ts_{0}; + size_t idx_{0}; +}; + +struct Returned +{ + bool ddl; + uint64_t ts; + std::string key; + + bool operator==(const Returned &o) const + { + return ddl == o.ddl && ts == o.ts && key == o.key; + } +}; + +/** + * @brief The records of one page in merged order, rebuilt the way a caller + * would: both lists are ascending, and at one timestamp data comes first. + * + * @param page The page. + * @return Its records in merged order. + */ +std::vector MergedPage(const CdcScanResponse &page) +{ + std::vector out; + int d = 0, l = 0; + while (d < page.data_size() || l < page.ddl_size()) + { + const bool take_data = + d < page.data_size() && + (l == page.ddl_size() || + page.data(d).commit_ts() <= page.ddl(l).ddl_ts()); + if (take_data) + { + out.push_back( + {false, page.data(d).commit_ts(), page.data(d).blob()}); + ++d; + } + else + { + out.push_back({true, + page.ddl(l).ddl_ts(), + page.ddl(l).schema_op().table_name_str()}); + ++l; + } + } + return out; +} + +/** + * @brief The whole merged sequence at or below a bound, as the scan should + * page it. + * + * @param data The data stream's records. + * @param ddl The DDL stream's records. + * @param bound The arrival_complete_ts of the scan. + * @return The records at or below `bound` in merged order. + */ +std::vector Expected(const std::vector &data, + const std::vector &ddl, + uint64_t bound) +{ + std::vector out; + for (const FakeRecord &r : data) + { + if (r.ts <= bound) + { + out.push_back({false, r.ts, r.key}); + } + } + for (const FakeRecord &r : ddl) + { + if (r.ts <= bound) + { + out.push_back({true, r.ts, r.key}); + } + } + std::sort(out.begin(), + out.end(), + [](const Returned &a, const Returned &b) + { + if (a.ts != b.ts) + { + return a.ts < b.ts; + } + if (a.ddl != b.ddl) + { + return !a.ddl; // data first + } + return a.key < b.key; + }); + return out; +} + +CdcScanResponse ScanPage(const std::vector &data, + const std::vector &ddl, + const CdcScanPosition &start, + uint64_t bound, + size_t page_bytes) +{ + VectorCursor data_cursor(data, false); + VectorCursor ddl_cursor(ddl, true); + CdcScanResponse page; + std::string error; + REQUIRE(RunCdcScan( + data_cursor, ddl_cursor, start, bound, page_bytes, &page, &error)); + return page; +} + +CdcScanPosition Decode(const std::string &continuation) +{ + CdcScanPosition position; + std::string error; + REQUIRE(DecodeCdcContinuation(continuation, &position, &error)); + return position; +} +} // namespace + +TEST_CASE("A continuation round-trips and is laid out big-endian", "[cdc_scan]") +{ + CdcScanPosition position; + position.data = KeyAt(7, std::string(12, '\x01')); // a 20-byte data key + position.ddl = CdcTimestampPosition(9) + "ns.coll"; // a 15-byte DDL key + const std::string bytes = EncodeCdcContinuation(position); + + REQUIRE(bytes.size() == 2 + 20 + 2 + 15); + CHECK(bytes.substr(0, 2) == std::string("\x00\x14", 2)); + CHECK(bytes.substr(2, 20) == position.data); + CHECK(bytes.substr(22, 2) == std::string("\x00\x0f", 2)); + CHECK(bytes.substr(24) == position.ddl); + + const CdcScanPosition decoded = Decode(bytes); + CHECK(decoded.data == position.data); + CHECK(decoded.ddl == position.ddl); + + // Every legal shape, including the empty and the timestamp positions. + for (const auto &[data, ddl] : + std::vector>{ + {"", ""}, + {CdcTimestampPosition(5), ""}, + {"", CdcTimestampPosition(5)}, + {CdcTimestampPosition(5), CdcTimestampPosition(6)}}) + { + const CdcScanPosition p = Decode(EncodeCdcContinuation({data, ddl})); + CHECK(p.data == data); + CHECK(p.ddl == ddl); + } +} + +TEST_CASE("A continuation this service could not have issued is rejected", + "[cdc_scan]") +{ + const auto rejected = [](const std::string &bytes) + { + CdcScanPosition position; + std::string error; + const bool ok = DecodeCdcContinuation(bytes, &position, &error); + CHECK_FALSE(error.empty()); + return !ok; + }; + CHECK(rejected("")); + CHECK(rejected(std::string("\x00", 1))); + // A length running past the end. + CHECK(rejected(std::string("\x00\x08", 2) + "1234")); + // Only the data position present. + CHECK(rejected(std::string("\x00\x00", 2))); + // Trailing bytes after both positions. + CHECK(rejected(EncodeCdcContinuation({}) + "x")); + // A data position that is neither a timestamp nor a data key. + CHECK(rejected(EncodeCdcContinuation({"12345", ""}))); + CHECK(rejected(EncodeCdcContinuation({std::string(21, 'k'), ""}))); + // A DDL position shorter than a timestamp. + CHECK(rejected(EncodeCdcContinuation({"", "abc"}))); +} + +TEST_CASE("A scan starts at its continuation, at from_ts, or at the beginning", + "[cdc_scan]") +{ + CdcScanPosition start; + std::string error; + + CdcScanRequest from_ts; + from_ts.set_from_ts(1234); + REQUIRE(ParseCdcScanStart(from_ts, &start, &error)); + CHECK(start.data == CdcTimestampPosition(1234)); + CHECK(start.ddl == CdcTimestampPosition(1234)); + + // Timestamp 0 precedes every record: the canonical empty position. + CdcScanRequest from_zero; + from_zero.set_from_ts(0); + REQUIRE(ParseCdcScanStart(from_zero, &start, &error)); + CHECK(start.data.empty()); + CHECK(start.ddl.empty()); + + CdcScanRequest neither; + start = {"stale", "stale"}; + REQUIRE(ParseCdcScanStart(neither, &start, &error)); + CHECK(start.data.empty()); + CHECK(start.ddl.empty()); + + CdcScanRequest resumed; + const CdcScanPosition issued{KeyAt(3, std::string(12, 'k')), + CdcTimestampPosition(2)}; + resumed.set_continuation(EncodeCdcContinuation(issued)); + REQUIRE(ParseCdcScanStart(resumed, &start, &error)); + CHECK(start.data == issued.data); + CHECK(start.ddl == issued.ddl); + + CdcScanRequest garbled; + garbled.set_continuation("garbled"); + CHECK_FALSE(ParseCdcScanStart(garbled, &start, &error)); +} + +TEST_CASE("The page size defaults and is capped", "[cdc_scan]") +{ + CHECK(CdcScanPageBytes(0) == 4 * 1024 * 1024); + CHECK(CdcScanPageBytes(1) == 1); + CHECK(CdcScanPageBytes(1 << 20) == 1 << 20); + CHECK(CdcScanPageBytes(UINT32_MAX) == 32 * 1024 * 1024); +} + +TEST_CASE("Every page size returns the whole merged sequence exactly once", + "[cdc_scan]") +{ + // Both streams share timestamps 3 and 5; timestamp 9 lies above the + // bound; sizes vary so that cuts land in every possible place. + const std::vector data{Rec(1, "a", 7), + Rec(1, "b", 13), + Rec(3, "a", 5), + Rec(5, "a", 11), + Rec(5, "b", 3), + Rec(5, "c", 17), + Rec(7, "a", 9), + Rec(9, "a", 4)}; + const std::vector ddl{Rec(3, "x", 6), + Rec(5, "x", 12), + Rec(5, "y", 8), + Rec(6, "x", 10), + Rec(9, "x", 2)}; + const uint64_t bound = 8; + const std::vector expected = Expected(data, ddl, bound); + size_t total = 0; + for (const FakeRecord &r : data) + { + total += r.size; + } + for (const FakeRecord &r : ddl) + { + total += r.size; + } + + for (size_t page_bytes = 1; page_bytes <= total + 1; ++page_bytes) + { + CAPTURE(page_bytes); + std::vector got; + CdcScanPosition position; + for (int pages = 0;; ++pages) + { + REQUIRE(pages <= static_cast(expected.size())); + const CdcScanResponse page = + ScanPage(data, ddl, position, bound, page_bytes); + CHECK(page.arrival_complete_ts() == bound); + CHECK(page.retention_floor_ts() == 0); + const std::vector records = MergedPage(page); + // A page is never empty while records remain. + CHECK(records.empty() == (got.size() == expected.size())); + got.insert(got.end(), records.begin(), records.end()); + // Every page ends on a prefix of the merged sequence. + REQUIRE(got.size() <= expected.size()); + CHECK(std::equal(got.begin(), got.end(), expected.begin())); + position = Decode(page.continuation()); + CHECK(page.has_more() == (got.size() < expected.size())); + if (!page.has_more()) + { + break; + } + } + CHECK(got == expected); + } +} + +TEST_CASE("A page cut inside one timestamp resumes without gap or duplicate", + "[cdc_scan]") +{ + const std::vector data{Rec(5, "a"), Rec(5, "b")}; + const std::vector ddl{Rec(5, "x"), Rec(5, "y")}; + + // Three records fit: both data records, then the first DDL record. + CdcScanResponse page = ScanPage(data, ddl, {}, 10, 30); + REQUIRE(page.data_size() == 2); + REQUIRE(page.ddl_size() == 1); + CHECK(page.ddl(0).schema_op().table_name_str() == KeyAt(5, "x")); + CHECK(page.has_more()); + CdcScanPosition position = Decode(page.continuation()); + CHECK(position.data == KeyAt(5, "b")); + CHECK(position.ddl == KeyAt(5, "x")); + + page = ScanPage(data, ddl, position, 10, 30); + REQUIRE(page.data_size() == 0); + REQUIRE(page.ddl_size() == 1); + CHECK(page.ddl(0).schema_op().table_name_str() == KeyAt(5, "y")); + CHECK_FALSE(page.has_more()); + + // Cut after the first data record: the DDL stream has returned nothing + // at 5, so it resumes there. + page = ScanPage(data, ddl, {}, 10, 10); + REQUIRE(page.data_size() == 1); + REQUIRE(page.ddl_size() == 0); + position = Decode(page.continuation()); + CHECK(position.data == KeyAt(5, "a")); + CHECK(position.ddl.empty()); + page = ScanPage(data, ddl, position, 10, 1000); + CHECK(MergedPage(page) == std::vector{{false, 5, KeyAt(5, "b")}, + {true, 5, KeyAt(5, "x")}, + {true, 5, KeyAt(5, "y")}}); +} + +TEST_CASE("Records above arrival_complete_ts wait for a later bound", + "[cdc_scan]") +{ + const std::vector data{Rec(2, "a"), Rec(6, "a")}; + const std::vector ddl{Rec(3, "x"), Rec(8, "x")}; + + CdcScanResponse page = ScanPage(data, ddl, {}, 4, 1000); + CHECK(MergedPage(page) == std::vector{{false, 2, KeyAt(2, "a")}, + {true, 3, KeyAt(3, "x")}}); + // Nothing at or below the bound remains, so the page was not cut. + CHECK_FALSE(page.has_more()); + + page = ScanPage(data, ddl, Decode(page.continuation()), 10, 1000); + CHECK(MergedPage(page) == std::vector{{false, 6, KeyAt(6, "a")}, + {true, 8, KeyAt(8, "x")}}); + CHECK(page.arrival_complete_ts() == 10); +} + +TEST_CASE("An empty page carries the bound and keeps the position", + "[cdc_scan]") +{ + const std::vector data{Rec(20, "a")}; + const CdcScanPosition start{KeyAt(4, std::string(12, 'k')), + CdcTimestampPosition(3)}; + const CdcScanResponse page = ScanPage(data, {}, start, 10, 1000); + CHECK(page.data_size() == 0); + CHECK(page.ddl_size() == 0); + CHECK(page.arrival_complete_ts() == 10); + CHECK_FALSE(page.has_more()); + CHECK(page.continuation() == EncodeCdcContinuation(start)); + + // No bound at all yet: still an empty, well-formed page. + const CdcScanResponse unbounded = ScanPage(data, {}, {}, 0, 1000); + CHECK(unbounded.data_size() == 0); + CHECK(unbounded.arrival_complete_ts() == 0); + CHECK_FALSE(unbounded.has_more()); +} + +TEST_CASE("A record larger than the page is returned on its own", "[cdc_scan]") +{ + const std::vector data{Rec(1, "big", 1000), Rec(2, "a", 1)}; + CdcScanResponse page = ScanPage(data, {}, {}, 10, 100); + REQUIRE(page.data_size() == 1); + CHECK(page.data(0).blob() == KeyAt(1, "big")); + CHECK(page.has_more()); + page = ScanPage(data, {}, Decode(page.continuation()), 10, 100); + REQUIRE(page.data_size() == 1); + CHECK(page.data(0).blob() == KeyAt(2, "a")); + CHECK_FALSE(page.has_more()); +} + +TEST_CASE("A stream that returns nothing keeps its from_ts position", + "[cdc_scan]") +{ + const CdcScanPosition start{CdcTimestampPosition(4), + CdcTimestampPosition(4)}; + + // Below from_ts nothing is returned; above the bound nothing yet. + std::vector data{Rec(2, "a"), Rec(6, "a")}; + CdcScanResponse page = ScanPage(data, {}, start, 5, 1000); + CHECK(page.data_size() == 0); + CdcScanPosition position = Decode(page.continuation()); + CHECK(position.data == CdcTimestampPosition(4)); + CHECK(position.ddl == CdcTimestampPosition(4)); + + // Records at from_ts itself are included; the DDL stream, still empty, + // keeps the eight-byte position. + data.push_back(Rec(4, "b")); + page = ScanPage(data, {}, position, 10, 1000); + CHECK(MergedPage(page) == std::vector{{false, 4, KeyAt(4, "b")}, + {false, 6, KeyAt(6, "a")}}); + position = Decode(page.continuation()); + CHECK(position.data == KeyAt(6, "a")); + CHECK(position.ddl == CdcTimestampPosition(4)); + + // A DDL record that arrives later, above the bound the scan saw, is + // found from the eight-byte position. + page = ScanPage(data, {Rec(11, "x")}, position, 12, 1000); + CHECK(MergedPage(page) == + std::vector{{true, 11, KeyAt(11, "x")}}); +} + +#if defined(LOG_STATE_TYPE_RKDB) +namespace +{ +/** @brief A RocksDB log state in a fresh directory, removed afterwards. */ +class ScratchLogState +{ +public: + explicit ScratchLogState(size_t sst_files_size_limit = size_t{1} << 40) + : path_(std::filesystem::temp_directory_path() / + ("cdc_scan_test_" + std::to_string(::getpid()) + "_" + + std::to_string(counter_++))) + { + std::filesystem::remove_all(path_); + state_ = std::make_unique( + (path_ / "rocksdb").string(), sst_files_size_limit, 1); + REQUIRE(state_->Start() == 0); + } + + /** + * Flushes the memtables into a new SST file, which also wakes the purge + * thread. The state offers no flush of its own; a snapshot's RocksDB + * checkpoint always flushes first. + */ + void Flush() + { + Snapshot(); + } + + /** + * @brief Writes a snapshot into a new directory, as a leader does for a + * follower that lags: copies the in-memory state, then writes it. + * Flushes the memtables first, like Flush(). + * + * @return The snapshot directory, and its files relative to it. + */ + std::pair> Snapshot() + { + const std::filesystem::path snapshot = + path_ / ("snapshot_" + std::to_string(snapshots_++)); + std::filesystem::create_directories(snapshot); + state_->BeginSnapshot(); + return {snapshot.string(), state_->WriteSnapshot(snapshot.string())}; + } + + ~ScratchLogState() + { + state_.reset(); + std::filesystem::remove_all(path_); + } + + LogStateRocksDBImpl &operator*() + { + return *state_; + } + LogStateRocksDBImpl *operator->() + { + return state_.get(); + } + +private: + static inline int counter_ = 0; + std::filesystem::path path_; + int snapshots_{0}; + std::unique_ptr state_; +}; + +CdcScanResponse StateScan(LogStateRocksDBImpl &state, + const CdcScanRequest &request) +{ + CdcScanResponse page; + std::string error; + REQUIRE(state.CdcScan(request, &page, &error) == CdcScanStatus::Ok); + CHECK(error.empty()); + return page; +} + +struct DataRow +{ + uint64_t ts; + uint32_t ng; + uint64_t txn; + std::string blob; + + bool operator==(const DataRow &o) const + { + return ts == o.ts && ng == o.ng && txn == o.txn && blob == o.blob; + } +}; + +std::vector Rows(const CdcScanResponse &page) +{ + std::vector out; + for (const CdcDataRecord &r : page.data()) + { + out.push_back({r.commit_ts(), r.cc_ng_id(), r.txn_number(), r.blob()}); + } + return out; +} +} // namespace + +TEST_CASE("The RocksDB log state scans its data log through min_ckpt_ts", + "[cdc_scan][rocksdb]") +{ + ScratchLogState state; + state->UpdateNgTerm(0, 1, "127.0.0.1", 1); + state->UpdateNgTerm(1, 1, "127.0.0.1", 2); + + // Binary blobs, two node groups sharing a timestamp. + const std::string zero_byte("a\0z", 3); + const std::vector rows{{100, 0, 11, zero_byte}, + {100, 1, 12, "b100"}, + {150, 0, 13, "a150"}, + {200, 1, 14, "b200"}, + {300, 0, 15, "a300"}}; + for (const DataRow &r : rows) + { + state->AddLogItem(r.ng, r.txn, r.ts, r.blob); + } + + CdcScanRequest from_start; + from_start.set_from_ts(0); + + // No checkpoint yet: nothing is known complete. + CdcScanResponse page = StateScan(*state, from_start); + CHECK(page.data_size() == 0); + CHECK(page.arrival_complete_ts() == 0); + CHECK_FALSE(page.has_more()); + + // arrival_complete_ts is the minimum over node groups. + state->UpdateCkptTs(0, 250); + state->UpdateCkptTs(1, 200); + page = StateScan(*state, from_start); + CHECK(page.arrival_complete_ts() == 200); + CHECK(Rows(page) == std::vector(rows.begin(), rows.begin() + 4)); + CHECK_FALSE(page.has_more()); + + // One record per page, through the continuation. + std::vector paged; + CdcScanRequest request = from_start; + request.set_max_bytes(1); + for (int i = 0; i < 4; ++i) + { + page = StateScan(*state, request); + REQUIRE(page.data_size() == 1); + paged.push_back(Rows(page).front()); + CHECK(page.has_more() == (i < 3)); + request.set_continuation(page.continuation()); + } + CHECK(paged == std::vector(rows.begin(), rows.begin() + 4)); + + // Exhausted: the same continuation returns nothing until the bound moves. + page = StateScan(*state, request); + CHECK(page.data_size() == 0); + state->UpdateCkptTs(0, 400); + state->UpdateCkptTs(1, 350); + page = StateScan(*state, request); + CHECK(page.arrival_complete_ts() == 350); + CHECK(Rows(page) == std::vector{rows[4]}); + + // from_ts starts at the first record at or above it. + CdcScanRequest from_150; + from_150.set_from_ts(150); + page = StateScan(*state, from_150); + CHECK(Rows(page) == std::vector(rows.begin() + 2, rows.end())); +} + +namespace +{ +/** + * @brief Logs an EloqDoc schema op's prepare and commit logs. + * + * @param state The log state. + * @param txn The op's tx. + * @param table The table name. + * @param op_type The txservice::OperationType. + * @param prepare_ts The prepare log's timestamp, the op's catalog version. + * @param commit_ts The commit log's timestamp. + */ +void LogSchemaOp(LogStateRocksDBImpl &state, + uint64_t txn, + const std::string &table, + uint32_t op_type, + uint64_t prepare_ts, + uint64_t commit_ts) +{ + SchemaOpMessage msg; + msg.set_table_name_str(table); + msg.set_table_engine(TableEngine::EloqDoc); + msg.set_table_type(CcTableType::Primary); + msg.mutable_table_op()->set_op_type(op_type); + msg.set_stage(SchemaOpMessage_Stage_PrepareSchema); + state.UpsertSchemaOp(txn, prepare_ts, msg); + msg.set_stage(SchemaOpMessage_Stage_CommitSchema); + state.UpsertSchemaOp(txn, commit_ts, msg); +} + +/** + * @param page A scan page. + * @return (ddl_ts, txn) of its DDL records. + */ +std::vector> DdlRows(const CdcScanResponse &page) +{ + std::vector> out; + for (const CdcDdlRecord &r : page.ddl()) + { + out.emplace_back(r.ddl_ts(), r.txn_number()); + } + return out; +} +} // namespace + +TEST_CASE("The RocksDB log state scans its DDL history with its data log", + "[cdc_scan][rocksdb]") +{ + ScratchLogState state; + state->UpdateNgTerm(0, 1, "127.0.0.1", 1); + // A window keeps the DDL records a checkpoint passes. + state->SetCdcRetentionSeconds(1); + for (uint64_t ts : {100, 200, 300}) + { + state->AddLogItem(0, ts, ts, "d" + std::to_string(ts)); + } + // A create committed at 200, the data record's timestamp, and an add + // index at 250 (txservice::OperationType CreateTable and AddIndex). + LogSchemaOp(*state, 50, "db.a", 5, 150, 200); + LogSchemaOp(*state, 51, "db.a", 8, 250, 250); + state->UpdateCkptTs(0, 260); + + CdcScanRequest from_start; + from_start.set_from_ts(0); + CdcScanResponse page = StateScan(*state, from_start); + CHECK(page.data_size() == 2); + CHECK(DdlRows(page) == + std::vector>{{200, 50}, {250, 51}}); + CHECK(page.ddl(0).catalog_ts() == 150); + CHECK(page.ddl(0).schema_op().table_name_str() == "db.a"); + CHECK_FALSE(page.has_more()); + + // One record per page: at 200 the data record comes first. + std::vector> merged; + CdcScanRequest request = from_start; + request.set_max_bytes(1); + for (int i = 0; i < 4; ++i) + { + page = StateScan(*state, request); + REQUIRE(page.data_size() + page.ddl_size() == 1); + merged.emplace_back(page.ddl_size() == 1, + page.ddl_size() == 1 ? page.ddl(0).ddl_ts() + : page.data(0).commit_ts()); + CHECK(page.has_more() == (i < 3)); + request.set_continuation(page.continuation()); + } + CHECK(merged == std::vector>{ + {false, 100}, {false, 200}, {true, 200}, {true, 250}}); + CdcScanPosition position; + std::string error; + REQUIRE(DecodeCdcContinuation(page.continuation(), &position, &error)); + CHECK(position.ddl == CdcDdlPosition(250, 51)); + page = StateScan(*state, request); + CHECK(page.data_size() + page.ddl_size() == 0); + + // A follower that installs the snapshot has the same history. + auto [snapshot_dir, files] = state.Snapshot(); + ScratchLogState follower; + follower->SetCdcRetentionSeconds(1); + REQUIRE(follower->ReadSnapshot(snapshot_dir, files) == 0); + // The bound is rebuilt at the next checkpoint. + follower->UpdateCkptTs(0, 261); + page = StateScan(*follower, from_start); + CHECK(page.data_size() == 2); + CHECK(DdlRows(page) == + std::vector>{{200, 50}, {250, 51}}); +} + +TEST_CASE("The CDC retention floor is the checkpoint less the window", + "[cdc_scan][rocksdb]") +{ + ScratchLogState state; + CHECK(state->CdcRetentionFloor(8'000'000) == 8'000'000); + state->SetCdcRetentionSeconds(4); + CHECK(state->CdcRetentionFloor(8'000'000) == 4'000'000); + CHECK(state->CdcRetentionFloor(4'000'000) == 0); + CHECK(state->CdcRetentionFloor(3'000'000) == 0); +} + +namespace +{ +/** + * @brief Writes one batch of records and flushes it into its own SST file. + * + * @param state The log state. + * @param base_ts The first record's timestamp; the others follow at + * `base_ts + 1`, `base_ts + 2`, ... + * @param records How many records to write. + */ +void WriteBatchFile(ScratchLogState &state, uint64_t base_ts, int records) +{ + for (int i = 0; i < records; ++i) + { + state->AddLogItem(0, base_ts + i, base_ts + i, "r"); + } + state.Flush(); +} + +/** + * @param state The log state. + * @return The commit timestamps a full scan through the checkpoint returns. + */ +std::vector ScannedTimestamps(LogStateRocksDBImpl &state) +{ + CdcScanRequest request; + request.set_from_ts(0); + std::vector out; + for (const DataRow &row : Rows(StateScan(state, request))) + { + out.push_back(row.ts); + } + return out; +} + +/** + * @brief Polls until a condition holds for the scanned timestamps, or ten + * seconds pass. + * + * @param state The log state. + * @param done The condition on ScannedTimestamps(). + * @return The last timestamps scanned. + */ +template +std::vector WaitForScan(LogStateRocksDBImpl &state, Pred done) +{ + std::vector ts = ScannedTimestamps(state); + for (int i = 0; i < 100 && !done(ts); ++i) + { + usleep(100 * 1000); + ts = ScannedTimestamps(state); + } + return ts; +} +} // namespace + +TEST_CASE("The RocksDB purge keeps the CDC retention window", + "[cdc_scan][rocksdb]") +{ + // A one-byte size limit makes every flush trigger a purge. + ScratchLogState state(1); + state->UpdateNgTerm(0, 1, "127.0.0.1", 1); + state->SetCdcRetentionSeconds(4); + + // Three files, timestamps in microseconds: A at 1 s, B at 5 s, C at 9 s. + WriteBatchFile(state, 1'000'000, 3); + WriteBatchFile(state, 5'000'000, 3); + WriteBatchFile(state, 9'000'000, 3); + + // Checkpoint at 8 s. The floor is 8 s - 4 s = 4 s: A lies wholly below + // it, B does not. The next flush wakes the purge. + state->UpdateCkptTs(0, 8'000'000); + WriteBatchFile(state, 9'500'000, 1); + std::vector ts = + WaitForScan(*state, + [](const std::vector &t) + { return !t.empty() && t.front() >= 5'000'000; }); + CHECK(ts == std::vector{5'000'000, 5'000'001, 5'000'002}); + + // With no window the purge takes everything checkpointed, B included. + // The checkpoint must move for the purge to run again. + state->SetCdcRetentionSeconds(0); + state->UpdateCkptTs(0, 8'100'000); + WriteBatchFile(state, 9'600'000, 1); + ts = WaitForScan(*state, + [](const std::vector &t) { return t.empty(); }); + CHECK(ts.empty()); + + // Everything above the checkpoint survived both purges. No flush follows, + // so no purge runs after the checkpoint moves past it. + state->UpdateCkptTs(0, 9'700'000); + CHECK(ScannedTimestamps(*state) == + std::vector{ + 9'000'000, 9'000'001, 9'000'002, 9'500'000, 9'600'000}); +} + +namespace +{ +/** + * @param state The log state. + * @return The ddl_ts of every record in its CDC DDL history. + */ +std::vector HistoryTs(LogStateRocksDBImpl &state) +{ + std::vector records; + state.CopyCdcDdlHistory({}, UINT64_MAX, &records); + std::vector out; + for (const CdcDdlRecord &r : records) + { + out.push_back(r.ddl_ts()); + } + return out; +} + +/** + * @brief Polls until the CDC DDL history holds the given timestamps, or ten + * seconds pass. + * + * @param state The log state. + * @param expected The ddl_ts of every record, in order. + * @return The last history seen. + */ +std::vector WaitForHistory(LogStateRocksDBImpl &state, + const std::vector &expected) +{ + std::vector ts = HistoryTs(state); + for (int i = 0; i < 100 && ts != expected; ++i) + { + usleep(100 * 1000); + ts = HistoryTs(state); + } + return ts; +} +} // namespace + +TEST_CASE("The RocksDB purge trims the CDC DDL history by its floor", + "[cdc_scan][rocksdb]") +{ + // A one-byte size limit makes every flush trigger a purge. + ScratchLogState state(1); + state->UpdateNgTerm(0, 1, "127.0.0.1", 1); + // Creates committed at 1 s, 5 s and 9 s, in microseconds. + for (uint64_t s : {1, 5, 9}) + { + LogSchemaOp(*state, + 50 + s, + "db.a" + std::to_string(s), + 5, + s * 1'000'000 - 1, + s * 1'000'000); + } + + // With no retention window a checkpoint trims nothing: the records it + // passes are what a scan returns. + state->UpdateCkptTs(0, 8'000'000); + CdcScanRequest from_start; + from_start.set_from_ts(0); + CHECK(DdlRows(StateScan(*state, from_start)) == + std::vector>{{1'000'000, 51}, + {5'000'000, 55}}); + + // A purge by the floor 8 s - 4 s = 4 s, woken by the next flush, takes + // the record at 1 s. + state->SetCdcRetentionSeconds(4); + WriteBatchFile(state, 9'500'000, 1); + CHECK(WaitForHistory(*state, {5'000'000, 9'000'000}) == + std::vector{5'000'000, 9'000'000}); + + // With no window the next purge's floor is just below the checkpoint. + state->SetCdcRetentionSeconds(0); + state->UpdateCkptTs(0, 8'100'000); + WriteBatchFile(state, 9'600'000, 1); + CHECK(WaitForHistory(*state, {9'000'000}) == + std::vector{9'000'000}); +} + +TEST_CASE("A CDC scan never reads the RocksDB state while it is reopened", + "[cdc_scan][rocksdb]") +{ + ScratchLogState state; + state->UpdateNgTerm(0, 1, "127.0.0.1", 1); + constexpr int kRecords = 200; + for (uint64_t i = 1; i <= kRecords; ++i) + { + state->AddLogItem(0, i, i, std::string(100, 'v')); + } + state->UpdateCkptTs(0, kRecords); + + // A snapshot to install again and again, as a follower installs the + // leader's: each install closes db_, deletes its files, links the + // snapshot's in and reopens. The ng-info file is left out so that the + // bound the scans read stays put. + auto [snapshot_dir, files] = state.Snapshot(); + files.erase(std::remove(files.begin(), files.end(), "data"), files.end()); + + std::atomic stop{false}; + std::atomic complete_pages{0}; + std::atomic not_ready{0}; + std::atomic bad{0}; + std::vector scanners; + for (int t = 0; t < 3; ++t) + { + scanners.emplace_back( + [&] + { + CdcScanRequest request; + request.set_from_ts(0); + while (!stop.load()) + { + CdcScanResponse page; + std::string error; + switch (state->CdcScan(request, &page, &error)) + { + case CdcScanStatus::Ok: + // A scan that runs sees the whole, intact store. + if (page.data_size() == kRecords && + page.data(kRecords - 1).commit_ts() == kRecords) + { + ++complete_pages; + } + else + { + ++bad; + } + break; + case CdcScanStatus::NotReady: + ++not_ready; + break; + default: + ++bad; + } + } + }); + } + + for (int i = 0; i < 30; ++i) + { + REQUIRE(state->ReadSnapshot(snapshot_dir, files) == 0); + } + stop.store(true); + for (std::thread &t : scanners) + { + t.join(); + } + + CHECK(bad.load() == 0); + CHECK(complete_pages.load() > 0); + INFO("scans refused while the state was reopened: " << not_ready.load()); + // The store is readable again once the last install has finished. + CdcScanRequest request; + request.set_from_ts(0); + CHECK(Rows(StateScan(*state, request)).size() == kRecords); +} + +TEST_CASE("The RocksDB log state rejects a malformed continuation", + "[cdc_scan][rocksdb]") +{ + ScratchLogState state; + CdcScanRequest request; + request.set_continuation("not a continuation"); + CdcScanResponse page; + std::string error; + CHECK(state->CdcScan(request, &page, &error) == + CdcScanStatus::InvalidArgument); + CHECK_FALSE(error.empty()); +} +#endif diff --git a/eloq_log_service/test/LogStateClusterGate-Test.cpp b/eloq_log_service/test/LogStateClusterGate-Test.cpp new file mode 100644 index 000000000..6d474ce68 --- /dev/null +++ b/eloq_log_service/test/LogStateClusterGate-Test.cpp @@ -0,0 +1,290 @@ +/** + * Copyright (C) 2026 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ + +/** + * @file + * @brief The log state's side of the cluster gate (EloqDoc + * docs/cdc-change-stream.md §6.2; plan D5): the gate slot, which keeps the + * latest entry, is never purged, survives a snapshot and is replayed as one + * item. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "log.pb.h" +#include "log_state.h" + +// butil's logging, which the log state pulls in, defines a one-argument CHECK +// that would replace Catch's; nothing here uses butil's. Catch comes last so +// that its definition is the one in force. +#undef CHECK +#include + +using namespace txlog; + +namespace +{ +class TestLogState : public LogState +{ +public: + using LogState::CleanSnapshotState; + using LogState::GetClusterGateList; + using LogState::GetClusterScaleOpList; + using LogState::LoadClusterGateFrom; + using LogState::MakeCopyOfNgInfoAndCatalogOps; + using LogState::WriteSnapshotClusterGateTo; + + void AddLogItem(uint32_t, uint64_t, uint64_t, const std::string &) override + { + } + std::pair> GetLogReplayList( + uint32_t, uint64_t) override + { + return {false, nullptr}; + } + std::pair SearchTxDataLog(uint64_t, + uint32_t, + uint64_t) override + { + return {false, nullptr}; + } + void BeginSnapshot() override + { + } + int ReadSnapshot(const std::string &, + const std::vector &) override + { + return 0; + } + std::vector WriteSnapshot(const std::string &) override + { + return {}; + } +}; + +/** + * @brief A gate log entry's value. + * + * @param paused Whether the gate is closed. + * @return The value. + */ +ClusterGateLogMessage Gate(bool paused) +{ + ClusterGateLogMessage msg; + msg.set_mode(ClusterGateLogMessage_GateMode_GateDdl); + msg.set_paused(paused); + return msg; +} + +/** + * @brief The slot's value, version and transaction. + */ +struct Slot +{ + bool paused; + uint64_t version; + uint64_t txn; + + bool operator==(const Slot &o) const + { + return paused == o.paused && version == o.version && txn == o.txn; + } +}; + +/** + * @param state The log state. + * @return The slot, or none when it is empty. + */ +std::optional Read(const LogState &state) +{ + ClusterGateLogMessage msg; + uint64_t version = 0; + uint64_t txn = 0; + if (!state.GetClusterGate(&msg, &version, &txn)) + { + return std::nullopt; + } + return Slot{msg.paused(), version, txn}; +} +} // namespace + +TEST_CASE("The gate slot keeps the newest entry", "[log-state][cluster-gate]") +{ + TestLogState state; + CHECK_FALSE(Read(state).has_value()); + + state.UpdateClusterGate(11, 100, Gate(true)); + CHECK(Read(state) == Slot{true, 100, 11}); + + // A resume replaces the pause: the slot is overwritten in place. + state.UpdateClusterGate(12, 200, Gate(false)); + CHECK(Read(state) == Slot{false, 200, 12}); + + // An older entry, applied late, does not bring the pause back; nor does + // one at the same version. + state.UpdateClusterGate(11, 100, Gate(true)); + state.UpdateClusterGate(13, 200, Gate(true)); + CHECK(Read(state) == Slot{false, 200, 12}); + + // A retried write log finds whether its entry was applied. + CHECK(state.SearchTxClusterGate(12)); + CHECK_FALSE(state.SearchTxClusterGate(11)); +} + +TEST_CASE("The gate slot is never purged", "[log-state][cluster-gate]") +{ + TestLogState state; + state.UpdateNgTerm(0, 1, "127.0.0.1", 1); + state.UpdateClusterGate(21, 1'000, Gate(true)); + + // Checkpoints far past the entry, over an hour beyond it, which erase + // finished schema and cluster scale ops, and a purge's trim, leave a + // pause in place however old it is. + const uint64_t two_hours = 2ULL * 3600 * 1'000'000; + state.UpdateCkptTs(0, two_hours); + state.UpdateCkptTs(0, 2 * two_hours); + state.TrimCdcDdlHistory(2 * two_hours); + CHECK(Read(state) == Slot{true, 1'000, 21}); +} + +TEST_CASE("The gate slot is one replay item, after cluster scale's", + "[log-state][cluster-gate]") +{ + TestLogState state; + std::vector items; + state.GetClusterGateList(items); + CHECK(items.empty()); + + state.UpdateClusterGate(31, 300, Gate(true)); + state.UpdateClusterGate(32, 400, Gate(false)); + // The replay lists call the cluster scale list first; it asserts it holds + // at most one item, so the gate's comes after. + state.GetClusterScaleOpList(items); + state.GetClusterGateList(items); + REQUIRE(items.size() == 1); + const Item &item = *items[0]; + CHECK(item.item_type_ == LogItemType::ClusterGateLog); + CHECK(item.tx_number_ == 32); + CHECK(item.timestamp_ == 400); + ClusterGateLogMessage msg; + REQUIRE(msg.ParseFromString(item.log_message_)); + CHECK_FALSE(msg.paused()); +} + +TEST_CASE("The gate slot survives a snapshot", "[log-state][cluster-gate]") +{ + const std::filesystem::path file = + std::filesystem::temp_directory_path() / + ("cluster_gate_test_" + std::to_string(::getpid())); + + SECTION("a held pause") + { + TestLogState state; + state.UpdateClusterGate(41, 500, Gate(true)); + state.MakeCopyOfNgInfoAndCatalogOps(); + // Applied after the snapshot began: not in it. + state.UpdateClusterGate(42, 600, Gate(false)); + { + std::ofstream os(file, std::ios::binary); + state.WriteSnapshotClusterGateTo(os); + } + + TestLogState loaded; + // Whatever the loading state held is replaced. + loaded.UpdateClusterGate(99, 9'999, Gate(false)); + { + std::ifstream is(file, std::ios::binary); + REQUIRE(loaded.LoadClusterGateFrom(is)); + } + CHECK(Read(loaded) == Slot{true, 500, 41}); + + // A truncated file is refused, and the slot kept as it was. + std::filesystem::resize_file(file, + std::filesystem::file_size(file) - 1); + { + std::ifstream is(file, std::ios::binary); + CHECK_FALSE(loaded.LoadClusterGateFrom(is)); + } + CHECK(Read(loaded) == Slot{true, 500, 41}); + } + + SECTION("an empty slot") + { + TestLogState state; + state.MakeCopyOfNgInfoAndCatalogOps(); + { + std::ofstream os(file, std::ios::binary); + state.WriteSnapshotClusterGateTo(os); + } + TestLogState loaded; + loaded.UpdateClusterGate(51, 700, Gate(true)); + { + std::ifstream is(file, std::ios::binary); + REQUIRE(loaded.LoadClusterGateFrom(is)); + } + CHECK_FALSE(Read(loaded).has_value()); + } + + SECTION("a value whose length runs past the file's end") + { + { + std::ofstream os(file, std::ios::binary); + const uint8_t present = 1; + const uint64_t version = 1; + const uint64_t txn = 1; + const uint32_t length = 0xfffffff0; + os.write(reinterpret_cast(&present), sizeof(present)); + os.write(reinterpret_cast(&version), sizeof(version)); + os.write(reinterpret_cast(&txn), sizeof(txn)); + os.write(reinterpret_cast(&length), sizeof(length)); + } + TestLogState loaded; + loaded.UpdateClusterGate(61, 800, Gate(true)); + { + std::ifstream is(file, std::ios::binary); + CHECK_FALSE(loaded.LoadClusterGateFrom(is)); + } + CHECK(Read(loaded) == Slot{true, 800, 61}); + } + + std::filesystem::remove(file); +} + +TEST_CASE("Clearing the state empties the gate slot", + "[log-state][cluster-gate]") +{ + TestLogState state; + state.UpdateClusterGate(71, 900, Gate(true)); + state.MakeCopyOfNgInfoAndCatalogOps(); + state.CleanSnapshotState(); + CHECK(Read(state) == Slot{true, 900, 71}); + state.Clear(); + CHECK_FALSE(Read(state).has_value()); +} diff --git a/eloq_log_service/test/LogStateSchemaOp-Test.cpp b/eloq_log_service/test/LogStateSchemaOp-Test.cpp new file mode 100644 index 000000000..ae5facf0f --- /dev/null +++ b/eloq_log_service/test/LogStateSchemaOp-Test.cpp @@ -0,0 +1,615 @@ +/** + * @file LogStateSchemaOp-Test.cpp + * @brief The log state's schema op bookkeeping: the commit timestamp it keeps + * for recovery, and the CDC DDL history it appends to. + * + * CREATE, DROP and TRUNCATE TABLE choose their commit timestamp at the commit + * stage, above the prepare log's. The commit log's timestamp must therefore + * replace the prepare log's, or an op recovered from its commit log would + * resume at the prepare-stage timestamp. + * + * The CDC DDL history (docs/cdc-change-stream.md in the EloqDoc repo, §4.3) + * holds one record per EloqDoc create, drop and index op, at the timestamp of + * the log written under the op's catalog write lock. + */ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "log.pb.h" +#include "log_state.h" + +// butil's logging, which the log state pulls in, defines a one-argument CHECK +// that would replace Catch's; nothing here uses butil's. Catch comes last so +// that its definition is the one in force. +#undef CHECK +#include + +using namespace txlog; + +namespace +{ +/** + * @brief A log state that keeps no data log: the schema op bookkeeping under + * test lives in the LogState base, whatever the log state type. + */ +class TestLogState : public LogState +{ +public: + using LogState::GetSchemaOpList; + using LogState::LoadCdcDdlHistoryFrom; + using LogState::MakeCopyOfNgInfoAndCatalogOps; + using LogState::WriteSnapshotCdcDdlHistoryTo; + + void AddLogItem(uint32_t, uint64_t, uint64_t, const std::string &) override + { + } + std::pair> GetLogReplayList( + uint32_t, uint64_t) override + { + return {false, nullptr}; + } + std::pair SearchTxDataLog(uint64_t, + uint32_t, + uint64_t) override + { + return {false, nullptr}; + } + void BeginSnapshot() override + { + } + int ReadSnapshot(const std::string &, + const std::vector &) override + { + return 0; + } + std::vector WriteSnapshot(const std::string &) override + { + return {}; + } +}; + +/** + * @brief A schema op message for one stage of a CREATE TABLE. + * + * @param stage The stage. + * @return The message. + */ +SchemaOpMessage CreateTableOp(SchemaOpMessage_Stage stage) +{ + SchemaOpMessage msg; + msg.set_table_name_str("./db/coll"); + msg.set_table_type(CcTableType::Primary); + msg.set_stage(stage); + msg.mutable_table_op()->set_op_type(0); + return msg; +} + +/** + * @brief The commit timestamp a node recovering the op would resume with. + * + * @param state The log state. + * @param txn The op's tx. + * @return The timestamp, or nothing if the op is not replayed. + */ +std::optional ReplayedCommitTs(TestLogState &state, uint64_t txn) +{ + std::vector ops; + state.GetSchemaOpList(ops); + for (const Item::Pointer &op : ops) + { + if (op->tx_number_ == txn) + { + return op->timestamp_; + } + } + return std::nullopt; +} + +/** @brief txservice::OperationType values. */ +constexpr uint32_t kUpdate = 1, kCreateTable = 5, kDropTable = 6, + kTruncateTable = 7, kAddIndex = 8, kDropIndex = 9; + +/** + * @brief A schema op message as EloqDoc logs it. + * + * @param table The table name. + * @param op_type The txservice::OperationType. + * @param stage The stage. + * @param engine The table engine. + * @return The message. + */ +SchemaOpMessage DocOp(const std::string &table, + uint32_t op_type, + SchemaOpMessage_Stage stage, + TableEngine engine = TableEngine::EloqDoc) +{ + SchemaOpMessage msg; + msg.set_table_name_str(table); + msg.set_table_engine(engine); + msg.set_table_type(CcTableType::Primary); + msg.set_stage(stage); + msg.mutable_table_op()->set_op_type(op_type); + return msg; +} + +/** + * @brief Logs a schema op's prepare and commit logs. + * + * @param state The log state. + * @param txn The op's tx. + * @param msg The op's prepare message; its commit message differs only in + * stage and mark. + * @param prepare_ts The prepare log's timestamp, the op's catalog version. + * @param commit_ts The commit log's timestamp. + * @param rolled_back The commit log's rolled-back mark. + */ +void LogOp(TestLogState &state, + uint64_t txn, + SchemaOpMessage msg, + uint64_t prepare_ts, + uint64_t commit_ts, + bool rolled_back = false) +{ + msg.set_stage(SchemaOpMessage_Stage_PrepareSchema); + state.UpsertSchemaOp(txn, prepare_ts, msg); + msg.set_stage(SchemaOpMessage_Stage_CommitSchema); + msg.set_rolled_back(rolled_back); + state.UpsertSchemaOp(txn, commit_ts, msg); +} + +/** + * @param state The log state. + * @return The whole CDC DDL history, in position order. + */ +std::vector History(const LogState &state) +{ + std::vector records; + state.CopyCdcDdlHistory({}, UINT64_MAX, &records); + return records; +} + +/** + * @brief A DDL record reduced to what the tests compare. + */ +struct Ddl +{ + uint64_t ddl_ts; + uint64_t catalog_ts; + uint64_t txn; + std::string table; + uint32_t op_type; + bool rolled_back; + + bool operator==(const Ddl &o) const + { + return ddl_ts == o.ddl_ts && catalog_ts == o.catalog_ts && + txn == o.txn && table == o.table && op_type == o.op_type && + rolled_back == o.rolled_back; + } +}; + +std::ostream &operator<<(std::ostream &os, const Ddl &d) +{ + return os << "{" << d.ddl_ts << ", " << d.catalog_ts << ", " << d.txn + << ", " << d.table << ", " << d.op_type << ", " << d.rolled_back + << "}"; +} + +/** + * @param records DDL records. + * @return Their reduced form. + */ +std::vector Reduced(const std::vector &records) +{ + std::vector out; + for (const CdcDdlRecord &r : records) + { + out.push_back({r.ddl_ts(), + r.catalog_ts(), + r.txn_number(), + r.schema_op().table_name_str(), + r.schema_op().table_op().op_type(), + r.schema_op().rolled_back()}); + } + return out; +} +} // namespace + +TEST_CASE("A schema op recovers with its commit log's timestamp", + "[log-state][schema-op]") +{ + TestLogState state; + constexpr uint64_t kTxn = 7; + constexpr uint64_t kPrepareTs = 100; + constexpr uint64_t kCommitTs = 105; + + state.UpsertSchemaOp( + kTxn, kPrepareTs, CreateTableOp(SchemaOpMessage_Stage_PrepareSchema)); + REQUIRE(ReplayedCommitTs(state, kTxn) == kPrepareTs); + + // The commit stage re-assigned the timestamp above the prepare log's. + state.UpsertSchemaOp( + kTxn, kCommitTs, CreateTableOp(SchemaOpMessage_Stage_CommitSchema)); + REQUIRE(ReplayedCommitTs(state, kTxn) == kCommitTs); + + // A retried prepare log that arrives late changes nothing. + state.UpsertSchemaOp( + kTxn, kPrepareTs, CreateTableOp(SchemaOpMessage_Stage_PrepareSchema)); + REQUIRE(ReplayedCommitTs(state, kTxn) == kCommitTs); + + // Once cleaned, the op is no longer replayed. + state.UpsertSchemaOp( + kTxn, 0, CreateTableOp(SchemaOpMessage_Stage_CleanSchema)); + REQUIRE_FALSE(ReplayedCommitTs(state, kTxn).has_value()); +} + +TEST_CASE("A rolled-back add index recovers at its timestamp with its mark", + "[log-state][schema-op]") +{ + // An add index whose build broke a constraint commits at a real + // timestamp; recovery tells it from a successful one by the mark. + TestLogState state; + constexpr uint64_t kTxn = 9; + state.UpsertSchemaOp( + kTxn, 300, CreateTableOp(SchemaOpMessage_Stage_PrepareSchema)); + SchemaOpMessage commit = CreateTableOp(SchemaOpMessage_Stage_CommitSchema); + commit.set_rolled_back(true); + state.UpsertSchemaOp(kTxn, 305, commit); + + REQUIRE(ReplayedCommitTs(state, kTxn) == 305); + std::vector ops; + state.GetSchemaOpList(ops); + REQUIRE(ops.size() == 1); + SchemaOpMessage replayed; + REQUIRE(replayed.ParseFromString(ops[0]->log_message_)); + REQUIRE(replayed.rolled_back()); + REQUIRE(replayed.stage() == SchemaOpMessage_Stage_CommitSchema); +} + +TEST_CASE("A schema op that failed after its prepare log recovers at 0", + "[log-state][schema-op]") +{ + // A create that gave up on its kv tables, or an ALTER TABLE whose catalog + // write failed, is rolled back at timestamp 0: no change event, and + // recovery discards the dirty schema. + TestLogState state; + constexpr uint64_t kTxn = 8; + state.UpsertSchemaOp( + kTxn, 200, CreateTableOp(SchemaOpMessage_Stage_PrepareSchema)); + SchemaOpMessage commit = CreateTableOp(SchemaOpMessage_Stage_CommitSchema); + commit.set_rolled_back(true); + state.UpsertSchemaOp(kTxn, 0, commit); + + REQUIRE(ReplayedCommitTs(state, kTxn) == 0); + std::vector ops; + state.GetSchemaOpList(ops); + REQUIRE(ops.size() == 1); + SchemaOpMessage replayed; + REQUIRE(replayed.ParseFromString(ops[0]->log_message_)); + REQUIRE(replayed.rolled_back()); +} + +TEST_CASE("CREATE and DROP enter the CDC DDL history at their commit log", + "[log-state][cdc-ddl]") +{ + TestLogState state; + const SchemaOpMessage create = + DocOp("db.a", kCreateTable, SchemaOpMessage_Stage_PrepareSchema); + + // A create holds only an intent across its prepare log: no record yet. + state.UpsertSchemaOp(11, 100, create); + CHECK(History(state).empty()); + + // The commit log re-assigned the timestamp under the write lock. + SchemaOpMessage commit = create; + commit.set_stage(SchemaOpMessage_Stage_CommitSchema); + state.UpsertSchemaOp(11, 105, commit); + const std::vector records = History(state); + REQUIRE(records.size() == 1); + CHECK(Reduced(records) == + std::vector{{105, 100, 11, "db.a", kCreateTable, false}}); + CHECK(records[0].schema_op().stage() == SchemaOpMessage_Stage_CommitSchema); + + // A retried commit log, the clean log and a late prepare log change + // nothing. + state.UpsertSchemaOp(11, 105, commit); + commit.set_stage(SchemaOpMessage_Stage_CleanSchema); + state.UpsertSchemaOp(11, 0, commit); + state.UpsertSchemaOp(11, 100, create); + CHECK(Reduced(History(state)) == + std::vector{{105, 100, 11, "db.a", kCreateTable, false}}); + + LogOp(state, + 12, + DocOp("db.a", kDropTable, SchemaOpMessage_Stage_PrepareSchema), + 200, + 207); + CHECK(Reduced(History(state)) == + std::vector{{105, 100, 11, "db.a", kCreateTable, false}, + {207, 200, 12, "db.a", kDropTable, false}}); +} + +TEST_CASE("An index op enters the CDC DDL history at its prepare log", + "[log-state][cdc-ddl]") +{ + TestLogState state; + // One op adding two indexes is one record. + SchemaOpMessage add = + DocOp("db.a", kAddIndex, SchemaOpMessage_Stage_PrepareSchema); + add.set_alter_table_info_blob("two index names"); + state.UpsertSchemaOp(21, 300, add); + std::vector records = History(state); + REQUIRE(records.size() == 1); + CHECK(Reduced(records) == + std::vector{{300, 300, 21, "db.a", kAddIndex, false}}); + CHECK(records[0].schema_op().alter_table_info_blob() == "two index names"); + + // Its build and its commit log, at the same timestamp, add nothing. + SchemaOpMessage data = add; + data.set_stage(SchemaOpMessage_Stage_PrepareData); + state.UpsertSchemaOp(21, 300, data); + SchemaOpMessage commit = add; + commit.set_stage(SchemaOpMessage_Stage_CommitSchema); + state.UpsertSchemaOp(21, 300, commit); + CHECK(Reduced(History(state)) == + std::vector{{300, 300, 21, "db.a", kAddIndex, false}}); + + LogOp(state, + 22, + DocOp("db.a", kDropIndex, SchemaOpMessage_Stage_PrepareSchema), + 400, + 400); + CHECK(Reduced(History(state)) == + std::vector{{300, 300, 21, "db.a", kAddIndex, false}, + {400, 400, 22, "db.a", kDropIndex, false}}); +} + +TEST_CASE("A rolled-back add index has a record at each of its two logs", + "[log-state][cdc-ddl]") +{ + TestLogState state; + LogOp(state, + 31, + DocOp("db.a", kAddIndex, SchemaOpMessage_Stage_PrepareSchema), + 500, + 505, + true); + // createIndexes at the prepare log's timestamp, then the mark at the + // commit log's, from which the runtime makes dropIndexes. + CHECK(Reduced(History(state)) == + std::vector{{500, 500, 31, "db.a", kAddIndex, false}, + {505, 500, 31, "db.a", kAddIndex, true}}); +} + +TEST_CASE("An op rolled back at timestamp 0 leaves no CDC DDL record", + "[log-state][cdc-ddl]") +{ + TestLogState state; + // A create that gave up on its kv tables. + LogOp(state, + 41, + DocOp("db.a", kCreateTable, SchemaOpMessage_Stage_PrepareSchema), + 600, + 0, + true); + CHECK(History(state).empty()); + + // An index op's record, appended at its prepare log, is taken out again. + LogOp(state, + 42, + DocOp("db.a", kAddIndex, SchemaOpMessage_Stage_PrepareSchema), + 700, + 0, + true); + CHECK(History(state).empty()); +} + +TEST_CASE("Only EloqDoc's create, drop and index ops are recorded", + "[log-state][cdc-ddl]") +{ + TestLogState state; + // collMod reaches the tx service as an Update. + LogOp(state, + 51, + DocOp("db.a", kUpdate, SchemaOpMessage_Stage_PrepareSchema), + 800, + 805); + LogOp(state, + 52, + DocOp("db.a", kTruncateTable, SchemaOpMessage_Stage_PrepareSchema), + 810, + 815); + LogOp(state, + 53, + DocOp("t", + kCreateTable, + SchemaOpMessage_Stage_PrepareSchema, + TableEngine::EloqSql), + 820, + 825); + LogOp(state, + 54, + DocOp("t", + kAddIndex, + SchemaOpMessage_Stage_PrepareSchema, + TableEngine::EloqKv), + 830, + 830); + SchemaOpMessage column_op = + DocOp("db.a", kCreateTable, SchemaOpMessage_Stage_PrepareSchema); + column_op.mutable_column_op()->set_column_name("c"); + LogOp(state, 55, column_op, 840, 845); + CHECK(History(state).empty()); +} + +TEST_CASE("The CDC DDL history is copied by position", "[log-state][cdc-ddl]") +{ + TestLogState state; + // Two creates on different tables commit at one timestamp: the + // transaction number orders them. + LogOp(state, + 62, + DocOp("db.b", kCreateTable, SchemaOpMessage_Stage_PrepareSchema), + 900, + 950); + LogOp(state, + 61, + DocOp("db.a", kCreateTable, SchemaOpMessage_Stage_PrepareSchema), + 901, + 950); + LogOp(state, + 63, + DocOp("db.c", kCreateTable, SchemaOpMessage_Stage_PrepareSchema), + 960, + 970); + + const auto txns = [&](const std::string &after, uint64_t upper_ts) + { + std::vector records; + state.CopyCdcDdlHistory(after, upper_ts, &records); + std::vector out; + for (const CdcDdlRecord &r : records) + { + out.push_back(r.txn_number()); + } + return out; + }; + CHECK(txns({}, UINT64_MAX) == std::vector{61, 62, 63}); + // After a timestamp position: every record at that timestamp. + CHECK(txns(CdcTimestampPosition(950), UINT64_MAX) == + std::vector{61, 62, 63}); + // After a record's position: strictly after it. + CHECK(txns(CdcDdlPosition(950, 61), UINT64_MAX) == + std::vector{62, 63}); + CHECK(txns(CdcDdlPosition(950, 62), UINT64_MAX) == + std::vector{63}); + // Nothing above the upper bound. + CHECK(txns({}, 969) == std::vector{61, 62}); + CHECK(txns({}, 949).empty()); +} + +TEST_CASE("The CDC DDL history is trimmed by a purge, not by a checkpoint", + "[log-state][cdc-ddl]") +{ + TestLogState state; + state.UpdateNgTerm(0, 1, "127.0.0.1", 1); + // Records at 1 s, 5 s and 9 s, in microseconds. + for (uint64_t s : {1, 5, 9}) + { + LogOp(state, + s, + DocOp("db.a" + std::to_string(s), + kCreateTable, + SchemaOpMessage_Stage_PrepareSchema), + s * 1'000'000 - 1, + s * 1'000'000); + } + const auto ddl_ts = [&] + { + std::vector out; + for (const CdcDdlRecord &r : History(state)) + { + out.push_back(r.ddl_ts()); + } + return out; + }; + + // Checkpoints past every record, with no retention window, trim nothing: + // a scan returns a record only once the checkpoint passes it. + state.UpdateCkptTs(0, 8'000'000); + state.UpdateCkptTs(0, 10'000'000); + CHECK(ddl_ts() == std::vector{1'000'000, 5'000'000, 9'000'000}); + + // A purge trims below its floor; a record exactly at the floor stays. + state.TrimCdcDdlHistory(5'000'000); + CHECK(ddl_ts() == std::vector{5'000'000, 9'000'000}); + state.TrimCdcDdlHistory(9'000'001); + CHECK(ddl_ts().empty()); +} + +TEST_CASE("The CDC DDL history survives a snapshot", "[log-state][cdc-ddl]") +{ + TestLogState state; + LogOp(state, + 71, + DocOp("db.a", kCreateTable, SchemaOpMessage_Stage_PrepareSchema), + 1000, + 1005); + LogOp(state, + 72, + DocOp("db.a", kAddIndex, SchemaOpMessage_Stage_PrepareSchema), + 1010, + 1015, + true); + state.MakeCopyOfNgInfoAndCatalogOps(); + // Appended after the snapshot began: not in it. + LogOp(state, + 73, + DocOp("db.b", kCreateTable, SchemaOpMessage_Stage_PrepareSchema), + 1020, + 1025); + + const std::filesystem::path file = + std::filesystem::temp_directory_path() / + ("cdc_ddl_history_test_" + std::to_string(::getpid())); + { + std::ofstream os(file, std::ios::binary); + state.WriteSnapshotCdcDdlHistoryTo(os); + } + + TestLogState loaded; + // Whatever the loading state held is replaced. + LogOp(loaded, + 99, + DocOp("db.z", kCreateTable, SchemaOpMessage_Stage_PrepareSchema), + 1, + 2); + { + std::ifstream is(file, std::ios::binary); + REQUIRE(loaded.LoadCdcDdlHistoryFrom(is)); + } + std::vector expected = History(state); + expected.pop_back(); + CHECK(Reduced(History(loaded)) == Reduced(expected)); + CHECK(Reduced(History(loaded)).size() == 3); + // Positions are rebuilt: a copy after a record's position still works. + std::vector after; + loaded.CopyCdcDdlHistory(CdcDdlPosition(1010, 72), UINT64_MAX, &after); + CHECK(Reduced(after) == + std::vector{{1015, 1010, 72, "db.a", kAddIndex, true}}); + + // A truncated file is refused. + std::filesystem::resize_file(file, std::filesystem::file_size(file) - 1); + TestLogState truncated; + { + std::ifstream is(file, std::ios::binary); + CHECK_FALSE(truncated.LoadCdcDdlHistoryFrom(is)); + } + CHECK(History(truncated).empty()); + + // So is a record whose length runs past the file's end, before a buffer + // of that length is sized; the loading state keeps what it had. + { + std::ofstream os(file, std::ios::binary | std::ios::trunc); + const uint32_t cnt = 1; + const uint32_t length = 0xfffffff0; + os.write(reinterpret_cast(&cnt), sizeof(cnt)); + os.write(reinterpret_cast(&length), sizeof(length)); + os.write("abc", 3); + } + { + std::ifstream is(file, std::ios::binary); + CHECK_FALSE(loaded.LoadCdcDdlHistoryFrom(is)); + } + CHECK(Reduced(History(loaded)).size() == 3); + std::filesystem::remove(file); +} diff --git a/eloq_log_service/test/log_server_rocksdb_cloud_tests.cpp b/eloq_log_service/test/log_server_rocksdb_cloud_tests.cpp index 928fcdfb0..700fa6cd6 100644 --- a/eloq_log_service/test/log_server_rocksdb_cloud_tests.cpp +++ b/eloq_log_service/test/log_server_rocksdb_cloud_tests.cpp @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include "fault_inject.h" #include "log.pb.h" @@ -1754,6 +1756,590 @@ CATCH_TEST_CASE("test_A_B_replay", "[log_server_rocksdb_cloud_tests]") DropBucket(FLAGS_region, bucket_prefix, bucket_name); } +namespace +{ +/** + * The cloud state asks its observer for a snapshot when the in-memory queue + * grows past its high watermark; the test sets the watermark out of reach. + */ +class NoSnapshotObserver : public txlog::LogStateRocksDBCloudImplObserver +{ +public: + void OnInMemStateFull( + size_t, size_t, std::function done) const override + { + done(false, 0); + } +}; + +txlog::CdcScanRequest CdcScanFrom(uint64_t from_ts, uint32_t max_bytes = 0) +{ + txlog::CdcScanRequest request; + request.set_log_group_id(0); + request.set_from_ts(from_ts); + request.set_max_bytes(max_bytes); + return request; +} + +txlog::CdcScanResponse CdcScanOk(txlog::LogStateRocksDBCloudImpl &state, + const txlog::CdcScanRequest &request) +{ + txlog::CdcScanResponse page; + std::string error; + CATCH_REQUIRE(state.CdcScan(request, &page, &error) == + txlog::CdcScanStatus::Ok); + return page; +} +} // namespace + +/** + * The CDC log scan served by the RocksDB-Cloud state (cdc_scan.h): refused + * until the leader's cloud DB is in sync, then records written before the DB + * opened (buffered in memory) and after it are both read back from the DB, + * bounded by the checkpoint and paged through continuations. + */ +CATCH_TEST_CASE("test_cdc_scan_cloud_state", "[log_server_rocksdb_cloud_tests]") +{ + LOG(INFO) << "TEST_CASE: test_cdc_scan_cloud_state"; + std::string bucket_name = + "rocksdb-cloud-tests" + std::to_string(time(NULL)); + std::string bucket_prefix = "log-server-test-"; + PrepareBucket(FLAGS_region, bucket_prefix, bucket_name); + + txlog::RocksDBCloudConfig rocksdb_cloud_config; + PrepareCloudConfig(rocksdb_cloud_config, bucket_prefix, bucket_name); + rocksdb_cloud_config.object_path_ = "rocksdb_cloud"; + rocksdb_cloud_config.log_retention_days_ = 90; + rocksdb_cloud_config.log_purger_starting_hour_ = 0; + rocksdb_cloud_config.log_purger_starting_minute_ = 0; + rocksdb_cloud_config.log_purger_starting_second_ = 1; + // The state opens its DB itself; route it at the test store in CI, as + // PrepareCFSOptions does for the tests that open one directly. + rocksdb_cloud_config.endpoint_url_ = TestS3Endpoint(); + + std::filesystem::path current_dir = std::filesystem::current_path(); + std::string storage_path = current_dir / "log_server_rocksdb_tests/cdc0"; + PrepareLogStoragePaths({storage_path}); + + std::atomic term_if_is_lg_leader{-1}; + NoSnapshotObserver observer; + txlog::LogStateRocksDBCloudImpl state(storage_path + "/rocksdb", + rocksdb_cloud_config, + term_if_is_lg_leader, + &observer, + 1000 * 1000, + 8, + 8, + 64 * 1024 * 1024, + 1); + CATCH_REQUIRE(state.Start() == 0); + state.UpdateNgTerm(0, 1, "127.0.0.1", 8888); + + const std::vector ts{1000, 2000, 3000, 4000, 5000}; + const std::vector txn{11, 12, 13, 14, 15}; + std::vector blobs; + for (size_t i = 0; i < ts.size(); ++i) + { + blobs.push_back("cdc-" + std::to_string(i) + std::string(64, 'x')); + } + const auto check_record = [&](const txlog::CdcDataRecord &record, size_t i) + { + CATCH_REQUIRE(record.commit_ts() == ts[i]); + CATCH_REQUIRE(record.cc_ng_id() == 0); + CATCH_REQUIRE(record.txn_number() == txn[i]); + CATCH_REQUIRE(record.blob() == blobs[i]); + }; + + // Not the leader: refused. + { + txlog::CdcScanResponse page; + std::string error; + CATCH_REQUIRE(state.CdcScan(CdcScanFrom(0), &page, &error) == + txlog::CdcScanStatus::NotLeader); + } + + // The leader whose DB is not open yet: the scan waits a little for it, + // then is refused rather than served from a partial view. The first + // record goes to the in-memory queue. + term_if_is_lg_leader.store(1); + { + txlog::CdcScanResponse page; + std::string error; + CATCH_REQUIRE(state.CdcScan(CdcScanFrom(0), &page, &error) == + txlog::CdcScanStatus::NotReady); + CATCH_REQUIRE_FALSE(error.empty()); + } + state.AddLogItem(0, txn[0], ts[0], blobs[0]); + + // The DB opens asynchronously and the queue is dumped into it before the + // state reports itself in sync. + state.AsyncStartCloudDB(0, 1); + txlog::CdcScanStatus status = txlog::CdcScanStatus::NotReady; + for (int i = 0; i < 600 && status == txlog::CdcScanStatus::NotReady; ++i) + { + txlog::CdcScanResponse page; + std::string error; + status = state.CdcScan(CdcScanFrom(0), &page, &error); + if (status == txlog::CdcScanStatus::NotReady) + { + usleep(100 * 1000); + } + } + CATCH_REQUIRE(status == txlog::CdcScanStatus::Ok); + + // The rest go straight to the DB. + for (size_t i = 1; i < ts.size(); ++i) + { + state.AddLogItem(0, txn[i], ts[i], blobs[i]); + } + + // Nothing is known complete before a checkpoint. + txlog::CdcScanResponse page = CdcScanOk(state, CdcScanFrom(0)); + CATCH_REQUIRE(page.data_size() == 0); + CATCH_REQUIRE(page.arrival_complete_ts() == 0); + CATCH_REQUIRE_FALSE(page.has_more()); + + // Checkpoint through the fourth record: the first four come back in + // order, the one buffered before the DB opened included. + state.UpdateCkptTs(0, ts[3]); + page = CdcScanOk(state, CdcScanFrom(0)); + CATCH_REQUIRE(page.arrival_complete_ts() == ts[3]); + CATCH_REQUIRE(page.data_size() == 4); + for (int i = 0; i < 4; ++i) + { + check_record(page.data(i), i); + } + CATCH_REQUIRE_FALSE(page.has_more()); + + // One record per page through the continuation. + txlog::CdcScanRequest request = CdcScanFrom(0, 1); + for (int i = 0; i < 4; ++i) + { + page = CdcScanOk(state, request); + CATCH_REQUIRE(page.data_size() == 1); + check_record(page.data(0), i); + CATCH_REQUIRE(page.has_more() == (i < 3)); + request.set_continuation(page.continuation()); + } + + // Exhausted through the bound; the last record comes with the next + // checkpoint, from the same continuation. + page = CdcScanOk(state, request); + CATCH_REQUIRE(page.data_size() == 0); + state.UpdateCkptTs(0, ts[4]); + page = CdcScanOk(state, request); + CATCH_REQUIRE(page.arrival_complete_ts() == ts[4]); + CATCH_REQUIRE(page.data_size() == 1); + check_record(page.data(0), 4); + + // from_ts starts at the first record at or above it. + page = CdcScanOk(state, CdcScanFrom(ts[2])); + CATCH_REQUIRE(page.data_size() == 3); + for (int i = 0; i < 3; ++i) + { + check_record(page.data(i), i + 2); + } + + // Losing leadership stops the DB, and scans are refused again. A record + // applied meanwhile goes to the in-memory queue. + term_if_is_lg_leader.store(-1); + state.StopRocksDB(); + { + txlog::CdcScanResponse refused; + std::string error; + CATCH_REQUIRE(state.CdcScan(CdcScanFrom(0), &refused, &error) == + txlog::CdcScanStatus::NotLeader); + } + const uint64_t ts5 = 6000; + const std::string blob5 = "cdc-5" + std::string(64, 'x'); + state.AddLogItem(0, 16, ts5, blob5); + + // Leader again, in a new term with a new DB container: once that one is + // in sync, a scan sees the records the last term wrote and the one + // buffered in between. + term_if_is_lg_leader.store(2); + state.AsyncStartCloudDB(1, 2); + status = txlog::CdcScanStatus::NotReady; + for (int i = 0; i < 600 && status == txlog::CdcScanStatus::NotReady; ++i) + { + txlog::CdcScanResponse attempt; + std::string error; + status = state.CdcScan(CdcScanFrom(0), &attempt, &error); + } + CATCH_REQUIRE(status == txlog::CdcScanStatus::Ok); + state.UpdateCkptTs(0, ts5); + page = CdcScanOk(state, CdcScanFrom(0)); + CATCH_REQUIRE(page.data_size() == 6); + for (int i = 0; i < 5; ++i) + { + check_record(page.data(i), i); + } + CATCH_REQUIRE(page.data(5).commit_ts() == ts5); + CATCH_REQUIRE(page.data(5).blob() == blob5); + + term_if_is_lg_leader.store(-1); + state.StopRocksDB(); + DropBucket(FLAGS_region, bucket_prefix, bucket_name); +} + +namespace +{ +/** + * @param state The cloud log state. + * @return The commit timestamps a full scan through the checkpoint returns. + */ +std::vector CloudScannedTimestamps( + txlog::LogStateRocksDBCloudImpl &state) +{ + // Held in a local: a range-for over a temporary's member would outlive it. + const txlog::CdcScanResponse page = CdcScanOk(state, CdcScanFrom(0)); + std::vector out; + for (const txlog::CdcDataRecord &record : page.data()) + { + out.push_back(record.commit_ts()); + } + return out; +} + +/** + * @brief Polls until a condition holds for the scanned timestamps, or a minute + * passes. + * + * @param state The cloud log state. + * @param done The condition on CloudScannedTimestamps(). + * @return The last timestamps scanned. + */ +template +std::vector CloudWaitForScan(txlog::LogStateRocksDBCloudImpl &state, + Pred done) +{ + std::vector ts = CloudScannedTimestamps(state); + for (int i = 0; i < 600 && !done(ts); ++i) + { + usleep(100 * 1000); + ts = CloudScannedTimestamps(state); + } + return ts; +} +} // namespace + +namespace +{ +/** + * @brief Polls until the cloud state serves CdcScan, which it does once its + * DB container of the current term is open and in sync, or a minute passes. + * + * @param state The cloud log state, leading. + * @return Whether it serves. + */ +bool CloudWaitInSync(txlog::LogStateRocksDBCloudImpl &state) +{ + txlog::CdcScanStatus status = txlog::CdcScanStatus::NotReady; + for (int i = 0; i < 600 && status == txlog::CdcScanStatus::NotReady; ++i) + { + txlog::CdcScanResponse page; + std::string error; + status = state.CdcScan(CdcScanFrom(0), &page, &error); + if (status == txlog::CdcScanStatus::NotReady) + { + usleep(100 * 1000); + } + } + return status == txlog::CdcScanStatus::Ok; +} + +/** + * @brief Logs an EloqDoc schema op's prepare and commit logs. + * + * @param state The log state. + * @param txn The op's tx. + * @param op_type The txservice::OperationType. + * @param prepare_ts The prepare log's timestamp, the op's catalog version. + * @param commit_ts The commit log's timestamp. + */ +void CloudLogSchemaOp(txlog::LogStateRocksDBCloudImpl &state, + uint64_t txn, + uint32_t op_type, + uint64_t prepare_ts, + uint64_t commit_ts) +{ + txlog::SchemaOpMessage msg; + msg.set_table_name_str("db.cdc"); + msg.set_table_engine(txlog::TableEngine::EloqDoc); + msg.set_table_type(txlog::CcTableType::Primary); + msg.mutable_table_op()->set_op_type(op_type); + msg.set_stage(txlog::SchemaOpMessage_Stage_PrepareSchema); + state.UpsertSchemaOp(txn, prepare_ts, msg); + msg.set_stage(txlog::SchemaOpMessage_Stage_CommitSchema); + state.UpsertSchemaOp(txn, commit_ts, msg); +} + +/** + * @param page A scan page. + * @return (ddl_ts, txn) of its DDL records. + */ +std::vector> CloudDdlRows( + const txlog::CdcScanResponse &page) +{ + std::vector> out; + for (const txlog::CdcDdlRecord &record : page.ddl()) + { + out.emplace_back(record.ddl_ts(), record.txn_number()); + } + return out; +} +} // namespace + +/** + * The CDC retention floor in the cloud state's archive mover (Stage 1): it + * moves out of the live DB only files wholly below the checkpoint less the + * retention window, since CDC scans read the live DB and never the archive, + * and trims the CDC DDL history by the same floor. + */ +CATCH_TEST_CASE("test_cdc_retention_floor_cloud_state", + "[log_server_rocksdb_cloud_tests]") +{ + LOG(INFO) << "TEST_CASE: test_cdc_retention_floor_cloud_state"; + std::string bucket_name = + "rocksdb-cloud-tests" + std::to_string(time(NULL)); + std::string bucket_prefix = "log-server-test-"; + PrepareBucket(FLAGS_region, bucket_prefix, bucket_name); + + txlog::RocksDBCloudConfig rocksdb_cloud_config; + PrepareCloudConfig(rocksdb_cloud_config, bucket_prefix, bucket_name); + rocksdb_cloud_config.object_path_ = "rocksdb_cloud"; + rocksdb_cloud_config.log_retention_days_ = 90; + rocksdb_cloud_config.log_purger_starting_hour_ = 0; + rocksdb_cloud_config.log_purger_starting_minute_ = 0; + rocksdb_cloud_config.log_purger_starting_second_ = 1; + rocksdb_cloud_config.endpoint_url_ = TestS3Endpoint(); + // Run the mover every two seconds. One second would disable it: the + // cron job's first run time would not lie in the future. + rocksdb_cloud_config.archive_object_path_ = "rocksdb_cloud_archives"; + rocksdb_cloud_config.archive_move_interval_seconds_ = 2; + + std::filesystem::path current_dir = std::filesystem::current_path(); + std::string storage_path = current_dir / "log_server_rocksdb_tests/cdc1"; + PrepareLogStoragePaths({storage_path}); + + std::atomic term_if_is_lg_leader{1}; + NoSnapshotObserver observer; + txlog::LogStateRocksDBCloudImpl state(storage_path + "/rocksdb", + rocksdb_cloud_config, + term_if_is_lg_leader, + &observer, + 1000 * 1000, + 8, + 8, + 64 * 1024 * 1024, + 1); + CATCH_REQUIRE(state.Start() == 0); + state.UpdateNgTerm(0, 1, "127.0.0.1", 8888); + state.SetCdcRetentionSeconds(4); + state.AsyncStartCloudDB(0, 1); + txlog::CdcScanStatus status = txlog::CdcScanStatus::NotReady; + for (int i = 0; i < 600 && status == txlog::CdcScanStatus::NotReady; ++i) + { + txlog::CdcScanResponse page; + std::string error; + status = state.CdcScan(CdcScanFrom(0), &page, &error); + if (status == txlog::CdcScanStatus::NotReady) + { + usleep(100 * 1000); + } + } + CATCH_REQUIRE(status == txlog::CdcScanStatus::Ok); + + // Three files, timestamps in microseconds: A at 1 s, B at 5 s, C at 9 s. + const auto write_file = [&](uint64_t base_ts) + { + for (uint64_t i = 0; i < 3; ++i) + { + state.AddLogItem(0, base_ts + i, base_ts + i, "r"); + } + rocksdb::FlushOptions flush_options; + flush_options.wait = true; + CATCH_REQUIRE( + state.GetDBCloudContainer()->GetDBPtr()->Flush(flush_options).ok()); + }; + write_file(1'000'000); + write_file(5'000'000); + write_file(9'000'000); + // A DDL record beside each file (txservice::OperationType CreateTable). + for (uint64_t s : {1, 5, 9}) + { + CloudLogSchemaOp(state, 50 + s, 5, s * 1'000'000 - 1, s * 1'000'000); + } + const auto history = [&] + { + std::vector records; + state.CopyCdcDdlHistory({}, UINT64_MAX, &records); + std::vector out; + for (const txlog::CdcDdlRecord &record : records) + { + out.push_back(record.ddl_ts()); + } + return out; + }; + const auto wait_for_history = [&](const std::vector &expected) + { + std::vector ts = history(); + for (int i = 0; i < 600 && ts != expected; ++i) + { + usleep(100 * 1000); + ts = history(); + } + return ts; + }; + + // Checkpoint at 8 s. The floor is 8 s - 4 s = 4 s: A leaves the live DB, + // B stays, and so do the DDL records at 5 s and 9 s. + state.UpdateCkptTs(0, 8'000'000); + std::vector ts = + CloudWaitForScan(state, + [](const std::vector &t) + { return !t.empty() && t.front() >= 5'000'000; }); + CATCH_REQUIRE(ts == std::vector{5'000'000, 5'000'001, 5'000'002}); + CATCH_REQUIRE(wait_for_history({5'000'000, 9'000'000}) == + std::vector{5'000'000, 9'000'000}); + + // With no window the mover takes everything checkpointed, B included, and + // the DDL record at 5 s. + state.SetCdcRetentionSeconds(0); + ts = CloudWaitForScan( + state, [](const std::vector &t) { return t.empty(); }); + CATCH_REQUIRE(ts.empty()); + CATCH_REQUIRE(wait_for_history({9'000'000}) == + std::vector{9'000'000}); + + // C, above the checkpoint, survived. Widen the window before moving the + // checkpoint past it, so no later mover run can take it. + state.SetCdcRetentionSeconds(3600); + state.UpdateCkptTs(0, 9'500'000); + CATCH_REQUIRE(CloudScannedTimestamps(state) == + std::vector{9'000'000, 9'000'001, 9'000'002}); + + DropBucket(FLAGS_region, bucket_prefix, bucket_name); +} + +/** + * The CDC DDL history on the cloud log state (plan A8): the scan merges it + * with the data log in the DB, and a replica that installs the leader's raft + * snapshot, which carries the history in a file of its own, serves the same + * records once it leads the next term and its DB container is in sync. + */ +CATCH_TEST_CASE("test_cdc_ddl_history_cloud_state", + "[log_server_rocksdb_cloud_tests]") +{ + LOG(INFO) << "TEST_CASE: test_cdc_ddl_history_cloud_state"; + std::string bucket_name = + "rocksdb-cloud-tests" + std::to_string(time(NULL)); + std::string bucket_prefix = "log-server-test-"; + PrepareBucket(FLAGS_region, bucket_prefix, bucket_name); + + txlog::RocksDBCloudConfig rocksdb_cloud_config; + PrepareCloudConfig(rocksdb_cloud_config, bucket_prefix, bucket_name); + rocksdb_cloud_config.object_path_ = "rocksdb_cloud"; + rocksdb_cloud_config.log_retention_days_ = 90; + rocksdb_cloud_config.log_purger_starting_hour_ = 0; + rocksdb_cloud_config.log_purger_starting_minute_ = 0; + rocksdb_cloud_config.log_purger_starting_second_ = 1; + rocksdb_cloud_config.endpoint_url_ = TestS3Endpoint(); + + std::filesystem::path current_dir = std::filesystem::current_path(); + std::string leader_path = current_dir / "log_server_rocksdb_tests/cdc2"; + std::string follower_path = current_dir / "log_server_rocksdb_tests/cdc3"; + PrepareLogStoragePaths({leader_path, follower_path}); + + // The leader of term 1. + std::atomic leader_term{1}; + NoSnapshotObserver observer; + txlog::LogStateRocksDBCloudImpl leader(leader_path + "/rocksdb", + rocksdb_cloud_config, + leader_term, + &observer, + 1000 * 1000, + 8, + 8, + 64 * 1024 * 1024, + 1); + CATCH_REQUIRE(leader.Start() == 0); + leader.UpdateNgTerm(0, 1, "127.0.0.1", 8888); + // A window keeps the DDL records the checkpoint passes. + leader.SetCdcRetentionSeconds(3600); + leader.AsyncStartCloudDB(0, 1); + CATCH_REQUIRE(CloudWaitInSync(leader)); + + for (uint64_t ts : {1000, 2000, 3000}) + { + leader.AddLogItem(0, ts, ts, "d" + std::to_string(ts)); + } + // A create committed at 2000, the timestamp of a data record, and an add + // index at 2500 (txservice::OperationType CreateTable and AddIndex). + CloudLogSchemaOp(leader, 50, 5, 1500, 2000); + CloudLogSchemaOp(leader, 51, 8, 2500, 2500); + leader.UpdateCkptTs(0, 2600); + + const std::vector> ddl{{2000, 50}, + {2500, 51}}; + txlog::CdcScanResponse page = CdcScanOk(leader, CdcScanFrom(0)); + CATCH_REQUIRE(page.data_size() == 2); + CATCH_REQUIRE(CloudDdlRows(page) == ddl); + CATCH_REQUIRE(page.ddl(0).catalog_ts() == 1500); + + // One record per page: at 2000 the data record comes first. + std::vector> merged; + txlog::CdcScanRequest request = CdcScanFrom(0, 1); + for (int i = 0; i < 4; ++i) + { + page = CdcScanOk(leader, request); + CATCH_REQUIRE(page.data_size() + page.ddl_size() == 1); + merged.emplace_back(page.ddl_size() == 1, + page.ddl_size() == 1 ? page.ddl(0).ddl_ts() + : page.data(0).commit_ts()); + request.set_continuation(page.continuation()); + } + CATCH_REQUIRE( + merged == + std::vector>{ + {false, 1000}, {false, 2000}, {true, 2000}, {true, 2500}}); + + // The leader's raft snapshot, then its step-down. + const std::string snapshot_dir = leader_path + "/snapshot"; + std::filesystem::create_directories(snapshot_dir); + leader.BeginSnapshot(); + const std::vector files = leader.WriteSnapshot(snapshot_dir); + leader_term.store(-1); + leader.StopRocksDB(); + + // Another replica installs the snapshot and leads term 2, in a DB + // container of its own over the same cloud storage. + std::atomic follower_term{-1}; + txlog::LogStateRocksDBCloudImpl follower(follower_path + "/rocksdb", + rocksdb_cloud_config, + follower_term, + &observer, + 1000 * 1000, + 8, + 8, + 64 * 1024 * 1024, + 1); + follower.SetCdcRetentionSeconds(3600); + CATCH_REQUIRE(follower.ReadSnapshot(snapshot_dir, files) == 0); + follower_term.store(2); + follower.AsyncStartCloudDB(1, 2); + CATCH_REQUIRE(CloudWaitInSync(follower)); + // The bound is rebuilt at the next checkpoint. + follower.UpdateCkptTs(0, 2601); + page = CdcScanOk(follower, CdcScanFrom(0)); + CATCH_REQUIRE(page.data_size() == 2); + CATCH_REQUIRE(CloudDdlRows(page) == ddl); + + follower_term.store(-1); + follower.StopRocksDB(); + DropBucket(FLAGS_region, bucket_prefix, bucket_name); +} + int main(int argc, char **argv) { #if defined(LOG_STATE_TYPE_RKDB_S3) diff --git a/eloq_log_service/test/log_snapshot_stream_test.cpp b/eloq_log_service/test/log_snapshot_stream_test.cpp index 9d407e25d..611e7ed12 100644 --- a/eloq_log_service/test/log_snapshot_stream_test.cpp +++ b/eloq_log_service/test/log_snapshot_stream_test.cpp @@ -53,7 +53,10 @@ class SnapshotFixture 8, 64 * 1024 * 1024, 1); - files_ = {"nginfo_and_catalog", "in_mem_data_log"}; + files_ = {"nginfo_and_catalog", + "cdc_ddl_history", + "cluster_gate", + "in_mem_data_log"}; #elif defined(LOG_STATE_TYPE_MEM) state_ = std::make_unique(); files_ = {"data"}; @@ -140,7 +143,10 @@ CATCH_TEST_CASE( { SnapshotFixture fixture(GENERATE(size_t{4}, size_t{16384})); #if defined(LOG_STATE_TYPE_RKDB_CLOUD) - const auto file = GENERATE("nginfo_and_catalog", "in_mem_data_log"); + const auto file = GENERATE("nginfo_and_catalog", + "cdc_ddl_history", + "cluster_gate", + "in_mem_data_log"); #else const auto file = "data"; #endif diff --git a/eloq_log_service/test/log_snapshot_test.cpp b/eloq_log_service/test/log_snapshot_test.cpp index a840ab0d1..3e4973c16 100644 --- a/eloq_log_service/test/log_snapshot_test.cpp +++ b/eloq_log_service/test/log_snapshot_test.cpp @@ -437,7 +437,8 @@ CATCH_TEST_CASE( 0, false, 10, - 128 * 1024 * 1024); + 128 * 1024 * 1024, + /*cdc_retention_seconds=*/0); CATCH_REQUIRE(instance.Start() == 0); braft::StateMachine &fsm = instance; for (int attempt = 0; attempt != 4; ++attempt) @@ -506,7 +507,8 @@ CATCH_TEST_CASE( 0, false, 10, - 128 * 1024 * 1024); + 128 * 1024 * 1024, + /*cdc_retention_seconds=*/0); }; auto instance = make_instance(); CATCH_REQUIRE(instance->Start() == 0); @@ -667,7 +669,8 @@ CATCH_TEST_CASE( 0, false, 10, - 128 * 1024 * 1024); + 128 * 1024 * 1024, + /*cdc_retention_seconds=*/0); CATCH_REQUIRE(instance.Start() == 0); const auto deadline = std::chrono::steady_clock::now() + 10s; while (!instance.is_leader() && std::chrono::steady_clock::now() < deadline) diff --git a/store_handler/data_store_service_client.cpp b/store_handler/data_store_service_client.cpp index 7c0963dff..b2639b8d2 100644 --- a/store_handler/data_store_service_client.cpp +++ b/store_handler/data_store_service_client.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -2263,7 +2264,7 @@ bool DataStoreServiceClient::UpdateRangeSlices( */ bool DataStoreServiceClient::UpsertRanges( const txservice::TableName &table_name, - std::vector range_info, + const std::vector &range_info, uint64_t version) { assert(table_name.StringView() != txservice::empty_sv); @@ -2282,7 +2283,7 @@ bool DataStoreServiceClient::UpsertRanges( slice_plans.reserve(TotalRangeSlicesKvPartitions()); RangeMetadataAccumulator meta_acc; - for (auto &range : range_info) + for (const auto &range : range_info) { // Prepare slice batches for this range auto slice_plan = PrepareRangeSliceBatches( @@ -2709,6 +2710,19 @@ bool DataStoreServiceClient::FetchAllDatabase( return !callback_data->HasError(); } +bool DataStoreServiceClient::CreateKvTable( + const txservice::TableName &table_name, uint64_t version) +{ + // The data store service keeps no per-table state (its CreateTable is a + // no-op on every backend), so only a range-partitioned table has anything + // to write: its initial range record. + if (table_name.IsHashPartitioned()) + { + return true; + } + return InitTableRanges(table_name, version); +} + bool DataStoreServiceClient::DropKvTable(const std::string &kv_table_name) { SyncCallbackData *callback_data = sync_callback_data_pool_.NextObject(); @@ -4119,15 +4133,107 @@ void DataStoreServiceClient::OnStartFollowing(uint32_t ng_id, DLOG(INFO) << "UpdatePrimaryNode, dss_shard_id: " << dss_shard_id << ", DSSNode: " << dss_leader_node.host_name_ << ":" << dss_leader_node.port_; - // Pump the dss shard version + // The shard's version is the leader term it is hosted under: the one + // number every node derives for this leadership, so that versions from + // different nodes compare. Never lowered. cluster_manager.UpdateDSShardVersion( - dss_shard_id, cluster_manager.FetchDSShardVersion(dss_shard_id) + 1); + dss_shard_id, + std::max(cluster_manager.FetchDSShardVersion(dss_shard_id), + static_cast(term))); // Update the client config SetupConfig(cluster_manager); Connect(); } +void DataStoreServiceClient::OnLeaderChanged(uint32_t ng_id, + uint32_t leader_node_id, + int64_t term) +{ + if (!bind_data_shard_with_ng_) + { + return; + } + const uint32_t shard_id = ng_id; + if (shard_id >= dss_shards_.size()) + { + LOG(ERROR) << "OnLeaderChanged: shard_id " << shard_id + << " exceeds array bounds"; + return; + } + + auto ng_configs = txservice::Sharder::Instance().GetNodeGroupConfigs(); + auto ng_config_it = ng_configs.find(shard_id); + if (ng_config_it == ng_configs.end()) + { + LOG(WARNING) << "OnLeaderChanged: no config for node group " << ng_id; + return; + } + const txservice::NodeConfig *leader_config = nullptr; + for (const auto &node_config : ng_config_it->second) + { + if (node_config.node_id_ == leader_node_id) + { + leader_config = &node_config; + break; + } + } + if (leader_config == nullptr) + { + LOG(WARNING) << "OnLeaderChanged: node " << leader_node_id + << " is not a member of node group " << ng_id; + return; + } + const std::string &host_name = leader_config->host_name_; + const uint16_t port = TxPort2DssPort(leader_config->port_); + + // The shard's version is the leader term it is hosted under: every node + // derives the same number for the same leadership, so a redirect's + // version compares with this node's. Calls arrive in any order; one whose + // term is below the shard's version is out-dated and moves nothing. + if (term < 0) + { + LOG(WARNING) << "OnLeaderChanged: no term for node " << leader_node_id + << " leading node group " << ng_id << "; shard kept"; + return; + } + const uint64_t version = static_cast(term); + const uint64_t current_version = + dss_nodes_[dss_shards_[shard_id].load(std::memory_order_acquire)] + .ShardVersion(); + if (version < current_version) + { + LOG(INFO) << "Data shard " << shard_id << " stays: node " + << leader_node_id << " leads node group " << ng_id + << " at term " << term << ", below the shard's version " + << current_version; + return; + } + if (data_store_service_ != nullptr) + { + // Keep this node's data store service's view, which answers + // misrouted requests, in line. + data_store_service_->GetClusterManager().UpdatePrimaryNodeIfNotOlder( + shard_id, DSSNode(host_name, port), version); + } + for (int attempt = 0; attempt < 100; ++attempt) + { + // Checks the version again against the slot it replaces. + if (UpgradeShardVersion(shard_id, version, host_name, port, true)) + { + LOG(INFO) << "Data shard " << shard_id + << " now at its node group's " + << "leader, node " << leader_node_id << " (" << host_name + << ":" << port << "), shard version " << version; + return; + } + // Another thread is moving the shard; try again. + bthread_usleep(1000); + } + LOG(ERROR) << "Could not move data shard " << shard_id << " to node " + << leader_node_id << " (" << host_name << ":" << port << ")"; +} + /** * @brief Handles shutdown event. * @@ -4486,6 +4592,23 @@ bool DataStoreServiceClient::UpdateOwnerNodeIndexOfShard( } } +bool DataStoreServiceClient::CanRetryShardNow(uint32_t shard_id) +{ + if (IsLocalShard(shard_id)) + { + return true; + } + const uint32_t node_index = GetOwnerNodeIndexOfShard(shard_id); + if (node_index >= dss_nodes_.size()) + { + return false; + } + // brpc::Channel overrides CheckHealth() as protected; ChannelBase + // declares it public. + brpc::ChannelBase *channel = dss_nodes_[node_index].Channel(); + return channel->CheckHealth() == 0; +} + uint32_t DataStoreServiceClient::FindFreeNodeIndex() { uint64_t current_ts = @@ -4545,7 +4668,8 @@ void DataStoreServiceClient::HandleShardingError( bool DataStoreServiceClient::UpgradeShardVersion(uint32_t shard_id, uint64_t shard_version, const std::string &host_name, - uint16_t port) + uint16_t port, + bool move_if_owner_differs) { if (shard_id >= dss_shards_.size()) { @@ -4555,7 +4679,14 @@ bool DataStoreServiceClient::UpgradeShardVersion(uint32_t shard_id, uint32_t node_index = dss_shards_[shard_id].load(std::memory_order_acquire); auto &node_ref = dss_nodes_[node_index]; - if (node_ref.ShardVersion() < shard_version) + const uint64_t held_version = node_ref.ShardVersion(); + // Never lowered: a move at an older version changes nothing. Claiming the + // held slot below fails if another mover replaced it after this check, + // and the caller checks again. + const bool owner_differs = + move_if_owner_differs && shard_version == held_version && + (node_ref.HostName() != host_name || node_ref.Port() != port); + if (held_version < shard_version || owner_differs) { uint64_t expect_val = 0; uint64_t current_ts = diff --git a/store_handler/data_store_service_client.h b/store_handler/data_store_service_client.h index 73c9966c6..f79ab643c 100644 --- a/store_handler/data_store_service_client.h +++ b/store_handler/data_store_service_client.h @@ -371,7 +371,7 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler uint64_t range_version) override; bool UpsertRanges(const txservice::TableName &table_name, - std::vector range_info, + const std::vector &range_info, uint64_t version) override; std::string EncodeRangeKey(const txservice::CatalogFactory *catalog_factory, @@ -428,6 +428,18 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr) override; + /** + * @brief Creates a table's kv table. The data store service keeps no + * per-table state to create, so this writes the initial range record of a + * range-partitioned table, as UpsertTable does when it creates a table. + * + * @param table_name The table to create. + * @param version The table's schema version, the range's version. + * @return true on success. + */ + bool CreateKvTable(const txservice::TableName &table_name, + uint64_t version) override; + bool DropKvTable(const std::string &kv_table_name) override; void DropKvTableAsync(const std::string &kv_table_name) override; @@ -512,6 +524,33 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler int64_t standby_term, bool resubscribe) override; + /** + * @brief Records that the node group's data shard is hosted by the given + * node, the group's leader at a newer term (with data shards bound to the + * node groups), also when that node hosted it already: the shard's + * version follows the term. + * + * Points the shard at that node's data store service in this client's + * shard map and in this process's cluster config, with the leader term as + * the shard's version: every node derives the same version for the same + * leadership, so a redirect's version compares with this node's; a + * version computed from this node's own view would not. The client's + * later requests for the shard (the metadata reads every node makes, + * flushes to every shard) then go there; without it, a node that does not + * follow the group kept sending them to the old leader, which, when down, + * never answers that it no longer owns the shard. + * + * Calls may arrive in any order: one whose term is below the shard's + * version is out-dated and changes nothing, so the newest leader stays. + * + * @param ng_id The node group, and so the data shard. + * @param leader_node_id The node now leading the group. + * @param term Its leader term, the shard's version. + */ + void OnLeaderChanged(uint32_t ng_id, + uint32_t leader_node_id, + int64_t term) override; + void OnShutdown() override; /** @@ -900,14 +939,56 @@ class DataStoreServiceClient : public txservice::store::DataStoreHandler uint32_t old_node_index, uint32_t &new_node_index); + /** + * @brief Whether a request whose RPC to the shard just failed is worth + * sending again at once. + * + * It is when the shard is now local, or when the channel to the shard's + * current owner is usable (brpc::Channel::CheckHealth): the owner has + * moved to another node, or the channel UpdateOwnerNodeIndexOfShard + * rebuilt reaches a different address, such as a restarted node's new IP + * under the same host name. It is not while brpc holds the failed + * connection down until its health check reconnects (up to + * health_check_interval seconds): an attempt then fails at once without + * touching the network, so the caller gets the network error now and + * retries on its own schedule. + * + * @param shard_id The shard. + * @return true if an immediate retry can reach the shard. + */ + bool CanRetryShardNow(uint32_t shard_id); + void UpdateShardOwner(uint32_t shard_id, uint32_t node_id); uint32_t FindFreeNodeIndex(); void HandleShardingError(const ::EloqDS::remote::CommonResult &result); + /** + * @brief Points the client's requests for a shard at the node hosting it + * when the given shard version is newer than the one held, or, with + * move_if_owner_differs, when it is the same and the node differs. + * + * The shard's node slot is replaced by a fresh one with a channel to the + * new host and the given version. A version below the held one never + * moves the shard; one at it, with the same host, changes nothing. The + * check reads the held slot inside the move, which another mover cannot + * replace meanwhile, so concurrent moves leave the newest version. + * + * @param shard_id The shard. + * @param shard_version The version to install. + * @param host_name The host of the shard's data store service. + * @param port Its port. + * @param move_if_owner_differs Also move at the held version when only + * the host differs: for a leader learned from the tx service, which + * names the host of its term. + * @return false if another thread is moving the shard or no node slot is + * free, so that the caller tries again; true otherwise, also when nothing + * needed moving. + */ bool UpgradeShardVersion(uint32_t shard_id, uint64_t shard_version, const std::string &host_name, - uint16_t port); + uint16_t port, + bool move_if_owner_differs = false); txservice::EloqHashCatalogFactory hash_catalog_factory_{}; txservice::EloqRangeCatalogFactory range_catalog_factory_{}; diff --git a/store_handler/data_store_service_client_closure.cpp b/store_handler/data_store_service_client_closure.cpp index d07cf6017..78624dcf4 100644 --- a/store_handler/data_store_service_client_closure.cpp +++ b/store_handler/data_store_service_client_closure.cpp @@ -1117,10 +1117,12 @@ void FetchRangeSlicesCallback(void *data, } else if (result.error_code() != remote::DataStoreError::NO_ERROR) { - assert(false); - // Only read partial range slices info. Caller should retry. - LOG(ERROR) << "Fetch range slices failed: Partial result, keep " - "retring"; + // The read failed, e.g. the shard's node is restarting and brpc + // holds it down until its health check reconnects. The waiting + // requests fail with DATA_STORE_ERR and their callers retry. + LOG(ERROR) << "Fetch range slices failed, error: " + << static_cast(result.error_code()) << " " + << result.error_msg() << "; the requests retry"; fetch_req->SetFinish(txservice::CcErrorCode::DATA_STORE_ERR); txservice::Sharder::Instance().UnpinNodeGroupData(ng_id); return; diff --git a/store_handler/data_store_service_client_closure.h b/store_handler/data_store_service_client_closure.h index f36572bde..6257a3c66 100644 --- a/store_handler/data_store_service_client_closure.h +++ b/store_handler/data_store_service_client_closure.h @@ -999,8 +999,11 @@ class ReadClosure : public ::google::protobuf::Closure, public Poolable ds_service_client_->UpdateOwnerNodeIndexOfShard( shard_id_, remote_node_index_, new_node_index); - // Retry - if (retry_count_ < ds_service_client_->retry_limit_) + // Retry at once only if that can reach the shard + // (CanRetryShardNow); otherwise the caller gets the + // network error and retries on its own schedule. + if (retry_count_ < ds_service_client_->retry_limit_ && + ds_service_client_->CanRetryShardNow(shard_id_)) { self_guard.Release(); retry_count_++; @@ -1328,8 +1331,11 @@ class FlushDataClosure : public ::google::protobuf::Closure, public Poolable ds_service_client_->UpdateOwnerNodeIndexOfShard( shard_ids_.back(), remote_node_index_, new_node_index); - // Retry - if (retry_count_ < ds_service_client_->retry_limit_) + // Retry at once only if that can reach the shard + // (CanRetryShardNow); otherwise the caller gets the + // network error and retries on its own schedule. + if (retry_count_ < ds_service_client_->retry_limit_ && + ds_service_client_->CanRetryShardNow(shard_ids_.back())) { self_guard.Release(); retry_count_++; @@ -1557,8 +1563,11 @@ class DeleteRangeClosure : public ::google::protobuf::Closure, public Poolable ds_service_client_->UpdateOwnerNodeIndexOfShard( shard_id_, remote_node_index_, new_node_index); - // Retry - if (retry_count_ < ds_service_client_->retry_limit_) + // Retry at once only if that can reach the shard + // (CanRetryShardNow); otherwise the caller gets the + // network error and retries on its own schedule. + if (retry_count_ < ds_service_client_->retry_limit_ && + ds_service_client_->CanRetryShardNow(shard_id_)) { self_guard.Release(); retry_count_++; @@ -1783,8 +1792,11 @@ class DropTableClosure : public ::google::protobuf::Closure, public Poolable ds_service_client_->UpdateOwnerNodeIndexOfShard( shard_ids_.back(), remote_node_index_, new_node_index); - // Retry - if (retry_count_ < ds_service_client_->retry_limit_) + // Retry at once only if that can reach the shard + // (CanRetryShardNow); otherwise the caller gets the + // network error and retries on its own schedule. + if (retry_count_ < ds_service_client_->retry_limit_ && + ds_service_client_->CanRetryShardNow(shard_ids_.back())) { self_guard.Release(); retry_count_++; @@ -2048,14 +2060,18 @@ class BatchWriteRecordsClosure : public ::google::protobuf::Closure, ds_service_client_->UpdateOwnerNodeIndexOfShard( shard_id_, remote_node_index_, new_node_index); - need_retry = true; - } - else - { - result_.set_error_code( - EloqDS::remote::DataStoreError::NETWORK_ERROR); - result_.set_error_msg(cntl_.ErrorText()); + // Retry at once only if that can reach the shard + // (CanRetryShardNow); otherwise the caller gets the + // network error and retries on its own schedule. + need_retry = + ds_service_client_->CanRetryShardNow(shard_id_); } + // The error stands unless a retry replaces it: without it, a + // write that is not retried, or runs out of retries, would + // report the NO_ERROR that Reset() left in result_. + result_.set_error_code( + EloqDS::remote::DataStoreError::NETWORK_ERROR); + result_.set_error_msg(cntl_.ErrorText()); } else { @@ -2392,8 +2408,11 @@ class ScanNextClosure : public ::google::protobuf::Closure, public Poolable ds_service_client_->UpdateOwnerNodeIndexOfShard( shard_id_, remote_node_index_, new_node_index); - // Retry - if (retry_count_ < ds_service_client_->retry_limit_) + // Retry at once only if that can reach the shard + // (CanRetryShardNow); otherwise the caller gets the + // network error and retries on its own schedule. + if (retry_count_ < ds_service_client_->retry_limit_ && + ds_service_client_->CanRetryShardNow(shard_id_)) { self_guard.Release(); retry_count_++; @@ -2739,8 +2758,11 @@ class CreateSnapshotForBackupClosure : public ::google::protobuf::Closure, ds_service_client_->UpdateOwnerNodeIndexOfShard( shard_id, remote_node_index_, new_node_index); - // Retry - if (retry_count_ < ds_service_client_->retry_limit_) + // Retry at once only if that can reach the shard + // (CanRetryShardNow); otherwise the caller gets the + // network error and retries on its own schedule. + if (retry_count_ < ds_service_client_->retry_limit_ && + ds_service_client_->CanRetryShardNow(shard_id)) { self_guard.Release(); retry_count_++; diff --git a/store_handler/eloq_data_store_service/build_eloq_store.cmake b/store_handler/eloq_data_store_service/build_eloq_store.cmake index 782048ab3..54b0fe9c7 100644 --- a/store_handler/eloq_data_store_service/build_eloq_store.cmake +++ b/store_handler/eloq_data_store_service/build_eloq_store.cmake @@ -118,9 +118,11 @@ set(ELOQ_STORE_SOURCES ${ELOQ_STORE_SOURCE_DIR}/src/storage/shard.cpp ${ELOQ_STORE_SOURCE_DIR}/src/tasks/write_buffer_aggregator.cpp ${ELOQ_STORE_SOURCE_DIR}/src/tasks/archive_crond.cpp + ${ELOQ_STORE_SOURCE_DIR}/src/tasks/attach_shard_task.cpp ${ELOQ_STORE_SOURCE_DIR}/src/tasks/reopen_task.cpp ${ELOQ_STORE_SOURCE_DIR}/src/tasks/background_write.cpp ${ELOQ_STORE_SOURCE_DIR}/src/tasks/batch_write_task.cpp + ${ELOQ_STORE_SOURCE_DIR}/src/tasks/detach_shard_task.cpp ${ELOQ_STORE_SOURCE_DIR}/src/tasks/prewarm_task.cpp ${ELOQ_STORE_SOURCE_DIR}/src/tasks/read_task.cpp ${ELOQ_STORE_SOURCE_DIR}/src/tasks/scan_task.cpp diff --git a/store_handler/eloq_data_store_service/data_store.h b/store_handler/eloq_data_store_service/data_store.h index b60416a1e..79287c309 100644 --- a/store_handler/eloq_data_store_service/data_store.h +++ b/store_handler/eloq_data_store_service/data_store.h @@ -26,6 +26,7 @@ #include #include #include +#include #include namespace EloqDS { @@ -159,6 +160,26 @@ class DataStore (void) tag; } + /** + * @brief Sets the buckets of the data shard this store serves. + * + * DataStoreService::ConnectAndStartDataStore calls it before each start + * of the store, with the buckets of the node group that owns the data + * shard. A store whose storage also holds other data shards' partitions + * (EloqStore keeps every data shard's partitions under one path) limits + * the operations that sweep its storage to these buckets. A store that + * has a database of its own per data shard (RocksDB, RocksDB-Cloud) + * ignores it. + * + * @param bucket_ids The data shard's buckets; empty means every partition + * belongs to the data shard. + */ + virtual void SetPartitionBuckets( + const std::unordered_set &bucket_ids) + { + (void) bucket_ids; + } + /** * @brief Switch the data store to read only mode. */ diff --git a/store_handler/eloq_data_store_service/data_store_factory.h b/store_handler/eloq_data_store_service/data_store_factory.h index 9d875a59e..c9f07114f 100644 --- a/store_handler/eloq_data_store_service/data_store_factory.h +++ b/store_handler/eloq_data_store_service/data_store_factory.h @@ -23,7 +23,6 @@ #include #include -#include #include "data_store.h" @@ -111,16 +110,6 @@ class DataStoreFactory return false; } - /** - * @brief Set the partition filter function for eloqstore - * @param ng_id The node group id - * @param bucket_ids The bucket ids - */ - virtual void InitializePartitionFilter( - uint32_t ng_id, std::unordered_set &&bucket_ids) - { - } - /** * @brief Check if the data store is in cloud mode. * @return true if the data store is in cloud mode, false otherwise diff --git a/store_handler/eloq_data_store_service/data_store_fault_inject.h b/store_handler/eloq_data_store_service/data_store_fault_inject.h index 9818d2c6f..2ffa7ef54 100644 --- a/store_handler/eloq_data_store_service/data_store_fault_inject.h +++ b/store_handler/eloq_data_store_service/data_store_fault_inject.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -220,8 +221,40 @@ class FaultInject } private: + /** + * @brief Arms the faults the environment variable ELOQ_DSS_FAULT_INJECT + * names, in fault-injection builds: comma-separated entries, each "name" + * or "name:params" (params as InjectFault takes them). For tests that + * need a fault from the process's start, before its RPC serves; the RPC + * removes it as any other. + */ FaultInject() { +#ifdef WITH_FAULT_INJECT + const char *faults = std::getenv("ELOQ_DSS_FAULT_INJECT"); + std::string spec = faults == nullptr ? "" : faults; + size_t begin = 0; + while (begin < spec.size()) + { + size_t end = spec.find(',', begin); + if (end == std::string::npos) + { + end = spec.size(); + } + std::string item = spec.substr(begin, end - begin); + size_t colon = item.find(':'); + std::string name = item.substr(0, colon); + std::string paras = + colon == std::string::npos ? "" : item.substr(colon + 1); + if (!name.empty()) + { + LOG(INFO) << "FaultInject from ELOQ_DSS_FAULT_INJECT, name=" + << name << " paras=" << paras; + injected_fault_map_.try_emplace(name, FaultEntry(name, paras)); + } + begin = end + 1; + } +#endif } ~FaultInject() diff --git a/store_handler/eloq_data_store_service/data_store_service.cpp b/store_handler/eloq_data_store_service/data_store_service.cpp index 7f643dbcd..1b04336d6 100644 --- a/store_handler/eloq_data_store_service/data_store_service.cpp +++ b/store_handler/eloq_data_store_service/data_store_service.cpp @@ -276,6 +276,14 @@ DataStoreService::~DataStoreService() { it.ShutDown(); } + // The server and workers are gone, so no request reads a data store now. + // Destroy them before the members are destroyed in reverse order, since + // the factory, declared after the shards, would go first, and the + // EloqStore-backed data stores withdraw from it as they are destroyed. + for (auto &it : data_shards_) + { + it.data_store_.reset(); + } } DataStoreService::ReadSubmitGuard::ReadSubmitGuard(DataStoreService *service, @@ -373,15 +381,6 @@ bool DataStoreService::StartService(bool create_db_if_missing) auto dss_shards = cluster_manager_.GetShardsForThisNode(); LOG(INFO) << "DataStoreService start with shards: " << dss_shards.size(); -#ifdef DATA_STORE_TYPE_ELOQDSS_ELOQSTORE - if (dss_shards.size() > 1) - { - LOG(ERROR) - << "EloqStore-backed DataStoreService supports at most one shard."; - return false; - } -#endif - if (!dss_shards.empty()) { for (uint32_t shard_id : dss_shards) @@ -415,6 +414,7 @@ bool DataStoreService::StartService(bool create_db_if_missing) } ds_ref.shard_status_.store(open_mode, std::memory_order_release); + RunStartingWaiters(shard_id); LOG(INFO) << "Created data store on starting " "DataStoreService, shard id: " << shard_id; @@ -464,10 +464,12 @@ bool DataStoreService::StartService(bool create_db_if_missing) return true; } -bool DataStoreService::ConnectAndStartDataStore(uint32_t data_shard_id, - DSShardStatus open_mode, - bool create_db_if_missing, - int64_t term) +bool DataStoreService::ConnectAndStartDataStore( + uint32_t data_shard_id, + DSShardStatus open_mode, + bool create_db_if_missing, + int64_t term, + const std::unordered_set *partition_buckets) { if (open_mode == DSShardStatus::Closed) { @@ -505,35 +507,62 @@ bool DataStoreService::ConnectAndStartDataStore(uint32_t data_shard_id, << ", data_store_ is null:" << (shard_ref.data_store_ == nullptr); assert(data_store_factory_ != nullptr); - if (shard_ref.data_store_ == nullptr) + const bool created = shard_ref.data_store_ == nullptr; + if (created) { + // Created without starting it, so that the store has its buckets + // before it starts. shard_ref.data_store_ = data_store_factory_->CreateDataStore( - create_db_if_missing, data_shard_id, this, true, term); + create_db_if_missing, data_shard_id, this, false, term); if (shard_ref.data_store_ == nullptr) { LOG(ERROR) << "Failed to create data store"; + // Not Starting forever: requests waiting meanwhile get the + // sharding error, and a later open can try again. + shard_ref.shard_status_.store(DSShardStatus::Closed, + std::memory_order_release); + RunStartingWaiters(data_shard_id); return false; } } - else + if (partition_buckets != nullptr) + { + // The store is stopped: safe to change what its filter reads. + shard_ref.data_store_->SetPartitionBuckets(*partition_buckets); + } + if (!created) { + // CreateDataStore initializes a new store; a kept one is initialized + // again after its close. bool res = shard_ref.data_store_->Initialize(); if (!res) { LOG(ERROR) << "Failed to initialize data store"; + shard_ref.shard_status_.store(DSShardStatus::Closed, + std::memory_order_release); + RunStartingWaiters(data_shard_id); return false; } + } #ifdef DATA_STORE_TYPE_ELOQDSS_ELOQSTORE - res = shard_ref.data_store_->StartDB(term, data_shard_id); + bool started = shard_ref.data_store_->StartDB(term, data_shard_id); #else - res = shard_ref.data_store_->StartDB(term); + bool started = shard_ref.data_store_->StartDB(term); #endif - if (!res) + if (!started) + { + LOG(ERROR) << "Failed to start db instance in data store service"; + if (created) { - LOG(ERROR) << "Failed to start db instance in data store service"; - return false; + // As when CreateDataStore starts the store itself and fails: the + // next open creates a new one. + shard_ref.data_store_ = nullptr; } + shard_ref.shard_status_.store(DSShardStatus::Closed, + std::memory_order_release); + RunStartingWaiters(data_shard_id); + return false; } if (shard_ref.scan_iter_cache_ == nullptr) @@ -554,12 +583,65 @@ bool DataStoreService::ConnectAndStartDataStore(uint32_t data_shard_id, DSShardStatus::Closed); } + // Test only: keeps the shard Starting until the test removes the fault, + // so that requests meet the shard while it opens. + CODE_FAULT_INJECTOR("dss_hold_starting", { + LOG(INFO) << "FaultInject dss_hold_starting: holding DSS shard " + << data_shard_id << " in Starting"; + while (FaultInject::Entry("dss_hold_starting") != nullptr) + { + bthread_usleep(100000); + } + LOG(INFO) << "FaultInject dss_hold_starting: released DSS shard " + << data_shard_id; + }); + expect_status = DSShardStatus::Starting; shard_ref.shard_status_.compare_exchange_strong( expect_status, open_mode, std::memory_order_release); + RunStartingWaiters(data_shard_id); return true; } +void DataStoreService::WaitWhileStarting(uint32_t shard_id, + std::function retry) +{ + DataShard &ds_ref = data_shards_.at(shard_id); + { + std::lock_guard lk(ds_ref.starting_waiters_mux_); + if (ds_ref.shard_status_.load(std::memory_order_seq_cst) == + DSShardStatus::Starting) + { + ds_ref.starting_waiters_.push_back(std::move(retry)); + DLOG(INFO) << "A request waits on DSS shard " << shard_id + << " until it leaves Starting"; + return; + } + } + // The shard left Starting since the handler looked. + retry(); +} + +void DataStoreService::RunStartingWaiters(uint32_t shard_id) +{ + DataShard &ds_ref = data_shards_.at(shard_id); + std::vector> waiters; + { + std::lock_guard lk(ds_ref.starting_waiters_mux_); + waiters.swap(ds_ref.starting_waiters_); + } + if (waiters.empty()) + { + return; + } + LOG(INFO) << "DSS shard " << shard_id << " left Starting; running again " + << waiters.size() << " requests that waited"; + for (const std::function &retry : waiters) + { + retry(); + } +} + void DataStoreService::Read(::google::protobuf::RpcController *controller, const ::EloqDS::remote::ReadRequest *request, ::EloqDS::remote::ReadResponse *response, @@ -581,6 +663,13 @@ void DataStoreService::Read(::google::protobuf::RpcController *controller, if (shard_status != DSShardStatus::ReadOnly && shard_status != DSShardStatus::ReadWrite) { + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting(shard_id, + [this, controller, request, response, done]() + { Read(controller, request, response, done); }); + return; + } brpc::ClosureGuard done_guard(done); auto *result = response->mutable_result(); result->set_error_code(::EloqDS::remote::DataStoreError::DB_NOT_OPEN); @@ -620,6 +709,34 @@ void DataStoreService::Read(const std::string_view table_name, if (shard_status != DSShardStatus::ReadOnly && shard_status != DSShardStatus::ReadWrite) { + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting(shard_id, + [this, + table_name, + partition_id, + shard_id, + key, + reopen, + record, + ts, + ttl, + result, + done]() + { + Read(table_name, + partition_id, + shard_id, + key, + reopen, + record, + ts, + ttl, + result, + done); + }); + return; + } brpc::ClosureGuard done_guard(done); record->clear(); *ts = 0; @@ -666,6 +783,14 @@ void DataStoreService::FlushData( if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting( + shard_id, + [this, controller, request, response, done]() + { FlushData(controller, request, response, done); }); + return; + } brpc::ClosureGuard done_guard(done); ::EloqDS::remote::CommonResult *result = response->mutable_result(); if (shard_status == DSShardStatus::Closed) @@ -820,6 +945,14 @@ void DataStoreService::FlushData(const std::vector &kv_table_names, if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting( + shard_id, + [this, &kv_table_names, shard_id, &result, done]() + { FlushData(kv_table_names, shard_id, result, done); }); + return; + } brpc::ClosureGuard done_guard(done); if (shard_status == DSShardStatus::Closed) { @@ -866,6 +999,14 @@ void DataStoreService::DeleteRange( if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting( + shard_id, + [this, controller, request, response, done]() + { DeleteRange(controller, request, response, done); }); + return; + } // This object helps to call done->Run() in RAII style. If you need to // process the request asynchronously, pass done_guard.release(). brpc::ClosureGuard done_guard(done); @@ -916,6 +1057,30 @@ void DataStoreService::DeleteRange(const std::string_view table_name, if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting(shard_id, + [this, + table_name, + partition_id, + shard_id, + start_key, + end_key, + skip_wal, + &result, + done]() + { + DeleteRange(table_name, + partition_id, + shard_id, + start_key, + end_key, + skip_wal, + result, + done); + }); + return; + } brpc::ClosureGuard done_guard(done); if (shard_status == DSShardStatus::Closed) { @@ -970,6 +1135,14 @@ void DataStoreService::CreateTable( if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting( + shard_id, + [this, controller, request, response, done]() + { CreateTable(controller, request, response, done); }); + return; + } brpc::ClosureGuard done_guard(done); ::EloqDS::remote::CommonResult *result = response->mutable_result(); if (shard_status == DSShardStatus::Closed) @@ -1014,6 +1187,14 @@ void DataStoreService::CreateTable(const std::string_view table_name, if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting( + shard_id, + [this, table_name, shard_id, &result, done]() + { CreateTable(table_name, shard_id, result, done); }); + return; + } brpc::ClosureGuard done_guard(done); if (shard_status == DSShardStatus::Closed) { @@ -1060,6 +1241,14 @@ void DataStoreService::DropTable( if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting( + shard_id, + [this, controller, request, response, done]() + { DropTable(controller, request, response, done); }); + return; + } brpc::ClosureGuard done_guard(done); ::EloqDS::remote::CommonResult *result = response->mutable_result(); if (shard_status == DSShardStatus::Closed) @@ -1104,6 +1293,14 @@ void DataStoreService::DropTable(const std::string_view table_name, if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting(shard_id, + [this, table_name, shard_id, &result, done]() { + DropTable(table_name, shard_id, result, done); + }); + return; + } brpc::ClosureGuard done_guard(done); if (shard_status == DSShardStatus::Closed) { @@ -1151,6 +1348,14 @@ void DataStoreService::BatchWriteRecords( if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting( + shard_id, + [this, controller, request, response, done]() + { BatchWriteRecords(controller, request, response, done); }); + return; + } brpc::ClosureGuard done_guard(done); auto *result = response->mutable_result(); if (shard_status == DSShardStatus::Closed) @@ -1208,6 +1413,44 @@ void DataStoreService::ScanNext( if (shard_status != DSShardStatus::ReadWrite && shard_status != DSShardStatus::ReadOnly) { + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting(shard_id, + [this, + table_name, + partition_id, + shard_id, + start_key, + end_key, + inclusive_start, + inclusive_end, + scan_forward, + batch_size, + search_conditions, + items, + session_id, + generate_session_id, + result, + done]() + { + ScanNext(table_name, + partition_id, + shard_id, + start_key, + end_key, + inclusive_start, + inclusive_end, + scan_forward, + batch_size, + search_conditions, + items, + session_id, + generate_session_id, + result, + done); + }); + return; + } brpc::ClosureGuard done_guard(done); result->set_error_code(::EloqDS::remote::DataStoreError::DB_NOT_OPEN); result->set_error_msg("KV store not opened yet."); @@ -1257,6 +1500,14 @@ void DataStoreService::ScanNext(::google::protobuf::RpcController *controller, if (shard_status != DSShardStatus::ReadWrite && shard_status != DSShardStatus::ReadOnly) { + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting(shard_id, + [this, controller, request, response, done]() { + ScanNext(controller, request, response, done); + }); + return; + } brpc::ClosureGuard done_guard(done); auto *result = response->mutable_result(); result->set_error_code(::EloqDS::remote::DataStoreError::DB_NOT_OPEN); @@ -1292,6 +1543,14 @@ void DataStoreService::ScanClose(::google::protobuf::RpcController *controller, if (shard_status != DSShardStatus::ReadWrite && shard_status != DSShardStatus::ReadOnly) { + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting( + shard_id, + [this, controller, request, response, done]() + { ScanClose(controller, request, response, done); }); + return; + } assert(false); brpc::ClosureGuard done_guard(done); auto *result = response->mutable_result(); @@ -1328,6 +1587,25 @@ void DataStoreService::ScanClose(const std::string_view table_name, if (shard_status != DSShardStatus::ReadWrite && shard_status != DSShardStatus::ReadOnly) { + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting(shard_id, + [this, + table_name, + partition_id, + shard_id, + session_id, + result, + done]() { + ScanClose(table_name, + partition_id, + shard_id, + session_id, + result, + done); + }); + return; + } assert(false); brpc::ClosureGuard done_guard(done); result->set_error_code(::EloqDS::remote::DataStoreError::DB_NOT_OPEN); @@ -1449,6 +1727,40 @@ void DataStoreService::BatchWriteRecords( if (shard_status != DSShardStatus::ReadWrite) { DecreaseWriteReqCount(shard_id); + if (shard_status == DSShardStatus::Starting) + { + WaitWhileStarting(shard_id, + [this, + table_name, + partition_id, + shard_id, + &key_parts, + &record_parts, + &ts, + &ttl, + &op_types, + skip_wal, + &result, + done, + parts_cnt_per_key, + parts_cnt_per_record]() + { + BatchWriteRecords(table_name, + partition_id, + shard_id, + key_parts, + record_parts, + ts, + ttl, + op_types, + skip_wal, + result, + done, + parts_cnt_per_key, + parts_cnt_per_record); + }); + return; + } brpc::ClosureGuard done_guard(done); if (shard_status == DSShardStatus::Closed) { @@ -2438,12 +2750,6 @@ void DataStoreService::OpenDataStore(uint32_t shard_id, DLOG(INFO) << "OpenDataStore begin for DSS shard " << shard_id << ", term " << term << ", bucket_count " << bucket_count << ", shard_status " << static_cast(ds_ref.shard_status_.load()); - if (data_store_factory_ != nullptr) - { - data_store_factory_->InitializePartitionFilter(shard_id, - std::move(bucket_ids)); - } - auto start_time = std::chrono::steady_clock::now(); if (ds_ref.shard_status_.load() != DSShardStatus::Closed) { @@ -2462,8 +2768,10 @@ void DataStoreService::OpenDataStore(uint32_t shard_id, DSShardStatus open_mode = DSShardStatus::ReadWrite; bool create_db_if_missing = false; + // The node group's buckets, which the shard's store limits its sweeps to + // (DataStore::SetPartitionBuckets). auto res = ConnectAndStartDataStore( - shard_id, open_mode, create_db_if_missing, term); + shard_id, open_mode, create_db_if_missing, term, &bucket_ids); auto end_time = std::chrono::steady_clock::now(); auto use_time = std::chrono::duration_cast( end_time - start_time) @@ -3254,6 +3562,7 @@ bool DataStoreService::SwitchReadOnlyToClosed(uint32_t shard_id) std::memory_order_seq_cst); CHECK(switched); cluster_manager_.SwitchShardToClosed(shard_id, DSShardStatus::ReadOnly); + RunStartingWaiters(shard_id); } return true; } diff --git a/store_handler/eloq_data_store_service/data_store_service.h b/store_handler/eloq_data_store_service/data_store_service.h index 9ba9cbac0..cf8dd2e7c 100644 --- a/store_handler/eloq_data_store_service/data_store_service.h +++ b/store_handler/eloq_data_store_service/data_store_service.h @@ -651,6 +651,22 @@ class DataStoreService : EloqDS::remote::DataStoreRpcService return data_store_factory_.get(); } + /** + * @brief Counts a write about to be submitted to the shard's store. + * + * One half of a store-then-load handshake with SwitchReadWriteToReadOnly: + * the writer increments ongoing_write_requests_, then loads + * shard_status_; the switcher stores ReadOnly, then waits for the count to + * reach zero. Each side must see the other's store, or a write reaches a + * store already switched to read-only. + * + * @param shard_id The shard. + * @warning The handler's shard_status_ load after this call must be + * seq_cst, as must the counter updates and the switcher's accesses: C++ + * orders a store before a later load of another atomic only among + * seq_cst operations, and an acquire load may still return the old + * status. + */ void IncreaseWriteReqCount(uint32_t shard_id) { data_shards_.at(shard_id).ongoing_write_requests_.fetch_add( @@ -683,6 +699,21 @@ class DataStoreService : EloqDS::remote::DataStoreRpcService void ClearStandbySnapshotPayloadForEloqStore(uint32_t shard_id); void CloseDataStore(uint32_t shard_id); + + /** + * @brief Opens a data shard read-write for the node group bound to it: + * the co-located mode's entry point, where data shard g opens on node + * group g's leader. + * + * Called in-process by the data store client (DataStoreServiceClient:: + * OnLeaderStart, and RequestSyncSnapshot on a standby). Does nothing if + * the shard is not Closed. + * + * @param shard_id The data shard, equal to the node group's id. + * @param bucket_ids The node group's buckets, which the shard's store + * limits its sweeps of shared storage to (DataStore::SetPartitionBuckets). + * @param term The node group's leader term; 0 on a single node. + */ void OpenDataStore(uint32_t shard_id, std::unordered_set &&bucket_ids, int64_t term = 0); @@ -753,15 +784,94 @@ class DataStoreService : EloqDS::remote::DataStoreRpcService } } - bool ConnectAndStartDataStore(uint32_t data_shard_id, - DSShardStatus open_mode, - bool create_db_if_missing = false, - int64_t term = 0); + /** + * @brief Opens one data shard's data store: moves the shard from Closed + * through Starting to open_mode, creating the shard's DataStore or + * starting the one it kept from an earlier close. + * + * Requests that meet the shard while it is Starting wait and run again + * once it leaves that state (RunStartingWaiters), also when the open + * fails and the shard goes back to Closed. + * + * @param data_shard_id The data shard. + * @param open_mode ReadOnly or ReadWrite; Closed does nothing. + * @param create_db_if_missing Whether to create the database if absent. + * @param term The term the store starts at (EloqStore's cloud fencing). + * @param partition_buckets If set, the data shard's buckets, given to + * the store (DataStore::SetPartitionBuckets) before every start, so that + * a reopen picks up the current set; nullptr leaves the store's set as + * it is. + * @return true if the shard is open in open_mode. + */ + bool ConnectAndStartDataStore( + uint32_t data_shard_id, + DSShardStatus open_mode, + bool create_db_if_missing = false, + int64_t term = 0, + const std::unordered_set *partition_buckets = nullptr); + + /** + * @brief Puts a request that found its shard Starting on the shard's + * waiting list, until the shard leaves that state. + * + * A shard is Starting while it opens (its node becomes the node group's + * leader after a failover or a restart) and while it closes. A request + * arriving then waits rather than fail: once the shard is open it is + * served, and once it is closed it gets the sharding error and the client + * re-routes. No thread waits: the request sits on the list until the + * thread that moves the shard out of Starting runs it again + * (RunStartingWaiters), and its handler checks the status afresh. The + * handler calls this only after it has seen the shard Starting, so only a + * request that really waits builds retry. + * + * @param shard_id The shard. + * @param retry Calls the handler again with the arguments the request + * came with. An RPC request's controller, request and response stay valid + * until its closure runs: brpc's baidu_std server frees them in + * SendRpcResponse, which is the closure, and its HTTP server's closure + * owns them. A local request's arguments are the caller's, kept alive + * until its closure runs, as for any asynchronous request. + * @note If the shard has left Starting since the handler looked, retry + * runs here, at once. + */ + void WaitWhileStarting(uint32_t shard_id, std::function retry); + + /** + * @brief Runs again, on this thread, the requests that waited while the + * shard was Starting. + * + * Called right after every change of the shard's status out of Starting. + * A request put on the list before this call takes the list is run here; + * one that comes later finds the new status (it was stored before), so + * none is left behind. Each handler only submits its request to the data + * store, so this does not block. + * + * @param shard_id The shard. + */ + void RunStartingWaiters(uint32_t shard_id); bool SwitchReadWriteToReadOnly(uint32_t shard_id); bool SwitchReadOnlyToClosed(uint32_t shard_id); bool SwitchReadOnlyToReadWrite(uint32_t shard_id); uint32_t GetReadSubmitSlotIndex(); + + /** + * @brief Marks this thread as submitting a read to the shard's store, + * until the returned guard is destroyed. + * + * One half of a store-then-load handshake with SwitchReadOnlyToClosed: + * the reader stores its window depth, then loads shard_status_; the + * closer stores Starting, then loads every window depth, then shuts the + * store down. Each side must see the other's store, or a read reaches a + * store being shut down. + * + * @param shard_id The shard. + * @return The guard that leaves the window. + * @warning The handler's shard_status_ load after this call must be + * seq_cst, as must the depth store and the closer's accesses: C++ orders + * a store before a later load of another atomic only among seq_cst + * operations, and an acquire load may still return the old status. + */ ReadSubmitGuard EnterReadSubmitWindow(uint32_t shard_id); void LeaveReadSubmitWindow(uint32_t shard_id, uint32_t slot_idx); void WaitReadSubmitWindowsDrained(const DataShard &ds_ref) const; @@ -826,6 +936,16 @@ class DataStoreService : EloqDS::remote::DataStoreRpcService // Whether the file cache sync is running. Used to avoid concurrent // local ssd file operations between db and file sync worker. std::atomic is_file_sync_running_{false}; + + /** + * @brief Guards starting_waiters_. Held only to add to the list or + * take it, never across a handler: a local request may come from a + * TxProcessor's stack. + */ + std::mutex starting_waiters_mux_; + /** @brief Requests waiting while the shard is Starting + * (WaitWhileStarting), each a call of its handler. */ + std::vector> starting_waiters_; }; std::array data_shards_; diff --git a/store_handler/eloq_data_store_service/data_store_service_config.cpp b/store_handler/eloq_data_store_service/data_store_service_config.cpp index f331fb874..3fbfbe3dc 100644 --- a/store_handler/eloq_data_store_service/data_store_service_config.cpp +++ b/store_handler/eloq_data_store_service/data_store_service_config.cpp @@ -560,6 +560,19 @@ void DataStoreServiceClusterManager::UpdatePrimaryNode(uint32_t shard_id, topology_.UpdatePrimaryNode(shard_id, node); } +bool DataStoreServiceClusterManager::UpdatePrimaryNodeIfNotOlder( + uint32_t shard_id, const DSSNode &node, uint64_t shard_version) +{ + std::unique_lock lk(mutex_); + if (shard_version < topology_.FetchDSShardVersion(shard_id)) + { + return false; + } + topology_.UpdatePrimaryNode(shard_id, node); + topology_.UpdateDSShardVersion(shard_id, shard_version); + return true; +} + std::shared_ptr DataStoreServiceClusterManager::GetDataStoreServiceChannelByPartitionId( uint32_t partition_id) diff --git a/store_handler/eloq_data_store_service/data_store_service_config.h b/store_handler/eloq_data_store_service/data_store_service_config.h index 99caaca9d..822852ee7 100644 --- a/store_handler/eloq_data_store_service/data_store_service_config.h +++ b/store_handler/eloq_data_store_service/data_store_service_config.h @@ -424,6 +424,20 @@ class DataStoreServiceClusterManager */ void UpdatePrimaryNode(uint32_t shard_id, const DSSNode &node); + /** + * @brief Records a shard's primary node and version together, unless the + * version is below the shard's current one: a primary learned for an + * older version is out-dated. + * + * @param shard_id The shard. + * @param node Its primary node. + * @param shard_version The version that primary hosts the shard under. + * @return true if recorded. + */ + bool UpdatePrimaryNodeIfNotOlder(uint32_t shard_id, + const DSSNode &node, + uint64_t shard_version); + /** * @brief Returns if this node is the owner of a specific node group. */ diff --git a/store_handler/eloq_data_store_service/eloq_store_data_store.cpp b/store_handler/eloq_data_store_service/eloq_store_data_store.cpp index 266cd37ca..df9e21365 100644 --- a/store_handler/eloq_data_store_service/eloq_store_data_store.cpp +++ b/store_handler/eloq_data_store_service/eloq_store_data_store.cpp @@ -30,6 +30,7 @@ #include #include +#include "butil/third_party/murmurhash3/murmurhash3.h" #include "common.h" #include "eloq_store_data_store_factory.h" #include "internal_request.h" @@ -94,35 +95,30 @@ inline void BuildValue(const WriteRecordsRequest &write_req, } } -EloqStoreDataStore::EloqStoreDataStore(uint32_t shard_id, - DataStoreService *data_store_service) - : DataStore(shard_id, data_store_service) +bool EloqStoreDataStoreFactory::EnsureStoreStarted() { - const EloqStoreDataStoreFactory *factory = - dynamic_cast( - data_store_service->GetDataStoreFactory()); - assert(factory != nullptr); - ::eloqstore::KvOptions opts = - factory->eloq_store_configs_.eloqstore_configs_; - branch_name_ = factory->eloq_store_configs_.branch_name_; + std::lock_guard lk(store_mux_); + if (store_ != nullptr) + { + return !store_->IsStopped(); + } + ::eloqstore::KvOptions opts = eloq_store_configs_.eloqstore_configs_; + // A sweep runs for one data shard; its data store knows the buckets. + opts.partition_filter = [this](::eloqstore::DsShardId ds_shard_id, + const ::eloqstore::TableIdent &table_ident) + { return OwnsPartition(ds_shard_id, table_ident); }; DLOG(INFO) << "Create EloqStore storage with workers: " << opts.num_threads << ", store path: " << opts.store_path.front() << ", open files limit: " << opts.fd_limit << ", cloud store path: " << opts.cloud_store_path << ", buffer pool size per shard: " << opts.buffer_pool_size; - eloq_store_service_ = std::make_unique<::eloqstore::EloqStore>(opts); -} - -bool EloqStoreDataStore::Initialize() -{ + auto store = std::make_unique<::eloqstore::EloqStore>(opts); #ifdef ELOQSTORE_WITH_TXSERVICE if (metrics::enable_eloqstore_metrics) { - // Initialize metrics for eloqstore eloq_metrics_app::MetricsRegistryImpl::MetricsRegistryResult metrics_registry_result = eloq_metrics_app::MetricsRegistryImpl::GetRegistry(); - if (metrics_registry_result.not_ok_ != nullptr) { LOG(WARNING) << "Failed to get metrics registry, skipping " @@ -131,19 +127,112 @@ bool EloqStoreDataStore::Initialize() } else if (metrics_registry_result.metrics_registry_ != nullptr) { - // Create common labels - // Note: node_ip and node_port may not be available at Initialize() - // time For now, we use empty labels. The shard_id label will be - // added by EloqStore::InitializeMetrics() + // The meters are per worker shard of the one store; node labels + // are not known yet, so the labels stay empty. metrics::CommonLabels common_labels{}; - - // Call eloqstore's metrics initialization - // This will initialize metrics for all shards with shard_id labels - eloq_store_service_->InitializeMetrics( + store->InitializeMetrics( metrics_registry_result.metrics_registry_.get(), common_labels); } } #endif + ::eloqstore::KvError res = + store->StartInstance(eloq_store_configs_.branch_name_); + if (res != ::eloqstore::KvError::NoError) + { + LOG(ERROR) << "EloqStore start failed with error code: " + << static_cast(res); + return false; + } + store_ = std::move(store); + return true; +} + +bool EloqStoreDataStoreFactory::OwnsPartition( + ::eloqstore::DsShardId ds_shard_id, + const ::eloqstore::TableIdent &table_ident) const +{ + if (ds_shard_id >= shard_stores_.size()) + { + return false; + } + const EloqStoreDataStore *data_store = + shard_stores_[ds_shard_id].load(std::memory_order_acquire); + return data_store != nullptr && data_store->OwnsPartition(table_ident); +} + +void EloqStoreDataStoreFactory::RegisterShardStore( + uint32_t shard_id, EloqStoreDataStore *data_store) +{ + if (shard_id < shard_stores_.size()) + { + shard_stores_[shard_id].store(data_store, std::memory_order_release); + } +} + +EloqStoreDataStore::EloqStoreDataStore(uint32_t shard_id, + DataStoreService *data_store_service) + : DataStore(shard_id, data_store_service) +{ + EloqStoreDataStoreFactory *factory = + dynamic_cast( + const_cast( + data_store_service->GetDataStoreFactory())); + assert(factory != nullptr); + store_ = factory->Store(); + CHECK(store_ != nullptr) << "EloqStoreDataStore made before the store"; + factory->RegisterShardStore(shard_id, this); +} + +EloqStoreDataStore::~EloqStoreDataStore() +{ + EloqStoreDataStoreFactory *factory = + dynamic_cast( + const_cast( + data_store_service_->GetDataStoreFactory())); + if (factory != nullptr) + { + factory->RegisterShardStore(shard_id_, nullptr); + } +} + +void EloqStoreDataStore::SetPartitionBuckets( + const std::unordered_set &bucket_ids) +{ + partition_buckets_ = bucket_ids; +} + +bool EloqStoreDataStore::OwnsPartition( + const ::eloqstore::TableIdent &table_ident) const +{ + if (partition_buckets_.empty()) + { + return true; + } + const std::string &table_name = table_ident.tbl_name_; + uint16_t bucket_id; + if (table_name.compare(0, 8, "eloqdoc_") == 0 || + table_name.compare(0, 8, "eloqsql_") == 0 || + table_name.compare(0, 7, "irange_") == 0) + { + // This table engine uses RangePartition: the partition is a range. + uint32_t hash_val; + butil::MurmurHash3_x86_32(&table_ident.partition_id_, + sizeof(table_ident.partition_id_), + 9001, + &hash_val); + bucket_id = hash_val % 1024; + } + else + { + // HashPartition: the hash partition id equals the bucket id. + bucket_id = static_cast(table_ident.partition_id_); + } + return partition_buckets_.find(bucket_id) != partition_buckets_.end(); +} + +bool EloqStoreDataStore::Initialize() +{ + // The store, its metrics included, is set up once by the factory. return true; } @@ -165,10 +254,11 @@ void EloqStoreDataStore::Read(ReadRequest *read_req) ::eloqstore::ReadRequest &kv_read_req = read_op->EloqStoreRequest(); kv_read_req.SetArgs(eloq_store_table_id, key); + kv_read_req.SetDsShard(shard_id_); kv_read_req.SetReopen(read_req->GetReopen()); uint64_t user_data = reinterpret_cast(read_op); - if (!eloq_store_service_->ExecAsyn(&kv_read_req, user_data, OnRead)) + if (!store_->ExecAsyn(&kv_read_req, user_data, OnRead)) { LOG(ERROR) << "Send read request to EloqStore failed for table: " << kv_read_req.TableId(); @@ -266,10 +356,11 @@ void EloqStoreDataStore::BatchWriteRecords(WriteRecordsRequest *write_req) } kv_write_req.SetArgs(eloq_store_table_id, std::move(entries)); + kv_write_req.SetDsShard(shard_id_); uint64_t user_data = reinterpret_cast(write_op); - if (!eloq_store_service_->ExecAsyn(&kv_write_req, user_data, OnBatchWrite)) + if (!store_->ExecAsyn(&kv_write_req, user_data, OnBatchWrite)) { LOG(ERROR) << "Send write request to EloqStore failed for table: " << kv_write_req.TableId(); @@ -354,10 +445,10 @@ void EloqStoreDataStore::DeleteRange(DeleteRangeRequest *delete_range_req) ::eloqstore::TruncateRequest &kv_truncate_req = truncate_op->EloqStoreRequest(); kv_truncate_req.SetArgs(eloq_store_table_id, start_key); + kv_truncate_req.SetDsShard(shard_id_); uint64_t user_data = reinterpret_cast(truncate_op); - if (!eloq_store_service_->ExecAsyn( - &kv_truncate_req, user_data, OnDeleteRange)) + if (!store_->ExecAsyn(&kv_truncate_req, user_data, OnDeleteRange)) { LOG(ERROR) << "Send truncate request to EloqStore failed for table: " << kv_truncate_req.TableId(); @@ -425,10 +516,10 @@ void EloqStoreDataStore::DropTable(DropTableRequest *drop_table_req) drop_table_op->EloqStoreRequest(); kv_drop_table_req.SetArgs(std::string(drop_table_req->GetTableName())); + kv_drop_table_req.SetDsShard(shard_id_); uint64_t user_data = reinterpret_cast(drop_table_op); - if (!eloq_store_service_->ExecAsyn( - &kv_drop_table_req, user_data, OnDropTable)) + if (!store_->ExecAsyn(&kv_drop_table_req, user_data, OnDropTable)) { LOG(ERROR) << "Send drop table request to EloqStore failed for table: " << drop_table_req->GetTableName(); @@ -504,9 +595,10 @@ void EloqStoreDataStore::ScanNext(ScanRequest *scan_req) kv_scan_req.SetArgs( eloq_store_table_id, start_key, end_key, inclusive_start); kv_scan_req.SetPagination(batch_size, 0); + kv_scan_req.SetDsShard(shard_id_); uint64_t user_data = reinterpret_cast(scan_op); - if (!eloq_store_service_->ExecAsyn(&kv_scan_req, user_data, OnScanNext)) + if (!store_->ExecAsyn(&kv_scan_req, user_data, OnScanNext)) { LOG(ERROR) << "Send scan request to EloqStore failed for table: " << kv_scan_req.TableId(); @@ -610,13 +702,14 @@ void EloqStoreDataStore::CreateSnapshotForBackup( std::string_view backup_name = req->GetBackupName(); assert(req->GetBackupTs() != 0); - if (backup_name.empty() || backup_name == eloq_store_service_->Branch()) + if (backup_name.empty() || backup_name == store_->Branch()) { // If backup_name is empty or matches the current branch, create // snapshot for current branch. ::eloqstore::GlobalArchiveRequest global_archive_req; global_archive_req.SetTag(std::to_string(req->GetBackupTs())); - eloq_store_service_->ExecSync(&global_archive_req); + global_archive_req.SetDsShard(shard_id_); + store_->ExecSync(&global_archive_req); ::EloqDS::remote::DataStoreError ds_error; std::string error_msg; @@ -649,7 +742,8 @@ void EloqStoreDataStore::CreateSnapshotForBackup( ::eloqstore::GlobalCreateBranchRequest create_branch_req; create_branch_req.SetArgs(std::string(backup_name)); create_branch_req.SetSaltTimestamp(req->GetBackupTs()); - eloq_store_service_->ExecSync(&create_branch_req); + create_branch_req.SetDsShard(shard_id_); + store_->ExecSync(&create_branch_req); ::EloqDS::remote::DataStoreError ds_error; std::string error_msg; @@ -687,7 +781,8 @@ bool EloqStoreDataStore::CreateSnapshotForStandby(uint32_t ng_id, global_archive_req.SetAction( ::eloqstore::GlobalArchiveRequest::Action::Create); global_archive_req.SetTag(std::string(tag)); - eloq_store_service_->ExecSync(&global_archive_req); + global_archive_req.SetDsShard(shard_id_); + store_->ExecSync(&global_archive_req); if (global_archive_req.Error() != ::eloqstore::KvError::NoError) { LOG(WARNING) << "CreateSnapshotForStandby failed, tag=" << tag @@ -706,7 +801,8 @@ void EloqStoreDataStore::DeleteStandbySnapshot(uint64_t term, ::eloqstore::GlobalArchiveRequest::Action::Delete); global_archive_req.SetTerm(term); global_archive_req.SetTag(std::string(tag)); - eloq_store_service_->ExecSync(&global_archive_req); + global_archive_req.SetDsShard(shard_id_); + store_->ExecSync(&global_archive_req); if (global_archive_req.Error() != ::eloqstore::KvError::NoError) { LOG(WARNING) << "DeleteStandbySnapshot failed for tag=" << tag @@ -724,7 +820,8 @@ bool EloqStoreDataStore::ListArchiveTags(std::string_view prefix, } ::eloqstore::GlobalListArchiveTagsRequest req; req.SetPrefix(std::string(prefix)); - eloq_store_service_->ExecSync(&req); + req.SetDsShard(shard_id_); + store_->ExecSync(&req); if (req.Error() != ::eloqstore::KvError::NoError) { return false; @@ -750,16 +847,20 @@ void EloqStoreDataStore::ScanDelete(DeleteRangeRequest *delete_range_req) // Delete records from eloqstore async ScanDeleteOperationData *scan_del_op = eloq_store_scan_del_op_pool_.NextObject(); - scan_del_op->Reset(delete_range_req, eloq_store_service_.get()); + scan_del_op->Reset(delete_range_req, store_); PoolableGuard op_guard(scan_del_op); ::eloqstore::ScanRequest &kv_scan_req = scan_del_op->EloqStoreScanRequest(); kv_scan_req.SetArgs(eloq_store_table_id, start_key, end_key, true); kv_scan_req.SetPagination(1024, 0); + // Both halves of the scan-and-delete loop work for this data shard; the + // pooled operation may have served another one before. + kv_scan_req.SetDsShard(shard_id_); + scan_del_op->EloqStoreWriteRequest().SetDsShard(shard_id_); uint64_t user_data = reinterpret_cast(scan_del_op); - if (!eloq_store_service_->ExecAsyn(&kv_scan_req, user_data, OnScanDelete)) + if (!store_->ExecAsyn(&kv_scan_req, user_data, OnScanDelete)) { LOG(ERROR) << "Send scan request to EloqStore failed for table: " << kv_scan_req.TableId(); @@ -952,9 +1053,10 @@ void EloqStoreDataStore::Floor(ScanRequest *scan_req) ::eloqstore::FloorRequest &kv_floor_req = floor_op->EloqStoreRequest(); kv_floor_req.SetArgs(eloq_store_table_id, start_key); + kv_floor_req.SetDsShard(shard_id_); uint64_t user_data = reinterpret_cast(floor_op); - if (!eloq_store_service_->ExecAsyn(&kv_floor_req, user_data, OnFloor)) + if (!store_->ExecAsyn(&kv_floor_req, user_data, OnFloor)) { LOG(ERROR) << "Floor request to EloqStore failed for table: " << kv_floor_req.TableId(); @@ -1007,15 +1109,16 @@ bool EloqStoreDataStore::ReloadData(int64_t term, LOG(INFO) << "EloqStoreDataStore::ReloadData, term: " << term << ", snapshot_ts: " << snapshot_ts << ", from_snapshot: " << from_snapshot; - if (eloq_store_service_->Term() != static_cast(term)) + if (store_->ShardTerm(shard_id_) != static_cast(term)) { LOG(ERROR) << "EloqStoreDataStore::ReloadData, term mismatch, " "expected: " - << term << ", actual: " << eloq_store_service_->Term() + << term << ", actual: " << store_->ShardTerm(shard_id_) << ")"; return false; } ::eloqstore::GlobalReopenRequest global_reopen_req; + global_reopen_req.SetDsShard(shard_id_); if (from_snapshot) { const std::string reopen_tag = @@ -1031,7 +1134,7 @@ bool EloqStoreDataStore::ReloadData(int64_t term, << term << ", snapshot_ts: " << snapshot_ts << ", tag: "; } - eloq_store_service_->ExecSync(&global_reopen_req); + store_->ExecSync(&global_reopen_req); if (global_reopen_req.Error() != ::eloqstore::KvError::NoError) { @@ -1050,7 +1153,7 @@ void EloqStoreDataStore::UpdateStandbyMasterStorePaths( const std::vector &store_paths, const std::vector &store_path_weights) { - auto res = eloq_store_service_->UpdateStandbyMasterStorePaths( + auto res = store_->UpdateStandbyMasterStorePaths( std::vector(store_paths.begin(), store_paths.end()), std::vector(store_path_weights.begin(), store_path_weights.end())); @@ -1064,8 +1167,7 @@ void EloqStoreDataStore::UpdateStandbyMasterStorePaths( void EloqStoreDataStore::UpdateStandbyMasterAddr( const std::string &standby_master_addr) { - auto res = - eloq_store_service_->UpdateStandbyMasterAddr(standby_master_addr); + auto res = store_->UpdateStandbyMasterAddr(standby_master_addr); if (res != ::eloqstore::KvError::NoError) { LOG(WARNING) << "UpdateStandbyMasterAddr failed with error: " diff --git a/store_handler/eloq_data_store_service/eloq_store_data_store.h b/store_handler/eloq_data_store_service/eloq_store_data_store.h index b06a04180..9903d4567 100644 --- a/store_handler/eloq_data_store_service/eloq_store_data_store.h +++ b/store_handler/eloq_data_store_service/eloq_store_data_store.h @@ -171,27 +171,59 @@ class EloqStoreDataStore : public DataStore std::string tag; }; + /** + * @brief Makes the data shard's data store on the factory's store, which + * must be running (EloqStoreDataStoreFactory::CreateDataStore). + * @param shard_id The data shard. + * @param data_store_service The service, which holds the factory. + */ EloqStoreDataStore(uint32_t shard_id, DataStoreService *data_store_service); - ~EloqStoreDataStore() override = default; + ~EloqStoreDataStore() override; bool Initialize() override; + /** + * @brief Attaches the data shard to the store at its term + * (EloqStore::AttachShard): its partitions are served in that term from + * now on, in cloud mode after the shard's term file has been raised. + * + * @param term The data shard's term. + * @param data_shard_id The data shard, the store's data store shard id. + * @return Whether the shard is attached; false on ExpiredTerm, when a + * newer term owns the shard. + */ bool StartDB(int64_t term, uint32_t data_shard_id) override { - ::eloqstore::KvError res = - eloq_store_service_->Start(branch_name_, term, data_shard_id); + LOG(INFO) << "Attaching data shard " << data_shard_id + << " to the EloqStore at term " << term << ", owning " + << partition_buckets_.size() + << " buckets (0: every partition)"; + ::eloqstore::KvError res = store_->AttachShard(data_shard_id, term); if (res != ::eloqstore::KvError::NoError) { - LOG(ERROR) << "EloqStore start failed with error code: " + LOG(ERROR) << "Attaching data shard " << data_shard_id + << " to the EloqStore failed with error code: " << static_cast(res); } return res == ::eloqstore::KvError::NoError; } + /** + * @brief Detaches the data shard from the store (EloqStore::DetachShard): + * its requests are refused, the ones in flight finish, and the store + * drops what it holds for its partitions. The store itself keeps running + * for the other data shards. + */ void Shutdown() override { - eloq_store_service_->Stop(); + ::eloqstore::KvError res = store_->DetachShard(shard_id_); + if (res != ::eloqstore::KvError::NoError) + { + LOG(ERROR) << "Detaching data shard " << shard_id_ + << " from the EloqStore failed with error code: " + << static_cast(res); + } } /** @@ -259,6 +291,26 @@ class EloqStoreDataStore : public DataStore */ void SwitchToReadWrite() override; + /** + * @brief Sets the buckets of this data shard, which the store's partition + * filter checks for it (OwnsPartition). + * + * Every data shard's partitions share the store's storage: one cloud + * store path for the cluster, and in local mode one store path per node. + * So the operations that sweep it for a data shard (prewarm, archives for + * backups and standby snapshots, branches, reopen) must pick that shard's + * partitions. The set belongs to the data shard, not the process: the one + * store hosts a shard for each node group the node leads. + * + * @param bucket_ids The data shard's buckets; empty accepts every + * partition. + * @warning Call only while the data shard is detached (Starting, before + * StartDB), since the store reads the set without a lock once the shard + * is attached. + */ + void SetPartitionBuckets( + const std::unordered_set &bucket_ids) override; + void CreateSnapshotForBackup(CreateSnapshotForBackupRequest *req) override; bool CreateSnapshotForStandby(uint32_t ng_id, std::string_view tag); void DeleteStandbySnapshot(uint64_t term, std::string_view tag) override; @@ -286,10 +338,36 @@ class EloqStoreDataStore : public DataStore void ScanDelete(DeleteRangeRequest *delete_range_req); void Floor(ScanRequest *scan_req); - std::unique_ptr<::eloqstore::EloqStore> eloq_store_service_{nullptr}; + /** + * @brief Whether a partition belongs to this data shard; the store's + * partition filter asks the factory, which asks the shard's data store. + * + * Maps the partition to its bucket as the tx service does + * (Sharder::MapRangeIdToBucketId for range-partitioned tables, the hash + * partition itself for hash-partitioned ones) and looks the bucket up in + * partition_buckets_. + * + * @param table_ident The partition. + * @return true if it belongs to this data shard, or if no buckets are set. + */ + bool OwnsPartition(const ::eloqstore::TableIdent &table_ident) const; + + /** + * @brief Buckets of this data shard (SetPartitionBuckets); empty accepts + * every partition. + * + * Written only while the data shard is detached and read by the store + * through its partition filter while it is attached, so it needs no lock. + */ + std::unordered_set partition_buckets_; + + /** + * The store every data shard's data store shares, owned by the factory, + * which outlives this object's use of it (DataStoreService shuts the + * data shards down before it destroys the factory). + */ + ::eloqstore::EloqStore *store_{nullptr}; - // Branch name passed to EloqStore::Start(). Populated from the factory - // config in the EloqStoreDataStore constructor. - std::string branch_name_{"main"}; + friend class EloqStoreDataStoreFactory; }; } // namespace EloqDS diff --git a/store_handler/eloq_data_store_service/eloq_store_data_store_factory.h b/store_handler/eloq_data_store_service/eloq_store_data_store_factory.h index d51270cb1..403f6b96e 100644 --- a/store_handler/eloq_data_store_service/eloq_store_data_store_factory.h +++ b/store_handler/eloq_data_store_service/eloq_store_data_store_factory.h @@ -21,9 +21,11 @@ */ #pragma once +#include +#include #include +#include #include -#include #include #include "data_store_factory.h" @@ -31,6 +33,16 @@ namespace EloqDS { +/** + * @brief Makes the EloqStore-backed data stores, and owns the one EloqStore + * they share. + * + * The process runs one EloqStore, with one buffer pool, fd budget and local + * cache quota, and that store hosts every data shard this node leads: each + * EloqStoreDataStore attaches its data shard to the store when the shard is + * opened (EloqStore::AttachShard) and detaches it when the shard is closed. + * The store starts with the first data store made and stops with the factory. + */ class EloqStoreDataStoreFactory : public DataStoreFactory { public: @@ -39,6 +51,14 @@ class EloqStoreDataStoreFactory : public DataStoreFactory { } + ~EloqStoreDataStoreFactory() override + { + if (store_ != nullptr && !store_->IsStopped()) + { + store_->Stop(); + } + } + std::unique_ptr CreateDataStore( bool create_if_missing, uint32_t shard_id, @@ -46,6 +66,10 @@ class EloqStoreDataStoreFactory : public DataStoreFactory bool start_db = true, int64_t term = 0) override { + if (!EnsureStoreStarted()) + { + return nullptr; + } auto ds = std::make_unique(shard_id, data_store_service); ds->Initialize(); @@ -109,37 +133,6 @@ class EloqStoreDataStoreFactory : public DataStoreFactory return 0; // Not applicable for EloqStore } - void InitializePartitionFilter( - uint32_t ng_id, std::unordered_set &&bucket_ids) override - { - eloq_store_configs_.eloqstore_configs_.partition_filter = - [bucket_ids = std::move(bucket_ids)]( - const eloqstore::TableIdent &table_ident) -> bool - { - const std::string &table_name = table_ident.tbl_name_; - uint16_t bucket_id; - if (table_name.compare(0, 8, "eloqdoc_") == 0 || - table_name.compare(0, 8, "eloqsql_") == 0 || - table_name.compare(0, 7, "irange_") == 0) - { - // this table engine use RangePartition. - uint32_t hash_val; - butil::MurmurHash3_x86_32(&table_ident.partition_id_, - sizeof(table_ident.partition_id_), - 9001, - &hash_val); - bucket_id = hash_val % 1024; - } - else - { - // use HashPartition, hash_partition_id equals to bucket_id - bucket_id = static_cast(table_ident.partition_id_); - } - - return bucket_ids.find(bucket_id) != bucket_ids.end(); - }; - } - bool IsCloudMode() const override { return !eloq_store_configs_.eloqstore_configs_.cloud_store_path.empty(); @@ -166,8 +159,60 @@ class EloqStoreDataStoreFactory : public DataStoreFactory eloq_store_configs_.eloqstore_configs_.enable_local_standby = enable; } + /** + * @brief The one EloqStore every data shard's data store sends to. + * @return The store; null until the first data store is made. + */ + ::eloqstore::EloqStore *Store() const + { + return store_.get(); + } + private: + /** + * @brief Builds and starts the store on the first call (StartInstance: + * the store paths are opened and the worker shards built, with no data + * shard attached yet). The options are fixed then, including the standby + * settings; later changes to them reach the running store through + * EloqStoreDataStore::UpdateStandbyMaster*. + * @return Whether the store is running. + */ + bool EnsureStoreStarted(); + + /** + * @brief The store's partition filter: whether a partition belongs to a + * data shard this node hosts, answered by that shard's data store + * (EloqStoreDataStore::OwnsPartition) from its buckets. + * + * Called from the store's threads when a sweep (archive, reopen, branch, + * prewarm) runs for a shard; a shard without a data store yet owns + * nothing. + * + * @param ds_shard_id The data shard the sweep runs for. + * @param table_ident The partition. + * @return Whether the partition is the shard's. + */ + bool OwnsPartition(::eloqstore::DsShardId ds_shard_id, + const ::eloqstore::TableIdent &table_ident) const; + + /** + * @brief Lets a data shard's data store answer the filter for its id. + * @param shard_id The data shard. + * @param data_store Its data store; null to withdraw it. + */ + void RegisterShardStore(uint32_t shard_id, EloqStoreDataStore *data_store); + EloqStoreConfig eloq_store_configs_; + /** Serializes the store's lazy start. */ + std::mutex store_mux_; + std::unique_ptr<::eloqstore::EloqStore> store_{nullptr}; + /** + * The data shards' data stores, for the partition filter; each registers + * itself for its lifetime. Indexed by data shard id. + */ + std::array, + DataStoreService::kMaxShardCount> + shard_stores_{}; friend class EloqStoreDataStore; }; diff --git a/store_handler/eloq_data_store_service/eloqstore b/store_handler/eloq_data_store_service/eloqstore index 09a422702..82b774c62 160000 --- a/store_handler/eloq_data_store_service/eloqstore +++ b/store_handler/eloq_data_store_service/eloqstore @@ -1 +1 @@ -Subproject commit 09a4227028cc0de58bd5e4af4e10577438ca5563 +Subproject commit 82b774c62728b023b03c6e5e84e69416ffec9d74 diff --git a/store_handler/rocksdb_handler.cpp b/store_handler/rocksdb_handler.cpp index f07045b13..372d51d36 100644 --- a/store_handler/rocksdb_handler.cpp +++ b/store_handler/rocksdb_handler.cpp @@ -1794,7 +1794,7 @@ bool RocksDBHandler::UpdateRangeSlices( bool RocksDBHandler::UpsertRanges( const txservice::TableName &table_name, - std::vector range_info, + const std::vector &range_info, uint64_t version) { LOG(ERROR) << "RocksDBHandler::UpsertRanges not implemented"; diff --git a/store_handler/rocksdb_handler.h b/store_handler/rocksdb_handler.h index e697e88d4..626cca247 100644 --- a/store_handler/rocksdb_handler.h +++ b/store_handler/rocksdb_handler.h @@ -426,7 +426,7 @@ class RocksDBHandler : public txservice::store::DataStoreHandler const std::function *sync_yield_fptr = nullptr) override; bool UpsertRanges(const txservice::TableName &table_name, - std::vector range_info, + const std::vector &range_info, uint64_t version) override; bool FetchTable( diff --git a/tx_service/CMakeLists.txt b/tx_service/CMakeLists.txt index 8f452be9c..7baea56bf 100644 --- a/tx_service/CMakeLists.txt +++ b/tx_service/CMakeLists.txt @@ -235,6 +235,8 @@ SET(TxService_SOURCES ${TX_SERVICE_SOURCE_DIR}/src/tx_worker_pool.cpp ${TX_SERVICE_SOURCE_DIR}/src/dead_lock_check.cpp ${TX_SERVICE_SOURCE_DIR}/src/tx_index_operation.cpp + ${TX_SERVICE_SOURCE_DIR}/src/tx_cluster_gate_operation.cpp + ${TX_SERVICE_SOURCE_DIR}/src/tx_upsert_table_operation.cpp ${TX_SERVICE_SOURCE_DIR}/src/sk_generator.cpp ${TX_SERVICE_SOURCE_DIR}/src/data_sync_task.cpp ${TX_SERVICE_SOURCE_DIR}/src/standby.cpp diff --git a/tx_service/include/catalog_factory.h b/tx_service/include/catalog_factory.h index 014e813f0..b2a4b82d9 100644 --- a/tx_service/include/catalog_factory.h +++ b/tx_service/include/catalog_factory.h @@ -398,6 +398,31 @@ class CatalogFactory CatalogFactory() = default; virtual ~CatalogFactory() = default; + /** + * @brief Builds a table's schema object from its catalog image. + * + * An image records the version of each table it describes, the base table + * and every index. A table the image records no version for is one the + * image's DDL creates: the engine built the image before the DDL had a + * timestamp. The schema gives such a table @p version and keeps every + * table's version in its SchemaImage(), so the image it returns gives each + * table the same version whatever version it is loaded at. + * + * The data substrate relies on this. Once a schema op has chosen its + * prepare-stage timestamp D, SchemaOp::ResolveSchemaImages builds the new + * schema at D and keeps its image: the tables the op creates (every table + * of a CREATE TABLE, the new indexes of an add index) get D, and the + * tables that exist keep their versions. An engine must therefore record + * the version of every existing table in a DDL's new image, and no version + * for a table the DDL creates. TableKeySchemaTs::GetKeySchemaTs reads a + * missing version as 0, the unknown timestamp. + * + * @param table_name The base table. + * @param catalog_image The table's catalog image. + * @param version The schema's version, which is also the version of every + * table the image records none for. + * @return The schema. + */ virtual TableSchema::uptr CreateTableSchema( const TableName &table_name, const std::string &catalog_image, diff --git a/tx_service/include/catalog_key_record.h b/tx_service/include/catalog_key_record.h index 1fe48dfcc..6e8095da9 100644 --- a/tx_service/include/catalog_key_record.h +++ b/tx_service/include/catalog_key_record.h @@ -21,6 +21,7 @@ */ #pragma once +#include #include #include #include @@ -185,13 +186,21 @@ struct CatalogEntry } } - void CommitDirtySchema() + /** + * @brief Promotes the dirty schema to the current schema. + * + * @param commit_ts The op's commit timestamp, the schema record's new + * version. CREATE, DROP and TRUNCATE TABLE commit above the version the + * dirty schema was installed with (UpsertTableOp::AssignCommitStageTs); + * other ops commit at it. 0 keeps the installed version. + */ + void CommitDirtySchema(uint64_t commit_ts = 0) { std::unique_lock lk(s_mux_); if (dirty_schema_version_ > schema_version_) { schema_ = std::move(dirty_schema_); - schema_version_ = dirty_schema_version_; + schema_version_ = std::max(dirty_schema_version_, commit_ts); } else { @@ -200,11 +209,25 @@ struct CatalogEntry dirty_schema_version_ = 0; } - void RejectDirtySchema() + /** + * @brief Rolls back a schema op: drops the dirty schema and keeps the + * current schema, whose version becomes the op's commit timestamp if it + * has one. + * + * No table's version changes, so the schema object stays as it is. + * + * @param version The op's commit timestamp: 0 for an op that failed, which + * leaves the version alone; ignored unless above the current version. + */ + void RollBackDirtySchema(uint64_t version) { std::unique_lock lk(s_mux_); dirty_schema_.reset(); dirty_schema_version_ = 0; + if (version > schema_version_) + { + schema_version_ = version; + } } // FIXME(lokax): diff --git a/tx_service/include/cc/catalog_cc_map.h b/tx_service/include/cc/catalog_cc_map.h index 587eff247..2ce22163a 100644 --- a/tx_service/include/cc/catalog_cc_map.h +++ b/tx_service/include/cc/catalog_cc_map.h @@ -165,15 +165,16 @@ class CatalogCcMap }); TX_TRACE_DUMP(&req); + // Kills the node (*_panic) or stalls it for 10 seconds (*_sleep) during + // a schema op's PrepareCommit, or during the Rollback of a failed one. CODE_FAULT_INJECTOR("during_post_write_all", { std::string &action = FaultInject::Instance() .Entry("during_post_write_all") ->vctAction_.front(); if ((req.CommitType() == PostWriteType::PrepareCommit && action == "prepare_commit_panic") || - (req.CommitType() == PostWriteType::PostCommit && - req.CommitTs() == TransactionOperation::tx_op_failed_ts_ && - action == "post_commit_panic")) + (req.CommitType() == PostWriteType::Rollback && + action == "rollback_panic")) { int retval; sigset_t new_mask; @@ -191,10 +192,8 @@ class CatalogCcMap } else if ((req.CommitType() == PostWriteType::PrepareCommit && action == "prepare_commit_sleep") || - (req.CommitType() == PostWriteType::PostCommit && - req.CommitTs() == - TransactionOperation::tx_op_failed_ts_ && - action == "post_commit_sleep")) + (req.CommitType() == PostWriteType::Rollback && + action == "rollback_sleep")) { LOG(INFO) << "sleep 10 seconds: "; sleep(10); @@ -260,17 +259,16 @@ class CatalogCcMap if (cce_ptr == nullptr || cce_ptr->GetKeyLock() == nullptr || !cce_ptr->GetKeyLock()->HasWriteLockOrWriteIntent(req.Txn())) { - // When the catalog entry is null in the post-write-all - // phase, it means that (1) the cc req is a resend request and - // previous has successed, (2) the cc node group must have failed - // over once, and (3) there is no catalog op in the log (so that the - // recovered cc node group has no catalog entry). No catalog op in - // the log means that this schema op fails before the prepare log - // and this post-write-all request is to release the write - // lock/intent. Given that the cc node group has failed once, the - // previously acquired intent/lock has gone. There is no need to - // proceed to release the intent/lock. The request is set to - // finished. + // The tx holds no lock on the entry here, so the post write has + // nothing left to do and finishes. That is the case when (1) the + // request is a resend whose first copy succeeded; (2) the node + // group failed over while the schema op had no log yet, so the + // op failed before its prepare log and the request only releases + // an intent or lock the failover already dropped; or (3) the node + // group, a participant, recovered from the op's commit log, which + // installs the committed schema and restores no lock there. Only + // the coordinator's node group restores one, see + // Execute(ReplayLogCc&). if (shard_->core_id_ == shard_->core_cnt_ - 1) { req.Result()->SetFinished(); @@ -285,781 +283,487 @@ class CatalogCcMap } } - CatalogRecord *schema_rec = nullptr; - CatalogEntry *catalog_entry = nullptr; - - // First setup the schema_rec that will replace the current catalog rec. switch (req.CommitType()) { case PostWriteType::PrepareCommit: - { - if (req.CommitTs() == TransactionOperation::tx_op_failed_ts_) - { - // When the commit ts is 0, the request commits nothing and only - // removes the write intents/locks acquired earlier. - return TemplateCcMap::Execute(req); - } + return PrepareCommitSchema(req, *table_key, ng_term); + case PostWriteType::UpdateDirty: + return UpdateDirtySchema(req, *table_key); + case PostWriteType::PostCommit: + return PostCommitSchema(req, *table_key, *cce_ptr, ng_term); + case PostWriteType::Rollback: + return RollBackSchema(req, *table_key); + case PostWriteType::DowngradeLock: + // The request commits nothing and only downgrades the write locks + // to write intents. + return TemplateCcMap::Execute(req); + default: + // Schema operations always employ multi-stage commits. + assert(false); + return TemplateCcMap::Execute(req); + } + } - if (shard_->core_id_ == 0) + bool Execute(ReadCc &req) override + { + TX_TRACE_ACTION_WITH_CONTEXT( + (txservice::CcMap *) this, + &req, + [&req]() -> std::string { - // For prepare commit or commit, instantiates the dirty schema - // instance in LocalCcShards when processing the request at the - // first shard/core. - - if (req.Payload() != nullptr) - { - // When the request comes from a tx in the same node, the - // request references a schema record in the tx's space. - schema_rec = static_cast(req.Payload()); - } - else - { - // When the request comes from a remote tx, the request - // contains a serialized representation of the catalog, but - // not a reference of a schema record. Allocates a schema - // record. - assert(req.PayloadStr() != nullptr); - std::unique_ptr decoded_rec = - std::make_unique(); - if (req.OpType() != OperationType::DropTable) - { - size_t offset = 0; - decoded_rec->Deserialize(req.PayloadStr()->data(), - offset); - } - schema_rec = decoded_rec.get(); - req.SetDecodedPayload(std::move(decoded_rec)); - } - - catalog_entry = - shard_->CreateDirtyCatalog(table_key->Name(), - req.NodeGroupId(), - schema_rec->DirtySchemaImage(), - req.CommitTs()); - - // For alter index, we need to initialize meta data for the - // new indexes at prepare stage since the new indexes may - // receive cc requests during the 2 phase commit DDL tx.. - if ((req.OpType() == OperationType::AddIndex || - req.OpType() == OperationType::DropIndex)) - { - if (catalog_entry->schema_ == nullptr) - { - // For alter table, in some case, the current schema may - // not exists yet, so should create the current schema. - // For example, this node is the participant node of the - // alter table transaction, and does not execute any - // transaction about this table before this alter table - // tx since server start. - shard_->FetchCatalog(table_key->Name(), - req.NodeGroupId(), - ng_term, - &req); - return false; - } - - if (Sharder::Instance().GetLocalCcShards()->store_hd_ != - nullptr && - !table_key->Name().IsHashPartitioned() && - !shard_->LoadRangesAndStatisticsNx( - catalog_entry->schema_.get(), - req.NodeGroupId(), - ng_term, - &req)) - { - return false; - } - - // Bind statistics for the dirty schema. - catalog_entry->dirty_schema_->BindStatistics( - catalog_entry->schema_->StatisticsObject()); - if (req.OpType() == OperationType::AddIndex) - { - // For ALTER TABLE, set the dirty index name, and this - // info should be clean when commit dirty schema. For - // CREATE TABLE, there is no matter that do not set the - // dirty index info, because this table is invisible - // until create table transaction commit. - auto new_index_names = - catalog_entry->dirty_schema_->IndexNames(); - auto old_index_names = - catalog_entry->schema_->IndexNames(); - std::vector dirty_index_names; - for (const TableName &new_index_name : new_index_names) - { - if (std::find(old_index_names.begin(), - old_index_names.end(), - new_index_name) == - old_index_names.end()) - { - dirty_index_names.emplace_back( - new_index_name.StringView(), - new_index_name.Type(), - new_index_name.Engine()); - } - } - - // Load ranges for the new added indexes. We cannot - // simply initialize it with empty range table since we - // might have pre-defined range table based on the data - // distribution offered by caller. - for (const TableName &index_name : dirty_index_names) - { - TableName index_range_name{ - index_name.StringView(), - TableType::RangePartition, - index_name.Engine()}; + return std::string("\"cc_map_type\":\"template_cc_map\"") + .append(",\"tx_number\":") + .append(std::to_string(req.Txn())) + .append(",\"term\":") + .append(std::to_string(req.TxTerm())); + }); + TX_TRACE_DUMP(&req); - auto ranges = shard_->GetTableRangesForATable( - index_range_name, req.NodeGroupId()); - if (ranges == nullptr) - { - shard_->FetchTableRanges(index_range_name, - &req, - req.NodeGroupId(), - ng_term); - return false; - } + assert(req.IsLocal()); - for (auto &range : *ranges) - { - range.second->SetVersion(req.CommitTs()); - NodeGroupId range_owner = - shard_ - ->GetRangeOwner( - range.second->GetRangeInfo() - ->PartitionId(), - req.NodeGroupId()) - ->BucketOwner(); - if (range_owner == req.NodeGroupId() && - range.second->RangeSlices() == nullptr) - { - // The owner this this range, and this is a - // empty range. - - std::unique_lock heap_lk( - shard_->local_shards_ - .table_ranges_heap_mux_); - bool is_override_thd = - mi_is_override_thread(); - mi_threadid_t prev_thd = mi_override_thread( - shard_->local_shards_ - .GetTableRangesHeapThreadId()); - mi_heap_t *prev_heap = mi_heap_set_default( - shard_->local_shards_ - .GetTableRangesHeap()); + uint32_t ng_id = req.NodeGroupId(); + int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); + ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); -#if defined(WITH_JEMALLOC) - uint32_t prev_arena; - JemallocArenaSwitcher::ReadCurrentArena( - prev_arena); - auto table_range_arena_id = - shard_->local_shards_ - .GetTableRangesArenaId(); - JemallocArenaSwitcher::SwitchToArena( - table_range_arena_id); -#endif + if (req.IsInRecovering()) + { + ng_term = ng_term > 0 + ? ng_term + : Sharder::Instance().CandidateLeaderTerm(ng_id); + } - std::vector slices; - range.second->InitRangeSlices( - std::move(slices), - req.NodeGroupId(), - false, - true, - UINT64_MAX, - false); - - mi_heap_set_default(prev_heap); - if (is_override_thd) - { - mi_override_thread(prev_thd); - } - else - { - mi_restore_default_thread_id(); - } + CODE_FAULT_INJECTOR("term_CatalogCcMap_Execute_ReadCc", { + LOG(INFO) << "FaultInject term_CatalogCcMap_Execute_ReadCc"; + ng_term = -1; + FaultInject::Instance().InjectFault( + "term_CatalogCcMap_Execute_ReadCc", "remove"); + }); -#if defined(WITH_JEMALLOC) - JemallocArenaSwitcher::SwitchToArena( - prev_arena); + if (ng_term < 0) + { + req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); + return true; + } -#endif - heap_lk.unlock(); - } - } - } + const CatalogKey *table_key = + static_cast(req.Key()); + bool emplace = false; + Iterator it = FindEmplace(*table_key, &emplace, true, false); + CcEntry *cce = it->second; + if (cce->PayloadStatus() == RecordStatus::Unknown) + { + const CatalogEntry *catalog_entry = + shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); - for (const TableName &index_name : dirty_index_names) + // If the read toward the catalog cc entry acquires the read + // lock but the cc entry does not contain the value, checks if + // the catalog has been constructed at this node. If so, turns + // this request into a read-outside request that installs the + // value in the cc entry. + if (catalog_entry != nullptr && catalog_entry->schema_version_ > 0) + { + if (catalog_entry->schema_ != nullptr) + { + { +#ifdef STATISTICS + // Initialize table statistics before create ccmap. + if (Sharder::Instance().GetLocalCcShards()->store_hd_ != + nullptr && + !table_key->Name().IsHashPartitioned() && + !shard_->LoadRangesAndStatisticsNx( + catalog_entry->schema_.get(), + req.NodeGroupId(), + ng_term, + &req)) { - catalog_entry->schema_->StatisticsObject() - ->CreateIndex(index_name, - catalog_entry->dirty_schema_ - ->IndexKeySchema(index_name), - cc_ng_id_); + return false; } +#endif } - } - - schema_rec->Set(catalog_entry->schema_, - catalog_entry->dirty_schema_, - catalog_entry->schema_version_); - } - else - { - assert(req.Payload() != nullptr); - schema_rec = static_cast(req.Payload()); - catalog_entry = - shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); - - if (catalog_entry == nullptr) - { - // The table catalog haven't been initialized. Just return. - req.Result()->SetError( - CcErrorCode::REQUESTED_TABLE_NOT_EXISTS); - return true; - } - } - break; - } - case PostWriteType::UpdateDirty: - { - assert(req.OpType() == OperationType::AddIndex); - assert(req.CommitTs() != TransactionOperation::tx_op_failed_ts_); - if (shard_->core_id_ == 0) - { - // For update dirty, retrieves the current and dirty schema pair - // from the current shard. - if (req.Payload() != nullptr) - { - // When the request comes from a tx in the same node, the - // request references a schema record in the tx's space. - schema_rec = static_cast(req.Payload()); + // upload catalog record + cce->payload_.PassInCurrentPayload( + std::make_unique()); + cce->payload_.cur_payload_->Set( + catalog_entry->schema_, + catalog_entry->dirty_schema_, + catalog_entry->schema_version_); + cce->SetCommitTsPayloadStatus( + catalog_entry->schema_version_, RecordStatus::Normal); } else { - assert(req.PayloadStr() != nullptr); - // When the request comes from a remote tx, allocates a - // schema record, which acts as a container referencing the - // current and dirty schema pair. - std::unique_ptr decoded_rec = - std::make_unique(); - size_t offset = 0; - decoded_rec->Deserialize(req.PayloadStr()->data(), offset); - - schema_rec = decoded_rec.get(); - req.SetDecodedPayload(std::move(decoded_rec)); - } - - catalog_entry = - shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); - - if (catalog_entry == nullptr) - { - // The target table catalog haven't been initialized. Just - // return. - req.Result()->SetError( - CcErrorCode::REQUESTED_TABLE_NOT_EXISTS); - return true; - } - - if (catalog_entry->dirty_schema_->SchemaImage() != - schema_rec->DirtySchemaImage()) - { - assert(catalog_entry->dirty_schema_version_ == - req.CommitTs()); - std::shared_ptr old_dirty_schema = - catalog_entry->dirty_schema_; - shard_->UpdateDirtyCatalog(table_key->Name(), - schema_rec->DirtySchemaImage(), - catalog_entry); - catalog_entry->dirty_schema_->BindStatistics( - old_dirty_schema->StatisticsObject()); + cce->SetCommitTsPayloadStatus( + catalog_entry->schema_version_, RecordStatus::Deleted); } - - schema_rec->Set(catalog_entry->schema_, - catalog_entry->dirty_schema_, - catalog_entry->schema_version_); } else { - assert(req.Payload() != nullptr); - schema_rec = static_cast(req.Payload()); - catalog_entry = - shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); - - if (catalog_entry == nullptr) - { - // The table catalog haven't been initialized. Just return. - req.Result()->SetError( - CcErrorCode::REQUESTED_TABLE_NOT_EXISTS); - return true; - } + shard_->FetchCatalog( + table_key->Name(), req.NodeGroupId(), ng_term, &req); + return false; } - break; } - case PostWriteType::PostCommit: + + // Increase ref cnt so that when setFinish is called in + // TemplateCcMap::Execute, the waiters cannot be woken up. We still need + // to access sch_rec which is owned by the caller. + CcHandlerResult *hd_res = req.Result(); + hd_res->SetRefCnt(2); + bool success = TemplateCcMap::Execute(req); + + ReadKeyResult &result = req.Result()->Value(); + CatalogRecord *sch_rec = static_cast(req.Record()); + + // Returns the reader-writer control block to runtime when there is no + // writer and no pending dirty schema. The control block provides a fast + // path for runtime to keep a cache of the schema and to coordinate + // modifications of the schema. + if (success && shard_->IsNative(ng_id) && + result.rec_status_ == RecordStatus::Normal && + sch_rec->DirtySchema() == nullptr) { - if (shard_->IsNative(req.NodeGroupId())) + std::shared_ptr> sch_cntl = + shard_->FindEmplaceSchemaCntl(table_key->Name()); + if (sch_cntl->HasNoWriter()) { - const TableName &tbl_name = table_key->Name(); - std::shared_ptr> schema_cntl = - shard_->FindSchemaCntl(tbl_name); - if (schema_cntl != nullptr) + if (sch_cntl->GetObjectPtr() == nullptr) { - schema_cntl->FinishWriter(req.Txn()); - shard_->DeleteSchemaCntl(tbl_name); + sch_cntl->SetObject(sch_rec->CopySchema()); } + sch_rec->SetSchemaCntl(std::move(sch_cntl)); } + } + hd_res->SetFinished(); - catalog_entry = - shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); + return success; + } - if (req.CommitTs() == TransactionOperation::tx_op_failed_ts_) + bool Execute(ReplayLogCc &req) override + { + TX_TRACE_ACTION_WITH_CONTEXT( + (txservice::CcMap *) this, + &req, + [&req]() -> std::string { - // The catalog_entry could be null if the txn aborts after - // acquire write intent failure. + return std::string("\"cc_map_type\":\"template_cc_map\"") + .append(",\"tx_number\":") + .append(std::to_string(req.Txn())) + .append(",\"term\":") + .append("0"); + }); + TX_TRACE_DUMP(&req); + int64_t ng_term = + Sharder::Instance().CandidateLeaderTerm(req.NodeGroupId()); + if (ng_term < 0) + { + LOG(INFO) << "ReplayLogCc, node_group(#" << req.NodeGroupId() + << ") term < 0, tx:" << req.Txn(); + req.AbortCcRequest(CcErrorCode::REQUESTED_NODE_NOT_LEADER); + return true; + } - // For add index op, we create new sk ccmap, table ranges and - // table statistics for the new sk during prepare phase. If - // flush kv failed, should clean these up. But, if the dirty - // schema is nullptr, that is mean, this is the recovering - // transaction, and there is no need to drop the new sk ccmap. - if (req.OpType() == OperationType::AddIndex && - catalog_entry != nullptr && - catalog_entry->dirty_schema_ != nullptr) - { - std::vector new_index_names = - catalog_entry->dirty_schema_->IndexNames(); - std::vector old_index_names = - catalog_entry->schema_->IndexNames(); - for (const TableName &new_index_name : new_index_names) - { - if (std::find(old_index_names.begin(), - old_index_names.end(), - new_index_name) == old_index_names.end()) - { - shard_->DropCcm(new_index_name, req.NodeGroupId()); - // Clean up table ranges for new sk. - const TableName index_range_name{ - new_index_name.StringView(), - TableType::RangePartition, - new_index_name.Engine()}; - shard_->DropCcm(index_range_name, - req.NodeGroupId()); - if (shard_->core_id_ == shard_->core_cnt_ - 1) - { - shard_->CleanTableRange(index_range_name, - req.NodeGroupId()); - Statistics *statistics = - catalog_entry->dirty_schema_ - ->StatisticsObject() - .get(); - statistics->DropIndex(new_index_name); - } - } - } - } + ::txlog::SchemaOpMessage schema_op_msg; + const std::string_view &content = req.LogContentView(); + schema_op_msg.ParseFromArray(content.data(), content.length()); - // Flush kv fails, need to clear dirty CatalogEntry - if (shard_->core_id_ == shard_->core_cnt_ - 1 && - catalog_entry != nullptr) - { - catalog_entry->RejectDirtySchema(); - } + const CatalogEntry *catalog_entry = nullptr; - if (cce_ptr->payload_.cur_payload_) - { - cce_ptr->payload_.cur_payload_->ClearDirtySchema(); - cce_ptr->payload_.cur_payload_->SetDirtySchemaImage(""); - } - return TemplateCcMap::Execute(req); - } - - assert(catalog_entry != nullptr); - if (shard_->core_id_ == 0) - { - // For post commit, retrieves the current and dirty schema pair - // from the current shard. - if (req.Payload() != nullptr) - { - // When the request comes from a tx in the same node, the - // request references a schema record in the tx's space. - schema_rec = static_cast(req.Payload()); - } - else - { - assert(req.PayloadStr() != nullptr); - // When the request comes from a remote tx, allocates a - // schema record, which acts as a container referencing the - // current and dirty schema pair. - std::unique_ptr decoded_rec = - std::make_unique(); - if (req.OpType() != OperationType::DropTable) - { - size_t offset = 0; - decoded_rec->Deserialize(req.PayloadStr()->data(), - offset); - } - - schema_rec = decoded_rec.get(); - req.SetDecodedPayload(std::move(decoded_rec)); - } - - if (req.OpType() == OperationType::CreateTable) - { - std::vector range_init_vec; - std::string_view table_name_view = - table_key->Name().StringView(); - int init_partition_id = 0; - if (table_name_view != "./mysql/sequences") - { - size_t tbl_name_hash = - std::hash()(table_name_view); - init_partition_id = tbl_name_hash & 0xFFF; - } - - // Use nullptr to represent negative infinity key here. - range_init_vec.emplace_back( - TxKey(), init_partition_id, req.CommitTs()); - - TableName range_table_name(table_name_view, - TableType::RangePartition, - table_key->Name().Engine()); - shard_->local_shards_.InitTableRanges(range_table_name, - range_init_vec, - req.NodeGroupId(), - true); - - std::vector index_names = - catalog_entry->dirty_schema_->IndexNames(); - for (const TableName &index_name : index_names) - { - // Create range table for each sk index - TableName index_range_table_name{ - index_name.StringView(), - TableType::RangePartition, - index_name.Engine()}; - - size_t tbl_name_hash = std::hash()( - index_name.StringView()); - init_partition_id = tbl_name_hash & 0xFFF; - range_init_vec.clear(); - range_init_vec.emplace_back( - TxKey(), init_partition_id, req.CommitTs()); - shard_->local_shards_.InitTableRanges( - index_range_table_name, - range_init_vec, - req.NodeGroupId(), - true); - } - shard_->InitTableStatistics( - catalog_entry->dirty_schema_.get(), cc_ng_id_); - } - else if (req.OpType() == OperationType::Update) - { - assert(catalog_entry->schema_->StatisticsObject()); - catalog_entry->dirty_schema_->BindStatistics( - catalog_entry->schema_->StatisticsObject()); - } + // Need to parse the string if not include table type in protobuf + TableType table_type = ::txlog::ToLocalType::ConvertCcTableType( + schema_op_msg.table_type()); + TableEngine table_engine = ::txlog::ToLocalType::ConvertTableEngine( + schema_op_msg.table_engine()); + std::string_view table_name_sv{schema_op_msg.table_name_str()}; + TableName table_name{table_name_sv, table_type, table_engine}; - // If recoverring from commit log, there is no dirty schema, - // skip updating schema_rec - if (catalog_entry->dirty_schema_ != nullptr) - { - schema_rec->Set(catalog_entry->dirty_schema_, - nullptr, - catalog_entry->dirty_schema_version_); - } - } - else + if (shard_->core_id_ == 0) + { + CatalogKey table_key(table_name); + Iterator it = Find(table_key); + // Find() returns End() when the key is absent (template_cc_map.h), + // and End() is not dereferenceable. During replay the catalog entry + // is normally absent -- that is the whole point of replaying it -- + // so the iterator must be range-checked before it->second. + CcEntry *cce = + it == End() ? nullptr : it->second; + if (cce != nullptr) { - assert(req.Payload() != nullptr); - schema_rec = static_cast(req.Payload()); + req.SetFinish(); + return true; } - break; - } - case PostWriteType::Commit: - { - // Schema operations always employ multi-stage commits. - assert(true); - break; - } - case PostWriteType::DowngradeLock: - { - // the request commits nothing and only downgrade the write - // locks to write intent. - return TemplateCcMap::Execute(req); - } - default: - break; } - assert(catalog_entry != nullptr); + std::pair &upsert_kv_err_code = + req.UpsertKvErrCode(); + bool need_to_upsert_kv_table = upsert_kv_err_code.first; - // When the request commits the schema operation, modifies the cc - // map(s) at this shard. - const TableSchema *old_schema = catalog_entry->schema_.get(); - const TableSchema *new_schema = catalog_entry->dirty_schema_.get(); - if (req.CommitType() == PostWriteType::PostCommit && - catalog_entry->dirty_schema_version_ > 0) + // 1. Replay the table catalog and table schema. + // The first shard is in charge of creating catalog_entry. + if (shard_->core_id_ == 0) { - if (req.OpType() == OperationType::TruncateTable) - { - assert(table_key->Name().Engine() == TableEngine::EloqKv); - // A remote tx is allowed to acquire write intents/locks and - // drop a table, even if the table's schema has not been - // initialized at this node. The earlier acquiring-write-intent - // request creates a schema cc entry in the catalog cc map and a - // node-level schema view. The version timestamp of the schema - // is 0, if the schema is uninitialized (null). Or, the current - // schema must not be null. - - assert(catalog_entry->schema_version_ == 0 || - old_schema != nullptr); - assert(new_schema->Version() == - catalog_entry->dirty_schema_version_); - - shard_->UpdateCcmSchema(table_key->Name(), - req.NodeGroupId(), - catalog_entry->dirty_schema_.get(), - catalog_entry->dirty_schema_version_); - - // Sync ddl op to standby nodes. - if (shard_->core_id_ == 0 && - !shard_->GetSubscribedStandbys().empty()) + if (schema_op_msg.stage() == + ::txlog::SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_PrepareSchema || + schema_op_msg.stage() == ::txlog::SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_PrepareData) + { + // If we are recovering from prepare log, we need to restore to + // the state right before commit log is flushed, so both current + // and dirty schema are needed. + auto [success, new_catalog_entry] = shard_->CreateReplayCatalog( + table_name, + req.NodeGroupId(), + schema_op_msg.old_catalog_blob(), + schema_op_msg.new_catalog_blob(), + schema_op_msg.catalog_ts(), + req.CommitTs()); + if (!success) { - auto forward_entry_ptr = - std::make_unique(); - StandbyForwardEntry *forward_entry = - forward_entry_ptr.get(); - auto *forward_req = &forward_entry->Request(); - forward_req->set_primary_leader_term(ng_term); - forward_req->set_tx_number(req.Txn()); - forward_req->set_table_name(table_name_.String()); - forward_req->set_table_type( - remote::ToRemoteType::ConvertTableType( - table_name_.Type())); - forward_req->set_table_engine( - remote::ToRemoteType::ConvertTableEngine( - table_name_.Engine())); - forward_req->set_key_shard_code(cc_ng_id_ << 10); - std::string key_str; - table_key->Serialize(key_str); - forward_req->set_key(std::move(key_str)); - std::string rec_str; - - assert(catalog_entry->schema_ != nullptr); - assert(catalog_entry->dirty_schema_ != nullptr); - assert(catalog_entry->dirty_schema_version_ > 0); - - CatalogRecord temp_rec; - temp_rec.SetSchemaImage( - catalog_entry->schema_->SchemaImage()); - temp_rec.SetDirtySchemaImage( - catalog_entry->dirty_schema_->SchemaImage()); - assert(!temp_rec.SchemaImage().empty()); - assert(!temp_rec.DirtySchemaImage().empty()); - - temp_rec.Serialize(rec_str); - forward_req->add_cmd_list(std::move(rec_str)); - - forward_req->set_commit_ts( - catalog_entry->dirty_schema_version_); - forward_req->set_schema_version( - catalog_entry->schema_version_); - shard_->ForwardStandbyMessage(forward_entry_ptr.release()); + // create fail, the catalog to be created is out of date + LOG(INFO) + << "create catalog fails, table name: " << table_name_sv + << ", catalog entry of the same or higher " + "version exists, stop replaying this schema op"; + req.SetFinish(); + return true; } + catalog_entry = new_catalog_entry; } - else if (req.OpType() == OperationType::DropTable) + else { - // A remote tx is allowed to acquire write intents/locks and - // drop a table, even if the table's schema has not been - // initialized at this node. The earlier acquiring-write-intent - // request creates a schema cc entry in the catalog cc map and a - // node-level schema view. The version timestamp of the schema - // is 0, if the schema is uninitialized (null). Or, the current - // schema must not be null. - assert(catalog_entry->schema_version_ == 0 || - old_schema != nullptr); + /* + If we are recovering as participant from commit stage, + we don't need to do kv_upsert_table_op_ so we can directly + recover to the state before commit log is cleaned, that is: - // This is a DROP TABLE statement. Drops the cc maps - // associated with the table in the final commit step. - shard_->DropCcm(table_key->Name(), req.NodeGroupId()); + 1. if dirty_schema_commit_ts>0, dirty schema is commited as + current schema and write lock has been released. - if (!table_key->Name().IsHashPartitioned()) - { - // Drop range table if exist - TableName range_table_name{table_key->Name().StringView(), - TableType::RangePartition, - table_key->Name().Engine()}; - shard_->DropCcm(range_table_name, req.NodeGroupId()); - } + 2. if dirty_schema_commit_ts=0, old schema is restored as + current schema and write lock has been released. - if (old_schema != nullptr) + P.S. If it is create table statement and + dirty_schema_commit_ts=0, the restored CatalogEntry has schema_ + set to nullptr and schema_version_ set to 1. + */ + + uint64_t dirty_schema_commit_ts = req.CommitTs(); + uint64_t old_schema_commit_ts = schema_op_msg.catalog_ts(); + // A rolled-back add index commits at a real timestamp: the + // table keeps its current schema, at that timestamp. + const bool rolled_back = schema_op_msg.rolled_back(); + const std::string &committed_blob = + rolled_back ? schema_op_msg.old_catalog_blob() + : schema_op_msg.new_catalog_blob(); + + // upsert kv store first, then create catalog entry + if (need_to_upsert_kv_table && dirty_schema_commit_ts > 0 && + !Sharder::Instance() + .GetDataStoreHandler() + ->IsSharedStorage()) { - std::vector index_names = - old_schema->IndexNames(); - for (const TableName &index_name : index_names) + // When not using shared storage, the coordinator and + // participants must upsert the KV table during recovery + // process. Otherwise, there may be inconsistency between + // in-memory schema and kv store schema. + + TableSchema::uptr schema_ptr = + Sharder::Instance() + .GetLocalCcShards() + ->CreateTableSchemaFromImage( + table_name, + schema_op_msg.old_catalog_blob(), + old_schema_commit_ts); + TableSchema::uptr dirty_schema_ptr = + Sharder::Instance() + .GetLocalCcShards() + ->CreateTableSchemaFromImage( + table_name, + committed_blob, + dirty_schema_commit_ts); + + OperationType op_type = static_cast( + schema_op_msg.table_op().op_type()); + if (op_type == OperationType::CreateTable || rolled_back) { - shard_->DropCcm(index_name, req.NodeGroupId()); - // Drop range table if exist - TableName index_range_table_name{ - index_name.StringView(), - TableType::RangePartition, - index_name.Engine()}; - shard_->DropCcm(index_range_table_name, - req.NodeGroupId()); + // A create's kv tables exist since before its commit + // log (UpsertTableOp::CreateKvTables), with their + // ranges at the prepare-stage version; only the catalog + // row may be missing. Op type Update is the store's + // catalog-only write. A rolled-back add index likewise + // writes only its catalog row, the current schema; the + // recovered coordinator drops the index's kv tables. + op_type = OperationType::Update; } + + upsert_kv_err_code = {false, CcErrorCode::NO_ERROR}; + shard_->local_shards_.store_hd_->UpsertTable( + schema_ptr.get(), + dirty_schema_ptr.get(), + op_type, + req.CommitTs(), + cc_ng_id_, + ng_term, + nullptr, + nullptr, + &req, + shard_, + &upsert_kv_err_code.second); + + return false; + } + else if (!need_to_upsert_kv_table && + upsert_kv_err_code.second != CcErrorCode::NO_ERROR) + { + req.AbortCcRequest(upsert_kv_err_code.second); + return true; } - } - else if (req.OpType() == OperationType::CreateTable) - { - assert(catalog_entry->dirty_schema_version_ > 0 && - new_schema != nullptr); - // This is a CREATE TABLE statement. Creates the cc maps - // associated with the table in the final commit step. - shard_->CreateOrUpdatePkCcMap(table_key->Name(), - new_schema, - req.NodeGroupId(), - true, - true); + auto [success, new_catalog_entry] = shard_->CreateCatalog( + table_name, + req.NodeGroupId(), + dirty_schema_commit_ts > 0 + ? committed_blob + : schema_op_msg.old_catalog_blob(), + dirty_schema_commit_ts > 0 ? dirty_schema_commit_ts + : old_schema_commit_ts); - std::vector index_names = new_schema->IndexNames(); - for (const TableName &index_name : index_names) + assert(new_catalog_entry != nullptr); + if (!success) { - shard_->CreateOrUpdateSkCcMap( - index_name, new_schema, req.NodeGroupId()); + // create fail, the catalog to be created is out of date + LOG(INFO) + << "create catalog entry fails, table name: " + << table_name_sv + << ", catalog entry of the same or higher version " + "exists, stop replaying this schema op"; + req.SetFinish(); + return true; } + catalog_entry = new_catalog_entry; } - // Alter Table - else - { - assert(old_schema != nullptr && new_schema != nullptr); - // Update pk cc map using new schema. - shard_->CreateOrUpdatePkCcMap( - table_key->Name(), new_schema, req.NodeGroupId(), false); - if (!table_key->Name().IsHashPartitioned()) + if (catalog_entry->schema_) + { + if (Sharder::Instance().GetLocalCcShards()->store_hd_ != + nullptr && + !table_name.IsHashPartitioned() && + !shard_->LoadRangesAndStatisticsNx( + catalog_entry->schema_.get(), + req.NodeGroupId(), + ng_term, + &req)) { - // Update pk range table if exist. - TableName base_range_table_name{ - table_key->Name().StringView(), - TableType::RangePartition, - table_key->Name().Engine()}; - auto ranges = shard_->GetTableRangesForATable( - base_range_table_name, req.NodeGroupId()); - if (ranges != nullptr) - { - shard_->CreateOrUpdateRangeCcMap( - base_range_table_name, - new_schema, - req.NodeGroupId(), - catalog_entry->dirty_schema_version_, - false); - } + return false; } - if (req.OpType() == OperationType::AddIndex || - req.OpType() == OperationType::DropIndex || - req.OpType() == OperationType::Update) + // Load range and stats for the new added indexes. + if (catalog_entry->dirty_schema_) { std::vector new_index_names = - new_schema->IndexNames(); + catalog_entry->dirty_schema_->IndexNames(); std::vector old_index_names = - old_schema->IndexNames(); - - for (const TableName &old_index_name : old_index_names) + catalog_entry->schema_->IndexNames(); + for (const TableName &new_index_name : new_index_names) { - if (std::find(new_index_names.begin(), - new_index_names.end(), - old_index_name) != new_index_names.end()) + if (std::find(old_index_names.begin(), + old_index_names.end(), + new_index_name) == old_index_names.end()) { - // Update current sk cc map using new schema. - shard_->CreateOrUpdateSkCcMap(old_index_name, - new_schema, - req.NodeGroupId(), - false); - // Update current sk range table if exist. - TableName index_range_table_name{ - old_index_name.StringView(), + TableName index_range_name{ + new_index_name.StringView(), TableType::RangePartition, - old_index_name.Engine()}; + new_index_name.Engine()}; auto ranges = shard_->GetTableRangesForATable( - index_range_table_name, req.NodeGroupId()); - if (ranges != nullptr) + index_range_name, req.NodeGroupId()); + if (ranges == nullptr) { - shard_->CreateOrUpdateRangeCcMap( - index_range_table_name, - new_schema, - req.NodeGroupId(), - catalog_entry->dirty_schema_version_, - false); + shard_->FetchTableRanges(index_range_name, + &req, + req.NodeGroupId(), + ng_term); + return false; } - } - else - { - assert(req.OpType() == OperationType::DropIndex); - // Remove sk cc map for dropped index. - shard_->DropCcm(old_index_name, req.NodeGroupId()); - - // range table operation. - // Drop range table if exist - TableName old_index_range_table_name{ - old_index_name.StringView(), - TableType::RangePartition, - old_index_name.Engine()}; - shard_->DropCcm(old_index_range_table_name, - req.NodeGroupId()); + + catalog_entry->dirty_schema_->StatisticsObject() + ->CreateIndex( + new_index_name, + catalog_entry->dirty_schema_ + ->IndexKeySchema(new_index_name), + req.NodeGroupId()); } } - } // End of alter table index + } } - - UpdateTableLocks(table_key->Name().StringView(), - catalog_entry->dirty_schema_version_, - cce_ptr->GetKeyLock()); } - else if ((req.CommitType() == PostWriteType::PrepareCommit || - req.CommitType() == PostWriteType::UpdateDirty) && - catalog_entry->dirty_schema_version_ > 0) + else + { + // other cores + catalog_entry = shard_->GetCatalog(table_name, req.NodeGroupId()); + assert(catalog_entry != nullptr); + } + + // 2. Replay the table ccmap. + if (schema_op_msg.stage() == ::txlog::SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_PrepareSchema || + schema_op_msg.stage() == ::txlog::SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_PrepareData) { - // Prepare commit. For certain schema operations, e.g., create - // secondary index, the cc map is modified in the prepare commit - // step. - // ALTER TABLE statement (include CREATE/DROP INDEX) - if (req.OpType() == OperationType::AddIndex) + const TableSchema *old_schema = catalog_entry->schema_.get(); + const TableSchema *new_schema = catalog_entry->dirty_schema_.get(); + if (old_schema != nullptr && new_schema != nullptr) { - std::vector new_index_names = - new_schema->IndexNames(); + // Alter table index operation. + // Pk table ccmap using old schema. + shard_->CreateOrUpdatePkCcMap( + table_name, old_schema, req.NodeGroupId()); + + if (!table_name.IsHashPartitioned()) + { + // Pk range table ccmap + const TableName base_range_name{table_name.StringView(), + TableType::RangePartition, + table_name.Engine()}; + shard_->CreateOrUpdateRangeCcMap( + base_range_name, + old_schema, + req.NodeGroupId(), + catalog_entry->schema_version_); + } + + // Old sk table ccmap using old schema. std::vector old_index_names = old_schema->IndexNames(); - bool found = false; - for (const TableName &new_index_name : new_index_names) + for (const auto &old_index_name : old_index_names) { - found = false; - for (const auto &old_index_name : old_index_names) - { - if (!new_index_name.String().compare( - old_index_name.String())) - { - found = true; - break; - } - } - if (!found) + shard_->CreateOrUpdateSkCcMap( + old_index_name, old_schema, req.NodeGroupId()); + // old sk range table ccmap + const TableName old_index_range_name{ + old_index_name.StringView(), + TableType::RangePartition, + old_index_name.Engine()}; + shard_->CreateOrUpdateRangeCcMap( + old_index_range_name, + old_schema, + req.NodeGroupId(), + catalog_entry->schema_version_); + } + + // New sk table ccmap using new schema + std::vector new_index_names = + new_schema->IndexNames(); + for (const auto &new_index_name : new_index_names) + { + if (std::find(old_index_names.cbegin(), + old_index_names.cend(), + new_index_name) == old_index_names.cend()) { - // In this step, just create cc map for new sk. - // We will update current sk ccmap in PostCommit. - bool is_create = - req.CommitType() == PostWriteType::PrepareCommit; - shard_->CreateOrUpdateSkCcMap(new_index_name, - new_schema, - req.NodeGroupId(), - is_create); - - // New sk range cc map should use the dirty schema + shard_->CreateOrUpdateSkCcMap( + new_index_name, new_schema, req.NodeGroupId()); + + // New sk range cc maps should use the dirty schema const TableName new_index_range_name{ new_index_name.StringView(), TableType::RangePartition, @@ -1068,153 +772,307 @@ class CatalogCcMap new_index_range_name, new_schema, req.NodeGroupId(), - catalog_entry->dirty_schema_version_, - is_create); + catalog_entry->dirty_schema_version_); } } } } - if (req.CommitType() == PostWriteType::PostCommit && - shard_->core_id_ == shard_->core_cnt_ - 1 && - catalog_entry->dirty_schema_version_ > 0) + CatalogKey table_key(table_name); + Iterator it = FindEmplace(table_key); + CcEntry *cce = it->second; + CcPage *ccp = it.GetPage(); + + if (cce == nullptr) + { + shard_->Enqueue(shard_->LocalCoreId(), &req); + return false; + } + + // 3. Replay the table write intent/lock. + LockType lock_type = LockType::NoLock; + TableName base_table_name(table_name.GetBaseTableNameSV(), + TableType::Primary, + table_name.Engine()); + OperationType op_type = + static_cast(schema_op_msg.table_op().op_type()); + switch (schema_op_msg.stage()) + { + case ::txlog::SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_PrepareSchema: { - // If this is a drop table req, drop the range table also - // Drop table range before drop catalog - if (req.OpType() == OperationType::DropTable) + // The lock restored is the one the coordinating tx may rely on + // (docs/range-split-schema-op-recovery.md). The holder's term is + // 0: after the prepare log the lock and the op are recovered + // whatever fails, so no term marks the holder as failed. + // + // The coordinator's node group restores a write intent: the op, + // recovered here, redoes every step after its prepare log, its + // write locks among them, which drain the DML admitted meanwhile. + // + // A participant cannot tell how far the live coordinator got: it + // may already hold this node group's write lock, the last step + // before its commit log, and it does not ask again. So a + // participant restores the write lock, keeping the commit log's + // barrier, unless the coordinator cannot hold it here: + // - an add index holds only an intent between its prepare log and + // its prepare data log, the stage the record names; + // - a split of the table that this node group coordinates holds + // the table's catalog read lock here, from before its first log + // to after its clean log, and the read lock excludes every write + // lock. The coordinator's write lock request, lost with the old + // leader, fails its liveness check (NG_TERM_CHANGED) and is sent + // again; granted here, it drains the DML admitted meanwhile. + const uint32_t tx_node_id = (req.Txn() >> 32L) >> 10; + const bool is_coordinator = + tx_node_id == req.NodeGroupId() && + Sharder::Instance().CandidateLeaderTerm(tx_node_id) >= 0; + if (is_coordinator || op_type == OperationType::AddIndex || + req.RangeSplitting(base_table_name)) { - shard_->CleanTableStatistics(table_key->Name(), cc_ng_id_); - if (!table_key->Name().IsHashPartitioned()) - { - TableName range_table_name{table_key->Name().StringView(), - TableType::RangePartition, - table_key->Name().Engine()}; - shard_->CleanTableRange(range_table_name, - req.NodeGroupId()); - if (old_schema != nullptr) - { - std::vector index_names = - old_schema->IndexNames(); - for (const TableName &index_name : index_names) - { - // Drop range table if exist - TableName index_range_table_name{ - index_name.StringView(), - TableType::RangePartition, - index_name.Engine()}; - shard_->CleanTableRange(index_range_table_name, - req.NodeGroupId()); - } - } - } + lock_type = LockType::WriteIntent; } - else if (req.OpType() == OperationType::AddIndex || - req.OpType() == OperationType::DropIndex) + else { - std::vector new_index_names = - new_schema->IndexNames(); - std::vector old_index_names = - old_schema->IndexNames(); - for (const TableName &old_index_name : old_index_names) - { - // Drop old index range table if not exist any more - if (std::find(new_index_names.begin(), - new_index_names.end(), - old_index_name) == new_index_names.end()) - { - TableName old_index_range_table_name{ - old_index_name.StringView(), - TableType::RangePartition, - old_index_name.Engine()}; - shard_->CleanTableRange(old_index_range_table_name, - req.NodeGroupId()); - Statistics *statistics = - old_schema->StatisticsObject().get(); - statistics->DropIndex(old_index_name); - } - } + lock_type = LockType::WriteLock; } - catalog_entry->CommitDirtySchema(); + break; } - - return TemplateCcMap::Execute(req); - } - - bool Execute(ReadCc &req) override - { - TX_TRACE_ACTION_WITH_CONTEXT( - (txservice::CcMap *) this, - &req, - [&req]() -> std::string + case ::txlog::SchemaOpMessage_Stage::SchemaOpMessage_Stage_PrepareData: + { + // An add index writes this record after each batch of its build, + // under intents, and after the last one (last key +inf, kept by + // the log state) takes its commit-stage write locks. The + // coordinator's node group restores a write intent: the recovered + // op resumes from the record and takes those locks itself. A + // participant past the last record cannot tell whether the live + // coordinator already holds its write lock, so it restores the + // write lock, unless a split of the table that this node group + // coordinates is in flight, whose catalog read lock excludes that + // write lock (as for PrepareSchema above). Before the last record + // the build still runs under intents, so an intent. + const uint32_t tx_node_id = (req.Txn() >> 32L) >> 10; + const bool is_coordinator = + tx_node_id == req.NodeGroupId() && + Sharder::Instance().CandidateLeaderTerm(tx_node_id) >= 0; + const bool build_finished = + schema_op_msg.last_key_type() == + ::txlog::SchemaOpMessage_LastKeyType_PosInfKey; + if (!is_coordinator && build_finished && + !req.RangeSplitting(base_table_name)) { - return std::string("\"cc_map_type\":\"template_cc_map\"") - .append(",\"tx_number\":") - .append(std::to_string(req.Txn())) - .append(",\"term\":") - .append(std::to_string(req.TxTerm())); - }); - TX_TRACE_DUMP(&req); + lock_type = LockType::WriteLock; + } + else + { + lock_type = LockType::WriteIntent; + } + break; + } + case ::txlog::SchemaOpMessage_Stage::SchemaOpMessage_Stage_CommitSchema: + { + // Only the coordinator's node group restores a lock at the commit + // stage: a write intent on the schema, under which the recovered + // op redoes its steps after the commit log, data store writes + // among them, and which the op's PostCommit or Rollback releases. + // The intent keeps a newer DDL on the table out until then, so no + // late write of the recovered op lands after one. It is not a + // write lock: the entry already holds the committed schema (step + // 1), so the op's effect is in place for everyone else, and a read + // of the entry must not wait for the op. The op's PostWriteAll + // needs the node group's leader term, which comes only once the + // log replay has finished, and that replay reads this entry to + // replay the table's data (CcShard::InitCcm, which fails on a + // write lock). The condition is the one below that starts the + // recovered op. A participant installs the committed schema with + // no lock, and the op's later post writes, finding no lock of + // theirs, finish at once (Execute(PostWriteAllCc&)). + const uint32_t tx_node_id = (req.Txn() >> 32L) >> 10; + if (tx_node_id == req.NodeGroupId() && + Sharder::Instance().CandidateLeaderTerm(tx_node_id) >= 0) + { + lock_type = LockType::WriteIntent; + } + break; + } + default: + break; + } - assert(req.IsLocal()); + if (lock_type == LockType::WriteIntent) + { + auto lock_pair = AcquireCceKeyLock(cce, + cce->CommitTs(), + ccp, + cce->PayloadStatus(), + &req, + req.NodeGroupId(), + ng_term, + 0, + CcOperation::ReadForWrite, + IsolationLevel::RepeatableRead, + CcProtocol::OCC, + 0, + false); + assert(lock_pair.first == LockType::WriteIntent && + lock_pair.second == CcErrorCode::NO_ERROR); + // This silences the -Wunused-but-set-variable warning without any + // runtime overhead. + (void) lock_pair; + } + else if (lock_type == LockType::WriteLock) + { + auto lock_pair = AcquireCceKeyLock(cce, + cce->CommitTs(), + ccp, + cce->PayloadStatus(), + &req, + req.NodeGroupId(), + ng_term, + 0, + CcOperation::Write, + IsolationLevel::RepeatableRead, + CcProtocol::Locking, + 0, + false); + // When a cc node recovers, no one should be holding read locks. + // So, the acquire operation should always succeed. + assert(lock_pair.first == LockType::WriteLock && + lock_pair.second == CcErrorCode::NO_ERROR); + // This silences the -Wunused-but-set-variable warning without any + // runtime overhead. + (void) lock_pair; + } - uint32_t ng_id = req.NodeGroupId(); - int64_t ng_term = Sharder::Instance().LeaderTerm(ng_id); - ng_term = std::max(ng_term, Sharder::Instance().StandbyNodeTerm()); + if (cce->payload_.cur_payload_ == nullptr) + { + cce->payload_.PassInCurrentPayload( + std::make_unique()); + } + cce->payload_.cur_payload_->Set(catalog_entry->schema_, + catalog_entry->dirty_schema_, + catalog_entry->schema_version_); + if (catalog_entry->schema_ == nullptr && + catalog_entry->dirty_schema_ != nullptr) + { + assert(op_type == OperationType::CreateTable); + cce->SetCommitTsPayloadStatus(1, RecordStatus::Deleted); + } + else if ((op_type == OperationType::AddIndex || + op_type == OperationType::DropIndex) && + (schema_op_msg.stage() == + ::txlog::SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_PrepareSchema || + schema_op_msg.stage() == + ::txlog::SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_PrepareData)) + { + // Replaying a prepare-stage log restores the state after + // PrepareCommit, at which an index op published its timestamp on + // the entry under the prepare-stage write lock (TemplateCcMap, + // Execute(PostWriteAllCc&)). The recovered lock may be only an + // intent, but the entry must carry the same timestamp, so that a tx + // reading the dirty schema here still commits above the index op. + // The record set above is complete, so the entry is loaded now; an + // entry left Unknown must keep commit ts 0 (see TemplateCcMap). The + // commit ts only moves forward, so a repeated replay changes + // nothing. + cce->SetCommitTsPayloadStatus(catalog_entry->dirty_schema_version_, + RecordStatus::Normal); + } + else if (schema_op_msg.stage() == + ::txlog::SchemaOpMessage_Stage:: + SchemaOpMessage_Stage_CommitSchema && + catalog_entry->schema_version_ > 0) + { + // Replaying a commit log installs the committed schema, or the + // current one for an op the log rolled back, at its version and + // with no dirty schema. The record set above is complete, so the + // entry is loaded now, as a catalog read would load it. Left + // Unknown, it would take the commit ts of a Rollback the recovered + // op sends above 0 while keeping its status, and the first read + // could not load it at that same ts: the commit ts only moves + // forward. A PostCommit or Rollback at the same ts changes nothing. + cce->SetCommitTsPayloadStatus(catalog_entry->schema_version_, + catalog_entry->schema_ != nullptr + ? RecordStatus::Normal + : RecordStatus::Deleted); + } - if (req.IsInRecovering()) + if (shard_->core_id_ < shard_->core_cnt_ - 1) { - ng_term = ng_term > 0 - ? ng_term - : Sharder::Instance().CandidateLeaderTerm(ng_id); + req.ResetCcm(); + MoveRequest(&req, shard_->core_id_ + 1); } + else + { + uint32_t tx_node_id = (req.Txn() >> 32L) >> 10; + bool is_coordinator = tx_node_id == req.NodeGroupId(); + int64_t tx_candidate_term = + Sharder::Instance().CandidateLeaderTerm(tx_node_id); - CODE_FAULT_INJECTOR("term_CatalogCcMap_Execute_ReadCc", { - LOG(INFO) << "FaultInject term_CatalogCcMap_Execute_ReadCc"; - ng_term = -1; - FaultInject::Instance().InjectFault( - "term_CatalogCcMap_Execute_ReadCc", "remove"); - }); + if (is_coordinator && tx_candidate_term >= 0) + { + // If the coordinating tx is bound to the recoverying cc + // node, resumes the tx. This will spawn + // a worker thread that will restore the transaction. It will + // call SetFinish() after acqruing all needed locks. + shard_->local_shards_.CreateSchemaRecoveryTx( + req, schema_op_msg, tx_candidate_term); + } + else + { + req.SetFinish(); + } + } + + return true; + } + + bool Execute(BroadcastStatisticsCc &req) override + { + CcHandlerResult *hd_res = req.Result(); + int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); if (ng_term < 0) { - req.Result()->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); + hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); return true; } - const CatalogKey *table_key = - static_cast(req.Key()); - bool emplace = false; - Iterator it = FindEmplace(*table_key, &emplace, true, false); - CcEntry *cce = it->second; - if (cce->PayloadStatus() == RecordStatus::Unknown) - { - const CatalogEntry *catalog_entry = - shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); + const TableName &table_name = *req.SamplingTableName(); - // If the read toward the catalog cc entry acquires the read - // lock but the cc entry does not contain the value, checks if - // the catalog has been constructed at this node. If so, turns - // this request into a read-outside request that installs the - // value in the cc entry. - if (catalog_entry != nullptr && catalog_entry->schema_version_ > 0) + DLOG(INFO) << "Receive table statistics " << table_name.StringView() + << ". From ng #" << req.SamplePool()->ng_id() << ", to ng #" + << req.NodeGroupId() + << ". ng_records: " << req.SamplePool()->records() + << ", ng_samples: " << req.SamplePool()->samples_size(); + + TableName base_table_name(table_name.GetBaseTableNameSV(), + TableType::Primary, + table_name.Engine()); + CatalogKey table_key(base_table_name); + Iterator it = FindEmplace(table_key); + CcEntry *cce = it->second; + if (cce->PayloadStatus() == RecordStatus::Unknown) + { + const CatalogEntry *catalog_entry = + shard_->GetCatalog(base_table_name, req.NodeGroupId()); + if (catalog_entry != nullptr) { if (catalog_entry->schema_ != nullptr) { + // Initialize table statistics before create ccmap. + if (Sharder::Instance().GetLocalCcShards()->store_hd_ != + nullptr && + !table_name.IsHashPartitioned() && + !shard_->LoadRangesAndStatisticsNx( + catalog_entry->schema_.get(), + req.NodeGroupId(), + ng_term, + &req)) { -#ifdef STATISTICS - // Initialize table statistics before create ccmap. - if (Sharder::Instance().GetLocalCcShards()->store_hd_ != - nullptr && - !table_key->Name().IsHashPartitioned() && - !shard_->LoadRangesAndStatisticsNx( - catalog_entry->schema_.get(), - req.NodeGroupId(), - ng_term, - &req)) - { - return false; - } -#endif + return false; // Loading... } // upload catalog record @@ -1235,1341 +1093,1696 @@ class CatalogCcMap } else { - shard_->FetchCatalog( - table_key->Name(), req.NodeGroupId(), ng_term, &req); - return false; + if (Statistics::LeaderNodeGroup(base_table_name) != + req.NodeGroupId()) + { + hd_res->SetFinished(); + return true; + } + else + { + shard_->FetchCatalog( + base_table_name, req.NodeGroupId(), ng_term, &req); + return false; + } } } - // Increase ref cnt so that when setFinish is called in - // TemplateCcMap::Execute, the waiters cannot be woken up. We still need - // to access sch_rec which is owned by the caller. - CcHandlerResult *hd_res = req.Result(); - hd_res->SetRefCnt(2); - bool success = TemplateCcMap::Execute(req); - - ReadKeyResult &result = req.Result()->Value(); - CatalogRecord *sch_rec = static_cast(req.Record()); - - // Returns the reader-writer control block to runtime when there is no - // writer and no pending dirty schema. The control block provides a fast - // path for runtime to keep a cache of the schema and to coordinate - // modifications of the schema. - if (success && shard_->IsNative(ng_id) && - result.rec_status_ == RecordStatus::Normal && - sch_rec->DirtySchema() == nullptr) + if (cce->PayloadStatus() == RecordStatus::Normal) { - std::shared_ptr> sch_cntl = - shard_->FindEmplaceSchemaCntl(table_key->Name()); - if (sch_cntl->HasNoWriter()) + const StatisticsEntry *statistics_entry = + shard_->GetTableStatistics(base_table_name, req.NodeGroupId()); + assert(statistics_entry && statistics_entry->statistics_); + const auto table_schema = [&req, cce]() -> const TableSchema * { - if (sch_cntl->GetObjectPtr() == nullptr) + if (cce->payload_.cur_payload_->Schema() && + cce->payload_.cur_payload_->Schema()->Version() == + req.SchemaVersion()) { - sch_cntl->SetObject(sch_rec->CopySchema()); + return cce->payload_.cur_payload_->Schema(); } - sch_rec->SetSchemaCntl(std::move(sch_cntl)); + else if (cce->payload_.cur_payload_->DirtySchema() && + cce->payload_.cur_payload_->DirtySchema()->Version() == + req.SchemaVersion()) + { + return cce->payload_.cur_payload_->DirtySchema(); + } + else + { + return nullptr; + } + }(); + + if (table_schema) + { + statistics_entry->statistics_->OnRemoteStatisticsMessage( + table_name, table_schema, *req.SamplePool()); } } - hd_res->SetFinished(); - return success; + hd_res->SetFinished(); + return true; } - bool Execute(ReplayLogCc &req) override + bool Execute(KeyObjectStandbyForwardCc &req) override { - TX_TRACE_ACTION_WITH_CONTEXT( - (txservice::CcMap *) this, - &req, - [&req]() -> std::string - { - return std::string("\"cc_map_type\":\"template_cc_map\"") - .append(",\"tx_number\":") - .append(std::to_string(req.Txn())) - .append(",\"term\":") - .append("0"); - }); - TX_TRACE_DUMP(&req); - int64_t ng_term = - Sharder::Instance().CandidateLeaderTerm(req.NodeGroupId()); - if (ng_term < 0) - { - LOG(INFO) << "ReplayLogCc, node_group(#" << req.NodeGroupId() - << ") term < 0, tx:" << req.Txn(); - req.AbortCcRequest(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; - } - - ::txlog::SchemaOpMessage schema_op_msg; - const std::string_view &content = req.LogContentView(); - schema_op_msg.ParseFromArray(content.data(), content.length()); - - const CatalogEntry *catalog_entry = nullptr; + uint64_t commit_ts = req.CommitTs(); - // Need to parse the string if not include table type in protobuf - TableType table_type = ::txlog::ToLocalType::ConvertCcTableType( - schema_op_msg.table_type()); - TableEngine table_engine = ::txlog::ToLocalType::ConvertTableEngine( - schema_op_msg.table_engine()); - std::string_view table_name_sv{schema_op_msg.table_name_str()}; - TableName table_name{table_name_sv, table_type, table_engine}; + const CatalogKey *table_key = nullptr; + const std::string *key_str = req.KeyImage(); + assert(key_str != nullptr); + std::unique_ptr decoded_key = + std::make_unique(); + size_t offset = 0; + decoded_key->Deserialize(key_str->data(), offset, KeySchema()); + table_key = decoded_key.get(); - if (shard_->core_id_ == 0) + CcEntry *cce; + CcPage *ccp; + if (req.CcePtr()) { - CatalogKey table_key(table_name); - Iterator it = Find(table_key); - // Find() returns End() when the key is absent (template_cc_map.h), - // and End() is not dereferenceable. During replay the catalog entry - // is normally absent -- that is the whole point of replaying it -- - // so the iterator must be range-checked before it->second. - CcEntry *cce = - it == End() ? nullptr : it->second; - if (cce != nullptr) - { - req.SetFinish(); - return true; - } + cce = + static_cast *>( + req.CcePtr()); + ccp = static_cast *>( + cce->GetCcPage()); + } + else + { + Iterator it = FindEmplace(*table_key); + cce = it->second; + ccp = it.GetPage(); } - std::pair &upsert_kv_err_code = - req.UpsertKvErrCode(); - bool need_to_upsert_kv_table = upsert_kv_err_code.first; - - // 1. Replay the table catalog and table schema. - // The first shard is in charge of creating catalog_entry. - if (shard_->core_id_ == 0) + // If the current node has not initialized the catalog entry, we fetch + // the newest catalog from the primary node. + if (cce->PayloadStatus() == RecordStatus::Unknown) { - if (schema_op_msg.stage() == - ::txlog::SchemaOpMessage_Stage:: - SchemaOpMessage_Stage_PrepareSchema || - schema_op_msg.stage() == ::txlog::SchemaOpMessage_Stage:: - SchemaOpMessage_Stage_PrepareData) + const CatalogEntry *catalog_entry = + shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); + if (catalog_entry != nullptr && catalog_entry->schema_version_ > 0) { - // If we are recovering from prepare log, we need to restore to - // the state right before commit log is flushed, so both current - // and dirty schema are needed. - auto [success, new_catalog_entry] = shard_->CreateReplayCatalog( - table_name, - req.NodeGroupId(), - schema_op_msg.old_catalog_blob(), - schema_op_msg.new_catalog_blob(), - schema_op_msg.catalog_ts(), - req.CommitTs()); - if (!success) + if (catalog_entry->schema_ != nullptr) { - // create fail, the catalog to be created is out of date - LOG(INFO) - << "create catalog fails, table name: " << table_name_sv - << ", catalog entry of the same or higher " - "version exists, stop replaying this schema op"; - req.SetFinish(); - return true; + // upload catalog record + cce->payload_.PassInCurrentPayload( + std::make_unique()); + cce->payload_.cur_payload_->Set( + catalog_entry->schema_, + catalog_entry->dirty_schema_, + catalog_entry->schema_version_); + // update commit ts + cce->SetCommitTsPayloadStatus( + catalog_entry->schema_version_, RecordStatus::Normal); + } + else + { + cce->SetCommitTsPayloadStatus( + catalog_entry->schema_version_, RecordStatus::Deleted); } - catalog_entry = new_catalog_entry; } else { - /* - If we are recovering as participant from commit stage, - we don't need to do kv_upsert_table_op_ so we can directly - recover to the state before commit log is cleaned, that is: - - 1. if dirty_schema_commit_ts>0, dirty schema is commited as - current schema and write lock has been released. + // fetch catalog from primary node + shard_->FetchCatalog(table_key->Name(), + req.NodeGroupId(), + req.StandbyNodeTerm(), + &req); + return false; + } + } - 2. if dirty_schema_commit_ts=0, old schema is restored as - current schema and write lock has been released. - - P.S. If it is create table statement and - dirty_schema_commit_ts=0, the restored CatalogEntry has schema_ - set to nullptr and schema_version_ set to 1. - */ - - uint64_t dirty_schema_commit_ts = req.CommitTs(); - uint64_t old_schema_commit_ts = schema_op_msg.catalog_ts(); - - // upsert kv store first, then create catalog entry - if (need_to_upsert_kv_table && dirty_schema_commit_ts > 0 && - !Sharder::Instance() - .GetDataStoreHandler() - ->IsSharedStorage()) - { - // When not using shared storage, the coordinator and - // participants must upsert the KV table during recovery - // process. Otherwise, there may be inconsistency between - // in-memory schema and kv store schema. - - TableSchema::uptr schema_ptr = - Sharder::Instance() - .GetLocalCcShards() - ->CreateTableSchemaFromImage( - table_name, - schema_op_msg.old_catalog_blob(), - old_schema_commit_ts); - TableSchema::uptr dirty_schema_ptr = - Sharder::Instance() - .GetLocalCcShards() - ->CreateTableSchemaFromImage( - table_name, - schema_op_msg.new_catalog_blob(), - dirty_schema_commit_ts); + if (commit_ts < cce->CommitTs()) + { + // discard outdate request + if (shard_->core_id_ + 1 == shard_->core_cnt_) + { + return req.SetFinish(*shard_); + } + else + { + MoveRequest(&req, shard_->core_id_ + 1); + return false; + } + } - OperationType op_type = static_cast( - schema_op_msg.table_op().op_type()); + // The catalog entry might be directly fetched from the + // primary node, so we still need to execute UpsertTable in the store + // handler. Otherwise, the kv storage will not have the latest kv table. + bool same_commit_ts = (commit_ts == cce->CommitTs()); - upsert_kv_err_code = {false, CcErrorCode::NO_ERROR}; - shard_->local_shards_.store_hd_->UpsertTable( - schema_ptr.get(), - dirty_schema_ptr.get(), - op_type, - req.CommitTs(), - cc_ng_id_, - ng_term, - nullptr, - nullptr, - &req, - shard_, - &upsert_kv_err_code.second); + LockType acquired_lock = LockType::NoLock; + CcErrorCode err_code = CcErrorCode::NO_ERROR; - return false; - } - else if (!need_to_upsert_kv_table && - upsert_kv_err_code.second != CcErrorCode::NO_ERROR) - { - req.AbortCcRequest(upsert_kv_err_code.second); - return true; - } + CODE_FAULT_INJECTOR("standby_forward_ddl_trigger_start_following", { + // only trigger once + txservice::FaultInject::Instance().InjectFault( + "standby_forward_ddl_trigger_start_following", "remove"); + DLOG(INFO) + << "fault inject: standby_forward_ddl_trigger_start_following"; + NodeGroupId native_ng = Sharder::Instance().NativeNodeGroup(); + int64_t cur_prim_term = Sharder::Instance().PrimaryNodeTerm(); + assert(cur_prim_term > 0); + Sharder::Instance().OnStartFollowing( + native_ng, + cur_prim_term, + Sharder::Instance().LeaderNodeId(native_ng), + true); - auto [success, new_catalog_entry] = shard_->CreateCatalog( - table_name, - req.NodeGroupId(), - dirty_schema_commit_ts > 0 - ? schema_op_msg.new_catalog_blob() - : schema_op_msg.old_catalog_blob(), - dirty_schema_commit_ts > 0 ? dirty_schema_commit_ts - : old_schema_commit_ts); + return req.SetFinish(*shard_); + }); - assert(new_catalog_entry != nullptr); - if (!success) - { - // create fail, the catalog to be created is out of date - LOG(INFO) - << "create catalog entry fails, table name: " - << table_name_sv - << ", catalog entry of the same or higher version " - "exists, stop replaying this schema op"; - req.SetFinish(); - return true; - } - catalog_entry = new_catalog_entry; + if (req.GetDDLPhase() == + KeyObjectStandbyForwardCc::DDLPhase::AcquirePhase) + { + // acquire write lock on catalog rec + if (req.CcePtr()) + { + // resumed req. + std::tie(acquired_lock, err_code) = LockHandleForResumedRequest( + cce, + cce->CommitTs(), + cce->PayloadStatus(), + &req, + Sharder::Instance().NativeNodeGroup(), + req.StandbyNodeTerm(), + req.TxTerm(), + CcOperation::Write, + IsolationLevel::RepeatableRead, + CcProtocol::Locking, + 0, + false); + assert(acquired_lock == LockType::WriteLock && + err_code == CcErrorCode::NO_ERROR); + req.SetCcePtr(nullptr); } - - if (catalog_entry->schema_) + else { - if (Sharder::Instance().GetLocalCcShards()->store_hd_ != - nullptr && - !table_name.IsHashPartitioned() && - !shard_->LoadRangesAndStatisticsNx( - catalog_entry->schema_.get(), - req.NodeGroupId(), - ng_term, - &req)) + std::tie(acquired_lock, err_code) = + AcquireCceKeyLock(cce, + cce->CommitTs(), + ccp, + cce->PayloadStatus(), + &req, + req.NodeGroupId(), + req.StandbyNodeTerm(), + req.TxTerm(), + CcOperation::Write, + IsolationLevel::RepeatableRead, + CcProtocol::Locking, + 0, + false); + } + switch (err_code) + { + case CcErrorCode::NO_ERROR: + { + if (shard_->core_id_ != shard_->core_cnt_ - 1) { + MoveRequest(&req, shard_->core_id_ + 1); return false; } - - // Load range and stats for the new added indexes. - if (catalog_entry->dirty_schema_) - { - std::vector new_index_names = - catalog_entry->dirty_schema_->IndexNames(); - std::vector old_index_names = - catalog_entry->schema_->IndexNames(); - for (const TableName &new_index_name : new_index_names) - { - if (std::find(old_index_names.begin(), - old_index_names.end(), - new_index_name) == old_index_names.end()) - { - TableName index_range_name{ - new_index_name.StringView(), - TableType::RangePartition, - new_index_name.Engine()}; - auto ranges = shard_->GetTableRangesForATable( - index_range_name, req.NodeGroupId()); - if (ranges == nullptr) - { - shard_->FetchTableRanges(index_range_name, - &req, - req.NodeGroupId(), - ng_term); - return false; - } - - catalog_entry->dirty_schema_->StatisticsObject() - ->CreateIndex( - new_index_name, - catalog_entry->dirty_schema_ - ->IndexKeySchema(new_index_name), - req.NodeGroupId()); - } - } - } + break; } - } - else - { - // other cores - catalog_entry = shard_->GetCatalog(table_name, req.NodeGroupId()); - assert(catalog_entry != nullptr); + case CcErrorCode::ACQUIRE_LOCK_BLOCKED: + { + req.SetCcePtr(cce); + return false; + } + default: + { + // lock confilct: back off and retry. + assert(false); + } + } //-- end: switch } - // 2. Replay the table ccmap. - if (schema_op_msg.stage() == ::txlog::SchemaOpMessage_Stage:: - SchemaOpMessage_Stage_PrepareSchema || - schema_op_msg.stage() == ::txlog::SchemaOpMessage_Stage:: - SchemaOpMessage_Stage_PrepareData) + if (shard_->core_id_ == shard_->core_cnt_ - 1) { - const TableSchema *old_schema = catalog_entry->schema_.get(); - const TableSchema *new_schema = catalog_entry->dirty_schema_.get(); - if (old_schema != nullptr && new_schema != nullptr) + if (req.GetDDLPhase() == + KeyObjectStandbyForwardCc::DDLPhase::AcquirePhase) { - // Alter table index operation. - // Pk table ccmap using old schema. - shard_->CreateOrUpdatePkCcMap( - table_name, old_schema, req.NodeGroupId()); - - if (!table_name.IsHashPartitioned()) + // after locks are acquired on all cores, update catalog schema + // in local cc shards. + CatalogRecord tmp_rec; + const std::string_view payload_sv = req.CommandList()->front(); + size_t offset = 0; + tmp_rec.Deserialize(payload_sv.data(), offset); + auto res = + shard_->CreateDirtyCatalog(table_key->Name(), + cc_ng_id_, + tmp_rec.DirtySchemaImage(), + commit_ts); + const CatalogEntry *catalog_entry = res; + // If dirty schema is created successfully, it means that the + // schema needs to be updated in kv store. + if (!txservice_skip_kv && + !shard_->local_shards_.store_hd_->IsSharedStorage()) { - // Pk range table ccmap - const TableName base_range_name{table_name.StringView(), - TableType::RangePartition, - table_name.Engine()}; - shard_->CreateOrUpdateRangeCcMap( - base_range_name, - old_schema, - req.NodeGroupId(), - catalog_entry->schema_version_); - } - - // Old sk table ccmap using old schema. - std::vector old_index_names = - old_schema->IndexNames(); - for (const auto &old_index_name : old_index_names) - { - shard_->CreateOrUpdateSkCcMap( - old_index_name, old_schema, req.NodeGroupId()); - // old sk range table ccmap - const TableName old_index_range_name{ - old_index_name.StringView(), - TableType::RangePartition, - old_index_name.Engine()}; - shard_->CreateOrUpdateRangeCcMap( - old_index_range_name, - old_schema, - req.NodeGroupId(), - catalog_entry->schema_version_); + if (same_commit_ts && + catalog_entry->schema_version_ == commit_ts) + { + req.SetOldTableSchema(Sharder::Instance() + .GetLocalCcShards() + ->CreateTableSchemaFromImage( + table_key->Name(), + tmp_rec.SchemaImage(), + req.SchemaVersion())); + req.SetDDLPhase( + KeyObjectStandbyForwardCc::DDLPhase::KvOpPhase); + shard_->local_shards_.store_hd_->UpsertTable( + req.GetOldTableSchema(), + catalog_entry->schema_.get(), + OperationType::TruncateTable, + commit_ts, + cc_ng_id_, + req.StandbyNodeTerm(), + nullptr, + nullptr, + &req, + shard_, + &req.DDLKvOpErrorCode()); + return false; + } + else if (catalog_entry->dirty_schema_) + { + // If using non-shared kv, we need to perform kv op on + // standby node as well. + req.SetDDLPhase( + KeyObjectStandbyForwardCc::DDLPhase::KvOpPhase); + shard_->local_shards_.store_hd_->UpsertTable( + catalog_entry->schema_.get(), + catalog_entry->dirty_schema_.get(), + OperationType::TruncateTable, + commit_ts, + cc_ng_id_, + req.StandbyNodeTerm(), + nullptr, + nullptr, + &req, + shard_, + &req.DDLKvOpErrorCode()); + return false; + } } - - // New sk table ccmap using new schema - std::vector new_index_names = - new_schema->IndexNames(); - for (const auto &new_index_name : new_index_names) + // If there's no need to update kv store, move to install & + // release phase and move the req back to core 0. + req.SetDDLPhase( + KeyObjectStandbyForwardCc::DDLPhase::ReleasePhase); + MoveRequest(&req, 0); + return false; + } + else if (req.GetDDLPhase() == + KeyObjectStandbyForwardCc::DDLPhase::KvOpPhase) + { + assert(!txservice_skip_kv && + !shard_->local_shards_.store_hd_->IsSharedStorage()); + if (req.DDLKvOpErrorCode() == CcErrorCode::DATA_STORE_ERR) { - if (std::find(old_index_names.cbegin(), - old_index_names.cend(), - new_index_name) == old_index_names.cend()) + auto catalog_entry = + shard_->GetCatalog(table_key->Name(), cc_ng_id_); + if (same_commit_ts && + catalog_entry->schema_version_ == commit_ts) { - shard_->CreateOrUpdateSkCcMap( - new_index_name, new_schema, req.NodeGroupId()); - - // New sk range cc maps should use the dirty schema - const TableName new_index_range_name{ - new_index_name.StringView(), - TableType::RangePartition, - new_index_name.Engine()}; - shard_->CreateOrUpdateRangeCcMap( - new_index_range_name, - new_schema, - req.NodeGroupId(), - catalog_entry->dirty_schema_version_); + // retry kv op + req.DDLKvOpErrorCode() = CcErrorCode::NO_ERROR; + shard_->local_shards_.store_hd_->UpsertTable( + req.GetOldTableSchema(), + catalog_entry->schema_.get(), + OperationType::TruncateTable, + commit_ts, + cc_ng_id_, + req.StandbyNodeTerm(), + nullptr, + nullptr, + &req, + shard_, + &req.DDLKvOpErrorCode()); + return false; + } + else + { + // retry kv op + req.DDLKvOpErrorCode() = CcErrorCode::NO_ERROR; + shard_->local_shards_.store_hd_->UpsertTable( + catalog_entry->schema_.get(), + catalog_entry->dirty_schema_.get(), + OperationType::TruncateTable, + commit_ts, + cc_ng_id_, + req.StandbyNodeTerm(), + nullptr, + nullptr, + &req, + shard_, + &req.DDLKvOpErrorCode()); + return false; } } + else if (req.DDLKvOpErrorCode() == CcErrorCode::NO_ERROR) + { + req.SetDDLPhase( + KeyObjectStandbyForwardCc::DDLPhase::ReleasePhase); + MoveRequest(&req, 0); + return false; + } + else if (req.DDLKvOpErrorCode() == CcErrorCode::NG_TERM_CHANGED) + { + // if ng term has changed, this cc should be rejected on + // ValidTermCheck() and should not reach here. + assert(false); + } + else + { + assert(false); + } } } - CatalogKey table_key(table_name); - Iterator it = FindEmplace(table_key); - CcEntry *cce = it->second; - CcPage *ccp = it.GetPage(); - - if (cce == nullptr) + if (req.GetDDLPhase() == + KeyObjectStandbyForwardCc::DDLPhase::ReleasePhase) { - shard_->Enqueue(shard_->LocalCoreId(), &req); - return false; - } + // Install new schema image in record and release the lock + if (cce->PayloadStatus() == RecordStatus::Unknown) + { + cce->payload_.PassInCurrentPayload( + std::make_unique()); + } + CatalogEntry *catalog_entry = + shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); + catalog_entry->CommitDirtySchema(); - // 3. Replay the table write intent/lock. - LockType lock_type = LockType::NoLock; - TableName base_table_name(table_name.GetBaseTableNameSV(), - TableType::Primary, - table_name.Engine()); - OperationType op_type = - static_cast(schema_op_msg.table_op().op_type()); - switch (schema_op_msg.stage()) - { - case ::txlog::SchemaOpMessage_Stage:: - SchemaOpMessage_Stage_PrepareSchema: - { - // If the prepare log has been flushed, the recovered cc ng leader - // replays all steps between the prepare log and the commit log, - // including the write lock on the schema. The recovered write lock - // is special in that the holding tx is not associated with the tx's - // term (always set to 0). This is because after the prepare log, - // the write lock on the schema and the coordinating tx are - // guaranteed to be recovered upon failures. The tx's term is not - // necessary here to mark whether or not if the coordinating tx has - // failed or not. - if (req.RangeSplitting(base_table_name) || - op_type == OperationType::AddIndex) - { - // If range splitting is also happening on this table, which - // must have acquired a read lock on the catalog entry, that - // means we only need to recover a write intent. - // For add index operation, between the prepare log and the - // prepare index table log, the recovered cc ng leader should - // recover the write intent on the schema. - lock_type = LockType::WriteIntent; + cce->payload_.cur_payload_->Set( + catalog_entry->schema_, nullptr, commit_ts); + cce->SetCommitTsPayloadStatus(commit_ts, RecordStatus::Normal); + // clean up data in ccm + CcMap *ccm = shard_->GetCcm(table_key->Name(), cc_ng_id_); + if (ccm != nullptr && ccm->SchemaTs() < commit_ts) + { + shard_->DropCcm(table_key->Name(), cc_ng_id_); + shard_->DequeueWaitListAfterSchemaUpdated(); + } + + ReleaseCceLock(cce->GetKeyLock(), + cce, + req.Txn(), + cc_ng_id_, + LockType::WriteLock, + false); + + UpdateTableLocks(table_key->Name().StringView(), + catalog_entry->schema_version_, + cce->GetKeyLock()); + + if (shard_->core_id_ == (shard_->core_cnt_ - 1)) + { + return req.SetFinish(*shard_); } else { - lock_type = LockType::WriteLock; + MoveRequest(&req, shard_->core_id_ + 1); + return false; } - break; - } - case ::txlog::SchemaOpMessage_Stage::SchemaOpMessage_Stage_PrepareData: - { - lock_type = LockType::WriteIntent; - break; } - case ::txlog::SchemaOpMessage_Stage::SchemaOpMessage_Stage_CommitSchema: + + // should not reach here + assert(false); + return true; + } + + bool Execute(InvalidateTableCacheCc &req) override + { + CcHandlerResult *hd_res = req.Result(); + + int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); + if (ng_term < 0) { - break; - } - default: - break; + hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); + return true; } - if (lock_type == LockType::WriteIntent) + const TableName &base_table_name = *req.invalidate_table_name_; + CatalogKey catalog_key(base_table_name); + Iterator it = + TemplateCcMap::Find( + catalog_key); + CcEntry *cce = + it == End() ? nullptr : it->second; + + if (cce == nullptr) { - auto lock_pair = AcquireCceKeyLock(cce, - cce->CommitTs(), - ccp, - cce->PayloadStatus(), - &req, - req.NodeGroupId(), - ng_term, - 0, - CcOperation::ReadForWrite, - IsolationLevel::RepeatableRead, - CcProtocol::OCC, - 0, - false); - assert(lock_pair.first == LockType::WriteIntent && - lock_pair.second == CcErrorCode::NO_ERROR); - // This silences the -Wunused-but-set-variable warning without any - // runtime overhead. - (void) lock_pair; - } - else if (lock_type == LockType::WriteLock) - { - auto lock_pair = AcquireCceKeyLock(cce, - cce->CommitTs(), - ccp, - cce->PayloadStatus(), - &req, - req.NodeGroupId(), - ng_term, - 0, - CcOperation::Write, - IsolationLevel::RepeatableRead, - CcProtocol::Locking, - 0, - false); - // When a cc node recovers, no one should be holding read locks. - // So, the acquire operation should always succeed. - assert(lock_pair.first == LockType::WriteLock && - lock_pair.second == CcErrorCode::NO_ERROR); - // This silences the -Wunused-but-set-variable warning without any - // runtime overhead. - (void) lock_pair; + // The catalog write lock was taken on the old leader by + // InvalidateTableCacheCompositeOp's acquire_all_lock_op_, and there + // is no leader-term re-check before this request is routed. If the + // node group failed over in between, this lands on a new leader + // whose catalog cc map has neither the entry nor the lock -- cc + // entries and locks do not survive failover. The ng_term check + // above still passes, because the new leader's term is valid. There + // is nothing cached to invalidate, so finish the request instead of + // dereferencing. + if (shard_->core_id_ < shard_->core_cnt_ - 1) + { + req.ResetCcm(); + MoveRequest(&req, shard_->core_id_ + 1); + return false; + } + hd_res->SetFinished(); + return true; } - if (cce->payload_.cur_payload_ == nullptr) + // Only meaningful when the entry survived, i.e. no failover happened. + assert(cce->GetKeyLock()->HasWriteLock(req.Txn())); + + if (cce->PayloadStatus() == RecordStatus::Unknown) { - cce->payload_.PassInCurrentPayload( - std::make_unique()); + const CatalogEntry *catalog_entry = + shard_->GetCatalog(base_table_name, req.NodeGroupId()); + if (catalog_entry != nullptr) + { + assert(catalog_entry->schema_); + // upload catalog record + cce->payload_.PassInCurrentPayload( + std::make_unique()); + cce->payload_.cur_payload_->Set(catalog_entry->schema_, + catalog_entry->dirty_schema_, + catalog_entry->schema_version_); + cce->SetCommitTsPayloadStatus(catalog_entry->schema_version_, + RecordStatus::Normal); + } + else + { + shard_->FetchCatalog( + base_table_name, req.NodeGroupId(), ng_term, &req); + return false; + } } - cce->payload_.cur_payload_->Set(catalog_entry->schema_, - catalog_entry->dirty_schema_, - catalog_entry->schema_version_); - if (catalog_entry->schema_ == nullptr && - catalog_entry->dirty_schema_ != nullptr) + + CatalogRecord *catalog_rec = cce->payload_.cur_payload_.get(); + const TableSchema *table_schema = catalog_rec->Schema(); + + TableName base_range_name(base_table_name.StringView(), + TableType::RangePartition, + base_table_name.Engine()); + shard_->DropCcm(base_table_name, cc_ng_id_); + shard_->DropCcm(base_range_name, cc_ng_id_); + + for (TableName index_table_name : table_schema->IndexNames()) { - assert(op_type == OperationType::CreateTable); - cce->SetCommitTsPayloadStatus(1, RecordStatus::Deleted); + TableName index_range_name(index_table_name.StringView(), + TableType::RangePartition, + index_table_name.Engine()); + shard_->DropCcm(index_table_name, cc_ng_id_); + shard_->DropCcm(index_range_name, cc_ng_id_); } if (shard_->core_id_ < shard_->core_cnt_ - 1) { req.ResetCcm(); MoveRequest(&req, shard_->core_id_ + 1); + return false; } else { - uint32_t tx_node_id = (req.Txn() >> 32L) >> 10; - bool is_coordinator = tx_node_id == req.NodeGroupId(); - int64_t tx_candidate_term = - Sharder::Instance().CandidateLeaderTerm(tx_node_id); + shard_->CleanTableRange(base_range_name, cc_ng_id_); + shard_->CleanTableStatistics(base_table_name, cc_ng_id_); - if (is_coordinator && tx_candidate_term >= 0) - { - // If the coordinating tx is bound to the recoverying cc - // node, resumes the tx. This will spawn - // a worker thread that will restore the transaction. It will - // call SetFinish() after acqruing all needed locks. - shard_->local_shards_.CreateSchemaRecoveryTx( - req, schema_op_msg, tx_candidate_term); - } - else + for (TableName index_table_name : table_schema->IndexNames()) { - req.SetFinish(); + TableName index_range_name(index_table_name.StringView(), + TableType::RangePartition, + index_table_name.Engine()); + shard_->CleanTableRange(index_range_name, cc_ng_id_); + shard_->CleanTableStatistics(index_table_name, cc_ng_id_); } + + hd_res->SetFinished(); + return true; } + } - return true; + TableType Type() const override + { + return TableType::Catalog; } - bool Execute(BroadcastStatisticsCc &req) override + bool HasWriteLock(const TableName &table_name, uint32_t node_group_id) { - CcHandlerResult *hd_res = req.Result(); + CatalogKey table_key{TableName{ + table_name.StringView(), TableType::Primary, table_name.Engine()}}; + TxKey catalog_tx_key(&table_key); + auto it = Find(table_key); + if (it == End()) + { + return false; + } - int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); - if (ng_term < 0) + CcEntry *catalog_cce = + it->second; + const auto keylock = catalog_cce->GetKeyLock(); + if (keylock == nullptr) { - hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; + return false; } + return keylock->HasWriteLock(); + } - const TableName &table_name = *req.SamplingTableName(); + /** + * GetTableSchema stored in each shard's CatalogCcMap. Used to initialize + * the Pk and Sk ccmaps. + * @param table_name + * @param cc_ng_id + * @param cc_ng_term + * @param requester + * @return InitCcmResult describing the schema lookup outcome. + */ + InitCcmResult GetTableSchema(const TableName &table_name, + NodeGroupId cc_ng_id, + int64_t cc_ng_term, + CcRequestBase *requester) + { + const TableName base_table_name{table_name.GetBaseTableNameSV(), + TableType::Primary, + table_name.Engine()}; - DLOG(INFO) << "Receive table statistics " << table_name.StringView() - << ". From ng #" << req.SamplePool()->ng_id() << ", to ng #" - << req.NodeGroupId() - << ". ng_records: " << req.SamplePool()->records() - << ", ng_samples: " << req.SamplePool()->samples_size(); + CatalogKey table_key{base_table_name}; - TableName base_table_name(table_name.GetBaseTableNameSV(), - TableType::Primary, - table_name.Engine()); - CatalogKey table_key(base_table_name); - Iterator it = FindEmplace(table_key); - CcEntry *cce = it->second; - if (cce->PayloadStatus() == RecordStatus::Unknown) + bool emplace = false; + Iterator it = FindEmplace(table_key, &emplace, true, false); + CcEntry *catalog_cce = + it->second; + + if (catalog_cce->PayloadStatus() == RecordStatus::Unknown) { const CatalogEntry *catalog_entry = - shard_->GetCatalog(base_table_name, req.NodeGroupId()); - if (catalog_entry != nullptr) + shard_->GetCatalog(table_key.Name(), cc_ng_id); + + // If the read toward the catalog cc entry acquires the read + // lock but the cc entry does not contain the value, checks if + // the catalog has been constructed at this node. If so, turns + // this request into a read-outside request that installs the + // value in the cc entry. + if (catalog_entry != nullptr && catalog_entry->schema_version_ > 0) { if (catalog_entry->schema_ != nullptr) { - // Initialize table statistics before create ccmap. - if (Sharder::Instance().GetLocalCcShards()->store_hd_ != - nullptr && - !table_name.IsHashPartitioned() && - !shard_->LoadRangesAndStatisticsNx( - catalog_entry->schema_.get(), - req.NodeGroupId(), - ng_term, - &req)) { - return false; // Loading... +#ifdef STATISTICS + assert(requester != nullptr); + // Initialize table statistics before create ccmap. + if (Sharder::Instance().GetLocalCcShards()->store_hd_ != + nullptr && + !table_name.IsHashPartitioned() && + !shard_->LoadRangesAndStatisticsNx( + catalog_entry->schema_.get(), + cc_ng_id, + cc_ng_term, + requester)) + { + return InitCcmResult::Retry(); + } +#endif } // upload catalog record - cce->payload_.PassInCurrentPayload( + catalog_cce->payload_.PassInCurrentPayload( std::make_unique()); - cce->payload_.cur_payload_->Set( + catalog_cce->payload_.cur_payload_->Set( catalog_entry->schema_, catalog_entry->dirty_schema_, catalog_entry->schema_version_); - cce->SetCommitTsPayloadStatus( + catalog_cce->SetCommitTsPayloadStatus( catalog_entry->schema_version_, RecordStatus::Normal); } else { - cce->SetCommitTsPayloadStatus( + catalog_cce->SetCommitTsPayloadStatus( catalog_entry->schema_version_, RecordStatus::Deleted); } } else { - if (Statistics::LeaderNodeGroup(base_table_name) != - req.NodeGroupId()) - { - hd_res->SetFinished(); - return true; - } - else - { - shard_->FetchCatalog( - base_table_name, req.NodeGroupId(), ng_term, &req); - return false; - } - } - } - - if (cce->PayloadStatus() == RecordStatus::Normal) + // Global CatalogEntry does not exist. + shard_->FetchCatalog( + table_key.Name(), cc_ng_id, cc_ng_term, requester); + return InitCcmResult::Retry(); + } + } + + const auto keylock = catalog_cce->GetKeyLock(); + if (keylock != nullptr && keylock->HasWriteLock()) { - const StatisticsEntry *statistics_entry = - shard_->GetTableStatistics(base_table_name, req.NodeGroupId()); - assert(statistics_entry && statistics_entry->statistics_); - const auto table_schema = [&req, cce]() -> const TableSchema * - { - if (cce->payload_.cur_payload_->Schema() && - cce->payload_.cur_payload_->Schema()->Version() == - req.SchemaVersion()) - { - return cce->payload_.cur_payload_->Schema(); - } - else if (cce->payload_.cur_payload_->DirtySchema() && - cce->payload_.cur_payload_->DirtySchema()->Version() == - req.SchemaVersion()) - { - return cce->payload_.cur_payload_->DirtySchema(); - } - else - { - return nullptr; - } - }(); + DLOG(WARNING) + << "InitCcm failure, target table: " << table_name.StringView() + << " has write lock on catalog_cc_map, which means, it is " + "being modified on this shard."; + return InitCcmResult::Failure(CcErrorCode::READ_CATALOG_CONFLICT); + } - if (table_schema) - { - statistics_entry->statistics_->OnRemoteStatisticsMessage( - table_name, table_schema, *req.SamplePool()); - } + RecordStatus payload_status = catalog_cce->PayloadStatus(); + if (payload_status == RecordStatus::Deleted) + { + return InitCcmResult::Failure( + CcErrorCode::REQUESTED_TABLE_NOT_EXISTS); } - hd_res->SetFinished(); - return true; + assert(payload_status == RecordStatus::Normal); + CatalogRecord *catalog_rec = catalog_cce->payload_.cur_payload_.get(); + assert(catalog_rec != nullptr); + assert(catalog_rec->Schema() != nullptr); + + return InitCcmResult::Success(catalog_rec->Schema()); } - bool Execute(KeyObjectStandbyForwardCc &req) override + std::tuple ReadTable( + const TableName &table_name, + uint32_t node_group_id, + int64_t ng_term, + TxNumber tx_number) { - uint64_t commit_ts = req.CommitTs(); - - const CatalogKey *table_key = nullptr; - const std::string *key_str = req.KeyImage(); - assert(key_str != nullptr); - std::unique_ptr decoded_key = - std::make_unique(); - size_t offset = 0; - decoded_key->Deserialize(key_str->data(), offset, KeySchema()); - table_key = decoded_key.get(); + bool read_success = false; + uint64_t schema_version = 0; + NonBlockingLock *lock_ptr = nullptr; - CcEntry *cce; - CcPage *ccp; - if (req.CcePtr()) - { - cce = - static_cast *>( - req.CcePtr()); - ccp = static_cast *>( - cce->GetCcPage()); - } - else + std::string_view table_name_sv = table_name.StringView(); + absl::string_view absl_table_name_sv(table_name_sv.data(), + table_name_sv.size()); + auto lock_it = table_locks_.find(absl_table_name_sv); + if (lock_it == table_locks_.end()) { - Iterator it = FindEmplace(*table_key); - cce = it->second; - ccp = it.GetPage(); - } + CatalogKey table_key{ + txservice::TableName{table_name_sv, + txservice::TableType::Primary, + table_name.Engine()}}; + TxKey catalog_tx_key(&table_key); + auto it = FindEmplace(table_key); + + CcEntry *catalog_cce = + it->second; + CcPage *catalog_ccp = + it.GetPage(); - // If the current node has not initialized the catalog entry, we fetch - // the newest catalog from the primary node. - if (cce->PayloadStatus() == RecordStatus::Unknown) - { const CatalogEntry *catalog_entry = - shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); - if (catalog_entry != nullptr && catalog_entry->schema_version_ > 0) + shard_->GetCatalog(table_key.Name(), node_group_id); + + if (catalog_entry == nullptr || catalog_entry->schema_ == nullptr) { - if (catalog_entry->schema_ != nullptr) - { - // upload catalog record - cce->payload_.PassInCurrentPayload( - std::make_unique()); - cce->payload_.cur_payload_->Set( - catalog_entry->schema_, - catalog_entry->dirty_schema_, - catalog_entry->schema_version_); - // update commit ts - cce->SetCommitTsPayloadStatus( - catalog_entry->schema_version_, RecordStatus::Normal); - } - else - { - cce->SetCommitTsPayloadStatus( - catalog_entry->schema_version_, RecordStatus::Deleted); - } + // The catalog entry hasn't been initialized, try to load it + // from data store. + shard_->FetchCatalog( + table_name, node_group_id, ng_term, nullptr); + return {CcErrorCode::READ_CATALOG_FAIL, nullptr, 0}; } - else + + assert(catalog_entry != nullptr && + catalog_entry->schema_version_ > 0); + assert(catalog_entry->schema_ != nullptr); + + schema_version = catalog_entry->schema_version_; + + lock_ptr = + &catalog_cce->GetOrCreateKeyLock(shard_, this, catalog_ccp); + auto [new_lock_it, _] = table_locks_.try_emplace( + std::string(table_name_sv), lock_ptr, schema_version); + read_success = + new_lock_it->second.first->AcquireReadLockFast(tx_number); + + if (read_success && + catalog_cce->PayloadStatus() == RecordStatus::Unknown) { - // fetch catalog from primary node - shard_->FetchCatalog(table_key->Name(), - req.NodeGroupId(), - req.StandbyNodeTerm(), - &req); - return false; + assert(schema_version == new_lock_it->second.second); + // upload catalog record + catalog_cce->payload_.PassInCurrentPayload( + std::make_unique()); + catalog_cce->payload_.cur_payload_->Set( + catalog_entry->schema_, + catalog_entry->dirty_schema_, + schema_version); + catalog_cce->SetCommitTsPayloadStatus(schema_version, + RecordStatus::Normal); } } - - if (commit_ts < cce->CommitTs()) + else { - // discard outdate request - if (shard_->core_id_ + 1 == shard_->core_cnt_) - { - return req.SetFinish(*shard_); - } - else - { - MoveRequest(&req, shard_->core_id_ + 1); - return false; - } + read_success = + lock_it->second.first->AcquireReadLockFast(tx_number); + lock_ptr = lock_it->second.first; + schema_version = lock_it->second.second; } - // The catalog entry might be directly fetched from the - // primary node, so we still need to execute UpsertTable in the store - // handler. Otherwise, the kv storage will not have the latest kv table. - bool same_commit_ts = (commit_ts == cce->CommitTs()); - - LockType acquired_lock = LockType::NoLock; - CcErrorCode err_code = CcErrorCode::NO_ERROR; + if (read_success) + { + assert(lock_ptr != nullptr); + assert(schema_version > 0); +#ifndef NDEBUG + shard_->UpsertLockHoldingTx(tx_number, + ng_term, + nullptr, + false, + node_group_id, + table_name_.Type()); +#endif + return {CcErrorCode::NO_ERROR, lock_ptr, schema_version}; + } + else + { + return {CcErrorCode::READ_CATALOG_CONFLICT, nullptr, 0}; + } + } - CODE_FAULT_INJECTOR("standby_forward_ddl_trigger_start_following", { - // only trigger once - txservice::FaultInject::Instance().InjectFault( - "standby_forward_ddl_trigger_start_following", "remove"); - DLOG(INFO) - << "fault inject: standby_forward_ddl_trigger_start_following"; - NodeGroupId native_ng = Sharder::Instance().NativeNodeGroup(); - int64_t cur_prim_term = Sharder::Instance().PrimaryNodeTerm(); - assert(cur_prim_term > 0); - Sharder::Instance().OnStartFollowing( - native_ng, - cur_prim_term, - Sharder::Instance().LeaderNodeId(native_ng), - true); +private: + /** @brief A cc entry of the catalog cc map. */ + using CatalogCce = CcEntry; - return req.SetFinish(*shard_); - }); + /** + * @brief PrepareCommit of a schema op: installs the op's dirty schema on + * this node and the record the template puts in the cc entry of every + * core. An index op also readies its table (PrepareIndexOp), and an add + * index creates its new indexes' cc maps. + * + * @param req The request. + * @param table_key The table's catalog key. + * @param ng_term The node group's leader term. + * @return As Execute(): whether the request is done with this core. + */ + bool PrepareCommitSchema(PostWriteAllCc &req, + const CatalogKey &table_key, + int64_t ng_term) + { + // Every sender prepares after its prepare log, or, for + // FlushUpdateTableOp, once its tx has committed: at a commit ts above + // 0. A post write at 0 is a Rollback or a DowngradeLock + // (PostWriteType::Rollback). + assert(req.CommitTs() != TransactionOperation::tx_op_failed_ts_); - if (req.GetDDLPhase() == - KeyObjectStandbyForwardCc::DDLPhase::AcquirePhase) + const NodeGroupId ng_id = req.NodeGroupId(); + CatalogEntry *catalog_entry = nullptr; + if (shard_->core_id_ == 0) { - // acquire write lock on catalog rec - if (req.CcePtr()) - { - // resumed req. - std::tie(acquired_lock, err_code) = LockHandleForResumedRequest( - cce, - cce->CommitTs(), - cce->PayloadStatus(), - &req, - Sharder::Instance().NativeNodeGroup(), - req.StandbyNodeTerm(), - req.TxTerm(), - CcOperation::Write, - IsolationLevel::RepeatableRead, - CcProtocol::Locking, - 0, - false); - assert(acquired_lock == LockType::WriteLock && - err_code == CcErrorCode::NO_ERROR); - req.SetCcePtr(nullptr); - } - else - { - std::tie(acquired_lock, err_code) = - AcquireCceKeyLock(cce, - cce->CommitTs(), - ccp, - cce->PayloadStatus(), - &req, - req.NodeGroupId(), - req.StandbyNodeTerm(), - req.TxTerm(), - CcOperation::Write, - IsolationLevel::RepeatableRead, - CcProtocol::Locking, - 0, - false); - } - switch (err_code) - { - case CcErrorCode::NO_ERROR: - { - if (shard_->core_id_ != shard_->core_cnt_ - 1) - { - MoveRequest(&req, shard_->core_id_ + 1); - return false; - } - break; - } - case CcErrorCode::ACQUIRE_LOCK_BLOCKED: + // The first core instantiates the dirty schema in LocalCcShards. + // A DROP TABLE's payload carries no image. + CatalogRecord *schema_rec = + RequestRecord(req, req.OpType() != OperationType::DropTable); + catalog_entry = + shard_->CreateDirtyCatalog(table_key.Name(), + ng_id, + schema_rec->DirtySchemaImage(), + req.CommitTs()); + if ((req.OpType() == OperationType::AddIndex || + req.OpType() == OperationType::DropIndex) && + !PrepareIndexOp(req, table_key, *catalog_entry, ng_term)) { - req.SetCcePtr(cce); return false; } - default: + + schema_rec->Set(catalog_entry->schema_, + catalog_entry->dirty_schema_, + catalog_entry->schema_version_); + } + else + { + // Core 0 set up the record the template installs. + assert(req.Payload() != nullptr); + catalog_entry = shard_->GetCatalog(table_key.Name(), ng_id); + if (catalog_entry == nullptr) { - // lock confilct: back off and retry. - assert(false); + // The table catalog haven't been initialized. Just return. + req.Result()->SetError(CcErrorCode::REQUESTED_TABLE_NOT_EXISTS); + return true; } - } //-- end: switch } - if (shard_->core_id_ == shard_->core_cnt_ - 1) + if (req.OpType() == OperationType::AddIndex && + catalog_entry->dirty_schema_version_ > 0) { - if (req.GetDDLPhase() == - KeyObjectStandbyForwardCc::DDLPhase::AcquirePhase) - { - // after locks are acquired on all cores, update catalog schema - // in local cc shards. - CatalogRecord tmp_rec; - const std::string_view payload_sv = req.CommandList()->front(); - size_t offset = 0; - tmp_rec.Deserialize(payload_sv.data(), offset); - auto res = - shard_->CreateDirtyCatalog(table_key->Name(), - cc_ng_id_, - tmp_rec.DirtySchemaImage(), - commit_ts); - const CatalogEntry *catalog_entry = res; - // If dirty schema is created successfully, it means that the - // schema needs to be updated in kv store. - if (!txservice_skip_kv && - !shard_->local_shards_.store_hd_->IsSharedStorage()) - { - if (same_commit_ts && - catalog_entry->schema_version_ == commit_ts) - { - req.SetOldTableSchema(Sharder::Instance() - .GetLocalCcShards() - ->CreateTableSchemaFromImage( - table_key->Name(), - tmp_rec.SchemaImage(), - req.SchemaVersion())); - req.SetDDLPhase( - KeyObjectStandbyForwardCc::DDLPhase::KvOpPhase); - shard_->local_shards_.store_hd_->UpsertTable( - req.GetOldTableSchema(), - catalog_entry->schema_.get(), - OperationType::TruncateTable, - commit_ts, - cc_ng_id_, - req.StandbyNodeTerm(), - nullptr, - nullptr, - &req, - shard_, - &req.DDLKvOpErrorCode()); - return false; - } - else if (catalog_entry->dirty_schema_) - { - // If using non-shared kv, we need to perform kv op on - // standby node as well. - req.SetDDLPhase( - KeyObjectStandbyForwardCc::DDLPhase::KvOpPhase); - shard_->local_shards_.store_hd_->UpsertTable( - catalog_entry->schema_.get(), - catalog_entry->dirty_schema_.get(), - OperationType::TruncateTable, - commit_ts, - cc_ng_id_, - req.StandbyNodeTerm(), - nullptr, - nullptr, - &req, - shard_, - &req.DDLKvOpErrorCode()); - return false; - } - } - // If there's no need to update kv store, move to install & - // release phase and move the req back to core 0. - req.SetDDLPhase( - KeyObjectStandbyForwardCc::DDLPhase::ReleasePhase); - MoveRequest(&req, 0); - return false; - } - else if (req.GetDDLPhase() == - KeyObjectStandbyForwardCc::DDLPhase::KvOpPhase) - { - assert(!txservice_skip_kv && - !shard_->local_shards_.store_hd_->IsSharedStorage()); - if (req.DDLKvOpErrorCode() == CcErrorCode::DATA_STORE_ERR) - { - auto catalog_entry = - shard_->GetCatalog(table_key->Name(), cc_ng_id_); - if (same_commit_ts && - catalog_entry->schema_version_ == commit_ts) - { - // retry kv op - req.DDLKvOpErrorCode() = CcErrorCode::NO_ERROR; - shard_->local_shards_.store_hd_->UpsertTable( - req.GetOldTableSchema(), - catalog_entry->schema_.get(), - OperationType::TruncateTable, - commit_ts, - cc_ng_id_, - req.StandbyNodeTerm(), - nullptr, - nullptr, - &req, - shard_, - &req.DDLKvOpErrorCode()); - return false; - } - else - { - // retry kv op - req.DDLKvOpErrorCode() = CcErrorCode::NO_ERROR; - shard_->local_shards_.store_hd_->UpsertTable( - catalog_entry->schema_.get(), - catalog_entry->dirty_schema_.get(), - OperationType::TruncateTable, - commit_ts, - cc_ng_id_, - req.StandbyNodeTerm(), - nullptr, - nullptr, - &req, - shard_, - &req.DDLKvOpErrorCode()); - return false; - } - } - else if (req.DDLKvOpErrorCode() == CcErrorCode::NO_ERROR) - { - req.SetDDLPhase( - KeyObjectStandbyForwardCc::DDLPhase::ReleasePhase); - MoveRequest(&req, 0); - return false; - } - else if (req.DDLKvOpErrorCode() == CcErrorCode::NG_TERM_CHANGED) - { - // if ng term has changed, this cc should be rejected on - // ValidTermCheck() and should not reach here. - assert(false); - } - else - { - assert(false); - } - } + SetAddedIndexCcms(*catalog_entry, ng_id, true); + } + return TemplateCcMap::Execute(req); + } + + /** + * @brief Readies an index op's table on this node for its dirty schema. + * Done once, on core 0, after the dirty schema is installed. + * + * 1. The current schema. The DDL read it on the coordinator's node group + * only, so another node group may not have loaded the table yet. + * Installing the dirty schema first is harmless: a current schema + * fetched later is older than it and keeps it + * (CatalogEntry::InitSchema), and CreateDirtyCatalog changes nothing when + * the request runs again. + * 2. The table's statistics (building them needs the table's ranges), + * which the dirty schema takes over: it becomes the current schema. + * 3. For an add index, the new indexes, which the table's cc maps and + * transactions may reach while the op runs: their ranges, at the op's + * version and with empty slices for the ranges this node group owns + * and has none for yet, and their entries in the statistics. + * + * A load is asynchronous: the request runs again once it completes. + * + * @param req The PrepareCommit request. + * @param table_key The table's catalog key. + * @param catalog_entry The table's catalog entry, holding the dirty schema. + * @param ng_term The node group's leader term. + * @return false while something is being fetched. + */ + bool PrepareIndexOp(PostWriteAllCc &req, + const CatalogKey &table_key, + CatalogEntry &catalog_entry, + int64_t ng_term) + { + const NodeGroupId ng_id = req.NodeGroupId(); + if (catalog_entry.schema_ == nullptr) + { + shard_->FetchCatalog(table_key.Name(), ng_id, ng_term, &req); + return false; + } + if (Sharder::Instance().GetLocalCcShards()->store_hd_ != nullptr && + !table_key.Name().IsHashPartitioned() && + shard_->LoadRangesAndStatisticsNx( + catalog_entry.schema_.get(), ng_id, ng_term, &req) == nullptr) + { + return false; + } + catalog_entry.dirty_schema_->BindStatistics( + catalog_entry.schema_->StatisticsObject()); + if (req.OpType() != OperationType::AddIndex) + { + return true; } - if (req.GetDDLPhase() == - KeyObjectStandbyForwardCc::DDLPhase::ReleasePhase) + const std::vector added_index_names = + IndexesNotIn(*catalog_entry.dirty_schema_, *catalog_entry.schema_); + // A new index may come with pre-defined ranges, based on the data + // distribution the caller offered, so its ranges are loaded rather + // than initialized empty. + for (const TableName &index_name : added_index_names) { - // Install new schema image in record and release the lock - if (cce->PayloadStatus() == RecordStatus::Unknown) + const TableName index_range_name = RangeTableNameOf(index_name); + auto ranges = + shard_->GetTableRangesForATable(index_range_name, ng_id); + if (ranges == nullptr) { - cce->payload_.PassInCurrentPayload( - std::make_unique()); + shard_->FetchTableRanges( + index_range_name, &req, ng_id, ng_term); + return false; } - CatalogEntry *catalog_entry = - shard_->GetCatalog(table_key->Name(), req.NodeGroupId()); - catalog_entry->CommitDirtySchema(); - cce->payload_.cur_payload_->Set( - catalog_entry->schema_, nullptr, commit_ts); - cce->SetCommitTsPayloadStatus(commit_ts, RecordStatus::Normal); - // clean up data in ccm - CcMap *ccm = shard_->GetCcm(table_key->Name(), cc_ng_id_); - if (ccm != nullptr && ccm->SchemaTs() < commit_ts) + for (auto &range : *ranges) { - shard_->DropCcm(table_key->Name(), cc_ng_id_); - shard_->DequeueWaitListAfterSchemaUpdated(); + range.second->SetVersion(req.CommitTs()); + NodeGroupId range_owner = + shard_ + ->GetRangeOwner( + range.second->GetRangeInfo()->PartitionId(), ng_id) + ->BucketOwner(); + if (range_owner == ng_id && + range.second->RangeSlices() == nullptr) + { + // This node group owns the range, which is empty. + InitEmptyRangeSlices(*range.second, ng_id); + } } + } - ReleaseCceLock(cce->GetKeyLock(), - cce, - req.Txn(), - cc_ng_id_, - LockType::WriteLock, - false); + for (const TableName &index_name : added_index_names) + { + catalog_entry.schema_->StatisticsObject()->CreateIndex( + index_name, + catalog_entry.dirty_schema_->IndexKeySchema(index_name), + cc_ng_id_); + } + return true; + } - UpdateTableLocks(table_key->Name().StringView(), - catalog_entry->schema_version_, - cce->GetKeyLock()); + /** + * @brief UpdateDirty of an add index: replaces the dirty schema if the + * request carries a new image of it, and moves the new indexes' cc maps to + * it. + * + * @param req The request. + * @param table_key The table's catalog key. + * @return As Execute(): whether the request is done with this core. + */ + bool UpdateDirtySchema(PostWriteAllCc &req, const CatalogKey &table_key) + { + assert(req.OpType() == OperationType::AddIndex); + assert(req.CommitTs() != TransactionOperation::tx_op_failed_ts_); + const NodeGroupId ng_id = req.NodeGroupId(); + CatalogEntry *catalog_entry = + shard_->GetCatalog(table_key.Name(), ng_id); + if (catalog_entry == nullptr) + { + // The table catalog haven't been initialized. Just return. + req.Result()->SetError(CcErrorCode::REQUESTED_TABLE_NOT_EXISTS); + return true; + } - if (shard_->core_id_ == (shard_->core_cnt_ - 1)) - { - return req.SetFinish(*shard_); + if (shard_->core_id_ == 0) + { + CatalogRecord *schema_rec = RequestRecord(req); + if (catalog_entry->dirty_schema_->SchemaImage() != + schema_rec->DirtySchemaImage()) + { + assert(catalog_entry->dirty_schema_version_ == req.CommitTs()); + std::shared_ptr old_dirty_schema = + catalog_entry->dirty_schema_; + shard_->UpdateDirtyCatalog(table_key.Name(), + schema_rec->DirtySchemaImage(), + catalog_entry); + catalog_entry->dirty_schema_->BindStatistics( + old_dirty_schema->StatisticsObject()); + } + + schema_rec->Set(catalog_entry->schema_, + catalog_entry->dirty_schema_, + catalog_entry->schema_version_); + } + else + { + // Core 0 set up the record the template installs. + assert(req.Payload() != nullptr); + } + + if (catalog_entry->dirty_schema_version_ > 0) + { + SetAddedIndexCcms(*catalog_entry, ng_id, false); + } + return TemplateCcMap::Execute(req); + } + + /** + * @brief Points the cc maps of an add index's new indexes at the dirty + * schema, on this core. The existing indexes' cc maps move at PostCommit. + * + * @param catalog_entry The table's catalog entry, holding the dirty schema. + * @param ng_id The node group. + * @param is_create Whether to create the cc maps: at PrepareCommit. + */ + void SetAddedIndexCcms(const CatalogEntry &catalog_entry, + NodeGroupId ng_id, + bool is_create) + { + for (const TableName &index_name : + IndexesNotIn(*catalog_entry.dirty_schema_, *catalog_entry.schema_)) + { + SetTableCcms(index_name, + catalog_entry.dirty_schema_.get(), + ng_id, + catalog_entry.dirty_schema_version_, + is_create); + } + } + + /** + * @brief PostCommit of a schema op: promotes the dirty schema at the + * commit timestamp. + * + * Core 0 fills the request's record from the node's catalog entry: the + * requester's record for a local request, one the request owns for a + * remote one, which carries none (RequestRecord). The template installs it + * in every core's cc entry, copy-on-write + * (VersionedPayload::SetCurrentPayload). Each core moves the table's cc + * maps; the last core cleans what the op removed and commits the catalog + * entry; the template publishes the commit timestamp and releases the + * lock. + * + * @param req The request. + * @param table_key The table's catalog key. + * @param cce The table's cc entry on this core. + * @param ng_term The node group's leader term. + * @return As Execute(): whether the request is done with this core. + */ + bool PostCommitSchema(PostWriteAllCc &req, + const CatalogKey &table_key, + CatalogCce &cce, + int64_t ng_term) + { + FinishSchemaWriter(req, table_key); + CatalogEntry *catalog_entry = + shard_->GetCatalog(table_key.Name(), req.NodeGroupId()); + + // Only a Rollback leaves the commit timestamp at 0. + const uint64_t commit_ts = req.CommitTs(); + assert(commit_ts != TransactionOperation::tx_op_failed_ts_); + assert(catalog_entry != nullptr); + if (shard_->core_id_ == 0) + { + CommitNodeLevelSchema(req, table_key, *catalog_entry); + // The record promotes the dirty schema, at the commit timestamp, + // which is the schema record's new version; DROP TABLE's dirty + // schema is null, so its record holds no schema. An entry without + // a dirty schema version has nothing to promote and keeps its + // current schema. + CatalogRecord *rec = RequestRecord(req, false); + if (catalog_entry->dirty_schema_version_ > 0) + { + rec->Set(catalog_entry->dirty_schema_, nullptr, commit_ts); } else { - MoveRequest(&req, shard_->core_id_ + 1); - return false; + rec->Set(catalog_entry->schema_, + nullptr, + catalog_entry->schema_version_); } } + else + { + // Core 0 filled the record the template installs. + assert(req.Payload() != nullptr); + } - // should not reach here - assert(false); - return true; + if (catalog_entry->dirty_schema_version_ > 0) + { + CommitTableCcms(req, table_key, *catalog_entry, cce, ng_term); + if (shard_->core_id_ == shard_->core_cnt_ - 1) + { + CleanRemovedTables(req, table_key, *catalog_entry); + catalog_entry->CommitDirtySchema(commit_ts); + } + } + return TemplateCcMap::Execute(req); } - bool Execute(InvalidateTableCacheCc &req) override + /** + * @brief Rollback of a schema op, its second phase instead of PostCommit: + * discards the dirty schema and undoes what the prepare stage did. + * + * An op that failed, before or after its prepare log, commits at timestamp + * 0, which leaves the catalog entry's version alone; an add index whose + * build broke a constraint commits at its commit-stage timestamp, which + * the catalog entry takes. No table's version changes. An add index's new + * cc maps, made at its prepare stage, go on every core, their ranges and + * statistics on the last. + * + * An op that failed before its prepare log installed no dirty schema, so + * there is nothing to discard: the request then carries no record and the + * template only releases the lock. Having failed to acquire its write + * intents, such an op may hold them on some cores only, core 0 or not. + * An op that installed a dirty schema holds every core; core 0 fills the + * record the template installs on each. + * + * @param req The request. + * @param table_key The table's catalog key. + * @return As Execute(): whether the request is done with this core. + */ + bool RollBackSchema(PostWriteAllCc &req, const CatalogKey &table_key) { - CcHandlerResult *hd_res = req.Result(); + FinishSchemaWriter(req, table_key); + CatalogEntry *catalog_entry = + shard_->GetCatalog(table_key.Name(), req.NodeGroupId()); + const uint64_t commit_ts = req.CommitTs(); + + // A dirty schema on this node can only be the op's own: another op + // installs one only once it holds the write intent on every core of + // every node group, which it cannot while this op holds any. The last + // core drops it, after every other core has looked. + // + // An entry without one keeps its schema, and the template only + // releases the lock. Either the op failed before installing a dirty + // schema, at commit ts 0, or this is the coordinator's node group + // recovered from a commit log that rolled the op back: replay + // installed the current schema at the commit ts, loaded the entry at + // it, and restored the write lock this request releases. + if (catalog_entry == nullptr || + catalog_entry->dirty_schema_version_ == 0) + { + assert(commit_ts == TransactionOperation::tx_op_failed_ts_ || + (catalog_entry != nullptr && + catalog_entry->schema_version_ == commit_ts)); + req.SetDecodedPayload(nullptr); + return TemplateCcMap::Execute(req); + } - int64_t ng_term = Sharder::Instance().LeaderTerm(req.NodeGroupId()); - if (ng_term < 0) + const NodeGroupId ng_id = req.NodeGroupId(); + const bool last_core = shard_->core_id_ == shard_->core_cnt_ - 1; + assert(commit_ts == TransactionOperation::tx_op_failed_ts_ || + catalog_entry->schema_ != nullptr); + if (req.OpType() == OperationType::AddIndex) { - hd_res->SetError(CcErrorCode::REQUESTED_NODE_NOT_LEADER); - return true; + assert(catalog_entry->schema_ != nullptr && + catalog_entry->dirty_schema_ != nullptr); + for (const TableName &index_name : IndexesNotIn( + *catalog_entry->dirty_schema_, *catalog_entry->schema_)) + { + DropTableCcms(index_name, ng_id); + if (last_core) + { + CleanIndexRangesAndStatistics( + index_name, + *catalog_entry->dirty_schema_->StatisticsObject(), + ng_id); + } + } } - const TableName &base_table_name = *req.invalidate_table_name_; - CatalogKey catalog_key(base_table_name); - Iterator it = - TemplateCcMap::Find( - catalog_key); - CcEntry *cce = - it == End() ? nullptr : it->second; + // The record the template installs on every core: the table's current + // schema and no dirty schema, at the commit timestamp if the op has + // one, else at the current version. + if (shard_->core_id_ == 0) + { + CatalogRecord *rec = RequestRecord(req, false); + rec->Set(catalog_entry->schema_, + nullptr, + commit_ts != TransactionOperation::tx_op_failed_ts_ + ? commit_ts + : catalog_entry->schema_version_); + } + else + { + // Core 0 filled the record the template installs. + assert(req.Payload() != nullptr); + } - if (cce == nullptr) + if (last_core) { - // The catalog write lock was taken on the old leader by - // InvalidateTableCacheCompositeOp's acquire_all_lock_op_, and there - // is no leader-term re-check before this request is routed. If the - // node group failed over in between, this lands on a new leader - // whose catalog cc map has neither the entry nor the lock -- cc - // entries and locks do not survive failover. The ng_term check - // above still passes, because the new leader's term is valid. There - // is nothing cached to invalidate, so finish the request instead of - // dereferencing. - if (shard_->core_id_ < shard_->core_cnt_ - 1) - { - req.ResetCcm(); - MoveRequest(&req, shard_->core_id_ + 1); - return false; + catalog_entry->RollBackDirtySchema(commit_ts); + } + return TemplateCcMap::Execute(req); + } + + /** + * @brief Ends the op's hold on the schema's reader-writer control block, + * which its AcquireAllCc took on the native node group, so that runtime + * queries may cache the schema again. Part of both second phases, + * PostCommit and Rollback. + * + * @param req The request. + * @param table_key The table's catalog key. + */ + void FinishSchemaWriter(const PostWriteAllCc &req, + const CatalogKey &table_key) + { + if (!shard_->IsNative(req.NodeGroupId())) + { + return; + } + const TableName &tbl_name = table_key.Name(); + std::shared_ptr> schema_cntl = + shard_->FindSchemaCntl(tbl_name); + if (schema_cntl != nullptr) + { + schema_cntl->FinishWriter(req.Txn()); + shard_->DeleteSchemaCntl(tbl_name); + } + } + + /** + * @brief The node-level work of committing a schema op, done once, on core + * 0, before any core moves its cc maps. + * + * TRUNCATE TABLE's table is a new one, whose version is the commit-stage + * timestamp (UpsertTableOp::AssignCommitStageTs), so its schema is rebuilt + * at it before it is promoted. CREATE TABLE installs the table's and its + * indexes' initial ranges and statistics, and ALTER TABLE binds the + * table's statistics to the dirty schema. Neither has anything to do when + * the op is recovering from its commit log: replay installed the committed + * schema, leaving no dirty one, and the coordinator's node group reaches + * here under the write lock replay restored. + * + * @param req The PostCommit request. + * @param table_key The table's catalog key. + * @param catalog_entry The table's catalog entry. + */ + void CommitNodeLevelSchema(const PostWriteAllCc &req, + const CatalogKey &table_key, + CatalogEntry &catalog_entry) + { + const uint64_t commit_ts = req.CommitTs(); + // The version the dirty schema was installed with at the prepare + // stage, which the kv tables of a CREATE TABLE and so the table's + // initial ranges carry: a node that loads them from the store gets the + // same. + const uint64_t prepared_version = catalog_entry.dirty_schema_version_; + if (req.OpType() == OperationType::TruncateTable && + catalog_entry.dirty_schema_ != nullptr && + commit_ts > prepared_version) + { + TableSchema::uptr at_commit = + shard_->local_shards_.CreateTableSchemaFromImage( + table_key.Name(), + catalog_entry.dirty_schema_->SchemaImage(), + commit_ts); + if (catalog_entry.dirty_schema_->StatisticsObject()) + { + at_commit->BindStatistics( + catalog_entry.dirty_schema_->StatisticsObject()); } - hd_res->SetFinished(); - return true; + catalog_entry.SetDirtySchema(std::move(at_commit), commit_ts); } - // Only meaningful when the entry survived, i.e. no failover happened. - assert(cce->GetKeyLock()->HasWriteLock(req.Txn())); + if (req.OpType() == OperationType::CreateTable && + catalog_entry.dirty_schema_ != nullptr) + { + InitCreatedTableRange( + table_key.Name(), prepared_version, req.NodeGroupId()); + for (const TableName &index_name : + catalog_entry.dirty_schema_->IndexNames()) + { + InitCreatedTableRange( + index_name, prepared_version, req.NodeGroupId()); + } + shard_->InitTableStatistics(catalog_entry.dirty_schema_.get(), + cc_ng_id_); + } + else if (req.OpType() == OperationType::Update && + catalog_entry.dirty_schema_ != nullptr) + { + assert(catalog_entry.schema_->StatisticsObject()); + catalog_entry.dirty_schema_->BindStatistics( + catalog_entry.schema_->StatisticsObject()); + } + } - if (cce->PayloadStatus() == RecordStatus::Unknown) + /** + * @brief Moves a committed schema op's cc maps on this core: TRUNCATE + * TABLE's to the new table, DROP TABLE's away, CREATE TABLE's into being, + * and ALTER TABLE's to the new schema, dropping those of a dropped index. + * + * @param req The PostCommit request. + * @param table_key The table's catalog key. + * @param catalog_entry The table's catalog entry, still holding the current + * and the dirty schema. + * @param cce The table's cc entry on this core. + * @param ng_term The node group's leader term. + */ + void CommitTableCcms(const PostWriteAllCc &req, + const CatalogKey &table_key, + const CatalogEntry &catalog_entry, + CatalogCce &cce, + int64_t ng_term) + { + const NodeGroupId ng_id = req.NodeGroupId(); + const TableName &table_name = table_key.Name(); + const TableSchema *old_schema = catalog_entry.schema_.get(); + const TableSchema *new_schema = catalog_entry.dirty_schema_.get(); + const uint64_t dirty_version = catalog_entry.dirty_schema_version_; + switch (req.OpType()) { - const CatalogEntry *catalog_entry = - shard_->GetCatalog(base_table_name, req.NodeGroupId()); - if (catalog_entry != nullptr) + case OperationType::TruncateTable: + { + assert(table_name.Engine() == TableEngine::EloqKv); + // Only the base table's cc map moves to the new schema: EloqKV + // tables have no indexes. A truncate of a table with indexes would + // have to move each index's map too, since CleanCcm empties them, + // and its data step, which drops only the base table's kv table, + // would have to reset ranges and statistics too + // (UpsertTableOp::TruncateUntilCommitLog). + // A remote tx is allowed to acquire write intents/locks and drop a + // table, even if the table's schema has not been initialized at + // this node. The earlier acquiring-write-intent request creates a + // schema cc entry in the catalog cc map and a node-level schema + // view. The version timestamp of the schema is 0, if the schema is + // uninitialized (null). Or, the current schema must not be null. + assert(catalog_entry.schema_version_ == 0 || old_schema != nullptr); + assert(new_schema->Version() == dirty_version); + shard_->UpdateCcmSchema(table_name, + ng_id, + catalog_entry.dirty_schema_.get(), + dirty_version); + if (shard_->core_id_ == 0 && + !shard_->GetSubscribedStandbys().empty()) + { + ForwardTruncateToStandbys( + req, table_key, catalog_entry, ng_term); + } + break; + } + case OperationType::DropTable: + { + // As for truncate, the current schema may be uninitialized here. + assert(catalog_entry.schema_version_ == 0 || old_schema != nullptr); + DropTableCcms(table_name, ng_id); + if (old_schema != nullptr) { - assert(catalog_entry->schema_); - // upload catalog record - cce->payload_.PassInCurrentPayload( - std::make_unique()); - cce->payload_.cur_payload_->Set(catalog_entry->schema_, - catalog_entry->dirty_schema_, - catalog_entry->schema_version_); - cce->SetCommitTsPayloadStatus(catalog_entry->schema_version_, - RecordStatus::Normal); + for (const TableName &index_name : old_schema->IndexNames()) + { + DropTableCcms(index_name, ng_id); + } } - else + break; + } + case OperationType::CreateTable: + { + assert(new_schema != nullptr); + shard_->CreateOrUpdatePkCcMap( + table_name, new_schema, ng_id, true, true); + for (const TableName &index_name : new_schema->IndexNames()) { - shard_->FetchCatalog( - base_table_name, req.NodeGroupId(), ng_term, &req); - return false; + shard_->CreateOrUpdateSkCcMap(index_name, new_schema, ng_id); } + break; } - - CatalogRecord *catalog_rec = cce->payload_.cur_payload_.get(); - const TableSchema *table_schema = catalog_rec->Schema(); - - TableName base_range_name(base_table_name.StringView(), - TableType::RangePartition, - base_table_name.Engine()); - shard_->DropCcm(base_table_name, cc_ng_id_); - shard_->DropCcm(base_range_name, cc_ng_id_); - - for (TableName index_table_name : table_schema->IndexNames()) + default: { - TableName index_range_name(index_table_name.StringView(), - TableType::RangePartition, - index_table_name.Engine()); - shard_->DropCcm(index_table_name, cc_ng_id_); - shard_->DropCcm(index_range_name, cc_ng_id_); + // ALTER TABLE: the table's cc maps take the new schema; an index + // the op dropped loses its cc maps. An add index's new indexes are + // not in the old schema and are left alone: their cc maps point at + // the dirty schema since PrepareCommit (SetAddedIndexCcms), at the + // op's version, and CommitDirtySchema promotes that schema object + // as it is. + assert(old_schema != nullptr && new_schema != nullptr); + SetTableCcms(table_name, new_schema, ng_id, dirty_version, false); + if (req.OpType() == OperationType::AddIndex || + req.OpType() == OperationType::DropIndex || + req.OpType() == OperationType::Update) + { + const std::vector new_index_names = + new_schema->IndexNames(); + for (const TableName &old_index_name : old_schema->IndexNames()) + { + if (std::find(new_index_names.begin(), + new_index_names.end(), + old_index_name) != new_index_names.end()) + { + SetTableCcms(old_index_name, + new_schema, + ng_id, + dirty_version, + false); + } + else + { + assert(req.OpType() == OperationType::DropIndex); + DropTableCcms(old_index_name, ng_id); + } + } + } + break; + } } - if (shard_->core_id_ < shard_->core_cnt_ - 1) + UpdateTableLocks( + table_name.StringView(), dirty_version, cce.GetKeyLock()); + } + + /** + * @brief Cleans the node-level structures of the tables and indexes a + * committed schema op removed: their ranges and statistics. Done once, on + * the last core, after every core dropped their cc maps. + * + * @param req The PostCommit request. + * @param table_key The table's catalog key. + * @param catalog_entry The table's catalog entry, still holding the current + * and the dirty schema. + */ + void CleanRemovedTables(const PostWriteAllCc &req, + const CatalogKey &table_key, + const CatalogEntry &catalog_entry) + { + const NodeGroupId ng_id = req.NodeGroupId(); + const TableName &table_name = table_key.Name(); + const TableSchema *old_schema = catalog_entry.schema_.get(); + if (req.OpType() == OperationType::DropTable) { - req.ResetCcm(); - MoveRequest(&req, shard_->core_id_ + 1); - return false; + shard_->CleanTableStatistics(table_name, cc_ng_id_); + if (!table_name.IsHashPartitioned()) + { + shard_->CleanTableRange(RangeTableNameOf(table_name), ng_id); + if (old_schema != nullptr) + { + for (const TableName &index_name : old_schema->IndexNames()) + { + shard_->CleanTableRange(RangeTableNameOf(index_name), + ng_id); + } + } + } } - else + else if (req.OpType() == OperationType::AddIndex || + req.OpType() == OperationType::DropIndex) { - shard_->CleanTableRange(base_range_name, cc_ng_id_); - shard_->CleanTableStatistics(base_table_name, cc_ng_id_); - - for (TableName index_table_name : table_schema->IndexNames()) + for (const TableName &index_name : + IndexesNotIn(*old_schema, *catalog_entry.dirty_schema_)) { - TableName index_range_name(index_table_name.StringView(), - TableType::RangePartition, - index_table_name.Engine()); - shard_->CleanTableRange(index_range_name, cc_ng_id_); - shard_->CleanTableStatistics(index_table_name, cc_ng_id_); + CleanIndexRangesAndStatistics( + index_name, *old_schema->StatisticsObject(), ng_id); } - - hd_res->SetFinished(); - return true; } } - TableType Type() const override + /** + * @brief The name of the range-partition table that holds a table's + * ranges. + * + * @param table A base or index table. + * @return A view of @p table's name, valid while @p table is. + */ + static TableName RangeTableNameOf(const TableName &table) { - return TableType::Catalog; + return TableName{ + table.StringView(), TableType::RangePartition, table.Engine()}; } - bool HasWriteLock(const TableName &table_name, uint32_t node_group_id) + /** + * @brief The indexes of one schema that another schema does not have. + * + * @param schema The schema whose indexes are listed. + * @param other The schema they are compared with. + * @return The names of @p schema's indexes absent from @p other, in + * @p schema's order. + */ + static std::vector IndexesNotIn(const TableSchema &schema, + const TableSchema &other) { - CatalogKey table_key{TableName{ - table_name.StringView(), TableType::Primary, table_name.Engine()}}; - TxKey catalog_tx_key(&table_key); - auto it = Find(table_key); - if (it == End()) + std::vector names = schema.IndexNames(); + const std::vector other_names = other.IndexNames(); + names.erase(std::remove_if(names.begin(), + names.end(), + [&other_names](const TableName &name) + { + return std::find(other_names.begin(), + other_names.end(), + name) != + other_names.end(); + }), + names.end()); + return names; + } + + /** + * @brief The catalog record of a post-write-all request. + * + * A local request references the requester's record. A remote one + * carries it serialized, or carries none (PostCommit, Rollback); a record + * the request owns holds it, which the later cores read as the request's + * payload. + * + * @param req The request. + * @param decode Whether to decode a remote payload; a DROP TABLE's + * PrepareCommit, a PostCommit and a Rollback carry nothing to decode. A + * remote request at commit timestamp 0 has no payload string at all. + * @return The record. + */ + static CatalogRecord *RequestRecord(PostWriteAllCc &req, bool decode = true) + { + if (req.Payload() != nullptr) { - return false; + return static_cast(req.Payload()); } - - CcEntry *catalog_cce = - it->second; - const auto keylock = catalog_cce->GetKeyLock(); - if (keylock == nullptr) + assert(!decode || req.PayloadStr() != nullptr); + std::unique_ptr decoded_rec = + std::make_unique(); + if (decode) { - return false; + size_t offset = 0; + decoded_rec->Deserialize(req.PayloadStr()->data(), offset); } - return keylock->HasWriteLock(); + CatalogRecord *rec = decoded_rec.get(); + req.SetDecodedPayload(std::move(decoded_rec)); + return rec; } /** - * GetTableSchema stored in each shard's CatalogCcMap. Used to initialize - * the Pk and Sk ccmaps. - * @param table_name - * @param cc_ng_id - * @param cc_ng_term - * @param requester - * @return InitCcmResult describing the schema lookup outcome. + * @brief Drops a table's cc map and, for a range-partitioned table, the cc + * map of its ranges, on this core. + * + * @param table A base or index table. + * @param ng_id The node group. */ - InitCcmResult GetTableSchema(const TableName &table_name, - NodeGroupId cc_ng_id, - int64_t cc_ng_term, - CcRequestBase *requester) + void DropTableCcms(const TableName &table, NodeGroupId ng_id) { - const TableName base_table_name{table_name.GetBaseTableNameSV(), - TableType::Primary, - table_name.Engine()}; - - CatalogKey table_key{base_table_name}; - - bool emplace = false; - Iterator it = FindEmplace(table_key, &emplace, true, false); - CcEntry *catalog_cce = - it->second; - - if (catalog_cce->PayloadStatus() == RecordStatus::Unknown) + shard_->DropCcm(table, ng_id); + if (!table.IsHashPartitioned()) { - const CatalogEntry *catalog_entry = - shard_->GetCatalog(table_key.Name(), cc_ng_id); - - // If the read toward the catalog cc entry acquires the read - // lock but the cc entry does not contain the value, checks if - // the catalog has been constructed at this node. If so, turns - // this request into a read-outside request that installs the - // value in the cc entry. - if (catalog_entry != nullptr && catalog_entry->schema_version_ > 0) - { - if (catalog_entry->schema_ != nullptr) - { - { -#ifdef STATISTICS - assert(requester != nullptr); - // Initialize table statistics before create ccmap. - if (Sharder::Instance().GetLocalCcShards()->store_hd_ != - nullptr && - !table_name.IsHashPartitioned() && - !shard_->LoadRangesAndStatisticsNx( - catalog_entry->schema_.get(), - cc_ng_id, - cc_ng_term, - requester)) - { - return InitCcmResult::Retry(); - } -#endif - } - - // upload catalog record - catalog_cce->payload_.PassInCurrentPayload( - std::make_unique()); - catalog_cce->payload_.cur_payload_->Set( - catalog_entry->schema_, - catalog_entry->dirty_schema_, - catalog_entry->schema_version_); - catalog_cce->SetCommitTsPayloadStatus( - catalog_entry->schema_version_, RecordStatus::Normal); - } - else - { - catalog_cce->SetCommitTsPayloadStatus( - catalog_entry->schema_version_, RecordStatus::Deleted); - } - } - else - { - // Global CatalogEntry does not exist. - shard_->FetchCatalog( - table_key.Name(), cc_ng_id, cc_ng_term, requester); - return InitCcmResult::Retry(); - } + shard_->DropCcm(RangeTableNameOf(table), ng_id); } + } - const auto keylock = catalog_cce->GetKeyLock(); - if (keylock != nullptr && keylock->HasWriteLock()) + /** + * @brief Points a table's cc map, and the cc map of its ranges if this + * node has them loaded, at a schema, on this core. + * + * @param table A base or index table. + * @param schema The schema the cc maps take. + * @param ng_id The node group. + * @param range_version The version the range cc map takes. + * @param is_create Whether to create a cc map that does not exist. + */ + void SetTableCcms(const TableName &table, + const TableSchema *schema, + NodeGroupId ng_id, + uint64_t range_version, + bool is_create) + { + if (table.IsBase()) { - DLOG(WARNING) - << "InitCcm failure, target table: " << table_name.StringView() - << " has write lock on catalog_cc_map, which means, it is " - "being modified on this shard."; - return InitCcmResult::Failure(CcErrorCode::READ_CATALOG_CONFLICT); + shard_->CreateOrUpdatePkCcMap(table, schema, ng_id, is_create); } - - RecordStatus payload_status = catalog_cce->PayloadStatus(); - if (payload_status == RecordStatus::Deleted) + else { - return InitCcmResult::Failure( - CcErrorCode::REQUESTED_TABLE_NOT_EXISTS); + shard_->CreateOrUpdateSkCcMap(table, schema, ng_id, is_create); + } + if (table.IsHashPartitioned()) + { + return; + } + const TableName range_table = RangeTableNameOf(table); + if (shard_->GetTableRangesForATable(range_table, ng_id) != nullptr) + { + shard_->CreateOrUpdateRangeCcMap( + range_table, schema, ng_id, range_version, is_create); } - - assert(payload_status == RecordStatus::Normal); - CatalogRecord *catalog_rec = catalog_cce->payload_.cur_payload_.get(); - assert(catalog_rec != nullptr); - assert(catalog_rec->Schema() != nullptr); - - return InitCcmResult::Success(catalog_rec->Schema()); } - std::tuple ReadTable( - const TableName &table_name, - uint32_t node_group_id, - int64_t ng_term, - TxNumber tx_number) + /** + * @brief Cleans the node-level structures of an index that is gone: its + * ranges, and its entry in the table's statistics. Done once, on the last + * core, after every core dropped the index's cc maps. + * + * @param index The index. + * @param statistics The table's statistics. + * @param ng_id The node group. + */ + void CleanIndexRangesAndStatistics(const TableName &index, + Statistics &statistics, + NodeGroupId ng_id) { - bool read_success = false; - uint64_t schema_version = 0; - NonBlockingLock *lock_ptr = nullptr; - - std::string_view table_name_sv = table_name.StringView(); - absl::string_view absl_table_name_sv(table_name_sv.data(), - table_name_sv.size()); - auto lock_it = table_locks_.find(absl_table_name_sv); - if (lock_it == table_locks_.end()) - { - CatalogKey table_key{ - txservice::TableName{table_name_sv, - txservice::TableType::Primary, - table_name.Engine()}}; - TxKey catalog_tx_key(&table_key); - auto it = FindEmplace(table_key); - - CcEntry *catalog_cce = - it->second; - CcPage *catalog_ccp = - it.GetPage(); - - const CatalogEntry *catalog_entry = - shard_->GetCatalog(table_key.Name(), node_group_id); + shard_->CleanTableRange(RangeTableNameOf(index), ng_id); + statistics.DropIndex(index); + } - if (catalog_entry == nullptr || catalog_entry->schema_ == nullptr) - { - // The catalog entry hasn't been initialized, try to load it - // from data store. - shard_->FetchCatalog( - table_name, node_group_id, ng_term, nullptr); - return {CcErrorCode::READ_CATALOG_FAIL, nullptr, 0}; - } + /** + * @brief Installs, on this node, the single initial range of a base or + * index table that CREATE TABLE made. + * + * The range spans the whole key space (a null start key is negative + * infinity), and its partition id is derived from the table's name; the + * sequences table's is 0. + * + * @param table The base or index table. + * @param version The range's version: the version the table's kv tables + * were created at. + * @param ng_id The node group. + */ + void InitCreatedTableRange(const TableName &table, + uint64_t version, + NodeGroupId ng_id) + { + const std::string_view name = table.StringView(); + const int partition_id = + name == "./mysql/sequences" + ? 0 + : static_cast(std::hash()(name) & 0xFFF); + std::vector range_init_vec; + range_init_vec.emplace_back(TxKey(), partition_id, version); + shard_->local_shards_.InitTableRanges( + RangeTableNameOf(table), range_init_vec, ng_id, true); + } - assert(catalog_entry != nullptr && - catalog_entry->schema_version_ > 0); - assert(catalog_entry->schema_ != nullptr); + /** + * @brief Gives an empty range, which this node group owns, its (empty) + * slices, allocated on the table-ranges heap. + * + * @param range The range. + * @param ng_id The node group. + */ + void InitEmptyRangeSlices(TableRangeEntry &range, NodeGroupId ng_id) + { + std::unique_lock heap_lk( + shard_->local_shards_.table_ranges_heap_mux_); + bool is_override_thd = mi_is_override_thread(); + mi_threadid_t prev_thd = mi_override_thread( + shard_->local_shards_.GetTableRangesHeapThreadId()); + mi_heap_t *prev_heap = + mi_heap_set_default(shard_->local_shards_.GetTableRangesHeap()); - schema_version = catalog_entry->schema_version_; +#if defined(WITH_JEMALLOC) + uint32_t prev_arena; + JemallocArenaSwitcher::ReadCurrentArena(prev_arena); + auto table_range_arena_id = + shard_->local_shards_.GetTableRangesArenaId(); + JemallocArenaSwitcher::SwitchToArena(table_range_arena_id); +#endif - lock_ptr = - &catalog_cce->GetOrCreateKeyLock(shard_, this, catalog_ccp); - auto [new_lock_it, _] = table_locks_.try_emplace( - std::string(table_name_sv), lock_ptr, schema_version); - read_success = - new_lock_it->second.first->AcquireReadLockFast(tx_number); + std::vector slices; + range.InitRangeSlices( + std::move(slices), ng_id, false, true, UINT64_MAX, false); - if (read_success && - catalog_cce->PayloadStatus() == RecordStatus::Unknown) - { - assert(schema_version == new_lock_it->second.second); - // upload catalog record - catalog_cce->payload_.PassInCurrentPayload( - std::make_unique()); - catalog_cce->payload_.cur_payload_->Set( - catalog_entry->schema_, - catalog_entry->dirty_schema_, - schema_version); - catalog_cce->SetCommitTsPayloadStatus(schema_version, - RecordStatus::Normal); - } + mi_heap_set_default(prev_heap); + if (is_override_thd) + { + mi_override_thread(prev_thd); } else { - read_success = - lock_it->second.first->AcquireReadLockFast(tx_number); - lock_ptr = lock_it->second.first; - schema_version = lock_it->second.second; + mi_restore_default_thread_id(); } - if (read_success) - { - assert(lock_ptr != nullptr); - assert(schema_version > 0); -#ifndef NDEBUG - shard_->UpsertLockHoldingTx(tx_number, - ng_term, - nullptr, - false, - node_group_id, - table_name_.Type()); +#if defined(WITH_JEMALLOC) + JemallocArenaSwitcher::SwitchToArena(prev_arena); #endif - return {CcErrorCode::NO_ERROR, lock_ptr, schema_version}; - } - else - { - return {CcErrorCode::READ_CATALOG_CONFLICT, nullptr, 0}; - } + heap_lk.unlock(); + } + + /** + * @brief Forwards a committed TRUNCATE TABLE to the standby nodes + * subscribed to this node group. + * + * @param req The PostCommit request. + * @param table_key The table's catalog key. + * @param catalog_entry The table's catalog entry, still holding the old + * and the new schema. + * @param ng_term The node group's leader term. + */ + void ForwardTruncateToStandbys(const PostWriteAllCc &req, + const CatalogKey &table_key, + const CatalogEntry &catalog_entry, + int64_t ng_term) + { + auto forward_entry_ptr = std::make_unique(); + StandbyForwardEntry *forward_entry = forward_entry_ptr.get(); + auto *forward_req = &forward_entry->Request(); + forward_req->set_primary_leader_term(ng_term); + forward_req->set_tx_number(req.Txn()); + forward_req->set_table_name(table_name_.String()); + forward_req->set_table_type( + remote::ToRemoteType::ConvertTableType(table_name_.Type())); + forward_req->set_table_engine( + remote::ToRemoteType::ConvertTableEngine(table_name_.Engine())); + forward_req->set_key_shard_code(cc_ng_id_ << 10); + std::string key_str; + table_key.Serialize(key_str); + forward_req->set_key(std::move(key_str)); + std::string rec_str; + + assert(catalog_entry.schema_ != nullptr); + assert(catalog_entry.dirty_schema_ != nullptr); + assert(catalog_entry.dirty_schema_version_ > 0); + + CatalogRecord temp_rec; + temp_rec.SetSchemaImage(catalog_entry.schema_->SchemaImage()); + temp_rec.SetDirtySchemaImage( + catalog_entry.dirty_schema_->SchemaImage()); + assert(!temp_rec.SchemaImage().empty()); + assert(!temp_rec.DirtySchemaImage().empty()); + + temp_rec.Serialize(rec_str); + forward_req->add_cmd_list(std::move(rec_str)); + + forward_req->set_commit_ts(catalog_entry.dirty_schema_version_); + forward_req->set_schema_version(catalog_entry.schema_version_); + shard_->ForwardStandbyMessage(forward_entry_ptr.release()); } -private: void UpdateTableLocks(std::string_view table_name_sv, uint64_t schema_version, NonBlockingLock *lock_ptr) diff --git a/tx_service/include/cc/cc_handler.h b/tx_service/include/cc/cc_handler.h index 74e7764b9..313b694fe 100644 --- a/tx_service/include/cc/cc_handler.h +++ b/tx_service/include/cc/cc_handler.h @@ -130,6 +130,26 @@ class CcHandler CcOperation cc_op, bool abort_if_oom) = 0; + /** + * @brief Post-processes a write key on every core of a node group, as + * @p post_write_type says: for example, PrepareCommit installs a dirty + * record, PostCommit commits it and releases the write lock or intent that + * AcquireWriteAll() took. + * + * @param table_name Name of the table the key belongs to + * @param key The written key + * @param rec The record to install + * @param ng_id The node group the key belongs to + * @param tx_number The tx that holds the write lock or intent + * @param tx_term The term of the tx's node group + * @param command_id The tx's command id + * @param ts The commit timestamp; 0 only for a Rollback, which then moves + * no commit timestamp, or a DowngradeLock + * @param hres Result handler of the request + * @param op_type The type of the write + * @param post_write_type The kind of post-processing + * @param abort_if_oom Whether or not to abort the request if out of memory + */ virtual void PostWriteAll(const TableName &table_name, const TxKey &key, TxRecord &rec, diff --git a/tx_service/include/cc/cc_req_misc.h b/tx_service/include/cc/cc_req_misc.h index 052b36679..3b9eef429 100644 --- a/tx_service/include/cc/cc_req_misc.h +++ b/tx_service/include/cc/cc_req_misc.h @@ -1523,19 +1523,49 @@ struct RestoreCcMapCc : public CcRequestBase bthread::Mutex req_mux_{}; }; +/** + * @brief The shard's eviction pass: evicts entries that are checkpointed, + * unlocked and unpinned from the LRU chain, and resumes requests parked for + * memory. + * + * One instance per shard (CcShard::shard_clean_cc_). It is scheduled when a + * request parks or a FindEmplace finds the heap full, re-enqueues itself + * behind every batch it releases, and stays in use while deferred between + * passes, so at most one pass or retry is ever pending. The decision rules + * are documented with Execute(). + */ struct ShardCleanCc : public CcRequestBase { public: - ShardCleanCc() : free_count_(0) - { - } + ShardCleanCc() = default; ShardCleanCc(ShardCleanCc &&rhs) = delete; bool Execute(CcShard &ccs) override; private: - size_t free_count_{0}; + /** Set from the first execution of a pass until the pass ends. */ + bool pass_started_{false}; + /** Entries freed by this pass so far. */ + size_t freed_in_pass_{0}; + /** + * Bytes this pass reclaimed so far, summed over its eviction batches. A + * pass that ends under FLAGS_shard_clean_release_kb is not keeping up: + * it asks for a checkpoint and its retry backs off. + */ + int64_t reclaimed_in_pass_{0}; + /** + * CcShard::MemoryParkCount() when this pass started or last released a + * batch, whichever is later; a later park means a request found the heap + * at its hard limit since then. + */ + uint64_t park_snapshot_{0}; + /** + * Bytes this pass reclaimed since it started or last released a batch; + * after a limit hit the next release waits for + * FLAGS_shard_clean_release_kb of them. + */ + int64_t reclaimed_since_release_{0}; }; struct FetchTableRangeSizeCc : public CcRequestBase diff --git a/tx_service/include/cc/cc_request.h b/tx_service/include/cc/cc_request.h index ec9cc363f..dec38c66e 100644 --- a/tx_service/include/cc/cc_request.h +++ b/tx_service/include/cc/cc_request.h @@ -3065,9 +3065,9 @@ struct CkptTsCc : public CcRequestBase /** * @brief The write-lock-holding tx that determined the node group's - * checkpoint timestamp, i.e. the tx with the minimum first-write-lock ts - * across all shards. txn_ == 0 when no such tx exists (the checkpoint ts - * came from the clock). + * checkpoint timestamp, i.e. the tx with the minimum wlock_ts (latest + * write-lock time) across all shards. txn_ == 0 when no such tx exists (the + * checkpoint ts came from the clock). */ PinningTxInfo GetPinningTx() const { @@ -3186,9 +3186,12 @@ struct CkptTsCc : public CcRequestBase std::vector standby_msg_seq_id_vec_; std::vector subscribed_node_ids_; std::vector heap_full_vec_; - // Per-shard tx (and its first-write-lock ts) that produced the shard's - // minimum in ActiveTxMinTs. 0 / UINT64_MAX when the shard has no - // write-lock-holding tx. Only filled on the leader path. + /** + * @brief Per-shard tx (and, in pinning_wlock_ts_vec_, its latest + * write-lock ts) that produced the shard's minimum in ActiveTxMinTs. + * 0 / UINT64_MAX when no tx in the shard holds the checkpoint back. Only + * filled on the leader path. + */ std::vector pinning_txn_vec_; std::vector pinning_wlock_ts_vec_; NodeGroupId cc_ng_id_; @@ -4719,6 +4722,7 @@ struct ReplayLogCc : public TemplatedCcRequest range_splitting_ = range_splitting; local_on_fly_cnt_ = nullptr; is_lock_recovery_ = is_lock_recovery; + is_cluster_gate_ = false; upsert_kv_err_code_ = {true, CcErrorCode::NO_ERROR}; parse_cc_ = parse_cc; split_ranges_ = nullptr; @@ -4917,6 +4921,14 @@ struct ReplayLogCc : public TemplatedCcRequest return range_split_started_; } + /** + * @brief Whether a split of the table that the recovering node group + * coordinates is in flight: it holds the table's catalog read lock on this + * node group, so no schema op holds the table's write lock here. + * + * @param table_name The base table. + * @return true if such a split is in flight. + */ bool RangeSplitting(const TableName &table_name) const { return range_splitting_ && @@ -4967,6 +4979,27 @@ struct ReplayLogCc : public TemplatedCcRequest return is_lock_recovery_; } + /** + * @brief Marks the request as the replay of a cluster gate slot, whose + * blob is a ClusterGateLogMessage, rather than of a cluster scale log. + * + * Call after Reset(), which clears the mark. + */ + void SetClusterGate() + { + is_cluster_gate_ = true; + } + + /** + * @brief Whether the request replays a cluster gate slot. + * + * @return true for a cluster gate slot, false for any other log. + */ + bool IsClusterGate() const + { + return is_cluster_gate_; + } + std::pair &UpsertKvErrCode() { return upsert_kv_err_code_; @@ -4992,6 +5025,7 @@ struct ReplayLogCc : public TemplatedCcRequest std::atomic_uint32_t *local_on_fly_cnt_{nullptr}; bool *recovery_error_; bool is_lock_recovery_; + bool is_cluster_gate_{false}; ///< See SetClusterGate(). uint16_t first_core_; uint16_t next_core_{UINT16_MAX}; const struct TableSchema *table_schema_{nullptr}; @@ -6443,9 +6477,16 @@ struct KickoutCcEntryCc : public TemplatedCcRequest // Target buckets to be cleaned if clean type is CleanBucketData. std::vector *bucket_ids_{nullptr}; - // 1. CleanForAlterTable: kickout all cce with commit ts <= clean_ts_ - // 2. CleanForTruncateTable: the commit ts of UpsertTableOp. We truncate ccm - // only if catalog_entry->DirtyVersion() == clean_ts_. + /** + * @brief The timestamp that bounds a clean. + * + * CleanForAlterTable: kicks out every cc entry committed at or below it. + * CleanCcm (DROP and TRUNCATE TABLE, DROP INDEX): the schema op's commit + * ts, which for a table op is its commit-stage timestamp P, not the dirty + * schema's version D. A node group whose catalog entry's schema version is + * at or above it, or a cc map whose schema ts is, treats the request as + * outdated and skips it. + */ uint64_t clean_ts_{0}; // Only used by migration int32_t range_id_{INT32_MAX}; diff --git a/tx_service/include/cc/cc_shard.h b/tx_service/include/cc/cc_shard.h index 6de474ea1..2978fb688 100644 --- a/tx_service/include/cc/cc_shard.h +++ b/tx_service/include/cc/cc_shard.h @@ -131,13 +131,36 @@ struct TxLockInfo { using uptr = std::unique_ptr; + /** + * @brief Whether a write lock on a table of this type holds the + * checkpoint back. + * + * Data tables do: ckpt_ts must stay below every commit still pending + * (07-durability-and-recovery.md). So do catalog entries: a DDL's catalog + * write lock orders it against DML, and a checkpoint taken while it is + * held must stay below it, so that the DDL's log record lands above the + * log group's min_ckpt_ts_ (EloqDoc's change data capture relies on it; + * 03-concurrency-control.md). Range, bucket and cluster-config write + * locks do not: split and migration operations hold them for long + * stretches, and nothing downstream needs them ordered. + * + * @param type The table type of the locked entry. + * @return True if the write lock must hold the checkpoint back. + */ + static bool WriteLockHoldsBackCheckpoint(TableType type) + { + return type != TableType::RangePartition && + type != TableType::RangeBucket && + type != TableType::ClusterConfig; + } + TxLockInfo() = delete; explicit TxLockInfo(int64_t tx_coord_term) : tx_coord_term_(tx_coord_term), wlock_ts_(0), + holds_ckpt_wlock_(false), last_recover_ts_(0), cce_list_(), - table_type_(TableType::Primary), next_(nullptr) { } @@ -156,17 +179,39 @@ struct TxLockInfo { tx_coord_term_ = tx_term; wlock_ts_ = 0; + holds_ckpt_wlock_ = false; last_recover_ts_ = 0; cce_list_.clear(); - table_type_ = TableType::Primary; next_ = nullptr; } // tx coordinator's term. int64_t tx_coord_term_; - // The timestamp when the tx acquires the first write lock in the cc shard. - // If tx has not acquired any write lock, set wlock_ts_ to 0. + /** + * @brief When the tx last acquired a write lock in this shard, on any + * table; 0 if it never took one. + * + * While the tx holds a write lock that counts (holds_ckpt_wlock_), + * ActiveTxMinTs keeps the checkpoint below this. That is the guarantee; + * outside the exclusive section the checkpoint may pass the tx. DML + * chooses its commit timestamp after its last write lock, and each grant + * forces the commit above the granting shard's clock (last_vali_ts), so + * for DML this is also the tightest lower bound on the commit. It is the + * most recent sign that the tx is alive. Also read by ActiveTxMaxTs, by + * orphan-lock recovery (the log search's lower bound and the + * committed-write window), and by the checkpoint stall warning. A + * downgrade leaves it unchanged: it acquires nothing. + */ uint64_t wlock_ts_; + /** + * @brief Whether the tx holds, in this shard, a write lock that holds the + * checkpoint back (WriteLockHoldsBackCheckpoint). + * + * Set when such a write lock is taken; cleared by OnWriteLockDowngraded + * once none remains. A tx's locks span many tables, so this is decided + * per write lock, from the locked entry's table. + */ + bool holds_ckpt_wlock_; // The last time when the tx is recovered or the tx acquired the latest // lock. Gates CheckRecoverTx to once per tx per 5s. The ctor/Reset zero // is never observable: UpsertLockHoldingTx — the only path that puts an @@ -175,9 +220,6 @@ struct TxLockInfo uint64_t last_recover_ts_; // A list of cc entries on which the tx has acquired write/read locks. absl::flat_hash_set cce_list_; - // This cc map type is used to skip the meta table(such as: catalog, range) - // during get ActiveTxMinTs() - TableType table_type_; std::unique_ptr next_{nullptr}; }; @@ -219,6 +261,12 @@ class CcShardHeap { static constexpr double high_water = 0.8; static constexpr double utilization = 0.8; + /** + * Headroom an eviction pass works towards once it has started: the + * cleaner keeps evicting until this fraction of the limit is free, so the + * requests it releases have room, and stops evicting above it. + */ + static constexpr double clean_target_free_ratio = 0.05; public: CcShardHeap(CcShard *cc_shard, size_t limit); @@ -235,6 +283,22 @@ class CcShardHeap bool NeedCleanShard(int64_t &alloc, int64_t &commit) const; + /** + * @brief Whether allocated memory is still above the eviction target. + * + * Full() is the hard limit at which requests park; this is the lower + * target an eviction pass works towards once started. A pass may end + * short of it when no more entries are evictable. + * @param alloc Bytes currently allocated from this heap. + * @return True while less than clean_target_free_ratio of the limit is + * free. + */ + bool NeedMoreCleaning(int64_t alloc) const + { + return alloc > static_cast(memory_limit_ * + (1.0 - clean_target_free_ratio)); + } + bool NeedDefragment(int64_t *alloc, int64_t *commit) const; mi_heap_t *Heap() const @@ -396,8 +460,15 @@ class CcShard void Enqueue(uint32_t thd_id, uint32_t shard_code, CcRequestBase *req); /** - * @brief Puts a cc request into the shard's request wait list until memory - * is avaliable. + * @brief Parks a request that found the shard heap full and schedules the + * cleaner if it is idle. + * + * Counts the park so the cleaner can tell that a request found the heap at + * its limit after its last release, whoever parked (see + * MemoryParkCount()). A cleaner that is + * already sweeping or deferred is left alone: the request joins the wait + * list and the pass or retry in progress covers it. Must run on the + * owning shard. * @param req The pointer to the cc request. */ void EnqueueWaitListIfMemoryFull(CcRequestBase *req); @@ -414,24 +485,111 @@ class CcShard size_t WaitListSizeForMemory(); + /** + * @brief Schedules the cleaner unless it is already in use. + * + * In use covers queued, executing and deferred (see DeferCleanPass()). In + * all three cases the pass or retry already pending sees the current + * state when it runs, so nothing is recorded. Must run on the owning + * shard. + */ void WakeUpShardCleanCc(); /** - * @brief Consumes the wake-up that arrived while the eviction pass was - * already running. + * @brief Number of requests parked for memory since the shard started. + * + * Every park is a request that found the heap at the hard limit. The + * cleaner snapshots the count when it releases a batch; a later change + * means a request hit the limit after that release, whether one of the + * released ones or a new one, which is the only distinction it needs. + * @return The monotonically increasing park count. + */ + uint64_t MemoryParkCount() const + { + return memory_park_count_; + } + + /** + * @brief Consumes the mark left by a flush or split that completed while + * an eviction pass was running. + * + * The pass in progress is not rewound. It finishes, then runs once more + * from the LRU head to reach entries the flush made evictable behind its + * cursor. The cleaner also clears the mark when it starts a pass, since a + * new pass visits every page anyway. + * @return Whether such a completion was recorded. + */ + bool TakeCleanRescanRequest() + { + return std::exchange(clean_rescan_requested_, false); + } + + /** + * @brief Defers the cleaner's next pass after one that ended with + * requests still parked and nothing more it could evict. + * + * The cleaner stays in use, so parkers only join the wait list. The timer + * thread enqueues it again once the deadline passes + * (FireCleanRetryIfDue()), or a flush or split completion does so earlier + * (OnDirtyDataFlushed()). Must run on the owning shard. + * @param productive Whether the pass reclaimed at least + * FLAGS_shard_clean_release_kb (see ShardCleanCc::Execute()). A pass + * under that doubles the delay up to kCleanRetryMaxBackoff steps; one at + * or above it resets the delay to the base. + */ + void DeferCleanPass(bool productive); + + /** + * @brief Clears the retry backoff when the cleaner goes idle. + * + * The backoff paces the retries of one episode of memory pressure. Once a + * pass ends with nothing parked that episode is over, so the next one + * starts from the base delay instead of inheriting a stale 1.6 s. Must + * run on the owning shard. + */ + void ResetCleanBackoff() + { + clean_backoff_ = 0; + } + + /** + * @brief Enqueues the deferred cleaner once its deadline has passed. * - * A wake-up cannot be delivered while ShardCleanCc is in use, and the pass - * in flight may have already scanned past the entries the wake-up is about - * -- the memory freed by a checkpoint flush that lands mid-pass is a case - * in point. Since that pass stops itself once it frees nothing, the - * wake-up would be lost and the shard would hold its parked requests until - * some later flush. Recording it lets the pass re-run instead. + * Called from the LocalCcShards timer thread on every tick. Claiming the + * deadline with a compare-and-swap keeps exactly one retry outstanding: + * whichever of this and OnDirtyDataFlushed() clears it enqueues the + * cleaner, and the other does nothing. + * @param now_us Current steady-clock time in microseconds. + */ + void FireCleanRetryIfDue(uint64_t now_us); + + /** + * @brief Wakes the cleaner for state that just became evictable: a + * checkpoint flush on this shard or a committed range split. * - * @return Whether a wake-up was dropped since the last call. + * A deferred cleaner runs now with its backoff reset. A cleaner that is + * queued or mid-pass is marked to run one more pass after the current + * one. An idle one has nothing parked and is left alone. Must run on the + * owning shard. */ - bool TakeShardCleanCcWakeUp() + void OnDirtyDataFlushed(); + + /** + * @brief Deadline of the deferred retry. + * @return Steady-clock microseconds, 0 when no retry is pending. + */ + uint64_t CleanRetryDeadline() const + { + return clean_retry_deadline_.load(std::memory_order_acquire); + } + + /** + * @brief Consecutive unproductive passes, capped at kCleanRetryMaxBackoff. + * @return The current backoff step. + */ + uint8_t CleanBackoff() const { - return std::exchange(shard_clean_cc_wake_pending_, false); + return clean_backoff_; } /** @@ -480,8 +638,18 @@ class CcShard /** * @brief Find an available TEntry in tranaction array and initialize it. * + * @param tx_ng_id The node group the transaction belongs to. + * @param log_group_id The transaction's log group. + * @param term The node group's term. + * @param read_snapshot Whether the transaction reads a snapshot at its + * start timestamp (snapshot isolation): its start timestamp is then no + * older than the wall clock, which also advances the timestamp base. + * @return The transaction's entry. */ - TEntry &NewTx(NodeGroupId tx_ng_id, uint32_t log_group_id, int64_t term); + TEntry &NewTx(NodeGroupId tx_ng_id, + uint32_t log_group_id, + int64_t term, + bool read_snapshot = false); /** * @brief Find an available NonBlockingLock in lock array and initialize it. @@ -606,6 +774,24 @@ class CcShard LruEntry *cce_ptr, NodeGroupId cc_ng_id); + /** + * @brief Called after the tx's write lock on an entry was downgraded to a + * write intent, so the lock stops holding the checkpoint back. + * + * Clears the tx's holds_ckpt_wlock_ unless another write lock of the tx + * in this shard still counts (TxLockInfo::WriteLockHoldsBackCheckpoint). + * A later upgrade of the intent back to a write lock sets it again and + * moves wlock_ts_ to the upgrade time. The downgrade itself leaves + * wlock_ts_ alone. + * + * @param txn The tx whose write lock was downgraded. + * @param cc_ng_id The node group of the entry. + * @param table_type The table type of the entry's cc map. + */ + void OnWriteLockDowngraded(TxNumber txn, + NodeGroupId cc_ng_id, + TableType table_type); + void DropLockHoldingTxs(NodeGroupId cc_ng_id); void VerifyOrphanLock(NodeGroupId cc_ng_id, TxNumber txn) @@ -673,7 +859,8 @@ class CcShard * still be running, and post-processing is what installs each pending * value and then releases its write lock. Write locks are therefore * left to it, unless committed_write_recover_window_seconds has - * passed since the write lock was granted (a wedged post-processing; + * passed since the tx's latest write-lock grant in this shard + * (@p wlock_ts; a wedged post-processing; * the pending write is then treated as lost per the no-WAL contract). * * Independent of the verdict: read locks, read intents and data write @@ -724,17 +911,15 @@ class CcShard continue; } - // Skip meta table because there is no need to do - // checkpoint for these type table. - if (!TableName::IsMeta(tx_it->second->table_type_) && - tx_it->second->wlock_ts_ != 0) + // Only data-table and catalog write locks count; see + // TxLockInfo::WriteLockHoldsBackCheckpoint. + const TxLockInfo &lk_info = *tx_it->second; + if (lk_info.holds_ckpt_wlock_ && lk_info.wlock_ts_ - 1 < min_ts) { - if (tx_it->second->wlock_ts_ - 1 < min_ts) - { - min_ts = tx_it->second->wlock_ts_ - 1; - *pinning_txn = tx_it->first; - *pinning_wlock_ts = tx_it->second->wlock_ts_; - } + assert(lk_info.wlock_ts_ != 0); + min_ts = lk_info.wlock_ts_ - 1; + *pinning_txn = tx_it->first; + *pinning_wlock_ts = lk_info.wlock_ts_; } ++tx_it; @@ -780,8 +965,7 @@ class CcShard { for (const auto &tx_pair : it->second) { - if (!TableName::IsMeta(tx_pair.second->table_type_) && - tx_pair.second->wlock_ts_ != 0) + if (tx_pair.second->holds_ckpt_wlock_) { max_ts = std::max(max_ts, tx_pair.second->wlock_ts_); } @@ -1269,15 +1453,6 @@ class CcShard bool EnableDefragment() const; - void OnDirtyDataFlushed() - { - ResetCleanStart(); - if (WaitListSizeForMemory() > 0) - { - WakeUpShardCleanCc(); - } - } - void ResetRangeSplittingStatus(const TableName &table_name, uint32_t ng_id, uint32_t range_id); @@ -1452,12 +1627,29 @@ class CcShard std::unordered_map> subscribed_standby_nodes_; std::unique_ptr retry_fwd_msg_cc_; - // Shard clean cc + /** + * The shard's single eviction request. In use while queued, executing or + * deferred; idle only when nothing is parked and the heap is at or below + * the eviction target. + */ std::unique_ptr shard_clean_cc_; - // Set when a wake-up arrives while shard_clean_cc_ is already in use, so - // that the pass in flight re-runs rather than dropping the wake-up. See - // TakeShardCleanCcWakeUp(). - bool shard_clean_cc_wake_pending_{false}; + /** + * Requests parked for memory since the shard started. See + * MemoryParkCount(). + */ + uint64_t memory_park_count_{0}; + /** + * Steady-clock deadline (us) of the deferred eviction retry, 0 when none. + * Written by this shard; claimed by the timer thread or by this shard. + */ + std::atomic clean_retry_deadline_{0}; + /** Consecutive unproductive passes; scales the retry delay. */ + uint8_t clean_backoff_{0}; + /** + * A flush or split completed while a pass was running. See + * TakeCleanRescanRequest(). + */ + bool clean_rescan_requested_{false}; // Standby forward msg related members used on follower node CcRequestPool key_obj_standby_msg_cc_pool_; @@ -1536,6 +1728,16 @@ class CcShard static constexpr uint64_t freeBatchSize = 10; // The maximum allowed duration(us) of one invocation of Clean(). static constexpr uint64_t maxDuration = 30; + /** + * Base delay (us) before a deferred eviction pass retries: one tick of + * the LocalCcShards timer thread. + */ + static constexpr uint64_t kCleanRetryBaseDelayUs = 100'000; + /** + * Cap on the backoff step. The delay is the base shifted left by the + * step, so retries are at most 1.6 s apart. + */ + static constexpr uint8_t kCleanRetryMaxBackoff = 4; // cache all tx info under SI isolation level in this shard, // format: {txn->start_ts} diff --git a/tx_service/include/cc/cluster_config_cc_map.h b/tx_service/include/cc/cluster_config_cc_map.h index f9d7c34d7..491ff5ab3 100644 --- a/tx_service/include/cc/cluster_config_cc_map.h +++ b/tx_service/include/cc/cluster_config_cc_map.h @@ -28,6 +28,7 @@ #include "tx_key.h" #include "tx_record.h" #include "tx_service.h" +#include "tx_service_metrics.h" namespace txservice { @@ -141,11 +142,14 @@ class ClusterConfigCcMap // Updates last_vali_ts such that it is no smaller than (1) all // read transactions that have read the item in all shards, and // (2) the local time. - acquire_all_result.last_vali_ts_ = shard_->LastReadTs(); - uint64_t lock_addr = - cce_ptr == nullptr ? 0 - : reinterpret_cast( - cce_ptr->GetKeyGapLockAndExtraData()); + acquire_all_result.last_vali_ts_ = + std::max(shard_->LastReadTs(), shard_->Now()); + // The entry's lock address, also on a first execution (cce_ptr is + // set only on resume): AcquireAllOp drops a read of the entry the + // tx made before from its read set by this address, and an address + // of 0 matches no read. + uint64_t lock_addr = reinterpret_cast( + cc_entry.GetKeyGapLockAndExtraData()); acquire_all_result.local_cce_addr_.SetCceLock( lock_addr, ng_term, req.NodeGroupId(), shard_->LocalCoreId()); acquire_all_result.commit_ts_ = cc_entry.CommitTs(); @@ -238,8 +242,16 @@ class ClusterConfigCcMap req.SetDecodedPayload(std::move(decoded_rec)); ACTION_FAULT_INJECTOR("cluster_config_PostWriteAll_remote"); } + // A cluster gate's post write all (EloqDoc docs/cdc-change-stream.md + // §6.2) carries no node group configs and changes no topology, so it + // skips Sharder: its commit timestamp, raised above the acquire-all + // results, is always above the topology version, and the update below + // would read the configs it does not carry. + const bool gate_only = config_rec->GetClusterGate().has_value(); + // First we need to update cluster configs in Sharder. - if (Sharder::Instance().ClusterConfigVersion() < req.CommitTs()) + if (!gate_only && + Sharder::Instance().ClusterConfigVersion() < req.CommitTs()) { // async call to host manager to update ng is made. cc req will be // put back in queue once it's done. @@ -251,8 +263,8 @@ class ClusterConfigCcMap return false; } - // Lastly release/downgrade lock. We do not need to update the record in - // cluster config cc map since it is always empty. + // Lastly release/downgrade lock. The record in cluster config cc map + // stays empty but for the cluster gate, stored below. LockType lk_type = LockType::NoLock; TxNumber txn = req.Txn(); if (lock != nullptr) @@ -272,10 +284,20 @@ class ClusterConfigCcMap if (lk_type != LockType::NoLock) { - bool was_dirty = neg_inf_.IsDirty(); + if (gate_only) + { + // Only while the request's tx owns the lock: a post write all + // retried after its tx completed elsewhere must not overwrite + // the value a newer gate op installed since, such as an old + // resume arriving after a newer pause. Updated in place, not + // replaced: cluster scale's replay caches node group configs + // in this payload for an asynchronous Sharder update. + neg_inf_.payload_.cur_payload_->SetClusterGate( + *config_rec->GetClusterGate()); + ReportClusterGate(config_rec->GetClusterGate()->paused); + } neg_inf_.SetCommitTsPayloadStatus(req.CommitTs(), RecordStatus::Normal); - OnCommittedUpdate(&neg_inf_, was_dirty); ReleaseCceLock(lock, &neg_inf_, txn, req.NodeGroupId()); } @@ -296,6 +318,10 @@ class ClusterConfigCcMap req.AbortCcRequest(CcErrorCode::REQUESTED_NODE_NOT_LEADER); return false; } + if (req.IsClusterGate()) + { + return ReplayClusterGate(req); + } shard_->SetBucketMigrating(true); const std::string_view &content = req.LogContentView(); ::txlog::ClusterScaleOpMessage scale_op_msg; @@ -528,12 +554,108 @@ class ClusterConfigCcMap cluster_scale_txn, tx_candidate_term, req.CommitTs()); txm->RecoverClusterScale(scale_op_msg, dm_started, dm_finished); } + neg_inf_.SetCommitTsPayloadStatus( + Sharder::Instance().ClusterConfigVersion(), RecordStatus::Normal); + + req.SetFinish(); + return true; + } + +private: + /** + * @brief Exports cluster_gate_paused for this map's node group: the value + * the map now holds. The node's leader stop resets it to 0, since the map + * is dropped then. + * + * @param paused Whether the gate the map holds is paused. + */ + void ReportClusterGate(bool paused) const + { + if (!metrics::enable_metrics) + { + return; + } + metrics::Meter *meter = shard_->local_shards_.GetNodeMeter(); + if (meter != nullptr) { - bool was_dirty = neg_inf_.IsDirty(); - neg_inf_.SetCommitTsPayloadStatus( - Sharder::Instance().ClusterConfigVersion(), - RecordStatus::Normal); - OnCommittedUpdate(&neg_inf_, was_dirty); + meter->Collect(metrics::NAME_CLUSTER_GATE_PAUSED, + paused ? 1.0 : 0.0, + std::to_string(cc_ng_id_)); + } + } + + /** + * @brief Replays log group 0's cluster gate slot (EloqDoc + * docs/cdc-change-stream.md §6.2): installs the slot's value, however old + * it is, so that a recovered node group enforces the same gate as the + * others. + * + * The slot holds the latest gate ever logged, and nothing installs a value + * it has not logged first, so the slot is never older than what the map + * holds. The value is set in place, as the gate's post write sets it: + * cluster scale's replay, which ran before this one, cached node group + * configs in the same payload for an asynchronous Sharder update. The + * entry's commit timestamp only grows: scale's replay may have left it at + * a config version above the gate's. + * + * On the node group that coordinated the gate tx, a recovery tx then + * re-sends the install to every node group, under the same tx, to release + * the cluster config write lock wherever the tx still holds it (orphan + * lock recovery leaves a logged gate tx's lock for this). It takes no lock + * or intent: nothing in a gate op follows its log but the install, which + * stores and releases only where the tx holds the lock and does nothing + * elsewhere. That install reaches this node group too, where the tx holds + * no lock after the failover, so the post write all's no-lock branch + * finishes without storing anything: this is why the replay installs the + * value here itself first. Holding the lock here would also block the + * schema op recovery that replays next, whose recovery tx reads the + * cluster config, while the install waits for this node group to lead. + * + * @param req The replay request; its blob is a ClusterGateLogMessage. + * @return true, the request is finished. + */ + bool ReplayClusterGate(ReplayLogCc &req) + { + const std::string_view &content = req.LogContentView(); + ::txlog::ClusterGateLogMessage gate_msg; + if (!gate_msg.ParseFromArray(content.data(), content.length())) + { + LOG(ERROR) << "Cluster gate replay: corrupt slot, txn: " + << req.Txn(); + req.AbortCcRequest(CcErrorCode::UNDEFINED_ERR); + return false; + } + ClusterGate gate; + gate.mode = gate_msg.mode() == + ::txlog::ClusterGateLogMessage_GateMode_GateDdlDml + ? ClusterGate::Mode::kDdlDml + : ClusterGate::Mode::kDdl; + gate.paused = gate_msg.paused(); + neg_inf_.payload_.cur_payload_->SetClusterGate(gate); + ReportClusterGate(gate.paused); + if (req.CommitTs() > neg_inf_.CommitTs()) + { + neg_inf_.SetCommitTsPayloadStatus(req.CommitTs(), + RecordStatus::Normal); + } + LOG(INFO) << "Cluster gate replay: installed " + << (gate.paused ? "paused" : "open") << " at " + << req.CommitTs() << ", txn: " << req.Txn(); + + const TxNumber gate_txn = req.Txn(); + if (((gate_txn >> 32L) >> 10) == req.NodeGroupId()) + { + LOG(INFO) << "Recovering cluster gate tx, txn: " << gate_txn + << ", term: " + << Sharder::Instance().CandidateLeaderTerm( + req.NodeGroupId()); + TxService *tx_service = shard_->local_shards_.GetTxService(); + TransactionExecution *txm = tx_service->NewTx(); + txm->SetRecoverTxState( + gate_txn, + Sharder::Instance().CandidateLeaderTerm(req.NodeGroupId()), + req.CommitTs()); + txm->RecoverClusterGate(gate_msg); } req.SetFinish(); diff --git a/tx_service/include/cc/local_cc_handler.h b/tx_service/include/cc/local_cc_handler.h index c60902c47..83723f928 100644 --- a/tx_service/include/cc/local_cc_handler.h +++ b/tx_service/include/cc/local_cc_handler.h @@ -182,7 +182,24 @@ class LocalCcHandler : public CcHandler int64_t ng_term, TxNumber tx_number) const; - bool ReleaseCatalogRead(NonBlockingLock *lock, TxNumber tx_number) const; + /** + * @brief Releases a catalog read lock taken on the fast path (ReadCatalog), + * on this handler's shard. + * + * A committed tx first publishes its commit ts in the shard's last read ts, + * as PostReadCc does for a regular read lock: a schema op that takes the + * catalog entry's write lock after this reader then commits above it + * (AcquireAllCc's last_vali_ts_). Without it, a TRUNCATE could commit + * below a write it erases, and replay would bring the write back. + * + * @param lock The catalog entry's lock. + * @param tx_number The reader. + * @param commit_ts The reader's commit ts; 0 if it has none. + * @return true. + */ + bool ReleaseCatalogRead(NonBlockingLock *lock, + TxNumber tx_number, + uint64_t commit_ts) const; bool ReadLocal(const TableName &table_name, const std::string &key_str, diff --git a/tx_service/include/cc/local_cc_shards.h b/tx_service/include/cc/local_cc_shards.h index 6a60d2e72..f00082d3c 100644 --- a/tx_service/include/cc/local_cc_shards.h +++ b/tx_service/include/cc/local_cc_shards.h @@ -83,6 +83,7 @@ class RemoteCcHandler; class Checkpointer; class TxService; struct ClusterScaleOp; +struct ClusterGateOp; struct DataMigrationOp; struct InvalidateTableCacheCompositeOp; class SkGenerator; @@ -645,6 +646,16 @@ class LocalCcShards const std::string &catalog_image, uint64_t commit_ts); + /** + * @brief Builds a table's schema object from its catalog image, through + * the table engine's CatalogFactory::CreateTableSchema, which documents + * how @p version applies to the tables the image records no version for. + * + * @param table_name The base table. + * @param catalog_image The table's catalog image. + * @param version The schema's version. + * @return The schema; null for an empty image. + */ TableSchema::uptr CreateTableSchemaFromImage( const TableName &table_name, const std::string &catalog_image, @@ -922,8 +933,18 @@ class LocalCcShards return new_range_entries; } + /** + * Frees the shard of core @p idx when its worker thread exits. + * + * Taken under timer_terminate_mux_: the timer thread holds that mutex + * while it walks cc_shards_ on every tick (LocalCcShards::TimerRun) to + * fire due eviction retries, so the slot turns null and the shard is + * destroyed only between two walks, never under one. + * @param idx The core, and shard index, whose worker is exiting. + */ void FreeCcShard(size_t idx) { + std::scoped_lock lk(timer_terminate_mux_); cc_shards_[idx] = nullptr; } @@ -1922,7 +1943,7 @@ class LocalCcShards response CcMessage should always be valid(memory not being freed). This is important in the case where network timeout happens and remote response(sent by retry) arrives after UpsertTableOp has finished(finished means txm has - been reset and schema_op_ unique pointer in txm has been set to null). + been reset and its composite op slot has been cleared). To avoid this pointer invalidation, the UpsertTableOp is moved from txm to local_cc_shards once finished the last step of the schema op to maintain @@ -1932,27 +1953,34 @@ class LocalCcShards */ std::vector> table_schema_op_pool_; - std::mutex table_schema_op_pool_mux_; std::vector> split_flush_range_op_pool_; - std::mutex split_flush_range_op_pool_mux_; std::vector> table_index_op_pool_; std::vector> sk_generator_pool_; - std::mutex table_index_op_pool_mux_; // Since there's only 1 cluster scale event at a time across the cluster, // we don't need a pool for it. We just need to make sure that the op is not // invalidated in case late remote cc response comes in. std::vector> cluster_scale_op_pool_; - std::mutex cluster_scale_op_mux_; + + /** + * @brief Finished cluster gate ops. An op outlives its request because a + * remote response that comes in after the op finished still reads the cc + * handler result of one of its stages, which the op owns. + */ + std::vector> cluster_gate_op_pool_; std::vector> invalidate_table_cache_op_pool_; - std::mutex invalidate_table_cache_op_mux_; - std::mutex data_migration_op_pool_mux_; std::vector> migration_op_pool_; + /** + * @brief Guards every op pool above and sk_generator_pool_. Each user holds + * it only to take or return one object, never while taking another lock. + */ + std::mutex op_pool_mux_; + // protects the table ranges heap to avoid concurrent memory requests by // multiple threads. std::mutex table_ranges_heap_mux_; @@ -2081,12 +2109,20 @@ class LocalCcShards const NodeGroupId ng_id_; std::vector> cc_shards_; - // The background thread that periodically advances the timers of the local - // shards to the current wall clock. + /** + * The background thread that periodically advances the timers of the local + * shards to the current wall clock and fires the shards' due eviction + * retries (LocalCcShards::TimerRun, one tick per 100 ms). + */ std::thread timer_thd_; - bool timer_terminate_; + bool timer_terminate_; ///< Set under timer_terminate_mux_ to stop it. + /** + * Guards timer_terminate_ and is held by the timer thread while it walks + * cc_shards_; FreeCcShard takes it so that a worker exiting at shutdown + * never empties a slot under that walk. + */ std::mutex timer_terminate_mux_; - std::condition_variable timer_terminate_cv_; + std::condition_variable timer_terminate_cv_; ///< Wakes the thread to stop. // std::atomic timer_terminate_; // Background thread that initializes table_ranges_heap. After init, it diff --git a/tx_service/include/cc/range_cc_map.h b/tx_service/include/cc/range_cc_map.h index 3a5180934..41ed184f8 100644 --- a/tx_service/include/cc/range_cc_map.h +++ b/tx_service/include/cc/range_cc_map.h @@ -440,16 +440,22 @@ class RangeCcMap : public TemplateCcMap return false; } - // When the commit ts is 0 or the commit type is DowngradeLock, the - // request commits nothing and only removes the write intents/locks - // acquired earlier. - if (req.CommitTs() == TransactionOperation::tx_op_failed_ts_ || - req.CommitType() == PostWriteType::DowngradeLock) + if (req.CommitType() == PostWriteType::Rollback) { + // A split rolls back only if it fails before its prepare log, so + // it installed no dirty range: there is no value to keep, only the + // write locks to release. The request's record is the split's + // own, which must not be installed. + assert(req.CommitTs() == TransactionOperation::tx_op_failed_ts_); + req.SetDecodedPayload(nullptr); return TemplateCcMap::Execute(req); } - - assert(req.CommitType() != PostWriteType::DowngradeLock); + if (req.CommitType() == PostWriteType::DowngradeLock) + { + // Downgrades the write locks to write intents; commits nothing. + return TemplateCcMap::Execute(req); + } + assert(req.CommitTs() > 0); // Prepare RangeRecord RangeRecord *upload_range_rec = nullptr; diff --git a/tx_service/include/cc/template_cc_map.h b/tx_service/include/cc/template_cc_map.h index cd962fc38..cb7c30de9 100644 --- a/tx_service/include/cc/template_cc_map.h +++ b/tx_service/include/cc/template_cc_map.h @@ -952,7 +952,8 @@ class TemplateCcMap : public CcMap // Updates last_vali_ts such that it is no smaller than (1) all // read transactions that have read the item in all shards, and // (2) the local time. - req.SetLastValidTs(shard_->LastReadTs()); + req.SetLastValidTs( + std::max(shard_->LastReadTs(), shard_->Now())); if (shard_->core_id_ == tx_core_id) { @@ -1078,11 +1079,13 @@ class TemplateCcMap : public CcMap { payload = static_cast(req.Payload()); } - // commit_ts = 0 means transaction failed (e.g. failed at prepare - // phase), we have nothing to upload, only need to release write intent. - else if (req.CommitTs() > 0 && - req.CommitType() != PostWriteType::DowngradeLock) + // A remote request that installs a value carries it serialized. A + // DowngradeLock installs nothing, and a Rollback installs only the + // value to keep that the cc map supplied, if any. + else if (req.CommitType() != PostWriteType::DowngradeLock && + req.CommitType() != PostWriteType::Rollback) { + assert(req.CommitTs() > 0); if (req.DecodedPayload() == nullptr) { const std::string *payload_str = req.PayloadStr(); @@ -1153,72 +1156,124 @@ class TemplateCcMap : public CcMap if (lk_type != LockType::NoLock) { - if (commit_ts > 0 && - req.CommitType() != PostWriteType::DowngradeLock) - { + // Do not recycle the lock on Catalog since it's frequently + // accessed. + const bool recycle_lock = + table_name_.Type() != TableType::Catalog; + switch (req.CommitType()) + { + case PostWriteType::PrepareCommit: + // Installs the dirty value and keeps the write intent, + // downgrading a write lock to one. Every op sends it after + // its prepare log, so commit_ts is set. The entry keeps its + // status and commit ts, except as below. + assert(commit_ts > 0 && payload != nullptr); cce_ptr->payload_.SetCurrentPayload(payload); - // A prepare commit request only installs the dirty value, - // and does not change the record status and commit_ts. - if (req.CommitType() == PostWriteType::Commit || - req.CommitType() == PostWriteType::PostCommit) + if (req.OpType() == OperationType::CreateTable) { - RecordStatus status = - (req.OpType() == OperationType::Delete || - req.OpType() == OperationType::DropTable) - ? RecordStatus::Deleted - : RecordStatus::Normal; - - bool was_dirty = cce_ptr->IsDirty(); - cce_ptr->SetCommitTsPayloadStatus(commit_ts, status); - OnCommittedUpdate(cce_ptr, was_dirty); + // For prepare commit of create table operation, set + // the status to Deleted. + cce_ptr->SetCommitTsPayloadStatus( + 1, RecordStatus::Deleted); } - else if (req.CommitType() == PostWriteType::PrepareCommit) + else if (table_name_.Type() == TableType::Catalog && + lk_type == LockType::WriteLock && + commit_ts > cce_ptr->CommitTs()) { - if (req.OpType() == OperationType::CreateTable) - { - // For prepare commit of create table operation, - // set the status to Deleted. - cce_ptr->SetCommitTsPayloadStatus( - 1, RecordStatus::Deleted); - } + // The DDL chose commit_ts under this write lock (an + // index op, which holds it across the prepare log). + // Publishing it as the entry's commit ts makes every + // tx that reads the entry once the lock is downgraded + // below, and so sees the dirty schema, commit above + // it: a catalog read enters the read set at the + // entry's commit ts. The node clocks alone would not + // order such a tx after the DDL. Replay of the prepare + // log does the same (CatalogCcMap, + // Execute(ReplayLogCc&)). + // + // The entry is marked Normal: the payload just + // installed is the complete record (CatalogCcMap fills + // it from the node's catalog entry). An entry left + // Unknown must keep commit ts 0, because the catalog + // load path could not mark it Normal at an unchanged + // timestamp. + cce_ptr->SetCommitTsPayloadStatus(commit_ts, + RecordStatus::Normal); } - } - - // When commit_ts = 0, the request removes the write lock - // without installing a new value. - - if (req.CommitType() == PostWriteType::PrepareCommit || - req.CommitType() == PostWriteType::DowngradeLock) - { - // For PrepareCommit and DowngradeLock, the - // post-write-all request keeps write intent or downgrades - // the write lock to the write intent. if (lk_type == LockType::WriteLock) { DowngradeCceKeyWriteLock(cce_ptr, txn); } - } - else if (req.CommitType() == PostWriteType::Commit || - req.CommitType() == PostWriteType::PostCommit) + break; + case PostWriteType::UpdateDirty: + // Replaces an add index's dirty value; the lock stays. + assert(req.OpType() == OperationType::AddIndex); + assert(commit_ts > 0 && payload != nullptr); + cce_ptr->payload_.SetCurrentPayload(payload); + break; + case PostWriteType::Commit: + case PostWriteType::PostCommit: { - // For PostCommit or Commit, the post-write-all request - // releases the write lock. - - // Do not recycle the lock on Catalog since it's frequently - // accessed. - bool recycle_lock = - table_name_.Type() != TableType::Catalog; + // Installs the committed value at commit_ts and releases + // the lock. A tx that failed rolls back instead. + assert(commit_ts > 0 && payload != nullptr); + cce_ptr->payload_.SetCurrentPayload(payload); + RecordStatus status = + (req.OpType() == OperationType::Delete || + req.OpType() == OperationType::DropTable) + ? RecordStatus::Deleted + : RecordStatus::Normal; + bool was_dirty = cce_ptr->IsDirty(); + cce_ptr->SetCommitTsPayloadStatus(commit_ts, status); + // The data-key stats count data tables only; today every + // post-write-all targets a meta table. + if (!table_name_.IsMeta()) + { + OnCommittedUpdate(cce_ptr, was_dirty); + } ReleaseCceLock(cce_ptr->GetKeyLock(), cce_ptr, txn, req.NodeGroupId(), lk_type, recycle_lock); + break; } - else - { - assert(req.CommitType() == PostWriteType::UpdateDirty && - req.OpType() == OperationType::AddIndex); + case PostWriteType::Rollback: + // Releases the lock. A payload is the value to keep, which + // the cc map supplies to replace the dirty value the tx + // installed; without one the tx installed nothing and the + // entry keeps its value. A commit_ts above 0 moves the + // entry's commit ts to it and keeps its status. + if (payload != nullptr) + { + cce_ptr->payload_.SetCurrentPayload(payload); + } + if (commit_ts > 0) + { + bool was_dirty = cce_ptr->IsDirty(); + cce_ptr->SetCommitTsPayloadStatus( + commit_ts, cce_ptr->PayloadStatus()); + if (!table_name_.IsMeta()) + { + OnCommittedUpdate(cce_ptr, was_dirty); + } + } + ReleaseCceLock(cce_ptr->GetKeyLock(), + cce_ptr, + txn, + req.NodeGroupId(), + lk_type, + recycle_lock); + break; + case PostWriteType::DowngradeLock: + // Downgrades a write lock to a write intent; commits + // nothing. + if (lk_type == LockType::WriteLock) + { + DowngradeCceKeyWriteLock(cce_ptr, txn); + } + break; } } } diff --git a/tx_service/include/checkpointer.h b/tx_service/include/checkpointer.h index 144bae104..4918b1512 100644 --- a/tx_service/include/checkpointer.h +++ b/tx_service/include/checkpointer.h @@ -101,6 +101,17 @@ class Checkpointer bool IsTerminated(); + /** + * @brief The periodic checkpoint interval this checkpointer runs with. + * + * @return The interval in seconds, after start-up raised it to the + * minimum. + */ + uint32_t CheckpointInterval() const + { + return checkpoint_interval_; + } + /** * @brief When TxService is stopping, this function will be called and * triggers checkpoint to flush data to KvStore. @@ -165,7 +176,22 @@ class Checkpointer std::chrono::system_clock::time_point last_checkpoint_ts_; std::atomic last_checkpoint_request_ts_; - uint32_t ckpt_delay_time_; // unit: Microsecond + /** + * @brief The most a round's checkpoint timestamp is lowered below the + * round's timestamp under MVCC, in microseconds: the + * checkpointer_delay_seconds setting (5 s by default). + * + * A round at timestamp T checkpoints at T when no snapshot transaction + * started below T, else at the oldest one's start, but never below T + * minus this delay (GetNewCheckpointTs). Entries updated above the + * checkpoint stay dirty, and the page cleaner evicts only checkpointed + * entries (CcEntry::IsFree), so they stay in memory with their version + * chains: a snapshot read in that window finds its version there instead + * of fetching it from the archives table. It is a cache residency choice + * only; recycling old versions follows the oldest snapshot transaction's + * start alone (CcEntry::KickOutArchiveRecords). + */ + uint32_t ckpt_delay_time_; std::atomic ongoing_data_sync_cnt_{0}; TxService *tx_service_; TxLog *log_agent_; diff --git a/tx_service/include/circular_queue.h b/tx_service/include/circular_queue.h index c50815058..2c52887a4 100644 --- a/tx_service/include/circular_queue.h +++ b/tx_service/include/circular_queue.h @@ -24,6 +24,7 @@ #ifndef CIRCULAR_QUEUE_H #define CIRCULAR_QUEUE_H +#include #include #include diff --git a/tx_service/include/cluster_config_record.h b/tx_service/include/cluster_config_record.h index 48ba3b5f3..b8f02f710 100644 --- a/tx_service/include/cluster_config_record.h +++ b/tx_service/include/cluster_config_record.h @@ -23,6 +23,7 @@ #include #include +#include #include #include "sharder.h" @@ -32,15 +33,56 @@ namespace txservice { +/** + * @brief The cluster gate: a cluster-wide flag that, while paused, makes every + * new schema op fail with TxErrorCode::CLUSTER_GATED (EloqDoc + * docs/cdc-change-stream.md §6.2). + * + * CDC bootstrap pauses it so that no DDL runs while it copies the catalog. The + * cluster config cc map keeps the latest value in its single entry's payload, + * where each schema op's cluster config read lock reads it. + */ +struct ClusterGate +{ + /** @brief What a pause rejects. */ + enum class Mode : uint8_t + { + kDdl = 0, ///< New create, drop, truncate and index ops. + kDdlDml = 1, ///< Also the first write of a transaction; reserved. + }; + + Mode mode{Mode::kDdl}; ///< What the gate rejects while paused. + bool paused{false}; ///< Whether the gate is closed. + + /** + * @brief Compares two gates. + * + * @param rhs The other gate. + * @return true if both have the same mode and state. + */ + bool operator==(const ClusterGate &rhs) const + { + return mode == rhs.mode && paused == rhs.paused; + } +}; + struct ClusterConfigRecord : public TxRecord { public: ClusterConfigRecord() = default; ClusterConfigRecord(const ClusterConfigRecord &rhs) - : is_config_owner_(rhs.is_config_owner_), version_(rhs.version_) + : is_config_owner_(rhs.is_config_owner_), + version_(rhs.version_), + cluster_gate_(rhs.cluster_gate_) { if (rhs.is_config_owner_) { + // A deserialized record owns its configs, and may have none: a + // cluster gate's carries none. + if (rhs.node_group_configs_uptr_ == nullptr) + { + return; + } node_group_configs_uptr_ = std::make_unique< std::unordered_map>>(); for (const auto &pair : *rhs.node_group_configs_uptr_) @@ -61,6 +103,7 @@ struct ClusterConfigRecord : public TxRecord return *this; } version_ = rhs.version_; + cluster_gate_ = rhs.cluster_gate_; if (is_config_owner_ && node_group_configs_uptr_ != nullptr) { node_group_configs_uptr_.reset(); @@ -68,11 +111,14 @@ struct ClusterConfigRecord : public TxRecord is_config_owner_ = rhs.is_config_owner_; if (rhs.is_config_owner_) { - node_group_configs_uptr_ = std::make_unique< - std::unordered_map>>(); - for (const auto &pair : *rhs.node_group_configs_uptr_) + if (rhs.node_group_configs_uptr_ != nullptr) { - node_group_configs_uptr_->emplace(pair.first, pair.second); + node_group_configs_uptr_ = std::make_unique< + std::unordered_map>>(); + for (const auto &pair : *rhs.node_group_configs_uptr_) + { + node_group_configs_uptr_->emplace(pair.first, pair.second); + } } } else @@ -126,6 +172,16 @@ struct ClusterConfigRecord : public TxRecord } } SerializeToStr(&version_, str); + // The gate goes last, behind its own has-bit, as the configs go behind + // has_ng_config. + bool has_gate = cluster_gate_.has_value(); + SerializeToStr(&has_gate, str); + if (has_gate) + { + uint8_t mode = static_cast(cluster_gate_->mode); + SerializeToStr(&mode, str); + SerializeToStr(&cluster_gate_->paused, str); + } } size_t SerializedLength() const override @@ -156,6 +212,11 @@ struct ClusterConfigRecord : public TxRecord } len += sizeof(uint64_t); + len += sizeof(bool); + if (cluster_gate_.has_value()) + { + len += sizeof(uint8_t) + sizeof(bool); + } return len; } @@ -195,12 +256,25 @@ struct ClusterConfigRecord : public TxRecord } } DesrializeFrom(buf, offset, &version_); + bool has_gate; + DesrializeFrom(buf, offset, &has_gate); + cluster_gate_.reset(); + if (has_gate) + { + uint8_t mode; + DesrializeFrom(buf, offset, &mode); + ClusterGate gate; + gate.mode = static_cast(mode); + DesrializeFrom(buf, offset, &gate.paused); + cluster_gate_ = gate; + } } TxRecord::Uptr Clone() const override { auto uptr = std::make_unique(); uptr->version_ = version_; + uptr->cluster_gate_ = cluster_gate_; uptr->is_config_owner_ = true; const std::unordered_map> *node_group_configs = nullptr; @@ -239,14 +313,18 @@ struct ClusterConfigRecord : public TxRecord const ClusterConfigRecord &rhs = dynamic_cast(record); version_ = rhs.version_; + cluster_gate_ = rhs.cluster_gate_; is_config_owner_ = rhs.is_config_owner_; if (rhs.is_config_owner_) { - node_group_configs_uptr_ = std::make_unique< - std::unordered_map>>(); - for (auto &pair : *rhs.node_group_configs_uptr_) + if (rhs.node_group_configs_uptr_ != nullptr) { - node_group_configs_uptr_->emplace(pair.first, pair.second); + node_group_configs_uptr_ = std::make_unique< + std::unordered_map>>(); + for (auto &pair : *rhs.node_group_configs_uptr_) + { + node_group_configs_uptr_->emplace(pair.first, pair.second); + } } } else @@ -310,6 +388,41 @@ struct ClusterConfigRecord : public TxRecord : *node_group_configs_ptr_; } + /** + * @brief The cluster gate the record carries. + * + * Absent and present with paused = false are different values: the + * first says nothing, the second is a resume. A record carrying a gate + * carries no node group configs. + * + * @return The gate, or none. + */ + const std::optional &GetClusterGate() const + { + return cluster_gate_; + } + + /** + * @brief Sets the cluster gate the record carries. + * + * @param gate The gate. + */ + void SetClusterGate(const ClusterGate &gate) + { + cluster_gate_ = gate; + } + + /** + * @brief Whether the record carries a closed gate in mode kDdl or + * kDdlDml, which rejects new schema ops. + * + * @return true if a new schema op must fail with CLUSTER_GATED. + */ + bool GatesSchemaOps() const + { + return cluster_gate_.has_value() && cluster_gate_->paused; + } + void Reset() { if (is_config_owner_ && node_group_configs_uptr_) @@ -319,6 +432,7 @@ struct ClusterConfigRecord : public TxRecord is_config_owner_ = false; node_group_configs_ptr_ = nullptr; version_ = 0; + cluster_gate_.reset(); } private: @@ -336,5 +450,11 @@ struct ClusterConfigRecord : public TxRecord }; bool is_config_owner_{false}; uint64_t version_{0}; + /** + * The cluster gate: in the cluster config cc map's entry, the latest value + * a gate op installed; in a gate op's post write all, the value to + * install. Absent everywhere else. + */ + std::optional cluster_gate_; }; } // namespace txservice diff --git a/tx_service/include/eloq_log_wrapper.h b/tx_service/include/eloq_log_wrapper.h index 4dee75ce6..217916a42 100644 --- a/tx_service/include/eloq_log_wrapper.h +++ b/tx_service/include/eloq_log_wrapper.h @@ -60,6 +60,23 @@ class EloqLogAgent : public TxLog log_group_id, cntl, &request, &response, &done); } + void CdcScan(uint32_t log_group_id, + brpc::Controller *cntl, + const ::txlog::CdcScanRequest &request, + ::txlog::CdcScanResponse &response, + std::optional *node_id, + google::protobuf::Closure &done) override + { + log_agent_.CdcScan( + log_group_id, cntl, &request, &response, node_id, &done); + } + + std::optional LogNodeByAddress( + uint32_t log_group_id, const std::string &address) override + { + return log_agent_.LogNodeByAddress(log_group_id, address); + } + ::txlog::CheckClusterScaleStatusResponse::Status CheckClusterScaleStatus( uint32_t log_group_id, const std::string &id) override { @@ -79,21 +96,19 @@ class EloqLogAgent : public TxLog log_agent_.RemoveCcNodeGroup(cc_node_group_id, term); } - void ReplayLog(uint32_t cc_node_group_id, + bool ReplayLog(uint32_t cc_node_group_id, int64_t term, const std::string &source_ip, uint16_t source_port, - int log_group, - uint64_t start_ts, - std::atomic &interrupt) override - { - log_agent_.ReplayLog(cc_node_group_id, - term, - source_ip, - source_port, - log_group, - start_ts, - interrupt); + uint32_t log_group_id, + uint64_t start_ts) override + { + return log_agent_.ReplayLogOnce(cc_node_group_id, + term, + source_ip, + source_port, + log_group_id, + start_ts); } txservice::RecoverTxStatus RecoverTx(uint64_t tx_number, diff --git a/tx_service/include/error_messages.h b/tx_service/include/error_messages.h index 6daf5a5dc..db3923f96 100644 --- a/tx_service/include/error_messages.h +++ b/tx_service/include/error_messages.h @@ -130,6 +130,19 @@ enum struct TxErrorCode : uint16_t UPDATE_SEQUENCE_TABLE_FAIL, INVALID_CURSOR, + + /** + * A new schema op met a paused cluster gate (EloqDoc + * docs/cdc-change-stream.md §6.2). Retriable once the gate resumes. + */ + CLUSTER_GATED, + + /** + * A pause met a gate that is already paused, by another bootstrap or an + * abandoned one (EloqDoc docs/cdc-change-stream.md §6.2). Not retriable: + * the operator decides whether to resume it. + */ + CLUSTER_GATE_HELD, }; static const std::unordered_map tx_error_messages{ @@ -217,6 +230,10 @@ static const std::unordered_map tx_error_messages{ {TxErrorCode::RPC_CALL_ERR, "RPC call error."}, {TxErrorCode::UPDATE_SEQUENCE_TABLE_FAIL, "Update sequence table fail."}, {TxErrorCode::INVALID_CURSOR, "ERR invalid cursor"}, + {TxErrorCode::CLUSTER_GATED, + "Schema changes are paused by the cluster gate; retry after it resumes."}, + {TxErrorCode::CLUSTER_GATE_HELD, + "The cluster gate is already paused; resume it before pausing again."}, }; static inline const std::string &TxErrorMessage(TxErrorCode err_code) diff --git a/tx_service/include/fault/cc_node.h b/tx_service/include/fault/cc_node.h index 48481e74f..44262dfd8 100644 --- a/tx_service/include/fault/cc_node.h +++ b/tx_service/include/fault/cc_node.h @@ -104,7 +104,9 @@ class CcNode bool PromoteStandbyTermIfCandidate(int64_t standby_term); private: - void NotifyNewLeaderStart(uint32_t leader_ng_id, uint32_t leader_node_id); + void NotifyNewLeaderStart(uint32_t leader_ng_id, + uint32_t leader_node_id, + int64_t term); void SubscribePrimaryNode(uint32_t node_id, int64_t term, bool resubscribe); void ClearCcNodeGroupData(); bool WaitForShardSyncWithRetry(const size_t shard_idx, diff --git a/tx_service/include/fault/fault_inject.h b/tx_service/include/fault/fault_inject.h index 88a402c86..7e254d29e 100644 --- a/tx_service/include/fault/fault_inject.h +++ b/tx_service/include/fault/fault_inject.h @@ -196,10 +196,34 @@ class FaultInject void TriggerAction(FaultEntry *entry); void InjectFault(std::string fault_name, std::string paras); + /** + * @brief Whether this build compiles the fault points in. + * + * The fault points (ACTION_FAULT_INJECTOR, CODE_FAULT_INJECTOR) expand to + * nothing unless the data substrate is built with WITH_FAULT_INJECT. The + * answer comes from the data substrate's own compilation, so a caller built + * with other flags (an API layer's command, a test) still gets it right. + * Without the fault points, InjectFault still records the fault, but the + * fault never fires. + * + * @return true if the fault points are compiled in. + */ + static bool CompiledIn(); + private: - FaultInject() - { - } + /** + * @brief Arms the faults the environment variable ELOQ_TX_FAULT_INJECT + * names, in fault-injection builds: comma-separated entries, each "name" + * or "name:params" (params as InjectFault takes them). For tests that need + * a fault from the process's start, before it serves commands that arm + * one; such a command removes it as any other. + * + * @note Defined in fault_inject.cpp, which the data substrate compiles + * with its WITH_FAULT_INJECT: an API layer that includes this header is + * built without it, and an inline definition compiled there could be the + * one the linker keeps. + */ + FaultInject(); ~FaultInject() { diff --git a/tx_service/include/fault/log_replay_service.h b/tx_service/include/fault/log_replay_service.h index 8d69f06dc..570ef0335 100644 --- a/tx_service/include/fault/log_replay_service.h +++ b/tx_service/include/fault/log_replay_service.h @@ -56,13 +56,11 @@ struct ReplayLogTask ReplayLogTask(uint32_t cc_ng_id, int64_t cc_ng_term, int log_group, - uint64_t replay_start_ts, - uint64_t queued_clock) + uint64_t replay_start_ts) : cc_ng_id_(cc_ng_id), cc_ng_term_(cc_ng_term), log_group_(log_group), - replay_start_ts_(replay_start_ts), - queued_clock_(queued_clock) + replay_start_ts_(replay_start_ts) { } @@ -70,9 +68,6 @@ struct ReplayLogTask int64_t cc_ng_term_; int log_group_; uint64_t replay_start_ts_; - // the clock when the request is put into the replay queue. - // set queued_clock_ to 0, if it's not a delayed request. - uint64_t queued_clock_; }; struct RecoverTxTask @@ -140,14 +135,14 @@ class RecoveryService : public brpc::StreamInputHandler, ::google::protobuf::Closure *done) override; /** - * @brief Send ReplayLogRequest to log groups. If log_group is negative, - * send to all log groups, else just the specified log group. + * @brief Queues a ReplayLogRequest to the log groups, sent by the notify + * thread (ProcessReplayLogTask). If log_group is negative, to every log + * group that has not finished the term's replay, else to that one. */ void ReplayLog(uint32_t cc_ng_id, int64_t cc_ng_term, int log_group = -1, - uint64_t replay_start_ts = 0, - bool delayed_request = false); + uint64_t replay_start_ts = 0); void RecoverTx(uint64_t tx_number, int64_t tx_term, @@ -164,24 +159,31 @@ class RecoveryService : public brpc::StreamInputHandler, void on_closed(brpc::StreamId id) override; private: + /** + * @brief The replay requests for a node group this node is the candidate + * leader of; used by the notify thread alone. + */ struct CandidateReplayWatch { - int64_t term{-1}; - uint64_t first_seen_ts{0}; - uint64_t last_replay_ts{0}; + int64_t term{-1}; ///< The candidate term the fields below are for. + /** The start timestamp of term's first request (the leader start's), + * which every later request reuses. */ + uint64_t replay_start_ts{0}; + /** Per log group: when a request was last sent to it, or it was last + * seen streaming the replay. */ + std::unordered_map last_send_ts; }; /** - * @brief ReplayNow() check whether the replay request can be processed - * immediately. When replay error happens, we will put the replay request - * into replay queue again, but with a delay (default 10 seconds). + * @brief Sends a queued ReplayLogRequest: one bounded attempt per log + * group (TxLog::ReplayLog), skipping log groups that have finished the + * term's replay, and none once the node is no longer the node group's + * candidate leader at that term. Records each send for the watchdog. + * + * @param task The request. */ - bool ReplayNow(ReplayLogTask &task); - void ProcessReplayLogTask(ReplayLogTask &task); - int ProcessDelayedReplayLogTask(); - void ProcessRecoverTxTask(RecoverTxTask &task); struct ConnectionInfo @@ -254,7 +256,11 @@ class RecoveryService : public brpc::StreamInputHandler, // Current active stream for each log group. std::unordered_map current_stream_; - // Cross log group aggregation: schema replay range splitting tables. + /** + * Cross log group aggregation, for schema replay: the base tables with + * a split in flight that the recovering node group coordinates, whose + * catalog read lock that split holds on it. + */ std::unordered_set range_split_tables_; // Cross log group aggregation: split range commit ts for data replay. @@ -275,6 +281,9 @@ class RecoveryService : public brpc::StreamInputHandler, std::unordered_map> inbound_connections_; int active_stream_cnt_ = 0; + /** @brief Test only: the replay stream the replay_drop_stream_once fault + * dropped, whose messages are ignored. */ + std::atomic fault_dropped_stream_{brpc::INVALID_STREAM_ID}; bthread::Mutex inbound_mux_; bthread::ConditionVariable inbound_cv_; @@ -291,16 +300,31 @@ class RecoveryService : public brpc::StreamInputHandler, std::thread notify_thread_; std::deque replay_log_queue_; - std::deque delayed_replay_queue_; std::deque recover_tx_queue_; + /** + * @brief Node group id -> its replay watch, for the node groups this node + * is recovering only: the watchdog visits these alone. + */ std::unordered_map candidate_replay_watch_; std::mutex queue_mux_; std::condition_variable queue_cv_; std::atomic finish_; - // Candidate leader has no replay stream for this duration, then trigger - // ReplayLog and keep retrying with the same interval until stream appears. + /** + * @brief How long a log group may go without a recovering replay stream + * after a request was sent to it (or its stream was last seen) before the + * request is sent to it again, and again at this interval until its + * stream appears. Counted from the send: a lost request, a log group with + * no leader and a lost reply look the same to the sender. + */ static constexpr uint64_t kReplayCheckIntervalUs = 5ULL * 1000 * 1000; + /** + * @brief How often the watchdog looks for log groups to ask again (when + * the notify thread has had no task for this long), so that a resend comes + * within this much after kReplayCheckIntervalUs has passed. It acts only + * for node groups replaying the log (with a candidate term). + */ + static constexpr int64_t kReplayWatchPeriodMs = 1000; // ip and port of log replay server of this node std::string ip_; diff --git a/tx_service/include/proto/cc_request.proto b/tx_service/include/proto/cc_request.proto index e0718e811..964d7c8ab 100644 --- a/tx_service/include/proto/cc_request.proto +++ b/tx_service/include/proto/cc_request.proto @@ -417,6 +417,9 @@ message FetchNodeInfoResponse{ uint32 node_id = 2 ; NodeRole role = 3 ; NodeStatus status = 4; + // The node group's leader term when the node leads it (role LeaderNode: + // Online once it serves, Loading while it replays the log); 0 otherwise. + int64 leader_term = 5; } message ResetStandbySequenceIdRequest{ @@ -757,6 +760,7 @@ enum CommitType { Commit = 2; DowngradeLock = 3; UpdateDirty = 4; + Rollback = 5; } enum PostReadType { diff --git a/tx_service/include/remote/remote_type.h b/tx_service/include/remote/remote_type.h index db6b3029d..8375dd64e 100644 --- a/tx_service/include/remote/remote_type.h +++ b/tx_service/include/remote/remote_type.h @@ -129,6 +129,8 @@ class ToRemoteType return CommitType::DowngradeLock; case PostWriteType::UpdateDirty: return CommitType::UpdateDirty; + case PostWriteType::Rollback: + return CommitType::Rollback; default: assert(false); return CommitType::PostCommit; @@ -358,6 +360,8 @@ class ToLocalType return PostWriteType::DowngradeLock; case CommitType::UpdateDirty: return PostWriteType::UpdateDirty; + case CommitType::Rollback: + return PostWriteType::Rollback; default: assert(false); return PostWriteType::PostCommit; diff --git a/tx_service/include/schema.h b/tx_service/include/schema.h index 5856aea1d..6fa4ef80a 100644 --- a/tx_service/include/schema.h +++ b/tx_service/include/schema.h @@ -434,7 +434,9 @@ struct TableKeySchemaTs * * @param table_name key schema name. * - * @return If the key is newly added, return 1. Otherwise, return normal ts. + * @return The key's recorded ts, or 0, the unknown timestamp, if the image + * records none: a key its DDL creates, which takes the version the schema + * is built at (CatalogFactory::CreateTableSchema). */ uint64_t GetKeySchemaTs(const txservice::TableName &table_name) const { @@ -445,12 +447,12 @@ struct TableKeySchemaTs else { auto v_it = sk_schemas_ts_.find(table_name); - return v_it == sk_schemas_ts_.end() ? 1 : v_it->second; + return v_it == sk_schemas_ts_.end() ? 0 : v_it->second; } } TableEngine table_engine_{TableEngine::None}; - uint64_t pk_schema_ts_{1}; + uint64_t pk_schema_ts_{0}; ///< The base table's ts; 0 if unknown. std::unordered_map sk_schemas_ts_; }; diff --git a/tx_service/include/sharder.h b/tx_service/include/sharder.h index 52ae9d2e4..eccbb1d07 100644 --- a/tx_service/include/sharder.h +++ b/tx_service/include/sharder.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -356,22 +357,28 @@ class Sharder /** * @brief When a remote cc node group fails over, this node is unaware of * the new leader. Its cc requests are directed to a non-leader node and - * return with errors. The method updates the leader of the input cc node - * group in the local cache. + * return with errors. The method asks the host manager for the leader of + * the input cc node group, then that node for its term (the host manager + * answers from braft's leader lookup, which has none), and updates the + * local cache with both. A node that no longer leads the group answers no + * term, and the lookup changes nothing. * * @param ng_id The cc node group ID. */ void UpdateLeader(uint32_t ng_id); /** - * @brief NotifyNewLeaderStart rpc will send the node_id of the new leader - * to each node. Update the leader cache without referring to hm service. + * @brief Updates the leader cache with a leader and its term: from the + * host manager's NotifyNewLeaderStart rpc, a log write's reply, or a + * lookup (UpdateLeader(ng_id)). An update at or below the newest term + * learned here is out-dated and skipped, so updates that arrive in any + * order leave the newest leader cached. * * @param ng_id The cc node group ID. * @param node_id The node_id of leader. * @param term The node group leader term. */ - void UpdateLeader(uint32_t ng_id, uint32_t node_id, int64_t term = -1); + void UpdateLeader(uint32_t ng_id, uint32_t node_id, int64_t term); bool OnLeaderStart(uint32_t ng_id, int64_t term, @@ -447,6 +454,23 @@ class Sharder uint32_t log_group_id, int64_t cc_ng_term); + /** + * @brief Asks every log group to replay a node group's log to this node, + * which has just become the node group's candidate leader. + * + * The node asks once, at its leader start (CcNode::OnLeaderStart); the + * log replay service then asks again only for a log group that has not + * answered (RecoveryService). Does nothing without a log service. + * + * @param cc_ng_id The node group. + * @param cc_ng_term The node group's new term. + * @param replay_start_ts The commit timestamp to replay from at the + * latest (0: from the log group's last checkpoint). + */ + void RequestLogReplay(uint32_t cc_ng_id, + int64_t cc_ng_term, + uint64_t replay_start_ts); + /** * @brief Wait for all the tx_service nodes to finish the log recovery * process and setup the cc_stream_sender. @@ -785,6 +809,18 @@ class Sharder inline void RebalanceNgMembers( std::unordered_map> &ng_configs); + /** + * @brief Asks a node for its leader term of a node group: this node's own + * terms when it is the node, else its FetchNodeInfo rpc. + * + * @param ng_id The cc node group ID. + * @param node_id The node a lookup named as the group's leader. + * @return The node's leader term of the group (its candidate term while + * it replays the log), or -1 if it does not lead the group or did not + * answer. + */ + int64_t FetchLeaderTerm(uint32_t ng_id, uint32_t node_id); + private: uint32_t node_id_; uint32_t native_ng_; @@ -801,11 +837,20 @@ class Sharder // The replicate number of node group. uint16_t rep_group_cnt_; + /** + * @brief Serializes the updates of the leader cache below, so that a node + * group's leader and term change together; readers of ng_leader_cache_ + * take no lock. + */ + std::mutex ng_leader_mux_; // Ng leader cache. We preallocate it to the max cluster size so that we // don't need to modify the size of it. std::atomic ng_leader_cache_[1000]; - // Ng leader term cache. Note this is not reliable. - std::atomic ng_leader_term_cache_[1000]; + /** + * @brief Per node group, the term of the cached leader, the newest learned + * here (-1: none yet). Under ng_leader_mux_. + */ + int64_t ng_leader_term_cache_[1000]; std::atomic leader_term_cache_[1000]; std::atomic candidate_leader_term_cache_[1000]; diff --git a/tx_service/include/store/data_store_handler.h b/tx_service/include/store/data_store_handler.h index 9632d6ca1..4eebe79b9 100644 --- a/tx_service/include/store/data_store_handler.h +++ b/tx_service/include/store/data_store_handler.h @@ -131,8 +131,8 @@ class DataStoreHandler /** * @param write_time Orders schema writes for idempotence. Commits use * commit_ts. A failed CREATE INDEX rollback normally uses - * max(kv_create_index_op_.write_time_, commit_ts_bound_) + 1 so it - * overwrites the original catalog write. During recovery, rollback uses + * max(the op's timestamp, commit_ts_bound_) + 1 so it overwrites any + * earlier catalog write for the op. During recovery, rollback uses * LocalCcHandler::GetTsBaseValue() instead. */ virtual void UpsertTable( @@ -260,11 +260,20 @@ class DataStoreHandler /** * @brief Upsert list of ranges into range table. This will also update * range slice sizes. + * + * @param table_name The table whose ranges are written. + * @param range_info The ranges to write. Only read: the caller keeps them, + * and retries the call with the same ranges after a failure. + * @param version The version the range entries are written at. + * @return true on success. */ virtual bool UpsertRanges(const TableName &table_name, - std::vector range_info, + const std::vector &range_info, uint64_t version) { + (void) table_name; + (void) range_info; + (void) version; return false; } @@ -299,6 +308,29 @@ class DataStoreHandler const std::function *yield_fptr = nullptr, const std::function *resume_fptr = nullptr) = 0; + /** + * @brief Creates a table's kv table, and nothing else: no catalog row. + * + * The counterpart of DropKvTable. Add index calls it for each index being + * added before its prepare log, and CREATE TABLE for the table and each + * of its indexes after its prepare log, so that the kv tables exist in + * every state recovery can reach from a commit log that commits the op; + * the new schema reaches the kv catalog only after the commit log. + * Synchronous and idempotent: a retry, or a later create of a table whose + * kv tables a failed op left behind, creates them again. The default suits + * a store that keeps no per-table state. + * + * @param table_name The table to create. + * @param version The table's schema version. + * @return true on success. + */ + virtual bool CreateKvTable(const TableName &table_name, uint64_t version) + { + (void) table_name; + (void) version; + return true; + } + virtual bool DropKvTable(const std::string &kv_table_name) = 0; virtual void DropKvTableAsync(const std::string &kv_table_name) = 0; @@ -511,6 +543,32 @@ class DataStoreHandler return true; } + /** + * @brief Notifies the store of a node group's leader at a newer term. + * + * Called on every node when it learns of a newer leader term + * (Sharder::UpdateLeader), also when the leader is the same node, + * whether or not the node is a member of the group. Does nothing by + * default; a store whose data shards follow the node groups, each hosted + * by its group's leader, records the shard's host and version. + * + * @param ng_id The node group. + * @param leader_node_id The node now leading it. + * @param term The node group's leader term of that node, the one + * number every node derives for the same leadership, so a store that + * versions its shards by ownership can take it as the shard's version. + * Calls may arrive in any order; such a store ignores one whose term is + * below the shard's version, which leaves the newest leader in place. + */ + virtual void OnLeaderChanged(uint32_t ng_id, + uint32_t leader_node_id, + int64_t term) + { + (void) ng_id; + (void) leader_node_id; + (void) term; + } + virtual std::string SnapshotSyncDestPath() const { return std::string(""); diff --git a/tx_service/include/tx_execution.h b/tx_service/include/tx_execution.h index 58cc5c067..c70873172 100644 --- a/tx_service/include/tx_execution.h +++ b/tx_service/include/tx_execution.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "cc/cc_handler.h" @@ -64,6 +65,7 @@ struct SplitFlushTxRequest; struct AnalyzeTableTxRequest; struct BroadcastStatisticsTxRequest; struct ClusterScaleTxRequest; +struct ClusterGateTxRequest; struct SchemaRecoveryTxRequest; struct RangeSplitRecoveryTxRequest; struct UnlockTuple; @@ -162,6 +164,12 @@ class TransactionExecution : public LinkedTransaction void ProcessTxRequest(AnalyzeTableTxRequest &analyze_req); void ProcessTxRequest(BroadcastStatisticsTxRequest &analyze_req); void ProcessTxRequest(ClusterScaleTxRequest &scale_req); + /** + * @brief Starts a ClusterGateOp that pauses or resumes the cluster gate. + * + * @param gate_req The request; its result is the op's commit timestamp. + */ + void ProcessTxRequest(ClusterGateTxRequest &gate_req); void ProcessTxRequest(SchemaRecoveryTxRequest &recover_req); void ProcessTxRequest(RangeSplitRecoveryTxRequest &recover_req); void ProcessTxRequest(BatchReadTxRequest &batch_read_req); @@ -301,6 +309,21 @@ class TransactionExecution : public LinkedTransaction bool dm_started, bool dm_finished); + /** + * @brief Resumes a cluster gate op whose log entry was written, at its + * install, on the node group that coordinated it (EloqDoc + * docs/cdc-change-stream.md §6.2). + * + * The caller has set the recovered tx's number, term and commit timestamp + * (SetRecoverTxState). The op takes no lock: it re-sends the install of + * the logged value to every node group, which stores it and releases the + * cluster config write lock wherever the tx still holds it, and does + * nothing elsewhere; then it commits the tx. Nobody waits on it. + * + * @param gate_msg The logged gate value. + */ + void RecoverClusterGate(const ::txlog::ClusterGateLogMessage &gate_msg); + /** * @brief Acquires the exclusive forward latch. * @@ -619,17 +642,43 @@ class TransactionExecution : public LinkedTransaction // can help us to locate the previous empty cc entry quickly. CcEntryAddr cache_miss_read_cce_addr_; - std::unique_ptr schema_op_; - - std::unique_ptr split_flush_op_; - - std::unique_ptr index_op_; - - std::unique_ptr cluster_scale_op_; - std::unique_ptr migration_op_; + /** + * @brief The composite op slot as a slot for an op of type Op. + * + * Switches the slot to Op first if it holds another kind, which drops + * that op: a txm runs one tx request at a time. + * + * @return The slot's pointer, null until an op is put in it. + */ + template + std::unique_ptr &CompositeOpSlot() + { + if (auto *slot = std::get_if>(&composite_op_)) + { + return *slot; + } + return composite_op_.template emplace>(); + } - std::unique_ptr - invalidate_table_cache_composite_op_; + /** + * @brief The composite op of the tx request this txm runs, if it needs + * one. + * + * A txm runs one tx request at a time, so one slot holds whichever kind + * of op that request needs. The ops come from LocalCcShards' pools and go + * back to them when they finish: a remote response that arrives after its + * op finished still reads the op's cc handler result, so an op must + * outlive its request. + */ + std::variant, + std::unique_ptr, + std::unique_ptr, + std::unique_ptr, + std::unique_ptr, + std::unique_ptr, + std::unique_ptr> + composite_op_; std::unordered_map< uint64_t, @@ -664,7 +713,8 @@ class TransactionExecution : public LinkedTransaction TxResult< std::tuple> *kvp_resp_; - // Scan open result + // Scan open result, and the commit timestamp of a ClusterGateTxRequest + // (null for a recovered gate op) TxResult *uint64_resp_; // Response whose returned result is vector TxResult> *int64_vec_resp_; @@ -780,6 +830,7 @@ class TransactionExecution : public LinkedTransaction friend struct AcquireAllOp; friend struct PostWriteAllOp; friend struct CatalogAcquireAllOp; + friend struct SchemaOp; friend struct UpsertTableOp; friend struct DsUpsertTableOp; friend struct FlushUpdateTableOp; @@ -792,6 +843,7 @@ class TransactionExecution : public LinkedTransaction friend struct BroadcastStatisticsOp; friend struct KickoutDataOp; friend struct ClusterScaleOp; + friend struct ClusterGateOp; friend struct NoOp; template friend struct AsyncOp; diff --git a/tx_service/include/tx_index_operation.h b/tx_service/include/tx_index_operation.h index ebb2a6424..497900ad3 100644 --- a/tx_service/include/tx_index_operation.h +++ b/tx_service/include/tx_index_operation.h @@ -79,7 +79,9 @@ struct UpsertTableIndexOp : public SchemaOp /** * @brief Acquire read lock on local cluster config ccmap to block cluster * config update during upsert table op. We cannot allow config update - * between acquire write all and post write all. + * between acquire write all and post write all. The op's first step; it + * holds the lock until its last post write all + * (unlock_cluster_config_op_). */ ReadLocalOperation lock_cluster_config_op_; /** @@ -93,6 +95,24 @@ struct UpsertTableIndexOp : public SchemaOp * and old tuples. */ AcquireAllOp upgrade_all_intent_to_lock_op_; + /** + * @brief For add index, creates the kv table of each index being added + * (DataStoreHandler::CreateKvTable), before the prepare log and under the + * prepare-stage write lock. + * + * Recovery returns a node to the state right after the prepare log, in + * which the dirty schema is installed and DML writes the new index, so its + * kv table must exist in every state recovery can reach. The kv catalog is + * written only after the commit log (kv_update_schema_image_op_). + */ + AsyncOp create_kv_table_op_; + /** + * @brief Drops the kv tables create_kv_table_op_ created, when the add + * index fails before its prepare log is flushed. A coordinator failure + * before it finishes leaves them orphaned, which is harmless: no schema + * refers to them. + */ + AsyncOp drop_kv_table_op_; /** * @brief Flushes the prepare log to the log service. The schema operation * is guaranteed to succeed after this stage. @@ -104,14 +124,10 @@ struct UpsertTableIndexOp : public SchemaOp */ PostWriteAllOp downgrade_all_lock_to_intent_op_; /** - * @brief Release cluster config read lock after post write all. + * @brief Release cluster config read lock after post write all: after the + * op's last one, post_all_lock_op_, before the clean log. */ PostReadOperation unlock_cluster_config_op_; - /** - * @brief Creates/deletes the data store table and persists/removes the - * binary representation of the catalog in the data store. - */ - DsUpsertTableOp kv_create_index_op_; /** * @brief Update the sequence table for range id info. */ @@ -160,8 +176,19 @@ struct UpsertTableIndexOp : public SchemaOp */ DsUpsertTableOp kv_rollback_create_index_op_; /** - * @brief For creating index, update the schema image inside kv-storage. - * Optional operation. + * @brief For creating index, writes the new schema (with the indexes' + * multikey attributes) to the kv catalog at the commit stage, after the + * commit log and under the commit-stage write lock. + * + * The index's kv table exists since before the prepare log + * (create_kv_table_op_), and until the commit log the prepare log carries + * the dirty schema for any node that recovers. Written at the op's + * timestamp, so the kv catalog version matches the in-memory schema + * version; retried until it succeeds, since after the commit log the op + * only rolls forward. The write lock keeps any later DDL on the table from + * writing its catalog row first: the DSS RocksDB backends apply writes in + * arrival order, not by version, and a later drop table deletes the row, + * leaving nothing a late write could be compared against. */ DsUpsertTableOp kv_update_schema_image_op_; /** @@ -192,6 +219,20 @@ struct UpsertTableIndexOp : public SchemaOp }; bool is_last_finished_key_str_; + /** + * @brief Whether the ADD INDEX was rolled back: its build broke a + * constraint after the prepare log. + * + * The op still commits, at its commit-stage timestamp, and the commit log + * carries this mark instead of the zero commit timestamp that used to mean + * the same: the index's kv tables are dropped, the catalog row keeps the + * table's current schema at the commit timestamp, and the second phase, a + * Rollback at that timestamp, drops the index's cc maps and keeps the + * current schema instead of promoting the dirty one. Recovery reads it + * from the commit log. + */ + bool rolled_back_{false}; + public: void RecoveryIndexesMultiKeyAttr(const TableSchema *dirty_schema); @@ -201,6 +242,81 @@ struct UpsertTableIndexOp : public SchemaOp void FillCommitLogRequest(TransactionExecution *txm); void ForceToFinish(TransactionExecution *txm); + /** + * @brief Starts create_kv_table_op_, which creates the kv table of each + * index being added. + * + * Called once `commit_ts_` is chosen under the prepare-stage write lock, + * before the prepare log. + * + * @param txm The transaction running this operation. + */ + void CreateKvTables(TransactionExecution *txm); + + /** + * @brief Starts drop_kv_table_op_, which drops the kv tables of the + * indexes being added, and then releases the locks. + * + * Called when the add index fails before its prepare log is flushed. + * + * @param txm The transaction running this operation. + */ + void DropKvTables(TransactionExecution *txm); + + /** + * @brief Writes the prepare log, once `commit_ts_` is chosen and, for add + * index, the kv tables exist. + * + * @param txm The transaction running this operation. + */ + void WritePrepareLog(TransactionExecution *txm); + + /** + * @brief Marks the op failed: commit timestamp 0, and a Rollback second + * phase that discards whatever dirty schema the op installed and leaves + * the catalog entry's version alone. + * + * @param txm The coordinating tx. + */ + void MarkFailed(TransactionExecution *txm); + + /** + * @brief Runs a data store step that failed after the prepare log again, + * KvRetryInterval() later (ArmKvRetryWait); ends the op if the node is no + * longer the leader. + * + * A failure before the prepare log (create_kv_table_op_) is not retried: + * the op fails. + * + * @param txm The coordinating tx. + * @param step The step to run again: update_sequence_table_op_, + * flush_all_old_tuples_sk_op_, kv_drop_index_op_, + * kv_rollback_create_index_op_ or kv_update_schema_image_op_. + */ + void RetryKvStep(TransactionExecution *txm, TransactionOperation *step); + + /** + * @brief Fault-injection hold (alter_table_index_hold_before_commit_lock): + * keeps an add index whose build has finished, after its last prepare data + * log, from taking its commit-stage write locks until the test disarms the + * hold, or for hold_s seconds (the fault's parameter "hold_s=N", 60 by + * default), still holding the cluster config read lock it took first. + * Arms the alter_table_index_held_before_commit_lock marker while it + * holds, so a test can tell the op got there. + * + * The wait is kv_retry_wait_op_'s blocking work on a worker thread, and + * the op's kv_retry_wait_op_ stage then runs acquire_all_lock_op_, so the + * TxProcessor keeps serving other requests meanwhile. Only fault-injection + * builds call it. + * + * @warning While the fault is armed it holds every add index of any table + * that finishes its build. A test arms it only around the one build it + * means to hold and disarms it in a finally. + * + * @param txm The coordinating tx. + */ + void HoldBeforeCommitLock(TransactionExecution *txm); + // Flush pk or sk data from ccmap into data store. void FlushDataIntoDataStore(const TableName &table_name, NodeGroupId ng_id, @@ -249,9 +365,6 @@ struct UpsertTableIndexOp : public SchemaOp static bool IndexesMergeMultiKeyAttr(std::vector &from, std::vector &to); - static bool AnyMultiKeyIndex( - const std::vector &indexes_multikey_attr); - static std::string RebuildSchemaImage( const TableSchema *dirty_schema, const std::vector &indexes_multikey_attr); @@ -273,6 +386,12 @@ struct UpsertTableIndexOp : public SchemaOp // operation bool is_force_finished_{false}; + /** + * The cluster gate refused the op at its first step: it holds nothing but + * the cluster config read lock, releases it and ends without a log. + */ + bool gated_{false}; + #ifdef NDEBUG static constexpr uint8_t scan_batch_range_size_{10}; #else diff --git a/tx_service/include/tx_operation.h b/tx_service/include/tx_operation.h index 6e3c415dd..7c7be5a6e 100644 --- a/tx_service/include/tx_operation.h +++ b/tx_service/include/tx_operation.h @@ -21,6 +21,8 @@ */ #pragma once +#include +#include #include #include #include @@ -43,10 +45,12 @@ #include "tx_key.h" #include "tx_operation_result.h" #include "tx_record.h" +#include "tx_req_result.h" #include "type.h" namespace txservice { +class FaultEntry; class TransactionExecution; struct ReadTxRequest; struct ReadOutsideTxRequest; @@ -143,6 +147,7 @@ struct ReadLocalOperation : TransactionOperation rec_ = rec; hd_result_ = hd_res; execute_immediately_ = true; + is_for_write_ = false; } void Forward(TransactionExecution *txm) override; @@ -153,6 +158,11 @@ struct ReadLocalOperation : TransactionOperation empty_sv, TableType::RangePartition, TableEngine::None}; TxRecord *rec_{}; bool execute_immediately_{true}; + /** + * @brief Whether to read for write: a write intent instead of a read + * lock, for a tx that writes the key next. + */ + bool is_for_write_{false}; // out-parameters, to pass result to caller operation CcHandlerResult *hd_result_{}; @@ -665,11 +675,13 @@ struct DsUpsertTableOp : public TransactionOperation CcHandlerResult hd_result_; txservice::AlterTableInfo *alter_table_info_{nullptr}; - // write_time_ is corresponding to kv-storage's write_time. Generally, - // write_time_ is assigned to commit_ts and commit_ts is regarded as new - // schema version. But for rollback create-index, write_time_ is assigned to - // `max(commit_ts, latest commit_ts_bound_) + 1` to overwrite catalog image - // in kv-storage. + /** + * The kv store's write time for the catalog image: the op's commit + * timestamp, which is the new schema version. A rolled-back add index + * writes its catalog image back at its commit-stage timestamp, above the + * prepare-stage one the image was first written at, so that the rollback + * overwrites it. + */ uint64_t write_time_{0}; }; @@ -684,6 +696,24 @@ struct AsyncOp : public TransactionOperation void Forward(TransactionExecution *txm) override; void Reset(); + /** + * @brief Sets op_func_ to run blocking work on worker_thread_ and finish + * hd_result_ with the code it returns. + * + * For work that must not run on the tx processor thread, which calls + * op_func_: a synchronous store call, an RPC, a sleep. Each run starts a + * new thread that calls the stored work; Forward() joins it once + * hd_result_ is finished. Work with a result value sets it on hd_result_ + * before returning. + * + * @param work Runs on the worker thread; returns CcErrorCode::NO_ERROR on + * success, else the error hd_result_ gets. + * @warning A retry runs the same work again, so the work must leave what + * it captured intact: pass captured data to callees by reference, never + * by std::move. + */ + void SetBlockingWork(std::function work); + std::function &async_op)> op_func_; // hd_result_ represents the async result, and op_func_ should finish it @@ -702,7 +732,8 @@ struct SchemaOp : public TransactionOperation const std::string ¤t_image, const std::string &dirty_image, uint64_t schema_ts, - OperationType op_type); + OperationType op_type, + TransactionExecution *txm); void FillPrepareLogRequestCommon(TransactionExecution *txm, WriteToLogOp &prepare_log_op); @@ -711,14 +742,177 @@ struct SchemaOp : public TransactionOperation void FillCleanLogRequestCommon(TransactionExecution *txm, WriteToLogOp &clean_log_op); + /** + * @brief Replaces the op's schema images, as the engine built them, with + * the images of schema objects built at known timestamps. + * + * An engine builds a DDL's new image before the DDL has a timestamp, so it + * may record a table the DDL creates without one (0, the unknown + * timestamp). Once the prepare stage has chosen the op's timestamp, and + * before anything is logged or installed, the op builds the new schema at + * that timestamp and the current schema at its version and keeps their + * images, in which the engine records every base and index table's real + * timestamp. Every log + * record, catalog row and copy sent to another node then carries each + * table's own version, whatever version a node later loads it at. + * CatalogFactory::CreateTableSchema states what each engine must do for + * this. + * + * @param dirty_version The op's prepare-stage timestamp: the version of a + * table the op creates. + */ + void ResolveSchemaImages(uint64_t dirty_version); + CatalogKey table_key_; // string owner CatalogRecord catalog_rec_; std::string image_str_{""}; std::string dirty_image_str_{""}; uint64_t curr_schema_ts_; OperationType op_type_{OperationType::Insert}; + + /** + * @brief Sets the pause before a failed data store step that must succeed + * runs again: the ddl_kv_retry_interval_seconds setting. Called at + * start-up, before any schema op runs. + * + * @param interval The pause, at least one second. + */ + static void SetKvRetryInterval(std::chrono::seconds interval) + { + assert(interval.count() > 0); + kv_retry_interval_ = interval; + } + + /** + * @return The pause before a failed data store step that must succeed runs + * again. + */ + static std::chrono::seconds KvRetryInterval() + { + return kv_retry_interval_; + } + + /** + * @brief Waits KvRetryInterval() before a failed data store step runs + * again (ArmKvRetryWait). + */ + AsyncOp kv_retry_wait_op_; + /** @brief The step to run again once kv_retry_wait_op_ ends. */ + TransactionOperation *kv_retry_step_{nullptr}; + /** @brief Failed attempts of kv_retry_step_, for the log. */ + uint32_t kv_retry_failures_{0}; + +protected: + /** + * @brief Starts a data store step as a single attempt. + * + * A DsUpsertTableOp retries on its own unless it is pushed with retry + * count 0; the schema op alone decides whether a failed step runs again. + * + * @param txm The coordinating tx. + * @param step The step. + */ + void StartKvStep(TransactionExecution *txm, DsUpsertTableOp &step); + + /** + * @brief Arms kv_retry_wait_op_ to wait KvRetryInterval() before a failed + * data store step runs again, and logs a warning. + * + * A data store step that fails before the op's point of no return, the + * commit log of UpsertTableOp or the prepare log of UpsertTableIndexOp, is + * not retried: the op rolls back. TRUNCATE TABLE, which never rolls back + * after its prepare log, retries the steps it runs before its commit log. + * A step that fails after the point of no return must succeed: + * the op retries it indefinitely, KvRetryInterval() apart, while its node + * is the leader, holding the table's catalog lock and keeping the client + * waiting. A node that loses leadership stops, and whoever recovers the op + * retries the step the same way. The caller makes kv_retry_wait_op_ its + * current stage, and once the wait ends runs the step again + * (RunKvStepAgain). + * + * @param txm The coordinating tx. + * @param step The failed step. + * @param step_name The step, for the log. + * @param error The step's error message, for the log. + */ + void ArmKvRetryWait(TransactionExecution *txm, + TransactionOperation *step, + std::string_view step_name, + const std::string &error); + + /** + * @brief Runs kv_retry_step_ again, as a single attempt, once + * kv_retry_wait_op_ has ended. + * + * @param txm The coordinating tx. + */ + void RunKvStepAgain(TransactionExecution *txm); + + /** + * @brief Clears the retry state, when a pooled op is reused. + * + * @param txm The tx the op now runs in. + */ + void ResetKvRetry(TransactionExecution *txm); + + /** + * @brief Releases the cluster config read lock, which the op took at its + * first step, or which recovery took before resuming it + * (LocalCcShards::CreateSchemaRecoveryTx). + * + * A schema op holds the lock until its last post write all, so that each + * step it sends to all node groups reaches the same node groups: a cluster + * scale waits for it. + * + * @param txm The coordinating tx. + * @param unlock_op The op's unlock step, which the caller made its current + * stage. + */ + void StartUnlockClusterConfig(TransactionExecution *txm, + PostReadOperation &unlock_op); + +private: + /** + * @brief The pause before a failed data store step that must succeed runs + * again (KvRetryInterval()); 30 seconds unless the + * ddl_kv_retry_interval_seconds setting says otherwise. + */ + static inline std::chrono::seconds kv_retry_interval_{30}; }; +/** + * @brief The schema op of CREATE, DROP, TRUNCATE and ALTER TABLE + * (tx_upsert_table_operation.cpp). + * + * The op runs as a stackful coroutine (boost::context) on a stack of its own: + * the first Forward() starts it and every later one resumes it, once the step + * it awaits has finished. Its path is written out flat, in three segments cut + * at the two log records, which are where recovery resumes an op: + * RunUntilPrepareLog (the cluster config read lock, the write intents, the + * prepare-stage timestamp, the prepare log), RunUntilCommitLog (the dirty + * schema's install, the op's own steps before its commit log, the write + * locks, the commit log) and RunAfterCommitLog (the op's own steps after its + * commit log, then RunEnding: the post write all, the release of the cluster + * config lock, which the op holds until then, and the clean log). + * + * Return values. A function on the coroutine's path that returns bool says + * whether the op proceeds to its following steps: + * - true: the function's steps are done, and the caller proceeds. That is not + * always success: a create that gave up, or an ALTER whose catalog row write + * failed, proceeds once its commit log at timestamp 0 is written, and its + * ending rolls it back. + * - false: the op has ended and must not proceed: it failed at or before its + * prepare log and was rolled back, or the node lost its leadership + * (ForceFinish). result_ is set. Every caller returns at once, up to Run(), + * and Forward() calls Finish(). + * + * KvWriteFailed and PrepareSequenceStep run no step and are outside this rule: + * they return whether the write failed and whether the table has sequences. + * + * @warning The coroutine may be resumed on another thread than the one that + * suspended it, so its body must not access thread_local state directly: an + * address computed before a suspension would be the old thread's. + */ struct UpsertTableOp : public SchemaOp { UpsertTableOp() = delete; @@ -741,14 +935,28 @@ struct UpsertTableOp : public SchemaOp void Forward(TransactionExecution *txm) override; /** - * @brief The current stage of this multi-stage schema operation. - * + * @brief Where a recovered op resumes: after the log record it was + * recovered from. + */ + enum class Entry + { + kStart, ///< A new op, from its first step. + kAfterPrepareLog, ///< Recovered from its prepare log. + kAfterCommitLog ///< Recovered from its commit log. + }; + + /** + * @brief The step the coroutine awaits, for tracing. */ TransactionOperation *op_{nullptr}; + /** @brief Where Run() starts; recovery sets it. */ + Entry entry_{Entry::kStart}; /** * @brief Acquire read lock on local cluster config ccmap to block cluster * config update during upsert table op. We cannot allow config update - * between acquire write all and post write all. + * between acquire write all and post write all. The first step of every + * op, which holds the lock until its last post write all + * (unlock_cluster_config_op_). */ ReadLocalOperation lock_cluster_config_op_; /** @@ -770,18 +978,66 @@ struct UpsertTableOp : public SchemaOp */ PostWriteAllOp post_all_intent_op_; /** - * @brief Release cluster config read lock after post write all. + * @brief Release cluster config read lock after post write all: after the + * op's last one, post_all_lock_op_, before the clean log. */ PostReadOperation unlock_cluster_config_op_; /** - * @brief Creates/deletes the data store table and persists/removes the - * binary representation of the catalog in the data store. + * @brief CREATE TABLE only: creates the kv tables of the new table and of + * its indexes, after the prepare log and under the write intent. + * + * On the data store service that is each range-partitioned table's + * initial range record, at the prepare-stage timestamp, which the + * in-memory ranges match (PostCommit). It precedes the commit log, so a + * failure is not retried: the create gives up (AbandonCreate). Every state + * recovery can reach from a commit log that commits the create has the kv + * tables. + */ + AsyncOp create_kv_table_op_; + /** + * @brief CREATE TABLE only: once the create gives up, removes what + * create_kv_table_op_ created, before the commit log that fails the op. + * + * The store's drop-table path, over the dirty schema at the prepare-stage + * timestamp: the kv tables, their ranges and the catalog key, which the + * create has not written. One attempt: a failure leaves them orphaned, + * which is accepted: they hold no data, no schema refers to them, and a + * later create of the same name writes them again (CreateKvTable is + * idempotent). + */ + DsUpsertTableOp drop_kv_table_op_; + /** + * @brief TRUNCATE TABLE on shared storage: drops the base table's kv table + * (DataStoreHandler::DropKvTable), under the write locks and before the + * commit log, retried until it succeeds; the catalog row follows after the + * commit log (upsert_kv_table_op_). + * + * That is all the store's own truncate (UpsertTable, TruncateTable) does to + * an EloqKV table's data, the only kind a truncate runs on: EloqKV tables + * are hash partitioned, have no indexes and keep no statistics, so there + * are no ranges or statistics to reset. The kv table keeps its name: the + * store keeps no per-table state. + */ + AsyncOp truncate_kv_table_op_; + /** + * @brief Writes the data store side of the op; each run names its store + * operation (AwaitUpsertKvTable). * + * CREATE TABLE and, on shared storage, TRUNCATE TABLE: the catalog row + * only, after the commit log, at the commit-stage timestamp (op type + * Update). DROP TABLE: deletes the kv tables, after the commit log. Those + * are retried until they succeed (AwaitKvRetry). ALTER TABLE (Update): the + * new catalog row, before the commit log, rolled back on failure. */ DsUpsertTableOp upsert_kv_table_op_; /** * @brief Update the sequence table for range id info and the auto increment * value if this table has auto_increment column. + * + * CREATE TABLE runs it before the commit log, and gives up on a failure + * (AbandonCreate). TRUNCATE TABLE on shared storage runs it before the + * commit log and DROP TABLE after it, both retrying it until it succeeds + * (AwaitKvRetry). */ AsyncOp update_sequence_table_op_; /** @@ -789,6 +1045,12 @@ struct UpsertTableOp : public SchemaOp * */ AcquireAllOp acquire_all_lock_op_; + /** + * @brief Downgrades the write locks to write intents when + * acquire_all_lock_op_ deadlocks, so that transactions waiting to read the + * catalog go ahead; the op then acquires the write locks again. + */ + PostWriteAllOp downgrade_lock_op_; /** * @brief Flushes the commit log to the log service. The commit log confirms * that the data store operation succeeds and does not need redo upon @@ -798,8 +1060,7 @@ struct UpsertTableOp : public SchemaOp WriteToLogOp commit_log_op_; /** - * @brief Clean ccmap on all node groups - * + * @brief Empties the table's cc maps on all node groups (CleanCcm). */ KickoutDataAllOp clean_ccm_op_; @@ -820,10 +1081,493 @@ struct UpsertTableOp : public SchemaOp ClusterConfigRecord cluster_conf_rec_; private: + /** @brief The state of the op's coroutine. */ + enum class CoroStatus + { + kEmpty, ///< Not started. + kRunning, ///< Started, suspended while a step runs. + kFinished ///< Its body has returned; Forward() ends the op. + }; + + /** + * @brief The coroutine's stack size: sub-operations can run cc requests + * inline and send remote requests on it. + */ + static constexpr size_t kCoroStackSize = 128 * 1024; + void FillPrepareLogRequest(TransactionExecution *txm); void FillCommitLogRequest(TransactionExecution *txm); - void ForceToFinish(TransactionExecution *txm); - // Due to term or other error, called ForceToFinish to terminate this + + /** + * @brief Suspends the coroutine, back to Forward(), until a step has + * finished; returns at once if it already has. + * + * A step has finished once it has left the tx's stack: its PostProcess + * pops it, and the tx then forwards the op again, which resumes the + * coroutine. A step can finish inside its Process() already, and then the + * coroutine goes on without a suspension. + * + * @param txm The coordinating tx. + * @param op The step, which the caller pushed and started. + */ + void SuspendUntilDone(TransactionExecution *txm, + const TransactionOperation &op); + + /** + * @brief Runs a sub-operation to its end: pushes and processes it, then + * suspends until it has finished and left the tx's stack + * (SuspendUntilDone). + * + * @note A retry awaits the same sub-operation again without resetting it: + * the tx's Process() for every sub-operation this op runs resets its + * result (and, for an AcquireAllOp, each result and the counters) before + * sending anything. The exception is the unlock step, a PostReadOperation, + * which StartUnlockClusterConfig resets and which never runs twice. + * + * @param txm The coordinating tx. + * @param op The sub-operation. + */ + template + void Await(TransactionExecution *txm, OpT &op); + + /** + * @brief Await(txm, op) with a fault point: while the fault point is + * armed, the sub-operation fails with fault_error without running. + * + * A sub-operation failed this way is neither pushed nor awaited: its + * hd_result_ holds the error when the call returns, and the coroutine goes + * on without a suspension. + * + * For a sub-operation whose result is a single hd_result_; one without + * such a result does not compile with it. + * + * @param txm The coordinating tx. + * @param op The sub-operation. + * @param fault_point The fault point's name. + * @param fault_error The error the sub-operation reports when it fires. + */ + template + void Await(TransactionExecution *txm, + OpT &op, + const char *fault_point, + CcErrorCode fault_error); + + /** + * @brief Runs a data store step to its end as a single attempt + * (StartKvStep). + * + * @param txm The coordinating tx. + * @param step The step. + */ + void AwaitKvStep(TransactionExecution *txm, DsUpsertTableOp &step); + + /** + * @brief The coroutine's body: the op's path, entered at entry_. + * + * @param txm The coordinating tx. + */ + void Run(TransactionExecution *txm); + + /** + * @brief The first segment, the same for every op: the cluster config read + * lock, the write intents, the prepare-stage timestamp D, the prepare log. + * A failure rolls the op back. + * + * @param txm The coordinating tx. + * @return true if the prepare log is written: the op proceeds to + * RunUntilCommitLog. false if the op has ended and must not proceed: it + * failed at or before the prepare log and was rolled back, or the node + * lost its leadership. + */ + bool RunUntilPrepareLog(TransactionExecution *txm); + + /** + * @brief The second segment: the dirty schema's install, then the op's own + * steps up to and including its commit log. + * + * @param txm The coordinating tx. + * @return true if the commit log is written: the op proceeds to + * RunAfterCommitLog, also when the commit log, at timestamp 0, rolls a + * failed op back. false if the op has ended and must not proceed: the node + * lost its leadership. + */ + bool RunUntilCommitLog(TransactionExecution *txm); + + /** + * @brief CREATE TABLE before its commit log: the kv tables, the sequence + * table, the write locks at the commit-stage timestamp P, the commit log. + * A data store failure gives the create up (AbandonCreate). + * + * @param txm The coordinating tx. + * @return true if the commit log is written, at the commit timestamp or, + * for a create that gave up, at timestamp 0: the op proceeds either way. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool CreateUntilCommitLog(TransactionExecution *txm); + + /** + * @brief DROP TABLE before its commit log: the write locks at the + * commit-stage timestamp P, the commit log. + * + * @param txm The coordinating tx. + * @return true if the commit log is written: the op proceeds. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool DropUntilCommitLog(TransactionExecution *txm); + + /** + * @brief TRUNCATE TABLE before its commit log: the write locks at the + * commit-stage timestamp P, on shared storage the table's data, each step + * retried until it succeeds, then the commit log. + * + * On shared storage the data is the cc maps (CleanCcm), then the kv table + * (truncate_kv_table_op_), then the sequences' reset; the catalog row waits + * for the commit log (TruncateAfterCommitLog). A participant that recovers + * from the commit log, and takes writes into the emptied table, then finds + * no later step that wipes them. Without shared storage every step comes + * after the commit log, as each node group truncates its own store: one + * that recovers from the commit log truncates its store in the replay, + * before it takes writes, and the coordinator's later request skips it. + * + * @param txm The coordinating tx. + * @return true if the commit log is written: the op proceeds. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool TruncateUntilCommitLog(TransactionExecution *txm); + + /** + * @brief ALTER TABLE before its commit log: the new catalog row, the write + * locks, the commit log. A failed catalog row write rolls the op back with + * a commit log at timestamp 0. + * + * @param txm The coordinating tx. + * @return true if the commit log is written, at the commit timestamp or, + * after a failed catalog row write, at timestamp 0: the op proceeds either + * way. false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool AlterUntilCommitLog(TransactionExecution *txm); + + /** + * @brief The third segment: the op's own steps after its commit log, then + * RunEnding. + * + * @param txm The coordinating tx. + */ + void RunAfterCommitLog(TransactionExecution *txm); + + /** + * @brief CREATE TABLE after its commit log: the catalog row, retried until + * it is written; nothing for a create that gave up. + * + * @param txm The coordinating tx. + * @return true if the catalog row is written, or the create gave up and + * has none to write: the op proceeds to RunEnding. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool CreateAfterCommitLog(TransactionExecution *txm); + + /** + * @brief DROP TABLE after its commit log: the kv table's deletion and the + * sequences' deletion, each retried until it succeeds, then the cc maps' + * emptying (CleanCcm), which PostCommit follows by dropping them. + * + * @param txm The coordinating tx. + * @return true if the kv table and the sequences are gone and the cc maps + * are empty: the op proceeds to RunEnding. false if the op has ended and + * must not proceed: the node lost its leadership. + */ + bool DropAfterCommitLog(TransactionExecution *txm); + + /** + * @brief TRUNCATE TABLE after its commit log: on shared storage, the + * catalog row at the commit timestamp, retried until it is written; without + * shared storage, each node group empties its cc maps and truncates its own + * store, catalog row included (CleanCcm). + * + * @param txm The coordinating tx. + * @return true if the catalog row is written, or the node groups truncated + * their stores: the op proceeds to RunEnding. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool TruncateAfterCommitLog(TransactionExecution *txm); + + /** + * @brief The ending every op shares: the post write all (PostCommit, or + * Rollback for an op that failed), the release of the cluster config lock, + * the clean log; then sets result_. + * + * @param txm The coordinating tx. + */ + void RunEnding(TransactionExecution *txm); + + /** + * @brief Upgrades the write intents to write locks on every node group, + * retried until held; a deadlock first downgrades the locks + * (downgrade_lock_op_). + * + * @param txm The coordinating tx. + * @return true if the write locks are held: the op proceeds. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool AcquireWriteLocks(TransactionExecution *txm); + + /** + * @brief Writes the commit log at the op's commit timestamp, 0 for an op + * that failed, retried until it is written. + * + * @param txm The coordinating tx. + * @return true if the commit log is written: the op proceeds. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool WriteCommitLog(TransactionExecution *txm); + + /** + * @brief Runs post_all_lock_op_: PostCommit, or Rollback for an op that + * failed. + * + * @param txm The coordinating tx. + * @param retry Whether to retry a failed attempt until one succeeds; if + * false, one attempt, whatever its outcome. + * @return true if the post write all has run: it succeeded, or, with + * retry false, it was attempted; the op proceeds. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool RunPostWriteAll(TransactionExecution *txm, bool retry); + + /** + * @brief Releases the cluster config read lock, after the op's last post + * write all. + * + * @param txm The coordinating tx. + * @return true if the lock is released: the op proceeds. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool ReleaseClusterConfig(TransactionExecution *txm); + + /** + * @brief Runs upsert_kv_table_op_ once over the current and dirty schema, + * at the op's commit timestamp. + * + * @param txm The coordinating tx. + * @param op_type What the store does: Update writes the catalog row only + * (CREATE and TRUNCATE TABLE after their commit log, ALTER TABLE); + * DropTable deletes the kv tables, their ranges and statistics and the + * catalog row. + */ + void AwaitUpsertKvTable(TransactionExecution *txm, OperationType op_type); + + /** + * @brief Whether upsert_kv_table_op_ failed, counting the + * upsert_table_kv_write_fail fault point. + * + * @param txm The coordinating tx, for the log. + * @return true if the write failed. + */ + bool KvWriteFailed(TransactionExecution *txm); + + /** + * @brief Holds the op between its prepare log and its commit-stage write + * locks while a test keeps the upsert_table_hold_before_commit_lock fault + * armed, so that the test can commit DML in that window (EloqDoc + * docs/cdc-change-stream.md §4.4(b), plan B6). + * + * Arms upsert_table_held_before_commit_lock, which the test polls to learn + * that the op is held, then waits on kv_retry_wait_op_, idle before the + * commit log, whose worker thread polls the hold fault every 100 ms; the + * coroutine is suspended meanwhile, so the TxProcessor keeps serving other + * requests. The wait ends when the test disarms the hold, or after 60 s, + * and then disarms the marker. Only fault-injection builds call it. + * + * @warning The fault point sits before RunUntilCommitLog's op-type switch, + * so while the fault is armed it holds every table op that gets there, of + * any table: creates, drops, truncates and ALTERs (index ops run another + * state machine and are not held), each for up to 60 s. The marker does not + * say which op is held. A test arms the fault only around the one DDL it + * means to hold and disarms it in a finally (as EloqDoc's + * eloq_fault/ddl_drop_order.js does), or every later DDL of its suite + * stalls. + * + * @param txm The coordinating tx. + */ + void HoldBeforeCommitLock(TransactionExecution *txm); + + /** + * @brief Waits KvRetryInterval() before a data store step that failed + * after the commit log runs again, and logs a warning (ArmKvRetryWait). + * + * @param txm The coordinating tx. + * @param step The failed step. + * @param step_name The step, for the log. + * @param error The step's error message, for the log. + * @return true if the pause is over: the caller runs the step again. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool AwaitKvRetry(TransactionExecution *txm, + TransactionOperation *step, + std::string_view step_name, + const std::string &error); + + /** + * @brief Sets update_sequence_table_op_'s work if the table has sequences. + * + * @return false if the table has none, so the step is skipped. + */ + bool PrepareSequenceStep(); + + /** + * @brief Sets truncate_kv_table_op_'s work: drops the base table's kv + * table, which keeps its name. + * + * @param txm The coordinating tx, for the log. + */ + void PrepareTruncateKvTable(TransactionExecution *txm); + + /** + * @brief Sets create_kv_table_op_'s work: creates the kv tables of the new + * table and of its indexes, at the op's prepare-stage timestamp. + * + * @param txm The coordinating tx. + */ + void PrepareCreateKvTables(TransactionExecution *txm); + + /** + * @brief Whether an armed fault applies to this op's table: its + * table_name parameter is empty or names the table. The server runs its + * own DDLs (its feature document) beside a test's, so a fault armed + * for a test's table must not fire on them. + * + * @param entry The armed fault. + * @return Whether it applies. + */ + bool FaultAppliesToTable(const FaultEntry &entry) const; + + /** + * @brief Runs an armed action fault (crash and the like) if it applies to + * this op's table (FaultAppliesToTable); a no-op otherwise, and in builds + * without fault injection. + * + * @param fault_name The fault point. + */ + void TriggerTableFault(const std::string &fault_name); + + /** + * @brief Empties the table's cc maps on every node group + * (clean_ccm_op_), retried until it succeeds. + * + * Each core frees a map's pages 32 at a time and yields between batches + * (CcShard::CleanCcmPages). For DROP TABLE, PostCommit then drops the + * emptied maps: a full map would free all its pages inside that one + * post-write request, which cannot yield, stalling the core. For TRUNCATE + * TABLE, whose maps stay, this is what empties them; without shared + * storage, core 0 of each node group also truncates its own kv table in + * the same request. + * + * @param txm The coordinating tx. + * @param base_table_only Whether to clean only the base table: a truncate + * without shared storage, whose node groups truncate their kv tables as + * part of it. + * @return true if the cc maps are cleaned: the op proceeds. + * false if the op has ended and must not proceed: the node lost its + * leadership. + */ + bool CleanCcm(TransactionExecution *txm, bool base_table_only); + + /** + * @brief Re-assigns the commit timestamp after the commit-stage write lock: + * P = max(the prepare-stage timestamp, the lock results' last_vali_ts + 1 + * and commit_ts + 1). + * + * For CREATE, DROP and TRUNCATE TABLE, whose prepare stage holds only a + * write intent: DML of the old table can commit after the prepare stage, + * and the checkpoint can pass its timestamp. Under the lock, P is above + * every DML the lock drained and above the checkpoint, so the commit log, + * the DDL record and PostCommit all carry a timestamp that orders the op + * after the old table's writes and before the new table's. + * + * @param txm The coordinating tx. + */ + void AssignCommitStageTs(TransactionExecution *txm); + + /** + * @brief Gives up a CREATE TABLE whose kv tables or sequence table update + * failed, before its commit log: marks the op failed and removes the kv + * tables it created. + * + * The op has only a prepare log, so it may still fail, as it did before + * the prepare log: drop_kv_table_op_ removes whatever was created, once, + * then the caller writes a commit log with commit timestamp 0, which fails + * the op; the ending rejects the dirty schema and releases the write + * intents, and the client gets DATA_STORE_ERROR. The failed op appends no + * DDL record. Sequence entries it wrote stay, which is harmless: a later + * create of the same name writes them again. A node that recovers the op + * from its prepare log tries its data store steps again, once. + * + * @param txm The coordinating tx. + * @return true if the kv tables' removal has run, whatever its outcome: + * the op proceeds to its commit log at timestamp 0. false if the op has + * ended and must not proceed: the node lost its leadership. + */ + bool AbandonCreate(TransactionExecution *txm); + + /** + * @brief Marks the op failed: commit timestamp 0, and a Rollback second + * phase that discards whatever dirty schema the op installed and leaves + * the catalog entry's version alone. + * + * @param txm The coordinating tx. + */ + void MarkFailed(TransactionExecution *txm); + + /** + * @brief Ends the op on the node's loss of leadership: whoever recovers it + * takes over. + * + * @param txm The coordinating tx. + */ + void ForceFinish(TransactionExecution *txm); + + /** + * @brief The result of an op that got to its clean log, or that ended on a + * loss of leadership: Failed for a failed op, Unverified for one forced + * to finish, Succeeded otherwise. + * + * @param txm The coordinating tx. + * @return The result. + */ + UpsertResult CleanLogResult(TransactionExecution *txm); + + /** + * @brief Ends the op once its coroutine has finished: answers the client + * with result_, pops the op and returns it to the pool. Runs on the tx's + * own stack, not the coroutine's, which the pooled op owns. + * + * @param txm The coordinating tx. + */ + void Finish(TransactionExecution *txm); + + /** @brief The coroutine, while it is suspended. */ + boost::context::continuation coro_; + /** + * @brief Forward()'s side of the switch, which lives on the coroutine's + * stack: set from the coroutine's start until its body returns. + */ + boost::context::continuation *caller_{nullptr}; + CoroStatus coro_status_{CoroStatus::kEmpty}; ///< The coroutine's state. + /** @brief The op's result, which Finish() reports. */ + UpsertResult result_{UpsertResult::Failed}; + + // Due to term or other error, called ForceFinish to terminate this // operation bool is_force_finished; }; @@ -1392,6 +2136,174 @@ struct ClusterScaleOp : public CompositeTransactionOperation TxNumber txn_; }; +/** + * @brief Pauses or resumes the cluster gate (EloqDoc docs/cdc-change-stream.md + * §6.2, "Translation into the tx service"). + * + * Four stages, the second taken in two steps as a schema op takes its catalog + * locks: + * 1. read_gate_op_: the record on this node group, read for write, on which + * the precondition is checked: a pause fails with CLUSTER_GATE_HELD on a + * paused gate, before anything else is locked. The write intent lets + * schema ops' read locks through, and a second gate op coordinated by this + * node group waits for it holding nothing. + * 2a. acquire_intent_op_: write intents on every node group, with OCC. They + * pass schema ops' read locks, so only another gate op or a cluster scale + * makes this fail, and it fails rather than waits: the op then releases + * everything, backs off 1 to 3 seconds and starts again from stage 1. + * Never waiting while holding the entry anywhere is what keeps gate ops + * and cluster scale from deadlocking. AcquireAllOp drops the read from the + * read set here (DedupRead), as for any key a tx reads before writing, so + * the commit does not validate the version read, which stage 4 replaces. + * 2b. acquire_write_op_: write locks on every node group, upgrading the + * intents, with Locking. Once every intent is held no other gate op or + * cluster scale can hold the entry anywhere, so this waits only for schema + * ops already admitted, which hold the read lock through their last post + * write; the queued upgrade keeps new ones out. It is the pause's drain. + * The commit timestamp is raised above its results, so it is above every + * schema op committed before. A failure here also releases and retries. + * 3. gate_log_op_: one cluster_gate_log entry, to the tx's log group, which is + * log group 0. It carries the terms of the node groups that granted the + * lock, so a node group that failed over since, and lost the lock with no + * record of it, fails the log rather than miss the install. + * 4. install_op_: the post write all of a record that carries the new gate; + * each node group stores it and releases the lock. + * A refusal after stage 1 ends the tx, whose abort releases the intent and + * fails the request. A refused log, for instance after a node group failed + * over, starts over like a conflict. Once the log is written the op only rolls + * forward, + * retrying the install a second apart; if the coordinator loses leadership + * meanwhile, the node group's replay installs the logged value and re-runs + * stage 4 alone, with no lock (TransactionExecution::RecoverClusterGate). + */ +struct ClusterGateOp : public CompositeTransactionOperation +{ +public: + ClusterGateOp() = delete; + + /** + * @brief Builds the op's stages for a transaction. + * + * @param txm The transaction that runs the op. + */ + explicit ClusterGateOp(TransactionExecution *txm); + + /** + * @brief Prepares the op for a new request on a transaction. + * + * @param op_type Pause or resume. + * @param mode What a pause rejects. + * @param txm The transaction that runs the op. + */ + void Reset(ClusterGateOpType op_type, + ClusterGate::Mode mode, + TransactionExecution *txm); + + void Forward(TransactionExecution *txm) override; + + /** + * @brief Positions a recovered op right after its log entry, so that the + * next Forward() runs the install. + * + * @param gate The logged value. + * @param commit_ts The entry's commit timestamp, the gate's version. + */ + void ResumeAtInstall(const ClusterGate &gate, uint64_t commit_ts); + + ReadLocalOperation read_gate_op_; ///< Stage 1: the read, for write. + AcquireAllOp acquire_intent_op_; ///< Stage 2a: write intents, OCC. + AcquireAllOp acquire_write_op_; ///< Stage 2b: the write locks. + WriteToLogOp gate_log_op_; ///< Stage 3: the gate's log entry. + PostWriteAllOp install_op_; ///< Stage 4, or a release. + +private: + /** + * @brief Makes a stage current and runs it, or runs it again. + * + * The op's own copy of ForwardToSubOperation(), whose definition is local + * to tx_operation.cpp. + * + * @param txm The transaction that runs the op. + * @param stage The stage to run. + */ + template + void RunStage(TransactionExecution *txm, Op *stage); + + /** + * @brief Gives up this attempt before the log is written: releases every + * lock the tx holds on the entry with a post write all at + * tx_op_failed_ts_; once that + * is done, the op backs off 1 to 3 seconds and starts again from stage 1, + * re-checking the precondition. + * + * @param txm The transaction that runs the op. + * @param stage What failed, for the log. + */ + void RestartLater(TransactionExecution *txm, const char *stage); + + /** + * @brief The second half of RestartLater(), once the release is done: + * drops the read, whose lock went with the release, resets the stages and + * sleeps before stage 1. + * + * @param txm The transaction that runs the op. + */ + void BackOffAndRestart(TransactionExecution *txm); + + /** + * @brief Runs stage 4 again after a second, as long as it fails: on a + * node group that has not finished its replay, which refuses the post + * write until it leads, or on one that is failing over. + * + * @param txm The transaction that runs the op. + */ + void RetryInstallLater(TransactionExecution *txm); + + /** + * @brief Fills the log entry: the new gate, the commit timestamp and the + * terms of the node groups whose locks the tx holds. + * + * @param txm The transaction that runs the op. + */ + void FillGateLogRequest(TransactionExecution *txm); + + /** + * @brief Ends the op without a post write all: after stage 1, when only + * the write intent there is held, which the tx's abort releases, or when + * the coordinator is no longer leader, when orphan lock recovery or the + * node group's replay resolves what the tx still holds. + * + * @param txm The transaction that runs the op. + * @param err What the request reports. + */ + void ForceToFinish(TransactionExecution *txm, TxErrorCode err); + + /** + * @brief Pops the op, returns it to LocalCcShards' pool, answers the + * request and commits the tx, or aborts it if the op failed. + * + * @warning Nothing may read the op after the pool push: another request + * may take it and reset it. What the answer needs is copied before. + * + * @param txm The transaction that runs the op. + */ + void Finish(TransactionExecution *txm); + + ClusterGateOpType op_type_{ClusterGateOpType::kPause}; + ClusterGate::Mode mode_{ClusterGate::Mode::kDdl}; + /** @brief What the request reports if the op does not succeed. */ + TxErrorCode error_{TxErrorCode::NO_ERROR}; + /** @brief The release under way is RestartLater()'s, not a failure's. */ + bool restart_after_release_{false}; + /** @brief Attempts so far, for the log. */ + uint32_t attempt_{0}; + + CcHandlerResult read_result_; ///< Stage 1's result. + ClusterConfigRecord read_rec_; ///< Stage 1's copy of the record. + /** @brief The record stage 4 installs: the new gate, no configs. */ + ClusterConfigRecord install_rec_; +}; + struct DataMigrationOp : public CompositeTransactionOperation { /** diff --git a/tx_service/include/tx_request.h b/tx_service/include/tx_request.h index f88d45082..d71342589 100644 --- a/tx_service/include/tx_request.h +++ b/tx_service/include/tx_request.h @@ -1226,6 +1226,50 @@ struct ClusterScaleTxRequest const std::vector *is_candidate_{nullptr}; }; +/** + * @brief Pauses or resumes the cluster gate, which keeps new schema ops out + * while EloqDoc's change data capture bootstrap copies the catalog (EloqDoc + * docs/cdc-change-stream.md §6.2). + * + * Runs a ClusterGateOp: the cluster config write lock on every node group, + * the precondition read under it, one cluster gate log entry, and the install + * on every node group. The request is the whole transaction: the op commits or + * aborts it on its own, so the caller only waits. Its result is the op's + * commit timestamp, the gate's new version; a pause reports it as pause_ts, + * which is above every schema op committed before the gate closed. + * + * @warning Submit it on a transaction initialized with log_group_id_ = 0, so + * that every gate log entry lands in log group 0, whose slot is the gate's + * only history. + */ +struct ClusterGateTxRequest + : public TemplateTxRequest +{ + /** + * @brief Builds the request. + * + * @param op_type Pause or resume. + * @param mode What a pause rejects; ignored by a resume. + * @param yield_fptr Yields the caller's coroutine while it waits, or null + * to block the thread. + * @param resume_fptr Resumes the caller's coroutine. + * @param txm The transaction the request runs on. + */ + ClusterGateTxRequest(ClusterGateOpType op_type, + ClusterGate::Mode mode, + const std::function *yield_fptr = nullptr, + const std::function *resume_fptr = nullptr, + TransactionExecution *txm = nullptr) + : TemplateTxRequest(yield_fptr, resume_fptr, txm), + op_type_(op_type), + mode_(mode) + { + } + + ClusterGateOpType op_type_; ///< Pause or resume. + ClusterGate::Mode mode_; ///< What a pause rejects. +}; + struct SchemaRecoveryTxRequest : public TemplateTxRequest { diff --git a/tx_service/include/tx_service_metrics.h b/tx_service/include/tx_service_metrics.h index e3b8d2344..e8588bfd8 100644 --- a/tx_service/include/tx_service_metrics.h +++ b/tx_service/include/tx_service_metrics.h @@ -79,6 +79,14 @@ inline const metrics::Name NAME_CHECKPOINT_FAILURES_TOTAL{ inline const metrics::HistogramBuckets CHECKPOINT_INTERVAL_SECONDS_BUCKETS{ 1, 2, 5, 10, 15, 30, 60, 120, 300, 600, 900, 1800, 3600}; +/** + * 1 while this node's cluster config map of a node group holds a paused + * cluster gate (EloqDoc docs/cdc-change-stream.md §6.2), else 0; 0 once the + * node stops leading the group, which drops the map. Its maximum over the + * cluster is what a gate-age alert watches: no pause ends on its own. + */ +inline const metrics::Name NAME_CLUSTER_GATE_PAUSED{"cluster_gate_paused"}; + inline const metrics::Name NAME_LEADER_CHANGES{"leader_changes_seen_total"}; inline const metrics::Name NAME_IS_LEADER{"is_leader"}; diff --git a/tx_service/include/txlog.h b/tx_service/include/txlog.h index 5673adf1f..8521695a6 100644 --- a/tx_service/include/txlog.h +++ b/tx_service/include/txlog.h @@ -23,6 +23,7 @@ #include +#include #include #include @@ -70,6 +71,59 @@ class TxLog virtual ::txlog::CheckClusterScaleStatusResponse::Status CheckClusterScaleStatus(uint32_t log_group_id, const std::string &id) = 0; + /** + * @brief Sends a change-data scan (CdcScan) to the log group's cached + * leader, asynchronously. + * + * The default fails the call at once, for a log without change data + * capture. + * + * @param log_group_id The log group to scan. + * @param cntl The call's controller; its timeout bounds the call. + * @param request The scan request. + * @param[out] response The scan response. + * @param[out] node_id Set to the node ID the call is sent to, before it + * is sent; left empty when no leader is known and the call fails at once. + * A NotLeader redirect is compared with it by node ID, looked up with + * LogNodeByAddress, rather than by address format. + * @param done Run exactly once when the call completes, possibly before + * this returns. + */ + virtual void CdcScan(uint32_t log_group_id, + brpc::Controller *cntl, + const ::txlog::CdcScanRequest &request, + ::txlog::CdcScanResponse &response, + std::optional *node_id, + google::protobuf::Closure &done) + { + (void) log_group_id; + (void) request; + (void) response; + (void) node_id; + cntl->SetFailed("CdcScan is not supported by this log"); + done.Run(); + } + + /** + * @brief The node ID configured with the given "host:port" in the log + * group, as a CdcScan NotLeader response's redirect names it. + * + * Following a redirect is then UpdateLeaderCache with that node ID. The + * default finds none, for a log without a node configuration. + * + * @param log_group_id The log group. + * @param address A "host:port". + * @return The node ID, or empty if no node ID of the group is configured + * with that address. + */ + virtual std::optional LogNodeByAddress(uint32_t log_group_id, + const std::string &address) + { + (void) log_group_id; + (void) address; + return std::nullopt; + } + // Invoked by a cc node group's leader to update its checkpoint // timestamp in all log groups (so that the transaction logs before // checkpoint timestamp can be ignored and truncated). @@ -81,17 +135,27 @@ class TxLog // groups on cluster scaling down. virtual void RemoveCcNodeGroup(uint32_t cc_node_group_id, int64_t term) = 0; - // Invoked by a failing over cc node group to replay log and notify all log - // groups the raft term of the group's new leader. - // If log_group >= 0, send ReplayLogRequest to specified log_group, else - // send to all log groups. - virtual void ReplayLog(uint32_t cc_node_group_id, + /** + * @brief Asks one log group, once, to register a failing-over node group's + * new term and replay its log to the node (LogAgent::ReplayLogOnce). + * + * A bounded attempt: the caller learns whether the log group got the + * request from the replay stream it opens, and asks again if none comes. + * + * @param cc_node_group_id The recovering node group. + * @param term The node group's new term. + * @param source_ip The address of the node's log replay service. + * @param source_port The port of the node's log replay service. + * @param log_group_id The log group. + * @param start_ts The commit timestamp to replay from at the latest. + * @return true if the log group's leader accepted the request. + */ + virtual bool ReplayLog(uint32_t cc_node_group_id, int64_t term, const std::string &source_ip, uint16_t source_port, - int log_group, - uint64_t start_ts, - std::atomic &interrupt) = 0; + uint32_t log_group_id, + uint64_t start_ts) = 0; virtual RecoverTxStatus RecoverTx(uint64_t tx_number, int64_t tx_term, diff --git a/tx_service/include/type.h b/tx_service/include/type.h index 3dc9b6ab7..da59c6d99 100644 --- a/tx_service/include/type.h +++ b/tx_service/include/type.h @@ -99,6 +99,18 @@ enum class ClusterScaleOpType AddNodeGroupPeers }; +/** + * @brief What a ClusterGateTxRequest does to the cluster gate (EloqDoc + * docs/cdc-change-stream.md §6.2). + */ +enum class ClusterGateOpType : uint8_t +{ + /// Close the gate; fails with CLUSTER_GATE_HELD if it is already paused. + kPause = 0, + /// Open the gate; opening an open gate changes nothing. + kResume = 1, +}; + enum class TxnStatus { Ongoing = 0, @@ -587,6 +599,15 @@ enum class PostWriteType // Update dirty value. Takes no effect on lock. An optional medium stage // between PrepareCommit and PostCommit. UpdateDirty, + /** + * Releases the write lock/intent of a tx that failed. If the tx installed + * a dirty value, the request's payload is the value to keep, which the cc + * map supplies; otherwise the request has none and the entry keeps its + * value. A commit ts above 0 moves the entry's commit ts to it (a schema op + * rolled back at its commit-stage timestamp); 0 leaves it alone. Every + * post-write-all at commit ts 0 is a Rollback or a DowngradeLock. + */ + Rollback, }; enum class PostReadType diff --git a/tx_service/raft_host_manager/include/raft_host_manager.h b/tx_service/raft_host_manager/include/raft_host_manager.h index 3f4fab244..d22792fee 100644 --- a/tx_service/raft_host_manager/include/raft_host_manager.h +++ b/tx_service/raft_host_manager/include/raft_host_manager.h @@ -354,7 +354,6 @@ class RaftHostManager void InitLogAgent(std::vector &ip_list, std::vector &port_list, uint32_t log_group_replica_num); - void NotifyLogReplay(uint32_t ng_id, int64_t term, uint64_t start_ts); void NotifyNewLeaderStart(uint32_t ng_id, int64_t term); int GetLeader(uint32_t ng_id); void UpdateClusterConfig( diff --git a/tx_service/raft_host_manager/src/raft_host_manager.cpp b/tx_service/raft_host_manager/src/raft_host_manager.cpp index e3d24a456..7689fa4d2 100644 --- a/tx_service/raft_host_manager/src/raft_host_manager.cpp +++ b/tx_service/raft_host_manager/src/raft_host_manager.cpp @@ -965,26 +965,6 @@ void RaftHostManager::InitLogAgent(std::vector &ip_list, } } -void RaftHostManager::NotifyLogReplay(uint32_t ng_id, - int64_t term, - uint64_t start_ts) -{ - if (log_agent_) - { - LOG(INFO) << "Notified log groups to start log replay for ng#" << ng_id - << "with term " << term << ", start ts " << start_ts << "..."; - log_agent_->ReplayLog(ng_id, - term, - node_config_.host_name_, - node_config_.port_ + 2, - -1, - start_ts, - log_agent_interrupt_); - LOG(INFO) << "Notified log groups to start log replay for ng#" << ng_id - << " with term " << term; - } -} - void RaftHostManager::NotifyNewLeaderStart(uint32_t leader_ng_id, int64_t term) { uint32_t node_id; @@ -1675,7 +1655,8 @@ void RaftNode::on_leader_start(int64_t term) << " escalated to leader of node group " << ng_id_; host_manager_.NotifyNewLeaderStart(ng_id_, term); - host_manager_.NotifyLogReplay(ng_id_, term, resp.log_replay_start_ts()); + // The tx node asks the log groups to replay the log itself, at the + // end of its OnLeaderStart, and asks again if they do not answer. } } diff --git a/tx_service/src/cc/cc_map.cpp b/tx_service/src/cc/cc_map.cpp index ede1962c1..21cb0961e 100644 --- a/tx_service/src/cc/cc_map.cpp +++ b/tx_service/src/cc/cc_map.cpp @@ -362,6 +362,7 @@ void CcMap::DowngradeCceKeyWriteLock(LruEntry *cce, TxNumber tx_number) if (lock != nullptr) { lock->DowngradeWriteLock(tx_number, shard_); + shard_->OnWriteLockDowngraded(tx_number, cc_ng_id_, table_name_.Type()); } } diff --git a/tx_service/src/cc/cc_req_misc.cpp b/tx_service/src/cc/cc_req_misc.cpp index 19a0c298e..3ab95960f 100644 --- a/tx_service/src/cc/cc_req_misc.cpp +++ b/tx_service/src/cc/cc_req_misc.cpp @@ -22,6 +22,7 @@ #include "cc/cc_req_misc.h" #include +#include #include #include @@ -46,6 +47,14 @@ #include "tx_service.h" #include "type.h" +DEFINE_uint64(shard_clean_release_kb, + 256, + "Kilobytes the shard cleaner must reclaim, after a request found " + "the shard heap at its limit, before it releases parked requests " + "again. Released requests then follow the pace of eviction; a " + "larger value releases fewer, larger batches. Zero means any " + "reclaimed amount."); + namespace txservice { FetchCc::FetchCc(CcShard &ccs, NodeGroupId cc_ng_id, int64_t cc_ng_term) @@ -1666,105 +1675,179 @@ void RestoreCcMapCc::DecodedDataItem( std::move(key), std::move(record), version_ts, is_deleted); } +/** + * One execution of the shard cleaner's pass. + * + * Terms: + * hard limit CcShardHeap::Full(): allocated at the limit (or, with + * defragmentation, committed over it). Requests park here. + * target CcShardHeap::NeedMoreCleaning(): less than 5% of the limit + * free. A pass that has started evicts down to it. + * tail the pass has visited every page of the LRU chain. + * + * Each execution does at most one eviction batch (CcShard::Clean, 10 pages or + * 30 us), then at most one release of 20 parked requests, then decides: + * evict while pages remain and the heap is above the target, or a + * request found the heap at its hard limit since this pass started + * or last released a batch, or the defrag path's rule asks for it. + * release below the hard limit, with requests parked, unless a request + * found the heap at its hard limit since this pass started or last + * released a batch (the park count reveals it, whoever parked): + * the room handed out is used up, so the next batch goes out once + * the pass has reclaimed FLAGS_shard_clean_release_kb more. The + * waiters thus follow the pace of eviction instead of waiting for + * a share of the heap. At the tail no further batch can add to + * that, so whatever the pass reclaimed since its last release goes + * out before the pass ends; an execution that starts at the tail + * reclaims nothing, releases nothing and lets the pass end. + * idle nothing parked and nothing to evict. The retry backoff clears: + * the episode of pressure it paced is over. + * continue pages remain and there is something to evict. + * end otherwise: at the tail with requests parked, or at the hard + * limit with nothing this pass may evict. Abort the requests that + * opted into OOM failure if the heap is full or the pass freed + * nothing; if the pass reclaimed less than + * FLAGS_shard_clean_release_kb, eviction is not keeping up, so + * ask for a checkpoint and lengthen the retry delay; then run once + * more if a flush landed mid-pass, stop (clearing the backoff) if + * nothing is parked, or defer (CcShard::DeferCleanPass) otherwise. + * The continuation is enqueued with the shard's own producer token, as the + * released batch is, so it runs after every released request has either + * finished or parked again. + * @param ccs The shard whose pass this execution advances. + * @return true once the cleaner is idle and a later park may schedule it + * again; false while it stays in use, re-enqueued or deferred. + */ bool ShardCleanCc::Execute(CcShard &ccs) { CcShardHeap *shard_heap = ccs.GetShardHeap(); - int64_t heap_alloc, heap_commit; - if (shard_heap != nullptr && shard_heap->Full(&heap_alloc, &heap_commit)) + if (shard_heap == nullptr) { - assert(txservice_enable_cache_replacement); - bool need_yield = false; - if (shard_heap->NeedCleanShard(heap_alloc, heap_commit)) - { - size_t free_size = 0; - std::tie(free_size, need_yield) = ccs.Clean(); - free_count_ += free_size; - } - - if (shard_heap->Full(&heap_alloc, &heap_commit) && - shard_heap->NeedCleanShard(heap_alloc, heap_commit)) - { - if (need_yield) - { - // Continue to clean in the next run one round. - ccs.Enqueue(this); - return false; - } - else - { - // Reached the tail ccpage with the allocated memory still - // above the heap threshold. Abort only the parked requests - // that opted in via AbortIfOom() -- txs holding range read - // locks, which would deadlock reclaim by blocking the very - // data sync that frees memory. Ordinary requests stay parked - // until eviction or a flush makes room. - ccs.AbortRequestsAfterMemoryFree(); - - // Notify the checkpointer thread to do checkpoint if there - // is not freeable entries to be kicked out from ccmap and - // if the shard is not doing defrag. - if (free_count_ == 0 && !shard_heap->IsDefragHeapCcOnFly() && - !Sharder::Instance().GetCheckpointer()->IsOngoingDataSync()) - { - ccs.NotifyCkpt(true, "memory_pressure"); - } - - free_count_ = 0; + // No heap accounting, so nothing is ever full: let everything parked + // run and finish. + ccs.DequeueWaitListAfterMemoryFree(/*deque_all=*/true); + return true; + } - // This pass decided there was nothing to free from state it - // gathered before any wake-up that arrived while it ran -- a - // checkpoint flush turning dirty entries evictable, say. Such - // a wake-up could not be delivered because this request was in - // use, so re-run rather than stopping with requests parked and - // nothing left to schedule another pass. - if (ccs.TakeShardCleanCcWakeUp() && - ccs.WaitListSizeForMemory() > 0) - { - ccs.Enqueue(this); - return false; - } + int64_t heap_alloc = 0, heap_commit = 0; + bool heap_full = shard_heap->Full(&heap_alloc, &heap_commit); + if (!pass_started_) + { + // A pass starts from the LRU head and visits every page, so a flush + // that completed before this point needs no separate mark. + pass_started_ = true; + freed_in_pass_ = 0; + reclaimed_in_pass_ = 0; + park_snapshot_ = ccs.MemoryParkCount(); + reclaimed_since_release_ = 0; + ccs.ResetCleanStart(); + ccs.TakeCleanRescanRequest(); + } + + // A park means a request found the heap at its hard limit at that moment, + // whoever parked: the room this pass had when it started, or handed out + // at its last release, is used up. + const bool parked_since_release = ccs.MemoryParkCount() != park_snapshot_; + // With cache replacement disabled there is nothing to sweep: the pass is + // over before it starts, and the cleaner only releases or defers. + bool at_tail = !txservice_enable_cache_replacement || ccs.OutOfMemory(); + bool below_target = shard_heap->NeedMoreCleaning(heap_alloc); + // Full() with allocated memory under the target happens only with + // defragmentation, when committed memory is over the limit; + // NeedCleanShard() keeps that path's rule of evicting only above its + // high-water mark. + auto want_evict = [&]() + { + return below_target || parked_since_release || + (heap_full && + shard_heap->NeedCleanShard(heap_alloc, heap_commit)); + }; + + if (!at_tail && want_evict()) + { + const int64_t alloc_before = heap_alloc; + auto [freed, more] = ccs.Clean(); + freed_in_pass_ += freed; + at_tail = !more; + heap_full = shard_heap->Full(&heap_alloc, &heap_commit); + below_target = shard_heap->NeedMoreCleaning(heap_alloc); + const int64_t reclaimed = + std::max(0, alloc_before - heap_alloc); + reclaimed_since_release_ += reclaimed; + reclaimed_in_pass_ += reclaimed; + } + + // After a limit hit, the next release waits until the pass has reclaimed + // the configured amount since the last one, so the waiters follow the + // pace of eviction, however small each step. At the tail no further batch + // can add to that, so whatever was reclaimed goes out before the pass + // ends; an execution that starts at the tail reclaims nothing, so a + // request that keeps parking there cannot keep the pass alive: the pass + // ends and the retry is paced by the backoff. + // Zero means any reclaimed amount: at least one byte, so that nothing + // reclaimed never passes for enough and a request that keeps parking at + // the tail cannot be released again and again. + const int64_t release_bytes = std::max( + 1, static_cast(FLAGS_shard_clean_release_kb) * 1024); + if (!heap_full && ccs.WaitListSizeForMemory() > 0 && + (!parked_since_release || reclaimed_since_release_ >= release_bytes || + (at_tail && reclaimed_since_release_ > 0))) + { + ccs.DequeueWaitListAfterMemoryFree(); + park_snapshot_ = ccs.MemoryParkCount(); + reclaimed_since_release_ = 0; + ccs.Enqueue(ccs.core_id_, this); + return false; + } - // Return true will set the request as free, which means the - // request is not in working state. - return true; - } - } - else - { - // Get the free memory, re-run a batch of the waiting ccrequest. - bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); - if (!wait_list_empty) - { - ccs.Enqueue(this); - } + const bool more_to_evict = want_evict(); + if (!more_to_evict && ccs.WaitListSizeForMemory() == 0) + { + pass_started_ = false; + ccs.ResetCleanStart(); + ccs.ResetCleanBackoff(); + return true; + } + if (more_to_evict && !at_tail) + { + ccs.Enqueue(ccs.core_id_, this); + return false; + } - // Reset the value if the ccrequest is finished. - free_count_ = (wait_list_empty) ? 0 : free_count_; - if (wait_list_empty) - { - // Nothing is parked any more, so a wake-up that arrived during - // this pass has already been served. - ccs.TakeShardCleanCcWakeUp(); - } - return wait_list_empty; - } + // End of pass. + if (heap_full || freed_in_pass_ == 0) + { + // Only requests that opted in: their range read locks can block the + // very data sync that would make entries evictable. + ccs.AbortRequestsAfterMemoryFree(); } - else + // A pass that reclaimed less than one release-worth is not keeping up: + // what is left on the chain is dirty, locked or pinned, and a flush is + // what changes that. Ask for a checkpoint now and slow the retries. + const bool productive = reclaimed_in_pass_ >= release_bytes; + if (!productive && !shard_heap->IsDefragHeapCcOnFly()) { - // There is available memory on this shard, re-run a batch of the - // waiting ccrequest if has any waiting request, otherwise, finish - // this shard clean ccrequests. - bool wait_list_empty = ccs.DequeueWaitListAfterMemoryFree(); - if (!wait_list_empty) + Checkpointer *ckpter = Sharder::Instance().GetCheckpointer(); + if (ckpter != nullptr && !ckpter->IsOngoingDataSync()) { - ccs.Enqueue(this); - } - else - { - ccs.TakeShardCleanCcWakeUp(); + ccs.NotifyCkpt(true, "memory_pressure"); } - return wait_list_empty; } + pass_started_ = false; + ccs.ResetCleanStart(); + + if (ccs.TakeCleanRescanRequest()) + { + ccs.Enqueue(ccs.core_id_, this); + return false; + } + if (ccs.WaitListSizeForMemory() == 0) + { + ccs.ResetCleanBackoff(); + return true; + } + ccs.DeferCleanPass(productive); + return false; } void FetchTableRangeSizeCc::Reset(const TableName &table_name, diff --git a/tx_service/src/cc/cc_shard.cpp b/tx_service/src/cc/cc_shard.cpp index 9a834889f..5ea09421d 100644 --- a/tx_service/src/cc/cc_shard.cpp +++ b/tx_service/src/cc/cc_shard.cpp @@ -568,6 +568,11 @@ void CcShard::Enqueue(uint32_t thd_id, uint32_t shard_code, CcRequestBase *req) void CcShard::EnqueueWaitListIfMemoryFull(CcRequestBase *req) { cc_wait_list_for_memory_.PushBack(req); + // Counted even when cache replacement is off: the cleaner then evicts + // nothing, but it still releases parked requests once the heap is below + // the limit and defers while it is not. + ++memory_park_count_; + WakeUpShardCleanCc(); } bool CcShard::DequeueWaitListAfterMemoryFree(bool deque_all) @@ -620,14 +625,56 @@ void CcShard::WakeUpShardCleanCc() if (!shard_clean_cc_->InUse()) { shard_clean_cc_->Use(); - Enqueue(shard_clean_cc_.get()); + Enqueue(core_id_, shard_clean_cc_.get()); } - else +} + +void CcShard::OnDirtyDataFlushed() +{ + if (clean_retry_deadline_.exchange(0, std::memory_order_acq_rel) != 0) + { + // Deferred, and the retry is now ours to run. Newly evictable entries + // are a reason to start from the base delay again. + clean_backoff_ = 0; + Enqueue(core_id_, shard_clean_cc_.get()); + } + else if (shard_clean_cc_->InUse()) { - // The pass in flight may already have scanned past whatever this - // wake-up is about. Remember it so the pass re-runs instead of - // stopping with requests still parked. - shard_clean_cc_wake_pending_ = true; + // Queued or mid-pass -- or deferred with the timer having just claimed + // the retry. A pass is coming either way; mark it to run once more so + // entries behind its cursor are reached. + clean_rescan_requested_ = true; + } + // Idle: nothing is parked, so there is nothing to resume. +} + +void CcShard::DeferCleanPass(bool productive) +{ + clean_backoff_ = productive + ? 0 + : std::min(static_cast(clean_backoff_ + 1), + kCleanRetryMaxBackoff); + const uint64_t now_us = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + clean_retry_deadline_.store( + now_us + (kCleanRetryBaseDelayUs << clean_backoff_), + std::memory_order_release); +} + +void CcShard::FireCleanRetryIfDue(uint64_t now_us) +{ + uint64_t deadline = clean_retry_deadline_.load(std::memory_order_acquire); + if (deadline == 0 || now_us < deadline) + { + return; + } + if (clean_retry_deadline_.compare_exchange_strong( + deadline, 0, std::memory_order_acq_rel)) + { + // From the timer thread: no producer token of this shard's. + Enqueue(shard_clean_cc_.get()); } } @@ -868,10 +915,28 @@ void CcShard::AbortCcRequests(std::vector &&reqs, TEntry &CcShard::NewTx(NodeGroupId tx_ng_id, uint32_t log_group_id, - int64_t term) + int64_t term, + bool read_snapshot) { // allocate start timestamp. uint64_t start_ts = Now(); + if (read_snapshot) + { + // A snapshot read must see what other nodes acknowledged before it + // started. The timestamp base advances from the wall clock only every + // 100 ms (LocalCcShards::TimerRun), while another node's commits take + // timestamps above its own base; so read the wall clock, and advance + // the base with it. + const uint64_t wall_ts = + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + if (wall_ts > start_ts) + { + UpdateTsBase(wall_ts); + start_ts = wall_ts; + } + } // Cicurlar iteration to find an available transaction entry. size_t cnt = 0; @@ -1217,20 +1282,63 @@ TxLockInfo *CcShard::UpsertLockHoldingTx(TxNumber txn, { tx_it->second->cce_list_.emplace(cce_ptr); } - tx_it->second->last_recover_ts_ = Now(); - tx_it->second->table_type_ = table_type; + const uint64_t now = Now(); + tx_it->second->last_recover_ts_ = now; - if (is_key_write_lock && tx_it->second->wlock_ts_ == 0) + if (is_key_write_lock) { - // write lock should update ts if the txn exists, or the CkptTsCc - // request may get an older ckpt_ts. - // For txn that acquires write lock more than once, do not update - // wlock_ts_ with bigger Now(). - tx_it->second->wlock_ts_ = Now(); + // Every write lock moves the stamp: while a counting write lock is + // held, the checkpoint stays below the latest grant + // (TxLockInfo::wlock_ts_). + tx_it->second->wlock_ts_ = now; + // Decided per write lock, from the locked entry's table: the tx's + // locks span many tables. + if (!tx_it->second->holds_ckpt_wlock_ && + TxLockInfo::WriteLockHoldsBackCheckpoint(table_type)) + { + tx_it->second->holds_ckpt_wlock_ = true; + } } return tx_it->second.get(); } +void CcShard::OnWriteLockDowngraded(TxNumber txn, + NodeGroupId cc_ng_id, + TableType table_type) +{ + if (!TxLockInfo::WriteLockHoldsBackCheckpoint(table_type)) + { + // This lock never set holds_ckpt_wlock_. + return; + } + + auto ng_it = lock_holding_txs_.find(cc_ng_id); + if (ng_it == lock_holding_txs_.end()) + { + return; + } + auto tx_it = ng_it->second.find(txn); + if (tx_it == ng_it->second.end()) + { + return; + } + TxLockInfo &lk_info = *tx_it->second; + + // holds_ckpt_wlock_ covers all of the tx's write locks in the shard. Keep + // it while another counting write lock remains. + for (LruEntry *cce : lk_info.cce_list_) + { + const NonBlockingLock *lock = cce->GetKeyLock(); + const CcMap *ccm = cce->GetCcMap(); + if (lock != nullptr && ccm != nullptr && lock->HasWriteLock(txn) && + TxLockInfo::WriteLockHoldsBackCheckpoint(ccm->table_name_.Type())) + { + return; + } + } + lk_info.holds_ckpt_wlock_ = false; +} + void CcShard::DeleteLockHoldingTx(TxNumber txn, LruEntry *cce_ptr, NodeGroupId cc_ng_id) @@ -1431,14 +1539,17 @@ void CcShard::ClearTx(TxNumber txn) NonBlockingLock *lock = lru_ptr->GetKeyLock(); if (lock != nullptr) { + // The entry's own table, read before the lock is cleared: the + // tx's locks span many tables. + const CcMap *ccm = lru_ptr->GetCcMap(); LockType lt = lock->ClearTx(txn, this); if (lt == LockType::WriteLock) { lru_ptr->GetKeyGapLockAndExtraData()->ClearTx(); } - if (FLAGS_cmd_read_catalog && - lk_info->table_type_ == TableType::Catalog) + if (FLAGS_cmd_read_catalog && ccm != nullptr && + ccm->table_name_.Type() == TableType::Catalog) { // Do not recycle the lock on Catalog since it's frequently // accessed. diff --git a/tx_service/src/cc/local_cc_handler.cpp b/tx_service/src/cc/local_cc_handler.cpp index 674f719b1..8d4711736 100644 --- a/tx_service/src/cc/local_cc_handler.cpp +++ b/tx_service/src/cc/local_cc_handler.cpp @@ -720,9 +720,15 @@ txservice::LocalCcHandler::ReadCatalog(const TableName &table_name, } bool txservice::LocalCcHandler::ReleaseCatalogRead(NonBlockingLock *lock, - TxNumber tx_number) const + TxNumber tx_number, + uint64_t commit_ts) const { CcShard *shard_ = cc_shards_.cc_shards_[thd_id_].get(); + if (commit_ts != 0) + { + // Before the release, which may resume a writer blocked on the lock. + shard_->UpdateLastReadTs(commit_ts); + } return lock->ReleaseReadLockFast(shard_, tx_number); } @@ -1394,7 +1400,10 @@ void txservice::LocalCcHandler::NewTxn(CcHandlerResult &hres, // NewTx reads each ccshard's next_tx_ident_, which is set concurrently // by log replay thread when native cc node finishes log replay. The two // events are synchronized by leader_term_. - TEntry &tx = ccs.NewTx(tx_ng_id, log_group_id, term); + TEntry &tx = ccs.NewTx(tx_ng_id, + log_group_id, + term, + iso_level == IsolationLevel::Snapshot); InitTxResult &init_tx_res = hres.Value(); init_tx_res.txid_ = tx.GetTxId(ccs.GlobalCoreId(tx_ng_id)); TxNumber txn = init_tx_res.txid_.TxNumber(); diff --git a/tx_service/src/cc/local_cc_shards.cpp b/tx_service/src/cc/local_cc_shards.cpp index 317b972fc..341122087 100644 --- a/tx_service/src/cc/local_cc_shards.cpp +++ b/tx_service/src/cc/local_cc_shards.cpp @@ -211,8 +211,6 @@ LocalCcShards::LocalCcShards( .count(); ts_base_.store(ts_base); local_clock.store(ts_base); - timer_thd_ = std::thread([this] { TimerRun(); }); - pthread_setname_np(timer_thd_.native_handle(), "tx_timer"); // Start background thread to initialize table_ranges_heap. offload heap // init to avoid concurrent access to heap with main thread. @@ -283,6 +281,11 @@ LocalCcShards::LocalCcShards( fast_meta_data_mux_.mux_ptrs_[i] = &cc_shards_[i]->meta_data_mux_; } + // The timer thread walks cc_shards_ on every tick, so it starts only once + // the vector is complete. The clocks it refreshes were seeded above. + timer_thd_ = std::thread([this] { TimerRun(); }); + pthread_setname_np(timer_thd_.native_handle(), "tx_timer"); + InitRangeBuckets(ng_id_, ng_ids, cluster_config_version); if (prebuilt_tables) @@ -320,6 +323,8 @@ LocalCcShards::LocalCcShards( leader_changes_metric_labels.emplace_back( "ng_id", std::move(ng_id_label_values)); auto is_leader_metric_labels = leader_changes_metric_labels; + // The cluster config map of every node group this node may lead. + auto gate_paused_metric_labels = leader_changes_metric_labels; node_meter_->Register(metrics::NAME_LEADER_CHANGES, metrics::Type::Counter, std::move(leader_changes_metric_labels)); @@ -344,6 +349,9 @@ LocalCcShards::LocalCcShards( "persist", "metadata", "unknown"}}}); + node_meter_->Register(metrics::NAME_CLUSTER_GATE_PAUSED, + metrics::Type::Gauge, + std::move(gate_paused_metric_labels)); } } @@ -546,6 +554,23 @@ void LocalCcShards::TimerRun() } UpdateTsBase(clock_ts); + // Deferred eviction retries: a shard whose last pass ended with + // requests parked and nothing to evict re-checks on this tick. The + // walk runs under lk (timer_terminate_mux_); a worker that exits at + // shutdown frees its shard under the same mutex (FreeCcShard), so a + // slot is either a live shard or null here, never one being destroyed. + const uint64_t steady_now_us = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + for (auto &shard : cc_shards_) + { + if (shard != nullptr) + { + shard->FireCleanRetryIfDue(steady_now_us); + } + } + timer_terminate_cv_.wait_for( lk, 100ms, [this]() { return timer_terminate_ == true; }); } while (!timer_terminate_); diff --git a/tx_service/src/checkpointer.cpp b/tx_service/src/checkpointer.cpp index 6179f9da4..d273a961d 100644 --- a/tx_service/src/checkpointer.cpp +++ b/tx_service/src/checkpointer.cpp @@ -367,7 +367,7 @@ void Checkpointer::WarnIfCkptStalled(uint32_t node_group, << " <= last_ckpt_ts=" << last_ckpt_ts << ". Pinned by txn " << pinning_tx.txn_ << " on core #" << pinning_tx.core_id_ - << ", which acquired its first write lock " << age_seconds + << ", which last acquired a write lock " << age_seconds << "s ago (wlock_ts=" << pinning_tx.wlock_ts_ << "). If this transaction never finishes, dirty data " "cannot be flushed and memory cannot be reclaimed."; diff --git a/tx_service/src/fault/cc_node.cpp b/tx_service/src/fault/cc_node.cpp index 845d300d8..7ec610ccc 100644 --- a/tx_service/src/fault/cc_node.cpp +++ b/tx_service/src/fault/cc_node.cpp @@ -200,11 +200,17 @@ bool CcNode::UpdateCkptTs(uint64_t new_ckpt_ts) * @brief Notify all the nodes that the node_id of the new leader in node * group leader_ng_id, and request these nodes to update their leader cache. * - * @param leader_ng_id - * @param leader_node_id + * @note No caller: the host manager sends the notification + * (CcRpcService::NotifyNewLeaderStart). A leader update always carries its + * term, which orders it against the others. + * + * @param leader_ng_id The node group. + * @param leader_node_id Its new leader. + * @param term The new leader's term. */ void CcNode::NotifyNewLeaderStart(uint32_t leader_ng_id, - uint32_t leader_node_id) + uint32_t leader_node_id, + int64_t term) { auto all_nodes = Sharder::Instance().GetAllNodesConfigs(); @@ -212,7 +218,8 @@ void CcNode::NotifyNewLeaderStart(uint32_t leader_ng_id, { if (node_id == leader_node_id) { - Sharder::Instance().UpdateLeader(leader_ng_id, leader_node_id); + Sharder::Instance().UpdateLeader( + leader_ng_id, leader_node_id, term); continue; } @@ -232,6 +239,7 @@ void CcNode::NotifyNewLeaderStart(uint32_t leader_ng_id, remote::NotifyNewLeaderStartRequest req; req.set_ng_id(leader_ng_id); req.set_node_id(leader_node_id); + req.set_term(term); remote::NotifyNewLeaderStartResponse res; res.set_error(false); @@ -479,6 +487,10 @@ bool CcNode::OnLeaderStart(int64_t term, local_cc_shards_.store_hd_->RestoreTxCache(ng_id_, term); } + // Ready to replay: ask the log groups to ship this term's log. Only this + // node asks; the log replay service asks again for a log group that does + // not answer. + Sharder::Instance().RequestLogReplay(ng_id_, term, replay_start_ts); return true; } @@ -536,6 +548,16 @@ bool CcNode::OnLeaderStop(int64_t term) pinning_threads_cv_.wait(lk, [this] { return pinning_threads_ == 0; }); } ClearCcNodeGroupData(); + if (metrics::enable_metrics) + { + // Stepping down drops the node group's cluster config map, and the + // gate it held with it; the next leader's replay reinstalls it. Only + // now: a gate install or replay queued before the clear ran before + // it on core 0 and may have reported 1, and none can report later, + // since a map made afresh holds no gate tx's lock. + Sharder::Instance().GetLocalCcShards()->GetNodeMeter()->Collect( + metrics::NAME_CLUSTER_GATE_PAUSED, 0, std::to_string(ng_id_)); + } return true; } diff --git a/tx_service/src/fault/fault_inject.cpp b/tx_service/src/fault/fault_inject.cpp index fc70e703b..cf37ea526 100644 --- a/tx_service/src/fault/fault_inject.cpp +++ b/tx_service/src/fault/fault_inject.cpp @@ -21,6 +21,8 @@ */ #include "fault/fault_inject.h" +#include + #if defined(WITH_LOG_SERVICE) && defined(LOG_STATE_TYPE_RKDB_S3) #include "../../../eloq_log_service/include/fault_inject.h" // txlog::FaultInject from eloq_log_service #endif @@ -250,6 +252,44 @@ void FaultInject::TriggerAction(FaultEntry *entry) } } +bool FaultInject::CompiledIn() +{ +#ifdef WITH_FAULT_INJECT + return true; +#else + return false; +#endif +} + +FaultInject::FaultInject() +{ +#ifdef WITH_FAULT_INJECT + const char *faults = std::getenv("ELOQ_TX_FAULT_INJECT"); + std::string spec = faults == nullptr ? "" : faults; + size_t begin = 0; + while (begin < spec.size()) + { + size_t end = spec.find(',', begin); + if (end == std::string::npos) + { + end = spec.size(); + } + std::string item = spec.substr(begin, end - begin); + size_t colon = item.find(':'); + std::string name = item.substr(0, colon); + std::string paras = + colon == std::string::npos ? "" : item.substr(colon + 1); + if (!name.empty()) + { + LOG(INFO) << "FaultInject from ELOQ_TX_FAULT_INJECT, name=" << name + << " paras=" << paras; + injected_fault_map_.try_emplace(name, FaultEntry(name, paras)); + } + begin = end + 1; + } +#endif +} + void FaultInject::InjectFault(std::string fault_name, std::string paras) { LOG(INFO) << "FaultInject name=" << fault_name << " paras=" << paras; diff --git a/tx_service/src/fault/log_replay_service.cpp b/tx_service/src/fault/log_replay_service.cpp index 813355121..d5a815bb7 100644 --- a/tx_service/src/fault/log_replay_service.cpp +++ b/tx_service/src/fault/log_replay_service.cpp @@ -25,10 +25,12 @@ #include #include +#include #include #include "cc/cc_request.h" #include "cc/local_cc_shards.h" +#include "fault/fault_inject.h" #include "log.pb.h" #include "log_type.h" #include "proto/cc_request.pb.h" @@ -42,15 +44,21 @@ * and request leader transfer if not. * * Here is a brief replay protocol description: - * 1. ccnode becomes a raft leader and begins to send ReplayLogRequest to all - * the log groups. If log group's leader is not elected, the ccnode will call - * braft API to wait for the log leader to be elected. + * 1. ccnode becomes a raft leader and, at the end of its leader start + * (CcNode::OnLeaderStart), sends ReplayLogRequest to all the log groups; it is + * the only node that asks. Each send is one bounded attempt (a leader lookup + * and an RPC with a deadline): a lost request, a log group with no leader and + * a lost reply look the same to the sender, so the proof that a log group got + * the request is the replay stream it opens. A log group that has neither + * finished the replay nor a replay stream kReplayCheckIntervalUs after the + * last send to it is asked again, at that interval, until it does. * 2. Log group will connect to log_replay_service and send replay logs to * ccnode. * 3. ccnode receives and replays the logs from the stream. * 4. If ccnode fails to receive new logs after a timeout(2000ms), it will close * the stream and send a new ReplayLogRequest to log group leader. And replay - * from the beginning. + * from the beginning. A replay that fails on the ccnode closes its streams; + * the log groups still owing the replay are then asked again as in 1. * 5. ccnode finishes the log replay and become the actual leader from candidate * leader. Then it will close the log replay stream. * 6. Later when an orphan lock is detected, will send RecoveryTx request to log @@ -87,7 +95,7 @@ RecoveryService::RecoveryService(LocalCcShards &local_shards, std::unique_lock lk(queue_mux_); bool timed_out = !queue_cv_.wait_for( lk, - std::chrono::seconds(5), + std::chrono::milliseconds(kReplayWatchPeriodMs), [this] { return !replay_log_queue_.empty() || @@ -98,31 +106,46 @@ RecoveryService::RecoveryService(LocalCcShards &local_shards, // Only check for orphaned ng recovery on timeout, not on notify if (timed_out) { - // Periodically check candidate leaders that still do not - // have replay stream connections. + // A candidate leader's first replay request comes from its + // leader start (Sharder::RequestLogReplay). Ask a log + // group again only while it owes the replay -- it has + // neither finished this term's replay nor a recovering + // stream to this node -- kReplayCheckIntervalUs after the + // last send to it. The stream is the only proof that a log + // group got a request: a lost request, a log group with no + // leader and a lost reply look the same here. A stream + // that opened and then stalls is left to its idle timeout + // (on_idle_timeout). + // + // Only the node groups this node is recovering have a + // watch: a candidate term's first request creates it + // (ProcessReplayLogTask), and it is dropped here once the + // node is no longer a candidate leader of the node group. lk.unlock(); uint64_t now_ts = LocalCcShards::ClockTs(); - auto ngs = Sharder::Instance().AllNodeGroups(); - for (auto ng_id : *ngs) + const uint32_t log_group_cnt = + log_agent_ != nullptr ? log_agent_->LogGroupCount() : 0; + for (auto watch_it = candidate_replay_watch_.begin(); + watch_it != candidate_replay_watch_.end();) { - int64_t candidate_term = + const uint32_t ng_id = watch_it->first; + CandidateReplayWatch &watch = watch_it->second; + const int64_t candidate_term = Sharder::Instance().CandidateLeaderTerm(ng_id); if (candidate_term < 0) { - candidate_replay_watch_.erase(ng_id); + watch_it = candidate_replay_watch_.erase(watch_it); continue; } - - CandidateReplayWatch &watch = - candidate_replay_watch_[ng_id]; if (watch.term != candidate_term) { - watch.term = candidate_term; - watch.first_seen_ts = now_ts; - watch.last_replay_ts = 0; + // A newer candidate term whose leader start's + // request is not sent yet. + ++watch_it; + continue; } - bool has_replay_connection = false; + std::unordered_set streaming; { std::unique_lock inbound_lk( inbound_mux_); @@ -133,40 +156,39 @@ RecoveryService::RecoveryService(LocalCcShards &local_shards, candidate_term && entry.second->recovering_) { - has_replay_connection = true; - break; + streaming.insert( + entry.second->log_group_id_); } } } - if (has_replay_connection) - { - watch.first_seen_ts = 0; - continue; - } - - if (watch.first_seen_ts == 0) + for (uint32_t lg_id = 0; lg_id < log_group_cnt; ++lg_id) { - watch.first_seen_ts = now_ts; - } - - if (now_ts - watch.first_seen_ts < - kReplayCheckIntervalUs) - { - continue; - } - - if (watch.last_replay_ts != 0 && - now_ts - watch.last_replay_ts < - kReplayCheckIntervalUs) - { - continue; + if (streaming.count(lg_id) > 0) + { + watch.last_send_ts[lg_id] = now_ts; + continue; + } + if (Sharder::Instance().CheckLogGroupReplayFinished( + ng_id, lg_id, candidate_term) || + now_ts - watch.last_send_ts[lg_id] < + kReplayCheckIntervalUs) + { + continue; + } + LOG(INFO) + << "Requesting log replay for ng: " << ng_id + << " at term: " << candidate_term + << " from log group " << lg_id + << " again: no recovery connection " + << kReplayCheckIntervalUs / 1000 + << " ms after the last request"; + ReplayLog(ng_id, + candidate_term, + static_cast(lg_id), + watch.replay_start_ts); + watch.last_send_ts[lg_id] = now_ts; } - - LOG(INFO) << "Requesting log replay for ng: " << ng_id - << " at term: " << candidate_term - << " as there is no recovery connection"; - ReplayLog(ng_id, candidate_term, -1, 0, false); - watch.last_replay_ts = now_ts; + ++watch_it; } lk.lock(); } @@ -184,16 +206,6 @@ RecoveryService::RecoveryService(LocalCcShards &local_shards, ProcessReplayLogTask(task); continue; } - if (!delayed_replay_queue_.empty()) - { - int ready_cnt = ProcessDelayedReplayLogTask(); - if (ready_cnt > 0) - { - // ReplayLog task is of the highest priority, continue - // to next round and process the ready ReplayTask - continue; - } - } if (!recover_tx_queue_.empty()) { RecoverTxTask task = recover_tx_queue_.front(); @@ -210,30 +222,6 @@ RecoveryService::RecoveryService(LocalCcShards &local_shards, pthread_setname_np(notify_thread_.native_handle(), "replay_notify"); } -bool RecoveryService::ReplayNow(ReplayLogTask &task) -{ - // queued_clock_ == 0 means it's not a delayed request. - if (task.queued_clock_ == 0) - { - return true; - } - - // check whether the delayed request can be executed now. - uint64_t now_ts = LocalCcShards::ClockTs(); - using namespace std::chrono_literals; - uint64_t duration = std::chrono::duration_cast( - std::chrono::seconds(30)) - .count(); - if (now_ts - task.queued_clock_ > duration) - { - return true; - } - else - { - return false; - } -} - void RecoveryService::Shutdown() { // reap background thread @@ -397,21 +385,11 @@ void RecoveryService::NotifyCheckpointer( void RecoveryService::ReplayLog(uint32_t cc_ng_id, int64_t cc_ng_term, int log_group, - uint64_t replay_start_ts, - bool delayed_request) + uint64_t replay_start_ts) { std::unique_lock lk(queue_mux_); - if (delayed_request) - { - uint64_t queued_clock = LocalCcShards::ClockTs(); - delayed_replay_queue_.emplace_back( - cc_ng_id, cc_ng_term, log_group, replay_start_ts, queued_clock); - } - else - { - replay_log_queue_.emplace_back( - cc_ng_id, cc_ng_term, log_group, replay_start_ts, 0); - } + replay_log_queue_.emplace_back( + cc_ng_id, cc_ng_term, log_group, replay_start_ts); queue_cv_.notify_one(); } @@ -431,6 +409,24 @@ int RecoveryService::on_received_messages(brpc::StreamId stream_id, butil::IOBuf *const messages[], size_t size) { +#ifdef WITH_FAULT_INJECT + // Test only: drops one replay stream, as a broken connection would. Its + // log group then goes silent, and the watchdog asks it again. + if (stream_id == fault_dropped_stream_.load(std::memory_order_relaxed)) + { + return 0; + } + CODE_FAULT_INJECTOR("replay_drop_stream_once", { + FaultInject::Instance().InjectFault("replay_drop_stream_once", + "remove"); + fault_dropped_stream_.store(stream_id, std::memory_order_relaxed); + LOG(INFO) << "FaultInject replay_drop_stream_once: dropping replay " + "stream " + << stream_id; + brpc::StreamClose(stream_id); + return 0; + }); +#endif std::unordered_map> table_range_split_cnt; std::unordered_map> @@ -465,12 +461,26 @@ int RecoveryService::on_received_messages(brpc::StreamId stream_id, { std::unordered_set range_split_tables; // All of the shema and range split logs should be in the first msg. - // Collect which tables are range splitting. For these table we only - // need to recover write intent on catalog entry instead of write - // lock when processing schema op msg. + // Collect the tables with a split in flight that this node group + // coordinates. Such a split holds the table's catalog read lock + // here (the data sync running it reads the catalog locally), and + // its recovery takes it again; so no schema op holds the table's + // write lock here, and a participant restores only a write intent + // (catalog_cc_map.h, Execute(ReplayLogCc&)). A split coordinated + // by another node group locks nothing here and proves nothing. + const uint32_t recovering_ng_id = msg.cc_node_group_id(); for (const ::txlog::ReplaySplitRangeMsg &split_range_msg : msg.split_range_op_msgs()) { + const uint32_t split_tx_ng_id = + (split_range_msg.txn() >> 32L) >> 10; + if (split_tx_ng_id != recovering_ng_id || + Sharder::Instance().CandidateLeaderTerm(split_tx_ng_id) < 0) + { + // The condition range_cc_map.h's Execute(ReplayLogCc&) + // uses to recover the split here. + continue; + } const std::string &split_range_op_blob = split_range_msg.split_range_op_blob(); size_t blob_offset = 0; @@ -499,8 +509,9 @@ int RecoveryService::on_received_messages(brpc::StreamId stream_id, range_split_tables.insert(base_table_name); } - // Barrier #1: collect global range splitting tables (base - // tables) before starting schema replay. + // Barrier #1: collect, from every log group, the base tables with + // a split in flight that this node group coordinates before + // starting schema replay. if (barrier != nullptr && !barrier->WaitSplitTableCollected( info, stream_id, std::move(range_split_tables))) @@ -574,6 +585,41 @@ int RecoveryService::on_received_messages(brpc::StreamId stream_id, } } + // The cluster gate slot (EloqDoc docs/cdc-change-stream.md §6.2), + // after cluster scale's replay, which caches node group configs in the + // same cc entry's payload, and before any schema op, so that the + // recovered node group enforces the gate before it serves DDL. + if (msg.has_cluster_gate_msg()) + { + const std::string &gate_blob = + msg.cluster_gate_msg().cluster_gate_blob(); + ReplayLogCc *cc_req = replay_cc_pool_.NextRequest(); + cc_req->Reset( + cc_ng_id, + cc_ng_term, + cluster_config_ccm_name_sv, + TableType::ClusterConfig, + TableEngine::None, + std::string_view(gate_blob.data(), gate_blob.length()), + msg.cluster_gate_msg().commit_ts(), + msg.cluster_gate_msg().txn(), + mux, + status, + on_fly_cnt, + recovery_error, + is_lock_recovery); + cc_req->SetClusterGate(); + + on_fly_cnt.fetch_add(1, std::memory_order_release); + local_shards_.EnqueueCcRequest(0, cc_req); + WaitAndClearRequests( + stream_id, mux, on_fly_cnt, status, recovery_error); + if (recovery_error) + { + return 0; + } + } + // process schema ops before processing data ops for (const ::txlog::ReplaySchemaMsg &schema_op_msg : msg.schema_op_msgs()) @@ -940,24 +986,18 @@ void RecoveryService::WaitAndClearRequests(brpc::StreamId stream_id, error_node_group_id = it->second->cc_ng_id_; error_term = it->second->cc_ng_term_; - uint64_t replay_start_ts = 0; - // close all the streams belonging to the current node group and term. + // The replay failed: close all the streams belonging to the current + // node group and term. The log groups that have not finished the + // term's replay then have no stream, and the watchdog asks them again. for (const auto &[stream_id, info] : inbound_connections_) { - assert(replay_start_ts == 0 || - replay_start_ts == info->replay_start_ts_); - replay_start_ts = info->replay_start_ts_; if (info->cc_ng_id_ == error_node_group_id && info->cc_ng_term_ == error_term) { brpc::StreamClose(stream_id); } } - // put the replay log request back to the replay queue, but the replay - // request will be scheduled with 10 senconds delay. log_id = -1 means - // replay from all the log groups. - ReplayLog(error_node_group_id, error_term, -1, replay_start_ts, true); } } @@ -976,42 +1016,55 @@ void RecoveryService::ClearTx(uint64_t tx_number) void RecoveryService::ProcessReplayLogTask(ReplayLogTask &task) { - if (Sharder::Instance().CandidateLeaderTerm(task.cc_ng_id_) < 0) + const uint32_t ng_id = task.cc_ng_id_; + const int64_t term = task.cc_ng_term_; + if (Sharder::Instance().CandidateLeaderTerm(ng_id) != term) { - LOG(INFO) << "node group is not recovering, skip replay."; - // node group is not recovering, skip replay. + LOG(INFO) << "node group " << ng_id << " is not recovering term " + << term << ", skip replay request."; return; } - // call log agent replay log api - log_agent_->ReplayLog(task.cc_ng_id_, - task.cc_ng_term_, - ip_, - port_, - task.log_group_, - task.replay_start_ts_, - finish_); -} + CandidateReplayWatch &watch = candidate_replay_watch_[ng_id]; + if (watch.term != term) + { + // The term's first request; every later one reuses its start + // timestamp. + watch = CandidateReplayWatch{}; + watch.term = term; + watch.replay_start_ts = task.replay_start_ts_; + } -int RecoveryService::ProcessDelayedReplayLogTask() -{ - // move the delayed replay requests whose timer is fired to - // replay_log_queue_ - int ready_cnt = 0; - for (auto it = delayed_replay_queue_.begin(); - it != delayed_replay_queue_.end();) + std::vector log_groups; + if (task.log_group_ >= 0) { - if (ReplayNow(*it)) + log_groups.push_back(static_cast(task.log_group_)); + } + else + { + for (uint32_t lg_id = 0; lg_id < log_agent_->LogGroupCount(); ++lg_id) { - replay_log_queue_.emplace_back(*it); - it = delayed_replay_queue_.erase(it); - ready_cnt++; + log_groups.push_back(lg_id); } - else + } + for (uint32_t lg_id : log_groups) + { + // Checked before each send, which takes up to a few seconds. + if (Sharder::Instance().CandidateLeaderTerm(ng_id) != term) + { + LOG(INFO) << "node group " << ng_id << " stopped recovering term " + << term << ", skip the rest of its replay request."; + return; + } + if (Sharder::Instance().CheckLogGroupReplayFinished(ng_id, lg_id, term)) { - it++; + continue; } + // One bounded attempt; whatever its outcome, the watchdog asks again + // if no replay stream comes from the log group. + log_agent_->ReplayLog( + ng_id, term, ip_, port_, lg_id, watch.replay_start_ts); + watch.last_send_ts[lg_id] = LocalCcShards::ClockTs(); } - return ready_cnt; } void RecoveryService::ProcessRecoverTxTask(RecoverTxTask &task) diff --git a/tx_service/src/remote/cc_node_service.cpp b/tx_service/src/remote/cc_node_service.cpp index 1075e9a8a..104e36b4f 100644 --- a/tx_service/src/remote/cc_node_service.cpp +++ b/tx_service/src/remote/cc_node_service.cpp @@ -1203,8 +1203,7 @@ void CcNodeService::GenerateSkFromPk( LocalCcShards *cc_shards = Sharder::Instance().GetLocalCcShards(); std::unique_ptr sk_generator = nullptr; { - std::lock_guard lk( - cc_shards->table_index_op_pool_mux_); + std::lock_guard lk(cc_shards->op_pool_mux_); if (cc_shards->sk_generator_pool_.empty()) { sk_generator = std::make_unique(); @@ -1270,8 +1269,7 @@ void CcNodeService::GenerateSkFromPk( // recycle skgenerator { - std::lock_guard lk( - cc_shards->table_index_op_pool_mux_); + std::lock_guard lk(cc_shards->op_pool_mux_); cc_shards->sk_generator_pool_.emplace_back( std::move(sk_generator)); } @@ -2185,16 +2183,21 @@ void CcNodeService::FetchNodeInfo( response->set_ng_id(ng_id); response->set_node_id(node_id); - if (Sharder::Instance().LeaderTerm(ng_id) > 0) + const int64_t leader_term = Sharder::Instance().LeaderTerm(ng_id); + const int64_t candidate_term = + Sharder::Instance().CandidateLeaderTerm(ng_id); + if (leader_term > 0) { response->set_role(NodeRole::LeaderNode); response->set_status(NodeStatus::Online); + response->set_leader_term(leader_term); return; } - else if (Sharder::Instance().CandidateLeaderTerm(ng_id) > 0) + else if (candidate_term > 0) { response->set_role(NodeRole::LeaderNode); response->set_status(NodeStatus::Loading); + response->set_leader_term(candidate_term); return; } else if (Sharder::Instance().NativeNodeGroup() == ng_id) diff --git a/tx_service/src/remote/remote_cc_handler.cpp b/tx_service/src/remote/remote_cc_handler.cpp index c7a14cb0d..515116463 100644 --- a/tx_service/src/remote/remote_cc_handler.cpp +++ b/tx_service/src/remote/remote_cc_handler.cpp @@ -255,8 +255,13 @@ void txservice::remote::RemoteCcHandler::PostWriteAll( post_write_all->set_commit_ts(commit_ts); post_write_all->clear_record(); + // A catalog PostCommit or Rollback carries no record: the receiver's + // catalog cc map fills one of its own from the node's catalog entry. if (commit_ts > 0 && op_type != OperationType::Delete && - op_type != OperationType::DropTable) + op_type != OperationType::DropTable && + !(table_name.Type() == TableType::Catalog && + (post_write_type == PostWriteType::PostCommit || + post_write_type == PostWriteType::Rollback))) { // The commit ts is 0, if the post-write request is used to clear the // write lock when the tx aborts. diff --git a/tx_service/src/sharder.cpp b/tx_service/src/sharder.cpp index d17158604..aa29bc9bd 100644 --- a/tx_service/src/sharder.cpp +++ b/tx_service/src/sharder.cpp @@ -168,7 +168,7 @@ int Sharder::Init( for (uint32_t nid = 0; nid < 1000; nid++) { ng_leader_cache_[nid].store(nid); - ng_leader_term_cache_[nid].store(-1); + ng_leader_term_cache_[nid] = -1; leader_term_cache_[nid].store(-1); candidate_leader_term_cache_[nid].store(-1); } @@ -477,21 +477,11 @@ int Sharder::Init( { bool retry = false; uint64_t start_ts = 0; + // OnLeaderStart asks the log service to replay the log. cluster_config_.cc_nodes_.at(native_ng_) ->OnLeaderStart(1, start_ts, retry); assert(!retry); - // Notify log service to replay log - if (log_agent_) - { - log_agent_->ReplayLog(native_ng_, - 1, - host_name_, - GET_LOG_REPLAY_RPC_PORT(port_), - -1, - start_ts, - log_agent_interrupt_); - } - else + if (!log_agent_) { // Test env. Directly finish log replay. Sharder::Instance().SetLeaderTerm(native_ng_, 1); @@ -668,42 +658,83 @@ void Sharder::UpdateLeader(uint32_t ng_id) remote::GetLeaderResponse resp; remote::HostMangerService_Stub stub(&hm_channel_); stub.GetLeader(&cntl, &req, &resp, nullptr); - if (!cntl.Failed() && !resp.error()) + if (cntl.Failed() || resp.error()) { - UpdateLeader(ng_id, resp.node_id()); + return; } + // The host manager names the leader without its term; the + // leader itself knows it. + const uint32_t leader_node_id = resp.node_id(); + const int64_t term = FetchLeaderTerm(ng_id, leader_node_id); + if (term < 0) + { + LOG(INFO) << "leader lookup of ng" << ng_id + << " named node_id " << leader_node_id + << ", which does not lead it now; cache kept"; + return; + } + UpdateLeader(ng_id, leader_node_id, term); }); } } -void Sharder::UpdateLeader(uint32_t ng_id, uint32_t node_id, int64_t term) +int64_t Sharder::FetchLeaderTerm(uint32_t ng_id, uint32_t node_id) { - auto cached_leader_term = - ng_leader_term_cache_[ng_id].load(std::memory_order_relaxed); - // If the term is passed and out-dated, skip it. - if (term != -1 && cached_leader_term != -1 && term <= cached_leader_term) + if (node_id == node_id_) { - DLOG(INFO) << "skip out-dated node group leader update, ng: " << ng_id - << ", term: " << term - << ", cached term: " << cached_leader_term; - return; + const int64_t leader_term = LeaderTerm(ng_id); + return leader_term > 0 ? leader_term : CandidateLeaderTerm(ng_id); + } + std::shared_ptr channel = GetCcNodeServiceChannel(node_id); + if (channel == nullptr) + { + return -1; } + remote::CcRpcService_Stub stub(channel.get()); + remote::FetchNodeInfoRequest req; + req.set_ng_id(ng_id); + req.set_node_id(node_id); + remote::FetchNodeInfoResponse resp; + brpc::Controller cntl; + cntl.set_timeout_ms(1000); + stub.FetchNodeInfo(&cntl, &req, &resp, nullptr); + if (cntl.Failed()) + { + UpdateCcNodeServiceChannel(node_id, channel); + return -1; + } + return resp.role() == remote::NodeRole::LeaderNode && resp.leader_term() > 0 + ? resp.leader_term() + : -1; +} - while (!ng_leader_term_cache_[ng_id].compare_exchange_weak( - cached_leader_term, term)) +void Sharder::UpdateLeader(uint32_t ng_id, uint32_t node_id, int64_t term) +{ { - if (term != -1 && cached_leader_term != -1 && - term <= cached_leader_term) + std::lock_guard lk(ng_leader_mux_); + const int64_t cached_leader_term = ng_leader_term_cache_[ng_id]; + if (term <= cached_leader_term) { DLOG(INFO) << "skip out-dated node group leader update, ng: " << ng_id << ", term: " << term << ", cached term: " << cached_leader_term; return; } + ng_leader_term_cache_[ng_id] = term; + ng_leader_cache_[ng_id].store(node_id, std::memory_order_release); } LOG(INFO) << "update node group ng" << ng_id - << " leader to node_id: " << node_id; - ng_leader_cache_[ng_id].store(node_id, std::memory_order_release); + << " leader to node_id: " << node_id << ", term: " << term; + if (local_shards_ != nullptr && local_shards_->store_hd_ != nullptr) + { + // The data store's direct accesses to the group follow its leader. The + // term is the version of the group's shard, the one every node derives + // alike, and orders these moves however they arrive there. Passed on + // even when the leader is the same node: the shard's version must keep + // up with the term, or a delayed redirect at a version between the two + // (a leader this node missed) would still be accepted. + local_shards_->store_hd_->OnLeaderChanged(ng_id, node_id, term); + } } void Sharder::FinishLogReplay(uint32_t cc_ng_id, @@ -747,6 +778,20 @@ bool Sharder::CheckLogGroupReplayFinished(uint32_t cc_ng_id, cc_ng_term); } +void Sharder::RequestLogReplay(uint32_t cc_ng_id, + int64_t cc_ng_term, + uint64_t replay_start_ts) +{ + if (log_agent_ == nullptr || recovery_service_ == nullptr) + { + return; + } + LOG(INFO) << "Requesting log replay for ng: " << cc_ng_id + << " at term: " << cc_ng_term << " from ts: " << replay_start_ts + << " on its leader start"; + recovery_service_->ReplayLog(cc_ng_id, cc_ng_term, -1, replay_start_ts); +} + void Sharder::WaitClusterReady() { bool recovery_all_finished = false; diff --git a/tx_service/src/tx_cluster_gate_operation.cpp b/tx_service/src/tx_cluster_gate_operation.cpp new file mode 100644 index 000000000..706b10677 --- /dev/null +++ b/tx_service/src/tx_cluster_gate_operation.cpp @@ -0,0 +1,462 @@ +/** + * Copyright (C) 2026 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ + +/** + * @file + * @brief ClusterGateOp, the transaction that pauses or resumes the cluster + * gate (EloqDoc docs/cdc-change-stream.md §6.2), and its entry points on + * TransactionExecution. + */ + +#include + +#include +#include +#include +#include +#include + +#include "error_messages.h" +#include "fault/fault_inject.h" +#include "local_cc_shards.h" +#include "log_type.h" +#include "sharder.h" +#include "tx_execution.h" +#include "tx_operation.h" +#include "tx_request.h" +#include "tx_trace.h" + +namespace txservice +{ + +template +void ClusterGateOp::RunStage(TransactionExecution *txm, Op *stage) +{ + op_ = stage; + txm->PushOperation(stage); + txm->Process(*stage); +} + +ClusterGateOp::ClusterGateOp(TransactionExecution *txm) + : CompositeTransactionOperation(), + read_gate_op_(), + acquire_intent_op_(txm), + acquire_write_op_(txm), + gate_log_op_(txm), + install_op_(txm), + read_result_(txm) +{ + acquire_intent_op_.table_name_ = &cluster_config_ccm_name; + acquire_intent_op_.keys_.emplace_back(TxKey(VoidKey::NegativeInfinity())); + acquire_intent_op_.cc_op_ = CcOperation::ReadForWrite; + acquire_intent_op_.protocol_ = CcProtocol::OCC; + + acquire_write_op_.table_name_ = &cluster_config_ccm_name; + acquire_write_op_.keys_.emplace_back(TxKey(VoidKey::NegativeInfinity())); + acquire_write_op_.cc_op_ = CcOperation::Write; + acquire_write_op_.protocol_ = CcProtocol::Locking; + + read_gate_op_.table_name_ = TableName(cluster_config_ccm_name_sv, + TableType::ClusterConfig, + TableEngine::None); + read_gate_op_.key_ = VoidKey::NegInfTxKey(); + read_gate_op_.hd_result_ = &read_result_; + read_gate_op_.rec_ = &read_rec_; + // A write intent: schema ops' read locks pass it, another gate op on this + // node group waits for it, and stage 2b upgrades it to the write lock. + read_gate_op_.is_for_write_ = true; + + install_op_.table_name_ = &cluster_config_ccm_name; + install_op_.keys_.emplace_back(TxKey(VoidKey::NegativeInfinity())); + install_op_.recs_.push_back(&install_rec_); + install_op_.op_type_ = OperationType::Update; + install_op_.write_type_ = PostWriteType::Commit; +} + +void ClusterGateOp::Reset(ClusterGateOpType op_type, + ClusterGate::Mode mode, + TransactionExecution *txm) +{ + op_ = nullptr; + op_type_ = op_type; + mode_ = mode; + error_ = TxErrorCode::NO_ERROR; + restart_after_release_ = false; + attempt_ = 0; + + acquire_intent_op_.ResetHandlerTxm(txm); + acquire_write_op_.ResetHandlerTxm(txm); + read_result_.ResetTxm(txm); + read_result_.Reset(); + read_rec_.Reset(); + gate_log_op_.Reset(); + gate_log_op_.ResetHandlerTxm(txm); + install_op_.ResetHandlerTxm(txm); + install_rec_.Reset(); +} + +void ClusterGateOp::ResumeAtInstall(const ClusterGate &gate, uint64_t commit_ts) +{ + op_type_ = + gate.paused ? ClusterGateOpType::kPause : ClusterGateOpType::kResume; + mode_ = gate.mode; + // What FillGateLogRequest() set before the crash. A record without a gate + // would take the map's topology path. + install_rec_.SetClusterGate(gate); + install_rec_.SetVersion(commit_ts); + op_ = &gate_log_op_; + gate_log_op_.hd_result_.SetFinished(); +} + +void ClusterGateOp::Forward(TransactionExecution *txm) +{ + if (op_ == nullptr) + { + ++attempt_; + LOG(INFO) << "Cluster gate " + << (op_type_ == ClusterGateOpType::kPause ? "pause" + : "resume") + << ", attempt " << attempt_ + << ": reading the gate for write, txn: " << txm->TxNumber(); + txm->commit_ts_ = txm->commit_ts_bound_ + 1; + RunStage(txm, &read_gate_op_); + } + else if (op_ == &read_gate_op_) + { + if (read_result_.IsError()) + { + LOG(ERROR) << "Cluster gate failed to read the cluster config, " + "txn: " + << txm->TxNumber(); + ForceToFinish( + txm, + TransactionExecution::ConvertCcError(read_result_.ErrorCode())); + return; + } + if (op_type_ == ClusterGateOpType::kPause && read_rec_.GatesSchemaOps()) + { + // Only the write intent here is held; the abort releases it. + LOG(WARNING) << "Cluster gate pause refused: the gate is already " + "paused, txn: " + << txm->TxNumber(); + ForceToFinish(txm, TxErrorCode::CLUSTER_GATE_HELD); + return; + } + RunStage(txm, &acquire_intent_op_); + } + else if (op_ == &acquire_intent_op_) + { + if (!txm->CheckLeaderTerm()) + { + ForceToFinish(txm, TxErrorCode::TRANSACTION_NODE_NOT_LEADER); + return; + } + bool conflict = + acquire_intent_op_.fail_cnt_.load(std::memory_order_relaxed) > 0; + // One conflict on demand, for tests that run a single node group, + // where no other gate op or cluster scale can hold the entry. + CODE_FAULT_INJECTOR("cluster_gate_intent_conflict_once", { + FaultInject::Instance().InjectFault( + "cluster_gate_intent_conflict_once", "remove"); + conflict = true; + }); + if (conflict) + { + // Another gate op or a cluster scale holds the entry somewhere: + // OCC failed rather than waited, so nothing waits on this tx. + RestartLater(txm, "write intents"); + return; + } + RunStage(txm, &acquire_write_op_); + } + else if (op_ == &acquire_write_op_) + { + if (!txm->CheckLeaderTerm()) + { + ForceToFinish(txm, TxErrorCode::TRANSACTION_NODE_NOT_LEADER); + return; + } + if (acquire_write_op_.fail_cnt_.load(std::memory_order_relaxed) > 0) + { + // Not a lock conflict, which this stage waits out, but a node + // group without a leader or failing over. + RestartLater(txm, "write locks"); + return; + } + txm->commit_ts_ = std::max(txm->commit_ts_, acquire_write_op_.MaxTs()); + FillGateLogRequest(txm); + RunStage(txm, &gate_log_op_); + } + else if (op_ == &gate_log_op_) + { + if (!txm->CheckLeaderTerm()) + { + // Whether the entry was written is for the log to say: if it was, + // this node group's replay resumes the op at its install. + ForceToFinish(txm, TxErrorCode::TRANSACTION_NODE_NOT_LEADER); + return; + } + if (gate_log_op_.hd_result_.IsError()) + { + if (gate_log_op_.hd_result_.ErrorCode() == + CcErrorCode::LOG_CLOSURE_RESULT_UNKNOWN_ERR) + { + // Retried with the retry flag, so that the log service looks + // the entry up in its slot before rejecting it on a term. + gate_log_op_.log_closure_.LogRequest() + .mutable_write_log_request() + ->set_retry(true); + RunStage(txm, &gate_log_op_); + return; + } + // Refused, for instance because a node group failed over after + // granting its lock: the entry is not in the log, so the op can + // start over like after a conflict. + RestartLater(txm, "log entry"); + return; + } + ACTION_FAULT_INJECTOR("cluster_gate_after_log"); + RunStage(txm, &install_op_); + } + else if (op_ == &install_op_) + { + if (!txm->CheckLeaderTerm()) + { + ForceToFinish(txm, TxErrorCode::TRANSACTION_NODE_NOT_LEADER); + return; + } + if (install_op_.hd_result_.IsError()) + { + // Both the install and the release are idempotent: a node group + // that already ran it no longer holds the lock for this tx and + // ignores the retry. + RetryInstallLater(txm); + return; + } + if (restart_after_release_) + { + BackOffAndRestart(txm); + return; + } + Finish(txm); + } +} + +void ClusterGateOp::RestartLater(TransactionExecution *txm, const char *stage) +{ + LOG(WARNING) << "Cluster gate attempt " << attempt_ << " failed at its " + << stage + << "; releasing and retrying, txn: " << txm->TxNumber(); + restart_after_release_ = true; + // The cluster config map releases the tx's locks and commits nothing on + // tx_op_failed_ts_, on node groups where nothing was granted too. + txm->commit_ts_ = tx_op_failed_ts_; + RunStage(txm, &install_op_); +} + +void ClusterGateOp::BackOffAndRestart(TransactionExecution *txm) +{ + restart_after_release_ = false; + // The release dropped the read's lock with every other; if the read is + // still in the read set, its address may name a recycled lock object. + txm->rw_set_.DedupRead(read_result_.Value().cce_addr_); + read_result_.Reset(); + read_rec_.Reset(); + op_ = nullptr; + // Jittered, so that two gate ops that keep meeting do not retry in step. + // The sleep op pops itself when the time is up, and Forward() of this op, + // now on top again, starts over at stage 1. + txm->sleep_op_.sleep_secs_ = 1 + butil::fast_rand_less_than(3); + txm->PushOperation(&txm->sleep_op_); + txm->StartTiming(); +} + +void ClusterGateOp::RetryInstallLater(TransactionExecution *txm) +{ + op_ = &install_op_; + txm->PushOperation(&install_op_); + install_op_.is_running_ = false; + // The sleep op pops itself when the second is up, and the install, now + // on top, runs again. + txm->sleep_op_.sleep_secs_ = 1; + txm->PushOperation(&txm->sleep_op_); + txm->StartTiming(); +} + +void ClusterGateOp::FillGateLogRequest(TransactionExecution *txm) +{ + ClusterGate gate; + gate.mode = mode_; + gate.paused = op_type_ == ClusterGateOpType::kPause; + install_rec_.SetClusterGate(gate); + install_rec_.SetVersion(txm->commit_ts_); + + gate_log_op_.log_type_ = TxLogType::COMMIT; + gate_log_op_.log_closure_.LogRequest().Clear(); + ::txlog::WriteLogRequest *log_rec = + gate_log_op_.log_closure_.LogRequest().mutable_write_log_request(); + log_rec->set_tx_term(txm->tx_term_); + log_rec->set_txn_number(txm->TxNumber()); + log_rec->set_commit_timestamp(txm->commit_ts_); + ::txlog::ClusterGateLogMessage *gate_msg = + log_rec->mutable_log_content()->mutable_cluster_gate_log(); + gate_msg->set_mode(gate.mode == ClusterGate::Mode::kDdlDml + ? ::txlog::ClusterGateLogMessage_GateMode_GateDdlDml + : ::txlog::ClusterGateLogMessage_GateMode_GateDdl); + gate_msg->set_paused(gate.paused); + + // Unlike cluster scale, the gate writes no log before it locks, so a node + // group's replay could not restore a lock it lost by failing over. The + // terms make the log service refuse the entry instead. + auto &node_terms = *log_rec->mutable_node_terms(); + node_terms.clear(); + for (size_t idx = 0; idx < acquire_write_op_.upload_cnt_; ++idx) + { + const AcquireAllResult &res = + acquire_write_op_.hd_results_[idx].Value(); + node_terms[res.local_cce_addr_.NodeGroupId()] = res.node_term_; + } +} + +void ClusterGateOp::ForceToFinish(TransactionExecution *txm, TxErrorCode err) +{ + error_ = err; + txm->commit_ts_ = tx_op_failed_ts_; + Finish(txm); +} + +void ClusterGateOp::Finish(TransactionExecution *txm) +{ + // Everything read from the op is copied first: once it is back in the + // pool, another request may take it and reset it. + const bool succeeded = txm->commit_ts_ != tx_op_failed_ts_; + const uint64_t commit_ts = txm->commit_ts_; + const TxErrorCode error = error_; + TxResult *resp = txm->uint64_resp_; + txm->uint64_resp_ = nullptr; + txm->state_stack_.pop_back(); + + if (succeeded) + { + LOG(INFO) << "Cluster gate " + << (op_type_ == ClusterGateOpType::kPause ? "paused" + : "resumed") + << " at " << commit_ts << ", txn: " << txm->TxNumber(); + } + + { + // Back to the pool, not freed: a remote response that comes in after + // the op finished still reads one of its stages' cc handler results. + LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); + std::lock_guard lk(shards->op_pool_mux_); + shards->cluster_gate_op_pool_.push_back( + std::move(txm->CompositeOpSlot())); + } + + if (resp != nullptr) + { + if (succeeded) + { + resp->Finish(commit_ts); + } + else + { + resp->FinishError(error); + } + } + + if (succeeded) + { + txm->Commit(); + } + else + { + txm->Abort(); + } +} + +namespace +{ +/** + * @brief Takes a cluster gate op from LocalCcShards' pool, or builds one when + * the pool is empty. + * + * @param txm The transaction that will run it. + * @return The op, still to be reset for its request. + */ +std::unique_ptr TakeClusterGateOp(TransactionExecution *txm) +{ + LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); + std::lock_guard lk(shards->op_pool_mux_); + if (shards->cluster_gate_op_pool_.empty()) + { + return std::make_unique(txm); + } + std::unique_ptr op = + std::move(shards->cluster_gate_op_pool_.back()); + shards->cluster_gate_op_pool_.pop_back(); + return op; +} +} // namespace + +void TransactionExecution::ProcessTxRequest(ClusterGateTxRequest &req) +{ + TX_TRACE_ACTION_WITH_CONTEXT( + this, + &req, + [this]() -> std::string + { + return std::string("\"tx_number\":") + .append(std::to_string(this->TxNumber())) + .append("\"tx_term\":") + .append(std::to_string(this->tx_term_)); + }); + + // The request's result shares the scan open response, of the same type. + static_assert(std::is_same_v); + uint64_resp_ = &req.tx_result_; + std::unique_ptr &op = CompositeOpSlot(); + op = TakeClusterGateOp(this); + op->Reset(req.op_type_, req.mode_, this); + PushOperation(op.get()); +} + +void TransactionExecution::RecoverClusterGate( + const ::txlog::ClusterGateLogMessage &gate_msg) +{ + ClusterGate gate; + gate.mode = + gate_msg.mode() == ::txlog::ClusterGateLogMessage_GateMode_GateDdlDml + ? ClusterGate::Mode::kDdlDml + : ClusterGate::Mode::kDdl; + gate.paused = gate_msg.paused(); + std::unique_ptr &op = CompositeOpSlot(); + op = TakeClusterGateOp(this); + op->Reset( + gate.paused ? ClusterGateOpType::kPause : ClusterGateOpType::kResume, + gate.mode, + this); + op->ResumeAtInstall(gate, commit_ts_); + PushOperation(op.get()); +} + +} // namespace txservice diff --git a/tx_service/src/tx_execution.cpp b/tx_service/src/tx_execution.cpp index 01e337b75..45557ca66 100644 --- a/tx_service/src/tx_execution.cpp +++ b/tx_service/src/tx_execution.cpp @@ -148,10 +148,7 @@ void TransactionExecution::Reset() bool_resp_ = nullptr; kvp_resp_ = nullptr; uint64_resp_ = nullptr; - schema_op_ = nullptr; - split_flush_op_ = nullptr; - index_op_ = nullptr; - invalidate_table_cache_composite_op_ = nullptr; + composite_op_ = std::monostate{}; scan_alias_cnt_ = 0; @@ -315,13 +312,16 @@ void TransactionExecution::ReleaseCatalogsRead() } uint64_t tx_number = TxNumber(); + // The commit ts, if the tx got one: 0 from its start until then, and + // UINT64_MAX before its first start. + const uint64_t commit_ts = commit_ts_ == UINT64_MAX ? 0 : commit_ts_; for (auto &db_idx : locked_db_) { if (db_idx.first != nullptr) { LocalCcHandler *local_hd = static_cast(cc_handler_); - local_hd->ReleaseCatalogRead(db_idx.first, tx_number); + local_hd->ReleaseCatalogRead(db_idx.first, tx_number, commit_ts); } } locked_db_ = {}; @@ -975,8 +975,7 @@ void TransactionExecution::ProcessTxRequest(UpsertTableTxRequest &req) req.op_type_ == OperationType::DropTable || req.op_type_ == OperationType::TruncateTable) { - std::unique_lock lk( - local_shards->table_schema_op_pool_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); if (local_shards->table_schema_op_pool_.empty()) { std::unique_ptr table_op = nullptr; @@ -988,30 +987,32 @@ void TransactionExecution::ProcessTxRequest(UpsertTableTxRequest &req) *req.dirty_image_, req.op_type_, this); - schema_op_ = std::move(table_op); + CompositeOpSlot() = std::move(table_op); } else { assert(local_shards->table_schema_op_pool_.back() != nullptr); - schema_op_ = std::move(local_shards->table_schema_op_pool_.back()); + CompositeOpSlot() = + std::move(local_shards->table_schema_op_pool_.back()); local_shards->table_schema_op_pool_.pop_back(); - schema_op_->Reset(req.table_name_->StringView(), - req.table_name_->Engine(), - *req.curr_image_, - req.curr_schema_ts_, - *req.dirty_image_, - req.op_type_, - this); + CompositeOpSlot()->Reset( + req.table_name_->StringView(), + req.table_name_->Engine(), + *req.curr_image_, + req.curr_schema_ts_, + *req.dirty_image_, + req.op_type_, + this); } lk.unlock(); - PushOperation(schema_op_.get()); + PushOperation(CompositeOpSlot().get()); } else if (req.op_type_ == OperationType::AddIndex || req.op_type_ == OperationType::DropIndex) { - std::unique_lock lk(local_shards->table_index_op_pool_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); if (local_shards->table_index_op_pool_.empty()) { std::unique_ptr index_op = @@ -1026,27 +1027,29 @@ void TransactionExecution::ProcessTxRequest(UpsertTableTxRequest &req) req.pack_sk_err_.get(), this); - index_op_ = std::move(index_op); + CompositeOpSlot() = std::move(index_op); } else { assert(local_shards->table_index_op_pool_.back() != nullptr); - index_op_ = std::move(local_shards->table_index_op_pool_.back()); + CompositeOpSlot() = + std::move(local_shards->table_index_op_pool_.back()); local_shards->table_index_op_pool_.pop_back(); - index_op_->Reset(req.table_name_->StringView(), - req.table_name_->Engine(), - *req.curr_image_, - req.curr_schema_ts_, - *req.dirty_image_, - *req.alter_table_info_image_, - req.op_type_, - req.pack_sk_err_.get(), - this); + CompositeOpSlot()->Reset( + req.table_name_->StringView(), + req.table_name_->Engine(), + *req.curr_image_, + req.curr_schema_ts_, + *req.dirty_image_, + *req.alter_table_info_image_, + req.op_type_, + req.pack_sk_err_.get(), + this); } lk.unlock(); - PushOperation(index_op_.get()); + PushOperation(CompositeOpSlot().get()); } else { @@ -1147,24 +1150,24 @@ void TransactionExecution::ProcessTxRequest(InvalidateTableCacheTxRequest &req) void_resp_ = &req.tx_result_; LocalCcShards *local_shards = Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk( - local_shards->invalidate_table_cache_op_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); if (local_shards->invalidate_table_cache_op_pool_.empty()) { - invalidate_table_cache_composite_op_ = + CompositeOpSlot() = std::make_unique(req.table_name_, this); } else { - invalidate_table_cache_composite_op_ = + CompositeOpSlot() = std::move(local_shards->invalidate_table_cache_op_pool_.back()); local_shards->invalidate_table_cache_op_pool_.pop_back(); - invalidate_table_cache_composite_op_->Reset(req.table_name_, this); + CompositeOpSlot()->Reset( + req.table_name_, this); } lk.unlock(); - PushOperation(invalidate_table_cache_composite_op_.get()); + PushOperation(CompositeOpSlot().get()); } void TransactionExecution::ProcessTxRequest(SplitFlushTxRequest &req) @@ -1183,11 +1186,10 @@ void TransactionExecution::ProcessTxRequest(SplitFlushTxRequest &req) bool_resp_ = &req.tx_result_; LocalCcShards *local_shards = Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk( - local_shards->split_flush_range_op_pool_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); if (local_shards->split_flush_range_op_pool_.empty()) { - split_flush_op_ = + CompositeOpSlot() = std::make_unique(*req.table_name_, std::move(req.schema_), req.range_entry_, @@ -1197,20 +1199,21 @@ void TransactionExecution::ProcessTxRequest(SplitFlushTxRequest &req) } else { - split_flush_op_ = + CompositeOpSlot() = std::move(local_shards->split_flush_range_op_pool_.back()); local_shards->split_flush_range_op_pool_.pop_back(); - assert(split_flush_op_ != nullptr); - split_flush_op_->Reset(*req.table_name_, - std::move(req.schema_), - req.range_entry_, - std::move(req.new_range_info_), - this, - req.is_dirty_); + assert(CompositeOpSlot() != nullptr); + CompositeOpSlot()->Reset( + *req.table_name_, + std::move(req.schema_), + req.range_entry_, + std::move(req.new_range_info_), + this, + req.is_dirty_); } lk.unlock(); - PushOperation(split_flush_op_.get()); + PushOperation(CompositeOpSlot().get()); } void TransactionExecution::ProcessTxRequest( @@ -1219,18 +1222,20 @@ void TransactionExecution::ProcessTxRequest( void_resp_ = &data_migration_req.tx_result_; LocalCcShards *local_shards = Sharder::Instance().GetLocalCcShards(); - std::lock_guard lk(local_shards->data_migration_op_pool_mux_); + std::lock_guard lk(local_shards->op_pool_mux_); if (local_shards->migration_op_pool_.empty()) { - migration_op_ = + CompositeOpSlot() = std::make_unique(this, data_migration_req.status_); } else { - migration_op_ = std::move(local_shards->migration_op_pool_.back()); + CompositeOpSlot() = + std::move(local_shards->migration_op_pool_.back()); local_shards->migration_op_pool_.pop_back(); - migration_op_->Reset(this, data_migration_req.status_); + CompositeOpSlot()->Reset(this, + data_migration_req.status_); } if (data_migration_req.status_->next_bucket_idx_ != 0) @@ -1240,7 +1245,7 @@ void TransactionExecution::ProcessTxRequest( void_resp_->Finish(void_); } - PushOperation(migration_op_.get()); + PushOperation(CompositeOpSlot().get()); Forward(); } @@ -1342,23 +1347,23 @@ void TransactionExecution::ProcessTxRequest(ClusterScaleTxRequest &req) return; } - std::unique_lock lk(local_shards->cluster_scale_op_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); if (local_shards->cluster_scale_op_pool_.empty()) { - cluster_scale_op_ = std::make_unique( + CompositeOpSlot() = std::make_unique( req.id_, req.scale_type_, std::move(new_ng_config), this); } else { - cluster_scale_op_ = + CompositeOpSlot() = std::move(local_shards->cluster_scale_op_pool_.back()); local_shards->cluster_scale_op_pool_.pop_back(); - cluster_scale_op_->Reset( + CompositeOpSlot()->Reset( req.id_, req.scale_type_, std::move(new_ng_config), this); } lk.unlock(); - PushOperation(cluster_scale_op_.get()); + PushOperation(CompositeOpSlot().get()); } void TransactionExecution::ProcessTxRequest( @@ -1383,8 +1388,7 @@ void TransactionExecution::ProcessTxRequest( case OperationType::TruncateTable: case OperationType::Update: { - std::unique_lock lk( - local_shards->table_schema_op_pool_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); if (Sharder::Instance() .GetLocalCcShards() ->table_schema_op_pool_.empty()) @@ -1401,16 +1405,17 @@ void TransactionExecution::ProcessTxRequest( schema_op_msg.new_catalog_blob(), operation_type, this); - schema_op_ = std::move(table_op); + CompositeOpSlot() = std::move(table_op); } else { assert(Sharder::Instance() .GetLocalCcShards() ->table_schema_op_pool_.back() != nullptr); - schema_op_ = std::move(Sharder::Instance() - .GetLocalCcShards() - ->table_schema_op_pool_.back()); + CompositeOpSlot() = + std::move(Sharder::Instance() + .GetLocalCcShards() + ->table_schema_op_pool_.back()); Sharder::Instance() .GetLocalCcShards() ->table_schema_op_pool_.pop_back(); @@ -1418,22 +1423,24 @@ void TransactionExecution::ProcessTxRequest( TableEngine table_engine = ::txlog::ToLocalType::ConvertTableEngine( schema_op_msg.table_engine()); - schema_op_->Reset(schema_op_msg.table_name_str(), - table_engine, - schema_op_msg.old_catalog_blob(), - schema_op_msg.catalog_ts(), - schema_op_msg.new_catalog_blob(), - operation_type, - this); + CompositeOpSlot()->Reset( + schema_op_msg.table_name_str(), + table_engine, + schema_op_msg.old_catalog_blob(), + schema_op_msg.catalog_ts(), + schema_op_msg.new_catalog_blob(), + operation_type, + this); } lk.unlock(); + UpsertTableOp *table_op = CompositeOpSlot().get(); if (schema_op_msg.stage() == ::txlog::SchemaOpMessage::Stage:: SchemaOpMessage_Stage_PrepareSchema) { - schema_op_->prepare_log_op_.hd_result_.SetFinished(); - schema_op_->op_ = &schema_op_->prepare_log_op_; + // The op resumes after its prepare log. + table_op->entry_ = UpsertTableOp::Entry::kAfterPrepareLog; } else { @@ -1465,20 +1472,24 @@ void TransactionExecution::ProcessTxRequest( schema_op_msg.new_catalog_blob(), commit_ts_); - schema_op_->catalog_rec_.Set( + table_op->catalog_rec_.Set( schema_ptr, dirty_schema_ptr, schema_ts); - schema_op_->commit_log_op_.hd_result_.SetFinished(); - schema_op_->op_ = &schema_op_->commit_log_op_; + // A commit log that rolled the op back, at timestamp 0: its + // second phase discards the dirty schema. + table_op->post_all_lock_op_.write_type_ = + schema_op_msg.rolled_back() ? PostWriteType::Rollback + : PostWriteType::PostCommit; + // The op resumes after its commit log. + table_op->entry_ = UpsertTableOp::Entry::kAfterCommitLog; } - PushOperation(schema_op_.get()); + PushOperation(table_op); break; } case OperationType::AddIndex: case OperationType::DropIndex: { - std::unique_lock lk( - local_shards->table_index_op_pool_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); if (local_shards->table_index_op_pool_.empty()) { @@ -1497,36 +1508,39 @@ void TransactionExecution::ProcessTxRequest( nullptr, this); - index_op_ = std::move(index_op); + CompositeOpSlot() = std::move(index_op); } else { assert(local_shards->table_index_op_pool_.back() != nullptr); - index_op_ = + CompositeOpSlot() = std::move(local_shards->table_index_op_pool_.back()); local_shards->table_index_op_pool_.pop_back(); TableEngine table_engine = ::txlog::ToLocalType::ConvertTableEngine( schema_op_msg.table_engine()); - index_op_->Reset(schema_op_msg.table_name_str(), - table_engine, - schema_op_msg.old_catalog_blob(), - schema_op_msg.catalog_ts(), - schema_op_msg.new_catalog_blob(), - schema_op_msg.alter_table_info_blob(), - operation_type, - nullptr, - this); + CompositeOpSlot()->Reset( + schema_op_msg.table_name_str(), + table_engine, + schema_op_msg.old_catalog_blob(), + schema_op_msg.catalog_ts(), + schema_op_msg.new_catalog_blob(), + schema_op_msg.alter_table_info_blob(), + operation_type, + nullptr, + this); } lk.unlock(); + UpsertTableIndexOp *index_op = + CompositeOpSlot().get(); if (schema_op_msg.stage() == ::txlog::SchemaOpMessage::Stage:: SchemaOpMessage_Stage_PrepareSchema) { - index_op_->prepare_log_op_.hd_result_.SetFinished(); - index_op_->op_ = &index_op_->prepare_log_op_; + index_op->prepare_log_op_.hd_result_.SetFinished(); + index_op->op_ = &index_op->prepare_log_op_; } else if (schema_op_msg.stage() == ::txlog::SchemaOpMessage::Stage:: @@ -1540,18 +1554,18 @@ void TransactionExecution::ProcessTxRequest( ::txlog::ToLocalType::ConvertTableEngine( schema_op_msg.table_engine()); // The positive inf key - index_op_->last_finished_end_key_ = + index_op->last_finished_end_key_ = Sharder::Instance() .GetLocalCcShards() ->GetCatalogFactory(table_engine) ->PositiveInfKey(); - index_op_->is_last_finished_key_str_ = false; + index_op->is_last_finished_key_str_ = false; } else { - index_op_->last_finished_end_key_str_ = + index_op->last_finished_end_key_str_ = &schema_op_msg.last_key_value(); - index_op_->is_last_finished_key_str_ = true; + index_op->is_last_finished_key_str_ = true; } // - Recovery catalog_rec_ to rebuild dirty schema image. @@ -1565,20 +1579,20 @@ void TransactionExecution::ProcessTxRequest( TableName table_name{table_name_sv, table_type, table_engine}; CatalogEntry *catalog_entry = local_shards->GetCatalog(table_name, TxCcNodeId()); - index_op_->catalog_rec_.Set(catalog_entry->schema_, - catalog_entry->dirty_schema_, - catalog_entry->schema_version_); - index_op_->catalog_rec_.SetSchemaImage( + index_op->catalog_rec_.Set(catalog_entry->schema_, + catalog_entry->dirty_schema_, + catalog_entry->schema_version_); + index_op->catalog_rec_.SetSchemaImage( catalog_entry->schema_->SchemaImage()); - index_op_->catalog_rec_.SetDirtySchemaImage( + index_op->catalog_rec_.SetDirtySchemaImage( catalog_entry->dirty_schema_->SchemaImage()); // Recovery indexes_multikey_attr_. - index_op_->RecoveryIndexesMultiKeyAttr( + index_op->RecoveryIndexesMultiKeyAttr( catalog_entry->dirty_schema_.get()); - index_op_->op_ = &index_op_->prepare_data_log_op_; - index_op_->prepare_data_log_op_.hd_result_.SetFinished(); + index_op->op_ = &index_op->prepare_data_log_op_; + index_op->prepare_data_log_op_.hd_result_.SetFinished(); } else { @@ -1612,21 +1626,28 @@ void TransactionExecution::ProcessTxRequest( schema_op_msg.new_catalog_blob(), commit_ts_); - index_op_->catalog_rec_.Set( + index_op->catalog_rec_.Set( schema_ptr, dirty_schema_ptr, schema_ts); - index_op_->catalog_rec_.SetSchemaImage( + index_op->catalog_rec_.SetSchemaImage( schema_ptr->SchemaImage()); - index_op_->catalog_rec_.SetDirtySchemaImage( + index_op->catalog_rec_.SetDirtySchemaImage( dirty_schema_ptr->SchemaImage()); // Recovery indexes_multikey_attr_. - index_op_->RecoveryIndexesMultiKeyAttr(dirty_schema_ptr.get()); + index_op->RecoveryIndexesMultiKeyAttr(dirty_schema_ptr.get()); + + // A rolled-back add index commits at a real timestamp; the + // commit log's mark tells the recovered op to drop the index. + index_op->rolled_back_ = schema_op_msg.rolled_back(); + index_op->post_all_lock_op_.write_type_ = + schema_op_msg.rolled_back() ? PostWriteType::Rollback + : PostWriteType::PostCommit; - index_op_->commit_log_op_.hd_result_.SetFinished(); - index_op_->op_ = &index_op_->commit_log_op_; + index_op->commit_log_op_.hd_result_.SetFinished(); + index_op->op_ = &index_op->commit_log_op_; } - PushOperation(index_op_.get()); + PushOperation(index_op); break; } default: @@ -1667,8 +1688,7 @@ void TransactionExecution::ProcessTxRequest( LocalCcShards *local_shards = Sharder::Instance().GetLocalCcShards(); std::unique_ptr split_range_op = nullptr; - std::unique_lock lk( - local_shards->split_flush_range_op_pool_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); if (local_shards->split_flush_range_op_pool_.empty()) { split_range_op = std::make_unique( @@ -1712,8 +1732,8 @@ void TransactionExecution::ProcessTxRequest( LOG(INFO) << "Recovering split flush tx " << TxNumber() << " on table " << table_name.StringView() << ", range id " << recover_req.range_entry_->GetRangeInfo()->PartitionId(); - split_flush_op_ = std::move(split_range_op); - PushOperation(split_flush_op_.get()); + CompositeOpSlot() = std::move(split_range_op); + PushOperation(CompositeOpSlot().get()); } void TransactionExecution::Process(InitTxnOperation &init_txn) @@ -2305,7 +2325,7 @@ void TransactionExecution::Process(ReadLocalOperation &lock_local) *lock_local.hd_result_, IsolationLevel::RepeatableRead, CcProtocol::Locking, - false, + lock_local.is_for_write_, false, lock_local.execute_immediately_); if (finished) @@ -4519,20 +4539,18 @@ bool TransactionExecution::FillDataLogRequest(WriteToLogOp &write_log) wset_entry.forward_addr_) { // If the wset entry needs to be double written into different - // ngs, write log for both ngs. + // ngs, write log for both ngs: every such key of the tx goes + // into the other ng's blob, which is all that ng's recovery + // has of it until that ng checkpoints it (a range split moving + // the key to that ng). uint32_t forward_ng_id = Sharder::Instance().ShardToCcNodeGroup(forward_shard_code); - auto [table_rec_it, inserted] = - ng_table_set.try_emplace(forward_ng_id); - if (!inserted) - { - continue; - } + auto table_rec_it = ng_table_set.try_emplace(forward_ng_id); std::unordered_map< TableName, std::vector< std::pair>> - &table_rec_set = table_rec_it->second.second; + &table_rec_set = table_rec_it.first->second.second; auto rec_vec_it = table_rec_set.emplace( std::piecewise_construct, @@ -4540,7 +4558,18 @@ bool TransactionExecution::FillDataLogRequest(WriteToLogOp &write_log) table_name.Type(), table_name.Engine()), std::forward_as_tuple()); - rec_vec_it.first->second.emplace_back(&write_key, &wset_entry); + std::vector> + &rec_vec = rec_vec_it.first->second; + // A key double written to several cores of one ng is logged + // there once. The key's own ng and its forward ngs all get it + // in this pass alone, so an ng that has it already has it + // last: its own ng, or a forward ng an earlier forward + // address of the key named. + if (!rec_vec.empty() && rec_vec.back().first == &write_key) + { + continue; + } + rec_vec.emplace_back(&write_key, &wset_entry); } } } @@ -5704,9 +5733,14 @@ void TransactionExecution::ReleaseMetaDataReadLock( void TransactionExecution::ReleaseCatalogWriteAll( CcHandlerResult &catalog_post_all_hd_result) { - uint64_t commit_ts = TxStatus() == TxnStatus::Committed - ? commit_ts_ - : TransactionOperation::tx_op_failed_ts_; + // A committed tx promotes the dirty schema its FlushUpdateTableOp + // installed; an aborted one prepared nothing on the catalog and rolls back + // at timestamp 0, only releasing its locks. + const bool committed = TxStatus() == TxnStatus::Committed; + const uint64_t commit_ts = + committed ? commit_ts_ : TransactionOperation::tx_op_failed_ts_; + const PostWriteType write_type = + committed ? PostWriteType::PostCommit : PostWriteType::Rollback; std::shared_ptr> all_node_groups = Sharder::Instance().AllNodeGroups(); @@ -5733,7 +5767,7 @@ void TransactionExecution::ReleaseCatalogWriteAll( commit_ts, catalog_post_all_hd_result, OperationType::Update, - PostWriteType::PostCommit, + write_type, abort_if_oom); } } @@ -5750,7 +5784,7 @@ void TransactionExecution::ReleaseCatalogWriteAll( commit_ts, catalog_post_all_hd_result, OperationType::Update, - PostWriteType::PostCommit, + write_type, abort_if_oom); } @@ -7908,42 +7942,44 @@ void TransactionExecution::RecoverDataMigration( { LocalCcShards *local_shards = Sharder::Instance().GetLocalCcShards(); - std::lock_guard lk(local_shards->data_migration_op_pool_mux_); + std::lock_guard lk(local_shards->op_pool_mux_); if (local_shards->migration_op_pool_.empty()) { - migration_op_ = std::make_unique(this, status); + CompositeOpSlot() = + std::make_unique(this, status); } else { - migration_op_ = std::move(local_shards->migration_op_pool_.back()); + CompositeOpSlot() = + std::move(local_shards->migration_op_pool_.back()); local_shards->migration_op_pool_.pop_back(); - migration_op_->Reset(this, status); + CompositeOpSlot()->Reset(this, status); } + DataMigrationOp *migration_op = CompositeOpSlot().get(); if (migrate_msg) { - migration_op_->migrate_bucket_idx_ = cur_idx; - migration_op_->PrepareNextRoundBuckets(); + migration_op->migrate_bucket_idx_ = cur_idx; + migration_op->PrepareNextRoundBuckets(); switch (migrate_msg->stage()) { case ::txlog::BucketMigrateStage::BeforeLocking: { - migration_op_->op_ = &migration_op_->write_before_locking_log_op_; - migration_op_->write_before_locking_log_op_.hd_result_ - .SetFinished(); + migration_op->op_ = &migration_op->write_before_locking_log_op_; + migration_op->write_before_locking_log_op_.hd_result_.SetFinished(); break; } case ::txlog::BucketMigrateStage::PrepareMigrate: { - migration_op_->op_ = &migration_op_->prepare_log_op_; - migration_op_->prepare_log_op_.hd_result_.SetFinished(); + migration_op->op_ = &migration_op->prepare_log_op_; + migration_op->prepare_log_op_.hd_result_.SetFinished(); break; } case ::txlog::BucketMigrateStage::CommitMigrate: { - migration_op_->op_ = &migration_op_->commit_log_op_; - migration_op_->commit_log_op_.hd_result_.SetFinished(); + migration_op->op_ = &migration_op->commit_log_op_; + migration_op->commit_log_op_.hd_result_.SetFinished(); break; } default: @@ -7953,7 +7989,7 @@ void TransactionExecution::RecoverDataMigration( } } - PushOperation(migration_op_.get()); + PushOperation(migration_op); } void TransactionExecution::RecoverClusterScale( @@ -7993,7 +8029,7 @@ void TransactionExecution::RecoverClusterScale( } LocalCcShards *local_shards = Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk(local_shards->cluster_scale_op_mux_); + std::unique_lock lk(local_shards->op_pool_mux_); ClusterScaleOpType op_type = ClusterScaleOpType::AddNodeGroup; switch (scale_op_msg.event_type()) { @@ -8011,19 +8047,19 @@ void TransactionExecution::RecoverClusterScale( } if (local_shards->cluster_scale_op_pool_.empty()) { - cluster_scale_op_ = std::make_unique( + CompositeOpSlot() = std::make_unique( scale_op_msg.id(), op_type, std::move(new_ng_configs), this); } else { - cluster_scale_op_ = + CompositeOpSlot() = std::move(local_shards->cluster_scale_op_pool_.back()); local_shards->cluster_scale_op_pool_.pop_back(); - cluster_scale_op_->Reset( + CompositeOpSlot()->Reset( scale_op_msg.id(), op_type, std::move(new_ng_configs), this); } - ClusterScaleOp *op = cluster_scale_op_.get(); + ClusterScaleOp *op = CompositeOpSlot().get(); if (op_type == ClusterScaleOpType::AddNodeGroup) { diff --git a/tx_service/src/tx_index_operation.cpp b/tx_service/src/tx_index_operation.cpp index f95f8be75..0af3a01f2 100644 --- a/tx_service/src/tx_index_operation.cpp +++ b/tx_service/src/tx_index_operation.cpp @@ -58,14 +58,16 @@ UpsertTableIndexOp::UpsertTableIndexOp( current_image, dirty_image, curr_schema_ts, - op_type), + op_type, + txm), lock_cluster_config_op_(), acquire_all_intent_op_(txm), upgrade_all_intent_to_lock_op_(txm), + create_kv_table_op_(txm), + drop_kv_table_op_(txm), prepare_log_op_(txm), downgrade_all_lock_to_intent_op_(txm), unlock_cluster_config_op_(txm), - kv_create_index_op_(&table_key_.Name(), OperationType::AddIndex, txm), update_sequence_table_op_(txm), generate_sk_parallel_op_(txm), flush_all_old_tuples_sk_op_(txm), @@ -134,6 +136,8 @@ UpsertTableIndexOp::UpsertTableIndexOp( table_engine); is_force_finished_ = false; + gated_ = false; + rolled_back_ = false; new_indexes_name_.reserve(alter_table_info_.index_add_count_); for (auto index_it = alter_table_info_.index_add_names_.cbegin(); @@ -167,11 +171,12 @@ UpsertTableIndexOp::UpsertTableIndexOp( TX_TRACE_ASSOCIATE(this, &upgrade_all_intent_to_lock_op_, "upgrade_all_intent_to_lock_op_"); + TX_TRACE_ASSOCIATE(this, &create_kv_table_op_, "create_kv_table_op_"); + TX_TRACE_ASSOCIATE(this, &drop_kv_table_op_, "drop_kv_table_op_"); TX_TRACE_ASSOCIATE(this, &prepare_log_op_, "prepare_log_op_"); TX_TRACE_ASSOCIATE(this, &downgrade_all_lock_to_intent_op_, "downgrade_all_lock_to_intent_op_"); - TX_TRACE_ASSOCIATE(this, &kv_create_index_op_, "kv_create_index_op_"); TX_TRACE_ASSOCIATE( this, &update_sequence_table_op_, "update_sequence_table_op_"); TX_TRACE_ASSOCIATE( @@ -186,6 +191,7 @@ UpsertTableIndexOp::UpsertTableIndexOp( this, &kv_rollback_create_index_op_, "kv_rollback_create_index_op_"); TX_TRACE_ASSOCIATE( this, &kv_update_schema_image_op_, "kv_update_schema_image_op_"); + TX_TRACE_ASSOCIATE(this, &kv_retry_wait_op_, "kv_retry_wait_op_"); TX_TRACE_ASSOCIATE(this, &clean_ccm_op_, "clean_ccm_op_"); TX_TRACE_ASSOCIATE(this, &post_all_lock_op_, "post_all_lock_op_"); TX_TRACE_ASSOCIATE(this, &clean_log_op_, "clean_log_op_"); @@ -207,37 +213,36 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) { LOG(ERROR) << "Alter Table Index read cluster config failed, txn:" << txm->TxNumber(); - if (!prepare_log_op_.hd_result_.IsFinished() && - !prepare_data_log_op_.hd_result_.IsFinished()) - { - txm->commit_ts_ = tx_op_failed_ts_; - } + MarkFailed(txm); ForceToFinish(txm); return; } - if (prepare_log_op_.hd_result_.IsFinished() || - prepare_data_log_op_.hd_result_.IsFinished()) - { - assert(op_type_ == OperationType::AddIndex); - LOG(INFO) << "Alter Table Index transaction post acquire all" - << " write lock, txn: " << txm->TxNumber(); - op_ = &acquire_all_lock_op_; - txm->PushOperation(&acquire_all_lock_op_); - txm->Process(acquire_all_lock_op_); - } - else - { - // Acquire write intent, and then upgrade to write lock(Locking), - // rather than acquire write lock(OCC) directly, aim to avoid two - // situations: (1) concurrent DDL deadlock. (2) concurrent DML cause - // to always abort this tx. - LOG(INFO) - << "Alter Table Index transaction prepare acquire all write" - << " intent, txn: " << txm->TxNumber(); - op_ = &acquire_all_intent_op_; - txm->PushOperation(&acquire_all_intent_op_); - txm->Process(acquire_all_intent_op_); + // The cluster gate admits new ops only here, where an op takes the + // cluster config read lock (EloqDoc docs/cdc-change-stream.md §6.2). + // An op past this step, or a recovered one, which enters after it, is + // never gated: those are what a pause drains. Nothing but the lock is + // held yet, so the op releases it and ends without a log to clean. + if (cluster_conf_rec_.GatesSchemaOps()) + { + LOG(WARNING) << "Alter Table Index on " << table_key_.Name().Trace() + << " refused: the cluster gate is paused, txn: " + << txm->TxNumber(); + txm->upsert_resp_->SetErrorCode(TxErrorCode::CLUSTER_GATED); + MarkFailed(txm); + gated_ = true; + op_ = &unlock_cluster_config_op_; + StartUnlockClusterConfig(txm, unlock_cluster_config_op_); + return; } + // Acquire write intent, and then upgrade to write lock(Locking), + // rather than acquire write lock(OCC) directly, aim to avoid two + // situations: (1) concurrent DDL deadlock. (2) concurrent DML cause + // to always abort this tx. + LOG(INFO) << "Alter Table Index transaction prepare acquire all write" + << " intent, txn: " << txm->TxNumber(); + op_ = &acquire_all_intent_op_; + txm->PushOperation(&acquire_all_intent_op_); + txm->Process(acquire_all_intent_op_); } else if (op_ == &acquire_all_intent_op_) { @@ -256,7 +261,7 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) // the schema operation. Set the commit ts to 0 to signal that // the following post write operation releases all write // intents. - txm->commit_ts_ = tx_op_failed_ts_; + MarkFailed(txm); // Moves to the last operation that removes all write // intents/locks. op_ = &post_all_lock_op_; @@ -285,7 +290,7 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) << txm->TxNumber(); // Set the commit ts to 0 to signal that the following post write // operation releases all write intents. - txm->commit_ts_ = tx_op_failed_ts_; + MarkFailed(txm); // Moves to the last operation that removes all write // intents/locks. op_ = &post_all_lock_op_; @@ -310,14 +315,71 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) upgrade_all_res.last_vali_ts_ + 1); txm->commit_ts_ = std::max(txm->commit_ts_, ts); } + ResolveSchemaImages(txm->commit_ts_); + + if (op_type_ == OperationType::AddIndex) + { + // Recovery returns to the state right after the prepare log, with + // the dirty schema installed, so the index's kv table must exist + // before the prepare log. + CreateKvTables(txm); + } + else + { + WritePrepareLog(txm); + } + } + else if (op_ == &create_kv_table_op_) + { + if (!txm->CheckLeaderTerm()) + { + // The locks went with the leader. Whatever was created stays + // orphaned, which is accepted (drop_kv_table_op_). + MarkFailed(txm); + ForceToFinish(txm); + } + else if (create_kv_table_op_.hd_result_.IsError()) + { + LOG(ERROR) << "Alter Table Index for table: " + << table_key_.Name().StringView() + << ", failed to create the index kv tables, txn: " + << txm->TxNumber(); + // No log exists yet: the op fails, and the locks are released + // once whatever was created is dropped again. + MarkFailed(txm); + txm->upsert_resp_->SetErrorCode(TxErrorCode::DATA_STORE_ERROR); + DropKvTables(txm); + } + else + { + WritePrepareLog(txm); + } + } + else if (op_ == &drop_kv_table_op_) + { + if (drop_kv_table_op_.hd_result_.IsError()) + { + // Accepted: the kv tables stay orphaned. No schema refers to them. + LOG(WARNING) << "Alter Table Index for table: " + << table_key_.Name().StringView() + << ", failed to drop the kv tables of a failed add " + "index, which stay orphaned, txn: " + << txm->TxNumber(); + } - LOG(INFO) << "Alter Table Index transaction write prepare log, txn: " - << txm->TxNumber() - << ". The schema version: " << txm->commit_ts_; - op_ = &prepare_log_op_; - FillPrepareLogRequest(txm); - txm->PushOperation(&prepare_log_op_); - txm->Process(prepare_log_op_); + if (txm->CheckLeaderTerm()) + { + // commit_ts_ is tx_op_failed_ts_: the post write only removes the + // write intents/locks. + assert(txm->commit_ts_ == tx_op_failed_ts_); + op_ = &post_all_lock_op_; + txm->PushOperation(&post_all_lock_op_); + txm->Process(post_all_lock_op_); + } + else + { + ForceToFinish(txm); + } } else if (op_ == &prepare_log_op_) { @@ -345,7 +407,7 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) } else { - txm->commit_ts_ = tx_op_failed_ts_; + MarkFailed(txm); ForceToFinish(txm); } } @@ -360,16 +422,24 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) // considered failed if the prepare log is not flushed. The // commit ts is set to 0 to signal that the following post write // operation releases all write intents. - txm->commit_ts_ = tx_op_failed_ts_; - // Moves to the last operation that removes all write - // intents/locks. - op_ = &post_all_lock_op_; - + MarkFailed(txm); txm->upsert_resp_->SetErrorCode( TxErrorCode::UPSERT_TABLE_PREPARE_FAIL); - txm->PushOperation(&post_all_lock_op_); - txm->Process(post_all_lock_op_); + if (op_type_ == OperationType::AddIndex) + { + // Drop the kv tables created before the prepare log, then + // remove the write intents/locks. + DropKvTables(txm); + } + else + { + // Moves to the last operation that removes all write + // intents/locks. + op_ = &post_all_lock_op_; + txm->PushOperation(&post_all_lock_op_); + txm->Process(post_all_lock_op_); + } } } else @@ -430,150 +500,13 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) txm->Process(acquire_all_lock_op_); } else - { - // Release cluster config op before doing data store op. - op_ = &unlock_cluster_config_op_; - // Get cce addr of cluster config read lock from rset. - const CcEntryAddr *cluster_config_addr = nullptr; - const ReadSetEntry *cluster_config_entry = nullptr; - const auto &meta_rset = txm->rw_set_.MetaDataReadSet(); - for (const auto &[cce_addr, read_entry_pair] : meta_rset) - { - if (read_entry_pair.second == cluster_config_ccm_name_sv) - { - cluster_config_addr = &cce_addr; - cluster_config_entry = &read_entry_pair.first; - break; - } - } - assert(cluster_config_addr && cluster_config_entry); - unlock_cluster_config_op_.Reset(cluster_config_addr, - cluster_config_entry); - LOG(INFO) << "Alter Table Index transaction release cluster config " - "lock, txn: " - << txm->TxNumber(); - txm->PushOperation(&unlock_cluster_config_op_); - txm->Process(unlock_cluster_config_op_); - } - } - else if (op_ == &unlock_cluster_config_op_) - { - if (unlock_cluster_config_op_.hd_result_.IsError()) - { - if (txm->CheckLeaderTerm()) - { - const CcEntryAddr *cluster_config_addr = nullptr; - const auto &meta_rset = txm->rw_set_.MetaDataReadSet(); - for (const auto &[cce_addr, read_entry_pair] : meta_rset) - { - if (read_entry_pair.second == cluster_config_ccm_name_sv) - { - cluster_config_addr = &cce_addr; - break; - } - } - - if (cluster_config_addr) - { - DLOG(INFO) - << "Alter Table Index transaction unlock cluster " - "config " - "lock failed, txn: " - << txm->TxNumber() << ", error code: " - << (int) - unlock_cluster_config_op_.hd_result_.ErrorCode() - << ", error message: " - << unlock_cluster_config_op_.hd_result_.ErrorMsg() - << ", cluster config addr term" - << cluster_config_addr->Term() - << ", cluster config addr node group id: " - << cluster_config_addr->NodeGroupId() - << ", node group term: " - << Sharder::Instance().LeaderTerm( - cluster_config_addr->NodeGroupId()); - } - // Releasing a local read lock should never fail - assert(false); - } - else - { - ForceToFinish(txm); - return; - } - } - else { assert(op_type_ == OperationType::AddIndex); - LOG(INFO) << "Alter Table Index transaction upsert data store" - << " info for base table: " - << table_key_.Name().StringView() - << ", txn: " << txm->TxNumber(); - uint16_t retry_times = 100; - CODE_FAULT_INJECTOR("trigger_flush_kv_error", { retry_times = 3; }); - op_ = &kv_create_index_op_; - // The post write request right after flushing the prepare log - // installs the dirty schema in the tx service and returns a - // local view (pointer) of the committed and dirty schema. - kv_create_index_op_.table_schema_old_ = catalog_rec_.Schema(); - kv_create_index_op_.table_schema_ = catalog_rec_.DirtySchema(); - kv_create_index_op_.alter_table_info_ = &alter_table_info_; - kv_create_index_op_.write_time_ = txm->commit_ts_; - txm->PushOperation(&kv_create_index_op_, retry_times); - txm->Process(kv_create_index_op_); - } - } - else if (op_ == &kv_create_index_op_) - { - if (kv_create_index_op_.hd_result_.IsError()) - { - // The data store operation failed. Retries the operation if the - // tx node is the leader or the tx is in the recovery mode and - // the cc node is a leader candidate. - if (txm->CheckLeaderTerm()) - { - // NOTE: The logic of this part is consistent with the logic in - // UpsertTableOp::Forward. - LOG(ERROR) << "Alter Table Index for table: " - << table_key_.Name().StringView() - << ", Failed to create tables in kv store" - << ". Txn: " << txm->TxNumber(); - // Set txm->commit_ts_ to 0 to indicate there is a flush - // error during upsert_kv_table_op_. - txm->commit_ts_ = tx_op_failed_ts_; - - /* - After upsert kv fails, we need to flush a commit log to - indicate this error. - If we skip this commit log and jump to post_all_lock_op_ - directly, once the participant crashes at the point between - it releases write intent and the coordinator flushes - clean_log, then during recovery, the participant sees a - prepare_log(whose commit_ts is not 0) and recovers write - lock and dirty_catalog. - Since the coordinator has finished its job, the write - lock recovered by participant becomes orphan lock, and the - dirty catalog can not be rejected either. - Also, in the current design, post_all_intent_op_ does not - release the write intent, which means the write intent is - still being held after upsert_kv_table_op_(during - CreateTable or AddIndex). If create table or add index in kv - fails, the only thing we should do after writing commit_log - is to reject dirty schema. So there is no need to upgrade - write intent to write lock, and it is safe to skip - acquire_all_lock_op_ and jump directly to commit_log_op_. - */ - op_ = &commit_log_op_; - FillCommitLogRequest(txm); - txm->PushOperation(&commit_log_op_); - txm->Process(commit_log_op_); - } - else - { - ForceToFinish(txm); - } - } - else - { + // The index's kv table exists since before the prepare log + // (create_kv_table_op_), and the kv catalog gets the new schema + // after the commit log (kv_update_schema_image_op_). The range-id + // sequence is seeded before the build, whose uploads can split the + // index's ranges. LOG(INFO) << "Alter Table Index transaction init range id " << "for table: " << table_key_.Name().StringView() << ", txn: " << txm->TxNumber(); @@ -625,26 +558,11 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) { if (update_sequence_table_op_.hd_result_.IsError()) { - if (txm->CheckLeaderTerm()) - { - DLOG(INFO) << "Alter Table Index transaction retry update " - "sequence table, base table: " - << table_key_.Name().StringView() - << ", txn: " << txm->TxNumber(); - txm->PushOperation(&update_sequence_table_op_); - txm->Process(update_sequence_table_op_); - } - else - { - DLOG(INFO) << "Alter Table Index transaction ForceToFinish " - "update sequence table, base table: " - << table_key_.Name().StringView() - << ", txn: " << txm->TxNumber(); - ForceToFinish(txm); - } + // After the prepare log (add index) or the commit log (drop index, + // rolled-back add index): retried until it succeeds. + RetryKvStep(txm, &update_sequence_table_op_); } - else if ((op_type_ == OperationType::AddIndex && - txm->CommitTs() != tx_op_failed_ts_)) + else if (op_type_ == OperationType::AddIndex && !rolled_back_) { if (txm->TxStatus() == TxnStatus::Recovering && Sharder::Instance().CandidateLeaderTerm(txm->TxCcNodeId()) > 0) @@ -691,8 +609,7 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) txm->Process(generate_sk_parallel_op_); } else if (op_type_ == OperationType::DropIndex || - (op_type_ == OperationType::AddIndex && - txm->CommitTs() == tx_op_failed_ts_)) + (op_type_ == OperationType::AddIndex && rolled_back_)) { LOG(INFO) << "Alter Table Index transaction clean cc map, txn: " << txm->TxNumber(); @@ -720,8 +637,7 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) else if (op_ == &clean_ccm_op_) { assert(op_type_ == OperationType::DropIndex || - (op_type_ == OperationType::AddIndex && - txm->CommitTs() == tx_op_failed_ts_)); + (op_type_ == OperationType::AddIndex && rolled_back_)); if (clean_ccm_op_.hd_result_.IsError()) { @@ -783,14 +699,14 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) // If creating index violates constraints, then the failure // is unrecoverable, and the tx should do rollback. // - // Besides generate_sk_parallel_op_ set commit_ts to - // tx_op_failed_ts_ when constrains violation, - // kv_create_index_op_ also set commit_ts to - // tx_op_failed_ts_ when flush kv failed. As a result, - // `op_type == Operation::AddIndex && txm->commit_ts_ == - // tx_op_failed_ts_` doesn't means abort by constrains - // violation. - txm->commit_ts_ = tx_op_failed_ts_; + // The only rollback after the prepare log: a failure to + // create the kv tables happens before it and writes no log + // (create_kv_table_op_). The rollback still commits, at a + // timestamp chosen under the commit-stage lock + // (acquire_all_lock_op_), with the mark in its commit log + // and its PostCommit. + rolled_back_ = true; + post_all_lock_op_.write_type_ = PostWriteType::Rollback; txm->upsert_resp_->SetErrorCode( TransactionExecution::ConvertCcError(err_code)); @@ -971,17 +887,8 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) } else { - LOG(WARNING) - << "Alter table index flush all old sk tuples failed " - << "for table: " << table_key_.Name().StringView() - << " with error message: " - << flush_all_old_tuples_sk_op_.hd_result_.ErrorMsg() - << ". Retry flush old sk operation, txn: " - << txm->TxNumber(); - - op_ = &flush_all_old_tuples_sk_op_; - txm->PushOperation(&flush_all_old_tuples_sk_op_); - txm->Process(flush_all_old_tuples_sk_op_); + // After the prepare log: retried until it succeeds. + RetryKvStep(txm, &flush_all_old_tuples_sk_op_); } } else @@ -1055,12 +962,16 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) if (!is_last_finished_key_str_ && (last_finished_end_key_.Type() == KeyType::PositiveInf)) { - // Reach the end. - LOG(INFO) << "Alter Table Index transaction lock cluster config" - << ", txn: " << txm->TxNumber(); - op_ = &lock_cluster_config_op_; - txm->PushOperation(&lock_cluster_config_op_); - txm->Process(lock_cluster_config_op_); + // Reach the end: the commit-stage write locks. + LOG(INFO) << "Alter Table Index transaction post acquire all" + << " write lock, txn: " << txm->TxNumber(); + CODE_FAULT_INJECTOR("alter_table_index_hold_before_commit_lock", { + HoldBeforeCommitLock(txm); + return; + }); + op_ = &acquire_all_lock_op_; + txm->PushOperation(&acquire_all_lock_op_); + txm->Process(acquire_all_lock_op_); } else { @@ -1116,12 +1027,16 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) if (!is_last_finished_key_str_ && (last_finished_end_key_.Type() == KeyType::PositiveInf)) { - // Reach the end. - LOG(INFO) << "Alter Table Index transaction lock cluster config" - << ", txn: " << txm->TxNumber(); - op_ = &lock_cluster_config_op_; - txm->PushOperation(&lock_cluster_config_op_); - txm->Process(lock_cluster_config_op_); + // Reach the end: the commit-stage write locks. + LOG(INFO) << "Alter Table Index transaction post acquire all" + << " write lock, txn: " << txm->TxNumber(); + CODE_FAULT_INJECTOR("alter_table_index_hold_before_commit_lock", { + HoldBeforeCommitLock(txm); + return; + }); + op_ = &acquire_all_lock_op_; + txm->PushOperation(&acquire_all_lock_op_); + txm->Process(acquire_all_lock_op_); } else { @@ -1198,6 +1113,23 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) } else { + if (rolled_back_) + { + // The rollback commits above the index's own timestamp D, + // where the index appeared, and above every DML the lock + // drained: P = max(D + 1, last_vali_ts + 1, commit_ts + 1). + txm->commit_ts_ += 1; + for (size_t idx = 0; idx < acquire_all_lock_op_.upload_cnt_; + ++idx) + { + const AcquireAllResult &acq_all_res = + acquire_all_lock_op_.hd_results_[idx].Value(); + txm->commit_ts_ = + std::max(txm->commit_ts_, + std::max(acq_all_res.last_vali_ts_ + 1, + acq_all_res.commit_ts_ + 1)); + } + } LOG(INFO) << "Alter Table Index transaction write commit log" << ", txn: " << txm->TxNumber(); op_ = &commit_log_op_; @@ -1238,120 +1170,74 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) kv_drop_index_op_.table_schema_ = catalog_rec_.DirtySchema(); kv_drop_index_op_.alter_table_info_ = &alter_table_info_; kv_drop_index_op_.write_time_ = txm->commit_ts_; - txm->PushOperation(&kv_drop_index_op_); - txm->Process(kv_drop_index_op_); - } - else if (op_type_ == OperationType::AddIndex && - txm->CommitTs() == tx_op_failed_ts_) - { - if (txm->TxStatus() == TxnStatus::Recovering || - generate_sk_parallel_op_.hd_result_.IsError()) - { - // Rollback due to generate_sk_parallel_op_ break violations. - LOG(WARNING) - << "Rollback Index transaction, txm: " << txm->TxNumber(); - - op_ = &kv_rollback_create_index_op_; - kv_rollback_create_index_op_.table_schema_old_ = - catalog_rec_.Schema(); - kv_rollback_create_index_op_.table_schema_ = - catalog_rec_.DirtySchema(); - kv_rollback_create_index_op_.alter_table_info_ = - &alter_table_info_; - - // Undo the CREATE INDEX operation with a DROP INDEX op. - assert(kv_rollback_create_index_op_.op_type_ == - OperationType::DropIndex); - std::swap(kv_rollback_create_index_op_.table_schema_old_, - kv_rollback_create_index_op_.table_schema_); - std::swap(kv_rollback_create_index_op_.alter_table_info_ - ->index_add_names_, - kv_rollback_create_index_op_.alter_table_info_ - ->index_drop_names_); - std::swap(kv_rollback_create_index_op_.alter_table_info_ - ->index_add_count_, - kv_rollback_create_index_op_.alter_table_info_ - ->index_drop_count_); - if (txm->TxStatus() != TxnStatus::Recovering) - { - kv_rollback_create_index_op_.write_time_ = - std::max(kv_create_index_op_.write_time_, - txm->commit_ts_bound_) + - 1; - } - else - { - kv_rollback_create_index_op_.write_time_ = - static_cast(txm->cc_handler_) - ->GetTsBaseValue(); - } - txm->PushOperation(&kv_rollback_create_index_op_); - txm->Process(kv_rollback_create_index_op_); - } - else - { - ACTION_FAULT_INJECTOR("term_AlterTableIndex_PostCommitAllWLOp"); - LOG(INFO) << "Alter Table Index transaction commit dirty table" - << " schema, txn: " << txm->TxNumber(); - - assert(kv_create_index_op_.hd_result_.IsError()); - assert(post_all_lock_op_.write_type_ == - PostWriteType::PostCommit); - op_ = &post_all_lock_op_; - txm->PushOperation(&post_all_lock_op_); - txm->Process(post_all_lock_op_); - } - } - else if (op_type_ == OperationType::AddIndex && txm->CommitTs() > 0 && - AnyMultiKeyIndex(indexes_multikey_attr_)) - { + StartKvStep(txm, kv_drop_index_op_); + } + else if (op_type_ == OperationType::AddIndex && rolled_back_) + { + // Crashes the node with the rollback's commit log written and its + // data store step not yet done, which recovery must roll forward. + ACTION_FAULT_INJECTOR("alter_table_index_crash_after_commit_log"); + // An add index is rolled back only when the build broke a + // constraint (generate_sk_parallel_op_), after the dirty schema was + // installed. + assert(txm->TxStatus() == TxnStatus::Recovering || + generate_sk_parallel_op_.hd_result_.IsError()); + LOG(WARNING) << "Rollback Index transaction, txm: " + << txm->TxNumber(); + + op_ = &kv_rollback_create_index_op_; + kv_rollback_create_index_op_.table_schema_old_ = + catalog_rec_.Schema(); + kv_rollback_create_index_op_.table_schema_ = + catalog_rec_.DirtySchema(); + kv_rollback_create_index_op_.alter_table_info_ = &alter_table_info_; + + // Undo the CREATE INDEX operation with a DROP INDEX op. + assert(kv_rollback_create_index_op_.op_type_ == + OperationType::DropIndex); + std::swap(kv_rollback_create_index_op_.table_schema_old_, + kv_rollback_create_index_op_.table_schema_); + std::swap(kv_rollback_create_index_op_.alter_table_info_ + ->index_add_names_, + kv_rollback_create_index_op_.alter_table_info_ + ->index_drop_names_); + std::swap(kv_rollback_create_index_op_.alter_table_info_ + ->index_add_count_, + kv_rollback_create_index_op_.alter_table_info_ + ->index_drop_count_); + // The catalog row keeps the current schema, at the rollback's + // commit timestamp, which PostCommit installs in memory too. A + // recovered op has the same timestamp from the commit log. + kv_rollback_create_index_op_.write_time_ = txm->commit_ts_; + StartKvStep(txm, kv_rollback_create_index_op_); + } + else if (op_type_ == OperationType::AddIndex) + { + // The op's only kv catalog write: the new schema, with the + // indexes' multikey attributes, at the op's timestamp, so the kv + // catalog version matches the in-memory one. Until now the + // prepare log carried the dirty schema for any node that recovers. + // It stays under the commit-stage write lock, so that no later DDL + // on the table writes its catalog row first: the DSS RocksDB + // backends apply writes in arrival order, not by version, and a + // later drop table deletes the row, leaving nothing a late write + // could be compared against. + assert(txm->CommitTs() > 0); op_ = &kv_update_schema_image_op_; kv_update_schema_image_op_.table_schema_old_ = catalog_rec_.Schema(); kv_update_schema_image_op_.table_schema_ = catalog_rec_.DirtySchema(); - if (txm->TxStatus() != TxnStatus::Recovering) - { - kv_update_schema_image_op_.write_time_ = - std::max({kv_update_schema_image_op_.write_time_, - kv_create_index_op_.write_time_, - txm->commit_ts_bound_}) + - 1; - } - else - { - kv_update_schema_image_op_.write_time_ = - static_cast(txm->cc_handler_) - ->GetTsBaseValue(); - } - txm->PushOperation(&kv_update_schema_image_op_); - txm->Process(kv_update_schema_image_op_); - } - else - { - ACTION_FAULT_INJECTOR("term_AlterTableIndex_PostCommitAllWLOp"); - LOG(INFO) << "Alter Table Index transaction commit dirty table" - << " schema, txn: " << txm->TxNumber(); - - assert(post_all_lock_op_.write_type_ == PostWriteType::PostCommit); - op_ = &post_all_lock_op_; - txm->PushOperation(&post_all_lock_op_); - txm->Process(post_all_lock_op_); + kv_update_schema_image_op_.write_time_ = txm->commit_ts_; + StartKvStep(txm, kv_update_schema_image_op_); } } else if (op_ == &kv_drop_index_op_) { if (kv_drop_index_op_.hd_result_.IsError()) { - if (txm->CheckLeaderTerm()) - { - txm->PushOperation(&kv_drop_index_op_); - txm->Process(kv_drop_index_op_); - } - else - { - ForceToFinish(txm); - } + // After the commit log: retried until it succeeds. + RetryKvStep(txm, &kv_drop_index_op_); } else { @@ -1400,15 +1286,8 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) { if (kv_rollback_create_index_op_.hd_result_.IsError()) { - if (txm->CheckLeaderTerm()) - { - txm->PushOperation(&kv_rollback_create_index_op_); - txm->Process(kv_rollback_create_index_op_); - } - else - { - ForceToFinish(txm); - } + // After the commit log: retried until it succeeds. + RetryKvStep(txm, &kv_rollback_create_index_op_); } else { @@ -1455,20 +1334,24 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) } else if (op_ == &kv_update_schema_image_op_) { - if (kv_update_schema_image_op_.hd_result_.IsError()) + bool write_failed = kv_update_schema_image_op_.hd_result_.IsError(); + // Fails one write, then disarms itself, so that a retry must write. + CODE_FAULT_INJECTOR("alter_table_index_update_schema_image_fail", { + LOG(INFO) << "FaultInject " + "alter_table_index_update_schema_image_fail, txn: " + << txm->TxNumber(); + FaultInject::Instance().InjectFault( + "alter_table_index_update_schema_image_fail", "remove"); + write_failed = true; + }); + if (write_failed) { - if (txm->CheckLeaderTerm()) - { - txm->PushOperation(&kv_update_schema_image_op_); - txm->Process(kv_update_schema_image_op_); - } - else - { - ForceToFinish(txm); - } + // After the commit log: retried until it succeeds. + RetryKvStep(txm, &kv_update_schema_image_op_); } else { + ACTION_FAULT_INJECTOR("term_AlterTableIndex_PostCommitAllWLOp"); LOG(INFO) << "Alter Table Index transaction commit dirty table " "schema, txn: " << txm->TxNumber(); @@ -1478,6 +1361,17 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) txm->Process(post_all_lock_op_); } } + else if (op_ == &kv_retry_wait_op_) + { + // The pause is over: run the failed step again. + if (!txm->CheckLeaderTerm()) + { + ForceToFinish(txm); + return; + } + op_ = kv_retry_step_; + RunKvStepAgain(txm); + } else if (op_ == &post_all_lock_op_) { if (!txm->CheckLeaderTerm()) @@ -1490,17 +1384,11 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) // The schema operation failed without flushing the prepare log. // Do not retry post-processing (release write intents) even if // it fails. Remaining write intents on the schema, if there are - // any, will be recovered by individual cc nodes separately. - txm->upsert_resp_->Finish(UpsertResult::Failed); - - txm->state_stack_.pop_back(); - assert(txm->state_stack_.empty()); - LocalCcShards *local_cc_shards = - Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk( - local_cc_shards->table_index_op_pool_mux_); - local_cc_shards->table_index_op_pool_.emplace_back( - std::move(txm->index_op_)); + // any, will be recovered by individual cc nodes separately. The + // cluster config lock is released next, then the op ends without + // a clean log. + op_ = &unlock_cluster_config_op_; + StartUnlockClusterConfig(txm, unlock_cluster_config_op_); } else if (post_all_lock_op_.hd_result_.IsError()) { @@ -1553,14 +1441,81 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) { assert(post_all_lock_op_.write_type_ != PostWriteType::DowngradeLock); - // Flush kv failed, or it is recovering from a flush kv failure. - // This schema op has already been rolled back by now, only need - // to flush clean log here. - // Also, flush kv failure does not require lock upgrade(write - // intent to write lock). So the CcEntryAddr needs to be kept in - // rset in order to release the read lock when committing. + // The op failed before its prepare log (the kv tables or the + // prepare log itself) and took no write lock, so the + // CcEntryAddr stays in the read set, whose read lock the tx + // releases when it commits. Only the release of the cluster + // config lock and the clean log are left. A rolled-back op + // commits at a real timestamp under the write lock and takes + // the branch above. } + // The op's last post write all: the cluster config lock can go. + op_ = &unlock_cluster_config_op_; + StartUnlockClusterConfig(txm, unlock_cluster_config_op_); + } + } + else if (op_ == &unlock_cluster_config_op_) + { + if (unlock_cluster_config_op_.hd_result_.IsError()) + { + if (txm->CheckLeaderTerm()) + { + const CcEntryAddr *cluster_config_addr = nullptr; + const auto &meta_rset = txm->rw_set_.MetaDataReadSet(); + for (const auto &[cce_addr, read_entry_pair] : meta_rset) + { + if (read_entry_pair.second == cluster_config_ccm_name_sv) + { + cluster_config_addr = &cce_addr; + break; + } + } + + if (cluster_config_addr) + { + DLOG(INFO) + << "Alter Table Index transaction unlock cluster " + "config " + "lock failed, txn: " + << txm->TxNumber() << ", error code: " + << (int) + unlock_cluster_config_op_.hd_result_.ErrorCode() + << ", error message: " + << unlock_cluster_config_op_.hd_result_.ErrorMsg() + << ", cluster config addr term" + << cluster_config_addr->Term() + << ", cluster config addr node group id: " + << cluster_config_addr->NodeGroupId() + << ", node group term: " + << Sharder::Instance().LeaderTerm( + cluster_config_addr->NodeGroupId()); + } + // Releasing a local read lock should never fail + assert(false); + } + else + { + ForceToFinish(txm); + return; + } + } + if (gated_ || acquire_all_intent_op_.fail_cnt_.load( + std::memory_order_relaxed) > 0) + { + // The write intents were never acquired: no log to clean. + txm->upsert_resp_->Finish(UpsertResult::Failed); + + txm->state_stack_.pop_back(); + assert(txm->state_stack_.empty()); + LocalCcShards *local_cc_shards = + Sharder::Instance().GetLocalCcShards(); + std::unique_lock lk(local_cc_shards->op_pool_mux_); + local_cc_shards->table_index_op_pool_.emplace_back( + std::move(txm->CompositeOpSlot())); + } + else + { LOG(INFO) << "Alter Table Index transaction write clean log" << ", txn: " << txm->TxNumber(); op_ = &clean_log_op_; @@ -1592,11 +1547,10 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) LOG(INFO) << "FaultInject alter_schema_term_changed"; is_force_finished_ = true; }); - if (txm->commit_ts_ == tx_op_failed_ts_) + if (txm->commit_ts_ == tx_op_failed_ts_ || rolled_back_) { - // - Flush kv error. - // - Fail to flush prepare_log. - // - Violate constraints. + // - Fail to create the kv tables or flush prepare_log. + // - Violate constraints (rolled back). txm->upsert_resp_->Finish(UpsertResult::Failed); } else @@ -1615,13 +1569,12 @@ void UpsertTableIndexOp::Forward(TransactionExecution *txm) txm->state_stack_.pop_back(); assert(txm->state_stack_.empty()); - txm->index_op_->catalog_rec_.Reset(); + txm->CompositeOpSlot()->catalog_rec_.Reset(); LocalCcShards *local_cc_shards = Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk( - local_cc_shards->table_index_op_pool_mux_); + std::unique_lock lk(local_cc_shards->op_pool_mux_); local_cc_shards->table_index_op_pool_.emplace_back( - std::move(txm->index_op_)); + std::move(txm->CompositeOpSlot())); } } else @@ -1682,9 +1635,10 @@ void UpsertTableIndexOp::Reset(const std::string_view table_name_str, indexes_multikey_attr_.clear(); + create_kv_table_op_.Reset(); + drop_kv_table_op_.Reset(); prepare_log_op_.Reset(); unlock_cluster_config_op_.Reset(); - kv_create_index_op_.Reset(); update_sequence_table_op_.Reset(); generate_sk_parallel_op_.Reset(); flush_all_old_tuples_sk_op_.Reset(); @@ -1693,6 +1647,7 @@ void UpsertTableIndexOp::Reset(const std::string_view table_name_str, kv_drop_index_op_.Reset(); kv_rollback_create_index_op_.Reset(); kv_update_schema_image_op_.Reset(); + ResetKvRetry(txm); clean_log_op_.Reset(); clean_ccm_op_.Clear(); @@ -1718,10 +1673,6 @@ void UpsertTableIndexOp::Reset(const std::string_view table_name_str, downgrade_all_lock_to_intent_op_.op_type_ = op_type_; downgrade_all_lock_to_intent_op_.write_type_ = PostWriteType::PrepareCommit; - kv_create_index_op_.alter_table_info_ = &alter_table_info_; - kv_create_index_op_.op_type_ = op_type_; - kv_create_index_op_.write_time_ = 0; - broadcast_dirty_schema_image_op_.table_name_ = &catalog_ccm_name; broadcast_dirty_schema_image_op_.keys_.clear(); broadcast_dirty_schema_image_op_.keys_.emplace_back(&table_key_); @@ -1759,10 +1710,11 @@ void UpsertTableIndexOp::Reset(const std::string_view table_name_str, // Reset cc_handler_res txm acquire_all_intent_op_.ResetHandlerTxm(txm); upgrade_all_intent_to_lock_op_.ResetHandlerTxm(txm); + create_kv_table_op_.ResetHandlerTxm(txm); + drop_kv_table_op_.ResetHandlerTxm(txm); prepare_log_op_.ResetHandlerTxm(txm); downgrade_all_lock_to_intent_op_.ResetHandlerTxm(txm); unlock_cluster_config_op_.ResetHandlerTxm(txm); - kv_create_index_op_.ResetHandlerTxm(txm); update_sequence_table_op_.ResetHandlerTxm(txm); generate_sk_parallel_op_.ResetHandlerTxm(txm); flush_all_old_tuples_sk_op_.ResetHandlerTxm(txm); @@ -1777,6 +1729,8 @@ void UpsertTableIndexOp::Reset(const std::string_view table_name_str, post_all_lock_op_.ResetHandlerTxm(txm); clean_log_op_.ResetHandlerTxm(txm); is_force_finished_ = false; + gated_ = false; + rolled_back_ = false; new_indexes_name_.clear(); new_indexes_name_.reserve(alter_table_info_.index_add_count_); @@ -1880,10 +1834,193 @@ void UpsertTableIndexOp::FillCommitLogRequest(TransactionExecution *txm) ::txlog::WriteLogRequest *commit_log_rec = commit_log_op_.log_closure_.LogRequest().mutable_write_log_request(); - assert(txm->commit_ts_ != tx_op_failed_ts_ || - kv_create_index_op_.hd_result_.IsError() || - generate_sk_parallel_op_.hd_result_.IsError()); + assert(txm->commit_ts_ != tx_op_failed_ts_); commit_log_rec->set_commit_timestamp(txm->commit_ts_); + commit_log_rec->mutable_log_content() + ->mutable_schema_log() + ->set_rolled_back(rolled_back_); +} + +void UpsertTableIndexOp::CreateKvTables(TransactionExecution *txm) +{ + assert(op_type_ == OperationType::AddIndex); + LOG(INFO) << "Alter Table Index transaction create the index kv tables " + << "for base table: " << table_key_.Name().StringView() + << ", txn: " << txm->TxNumber(); + + const uint64_t schema_version = txm->commit_ts_; + create_kv_table_op_.SetBlockingWork( + [this, schema_version, txn = txm->TxNumber()]() + { + store::DataStoreHandler *const store_hd = + Sharder::Instance().GetLocalCcShards()->store_hd_; + // One attempt, no retry: this runs under the prepare-stage write + // lock, which blocks DML on the table and holds the checkpoint + // back, and a failure only fails the command. + bool succ = + store_hd == nullptr || store_hd->ByPassDataStore() || + std::all_of(alter_table_info_.index_add_names_.begin(), + alter_table_info_.index_add_names_.end(), + [store_hd, schema_version]( + const std::pair &p) { + return store_hd->CreateKvTable(p.first, + schema_version); + }); + // Fails the creation after the tables are created, so the cleanup + // has real tables to drop. + CODE_FAULT_INJECTOR("alter_table_index_create_kv_table_fail", { + LOG(INFO) << "FaultInject " + "alter_table_index_create_kv_table_fail, txn: " + << txn; + succ = false; + }); + return succ ? CcErrorCode::NO_ERROR : CcErrorCode::DATA_STORE_ERR; + }); + + op_ = &create_kv_table_op_; + txm->PushOperation(&create_kv_table_op_); + txm->Process(create_kv_table_op_); +} + +void UpsertTableIndexOp::DropKvTables(TransactionExecution *txm) +{ + assert(op_type_ == OperationType::AddIndex); + LOG(INFO) << "Alter Table Index transaction drop the kv tables of a " + << "failed add index, base table: " + << table_key_.Name().StringView() << ", txn: " << txm->TxNumber(); + + drop_kv_table_op_.SetBlockingWork( + [this]() + { + store::DataStoreHandler *const store_hd = + Sharder::Instance().GetLocalCcShards()->store_hd_; + // One attempt: a failure leaves the tables orphaned, which is + // accepted, and the locks must not wait on it. + bool succ = + store_hd == nullptr || store_hd->ByPassDataStore() || + std::all_of( + alter_table_info_.index_add_names_.begin(), + alter_table_info_.index_add_names_.end(), + [store_hd](const std::pair &p) + { return store_hd->DropKvTable(p.second); }); + return succ ? CcErrorCode::NO_ERROR : CcErrorCode::DATA_STORE_ERR; + }); + + op_ = &drop_kv_table_op_; + txm->PushOperation(&drop_kv_table_op_); + txm->Process(drop_kv_table_op_); +} + +void UpsertTableIndexOp::HoldBeforeCommitLock(TransactionExecution *txm) +{ + static constexpr char kHold[] = "alter_table_index_hold_before_commit_lock"; + static constexpr char kHeld[] = "alter_table_index_held_before_commit_lock"; + // At most hold_s seconds (the fault's parameter "hold_s=N"), 60 by default. + // Read before kHeld is set: a test removes kHold only after seeing kHeld. + int hold_s = 60; + if (FaultEntry *entry = FaultInject::Entry(kHold); entry != nullptr) + { + auto it = entry->map_para_.find("hold_s"); + if (it != entry->map_para_.end()) + { + hold_s = std::stoi(it->second); + } + } + LOG(INFO) << "FaultInject " << kHold << ": holding the index op on table " + << table_key_.Name().StringView() + << " before its commit-stage write locks for at most " << hold_s + << " s, txn: " << txm->TxNumber(); + FaultInject::Instance().InjectFault(kHeld, ""); + kv_retry_wait_op_.SetBlockingWork( + [hold_s]() + { + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(hold_s); + while (FaultInject::Entry(kHold) != nullptr && + std::chrono::steady_clock::now() < deadline) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + FaultInject::Instance().InjectFault(kHeld, "remove"); + return CcErrorCode::NO_ERROR; + }); + // The kv_retry_wait_op_ stage runs kv_retry_step_ once the wait ends. + kv_retry_step_ = &acquire_all_lock_op_; + op_ = &kv_retry_wait_op_; + txm->PushOperation(&kv_retry_wait_op_); + txm->Process(kv_retry_wait_op_); +} + +void UpsertTableIndexOp::MarkFailed(TransactionExecution *txm) +{ + txm->commit_ts_ = tx_op_failed_ts_; + post_all_lock_op_.write_type_ = PostWriteType::Rollback; +} + +void UpsertTableIndexOp::RetryKvStep(TransactionExecution *txm, + TransactionOperation *step) +{ + if (!txm->CheckLeaderTerm()) + { + ForceToFinish(txm); + return; + } + if (step == &update_sequence_table_op_) + { + ArmKvRetryWait(txm, + step, + "sequence table update", + update_sequence_table_op_.hd_result_.ErrorMsg()); + } + else if (step == &flush_all_old_tuples_sk_op_) + { + ArmKvRetryWait(txm, + step, + "index data flush", + flush_all_old_tuples_sk_op_.hd_result_.ErrorMsg()); + } + else if (step == &kv_drop_index_op_) + { + ArmKvRetryWait(txm, + step, + "index kv table drop", + kv_drop_index_op_.hd_result_.ErrorMsg()); + } + else if (step == &kv_rollback_create_index_op_) + { + ArmKvRetryWait(txm, + step, + "rolled-back index's kv table drop", + kv_rollback_create_index_op_.hd_result_.ErrorMsg()); + } + else + { + assert(step == &kv_update_schema_image_op_); + ArmKvRetryWait(txm, + step, + "catalog row write", + kv_update_schema_image_op_.hd_result_.ErrorMsg()); + } + op_ = &kv_retry_wait_op_; + txm->PushOperation(&kv_retry_wait_op_); + txm->Process(kv_retry_wait_op_); +} + +void UpsertTableIndexOp::WritePrepareLog(TransactionExecution *txm) +{ + LOG(INFO) << "Alter Table Index transaction write prepare log, txn: " + << txm->TxNumber() << ". The schema version: " << txm->commit_ts_; + op_ = &prepare_log_op_; + FillPrepareLogRequest(txm); + txm->PushOperation(&prepare_log_op_); + CODE_FAULT_INJECTOR("alter_table_index_prepare_log_fail", { + LOG(INFO) << "FaultInject alter_table_index_prepare_log_fail, txn: " + << txm->TxNumber(); + prepare_log_op_.is_running_ = true; + prepare_log_op_.hd_result_.SetError(CcErrorCode::WRITE_LOG_FAILED); + return; + }); + txm->Process(prepare_log_op_); } void UpsertTableIndexOp::ForceToFinish(TransactionExecution *txm) @@ -2352,8 +2489,7 @@ void UpsertTableIndexOp::HandleRangeTask( Sharder::Instance().GetLocalCcShards(); std::unique_ptr sk_generator = nullptr; { - std::lock_guard lk( - cc_shards->table_index_op_pool_mux_); + std::lock_guard lk(cc_shards->op_pool_mux_); if (cc_shards->sk_generator_pool_.empty()) { sk_generator = std::make_unique(); @@ -2390,8 +2526,7 @@ void UpsertTableIndexOp::HandleRangeTask( // recycle skgenerator { - std::lock_guard lk( - cc_shards->table_index_op_pool_mux_); + std::lock_guard lk(cc_shards->op_pool_mux_); cc_shards->sk_generator_pool_.emplace_back( std::move(sk_generator)); } @@ -2515,8 +2650,7 @@ void UpsertTableIndexOp::HandleRangeTask( // recycle skgenerator { - std::lock_guard lk( - cc_shards->table_index_op_pool_mux_); + std::lock_guard lk(cc_shards->op_pool_mux_); cc_shards->sk_generator_pool_.emplace_back( std::move(sk_generator)); } @@ -2667,14 +2801,6 @@ bool UpsertTableIndexOp::IndexesMergeMultiKeyAttr( return changed; } -bool UpsertTableIndexOp::AnyMultiKeyIndex( - const std::vector &indexes_multikey_attr) -{ - return std::any_of(indexes_multikey_attr.begin(), - indexes_multikey_attr.end(), - [](const MultiKeyAttr &attr) { return attr.multikey_; }); -} - std::string UpsertTableIndexOp::RebuildSchemaImage( const TableSchema *dirty_schema, const std::vector &indexes_multikey_attr) diff --git a/tx_service/src/tx_operation.cpp b/tx_service/src/tx_operation.cpp index 1cbb5eb6e..3c97e10db 100644 --- a/tx_service/src/tx_operation.cpp +++ b/tx_service/src/tx_operation.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -51,6 +52,7 @@ #include "tx_request.h" #include "tx_service.h" #include "tx_service_common.h" +#include "tx_service_metrics.h" #include "tx_trace.h" #include "tx_util.h" #include "tx_worker_pool.h" @@ -143,8 +145,13 @@ void ReadOperation::Forward(TransactionExecution *txm) { if (!read_tx_req_->read_local_) { - // Just returned from lock_bucket_op_, check lock_bucket_result_. - assert(lock_range_bucket_result_->IsFinished()); + // Back from lock_range_op_ or lock_bucket_op_, or from the wait + // before a retry (ReRunOp). A hash-partitioned key whose bucket + // was already known (FastToGetBucket) runs no lock op, so a retry + // of its read finds the lock result still reset: neither finished + // nor errored, and Process() looks the bucket up again. + assert(lock_range_bucket_result_->IsFinished() || + read_tx_req_->tab_name_->IsHashPartitioned()); if (lock_range_bucket_result_->IsError()) { // There is an error when getting the input key's bucket or @@ -287,6 +294,7 @@ void ReadLocalOperation::Reset() rec_ = nullptr; hd_result_ = nullptr; execute_immediately_ = true; + is_for_write_ = false; } void ReadLocalOperation::Forward(txservice::TransactionExecution *txm) @@ -2363,11 +2371,13 @@ SchemaOp::SchemaOp(const std::string_view table_name_sv, const std::string ¤t_image, const std::string &dirty_image, uint64_t schema_ts, - OperationType op_type) + OperationType op_type, + TransactionExecution *txm) : table_key_(TableName(table_name_sv.data(), table_name_sv.size(), TableType::Primary, - table_engine)) + table_engine)), + kv_retry_wait_op_(txm) { catalog_rec_.SetSchemaImage(current_image); catalog_rec_.SetDirtySchemaImage(dirty_image); @@ -2377,1188 +2387,186 @@ SchemaOp::SchemaOp(const std::string_view table_name_sv, op_type_ = op_type; } -void SchemaOp::FillPrepareLogRequestCommon(TransactionExecution *txm, - WriteToLogOp &prepare_log_op) -{ - prepare_log_op.log_type_ = TxLogType::PREPARE; - - prepare_log_op.log_closure_.LogRequest().Clear(); - - ::txlog::WriteLogRequest *prepare_log_rec = - prepare_log_op.log_closure_.LogRequest().mutable_write_log_request(); - - prepare_log_rec->set_tx_term(txm->TxTerm()); - prepare_log_rec->set_txn_number(txm->TxNumber()); - prepare_log_rec->set_commit_timestamp(txm->CommitTs()); - - ::txlog::SchemaOpMessage *prepare_schema_msg = - prepare_log_rec->mutable_log_content()->mutable_schema_log(); - prepare_schema_msg->set_table_name_str(table_key_.Name().String()); - prepare_schema_msg->set_table_type( - ::txlog::ToRemoteType::ConvertTableType(table_key_.Name().Type())); - prepare_schema_msg->set_table_engine( - ::txlog::ToRemoteType::ConvertTableEngine(table_key_.Name().Engine())); - prepare_schema_msg->set_old_catalog_blob(catalog_rec_.SchemaImage()); - prepare_schema_msg->set_catalog_ts(curr_schema_ts_); - prepare_schema_msg->set_new_catalog_blob(catalog_rec_.DirtySchemaImage()); - prepare_schema_msg->mutable_table_op()->set_op_type( - static_cast<::google::protobuf::uint32>(op_type_)); - prepare_schema_msg->set_stage(::txlog::SchemaOpMessage_Stage_PrepareSchema); -} - -void SchemaOp::FillCommitLogRequestCommon(TransactionExecution *txm, - WriteToLogOp &commit_log_op) -{ - commit_log_op.log_type_ = TxLogType::COMMIT; - - commit_log_op.log_closure_.LogRequest().Clear(); - - ::txlog::WriteLogRequest *commit_log_rec = - commit_log_op.log_closure_.LogRequest().mutable_write_log_request(); - - commit_log_rec->set_tx_term(txm->TxTerm()); - commit_log_rec->set_txn_number(txm->TxNumber()); - - ::txlog::SchemaOpMessage *commit_schema_msg = - commit_log_rec->mutable_log_content()->mutable_schema_log(); - - commit_schema_msg->set_table_name_str(table_key_.Name().String()); - commit_schema_msg->set_table_type( - ::txlog::ToRemoteType::ConvertTableType(table_key_.Name().Type())); - commit_schema_msg->set_stage(::txlog::SchemaOpMessage_Stage_CommitSchema); - - // The prepare log keeps all cc nodes' terms and match them in the log - // service to detect invalidated write intents. The commit log, however, - // does not match terms in the log service. This is because all - // operations after the prepare log are retried or replayed upon - // failures to guarantee that the schema operation always roll forward. - // If a cc node fails over, the new node must restore write intents - // gained prior to the prepare log and then replay operations between - // the prepare log and the commit log, which in this case upgrade write - // intents to write locks. So, there is no need to check the liveness of - // write locks when flushing the commit log. - commit_log_rec->mutable_node_terms()->clear(); -} - -void SchemaOp::FillCleanLogRequestCommon(TransactionExecution *txm, - WriteToLogOp &clean_log_op) -{ - clean_log_op.log_type_ = TxLogType::CLEAN; - - clean_log_op.log_closure_.LogRequest().Clear(); - - ::txlog::WriteLogRequest *clean_log_rec = - clean_log_op.log_closure_.LogRequest().mutable_write_log_request(); - - clean_log_rec->set_tx_term(txm->TxTerm()); - clean_log_rec->set_txn_number(txm->TxNumber()); - - ::txlog::SchemaOpMessage *clean_schema_msg = - clean_log_rec->mutable_log_content()->mutable_schema_log(); - - clean_schema_msg->set_table_name_str(table_key_.Name().String()); - clean_schema_msg->set_table_type( - ::txlog::ToRemoteType::ConvertTableType(table_key_.Name().Type())); - clean_schema_msg->set_stage(::txlog::SchemaOpMessage_Stage_CleanSchema); - clean_log_rec->mutable_node_terms()->clear(); -} - -UpsertTableOp::UpsertTableOp(const std::string_view table_name_str, - TableEngine table_engine, - const std::string ¤t_image, - uint64_t curr_schema_ts, - const std::string &dirty_image, - OperationType op_type, - TransactionExecution *txm) - : SchemaOp(table_name_str, - table_engine, - current_image, - dirty_image, - curr_schema_ts, - op_type), - lock_cluster_config_op_(), - acquire_all_intent_op_(txm), - prepare_log_op_(txm), - post_all_intent_op_(txm), - unlock_cluster_config_op_(txm), - upsert_kv_table_op_(&table_key_.Name(), op_type, txm), - update_sequence_table_op_(txm), - acquire_all_lock_op_(txm), - commit_log_op_(txm), - clean_ccm_op_(txm), - post_all_lock_op_(txm), - clean_log_op_(txm), - read_cluster_result_(txm) -{ - assert(op_type_ == OperationType::CreateTable || - op_type_ == OperationType::DropTable || - op_type_ == OperationType::TruncateTable || - op_type_ == OperationType::Update); - - lock_cluster_config_op_.table_name_ = TableName(cluster_config_ccm_name_sv, - TableType::ClusterConfig, - TableEngine::None); - lock_cluster_config_op_.key_ = VoidKey::NegInfTxKey(); - lock_cluster_config_op_.rec_ = &cluster_conf_rec_; - lock_cluster_config_op_.hd_result_ = &read_cluster_result_; - - acquire_all_intent_op_.table_name_ = &catalog_ccm_name; - acquire_all_intent_op_.keys_.emplace_back(&table_key_); - acquire_all_intent_op_.cc_op_ = CcOperation::ReadForWrite; - acquire_all_intent_op_.protocol_ = CcProtocol::OCC; - - post_all_intent_op_.table_name_ = &catalog_ccm_name; - post_all_intent_op_.keys_.emplace_back(&table_key_); - post_all_intent_op_.recs_.push_back(&catalog_rec_); - post_all_intent_op_.op_type_ = op_type_; - post_all_intent_op_.write_type_ = PostWriteType::PrepareCommit; - - acquire_all_lock_op_.table_name_ = &catalog_ccm_name; - acquire_all_lock_op_.keys_.emplace_back(&table_key_); - acquire_all_lock_op_.cc_op_ = CcOperation::Write; - acquire_all_lock_op_.protocol_ = CcProtocol::Locking; - - post_all_lock_op_.table_name_ = &catalog_ccm_name; - post_all_lock_op_.keys_.emplace_back(&table_key_); - post_all_lock_op_.recs_.push_back(&catalog_rec_); - post_all_lock_op_.op_type_ = op_type_; - post_all_lock_op_.write_type_ = PostWriteType::PostCommit; - - clean_ccm_op_.Clear(); - - TX_TRACE_ASSOCIATE(this, &acquire_all_intent_op_, "acquire_all_intent_op_"); - TX_TRACE_ASSOCIATE(this, &prepare_log_op_, "prepare_log_op_"); - TX_TRACE_ASSOCIATE(this, &post_all_intent_op_, "post_all_intent_op_"); - TX_TRACE_ASSOCIATE(this, &upsert_kv_table_op_, "upsert_kv_table_op_"); - TX_TRACE_ASSOCIATE( - this, &update_sequence_table_op_, "update_sequence_table_op_"); - TX_TRACE_ASSOCIATE(this, &acquire_all_lock_op_, "acquire_all_lock_op_"); - TX_TRACE_ASSOCIATE(this, &commit_log_op_, "commit_log_op_"); - TX_TRACE_ASSOCIATE(this, &post_all_lock_op_, "post_all_lock_op_"); - TX_TRACE_ASSOCIATE(this, &clean_log_op_, "clean_log_op_"); - - is_force_finished = false; -} - -void UpsertTableOp::Forward(TransactionExecution *txm) -{ - if (op_ == nullptr) - { - op_ = &lock_cluster_config_op_; - txm->PushOperation(&lock_cluster_config_op_); - txm->Process(lock_cluster_config_op_); - } - else if (op_ == &lock_cluster_config_op_) - { - if (lock_cluster_config_op_.hd_result_->IsError()) - { - LOG(ERROR) << "Upsert table read cluster config failed, tx_number:" - << txm->TxNumber(); - if (!prepare_log_op_.hd_result_.IsFinished()) - { - txm->commit_ts_ = tx_op_failed_ts_; - } - ForceToFinish(txm); - return; - } - - if (prepare_log_op_.hd_result_.IsFinished()) - { - assert(op_type_ == OperationType::CreateTable || - op_type_ == OperationType::Update); - op_ = &acquire_all_lock_op_; - txm->PushOperation(&acquire_all_lock_op_); - txm->Process(acquire_all_lock_op_); - } - else - { - op_ = &acquire_all_intent_op_; - txm->PushOperation(&acquire_all_intent_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process acquire_all_intent_op_"; - txm->Process(acquire_all_intent_op_); - } - } - else if (op_ == &acquire_all_intent_op_) - { - if (acquire_all_intent_op_.fail_cnt_.load(std::memory_order_relaxed) > - 0) - { - LOG(ERROR) << "Upsert table acquire write intent failed, tx_number:" - << txm->TxNumber(); - txm->upsert_resp_->SetErrorCode( - TxErrorCode::UPSERT_TABLE_ACQUIRE_WRITE_INTENT_FAIL); - // Fails to acquire the write intent on the schema. Since write - // intents only conflict with other writes, there must be - // another tx trying to modify the same table's schema. Stops - // the schema operation. Set the commit ts to 0 to signal that - // the following post write operation releases all write - // intents. - txm->commit_ts_ = tx_op_failed_ts_; - // Moves to the last operation that removes all write - // intents/locks. - op_ = &post_all_lock_op_; - txm->PushOperation(&post_all_lock_op_); - txm->Process(post_all_lock_op_); - return; - } - - // Assigns a commit timestamp to the tx state machine as the version - // of the new schema. Unlike the conventional commit ts, the - // schema's commit ts is not chosen via the SetTs stage. But it - // follows a similar set of rules: the new schema's version should - // be greater than 1) the current version, 2) the maximal commit ts - // of all tx's that have read the schema, and 3) the local time when - // the tx starts. - txm->commit_ts_ = txm->commit_ts_bound_ + 1; - - for (size_t idx = 0; idx < acquire_all_intent_op_.upload_cnt_; ++idx) - { - const AcquireAllResult &acq_all_res = - acquire_all_intent_op_.hd_results_[idx].Value(); - uint64_t ts = std::max(acq_all_res.commit_ts_ + 1, - acq_all_res.last_vali_ts_ + 1); - txm->commit_ts_ = std::max(txm->commit_ts_, ts); - } - - op_ = &prepare_log_op_; - FillPrepareLogRequest(txm); - txm->PushOperation(&prepare_log_op_); - CODE_FAULT_INJECTOR("upsert_table_prepare_log_fail", { - LOG(INFO) << "FaultInject upsert_table_prepare_log_fail"; - prepare_log_op_.is_running_ = true; - prepare_log_op_.hd_result_.SetError(CcErrorCode::WRITE_LOG_FAILED); - return; - }); - DLOG(INFO) << "txn: " << txm->TxNumber() << " process prepare_log_op_"; - txm->Process(prepare_log_op_); - } - else if (op_ == &prepare_log_op_) - { - if (prepare_log_op_.hd_result_.IsError()) - { - if (prepare_log_op_.hd_result_.ErrorCode() == - CcErrorCode::LOG_CLOSURE_RESULT_UNKNOWN_ERR) - { - // prepare log result unknown, keep retrying until getting a - // clear response, either success or failure, or the - // coordinator itself is no longer leader - if (txm->CheckLeaderTerm()) - { - LOG(WARNING) - << "Upsert table write prepare log result unknown, " - "tx_number:" - << txm->TxNumber() << ", keep retrying"; - // set retry flag and retry prepare log - ::txlog::WriteLogRequest *log_req = - prepare_log_op_.log_closure_.LogRequest() - .mutable_write_log_request(); - log_req->set_retry(true); - txm->PushOperation(&prepare_log_op_); - txm->Process(prepare_log_op_); - } - else - { - LOG(ERROR) << "Upsert table write prepare log result " - "unknown, tx_number:" - << txm->TxNumber() - << ", not leader any more, stop retrying"; - // Not leader anymore, just quit. New leader will know - // whether prepare log succeeds and continue the rest if - // it does. Should not release the write intents. If - // prepare log is not written, the write intents will be - // released individually via orphan lock recovery - // mechanism. - txm->upsert_resp_->SetErrorCode( - TxErrorCode::LOG_SERVICE_UNREACHABLE); - - txm->upsert_resp_->Finish(UpsertResult::Failed); - txm->state_stack_.pop_back(); - assert(txm->state_stack_.empty()); - LocalCcShards *local_shards = - Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk( - local_shards->table_schema_op_pool_mux_); - local_shards->table_schema_op_pool_.emplace_back( - std::move(txm->schema_op_)); - } - } - else - { - LOG(ERROR) - << "Upsert table write prepare log failed, tx_number:" - << txm->TxNumber(); - // Fails to flush the prepare log. The schema operation is - // considered failed if the prepare log is not flushed. The - // commit ts is set to 0 to signal that the following post - // write operation releases all write intents. - txm->commit_ts_ = tx_op_failed_ts_; - // Moves to the last operation that removes all write - // intents/locks. - op_ = &post_all_lock_op_; - - txm->upsert_resp_->SetErrorCode( - TxErrorCode::UPSERT_TABLE_PREPARE_FAIL); - - txm->PushOperation(&post_all_lock_op_); - txm->Process(post_all_lock_op_); - } - } - else - { - ACTION_FAULT_INJECTOR("upsert_table_crash_after_prepare_log"); - op_ = &post_all_intent_op_; - - txm->PushOperation(&post_all_intent_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process post_all_intent_op_"; - txm->Process(post_all_intent_op_); - } - } - else if (op_ == &post_all_intent_op_) - { - if (post_all_intent_op_.IsFailed()) - { - // After the prepare log is flushed, the schema op is guaranteed - // to proceed. Retry this step to install the dirty schema in the tx - // service, if the tx node is still the leader. The tx is also - // allowed to proceed if the tx is in the recovery mode and the tx - // node is a leader candidate. - - if (txm->CheckLeaderTerm()) - { - // set catalog_rec_'s binary_value_ to image_str since it - // could be set to TableSchemaView pointer in localshard. - catalog_rec_.SetSchemaImage(image_str_); - - txm->PushOperation(&post_all_intent_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process post_all_intent_op_"; - txm->Process(post_all_intent_op_); - } - else - { - ForceToFinish(txm); - } - } - else if (op_type_ == OperationType::DropTable || - op_type_ == OperationType::TruncateTable) - { - // For DROP TABLE and TRUNCATE TABLE operations, the data store - // operation of deleting the k-v table happens after the commit log - // is flushed. - op_ = &acquire_all_lock_op_; - txm->PushOperation(&acquire_all_lock_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process acquire_all_lock_op_"; - txm->Process(acquire_all_lock_op_); - } - else - { - // Release cluster config op before doing data store op. - op_ = &unlock_cluster_config_op_; - // Get cce addr of cluster config read lock from rset. - const CcEntryAddr *cluster_config_addr = nullptr; - const ReadSetEntry *cluster_config_entry = nullptr; - const auto &meta_rset = txm->rw_set_.MetaDataReadSet(); - for (const auto &[cce_addr, read_entry_pair] : meta_rset) - { - if (read_entry_pair.second == cluster_config_ccm_name_sv) - { - cluster_config_addr = &cce_addr; - cluster_config_entry = &read_entry_pair.first; - break; - } - } - assert(cluster_config_addr && cluster_config_entry); - unlock_cluster_config_op_.Reset(cluster_config_addr, - cluster_config_entry); - txm->PushOperation(&unlock_cluster_config_op_); - txm->Process(unlock_cluster_config_op_); - } - } - else if (op_ == &unlock_cluster_config_op_) - { - assert(op_type_ == OperationType::CreateTable || - op_type_ == OperationType::Update || - op_type_ == OperationType::TruncateTable); - if (unlock_cluster_config_op_.hd_result_.IsError()) - { - if (txm->CheckLeaderTerm()) - { - // Releasing a local read lock should never fail - assert(false); - } - else - { - ForceToFinish(txm); - return; - } - } - op_ = &upsert_kv_table_op_; - // The post write request right after flushing the prepare log - // installs the dirty schema in the tx service and returns a - // local view (pointer) of the committed and dirty schema. - upsert_kv_table_op_.table_schema_old_ = catalog_rec_.Schema(); - upsert_kv_table_op_.table_schema_ = catalog_rec_.DirtySchema(); - upsert_kv_table_op_.alter_table_info_ = nullptr; - upsert_kv_table_op_.write_time_ = txm->commit_ts_; - txm->PushOperation(&upsert_kv_table_op_); - txm->Process(upsert_kv_table_op_); - } - else if (op_ == &upsert_kv_table_op_) - { - if (upsert_kv_table_op_.hd_result_.IsError()) - { - if (txm->CheckLeaderTerm()) - { - // Keep retrying if it is DropTable, TruncateTable. - if (op_type_ == OperationType::DropTable || - op_type_ == OperationType::TruncateTable) - { - txm->PushOperation(&upsert_kv_table_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process upsert_kv_table_op_"; - txm->Process(upsert_kv_table_op_); - } - else - { - // Set txm->commit_ts_ to 0 to indicate there is a flush - // error during upsert_kv_table_op_. - txm->commit_ts_ = tx_op_failed_ts_; - - /* - After upsert kv fails, we need to flush a commit log to - indicate this error. - - If we skip this commit log and jump to post_all_lock_op_ - directly, once the participant crashes at the point between - it releases write intent and the coordinator flushes - clean_log, then during recovery, the participant sees a - prepare_log(whose commit_ts is not 0) and recovers write - lock and dirty_catalog. - - Since the coordinator has finished its job, the write - lock recovered by participant becomes orphan lock, and the - dirty catalog can not be rejected either. - - Also, in the current design, post_all_intent_op_ does not - release the write intent, which means the write intent is - still being held after upsert_kv_table_op_(during - CreateTable or AddIndex). If create table or add index in kv - fails, the only thing we should do after writing commit_log - is to reject dirty schema. So there is no need to upgrade - write intent to write lock, and it is safe to skip - acquire_all_lock_op_ and jump directly to commit_log_op_. - - */ - - op_ = &commit_log_op_; - FillCommitLogRequest(txm); - txm->PushOperation(&commit_log_op_); - txm->Process(commit_log_op_); - } - } - else - { - ForceToFinish(txm); - } - } - else if (op_type_ == OperationType::CreateTable || - op_type_ == OperationType::DropTable || - op_type_ == OperationType::TruncateTable) - { - bool need_update_sequence_table = - !table_key_.Name().IsHashPartitioned() || - (op_type_ == OperationType::CreateTable - ? catalog_rec_.DirtySchema()->HasAutoIncrement() - : catalog_rec_.Schema()->HasAutoIncrement()); - if (need_update_sequence_table) - { - update_sequence_table_op_.op_func_ = - [this](AsyncOp &async_op) mutable - { - CcHandlerResult &hd_res = async_op.hd_result_; -#ifdef EXT_TX_PROC_ENABLED - hd_res.SetToBlock(); - // The memory fence ensures that the block flag is set - // before the background thread is executed. - std::atomic_thread_fence(std::memory_order_release); -#endif - assert(txservice::Sequences::Initialized()); - // Launch a new thread instead of sending it to workerpool - // to avoid being blocked during write lock is held. - async_op.worker_thread_ = std::thread( - [this, &hd_res]() mutable - { - bool succ = true; - const TableName &table_name = table_key_.Name(); - // Update sequence table about the range id info. - std::vector index_names = - op_type_ == OperationType::CreateTable - ? catalog_rec_.DirtySchema()->IndexNames() - : catalog_rec_.Schema()->IndexNames(); - if (op_type_ == OperationType::CreateTable || - op_type_ == OperationType::TruncateTable) - { - if (!table_name.IsHashPartitioned()) - { - int32_t init_range_id = - Sequences::InitialRangePartitionIdOf( - table_name); - succ = - Sequences::InitIdOfTableRangePartition( - table_name, init_range_id); - } - - // secondary index are always range partitioned. - if (succ && index_names.size() > 0) - { - succ = std::all_of( - index_names.begin(), - index_names.end(), - [](const TableName &index_name) - { - int32_t init_range_id = Sequences:: - InitialRangePartitionIdOf( - index_name); - return Sequences:: - InitIdOfTableRangePartition( - index_name, init_range_id); - }); - } - } - else if (op_type_ == OperationType::DropTable) - { - if (!table_name.IsHashPartitioned()) - { - succ = Sequences::DeleteSequence( - table_name, - SequenceType::RangePartitionId); - } - - if (succ && index_names.size() > 0) - { - succ = std::all_of( - index_names.begin(), - index_names.end(), - [](const TableName &index_name) - { - return Sequences::DeleteSequence( - index_name, - SequenceType::RangePartitionId); - }); - } - } - - bool has_auto_increment = - op_type_ == OperationType::CreateTable - ? catalog_rec_.DirtySchema() - ->HasAutoIncrement() - : catalog_rec_.Schema()->HasAutoIncrement(); - if (succ && has_auto_increment) - { - // Update sequence table about the auto - // increment info. - if (op_type_ == OperationType::CreateTable || - op_type_ == OperationType::TruncateTable) - { - succ = - Sequences::InitIdOfAutoIncrementColumn( - table_name); - } - else if (op_type_ == OperationType::DropTable) - { - succ = Sequences::DeleteSequence( - table_name, - SequenceType::AutoIncrementColumn); - } - } - - if (succ) - { - hd_res.SetFinished(); - } - else - { - LOG(ERROR) << "UpsertTableOp: failed to update" - " sequence info for table: " - << table_name.StringView(); - hd_res.SetError( - CcErrorCode::UPDATE_SEQUENCE_TABLE_FAIL); - } - }); - }; - - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process update_sequence_table_op_"; - op_ = &update_sequence_table_op_; - txm->PushOperation(&update_sequence_table_op_); - txm->Process(update_sequence_table_op_); - } - else if (op_type_ == OperationType::DropTable || - op_type_ == OperationType::TruncateTable) - { - const TableSchema *table_old_schema = catalog_rec_.Schema(); - assert(table_old_schema->GetBaseTableName() == - table_key_.Name()); - assert(clean_ccm_op_.table_names_.empty()); - - auto clean_ccm_names = table_old_schema->IndexNames(); - - clean_ccm_names.emplace_back( - table_old_schema->GetBaseTableName().StringView().data(), - table_old_schema->GetBaseTableName().StringView().size(), - table_old_schema->GetBaseTableName().Type(), - table_old_schema->GetBaseTableName().Engine()); - clean_ccm_op_.table_names_ = std::move(clean_ccm_names); - clean_ccm_op_.clean_type_ = CleanType::CleanCcm; - clean_ccm_op_.commit_ts_ = txm->CommitTs(); - - LOG(INFO) << "UpsertTableOp: Clean all ccmap on all node " - "groups, txn: " - << txm->TxNumber(); - - op_ = &clean_ccm_op_; - txm->PushOperation(&clean_ccm_op_); - txm->Process(clean_ccm_op_); - } - else - { - op_ = &lock_cluster_config_op_; - txm->PushOperation(&lock_cluster_config_op_); - txm->Process(lock_cluster_config_op_); - } - } - else - { - assert(op_type_ == OperationType::Update); - op_ = &lock_cluster_config_op_; - txm->PushOperation(&lock_cluster_config_op_); - txm->Process(lock_cluster_config_op_); - } - } - else if (op_ == &update_sequence_table_op_) - { - if (update_sequence_table_op_.hd_result_.IsError()) - { - if (txm->CheckLeaderTerm()) - { - // Retry - LOG(WARNING) - << "Upsert table schema transaction retry to update" - " sequence table, tx_number:" - << txm->TxNumber(); - txm->PushOperation(&update_sequence_table_op_); - txm->Process(update_sequence_table_op_); - } - else - { - ForceToFinish(txm); - } - return; - } - - if (op_type_ == OperationType::CreateTable) - { - op_ = &lock_cluster_config_op_; - txm->PushOperation(&lock_cluster_config_op_); - txm->Process(lock_cluster_config_op_); - } - else - { - assert(op_type_ == OperationType::DropTable || - op_type_ == OperationType::TruncateTable); - - const TableSchema *table_old_schema = catalog_rec_.Schema(); - assert(table_old_schema->GetBaseTableName() == table_key_.Name()); - assert(clean_ccm_op_.table_names_.empty()); - - auto clean_ccm_names = table_old_schema->IndexNames(); - - clean_ccm_names.emplace_back( - table_old_schema->GetBaseTableName().StringView().data(), - table_old_schema->GetBaseTableName().StringView().size(), - table_old_schema->GetBaseTableName().Type(), - table_old_schema->GetBaseTableName().Engine()); - clean_ccm_op_.table_names_ = std::move(clean_ccm_names); - - clean_ccm_op_.clean_type_ = CleanType::CleanCcm; - clean_ccm_op_.commit_ts_ = txm->CommitTs(); - - LOG(INFO) - << "UpsertTableOp: Clean all ccmap on all node groups, txn: " - << txm->TxNumber(); - - op_ = &clean_ccm_op_; - txm->PushOperation(&clean_ccm_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process clean_ccm_op_"; - txm->Process(clean_ccm_op_); - } - } - else if (op_ == &acquire_all_lock_op_) - { - if (acquire_all_lock_op_.fail_cnt_.load(std::memory_order_relaxed) > 0) - { - // Fails to acquire the write lock. The schema operation can - // only roll forward after flushing the prepare log. Retries the - // request if the tx node is still the leader or the tx is in - // the recovery mode and the cc node is a leader candidate. - if (txm->CheckLeaderTerm()) - { - LOG(ERROR) << "Upsert table schema transaction failed to " - "acquire write lock, tx_number:" - << txm->TxNumber(); - // We downgrade catalog write lock to avoid blocking other - // transaction which acquiring catalog read lock. And then retry - // to acquire catalog write lock. - if (acquire_all_lock_op_.IsDeadlock()) - { - LOG(ERROR) << "Upsert table schema transaction deadlocks " - "with other transaction, downgrade write " - "lock, tx_number: " - << txm->TxNumber(); - post_all_lock_op_.write_type_ = - PostWriteType::DowngradeLock; - op_ = &post_all_lock_op_; - txm->PushOperation(&post_all_lock_op_); - txm->Process(post_all_lock_op_); - } - else - { - txm->PushOperation(&acquire_all_lock_op_); - txm->Process(acquire_all_lock_op_); - } - } - else - { - ForceToFinish(txm); - } - } - else - { - op_ = &commit_log_op_; - FillCommitLogRequest(txm); - txm->PushOperation(&commit_log_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process commit_log_op_"; - txm->Process(commit_log_op_); - } - } - else if (op_ == &commit_log_op_) - { - if (commit_log_op_.hd_result_.IsError()) - { - // Fails to flush the commit log. Retries the operation if the - // tx node is still the leader or the tx is in the recovery - // mode and the cc node is a leader candidate. - if (txm->CheckLeaderTerm()) - { - // set retry flag and retry commit log - ::txlog::WriteLogRequest *log_req = - commit_log_op_.log_closure_.LogRequest() - .mutable_write_log_request(); - log_req->set_retry(true); - txm->PushOperation(&commit_log_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process commit_log_op_"; - txm->Process(commit_log_op_); - } - else - { - ForceToFinish(txm); - } - } - else - { - if (op_type_ == OperationType::DropTable || - op_type_ == OperationType::TruncateTable) - { - // `clean_ccm_op` will notify each node group leader performs - // `UpsertTable` and `KickoutData` independently for rocksdb. - if (op_type_ == OperationType::TruncateTable && - !txservice_skip_kv && - !Sharder::Instance() - .GetDataStoreHandler() - ->IsSharedStorage()) - { - assert(clean_ccm_op_.table_names_.empty()); - clean_ccm_op_.table_names_.emplace_back( - table_key_.Name().StringView().data(), - table_key_.Name().StringView().size(), - table_key_.Name().Type(), - table_key_.Name().Engine()); - clean_ccm_op_.clean_type_ = CleanType::CleanCcm; - clean_ccm_op_.commit_ts_ = txm->CommitTs(); - - LOG(INFO) << "UpsertTableOp: Clean all ccmap on all node " - "groups, txn: " - << txm->TxNumber(); - - op_ = &clean_ccm_op_; - txm->PushOperation(&clean_ccm_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process clean_ccm_op_"; - txm->Process(clean_ccm_op_); - } - else - { - op_ = &upsert_kv_table_op_; - upsert_kv_table_op_.table_schema_old_ = - catalog_rec_.Schema(); - upsert_kv_table_op_.table_schema_ = - catalog_rec_.DirtySchema(); - upsert_kv_table_op_.alter_table_info_ = nullptr; - upsert_kv_table_op_.write_time_ = txm->commit_ts_; - txm->PushOperation(&upsert_kv_table_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process upsert_kv_table_op_"; - txm->Process(upsert_kv_table_op_); - } - } - else - { - ACTION_FAULT_INJECTOR("upsert_table_post_all_lock"); - assert(post_all_lock_op_.write_type_ == - PostWriteType::PostCommit); - catalog_rec_.SetSchemaImage(catalog_rec_.DirtySchemaImage()); - catalog_rec_.ClearDirtySchema(); - op_ = &post_all_lock_op_; - txm->PushOperation(&post_all_lock_op_); - txm->Process(post_all_lock_op_); - } - } - } - else if (op_ == &clean_ccm_op_) +void SchemaOp::StartKvStep(TransactionExecution *txm, DsUpsertTableOp &step) +{ + txm->PushOperation(&step, 0); + txm->Process(step); +} + +void SchemaOp::ArmKvRetryWait(TransactionExecution *txm, + TransactionOperation *step, + std::string_view step_name, + const std::string &error) +{ + if (step != kv_retry_step_) { - assert(op_type_ == OperationType::TruncateTable || - op_type_ == OperationType::DropTable); - if (clean_ccm_op_.hd_result_.IsError()) + kv_retry_step_ = step; + kv_retry_failures_ = 0; + } + ++kv_retry_failures_; + const std::chrono::seconds interval = KvRetryInterval(); + LOG(WARNING) << "Schema op on table " << table_key_.Name().StringView() + << ": the data store failed the " << step_name << " (attempt " + << kv_retry_failures_ << ", error: " << error + << "); retrying in " << interval.count() + << " s while holding the table's catalog lock, txn: " + << txm->TxNumber(); + kv_retry_wait_op_.SetBlockingWork( + [interval]() { - if (txm->CheckLeaderTerm()) - { - LOG(ERROR) - << "UpsertTableOp: failed to truncate table, err code: " - << (int) clean_ccm_op_.hd_result_.ErrorCode() - << ", err msg: " << clean_ccm_op_.hd_result_.ErrorMsg() - << ", txn: " << txm->TxNumber() << ", keep retrying"; - op_ = &clean_ccm_op_; - txm->PushOperation(&clean_ccm_op_); - txm->Process(clean_ccm_op_); - } - else - { - ForceToFinish(txm); - } - - return; - } - - clean_ccm_op_.Clear(); + std::this_thread::sleep_for(interval); + return CcErrorCode::NO_ERROR; + }); +} - // Clear write set before commit dirty schema. - txm->rw_set_.ClearWriteSet(table_key_.Name()); - txm->rw_set_.ClearReadSet(table_key_.Name()); +void SchemaOp::RunKvStepAgain(TransactionExecution *txm) +{ + assert(kv_retry_step_ != nullptr); + // The step is not running, so the tx processes it as soon as it is on top + // of the stack; retry count 0 keeps a DsUpsertTableOp to one attempt. + txm->PushOperation(kv_retry_step_, 0); +} - // Update catalog record to the latest schema. - catalog_rec_.SetSchemaImage(catalog_rec_.DirtySchemaImage()); - catalog_rec_.ClearDirtySchema(); +void SchemaOp::ResetKvRetry(TransactionExecution *txm) +{ + kv_retry_wait_op_.Reset(); + kv_retry_wait_op_.ResetHandlerTxm(txm); + kv_retry_step_ = nullptr; + kv_retry_failures_ = 0; +} - // For DROP TABLE and TRUNCATE TABLE, the data store operation - // happens after all write locks are acquired and commit log is - // flushed. - op_ = &post_all_lock_op_; - txm->PushOperation(&post_all_lock_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process post_all_lock_op_"; - txm->Process(post_all_lock_op_); - } - else if (op_ == &post_all_lock_op_) +void SchemaOp::StartUnlockClusterConfig(TransactionExecution *txm, + PostReadOperation &unlock_op) +{ + // Get cce addr of cluster config read lock from rset. + const CcEntryAddr *cluster_config_addr = nullptr; + const ReadSetEntry *cluster_config_entry = nullptr; + for (const auto &[cce_addr, read_entry_pair] : + txm->rw_set_.MetaDataReadSet()) { - if (!txm->CheckLeaderTerm()) - { - // The tx node is no longer the leader or leader candidate(during - // recovery), ForceToFinish. - ForceToFinish(txm); - return; - } - - // The tx's modification of the schema has finished. If the tx has - // previously read the same schema and keeps a pointer in the read set - // to the cc entry of the schema, removes it from the read set. As a - // result, the tx will not try to release the read lock of the schema - // when committing. - for (size_t idx = 0; idx < acquire_all_intent_op_.upload_cnt_; ++idx) + if (read_entry_pair.second == cluster_config_ccm_name_sv) { - const CcEntryAddr &schema_entry_addr = - acquire_all_intent_op_.hd_results_[idx].Value().local_cce_addr_; - if (schema_entry_addr.NodeGroupId() == txm->TxCcNodeId()) - { - txm->rw_set_.DedupRead(schema_entry_addr); - } + cluster_config_addr = &cce_addr; + cluster_config_entry = &read_entry_pair.first; + break; } + } + assert(cluster_config_addr && cluster_config_entry); + unlock_op.Reset(cluster_config_addr, cluster_config_entry); + txm->PushOperation(&unlock_op); + txm->Process(unlock_op); +} - if (acquire_all_intent_op_.fail_cnt_.load(std::memory_order_relaxed) > - 0) - { - // The schema operation failed at acquire_all_intent_op_, without - // flushing the prepare log. Do not retry post-processing (release - // write intents) even if it fails. Remaining write intents on the - // schema, if there are any, will be recovered by individual cc - // nodes separately. - - txm->upsert_resp_->Finish(UpsertResult::Failed); - - txm->state_stack_.pop_back(); - assert(txm->state_stack_.empty()); - - LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk(shards->table_schema_op_pool_mux_); - shards->table_schema_op_pool_.emplace_back( - std::move(txm->schema_op_)); - } - else if (post_all_lock_op_.hd_result_.IsError()) - { - // post_all_lock_op_ returns an error: - // 1. if flush kv succeeds, the schema op is guaranteed to succeed - // and can only roll forward. Retry this step to install the - // committed schema and remove write locks; - // 2. if flush kx fails, the schema op has to roll backward. Retry - // this step to reject dirty schema and remove write locks. - txm->PushOperation(&post_all_lock_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process post_all_lock_op_, " - << post_all_lock_op_.hd_result_.ErrorMsg(); - txm->Process(post_all_lock_op_); - return; - } - else +void SchemaOp::ResolveSchemaImages(uint64_t dirty_version) +{ + LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); + const TableName &table_name = table_key_.Name(); + if (!image_str_.empty()) + { + TableSchema::uptr current = shards->CreateTableSchemaFromImage( + table_name, image_str_, curr_schema_ts_); + if (current != nullptr) { - // post_all_lock_op_ has finished without an error. - assert(!post_all_lock_op_.IsFailed()); - - if (post_all_lock_op_.write_type_ == PostWriteType::DowngradeLock) - { - // Reset write type to PostCommit - post_all_lock_op_.write_type_ = PostWriteType::PostCommit; - op_ = &acquire_all_lock_op_; - txm->PushOperation(&acquire_all_lock_op_); - txm->Process(acquire_all_lock_op_); - return; - } - else if (txm->commit_ts_ != tx_op_failed_ts_) - { - assert(post_all_lock_op_.write_type_ != - PostWriteType::DowngradeLock); - } - else - { - assert(post_all_lock_op_.write_type_ != - PostWriteType::DowngradeLock); - // Flush kv failed, or it is recovering from a flush kv failure. - // This schema op has already been rolled back by now, only need - // to flush clean log here. - // Also, flush kv failure does not require lock upgrade(write - // intent to write lock). So the CcEntryAddr needs to be kept in - // rset in order to release the read lock when committing. - } - - op_ = &clean_log_op_; - FillCleanLogRequestCommon(txm, clean_log_op_); - txm->PushOperation(&clean_log_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process clean_log_op_"; - txm->Process(clean_log_op_); + image_str_ = current->SchemaImage(); } } - else if (op_ == &clean_log_op_) + if (!dirty_image_str_.empty()) { - LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); - if (clean_log_op_.hd_result_.IsError() && txm->CheckLeaderTerm()) - { - // set retry flag and retry clean log - ::txlog::WriteLogRequest *log_req = - clean_log_op_.log_closure_.LogRequest() - .mutable_write_log_request(); - log_req->set_retry(true); - txm->PushOperation(&clean_log_op_); - DLOG(INFO) << "txn: " << txm->TxNumber() - << " process clean_log_op_"; - txm->Process(clean_log_op_); - } - else + TableSchema::uptr dirty = shards->CreateTableSchemaFromImage( + table_name, dirty_image_str_, dirty_version); + if (dirty != nullptr) { - CODE_FAULT_INJECTOR("alter_schema_term_changed", { - LOG(INFO) << "FaultInject alter_schema_term_changed"; - is_force_finished = true; - }); - - if (txm->commit_ts_ == tx_op_failed_ts_) - { - // Flush kv error or fail to flush prepare_log. - txm->upsert_resp_->Finish(UpsertResult::Failed); - } - else - { - assert(txm->commit_ts_ > 0); - if (is_force_finished) - { - txm->upsert_resp_->Finish(UpsertResult::Unverified); - } - else - { - txm->upsert_resp_->Finish(UpsertResult::Succeeded); - } - } - - txm->state_stack_.pop_back(); - assert(txm->state_stack_.empty()); - - txm->schema_op_->catalog_rec_.Reset(); - std::unique_lock lk(shards->table_schema_op_pool_mux_); - shards->table_schema_op_pool_.emplace_back( - std::move(txm->schema_op_)); + dirty_image_str_ = dirty->SchemaImage(); } } + catalog_rec_.SetSchemaImage(image_str_); + catalog_rec_.SetDirtySchemaImage(dirty_image_str_); } -void UpsertTableOp::Reset(const std::string_view table_name_str, - TableEngine table_engine, - const std::string ¤t_image, - uint64_t curr_schema_ts, - const std::string &dirty_image, - OperationType op_type, - TransactionExecution *txm) +void SchemaOp::FillPrepareLogRequestCommon(TransactionExecution *txm, + WriteToLogOp &prepare_log_op) { - assert(op_type_ == OperationType::CreateTable || - op_type_ == OperationType::Update || - op_type_ == OperationType::DropTable || - op_type_ == OperationType::TruncateTable); - - // reset TransactionOperation - retry_num_ = RETRY_NUM; - is_running_ = false; + prepare_log_op.log_type_ = TxLogType::PREPARE; - // reset SchemaOp - table_key_.Name() = TableName(table_name_str.data(), - table_name_str.size(), - TableType::Primary, - table_engine); - catalog_rec_.SetSchemaImage(current_image); - catalog_rec_.SetDirtySchemaImage(dirty_image); - image_str_ = current_image; - dirty_image_str_ = dirty_image; - curr_schema_ts_ = curr_schema_ts; + prepare_log_op.log_closure_.LogRequest().Clear(); - // reset UpsertTableOp - op_type_ = op_type; - op_ = nullptr; + ::txlog::WriteLogRequest *prepare_log_rec = + prepare_log_op.log_closure_.LogRequest().mutable_write_log_request(); - // reset op - read_cluster_result_.Reset(); - cluster_conf_rec_.Reset(); - lock_cluster_config_op_.Reset(); - lock_cluster_config_op_.key_ = VoidKey::NegInfTxKey(); - lock_cluster_config_op_.table_name_ = TableName(cluster_config_ccm_name_sv, - TableType::ClusterConfig, - TableEngine::None); - lock_cluster_config_op_.rec_ = &cluster_conf_rec_; - lock_cluster_config_op_.hd_result_ = &read_cluster_result_; - prepare_log_op_.Reset(); - unlock_cluster_config_op_.Reset(); - upsert_kv_table_op_.Reset(); - update_sequence_table_op_.Reset(); - commit_log_op_.Reset(); - clean_log_op_.Reset(); + prepare_log_rec->set_tx_term(txm->TxTerm()); + prepare_log_rec->set_txn_number(txm->TxNumber()); + prepare_log_rec->set_commit_timestamp(txm->CommitTs()); - acquire_all_intent_op_.table_name_ = &catalog_ccm_name; - acquire_all_intent_op_.keys_.clear(); - acquire_all_intent_op_.keys_.emplace_back(&table_key_); - acquire_all_intent_op_.cc_op_ = CcOperation::ReadForWrite; - acquire_all_intent_op_.protocol_ = CcProtocol::OCC; + ::txlog::SchemaOpMessage *prepare_schema_msg = + prepare_log_rec->mutable_log_content()->mutable_schema_log(); + prepare_schema_msg->set_table_name_str(table_key_.Name().String()); + prepare_schema_msg->set_table_type( + ::txlog::ToRemoteType::ConvertTableType(table_key_.Name().Type())); + prepare_schema_msg->set_table_engine( + ::txlog::ToRemoteType::ConvertTableEngine(table_key_.Name().Engine())); + prepare_schema_msg->set_old_catalog_blob(catalog_rec_.SchemaImage()); + prepare_schema_msg->set_catalog_ts(curr_schema_ts_); + prepare_schema_msg->set_new_catalog_blob(catalog_rec_.DirtySchemaImage()); + prepare_schema_msg->mutable_table_op()->set_op_type( + static_cast<::google::protobuf::uint32>(op_type_)); + prepare_schema_msg->set_stage(::txlog::SchemaOpMessage_Stage_PrepareSchema); +} - post_all_intent_op_.table_name_ = &catalog_ccm_name; - post_all_intent_op_.keys_.clear(); - post_all_intent_op_.keys_.emplace_back(&table_key_); - post_all_intent_op_.recs_.clear(); - post_all_intent_op_.recs_.push_back(&catalog_rec_); - post_all_intent_op_.op_type_ = op_type_; - post_all_intent_op_.write_type_ = PostWriteType::PrepareCommit; +void SchemaOp::FillCommitLogRequestCommon(TransactionExecution *txm, + WriteToLogOp &commit_log_op) +{ + commit_log_op.log_type_ = TxLogType::COMMIT; - upsert_kv_table_op_.alter_table_info_ = nullptr; - upsert_kv_table_op_.op_type_ = op_type_; - upsert_kv_table_op_.write_time_ = 0; + commit_log_op.log_closure_.LogRequest().Clear(); - acquire_all_lock_op_.table_name_ = &catalog_ccm_name; - acquire_all_lock_op_.keys_.clear(); - acquire_all_lock_op_.keys_.emplace_back(&table_key_); - acquire_all_lock_op_.cc_op_ = CcOperation::Write; - acquire_all_lock_op_.protocol_ = CcProtocol::Locking; + ::txlog::WriteLogRequest *commit_log_rec = + commit_log_op.log_closure_.LogRequest().mutable_write_log_request(); - post_all_lock_op_.table_name_ = &catalog_ccm_name; - post_all_lock_op_.keys_.clear(); - post_all_lock_op_.keys_.emplace_back(&table_key_); - post_all_lock_op_.recs_.clear(); - post_all_lock_op_.recs_.push_back(&catalog_rec_); - post_all_lock_op_.op_type_ = op_type_; - post_all_lock_op_.write_type_ = PostWriteType::PostCommit; + commit_log_rec->set_tx_term(txm->TxTerm()); + commit_log_rec->set_txn_number(txm->TxNumber()); - clean_ccm_op_.Clear(); - clean_ccm_op_.hd_result_.Reset(); + ::txlog::SchemaOpMessage *commit_schema_msg = + commit_log_rec->mutable_log_content()->mutable_schema_log(); - // reset cc_handler_res txm - read_cluster_result_.ResetTxm(txm); - acquire_all_intent_op_.ResetHandlerTxm(txm); - prepare_log_op_.ResetHandlerTxm(txm); - post_all_intent_op_.ResetHandlerTxm(txm); - unlock_cluster_config_op_.ResetHandlerTxm(txm); - upsert_kv_table_op_.ResetHandlerTxm(txm); - update_sequence_table_op_.ResetHandlerTxm(txm); - acquire_all_lock_op_.ResetHandlerTxm(txm); - commit_log_op_.ResetHandlerTxm(txm); - clean_ccm_op_.ResetHandlerTxm(txm); - post_all_lock_op_.ResetHandlerTxm(txm); - clean_log_op_.ResetHandlerTxm(txm); + commit_schema_msg->set_table_name_str(table_key_.Name().String()); + commit_schema_msg->set_table_type( + ::txlog::ToRemoteType::ConvertTableType(table_key_.Name().Type())); + commit_schema_msg->set_stage(::txlog::SchemaOpMessage_Stage_CommitSchema); - is_force_finished = false; + // The prepare log keeps all cc nodes' terms and match them in the log + // service to detect invalidated write intents. The commit log, however, + // does not match terms in the log service. This is because all + // operations after the prepare log are retried or replayed upon + // failures to guarantee that the schema operation always roll forward. + // If a cc node fails over, the new node must restore write intents + // gained prior to the prepare log and then replay operations between + // the prepare log and the commit log, which in this case upgrade write + // intents to write locks. So, there is no need to check the liveness of + // write locks when flushing the commit log. + commit_log_rec->mutable_node_terms()->clear(); } -void UpsertTableOp::FillPrepareLogRequest(TransactionExecution *txm) +void SchemaOp::FillCleanLogRequestCommon(TransactionExecution *txm, + WriteToLogOp &clean_log_op) { - FillPrepareLogRequestCommon(txm, prepare_log_op_); - ::txlog::WriteLogRequest *prepare_log_rec = - prepare_log_op_.log_closure_.LogRequest().mutable_write_log_request(); + clean_log_op.log_type_ = TxLogType::CLEAN; - auto &node_terms = *prepare_log_rec->mutable_node_terms(); - node_terms.clear(); - for (size_t idx = 0; idx < acquire_all_intent_op_.upload_cnt_; ++idx) - { - const AcquireAllResult &hres_val = - acquire_all_intent_op_.hd_results_[idx].Value(); - uint32_t ng_id = hres_val.local_cce_addr_.NodeGroupId(); - node_terms[ng_id] = hres_val.node_term_; - } -} + clean_log_op.log_closure_.LogRequest().Clear(); -void UpsertTableOp::FillCommitLogRequest(TransactionExecution *txm) -{ - FillCommitLogRequestCommon(txm, commit_log_op_); + ::txlog::WriteLogRequest *clean_log_rec = + clean_log_op.log_closure_.LogRequest().mutable_write_log_request(); - ::txlog::WriteLogRequest *commit_log_rec = - commit_log_op_.log_closure_.LogRequest().mutable_write_log_request(); + clean_log_rec->set_tx_term(txm->TxTerm()); + clean_log_rec->set_txn_number(txm->TxNumber()); - assert(txm->commit_ts_ != tx_op_failed_ts_ || - upsert_kv_table_op_.hd_result_.IsError()); - commit_log_rec->set_commit_timestamp(txm->commit_ts_); -} + ::txlog::SchemaOpMessage *clean_schema_msg = + clean_log_rec->mutable_log_content()->mutable_schema_log(); -void UpsertTableOp::ForceToFinish(TransactionExecution *txm) -{ - clean_log_op_.hd_result_.SetFinished(); - op_ = &clean_log_op_; - is_force_finished = true; - Forward(txm); + clean_schema_msg->set_table_name_str(table_key_.Name().String()); + clean_schema_msg->set_table_type( + ::txlog::ToRemoteType::ConvertTableType(table_key_.Name().Type())); + clean_schema_msg->set_stage(::txlog::SchemaOpMessage_Stage_CleanSchema); + clean_log_rec->mutable_node_terms()->clear(); } FlushUpdateTableOp::FlushUpdateTableOp(TransactionExecution *txm) @@ -3922,6 +2930,36 @@ void AsyncOp::Reset() } } +template +void AsyncOp::SetBlockingWork(std::function work) +{ + op_func_ = [work = std::move(work)](AsyncOp &async_op) + { + // Forward() or Reset() joined the thread of an earlier run. + assert(!async_op.worker_thread_.joinable()); + CcHandlerResult &hd_res = async_op.hd_result_; +#ifdef EXT_TX_PROC_ENABLED + // Starting the thread orders the flag before the worker reads it. + hd_res.SetToBlock(); +#endif + // The thread uses the work stored in op_func_: op_func_ outlives the + // thread, which Forward(), Reset() and ~AsyncOp() join. + async_op.worker_thread_ = std::thread( + [&work, &hd_res]() + { + CcErrorCode err = work(); + if (err == CcErrorCode::NO_ERROR) + { + hd_res.SetFinished(); + } + else + { + hd_res.SetError(err); + } + }); + }; +} + template struct AsyncOp; template struct AsyncOp; template struct AsyncOp; @@ -4328,13 +3366,12 @@ void SplitFlushRangeOp::Forward(TransactionExecution *txm) txm->state_stack_.pop_back(); assert(txm->state_stack_.empty()); - assert(this == txm->split_flush_op_.get()); + assert(this == txm->CompositeOpSlot().get()); LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk( - shards->split_flush_range_op_pool_mux_); + std::unique_lock lk(shards->op_pool_mux_); shards->split_flush_range_op_pool_.emplace_back( - std::move(txm->split_flush_op_)); - assert(txm->split_flush_op_ == nullptr); + std::move(txm->CompositeOpSlot())); + assert(txm->CompositeOpSlot() == nullptr); } return; } @@ -4421,9 +3458,10 @@ void SplitFlushRangeOp::Forward(TransactionExecution *txm) "lock, tx_number:" << txm->TxNumber(); - // Set commit ts to 0 to indicate transaction failure. - // post_all_lock_op_ will release locks acquired. + // The split fails: post_all_lock_op_ rolls back at commit ts 0, + // releasing the locks acquired. txm->commit_ts_ = tx_op_failed_ts_; + post_all_lock_op_.write_type_ = PostWriteType::Rollback; ForwardToSubOperation(txm, &post_all_lock_op_); return; } @@ -4492,9 +3530,10 @@ void SplitFlushRangeOp::Forward(TransactionExecution *txm) } else { - // Set commit ts to 0 to indicate transaction failure. - // post_all_lock_op_ will release locks acquired. + // The split fails: post_all_lock_op_ rolls back at commit ts + // 0, releasing the locks acquired. txm->commit_ts_ = tx_op_failed_ts_; + post_all_lock_op_.write_type_ = PostWriteType::Rollback; ForwardToSubOperation(txm, &post_all_lock_op_); } return; @@ -4854,41 +3893,35 @@ void SplitFlushRangeOp::Forward(TransactionExecution *txm) GenSplittedRangeInfos(); // Insert new ranges into data store range table. Update - // range slice size of the old range. - ds_upsert_range_op_.op_func_ = + // range slice size of the old range. On its own thread, not the + // worker pool, so that it is not blocked while the write lock is + // held. A failed upsert is retried (RetrySubOperation) with the same + // range_info, which UpsertRanges only reads. + ds_upsert_range_op_.SetBlockingWork( [&table_name = table_name_, range_info = std::move(splitted_range_info), - tx_ts = txm->commit_ts_](AsyncOp &async_op) mutable - { - CcHandlerResult &hd_res = async_op.hd_result_; -#ifdef EXT_TX_PROC_ENABLED - hd_res.SetToBlock(); - // The memory fence ensures that the block flag is set before the - // background data store thread is executed. - std::atomic_thread_fence(std::memory_order_release); -#endif - // Launch a new thread instead of sending it to workerpool to - // avoid being blocked during write lock is held. - async_op.worker_thread_ = std::thread( - [table_name, - range_info = std::move(range_info), - tx_ts, - &hd_res]() mutable - { - store::DataStoreHandler *const store_hd = - Sharder::Instance().GetLocalCcShards()->store_hd_; - bool succ = store_hd->UpsertRanges( - table_name, std::move(range_info), tx_ts); - if (succ) - { - hd_res.SetFinished(); - } - else - { - hd_res.SetError(CcErrorCode::DATA_STORE_ERR); - } + tx_ts = txm->commit_ts_]() + { + store::DataStoreHandler *const store_hd = + Sharder::Instance().GetLocalCcShards()->store_hd_; + // Fails one upsert before it writes, so that the retry must + // write the ranges. + CODE_FAULT_INJECTOR("split_flush_upsert_ranges_fail", { + LOG(INFO) << "FaultInject split_flush_upsert_ranges_fail, " + "ranges: " + << range_info.size(); + FaultInject::Instance().InjectFault( + "split_flush_upsert_ranges_fail", "remove"); + return CcErrorCode::DATA_STORE_ERR; }); - }; + LOG(INFO) << "Split Flush transaction upsert " + << range_info.size() + << " ranges into the data store, table: " + << table_name.StringView(); + return store_hd->UpsertRanges(table_name, range_info, tx_ts) + ? CcErrorCode::NO_ERROR + : CcErrorCode::DATA_STORE_ERR; + }); LOG(INFO) << "Split Flush transaction upsert new range spec, range id " << range_info_->PartitionId() << ", txn: " << txm->TxNumber(); @@ -5090,13 +4123,12 @@ void SplitFlushRangeOp::Forward(TransactionExecution *txm) txm->state_stack_.pop_back(); assert(txm->state_stack_.empty()); - assert(this == txm->split_flush_op_.get()); + assert(this == txm->CompositeOpSlot().get()); LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); - std::unique_lock lk( - shards->split_flush_range_op_pool_mux_); + std::unique_lock lk(shards->op_pool_mux_); shards->split_flush_range_op_pool_.emplace_back( - std::move(txm->split_flush_op_)); - assert(txm->split_flush_op_ == nullptr); + std::move(txm->CompositeOpSlot())); + assert(txm->CompositeOpSlot() == nullptr); } } } @@ -6686,9 +5718,9 @@ void ClusterScaleOp::Forward(TransactionExecution *txm) { auto shards = Sharder::Instance().GetLocalCcShards(); - std::lock_guard lk(shards->cluster_scale_op_mux_); + std::lock_guard lk(shards->op_pool_mux_); shards->cluster_scale_op_pool_.push_back( - std::move(txm->cluster_scale_op_)); + std::move(txm->CompositeOpSlot())); } if (txm->CommitTs() != tx_op_failed_ts_) @@ -7400,10 +6432,9 @@ void DataMigrationOp::Forward(TransactionExecution *txm) { auto shards = Sharder::Instance().GetLocalCcShards(); - std::lock_guard lk( - shards->data_migration_op_pool_mux_); + std::lock_guard lk(shards->op_pool_mux_); shards->migration_op_pool_.push_back( - std::move(txm->migration_op_)); + std::move(txm->CompositeOpSlot())); } // Abort and recyle txm txm->Abort(); @@ -8247,8 +7278,9 @@ void DataMigrationOp::Forward(TransactionExecution *txm) { auto shards = Sharder::Instance().GetLocalCcShards(); - std::lock_guard lk(shards->data_migration_op_pool_mux_); - shards->migration_op_pool_.push_back(std::move(txm->migration_op_)); + std::lock_guard lk(shards->op_pool_mux_); + shards->migration_op_pool_.push_back( + std::move(txm->CompositeOpSlot())); } // Commit the tx and recyle txm txm->Commit(); @@ -8428,7 +7460,13 @@ void BatchReadOperation::Forward(TransactionExecution *txm) { if (!is_running_) { - assert(lock_range_bucket_result_->IsFinished()); + // Back from lock_range_op_, or from the wait before a retry + // (ReRunOp). A hash-partitioned batch whose buckets were all already + // known (FastToGetBucket) runs no lock op, so a retry finds the lock + // result still reset: neither finished nor errored, and every key's + // node group already set (lock_index_ is at the end). + assert(lock_range_bucket_result_->IsFinished() || + batch_read_tx_req_->tab_name_->IsHashPartitioned()); std::vector &read_batch = batch_read_tx_req_->read_batch_; @@ -8668,7 +7706,9 @@ InvalidateTableCacheCompositeOp::InvalidateTableCacheCompositeOp( post_all_lock_op_.keys_.emplace_back(&catalog_key_); post_all_lock_op_.recs_.push_back(&catalog_rec_); post_all_lock_op_.op_type_ = OperationType::Update; - post_all_lock_op_.write_type_ = PostWriteType::PostCommit; + // Its second phase only releases the write locks, rolling back at + // timestamp 0: nothing was prepared on the catalog entry. + post_all_lock_op_.write_type_ = PostWriteType::Rollback; } void InvalidateTableCacheCompositeOp::Reset(const TableName *table_name, @@ -8696,7 +7736,9 @@ void InvalidateTableCacheCompositeOp::Reset(const TableName *table_name, post_all_lock_op_.recs_.clear(); post_all_lock_op_.recs_.push_back(&catalog_rec_); post_all_lock_op_.op_type_ = OperationType::Update; - post_all_lock_op_.write_type_ = PostWriteType::PostCommit; + // Its second phase only releases the write locks, rolling back at + // timestamp 0: nothing was prepared on the catalog entry. + post_all_lock_op_.write_type_ = PostWriteType::Rollback; acquire_all_lock_op_.ResetHandlerTxm(txm); invalidate_table_cache_op_.ResetHandlerTxm(txm); @@ -8778,9 +7820,9 @@ void InvalidateTableCacheCompositeOp::Forward(TransactionExecution *txm) txm->state_stack_.pop_back(); LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); assert(txm->state_stack_.empty()); - std::unique_lock lk(shards->invalidate_table_cache_op_mux_); + std::unique_lock lk(shards->op_pool_mux_); shards->invalidate_table_cache_op_pool_.emplace_back( - std::move(txm->invalidate_table_cache_composite_op_)); + std::move(txm->CompositeOpSlot())); } } diff --git a/tx_service/src/tx_trace.cpp b/tx_service/src/tx_trace.cpp index 62b73dbab..db8548c99 100644 --- a/tx_service/src/tx_trace.cpp +++ b/tx_service/src/tx_trace.cpp @@ -1403,6 +1403,9 @@ std::ostream &operator<<(std::ostream &outs, txservice::PostWriteType r) case txservice::PostWriteType::PostCommit: pwt = "PostCommit"; break; + case txservice::PostWriteType::Rollback: + pwt = "Rollback"; + break; default: pwt = "[Unknown Post Write Type]"; }; diff --git a/tx_service/src/tx_upsert_table_operation.cpp b/tx_service/src/tx_upsert_table_operation.cpp new file mode 100644 index 000000000..72cd2dc03 --- /dev/null +++ b/tx_service/src/tx_upsert_table_operation.cpp @@ -0,0 +1,1474 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +/** + * @file tx_upsert_table_operation.cpp + * @brief UpsertTableOp, the schema op of CREATE, DROP, TRUNCATE and ALTER + * TABLE: the steps every op shares, and each op's own path through its data + * store steps. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fault/fault_inject.h" +#include "local_cc_shards.h" +#include "sequences/sequences.h" +#include "sharder.h" +#include "store/data_store_handler.h" +#include "tx_execution.h" +#include "tx_operation.h" +#include "tx_service_common.h" +#include "tx_trace.h" + +namespace txservice +{ +namespace +{ +#ifndef NDEBUG +/** @brief Coroutine stacks with a guard page, to catch an overflow. */ +using CoroStackAllocator = boost::context::protected_fixedsize_stack; +#else +/** @brief Coroutine stacks. */ +using CoroStackAllocator = boost::context::fixedsize_stack; +#endif +} // namespace + +UpsertTableOp::UpsertTableOp(const std::string_view table_name_str, + TableEngine table_engine, + const std::string ¤t_image, + uint64_t curr_schema_ts, + const std::string &dirty_image, + OperationType op_type, + TransactionExecution *txm) + : SchemaOp(table_name_str, + table_engine, + current_image, + dirty_image, + curr_schema_ts, + op_type, + txm), + lock_cluster_config_op_(), + acquire_all_intent_op_(txm), + prepare_log_op_(txm), + post_all_intent_op_(txm), + unlock_cluster_config_op_(txm), + create_kv_table_op_(txm), + drop_kv_table_op_(&table_key_.Name(), OperationType::DropTable, txm), + truncate_kv_table_op_(txm), + upsert_kv_table_op_(&table_key_.Name(), op_type, txm), + update_sequence_table_op_(txm), + acquire_all_lock_op_(txm), + downgrade_lock_op_(txm), + commit_log_op_(txm), + clean_ccm_op_(txm), + post_all_lock_op_(txm), + clean_log_op_(txm), + read_cluster_result_(txm) +{ + assert(op_type_ == OperationType::CreateTable || + op_type_ == OperationType::DropTable || + op_type_ == OperationType::TruncateTable || + op_type_ == OperationType::Update); + + lock_cluster_config_op_.table_name_ = TableName(cluster_config_ccm_name_sv, + TableType::ClusterConfig, + TableEngine::None); + lock_cluster_config_op_.key_ = VoidKey::NegInfTxKey(); + lock_cluster_config_op_.rec_ = &cluster_conf_rec_; + lock_cluster_config_op_.hd_result_ = &read_cluster_result_; + + acquire_all_intent_op_.table_name_ = &catalog_ccm_name; + acquire_all_intent_op_.keys_.emplace_back(&table_key_); + acquire_all_intent_op_.cc_op_ = CcOperation::ReadForWrite; + acquire_all_intent_op_.protocol_ = CcProtocol::OCC; + + post_all_intent_op_.table_name_ = &catalog_ccm_name; + post_all_intent_op_.keys_.emplace_back(&table_key_); + post_all_intent_op_.recs_.push_back(&catalog_rec_); + post_all_intent_op_.op_type_ = op_type_; + post_all_intent_op_.write_type_ = PostWriteType::PrepareCommit; + + acquire_all_lock_op_.table_name_ = &catalog_ccm_name; + acquire_all_lock_op_.keys_.emplace_back(&table_key_); + acquire_all_lock_op_.cc_op_ = CcOperation::Write; + acquire_all_lock_op_.protocol_ = CcProtocol::Locking; + + downgrade_lock_op_.table_name_ = &catalog_ccm_name; + downgrade_lock_op_.keys_.emplace_back(&table_key_); + downgrade_lock_op_.recs_.push_back(&catalog_rec_); + downgrade_lock_op_.op_type_ = op_type_; + downgrade_lock_op_.write_type_ = PostWriteType::DowngradeLock; + + post_all_lock_op_.table_name_ = &catalog_ccm_name; + post_all_lock_op_.keys_.emplace_back(&table_key_); + post_all_lock_op_.recs_.push_back(&catalog_rec_); + post_all_lock_op_.op_type_ = op_type_; + post_all_lock_op_.write_type_ = PostWriteType::PostCommit; + + clean_ccm_op_.Clear(); + + TX_TRACE_ASSOCIATE(this, &acquire_all_intent_op_, "acquire_all_intent_op_"); + TX_TRACE_ASSOCIATE(this, &prepare_log_op_, "prepare_log_op_"); + TX_TRACE_ASSOCIATE(this, &post_all_intent_op_, "post_all_intent_op_"); + TX_TRACE_ASSOCIATE(this, &create_kv_table_op_, "create_kv_table_op_"); + TX_TRACE_ASSOCIATE(this, &drop_kv_table_op_, "drop_kv_table_op_"); + TX_TRACE_ASSOCIATE(this, &truncate_kv_table_op_, "truncate_kv_table_op_"); + TX_TRACE_ASSOCIATE(this, &upsert_kv_table_op_, "upsert_kv_table_op_"); + TX_TRACE_ASSOCIATE(this, &kv_retry_wait_op_, "kv_retry_wait_op_"); + TX_TRACE_ASSOCIATE( + this, &update_sequence_table_op_, "update_sequence_table_op_"); + TX_TRACE_ASSOCIATE(this, &acquire_all_lock_op_, "acquire_all_lock_op_"); + TX_TRACE_ASSOCIATE(this, &downgrade_lock_op_, "downgrade_lock_op_"); + TX_TRACE_ASSOCIATE(this, &commit_log_op_, "commit_log_op_"); + TX_TRACE_ASSOCIATE(this, &post_all_lock_op_, "post_all_lock_op_"); + TX_TRACE_ASSOCIATE(this, &clean_log_op_, "clean_log_op_"); + + is_force_finished = false; +} + +void UpsertTableOp::Forward(TransactionExecution *txm) +{ + // Runs the coroutine, from where it stands on entry, until it suspends + // again or its body returns. + switch (coro_status_) + { + case CoroStatus::kEmpty: + coro_status_ = CoroStatus::kRunning; + coro_ = boost::context::callcc( + std::allocator_arg, + CoroStackAllocator(kCoroStackSize), + [this, txm](boost::context::continuation &&caller) + { + caller_ = &caller; + Run(txm); + caller_ = nullptr; + coro_status_ = CoroStatus::kFinished; + return std::move(caller); + }); + break; + case CoroStatus::kRunning: + // The step the coroutine awaits has finished. A running coroutine's + // handle has left coro_, so an empty one here would mean a Forward() + // from inside the coroutine. + assert(coro_); + coro_ = coro_.resume(); + break; + case CoroStatus::kFinished: + // Never an entry state: Finish() pops the op and pools it as soon as + // the body returns, and Reset() empties it before its next use. + assert(false); + return; + } + + // Back from the coroutine: it suspended (still kRunning), or its body + // returned (now kFinished). A returned body ends the op here, on the tx's + // own stack, since the coroutine and its stack are gone. + if (coro_status_ == CoroStatus::kFinished) + { + assert(!coro_); + Finish(txm); + } +} + +void UpsertTableOp::SuspendUntilDone(TransactionExecution *txm, + const TransactionOperation &op) +{ + assert(caller_ != nullptr); + // A step stays on top of the tx's stack until its PostProcess pops it, + // which can happen inside Process() already, e.g. a DsUpsertTableOp under + // txservice_skip_kv. Suspending then would leave the op itself on top: the + // tx's Forward() loop would stop there, and the op would resume only on the + // processor's next round. + if (txm->state_stack_.back() == &op) + { + *caller_ = caller_->resume(); + } + // Resumed by Forward(), which the tx calls on the op on top of its stack. + assert(txm->state_stack_.back() == this); +} + +template +void UpsertTableOp::Await(TransactionExecution *txm, OpT &op) +{ + op_ = &op; + txm->PushOperation(&op); + txm->Process(op); + SuspendUntilDone(txm, op); +} + +template +void UpsertTableOp::Await(TransactionExecution *txm, + OpT &op, + const char *fault_point, + CcErrorCode fault_error) +{ + bool inject = false; + CODE_FAULT_INJECTOR(fault_point, { + LOG(INFO) << "FaultInject " << fault_point + << ", txn: " << txm->TxNumber(); + inject = true; + }); + if (!inject) + { + Await(txm, op); + return; + } + // The step fails without running. Nothing runs, so nothing is pushed or + // awaited: the caller reads the error at once. The reset clears what a + // pooled op's previous use left in the result. + op_ = &op; + op.hd_result_.Reset(); + op.hd_result_.SetError(fault_error); +} + +void UpsertTableOp::AwaitKvStep(TransactionExecution *txm, + DsUpsertTableOp &step) +{ + op_ = &step; + StartKvStep(txm, step); + SuspendUntilDone(txm, step); +} + +void UpsertTableOp::Run(TransactionExecution *txm) +{ + switch (entry_) + { + case Entry::kStart: + if (!RunUntilPrepareLog(txm)) + { + return; + } + [[fallthrough]]; + case Entry::kAfterPrepareLog: + if (!RunUntilCommitLog(txm)) + { + return; + } + [[fallthrough]]; + case Entry::kAfterCommitLog: + RunAfterCommitLog(txm); + break; + } +} + +bool UpsertTableOp::RunUntilPrepareLog(TransactionExecution *txm) +{ + // The cluster config read lock, held until the op's last post write all, + // so that every step sent to all node groups reaches the same ones. + Await(txm, lock_cluster_config_op_); + if (lock_cluster_config_op_.hd_result_->IsError()) + { + LOG(ERROR) << "Upsert table read cluster config failed, tx_number:" + << txm->TxNumber(); + MarkFailed(txm); + ForceFinish(txm); + return false; + } + + // The cluster gate admits new ops only here, where an op takes the cluster + // config read lock (EloqDoc docs/cdc-change-stream.md §6.2). An op past + // this step, or a recovered one, which enters after it, is never gated: + // those are what a pause drains. Nothing but the lock is held yet. + if (cluster_conf_rec_.GatesSchemaOps()) + { + LOG(WARNING) << "Upsert table on " << table_key_.Name().StringView() + << " refused: the cluster gate is paused, tx_number:" + << txm->TxNumber(); + txm->upsert_resp_->SetErrorCode(TxErrorCode::CLUSTER_GATED); + MarkFailed(txm); + ReleaseClusterConfig(txm); + result_ = UpsertResult::Failed; + return false; + } + + // Write intents on the table's catalog entry, on every node group. + DLOG(INFO) << "txn: " << txm->TxNumber() + << " process acquire_all_intent_op_"; + Await(txm, acquire_all_intent_op_); + if (acquire_all_intent_op_.fail_cnt_.load(std::memory_order_relaxed) > 0) + { + LOG(ERROR) << "Upsert table acquire write intent failed, tx_number:" + << txm->TxNumber(); + txm->upsert_resp_->SetErrorCode( + TxErrorCode::UPSERT_TABLE_ACQUIRE_WRITE_INTENT_FAIL); + // Fails to acquire the write intent on the schema. Since write + // intents only conflict with other writes, there must be another tx + // trying to modify the same table's schema. Stops the schema + // operation, releasing the write intents acquired, once: remaining + // ones, if there are any, will be recovered by individual cc nodes + // separately. No log was written, so none is cleaned. + MarkFailed(txm); + if (RunPostWriteAll(txm, false)) + { + ReleaseClusterConfig(txm); + } + // The op fails whether or not the node kept its leadership: a + // ForceFinish above reports it Failed as well (commit timestamp 0). + result_ = UpsertResult::Failed; + return false; + } + + // Assigns a commit timestamp to the tx state machine as the version of + // the new schema. Unlike the conventional commit ts, the schema's commit + // ts is not chosen via the SetTs stage. But it follows a similar set of + // rules: the new schema's version should be greater than 1) the current + // version, 2) the maximal commit ts of all tx's that have read the schema, + // and 3) the local time when the tx starts. + txm->commit_ts_ = txm->commit_ts_bound_ + 1; + for (size_t idx = 0; idx < acquire_all_intent_op_.upload_cnt_; ++idx) + { + const AcquireAllResult &acq_all_res = + acquire_all_intent_op_.hd_results_[idx].Value(); + uint64_t ts = + std::max(acq_all_res.commit_ts_ + 1, acq_all_res.last_vali_ts_ + 1); + txm->commit_ts_ = std::max(txm->commit_ts_, ts); + } + ResolveSchemaImages(txm->commit_ts_); + + // The prepare log. The op is guaranteed to proceed once it is flushed. + FillPrepareLogRequest(txm); + DLOG(INFO) << "txn: " << txm->TxNumber() << " process prepare_log_op_"; + Await(txm, + prepare_log_op_, + "upsert_table_prepare_log_fail", + CcErrorCode::WRITE_LOG_FAILED); + while (prepare_log_op_.hd_result_.IsError()) + { + if (prepare_log_op_.hd_result_.ErrorCode() != + CcErrorCode::LOG_CLOSURE_RESULT_UNKNOWN_ERR) + { + LOG(ERROR) << "Upsert table write prepare log failed, tx_number:" + << txm->TxNumber(); + // Fails to flush the prepare log. The schema operation is + // considered failed if the prepare log is not flushed. The commit + // ts is set to 0 to signal that the ending releases all write + // intents. + MarkFailed(txm); + txm->upsert_resp_->SetErrorCode( + TxErrorCode::UPSERT_TABLE_PREPARE_FAIL); + RunEnding(txm); + return false; + } + if (!txm->CheckLeaderTerm()) + { + LOG(ERROR) << "Upsert table write prepare log result unknown, " + "tx_number:" + << txm->TxNumber() + << ", not leader any more, stop retrying"; + // Not leader anymore, just quit. New leader will know whether + // prepare log succeeds and continue the rest if it does. Should + // not release the write intents. If prepare log is not written, + // the write intents will be released individually via orphan lock + // recovery mechanism. + txm->upsert_resp_->SetErrorCode( + TxErrorCode::LOG_SERVICE_UNREACHABLE); + result_ = UpsertResult::Failed; + return false; + } + // prepare log result unknown, keep retrying until getting a clear + // response, either success or failure, or the coordinator itself is no + // longer leader + LOG(WARNING) << "Upsert table write prepare log result unknown, " + "tx_number:" + << txm->TxNumber() << ", keep retrying"; + prepare_log_op_.log_closure_.LogRequest() + .mutable_write_log_request() + ->set_retry(true); + Await(txm, prepare_log_op_); + } + return true; +} + +bool UpsertTableOp::FaultAppliesToTable(const FaultEntry &entry) const +{ + return entry.table_name_.empty() || + entry.table_name_ == table_key_.Name().StringView(); +} + +void UpsertTableOp::TriggerTableFault( + [[maybe_unused]] const std::string &fault_name) +{ + CODE_FAULT_INJECTOR(fault_name, { + if (FaultAppliesToTable(*fault_entry)) + { + FaultInject::Instance().TriggerAction(fault_entry); + } + }); +} + +bool UpsertTableOp::RunUntilCommitLog(TransactionExecution *txm) +{ + TriggerTableFault("upsert_table_crash_after_prepare_log"); + + // Installs the dirty schema on every node group, and gets back a local + // view (pointer) of the committed and dirty schema. + DLOG(INFO) << "txn: " << txm->TxNumber() << " process post_all_intent_op_"; + Await(txm, post_all_intent_op_); + while (post_all_intent_op_.IsFailed()) + { + // After the prepare log is flushed, the schema op is guaranteed to + // proceed. Retry this step to install the dirty schema in the tx + // service, if the tx node is still the leader. The tx is also allowed + // to proceed if the tx is in the recovery mode and the tx node is a + // leader candidate. + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + // set catalog_rec_'s binary_value_ to image_str since it could be set + // to TableSchemaView pointer in localshard. + catalog_rec_.SetSchemaImage(image_str_); + Await(txm, post_all_intent_op_); + } + + CODE_FAULT_INJECTOR("upsert_table_hold_before_commit_lock", + { HoldBeforeCommitLock(txm); }); + + // The op's own steps, up to and including its commit log. + switch (op_type_) + { + case OperationType::CreateTable: + return CreateUntilCommitLog(txm); + case OperationType::Update: + return AlterUntilCommitLog(txm); + case OperationType::DropTable: + return DropUntilCommitLog(txm); + case OperationType::TruncateTable: + return TruncateUntilCommitLog(txm); + default: + // An UpsertTableOp runs only the four table ops. + assert(false); + return false; + } +} + +bool UpsertTableOp::CreateUntilCommitLog(TransactionExecution *txm) +{ + // The kv tables at the prepare-stage timestamp; the catalog row waits for + // the commit log, which fixes the timestamp it must carry. No commit log + // has decided the create yet, so a data store failure gives it up rather + // than retry. + PrepareCreateKvTables(txm); + Await(txm, create_kv_table_op_); + bool failed = create_kv_table_op_.hd_result_.IsError(); + + // The sequence table, also before the commit log. + if (!failed && PrepareSequenceStep()) + { + DLOG(INFO) << "txn: " << txm->TxNumber() + << " process update_sequence_table_op_"; + Await(txm, update_sequence_table_op_); + failed = update_sequence_table_op_.hd_result_.IsError(); + } + + if (failed) + { + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + if (!AbandonCreate(txm)) + { + return false; + } + // The commit log with commit timestamp 0 fails the op. The write + // intents are still held and rejecting the dirty schema needs no write + // lock, so the op skips the write locks, as ALTER TABLE's failure does. + return WriteCommitLog(txm); + } + + // The write locks, at the commit-stage timestamp, then the commit log. + if (!AcquireWriteLocks(txm)) + { + return false; + } + AssignCommitStageTs(txm); + return WriteCommitLog(txm); +} + +bool UpsertTableOp::DropUntilCommitLog(TransactionExecution *txm) +{ + // DROP TABLE: its data store steps come after the commit log. + if (!AcquireWriteLocks(txm)) + { + return false; + } + AssignCommitStageTs(txm); + return WriteCommitLog(txm); +} + +bool UpsertTableOp::TruncateUntilCommitLog(TransactionExecution *txm) +{ + // The table's data goes once no DML can reach it: after the write locks. + if (!AcquireWriteLocks(txm)) + { + return false; + } + AssignCommitStageTs(txm); + + if (!txservice_skip_kv && + !Sharder::Instance().GetDataStoreHandler()->IsSharedStorage()) + { + // Without shared storage each node group truncates its own store, + // after the commit log (TruncateAfterCommitLog). A node group that + // recovers from the commit log truncates its store in the replay, + // before it takes writes, and the later request finds it at the + // commit ts and skips it (KickoutCcEntryCc): no write it takes is + // wiped. + return WriteCommitLog(txm); + } + + // With shared storage the coordinator empties the table, before the commit + // log: a participant that recovers from the commit log, and takes writes + // into the emptied table, then finds no later step that wipes them. A + // truncate never rolls back after its prepare log, so each step is retried + // until it succeeds. + + // The cc maps first, so that no checkpoint writes a dirty entry of the old + // table into the kv table once it is dropped. + if (!CleanCcm(txm, false)) + { + return false; + } + + // Drops the kv table, which keeps its name. The catalog row waits for the + // commit log, which fixes the timestamp it carries. + PrepareTruncateKvTable(txm); + Await(txm, truncate_kv_table_op_); + while (truncate_kv_table_op_.hd_result_.IsError()) + { + if (!AwaitKvRetry(txm, + &truncate_kv_table_op_, + "kv table truncate", + truncate_kv_table_op_.hd_result_.ErrorMsg())) + { + return false; + } + Await(txm, truncate_kv_table_op_); + } + + // Seeds the table's sequences again. + if (PrepareSequenceStep()) + { + Await(txm, update_sequence_table_op_); + while (update_sequence_table_op_.hd_result_.IsError()) + { + if (!AwaitKvRetry(txm, + &update_sequence_table_op_, + "sequence table update", + update_sequence_table_op_.hd_result_.ErrorMsg())) + { + return false; + } + Await(txm, update_sequence_table_op_); + } + } + return WriteCommitLog(txm); +} + +bool UpsertTableOp::AlterUntilCommitLog(TransactionExecution *txm) +{ + // The new catalog row, before the commit log. The post write right after + // the prepare log installed the dirty schema in the tx service and + // returned a local view (pointer) of the committed and dirty schema. + AwaitUpsertKvTable(txm, OperationType::Update); + if (KvWriteFailed(txm)) + { + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + // ALTER TABLE writes its catalog row before the commit log, so a + // failure rolls the op back: commit timestamp 0 tells the commit log + // and the post write to reject the dirty schema. + // + // The commit log is still needed. Without it, a participant that fails + // between releasing its write intent and the coordinator's clean log + // would recover from the prepare log (whose commit timestamp is not 0) + // a write lock and a dirty schema that no coordinator releases or + // rejects. The write intent is still held here, and rejecting the + // dirty schema needs no write lock, so the op skips the write locks. + MarkFailed(txm); + return WriteCommitLog(txm); + } + + if (!AcquireWriteLocks(txm)) + { + return false; + } + return WriteCommitLog(txm); +} + +void UpsertTableOp::RunAfterCommitLog(TransactionExecution *txm) +{ + // Crashes the node with the commit log written and the op's data store + // step not yet done, which recovery must roll forward. + TriggerTableFault("upsert_table_crash_after_commit_log"); + bool go_on = true; + switch (op_type_) + { + case OperationType::CreateTable: + go_on = CreateAfterCommitLog(txm); + break; + case OperationType::DropTable: + go_on = DropAfterCommitLog(txm); + break; + case OperationType::TruncateTable: + go_on = TruncateAfterCommitLog(txm); + break; + case OperationType::Update: + // ALTER TABLE wrote its catalog row before the commit log: nothing + // else to do before the ending (PostCommit, or Rollback for a failed + // catalog row write). + ACTION_FAULT_INJECTOR("upsert_table_post_all_lock"); + break; + default: + // An UpsertTableOp runs only the four table ops. + assert(false); + break; + } + if (go_on) + { + RunEnding(txm); + } +} + +bool UpsertTableOp::CreateAfterCommitLog(TransactionExecution *txm) +{ + if (txm->commit_ts_ != tx_op_failed_ts_) + { + // The catalog row, now that the commit log fixed its timestamp; the + // kv tables exist since before the lock. Only the row: op type Update + // is the store's catalog-only write. Retried until the store answers. + AwaitUpsertKvTable(txm, OperationType::Update); + while (KvWriteFailed(txm)) + { + if (!AwaitKvRetry(txm, + &upsert_kv_table_op_, + "catalog row write", + upsert_kv_table_op_.hd_result_.ErrorMsg())) + { + return false; + } + AwaitUpsertKvTable(txm, OperationType::Update); + } + } + // The catalog row carries the commit-stage timestamp: the table is + // complete in the store. A create that gave up writes nothing and goes on + // to reject its dirty schema. + ACTION_FAULT_INJECTOR("upsert_table_post_all_lock"); + return true; +} + +bool UpsertTableOp::DropAfterCommitLog(TransactionExecution *txm) +{ + // Deletes the kv table, retried until the store answers. + AwaitUpsertKvTable(txm, OperationType::DropTable); + while (KvWriteFailed(txm)) + { + if (!AwaitKvRetry(txm, + &upsert_kv_table_op_, + "kv table drop", + upsert_kv_table_op_.hd_result_.ErrorMsg())) + { + return false; + } + AwaitUpsertKvTable(txm, OperationType::DropTable); + } + + // Deletes the table's sequences, retried until the store answers. + if (PrepareSequenceStep()) + { + Await(txm, update_sequence_table_op_); + while (update_sequence_table_op_.hd_result_.IsError()) + { + if (!AwaitKvRetry(txm, + &update_sequence_table_op_, + "sequence table update", + update_sequence_table_op_.hd_result_.ErrorMsg())) + { + return false; + } + Await(txm, update_sequence_table_op_); + } + } + + // Empties the table's cc maps on every node group; the ending's + // PostCommit then drops the empty maps. + return CleanCcm(txm, false); +} + +bool UpsertTableOp::TruncateAfterCommitLog(TransactionExecution *txm) +{ + if (!txservice_skip_kv && + !Sharder::Instance().GetDataStoreHandler()->IsSharedStorage()) + { + // Each node group empties its cc maps, then truncates its own store, + // catalog row included (clean_ccm_op_). + return CleanCcm(txm, true); + } + + // The catalog row, now that the commit log fixed its timestamp: only the + // row, op type Update, retried until the store answers. The data went + // before the commit log. + AwaitUpsertKvTable(txm, OperationType::Update); + while (KvWriteFailed(txm)) + { + if (!AwaitKvRetry(txm, + &upsert_kv_table_op_, + "catalog row write", + upsert_kv_table_op_.hd_result_.ErrorMsg())) + { + return false; + } + AwaitUpsertKvTable(txm, OperationType::Update); + } + return true; +} + +void UpsertTableOp::RunEnding(TransactionExecution *txm) +{ + // PostCommit, or Rollback for an op that failed, retried until it + // succeeds. + if (!RunPostWriteAll(txm, true)) + { + return; + } + + // The op's last post write all: the cluster config lock can go. + if (!ReleaseClusterConfig(txm)) + { + return; + } + + // The clean log, retried while the node is the leader. + FillCleanLogRequestCommon(txm, clean_log_op_); + DLOG(INFO) << "txn: " << txm->TxNumber() << " process clean_log_op_"; + Await(txm, clean_log_op_); + while (clean_log_op_.hd_result_.IsError() && txm->CheckLeaderTerm()) + { + clean_log_op_.log_closure_.LogRequest() + .mutable_write_log_request() + ->set_retry(true); + Await(txm, clean_log_op_); + } + result_ = CleanLogResult(txm); +} + +bool UpsertTableOp::AcquireWriteLocks(TransactionExecution *txm) +{ + DLOG(INFO) << "txn: " << txm->TxNumber() << " process acquire_all_lock_op_"; + Await(txm, acquire_all_lock_op_); + while (acquire_all_lock_op_.fail_cnt_.load(std::memory_order_relaxed) > 0) + { + // Fails to acquire the write lock. The schema operation can only roll + // forward after flushing the prepare log. Retries the request if the + // tx node is still the leader or the tx is in the recovery mode and + // the cc node is a leader candidate. + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + LOG(ERROR) << "Upsert table schema transaction failed to acquire write " + "lock, tx_number:" + << txm->TxNumber(); + if (acquire_all_lock_op_.IsDeadlock()) + { + // We downgrade catalog write lock to avoid blocking other + // transaction which acquiring catalog read lock. And then retry + // to acquire catalog write lock. + LOG(ERROR) << "Upsert table schema transaction deadlocks with " + "other transaction, downgrade write lock, " + "tx_number: " + << txm->TxNumber(); + Await(txm, downgrade_lock_op_); + while (true) + { + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + if (!downgrade_lock_op_.hd_result_.IsError()) + { + break; + } + Await(txm, downgrade_lock_op_); + } + } + Await(txm, acquire_all_lock_op_); + } + return true; +} + +bool UpsertTableOp::WriteCommitLog(TransactionExecution *txm) +{ + FillCommitLogRequest(txm); + DLOG(INFO) << "txn: " << txm->TxNumber() << " process commit_log_op_"; + Await(txm, commit_log_op_); + while (commit_log_op_.hd_result_.IsError()) + { + // Fails to flush the commit log. Retries the operation if the tx node + // is still the leader or the tx is in the recovery mode and the cc + // node is a leader candidate. + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + commit_log_op_.log_closure_.LogRequest() + .mutable_write_log_request() + ->set_retry(true); + Await(txm, commit_log_op_); + } + return true; +} + +bool UpsertTableOp::RunPostWriteAll(TransactionExecution *txm, bool retry) +{ + assert(post_all_lock_op_.write_type_ == (txm->commit_ts_ == tx_op_failed_ts_ + ? PostWriteType::Rollback + : PostWriteType::PostCommit)); + while (true) + { + DLOG(INFO) << "txn: " << txm->TxNumber() + << " process post_all_lock_op_"; + Await(txm, post_all_lock_op_); + if (!txm->CheckLeaderTerm()) + { + // The tx node is no longer the leader or leader candidate(during + // recovery), ForceFinish. + ForceFinish(txm); + return false; + } + + // The tx's modification of the schema has finished. If the tx has + // previously read the same schema and keeps a pointer in the read set + // to the cc entry of the schema, removes it from the read set. As a + // result, the tx will not try to release the read lock of the schema + // when committing. + for (size_t idx = 0; idx < acquire_all_intent_op_.upload_cnt_; ++idx) + { + const CcEntryAddr &schema_entry_addr = + acquire_all_intent_op_.hd_results_[idx].Value().local_cce_addr_; + if (schema_entry_addr.NodeGroupId() == txm->TxCcNodeId()) + { + txm->rw_set_.DedupRead(schema_entry_addr); + } + } + + if (!retry || !post_all_lock_op_.hd_result_.IsError()) + { + return true; + } + // post_all_lock_op_ returns an error: + // 1. if flush kv succeeds, the schema op is guaranteed to succeed and + // can only roll forward. Retry this step to install the committed + // schema and remove write locks; + // 2. if flush kx fails, the schema op has to roll backward. Retry this + // step to reject dirty schema and remove write locks. + DLOG(INFO) << "txn: " << txm->TxNumber() << " retry post_all_lock_op_, " + << post_all_lock_op_.hd_result_.ErrorMsg(); + } +} + +bool UpsertTableOp::ReleaseClusterConfig(TransactionExecution *txm) +{ + op_ = &unlock_cluster_config_op_; + StartUnlockClusterConfig(txm, unlock_cluster_config_op_); + SuspendUntilDone(txm, unlock_cluster_config_op_); + if (unlock_cluster_config_op_.hd_result_.IsError()) + { + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + // Releasing a local read lock should never fail + assert(false); + } + return true; +} + +void UpsertTableOp::HoldBeforeCommitLock(TransactionExecution *txm) +{ + static constexpr char kHold[] = "upsert_table_hold_before_commit_lock"; + static constexpr char kHeld[] = "upsert_table_held_before_commit_lock"; + LOG(INFO) << "FaultInject " << kHold << ": holding the schema op on table " + << table_key_.Name().StringView() + << " before its commit-stage write locks, txn: " + << txm->TxNumber(); + FaultInject::Instance().InjectFault(kHeld, ""); + kv_retry_wait_op_.SetBlockingWork( + []() + { + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (FaultInject::Entry(kHold) != nullptr && + std::chrono::steady_clock::now() < deadline) + { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return CcErrorCode::NO_ERROR; + }); + Await(txm, kv_retry_wait_op_); + FaultInject::Instance().InjectFault(kHeld, "remove"); + LOG(INFO) << "FaultInject " << kHold << ": released the schema op on table " + << table_key_.Name().StringView() << ", txn: " << txm->TxNumber(); +} + +bool UpsertTableOp::AwaitKvRetry(TransactionExecution *txm, + TransactionOperation *step, + std::string_view step_name, + const std::string &error) +{ + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + ArmKvRetryWait(txm, step, step_name, error); + Await(txm, kv_retry_wait_op_); + // The pause is over: the step runs again, if the node is still the leader. + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + return true; +} + +void UpsertTableOp::ForceFinish(TransactionExecution *txm) +{ + is_force_finished = true; + result_ = CleanLogResult(txm); +} + +UpsertResult UpsertTableOp::CleanLogResult(TransactionExecution *txm) +{ + CODE_FAULT_INJECTOR("alter_schema_term_changed", { + LOG(INFO) << "FaultInject alter_schema_term_changed"; + is_force_finished = true; + }); + if (txm->commit_ts_ == tx_op_failed_ts_) + { + // Flush kv error or fail to flush prepare_log. + return UpsertResult::Failed; + } + return is_force_finished ? UpsertResult::Unverified + : UpsertResult::Succeeded; +} + +void UpsertTableOp::Finish(TransactionExecution *txm) +{ + txm->upsert_resp_->Finish(result_); + txm->state_stack_.pop_back(); + assert(txm->state_stack_.empty()); + + catalog_rec_.Reset(); + LocalCcShards *shards = Sharder::Instance().GetLocalCcShards(); + std::unique_lock lk(shards->op_pool_mux_); + // Moves this op into the pool: nothing may touch it afterwards. + shards->table_schema_op_pool_.emplace_back( + std::move(txm->CompositeOpSlot())); +} + +void UpsertTableOp::Reset(const std::string_view table_name_str, + TableEngine table_engine, + const std::string ¤t_image, + uint64_t curr_schema_ts, + const std::string &dirty_image, + OperationType op_type, + TransactionExecution *txm) +{ + assert(op_type_ == OperationType::CreateTable || + op_type_ == OperationType::Update || + op_type_ == OperationType::DropTable || + op_type_ == OperationType::TruncateTable); + + // reset TransactionOperation + retry_num_ = RETRY_NUM; + is_running_ = false; + + // reset SchemaOp + table_key_.Name() = TableName(table_name_str.data(), + table_name_str.size(), + TableType::Primary, + table_engine); + catalog_rec_.SetSchemaImage(current_image); + catalog_rec_.SetDirtySchemaImage(dirty_image); + image_str_ = current_image; + dirty_image_str_ = dirty_image; + curr_schema_ts_ = curr_schema_ts; + + // reset UpsertTableOp + op_type_ = op_type; + op_ = nullptr; + + // reset op + read_cluster_result_.Reset(); + cluster_conf_rec_.Reset(); + lock_cluster_config_op_.Reset(); + lock_cluster_config_op_.key_ = VoidKey::NegInfTxKey(); + lock_cluster_config_op_.table_name_ = TableName(cluster_config_ccm_name_sv, + TableType::ClusterConfig, + TableEngine::None); + lock_cluster_config_op_.rec_ = &cluster_conf_rec_; + lock_cluster_config_op_.hd_result_ = &read_cluster_result_; + prepare_log_op_.Reset(); + unlock_cluster_config_op_.Reset(); + create_kv_table_op_.Reset(); + drop_kv_table_op_.Reset(); + truncate_kv_table_op_.Reset(); + upsert_kv_table_op_.Reset(); + ResetKvRetry(txm); + update_sequence_table_op_.Reset(); + commit_log_op_.Reset(); + clean_log_op_.Reset(); + + acquire_all_intent_op_.table_name_ = &catalog_ccm_name; + acquire_all_intent_op_.keys_.clear(); + acquire_all_intent_op_.keys_.emplace_back(&table_key_); + acquire_all_intent_op_.cc_op_ = CcOperation::ReadForWrite; + acquire_all_intent_op_.protocol_ = CcProtocol::OCC; + + post_all_intent_op_.table_name_ = &catalog_ccm_name; + post_all_intent_op_.keys_.clear(); + post_all_intent_op_.keys_.emplace_back(&table_key_); + post_all_intent_op_.recs_.clear(); + post_all_intent_op_.recs_.push_back(&catalog_rec_); + post_all_intent_op_.op_type_ = op_type_; + post_all_intent_op_.write_type_ = PostWriteType::PrepareCommit; + + upsert_kv_table_op_.alter_table_info_ = nullptr; + upsert_kv_table_op_.write_time_ = 0; + + acquire_all_lock_op_.table_name_ = &catalog_ccm_name; + acquire_all_lock_op_.keys_.clear(); + acquire_all_lock_op_.keys_.emplace_back(&table_key_); + acquire_all_lock_op_.cc_op_ = CcOperation::Write; + acquire_all_lock_op_.protocol_ = CcProtocol::Locking; + + downgrade_lock_op_.table_name_ = &catalog_ccm_name; + downgrade_lock_op_.keys_.clear(); + downgrade_lock_op_.keys_.emplace_back(&table_key_); + downgrade_lock_op_.recs_.clear(); + downgrade_lock_op_.recs_.push_back(&catalog_rec_); + downgrade_lock_op_.op_type_ = op_type_; + downgrade_lock_op_.write_type_ = PostWriteType::DowngradeLock; + + post_all_lock_op_.table_name_ = &catalog_ccm_name; + post_all_lock_op_.keys_.clear(); + post_all_lock_op_.keys_.emplace_back(&table_key_); + post_all_lock_op_.recs_.clear(); + post_all_lock_op_.recs_.push_back(&catalog_rec_); + post_all_lock_op_.op_type_ = op_type_; + post_all_lock_op_.write_type_ = PostWriteType::PostCommit; + + clean_ccm_op_.Clear(); + clean_ccm_op_.hd_result_.Reset(); + + // reset cc_handler_res txm + read_cluster_result_.ResetTxm(txm); + acquire_all_intent_op_.ResetHandlerTxm(txm); + prepare_log_op_.ResetHandlerTxm(txm); + post_all_intent_op_.ResetHandlerTxm(txm); + unlock_cluster_config_op_.ResetHandlerTxm(txm); + create_kv_table_op_.ResetHandlerTxm(txm); + drop_kv_table_op_.ResetHandlerTxm(txm); + truncate_kv_table_op_.ResetHandlerTxm(txm); + upsert_kv_table_op_.ResetHandlerTxm(txm); + update_sequence_table_op_.ResetHandlerTxm(txm); + acquire_all_lock_op_.ResetHandlerTxm(txm); + downgrade_lock_op_.ResetHandlerTxm(txm); + commit_log_op_.ResetHandlerTxm(txm); + clean_ccm_op_.ResetHandlerTxm(txm); + post_all_lock_op_.ResetHandlerTxm(txm); + clean_log_op_.ResetHandlerTxm(txm); + + is_force_finished = false; + assert(!coro_ && caller_ == nullptr); + coro_status_ = CoroStatus::kEmpty; + entry_ = Entry::kStart; + result_ = UpsertResult::Failed; +} + +void UpsertTableOp::FillPrepareLogRequest(TransactionExecution *txm) +{ + FillPrepareLogRequestCommon(txm, prepare_log_op_); + ::txlog::WriteLogRequest *prepare_log_rec = + prepare_log_op_.log_closure_.LogRequest().mutable_write_log_request(); + + auto &node_terms = *prepare_log_rec->mutable_node_terms(); + node_terms.clear(); + for (size_t idx = 0; idx < acquire_all_intent_op_.upload_cnt_; ++idx) + { + const AcquireAllResult &hres_val = + acquire_all_intent_op_.hd_results_[idx].Value(); + uint32_t ng_id = hres_val.local_cce_addr_.NodeGroupId(); + node_terms[ng_id] = hres_val.node_term_; + } +} + +void UpsertTableOp::FillCommitLogRequest(TransactionExecution *txm) +{ + FillCommitLogRequestCommon(txm, commit_log_op_); + + ::txlog::WriteLogRequest *commit_log_rec = + commit_log_op_.log_closure_.LogRequest().mutable_write_log_request(); + + // Commit timestamp 0 only for ALTER TABLE's failed catalog write or a + // create that gave up on its kv tables or sequence table update. Such an + // op is rolled back at timestamp 0: its DDL record is dropped, with no + // change event. + assert(txm->commit_ts_ != tx_op_failed_ts_ || + upsert_kv_table_op_.hd_result_.IsError() || + create_kv_table_op_.hd_result_.IsError() || + update_sequence_table_op_.hd_result_.IsError()); + const bool rolled_back = + post_all_lock_op_.write_type_ == PostWriteType::Rollback; + assert(rolled_back == (txm->commit_ts_ == tx_op_failed_ts_)); + commit_log_rec->set_commit_timestamp(txm->commit_ts_); + commit_log_rec->mutable_log_content() + ->mutable_schema_log() + ->set_rolled_back(rolled_back); +} + +void UpsertTableOp::AwaitUpsertKvTable(TransactionExecution *txm, + OperationType op_type) +{ + upsert_kv_table_op_.op_type_ = op_type; + upsert_kv_table_op_.table_schema_old_ = catalog_rec_.Schema(); + upsert_kv_table_op_.table_schema_ = catalog_rec_.DirtySchema(); + upsert_kv_table_op_.alter_table_info_ = nullptr; + upsert_kv_table_op_.write_time_ = txm->commit_ts_; + DLOG(INFO) << "txn: " << txm->TxNumber() << " process upsert_kv_table_op_"; + AwaitKvStep(txm, upsert_kv_table_op_); +} + +bool UpsertTableOp::KvWriteFailed(TransactionExecution *txm) +{ + bool write_failed = upsert_kv_table_op_.hd_result_.IsError(); + // Fails one write, then disarms itself, so that a retry must write. + CODE_FAULT_INJECTOR("upsert_table_kv_write_fail", { + LOG(INFO) << "FaultInject upsert_table_kv_write_fail, txn: " + << txm->TxNumber(); + FaultInject::Instance().InjectFault("upsert_table_kv_write_fail", + "remove"); + write_failed = true; + }); + return write_failed; +} + +bool UpsertTableOp::PrepareSequenceStep() +{ + const TableSchema *schema = op_type_ == OperationType::CreateTable + ? catalog_rec_.DirtySchema() + : catalog_rec_.Schema(); + // The sequence table holds two kinds of sequences. Every range-partitioned + // table, a base table or an index (indexes are always range-partitioned), + // has a range-id sequence: a create seeds it with the id of the table's + // initial range (Sequences::InitialRangePartitionIdOf), a range split + // takes new ids from it (LocalCcShards::GetNextRangePartitionId), a drop + // deletes it and a truncate seeds it again. A table with an auto-increment + // column also has that column's sequence. A hash-partitioned table without + // indexes or an auto-increment column has neither. + const bool need_update_sequence_table = + !table_key_.Name().IsHashPartitioned() || + !schema->IndexNames().empty() || schema->HasAutoIncrement(); + if (!need_update_sequence_table) + { + return false; + } + + update_sequence_table_op_.SetBlockingWork( + [this, schema]() + { + assert(txservice::Sequences::Initialized()); + bool succ = true; + const TableName &table_name = table_key_.Name(); + const std::vector index_names = schema->IndexNames(); + if (op_type_ == OperationType::CreateTable || + op_type_ == OperationType::TruncateTable) + { + if (!table_name.IsHashPartitioned()) + { + succ = Sequences::InitIdOfTableRangePartition( + table_name, + Sequences::InitialRangePartitionIdOf(table_name)); + } + succ = succ && + std::all_of( + index_names.begin(), + index_names.end(), + [](const TableName &index_name) + { + return Sequences::InitIdOfTableRangePartition( + index_name, + Sequences::InitialRangePartitionIdOf( + index_name)); + }); + } + else + { + assert(op_type_ == OperationType::DropTable); + if (!table_name.IsHashPartitioned()) + { + succ = Sequences::DeleteSequence( + table_name, SequenceType::RangePartitionId); + } + succ = + succ && std::all_of(index_names.begin(), + index_names.end(), + [](const TableName &index_name) + { + return Sequences::DeleteSequence( + index_name, + SequenceType::RangePartitionId); + }); + } + + if (succ && schema->HasAutoIncrement()) + { + succ = op_type_ == OperationType::DropTable + ? Sequences::DeleteSequence( + table_name, SequenceType::AutoIncrementColumn) + : Sequences::InitIdOfAutoIncrementColumn(table_name); + } + + // Fails the update once, then disarms itself: a create gives up, a + // drop retries. + CODE_FAULT_INJECTOR("upsert_table_update_sequence_fail", { + LOG(INFO) << "FaultInject upsert_table_update_sequence_fail"; + FaultInject::Instance().InjectFault( + "upsert_table_update_sequence_fail", "remove"); + succ = false; + }); + + if (!succ) + { + LOG(ERROR) << "UpsertTableOp: failed to update the sequence " + "table for table: " + << table_name.StringView(); + return CcErrorCode::UPDATE_SEQUENCE_TABLE_FAIL; + } + return CcErrorCode::NO_ERROR; + }); + return true; +} + +void UpsertTableOp::PrepareCreateKvTables(TransactionExecution *txm) +{ + assert(op_type_ == OperationType::CreateTable); + LOG(INFO) << "UpsertTableOp: create the kv tables of table " + << table_key_.Name().StringView() << ", txn: " << txm->TxNumber(); + + // The prepare-stage timestamp, which the in-memory ranges also carry + // (PostCommit); recovery from the prepare log creates them again with the + // same one. + const uint64_t version = txm->commit_ts_; + create_kv_table_op_.SetBlockingWork( + [this, version, txn = txm->TxNumber()]() + { + store::DataStoreHandler *const store_hd = + Sharder::Instance().GetLocalCcShards()->store_hd_; + if (store_hd == nullptr || store_hd->ByPassDataStore()) + { + return CcErrorCode::NO_ERROR; + } + const TableSchema *schema = catalog_rec_.DirtySchema(); + assert(schema != nullptr); + bool succ = store_hd->CreateKvTable(table_key_.Name(), version); + for (const TableName &index_name : schema->IndexNames()) + { + succ = succ && store_hd->CreateKvTable(index_name, version); + } + // Fails the creation once, after the tables are created, so that + // the create gives up with real tables to drop; then disarms + // itself. A table_name parameter aims it at one table + // (FaultAppliesToTable); the server's own creates would consume + // it otherwise. + CODE_FAULT_INJECTOR("upsert_table_create_kv_table_fail", { + if (FaultAppliesToTable(*fault_entry)) + { + LOG(INFO) + << "FaultInject upsert_table_create_kv_table_fail, " + "txn: " + << txn; + FaultInject::Instance().InjectFault( + "upsert_table_create_kv_table_fail", "remove"); + succ = false; + } + }); + return succ ? CcErrorCode::NO_ERROR : CcErrorCode::DATA_STORE_ERR; + }); +} + +void UpsertTableOp::PrepareTruncateKvTable(TransactionExecution *txm) +{ + assert(op_type_ == OperationType::TruncateTable); + const TableSchema *schema = catalog_rec_.Schema(); + assert(schema != nullptr); + LOG(INFO) << "UpsertTableOp: truncate the kv table of table " + << table_key_.Name().StringView() << ", txn: " << txm->TxNumber(); + truncate_kv_table_op_.SetBlockingWork( + [kv_table_name = schema->GetKVCatalogInfo()->kv_table_name_]() + { + store::DataStoreHandler *const store_hd = + Sharder::Instance().GetLocalCcShards()->store_hd_; + if (txservice_skip_kv || store_hd == nullptr || + store_hd->ByPassDataStore()) + { + return CcErrorCode::NO_ERROR; + } + return store_hd->DropKvTable(kv_table_name) + ? CcErrorCode::NO_ERROR + : CcErrorCode::DATA_STORE_ERR; + }); +} + +bool UpsertTableOp::CleanCcm(TransactionExecution *txm, bool base_table_only) +{ + assert(clean_ccm_op_.table_names_.empty()); + if (base_table_only) + { + clean_ccm_op_.table_names_.emplace_back( + table_key_.Name().StringView().data(), + table_key_.Name().StringView().size(), + table_key_.Name().Type(), + table_key_.Name().Engine()); + } + else + { + const TableSchema *table_old_schema = catalog_rec_.Schema(); + assert(table_old_schema->GetBaseTableName() == table_key_.Name()); + auto clean_ccm_names = table_old_schema->IndexNames(); + clean_ccm_names.emplace_back( + table_old_schema->GetBaseTableName().StringView().data(), + table_old_schema->GetBaseTableName().StringView().size(), + table_old_schema->GetBaseTableName().Type(), + table_old_schema->GetBaseTableName().Engine()); + clean_ccm_op_.table_names_ = std::move(clean_ccm_names); + } + clean_ccm_op_.clean_type_ = CleanType::CleanCcm; + clean_ccm_op_.commit_ts_ = txm->CommitTs(); + + LOG(INFO) << "UpsertTableOp: Clean all ccmap on all node groups, txn: " + << txm->TxNumber(); + + Await(txm, clean_ccm_op_); + while (clean_ccm_op_.hd_result_.IsError()) + { + if (!txm->CheckLeaderTerm()) + { + ForceFinish(txm); + return false; + } + LOG(ERROR) << "UpsertTableOp: failed to truncate table, err code: " + << (int) clean_ccm_op_.hd_result_.ErrorCode() + << ", err msg: " << clean_ccm_op_.hd_result_.ErrorMsg() + << ", txn: " << txm->TxNumber() << ", keep retrying"; + Await(txm, clean_ccm_op_); + } + + clean_ccm_op_.Clear(); + + // Clear write set before commit dirty schema. + txm->rw_set_.ClearWriteSet(table_key_.Name()); + txm->rw_set_.ClearReadSet(table_key_.Name()); + return true; +} + +void UpsertTableOp::AssignCommitStageTs(TransactionExecution *txm) +{ + assert(op_type_ == OperationType::CreateTable || + op_type_ == OperationType::DropTable || + op_type_ == OperationType::TruncateTable); + const uint64_t prepare_ts = txm->commit_ts_; + for (size_t idx = 0; idx < acquire_all_lock_op_.upload_cnt_; ++idx) + { + const AcquireAllResult &acq_all_res = + acquire_all_lock_op_.hd_results_[idx].Value(); + txm->commit_ts_ = std::max(txm->commit_ts_, + std::max(acq_all_res.last_vali_ts_ + 1, + acq_all_res.commit_ts_ + 1)); + } + DLOG(INFO) << "UpsertTableOp: table " << table_key_.Name().StringView() + << " commits at " << txm->commit_ts_ << ", prepared at " + << prepare_ts << ", txn: " << txm->TxNumber(); +} + +bool UpsertTableOp::AbandonCreate(TransactionExecution *txm) +{ + assert(op_type_ == OperationType::CreateTable); + const uint64_t prepare_ts = txm->commit_ts_; + LOG(ERROR) << "UpsertTableOp: the data store failed the " + << (create_kv_table_op_.hd_result_.IsError() + ? "kv table creation" + : "sequence table update") + << " of table " << table_key_.Name().StringView() + << " before the commit log; the create fails, txn: " + << txm->TxNumber(); + MarkFailed(txm); + txm->upsert_resp_->SetErrorCode(TxErrorCode::DATA_STORE_ERROR); + + // The store's drop-table path over the schema the kv tables were created + // from, at the version they carry. One attempt. + drop_kv_table_op_.table_schema_old_ = catalog_rec_.DirtySchema(); + drop_kv_table_op_.table_schema_ = nullptr; + drop_kv_table_op_.alter_table_info_ = nullptr; + drop_kv_table_op_.write_time_ = prepare_ts; + AwaitKvStep(txm, drop_kv_table_op_); + if (drop_kv_table_op_.hd_result_.IsError()) + { + LOG(WARNING) << "UpsertTableOp: failed to drop the kv tables of table " + << table_key_.Name().StringView() + << " after its create gave up; they stay orphaned, txn: " + << txm->TxNumber(); + } + if (!txm->CheckLeaderTerm()) + { + // No commit log yet: whoever recovers the op from its prepare log + // tries the kv tables again. + ForceFinish(txm); + return false; + } + return true; +} + +void UpsertTableOp::MarkFailed(TransactionExecution *txm) +{ + txm->commit_ts_ = tx_op_failed_ts_; + post_all_lock_op_.write_type_ = PostWriteType::Rollback; +} + +} // namespace txservice diff --git a/tx_service/tests/CMakeLists.txt b/tx_service/tests/CMakeLists.txt index 41fed6301..10ccc14a1 100644 --- a/tx_service/tests/CMakeLists.txt +++ b/tx_service/tests/CMakeLists.txt @@ -82,6 +82,9 @@ set(CATCH_MAIN_TESTS ApplyResponseBoundary-Test RemoteUploadTxCommands-Test TransactionImageLifetime-Test + CkptLockRegistry-Test + ClusterGate-Test + LogAgentCdc-Test ) if(WITH_DATA_STORE STREQUAL "ELOQDSS_ROCKSDB" OR @@ -135,6 +138,19 @@ add_test(NAME RangeScanMemory-Test COMMAND RangeScanMemory-Test) set_tests_properties(RangeScanMemory-Test PROPERTIES LABELS "RangeScanMemory-Test" TIMEOUT 90) +# --- Log service unit tests --- +# Their sources live beside the log service they test; they need only the +# logservice library, not the tx-service harness. +set(LOG_SERVICE_TESTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../../eloq_log_service/test) +foreach(name CdcScan-Test CdcLongPoll-Test LogStateSchemaOp-Test + LogStateClusterGate-Test) + add_executable(${name} ${LOG_SERVICE_TESTS_DIR}/${name}.cpp) + target_link_libraries(${name} PRIVATE logservice Catch2::Catch2WithMain) + set_property(TARGET ${name} PROPERTY CXX_STANDARD 20) + set_property(TARGET ${name} PROPERTY CXX_EXTENSIONS OFF) + catch_discover_tests(${name} PROPERTIES LABELS "${name}") +endforeach() + # --- Phase 2 cross-NG cluster test (own main; links cluster_harness) --- # Drives a real 2-node out-of-process cluster, so it links cluster_harness (the # TestCluster driver + generated WorkloadService stub) and must be built after diff --git a/tx_service/tests/CcEntry-Test.cpp b/tx_service/tests/CcEntry-Test.cpp index 07529ea74..908f385ee 100644 --- a/tx_service/tests/CcEntry-Test.cpp +++ b/tx_service/tests/CcEntry-Test.cpp @@ -40,6 +40,34 @@ TEST_CASE("CcEntry Init", "[cc-entry]") REQUIRE(entry.ArchiveRecordsCount() == 0); } +TEST_CASE("VersionedPayload installs a payload copy-on-write", "[cc-entry]") +{ + CcEntry, CompositeRecord, true, true> entry; + const CompositeRecord first(1); + const CompositeRecord second(2); + const CompositeRecord third(3); + + // No payload yet: the entry gets its own copy. + entry.payload_.SetCurrentPayload(&first); + CompositeRecord *own = entry.payload_.cur_payload_.get(); + REQUIRE(own != &first); + REQUIRE(std::get<0>(own->Tuple()) == 1); + + // Held by the entry alone: the content is copied in place. + entry.payload_.SetCurrentPayload(&second); + REQUIRE(entry.payload_.cur_payload_.get() == own); + REQUIRE(std::get<0>(own->Tuple()) == 2); + + // Held by a reader too: the entry gets a new payload, and the reader keeps + // the value it has. + std::shared_ptr> reader = + entry.payload_.VersionedCurrentPayload(); + entry.payload_.SetCurrentPayload(&third); + REQUIRE(entry.payload_.cur_payload_.get() != reader.get()); + REQUIRE(std::get<0>(reader->Tuple()) == 2); + REQUIRE(std::get<0>(entry.payload_.cur_payload_->Tuple()) == 3); +} + TEST_CASE("CcEntry ArchiveBeforeUpdate", "[cc-entry]") { CcEntry, CompositeRecord, true, true> entry; diff --git a/tx_service/tests/CheckpointFlush-Test.cpp b/tx_service/tests/CheckpointFlush-Test.cpp index be9cb8d97..88f5055e8 100644 --- a/tx_service/tests/CheckpointFlush-Test.cpp +++ b/tx_service/tests/CheckpointFlush-Test.cpp @@ -7,6 +7,7 @@ */ #include +#include #include #include @@ -34,6 +35,7 @@ #define private public #include "cc/local_cc_shards.h" #undef private +#include "cc/template_cc_map.h" #include "data_store_service_client_closure.h" #include "data_sync_task.h" #include "eloq_basic_catalog_factory.h" @@ -44,6 +46,8 @@ #include "harness/test_node.h" #include "include/mock/mock_catalog_factory.h" +DECLARE_uint64(shard_clean_release_kb); + using namespace std::chrono_literals; using namespace txservice; @@ -151,6 +155,154 @@ class ShardCleanerFixture CcShard *shard_{nullptr}; }; +#ifndef WITH_JEMALLOC +/** Heap memory a fixture owns, allocated and freed through mimalloc so the + * shard heap accounts for it. */ +using CleanerBuffer = std::unique_ptr; + +/** + * A map of LRU pages for exercising the shard cleaner against real heap + * accounting and the production LRU and scheduling loop. + * + * Each page owns a fixed, deterministic number of reclaimable bytes. Entry + * eligibility is covered by the CC-map tests; here each page represents one + * already-durable allocation, or, when not evictable, one the sweep visits but + * must leave on the chain, like a dirty, locked or pinned page in production. + */ +class CleanerPages + : public TemplateCcMap, CompositeRecord, true, true> +{ + /** One LRU page owning 64 KB of heap memory, the bytes its eviction + * reclaims. */ + struct Page : LruPage + { + /** + * Allocates the page's memory. + * @param map The map the page belongs to. + */ + explicit Page(CcMap *map) + : LruPage(map), buffer(mi_malloc(64 * 1024), &mi_free) + { + } + CleanerBuffer buffer; ///< Freed when the page is evicted. + }; + +public: + /** + * Creates @p count pages and appends them to the shard's LRU chain. + * @param shard The shard whose LRU chain and heap the pages live on. + * @param count The number of pages to create. + * @param evictable Whether a sweep may evict the pages; when false the + * sweep visits them and leaves them on the chain. + */ + CleanerPages(CcShard &shard, size_t count, bool evictable = true) + : TemplateCcMap(&shard, + 0, + TableName(std::string_view("cleaner_pages"), + TableType::Primary, + TableEngine::EloqDoc), + 1), + owner_(shard), + evictable_(evictable) + { + pages_.reserve(count); + for (size_t i = 0; i < count; ++i) + { + pages_.push_back(std::make_unique(this)); + REQUIRE(pages_.back()->buffer != nullptr); + owner_.UpdateLruList(pages_.back().get(), true); + } + } + + /** Detaches the pages still on the LRU chain and resets the cleaner's + * cursor, which may point at one of them. */ + ~CleanerPages() override + { + for (auto &page : pages_) + { + if (page->lru_next_ != nullptr) + { + owner_.DetachLru(page.get()); + } + } + owner_.ResetCleanStart(); + } + + /** + * The sweep's visit of one page: evicts it when the pages are evictable, + * otherwise leaves it on the chain. + * @param page The page the sweep is at. + * @return The number of pages reclaimed (1 or 0) and the next page. + */ + std::pair CleanPageAndReBalance( + LruPage *page, KickoutCcEntryCc * = nullptr, bool * = nullptr) override + { + LruPage *next = page->lru_next_; + ++visited_; + if (!evictable_) + { + return {0, next}; + } + static_cast(page)->buffer.reset(); + owner_.DetachLru(page); + ++cleaned_; + return {1, next}; + } + + /** @return The number of pages evicted so far. */ + size_t Cleaned() const + { + return cleaned_; + } + + /** @return The number of pages the sweep visited, evicted or not. */ + size_t Visited() const + { + return visited_; + } + +private: + CcShard &owner_; ///< The shard the pages are on. + const bool evictable_; ///< Whether a sweep may evict. + std::vector> pages_; ///< Every page created. + size_t cleaned_{0}; ///< Pages evicted so far. + size_t visited_{0}; ///< Pages the sweep visited. +}; + +/** + * Allocates heap memory until the shard heap is at @p ratio of its limit. + * @param heap The shard heap to fill. + * @param ratio The fraction of the limit to reach; above 1.0 puts the heap + * over its hard limit. + * @return The buffers holding that memory; dropping them frees it. + */ +std::vector FillCleanerHeap(CcShardHeap &heap, double ratio) +{ + std::vector buffers; + const int64_t target = static_cast(heap.MemoryLimit() * ratio); + int64_t allocated = 0; + heap.Full(&allocated); + REQUIRE(allocated < target); + while (allocated < target) + { + buffers.emplace_back(mi_malloc(8192), &mi_free); + REQUIRE(buffers.back() != nullptr); + heap.Full(&allocated); + } + return buffers; +} + +/** Puts the cleaner's release threshold back when the test ends. */ +struct RestoreReleaseKb +{ + uint64_t previous = FLAGS_shard_clean_release_kb; ///< Saved value. + ~RestoreReleaseKb() + { + FLAGS_shard_clean_release_kb = previous; + } +}; +#endif + class Watchdog { public: @@ -456,72 +608,560 @@ TEST_CASE("CcRequestList preserves intrusive links across every removal shape", REQUIRE_FALSE(second.InUse()); } -TEST_CASE("shard cleaner preserves a wake received during an active pass", - "[checkpoint-flush][shard-clean][sticky-wake]") +TEST_CASE("parks and wake-ups for an in-use cleaner only count and join", + "[checkpoint-flush][shard-clean]") { // These requests outlive local_shards, whose unprocessed queue retains // their addresses until fixture teardown. - ShardRequest parked_request; - ShardCleanCc clean_pass; - ShardCleanerFixture fixture(/*node_memory_limit_mb=*/0); + ShardRequest first; + ShardRequest second; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/1000); CcShard &shard = fixture.Shard(); - // Initialization may already have queued the cleaner when the zero-byte - // heap became full. Either way, this call leaves the owned cleaner in-use - // and queued without a processor consuming it. + // Leaves the owned cleaner in use and queued, with no processor consuming + // it, whether or not initialization had queued it already. shard.WakeUpShardCleanCc(); const size_t active_cleaner_queue_size = shard.QueueSize(); REQUIRE(active_cleaner_queue_size > 0); - // This second call must record a sticky wake instead of enqueueing another - // copy of the cleaner. + // Neither a second wake-up nor a park enqueues another copy. Each park is + // counted: that count, not the list, is how the cleaner later tells that + // a request found the heap at its limit after a release. shard.WakeUpShardCleanCc(); + first.Use(); + shard.EnqueueWaitListIfMemoryFull(&first); + second.Use(); + shard.EnqueueWaitListIfMemoryFull(&second); REQUIRE(shard.QueueSize() == active_cleaner_queue_size); + REQUIRE(shard.WaitListSizeForMemory() == 2); + REQUIRE(shard.MemoryParkCount() == 2); + REQUIRE(shard.DequeueWaitListAfterMemoryFree(/*deque_all=*/true)); +} +TEST_CASE( + "without cache replacement the cleaner releases below the limit and " + "sweeps nothing", + "[checkpoint-flush][shard-clean][headroom]") +{ + ShardRequest parked_request; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/1000); + CcShard &shard = fixture.Shard(); + /** Puts the cache-replacement flag back when the test ends. */ + struct RestoreCacheReplacement + { + bool previous = txservice_enable_cache_replacement; ///< Saved value. + ~RestoreCacheReplacement() + { + txservice_enable_cache_replacement = previous; + } + } restore; + txservice_enable_cache_replacement = false; + const size_t queued = shard.QueueSize(); parked_request.Use(); shard.EnqueueWaitListIfMemoryFull(&parked_request); REQUIRE(shard.WaitListSizeForMemory() == 1); + REQUIRE(shard.QueueSize() == queued + 1); - // A zero-byte heap is deterministically full. Marking defrag in-flight - // keeps the unproductive-pass branch from notifying a checkpointer, which - // is intentionally absent from this standalone fixture. - shard.GetShardHeap()->SetDefragHeapCcOnFly(true); + // The heap is far below the limit: the pass has nothing to sweep, releases + // at once, and its continuation finds nothing to do. The cursor never + // moves and no retry is deferred. clean_pass.Use(); REQUIRE_FALSE(clean_pass.Execute(shard)); - - // Execute consumed the sticky wake and queued another pass rather than - // returning true while the ordinary request remained parked. - REQUIRE(shard.WaitListSizeForMemory() == 1); - REQUIRE(shard.QueueSize() == active_cleaner_queue_size + 1); - REQUIRE_FALSE(shard.TakeShardCleanCcWakeUp()); + REQUIRE(shard.WaitListSizeForMemory() == 0); + REQUIRE(shard.CleanStart() == nullptr); + REQUIRE(clean_pass.Execute(shard)); + REQUIRE(shard.CleanRetryDeadline() == 0); } -TEST_CASE("shard cleaner clears a sticky wake after draining parked requests", - "[checkpoint-flush][shard-clean][sticky-wake]") +#if !defined(WITH_JEMALLOC) && !defined(RUNNING_TXSERVICE_CTEST) +TEST_CASE("shard cleaner evicts to the target and releases below the limit", + "[checkpoint-flush][shard-clean][headroom]") { ShardRequest parked_request; ShardCleanCc clean_pass; - ShardCleanerFixture fixture(/*node_memory_limit_mb=*/1000); + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + CleanerPages pages(shard, 64); + auto ballast = FillCleanerHeap(heap, 1.001); + REQUIRE(heap.Full()); + bool parked = false; + SECTION( + "a parked request is released as soon as the heap is below the " + "limit") + { + parked_request.Use(); + shard.EnqueueWaitListIfMemoryFull(&parked_request); + parked = true; + } + SECTION("a full heap starts cleaning without a parked request") + { + } - shard.WakeUpShardCleanCc(); - const size_t active_cleaner_queue_size = shard.QueueSize(); - REQUIRE(active_cleaner_queue_size > 0); - shard.WakeUpShardCleanCc(); - REQUIRE(shard.QueueSize() == active_cleaner_queue_size); + clean_pass.Use(); + size_t rounds = 0; + bool finished = false; + do + { + const size_t cleaned_before = pages.Cleaned(); + finished = clean_pass.Execute(shard); + REQUIRE(pages.Cleaned() - cleaned_before <= 10); + int64_t allocated = 0; + const bool full = heap.Full(&allocated); + if (!finished) + { + // Eviction continues to the target. The request, if any, is + // released on the first execution that finds the heap below the + // limit, and nothing parks after that, so it stays released. + REQUIRE(allocated > + static_cast(heap.MemoryLimit() * 0.95)); + if (parked && !full) + { + REQUIRE(shard.WaitListSizeForMemory() == 0); + } + } + else + { + REQUIRE(allocated <= + static_cast(heap.MemoryLimit() * 0.95)); + REQUIRE(shard.WaitListSizeForMemory() == 0); + } + REQUIRE(++rounds <= 64); + } while (!finished); + REQUIRE(pages.Cleaned() > 10); + REQUIRE(pages.Cleaned() < 64); + REQUIRE(shard.CleanRetryDeadline() == 0); + REQUIRE(parked_request.AbortError() == CcErrorCode::NO_ERROR); +} - parked_request.Use(); - shard.EnqueueWaitListIfMemoryFull(&parked_request); +TEST_CASE("shard cleaner drains more than twenty waiters after a short sweep", + "[checkpoint-flush][shard-clean][headroom]") +{ + std::vector> requests; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); + CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + CleanerPages pages(shard, 1); + for (size_t i = 0; i < 45; ++i) + { + requests.push_back(std::make_unique(true)); + requests.back()->Use(); + shard.EnqueueWaitListIfMemoryFull(requests.back().get()); + } + auto ballast = FillCleanerHeap(heap, 0.97); + REQUIRE_FALSE(heap.Full()); clean_pass.Use(); + // The only page goes in the first execution and the sweep is at the tail, + // still above the target. Nothing parks after a release, so a batch of 20 + // goes out per execution until the list is empty. + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(pages.Cleaned() == 1); + REQUIRE(shard.WaitListSizeForMemory() == 25); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.WaitListSizeForMemory() == 5); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.WaitListSizeForMemory() == 0); + // Nothing parked and nothing left to evict: the pass ends and the cleaner + // goes idle without deferring a retry. One page is under the release + // threshold, so the pass would ask for a checkpoint (none exists in this + // fixture, hence the defrag flag). + heap.SetDefragHeapCcOnFly(true); REQUIRE(clean_pass.Execute(shard)); + REQUIRE(shard.CleanRetryDeadline() == 0); + for (const auto &request : requests) + { + REQUIRE(request->AbortError() == CcErrorCode::NO_ERROR); + } +} + +TEST_CASE( + "an unproductive pass releases once, then aborts opt-in requests " + "that park again and defers", + "[checkpoint-flush][shard-clean][headroom]") +{ + ShardRequest upload(/*abort_if_oom=*/true); + ShardRequest ordinary; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); + CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + // No checkpointer exists in this fixture. Simulate one already handling + // pressure so the test isolates waiter handling at the end of the pass. + heap.SetDefragHeapCcOnFly(true); + auto ballast = FillCleanerHeap(heap, 0.99); + upload.Use(); + ordinary.Use(); + shard.EnqueueWaitListIfMemoryFull(&upload); + shard.EnqueueWaitListIfMemoryFull(&ordinary); + REQUIRE_FALSE(heap.Full()); + clean_pass.Use(); + + // Nothing is evictable, but the heap is below the limit and nothing has + // parked since a release, so both requests get one try. + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.WaitListSizeForMemory() == 0); + REQUIRE(upload.AbortError() == CcErrorCode::NO_ERROR); + + // Both find the heap full again and park. At the tail nothing backs a + // second try: the pass ends, the opt-in request is aborted because the + // pass freed nothing, the ordinary one stays parked, and the cleaner + // defers instead of releasing again. + shard.EnqueueWaitListIfMemoryFull(&upload); + shard.EnqueueWaitListIfMemoryFull(&ordinary); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(upload.AbortError() == CcErrorCode::OUT_OF_MEMORY); + REQUIRE_FALSE(upload.InUse()); + REQUIRE(ordinary.AbortError() == CcErrorCode::NO_ERROR); + REQUIRE(shard.WaitListSizeForMemory() == 1); + REQUIRE(shard.CleanBackoff() == 1); + const uint64_t deadline = shard.CleanRetryDeadline(); + REQUIRE(deadline != 0); + + // The timer thread enqueues the shard's cleaner once the deadline passes, + // and only then. + const size_t queued = shard.QueueSize(); + shard.FireCleanRetryIfDue(deadline - 1); + REQUIRE(shard.CleanRetryDeadline() == deadline); + REQUIRE(shard.QueueSize() == queued); + shard.FireCleanRetryIfDue(deadline); + REQUIRE(shard.CleanRetryDeadline() == 0); + REQUIRE(shard.QueueSize() == queued + 1); +} + +TEST_CASE( + "after a request hits the limit, the next release waits for the " + "configured reclaim, not for the target", + "[checkpoint-flush][shard-clean][headroom]") +{ + ShardRequest request; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); + CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + CleanerPages pages(shard, 64); + RestoreReleaseKb restore; + // 1 MB: sixteen of the fixture's 64 KB pages, more than one batch of ten + // frees, and far less than the 5% target (3.2 MB). + FLAGS_shard_clean_release_kb = 1024; + // Just over the limit, so that even a single evicted page brings the heap + // back below it. + auto ballast = FillCleanerHeap(heap, 1.0005); + REQUIRE(heap.Full()); + request.Use(); + shard.EnqueueWaitListIfMemoryFull(&request); + clean_pass.Use(); + + // The first batch of pages brings the heap below the limit and the + // request is released at once: nothing hit the limit since the pass + // started. + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(pages.Cleaned() >= 1); + REQUIRE(pages.Cleaned() <= 10); + REQUIRE_FALSE(heap.Full()); + REQUIRE(shard.WaitListSizeForMemory() == 0); + + // It parks again: the room that release handed out is used up. The next + // release waits until the pass has reclaimed the threshold, which takes + // more than one batch of pages; the heap stays below the target all along. + shard.EnqueueWaitListIfMemoryFull(&request); + const size_t cleaned_at_park = pages.Cleaned(); + size_t rounds = 0; + while (shard.WaitListSizeForMemory() == 1) + { + const size_t cleaned_before = pages.Cleaned(); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(pages.Cleaned() > cleaned_before); + int64_t allocated = 0; + heap.Full(&allocated); + REQUIRE(heap.NeedMoreCleaning(allocated)); + const size_t reclaimed_pages = pages.Cleaned() - cleaned_at_park; + if (reclaimed_pages <= 12) + { + // At most 768 KB reclaimed: still held. + REQUIRE(shard.WaitListSizeForMemory() == 1); + } + else if (reclaimed_pages >= 20) + { + // 1.25 MB or more reclaimed: released. + REQUIRE(shard.WaitListSizeForMemory() == 0); + } + REQUIRE(++rounds <= 8); + } + REQUIRE(rounds >= 2); + + // Nothing parked any more: the pass evicts on and goes idle without + // deferring a retry. + rounds = 0; + while (!clean_pass.Execute(shard)) + { + REQUIRE(shard.WaitListSizeForMemory() == 0); + REQUIRE(++rounds <= 64); + } + REQUIRE(shard.CleanRetryDeadline() == 0); + REQUIRE(request.AbortError() == CcErrorCode::NO_ERROR); +} + +TEST_CASE("at the tail a batch that parks again defers the cleaner", + "[checkpoint-flush][shard-clean][headroom]") +{ + ShardRequest request; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); + CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + CleanerPages pages(shard, 1); + auto ballast = FillCleanerHeap(heap, 0.97); + request.Use(); + shard.EnqueueWaitListIfMemoryFull(&request); + clean_pass.Use(); + + // The only page goes, the sweep is at the tail, and the request gets one + // try with the room that made. + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(pages.Cleaned() == 1); + REQUIRE(shard.WaitListSizeForMemory() == 0); + + // It parks again. Nothing backs another try at the tail, so the pass ends + // and the cleaner defers. The pass freed something, so no request is + // aborted; one page is under the release threshold, so the pass counts as + // not keeping up: it would ask for a checkpoint (none exists in this + // fixture, hence the defrag flag) and its retry backs off. + heap.SetDefragHeapCcOnFly(true); + shard.EnqueueWaitListIfMemoryFull(&request); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.WaitListSizeForMemory() == 1); + REQUIRE(request.AbortError() == CcErrorCode::NO_ERROR); + REQUIRE(shard.CleanRetryDeadline() != 0); + REQUIRE(shard.CleanBackoff() == 1); +} + +TEST_CASE( + "a zero release threshold means any reclaim and still ends the pass at " + "the tail", + "[checkpoint-flush][shard-clean][headroom]") +{ + ShardRequest request; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); + CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + CleanerPages pages(shard, 1); + RestoreReleaseKb restore; + FLAGS_shard_clean_release_kb = 0; + // No checkpointer exists in this fixture; see above. + heap.SetDefragHeapCcOnFly(true); + auto ballast = FillCleanerHeap(heap, 0.97); + request.Use(); + shard.EnqueueWaitListIfMemoryFull(&request); + clean_pass.Use(); + + // The only page goes and the request gets its try with that room. + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(pages.Cleaned() == 1); + REQUIRE(shard.WaitListSizeForMemory() == 0); + + // It parks again at the tail. Zero reclaimed since that release is not + // "any amount": nothing is released, the pass ends and the cleaner defers + // instead of releasing the same request over and over. The one page it + // did reclaim is any amount, so the pass counts as productive. + shard.EnqueueWaitListIfMemoryFull(&request); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.WaitListSizeForMemory() == 1); + REQUIRE(shard.CleanStart() == nullptr); + REQUIRE(shard.CleanRetryDeadline() != 0); + REQUIRE(shard.CleanBackoff() == 0); +} + +TEST_CASE( + "after a limit hit the batch that reaches the tail releases what the pass " + "reclaimed even under the threshold, and a park at the tail defers", + "[checkpoint-flush][shard-clean][headroom]") +{ + ShardRequest request; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); + CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + // Fourteen 64 KB pages are 896 KB, under the 1 MB threshold set below, so + // the only release after the limit hit is the one the tail rule allows. + CleanerPages pages(shard, 14); + RestoreReleaseKb restore; + FLAGS_shard_clean_release_kb = 1024; + // Above the target from the start: eviction here is driven by the limit + // hit alone. + auto ballast = FillCleanerHeap(heap, 0.93); + int64_t allocated = 0; + REQUIRE_FALSE(heap.Full(&allocated)); + REQUIRE_FALSE(heap.NeedMoreCleaning(allocated)); + request.Use(); + shard.EnqueueWaitListIfMemoryFull(&request); + clean_pass.Use(); - // Available memory released the parked request. The concurrent wake has - // therefore already been served and must not survive into a later pass. + // Nothing to evict above the target: the request is released as is. + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(pages.Cleaned() == 0); REQUIRE(shard.WaitListSizeForMemory() == 0); - REQUIRE(shard.QueueSize() == active_cleaner_queue_size + 1); - REQUIRE_FALSE(shard.TakeShardCleanCcWakeUp()); + + // It parks again. Every execution evicts a batch and holds the release, + // since these pages cannot reach the threshold, until the batch that + // reaches the tail: no later batch could add to the reclaim, so what was + // reclaimed goes out instead of waiting for the retry. + shard.EnqueueWaitListIfMemoryFull(&request); + size_t rounds = 0; + do + { + const size_t cleaned_before = pages.Cleaned(); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(pages.Cleaned() > cleaned_before); + REQUIRE(shard.WaitListSizeForMemory() == (shard.OutOfMemory() ? 0 : 1)); + REQUIRE(++rounds <= 14); + } while (!shard.OutOfMemory()); + REQUIRE(pages.Cleaned() == 14); + + // Parked once more at the tail. Nothing was reclaimed since that release, + // so nothing is released: the pass ends and the cleaner defers. It freed + // memory and the heap is below the limit, so the request is not aborted; + // 896 KB is under the 1 MB threshold, so the pass counts as not keeping + // up: it would ask for a checkpoint (none exists in this fixture, hence + // the defrag flag) and its retry backs off. + heap.SetDefragHeapCcOnFly(true); + shard.EnqueueWaitListIfMemoryFull(&request); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.WaitListSizeForMemory() == 1); + REQUIRE(request.AbortError() == CcErrorCode::NO_ERROR); + REQUIRE(shard.CleanStart() == nullptr); + REQUIRE(shard.CleanRetryDeadline() != 0); + REQUIRE(shard.CleanBackoff() == 1); } +TEST_CASE("the backoff clears when the cleaner goes idle", + "[checkpoint-flush][shard-clean][headroom]") +{ + ShardRequest request; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); + CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + // Pages the sweep visits but cannot evict. + CleanerPages pages(shard, 2, /*evictable=*/false); + // No checkpointer exists in this fixture; see above. + heap.SetDefragHeapCcOnFly(true); + auto ballast = FillCleanerHeap(heap, 1.001); + REQUIRE(heap.Full()); + request.Use(); + shard.EnqueueWaitListIfMemoryFull(&request); + clean_pass.Use(); + + // Two passes that reclaim nothing: each visits both pages, ends at the + // tail with the request still parked, and backs off one step further. + for (uint8_t expected_backoff = 1; expected_backoff <= 2; + ++expected_backoff) + { + size_t rounds = 0; + do + { + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(++rounds <= 8); + } while (shard.CleanStart() != nullptr); + REQUIRE(shard.WaitListSizeForMemory() == 1); + REQUIRE(shard.CleanBackoff() == expected_backoff); + } + + // Traffic frees memory and the heap drops far below the limit. The next + // pass releases the request at once, then finds nothing parked and + // nothing to evict and goes idle: the episode is over and the backoff + // clears, so the next one starts from the base delay. + ballast.clear(); + REQUIRE_FALSE(heap.Full()); + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.WaitListSizeForMemory() == 0); + REQUIRE(clean_pass.Execute(shard)); + REQUIRE(shard.CleanBackoff() == 0); + REQUIRE(request.AbortError() == CcErrorCode::NO_ERROR); +} + +TEST_CASE( + "a flush during a pass adds one pass, unproductive passes back off, " + "and a flush wake resets the backoff", + "[checkpoint-flush][shard-clean][headroom]") +{ + ShardRequest request; + ShardCleanCc clean_pass; + ShardCleanerFixture fixture(/*node_memory_limit_mb=*/64); + CcShard &shard = fixture.Shard(); + CcShardHeap &heap = *shard.GetShardHeap(); + // Pages the sweep visits but cannot evict. + CleanerPages pages(shard, 64, /*evictable=*/false); + // No checkpointer exists in this fixture; see above. + heap.SetDefragHeapCcOnFly(true); + auto ballast = FillCleanerHeap(heap, 1.001); + REQUIRE(heap.Full()); + request.Use(); + shard.EnqueueWaitListIfMemoryFull(&request); + clean_pass.Use(); + + // Drives the pass to its end, which resets the cursor. The heap stays + // full, so the request stays parked throughout. + auto run_pass_to_end = [&]() + { + size_t rounds = 0; + do + { + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.WaitListSizeForMemory() == 1); + REQUIRE(++rounds <= 200); + } while (shard.CleanStart() != nullptr); + }; + + // A flush completes mid-pass. The pass is not rewound: it finishes its + // sweep, then runs once more from the head instead of deferring. + REQUIRE_FALSE(clean_pass.Execute(shard)); + REQUIRE(shard.CleanStart() != nullptr); + REQUIRE_FALSE(shard.OutOfMemory()); + shard.OnDirtyDataFlushed(); + run_pass_to_end(); + REQUIRE(pages.Visited() == 64); + REQUIRE(shard.CleanRetryDeadline() == 0); + run_pass_to_end(); + REQUIRE(pages.Visited() == 128); + + // That second pass was unproductive and nothing arrived during it: the + // cleaner defers with the first backoff step. + REQUIRE(shard.CleanRetryDeadline() != 0); + REQUIRE(shard.CleanBackoff() == 1); + + // A flush completion while deferred runs the cleaner now and resets the + // backoff. + const size_t queued = shard.QueueSize(); + shard.OnDirtyDataFlushed(); + REQUIRE(shard.CleanRetryDeadline() == 0); + REQUIRE(shard.CleanBackoff() == 0); + REQUIRE(shard.QueueSize() == queued + 1); + + // Consecutive unproductive passes double the delay up to the cap. + for (uint8_t expected_backoff : {1, 2, 3, 4, 4}) + { + run_pass_to_end(); + REQUIRE(shard.CleanBackoff() == expected_backoff); + const uint64_t deadline = shard.CleanRetryDeadline(); + REQUIRE(deadline != 0); + const uint64_t now_us = + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); + const uint64_t delay_us = uint64_t{100'000} << expected_backoff; + REQUIRE(deadline > now_us); + REQUIRE(deadline - now_us <= delay_us); + REQUIRE(deadline - now_us > delay_us / 2); + shard.FireCleanRetryIfDue(deadline); + REQUIRE(shard.CleanRetryDeadline() == 0); + } + REQUIRE(request.AbortError() == CcErrorCode::NO_ERROR); +} +#endif + TEST_CASE("checkpoint publication honors backend and MVCC durability bounds", "[checkpoint-flush][durability]") { @@ -1993,21 +2633,36 @@ TEST_CASE("live shard dispatch covers checkpoint and intrusive wait lists", abort_first.Use(); retained.Use(); abort_last.Use(); - shard->EnqueueWaitListIfMemoryFull(&abort_first); - shard->EnqueueWaitListIfMemoryFull(&retained); - shard->EnqueueWaitListIfMemoryFull(&abort_last); - REQUIRE(shard->WaitListSizeForMemory() == 3); - - shard->AbortRequestsAfterMemoryFree(); - REQUIRE(abort_first.AbortError() == CcErrorCode::OUT_OF_MEMORY); - REQUIRE(abort_last.AbortError() == CcErrorCode::OUT_OF_MEMORY); - REQUIRE(shard->WaitListSizeForMemory() == 1); - REQUIRE(shard->DequeueWaitListAfterMemoryFree(/*deque_all=*/true)); - ShardRequest schema_waiter; schema_waiter.Use(); - shard->EnqueueWaitListIfSchemaMismatch(&schema_waiter); - shard->DequeueWaitListAfterSchemaUpdated(); + size_t parked_before_abort = 0; + size_t parked_after_abort = 0; + bool memory_wait_list_drained = false; + // Parking schedules the cleaner. Manipulate the intrusive lists on their + // owning shard so it cannot drain them concurrently with this test. + WaitableCc manage_waiters( + [&](CcShard &owner) + { + owner.EnqueueWaitListIfMemoryFull(&abort_first); + owner.EnqueueWaitListIfMemoryFull(&retained); + owner.EnqueueWaitListIfMemoryFull(&abort_last); + parked_before_abort = owner.WaitListSizeForMemory(); + owner.AbortRequestsAfterMemoryFree(); + parked_after_abort = owner.WaitListSizeForMemory(); + memory_wait_list_drained = + owner.DequeueWaitListAfterMemoryFree(/*deque_all=*/true); + owner.EnqueueWaitListIfSchemaMismatch(&schema_waiter); + owner.DequeueWaitListAfterSchemaUpdated(); + return true; + }); + shards->EnqueueToCcShard(0, &manage_waiters); + manage_waiters.Wait(); + REQUIRE_FALSE(manage_waiters.IsError()); + REQUIRE(parked_before_abort == 3); + REQUIRE(abort_first.AbortError() == CcErrorCode::OUT_OF_MEMORY); + REQUIRE(abort_last.AbortError() == CcErrorCode::OUT_OF_MEMORY); + REQUIRE(parked_after_abort == 1); + REQUIRE(memory_wait_list_drained); const auto deadline = std::chrono::steady_clock::now() + 5s; while ( diff --git a/tx_service/tests/CkptLockRegistry-Test.cpp b/tx_service/tests/CkptLockRegistry-Test.cpp new file mode 100644 index 000000000..d006396df --- /dev/null +++ b/tx_service/tests/CkptLockRegistry-Test.cpp @@ -0,0 +1,527 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +// Let Catch provide main(): +#include +#include +#include +#include +#include +#include +#include + +#include "cc/cc_req_base.h" +#include "cc/non_blocking_lock.h" +#include "cc_entry.h" +#include "cc_protocol.h" +#include "cc_shard.h" +#include "include/mock/mock_catalog_factory.h" +#include "local_cc_shards.h" +#include "template_cc_map.h" +#include "tx_key.h" // CompositeKey +#include "tx_record.h" // CompositeRecord +#include "type.h" + +/** + * @file + * @brief Which write locks hold the checkpoint back (CcShard::ActiveTxMinTs / + * ActiveTxMaxTs), and for how long. + * + * Write locks on data tables and catalog entries count, until release or + * downgrade to a write intent. Range, bucket and cluster-config write locks, + * intents and read locks never count, and no later registration of another + * lock changes either. While a tx counts, the bound is its latest write-lock + * grant in the shard, on any table: the tx commits above every grant. + */ + +namespace txservice +{ +namespace +{ +constexpr NodeGroupId kNg = 0; +constexpr int64_t kTerm = 1; +/** @brief Marks an ActiveTxMinTs out-parameter the call did not set. */ +constexpr TxNumber kNoTxn = UINT64_MAX; + +MockCatalogFactory mock_catalog_factory{}; + +using TestKey = CompositeKey; +using TestRec = CompositeRecord; +using TestBaseMap = TemplateCcMap; + +/** @brief Every (tx, entry) lock the test has taken and not yet released. */ +std::vector> &Held() +{ + static std::vector> held; + return held; +} + +/** + * @brief Minimal CcRequestBase carrying a tx number. NonBlockingLock only + * reads Txn() off the requests it is given, so Execute() is never reached. + */ +struct StubCcRequest : public CcRequestBase +{ + explicit StubCcRequest(TxNumber txn) + { + tx_number_ = txn; + proto_ = CcProtocol::Locking; + isolation_level_ = IsolationLevel::RepeatableRead; + } + + bool Execute(CcShard &) override + { + return false; + } +}; + +/** + * @brief A cc map of a chosen table type whose entries the test locks + * directly, registering each lock the way CcMap::AcquireCceKeyLock does. + */ +class RegistryTestMap : public TestBaseMap +{ +public: + RegistryTestMap(CcShard *shard, TableType type) + : TestBaseMap(shard, + kNg, + TableName(std::string("b1_") + + std::to_string(static_cast(type)), + type, + TableEngine::EloqSql), + 1, + nullptr, + true) + { + } + + /** + * @brief Takes a lock on a key of this map and registers it in the + * shard's lock registry. + * + * @param k The key's integer part. + * @param txn The tx taking the lock. + * @param type The lock type: WriteLock, WriteIntent or ReadLock. + * @return The locked entry and the tx's registry entry. + */ + std::pair Lock(int k, TxNumber txn, LockType type) + { + TestKey key(std::make_tuple(std::string("k"), k)); + auto it = FindEmplace(key); + auto *cce = it->second; + NonBlockingLock &lock = + cce->GetOrCreateKeyLock(shard_, this, it.GetPage()); + StubCcRequest req{txn}; + // The entry point CcMap::AcquireCceKeyLock uses for every lock type. + REQUIRE(lock.AcquireLock(&req, CcProtocol::Locking, type) == + LockOpStatus::Successful); + Held().emplace_back(txn, cce); + TxLockInfo *info = + shard_->UpsertLockHoldingTx(txn, + kTerm, + cce, + type == LockType::WriteLock, + cc_ng_id_, + table_name_.Type()); + return {cce, info}; + } + + using CcMap::DowngradeCceKeyWriteLock; +}; + +/** + * @brief One shard with a map per table type, built on first use and leaked, + * so it outlives every test case the way a TxProcessor's shard does. + */ +struct Fixture +{ + Fixture() + : ng_configs{{kNg, {NodeConfig(0, "127.0.0.1", 8600)}}}, + tx_cnf{ + {"node_memory_limit_mb", 1000}, + {"enable_key_cache", 0}, + {"reltime_sampling", 0}, + {"range_split_worker_num", 1}, + {"range_slice_memory_limit_percent", 20}, + {"core_num", 1}, + {"realtime_sampling", 0}, + {"checkpointer_interval", 10}, + {"checkpointer_delay_seconds", 0}, + {"checkpointer_min_ckpt_request_interval", 5}, + {"enable_shard_heap_defragment", 0}, + {"node_log_limit_mb", 1000}, + {"collect_active_tx_ts_interval_seconds", 2}, + {"rep_group_cnt", 1}, + }, + catalog_factory{&mock_catalog_factory, + &mock_catalog_factory, + &mock_catalog_factory, + &mock_catalog_factory, + &mock_catalog_factory}, + local_cc_shards(0, + 0, + tx_cnf, + catalog_factory, + nullptr, + &ng_configs, + 2, + nullptr, + nullptr, + true), + shard(0, + 1, + 10000, + false, + 0, + local_cc_shards, + catalog_factory, + nullptr, + &ng_configs, + 2) + { + // As TxProcessor::Run does before touching its shard. + local_cc_shards.BindThreadToFastMetaDataShard(0); + shard.Init(); + + for (TableType type : {TableType::Primary, + TableType::Secondary, + TableType::Catalog, + TableType::RangePartition, + TableType::RangeBucket, + TableType::ClusterConfig}) + { + maps.emplace(type, std::make_unique(&shard, type)); + } + } + + RegistryTestMap &Map(TableType type) + { + return *maps.at(type); + } + + std::unordered_map> ng_configs; + std::map tx_cnf; + CatalogFactory *catalog_factory[5]; + LocalCcShards local_cc_shards; + CcShard shard; + std::map> maps; +}; + +Fixture &F() +{ + static Fixture *fixture = new Fixture(); + return *fixture; +} + +/** + * @brief Releases every lock the test took for a tx and drops its registry + * entry. + * + * Does what CcShard::ClearTx does, minus recycling the lock objects: that + * stamps them from Sharder's LocalCcShards, which only Sharder::Init sets. + * + * @param txn The tx whose locks to release. + */ +void Release(TxNumber txn) +{ + auto &held = Held(); + for (auto it = held.begin(); it != held.end();) + { + if (it->first != txn) + { + ++it; + continue; + } + it->second->GetKeyLock()->ClearTx(txn, &F().shard); + F().shard.DeleteLockHoldingTx(txn, it->second, kNg); + it = held.erase(it); + } +} + +/** + * @brief Releases a tx's locks when a test case exits, including when a + * failed REQUIRE throws, so one failing case cannot pin the checkpoint for + * the cases after it. + */ +struct ReleaseOnExit +{ + ~ReleaseOnExit() + { + Release(txn); + } + + TxNumber txn; ///< The tx whose locks to release. +}; + +/** + * @brief Moves the shard clock forward. + * + * @return The new shard time. + */ +uint64_t Tick() +{ + uint64_t t = F().shard.Now() + 1000; + F().shard.UpdateTsBase(t); + return t; +} + +/** @brief One ActiveTxMinTs round and what it reported. */ +struct MinTs +{ + uint64_t ts{0}; + TxNumber pinning_txn{kNoTxn}; + uint64_t pinning_wlock_ts{UINT64_MAX}; +}; + +MinTs ActiveMin() +{ + MinTs r; + r.ts = F().shard.ActiveTxMinTs(kNg, &r.pinning_txn, &r.pinning_wlock_ts); + return r; +} + +/** @brief Requires that no lock of the tx holds the checkpoint back. */ +void RequireNotHoldingBack() +{ + MinTs r = ActiveMin(); + REQUIRE(r.pinning_txn == kNoTxn); + REQUIRE(F().shard.ActiveTxMaxTs(kNg) == 0); +} + +/** + * @brief Requires that a tx holds the checkpoint below its latest write-lock + * grant. + * + * @param txn The tx expected to pin the checkpoint. + * @param lock_ts When it last acquired a write lock in the shard. + */ +void RequireHoldingBack(TxNumber txn, uint64_t lock_ts) +{ + MinTs r = ActiveMin(); + REQUIRE(r.pinning_txn == txn); + REQUIRE(r.pinning_wlock_ts == lock_ts); + REQUIRE(r.ts == lock_ts - 1); + REQUIRE(r.ts < lock_ts); + REQUIRE(F().shard.ActiveTxMaxTs(kNg) == lock_ts); +} + +} // namespace + +TEST_CASE("a data write lock holds the checkpoint below its acquisition", + "[ckpt-lock-registry]") +{ + constexpr TxNumber kTxn = 101; + const ReleaseOnExit release_on_exit{kTxn}; + for (TableType type : {TableType::Primary, TableType::Secondary}) + { + uint64_t t = Tick(); + F().Map(type).Lock(1, kTxn, LockType::WriteLock); + RequireHoldingBack(kTxn, t); + + Release(kTxn); + RequireNotHoldingBack(); + } +} + +TEST_CASE("each write-lock grant moves the bound up, reads and intents do not", + "[ckpt-lock-registry]") +{ + // The tx chooses its commit timestamp after its last write lock, so the + // latest grant bounds the commit, and the checkpoint may advance to it. + constexpr TxNumber kTxn = 151; + const ReleaseOnExit release_on_exit{kTxn}; + RegistryTestMap &data = F().Map(TableType::Primary); + + uint64_t t1 = Tick(); + data.Lock(1, kTxn, LockType::WriteLock); + RequireHoldingBack(kTxn, t1); + + Tick(); + data.Lock(2, kTxn, LockType::ReadLock); + Tick(); + data.Lock(3, kTxn, LockType::WriteIntent); + RequireHoldingBack(kTxn, t1); + + uint64_t t4 = Tick(); + data.Lock(4, kTxn, LockType::WriteLock); + RequireHoldingBack(kTxn, t4); + + Release(kTxn); + RequireNotHoldingBack(); +} + +TEST_CASE("a catalog write lock holds the checkpoint below its acquisition", + "[ckpt-lock-registry]") +{ + // What EloqDoc's change data capture relies on: a checkpoint computed + // while a DDL holds its catalog write lock stays below the lock time, + // so the DDL's log record lands above the log group's checkpoint. + constexpr TxNumber kTxn = 201; + const ReleaseOnExit release_on_exit{kTxn}; + uint64_t t = Tick(); + F().Map(TableType::Catalog).Lock(1, kTxn, LockType::WriteLock); + RequireHoldingBack(kTxn, t); + + Release(kTxn); + RequireNotHoldingBack(); +} + +TEST_CASE("range, bucket and cluster-config write locks do not hold it back", + "[ckpt-lock-registry]") +{ + constexpr TxNumber kTxn = 301; + const ReleaseOnExit release_on_exit{kTxn}; + for (TableType type : {TableType::RangePartition, + TableType::RangeBucket, + TableType::ClusterConfig}) + { + Tick(); + TxLockInfo *info = + F().Map(type).Lock(1, kTxn, LockType::WriteLock).second; + // Still recorded for orphan-lock recovery. + REQUIRE(info->wlock_ts_ != 0); + RequireNotHoldingBack(); + + Release(kTxn); + } +} + +TEST_CASE("intents and read locks do not hold the checkpoint back", + "[ckpt-lock-registry]") +{ + constexpr TxNumber kTxn = 401; + const ReleaseOnExit release_on_exit{kTxn}; + Tick(); + F().Map(TableType::Primary).Lock(1, kTxn, LockType::WriteIntent); + F().Map(TableType::Primary).Lock(2, kTxn, LockType::ReadLock); + F().Map(TableType::Catalog).Lock(1, kTxn, LockType::WriteIntent); + F().Map(TableType::Catalog).Lock(2, kTxn, LockType::ReadLock); + RequireNotHoldingBack(); + + Release(kTxn); +} + +TEST_CASE("a later meta lock does not hide a data or catalog write lock", + "[ckpt-lock-registry]") +{ + // The registry once kept one table type per tx, overwritten by every + // registration, and the checkpoint read it: a data or catalog write lock + // followed by any lock on a range, bucket or cluster-config entry dropped + // out of the checkpoint. A later meta write lock is still a write-lock + // grant, so it moves the bound up to its own time. + constexpr TxNumber kTxn = 501; + const ReleaseOnExit release_on_exit{kTxn}; + for (TableType type : {TableType::Primary, TableType::Catalog}) + { + uint64_t t = Tick(); + F().Map(type).Lock(1, kTxn, LockType::WriteLock); + + Tick(); + F().Map(TableType::ClusterConfig).Lock(1, kTxn, LockType::ReadLock); + RequireHoldingBack(kTxn, t); + + uint64_t t_range = Tick(); + F().Map(TableType::RangePartition).Lock(1, kTxn, LockType::WriteLock); + RequireHoldingBack(kTxn, t_range); + + Release(kTxn); + RequireNotHoldingBack(); + } +} + +TEST_CASE("a meta write lock alone does not count, a later data lock does", + "[ckpt-lock-registry]") +{ + constexpr TxNumber kTxn = 601; + const ReleaseOnExit release_on_exit{kTxn}; + uint64_t t_range = Tick(); + F().Map(TableType::RangePartition).Lock(1, kTxn, LockType::WriteLock); + RequireNotHoldingBack(); + + uint64_t t_data = Tick(); + TxLockInfo *info = + F().Map(TableType::Primary).Lock(1, kTxn, LockType::WriteLock).second; + REQUIRE(t_range < t_data); + REQUIRE(info->wlock_ts_ == t_data); + RequireHoldingBack(kTxn, t_data); + + Release(kTxn); +} + +TEST_CASE("a downgraded catalog write lock stops holding the checkpoint back", + "[ckpt-lock-registry]") +{ + // A DDL's prepare stage downgrades its catalog write lock to a write + // intent (PostWriteType::PrepareCommit) and its commit stage upgrades it + // again. The checkpoint is not held while only the intent remains, and + // is held below the upgrade once it is taken. + constexpr TxNumber kTxn = 701; + const ReleaseOnExit release_on_exit{kTxn}; + RegistryTestMap &catalog = F().Map(TableType::Catalog); + + uint64_t t_first = Tick(); + LruEntry *cce = catalog.Lock(1, kTxn, LockType::WriteLock).first; + RequireHoldingBack(kTxn, t_first); + + Tick(); + catalog.DowngradeCceKeyWriteLock(cce, kTxn); + REQUIRE(cce->GetKeyLock()->SearchLock(kTxn) == LockType::WriteIntent); + RequireNotHoldingBack(); + + uint64_t t_upgrade = Tick(); + catalog.Lock(1, kTxn, LockType::WriteLock); + RequireHoldingBack(kTxn, t_upgrade); + + Release(kTxn); + RequireNotHoldingBack(); +} + +TEST_CASE("a downgrade keeps the tx counted while a counting lock remains", + "[ckpt-lock-registry]") +{ + constexpr TxNumber kTxn = 801; + const ReleaseOnExit release_on_exit{kTxn}; + RegistryTestMap &catalog = F().Map(TableType::Catalog); + RegistryTestMap &range = F().Map(TableType::RangePartition); + + Tick(); + LruEntry *first = catalog.Lock(1, kTxn, LockType::WriteLock).first; + Tick(); + LruEntry *second = catalog.Lock(2, kTxn, LockType::WriteLock).first; + uint64_t t_last = Tick(); + LruEntry *range_entry = range.Lock(1, kTxn, LockType::WriteLock).first; + + // Downgrading a lock that never counted changes nothing. + range.DowngradeCceKeyWriteLock(range_entry, kTxn); + RequireHoldingBack(kTxn, t_last); + + // Another counting write lock remains; a downgrade acquires nothing, so + // the bound stays at the latest grant. + catalog.DowngradeCceKeyWriteLock(first, kTxn); + RequireHoldingBack(kTxn, t_last); + + catalog.DowngradeCceKeyWriteLock(second, kTxn); + RequireNotHoldingBack(); + + Release(kTxn); +} + +} // namespace txservice diff --git a/tx_service/tests/ClusterGate-Test.cpp b/tx_service/tests/ClusterGate-Test.cpp new file mode 100644 index 000000000..1f2f1c29b --- /dev/null +++ b/tx_service/tests/ClusterGate-Test.cpp @@ -0,0 +1,585 @@ +/** + * Copyright (C) 2026 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +// Before Catch2, whose CHECK replaces the glog CHECK that brpc headers use. +#include +#include +#include +#include + +// Let Catch provide main(): +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// For Sharder's local shards and candidate terms, which only Sharder::Init +// sets (see Fixture). +#define private public +#include "cc/cc_handler_result.h" +#include "cc/cc_req_base.h" +#include "cc/cc_request.h" +#include "cc/cluster_config_cc_map.h" +#include "cc/non_blocking_lock.h" +#include "cc_shard.h" +#include "cluster_config_record.h" +#include "fault/log_replay_service.h" +#include "include/mock/mock_catalog_factory.h" +#include "local_cc_shards.h" +#include "log.pb.h" +#include "sharder.h" +#undef private + +/** + * @file + * @brief The cluster gate's carrier and replay (EloqDoc + * docs/cdc-change-stream.md §6.2; plan D1, D2, D6): the optional gate a + * ClusterConfigRecord carries through every copy and its serialization, the + * cluster config cc map's gate-only post write all, which stores the gate only + * while the request's tx owns the lock and never touches Sharder, and the + * map's replay of log group 0's gate slot, which installs the value in place. + */ + +namespace txservice +{ +namespace +{ +constexpr NodeGroupId kNg = 0; +constexpr int64_t kTerm = 1; + +MockCatalogFactory mock_catalog_factory{}; + +/** + * @brief A gate value. + * + * @param paused Whether it is closed. + * @param mode What it rejects. + * @return The gate. + */ +ClusterGate Gate(bool paused, ClusterGate::Mode mode = ClusterGate::Mode::kDdl) +{ + ClusterGate gate; + gate.paused = paused; + gate.mode = mode; + return gate; +} + +/** + * @brief A record round-tripped through Serialize and Deserialize, as a + * remote post write all carries it. + * + * @param rec The record. + * @return The decoded record. + */ +std::unique_ptr RoundTrip(const ClusterConfigRecord &rec) +{ + std::string str; + rec.Serialize(str); + REQUIRE(str.size() == rec.SerializedLength()); + auto decoded = std::make_unique(); + size_t offset = 0; + decoded->Deserialize(str.data(), offset); + REQUIRE(offset == str.size()); + return decoded; +} + +TEST_CASE("a record without a gate round-trips without one", "[ClusterGate]") +{ + ClusterConfigRecord rec; + rec.SetVersion(42); + auto decoded = RoundTrip(rec); + CHECK(decoded->Version() == 42); + CHECK_FALSE(decoded->GetClusterGate().has_value()); + CHECK_FALSE(decoded->GatesSchemaOps()); +} + +TEST_CASE("a gate round-trips, a resume as a present value", "[ClusterGate]") +{ + for (const ClusterGate &gate : + {Gate(true), Gate(false), Gate(true, ClusterGate::Mode::kDdlDml)}) + { + ClusterConfigRecord rec; + rec.SetVersion(7); + rec.SetClusterGate(gate); + auto decoded = RoundTrip(rec); + CHECK(decoded->Version() == 7); + REQUIRE(decoded->GetClusterGate().has_value()); + CHECK(*decoded->GetClusterGate() == gate); + CHECK(decoded->GatesSchemaOps() == gate.paused); + } +} + +TEST_CASE("every copy carries the gate", "[ClusterGate]") +{ + ClusterConfigRecord rec; + rec.SetVersion(9); + rec.SetClusterGate(Gate(true)); + + ClusterConfigRecord constructed(rec); + CHECK(*constructed.GetClusterGate() == Gate(true)); + + ClusterConfigRecord assigned; + assigned = rec; + CHECK(*assigned.GetClusterGate() == Gate(true)); + + TxRecord::Uptr cloned = rec.Clone(); + CHECK(*static_cast(*cloned).GetClusterGate() == + Gate(true)); + + ClusterConfigRecord copied; + copied.Copy(rec); + CHECK(*copied.GetClusterGate() == Gate(true)); + + copied.Reset(); + CHECK_FALSE(copied.GetClusterGate().has_value()); + CHECK(copied.Version() == 0); +} + +TEST_CASE("a decoded gate record, which owns no configs, copies safely", + "[ClusterGate]") +{ + // Deserialize makes a record the owner of its configs even when it has + // none, which every gate record is; copying one used to dereference null. + ClusterConfigRecord rec; + rec.SetClusterGate(Gate(false)); + auto decoded = RoundTrip(rec); + + ClusterConfigRecord constructed(*decoded); + ClusterConfigRecord assigned; + assigned = *decoded; + ClusterConfigRecord copied; + copied.Copy(*decoded); + for (const ClusterConfigRecord *copy : {&constructed, &assigned, &copied}) + { + REQUIRE(copy->GetClusterGate().has_value()); + CHECK(*copy->GetClusterGate() == Gate(false)); + } +} + +/** + * @brief Minimal CcRequestBase carrying a tx number, for taking the map's + * write lock directly. NonBlockingLock only reads Txn() off it. + */ +struct StubCcRequest : public CcRequestBase +{ + explicit StubCcRequest(TxNumber txn) + { + tx_number_ = txn; + proto_ = CcProtocol::Locking; + isolation_level_ = IsolationLevel::RepeatableRead; + } + + bool Execute(CcShard &) override + { + return false; + } +}; + +/** + * @brief The cluster config cc map, with a way to take its write lock as a + * gate op's acquire-all does. + */ +class GateTestMap : public ClusterConfigCcMap +{ +public: + explicit GateTestMap(CcShard *shard) : ClusterConfigCcMap(shard, kNg, 1) + { + } + + /** + * @brief Takes the entry's write lock for a tx and registers it. + * + * @param txn The tx. + */ + void WriteLock(TxNumber txn) + { + NonBlockingLock &lock = + neg_inf_.GetOrCreateKeyLock(shard_, this, &neg_inf_page_); + StubCcRequest req{txn}; + REQUIRE( + lock.AcquireLock(&req, CcProtocol::Locking, LockType::WriteLock) == + LockOpStatus::Successful); + shard_->UpsertLockHoldingTx( + txn, kTerm, &neg_inf_, true, cc_ng_id_, TableType::ClusterConfig); + } + + /** + * @brief Which tx holds the entry's write lock. + * + * @return The tx, or 0 when none does. + */ + TxNumber WriteLockHolder() + { + NonBlockingLock *lock = neg_inf_.GetKeyLock(); + if (lock == nullptr) + { + return 0; + } + auto [txn, type] = lock->WriteTx(); + return type == NonBlockingLock::WriteLockType::NoWritelock ? 0 : txn; + } + + /** + * @brief The gate the entry stores. + * + * @return The gate, or none. + */ + std::optional StoredGate() + { + return neg_inf_.payload_.cur_payload_->GetClusterGate(); + } + + /** + * @brief The record a schema op's cluster config read gets: a copy of + * the entry's payload, as Execute(ReadCc&) makes it. + * + * @return The copy. + */ + ClusterConfigRecord ReadCopy() + { + ClusterConfigRecord rec; + rec = *neg_inf_.payload_.cur_payload_; + return rec; + } + + /** + * @brief The entry's commit timestamp. + * + * @return The timestamp. + */ + uint64_t EntryCommitTs() + { + return neg_inf_.CommitTs(); + } + + /** + * @brief Sets the entry's commit timestamp, as a topology change leaves + * it. + * + * @param ts The timestamp. + */ + void SetEntryCommitTs(uint64_t ts) + { + neg_inf_.SetCommitTsPayloadStatus(ts, RecordStatus::Normal); + } + + /** + * @brief Caches node group configs in the payload, as cluster scale's + * replay does for its asynchronous Sharder update. + * + * @param ng_cnt How many node groups the configs name. + */ + void CacheConfigs(uint32_t ng_cnt) + { + auto configs = std::make_unique< + std::unordered_map>>(); + for (uint32_t ng = 0; ng < ng_cnt; ++ng) + { + configs->try_emplace( + ng, std::vector{NodeConfig(ng, "127.0.0.1", 8600)}); + } + neg_inf_.payload_.cur_payload_->SetNodeGroupConfigs(std::move(configs)); + } + + /** + * @brief How many node groups the payload's cached configs name. + * + * @return The count. + */ + size_t CachedConfigCount() + { + return neg_inf_.payload_.cur_payload_->GetNodeGroupConfigs().size(); + } +}; + +/** + * @brief One shard and the map, built on first use and leaked, so they + * outlive every test case the way a TxProcessor's shard does. + */ +struct Fixture +{ + Fixture() + : ng_configs{{kNg, {NodeConfig(0, "127.0.0.1", 8600)}}}, + tx_cnf{ + {"node_memory_limit_mb", 1000}, + {"enable_key_cache", 0}, + {"reltime_sampling", 0}, + {"range_split_worker_num", 1}, + {"range_slice_memory_limit_percent", 20}, + {"core_num", 1}, + {"realtime_sampling", 0}, + {"checkpointer_interval", 10}, + {"checkpointer_delay_seconds", 0}, + {"checkpointer_min_ckpt_request_interval", 5}, + {"enable_shard_heap_defragment", 0}, + {"node_log_limit_mb", 1000}, + {"collect_active_tx_ts_interval_seconds", 2}, + {"rep_group_cnt", 1}, + }, + catalog_factory{&mock_catalog_factory, + &mock_catalog_factory, + &mock_catalog_factory, + &mock_catalog_factory, + &mock_catalog_factory}, + local_cc_shards(0, + 0, + tx_cnf, + catalog_factory, + nullptr, + &ng_configs, + 2, + nullptr, + nullptr, + true), + shard(0, + 1, + 10000, + false, + 0, + local_cc_shards, + catalog_factory, + nullptr, + &ng_configs, + 2) + { + // As TxProcessor::Run does before touching its shard. + local_cc_shards.BindThreadToFastMetaDataShard(0); + shard.Init(); + // Releasing a lock recycles its object, which stamps it with the time + // from Sharder's LocalCcShards. Only Sharder::Init sets that, and it + // also starts the node's streams and host manager. + Sharder::Instance().local_shards_ = &local_cc_shards; + // A replay runs while its node group is a candidate leader. The + // terms are read only once the node's cc nodes are up. + Sharder::Instance().candidate_leader_term_cache_[kNg].store(kTerm); + Sharder::Instance().cc_nodes_init_.store(true); + map = std::make_unique(&shard); + } + + std::unordered_map> ng_configs; + std::map tx_cnf; + CatalogFactory *catalog_factory[5]; + LocalCcShards local_cc_shards; + CcShard shard; + std::unique_ptr map; +}; + +Fixture &F() +{ + static Fixture *fixture = new Fixture(); + return *fixture; +} + +/** + * @brief Runs a gate op's commit post write all on the map, from the same + * node, as stage (4) of a ClusterGateOp sends it. + * + * @param txn The gate tx. + * @param commit_ts Its commit timestamp, the gate's version. + * @param gate The value to install. + * @return Whether the request finished. + */ +bool PostWriteGate(TxNumber txn, uint64_t commit_ts, const ClusterGate &gate) +{ + ClusterConfigRecord rec; + rec.SetClusterGate(gate); + CcHandlerResult result(nullptr); + PostWriteAllCc req; + req.Reset(&cluster_config_ccm_name, + VoidKey::NegInfTxKey(), + kNg, + txn, + commit_ts, + &rec, + OperationType::Update, + &result, + PostWriteType::Commit, + kTerm, + false); + REQUIRE(F().map->Execute(req)); + return result.IsFinished() && !result.IsError(); +} + +TEST_CASE( + "a gate's post write all stores the gate and releases the lock, without " + "Sharder", + "[ClusterGate]") +{ + const uint64_t topology_version = + Sharder::Instance().ClusterConfigVersion(); + const TxNumber txn = 101; + F().map->WriteLock(txn); + REQUIRE(F().map->WriteLockHolder() == txn); + + // Above the topology version, where a record without a gate would update + // Sharder from node group configs this record does not carry. + REQUIRE(PostWriteGate(txn, topology_version + 1000, Gate(true))); + CHECK(Sharder::Instance().ClusterConfigVersion() == topology_version); + CHECK(F().map->WriteLockHolder() == 0); + REQUIRE(F().map->StoredGate().has_value()); + CHECK(*F().map->StoredGate() == Gate(true)); + // What the next schema op's cluster config read lock reads. + CHECK(F().map->ReadCopy().GatesSchemaOps()); + + // A resume clears it the same way. + F().map->WriteLock(txn + 1); + REQUIRE(PostWriteGate(txn + 1, topology_version + 2000, Gate(false))); + CHECK(*F().map->StoredGate() == Gate(false)); + CHECK_FALSE(F().map->ReadCopy().GatesSchemaOps()); +} + +TEST_CASE("a post write all whose tx no longer holds the lock stores nothing", + "[ClusterGate]") +{ + const uint64_t base = Sharder::Instance().ClusterConfigVersion() + 10000; + + // A newer pause takes the lock and installs its value. + const TxNumber stale_resume = 201; + const TxNumber newer_pause = 202; + F().map->WriteLock(newer_pause); + REQUIRE(PostWriteGate(newer_pause, base + 2, Gate(true))); + REQUIRE(*F().map->StoredGate() == Gate(true)); + + // An older resume's post write all, retried after its tx completed + // elsewhere, finds the lock free: it must not reopen the gate. + REQUIRE(PostWriteGate(stale_resume, base + 1, Gate(false))); + CHECK(*F().map->StoredGate() == Gate(true)); + + // Nor while a third gate tx holds the lock. + const TxNumber holder = 203; + F().map->WriteLock(holder); + REQUIRE(PostWriteGate(stale_resume, base + 1, Gate(false))); + CHECK(*F().map->StoredGate() == Gate(true)); + CHECK(F().map->WriteLockHolder() == holder); + REQUIRE(PostWriteGate(holder, base + 3, Gate(true))); + CHECK(F().map->WriteLockHolder() == 0); +} +/** + * @brief A gate tx coordinated by another node group, whose replay here only + * installs the value: the coordinator's replay also starts a recovery tx, + * which needs a running TxService. + * + * @param seq The tx's sequence number within its node group. + * @return The tx number. + */ +TxNumber RemoteGateTxn(uint32_t seq) +{ + return (static_cast(kNg + 1) << 42) | seq; +} + +/** + * @brief Replays a cluster gate slot into the map, as RecoveryService does + * for the item log group 0 ships, and waits for it. + * + * @param txn The gate tx that set the slot. + * @param commit_ts The slot's version. + * @param blob The slot's value, a serialized ClusterGateLogMessage. + * @return false if the replay reported a recovery error. + */ +bool ReplayGate(TxNumber txn, uint64_t commit_ts, const std::string &blob) +{ + bthread::Mutex mux; + std::atomic status{ + fault::RecoveryService::WaitingStatus::Active}; + std::atomic on_fly_cnt{1}; + bool recovery_error = false; + ReplayLogCc req; + req.Reset(kNg, + kTerm, + cluster_config_ccm_name_sv, + TableType::ClusterConfig, + TableEngine::None, + blob, + commit_ts, + txn, + mux, + status, + on_fly_cnt, + recovery_error); + req.SetClusterGate(); + F().map->Execute(req); + CHECK(on_fly_cnt.load() == 0); + return !recovery_error; +} + +/** + * @brief A slot's value as the log service ships it. + * + * @param paused Whether the gate is paused. + * @return The serialized ClusterGateLogMessage. + */ +std::string SlotBlob(bool paused) +{ + ::txlog::ClusterGateLogMessage msg; + msg.set_mode(::txlog::ClusterGateLogMessage_GateMode_GateDdl); + msg.set_paused(paused); + std::string blob; + msg.SerializeToString(&blob); + return blob; +} + +TEST_CASE( + "a replayed slot installs its gate in place, keeping the configs scale's " + "replay cached and the larger commit timestamp", + "[ClusterGate]") +{ + const uint64_t base = F().map->EntryCommitTs() + 100000; + // What cluster scale's replay, which runs first, leaves behind. + F().map->CacheConfigs(2); + F().map->SetEntryCommitTs(base + 100); + + // A pause logged before that topology change. + REQUIRE(ReplayGate(RemoteGateTxn(301), base + 50, SlotBlob(true))); + REQUIRE(F().map->StoredGate().has_value()); + CHECK(*F().map->StoredGate() == Gate(true)); + CHECK(F().map->ReadCopy().GatesSchemaOps()); + CHECK(F().map->CachedConfigCount() == 2); + CHECK(F().map->EntryCommitTs() == base + 100); + + // A resume logged after it raises the commit timestamp. + REQUIRE(ReplayGate(RemoteGateTxn(302), base + 200, SlotBlob(false))); + CHECK(*F().map->StoredGate() == Gate(false)); + CHECK_FALSE(F().map->ReadCopy().GatesSchemaOps()); + CHECK(F().map->CachedConfigCount() == 2); + CHECK(F().map->EntryCommitTs() == base + 200); + + // Replay takes no lock: the next gate op or schema op finds it free. + CHECK(F().map->WriteLockHolder() == 0); +} + +TEST_CASE("a corrupt slot fails the replay rather than install nothing", + "[ClusterGate]") +{ + const std::optional before = F().map->StoredGate(); + // A length-delimited field whose length runs past the end. + CHECK_FALSE(ReplayGate(RemoteGateTxn(303), + F().map->EntryCommitTs() + 1, + std::string("\x12\x05" + "ab"))); + CHECK(F().map->StoredGate() == before); +} +} // namespace +} // namespace txservice diff --git a/tx_service/tests/FetchRecordCc-Test.cpp b/tx_service/tests/FetchRecordCc-Test.cpp index c8e50273a..52d5c2e13 100644 --- a/tx_service/tests/FetchRecordCc-Test.cpp +++ b/tx_service/tests/FetchRecordCc-Test.cpp @@ -510,7 +510,7 @@ struct FetchRecordFixture sharder.native_ng_ = 0; sharder.local_shards_ = &local_cc_shards; sharder.ng_leader_cache_[0].store(0, std::memory_order_relaxed); - sharder.ng_leader_term_cache_[0].store(-1, std::memory_order_relaxed); + sharder.ng_leader_term_cache_[0] = -1; sharder.leader_term_cache_[0].store(-1, std::memory_order_relaxed); sharder.candidate_leader_term_cache_[0].store( -1, std::memory_order_relaxed); diff --git a/tx_service/tests/LogAgentCdc-Test.cpp b/tx_service/tests/LogAgentCdc-Test.cpp new file mode 100644 index 000000000..06ae1e8c8 --- /dev/null +++ b/tx_service/tests/LogAgentCdc-Test.cpp @@ -0,0 +1,192 @@ +/** + * Copyright (C) 2025 EloqData Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under either of the following two licenses: + * 1. GNU Affero General Public License, version 3, as published by the Free + * Software Foundation. + * 2. GNU General Public License as published by the Free Software + * Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License or GNU General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * and GNU General Public License V2 along with this program. If not, see + * . + * + */ +// Let Catch provide main(): +// clang-format off +// log_agent.h first: it includes braft and brpc in the order their logging +// setup needs, and include sorting would move it after Catch2. +#include "log_agent.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "log.pb.h" +// clang-format on + +/** + * @file + * @brief The LogAgent side of change data capture: CdcScan on the cached + * leader, which reports the node ID it sends to, and LogNodeByAddress, which + * looks up the node ID of a NotLeader redirect. + * + * No log server runs. The agent is configured with one group of three nodes + * on ports nothing listens on, so the leader cache is exercised as pure + * bookkeeping and a scan can only fail; what the scan tests pin down is that + * its closure runs exactly once, as an asynchronous caller relies on. + */ + +namespace +{ +/// Ports of the three configured nodes; nothing listens on them. +const std::vector kPorts = {1, 2, 3}; + +/** + * @brief Initializes a LogAgent with one log group, lg0, of three nodes on + * 127.0.0.1 and kPorts; node ids are 0, 1 and 2, and the cache starts at 0. + * + * @param[in,out] agent The agent to initialize. + */ +void InitAgent(txlog::LogAgent &agent) +{ + std::vector ips(kPorts.size(), "127.0.0.1"); + std::vector ports = kPorts; + agent.Init(ips, ports, 0, static_cast(kPorts.size())); +} + +/** + * @brief A closure that counts its runs and signals the first. + */ +class CountingClosure : public google::protobuf::Closure +{ +public: + void Run() override + { + if (runs_.fetch_add(1) == 0) + { + ran_.set_value(); + } + } + + /** + * @brief Waits for the first run. + * + * @return true if the closure ran within the timeout. + */ + bool WaitForRun() + { + return ran_future_.wait_for(std::chrono::seconds(10)) == + std::future_status::ready; + } + + std::atomic runs_{0}; ///< How many times Run() was called. + +private: + std::promise ran_; + std::future ran_future_{ran_.get_future()}; +}; +} // namespace + +TEST_CASE( + "LogNodeByAddress finds the group's node ID configured with that address", + "[LogAgentCdc]") +{ + txlog::LogAgent agent; + InitAgent(agent); + + REQUIRE(agent.LogNodeByAddress(0, "127.0.0.1:1") == 0u); + REQUIRE(agent.LogNodeByAddress(0, "127.0.0.1:2") == 1u); + REQUIRE(agent.LogNodeByAddress(0, "127.0.0.1:3") == 2u); + // A lookup only: following the node is the caller's UpdateLeaderCache. + REQUIRE(agent.TEST_LogGroupLeaderNode(0) == 0); +} + +TEST_CASE("LogNodeByAddress finds nothing for an address it cannot match", + "[LogAgentCdc]") +{ + txlog::LogAgent agent; + InitAgent(agent); + + const std::vector unmatched = { + "", // no leader known + "127.0.0.2:2", // another host + "127.0.0.1:4", // a port no node has + "127.0.0.1", // no port + ":2", // no host + "127.0.0.1:", // empty port + "127.0.0.1:2x", // not a number + "127.0.0.1:65538", // past 16 bits; wraps to 2 if parsed naively + }; + for (const std::string &address : unmatched) + { + INFO("address " << address); + REQUIRE_FALSE(agent.LogNodeByAddress(0, address).has_value()); + } + + // A group the agent does not know. + REQUIRE_FALSE(agent.LogNodeByAddress(7, "127.0.0.1:3").has_value()); +} + +TEST_CASE( + "CdcScan on an unknown group fails and runs its closure once, " + "before returning", + "[LogAgentCdc]") +{ + txlog::LogAgent agent; + InitAgent(agent); + + brpc::Controller cntl; + txlog::CdcScanRequest request; + request.set_log_group_id(7); + txlog::CdcScanResponse response; + std::optional node_id; + CountingClosure done; + agent.CdcScan(7, &cntl, &request, &response, &node_id, &done); + + REQUIRE(done.runs_.load() == 1); + REQUIRE(cntl.Failed()); + // Nothing was sent, so no node is reported. + REQUIRE_FALSE(node_id.has_value()); +} + +TEST_CASE( + "CdcScan reports the cached leader it sends to, and runs its closure " + "once when the leader is unreachable", + "[LogAgentCdc]") +{ + txlog::LogAgent agent; + InitAgent(agent); + agent.UpdateLeaderCache(0, 2); + + brpc::Controller cntl; + cntl.set_timeout_ms(2000); + cntl.set_max_retry(0); + txlog::CdcScanRequest request; + request.set_log_group_id(0); + request.set_from_ts(1); + txlog::CdcScanResponse response; + std::optional node_id; + CountingClosure done; + agent.CdcScan(0, &cntl, &request, &response, &node_id, &done); + + REQUIRE(done.WaitForRun()); + REQUIRE(node_id == 2u); + REQUIRE(cntl.Failed()); + // A second run would be a double completion of the caller's state. + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + REQUIRE(done.runs_.load() == 1); +} diff --git a/tx_service/tests/include/mock/mock_log_agent.h b/tx_service/tests/include/mock/mock_log_agent.h index 4013edac1..3f7b06082 100644 --- a/tx_service/tests/include/mock/mock_log_agent.h +++ b/tx_service/tests/include/mock/mock_log_agent.h @@ -39,21 +39,19 @@ class MockLogAgent : public txservice::TxLog cc_node_group_id, term, checkpoint_timestamp); } - void ReplayLog(uint32_t cc_node_group_id, + bool ReplayLog(uint32_t cc_node_group_id, int64_t term, const std::string &source_ip, uint16_t source_port, - int log_group, - uint64_t start_ts, - std::atomic &interrupt) override + uint32_t log_group_id, + uint64_t start_ts) override { - log_agent_.ReplayLog(cc_node_group_id, - term, - source_ip, - source_port, - log_group, - start_ts, - interrupt); + return log_agent_.ReplayLogOnce(cc_node_group_id, + term, + source_ip, + source_port, + log_group_id, + start_ts); } txservice::RecoverTxStatus RecoverTx(uint64_t tx_number, diff --git a/tx_service/tx-log-protos/log.proto b/tx_service/tx-log-protos/log.proto index 63389b3bc..a5bafe958 100644 --- a/tx_service/tx-log-protos/log.proto +++ b/tx_service/tx-log-protos/log.proto @@ -70,6 +70,10 @@ message SchemaOpMessage { } LastKeyType last_key_type = 13; bytes last_key_value = 14; + // Commit log of an ADD INDEX whose build broke a constraint: the op commits + // at its commit-stage timestamp and drops the index instead of installing + // it. Replaces the zero commit timestamp that used to mean the same. + bool rolled_back = 15; } message UpdateCatalogMessage { @@ -223,6 +227,25 @@ message ClusterScaleOpMessage { map node_group_bucket_migrate_process = 6; } +// Pauses and resumes the cluster gate, which CDC bootstrap closes so that no DDL runs while it +// copies the catalog. Written by ClusterGateOp and applied by LogState::UpdateClusterGate; the +// entry is not retained, like UpdateCkptTs it updates state rather than being kept. Every gate +// transaction is pinned to log group 0, so that group's single slot is the ordered history of +// the gate, overwritten in place by each entry. The slot never expires: only a resume clears a +// pause. +message ClusterGateLogMessage { + enum GateMode { + // Reject new create, drop and index operations. What CDC bootstrap uses. + GateDdl = 0; + // Also reject the first write of a transaction. For a consistent snapshot; not implemented. + GateDdlDml = 1; + } + GateMode mode = 1; + // Absent and "present with paused = false" are different values on purpose: the first says + // nothing, the second is a resume. + bool paused = 2; +} + message DataLogMessage { // A map mapping the cc node group ID to the records // the tx commits in this cc node group. @@ -241,6 +264,7 @@ message LogContentMessage { SplitRangeOpMessage split_range_log = 3; ClusterScaleOpMessage cluster_scale_log = 4; DataMigrateTxLogMessage migration_log = 5; + ClusterGateLogMessage cluster_gate_log = 6; } } @@ -450,6 +474,122 @@ message GetLogGroupConfigResponse { repeated uint32 port = 3; } +// --------------------------------------------------------------------------- +// Change data capture (docs/cdc-change-stream.md in the EloqDoc repo, §4.1). +// +// CdcScan reads a log group's two timestamp-ordered streams -- committed data records and the +// DDL history -- as one merged sequence, so that a caller pulling from every group can order +// events across the cluster. It returns records as stored; the logger decodes nothing. +// +// The merged order is by timestamp, and at one timestamp every data record precedes every DDL +// record. Each page is a prefix of that order. +// +// `continuation` byte layout, frozen here because the EloqDoc runtime's resume tokens use the +// same two-key form for their per-group entries (§5.6), so the two must agree: +// +// u16 len | data-stream key | u16 len | ddl-stream key +// +// with both lengths big-endian, matching the token encoding. Each key is where that stream +// resumes, strictly after it in the store's bytewise key order, and takes one of three shapes: +// - empty: the stream has returned nothing and starts at the beginning of complete history; +// - 8 bytes, a big-endian timestamp: the stream has returned nothing since a scan began at +// from_ts = that timestamp. Every key starts with its big-endian timestamp and is longer, so +// this sorts before each key at that timestamp and after each key below it; +// - the key of a record the stream returned: for the data stream its data CF key, +// bswap64(commit_ts) | bswap32(ng) | bswap64(txn); for the DDL stream its position in the +// DDL history, bswap64(ddl_ts) | bswap64(txn_number), which the caller can rebuild from the +// record. +// The caller stores a continuation and passes it back, never interpreting it. A token entry is +// built from records the runtime produced, so its keys always have the third shape. +// --------------------------------------------------------------------------- + +message CdcScanRequest { + uint32 log_group_id = 1; + // Where to start. Exactly one is set; neither set means the beginning of complete history, + // which the logger positions at retention_floor_ts and never below it. + oneof start { + // Returned by the previous response for this group: the position of each stream (above), + // after which both resume, so pages neither omit nor duplicate records. + bytes continuation = 2; + // First storage key with timestamp >= from_ts, in both streams. Used to open a fresh stream + // and to resume from a token, which needs the record *at* a key rather than after it. + uint64 from_ts = 3; + } + // Soft cap on the response size. A page is cut at this, except that at least one record is + // always returned even when it alone exceeds the cap. + uint32 max_bytes = 4; + // Long poll: hold the request until something becomes emittable, or until this expires. Only + // an advance of min_ckpt_ts_ can make a record emittable, so only that and the deadline + // complete a parked request with a page; a step-down answers it NotLeader. The logger may + // still answer early with an empty page (it holds no more parked requests, its own cap on the + // wait is shorter, or it does not park at all); a caller backs off before scanning the group + // again rather than re-issuing at once. + uint32 wait_ms = 5; +} + +message CdcDataRecord { + uint64 commit_ts = 1; + uint32 cc_ng_id = 2; + uint64 txn_number = 3; + // The per-node-group blob exactly as stored, undecoded. + bytes blob = 4; +} + +// One record per recorded schema op: EloqDoc's create and drop collection and its index +// operations. An index operation's message is a table_op whose index names are in +// alter_table_info_blob. An add index whose build broke a constraint has a second record, at its +// commit-stage timestamp, whose schema_op.rolled_back is set: the indexes it added are dropped +// there. +message CdcDdlRecord { + // The record's stream timestamp: the commit log's timestamp for create and drop and for a + // rolled-back add index's second record, and the prepare log's timestamp (the op's catalog + // version) for index operations otherwise. + uint64 ddl_ts = 1; + // The op's catalog version: the timestamp its prepare log carried, at which it installed its + // dirty schema. Create and drop commit the catalog entry at ddl_ts instead; their tables keep + // this version. + uint64 catalog_ts = 2; + uint64 txn_number = 3; + // The op's message as the log state holds it when the record is appended: the prepare log's, + // with the commit log's stage and rolled-back mark for a record appended at the commit log. + SchemaOpMessage schema_op = 4; +} + +message CdcScanResponse { + enum Status { + Success = 0; + NotLeader = 1; + // The requested position is at or below retention_floor_ts, so history from it is + // incomplete. Defined here but never returned while retention_floor_ts is untracked. + PositionTruncated = 2; + } + Status status = 1; + // On NotLeader, "host:port" of the group's leader as the answering node knows it; the + // answering node's own address when it leads but cannot serve a consistent view yet (retry + // there after a backoff); empty when no leader is known. + string redirect = 2; + + // Ascending by (commit_ts, ng, txn); every commit_ts <= arrival_complete_ts. + repeated CdcDataRecord data = 3; + // Ascending by (ddl_ts, txn_number); every ddl_ts <= arrival_complete_ts. + repeated CdcDdlRecord ddl = 4; + + // This group's min_ckpt_ts_, read *before* either stream is scanned. Nothing at or below it + // can still arrive, so a caller that has consumed through it from every group holds a complete + // prefix. Returned on every response, including empty ones, which is what lets an idle stream + // keep advancing. + uint64 arrival_complete_ts = 5; + // History is complete above this. 0 means the boundary is not tracked, which is the case + // today; correctness then assumes the requested history is still retained. + uint64 retention_floor_ts = 6; + // Pass back to resume both streams exactly after this page. + bytes continuation = 7; + // True when the page was cut: records at or below arrival_complete_ts remain beyond + // `continuation`, possibly at the page's last timestamp. A caller derives its completeness + // boundary from this -- last_ts - 1 when set, arrival_complete_ts when clear. + bool has_more = 8; +} + service LogService { rpc WriteLog(LogRequest) returns (LogResponse); rpc ReplayLog(LogRequest) returns (LogResponse); @@ -458,6 +598,8 @@ service LogService { rpc RecoverTx(RecoverTxRequest) returns (RecoverTxResponse); rpc CheckMigrationIsFinished(CheckMigrationIsFinishedRequest) returns (CheckMigrationIsFinishedResponse); rpc CheckClusterScaleStatus(LogRequest) returns (LogResponse); + // Change data capture. Leader only; followers answer NotLeader with a redirect. + rpc CdcScan(CdcScanRequest) returns (CdcScanResponse); // Add this function for test aim, but it can be used with other aims. // Call this method will transfer the log group leader to the pointed node. rpc TransferLeader(TransferRequest) returns (TransferResponse); @@ -492,6 +634,18 @@ message ReplayClusterScaleMsg { uint64 txn = 3; } +// A log group's cluster gate slot, shipped to a recovering node group so that +// it can reinstall the gate however old it is. Only log group 0 ever holds +// one, so a replay carries at most one. +message ReplayClusterGateMsg { + // The version: the commit timestamp of the entry that set the slot. + uint64 commit_ts = 1; + // The serialized ClusterGateLogMessage. + bytes cluster_gate_blob = 2; + // The gate transaction that set it; its coordinator's node group resumes it. + uint64 txn = 3; +} + message ReplayFinishMsg { uint32 log_group_id = 1; uint32 latest_txn_no = 2; @@ -508,6 +662,7 @@ message ReplayMessage { ReplayClusterScaleMsg cluster_scale_op_msg = 6; ReplayFinishMsg finish = 7; bool is_lock_recovery = 8; + ReplayClusterGateMsg cluster_gate_msg = 9; } message LogReplayConnectRequest { diff --git a/tx_service/tx-log-protos/log_agent.cpp b/tx_service/tx-log-protos/log_agent.cpp index 8af0e89a4..206266a35 100644 --- a/tx_service/tx-log-protos/log_agent.cpp +++ b/tx_service/tx-log-protos/log_agent.cpp @@ -374,6 +374,42 @@ void LogAgent::CheckMigrationIsFinished( stub.CheckMigrationIsFinished(controller, request, response, done); } +void LogAgent::CdcScan(uint32_t log_group_id, + brpc::Controller *controller, + const CdcScanRequest *request, + CdcScanResponse *response, + std::optional *node_id, + ::google::protobuf::Closure *done) +{ + std::shared_ptr channel; + uint32_t leader = 0; + { + std::shared_lock lock(config_map_mutex_); + auto cache_it = lg_leader_cache_.find(log_group_id); + if (cache_it != lg_leader_cache_.end()) + { + leader = cache_it->second.load(std::memory_order_acquire); + auto channel_it = log_channel_map_.find(leader); + if (channel_it != log_channel_map_.end()) + { + channel = channel_it->second; + } + } + } + if (channel == nullptr) + { + controller->SetFailed("Log channel not found for group " + + std::to_string(log_group_id)); + done->Run(); + return; + } + // Before the call is sent: done may run on another thread before the stub + // returns, and the caller reads the node ID from it. + *node_id = leader; + LogService_Stub stub(channel.get()); + stub.CdcScan(controller, request, response, done); +} + CheckClusterScaleStatusResponse::Status LogAgent::CheckClusterScaleStatus( uint32_t log_group_id, const std::string &id) { @@ -554,6 +590,62 @@ void LogAgent::ReplayLog(uint32_t cc_node_group_id, } } +bool LogAgent::ReplayLogOnce(uint32_t cc_node_group_id, + int64_t term, + const std::string &source_ip, + uint16_t source_port, + uint32_t log_group_id, + uint64_t start_ts) +{ + if (RefreshLeader(log_group_id) != 0) + { + LOG(WARNING) << "ReplayLog for ng " << cc_node_group_id << " term " + << term << " not sent: log group " << log_group_id + << " has no known leader"; + return false; + } + auto channel = GetChannel(log_group_id); + if (channel == nullptr) + { + LOG(WARNING) << "ReplayLog for ng " << cc_node_group_id << " term " + << term << " not sent: no channel to log group " + << log_group_id << "'s leader"; + return false; + } + + LogRequest req; + ReplayLogRequest *replay_log_req = req.mutable_replay_log_request(); + replay_log_req->set_cc_node_group_id(cc_node_group_id); + replay_log_req->set_term(term); + replay_log_req->set_source_ip(source_ip); + replay_log_req->set_source_port(source_port); + replay_log_req->set_start_ts(start_ts); + replay_log_req->set_log_group_id(log_group_id); + LogResponse resp; + brpc::Controller cntl; + cntl.set_timeout_ms(kReplayRequestTimeoutMs); + LogService_Stub stub(channel.get()); + stub.ReplayLog(&cntl, &req, &resp, nullptr); + if (cntl.Failed()) + { + LOG(WARNING) << "ReplayLog for ng " << cc_node_group_id << " term " + << term << " to log group " << log_group_id + << ": no reply (" << cntl.ErrorText() << ")"; + return false; + } + if (resp.response_status() != + LogResponse::ResponseStatus::LogResponse_ResponseStatus_Success) + { + LOG(WARNING) << "ReplayLog for ng " << cc_node_group_id << " term " + << term << " to log group " << log_group_id + << ": refused, status " << resp.response_status(); + return false; + } + LOG(INFO) << "ReplayLog for ng " << cc_node_group_id << " term " << term + << " accepted by log group " << log_group_id; + return true; +} + void LogAgent::RemoveCcNodeGroup(uint32_t cc_node_group_id, int64_t term) { brpc::Controller cntl; @@ -704,6 +796,50 @@ void LogAgent::UpdateLeaderCache(uint32_t lg_id, uint32_t node_id) lg_leader_cache_.at(lg_id).store(node_id, std::memory_order_release); } +std::optional LogAgent::LogNodeByAddress(uint32_t log_group_id, + const std::string &address) +{ + const size_t colon_pos = address.rfind(':'); + if (colon_pos == std::string::npos || colon_pos == 0 || + colon_pos + 1 == address.size()) + { + return std::nullopt; + } + const std::string host = address.substr(0, colon_pos); + uint32_t port = 0; + for (size_t i = colon_pos + 1; i < address.size(); ++i) + { + const char c = address[i]; + if (c < '0' || c > '9') + { + return std::nullopt; + } + port = port * 10 + static_cast(c - '0'); + if (port > UINT16_MAX) + { + return std::nullopt; + } + } + + std::shared_lock lock(config_map_mutex_); + auto config_it = log_group_config_map_.find(log_group_id); + if (config_it == log_group_config_map_.end()) + { + return std::nullopt; + } + // The group's config holds node IDs only; their addresses are in + // log_nodes_, which UpdateLogGroupConfig updates with it. + for (uint32_t node_id : config_it->second.group_nodes_) + { + const std::pair &node = log_nodes_.at(node_id); + if (node.first == host && node.second == port) + { + return node_id; + } + } + return std::nullopt; +} + bool LogAgent::UpdateLogGroupConfig(std::vector &ips, std::vector &ports, uint32_t log_group_id) diff --git a/tx_service/tx-log-protos/log_agent.h b/tx_service/tx-log-protos/log_agent.h index b72821d4b..9cdd8028a 100644 --- a/tx_service/tx-log-protos/log_agent.h +++ b/tx_service/tx-log-protos/log_agent.h @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -109,6 +110,30 @@ class LogAgent CheckMigrationIsFinishedResponse *response, ::google::protobuf::Closure *done); + /** + * @brief Sends a CdcScan to the group's cached leader, asynchronously. + * + * The caller handles NotLeader: it looks up the redirect's node ID with + * LogNodeByAddress, points the cache at it with UpdateLeaderCache or asks + * for a refresh, and scans again. + * + * @param log_group_id The log group to scan. + * @param controller The call's controller; its timeout bounds the call. + * @param request The scan request. + * @param[out] response The scan response. + * @param[out] node_id Set to the node ID the call is sent to, before it + * is sent; left as is when the group has no channel. + * @param done Run exactly once when the call completes. When the group has + * no channel, the controller reports the failure and done runs before this + * returns. + */ + void CdcScan(uint32_t log_group_id, + brpc::Controller *controller, + const CdcScanRequest *request, + CdcScanResponse *response, + std::optional *node_id, + ::google::protobuf::Closure *done); + CheckClusterScaleStatusResponse::Status CheckClusterScaleStatus( uint32_t log_group_id, const std::string &id); @@ -139,6 +164,36 @@ class LogAgent std::atomic &interrupt, bool no_replay = false); + /** + * @brief Sends a ReplayLogRequest to one log group, once: looks up the + * group's leader (at most 1 s) and sends the request (deadline + * kReplayRequestTimeoutMs). + * + * A request or its reply may be lost without a trace, so the outcome + * proves little: false if no leader was found or no reply came, true if + * the leader accepted. Either way the caller learns that the log group got + * the request from the replay stream it opens, and asks again if none + * comes. + * + * @param cc_node_group_id The recovering node group. + * @param term The node group's new term. + * @param source_ip The address of the node's log replay service. + * @param source_port The port of the node's log replay service. + * @param log_group_id The log group. + * @param start_ts The commit timestamp to replay from at the latest (0: + * from the log group's last checkpoint). + * @return true if the log group's leader accepted the request. + */ + bool ReplayLogOnce(uint32_t cc_node_group_id, + int64_t term, + const std::string &source_ip, + uint16_t source_port, + uint32_t log_group_id, + uint64_t start_ts); + + /** @brief The deadline of one ReplayLogOnce request, in milliseconds. */ + static constexpr int32_t kReplayRequestTimeoutMs = 2000; + /* * RemoveCcNodeGroup request informs all log groups of this node * group is removed in cluster. @@ -210,6 +265,22 @@ class LogAgent void UpdateLeaderCache(uint32_t lg_id, uint32_t node_id); + /** + * @brief The node ID configured with the given address in the log group. + * + * For a CdcScan NotLeader response, whose redirect names the leader as the + * log service formats a peer address, "host:port"; matched against the + * addresses of the group's node IDs as RefreshLeader matches braft's + * leader. + * + * @param log_group_id The log group. + * @param address A "host:port". + * @return The node ID, or empty if no node ID of the group is configured + * with that address or the address is malformed. + */ + std::optional LogNodeByAddress(uint32_t log_group_id, + const std::string &address); + bool UpdateLogGroupConfig(std::vector &ips, std::vector &ports, uint32_t log_group_id);