diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d95a20db..af7ae5630 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- Identity creation and top-up accounts are saved before payment and restored + after restarting the app. Temporary save failures are retried a few times; + if saving still fails, the payment stops and the app asks you to try again. + - **Closing password-entry screens no longer aborts debug builds**: secret buffers now own separate guarded memory pages, so releasing one password field cannot interfere with another field that is still in use. diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index d00b8c18c..2959848db 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -185,6 +185,18 @@ pub enum TaskError { source: dash_sdk::dpp::key_wallet::Error, }, + /// An identity-funding account was derived but could not be saved. Saving + /// it is what lets a restart find the account again, so the operation is + /// stopped rather than left able to strand a funding lock the app could no + /// longer spend. The technical cause lives in `Debug` and the logs. + #[error( + "Your wallet could not save the account this payment needs. Check that your disk is not full, then try again." + )] + IdentityFundingAccountPersistFailed { + #[source] + source: Box, + }, + /// Single-key wallets are not supported in this version. Their data is /// preserved; HD (recovery-phrase) wallets remain fully functional. #[error( diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 97660d094..978326abc 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -9,6 +9,7 @@ use crate::model::secret::Secret; use crate::utils::egui_mpsc::SenderAsync; use crate::utils::tasks::TaskManager; use crate::wallet_backend::wallet_database_path; +use platform_wallet::changeset::PersistenceErrorKind; /// Build an offline `AppContext` for testnet in an isolated temp dir. No /// network I/O happens at construction: the SDK and Core client are built @@ -3815,6 +3816,209 @@ async fn ensure_identity_funding_accounts_succeeds_on_cold_booted_watch_only_wal backend2.shutdown().await; } +/// A provisioned identity top-up account must survive a restart. +/// +/// `load()` rebuilds `Wallet.accounts` from `account_registrations` alone, and +/// the upstream creator that would otherwise write that row skips it once both +/// in-memory collections already hold the account — which DET's own +/// provisioning puts there first. A memory-only account leaves a restart +/// between an asset-lock broadcast and its consumption unable to re-derive the +/// credit-output path, stranding the lock and the funds in it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_provisioned_identity_topup_account_survives_a_restart() { + let _guard = backend_reopen_lock().await; + let source_dir = tempfile::tempdir().expect("source tempdir"); + let seed = [0xF3u8; 64]; + let registration_index = 7u32; + let account_type = dash_sdk::dpp::key_wallet::AccountType::IdentityTopUp { registration_index }; + + let (seed_hash, provisioned) = { + let wallet = + crate::model::wallet::Wallet::new_from_seed(seed, Network::Testnet, None, None) + .expect("build wallet"); + let seed_hash = wallet.seed_hash(); + + let (ctx, sender) = offline_testnet_context_at(source_dir.path()); + ctx.register_wallet(wallet, &seed, WalletOrigin::Fresh) + .expect("register wallet"); + ctx.ensure_wallet_backend(sender) + .await + .expect("wire backend offline"); + let backend = ctx.wallet_backend().expect("backend"); + backend + .register_wallet_from_seed(&seed_hash, &seed, Some(0)) + .await + .expect("upstream register"); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, registration_index) + .await + .expect("provision identity funding accounts"); + let provisioned = backend + .identity_funding_account(&seed_hash, account_type) + .await + .expect("probe the provisioned funding account") + .expect("the top-up account must be live in memory after provisioning"); + backend.shutdown().await; + (seed_hash, provisioned) + }; + + let cold_dir = tempfile::tempdir().expect("cold tempdir"); + copy_dir_recursive(source_dir.path(), cold_dir.path()); + + // The manifest is the only thing `load()` rebuilds `Wallet.accounts` from, + // so assert the row itself rather than a downstream in-memory effect. + assert_eq!( + persisted_account_registration_rows(cold_dir.path(), "identity_topup", registration_index), + 1, + "the provisioned top-up account must be in the persisted manifest, or a \ + broadcast asset lock cannot be resumed after a restart", + ); + + let (ctx2, sender2) = offline_testnet_context_at(cold_dir.path()); + ctx2.ensure_wallet_backend(sender2) + .await + .expect("cold boot must load the persisted wallet"); + let backend2 = ctx2.wallet_backend().expect("backend"); + assert!( + backend2.is_wallet_registered(&seed_hash), + "the wallet must still come back registered with the extra account row", + ); + + // A row on disk is only half the contract: the loader has to turn it back + // into the indexed account both funding-address collections read. A loader + // that dropped it — or rebuilt it at the wrong index, or from a different + // xpub — would still leave the asset lock unspendable. + let restored = backend2 + .identity_funding_account(&seed_hash, account_type) + .await + .expect("probe the reloaded funding account") + .expect("the reloaded wallet must hold the provisioned top-up account"); + assert_eq!( + restored, provisioned, + "the reloaded top-up account must match the one provisioned before the restart", + ); + + backend2.shutdown().await; +} + +/// The index-less top-up funding account must survive a restart too. +/// +/// It reaches the manifest as `identity_topup_unbound` and is the only +/// credit-output source for topping up an identity this wallet does not own. +/// In-process idempotency proves nothing about a cold boot: if the row is +/// missing, upstream's presence guards make the next launch skip the write +/// again and the account stays memory-only forever. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unbound_topup_funding_account_survives_a_restart() { + let _guard = backend_reopen_lock().await; + let source_dir = tempfile::tempdir().expect("source tempdir"); + let seed = [0xB4u8; 64]; + let account_type = dash_sdk::dpp::key_wallet::AccountType::IdentityTopUpNotBoundToIdentity; + + let (seed_hash, provisioned) = { + let wallet = + crate::model::wallet::Wallet::new_from_seed(seed, Network::Testnet, None, None) + .expect("build wallet"); + let seed_hash = wallet.seed_hash(); + + let (ctx, sender) = offline_testnet_context_at(source_dir.path()); + ctx.register_wallet(wallet, &seed, WalletOrigin::Fresh) + .expect("register wallet"); + ctx.ensure_wallet_backend(sender) + .await + .expect("wire backend offline"); + let backend = ctx.wallet_backend().expect("backend"); + backend + .register_wallet_from_seed(&seed_hash, &seed, Some(0)) + .await + .expect("upstream register"); + backend + .ensure_unbound_topup_funding_account(&seed_hash, &seed) + .await + .expect("provision the index-less top-up account"); + let provisioned = backend + .identity_funding_account(&seed_hash, account_type) + .await + .expect("probe the provisioned funding account") + .expect("the index-less top-up account must be live in memory"); + backend.shutdown().await; + (seed_hash, provisioned) + }; + + let cold_dir = tempfile::tempdir().expect("cold tempdir"); + copy_dir_recursive(source_dir.path(), cold_dir.path()); + + assert_eq!( + persisted_account_registration_rows(cold_dir.path(), "identity_topup_unbound", 0), + 1, + "the index-less top-up account must be in the persisted manifest, or a \ + top-up of an identity this wallet does not own cannot be resumed", + ); + + let (ctx2, sender2) = offline_testnet_context_at(cold_dir.path()); + ctx2.ensure_wallet_backend(sender2) + .await + .expect("cold boot must load the persisted wallet"); + let backend2 = ctx2.wallet_backend().expect("backend"); + let restored = backend2 + .identity_funding_account(&seed_hash, account_type) + .await + .expect("probe the reloaded funding account") + .expect("the reloaded wallet must hold the index-less top-up account"); + assert_eq!( + restored, provisioned, + "the reloaded index-less top-up account must match the one provisioned \ + before the restart", + ); + + backend2.shutdown().await; +} + +/// Rows in the persisted `account_registrations` manifest for one account +/// type / index across every wallet — what `load()` rebuilds +/// `Wallet.accounts` from. +fn persisted_account_registration_rows( + data_dir: &std::path::Path, + account_type: &str, + account_index: u32, +) -> i64 { + persisted_account_registration_rows_for(data_dir, None, account_type, account_index) +} + +/// As [`persisted_account_registration_rows`], but scoped to one upstream +/// `wallet_id` when given. Multi-wallet tests must scope: the manifest is +/// shared, so an unscoped count cannot tell whose registration it found. +fn persisted_account_registration_rows_for( + data_dir: &std::path::Path, + wallet_id: Option<[u8; 32]>, + account_type: &str, + account_index: u32, +) -> i64 { + let conn = rusqlite::Connection::open_with_flags( + wallet_database_path(data_dir, Network::Testnet), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .expect("open persisted store"); + match wallet_id { + Some(wallet_id) => conn + .query_row( + "SELECT COUNT(*) FROM account_registrations \ + WHERE wallet_id = ?1 AND account_type = ?2 AND account_index = ?3", + rusqlite::params![&wallet_id[..], account_type, account_index], + |row| row.get(0), + ) + .expect("count persisted account registrations for wallet"), + None => conn + .query_row( + "SELECT COUNT(*) FROM account_registrations \ + WHERE account_type = ?1 AND account_index = ?2", + rusqlite::params![account_type, account_index], + |row| row.get(0), + ) + .expect("count persisted account registrations"), + } +} + /// A shielded viewing-key row that will not decode fails the whole seedless /// cold boot, with the dedicated fatal local-data error naming the column. /// @@ -5645,6 +5849,689 @@ async fn unbound_topup_funding_account_provisions_on_the_watch_only_wallet() { backend.shutdown().await; } +/// A committed registration write must not be followed by a second one. +/// +/// DET's persister answers `store_commits_inline`, so a successful `store` has +/// already committed. A follow-up `flush` would add no durability and would +/// take the per-wallet buffer *including whatever an unrelated writer parked +/// there*: that stranger's terminal failure would come back as this +/// registration's, evicting an account whose row is already on disk. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_committed_registration_write_is_not_followed_by_a_second_write() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xCBu8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "one-write").await; + + assert!( + backend + .identity_funding_account( + &seed_hash, + dash_sdk::dpp::key_wallet::AccountType::IdentityRegistration, + ) + .await + .expect("probe the registration account") + .is_some(), + "`key-wallet` must already hold the identity-registration account from \ + a wallet's default set. Without it the call below provisions two \ + accounts, and the `store` count it asserts measures upstream's \ + defaults rather than DET's write shape", + ); + + assert!( + backend.registration_persist_commits_inline(), + "DET's persister must report `store` as the commit. A buffering backend \ + would have to flush, and the flush count asserted below would be 1", + ); + + let stores_before = backend.registration_persist_store_calls(); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("an unfaulted provisioning must succeed"); + + assert_eq!( + backend.registration_persist_store_calls() - stores_before, + 1, + "the registration must reach disk in exactly one `store`", + ); + assert_eq!( + backend.registration_persist_flush_calls(), + 0, + "a committed `store` must not be followed by a `flush` — it adds no \ + durability, and the shared buffer it would adopt lets an unrelated \ + writer's terminal failure discard this registration", + ); + + let view = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("the provisioned account must be live in memory"); + assert!(view.managed, "both collections must hold the account"); + assert!( + !backend.registration_persist_buffer_is_staged(), + "a committed write must leave nothing staged", + ); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the single `store` must have made the registration durable", + ); +} + +/// A transient persister failure must be ridden out, not surfaced. +/// +/// The persistence contract says a transient `store` keeps the changeset +/// buffered and a bare `flush` completes it. Rolling the account out of memory +/// instead leaves the staged row able to land later under a live wallet that no +/// longer has the account — a manifest and a wallet that disagree about which +/// accounts exist. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_transient_registration_persist_failure_is_retried_and_keeps_the_account() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC1u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "transient").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("a transient persist failure must be retried, not reported to the user"); + + let view = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("the account must survive a retried transient failure"); + assert!( + view.managed, + "the managed collection must keep the account too — funding-address \ + derivation reads both sides", + ); + assert!( + !backend.registration_persist_buffer_is_staged(), + "a successful retry must leave nothing staged", + ); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the retried write must have made the registration durable", + ); +} + +/// When the retry budget runs out while the failure is still transient, the +/// account stays in memory and the next provisioning attempt finishes the +/// write instead of taking the idempotent early return. +/// +/// Reporting success on the second call would leave the row staged forever: +/// the account is live in memory, absent from the manifest, and every launch +/// after this one re-derives nothing. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_exhausted_transient_persist_budget_keeps_memory_and_the_next_attempt_rewrites_it() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC2u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "exhausted").await; + + // One fault per `store` the retry budget allows — every attempt resupplies + // the entries. Sizing the queue to the budget exactly leaves nothing armed + // for the follow-up call. + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); + let error = backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("an exhausted retry budget must surface as a typed error"); + assert!( + matches!(error, TaskError::IdentityFundingAccountPersistFailed { .. }), + "the persist failure must keep its dedicated variant, got {error:?}", + ); + + let view = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("a still-staged registration must not have its account removed"); + assert!(view.managed, "both collections must keep the account"); + assert!( + backend.registration_persist_buffer_is_staged(), + "the changeset must still be staged for a later rewrite", + ); + + // The next attempt sees both collections populated. It must rewrite the + // pending registration rather than short-circuit on the presence guards. + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the follow-up attempt must rewrite the pending registration"); + assert!( + !backend.registration_persist_buffer_is_staged(), + "the follow-up attempt must leave nothing staged", + ); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the staged registration must reach the manifest once the blip clears", + ); +} + +/// A terminal persister failure discards the staged changeset, so the account +/// must leave memory — otherwise the presence guards make every later attempt +/// report success on a row that was never written. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terminal_registration_persist_failure_rolls_back_the_in_memory_account() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC3u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "terminal").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Fatal]); + let error = backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("a terminal persist failure must surface as a typed error"); + assert!( + matches!(error, TaskError::IdentityFundingAccountPersistFailed { .. }), + "the persist failure must keep its dedicated variant, got {error:?}", + ); + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .is_none(), + "a discarded changeset must take the in-memory account with it", + ); + + // With the fault queue empty, a retry re-creates and re-persists. + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the retry must re-create and re-persist the account"); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the retry must have made the registration durable", + ); +} + +/// A terminal failure must only evict what the failing call itself created. +/// +/// A call rewrites every registration still pending on the wallet, not just +/// its own, so its `pending` set includes entries inherited from earlier +/// attempts. Its failure says nothing about those: a foreign flush may already +/// have made them durable, and the next attempt rewrites them regardless. +/// Evicting them as collateral would drop an account that can be on disk — +/// memory-without-disk's mirror image — so they keep both their account and +/// their pending marker, and a durable account is never touched at all. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terminal_failure_keeps_inherited_pending_registrations_for_the_next_attempt() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC4u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "inherited").await; + + // Leave the top-up registration pending. + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("the exhausted budget must surface"); + + // The next call creates nothing — both accounts are already in memory — so + // its only work is rewriting the inherited pending registration, and that + // is what fails terminally here. + backend.arm_registration_persist_faults([PersistenceErrorKind::Fatal]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("a terminal rewrite must surface"); + + assert!( + backend + .identity_funding_account( + &seed_hash, + dash_sdk::dpp::key_wallet::AccountType::IdentityRegistration, + ) + .await + .expect("probe the registration account") + .is_some(), + "a durable account must survive another account's failed rewrite", + ); + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the top-up account") + .is_some(), + "an inherited pending account must not be evicted as collateral — its \ + row may already be durable by another route", + ); + assert!( + backend.has_pending_account_registrations(&seed_hash), + "the inherited registration must stay pending so the next attempt \ + rewrites it", + ); + + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("a clean retry must rewrite the pending top-up registration"); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the retry must have made the top-up registration durable", + ); +} + +/// A write that another caller drains and loses must not be reported as saved. +/// +/// The persister buffer is shared per wallet, not owned by this call site, and +/// `flush` answers `Ok(())` when it finds nothing staged. Upstream writes into +/// that buffer constantly and swallows its own terminal failures, which take +/// the whole merged changeset — this registration included — with them. A +/// retry that only re-flushes therefore cannot tell "my row was committed" +/// from "my row was thrown away by somebody else", and reporting success there +/// strands an asset lock against a credit-output path that exists only in RAM. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_registration_lost_by_a_foreign_buffer_drain_is_rewritten_not_reported_saved() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC7u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "foreign").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("the exhausted budget must surface"); + + // An unrelated wallet write drains the shared buffer and dies terminally: + // the staged registration is gone and this call site is never told. + backend.discard_staged_registration_buffer(); + + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the follow-up attempt must rewrite the lost registration"); + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .is_some(), + "the account must still be live in memory after a successful rewrite", + ); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "success must mean the row is in the manifest, not merely that some \ + flush returned Ok", + ); +} + +/// The same loss, inside a single retry loop, must also be rewritten. +/// +/// After a transient `store` parks the changeset, the backoff window belongs +/// to everyone: an unrelated writer can drain the shared buffer and lose it +/// terminally before this call's next attempt. A retry that only re-flushes +/// would read `Ok(())` off an empty buffer and report a saved registration +/// that no longer exists anywhere. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_registration_lost_between_retries_is_rewritten_not_reported_saved() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC9u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "between").await; + + // The first `store` parks the changeset; before the retry runs, an + // unrelated writer drains the shared buffer and loses it. + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient]); + backend.discard_staged_registration_buffer_before_write(2); + + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the retry must rewrite the lost registration, not ride the buffer"); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "reporting success means the row is in the manifest — a flush that found \ + an empty buffer proves nothing", + ); +} + +/// `Constraint` is the third terminal kind and must behave exactly like +/// `Fatal`. `PersistenceErrorKind` is deliberately not `#[non_exhaustive]` so +/// that a new kind breaks every consumer match — pinning this branch is what +/// makes that guarantee worth something here. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_constraint_registration_persist_failure_rolls_back_the_in_memory_account() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC5u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "constraint").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Constraint]); + let error = backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("a constraint violation must surface as a typed error"); + assert!( + matches!(error, TaskError::IdentityFundingAccountPersistFailed { .. }), + "the persist failure must keep its dedicated variant, got {error:?}", + ); + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .is_none(), + "a constraint violation is terminal, so the account must leave memory", + ); + assert!( + !backend.has_pending_account_registrations(&seed_hash), + "a terminal failure must leave nothing pending to rewrite", + ); + + backend.shutdown().await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terminal_retry_preserves_a_registration_committed_by_another_writer() { + assert_terminal_retry_preserves_registration(true).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terminal_retry_keeps_a_discarded_registration_pending_for_rewrite() { + assert_terminal_retry_preserves_registration(false).await; +} + +async fn assert_terminal_retry_preserves_registration(foreign_commit: bool) { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC6u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "escalate").await; + + // Transient on the first `store`, then terminal on the retry. + backend.arm_registration_persist_faults([ + PersistenceErrorKind::Transient, + PersistenceErrorKind::Fatal, + ]); + if foreign_commit { + backend.commit_staged_registration_buffer_before_write(2); + } + let error = backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("the escalated failure must surface"); + assert!(matches!( + error, + TaskError::IdentityFundingAccountPersistFailed { .. } + )); + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + i64::from(foreign_commit), + "the foreign writer determines whether the row is already durable", + ); + + let account = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("an earlier transient failure makes durability ambiguous"); + assert!(account.managed); + assert!( + !backend.registration_persist_buffer_is_staged(), + "a terminal failure must take the staged changeset with it", + ); + assert!( + backend.has_pending_account_registrations(&seed_hash), + "an ambiguous write must remain pending for an idempotent rewrite", + ); + + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the next attempt must rewrite the pending registration"); + assert!(!backend.has_pending_account_registrations(&seed_hash)); + assert_eq!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .unwrap(), + Some(account), + ); + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the rewrite must leave exactly one durable registration", + ); +} + +/// Removing a wallet must drop its pending registrations. +/// +/// A same-seed re-import computes the same upstream `WalletId`, so a marker +/// left behind would be inherited by the fresh wallet and rewritten against +/// accounts it never provisioned. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn removing_a_wallet_drops_its_pending_account_registrations() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC8u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "removed").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("the exhausted budget must surface"); + assert!( + backend.has_pending_account_registrations(&seed_hash), + "the registration must be recorded as pending before removal", + ); + + let wallet_id = backend.registered_wallet_id(&seed_hash).expect("wallet id"); + backend + .forget_wallet_local_state(&seed_hash, Some(wallet_id)) + .expect("forget wallet local state"); + + assert!( + !backend.has_pending_account_registrations(&seed_hash), + "wallet removal must drop the pending registration, or a same-seed \ + re-import inherits it", + ); + + backend.shutdown().await; +} + +/// Provisioning two funding accounts on one wallet must not overlap. +/// +/// Each call reads the wallet's pending-registration set, releases the manager +/// guard for the persist, and writes the set back. Overlapping calls interleave +/// those read-modify-write cycles, so one call's bookkeeping lands on top of +/// the other's and a registration that never reached the manifest is left with +/// nothing recorded to rewrite it. Serialising per wallet is what makes the +/// pending set safe to reason about; the manager guard used to do it by +/// accident, and no longer does. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn provisioning_two_funding_accounts_on_one_wallet_does_not_overlap() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xCEu8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "concurrent").await; + + // Enough transient faults that both calls sit in their retry windows at + // the same time if nothing serialises them. + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 8]); + + let first = { + let backend = backend.clone(); + tokio::spawn(async move { + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, 11) + .await + }) + }; + let second = { + let backend = backend.clone(); + tokio::spawn(async move { + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, 12) + .await + }) + }; + let _ = first.await.expect("join first"); + let _ = second.await.expect("join second"); + + assert_eq!( + backend.provisioning_high_water(), + 1, + "two provisioning calls on one wallet overlapped; their pending-registration \ + bookkeeping can then clobber each other", + ); + + backend.shutdown().await; +} + +/// A pending registration must not outlive the wallet it belongs to. +/// +/// `forget_wallet_local_state` is the synchronous cleanup path and cannot take +/// the (async) provisioning lock, so a provisioning call already inside its +/// retry window can still be running when the wallet is removed and record its +/// marker afterwards. A same-seed re-import recomputes the identical +/// `WalletId`, so that leftover would be inherited by the fresh wallet and +/// rewritten against accounts it never provisioned — exactly what the prune +/// exists to stop. Recording is therefore refused once the wallet is gone. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_pending_registration_recorded_after_wallet_removal_is_refused() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xCAu8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "outlive").await; + let wallet_id = backend.registered_wallet_id(&seed_hash).expect("wallet id"); + + // Provision once so a real account xpub is available to record. + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("provision identity funding accounts"); + let account_xpub = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("the top-up account must be live in memory") + .account_xpub; + + backend + .forget_wallet_local_state(&seed_hash, Some(wallet_id)) + .expect("forget wallet local state"); + + // The tail of a provisioning call that was still in flight when the wallet + // was removed: it reaches its bookkeeping after the prune has run. + backend + .record_pending_account_registration_for_test(wallet_id, FAULT_TOPUP_INDEX, account_xpub) + .expect("recording must not error, only decline"); + + assert!( + !backend.has_pending_account_registrations_for(wallet_id), + "a pending registration recorded after the wallet was removed outlives \ + it, and a same-seed re-import inherits it", + ); + + backend.shutdown().await; +} + +/// Identity index the persist-failure tests provision a top-up account for. +/// +/// Per-index top-up accounts are the funding accounts DET actually creates: +/// every other identity-funding account type is pre-created and persisted when +/// the wallet is registered, so provisioning them takes the idempotent early +/// return and never reaches the persist path under test. +const FAULT_TOPUP_INDEX: u32 = 11; + +fn fault_topup_account_type() -> dash_sdk::dpp::key_wallet::AccountType { + dash_sdk::dpp::key_wallet::AccountType::IdentityTopUp { + registration_index: FAULT_TOPUP_INDEX, + } +} + /// An identity publishing real public keys. `Identity::create_basic_identity` /// publishes none, and a key-less identity persists no `identity_keys` rows — /// the very rows whose mis-filing bricks a wallet at load time. diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index 152476013..103b27f6c 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -10,15 +10,32 @@ use crate::backend_task::error::TaskError; use crate::model::wallet::WalletSeedHash; -use platform_wallet::changeset::PlatformWalletPersistence; -use platform_wallet::wallet::platform_wallet::WalletId; use std::sync::Arc; use super::{ - DetPlatformSigner, DetSigner, PlatformPathIndex, WalletBackend, map_identity_register_error, - map_identity_top_up_error, map_platform_address_fund_error, + DetPlatformSigner, DetSigner, PlatformPathIndex, WalletBackend, WalletId, + map_identity_register_error, map_identity_top_up_error, map_platform_address_fund_error, }; +use dash_sdk::dpp::key_wallet::account::Account; +use dash_sdk::dpp::key_wallet::bip32::ExtendedPubKey; +use platform_wallet::changeset::{ + AccountRegistrationEntry, PersistenceError, PlatformWalletChangeSet, PlatformWalletPersistence, +}; + +/// Total attempts for a transient persister failure on the funding-account +/// path. Mirrors upstream's `PERSIST_RETRY_MAX_ATTEMPTS` on the +/// wallet-registration path, which is `pub(super)` there and so unreachable +/// from DET. +const PERSIST_RETRY_MAX_ATTEMPTS: u32 = 4; + +/// Backoff before the first retry; doubles up to [`PERSIST_RETRY_MAX_BACKOFF`]. +const PERSIST_RETRY_INITIAL_BACKOFF: std::time::Duration = std::time::Duration::from_millis(20); + +/// Ceiling for the doubling backoff, so a busy database still surfaces +/// promptly (worst case with the constants above: 20 + 40 + 80 ≈ 140 ms). +const PERSIST_RETRY_MAX_BACKOFF: std::time::Duration = std::time::Duration::from_millis(200); + /// The all-zero `WalletId` upstream storage reserves as the spelling of /// "owned by no wallet": every scope-aware reader and writer maps it to a /// NULL `wallet_id`. Never a real wallet id. @@ -29,8 +46,8 @@ const UNOWNED_WALLET_SCOPE: WalletId = [0u8; 32]; /// removes the repeated `AccountType` matches — and their `unreachable!` arms — /// inside [`WalletBackend::provision_identity_funding_account`], and makes the /// unsupported-account-type case unrepresentable. -#[derive(Clone, Copy)] -enum Funding { +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum Funding { /// The identity-registration funding account. Registration, /// The per-identity top-up funding account at the given registration index. @@ -41,6 +58,227 @@ enum Funding { TopUpNotBound, } +impl Funding { + /// The upstream account type this funding flavour derives and registers. + fn account_type(self) -> dash_sdk::dpp::key_wallet::AccountType { + use dash_sdk::dpp::key_wallet::AccountType; + match self { + Self::Registration => AccountType::IdentityRegistration, + Self::TopUp(registration_index) => AccountType::IdentityTopUp { registration_index }, + Self::TopUpNotBound => AccountType::IdentityTopUpNotBoundToIdentity, + } + } + + /// Whether the managed (info-side) collection holds this funding account. + fn is_managed( + self, + info: &platform_wallet::wallet::platform_wallet::PlatformWalletInfo, + ) -> bool { + match self { + Self::Registration => info.core_wallet.accounts.identity_registration.is_some(), + Self::TopUp(registration_index) => info + .core_wallet + .accounts + .identity_topup + .contains_key(®istration_index), + Self::TopUpNotBound => info.core_wallet.accounts.identity_topup_not_bound.is_some(), + } + } + + /// This funding account as the key-wallet holds it, if at all. + fn account(self, kw: &dash_sdk::dpp::key_wallet::wallet::Wallet) -> Option<&Account> { + match self { + Self::Registration => kw.accounts.identity_registration.as_ref(), + Self::TopUp(registration_index) => kw.accounts.identity_topup.get(®istration_index), + Self::TopUpNotBound => kw.accounts.identity_topup_not_bound.as_ref(), + } + } + + /// Drop this funding account from both in-memory collections, so the next + /// provisioning attempt re-creates and re-persists it instead of + /// short-circuiting on the presence guards. + fn remove( + self, + kw: &mut dash_sdk::dpp::key_wallet::wallet::Wallet, + info: &mut platform_wallet::wallet::platform_wallet::PlatformWalletInfo, + ) { + match self { + Self::Registration => { + kw.accounts.identity_registration = None; + info.core_wallet.accounts.identity_registration = None; + } + Self::TopUp(registration_index) => { + kw.accounts.identity_topup.remove(®istration_index); + info.core_wallet + .accounts + .identity_topup + .remove(®istration_index); + } + Self::TopUpNotBound => { + kw.accounts.identity_topup_not_bound = None; + info.core_wallet.accounts.identity_topup_not_bound = None; + } + } + } +} + +/// One provisioned identity-funding account as the wallet holds it, for +/// asserting that a restart restored the same account rather than a +/// same-shaped placeholder. +#[cfg(test)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FundingAccountView { + /// Account type on the key-wallet account — carries the registration index + /// for [`AccountType::IdentityTopUp`](dash_sdk::dpp::key_wallet::AccountType::IdentityTopUp). + pub account_type: dash_sdk::dpp::key_wallet::AccountType, + /// The account xpub funding-address derivation reads. + pub account_xpub: dash_sdk::dpp::key_wallet::bip32::ExtendedPubKey, + /// Whether the managed (info-side) collection also holds the account. + pub managed: bool, +} + +/// Per-wallet serialisation for identity-funding provisioning. +/// +/// A provisioning call reads the wallet's pending-registration set, releases +/// the manager guard for the persist, then writes the set back. Overlapping +/// calls interleave those read-modify-write cycles and one call's bookkeeping +/// lands on top of the other's — leaving a registration that never reached the +/// manifest with nothing recorded to rewrite it. The manager guard used to +/// serialise these by accident; it is deliberately released for the persist +/// now (holding a process-wide lock across a retry backoff stalls every +/// wallet), so this carries the invariant explicitly, and per wallet. +/// +/// Lock order is provisioning lock → manager guard, never the reverse. +#[derive(Default)] +pub(super) struct FundingProvisionLocks { + locks: std::sync::Mutex< + std::collections::BTreeMap>>, + >, +} + +impl FundingProvisionLocks { + async fn lock(&self, wallet_id: WalletId) -> tokio::sync::OwnedMutexGuard<()> { + let wallet_lock = { + let mut locks = self + .locks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + locks.retain(|_, lock| lock.strong_count() > 0); + match locks.get(&wallet_id).and_then(std::sync::Weak::upgrade) { + Some(lock) => lock, + None => { + let lock = Arc::new(tokio::sync::Mutex::new(())); + locks.insert(wallet_id, Arc::downgrade(&lock)); + lock + } + } + }; + wallet_lock.lock_owned().await + } +} + +/// Whether a failed registration write permits removing the account from memory. +enum RegistrationPersistFailure { + /// A transient attempt exposed the registration to the shared buffer. + /// It may already be durable; retain the account and its pending rewrite. + Staged(PersistenceError), + /// The first attempt failed terminally, without an earlier buffered write. + Discarded(PersistenceError), +} + +impl RegistrationPersistFailure { + fn into_task_error(self) -> TaskError { + let (Self::Staged(source) | Self::Discarded(source)) = self; + TaskError::IdentityFundingAccountPersistFailed { + source: Box::new(source), + } + } +} + +/// Retry `op` while it fails transiently, with bounded exponential backoff, +/// and classify whatever error survives. +/// +/// Mirrors upstream's `retry_transient` on the wallet-registration path. The +/// sleep is async so it yields the Tokio worker rather than spinning. +async fn retry_transient(mut op: F) -> Result<(), RegistrationPersistFailure> +where + F: FnMut() -> Result<(), PersistenceError>, +{ + let mut backoff = PERSIST_RETRY_INITIAL_BACKOFF; + let mut attempt: u32 = 1; + loop { + match op() { + Ok(()) => return Ok(()), + Err(e) if !e.is_transient() && attempt == 1 => { + return Err(RegistrationPersistFailure::Discarded(e)); + } + // A previous transient attempt may have been committed by another writer. + Err(e) if !e.is_transient() || attempt >= PERSIST_RETRY_MAX_ATTEMPTS => { + return Err(RegistrationPersistFailure::Staged(e)); + } + Err(e) => { + tracing::debug!( + attempt, + max_attempts = PERSIST_RETRY_MAX_ATTEMPTS, + backoff_ms = backoff.as_millis() as u64, + error = %e, + "identity-funding account registration hit a transient persister failure; backing off before rewriting the registration" + ); + tokio::time::sleep(backoff).await; + backoff = backoff.saturating_mul(2).min(PERSIST_RETRY_MAX_BACKOFF); + attempt += 1; + } + } + } +} + +/// Write these account registrations through `persister`, so a cold boot +/// rebuilds the accounts from the manifest. +/// +/// A `store` the backend answers +/// [`store_commits_inline`](PlatformWalletPersistence::store_commits_inline) +/// for is the whole write, and no `flush` follows it. Such a `store` has +/// already committed its transaction, so the follow-up could add no +/// durability, only exposure: the buffer is shared per wallet, so that call +/// would adopt whatever another writer had parked in it and hand us that +/// stranger's failure — classifying a registration whose row is already on +/// disk as discarded, and evicting its account from memory. DET's SQLite +/// persister answers `true` (it runs in `FlushMode::Immediate`), which is +/// asked here rather than assumed, so selecting another flush mode at the +/// construction site changes the write shape instead of silently costing the +/// durability this function reports. +/// +/// A backend that only buffers is flushed in the same attempt: there, nothing +/// is durable until the flush, and a failure at either step is one failure of +/// one attempt — retried and classified together. +/// +/// Every attempt — the first and each retry — **resupplies** the entries +/// rather than issuing a bare `flush`, which answers `Ok(())` on an empty +/// buffer and so cannot distinguish "committed" from "thrown away by somebody +/// else". Resupplying is safe precisely here, and the general "re-`store` +/// would double-merge" caution does not bind: `account_registrations` merges +/// by `extend` and applies through `UPSERT_ACCOUNT_SQL`, an +/// `ON CONFLICT(wallet_id, account_type, account_index, …) DO UPDATE` keyed on +/// the account's identity, so writing the same entry twice is idempotent. +async fn persist_account_registrations( + persister: &dyn PlatformWalletPersistence, + wallet_id: WalletId, + entries: &[AccountRegistrationEntry], +) -> Result<(), RegistrationPersistFailure> { + retry_transient(|| { + let changeset = PlatformWalletChangeSet { + account_registrations: entries.to_vec(), + ..Default::default() + }; + persister.store(wallet_id, changeset)?; + if persister.store_commits_inline() { + return Ok(()); + } + persister.flush(wallet_id) + }) + .await +} + impl WalletBackend { /// Register a new identity on Platform funded by an asset lock built and /// tracked-to-finality by the upstream `AssetLockManager`. Returns the @@ -577,7 +815,9 @@ impl WalletBackend { // `IdentityTopUp{registration_index}` enters the manifest only after a // register/top-up, so a reloaded already-registered identity needs this // re-provision. Idempotent: probes both collections and no-ops if present - // (no error-string parsing — direct membership checks). + // (no error-string parsing — direct membership checks) — except when a + // previous transient persist failure left the registration staged, which + // this call finishes rather than reporting success on an absent row. // // `seed` must be the wallet's HD seed so the hardened account xpub can be // derived — the live wallet is watch-only and cannot derive hardened paths @@ -588,77 +828,285 @@ impl WalletBackend { seed: &[u8; 64], funding: Funding, ) -> Result<(), TaskError> { - use dash_sdk::dpp::key_wallet::AccountType; use dash_sdk::dpp::key_wallet::managed_account::ManagedCoreKeysAccount; let wallet = self.resolve_wallet(seed_hash).await?; let wallet_id = wallet.wallet_id(); - let mut wm = wallet.wallet_manager().write().await; - let (kw, info) = wm - .get_wallet_mut_and_info_mut(&wallet_id) - .ok_or(TaskError::WalletStateInconsistent)?; - let (in_wallet, in_managed) = match funding { - Funding::Registration => ( - kw.accounts.identity_registration.is_some(), - info.core_wallet.accounts.identity_registration.is_some(), - ), - Funding::TopUp(registration_index) => ( - kw.accounts.identity_topup.contains_key(®istration_index), + // Serialise per wallet: the pending-registration set below is a + // read-modify-write spanning an unlocked persist. + let _provisioning = self.inner.funding_provision_locks.lock(wallet_id).await; + + #[cfg(test)] + let _overlap = self.inner.provision_overlap.enter(); + + // Everything that touches wallet state happens under the manager + // guard; the persist below deliberately does not. `wallet_manager()` + // hands out ONE `RwLock` shared by every wallet in the + // process, and it is write-preferring, so sleeping a retry backoff + // under it would stall balance reads, sync callbacks and wallet + // removal for every other wallet — upstream warns about exactly this. + let (pending, created) = { + let mut wm = wallet.wallet_manager().write().await; + let (kw, info) = wm + .get_wallet_mut_and_info_mut(&wallet_id) + .ok_or(TaskError::WalletStateInconsistent)?; + + let in_wallet = funding.account(kw).is_some(); + let in_managed = funding.is_managed(info); + + // Registrations a previous attempt could not make durable. They + // are rewritten below: returning success while one is outstanding + // would leave its account live in memory and absent from the + // manifest. + let mut pending = self.staged_account_registrations(&wallet_id)?; + if in_wallet && in_managed && pending.is_empty() { + return Ok(()); + } + + if !in_wallet { + let account_type = funding.account_type(); + // The live wallet is watch-only: calling `add_account(…, None)` would + // try to derive a hardened path from an absent private key and fail + // with "Watch-only wallet has no private key". Derive the xpub from a + // short-lived seed wallet instead and pass it as `Some(xpub)`. + let seed_wallet = self.seed_wallet(seed)?; + + let path = account_type + .derivation_path(self.inner.network) + .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { + source, + })?; + + let account_xpub = + seed_wallet + .derive_extended_public_key(&path) + .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { + source, + })?; + + kw.add_account(account_type, Some(account_xpub)) + .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { + source, + })?; + } + + let mut created = None; + if !(in_wallet && in_managed) { + let derived = funding + .account(kw) + .ok_or(TaskError::WalletStateInconsistent)?; + let account_xpub = derived.account_xpub; + let managed = ManagedCoreKeysAccount::from_account(derived); info.core_wallet .accounts - .identity_topup - .contains_key(®istration_index), - ), - Funding::TopUpNotBound => ( - kw.accounts.identity_topup_not_bound.is_some(), - info.core_wallet.accounts.identity_topup_not_bound.is_some(), - ), + .insert_keys_bearing_account(managed) + .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { + source, + })?; + pending.insert(funding, account_xpub); + created = Some(funding); + } + + (pending, created) + }; + + // In test builds the write goes through a fault-injecting decorator + // that is inert until a test arms it; production talks to the + // persister directly. + #[cfg(not(test))] + let persister: &dyn PlatformWalletPersistence = self.inner.wallet_persister.as_ref(); + #[cfg(test)] + let injector = super::persist_fault_test_support::PersistFaultInjector::new( + &self.inner.wallet_persister, + &self.inner.persist_faults, + ); + #[cfg(test)] + let persister: &dyn PlatformWalletPersistence = &injector; + + // Persist the registrations. `load()` rebuilds `Wallet.accounts` from + // `account_registrations` alone, and the upstream creator that would + // normally write these rows skips them once both in-memory collections + // hold the account — which they do by the time it runs, because of the + // inserts above. Without this the account is memory-only: a restart + // between an asset-lock broadcast and its consumption cannot re-derive + // the credit-output path, stranding the lock and its funds. + let entries: Vec = pending + .iter() + .map(|(funding, account_xpub)| AccountRegistrationEntry { + account_type: funding.account_type(), + account_xpub: *account_xpub, + }) + .collect(); + + let failure = match persist_account_registrations(persister, wallet_id, &entries).await { + // Only the entries this call actually wrote stop being pending. + // Another call's marker is not this call's to clear. + Ok(()) => { + let written: Vec = pending.keys().copied().collect(); + return self.clear_staged_account_registrations(&wallet_id, &written); + } + Err(failure) => failure, }; - if in_wallet && in_managed { - return Ok(()); - } - if !in_wallet { - let account_type = match funding { - Funding::Registration => AccountType::IdentityRegistration, - Funding::TopUp(registration_index) => { - AccountType::IdentityTopUp { registration_index } + // Keep the accounts only when the write is still worth retrying AND + // that intent was actually recorded. A marker we could not write is a + // silent dead end: the account would stay in memory with nothing + // scheduled to persist it, and the next call would early-return + // success on a row that is not there. + let recoverable = matches!(failure, RegistrationPersistFailure::Staged(_)) + && match self.mark_account_registrations_staged(wallet_id, &pending) { + Ok(()) => true, + Err(e) => { + tracing::error!( + wallet = %hex::encode(wallet_id), + error = ?e, + "could not record a pending identity-funding registration; dropping the account so a retry rebuilds it" + ); + false } - Funding::TopUpNotBound => AccountType::IdentityTopUpNotBoundToIdentity, }; - // The live wallet is watch-only: calling `add_account(…, None)` would - // try to derive a hardened path from an absent private key and fail - // with "Watch-only wallet has no private key". Derive the xpub from a - // short-lived seed wallet instead and pass it as `Some(xpub)`. - let seed_wallet = self.seed_wallet(seed)?; - - let path = account_type - .derivation_path(self.inner.network) - .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; - - let account_xpub = seed_wallet - .derive_extended_public_key(&path) - .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; - - kw.add_account(account_type, Some(account_xpub)) - .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; - } - let derived = match funding { - Funding::Registration => kw.accounts.identity_registration.as_ref(), - Funding::TopUp(registration_index) => { - kw.accounts.identity_topup.get(®istration_index) + if let (false, Some(created)) = (recoverable, created) { + // Only the account this call created is known not to be durable, + // so only it leaves memory — otherwise the presence guards would + // short-circuit a persist that never happened. Entries inherited + // from an earlier attempt keep both their account and their + // marker: this failure says nothing about them, a foreign flush + // may already have committed them, and the next attempt rewrites + // them either way. Evicting on their behalf would drop an account + // that can be on disk. + self.evict_funding_account(&wallet, wallet_id, created) + .await; + if let Err(e) = self.clear_staged_account_registrations(&wallet_id, &[created]) { + tracing::error!( + wallet = %hex::encode(wallet_id), + error = ?e, + "could not clear a pending identity-funding registration after a terminal persist failure" + ); } - Funding::TopUpNotBound => kw.accounts.identity_topup_not_bound.as_ref(), } - .ok_or(TaskError::WalletStateInconsistent)?; - let managed = ManagedCoreKeysAccount::from_account(derived); - info.core_wallet - .accounts - .insert_keys_bearing_account(managed) - .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; + Err(failure.into_task_error()) + } + + /// Drop a funding account this call created but could not persist, from + /// both in-memory collections, so the next attempt re-creates and + /// re-persists it. + /// + /// Re-acquires the manager guard, which was released for the persist. + /// Removing unconditionally is sound because the per-wallet provisioning + /// lock is still held, so no concurrent provisioning can have re-created + /// this account in the meantime. An xpub comparison could not establish + /// that in any case: the xpub is a deterministic derivation from the seed + /// and the account's path, so a concurrent provisioning of the same + /// account yields the identical value. + async fn evict_funding_account( + &self, + wallet: &Arc, + wallet_id: WalletId, + funding: Funding, + ) { + let mut wm = wallet.wallet_manager().write().await; + let Some((kw, info)) = wm.get_wallet_mut_and_info_mut(&wallet_id) else { + return; + }; + funding.remove(kw, info); + } + + /// The funding accounts whose registration a previous attempt could not + /// make durable, with the account xpub each one still needs written. + fn staged_account_registrations( + &self, + wallet_id: &WalletId, + ) -> Result, TaskError> { + Ok(self + .inner + .buffered_account_registrations + .lock()? + .get(wallet_id) + .cloned() + .unwrap_or_default()) + } + + /// Record registrations for the next provisioning attempt on this wallet + /// to rewrite, instead of early-returning success on absent rows. + fn mark_account_registrations_staged( + &self, + wallet_id: WalletId, + pending: &std::collections::BTreeMap, + ) -> Result<(), TaskError> { + let mut buffered = self.inner.buffered_account_registrations.lock()?; + + // Declining for a wallet that is gone is what keeps a pending + // registration from outliving it — `forget_wallet_local_state` is the + // synchronous cleanup path and cannot take the (async) provisioning + // lock, so a call already inside its retry window can reach this point + // after the removal has pruned. A same-seed re-import recomputes the + // identical `WalletId` and would inherit the leftover. + // + // The `wallets` read below is taken while this pending-set guard is + // held; removal takes the two the other way round, so it must not hold + // its `wallets` guard across its prune (noted at that site). + // + // Sound for every interleaving because removal drops the wallet from + // `Inner::wallets` BEFORE pruning here, and this liveness check is + // taken while holding the pending-set lock the prune also needs: + // seeing the wallet still present means the prune has not run yet and + // must wait for this lock, so it will remove what is inserted below; + // seeing it absent means the removal already happened and there is + // nothing worth recording. + if !self.inner.wallets.read()?.contains_key(&wallet_id) { + tracing::debug!( + wallet = %hex::encode(wallet_id), + "declining to record a pending identity-funding registration for a removed wallet" + ); + return Ok(()); + } + + buffered + .entry(wallet_id) + .or_default() + .extend(pending.iter().map(|(f, x)| (*f, *x))); + Ok(()) + } + + /// Record a pending funding-account registration through the real + /// bookkeeping path, so a test can drive the ordering a concurrent + /// provisioning would produce without racing it. + #[cfg(test)] + pub(crate) fn record_pending_account_registration_for_test( + &self, + wallet_id: WalletId, + registration_index: u32, + account_xpub: ExtendedPubKey, + ) -> Result<(), TaskError> { + let pending = + std::collections::BTreeMap::from([(Funding::TopUp(registration_index), account_xpub)]); + self.mark_account_registrations_staged(wallet_id, &pending) + } + + /// Forget the given registrations, once they are durable or definitively + /// lost. Scoped to `keys` rather than the whole wallet: a concurrent + /// attempt's pending registration is not this call's to forget, and + /// dropping it would leave that account in memory with nothing left to + /// rewrite it. + fn clear_staged_account_registrations( + &self, + wallet_id: &WalletId, + keys: &[Funding], + ) -> Result<(), TaskError> { + let mut pending = self.inner.buffered_account_registrations.lock()?; + let std::collections::btree_map::Entry::Occupied(mut entry) = pending.entry(*wallet_id) + else { + return Ok(()); + }; + for key in keys { + entry.get_mut().remove(key); + } + if entry.get().is_empty() { + entry.remove(); + } Ok(()) } @@ -680,6 +1128,54 @@ impl WalletBackend { .await } + /// The provisioned identity-funding account for `account_type`, as seen by + /// the two collections that must both hold it. + /// + /// `peek_next_funding_address` reads the account xpub from + /// `Wallet.accounts` and the address pool from the managed collection, so + /// a restore that repopulates only one side leaves funding broken. Returns + /// `None` when the key-wallet side has no such account. + #[cfg(test)] + pub(crate) async fn identity_funding_account( + &self, + seed_hash: &WalletSeedHash, + account_type: dash_sdk::dpp::key_wallet::AccountType, + ) -> Result, TaskError> { + use dash_sdk::dpp::key_wallet::AccountType; + + let wallet = self.resolve_wallet(seed_hash).await?; + let wallet_id = wallet.wallet_id(); + let wm = wallet.wallet_manager().read().await; + let (kw, info) = wm + .get_wallet_and_info(&wallet_id) + .ok_or(TaskError::WalletStateInconsistent)?; + + let (derived, managed) = match account_type { + AccountType::IdentityRegistration => ( + kw.accounts.identity_registration.as_ref(), + info.core_wallet.accounts.identity_registration.is_some(), + ), + AccountType::IdentityTopUp { registration_index } => ( + kw.accounts.identity_topup.get(®istration_index), + info.core_wallet + .accounts + .identity_topup + .contains_key(®istration_index), + ), + AccountType::IdentityTopUpNotBoundToIdentity => ( + kw.accounts.identity_topup_not_bound.as_ref(), + info.core_wallet.accounts.identity_topup_not_bound.is_some(), + ), + _ => return Err(TaskError::WalletStateInconsistent), + }; + + Ok(derived.map(|account| FundingAccountView { + account_type: account.account_type, + account_xpub: account.account_xpub, + managed, + })) + } + /// Provision the index-less top-up funding account — the credit-output /// source for topping up an identity this wallet does not own. Idempotent; /// `seed` must be held for the duration of the call. @@ -692,3 +1188,121 @@ impl WalletBackend { .await } } + +#[cfg(test)] +mod tests { + use super::*; + use platform_wallet::changeset::ClientStartState; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Counts the writes [`persist_account_registrations`] issues, answering + /// the inline-commit question however the test asks it to. + struct CountingPersister { + commits_inline: bool, + stores: AtomicUsize, + flushes: AtomicUsize, + } + + impl CountingPersister { + fn new(commits_inline: bool) -> Self { + Self { + commits_inline, + stores: AtomicUsize::new(0), + flushes: AtomicUsize::new(0), + } + } + } + + impl PlatformWalletPersistence for CountingPersister { + fn store_commits_inline(&self) -> bool { + self.commits_inline + } + + fn store( + &self, + _wallet_id: WalletId, + _changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + self.stores.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { + self.flushes.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn load(&self) -> Result { + unimplemented!("the registration write never loads") + } + } + + fn one_entry() -> Vec { + // A BIP32 serialized xpub: version, depth, parent fingerprint, child + // number, chain code, then the compressed point (secp256k1's + // generator, so the key itself is valid). + let mut raw = Vec::with_capacity(78); + raw.extend_from_slice(&[0x04, 0x88, 0xB2, 0x1E]); + raw.push(0); + raw.extend_from_slice(&[0u8; 4]); + raw.extend_from_slice(&[0u8; 4]); + raw.extend_from_slice(&[1u8; 32]); + raw.extend_from_slice(&[ + 0x02, 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, + 0x87, 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, + 0x5B, 0x16, 0xF8, 0x17, 0x98, + ]); + let xpub = ExtendedPubKey::decode(&raw).expect("decode a well-formed xpub"); + vec![AccountRegistrationEntry { + account_type: dash_sdk::dpp::key_wallet::AccountType::IdentityRegistration, + account_xpub: xpub, + }] + } + + /// A backend whose `store` commits inline gets exactly one write. The + /// follow-up `flush` would add no durability, and the per-wallet buffer it + /// adopts would let an unrelated writer's terminal failure discard a + /// registration already on disk. + #[tokio::test] + async fn an_inline_committing_store_is_the_whole_write() { + let persister = CountingPersister::new(true); + persist_account_registrations(&persister, [7u8; 32], &one_entry()) + .await + .map_err(|failure| failure.into_task_error()) + .expect("an unfaulted write must succeed"); + + assert_eq!( + persister.stores.load(Ordering::SeqCst), + 1, + "the registration must reach disk in exactly one `store`", + ); + assert_eq!( + persister.flushes.load(Ordering::SeqCst), + 0, + "a `store` that commits inline must not be followed by a `flush`", + ); + } + + /// A backend that only buffers is flushed, in the same attempt: without + /// it the registration would be reported durable while still in memory. + #[tokio::test] + async fn a_buffering_store_is_completed_by_a_flush() { + let persister = CountingPersister::new(false); + persist_account_registrations(&persister, [7u8; 32], &one_entry()) + .await + .map_err(|failure| failure.into_task_error()) + .expect("an unfaulted write must succeed"); + + assert_eq!( + persister.stores.load(Ordering::SeqCst), + 1, + "the entries must be supplied once per attempt", + ); + assert_eq!( + persister.flushes.load(Ordering::SeqCst), + 1, + "nothing is durable on a buffering backend until the flush that \ + completes the same attempt", + ); + } +} diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 347da211c..f11fa6aee 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -47,6 +47,8 @@ pub(crate) mod kv_test_support; pub(crate) mod leak_test_support; mod loader; mod payments; +#[cfg(test)] +pub(crate) mod persist_fault_test_support; pub(crate) mod poison; pub mod secret_access; pub mod secret_prompt; @@ -90,6 +92,7 @@ pub use secret_prompt::{ pub use secret_seam::SecretSeam; use coordinator_gate::CoordinatorGate; +use identity_ops::Funding; pub use auth_pubkey_cache::AuthPubkeyCacheView; pub use avatar_cache::AvatarCacheView; @@ -109,6 +112,49 @@ pub use token_balance::UpstreamTokenBalances; pub use wallet_meta::WalletMetaView; pub use wallet_seed_store::WalletSeedView; +/// Test-only detector for overlapping identity-funding provisioning. +/// +/// Counts concurrent entries into the provisioning body and remembers the +/// high-water mark, so a test can assert that two calls on one wallet are +/// serialised instead of racing over the shared pending-registration set. +/// Counts globally, so tests that use it must drive a single wallet. +#[cfg(test)] +#[derive(Default)] +pub(crate) struct ProvisionOverlap { + in_flight: std::sync::atomic::AtomicUsize, + high_water: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +impl ProvisionOverlap { + /// Record an entry; the returned guard records the matching exit. + fn enter(&self) -> ProvisionOverlapGuard<'_> { + use std::sync::atomic::Ordering; + let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1; + self.high_water.fetch_max(now, Ordering::SeqCst); + ProvisionOverlapGuard { owner: self } + } + + /// Greatest number of provisioning calls observed in flight at once. + pub(crate) fn high_water(&self) -> usize { + self.high_water.load(std::sync::atomic::Ordering::SeqCst) + } +} + +#[cfg(test)] +struct ProvisionOverlapGuard<'a> { + owner: &'a ProvisionOverlap, +} + +#[cfg(test)] +impl Drop for ProvisionOverlapGuard<'_> { + fn drop(&mut self) { + self.owner + .in_flight + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum TokenBalanceSyncOutcome { Performed, @@ -344,6 +390,23 @@ struct Inner { clear_shielded_test_failure: std::sync::atomic::AtomicBool, #[cfg(test)] forget_wallet_local_state_test_failure: std::sync::atomic::AtomicBool, + /// Injected persister faults for the identity-funding account + /// registration write. Inert until a test arms it. + #[cfg(test)] + persist_faults: persist_fault_test_support::PersistFaults, + #[cfg(test)] + provision_overlap: ProvisionOverlap, + /// Per-wallet serialisation for identity-funding provisioning. See + /// [`identity_ops::FundingProvisionLocks`]. + funding_provision_locks: identity_ops::FundingProvisionLocks, + /// Registrations with uncertain durability, rewritten before funding can proceed. + /// Retained after terminal retries too; cleared on successful write or wallet removal. + buffered_account_registrations: std::sync::Mutex< + std::collections::BTreeMap< + WalletId, + std::collections::BTreeMap, + >, + >, /// Forces the next unowned-scope read to fail, so a test can tell a /// wallet-store failure apart from a mirror upstream genuinely refused. #[cfg(test)] @@ -569,6 +632,14 @@ impl WalletBackend { #[cfg(test)] forget_wallet_local_state_test_failure: std::sync::atomic::AtomicBool::new(false), #[cfg(test)] + persist_faults: persist_fault_test_support::PersistFaults::default(), + #[cfg(test)] + provision_overlap: ProvisionOverlap::default(), + funding_provision_locks: identity_ops::FundingProvisionLocks::default(), + buffered_account_registrations: std::sync::Mutex::new( + std::collections::BTreeMap::new(), + ), + #[cfg(test)] unowned_read_test_failure: std::sync::atomic::AtomicBool::new(false), #[cfg(test)] swallow_next_unowned_write: std::sync::atomic::AtomicBool::new(false), @@ -1123,6 +1194,100 @@ impl WalletBackend { .store(fail, std::sync::atomic::Ordering::Relaxed); } + /// Queue persister faults for the identity-funding account registration + /// write, served one per `store` call in order. + #[cfg(test)] + pub(crate) fn arm_registration_persist_faults( + &self, + kinds: impl IntoIterator, + ) { + self.inner.persist_faults.arm(kinds); + } + + /// `true` while an injected transient failure still holds an uncommitted + /// account-registration changeset in the persister buffer. + #[cfg(test)] + pub(crate) fn registration_persist_buffer_is_staged(&self) -> bool { + self.inner.persist_faults.has_staged_changeset() + } + + /// Drop the staged changeset the way an unrelated writer's terminal flush + /// does — the buffer is shared per wallet, and its loss is never reported + /// to the account-registration caller. + #[cfg(test)] + pub(crate) fn discard_staged_registration_buffer(&self) { + self.inner.persist_faults.discard_staged(); + } + + /// Arm a one-shot foreign drain of the staged buffer immediately before the + /// `write`-th `store` on the registration path. Write 1 is the first + /// attempt's `store`, so arming write 2 puts the drain in the backoff + /// window between the first attempt and its retry. + #[cfg(test)] + pub(crate) fn discard_staged_registration_buffer_before_write(&self, write: usize) { + self.inner.persist_faults.discard_staged_before_write(write); + } + + /// Commit an injected pending registration before the selected retry. + #[cfg(test)] + pub(crate) fn commit_staged_registration_buffer_before_write(&self, write: usize) { + self.inner.persist_faults.commit_staged_before_write(write); + } + + /// `store` calls the identity-funding registration path has made since this + /// backend was built — one per persist attempt. + #[cfg(test)] + pub(crate) fn registration_persist_store_calls(&self) -> usize { + self.inner.persist_faults.store_calls() + } + + /// `flush` calls the identity-funding registration path has made since this + /// backend was built. The path makes none while + /// [`Self::registration_persist_commits_inline`] holds: a successful + /// `store` has already committed. + #[cfg(test)] + pub(crate) fn registration_persist_flush_calls(&self) -> usize { + self.inner.persist_faults.flush_calls() + } + + /// Whether DET's persister reports a successful `store` as already + /// committed — the property the flush-free registration write rests on. + #[cfg(test)] + pub(crate) fn registration_persist_commits_inline(&self) -> bool { + use platform_wallet::changeset::PlatformWalletPersistence; + self.inner.wallet_persister.store_commits_inline() + } + + /// Greatest number of identity-funding provisioning calls seen in flight at + /// once since this backend was built. + #[cfg(test)] + pub(crate) fn provisioning_high_water(&self) -> usize { + self.inner.provision_overlap.high_water() + } + + /// `true` when a pending funding-account registration is recorded for this + /// wallet, awaiting a rewrite by the next provisioning attempt. + #[cfg(test)] + pub(crate) fn has_pending_account_registrations(&self, seed_hash: &WalletSeedHash) -> bool { + let Some(wallet_id) = self.registered_wallet_id(seed_hash) else { + return false; + }; + self.has_pending_account_registrations_for(wallet_id) + } + + /// As [`Self::has_pending_account_registrations`], but keyed by the + /// upstream id. A removed wallet is gone from `id_map`, so the seed-hash + /// form cannot see leftovers that outlived it — which is the state worth + /// asserting after a removal. + #[cfg(test)] + pub(crate) fn has_pending_account_registrations_for(&self, wallet_id: WalletId) -> bool { + self.inner + .buffered_account_registrations + .lock() + .map(|pending| pending.contains_key(&wallet_id)) + .unwrap_or(false) + } + /// Resolve one just-registered upstream wallet into the DET-keyed maps via /// the account-xpub fund-routing gate, identical in spirit to the gate the /// seedless loader applies per wallet. @@ -1419,8 +1584,28 @@ impl WalletBackend { // In-memory maps + snapshot registration. if let Some(wallet_id) = wallet_id { self.inner.id_map.write()?.remove(seed_hash); + // This guard MUST stay a statement temporary, dropped at the + // semicolon. `mark_account_registrations_staged` takes the locks + // the other way round (pending set, then a `wallets` read), so + // holding this one across the prune below — e.g. by binding it to + // a `let` that outlives this line — closes the cycle and hangs + // wallet removal against a concurrent provisioning. self.inner.wallets.write()?.remove(&wallet_id); self.inner.snapshots.forget_wallet(seed_hash, &wallet_id); + // A same-seed re-import computes the same `WalletId`, so a pending + // registration left behind here would be inherited by the fresh + // wallet and rewritten against accounts it never provisioned. + // + // Order matters: `wallets` is cleared above BEFORE this prune, and + // `mark_account_registrations_staged` checks `wallets` while + // holding this same lock. That pairing is what stops a + // provisioning call still inside its retry window from recording a + // marker that outlives the wallet — this path is synchronous and + // cannot take the wallet's async provisioning lock. + self.inner + .buffered_account_registrations + .lock()? + .remove(&wallet_id); } match first_error { diff --git a/src/wallet_backend/persist_fault_test_support.rs b/src/wallet_backend/persist_fault_test_support.rs new file mode 100644 index 000000000..128997d59 --- /dev/null +++ b/src/wallet_backend/persist_fault_test_support.rs @@ -0,0 +1,242 @@ +//! Fault injection for the account-registration persist path, faithful to the +//! [`PlatformWalletPersistence`] retry contract. +//! +//! Decorates the real [`SqlitePersister`] so a test can make individual +//! `store` calls fail with a chosen [`PersistenceErrorKind`] while every other +//! call reaches the real database. The injected failures mirror what the real +//! persister does with the buffer: a `Transient` failure parks the changeset +//! so a later write still commits it, and a terminal failure drops the whole +//! per-wallet buffer — including anything a previous transient failure had +//! staged, because `flush_inner` takes the merged buffer and +//! `handle_flush_error` restores it only for transient errors. Assertions can +//! therefore be made against real persisted rows rather than a fake's +//! bookkeeping. +//! +//! The staged buffer is keyed by [`WalletId`], matching the real `Buffer`'s +//! `HashMap`: an unkeyed buffer would let a +//! two-wallet test commit one wallet's changeset under another wallet's id and +//! still pass. +//! +//! Only `store` faults, and only `store_commits_inline`, `store`, `flush` and +//! `load` are routed at all — the trait's defaults cover the rest. The +//! decorator has a single call site, the identity-funding account registration +//! write, and that path issues one `store` per attempt and never flushes: the +//! decorated persister answers `store_commits_inline`, so a successful `store` +//! is already the commit. A +//! flush-time fault would therefore be unreachable rather than merely unused, +//! so `flush` is a plain passthrough — it still completes a changeset an +//! injected transient failure parked, per the trait's contract — and +//! [`store_calls`](PersistFaults::store_calls) / +//! [`flush_calls`](PersistFaults::flush_calls) are what a test pins that shape +//! with. + +use std::collections::{HashMap, VecDeque}; +use std::sync::Mutex; + +use platform_wallet::changeset::{ + ClientStartState, Merge, PersistenceCapabilities, PersistenceError, PersistenceErrorKind, + PlatformWalletChangeSet, PlatformWalletPersistence, +}; +use platform_wallet::wallet::platform_wallet::WalletId; +use platform_wallet_storage::SqlitePersister; + +/// Queued faults plus the per-wallet changesets a transient fault left staged. +/// +/// Inert until [`arm`](Self::arm) queues something, so production-shaped tests +/// pay nothing for its presence. +#[derive(Default)] +pub(crate) struct PersistFaults { + queue: Mutex>, + staged: Mutex>, + /// One-based index of the `store` call before which the staged buffer is + /// dropped; `0` disables the probe. + discard_before_write: std::sync::atomic::AtomicUsize, + commit_before_write: std::sync::atomic::AtomicUsize, + store_calls: std::sync::atomic::AtomicUsize, + flush_calls: std::sync::atomic::AtomicUsize, +} + +impl PersistFaults { + /// Queue `kinds`, served one per `store` call in order. + pub(crate) fn arm(&self, kinds: impl IntoIterator) { + self.queue.lock().expect("fault queue").extend(kinds); + } + + /// `true` while an injected transient failure still holds an uncommitted + /// changeset for any wallet — the persister-buffer state a caller must not + /// assume it owns. + pub(crate) fn has_staged_changeset(&self) -> bool { + self.staged + .lock() + .expect("staged changeset") + .values() + .any(|changeset| !changeset.is_empty()) + } + + /// Drop every staged changeset the way an unrelated writer's terminal + /// flush does: `flush_inner` takes the whole per-wallet buffer, and a + /// terminal failure discards it without the account-registration caller + /// ever seeing the error. + pub(crate) fn discard_staged(&self) { + self.staged.lock().expect("staged changeset").clear(); + } + + /// Arm a one-shot foreign drain immediately before the `write`-th `store` + /// call, as if an unrelated writer had taken the shared buffer and lost it + /// terminally in that window. Counting calls (rather than firing on the + /// next one) is what places the drain *between* two attempts of one retry + /// loop. + pub(crate) fn discard_staged_before_write(&self, write: usize) { + self.discard_before_write + .store(write, std::sync::atomic::Ordering::Relaxed); + } + + /// Commit the staged buffer before the selected retry, as a foreign writer would. + pub(crate) fn commit_staged_before_write(&self, write: usize) { + self.commit_before_write + .store(write, std::sync::atomic::Ordering::Relaxed); + } + + /// Count this write and, if it is the armed one, drop the staged buffer. + fn apply_pending_discard(&self) { + let remaining = self + .discard_before_write + .load(std::sync::atomic::Ordering::Relaxed); + if remaining == 0 { + return; + } + self.discard_before_write + .store(remaining - 1, std::sync::atomic::Ordering::Relaxed); + if remaining == 1 { + self.discard_staged(); + } + } + + fn next(&self) -> Option { + self.queue.lock().expect("fault queue").pop_front() + } + + /// `store` calls the registration path has made since this backend was + /// built. One per persist attempt. + pub(crate) fn store_calls(&self) -> usize { + self.store_calls.load(std::sync::atomic::Ordering::Relaxed) + } + + /// `flush` calls the registration path has made since this backend was + /// built. Zero unless a follow-up write has crept back in: a committed + /// `store` needs none, and one would expose the registration to an + /// unrelated writer's failure. + pub(crate) fn flush_calls(&self) -> usize { + self.flush_calls.load(std::sync::atomic::Ordering::Relaxed) + } + + fn record_store(&self) { + self.store_calls + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + fn record_flush(&self) { + self.flush_calls + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + fn stage(&self, wallet_id: WalletId, changeset: PlatformWalletChangeSet) { + self.staged + .lock() + .expect("staged changeset") + .entry(wallet_id) + .or_default() + .merge(changeset); + } + + fn take_staged(&self, wallet_id: &WalletId) -> Option { + self.staged + .lock() + .expect("staged changeset") + .remove(wallet_id) + .filter(|changeset| !changeset.is_empty()) + } +} + +/// Borrowed decorator over the real persister; cheap to build per call. +pub(crate) struct PersistFaultInjector<'a> { + inner: &'a SqlitePersister, + faults: &'a PersistFaults, +} + +impl<'a> PersistFaultInjector<'a> { + pub(crate) fn new(inner: &'a SqlitePersister, faults: &'a PersistFaults) -> Self { + Self { inner, faults } + } + + fn injected(kind: PersistenceErrorKind) -> PersistenceError { + PersistenceError::backend_with_kind(kind, "injected persistence fault") + } +} + +impl PlatformWalletPersistence for PersistFaultInjector<'_> { + /// Forwarded, not defaulted: the trait's conservative `false` would put + /// the registration path on its buffered-backend branch and test a write + /// shape DET never runs. + fn store_commits_inline(&self) -> bool { + self.inner.store_commits_inline() + } + + fn persistence_capabilities(&self) -> PersistenceCapabilities { + self.inner.persistence_capabilities() + } + + fn store( + &self, + wallet_id: WalletId, + changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + self.faults.record_store(); + self.faults.apply_pending_discard(); + if self.faults.commit_before_write.fetch_update( + std::sync::atomic::Ordering::Relaxed, + std::sync::atomic::Ordering::Relaxed, + |remaining| remaining.checked_sub(1), + ) == Ok(1) + && let Some(staged) = self.faults.take_staged(&wallet_id) + { + self.inner.store(wallet_id, staged)?; + } + match self.faults.next() { + // Contract: a transient failure preserves the changeset, so a + // later write still commits it. + Some(PersistenceErrorKind::Transient) => { + self.faults.stage(wallet_id, changeset); + Err(Self::injected(PersistenceErrorKind::Transient)) + } + // A terminal failure takes the whole merged buffer with it, not + // just the incoming delta. + Some(kind) => { + self.faults.take_staged(&wallet_id); + Err(Self::injected(kind)) + } + None => { + self.faults.stage(wallet_id, changeset); + match self.faults.take_staged(&wallet_id) { + Some(staged) => self.inner.store(wallet_id, staged), + None => Ok(()), + } + } + } + } + + /// Passthrough — the queue serves `store` only. The registration path + /// makes no `flush` call, so faulting one would test nothing; the counter + /// is what a test asserts on instead. + fn flush(&self, wallet_id: WalletId) -> Result<(), PersistenceError> { + self.faults.record_flush(); + if let Some(staged) = self.faults.take_staged(&wallet_id) { + self.inner.store(wallet_id, staged)?; + } + self.inner.flush(wallet_id) + } + + fn load(&self) -> Result { + self.inner.load() + } +}