From feb25be900f8acf78757fe3cbd444b2f05fdd467 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 21 Aug 2026 08:53:54 +0000 Subject: [PATCH 01/11] fix(wallet): persist the identity funding account DET provisions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A restart between an asset-lock broadcast and its consumption could strand the lock and the funds in it: `resume_asset_lock` fails to re-derive the credit output with "Funding account IdentityTopUp not found for re-derivation". `load()` rebuilds `Wallet.accounts` from `account_registrations` alone. `provision_identity_funding_account` created the account in both upstream in-memory collections and persisted nothing, so upstream's own creator — the only writer of that row — then hit its `contains_*` guards, concluded both sides already existed, and took its early return, skipping the registration and address-pool store its docs call load-bearing for crash recovery. The account was live in memory and absent from disk on every launch. DET's provisioning now writes the `AccountRegistrationEntry` through the persister and flushes it, and rolls all three in-memory funding slots back on a store failure so a retry re-creates and re-persists rather than short-circuiting on the presence guards. Failures surface as the dedicated `TaskError::IdentityFundingAccountPersistFailed`. Residual: the paired address-pool snapshot upstream writes alongside the registration is not reachable from here (`account_address_pool_entries` is crate-private), so pool depth for that account re-warms on the next sync instead of restoring. The account itself — what re-derivation needs — is restored. Test: `a_provisioned_identity_topup_account_survives_a_restart`, confirmed RED before the fix (0 persisted rows, expected 1). Co-Authored-By: Claude Opus 5 --- src/backend_task/error.rs | 12 +++++ src/context/wallet_lifecycle/tests.rs | 76 +++++++++++++++++++++++++++ src/wallet_backend/identity_ops.rs | 63 +++++++++++++++++++++- 3 files changed, 150 insertions(+), 1 deletion(-) diff --git a/src/backend_task/error.rs b/src/backend_task/error.rs index 48aa18d90..e28eacda9 100644 --- a/src/backend_task/error.rs +++ b/src/backend_task/error.rs @@ -185,6 +185,18 @@ pub enum TaskError { source: dash_sdk::dpp::key_wallet::Error, }, + /// An identity-funding account was derived but could not be saved. Saving + /// it is what lets a restart find the account again, so the operation is + /// stopped rather than left able to strand a funding lock the app could no + /// longer spend. The technical cause lives in `Debug` and the logs. + #[error( + "Your wallet could not save the account this payment needs. Check that your disk is not full, then try again." + )] + IdentityFundingAccountPersistFailed { + #[source] + source: Box, + }, + /// Single-key wallets are not supported in this version. Their data is /// preserved; HD (recovery-phrase) wallets remain fully functional. #[error( diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 4871d7ed3..ad78d3835 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -3786,6 +3786,82 @@ async fn ensure_identity_funding_accounts_succeeds_on_cold_booted_watch_only_wal backend2.shutdown().await; } +/// A provisioned identity top-up account must survive a restart. +/// +/// `load()` rebuilds `Wallet.accounts` from `account_registrations` alone, and +/// the upstream creator that would otherwise write that row skips it once both +/// in-memory collections already hold the account — which DET's own +/// provisioning puts there first. A memory-only account leaves a restart +/// between an asset-lock broadcast and its consumption unable to re-derive the +/// credit-output path, stranding the lock and the funds in it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_provisioned_identity_topup_account_survives_a_restart() { + let _guard = backend_reopen_lock().await; + let source_dir = tempfile::tempdir().expect("source tempdir"); + let seed = [0xF3u8; 64]; + let registration_index = 7u32; + + let seed_hash = { + let wallet = + crate::model::wallet::Wallet::new_from_seed(seed, Network::Testnet, None, None) + .expect("build wallet"); + let seed_hash = wallet.seed_hash(); + + let (ctx, sender) = offline_testnet_context_at(source_dir.path()); + ctx.register_wallet(wallet, &seed, WalletOrigin::Fresh) + .expect("register wallet"); + ctx.ensure_wallet_backend(sender) + .await + .expect("wire backend offline"); + let backend = ctx.wallet_backend().expect("backend"); + backend + .register_wallet_from_seed(&seed_hash, &seed, Some(0)) + .await + .expect("upstream register"); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, registration_index) + .await + .expect("provision identity funding accounts"); + backend.shutdown().await; + seed_hash + }; + + let cold_dir = tempfile::tempdir().expect("cold tempdir"); + copy_dir_recursive(source_dir.path(), cold_dir.path()); + + // The manifest is the only thing `load()` rebuilds `Wallet.accounts` from, + // so assert the row itself rather than a downstream in-memory effect. + let persisted_topup_rows: i64 = rusqlite::Connection::open_with_flags( + wallet_database_path(cold_dir.path(), Network::Testnet), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .expect("open persisted store") + .query_row( + "SELECT COUNT(*) FROM account_registrations \ + WHERE account_type = 'identity_topup' AND account_index = ?1", + [registration_index], + |row| row.get(0), + ) + .expect("count persisted top-up registrations"); + assert_eq!( + persisted_topup_rows, 1, + "the provisioned top-up account must be in the persisted manifest, or a \ + broadcast asset lock cannot be resumed after a restart", + ); + + let (ctx2, sender2) = offline_testnet_context_at(cold_dir.path()); + ctx2.ensure_wallet_backend(sender2) + .await + .expect("cold boot must load the persisted wallet"); + let backend2 = ctx2.wallet_backend().expect("backend"); + assert!( + backend2.is_wallet_registered(&seed_hash), + "the wallet must still come back registered with the extra account row", + ); + + backend2.shutdown().await; +} + /// A malformed Orchard viewing key must be isolated to its own wallet during /// the real seedless cold-boot load. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index da3f2639e..7115c4f04 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -431,12 +431,73 @@ impl WalletBackend { } .ok_or(TaskError::WalletStateInconsistent)?; + let account_type = derived.account_type; + let account_xpub = derived.account_xpub; + let managed = ManagedCoreKeysAccount::from_account(derived); info.core_wallet .accounts .insert_keys_bearing_account(managed) .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; - Ok(()) + + // Persist the registration. `load()` rebuilds `Wallet.accounts` from + // `account_registrations` alone, and the upstream creator that would + // normally write this row skips it once both in-memory collections hold + // the account — which they do by the time it runs, because of the + // inserts above. Without this the account is memory-only: a restart + // between an asset-lock broadcast and its consumption cannot re-derive + // the credit-output path, stranding the lock and its funds. + self.persist_account_registration(&wallet_id, account_type, account_xpub) + .inspect_err(|_| { + // Roll back both sides so a retry re-creates and re-persists, + // rather than the `in_wallet && in_managed` guard above short- + // circuiting a persist that never happened. + match funding { + Funding::Registration => { + kw.accounts.identity_registration = None; + info.core_wallet.accounts.identity_registration = None; + } + Funding::TopUp(registration_index) => { + kw.accounts.identity_topup.remove(®istration_index); + info.core_wallet + .accounts + .identity_topup + .remove(®istration_index); + } + Funding::TopUpNotBound => { + kw.accounts.identity_topup_not_bound = None; + info.core_wallet.accounts.identity_topup_not_bound = None; + } + } + }) + } + + /// Write one account registration through the upstream persister and flush + /// it, so a cold boot rebuilds the account from the manifest. + fn persist_account_registration( + &self, + wallet_id: &platform_wallet::wallet::platform_wallet::WalletId, + account_type: dash_sdk::dpp::key_wallet::AccountType, + account_xpub: dash_sdk::dpp::key_wallet::bip32::ExtendedPubKey, + ) -> Result<(), TaskError> { + use platform_wallet::changeset::{ + AccountRegistrationEntry, PlatformWalletChangeSet, PlatformWalletPersistence, + }; + + let changeset = PlatformWalletChangeSet { + account_registrations: vec![AccountRegistrationEntry { + account_type, + account_xpub, + }], + ..Default::default() + }; + self.inner + .wallet_persister + .store(*wallet_id, changeset) + .and_then(|()| self.inner.wallet_persister.flush(*wallet_id)) + .map_err(|source| TaskError::IdentityFundingAccountPersistFailed { + source: Box::new(source), + }) } /// Provision the identity-registration funding account and the per- From 1330c1be04e232f8febe6098d977f3ef55ca1bf7 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:52:17 +0000 Subject: [PATCH 02/11] fix(wallet): retry a transient funding-account persist before dropping memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SqlitePersister` classifies a failed write `Transient`, `Fatal` or `Constraint`, and the `PlatformWalletPersistence` contract is explicit about what each means: a `Transient` `store` or `flush` MUST have preserved the staged changeset, and the caller completes the write with a **bare** `flush` — re-`store`ing the same delta would double-merge it. Only a terminal failure discards the buffer. The previous `store(..).and_then(flush)` did the rollback half of that contract without the retry half. A transient blip therefore removed the account from both in-memory collections while its registration stayed buffered, so a later unrelated flush could commit the row under a live wallet that no longer had the account — a manifest and a wallet disagreeing about which accounts exist. The write now retries the bare `flush` with bounded exponential backoff, mirroring upstream's own `retry_transient` on the wallet-registration path (4 attempts, 20 ms doubling to 200 ms; that helper is `pub(super)` and cannot be reused). The surviving error is classified: terminal failures roll memory back as before, while a still-transient failure keeps the account and records the staged registration. The idempotent early return then finishes that write instead of reporting success on a row that never landed. The staged marker is keyed by account, not just by wallet. `flush` is wallet-scoped, so the drain usually runs while provisioning some *other* funding account; evicting the account being provisioned would strip a durable account from the live wallet and strand the staged one with nothing left to retry it. Persist-failure tests drive a fault-injecting decorator over the real `SqlitePersister` that honours the trait contract — a transient fault parks the changeset so a later bare `flush` still commits it — so their assertions land on real persisted rows. They target the per-index top-up account, the only identity-funding account DET actually provisions: every other variant is created and persisted when the wallet is registered, so provisioning it takes the idempotent early return and never reaches the persist path. Restart coverage is widened alongside. The top-up restart test asserted the manifest row and that the wallet came back registered, which a loader silently dropping the account would still pass; it now asserts the reloaded account's type, index and xpub in both collections, via a test-only accessor. The index-less top-up variant, which had no cold-boot coverage at all, gets an equivalent test. Confirmed RED against the previous behaviour: `a_transient_registration_persist_failure_retries_the_bare_flush_and_keeps_the_account` and `an_exhausted_transient_persist_budget_keeps_memory_and_the_next_attempt_flushes_the_buffer`; `a_provisioned_identity_topup_account_survives_a_restart` still goes RED with the persist removed. Co-Authored-By: Claude Opus 5 --- src/context/wallet_lifecycle/tests.rs | 373 +++++++++++++++++- src/wallet_backend/identity_ops.rs | 340 +++++++++++++--- src/wallet_backend/mod.rs | 42 ++ .../persist_fault_test_support.rs | 129 ++++++ 4 files changed, 819 insertions(+), 65 deletions(-) create mode 100644 src/wallet_backend/persist_fault_test_support.rs diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index ad78d3835..115540dfd 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -9,6 +9,7 @@ use crate::model::secret::Secret; use crate::utils::egui_mpsc::SenderAsync; use crate::utils::tasks::TaskManager; use crate::wallet_backend::wallet_database_path; +use platform_wallet::changeset::PersistenceErrorKind; /// Build an offline `AppContext` for testnet in an isolated temp dir. No /// network I/O happens at construction: the SDK and Core client are built @@ -3800,8 +3801,9 @@ async fn a_provisioned_identity_topup_account_survives_a_restart() { let source_dir = tempfile::tempdir().expect("source tempdir"); let seed = [0xF3u8; 64]; let registration_index = 7u32; + let account_type = dash_sdk::dpp::key_wallet::AccountType::IdentityTopUp { registration_index }; - let seed_hash = { + let (seed_hash, provisioned) = { let wallet = crate::model::wallet::Wallet::new_from_seed(seed, Network::Testnet, None, None) .expect("build wallet"); @@ -3822,8 +3824,13 @@ async fn a_provisioned_identity_topup_account_survives_a_restart() { .ensure_identity_funding_accounts(&seed_hash, &seed, registration_index) .await .expect("provision identity funding accounts"); + let provisioned = backend + .identity_funding_account(&seed_hash, account_type) + .await + .expect("probe the provisioned funding account") + .expect("the top-up account must be live in memory after provisioning"); backend.shutdown().await; - seed_hash + (seed_hash, provisioned) }; let cold_dir = tempfile::tempdir().expect("cold tempdir"); @@ -3831,20 +3838,9 @@ async fn a_provisioned_identity_topup_account_survives_a_restart() { // The manifest is the only thing `load()` rebuilds `Wallet.accounts` from, // so assert the row itself rather than a downstream in-memory effect. - let persisted_topup_rows: i64 = rusqlite::Connection::open_with_flags( - wallet_database_path(cold_dir.path(), Network::Testnet), - rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, - ) - .expect("open persisted store") - .query_row( - "SELECT COUNT(*) FROM account_registrations \ - WHERE account_type = 'identity_topup' AND account_index = ?1", - [registration_index], - |row| row.get(0), - ) - .expect("count persisted top-up registrations"); assert_eq!( - persisted_topup_rows, 1, + persisted_account_registration_rows(cold_dir.path(), "identity_topup", registration_index), + 1, "the provisioned top-up account must be in the persisted manifest, or a \ broadcast asset lock cannot be resumed after a restart", ); @@ -3859,9 +3855,117 @@ async fn a_provisioned_identity_topup_account_survives_a_restart() { "the wallet must still come back registered with the extra account row", ); + // A row on disk is only half the contract: the loader has to turn it back + // into the indexed account both funding-address collections read. A loader + // that dropped it — or rebuilt it at the wrong index, or from a different + // xpub — would still leave the asset lock unspendable. + let restored = backend2 + .identity_funding_account(&seed_hash, account_type) + .await + .expect("probe the reloaded funding account") + .expect("the reloaded wallet must hold the provisioned top-up account"); + assert_eq!( + restored, provisioned, + "the reloaded top-up account must match the one provisioned before the restart", + ); + + backend2.shutdown().await; +} + +/// The index-less top-up funding account must survive a restart too. +/// +/// It reaches the manifest as `identity_topup_unbound` and is the only +/// credit-output source for topping up an identity this wallet does not own. +/// In-process idempotency proves nothing about a cold boot: if the row is +/// missing, upstream's presence guards make the next launch skip the write +/// again and the account stays memory-only forever. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unbound_topup_funding_account_survives_a_restart() { + let _guard = backend_reopen_lock().await; + let source_dir = tempfile::tempdir().expect("source tempdir"); + let seed = [0xB4u8; 64]; + let account_type = dash_sdk::dpp::key_wallet::AccountType::IdentityTopUpNotBoundToIdentity; + + let (seed_hash, provisioned) = { + let wallet = + crate::model::wallet::Wallet::new_from_seed(seed, Network::Testnet, None, None) + .expect("build wallet"); + let seed_hash = wallet.seed_hash(); + + let (ctx, sender) = offline_testnet_context_at(source_dir.path()); + ctx.register_wallet(wallet, &seed, WalletOrigin::Fresh) + .expect("register wallet"); + ctx.ensure_wallet_backend(sender) + .await + .expect("wire backend offline"); + let backend = ctx.wallet_backend().expect("backend"); + backend + .register_wallet_from_seed(&seed_hash, &seed, Some(0)) + .await + .expect("upstream register"); + backend + .ensure_unbound_topup_funding_account(&seed_hash, &seed) + .await + .expect("provision the index-less top-up account"); + let provisioned = backend + .identity_funding_account(&seed_hash, account_type) + .await + .expect("probe the provisioned funding account") + .expect("the index-less top-up account must be live in memory"); + backend.shutdown().await; + (seed_hash, provisioned) + }; + + let cold_dir = tempfile::tempdir().expect("cold tempdir"); + copy_dir_recursive(source_dir.path(), cold_dir.path()); + + assert_eq!( + persisted_account_registration_rows(cold_dir.path(), "identity_topup_unbound", 0), + 1, + "the index-less top-up account must be in the persisted manifest, or a \ + top-up of an identity this wallet does not own cannot be resumed", + ); + + let (ctx2, sender2) = offline_testnet_context_at(cold_dir.path()); + ctx2.ensure_wallet_backend(sender2) + .await + .expect("cold boot must load the persisted wallet"); + let backend2 = ctx2.wallet_backend().expect("backend"); + let restored = backend2 + .identity_funding_account(&seed_hash, account_type) + .await + .expect("probe the reloaded funding account") + .expect("the reloaded wallet must hold the index-less top-up account"); + assert_eq!( + restored, provisioned, + "the reloaded index-less top-up account must match the one provisioned \ + before the restart", + ); + backend2.shutdown().await; } +/// Rows in the persisted `account_registrations` manifest for one account +/// type / index — what `load()` rebuilds `Wallet.accounts` from. +fn persisted_account_registration_rows( + data_dir: &std::path::Path, + account_type: &str, + account_index: u32, +) -> i64 { + rusqlite::Connection::open_with_flags( + wallet_database_path(data_dir, Network::Testnet), + rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .expect("open persisted store") + .query_row( + "SELECT COUNT(*) FROM account_registrations \ + WHERE account_type = ?1 AND account_index = ?2", + rusqlite::params![account_type, account_index], + |row| row.get(0), + ) + .expect("count persisted account registrations") +} + /// A malformed Orchard viewing key must be isolated to its own wallet during /// the real seedless cold-boot load. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -4433,6 +4537,245 @@ async fn unbound_topup_funding_account_provisions_on_the_watch_only_wallet() { backend.shutdown().await; } +/// A transient persister failure must be ridden out, not surfaced. +/// +/// The persistence contract says a transient `store` keeps the changeset +/// buffered and a bare `flush` completes it. Rolling the account out of memory +/// instead leaves the staged row able to land later under a live wallet that no +/// longer has the account — a manifest and a wallet that disagree about which +/// accounts exist. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_transient_registration_persist_failure_retries_the_bare_flush_and_keeps_the_account() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC1u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "transient").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("a transient persist failure must be retried, not reported to the user"); + + let view = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("the account must survive a retried transient failure"); + assert!( + view.managed, + "the managed collection must keep the account too — funding-address \ + derivation reads both sides", + ); + assert!( + !backend.registration_persist_buffer_is_staged(), + "the bare-flush retry must have drained the persister buffer", + ); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the retried write must have made the registration durable", + ); +} + +/// When the retry budget runs out while the failure is still transient, the +/// account stays in memory and the next provisioning attempt finishes the +/// write instead of taking the idempotent early return. +/// +/// Reporting success on the second call would leave the row staged forever: +/// the account is live in memory, absent from the manifest, and every launch +/// after this one re-derives nothing. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_exhausted_transient_persist_budget_keeps_memory_and_the_next_attempt_flushes_the_buffer() + { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC2u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "exhausted").await; + + // One `store` plus every bare-flush retry the budget allows. Sizing the + // queue to the budget exactly leaves nothing armed for the follow-up call. + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); + let error = backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("an exhausted retry budget must surface as a typed error"); + assert!( + matches!(error, TaskError::IdentityFundingAccountPersistFailed { .. }), + "the persist failure must keep its dedicated variant, got {error:?}", + ); + + let view = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("a still-staged registration must not have its account removed"); + assert!(view.managed, "both collections must keep the account"); + assert!( + backend.registration_persist_buffer_is_staged(), + "the changeset must still be staged for a later flush", + ); + + // The next attempt sees both collections populated. It must complete the + // staged write rather than short-circuit on the presence guards. + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the follow-up attempt must flush the staged registration"); + assert!( + !backend.registration_persist_buffer_is_staged(), + "the follow-up attempt must have drained the persister buffer", + ); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the staged registration must reach the manifest once the blip clears", + ); +} + +/// A terminal persister failure discards the staged changeset, so the account +/// must leave memory — otherwise the presence guards make every later attempt +/// report success on a row that was never written. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terminal_registration_persist_failure_rolls_back_the_in_memory_account() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC3u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "terminal").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Fatal]); + let error = backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("a terminal persist failure must surface as a typed error"); + assert!( + matches!(error, TaskError::IdentityFundingAccountPersistFailed { .. }), + "the persist failure must keep its dedicated variant, got {error:?}", + ); + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .is_none(), + "a discarded changeset must take the in-memory account with it", + ); + + // With the fault queue empty, a retry re-creates and re-persists. + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the retry must re-create and re-persist the account"); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the retry must have made the registration durable", + ); +} + +/// A drain that fails terminally must evict the accounts whose registrations +/// it lost — and only those. +/// +/// `flush` is wallet-scoped, so the drain runs on whichever funding account is +/// provisioned next, which is usually not the one that was staged. Evicting +/// the account being provisioned instead of the staged one would strip a +/// perfectly durable account from the live wallet and leave the staged one +/// memory-only with nothing left to retry it. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terminal_drain_evicts_only_the_accounts_whose_registrations_it_lost() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC4u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "drain").await; + + // Strand the top-up registration in the buffer. + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("the exhausted budget must surface"); + + // The next call provisions the registration account first, so its drain of + // the staged top-up write is what fails terminally here. + backend.arm_registration_persist_faults([PersistenceErrorKind::Fatal]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("a terminal drain must surface"); + + assert!( + backend + .identity_funding_account( + &seed_hash, + dash_sdk::dpp::key_wallet::AccountType::IdentityRegistration, + ) + .await + .expect("probe the registration account") + .is_some(), + "the registration account is durable and must survive another account's failed drain", + ); + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the top-up account") + .is_none(), + "the account whose registration the drain discarded must leave memory", + ); + + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("a clean retry must re-create and re-persist the top-up account"); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the retry must have made the top-up registration durable", + ); +} + +/// Identity index the persist-failure tests provision a top-up account for. +/// +/// Per-index top-up accounts are the funding accounts DET actually creates: +/// every other identity-funding account type is pre-created and persisted when +/// the wallet is registered, so provisioning them takes the idempotent early +/// return and never reaches the persist path under test. +const FAULT_TOPUP_INDEX: u32 = 11; + +fn fault_topup_account_type() -> dash_sdk::dpp::key_wallet::AccountType { + dash_sdk::dpp::key_wallet::AccountType::IdentityTopUp { + registration_index: FAULT_TOPUP_INDEX, + } +} + /// An identity publishing real public keys. `Identity::create_basic_identity` /// publishes none, and a key-less identity persists no `identity_keys` rows — /// the very rows whose mis-filing bricks a wallet at load time. diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index 7115c4f04..ba7d765a1 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -13,17 +13,34 @@ use crate::model::wallet::WalletSeedHash; use std::sync::Arc; use super::{ - DetPlatformSigner, DetSigner, PlatformPathIndex, WalletBackend, map_identity_register_error, - map_identity_top_up_error, map_platform_address_fund_error, + DetPlatformSigner, DetSigner, PlatformPathIndex, WalletBackend, WalletId, + map_identity_register_error, map_identity_top_up_error, map_platform_address_fund_error, }; +use platform_wallet::changeset::{ + AccountRegistrationEntry, PersistenceError, PlatformWalletChangeSet, PlatformWalletPersistence, +}; + +/// Total attempts (initial write plus bare-`flush` retries) for a transient +/// persister failure on the funding-account path. Mirrors upstream's +/// `PERSIST_RETRY_MAX_ATTEMPTS` on the wallet-registration path, which is +/// `pub(super)` there and so unreachable from DET. +const PERSIST_RETRY_MAX_ATTEMPTS: u32 = 4; + +/// Backoff before the first retry; doubles up to [`PERSIST_RETRY_MAX_BACKOFF`]. +const PERSIST_RETRY_INITIAL_BACKOFF: std::time::Duration = std::time::Duration::from_millis(20); + +/// Ceiling for the doubling backoff, so a busy database still surfaces +/// promptly (worst case with the constants above: 20 + 40 + 80 ≈ 140 ms). +const PERSIST_RETRY_MAX_BACKOFF: std::time::Duration = std::time::Duration::from_millis(200); + /// The two identity-funding account flavours DET provisions. Parsing the /// caller's intent into this once (at [`WalletBackend::ensure_identity_funding_accounts`]) /// removes the repeated `AccountType` matches — and their `unreachable!` arms — /// inside [`WalletBackend::provision_identity_funding_account`], and makes the /// unsupported-account-type case unrepresentable. -#[derive(Clone, Copy)] -enum Funding { +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub(super) enum Funding { /// The identity-registration funding account. Registration, /// The per-identity top-up funding account at the given registration index. @@ -34,6 +51,135 @@ enum Funding { TopUpNotBound, } +impl Funding { + /// Drop this funding account from both in-memory collections, so the next + /// provisioning attempt re-creates and re-persists it instead of + /// short-circuiting on the presence guards. + fn remove( + self, + kw: &mut dash_sdk::dpp::key_wallet::wallet::Wallet, + info: &mut platform_wallet::wallet::platform_wallet::PlatformWalletInfo, + ) { + match self { + Self::Registration => { + kw.accounts.identity_registration = None; + info.core_wallet.accounts.identity_registration = None; + } + Self::TopUp(registration_index) => { + kw.accounts.identity_topup.remove(®istration_index); + info.core_wallet + .accounts + .identity_topup + .remove(®istration_index); + } + Self::TopUpNotBound => { + kw.accounts.identity_topup_not_bound = None; + info.core_wallet.accounts.identity_topup_not_bound = None; + } + } + } +} + +/// One provisioned identity-funding account as the wallet holds it, for +/// asserting that a restart restored the same account rather than a +/// same-shaped placeholder. +#[cfg(test)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct FundingAccountView { + /// Account type on the key-wallet account — carries the registration index + /// for [`AccountType::IdentityTopUp`](dash_sdk::dpp::key_wallet::AccountType::IdentityTopUp). + pub account_type: dash_sdk::dpp::key_wallet::AccountType, + /// The account xpub funding-address derivation reads. + pub account_xpub: dash_sdk::dpp::key_wallet::bip32::ExtendedPubKey, + /// Whether the managed (info-side) collection also holds the account. + pub managed: bool, +} + +/// Why an account registration did not reach disk — and therefore what the +/// caller must do with the account it just put in memory. +enum RegistrationPersistFailure { + /// The retry budget ran out while the failure was still transient. The + /// persister's contract keeps the changeset buffered, so the row can still + /// land: the account MUST stay in memory, or a later unrelated flush would + /// commit a registration the live wallet no longer has. A bare `flush` + /// completes the write. + Staged(PersistenceError), + /// A terminal failure discarded the staged changeset. The row will never + /// land, so the account must leave memory. + Discarded(PersistenceError), +} + +impl RegistrationPersistFailure { + fn into_task_error(self) -> TaskError { + let (Self::Staged(source) | Self::Discarded(source)) = self; + TaskError::IdentityFundingAccountPersistFailed { + source: Box::new(source), + } + } +} + +/// Retry `op` while it fails transiently, with bounded exponential backoff, +/// and classify whatever error survives. +/// +/// Mirrors upstream's `retry_transient` on the wallet-registration path. The +/// sleep is async so it yields the Tokio worker rather than spinning. +async fn retry_transient(mut op: F) -> Result<(), RegistrationPersistFailure> +where + F: FnMut() -> Result<(), PersistenceError>, +{ + let mut backoff = PERSIST_RETRY_INITIAL_BACKOFF; + let mut attempt: u32 = 1; + loop { + match op() { + Ok(()) => return Ok(()), + Err(e) if !e.is_transient() => return Err(RegistrationPersistFailure::Discarded(e)), + Err(e) if attempt >= PERSIST_RETRY_MAX_ATTEMPTS => { + return Err(RegistrationPersistFailure::Staged(e)); + } + Err(e) => { + tracing::debug!( + attempt, + max_attempts = PERSIST_RETRY_MAX_ATTEMPTS, + backoff_ms = backoff.as_millis() as u64, + error = %e, + "identity-funding account registration hit a transient persister failure; backing off before retrying the flush" + ); + tokio::time::sleep(backoff).await; + backoff = backoff.saturating_mul(2).min(PERSIST_RETRY_MAX_BACKOFF); + attempt += 1; + } + } + } +} + +/// Write one account registration through `persister` and flush it, so a cold +/// boot rebuilds the account from the manifest. +/// +/// The first attempt is `store` then `flush`; every retry is a **bare** +/// `flush`, because the persister's contract preserves the changeset across a +/// transient failure and re-`store`ing the same delta would double-merge it. +async fn persist_account_registration( + persister: &dyn PlatformWalletPersistence, + wallet_id: WalletId, + account_type: dash_sdk::dpp::key_wallet::AccountType, + account_xpub: dash_sdk::dpp::key_wallet::bip32::ExtendedPubKey, +) -> Result<(), RegistrationPersistFailure> { + let mut changeset = Some(PlatformWalletChangeSet { + account_registrations: vec![AccountRegistrationEntry { + account_type, + account_xpub, + }], + ..Default::default() + }); + retry_transient(|| match changeset.take() { + Some(changeset) => persister + .store(wallet_id, changeset) + .and_then(|()| persister.flush(wallet_id)), + None => persister.flush(wallet_id), + }) + .await +} + impl WalletBackend { /// Register a new identity on Platform funded by an asset lock built and /// tracked-to-finality by the upstream `AssetLockManager`. Returns the @@ -354,7 +500,9 @@ impl WalletBackend { // `IdentityTopUp{registration_index}` enters the manifest only after a // register/top-up, so a reloaded already-registered identity needs this // re-provision. Idempotent: probes both collections and no-ops if present - // (no error-string parsing — direct membership checks). + // (no error-string parsing — direct membership checks) — except when a + // previous transient persist failure left the registration staged, which + // this call finishes rather than reporting success on an absent row. // // `seed` must be the wallet's HD seed so the hardened account xpub can be // derived — the live wallet is watch-only and cannot derive hardened paths @@ -375,6 +523,19 @@ impl WalletBackend { .get_wallet_mut_and_info_mut(&wallet_id) .ok_or(TaskError::WalletStateInconsistent)?; + // In test builds the write goes through a fault-injecting decorator + // that is inert until a test arms it; production talks to the + // persister directly. + #[cfg(not(test))] + let persister: &dyn PlatformWalletPersistence = self.inner.wallet_persister.as_ref(); + #[cfg(test)] + let injector = super::persist_fault_test_support::PersistFaultInjector::new( + &self.inner.wallet_persister, + &self.inner.persist_faults, + ); + #[cfg(test)] + let persister: &dyn PlatformWalletPersistence = &injector; + let (in_wallet, in_managed) = match funding { Funding::Registration => ( kw.accounts.identity_registration.is_some(), @@ -393,7 +554,25 @@ impl WalletBackend { ), }; if in_wallet && in_managed { - return Ok(()); + // A previous attempt may have left registrations staged in the + // persister buffer after a transient failure. Returning success + // here would leave those accounts live in memory and absent from + // the manifest — precisely the state this path exists to prevent. + if self.staged_account_registrations(&wallet_id)?.is_empty() { + return Ok(()); + } + return match retry_transient(|| persister.flush(wallet_id)).await { + Ok(()) => self.clear_staged_account_registrations(&wallet_id), + Err(failure) => { + if matches!(failure, RegistrationPersistFailure::Discarded(_)) { + for staged in self.staged_account_registrations(&wallet_id)? { + staged.remove(kw, info); + } + self.clear_staged_account_registrations(&wallet_id)?; + } + Err(failure.into_task_error()) + } + }; } if !in_wallet { @@ -447,57 +626,70 @@ impl WalletBackend { // inserts above. Without this the account is memory-only: a restart // between an asset-lock broadcast and its consumption cannot re-derive // the credit-output path, stranding the lock and its funds. - self.persist_account_registration(&wallet_id, account_type, account_xpub) - .inspect_err(|_| { - // Roll back both sides so a retry re-creates and re-persists, - // rather than the `in_wallet && in_managed` guard above short- - // circuiting a persist that never happened. - match funding { - Funding::Registration => { - kw.accounts.identity_registration = None; - info.core_wallet.accounts.identity_registration = None; - } - Funding::TopUp(registration_index) => { - kw.accounts.identity_topup.remove(®istration_index); - info.core_wallet - .accounts - .identity_topup - .remove(®istration_index); - } - Funding::TopUpNotBound => { - kw.accounts.identity_topup_not_bound = None; - info.core_wallet.accounts.identity_topup_not_bound = None; - } + match persist_account_registration(persister, wallet_id, account_type, account_xpub).await { + // A successful flush is wallet-wide: everything staged landed too. + Ok(()) => self.clear_staged_account_registrations(&wallet_id), + Err(RegistrationPersistFailure::Staged(source)) => { + // The row can still land, so keeping the account is what makes + // disk and the live wallet agree; the marker makes the next + // attempt for this wallet finish the write. + self.mark_account_registration_staged(wallet_id, funding)?; + Err(RegistrationPersistFailure::Staged(source).into_task_error()) + } + Err(RegistrationPersistFailure::Discarded(source)) => { + // A terminal failure drops the whole wallet buffer, so this + // account and every previously staged one must leave memory — + // otherwise the presence guards short-circuit persists that + // never happened. + funding.remove(kw, info); + for staged in self.staged_account_registrations(&wallet_id)? { + staged.remove(kw, info); } - }) + self.clear_staged_account_registrations(&wallet_id)?; + Err(RegistrationPersistFailure::Discarded(source).into_task_error()) + } + } } - /// Write one account registration through the upstream persister and flush - /// it, so a cold boot rebuilds the account from the manifest. - fn persist_account_registration( + /// The funding accounts whose registration a transient failure left staged + /// in this wallet's persister buffer. + fn staged_account_registrations( &self, - wallet_id: &platform_wallet::wallet::platform_wallet::WalletId, - account_type: dash_sdk::dpp::key_wallet::AccountType, - account_xpub: dash_sdk::dpp::key_wallet::bip32::ExtendedPubKey, + wallet_id: &WalletId, + ) -> Result, TaskError> { + Ok(self + .inner + .buffered_account_registrations + .lock()? + .get(wallet_id) + .map(|staged| staged.iter().copied().collect()) + .unwrap_or_default()) + } + + /// Record a staged registration for the idempotent early return in + /// [`Self::provision_identity_funding_account`] to finish later. + fn mark_account_registration_staged( + &self, + wallet_id: WalletId, + funding: Funding, ) -> Result<(), TaskError> { - use platform_wallet::changeset::{ - AccountRegistrationEntry, PlatformWalletChangeSet, PlatformWalletPersistence, - }; + self.inner + .buffered_account_registrations + .lock()? + .entry(wallet_id) + .or_default() + .insert(funding); + Ok(()) + } - let changeset = PlatformWalletChangeSet { - account_registrations: vec![AccountRegistrationEntry { - account_type, - account_xpub, - }], - ..Default::default() - }; + /// Forget this wallet's staged registrations, once they are durable or + /// definitively lost. + fn clear_staged_account_registrations(&self, wallet_id: &WalletId) -> Result<(), TaskError> { self.inner - .wallet_persister - .store(*wallet_id, changeset) - .and_then(|()| self.inner.wallet_persister.flush(*wallet_id)) - .map_err(|source| TaskError::IdentityFundingAccountPersistFailed { - source: Box::new(source), - }) + .buffered_account_registrations + .lock()? + .remove(wallet_id); + Ok(()) } /// Provision the identity-registration funding account and the per- @@ -518,6 +710,54 @@ impl WalletBackend { .await } + /// The provisioned identity-funding account for `account_type`, as seen by + /// the two collections that must both hold it. + /// + /// `peek_next_funding_address` reads the account xpub from + /// `Wallet.accounts` and the address pool from the managed collection, so + /// a restore that repopulates only one side leaves funding broken. Returns + /// `None` when the key-wallet side has no such account. + #[cfg(test)] + pub(crate) async fn identity_funding_account( + &self, + seed_hash: &WalletSeedHash, + account_type: dash_sdk::dpp::key_wallet::AccountType, + ) -> Result, TaskError> { + use dash_sdk::dpp::key_wallet::AccountType; + + let wallet = self.resolve_wallet(seed_hash).await?; + let wallet_id = wallet.wallet_id(); + let wm = wallet.wallet_manager().read().await; + let (kw, info) = wm + .get_wallet_and_info(&wallet_id) + .ok_or(TaskError::WalletStateInconsistent)?; + + let (derived, managed) = match account_type { + AccountType::IdentityRegistration => ( + kw.accounts.identity_registration.as_ref(), + info.core_wallet.accounts.identity_registration.is_some(), + ), + AccountType::IdentityTopUp { registration_index } => ( + kw.accounts.identity_topup.get(®istration_index), + info.core_wallet + .accounts + .identity_topup + .contains_key(®istration_index), + ), + AccountType::IdentityTopUpNotBoundToIdentity => ( + kw.accounts.identity_topup_not_bound.as_ref(), + info.core_wallet.accounts.identity_topup_not_bound.is_some(), + ), + _ => return Err(TaskError::WalletStateInconsistent), + }; + + Ok(derived.map(|account| FundingAccountView { + account_type: account.account_type, + account_xpub: account.account_xpub, + managed, + })) + } + /// Provision the index-less top-up funding account — the credit-output /// source for topping up an identity this wallet does not own. Idempotent; /// `seed` must be held for the duration of the call. diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 3f4fab003..d0e83db36 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -47,6 +47,8 @@ pub(crate) mod kv_test_support; pub(crate) mod leak_test_support; mod loader; mod payments; +#[cfg(test)] +pub(crate) mod persist_fault_test_support; pub(crate) mod poison; pub mod secret_access; pub mod secret_prompt; @@ -90,6 +92,7 @@ pub use secret_prompt::{ pub use secret_seam::SecretSeam; use coordinator_gate::CoordinatorGate; +use identity_ops::Funding; pub use auth_pubkey_cache::AuthPubkeyCacheView; pub use avatar_cache::AvatarCacheView; @@ -344,6 +347,23 @@ struct Inner { clear_shielded_test_failure: std::sync::atomic::AtomicBool, #[cfg(test)] forget_wallet_local_state_test_failure: std::sync::atomic::AtomicBool, + /// Injected persister faults for the identity-funding account + /// registration write. Inert until a test arms it. + #[cfg(test)] + persist_faults: persist_fault_test_support::PersistFaults, + /// Per wallet, the identity-funding accounts whose registration is staged + /// in the persister buffer but not yet durable, because a transient write + /// failure exhausted its retry budget. Each account stays live in memory + /// (its row can still land), so the next provisioning attempt for that + /// wallet must complete the write with a bare `flush` instead of taking + /// the idempotent early return and reporting success on absent rows. + /// + /// Keyed by account and not just by wallet: `flush` is wallet-scoped, so + /// one failed drain discards every registration staged for that wallet and + /// all of them — not merely the account being provisioned right now — have + /// to leave memory. + buffered_account_registrations: + std::sync::Mutex>>, /// Per-wallet shared-result flights for upstream registration. Every caller /// that joins an active flight awaits the same success or typed error. registration_flights: @@ -554,6 +574,11 @@ impl WalletBackend { clear_shielded_test_failure: std::sync::atomic::AtomicBool::new(false), #[cfg(test)] forget_wallet_local_state_test_failure: std::sync::atomic::AtomicBool::new(false), + #[cfg(test)] + persist_faults: persist_fault_test_support::PersistFaults::default(), + buffered_account_registrations: std::sync::Mutex::new( + std::collections::BTreeMap::new(), + ), registration_flights: std::sync::Mutex::new(std::collections::BTreeMap::new()), dashpay_request_action_locks: dashpay::ContactRequestActionLocks::default(), wallets: std::sync::RwLock::new(std::collections::BTreeMap::new()), @@ -1082,6 +1107,23 @@ impl WalletBackend { .store(fail, std::sync::atomic::Ordering::Relaxed); } + /// Queue persister faults for the identity-funding account registration + /// write, served one per `store` / `flush` call in order. + #[cfg(test)] + pub(crate) fn arm_registration_persist_faults( + &self, + kinds: impl IntoIterator, + ) { + self.inner.persist_faults.arm(kinds); + } + + /// `true` while an injected transient failure still holds an uncommitted + /// account-registration changeset in the persister buffer. + #[cfg(test)] + pub(crate) fn registration_persist_buffer_is_staged(&self) -> bool { + self.inner.persist_faults.has_staged_changeset() + } + /// Resolve one just-registered upstream wallet into the DET-keyed maps via /// the account-xpub fund-routing gate, identical in spirit to the gate the /// seedless loader applies per wallet. diff --git a/src/wallet_backend/persist_fault_test_support.rs b/src/wallet_backend/persist_fault_test_support.rs new file mode 100644 index 000000000..12b006ca2 --- /dev/null +++ b/src/wallet_backend/persist_fault_test_support.rs @@ -0,0 +1,129 @@ +//! Fault injection for the account-registration persist path, faithful to the +//! [`PlatformWalletPersistence`] retry contract. +//! +//! Decorates the real [`SqlitePersister`] so a test can make individual +//! `store` / `flush` calls fail with a chosen [`PersistenceErrorKind`] while +//! every other call reaches the real database. The injected failures honour the +//! trait's own contract: a `Transient` failure parks the changeset so a later +//! bare `flush` still commits it, a terminal failure discards it. Assertions +//! can therefore be made against real persisted rows rather than a fake's +//! bookkeeping. +//! +//! Only `store`, `flush` and `load` are routed — the decorator is used at one +//! call site (the identity-funding account registration write) and the trait's +//! defaults cover the rest. + +use std::collections::VecDeque; +use std::sync::Mutex; + +use platform_wallet::changeset::{ + ClientStartState, Merge, PersistenceCapabilities, PersistenceError, PersistenceErrorKind, + PlatformWalletChangeSet, PlatformWalletPersistence, +}; +use platform_wallet::wallet::platform_wallet::WalletId; +use platform_wallet_storage::SqlitePersister; + +/// Queued faults plus the changeset a transient fault left staged. +/// +/// Inert until [`arm`](Self::arm) queues something, so production-shaped tests +/// pay nothing for its presence. +#[derive(Default)] +pub(crate) struct PersistFaults { + queue: Mutex>, + staged: Mutex, +} + +impl PersistFaults { + /// Queue `kinds`, served one per `store` / `flush` call in order. + pub(crate) fn arm(&self, kinds: impl IntoIterator) { + self.queue.lock().expect("fault queue").extend(kinds); + } + + /// `true` while an injected transient failure still holds an uncommitted + /// changeset — the persister-buffer state a caller must not discard. + pub(crate) fn has_staged_changeset(&self) -> bool { + !self.staged.lock().expect("staged changeset").is_empty() + } + + fn next(&self) -> Option { + self.queue.lock().expect("fault queue").pop_front() + } + + fn stage(&self, changeset: PlatformWalletChangeSet) { + self.staged + .lock() + .expect("staged changeset") + .merge(changeset); + } + + fn take_staged(&self) -> PlatformWalletChangeSet { + std::mem::take(&mut *self.staged.lock().expect("staged changeset")) + } +} + +/// Borrowed decorator over the real persister; cheap to build per call. +pub(crate) struct PersistFaultInjector<'a> { + inner: &'a SqlitePersister, + faults: &'a PersistFaults, +} + +impl<'a> PersistFaultInjector<'a> { + pub(crate) fn new(inner: &'a SqlitePersister, faults: &'a PersistFaults) -> Self { + Self { inner, faults } + } + + fn injected(kind: PersistenceErrorKind) -> PersistenceError { + PersistenceError::backend_with_kind(kind, "injected persistence fault") + } +} + +impl PlatformWalletPersistence for PersistFaultInjector<'_> { + fn persistence_capabilities(&self) -> PersistenceCapabilities { + self.inner.persistence_capabilities() + } + + fn store( + &self, + wallet_id: WalletId, + changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + match self.faults.next() { + // Contract: a transient `store` MUST preserve the changeset so a + // later bare `flush` completes the write without re-supplying it. + Some(PersistenceErrorKind::Transient) => { + self.faults.stage(changeset); + Err(Self::injected(PersistenceErrorKind::Transient)) + } + Some(kind) => Err(Self::injected(kind)), + None => { + self.faults.stage(changeset); + self.inner.store(wallet_id, self.faults.take_staged()) + } + } + } + + fn flush(&self, wallet_id: WalletId) -> Result<(), PersistenceError> { + match self.faults.next() { + Some(PersistenceErrorKind::Transient) => { + Err(Self::injected(PersistenceErrorKind::Transient)) + } + // Contract: a terminal failure drops the buffer — the staged + // changeset is gone and no retry can recover it. + Some(kind) => { + self.faults.take_staged(); + Err(Self::injected(kind)) + } + None => { + let staged = self.faults.take_staged(); + if !staged.is_empty() { + self.inner.store(wallet_id, staged)?; + } + self.inner.flush(wallet_id) + } + } + } + + fn load(&self) -> Result { + self.inner.load() + } +} From 4b086264c265fccab6134ffe1d93c31060a2e234 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:29:44 +0000 Subject: [PATCH 03/11] fix(wallet): rewrite pending funding-account registrations instead of re-flushing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The persister buffer is shared per wallet, not owned by the registration call site. `SqlitePersister::flush_inner` returns `Ok(())` when `take_for_flush` finds nothing staged, and any other writer's terminal flush takes the whole merged buffer — these entries included — via `handle_flush_error`, which restores only on transient errors. Upstream writes into that buffer constantly and swallows the error (`core_bridge.rs`, `platform_addresses/sync.rs`). So a bare `flush` returning `Ok(())` cannot distinguish "my row was committed" from "my row was thrown away by somebody else". The previous retry inferred the first, cleared its marker and reported success — leaving the funding account live in memory and absent from `account_registrations`, which is the exact manifest/wallet disagreement this work exists to prevent, moved one step out. Two windows were reachable: between two attempts of one retry loop, and between a failed attempt and the next provisioning call that drained its marker. Every attempt now resupplies the entries through `store` instead of re-flushing, so success means this write committed. Resupplying is safe precisely here and the general "re-`store` would double-merge" caution does not bind: `account_registrations` merges by `extend` and applies through `UPSERT_ACCOUNT_SQL`, an `ON CONFLICT(wallet_id, account_type, account_index, …) DO UPDATE` keyed on the account's identity, so the same entry written twice is idempotent. The staged-marker drain is no longer a separate mechanism: pending registrations are simply part of the entry set the next attempt writes, so both windows close together. The retry backoff no longer sleeps under the manager guard. `wallet_manager()` hands out ONE `RwLock` shared by every wallet in the process, and it is write-preferring; upstream warns that holding it stalls removal and reads for every wallet. Wallet state is now prepared under the guard, the guard is released for the persist, and it is re-acquired only to evict on a terminal failure — evicting solely accounts still carrying the xpub whose write was lost, so a concurrent provisioning that re-created one is not collateral. A pending marker that cannot be recorded now evicts the account rather than leaving it in memory with nothing scheduled to persist it, and markers are dropped when a wallet is removed: a same-seed re-import computes the same `WalletId` and would otherwise inherit them. Fault-injector fidelity, which the assertions depend on: a terminal `store` fault now drops the whole staged buffer as the real persister does (previously only its `flush` arm did), and the staged buffer is keyed by `WalletId` like the real `Buffer`'s map — unkeyed, a two-wallet test could commit one wallet's registration under another's id and still pass. Confirmed RED against the previous behaviour, each in its own window: `a_registration_lost_between_retries_is_rewritten_not_reported_saved` and `a_registration_lost_by_a_foreign_buffer_drain_is_rewritten_not_reported_saved` (both `outcome=Ok, persisted_rows=0` before the fix). Adds coverage for the `Constraint` terminal kind, transient-to-terminal escalation, and marker cleanup on wallet removal. Reported by Marvin (QA-001 executed repro, QA-002..006). Co-Authored-By: Claude Opus 5 --- src/context/wallet_lifecycle/tests.rs | 283 +++++++++++++- src/wallet_backend/identity_ops.rs | 364 +++++++++++------- src/wallet_backend/mod.rs | 71 +++- .../persist_fault_test_support.rs | 108 ++++-- 4 files changed, 628 insertions(+), 198 deletions(-) diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 115540dfd..7d6e0b846 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -3946,24 +3946,48 @@ async fn an_unbound_topup_funding_account_survives_a_restart() { } /// Rows in the persisted `account_registrations` manifest for one account -/// type / index — what `load()` rebuilds `Wallet.accounts` from. +/// type / index across every wallet — what `load()` rebuilds +/// `Wallet.accounts` from. fn persisted_account_registration_rows( data_dir: &std::path::Path, account_type: &str, account_index: u32, ) -> i64 { - rusqlite::Connection::open_with_flags( + persisted_account_registration_rows_for(data_dir, None, account_type, account_index) +} + +/// As [`persisted_account_registration_rows`], but scoped to one upstream +/// `wallet_id` when given. Multi-wallet tests must scope: the manifest is +/// shared, so an unscoped count cannot tell whose registration it found. +fn persisted_account_registration_rows_for( + data_dir: &std::path::Path, + wallet_id: Option<[u8; 32]>, + account_type: &str, + account_index: u32, +) -> i64 { + let conn = rusqlite::Connection::open_with_flags( wallet_database_path(data_dir, Network::Testnet), rusqlite::OpenFlags::SQLITE_OPEN_READ_ONLY, ) - .expect("open persisted store") - .query_row( - "SELECT COUNT(*) FROM account_registrations \ - WHERE account_type = ?1 AND account_index = ?2", - rusqlite::params![account_type, account_index], - |row| row.get(0), - ) - .expect("count persisted account registrations") + .expect("open persisted store"); + match wallet_id { + Some(wallet_id) => conn + .query_row( + "SELECT COUNT(*) FROM account_registrations \ + WHERE wallet_id = ?1 AND account_type = ?2 AND account_index = ?3", + rusqlite::params![&wallet_id[..], account_type, account_index], + |row| row.get(0), + ) + .expect("count persisted account registrations for wallet"), + None => conn + .query_row( + "SELECT COUNT(*) FROM account_registrations \ + WHERE account_type = ?1 AND account_index = ?2", + rusqlite::params![account_type, account_index], + |row| row.get(0), + ) + .expect("count persisted account registrations"), + } } /// A malformed Orchard viewing key must be isolated to its own wallet during @@ -4545,7 +4569,7 @@ async fn unbound_topup_funding_account_provisions_on_the_watch_only_wallet() { /// longer has the account — a manifest and a wallet that disagree about which /// accounts exist. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_transient_registration_persist_failure_retries_the_bare_flush_and_keeps_the_account() { +async fn a_transient_registration_persist_failure_is_retried_and_keeps_the_account() { let dir = tempfile::tempdir().expect("tempdir"); let (ctx, sender) = offline_testnet_context_at(dir.path()); ctx.ensure_wallet_backend(sender) @@ -4574,7 +4598,7 @@ async fn a_transient_registration_persist_failure_retries_the_bare_flush_and_kee ); assert!( !backend.registration_persist_buffer_is_staged(), - "the bare-flush retry must have drained the persister buffer", + "a successful retry must leave nothing staged", ); backend.shutdown().await; @@ -4593,8 +4617,7 @@ async fn a_transient_registration_persist_failure_retries_the_bare_flush_and_kee /// the account is live in memory, absent from the manifest, and every launch /// after this one re-derives nothing. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn an_exhausted_transient_persist_budget_keeps_memory_and_the_next_attempt_flushes_the_buffer() - { +async fn an_exhausted_transient_persist_budget_keeps_memory_and_the_next_attempt_rewrites_it() { let dir = tempfile::tempdir().expect("tempdir"); let (ctx, sender) = offline_testnet_context_at(dir.path()); ctx.ensure_wallet_backend(sender) @@ -4625,18 +4648,18 @@ async fn an_exhausted_transient_persist_budget_keeps_memory_and_the_next_attempt assert!(view.managed, "both collections must keep the account"); assert!( backend.registration_persist_buffer_is_staged(), - "the changeset must still be staged for a later flush", + "the changeset must still be staged for a later rewrite", ); - // The next attempt sees both collections populated. It must complete the - // staged write rather than short-circuit on the presence guards. + // The next attempt sees both collections populated. It must rewrite the + // pending registration rather than short-circuit on the presence guards. backend .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) .await - .expect("the follow-up attempt must flush the staged registration"); + .expect("the follow-up attempt must rewrite the pending registration"); assert!( !backend.registration_persist_buffer_is_staged(), - "the follow-up attempt must have drained the persister buffer", + "the follow-up attempt must leave nothing staged", ); backend.shutdown().await; @@ -4762,6 +4785,228 @@ async fn a_terminal_drain_evicts_only_the_accounts_whose_registrations_it_lost() ); } +/// A write that another caller drains and loses must not be reported as saved. +/// +/// The persister buffer is shared per wallet, not owned by this call site, and +/// `flush` answers `Ok(())` when it finds nothing staged. Upstream writes into +/// that buffer constantly and swallows its own terminal failures, which take +/// the whole merged changeset — this registration included — with them. A +/// retry that only re-flushes therefore cannot tell "my row was committed" +/// from "my row was thrown away by somebody else", and reporting success there +/// strands an asset lock against a credit-output path that exists only in RAM. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_registration_lost_by_a_foreign_buffer_drain_is_rewritten_not_reported_saved() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC7u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "foreign").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("the exhausted budget must surface"); + + // An unrelated wallet write drains the shared buffer and dies terminally: + // the staged registration is gone and this call site is never told. + backend.discard_staged_registration_buffer(); + + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the follow-up attempt must rewrite the lost registration"); + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .is_some(), + "the account must still be live in memory after a successful rewrite", + ); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "success must mean the row is in the manifest, not merely that some \ + flush returned Ok", + ); +} + +/// The same loss, inside a single retry loop, must also be rewritten. +/// +/// After a transient `store` parks the changeset, the backoff window belongs +/// to everyone: an unrelated writer can drain the shared buffer and lose it +/// terminally before this call's next attempt. A retry that only re-flushes +/// would read `Ok(())` off an empty buffer and report a saved registration +/// that no longer exists anywhere. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_registration_lost_between_retries_is_rewritten_not_reported_saved() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC9u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "between").await; + + // The first `store` parks the changeset; before the retry runs, an + // unrelated writer drains the shared buffer and loses it. + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient]); + backend.discard_staged_registration_buffer_before_write(2); + + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the retry must rewrite the lost registration, not ride the buffer"); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "reporting success means the row is in the manifest — a flush that found \ + an empty buffer proves nothing", + ); +} + +/// `Constraint` is the third terminal kind and must behave exactly like +/// `Fatal`. `PersistenceErrorKind` is deliberately not `#[non_exhaustive]` so +/// that a new kind breaks every consumer match — pinning this branch is what +/// makes that guarantee worth something here. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_constraint_registration_persist_failure_rolls_back_the_in_memory_account() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC5u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "constraint").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Constraint]); + let error = backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("a constraint violation must surface as a typed error"); + assert!( + matches!(error, TaskError::IdentityFundingAccountPersistFailed { .. }), + "the persist failure must keep its dedicated variant, got {error:?}", + ); + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .is_none(), + "a constraint violation is terminal, so the account must leave memory", + ); + assert!( + !backend.has_pending_account_registrations(&seed_hash), + "a terminal failure must leave nothing pending to rewrite", + ); + + backend.shutdown().await; +} + +/// A write that starts transient and then fails terminally is terminal: the +/// staged changeset is gone, so the account must be rolled back rather than +/// kept on the strength of the first, softer error. +/// +/// This is the one ordering where misclassifying strands funds — keeping an +/// account whose registration was discarded means the next attempt +/// early-returns success on a row that will never exist. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_transient_write_that_escalates_to_a_terminal_one_rolls_back() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC6u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "escalate").await; + + // Transient on the first `store`, then terminal on the retry. + backend.arm_registration_persist_faults([ + PersistenceErrorKind::Transient, + PersistenceErrorKind::Fatal, + ]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("the escalated failure must surface"); + + assert!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .is_none(), + "escalation to a terminal failure must roll the account back", + ); + assert!( + !backend.registration_persist_buffer_is_staged(), + "a terminal failure must take the staged changeset with it", + ); + assert!( + !backend.has_pending_account_registrations(&seed_hash), + "a terminal failure must leave nothing pending to rewrite", + ); + + backend.shutdown().await; +} + +/// Removing a wallet must drop its pending registrations. +/// +/// A same-seed re-import computes the same upstream `WalletId`, so a marker +/// left behind would be inherited by the fresh wallet and rewritten against +/// accounts it never provisioned. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn removing_a_wallet_drops_its_pending_account_registrations() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xC8u8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "removed").await; + + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect_err("the exhausted budget must surface"); + assert!( + backend.has_pending_account_registrations(&seed_hash), + "the registration must be recorded as pending before removal", + ); + + let wallet_id = backend.registered_wallet_id(&seed_hash).expect("wallet id"); + backend + .forget_wallet_local_state(&seed_hash, Some(wallet_id)) + .expect("forget wallet local state"); + + assert!( + !backend.has_pending_account_registrations(&seed_hash), + "wallet removal must drop the pending registration, or a same-seed \ + re-import inherits it", + ); + + backend.shutdown().await; +} + /// Identity index the persist-failure tests provision a top-up account for. /// /// Per-index top-up accounts are the funding accounts DET actually creates: diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index ba7d765a1..f2bf9e52c 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -17,14 +17,16 @@ use super::{ map_identity_register_error, map_identity_top_up_error, map_platform_address_fund_error, }; +use dash_sdk::dpp::key_wallet::account::Account; +use dash_sdk::dpp::key_wallet::bip32::ExtendedPubKey; use platform_wallet::changeset::{ AccountRegistrationEntry, PersistenceError, PlatformWalletChangeSet, PlatformWalletPersistence, }; -/// Total attempts (initial write plus bare-`flush` retries) for a transient -/// persister failure on the funding-account path. Mirrors upstream's -/// `PERSIST_RETRY_MAX_ATTEMPTS` on the wallet-registration path, which is -/// `pub(super)` there and so unreachable from DET. +/// Total attempts for a transient persister failure on the funding-account +/// path. Mirrors upstream's `PERSIST_RETRY_MAX_ATTEMPTS` on the +/// wallet-registration path, which is `pub(super)` there and so unreachable +/// from DET. const PERSIST_RETRY_MAX_ATTEMPTS: u32 = 4; /// Backoff before the first retry; doubles up to [`PERSIST_RETRY_MAX_BACKOFF`]. @@ -52,6 +54,41 @@ pub(super) enum Funding { } impl Funding { + /// The upstream account type this funding flavour derives and registers. + fn account_type(self) -> dash_sdk::dpp::key_wallet::AccountType { + use dash_sdk::dpp::key_wallet::AccountType; + match self { + Self::Registration => AccountType::IdentityRegistration, + Self::TopUp(registration_index) => AccountType::IdentityTopUp { registration_index }, + Self::TopUpNotBound => AccountType::IdentityTopUpNotBoundToIdentity, + } + } + + /// Whether the managed (info-side) collection holds this funding account. + fn is_managed( + self, + info: &platform_wallet::wallet::platform_wallet::PlatformWalletInfo, + ) -> bool { + match self { + Self::Registration => info.core_wallet.accounts.identity_registration.is_some(), + Self::TopUp(registration_index) => info + .core_wallet + .accounts + .identity_topup + .contains_key(®istration_index), + Self::TopUpNotBound => info.core_wallet.accounts.identity_topup_not_bound.is_some(), + } + } + + /// This funding account as the key-wallet holds it, if at all. + fn account(self, kw: &dash_sdk::dpp::key_wallet::wallet::Wallet) -> Option<&Account> { + match self { + Self::Registration => kw.accounts.identity_registration.as_ref(), + Self::TopUp(registration_index) => kw.accounts.identity_topup.get(®istration_index), + Self::TopUpNotBound => kw.accounts.identity_topup_not_bound.as_ref(), + } + } + /// Drop this funding account from both in-memory collections, so the next /// provisioning attempt re-creates and re-persists it instead of /// short-circuiting on the presence guards. @@ -99,10 +136,10 @@ pub(crate) struct FundingAccountView { /// caller must do with the account it just put in memory. enum RegistrationPersistFailure { /// The retry budget ran out while the failure was still transient. The - /// persister's contract keeps the changeset buffered, so the row can still - /// land: the account MUST stay in memory, or a later unrelated flush would - /// commit a registration the live wallet no longer has. A bare `flush` - /// completes the write. + /// changeset may still be buffered and land later, so the account MUST + /// stay in memory — dropping it while the row can still appear would leave + /// a manifest the live wallet disagrees with. The registration is recorded + /// as pending and rewritten by the next attempt. Staged(PersistenceError), /// A terminal failure discarded the staged changeset. The row will never /// land, so the account must leave memory. @@ -142,7 +179,7 @@ where max_attempts = PERSIST_RETRY_MAX_ATTEMPTS, backoff_ms = backoff.as_millis() as u64, error = %e, - "identity-funding account registration hit a transient persister failure; backing off before retrying the flush" + "identity-funding account registration hit a transient persister failure; backing off before rewriting the registration" ); tokio::time::sleep(backoff).await; backoff = backoff.saturating_mul(2).min(PERSIST_RETRY_MAX_BACKOFF); @@ -152,30 +189,36 @@ where } } -/// Write one account registration through `persister` and flush it, so a cold -/// boot rebuilds the account from the manifest. +/// Write these account registrations through `persister` and flush them, so a +/// cold boot rebuilds the accounts from the manifest. /// -/// The first attempt is `store` then `flush`; every retry is a **bare** -/// `flush`, because the persister's contract preserves the changeset across a -/// transient failure and re-`store`ing the same delta would double-merge it. -async fn persist_account_registration( +/// Every attempt — the first and each retry — **resupplies** the entries via +/// `store` rather than issuing a bare `flush`. The persister's buffer is +/// shared per wallet, not owned by this call site: `flush` answers `Ok(())` +/// when it finds nothing staged, and any other writer's terminal flush takes +/// the whole merged buffer (these entries included) without telling us. A bare +/// `flush` therefore cannot distinguish "committed" from "thrown away by +/// somebody else", and inferring durability from it is how a funding account +/// ends up live in memory and absent from the manifest. +/// +/// Resupplying is safe precisely here, and the general "re-`store` would +/// double-merge" caution does not bind: `account_registrations` merges by +/// `extend` and applies through `UPSERT_ACCOUNT_SQL`, an +/// `ON CONFLICT(wallet_id, account_type, account_index, …) DO UPDATE` keyed on +/// the account's identity, so writing the same entry twice is idempotent. +async fn persist_account_registrations( persister: &dyn PlatformWalletPersistence, wallet_id: WalletId, - account_type: dash_sdk::dpp::key_wallet::AccountType, - account_xpub: dash_sdk::dpp::key_wallet::bip32::ExtendedPubKey, + entries: &[AccountRegistrationEntry], ) -> Result<(), RegistrationPersistFailure> { - let mut changeset = Some(PlatformWalletChangeSet { - account_registrations: vec![AccountRegistrationEntry { - account_type, - account_xpub, - }], - ..Default::default() - }); - retry_transient(|| match changeset.take() { - Some(changeset) => persister + retry_transient(|| { + let changeset = PlatformWalletChangeSet { + account_registrations: entries.to_vec(), + ..Default::default() + }; + persister .store(wallet_id, changeset) - .and_then(|()| persister.flush(wallet_id)), - None => persister.flush(wallet_id), + .and_then(|()| persister.flush(wallet_id)) }) .await } @@ -513,15 +556,79 @@ impl WalletBackend { seed: &[u8; 64], funding: Funding, ) -> Result<(), TaskError> { - use dash_sdk::dpp::key_wallet::AccountType; use dash_sdk::dpp::key_wallet::managed_account::ManagedCoreKeysAccount; let wallet = self.resolve_wallet(seed_hash).await?; let wallet_id = wallet.wallet_id(); - let mut wm = wallet.wallet_manager().write().await; - let (kw, info) = wm - .get_wallet_mut_and_info_mut(&wallet_id) - .ok_or(TaskError::WalletStateInconsistent)?; + + // Everything that touches wallet state happens under the manager + // guard; the persist below deliberately does not. `wallet_manager()` + // hands out ONE `RwLock` shared by every wallet in the + // process, and it is write-preferring, so sleeping a retry backoff + // under it would stall balance reads, sync callbacks and wallet + // removal for every other wallet — upstream warns about exactly this. + let pending = { + let mut wm = wallet.wallet_manager().write().await; + let (kw, info) = wm + .get_wallet_mut_and_info_mut(&wallet_id) + .ok_or(TaskError::WalletStateInconsistent)?; + + let in_wallet = funding.account(kw).is_some(); + let in_managed = funding.is_managed(info); + + // Registrations a previous attempt could not make durable. They + // are rewritten below: returning success while one is outstanding + // would leave its account live in memory and absent from the + // manifest. + let mut pending = self.staged_account_registrations(&wallet_id)?; + if in_wallet && in_managed && pending.is_empty() { + return Ok(()); + } + + if !in_wallet { + let account_type = funding.account_type(); + // The live wallet is watch-only: calling `add_account(…, None)` would + // try to derive a hardened path from an absent private key and fail + // with "Watch-only wallet has no private key". Derive the xpub from a + // short-lived seed wallet instead and pass it as `Some(xpub)`. + let seed_wallet = self.seed_wallet(seed)?; + + let path = account_type + .derivation_path(self.inner.network) + .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { + source, + })?; + + let account_xpub = + seed_wallet + .derive_extended_public_key(&path) + .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { + source, + })?; + + kw.add_account(account_type, Some(account_xpub)) + .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { + source, + })?; + } + + if !(in_wallet && in_managed) { + let derived = funding + .account(kw) + .ok_or(TaskError::WalletStateInconsistent)?; + let account_xpub = derived.account_xpub; + let managed = ManagedCoreKeysAccount::from_account(derived); + info.core_wallet + .accounts + .insert_keys_bearing_account(managed) + .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { + source, + })?; + pending.insert(funding, account_xpub); + } + + pending + }; // In test builds the write goes through a fault-injecting decorator // that is inert until a test arms it; production talks to the @@ -536,153 +643,122 @@ impl WalletBackend { #[cfg(test)] let persister: &dyn PlatformWalletPersistence = &injector; - let (in_wallet, in_managed) = match funding { - Funding::Registration => ( - kw.accounts.identity_registration.is_some(), - info.core_wallet.accounts.identity_registration.is_some(), - ), - Funding::TopUp(registration_index) => ( - kw.accounts.identity_topup.contains_key(®istration_index), - info.core_wallet - .accounts - .identity_topup - .contains_key(®istration_index), - ), - Funding::TopUpNotBound => ( - kw.accounts.identity_topup_not_bound.is_some(), - info.core_wallet.accounts.identity_topup_not_bound.is_some(), - ), + // Persist the registrations. `load()` rebuilds `Wallet.accounts` from + // `account_registrations` alone, and the upstream creator that would + // normally write these rows skips them once both in-memory collections + // hold the account — which they do by the time it runs, because of the + // inserts above. Without this the account is memory-only: a restart + // between an asset-lock broadcast and its consumption cannot re-derive + // the credit-output path, stranding the lock and its funds. + let entries: Vec = pending + .iter() + .map(|(funding, account_xpub)| AccountRegistrationEntry { + account_type: funding.account_type(), + account_xpub: *account_xpub, + }) + .collect(); + + let failure = match persist_account_registrations(persister, wallet_id, &entries).await { + Ok(()) => return self.clear_staged_account_registrations(&wallet_id), + Err(failure) => failure, }; - if in_wallet && in_managed { - // A previous attempt may have left registrations staged in the - // persister buffer after a transient failure. Returning success - // here would leave those accounts live in memory and absent from - // the manifest — precisely the state this path exists to prevent. - if self.staged_account_registrations(&wallet_id)?.is_empty() { - return Ok(()); - } - return match retry_transient(|| persister.flush(wallet_id)).await { - Ok(()) => self.clear_staged_account_registrations(&wallet_id), - Err(failure) => { - if matches!(failure, RegistrationPersistFailure::Discarded(_)) { - for staged in self.staged_account_registrations(&wallet_id)? { - staged.remove(kw, info); - } - self.clear_staged_account_registrations(&wallet_id)?; - } - Err(failure.into_task_error()) - } - }; - } - if !in_wallet { - let account_type = match funding { - Funding::Registration => AccountType::IdentityRegistration, - Funding::TopUp(registration_index) => { - AccountType::IdentityTopUp { registration_index } + // Keep the accounts only when the write is still worth retrying AND + // that intent was actually recorded. A marker we could not write is a + // silent dead end: the account would stay in memory with nothing + // scheduled to persist it, and the next call would early-return + // success on a row that is not there. + let recoverable = matches!(failure, RegistrationPersistFailure::Staged(_)) + && match self.mark_account_registrations_staged(wallet_id, &pending) { + Ok(()) => true, + Err(e) => { + tracing::error!( + wallet = %hex::encode(wallet_id), + error = ?e, + "could not record a pending identity-funding registration; dropping the account so a retry rebuilds it" + ); + false } - Funding::TopUpNotBound => AccountType::IdentityTopUpNotBoundToIdentity, }; - // The live wallet is watch-only: calling `add_account(…, None)` would - // try to derive a hardened path from an absent private key and fail - // with "Watch-only wallet has no private key". Derive the xpub from a - // short-lived seed wallet instead and pass it as `Some(xpub)`. - let seed_wallet = self.seed_wallet(seed)?; - - let path = account_type - .derivation_path(self.inner.network) - .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; - - let account_xpub = seed_wallet - .derive_extended_public_key(&path) - .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; - - kw.add_account(account_type, Some(account_xpub)) - .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; - } - let derived = match funding { - Funding::Registration => kw.accounts.identity_registration.as_ref(), - Funding::TopUp(registration_index) => { - kw.accounts.identity_topup.get(®istration_index) + if !recoverable { + // The staged changeset is gone (or unrecoverable), so every + // account it covered must leave memory — otherwise the presence + // guards short-circuit persists that never happened. + self.evict_funding_accounts(&wallet, wallet_id, &pending) + .await; + if let Err(e) = self.clear_staged_account_registrations(&wallet_id) { + tracing::error!( + wallet = %hex::encode(wallet_id), + error = ?e, + "could not clear pending identity-funding registrations after a terminal persist failure" + ); } - Funding::TopUpNotBound => kw.accounts.identity_topup_not_bound.as_ref(), } - .ok_or(TaskError::WalletStateInconsistent)?; - - let account_type = derived.account_type; - let account_xpub = derived.account_xpub; - let managed = ManagedCoreKeysAccount::from_account(derived); - info.core_wallet - .accounts - .insert_keys_bearing_account(managed) - .map_err(|source| TaskError::IdentityFundingAccountProvisionFailed { source })?; + Err(failure.into_task_error()) + } - // Persist the registration. `load()` rebuilds `Wallet.accounts` from - // `account_registrations` alone, and the upstream creator that would - // normally write this row skips it once both in-memory collections hold - // the account — which they do by the time it runs, because of the - // inserts above. Without this the account is memory-only: a restart - // between an asset-lock broadcast and its consumption cannot re-derive - // the credit-output path, stranding the lock and its funds. - match persist_account_registration(persister, wallet_id, account_type, account_xpub).await { - // A successful flush is wallet-wide: everything staged landed too. - Ok(()) => self.clear_staged_account_registrations(&wallet_id), - Err(RegistrationPersistFailure::Staged(source)) => { - // The row can still land, so keeping the account is what makes - // disk and the live wallet agree; the marker makes the next - // attempt for this wallet finish the write. - self.mark_account_registration_staged(wallet_id, funding)?; - Err(RegistrationPersistFailure::Staged(source).into_task_error()) - } - Err(RegistrationPersistFailure::Discarded(source)) => { - // A terminal failure drops the whole wallet buffer, so this - // account and every previously staged one must leave memory — - // otherwise the presence guards short-circuit persists that - // never happened. + /// Drop funding accounts whose registrations did not reach disk from both + /// in-memory collections, so the next attempt re-creates and re-persists + /// them. + /// + /// Re-acquires the manager guard, which was released for the persist, and + /// only removes an account still carrying the xpub we tried to write — a + /// concurrent provisioning that re-created it has its own durable row and + /// must not be evicted on our behalf. + async fn evict_funding_accounts( + &self, + wallet: &Arc, + wallet_id: WalletId, + pending: &std::collections::BTreeMap, + ) { + let mut wm = wallet.wallet_manager().write().await; + let Some((kw, info)) = wm.get_wallet_mut_and_info_mut(&wallet_id) else { + return; + }; + for (funding, account_xpub) in pending { + if funding + .account(kw) + .is_some_and(|a| a.account_xpub == *account_xpub) + { funding.remove(kw, info); - for staged in self.staged_account_registrations(&wallet_id)? { - staged.remove(kw, info); - } - self.clear_staged_account_registrations(&wallet_id)?; - Err(RegistrationPersistFailure::Discarded(source).into_task_error()) } } } - /// The funding accounts whose registration a transient failure left staged - /// in this wallet's persister buffer. + /// The funding accounts whose registration a previous attempt could not + /// make durable, with the account xpub each one still needs written. fn staged_account_registrations( &self, wallet_id: &WalletId, - ) -> Result, TaskError> { + ) -> Result, TaskError> { Ok(self .inner .buffered_account_registrations .lock()? .get(wallet_id) - .map(|staged| staged.iter().copied().collect()) + .cloned() .unwrap_or_default()) } - /// Record a staged registration for the idempotent early return in - /// [`Self::provision_identity_funding_account`] to finish later. - fn mark_account_registration_staged( + /// Record registrations for the next provisioning attempt on this wallet + /// to rewrite, instead of early-returning success on absent rows. + fn mark_account_registrations_staged( &self, wallet_id: WalletId, - funding: Funding, + pending: &std::collections::BTreeMap, ) -> Result<(), TaskError> { self.inner .buffered_account_registrations .lock()? .entry(wallet_id) .or_default() - .insert(funding); + .extend(pending.iter().map(|(f, x)| (*f, *x))); Ok(()) } - /// Forget this wallet's staged registrations, once they are durable or + /// Forget this wallet's pending registrations, once they are durable or /// definitively lost. fn clear_staged_account_registrations(&self, wallet_id: &WalletId) -> Result<(), TaskError> { self.inner diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index d0e83db36..842af72c5 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -351,19 +351,28 @@ struct Inner { /// registration write. Inert until a test arms it. #[cfg(test)] persist_faults: persist_fault_test_support::PersistFaults, - /// Per wallet, the identity-funding accounts whose registration is staged - /// in the persister buffer but not yet durable, because a transient write - /// failure exhausted its retry budget. Each account stays live in memory - /// (its row can still land), so the next provisioning attempt for that - /// wallet must complete the write with a bare `flush` instead of taking - /// the idempotent early return and reporting success on absent rows. + /// Per wallet, the identity-funding accounts whose registration a + /// transient write failure could not make durable, each with the account + /// xpub still owed to the manifest. The accounts stay live in memory (the + /// rows can still land), so the next provisioning attempt for that wallet + /// rewrites them instead of taking the idempotent early return and + /// reporting success on rows that are not there. /// - /// Keyed by account and not just by wallet: `flush` is wallet-scoped, so - /// one failed drain discards every registration staged for that wallet and - /// all of them — not merely the account being provisioned right now — have - /// to leave memory. - buffered_account_registrations: - std::sync::Mutex>>, + /// Carries the xpub, not just the account identity, so the rewrite + /// resupplies the entry rather than betting on a shared persister buffer + /// this call site does not own — and so a terminal failure only evicts an + /// account still carrying the xpub whose write was lost. + /// + /// Entries are dropped when the write succeeds, when it fails terminally, + /// and when the wallet is removed ([`Self::forget_wallet_local_state`]) — + /// a stale entry would otherwise be inherited by a same-seed re-import, + /// which computes the same `WalletId`. + buffered_account_registrations: std::sync::Mutex< + std::collections::BTreeMap< + WalletId, + std::collections::BTreeMap, + >, + >, /// Per-wallet shared-result flights for upstream registration. Every caller /// that joins an active flight awaits the same success or typed error. registration_flights: @@ -1124,6 +1133,37 @@ impl WalletBackend { self.inner.persist_faults.has_staged_changeset() } + /// Drop the staged changeset the way an unrelated writer's terminal flush + /// does — the buffer is shared per wallet, and its loss is never reported + /// to the account-registration caller. + #[cfg(test)] + pub(crate) fn discard_staged_registration_buffer(&self) { + self.inner.persist_faults.discard_staged(); + } + + /// Arm a one-shot foreign drain of the staged buffer immediately before the + /// `write`-th `store`/`flush` on the registration path. Write 1 is the + /// first attempt's `store`, so arming write 2 puts the drain in the + /// backoff window between the first attempt and its retry. + #[cfg(test)] + pub(crate) fn discard_staged_registration_buffer_before_write(&self, write: usize) { + self.inner.persist_faults.discard_staged_before_write(write); + } + + /// `true` when a pending funding-account registration is recorded for this + /// wallet, awaiting a rewrite by the next provisioning attempt. + #[cfg(test)] + pub(crate) fn has_pending_account_registrations(&self, seed_hash: &WalletSeedHash) -> bool { + let Some(wallet_id) = self.registered_wallet_id(seed_hash) else { + return false; + }; + self.inner + .buffered_account_registrations + .lock() + .map(|pending| pending.contains_key(&wallet_id)) + .unwrap_or(false) + } + /// Resolve one just-registered upstream wallet into the DET-keyed maps via /// the account-xpub fund-routing gate, identical in spirit to the gate the /// seedless loader applies per wallet. @@ -1422,6 +1462,13 @@ impl WalletBackend { self.inner.id_map.write()?.remove(seed_hash); self.inner.wallets.write()?.remove(&wallet_id); self.inner.snapshots.forget_wallet(seed_hash, &wallet_id); + // A same-seed re-import computes the same `WalletId`, so a pending + // registration left behind here would be inherited by the fresh + // wallet and rewritten against accounts it never provisioned. + self.inner + .buffered_account_registrations + .lock()? + .remove(&wallet_id); } match first_error { diff --git a/src/wallet_backend/persist_fault_test_support.rs b/src/wallet_backend/persist_fault_test_support.rs index 12b006ca2..69835c014 100644 --- a/src/wallet_backend/persist_fault_test_support.rs +++ b/src/wallet_backend/persist_fault_test_support.rs @@ -3,17 +3,25 @@ //! //! Decorates the real [`SqlitePersister`] so a test can make individual //! `store` / `flush` calls fail with a chosen [`PersistenceErrorKind`] while -//! every other call reaches the real database. The injected failures honour the -//! trait's own contract: a `Transient` failure parks the changeset so a later -//! bare `flush` still commits it, a terminal failure discards it. Assertions -//! can therefore be made against real persisted rows rather than a fake's +//! every other call reaches the real database. The injected failures mirror +//! what the real persister does with the buffer: a `Transient` failure parks +//! the changeset so a later write still commits it, and a terminal failure +//! drops the whole per-wallet buffer — including anything a previous transient +//! failure had staged, because `flush_inner` takes the merged buffer and +//! `handle_flush_error` restores it only for transient errors. Assertions can +//! therefore be made against real persisted rows rather than a fake's //! bookkeeping. //! +//! The staged buffer is keyed by [`WalletId`], matching the real `Buffer`'s +//! `HashMap`: an unkeyed buffer would let a +//! two-wallet test commit one wallet's changeset under another wallet's id and +//! still pass. +//! //! Only `store`, `flush` and `load` are routed — the decorator is used at one //! call site (the identity-funding account registration write) and the trait's //! defaults cover the rest. -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::sync::Mutex; use platform_wallet::changeset::{ @@ -23,14 +31,17 @@ use platform_wallet::changeset::{ use platform_wallet::wallet::platform_wallet::WalletId; use platform_wallet_storage::SqlitePersister; -/// Queued faults plus the changeset a transient fault left staged. +/// Queued faults plus the per-wallet changesets a transient fault left staged. /// /// Inert until [`arm`](Self::arm) queues something, so production-shaped tests /// pay nothing for its presence. #[derive(Default)] pub(crate) struct PersistFaults { queue: Mutex>, - staged: Mutex, + staged: Mutex>, + /// One-based index of the `store`/`flush` call before which the staged + /// buffer is dropped; `0` disables the probe. + discard_before_write: std::sync::atomic::AtomicUsize, } impl PersistFaults { @@ -40,24 +51,68 @@ impl PersistFaults { } /// `true` while an injected transient failure still holds an uncommitted - /// changeset — the persister-buffer state a caller must not discard. + /// changeset for any wallet — the persister-buffer state a caller must not + /// assume it owns. pub(crate) fn has_staged_changeset(&self) -> bool { - !self.staged.lock().expect("staged changeset").is_empty() + self.staged + .lock() + .expect("staged changeset") + .values() + .any(|changeset| !changeset.is_empty()) + } + + /// Drop every staged changeset the way an unrelated writer's terminal + /// flush does: `flush_inner` takes the whole per-wallet buffer, and a + /// terminal failure discards it without the account-registration caller + /// ever seeing the error. + pub(crate) fn discard_staged(&self) { + self.staged.lock().expect("staged changeset").clear(); + } + + /// Arm a one-shot foreign drain immediately before the `write`-th + /// `store`/`flush` call, as if an unrelated writer had taken the shared + /// buffer and lost it terminally in that window. Counting calls (rather + /// than firing on the next one) is what places the drain *between* two + /// attempts of one retry loop. + pub(crate) fn discard_staged_before_write(&self, write: usize) { + self.discard_before_write + .store(write, std::sync::atomic::Ordering::Relaxed); + } + + /// Count this write and, if it is the armed one, drop the staged buffer. + fn apply_pending_discard(&self) { + let remaining = self + .discard_before_write + .load(std::sync::atomic::Ordering::Relaxed); + if remaining == 0 { + return; + } + self.discard_before_write + .store(remaining - 1, std::sync::atomic::Ordering::Relaxed); + if remaining == 1 { + self.discard_staged(); + } } fn next(&self) -> Option { self.queue.lock().expect("fault queue").pop_front() } - fn stage(&self, changeset: PlatformWalletChangeSet) { + fn stage(&self, wallet_id: WalletId, changeset: PlatformWalletChangeSet) { self.staged .lock() .expect("staged changeset") + .entry(wallet_id) + .or_default() .merge(changeset); } - fn take_staged(&self) -> PlatformWalletChangeSet { - std::mem::take(&mut *self.staged.lock().expect("staged changeset")) + fn take_staged(&self, wallet_id: &WalletId) -> Option { + self.staged + .lock() + .expect("staged changeset") + .remove(wallet_id) + .filter(|changeset| !changeset.is_empty()) } } @@ -87,35 +142,42 @@ impl PlatformWalletPersistence for PersistFaultInjector<'_> { wallet_id: WalletId, changeset: PlatformWalletChangeSet, ) -> Result<(), PersistenceError> { + self.faults.apply_pending_discard(); match self.faults.next() { - // Contract: a transient `store` MUST preserve the changeset so a - // later bare `flush` completes the write without re-supplying it. + // Contract: a transient failure preserves the changeset, so a + // later write still commits it. Some(PersistenceErrorKind::Transient) => { - self.faults.stage(changeset); + self.faults.stage(wallet_id, changeset); Err(Self::injected(PersistenceErrorKind::Transient)) } - Some(kind) => Err(Self::injected(kind)), + // A terminal failure takes the whole merged buffer with it, not + // just the incoming delta. + Some(kind) => { + self.faults.take_staged(&wallet_id); + Err(Self::injected(kind)) + } None => { - self.faults.stage(changeset); - self.inner.store(wallet_id, self.faults.take_staged()) + self.faults.stage(wallet_id, changeset); + match self.faults.take_staged(&wallet_id) { + Some(staged) => self.inner.store(wallet_id, staged), + None => Ok(()), + } } } } fn flush(&self, wallet_id: WalletId) -> Result<(), PersistenceError> { + self.faults.apply_pending_discard(); match self.faults.next() { Some(PersistenceErrorKind::Transient) => { Err(Self::injected(PersistenceErrorKind::Transient)) } - // Contract: a terminal failure drops the buffer — the staged - // changeset is gone and no retry can recover it. Some(kind) => { - self.faults.take_staged(); + self.faults.take_staged(&wallet_id); Err(Self::injected(kind)) } None => { - let staged = self.faults.take_staged(); - if !staged.is_empty() { + if let Some(staged) = self.faults.take_staged(&wallet_id) { self.inner.store(wallet_id, staged)?; } self.inner.flush(wallet_id) From 6b8a2282023a1a2debf1361c8af7f2b5d459a5e6 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:55:25 +0000 Subject: [PATCH 04/11] fix(wallet): serialise funding provisioning per wallet and scope its bookkeeping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Releasing the manager guard for the persist removed the accidental serialisation that made the pending-registration set safe to reason about. Nothing else gated it: `with_secret_session` drops its lock before running the closure, and provisioning is reachable from three independent async entry points. Two concurrent calls on one wallet then interleave a read-modify-write of that set — call A snapshots it, releases the guard, persists, and clears markers for the whole wallet, including one call B recorded in the meantime. B's account is left live in memory, absent from `account_registrations`, and unmarked, so the next call takes the idempotent early return and reports success. That is the same silent loss this work exists to prevent, one level further out. Provisioning now takes a per-wallet async lock for the whole call, mirroring the `ContactRequestActionLocks` pattern already used for paid DashPay actions (`Weak` values, self-pruning). It is per wallet and never held in the reverse order against the manager guard, so it reintroduces none of the process-wide contention that releasing the manager guard was meant to fix. Marker bookkeeping is scoped to the entries a call actually accounted for rather than the whole wallet, so a concurrent attempt's pending registration is never cleared by someone else — defence in depth, since the lock now prevents the interleaving outright. Eviction on a terminal failure is narrowed to the account the failing call itself created. Inherited pending entries keep both their account and their marker: the failure says nothing about them, a foreign flush may already have made them durable, and the next attempt rewrites them regardless. Evicting them as collateral produced memory-without-disk's mirror image — disk holding an account the live wallet had dropped. The xpub gate that previously guarded eviction is removed rather than repaired. It could not do what its comment claimed: the xpub is a deterministic BIP-32 derivation from the seed and the account's path, so a concurrent provisioning of the same account yields the identical value and the comparison is true in exactly the case it was meant to exclude. The provisioning lock is what makes unconditional removal sound, and the doc now says so. `a_terminal_drain_evicts_only_the_accounts_whose_registrations_it_lost` asserted the old rule and is rewritten, not deleted: it now pins that an inherited pending registration survives another call's terminal failure with its marker intact. Its name and comments also still described the drain mechanism removed in the previous commit. Confirmed RED against the previous behaviour: `provisioning_two_funding_accounts_on_one_wallet_does_not_overlap` observed a high-water mark of 2 concurrent provisioning calls on one wallet before the lock, 1 after. Reported by Marvin (QA-007 executed repro, QA-008..010). Co-Authored-By: Claude Opus 5 --- src/context/wallet_lifecycle/tests.rs | 96 ++++++++++++++---- src/wallet_backend/identity_ops.rs | 137 +++++++++++++++++++------- src/wallet_backend/mod.rs | 58 +++++++++++ 3 files changed, 240 insertions(+), 51 deletions(-) diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 7d6e0b846..2172a50c4 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -4717,16 +4717,17 @@ async fn a_terminal_registration_persist_failure_rolls_back_the_in_memory_accoun ); } -/// A drain that fails terminally must evict the accounts whose registrations -/// it lost — and only those. +/// A terminal failure must only evict what the failing call itself created. /// -/// `flush` is wallet-scoped, so the drain runs on whichever funding account is -/// provisioned next, which is usually not the one that was staged. Evicting -/// the account being provisioned instead of the staged one would strip a -/// perfectly durable account from the live wallet and leave the staged one -/// memory-only with nothing left to retry it. +/// A call rewrites every registration still pending on the wallet, not just +/// its own, so its `pending` set includes entries inherited from earlier +/// attempts. Its failure says nothing about those: a foreign flush may already +/// have made them durable, and the next attempt rewrites them regardless. +/// Evicting them as collateral would drop an account that can be on disk — +/// memory-without-disk's mirror image — so they keep both their account and +/// their pending marker, and a durable account is never touched at all. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_terminal_drain_evicts_only_the_accounts_whose_registrations_it_lost() { +async fn a_terminal_failure_keeps_inherited_pending_registrations_for_the_next_attempt() { let dir = tempfile::tempdir().expect("tempdir"); let (ctx, sender) = offline_testnet_context_at(dir.path()); ctx.ensure_wallet_backend(sender) @@ -4735,22 +4736,23 @@ async fn a_terminal_drain_evicts_only_the_accounts_whose_registrations_it_lost() let backend = ctx.wallet_backend().expect("backend wired"); let seed = [0xC4u8; 64]; - let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "drain").await; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "inherited").await; - // Strand the top-up registration in the buffer. + // Leave the top-up registration pending. backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); backend .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) .await .expect_err("the exhausted budget must surface"); - // The next call provisions the registration account first, so its drain of - // the staged top-up write is what fails terminally here. + // The next call creates nothing — both accounts are already in memory — so + // its only work is rewriting the inherited pending registration, and that + // is what fails terminally here. backend.arm_registration_persist_faults([PersistenceErrorKind::Fatal]); backend .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) .await - .expect_err("a terminal drain must surface"); + .expect_err("a terminal rewrite must surface"); assert!( backend @@ -4761,21 +4763,27 @@ async fn a_terminal_drain_evicts_only_the_accounts_whose_registrations_it_lost() .await .expect("probe the registration account") .is_some(), - "the registration account is durable and must survive another account's failed drain", + "a durable account must survive another account's failed rewrite", ); assert!( backend .identity_funding_account(&seed_hash, fault_topup_account_type()) .await .expect("probe the top-up account") - .is_none(), - "the account whose registration the drain discarded must leave memory", + .is_some(), + "an inherited pending account must not be evicted as collateral — its \ + row may already be durable by another route", + ); + assert!( + backend.has_pending_account_registrations(&seed_hash), + "the inherited registration must stay pending so the next attempt \ + rewrites it", ); backend .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) .await - .expect("a clean retry must re-create and re-persist the top-up account"); + .expect("a clean retry must rewrite the pending top-up registration"); backend.shutdown().await; assert_eq!( @@ -5007,6 +5015,60 @@ async fn removing_a_wallet_drops_its_pending_account_registrations() { backend.shutdown().await; } +/// Provisioning two funding accounts on one wallet must not overlap. +/// +/// Each call reads the wallet's pending-registration set, releases the manager +/// guard for the persist, and writes the set back. Overlapping calls interleave +/// those read-modify-write cycles, so one call's bookkeeping lands on top of +/// the other's and a registration that never reached the manifest is left with +/// nothing recorded to rewrite it. Serialising per wallet is what makes the +/// pending set safe to reason about; the manager guard used to do it by +/// accident, and no longer does. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn provisioning_two_funding_accounts_on_one_wallet_does_not_overlap() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xCEu8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "concurrent").await; + + // Enough transient faults that both calls sit in their retry windows at + // the same time if nothing serialises them. + backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 8]); + + let first = { + let backend = backend.clone(); + tokio::spawn(async move { + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, 11) + .await + }) + }; + let second = { + let backend = backend.clone(); + tokio::spawn(async move { + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, 12) + .await + }) + }; + let _ = first.await.expect("join first"); + let _ = second.await.expect("join second"); + + assert_eq!( + backend.provisioning_high_water(), + 1, + "two provisioning calls on one wallet overlapped; their pending-registration \ + bookkeeping can then clobber each other", + ); + + backend.shutdown().await; +} + /// Identity index the persist-failure tests provision a top-up account for. /// /// Per-index top-up accounts are the funding accounts DET actually creates: diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index f2bf9e52c..346c5079b 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -132,6 +132,46 @@ pub(crate) struct FundingAccountView { pub managed: bool, } +/// Per-wallet serialisation for identity-funding provisioning. +/// +/// A provisioning call reads the wallet's pending-registration set, releases +/// the manager guard for the persist, then writes the set back. Overlapping +/// calls interleave those read-modify-write cycles and one call's bookkeeping +/// lands on top of the other's — leaving a registration that never reached the +/// manifest with nothing recorded to rewrite it. The manager guard used to +/// serialise these by accident; it is deliberately released for the persist +/// now (holding a process-wide lock across a retry backoff stalls every +/// wallet), so this carries the invariant explicitly, and per wallet. +/// +/// Lock order is provisioning lock → manager guard, never the reverse. +#[derive(Default)] +pub(super) struct FundingProvisionLocks { + locks: std::sync::Mutex< + std::collections::BTreeMap>>, + >, +} + +impl FundingProvisionLocks { + async fn lock(&self, wallet_id: WalletId) -> tokio::sync::OwnedMutexGuard<()> { + let wallet_lock = { + let mut locks = self + .locks + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + locks.retain(|_, lock| lock.strong_count() > 0); + match locks.get(&wallet_id).and_then(std::sync::Weak::upgrade) { + Some(lock) => lock, + None => { + let lock = Arc::new(tokio::sync::Mutex::new(())); + locks.insert(wallet_id, Arc::downgrade(&lock)); + lock + } + } + }; + wallet_lock.lock_owned().await + } +} + /// Why an account registration did not reach disk — and therefore what the /// caller must do with the account it just put in memory. enum RegistrationPersistFailure { @@ -561,13 +601,20 @@ impl WalletBackend { let wallet = self.resolve_wallet(seed_hash).await?; let wallet_id = wallet.wallet_id(); + // Serialise per wallet: the pending-registration set below is a + // read-modify-write spanning an unlocked persist. + let _provisioning = self.inner.funding_provision_locks.lock(wallet_id).await; + + #[cfg(test)] + let _overlap = self.inner.provision_overlap.enter(); + // Everything that touches wallet state happens under the manager // guard; the persist below deliberately does not. `wallet_manager()` // hands out ONE `RwLock` shared by every wallet in the // process, and it is write-preferring, so sleeping a retry backoff // under it would stall balance reads, sync callbacks and wallet // removal for every other wallet — upstream warns about exactly this. - let pending = { + let (pending, created) = { let mut wm = wallet.wallet_manager().write().await; let (kw, info) = wm .get_wallet_mut_and_info_mut(&wallet_id) @@ -612,6 +659,7 @@ impl WalletBackend { })?; } + let mut created = None; if !(in_wallet && in_managed) { let derived = funding .account(kw) @@ -625,9 +673,10 @@ impl WalletBackend { source, })?; pending.insert(funding, account_xpub); + created = Some(funding); } - pending + (pending, created) }; // In test builds the write goes through a fault-injecting decorator @@ -659,7 +708,12 @@ impl WalletBackend { .collect(); let failure = match persist_account_registrations(persister, wallet_id, &entries).await { - Ok(()) => return self.clear_staged_account_registrations(&wallet_id), + // Only the entries this call actually wrote stop being pending. + // Another call's marker is not this call's to clear. + Ok(()) => { + let written: Vec = pending.keys().copied().collect(); + return self.clear_staged_account_registrations(&wallet_id, &written); + } Err(failure) => failure, }; @@ -681,17 +735,22 @@ impl WalletBackend { } }; - if !recoverable { - // The staged changeset is gone (or unrecoverable), so every - // account it covered must leave memory — otherwise the presence - // guards short-circuit persists that never happened. - self.evict_funding_accounts(&wallet, wallet_id, &pending) + if let (false, Some(created)) = (recoverable, created) { + // Only the account this call created is known not to be durable, + // so only it leaves memory — otherwise the presence guards would + // short-circuit a persist that never happened. Entries inherited + // from an earlier attempt keep both their account and their + // marker: this failure says nothing about them, a foreign flush + // may already have committed them, and the next attempt rewrites + // them either way. Evicting on their behalf would drop an account + // that can be on disk. + self.evict_funding_account(&wallet, wallet_id, created) .await; - if let Err(e) = self.clear_staged_account_registrations(&wallet_id) { + if let Err(e) = self.clear_staged_account_registrations(&wallet_id, &[created]) { tracing::error!( wallet = %hex::encode(wallet_id), error = ?e, - "could not clear pending identity-funding registrations after a terminal persist failure" + "could not clear a pending identity-funding registration after a terminal persist failure" ); } } @@ -699,32 +758,28 @@ impl WalletBackend { Err(failure.into_task_error()) } - /// Drop funding accounts whose registrations did not reach disk from both - /// in-memory collections, so the next attempt re-creates and re-persists - /// them. + /// Drop a funding account this call created but could not persist, from + /// both in-memory collections, so the next attempt re-creates and + /// re-persists it. /// - /// Re-acquires the manager guard, which was released for the persist, and - /// only removes an account still carrying the xpub we tried to write — a - /// concurrent provisioning that re-created it has its own durable row and - /// must not be evicted on our behalf. - async fn evict_funding_accounts( + /// Re-acquires the manager guard, which was released for the persist. + /// Removing unconditionally is sound because the per-wallet provisioning + /// lock is still held, so no concurrent provisioning can have re-created + /// this account in the meantime. An xpub comparison could not establish + /// that in any case: the xpub is a deterministic derivation from the seed + /// and the account's path, so a concurrent provisioning of the same + /// account yields the identical value. + async fn evict_funding_account( &self, wallet: &Arc, wallet_id: WalletId, - pending: &std::collections::BTreeMap, + funding: Funding, ) { let mut wm = wallet.wallet_manager().write().await; let Some((kw, info)) = wm.get_wallet_mut_and_info_mut(&wallet_id) else { return; }; - for (funding, account_xpub) in pending { - if funding - .account(kw) - .is_some_and(|a| a.account_xpub == *account_xpub) - { - funding.remove(kw, info); - } - } + funding.remove(kw, info); } /// The funding accounts whose registration a previous attempt could not @@ -758,13 +813,27 @@ impl WalletBackend { Ok(()) } - /// Forget this wallet's pending registrations, once they are durable or - /// definitively lost. - fn clear_staged_account_registrations(&self, wallet_id: &WalletId) -> Result<(), TaskError> { - self.inner - .buffered_account_registrations - .lock()? - .remove(wallet_id); + /// Forget the given registrations, once they are durable or definitively + /// lost. Scoped to `keys` rather than the whole wallet: a concurrent + /// attempt's pending registration is not this call's to forget, and + /// dropping it would leave that account in memory with nothing left to + /// rewrite it. + fn clear_staged_account_registrations( + &self, + wallet_id: &WalletId, + keys: &[Funding], + ) -> Result<(), TaskError> { + let mut pending = self.inner.buffered_account_registrations.lock()?; + let std::collections::btree_map::Entry::Occupied(mut entry) = pending.entry(*wallet_id) + else { + return Ok(()); + }; + for key in keys { + entry.get_mut().remove(key); + } + if entry.get().is_empty() { + entry.remove(); + } Ok(()) } diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 842af72c5..334a27a94 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -112,6 +112,49 @@ pub use token_balance::UpstreamTokenBalances; pub use wallet_meta::WalletMetaView; pub use wallet_seed_store::WalletSeedView; +/// Test-only detector for overlapping identity-funding provisioning. +/// +/// Counts concurrent entries into the provisioning body and remembers the +/// high-water mark, so a test can assert that two calls on one wallet are +/// serialised instead of racing over the shared pending-registration set. +/// Counts globally, so tests that use it must drive a single wallet. +#[cfg(test)] +#[derive(Default)] +pub(crate) struct ProvisionOverlap { + in_flight: std::sync::atomic::AtomicUsize, + high_water: std::sync::atomic::AtomicUsize, +} + +#[cfg(test)] +impl ProvisionOverlap { + /// Record an entry; the returned guard records the matching exit. + fn enter(&self) -> ProvisionOverlapGuard<'_> { + use std::sync::atomic::Ordering; + let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1; + self.high_water.fetch_max(now, Ordering::SeqCst); + ProvisionOverlapGuard { owner: self } + } + + /// Greatest number of provisioning calls observed in flight at once. + pub(crate) fn high_water(&self) -> usize { + self.high_water.load(std::sync::atomic::Ordering::SeqCst) + } +} + +#[cfg(test)] +struct ProvisionOverlapGuard<'a> { + owner: &'a ProvisionOverlap, +} + +#[cfg(test)] +impl Drop for ProvisionOverlapGuard<'_> { + fn drop(&mut self) { + self.owner + .in_flight + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum TokenBalanceSyncOutcome { Performed, @@ -351,6 +394,11 @@ struct Inner { /// registration write. Inert until a test arms it. #[cfg(test)] persist_faults: persist_fault_test_support::PersistFaults, + #[cfg(test)] + provision_overlap: ProvisionOverlap, + /// Per-wallet serialisation for identity-funding provisioning. See + /// [`identity_ops::FundingProvisionLocks`]. + funding_provision_locks: identity_ops::FundingProvisionLocks, /// Per wallet, the identity-funding accounts whose registration a /// transient write failure could not make durable, each with the account /// xpub still owed to the manifest. The accounts stay live in memory (the @@ -585,6 +633,9 @@ impl WalletBackend { forget_wallet_local_state_test_failure: std::sync::atomic::AtomicBool::new(false), #[cfg(test)] persist_faults: persist_fault_test_support::PersistFaults::default(), + #[cfg(test)] + provision_overlap: ProvisionOverlap::default(), + funding_provision_locks: identity_ops::FundingProvisionLocks::default(), buffered_account_registrations: std::sync::Mutex::new( std::collections::BTreeMap::new(), ), @@ -1150,6 +1201,13 @@ impl WalletBackend { self.inner.persist_faults.discard_staged_before_write(write); } + /// Greatest number of identity-funding provisioning calls seen in flight at + /// once since this backend was built. + #[cfg(test)] + pub(crate) fn provisioning_high_water(&self) -> usize { + self.inner.provision_overlap.high_water() + } + /// `true` when a pending funding-account registration is recorded for this /// wallet, awaiting a rewrite by the next provisioning attempt. #[cfg(test)] From aa48aef72f28ad09acade749274082a99944d430 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:12:55 +0000 Subject: [PATCH 05/11] fix(wallet): stop a pending registration outliving the wallet it belongs to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `forget_wallet_local_state` prunes the wallet's pending funding-account registrations, but it is the synchronous cleanup path and cannot take the wallet's async provisioning lock. A provisioning call already inside its retry window therefore runs underneath the removal and records its marker afterwards, so the entry survives the wallet. A same-seed re-import recomputes the identical `WalletId` and inherits it, which is precisely what the prune exists to prevent. Recording now declines for a wallet that is gone. The check is taken while holding the pending-set lock, and removal clears `Inner::wallets` before pruning that set, which makes every interleaving safe rather than merely narrowing the window: seeing the wallet still present means the prune has not run yet and must wait for the same lock, so it removes whatever is inserted; seeing it absent means the removal already happened and there is nothing worth recording. Both halves of that pairing are documented at their respective sites, since neither is safe alone. Taking the provisioning lock in `forget_wallet_local_state` would have been the more direct fix, but that path is deliberately synchronous — its doc calls it "the synchronous secret-bearing cleanup" — and making it async to close a phantom-state gap would ripple through every caller. Confirmed RED first. The regression test drives the removal-then-record ordering directly instead of racing it, and needed a `wallet_id`-keyed probe to see the leak at all: the seed-hash form resolves through `id_map`, which the removal has already cleared, so it reports "no pending registrations" for a removed wallet whether or not one leaked. Reported by Marvin (QA-011). Co-Authored-By: Claude Opus 5 --- src/context/wallet_lifecycle/tests.rs | 53 +++++++++++++++++++++++++++ src/wallet_backend/identity_ops.rs | 43 ++++++++++++++++++++-- src/wallet_backend/mod.rs | 16 ++++++++ 3 files changed, 109 insertions(+), 3 deletions(-) diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 2172a50c4..b450b1600 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -5069,6 +5069,59 @@ async fn provisioning_two_funding_accounts_on_one_wallet_does_not_overlap() { backend.shutdown().await; } +/// A pending registration must not outlive the wallet it belongs to. +/// +/// `forget_wallet_local_state` is the synchronous cleanup path and cannot take +/// the (async) provisioning lock, so a provisioning call already inside its +/// retry window can still be running when the wallet is removed and record its +/// marker afterwards. A same-seed re-import recomputes the identical +/// `WalletId`, so that leftover would be inherited by the fresh wallet and +/// rewritten against accounts it never provisioned — exactly what the prune +/// exists to stop. Recording is therefore refused once the wallet is gone. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_pending_registration_recorded_after_wallet_removal_is_refused() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xCAu8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "outlive").await; + let wallet_id = backend.registered_wallet_id(&seed_hash).expect("wallet id"); + + // Provision once so a real account xpub is available to record. + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("provision identity funding accounts"); + let account_xpub = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("the top-up account must be live in memory") + .account_xpub; + + backend + .forget_wallet_local_state(&seed_hash, Some(wallet_id)) + .expect("forget wallet local state"); + + // The tail of a provisioning call that was still in flight when the wallet + // was removed: it reaches its bookkeeping after the prune has run. + backend + .record_pending_account_registration_for_test(wallet_id, FAULT_TOPUP_INDEX, account_xpub) + .expect("recording must not error, only decline"); + + assert!( + !backend.has_pending_account_registrations_for(wallet_id), + "a pending registration recorded after the wallet was removed outlives \ + it, and a same-seed re-import inherits it", + ); + + backend.shutdown().await; +} + /// Identity index the persist-failure tests provision a top-up account for. /// /// Per-index top-up accounts are the funding accounts DET actually creates: diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index 346c5079b..163bd78ab 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -804,15 +804,52 @@ impl WalletBackend { wallet_id: WalletId, pending: &std::collections::BTreeMap, ) -> Result<(), TaskError> { - self.inner - .buffered_account_registrations - .lock()? + let mut buffered = self.inner.buffered_account_registrations.lock()?; + + // Declining for a wallet that is gone is what keeps a pending + // registration from outliving it — `forget_wallet_local_state` is the + // synchronous cleanup path and cannot take the (async) provisioning + // lock, so a call already inside its retry window can reach this point + // after the removal has pruned. A same-seed re-import recomputes the + // identical `WalletId` and would inherit the leftover. + // + // Sound for every interleaving because removal drops the wallet from + // `Inner::wallets` BEFORE pruning here, and this liveness check is + // taken while holding the pending-set lock the prune also needs: + // seeing the wallet still present means the prune has not run yet and + // must wait for this lock, so it will remove what is inserted below; + // seeing it absent means the removal already happened and there is + // nothing worth recording. + if !self.inner.wallets.read()?.contains_key(&wallet_id) { + tracing::debug!( + wallet = %hex::encode(wallet_id), + "declining to record a pending identity-funding registration for a removed wallet" + ); + return Ok(()); + } + + buffered .entry(wallet_id) .or_default() .extend(pending.iter().map(|(f, x)| (*f, *x))); Ok(()) } + /// Record a pending funding-account registration through the real + /// bookkeeping path, so a test can drive the ordering a concurrent + /// provisioning would produce without racing it. + #[cfg(test)] + pub(crate) fn record_pending_account_registration_for_test( + &self, + wallet_id: WalletId, + registration_index: u32, + account_xpub: ExtendedPubKey, + ) -> Result<(), TaskError> { + let pending = + std::collections::BTreeMap::from([(Funding::TopUp(registration_index), account_xpub)]); + self.mark_account_registrations_staged(wallet_id, &pending) + } + /// Forget the given registrations, once they are durable or definitively /// lost. Scoped to `keys` rather than the whole wallet: a concurrent /// attempt's pending registration is not this call's to forget, and diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 334a27a94..1dc0bb109 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -1215,6 +1215,15 @@ impl WalletBackend { let Some(wallet_id) = self.registered_wallet_id(seed_hash) else { return false; }; + self.has_pending_account_registrations_for(wallet_id) + } + + /// As [`Self::has_pending_account_registrations`], but keyed by the + /// upstream id. A removed wallet is gone from `id_map`, so the seed-hash + /// form cannot see leftovers that outlived it — which is the state worth + /// asserting after a removal. + #[cfg(test)] + pub(crate) fn has_pending_account_registrations_for(&self, wallet_id: WalletId) -> bool { self.inner .buffered_account_registrations .lock() @@ -1523,6 +1532,13 @@ impl WalletBackend { // A same-seed re-import computes the same `WalletId`, so a pending // registration left behind here would be inherited by the fresh // wallet and rewritten against accounts it never provisioned. + // + // Order matters: `wallets` is cleared above BEFORE this prune, and + // `mark_account_registrations_staged` checks `wallets` while + // holding this same lock. That pairing is what stops a + // provisioning call still inside its retry window from recording a + // marker that outlives the wallet — this path is synchronous and + // cannot take the wallet's async provisioning lock. self.inner .buffered_account_registrations .lock()? From c53b266aa4fc7d480757b4480604d504005d59b1 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Fri, 21 Aug 2026 15:23:55 +0000 Subject: [PATCH 06/11] docs(wallet): record the no-nesting requirement on the pending-registration locks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `mark_account_registrations_staged` and `forget_wallet_local_state` take the pending-registration mutex and the `wallets` lock in opposite orders. That is a lock cycle on paper; it does not deadlock only because the removal path's `wallets` write guard is a statement temporary dropped at its semicolon, so the two are never held at once. Nothing said so. The existing comments documented which order each site uses — the property that makes the liveness check sound — but not the requirement that the guards must not nest, which is what keeps that order from hanging. Binding the removal's guard to a `let` spanning the prune would close the cycle, look entirely reasonable, and deadlock wallet removal against a concurrent provisioning, with no failing case on the concurrent path to catch it. Comments only; no behaviour change and no new test, since there is no failing case to pin. Reported by Marvin (QA-012). Co-Authored-By: Claude Opus 5 --- src/wallet_backend/identity_ops.rs | 4 ++++ src/wallet_backend/mod.rs | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index 163bd78ab..ebd6e9e40 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -813,6 +813,10 @@ impl WalletBackend { // after the removal has pruned. A same-seed re-import recomputes the // identical `WalletId` and would inherit the leftover. // + // The `wallets` read below is taken while this pending-set guard is + // held; removal takes the two the other way round, so it must not hold + // its `wallets` guard across its prune (noted at that site). + // // Sound for every interleaving because removal drops the wallet from // `Inner::wallets` BEFORE pruning here, and this liveness check is // taken while holding the pending-set lock the prune also needs: diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 1dc0bb109..33240eb0c 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -1527,6 +1527,12 @@ impl WalletBackend { // In-memory maps + snapshot registration. if let Some(wallet_id) = wallet_id { self.inner.id_map.write()?.remove(seed_hash); + // This guard MUST stay a statement temporary, dropped at the + // semicolon. `mark_account_registrations_staged` takes the locks + // the other way round (pending set, then a `wallets` read), so + // holding this one across the prune below — e.g. by binding it to + // a `let` that outlives this line — closes the cycle and hangs + // wallet removal against a concurrent provisioning. self.inner.wallets.write()?.remove(&wallet_id); self.inner.snapshots.forget_wallet(seed_hash, &wallet_id); // A same-seed re-import computes the same `WalletId`, so a pending From 39695b103a3b7418c7a6f008de5ce860cb4669aa Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 25 Aug 2026 07:57:42 +0000 Subject: [PATCH 07/11] fix(wallet): commit a funding-account registration in a single store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The registration persist chained a second write onto one that had already committed. DET builds its persister with `SqlitePersisterConfig::new` and never selects a flush mode, so it runs in `FlushMode::Immediate`, where a successful `store` is the durability acknowledgement. The trailing `flush` could therefore add nothing — but it operated on a buffer shared with every other writer for that wallet, so an unrelated writer's terminal failure came back as ours and classified the registration `Discarded`, evicting a funding account whose manifest row was already on disk. Drop the trailing flush, and record on `persist_account_registrations` the flush-mode assumption the single store now rests on, and where it is established. Only failures that concern our own entries can classify the registration now. The fault-injection harness documented, in three places, that faults could be armed per `store` or `flush` call. The flush arms were unreachable: the queue had no "no fault this call" placeholder, and with the trailing flush gone the decorator's sole call site issues no flush at all — so a flush-time fault is unreachable rather than merely unused, whatever shape the queue takes. Remove the dead capability, make `flush` an honest passthrough, and correct every doc that claimed otherwise. In its place the harness counts `store` and `flush` calls, which is what lets a test pin the write shape. `a_committed_registration_write_is_not_followed_by_a_second_write` asserts exactly one `store` and no `flush`, plus the account live in both collections and one durable row. Restoring the chained flush fails it on the flush count, verified by reintroducing the call. Co-Authored-By: Claude Opus 5 --- src/context/wallet_lifecycle/tests.rs | 62 ++++++++++++- src/wallet_backend/identity_ops.rs | 33 +++---- src/wallet_backend/mod.rs | 23 ++++- .../persist_fault_test_support.rs | 91 ++++++++++++------- 4 files changed, 155 insertions(+), 54 deletions(-) diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index b450b1600..dc378c7ef 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -4561,6 +4561,63 @@ async fn unbound_topup_funding_account_provisions_on_the_watch_only_wallet() { backend.shutdown().await; } +/// A committed registration write must not be followed by a second one. +/// +/// The persister runs in `FlushMode::Immediate`, so a successful `store` has +/// already committed. A follow-up `flush` would add no durability and would +/// take the per-wallet buffer *including whatever an unrelated writer parked +/// there*: that stranger's terminal failure would come back as this +/// registration's, evicting an account whose row is already on disk. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_committed_registration_write_is_not_followed_by_a_second_write() { + let dir = tempfile::tempdir().expect("tempdir"); + let (ctx, sender) = offline_testnet_context_at(dir.path()); + ctx.ensure_wallet_backend(sender) + .await + .expect("ensure_wallet_backend should succeed offline"); + let backend = ctx.wallet_backend().expect("backend wired"); + + let seed = [0xCBu8; 64]; + let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "one-write").await; + + let stores_before = backend.registration_persist_store_calls(); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("an unfaulted provisioning must succeed"); + + assert_eq!( + backend.registration_persist_store_calls() - stores_before, + 1, + "the registration must reach disk in exactly one `store`", + ); + assert_eq!( + backend.registration_persist_flush_calls(), + 0, + "a committed `store` must not be followed by a `flush` — it adds no \ + durability, and the shared buffer it would adopt lets an unrelated \ + writer's terminal failure discard this registration", + ); + + let view = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("the provisioned account must be live in memory"); + assert!(view.managed, "both collections must hold the account"); + assert!( + !backend.registration_persist_buffer_is_staged(), + "a committed write must leave nothing staged", + ); + + backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the single `store` must have made the registration durable", + ); +} + /// A transient persister failure must be ridden out, not surfaced. /// /// The persistence contract says a transient `store` keeps the changeset @@ -4628,8 +4685,9 @@ async fn an_exhausted_transient_persist_budget_keeps_memory_and_the_next_attempt let seed = [0xC2u8; 64]; let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "exhausted").await; - // One `store` plus every bare-flush retry the budget allows. Sizing the - // queue to the budget exactly leaves nothing armed for the follow-up call. + // One fault per `store` the retry budget allows — every attempt resupplies + // the entries. Sizing the queue to the budget exactly leaves nothing armed + // for the follow-up call. backend.arm_registration_persist_faults([PersistenceErrorKind::Transient; 4]); let error = backend .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index ebd6e9e40..9a19e3dc3 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -229,21 +229,24 @@ where } } -/// Write these account registrations through `persister` and flush them, so a -/// cold boot rebuilds the accounts from the manifest. +/// Write these account registrations through `persister`, so a cold boot +/// rebuilds the accounts from the manifest. /// -/// Every attempt — the first and each retry — **resupplies** the entries via -/// `store` rather than issuing a bare `flush`. The persister's buffer is -/// shared per wallet, not owned by this call site: `flush` answers `Ok(())` -/// when it finds nothing staged, and any other writer's terminal flush takes -/// the whole merged buffer (these entries included) without telling us. A bare -/// `flush` therefore cannot distinguish "committed" from "thrown away by -/// somebody else", and inferring durability from it is how a funding account -/// ends up live in memory and absent from the manifest. +/// A single `store` is the whole write. [`WalletBackend::new`] builds the +/// persister with `SqlitePersisterConfig::new` and never selects a flush mode, +/// leaving the default `FlushMode::Immediate`, under which a successful +/// `store` has already committed its transaction. A follow-up `flush` could +/// therefore add no durability, only exposure: the buffer is shared per +/// wallet, so that call would adopt whatever another writer had parked in it +/// and hand us that stranger's failure — classifying a registration whose row +/// is already on disk as discarded, and evicting its account from memory. /// -/// Resupplying is safe precisely here, and the general "re-`store` would -/// double-merge" caution does not bind: `account_registrations` merges by -/// `extend` and applies through `UPSERT_ACCOUNT_SQL`, an +/// Every attempt — the first and each retry — **resupplies** the entries +/// rather than issuing a bare `flush`, which answers `Ok(())` on an empty +/// buffer and so cannot distinguish "committed" from "thrown away by somebody +/// else". Resupplying is safe precisely here, and the general "re-`store` +/// would double-merge" caution does not bind: `account_registrations` merges +/// by `extend` and applies through `UPSERT_ACCOUNT_SQL`, an /// `ON CONFLICT(wallet_id, account_type, account_index, …) DO UPDATE` keyed on /// the account's identity, so writing the same entry twice is idempotent. async fn persist_account_registrations( @@ -256,9 +259,7 @@ async fn persist_account_registrations( account_registrations: entries.to_vec(), ..Default::default() }; - persister - .store(wallet_id, changeset) - .and_then(|()| persister.flush(wallet_id)) + persister.store(wallet_id, changeset) }) .await } diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 33240eb0c..c7aaf3259 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -1168,7 +1168,7 @@ impl WalletBackend { } /// Queue persister faults for the identity-funding account registration - /// write, served one per `store` / `flush` call in order. + /// write, served one per `store` call in order. #[cfg(test)] pub(crate) fn arm_registration_persist_faults( &self, @@ -1193,14 +1193,29 @@ impl WalletBackend { } /// Arm a one-shot foreign drain of the staged buffer immediately before the - /// `write`-th `store`/`flush` on the registration path. Write 1 is the - /// first attempt's `store`, so arming write 2 puts the drain in the - /// backoff window between the first attempt and its retry. + /// `write`-th `store` on the registration path. Write 1 is the first + /// attempt's `store`, so arming write 2 puts the drain in the backoff + /// window between the first attempt and its retry. #[cfg(test)] pub(crate) fn discard_staged_registration_buffer_before_write(&self, write: usize) { self.inner.persist_faults.discard_staged_before_write(write); } + /// `store` calls the identity-funding registration path has made since this + /// backend was built — one per persist attempt. + #[cfg(test)] + pub(crate) fn registration_persist_store_calls(&self) -> usize { + self.inner.persist_faults.store_calls() + } + + /// `flush` calls the identity-funding registration path has made since this + /// backend was built. The path makes none: the persister runs in + /// `FlushMode::Immediate`, so a successful `store` has already committed. + #[cfg(test)] + pub(crate) fn registration_persist_flush_calls(&self) -> usize { + self.inner.persist_faults.flush_calls() + } + /// Greatest number of identity-funding provisioning calls seen in flight at /// once since this backend was built. #[cfg(test)] diff --git a/src/wallet_backend/persist_fault_test_support.rs b/src/wallet_backend/persist_fault_test_support.rs index 69835c014..a573db77d 100644 --- a/src/wallet_backend/persist_fault_test_support.rs +++ b/src/wallet_backend/persist_fault_test_support.rs @@ -2,12 +2,12 @@ //! [`PlatformWalletPersistence`] retry contract. //! //! Decorates the real [`SqlitePersister`] so a test can make individual -//! `store` / `flush` calls fail with a chosen [`PersistenceErrorKind`] while -//! every other call reaches the real database. The injected failures mirror -//! what the real persister does with the buffer: a `Transient` failure parks -//! the changeset so a later write still commits it, and a terminal failure -//! drops the whole per-wallet buffer — including anything a previous transient -//! failure had staged, because `flush_inner` takes the merged buffer and +//! `store` calls fail with a chosen [`PersistenceErrorKind`] while every other +//! call reaches the real database. The injected failures mirror what the real +//! persister does with the buffer: a `Transient` failure parks the changeset +//! so a later write still commits it, and a terminal failure drops the whole +//! per-wallet buffer — including anything a previous transient failure had +//! staged, because `flush_inner` takes the merged buffer and //! `handle_flush_error` restores it only for transient errors. Assertions can //! therefore be made against real persisted rows rather than a fake's //! bookkeeping. @@ -17,9 +17,17 @@ //! two-wallet test commit one wallet's changeset under another wallet's id and //! still pass. //! -//! Only `store`, `flush` and `load` are routed — the decorator is used at one -//! call site (the identity-funding account registration write) and the trait's -//! defaults cover the rest. +//! Only `store` faults, and only `store`, `flush` and `load` are routed at all +//! — the trait's defaults cover the rest. The decorator has a single call site, +//! the identity-funding account registration write, and that path issues one +//! `store` per attempt and never flushes: DET runs the persister in +//! `FlushMode::Immediate`, where a successful `store` is already the commit. A +//! flush-time fault would therefore be unreachable rather than merely unused, +//! so `flush` is a plain passthrough — it still completes a changeset an +//! injected transient failure parked, per the trait's contract — and +//! [`store_calls`](PersistFaults::store_calls) / +//! [`flush_calls`](PersistFaults::flush_calls) are what a test pins that shape +//! with. use std::collections::{HashMap, VecDeque}; use std::sync::Mutex; @@ -39,13 +47,15 @@ use platform_wallet_storage::SqlitePersister; pub(crate) struct PersistFaults { queue: Mutex>, staged: Mutex>, - /// One-based index of the `store`/`flush` call before which the staged - /// buffer is dropped; `0` disables the probe. + /// One-based index of the `store` call before which the staged buffer is + /// dropped; `0` disables the probe. discard_before_write: std::sync::atomic::AtomicUsize, + store_calls: std::sync::atomic::AtomicUsize, + flush_calls: std::sync::atomic::AtomicUsize, } impl PersistFaults { - /// Queue `kinds`, served one per `store` / `flush` call in order. + /// Queue `kinds`, served one per `store` call in order. pub(crate) fn arm(&self, kinds: impl IntoIterator) { self.queue.lock().expect("fault queue").extend(kinds); } @@ -69,11 +79,11 @@ impl PersistFaults { self.staged.lock().expect("staged changeset").clear(); } - /// Arm a one-shot foreign drain immediately before the `write`-th - /// `store`/`flush` call, as if an unrelated writer had taken the shared - /// buffer and lost it terminally in that window. Counting calls (rather - /// than firing on the next one) is what places the drain *between* two - /// attempts of one retry loop. + /// Arm a one-shot foreign drain immediately before the `write`-th `store` + /// call, as if an unrelated writer had taken the shared buffer and lost it + /// terminally in that window. Counting calls (rather than firing on the + /// next one) is what places the drain *between* two attempts of one retry + /// loop. pub(crate) fn discard_staged_before_write(&self, write: usize) { self.discard_before_write .store(write, std::sync::atomic::Ordering::Relaxed); @@ -98,6 +108,30 @@ impl PersistFaults { self.queue.lock().expect("fault queue").pop_front() } + /// `store` calls the registration path has made since this backend was + /// built. One per persist attempt. + pub(crate) fn store_calls(&self) -> usize { + self.store_calls.load(std::sync::atomic::Ordering::Relaxed) + } + + /// `flush` calls the registration path has made since this backend was + /// built. Zero unless a follow-up write has crept back in: a committed + /// `store` needs none, and one would expose the registration to an + /// unrelated writer's failure. + pub(crate) fn flush_calls(&self) -> usize { + self.flush_calls.load(std::sync::atomic::Ordering::Relaxed) + } + + fn record_store(&self) { + self.store_calls + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + + fn record_flush(&self) { + self.flush_calls + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + fn stage(&self, wallet_id: WalletId, changeset: PlatformWalletChangeSet) { self.staged .lock() @@ -142,6 +176,7 @@ impl PlatformWalletPersistence for PersistFaultInjector<'_> { wallet_id: WalletId, changeset: PlatformWalletChangeSet, ) -> Result<(), PersistenceError> { + self.faults.record_store(); self.faults.apply_pending_discard(); match self.faults.next() { // Contract: a transient failure preserves the changeset, so a @@ -166,23 +201,15 @@ impl PlatformWalletPersistence for PersistFaultInjector<'_> { } } + /// Passthrough — the queue serves `store` only. The registration path + /// makes no `flush` call, so faulting one would test nothing; the counter + /// is what a test asserts on instead. fn flush(&self, wallet_id: WalletId) -> Result<(), PersistenceError> { - self.faults.apply_pending_discard(); - match self.faults.next() { - Some(PersistenceErrorKind::Transient) => { - Err(Self::injected(PersistenceErrorKind::Transient)) - } - Some(kind) => { - self.faults.take_staged(&wallet_id); - Err(Self::injected(kind)) - } - None => { - if let Some(staged) = self.faults.take_staged(&wallet_id) { - self.inner.store(wallet_id, staged)?; - } - self.inner.flush(wallet_id) - } + self.faults.record_flush(); + if let Some(staged) = self.faults.take_staged(&wallet_id) { + self.inner.store(wallet_id, staged)?; } + self.inner.flush(wallet_id) } fn load(&self) -> Result { From ad7997929676d16c52bc6ffcbebf546f88b5ef58 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 25 Aug 2026 08:07:47 +0000 Subject: [PATCH 08/11] test(wallet): name the upstream default the single-store count rests on `ensure_identity_funding_accounts` provisions the registration account as well as the top-up one, and the store count of 1 holds only because `key-wallet` already creates `IdentityRegistration` in a wallet's default account set, so that half early-returns without a write. Were that default to change, the count would become 2 and the failure would read as DET writing twice. Assert the precondition before the call, so the red run names the upstream default instead of the write count downstream of it. Co-Authored-By: Claude Opus 5 --- src/context/wallet_lifecycle/tests.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index dc378c7ef..7b35c52e3 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -4580,6 +4580,21 @@ async fn a_committed_registration_write_is_not_followed_by_a_second_write() { let seed = [0xCBu8; 64]; let (seed_hash, _wallet_arc) = register_test_wallet(&ctx, &backend, seed, "one-write").await; + assert!( + backend + .identity_funding_account( + &seed_hash, + dash_sdk::dpp::key_wallet::AccountType::IdentityRegistration, + ) + .await + .expect("probe the registration account") + .is_some(), + "`key-wallet` must already hold the identity-registration account from \ + a wallet's default set. Without it the call below provisions two \ + accounts, and the `store` count it asserts measures upstream's \ + defaults rather than DET's write shape", + ); + let stores_before = backend.registration_persist_store_calls(); backend .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) From 243ce7c6fe9ca2642c5b6cf2cefcda7fee7376e6 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:10:10 +0000 Subject: [PATCH 09/11] fix(wallet): ask the persister whether a store is already the commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The funding-account registration write issued one `store` and no `flush` because DET happens to build its persister in `FlushMode::Immediate` — a convention held up by prose, which the day someone selects another flush mode at the construction site would silently downgrade a reported-durable registration to a buffered one. The pin at `4784de03` exposes exactly that property as `PlatformWalletPersistence::store_commits_inline()`, so ask it: an inline-committing backend keeps the single-store shape, and a buffering one is flushed inside the same attempt, where a failure at either step is one failure of one attempt. No unconditional flush returns — an already-committed `store` must not adopt a shared per-wallet buffer whose stranger's terminal failure would evict an account whose row is on disk. The fault-injecting test decorator forwards the answer instead of inheriting the trait's conservative `false`, so the regression test still measures DET's real write shape, and now says so. Co-Authored-By: Claude Opus 5 --- src/context/wallet_lifecycle/tests.rs | 8 +- src/wallet_backend/identity_ops.rs | 148 ++++++++++++++++-- src/wallet_backend/mod.rs | 13 +- .../persist_fault_test_support.rs | 18 ++- 4 files changed, 170 insertions(+), 17 deletions(-) diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 5293f2d31..8b48297e6 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -5822,7 +5822,7 @@ async fn unbound_topup_funding_account_provisions_on_the_watch_only_wallet() { /// A committed registration write must not be followed by a second one. /// -/// The persister runs in `FlushMode::Immediate`, so a successful `store` has +/// DET's persister answers `store_commits_inline`, so a successful `store` has /// already committed. A follow-up `flush` would add no durability and would /// take the per-wallet buffer *including whatever an unrelated writer parked /// there*: that stranger's terminal failure would come back as this @@ -5854,6 +5854,12 @@ async fn a_committed_registration_write_is_not_followed_by_a_second_write() { defaults rather than DET's write shape", ); + assert!( + backend.registration_persist_commits_inline(), + "DET's persister must report `store` as the commit. A buffering backend \ + would have to flush, and the flush count asserted below would be 1", + ); + let stores_before = backend.registration_persist_store_calls(); backend .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index 3fe5c3e44..dfc041082 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -237,14 +237,22 @@ where /// Write these account registrations through `persister`, so a cold boot /// rebuilds the accounts from the manifest. /// -/// A single `store` is the whole write. [`WalletBackend::new`] builds the -/// persister with `SqlitePersisterConfig::new` and never selects a flush mode, -/// leaving the default `FlushMode::Immediate`, under which a successful -/// `store` has already committed its transaction. A follow-up `flush` could -/// therefore add no durability, only exposure: the buffer is shared per -/// wallet, so that call would adopt whatever another writer had parked in it -/// and hand us that stranger's failure — classifying a registration whose row -/// is already on disk as discarded, and evicting its account from memory. +/// A `store` the backend answers +/// [`store_commits_inline`](PlatformWalletPersistence::store_commits_inline) +/// for is the whole write, and no `flush` follows it. Such a `store` has +/// already committed its transaction, so the follow-up could add no +/// durability, only exposure: the buffer is shared per wallet, so that call +/// would adopt whatever another writer had parked in it and hand us that +/// stranger's failure — classifying a registration whose row is already on +/// disk as discarded, and evicting its account from memory. DET's SQLite +/// persister answers `true` (it runs in `FlushMode::Immediate`), which is +/// asked here rather than assumed, so selecting another flush mode at the +/// construction site changes the write shape instead of silently costing the +/// durability this function reports. +/// +/// A backend that only buffers is flushed in the same attempt: there, nothing +/// is durable until the flush, and a failure at either step is one failure of +/// one attempt — retried and classified together. /// /// Every attempt — the first and each retry — **resupplies** the entries /// rather than issuing a bare `flush`, which answers `Ok(())` on an empty @@ -264,7 +272,11 @@ async fn persist_account_registrations( account_registrations: entries.to_vec(), ..Default::default() }; - persister.store(wallet_id, changeset) + persister.store(wallet_id, changeset)?; + if persister.store_commits_inline() { + return Ok(()); + } + persister.flush(wallet_id) }) .await } @@ -1178,3 +1190,121 @@ impl WalletBackend { .await } } + +#[cfg(test)] +mod tests { + use super::*; + use platform_wallet::changeset::ClientStartState; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Counts the writes [`persist_account_registrations`] issues, answering + /// the inline-commit question however the test asks it to. + struct CountingPersister { + commits_inline: bool, + stores: AtomicUsize, + flushes: AtomicUsize, + } + + impl CountingPersister { + fn new(commits_inline: bool) -> Self { + Self { + commits_inline, + stores: AtomicUsize::new(0), + flushes: AtomicUsize::new(0), + } + } + } + + impl PlatformWalletPersistence for CountingPersister { + fn store_commits_inline(&self) -> bool { + self.commits_inline + } + + fn store( + &self, + _wallet_id: WalletId, + _changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + self.stores.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { + self.flushes.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn load(&self) -> Result { + unimplemented!("the registration write never loads") + } + } + + fn one_entry() -> Vec { + // A BIP32 serialized xpub: version, depth, parent fingerprint, child + // number, chain code, then the compressed point (secp256k1's + // generator, so the key itself is valid). + let mut raw = Vec::with_capacity(78); + raw.extend_from_slice(&[0x04, 0x88, 0xB2, 0x1E]); + raw.push(0); + raw.extend_from_slice(&[0u8; 4]); + raw.extend_from_slice(&[0u8; 4]); + raw.extend_from_slice(&[1u8; 32]); + raw.extend_from_slice(&[ + 0x02, 0x79, 0xBE, 0x66, 0x7E, 0xF9, 0xDC, 0xBB, 0xAC, 0x55, 0xA0, 0x62, 0x95, 0xCE, + 0x87, 0x0B, 0x07, 0x02, 0x9B, 0xFC, 0xDB, 0x2D, 0xCE, 0x28, 0xD9, 0x59, 0xF2, 0x81, + 0x5B, 0x16, 0xF8, 0x17, 0x98, + ]); + let xpub = ExtendedPubKey::decode(&raw).expect("decode a well-formed xpub"); + vec![AccountRegistrationEntry { + account_type: dash_sdk::dpp::key_wallet::AccountType::IdentityRegistration, + account_xpub: xpub, + }] + } + + /// A backend whose `store` commits inline gets exactly one write. The + /// follow-up `flush` would add no durability, and the per-wallet buffer it + /// adopts would let an unrelated writer's terminal failure discard a + /// registration already on disk. + #[tokio::test] + async fn an_inline_committing_store_is_the_whole_write() { + let persister = CountingPersister::new(true); + persist_account_registrations(&persister, [7u8; 32], &one_entry()) + .await + .map_err(|failure| failure.into_task_error()) + .expect("an unfaulted write must succeed"); + + assert_eq!( + persister.stores.load(Ordering::SeqCst), + 1, + "the registration must reach disk in exactly one `store`", + ); + assert_eq!( + persister.flushes.load(Ordering::SeqCst), + 0, + "a `store` that commits inline must not be followed by a `flush`", + ); + } + + /// A backend that only buffers is flushed, in the same attempt: without + /// it the registration would be reported durable while still in memory. + #[tokio::test] + async fn a_buffering_store_is_completed_by_a_flush() { + let persister = CountingPersister::new(false); + persist_account_registrations(&persister, [7u8; 32], &one_entry()) + .await + .map_err(|failure| failure.into_task_error()) + .expect("an unfaulted write must succeed"); + + assert_eq!( + persister.stores.load(Ordering::SeqCst), + 1, + "the entries must be supplied once per attempt", + ); + assert_eq!( + persister.flushes.load(Ordering::SeqCst), + 1, + "nothing is durable on a buffering backend until the flush that \ + completes the same attempt", + ); + } +} diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 6cad20ef6..937b12751 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -1250,13 +1250,22 @@ impl WalletBackend { } /// `flush` calls the identity-funding registration path has made since this - /// backend was built. The path makes none: the persister runs in - /// `FlushMode::Immediate`, so a successful `store` has already committed. + /// backend was built. The path makes none while + /// [`Self::registration_persist_commits_inline`] holds: a successful + /// `store` has already committed. #[cfg(test)] pub(crate) fn registration_persist_flush_calls(&self) -> usize { self.inner.persist_faults.flush_calls() } + /// Whether DET's persister reports a successful `store` as already + /// committed — the property the flush-free registration write rests on. + #[cfg(test)] + pub(crate) fn registration_persist_commits_inline(&self) -> bool { + use platform_wallet::changeset::PlatformWalletPersistence; + self.inner.wallet_persister.store_commits_inline() + } + /// Greatest number of identity-funding provisioning calls seen in flight at /// once since this backend was built. #[cfg(test)] diff --git a/src/wallet_backend/persist_fault_test_support.rs b/src/wallet_backend/persist_fault_test_support.rs index a573db77d..07f4f3df1 100644 --- a/src/wallet_backend/persist_fault_test_support.rs +++ b/src/wallet_backend/persist_fault_test_support.rs @@ -17,11 +17,12 @@ //! two-wallet test commit one wallet's changeset under another wallet's id and //! still pass. //! -//! Only `store` faults, and only `store`, `flush` and `load` are routed at all -//! — the trait's defaults cover the rest. The decorator has a single call site, -//! the identity-funding account registration write, and that path issues one -//! `store` per attempt and never flushes: DET runs the persister in -//! `FlushMode::Immediate`, where a successful `store` is already the commit. A +//! Only `store` faults, and only `store_commits_inline`, `store`, `flush` and +//! `load` are routed at all — the trait's defaults cover the rest. The +//! decorator has a single call site, the identity-funding account registration +//! write, and that path issues one `store` per attempt and never flushes: the +//! decorated persister answers `store_commits_inline`, so a successful `store` +//! is already the commit. A //! flush-time fault would therefore be unreachable rather than merely unused, //! so `flush` is a plain passthrough — it still completes a changeset an //! injected transient failure parked, per the trait's contract — and @@ -167,6 +168,13 @@ impl<'a> PersistFaultInjector<'a> { } impl PlatformWalletPersistence for PersistFaultInjector<'_> { + /// Forwarded, not defaulted: the trait's conservative `false` would put + /// the registration path on its buffered-backend branch and test a write + /// shape DET never runs. + fn store_commits_inline(&self) -> bool { + self.inner.store_commits_inline() + } + fn persistence_capabilities(&self) -> PersistenceCapabilities { self.inner.persistence_capabilities() } From 29722d0304403a2c1a9f9aec861b51ac5b8dc3d1 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 25 Aug 2026 09:20:24 +0000 Subject: [PATCH 10/11] docs(changelog): record the identity funding-account persistence fix The branch had no Unreleased entry, unlike the sibling wallet fixes it lands alongside. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013y5kGhnD5WwwL3YKEFi7Av --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 479fbf804..eb454f06c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed +- **Restarting no longer loses the account an identity's funding is paid + through**: creating an identity, or topping one up, first sets up an account + on this device for the payment to travel through. That account was only ever + held in memory and never written down, so closing the app between sending the + payment and the identity appearing left nothing able to work out where the + money had gone — the payment was stranded, and the funds in it with it. This + is also what could leave a wallet unusable after updating the app. The + account is now written down as it is created, so it comes back exactly as it + was the next time the app starts. If writing it down does not succeed, the + app keeps trying; it only gives up when the write can never land, and then it + says so plainly instead of carrying on as though the account were there. + - **Masternodes and evonodes are now recorded in the wallet store too**: a node loaded from its ProTxHash was only ever written to this app's own records, so the wallet store the app shares with the rest of the wallet stack had no entry From ec87d7f64d2ea92ea12ea58415c7ca9de3a66aeb Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Tue, 8 Sep 2026 11:26:27 +0000 Subject: [PATCH 11/11] fix(wallet): retain funding accounts after ambiguous persistence retries Keep the account and pending rewrite when a terminal retry follows a transient failure, since another writer may already have committed it. Cover both committed and discarded shared buffers with regression tests. Co-Authored-By: Codex --- src/context/wallet_lifecycle/tests.rs | 67 +++++++++++++------ src/wallet_backend/identity_ops.rs | 20 +++--- src/wallet_backend/mod.rs | 24 +++---- .../persist_fault_test_support.rs | 16 +++++ 4 files changed, 81 insertions(+), 46 deletions(-) diff --git a/src/context/wallet_lifecycle/tests.rs b/src/context/wallet_lifecycle/tests.rs index 0ae322377..978326abc 100644 --- a/src/context/wallet_lifecycle/tests.rs +++ b/src/context/wallet_lifecycle/tests.rs @@ -6292,15 +6292,17 @@ async fn a_constraint_registration_persist_failure_rolls_back_the_in_memory_acco backend.shutdown().await; } -/// A write that starts transient and then fails terminally is terminal: the -/// staged changeset is gone, so the account must be rolled back rather than -/// kept on the strength of the first, softer error. -/// -/// This is the one ordering where misclassifying strands funds — keeping an -/// account whose registration was discarded means the next attempt -/// early-returns success on a row that will never exist. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn a_transient_write_that_escalates_to_a_terminal_one_rolls_back() { +async fn a_terminal_retry_preserves_a_registration_committed_by_another_writer() { + assert_terminal_retry_preserves_registration(true).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terminal_retry_keeps_a_discarded_registration_pending_for_rewrite() { + assert_terminal_retry_preserves_registration(false).await; +} + +async fn assert_terminal_retry_preserves_registration(foreign_commit: bool) { let dir = tempfile::tempdir().expect("tempdir"); let (ctx, sender) = offline_testnet_context_at(dir.path()); ctx.ensure_wallet_backend(sender) @@ -6316,29 +6318,56 @@ async fn a_transient_write_that_escalates_to_a_terminal_one_rolls_back() { PersistenceErrorKind::Transient, PersistenceErrorKind::Fatal, ]); - backend + if foreign_commit { + backend.commit_staged_registration_buffer_before_write(2); + } + let error = backend .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) .await .expect_err("the escalated failure must surface"); - - assert!( - backend - .identity_funding_account(&seed_hash, fault_topup_account_type()) - .await - .expect("probe the funding account") - .is_none(), - "escalation to a terminal failure must roll the account back", + assert!(matches!( + error, + TaskError::IdentityFundingAccountPersistFailed { .. } + )); + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + i64::from(foreign_commit), + "the foreign writer determines whether the row is already durable", ); + + let account = backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .expect("probe the funding account") + .expect("an earlier transient failure makes durability ambiguous"); + assert!(account.managed); assert!( !backend.registration_persist_buffer_is_staged(), "a terminal failure must take the staged changeset with it", ); assert!( - !backend.has_pending_account_registrations(&seed_hash), - "a terminal failure must leave nothing pending to rewrite", + backend.has_pending_account_registrations(&seed_hash), + "an ambiguous write must remain pending for an idempotent rewrite", ); + backend + .ensure_identity_funding_accounts(&seed_hash, &seed, FAULT_TOPUP_INDEX) + .await + .expect("the next attempt must rewrite the pending registration"); + assert!(!backend.has_pending_account_registrations(&seed_hash)); + assert_eq!( + backend + .identity_funding_account(&seed_hash, fault_topup_account_type()) + .await + .unwrap(), + Some(account), + ); backend.shutdown().await; + assert_eq!( + persisted_account_registration_rows(dir.path(), "identity_topup", FAULT_TOPUP_INDEX), + 1, + "the rewrite must leave exactly one durable registration", + ); } /// Removing a wallet must drop its pending registrations. diff --git a/src/wallet_backend/identity_ops.rs b/src/wallet_backend/identity_ops.rs index dfc041082..103b27f6c 100644 --- a/src/wallet_backend/identity_ops.rs +++ b/src/wallet_backend/identity_ops.rs @@ -177,17 +177,12 @@ impl FundingProvisionLocks { } } -/// Why an account registration did not reach disk — and therefore what the -/// caller must do with the account it just put in memory. +/// Whether a failed registration write permits removing the account from memory. enum RegistrationPersistFailure { - /// The retry budget ran out while the failure was still transient. The - /// changeset may still be buffered and land later, so the account MUST - /// stay in memory — dropping it while the row can still appear would leave - /// a manifest the live wallet disagrees with. The registration is recorded - /// as pending and rewritten by the next attempt. + /// A transient attempt exposed the registration to the shared buffer. + /// It may already be durable; retain the account and its pending rewrite. Staged(PersistenceError), - /// A terminal failure discarded the staged changeset. The row will never - /// land, so the account must leave memory. + /// The first attempt failed terminally, without an earlier buffered write. Discarded(PersistenceError), } @@ -214,8 +209,11 @@ where loop { match op() { Ok(()) => return Ok(()), - Err(e) if !e.is_transient() => return Err(RegistrationPersistFailure::Discarded(e)), - Err(e) if attempt >= PERSIST_RETRY_MAX_ATTEMPTS => { + Err(e) if !e.is_transient() && attempt == 1 => { + return Err(RegistrationPersistFailure::Discarded(e)); + } + // A previous transient attempt may have been committed by another writer. + Err(e) if !e.is_transient() || attempt >= PERSIST_RETRY_MAX_ATTEMPTS => { return Err(RegistrationPersistFailure::Staged(e)); } Err(e) => { diff --git a/src/wallet_backend/mod.rs b/src/wallet_backend/mod.rs index 1205654c3..f11fa6aee 100644 --- a/src/wallet_backend/mod.rs +++ b/src/wallet_backend/mod.rs @@ -399,22 +399,8 @@ struct Inner { /// Per-wallet serialisation for identity-funding provisioning. See /// [`identity_ops::FundingProvisionLocks`]. funding_provision_locks: identity_ops::FundingProvisionLocks, - /// Per wallet, the identity-funding accounts whose registration a - /// transient write failure could not make durable, each with the account - /// xpub still owed to the manifest. The accounts stay live in memory (the - /// rows can still land), so the next provisioning attempt for that wallet - /// rewrites them instead of taking the idempotent early return and - /// reporting success on rows that are not there. - /// - /// Carries the xpub, not just the account identity, so the rewrite - /// resupplies the entry rather than betting on a shared persister buffer - /// this call site does not own — and so a terminal failure only evicts an - /// account still carrying the xpub whose write was lost. - /// - /// Entries are dropped when the write succeeds, when it fails terminally, - /// and when the wallet is removed ([`Self::forget_wallet_local_state`]) — - /// a stale entry would otherwise be inherited by a same-seed re-import, - /// which computes the same `WalletId`. + /// Registrations with uncertain durability, rewritten before funding can proceed. + /// Retained after terminal retries too; cleared on successful write or wallet removal. buffered_account_registrations: std::sync::Mutex< std::collections::BTreeMap< WalletId, @@ -1242,6 +1228,12 @@ impl WalletBackend { self.inner.persist_faults.discard_staged_before_write(write); } + /// Commit an injected pending registration before the selected retry. + #[cfg(test)] + pub(crate) fn commit_staged_registration_buffer_before_write(&self, write: usize) { + self.inner.persist_faults.commit_staged_before_write(write); + } + /// `store` calls the identity-funding registration path has made since this /// backend was built — one per persist attempt. #[cfg(test)] diff --git a/src/wallet_backend/persist_fault_test_support.rs b/src/wallet_backend/persist_fault_test_support.rs index 07f4f3df1..128997d59 100644 --- a/src/wallet_backend/persist_fault_test_support.rs +++ b/src/wallet_backend/persist_fault_test_support.rs @@ -51,6 +51,7 @@ pub(crate) struct PersistFaults { /// One-based index of the `store` call before which the staged buffer is /// dropped; `0` disables the probe. discard_before_write: std::sync::atomic::AtomicUsize, + commit_before_write: std::sync::atomic::AtomicUsize, store_calls: std::sync::atomic::AtomicUsize, flush_calls: std::sync::atomic::AtomicUsize, } @@ -90,6 +91,12 @@ impl PersistFaults { .store(write, std::sync::atomic::Ordering::Relaxed); } + /// Commit the staged buffer before the selected retry, as a foreign writer would. + pub(crate) fn commit_staged_before_write(&self, write: usize) { + self.commit_before_write + .store(write, std::sync::atomic::Ordering::Relaxed); + } + /// Count this write and, if it is the armed one, drop the staged buffer. fn apply_pending_discard(&self) { let remaining = self @@ -186,6 +193,15 @@ impl PlatformWalletPersistence for PersistFaultInjector<'_> { ) -> Result<(), PersistenceError> { self.faults.record_store(); self.faults.apply_pending_discard(); + if self.faults.commit_before_write.fetch_update( + std::sync::atomic::Ordering::Relaxed, + std::sync::atomic::Ordering::Relaxed, + |remaining| remaining.checked_sub(1), + ) == Ok(1) + && let Some(staged) = self.faults.take_staged(&wallet_id) + { + self.inner.store(wallet_id, staged)?; + } match self.faults.next() { // Contract: a transient failure preserves the changeset, so a // later write still commits it.