From 1ac758ec557e1716957e0b4fc4c52c35d843c186 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 14 Nov 2026 00:28:03 +0700 Subject: [PATCH 1/7] fix(platform-wallet): trust a stored record only when no other flush dropped it Five places store a record, flush it on a backend that does not commit a store inline, and then act as if the record were on disk: the address reservation a contact payment stores before its broadcast, the address `reserve_payment_address` hands out, the invitation funding index, the row of a funding that pays a shielded recipient, and the record of a funded invitation voucher. A flush that succeeds proves nothing by itself: another flush of the wallet's records may take what was buffered and fail for good, dropping it, and the site's own flush then succeeds over a buffer that no longer holds the record. A restart would then pay a contact's address twice, export an invitation's one-time key twice, lose the address a shielded funding pays, or lose track of a funded voucher. Each of them now goes through `WalletPersister::store_durably`, which answers for the record only when no other flush can have dropped it. The sites that have sent nothing yet abandon what they built, as they do for a failed flush. The voucher's record is stored once more first, as the voucher is already funded, and only then reported as not persisted, with no link. `NotDurable::Backend` is split into `Store` and `Flush`, so a site keeps reporting a failed store and a failed flush the way it did. Wallet drains, the consumption-unknown marker, the cleared private payment marker, the balance retry and the manager's backfill replace are left as they are: a drain runs only where a store is the commit, and a record lost at the others is repaired by a later sync or fails closed. Co-Authored-By: Claude Fable 5.2 --- .../src/wallet/asset_lock/build.rs | 203 +++++++++++-- .../src/wallet/identity/network/invitation.rs | 257 +++++++++++++++-- .../src/wallet/identity/network/payments.rs | 269 +++++++++++++++++- .../identity/network/prepared_payment.rs | 44 ++- .../src/wallet/persister.rs | 19 +- 5 files changed, 721 insertions(+), 71 deletions(-) diff --git a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs index 57a6ae3b46a..9a2ed737f02 100644 --- a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs +++ b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs @@ -1383,17 +1383,30 @@ impl AssetLockManager { // `store()` alone is only a buffer hint under the persistence contract // (backends may defer I/O until `flush`), so the invitation gate also // drives `flush()` — the contract's durability boundary — before - // anything hits the wire. + // anything hits the wire. That flush succeeding is no proof by itself: + // another flush of the wallet's records may have taken the buffered + // pool and failed for good, dropping it, so the gate trusts only a + // store no other flush can have emptied + // (`WalletPersister::store_durably`). // Aborting BEFORE broadcast is harmless (no tx on the wire); the other // asset-lock accounts keep their keys on-device, so they stay best-effort. - let pool_durability = match self.persist_asset_lock_account_pools(funding_pools) { - Ok(()) - if funding_type == AssetLockFundingType::IdentityInvitation - || operation.is_some() => - { - self.persister.flush() - } - other => other, + let pool_durability = if operation.is_some() { + // A drain runs only where a store is the commit + // (`require_wallet_drain_persistence`), so no flush can drop its + // pools: the flush here only notifies the host. + self.persist_asset_lock_account_pools(funding_pools) + .and_then(|()| self.persister.flush()) + .map_err(|e| e.to_string()) + } else if funding_type == AssetLockFundingType::IdentityInvitation { + self.persister + .store_durably(PlatformWalletChangeSet { + account_address_pools: funding_pools, + ..Default::default() + }) + .map_err(|e| e.to_string()) + } else { + self.persist_asset_lock_account_pools(funding_pools) + .map_err(|e| e.to_string()) }; if let Err(e) = pool_durability { tracing::error!(error = %e, "failed to persist asset-lock funding index"); @@ -1503,17 +1516,19 @@ impl AssetLockManager { } let mut changeset = AssetLockChangeSet::default(); changeset.asset_locks.insert(out_point, (&tracked).into()); - let durable = self - .persister - .store(PlatformWalletChangeSet { - asset_locks: Some(changeset), - ..Default::default() - }) - .and_then(|()| self.persister.flush()); + // Durable only once no other flush of the wallet's records can + // have dropped the row (`WalletPersister::store_durably`): a flush + // that succeeded over a buffer another one emptied would send a + // lock whose recipient is on no disk. + let durable = self.persister.store_durably(PlatformWalletChangeSet { + asset_locks: Some(changeset), + ..Default::default() + }); if let Err(error) = durable { // The row may be buffered (a backend that restores a failed // flush to its buffer) or committed (a store that succeeded - // before a failed flush), and a later flush or a restart would + // before a failed flush, or one a flush still in progress + // took), and a later flush or a restart would // then surface a resumable row for inputs released below. Its // removal is queued behind it — stores apply removals after // upserts — so the abandoned row does not outlive this call @@ -1973,6 +1988,16 @@ mod tests { committed_stores: std::sync::atomic::AtomicUsize, fail_lock_store: std::sync::atomic::AtomicBool, failed_lock_stores: Mutex>, + /// Whether the backend also attests `ASSET_LOCK_SHIELDED_RECIPIENTS`. + records_shielded_recipients: bool, + /// Once set, the next flush is overtaken: before it commits, a flush + /// through this persister fails for good, as one from another thread + /// would, dropping what was buffered; the overtaken flush then + /// succeeds over nothing. + overtaken_by: Mutex>, + /// While set, a flush fails for good: the backend drops what it + /// buffered. + flush_drops_buffer: std::sync::atomic::AtomicBool, } impl CapturingPersistence { @@ -2021,7 +2046,14 @@ mod tests { } fn persistence_capabilities(&self) -> crate::changeset::PersistenceCapabilities { - crate::changeset::PersistenceCapabilities::WALLET_DRAIN + let drain = crate::changeset::PersistenceCapabilities::WALLET_DRAIN; + if self.records_shielded_recipients { + drain.union( + crate::changeset::PersistenceCapabilities::ASSET_LOCK_SHIELDED_RECIPIENTS, + ) + } else { + drain + } } fn store( @@ -2073,6 +2105,23 @@ mod tests { .flushes .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + // Taken in its own statement: the flush below comes back here. + let overtaking = self.overtaken_by.lock().unwrap().take(); + if let Some(persister) = overtaking { + self.flush_drops_buffer + .store(true, std::sync::atomic::Ordering::SeqCst); + let overtook = persister.flush(); + self.flush_drops_buffer + .store(false, std::sync::atomic::Ordering::SeqCst); + assert!(overtook.is_err_and(|error| !error.is_transient())); + return Ok(()); + } + if self + .flush_drops_buffer + .load(std::sync::atomic::Ordering::SeqCst) + { + return Err(PersistenceError::backend("the database is corrupt")); + } if self.fail_flush || self.fail_on_flush == Some(count) { return Err(PersistenceError::backend("simulated flush failure")); } @@ -3443,6 +3492,124 @@ mod tests { ); } + /// The invitation gate does not trust a flush that succeeded over + /// nothing: another flush of the wallet's records took the buffered + /// funding pool first and failed for good, dropping it, so the funding + /// index is on no disk and a restart would export the same voucher key + /// again. The build is abandoned before anything is broadcast, and its + /// inputs come back for the next one. + #[tokio::test] + async fn should_abort_an_invitation_whose_funding_index_another_flush_dropped() { + let persistence = Arc::new(CapturingPersistence::default()); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + let refused = manager + .broadcast_funded_asset_lock( + 1_000_000, + 0, + AssetLockFundingType::IdentityInvitation, + 0, + &signer, + ) + .await; + assert!( + matches!(&refused, Err(PlatformWalletError::AssetLockTransaction(message)) + if message.contains("aborted before broadcast")), + "{refused:?}" + ); + assert!( + persistence.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + + manager + .broadcast_funded_asset_lock( + 1_000_000, + 0, + AssetLockFundingType::IdentityInvitation, + 0, + &signer, + ) + .await + .expect("the abandoned build released its inputs, and this pool is durable"); + assert_eq!(broadcaster.calls(), 1); + } + + /// A funding that pays a shielded recipient is not sent when another + /// flush of the wallet's records took its row first and failed for good, + /// though the row's own flush then succeeded, over nothing: the recipient + /// is recorded nowhere else, so a lock recovered after a kill would pay + /// whatever address a resume names. The build is abandoned before + /// anything is broadcast, its row's removal queued, and its inputs come + /// back for the next one. + #[tokio::test] + async fn should_not_send_a_shielded_funding_whose_recipient_row_another_flush_dropped() { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + let fund = || { + manager.broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + &signer, + None, + None, + ) + }; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + let refused = fund().await; + assert!( + matches!(&refused, Err(PlatformWalletError::Persistence(message)) + if message.contains("shielded recipient was not made durable before dispatch")), + "{refused:?}" + ); + assert!( + persistence.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + assert_eq!( + persistence.removed_outpoints(), + persistence.stored_outpoints(), + "the abandoned row's removal is queued behind it" + ); + + let (_, out_point) = fund() + .await + .expect("the abandoned build released its inputs, and this row is durable"); + assert_eq!(broadcaster.calls(), 1); + assert_eq!( + manager + .list_tracked_locks() + .await + .iter() + .map(|lock| (lock.out_point, lock.shielded_recipient)) + .collect::>(), + vec![(out_point, Some([7; 43]))] + ); + } + /// The broadcast half returns when the broadcaster accepts the transaction: /// the tracked row is `Broadcast` (recoverable/resumable) and the invitation /// funding pool was persisted AND flushed — all BEFORE any proof wait (the diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs index 1c158cf7248..81db6588220 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs @@ -24,6 +24,7 @@ use key_wallet::wallet::managed_wallet_info::asset_lock_builder::AssetLockFundin use crate::changeset::{ InvitationChangeSet, InvitationEntry, InvitationStatus, PersistenceCapabilities, + PlatformWalletChangeSet, }; use dash_sdk::platform::transition::put_settings::PutSettings; @@ -35,6 +36,7 @@ pub use dash_sdk::platform::dashpay::{ use crate::error::PlatformWalletError; use crate::wallet::identity::crypto::{encode_invitation_uri, InviterInfo, ParsedInvitation}; use crate::wallet::identity::network::contact_requests::ContactCryptoProvider; +use crate::wallet::persister::NotDurable; use super::*; @@ -83,6 +85,40 @@ fn funding_index_from_path(path: &DerivationPath) -> Option { } impl IdentityWallet { + /// Record the inviter-side row of a voucher that is already funded, and + /// answer `Ok` only once it is on disk. + /// + /// A flush that succeeded is not trusted by itself + /// (`WalletPersister::store_durably`): another flush of the wallet's + /// records may have taken the record and failed for good, dropping it, + /// and the link handed out afterwards would then be for a voucher no + /// restart could list or reclaim. The voucher is funded by now, so a + /// record another flush dropped, or may have, is stored once more rather + /// than given up: the row is keyed by outpoint, so storing it again + /// changes nothing where the first one did land. Only when that fails + /// too is the failure reported, as a store or flush that failed is. + fn record_funded_invitation(&self, entry: InvitationEntry) -> Result<(), PlatformWalletError> { + let mut inv_cs = InvitationChangeSet::default(); + inv_cs.invitations.insert(entry.out_point, entry); + let record = || PlatformWalletChangeSet { + invitations: Some(inv_cs.clone()), + ..Default::default() + }; + let recorded = match self.persister.store_durably(record()) { + Err(NotDurable::Lost | NotDurable::OtherFlushInFlight) => { + self.persister.store_durably(record()) + } + recorded => recorded, + }; + recorded.map_err(|e| { + PlatformWalletError::AssetLockTransaction(format!( + "the invitation voucher is funded but its record could not be \ + persisted (it would be missing from Sent invitations and \ + unreclaimable): {e}" + )) + }) + } + /// Create a DashPay invitation: fund a one-time asset-lock voucher at the /// DIP-13 invitation path and return a shareable `dashpay://invite` link. /// @@ -198,32 +234,15 @@ impl IdentityWallet { // for the invitation account), warn-skip: an untrackable row can't be // reclaimed either way. if let Some(funding_index) = funding_index_from_path(&path) { - let mut inv_cs = InvitationChangeSet::default(); - inv_cs.invitations.insert( + self.record_funded_invitation(InvitationEntry { out_point, - InvitationEntry { - out_point, - funding_index, - amount_duffs, - expiry_unix, - created_at_secs, - has_inviter: inviter.is_some(), - status: InvitationStatus::Created, - }, - ); - self.persister - .store(crate::changeset::PlatformWalletChangeSet { - invitations: Some(inv_cs), - ..Default::default() - }) - .and_then(|_| self.persister.flush()) - .map_err(|e| { - PlatformWalletError::AssetLockTransaction(format!( - "the invitation voucher is funded but its record could not be \ - persisted (it would be missing from Sent invitations and \ - unreclaimable): {e}" - )) - })?; + funding_index, + amount_duffs, + expiry_unix, + created_at_secs, + has_inviter: inviter.is_some(), + status: InvitationStatus::Created, + })?; } else { tracing::warn!( "invitation funding path has no u32 index tail; \ @@ -560,4 +579,192 @@ mod tests { ); } } + + // --- create_invitation: the funded voucher's record --- + + mod funded_record { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + use dashcore::hashes::Hash; + use dashcore::{OutPoint, Txid}; + use key_wallet::mnemonic::Mnemonic; + use key_wallet::wallet::initialization::WalletAccountCreationOptions; + use key_wallet::Network; + + use crate::changeset::{ + ClientStartState, InvitationEntry, InvitationStatus, PersistenceError, + PlatformWalletChangeSet, PlatformWalletPersistence, + }; + use crate::events::{EventHandler, PlatformEventHandler}; + use crate::wallet::persister::WalletPersister; + use crate::wallet::platform_wallet::WalletId; + use crate::{PlatformWallet, PlatformWalletError}; + + const TEST_MNEMONIC: &str = "abandon abandon abandon abandon abandon abandon \ + abandon abandon abandon abandon abandon about"; + + struct NoopEventHandler; + impl EventHandler for NoopEventHandler {} + impl PlatformEventHandler for NoopEventHandler {} + + /// A backend that buffers what is stored until a flush writes it to + /// `durable`; a flush that fails for good drops the buffer. + #[derive(Default)] + struct BufferingPersistence { + buffered: Mutex>, + durable: Mutex>, + /// One handle per flush still to be overtaken: before such a + /// flush writes, a flush through the handle fails for good, as + /// one from another thread would, dropping what was buffered; + /// the overtaken flush then succeeds over nothing. + overtaken_by: Mutex>, + /// While set, a flush fails for good. + flush_drops_buffer: AtomicBool, + } + + impl BufferingPersistence { + /// The outpoints of the invitation records on disk, in order. + fn durable_invitations(&self) -> Vec { + self.durable + .lock() + .expect("durable") + .iter() + .filter_map(|changeset| changeset.invitations.as_ref()) + .flat_map(|invitations| invitations.invitations.keys().copied()) + .collect() + } + } + + impl PlatformWalletPersistence for BufferingPersistence { + fn store( + &self, + _wallet_id: WalletId, + changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + self.buffered.lock().expect("buffered").push(changeset); + Ok(()) + } + + fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { + if self.flush_drops_buffer.load(Ordering::SeqCst) { + self.buffered.lock().expect("buffered").clear(); + return Err(PersistenceError::backend("the database is corrupt")); + } + // Taken in its own statement: the flush below comes back here. + let overtaking = self.overtaken_by.lock().expect("overtaken").pop(); + if let Some(persister) = overtaking { + self.flush_drops_buffer.store(true, Ordering::SeqCst); + let overtook = persister.flush(); + self.flush_drops_buffer.store(false, Ordering::SeqCst); + assert!(overtook.is_err_and(|error| !error.is_transient())); + } + let flushed = std::mem::take(&mut *self.buffered.lock().expect("buffered")); + self.durable.lock().expect("durable").extend(flushed); + Ok(()) + } + + fn load(&self) -> Result { + Ok(ClientStartState::default()) + } + } + + async fn wallet_on(persistence: Arc) -> Arc { + let sdk = Arc::new(dash_sdk::SdkBuilder::new_mock().build().expect("mock sdk")); + let handler: Arc = Arc::new(NoopEventHandler); + let manager = Arc::new(crate::PlatformWalletManager::new(sdk, persistence, handler)); + let seed = Mnemonic::from_phrase(TEST_MNEMONIC) + .expect("valid mnemonic") + .to_seed(""); + manager + .create_wallet_from_seed_bytes( + Network::Testnet, + &seed, + WalletAccountCreationOptions::None, + Some(0), + ) + .await + .expect("wallet creation") + } + + fn funded_voucher() -> InvitationEntry { + InvitationEntry { + out_point: OutPoint::new(Txid::from_byte_array([7; 32]), 0), + funding_index: 0, + amount_duffs: 1_000_000, + expiry_unix: 1, + created_at_secs: 1, + has_inviter: false, + status: InvitationStatus::Created, + } + } + + /// A funded voucher's record is not taken as recorded on the strength + /// of a flush that succeeded over nothing, another flush of the + /// wallet's records having taken it first and failed for good: it is + /// stored once more, and is on disk when the call answers. + #[tokio::test] + async fn should_record_a_funded_invitation_again_when_another_flush_dropped_its_record() { + let persistence = Arc::new(BufferingPersistence::default()); + let wallet = wallet_on(Arc::clone(&persistence)).await; + let iw = wallet.identity(); + let voucher = funded_voucher(); + persistence + .overtaken_by + .lock() + .expect("overtaken") + .push(iw.persister.clone()); + + iw.record_funded_invitation(voucher.clone()) + .expect("stored once more, and durable"); + + assert!( + persistence + .overtaken_by + .lock() + .expect("overtaken") + .is_empty(), + "a flush was overtaken" + ); + assert_eq!( + persistence.durable_invitations(), + vec![voucher.out_point], + "the record is on disk" + ); + } + + /// When the record stored once more is dropped by another flush as + /// well, the voucher is reported funded and unrecorded, as a flush + /// that failed reports it, and no link follows: it is not reported + /// recorded while on no disk. + #[tokio::test] + async fn should_report_a_funded_invitation_unrecorded_when_another_flush_dropped_it_twice() + { + let persistence = Arc::new(BufferingPersistence::default()); + let wallet = wallet_on(Arc::clone(&persistence)).await; + let iw = wallet.identity(); + persistence + .overtaken_by + .lock() + .expect("overtaken") + .extend([iw.persister.clone(), iw.persister.clone()]); + + let unrecorded = iw.record_funded_invitation(funded_voucher()); + + assert!( + matches!(&unrecorded, Err(PlatformWalletError::AssetLockTransaction(message)) + if message.contains("funded but its record could not be persisted")), + "{unrecorded:?}" + ); + assert!( + persistence + .overtaken_by + .lock() + .expect("overtaken") + .is_empty(), + "both flushes were overtaken" + ); + assert!(persistence.durable_invitations().is_empty(), "on no disk"); + } + } } diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs b/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs index 5060aa6295d..3414c292223 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs @@ -22,6 +22,7 @@ use crate::broadcaster::TransactionBroadcaster; use crate::changeset::PlatformWalletChangeSet; use crate::error::PlatformWalletError; use crate::wallet::core::WalletGeneration; +use crate::wallet::persister::NotDurable; use crate::wallet::platform_wallet::{PlatformWalletInfo, WalletId}; use crate::wallet::reservations::release_reservation_after_rejected_broadcast; @@ -1562,15 +1563,19 @@ impl DashPayView<'_, B> { )? }; // Host persistence may re-enter wallet APIs, so never invoke it under - // the manager guard. Failure must not expose an unpersisted address. + // the manager guard. Failure must not expose an unpersisted address, + // and a flush that succeeded is no proof by itself: another flush of + // the wallet's records may have taken the reservation and failed for + // good, dropping it (`WalletPersister::store_durably`). The address + // then stays used in memory and is not handed out. self.persister - .store(changeset) - .map_err(|e| self.persister.classify_store_failure(e))?; - self.persister.flush().map_err(|e| { - PlatformWalletError::Persistence(format!( - "failed to flush payment-address reservation: {e}" - )) - })?; + .store_durably(changeset) + .map_err(|not_durable| match not_durable { + NotDurable::Store(e) => self.persister.classify_store_failure(e), + not_flushed => PlatformWalletError::Persistence(format!( + "failed to flush payment-address reservation: {not_flushed}" + )), + })?; Ok(address) } @@ -2196,6 +2201,14 @@ mod tests { /// When set, every round recording a DashPay payment fails /// transiently (and is not recorded). fail_payment_records: Mutex, + /// Once set, the next flush is overtaken: before it writes, a flush + /// through this persister fails for good, as one from another thread + /// would, dropping what was buffered; the overtaken flush then + /// succeeds over nothing. + overtaken_by: Mutex>, + /// While set, a flush fails for good: the backend drops what it + /// buffered. + flush_drops_buffer: std::sync::atomic::AtomicBool, } impl PlatformWalletPersistence for RecordingPersister { @@ -2283,6 +2296,23 @@ mod tests { fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { self.flushes .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + // Taken in its own statement: the flush below comes back here. + let overtaking = self.overtaken_by.lock().unwrap().take(); + if let Some(persister) = overtaking { + self.flush_drops_buffer + .store(true, std::sync::atomic::Ordering::SeqCst); + let overtook = persister.flush(); + self.flush_drops_buffer + .store(false, std::sync::atomic::Ordering::SeqCst); + assert!(overtook.is_err_and(|error| !error.is_transient())); + return Ok(()); + } + if self + .flush_drops_buffer + .load(std::sync::atomic::Ordering::SeqCst) + { + return Err(PersistenceError::backend("the database is corrupt")); + } if *self.fail_flushes.lock().unwrap() { return Err(PersistenceError::backend_with_kind( PersistenceErrorKind::Transient, @@ -12125,6 +12155,60 @@ mod tests { ); } + /// A withdrawal address is not handed out when another flush of the + /// wallet's records took its reservation first and failed for good, + /// though the reservation's own flush then succeeded, over nothing: a + /// restart would hand the same address to the next payment. The address + /// stays used in memory, so the next call hands out another one. + #[tokio::test] + async fn should_not_hand_out_a_payment_address_whose_reservation_another_flush_dropped() { + use crate::wallet::identity::network::contact_requests::SeedCryptoProvider; + let (manager, persister, wallet_id, owner, contact) = + register_sender_and_external_account().await; + let wallet = manager.get_wallet(&wallet_id).await.unwrap(); + let iw = wallet.identity(); + let seed = Mnemonic::from_phrase(TEST_MNEMONIC).unwrap().to_seed(""); + let provider = SeedCryptoProvider::from_seed(*seed, Network::Testnet); + *persister.buffered.lock().unwrap() = true; + *persister.overtaken_by.lock().unwrap() = Some(wallet.persister().clone()); + let refused = iw + .dashpay() + .reserve_payment_address(&owner, &contact, &provider) + .await; + assert!( + matches!(&refused, Err(PlatformWalletError::Persistence(message)) + if message.contains("failed to flush payment-address reservation")), + "{refused:?}" + ); + assert!( + persister.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!( + external_used_indices(&manager, wallet_id, owner, contact).await, + vec![0], + "the address that was not handed out stays used in memory" + ); + let next = iw + .dashpay() + .reserve_payment_address(&owner, &contact, &provider) + .await + .expect("a later reservation is durable"); + assert_eq!( + external_used_indices(&manager, wallet_id, owner, contact).await, + vec![0, 1] + ); + assert!(persister + .stores + .lock() + .unwrap() + .iter() + .rev() + .flat_map(|(_, cs)| &cs.account_address_pools) + .flat_map(|pool| &pool.addresses) + .any(|info| info.address == next && info.is_used())); + } + /// Persister that parks the FIRST external-pool store inside `store`, /// after the caller has captured its whole-pool snapshot, until the /// test releases it. Every store is recorded in arrival order so the @@ -14882,6 +14966,175 @@ mod tests { iw.dashpay().abandon_prepared_payment(retry).await; } + /// A prepared payment's reservation is not on disk when another flush of + /// the wallet's records took it first and failed for good, though its own + /// flush then succeeded, over nothing: nothing is sent, the address and + /// the inputs come back, and the error is a flush whose buffered round + /// was discarded. + #[tokio::test] + async fn should_send_nothing_when_another_flush_dropped_a_prepared_reservation() { + let (manager, persister, wallet_id, owner_id, contact_id) = + register_sender_and_external_account().await; + let wallet_arc = manager.get_wallet(&wallet_id).await.expect("wallet"); + let iw = wallet_arc.identity(); + // One coin, so the retry proves the inputs were released. + fund_bip44_account_0(&manager, wallet_id, 0xD1, 120_000).await; + let (signer, provider) = test_signer_and_provider(); + let probe = Arc::new(DurabilityProbeBroadcaster { + persister: Arc::clone(&persister), + seen: Mutex::new(None), + }); + let core = crate::CoreWallet::new( + Arc::clone(&wallet_arc.core().sdk), + Arc::clone(&wallet_arc.core().wallet_manager), + wallet_arc.core().wallet_id, + Arc::clone(&probe), + Arc::clone(wallet_arc.core().generation()), + ); + + let prepared = iw + .dashpay() + .prepare_payment( + &core, + &owner_id, + &contact_id, + 100_000, + None, + &signer, + &provider, + ) + .await + .expect("prepare"); + let first_output = payment_script(prepared.transaction(), 100_000); + *persister.buffered.lock().unwrap() = true; + *persister.overtaken_by.lock().unwrap() = Some(wallet_arc.persister().clone()); + let err = iw + .dashpay() + .send_prepared_payment(prepared) + .await + .expect_err("a reservation another flush dropped must not be broadcast"); + assert!( + matches!( + &err, + PlatformWalletError::PersisterFlush(source) + if source.kind() == Some(PersistenceErrorKind::Fatal) + ), + "got {err:?}" + ); + assert!( + persister.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert!( + probe.seen.lock().unwrap().is_none(), + "the broadcaster never ran" + ); + assert!( + external_used_indices(&manager, wallet_id, owner_id, contact_id) + .await + .is_empty() + ); + let retry = iw + .dashpay() + .prepare_payment( + &core, + &owner_id, + &contact_id, + 100_000, + None, + &signer, + &provider, + ) + .await + .expect("the failed send released the only coin"); + assert_eq!( + payment_script(retry.transaction(), 100_000), + first_output, + "the address was released, so the retry pays it" + ); + iw.dashpay() + .send_prepared_payment(retry) + .await + .expect("the retry's reservation is durable"); + assert!( + probe + .seen + .lock() + .unwrap() + .is_some_and(|(_, reserved)| reserved), + "the retry reached the network with its reservation stored" + ); + } + + /// A direct send's reservation is not on disk when another flush of the + /// wallet's records took it first and failed for good, though its own + /// flush then succeeded, over nothing: nothing is sent, and the address + /// and the inputs come back for the retry. + #[tokio::test] + async fn should_send_nothing_when_another_flush_dropped_a_send_s_reservation() { + let (manager, persister, wallet_id, owner_id, contact_id) = + register_sender_and_external_account().await; + let wallet_arc = manager.get_wallet(&wallet_id).await.expect("wallet"); + let iw = wallet_arc.identity(); + // One coin, so the retry proves the inputs were released. + fund_bip44_account_0(&manager, wallet_id, 0xD2, 120_000).await; + let (signer, provider) = test_signer_and_provider(); + let sending = with_identity_broadcaster( + iw, + DurabilityProbeBroadcaster { + persister: Arc::clone(&persister), + seen: Mutex::new(None), + }, + ); + *persister.buffered.lock().unwrap() = true; + *persister.overtaken_by.lock().unwrap() = Some(wallet_arc.persister().clone()); + let err = sending + .dashpay() + .send_payment(&owner_id, &contact_id, 100_000, None, &signer, &provider) + .await + .expect_err("a reservation another flush dropped must not be broadcast"); + assert!( + matches!( + &err, + PlatformWalletError::PersisterFlush(source) + if source.kind() == Some(PersistenceErrorKind::Fatal) + ), + "got {err:?}" + ); + assert!( + persister.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert!( + sending.broadcaster.seen.lock().unwrap().is_none(), + "the broadcaster never ran" + ); + assert!( + external_used_indices(&manager, wallet_id, owner_id, contact_id) + .await + .is_empty() + ); + sending + .dashpay() + .send_payment(&owner_id, &contact_id, 100_000, None, &signer, &provider) + .await + .expect("the failed send released the only coin"); + assert!( + sending + .broadcaster + .seen + .lock() + .unwrap() + .is_some_and(|(_, reserved)| reserved), + "the retry reached the network with its reservation stored" + ); + assert_eq!( + external_used_indices(&manager, wallet_id, owner_id, contact_id).await, + vec![0], + "the address was released, so the retry pays it" + ); + } + /// Broadcaster that, while it holds the transaction, checks whether a /// teardown of the funding generation could start, then accepts it. struct TeardownProbeBroadcaster { diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs b/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs index 83d0af0f00b..66f8a2fc34d 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs @@ -74,11 +74,14 @@ use super::payments::{ }; use super::DashPayView; use crate::broadcaster::TransactionBroadcaster; -use crate::changeset::{account_address_pool_entries, PlatformWalletChangeSet}; +use crate::changeset::{ + account_address_pool_entries, PersistenceError, PersistenceErrorKind, PlatformWalletChangeSet, +}; use crate::error::PlatformWalletError; use crate::util::now_secs; use crate::wallet::core::{CoreWallet, SignedCoreTransaction, WalletGeneration}; use crate::wallet::identity::types::dashpay::payment::PaymentEntry; +use crate::wallet::persister::NotDurable; use crate::wallet::platform_wallet::PlatformWalletInfo; use crate::wallet::reservations::reservation_expired; use crate::SEND_FUNDING_SOURCES; @@ -722,23 +725,38 @@ impl DashPayView<'_, B> { } } - /// Store `changeset` and, on a backend that buffers, flush it, so it is - /// on disk when this returns. A failed flush is `PersisterFlush` with the - /// backend's own kind: the round may still be buffered and commit at a - /// later flush, so it is not proof that nothing was stored. + /// Store `changeset` and answer `Ok` only once it is on disk + /// (`WalletPersister::store_durably`): at once on a backend that commits + /// a store inline, and otherwise after a flush that no other flush of the + /// wallet's records can have emptied. A failed flush is `PersisterFlush` + /// with the backend's own kind: the round may still be buffered and + /// commit at a later flush, so it is not proof that nothing was stored. + /// + /// A flush that succeeded is not trusted by itself: another flush may + /// have taken what the store buffered and failed for good, dropping it. + /// That is reported as the flush failure it is, `PersisterFlush` of kind + /// `Fatal`, the buffered round discarded; and a flush still in progress, + /// whose outcome is not known yet, as one of kind `Transient`, to be + /// tried again. pub(super) fn store_durably( &self, changeset: PlatformWalletChangeSet, ) -> Result<(), PlatformWalletError> { self.persister - .store(changeset) - .map_err(|e| self.persister.classify_store_failure(e))?; - if !self.persister.store_commits_inline() { - self.persister - .flush() - .map_err(PlatformWalletError::from_flush_failure)?; - } - Ok(()) + .store_durably(changeset) + .map_err(|not_durable| match not_durable { + NotDurable::Store(e) => self.persister.classify_store_failure(e), + NotDurable::Flush(e) => PlatformWalletError::from_flush_failure(e), + NotDurable::Lost => PlatformWalletError::from_flush_failure( + PersistenceError::backend_with_kind(PersistenceErrorKind::Fatal, not_durable), + ), + NotDurable::OtherFlushInFlight => { + PlatformWalletError::from_flush_failure(PersistenceError::backend_with_kind( + PersistenceErrorKind::Transient, + not_durable, + )) + } + }) } /// The wallet info of `generation`, or `WalletNotFound` when the wallet diff --git a/packages/rs-platform-wallet/src/wallet/persister.rs b/packages/rs-platform-wallet/src/wallet/persister.rs index a8ccb1b2887..9e646d25d48 100644 --- a/packages/rs-platform-wallet/src/wallet/persister.rs +++ b/packages/rs-platform-wallet/src/wallet/persister.rs @@ -179,9 +179,13 @@ pub(crate) enum NotDurable { had buffered" )] Lost, - /// The store or the flush this call made failed. + /// The store this call made failed, so nothing was flushed. #[error("{0}")] - Backend(PersistenceError), + Store(PersistenceError), + /// The flush this call made failed: a transient failure keeps what the + /// backend buffered, any other drops it. + #[error("{0}")] + Flush(PersistenceError), /// Another flush is in progress, and its outcome is not recorded yet. #[error( "another flush of the wallet's records is in progress, and its outcome is not known \ @@ -348,7 +352,7 @@ impl WalletPersister { } match self.flush_durably() { Ok(()) => self.note_read_back_private_entries(pending), - Err(NotDurable::Backend(error)) if error.is_transient() => { + Err(NotDurable::Flush(error)) if error.is_transient() => { tracing::warn!( %error, kept = pending.len(), @@ -367,7 +371,8 @@ impl WalletPersister { ); self.unflushed_read_back_entries().extend(pending); } - Err(NotDurable::Backend(error)) => { + // `flush_durably` stores nothing, so `Store` is never its answer. + Err(NotDurable::Flush(error) | NotDurable::Store(error)) => { // Any other failure dropped the buffer, the records these // notices rest on with it: the notices go, so their journal // records, and the pairs they hold, stay. Their read is not @@ -635,11 +640,11 @@ impl WalletPersister { changeset: PlatformWalletChangeSet, ) -> Result<(), NotDurable> { let lost_before = self.flush_ledger.lost(); - self.store(changeset).map_err(NotDurable::Backend)?; + self.store(changeset).map_err(NotDurable::Store)?; if self.store_commits_inline() { return Ok(()); } - self.flush().map_err(NotDurable::Backend)?; + self.flush().map_err(NotDurable::Flush)?; if self.flush_ledger.in_flight() != 0 { return Err(NotDurable::OtherFlushInFlight); } @@ -678,7 +683,7 @@ impl WalletPersister { return Err(NotDurable::Lost); } if !self.store_commits_inline() { - self.flush().map_err(NotDurable::Backend)?; + self.flush().map_err(NotDurable::Flush)?; if self.flush_ledger.in_flight() != 0 { return Err(NotDurable::OtherFlushInFlight); } From 95a02257843d256795762223c0b966edcd3c5d29 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 14 Nov 2026 00:38:07 +0700 Subject: [PATCH 2/7] test(platform-wallet): cover a record another flush took and put back The base branch now answers `NotDurable::OtherFlushUnsettled` when another flush of the wallet's records failed transiently while the call ran and put back what it had taken, as well as when one has not returned. Neither is a lost record: a later flush writes it. Two tests pin how the changed sites treat it. A prepared contact payment whose reservation another flush took and put back sends nothing and reports a flush to be tried again (`PersisterFlush` of kind `Transient`), not a discarded round. A funded invitation's record that another flush took and put back is stored once more, and the flush that follows writes it. The comments on both sites say what the answer covers now, and an unused import in the invitation tests goes. Co-Authored-By: Claude Fable 5.2 --- .../src/wallet/identity/network/invitation.rs | 98 ++++++++++--- .../src/wallet/identity/network/payments.rs | 134 ++++++++++++++++-- .../identity/network/prepared_payment.rs | 7 +- 3 files changed, 203 insertions(+), 36 deletions(-) diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs index 40affe15fdc..3994df9dc73 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs @@ -93,10 +93,12 @@ impl IdentityWallet { /// records may have taken the record and failed for good, dropping it, /// and the link handed out afterwards would then be for a voucher no /// restart could list or reclaim. The voucher is funded by now, so a - /// record another flush dropped, or may have, is stored once more rather - /// than given up: the row is keyed by outpoint, so storing it again - /// changes nothing where the first one did land. Only when that fails - /// too is the failure reported, as a store or flush that failed is. + /// record another flush dropped, or still holds or put back, is stored + /// once more rather than given up: the row is keyed by outpoint, so + /// storing it again changes nothing where the first one did land, and + /// the flush that follows writes what was put back. Only when that + /// fails too is the failure reported, as a store or flush that failed + /// is. fn record_funded_invitation(&self, entry: InvitationEntry) -> Result<(), PlatformWalletError> { let mut inv_cs = InvitationChangeSet::default(); inv_cs.invitations.insert(entry.out_point, entry); @@ -586,7 +588,6 @@ mod tests { use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; - use dashcore::hashes::Hash; use dashcore::{OutPoint, Txid}; use key_wallet::mnemonic::Mnemonic; use key_wallet::wallet::initialization::WalletAccountCreationOptions; @@ -594,7 +595,7 @@ mod tests { use crate::changeset::{ ClientStartState, InvitationEntry, InvitationStatus, PersistenceError, - PlatformWalletChangeSet, PlatformWalletPersistence, + PersistenceErrorKind, PlatformWalletChangeSet, PlatformWalletPersistence, }; use crate::events::{EventHandler, PlatformEventHandler}; use crate::wallet::persister::WalletPersister; @@ -609,18 +610,23 @@ mod tests { impl PlatformEventHandler for NoopEventHandler {} /// A backend that buffers what is stored until a flush writes it to - /// `durable`; a flush that fails for good drops the buffer. + /// `durable`; a flush that fails for good drops the buffer, and one + /// that fails transiently keeps it. #[derive(Default)] struct BufferingPersistence { buffered: Mutex>, durable: Mutex>, - /// One handle per flush still to be overtaken: before such a - /// flush writes, a flush through the handle fails for good, as - /// one from another thread would, dropping what was buffered; - /// the overtaken flush then succeeds over nothing. + /// One handle per flush still to be overtaken: a flush through + /// the handle, as one from another thread would, takes what was + /// buffered before such a flush writes, and fails; the overtaken + /// flush then succeeds over nothing. overtaken_by: Mutex>, - /// While set, a flush fails for good. - flush_drops_buffer: AtomicBool, + /// Whether the overtaking flush fails transiently, putting back + /// what it took once the overtaken one is done, or for good, + /// dropping it. + overtaking_puts_back: AtomicBool, + /// While set, a flush is the overtaking one, and fails. + overtaking: AtomicBool, } impl BufferingPersistence { @@ -647,17 +653,31 @@ mod tests { } fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { - if self.flush_drops_buffer.load(Ordering::SeqCst) { - self.buffered.lock().expect("buffered").clear(); - return Err(PersistenceError::backend("the database is corrupt")); + let puts_back = self.overtaking_puts_back.load(Ordering::SeqCst); + if self.overtaking.load(Ordering::SeqCst) { + return Err(if puts_back { + PersistenceError::backend_with_kind( + PersistenceErrorKind::Transient, + "the disk is busy", + ) + } else { + PersistenceError::backend("the database is corrupt") + }); } // Taken in its own statement: the flush below comes back here. let overtaking = self.overtaken_by.lock().expect("overtaken").pop(); if let Some(persister) = overtaking { - self.flush_drops_buffer.store(true, Ordering::SeqCst); + // The other flush holds what was buffered while this one + // runs, over nothing. + let taken = std::mem::take(&mut *self.buffered.lock().expect("buffered")); + self.overtaking.store(true, Ordering::SeqCst); let overtook = persister.flush(); - self.flush_drops_buffer.store(false, Ordering::SeqCst); - assert!(overtook.is_err_and(|error| !error.is_transient())); + self.overtaking.store(false, Ordering::SeqCst); + assert!(overtook.is_err_and(|error| error.is_transient() == puts_back)); + if puts_back { + self.buffered.lock().expect("buffered").extend(taken); + } + return Ok(()); } let flushed = std::mem::take(&mut *self.buffered.lock().expect("buffered")); self.durable.lock().expect("durable").extend(flushed); @@ -733,6 +753,46 @@ mod tests { ); } + /// A funded voucher's record is not taken as recorded while another + /// flush of the wallet's records holds it, a flush that then fails + /// transiently and puts it back: the record is stored once more, and + /// the flush that follows writes it. + #[tokio::test] + async fn should_record_a_funded_invitation_another_flush_took_and_put_back() { + let persistence = Arc::new(BufferingPersistence::default()); + let wallet = wallet_on(Arc::clone(&persistence)).await; + let iw = wallet.identity(); + let voucher = funded_voucher(); + persistence + .overtaking_puts_back + .store(true, Ordering::SeqCst); + persistence + .overtaken_by + .lock() + .expect("overtaken") + .push(iw.persister.clone()); + + iw.record_funded_invitation(voucher.clone()) + .expect("stored once more, and durable"); + + assert!( + persistence + .overtaken_by + .lock() + .expect("overtaken") + .is_empty(), + "a flush was overtaken" + ); + let on_disk = persistence.durable_invitations(); + assert!( + !on_disk.is_empty() + && on_disk + .iter() + .all(|out_point| *out_point == voucher.out_point), + "the record is on disk: {on_disk:?}" + ); + } + /// When the record stored once more is dropped by another flush as /// well, the voucher is reported funded and unrecorded, as a flush /// that failed reports it, and no link follows: it is not reported diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs b/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs index 3414c292223..50139d56f7d 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs @@ -2201,14 +2201,16 @@ mod tests { /// When set, every round recording a DashPay payment fails /// transiently (and is not recorded). fail_payment_records: Mutex, - /// Once set, the next flush is overtaken: before it writes, a flush - /// through this persister fails for good, as one from another thread - /// would, dropping what was buffered; the overtaken flush then + /// Once set, the next flush is overtaken: a flush through this + /// persister, as one from another thread would, takes what was + /// buffered before it writes, and fails; the overtaken flush then /// succeeds over nothing. overtaken_by: Mutex>, - /// While set, a flush fails for good: the backend drops what it - /// buffered. - flush_drops_buffer: std::sync::atomic::AtomicBool, + /// Whether the overtaking flush fails transiently, putting back what + /// it took, or for good, dropping it. + overtaking_puts_back: std::sync::atomic::AtomicBool, + /// While set, a flush is the overtaking one, and fails. + overtaking: std::sync::atomic::AtomicBool, } impl PlatformWalletPersistence for RecordingPersister { @@ -2298,20 +2300,27 @@ mod tests { .fetch_add(1, std::sync::atomic::Ordering::SeqCst); // Taken in its own statement: the flush below comes back here. let overtaking = self.overtaken_by.lock().unwrap().take(); + let puts_back = self + .overtaking_puts_back + .load(std::sync::atomic::Ordering::SeqCst); if let Some(persister) = overtaking { - self.flush_drops_buffer + self.overtaking .store(true, std::sync::atomic::Ordering::SeqCst); let overtook = persister.flush(); - self.flush_drops_buffer + self.overtaking .store(false, std::sync::atomic::Ordering::SeqCst); - assert!(overtook.is_err_and(|error| !error.is_transient())); + assert!(overtook.is_err_and(|error| error.is_transient() == puts_back)); return Ok(()); } - if self - .flush_drops_buffer - .load(std::sync::atomic::Ordering::SeqCst) - { - return Err(PersistenceError::backend("the database is corrupt")); + if self.overtaking.load(std::sync::atomic::Ordering::SeqCst) { + return Err(if puts_back { + PersistenceError::backend_with_kind( + PersistenceErrorKind::Transient, + "the disk is busy", + ) + } else { + PersistenceError::backend("the database is corrupt") + }); } if *self.fail_flushes.lock().unwrap() { return Err(PersistenceError::backend_with_kind( @@ -15066,6 +15075,103 @@ mod tests { ); } + /// A prepared payment's reservation is not on disk either while another + /// flush of the wallet's records holds it, one that then fails + /// transiently and puts it back: nothing is sent, the address and the + /// inputs come back, and the error is a flush to be tried again, not a + /// lost round. + #[tokio::test] + async fn should_send_nothing_when_another_flush_took_a_prepared_reservation_and_put_it_back() { + let (manager, persister, wallet_id, owner_id, contact_id) = + register_sender_and_external_account().await; + let wallet_arc = manager.get_wallet(&wallet_id).await.expect("wallet"); + let iw = wallet_arc.identity(); + // One coin, so the retry proves the inputs were released. + fund_bip44_account_0(&manager, wallet_id, 0xD3, 120_000).await; + let (signer, provider) = test_signer_and_provider(); + let probe = Arc::new(DurabilityProbeBroadcaster { + persister: Arc::clone(&persister), + seen: Mutex::new(None), + }); + let core = crate::CoreWallet::new( + Arc::clone(&wallet_arc.core().sdk), + Arc::clone(&wallet_arc.core().wallet_manager), + wallet_arc.core().wallet_id, + Arc::clone(&probe), + Arc::clone(wallet_arc.core().generation()), + ); + + let prepared = iw + .dashpay() + .prepare_payment( + &core, + &owner_id, + &contact_id, + 100_000, + None, + &signer, + &provider, + ) + .await + .expect("prepare"); + *persister.buffered.lock().unwrap() = true; + persister + .overtaking_puts_back + .store(true, std::sync::atomic::Ordering::SeqCst); + *persister.overtaken_by.lock().unwrap() = Some(wallet_arc.persister().clone()); + let err = iw + .dashpay() + .send_prepared_payment(prepared) + .await + .expect_err("a reservation another flush holds must not be broadcast"); + assert!( + matches!( + &err, + PlatformWalletError::PersisterFlush(source) + if source.kind() == Some(PersistenceErrorKind::Transient) + ), + "got {err:?}" + ); + assert!( + persister.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert!( + probe.seen.lock().unwrap().is_none(), + "the broadcaster never ran" + ); + assert!( + external_used_indices(&manager, wallet_id, owner_id, contact_id) + .await + .is_empty() + ); + let retry = iw + .dashpay() + .prepare_payment( + &core, + &owner_id, + &contact_id, + 100_000, + None, + &signer, + &provider, + ) + .await + .expect("the failed send released the only coin"); + iw.dashpay() + .send_prepared_payment(retry) + .await + .expect("the retry's reservation is durable"); + assert!( + probe + .seen + .lock() + .unwrap() + .is_some_and(|(_, reserved)| reserved), + "the retry reached the network with its reservation stored" + ); + } + /// A direct send's reservation is not on disk when another flush of the /// wallet's records took it first and failed for good, though its own /// flush then succeeded, over nothing: nothing is sent, and the address diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs b/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs index 11b43c5d48a..6f0faf1b725 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs @@ -735,9 +735,10 @@ impl DashPayView<'_, B> { /// A flush that succeeded is not trusted by itself: another flush may /// have taken what the store buffered and failed for good, dropping it. /// That is reported as the flush failure it is, `PersisterFlush` of kind - /// `Fatal`, the buffered round discarded; and a flush still in progress, - /// whose outcome is not known yet, as one of kind `Transient`, to be - /// tried again. + /// `Fatal`, the buffered round discarded. Another flush that is still in + /// progress, or that failed and put back what it took, leaves the round + /// buffered for a later flush: `PersisterFlush` of kind `Transient`, to + /// be tried again. pub(super) fn store_durably( &self, changeset: PlatformWalletChangeSet, From 133eaf0bbb6ed964997e967f316920a9d1f99f60 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 14 Nov 2026 02:15:28 +0700 Subject: [PATCH 3/7] fix(platform-wallet): write an abandoned shielded funding's row removal before releasing its inputs A funding that pays a shielded recipient is abandoned when its row is not known to be on disk. The row may be there all the same: its own flush wrote it while another flush of the wallet's records failed or was in progress beside it. The removal of the row was only queued, so a process that ended before the next flush left a Built row behind, and the restart sent a funding that had been reported as not made. The removal now goes through WalletPersister::store_durably before the inputs are released. When it is not known to be on disk either, the error says a restart may still resume the lock. The inputs are released in both cases: nothing was sent, and the reservation does not outlast a restart. Also documents why record_funded_invitation stores the record again for a flush another one overtook, but not for a store or flush of its own that the backend failed. Co-Authored-By: Claude Fable 5.2 --- .../src/wallet/asset_lock/build.rs | 245 ++++++++++++++++-- .../src/wallet/identity/network/invitation.rs | 10 + 2 files changed, 230 insertions(+), 25 deletions(-) diff --git a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs index 9a2ed737f02..d835d9710eb 100644 --- a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs +++ b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs @@ -1526,19 +1526,30 @@ impl AssetLockManager { }); if let Err(error) = durable { // The row may be buffered (a backend that restores a failed - // flush to its buffer) or committed (a store that succeeded - // before a failed flush, or one a flush still in progress - // took), and a later flush or a restart would - // then surface a resumable row for inputs released below. Its - // removal is queued behind it — stores apply removals after - // upserts — so the abandoned row does not outlive this call - // unless the backend refuses that write too. The funding index - // stays consumed, as for the pool gate above. - self.queue_asset_lock_changeset(AssetLockChangeSet { - removed: [out_point].into(), + // flush to its buffer) or on disk (a store that succeeded + // before a failed flush, the row's own flush where another + // one failed or was in progress beside it, or a flush still + // in progress that took the row), and a restart would then + // resume a funding reported here as not made. So its removal + // is made durable too, before the inputs are released: only + // queued behind the row, it would be lost with the process + // where the row was not. A backend writes what it buffered in + // the order it was stored, and applies removals after + // upserts, so once the removal is on disk the row is not. The + // funding index stays consumed, as for the pool gate above. + let removed = self.persister.store_durably(PlatformWalletChangeSet { + asset_locks: Some(AssetLockChangeSet { + removed: [out_point].into(), + ..Default::default() + }), ..Default::default() }); drop(build_persist_guard); + // Released either way: nothing was sent, and the reservation + // and the fence live in memory, which no restart that could + // resume the row keeps. Holding them would only make a retry + // here select other inputs, leaving the unsent transaction + // valid beside it. self.abandon_unsent_build( in_broadcast_pin, &funding_accounts, @@ -1546,9 +1557,25 @@ impl AssetLockManager { reservation_token, ) .await; - return Err(PlatformWalletError::Persistence(format!( - "shielded recipient was not made durable before dispatch: {error}" - ))); + return Err(PlatformWalletError::Persistence(match removed { + Ok(()) => { + format!("shielded recipient was not made durable before dispatch: {error}") + } + Err(removal) => { + tracing::error!( + %out_point, + %error, + %removal, + "an unsent shielded funding's row may be on disk, and its removal \ + is not known to be" + ); + format!( + "shielded recipient was not made durable before dispatch: {error}; \ + nothing was sent, but the removal of the unsent lock's row is not \ + known to be on disk, so a restart may still resume it: {removal}" + ) + } + })); } self.track_asset_lock(tracked).await; } else { @@ -1876,8 +1903,8 @@ mod tests { use crate::broadcaster::{BroadcastError, TransactionBroadcaster}; use crate::changeset::{ - ClientStartState, ClientWalletStartState, PersistenceError, PlatformWalletChangeSet, - PlatformWalletPersistence, + ClientStartState, ClientWalletStartState, PersistenceError, PersistenceErrorKind, + PlatformWalletChangeSet, PlatformWalletPersistence, }; use crate::test_support::{ funded_wallet_manager, funded_wallet_manager_dual_standard, @@ -1995,9 +2022,15 @@ mod tests { /// would, dropping what was buffered; the overtaken flush then /// succeeds over nothing. overtaken_by: Mutex>, + /// Whether the overtaking flush fails transiently instead, putting + /// back what it took: the overtaken flush then writes it. + overtaking_puts_back: std::sync::atomic::AtomicBool, /// While set, a flush fails for good: the backend drops what it /// buffered. flush_drops_buffer: std::sync::atomic::AtomicBool, + /// While set, a flush fails transiently: the backend keeps what it + /// buffered. + flush_keeps_buffer: std::sync::atomic::AtomicBool, } impl CapturingPersistence { @@ -2028,6 +2061,30 @@ mod tests { .collect() } + /// Outpoints of the asset-lock rows on disk, which a restart would + /// load: what the committed changesets leave, each one's removals + /// applied after its rows. + fn rows_on_disk(&self) -> Vec { + let mut rows = std::collections::BTreeSet::new(); + for locks in self + .stored + .lock() + .expect("capturing persistence mutex") + .iter() + .take( + self.committed_stores + .load(std::sync::atomic::Ordering::SeqCst), + ) + .filter_map(|cs| cs.asset_locks.as_ref()) + { + rows.extend(locks.asset_locks.keys().copied()); + for removed in &locks.removed { + rows.remove(removed); + } + } + rows.into_iter().collect() + } + /// Outpoints of asset-lock rows stored across all changesets. fn stored_outpoints(&self) -> Vec { self.stored @@ -2108,13 +2165,26 @@ mod tests { // Taken in its own statement: the flush below comes back here. let overtaking = self.overtaken_by.lock().unwrap().take(); if let Some(persister) = overtaking { - self.flush_drops_buffer - .store(true, std::sync::atomic::Ordering::SeqCst); - let overtook = persister.flush(); - self.flush_drops_buffer - .store(false, std::sync::atomic::Ordering::SeqCst); - assert!(overtook.is_err_and(|error| !error.is_transient())); - return Ok(()); + if self + .overtaking_puts_back + .load(std::sync::atomic::Ordering::SeqCst) + { + self.flush_keeps_buffer + .store(true, std::sync::atomic::Ordering::SeqCst); + let overtook = persister.flush(); + self.flush_keeps_buffer + .store(false, std::sync::atomic::Ordering::SeqCst); + assert!(overtook.is_err_and(|error| error.is_transient())); + // What it put back is written by this flush, below. + } else { + self.flush_drops_buffer + .store(true, std::sync::atomic::Ordering::SeqCst); + let overtook = persister.flush(); + self.flush_drops_buffer + .store(false, std::sync::atomic::Ordering::SeqCst); + assert!(overtook.is_err_and(|error| !error.is_transient())); + return Ok(()); + } } if self .flush_drops_buffer @@ -2122,6 +2192,15 @@ mod tests { { return Err(PersistenceError::backend("the database is corrupt")); } + if self + .flush_keeps_buffer + .load(std::sync::atomic::Ordering::SeqCst) + { + return Err(PersistenceError::backend_with_kind( + PersistenceErrorKind::Transient, + "the database is busy", + )); + } if self.fail_flush || self.fail_on_flush == Some(count) { return Err(PersistenceError::backend("simulated flush failure")); } @@ -3548,8 +3627,8 @@ mod tests { /// though the row's own flush then succeeded, over nothing: the recipient /// is recorded nowhere else, so a lock recovered after a kill would pay /// whatever address a resume names. The build is abandoned before - /// anything is broadcast, its row's removal queued, and its inputs come - /// back for the next one. + /// anything is broadcast, its row's removal stored behind it, and its + /// inputs come back for the next one. #[tokio::test] async fn should_not_send_a_shielded_funding_whose_recipient_row_another_flush_dropped() { let persistence = Arc::new(CapturingPersistence { @@ -3592,8 +3671,9 @@ mod tests { assert_eq!( persistence.removed_outpoints(), persistence.stored_outpoints(), - "the abandoned row's removal is queued behind it" + "the abandoned row's removal is stored behind it" ); + assert!(persistence.rows_on_disk().is_empty()); let (_, out_point) = fund() .await @@ -3610,6 +3690,121 @@ mod tests { ); } + /// A funding that pays a shielded recipient and is abandoned leaves no + /// row a restart would resume. Another flush of the wallet's records took + /// the row, failed transiently and put it back, and the row's own flush + /// then wrote it: the row is on disk, though not known to be, so the + /// build is abandoned. Its removal is written before the inputs are + /// released: only queued, it would be lost with the process, and the + /// restart would send a funding reported as not made. + #[tokio::test] + async fn should_leave_no_row_on_disk_for_a_shielded_funding_abandoned_after_its_row_was_written( + ) { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + overtaking_puts_back: std::sync::atomic::AtomicBool::new(true), + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + let refused = manager + .broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + &signer, + None, + None, + ) + .await; + assert!( + matches!(&refused, Err(PlatformWalletError::Persistence(message)) + if message.contains("shielded recipient was not made durable before dispatch") + && !message.contains("restart")), + "{refused:?}" + ); + assert!( + persistence.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + assert_eq!( + persistence.stored_outpoints().len(), + 1, + "the row was stored, and its own flush wrote it" + ); + assert!( + persistence.rows_on_disk().is_empty(), + "a restart finds no row to resume" + ); + } + + /// An abandoned shielded funding whose row may be on disk, and whose + /// removal is not known to be, is not reported as a plain failure: the + /// error says a restart may still resume it. Nothing was sent, so its + /// inputs come back for the next funding all the same. + #[tokio::test] + async fn should_say_a_restart_may_resume_an_abandoned_shielded_funding_whose_removal_is_not_durable( + ) { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + overtaking_puts_back: std::sync::atomic::AtomicBool::new(true), + // The row's flush, the one that overtakes it, then the removal's. + fail_on_flush: Some(3), + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + let fund = || { + manager.broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + &signer, + None, + None, + ) + }; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + let refused = fund().await; + assert!( + matches!(&refused, Err(PlatformWalletError::Persistence(message)) + if message.contains("shielded recipient was not made durable before dispatch") + && message.contains("a restart may still resume it")), + "{refused:?}" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the row is on disk, its removal is not" + ); + + fund() + .await + .expect("the abandoned build released its inputs"); + assert_eq!(broadcaster.calls(), 1); + } + /// The broadcast half returns when the broadcaster accepts the transaction: /// the tracked row is `Broadcast` (recoverable/resumable) and the invitation /// funding pool was persisted AND flushed — all BEFORE any proof wait (the diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs index 3994df9dc73..6a609296c9a 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs @@ -99,6 +99,16 @@ impl IdentityWallet { /// the flush that follows writes what was put back. Only when that /// fails too is the failure reported, as a store or flush that failed /// is. + /// + /// A store or flush of this call's own that the backend failed is not + /// tried again, as it never was here. There the outcome is known: a + /// flush that failed transiently kept the record buffered for a later + /// one to write, and any other failure is the backend refusing or + /// dropping what it was given, which its contract says not to retry + /// (`PlatformWalletPersistence::flush`). The two answers stored again + /// are the ones that leave it unknown whether a record this call stored + /// and flushed without an error is on disk, and the second store is + /// what settles that. fn record_funded_invitation(&self, entry: InvitationEntry) -> Result<(), PlatformWalletError> { let mut inv_cs = InvitationChangeSet::default(); inv_cs.invitations.insert(entry.out_point, entry); From e877a8a1475596129335b4521b45993bf8d135d8 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 14 Nov 2026 04:28:55 +0700 Subject: [PATCH 4/7] fix(platform-wallet): keep an abandoned shielded funding's inputs while its row may outlive its removal A funding that pays a shielded recipient is abandoned before its broadcast when its row is not known to be on disk, and the row's removal is then written before the inputs are released. Where that removal was not known to be on disk either, the inputs were released all the same and the call failed with a persistence error, while a restart could still find the row and send it: a caller told the funding failed builds it again, and the restart sends the first lock beside the second. The inputs now stay reserved and fenced there, and the call returns TransactionBroadcastUnconfirmed. They are still released where the row cannot outlive the call without its removal: the removal is on disk, the backend refused the row, or the row's own flush failed and the removal's did too, the two then being kept, written or dropped together. Co-Authored-By: Claude Fable 5.2 --- packages/rs-platform-wallet/src/error.rs | 6 +- .../src/wallet/asset_lock/build.rs | 202 ++++++++++++++---- 2 files changed, 163 insertions(+), 45 deletions(-) diff --git a/packages/rs-platform-wallet/src/error.rs b/packages/rs-platform-wallet/src/error.rs index eac4c960f44..ca0976bc8d4 100644 --- a/packages/rs-platform-wallet/src/error.rs +++ b/packages/rs-platform-wallet/src/error.rs @@ -194,8 +194,10 @@ pub enum PlatformWalletError { /// For Core transactions, the /// transaction may already have reached the network (transport timeout /// after delivery, partial peer send, or an internal multi-node retry - /// whose earlier attempt may have succeeded). The spent Core inputs are - /// intentionally kept out of the selectable set, so an immediate retry + /// whose earlier attempt may have succeeded). One case was not sent but + /// may still be: a shielded funding abandoned before its broadcast whose + /// row may be on disk, where a restart resumes it. The spent Core inputs + /// are intentionally kept out of the selectable set, so an immediate retry /// fails at input selection instead of double-spending. /// /// A Platform state transition may instead have been accepted but its diff --git a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs index d835d9710eb..a7ec7014c13 100644 --- a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs +++ b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs @@ -30,6 +30,7 @@ use crate::changeset::{ }; use crate::error::PlatformWalletError; use crate::wallet::core::InBroadcastPin; +use crate::wallet::persister::NotDurable; use crate::wallet::platform_wallet::PlatformWalletInfo; use crate::ASSET_LOCK_FUNDING_SOURCES; @@ -1258,8 +1259,8 @@ impl AssetLockManager { // competing build can sweep and re-reserve them across the durability // gate and the broadcast await below. Every // exit from here on settles it: released on the aborts that never - // reach the broadcaster and on a definitive rejection, left pending - // otherwise. + // reach the broadcaster and leave no row a restart could resume, + // and on a definitive rejection; left pending otherwise. let (tx, path, reservation_token, funding_accounts, in_broadcast_pin, funding_pools) = self .build_asset_lock_transaction_fenced( amount, @@ -1545,37 +1546,62 @@ impl AssetLockManager { ..Default::default() }); drop(build_persist_guard); - // Released either way: nothing was sent, and the reservation - // and the fence live in memory, which no restart that could - // resume the row keeps. Holding them would only make a retry - // here select other inputs, leaving the unsent transaction - // valid beside it. - self.abandon_unsent_build( - in_broadcast_pin, - &funding_accounts, - &tx, - reservation_token, - ) - .await; - return Err(PlatformWalletError::Persistence(match removed { - Ok(()) => { - format!("shielded recipient was not made durable before dispatch: {error}") - } - Err(removal) => { - tracing::error!( - %out_point, - %error, - %removal, - "an unsent shielded funding's row may be on disk, and its removal \ - is not known to be" - ); - format!( - "shielded recipient was not made durable before dispatch: {error}; \ - nothing was sent, but the removal of the unsent lock's row is not \ - known to be on disk, so a restart may still resume it: {removal}" + // Whether the row can outlive this call without its removal. + // Not once the removal is on disk. Not where the backend + // refused the row, which leaves nothing to remove. And not + // where the row's own flush failed and the removal's did + // too: the removal was then accepted behind the row, and the + // backend keeps, writes or drops the two together. Anything + // else leaves it open: the row's own flush succeeded beside + // another one, so the row may be on disk, or the removal was + // refused or met another flush while the row waited. + let removal = match (&error, removed) { + (_, Ok(())) + | (NotDurable::Store(_), Err(_)) + | (NotDurable::Flush(_), Err(NotDurable::Flush(_))) => { + // Nothing was sent and no row is left to resume: an + // abort like the pool gate's, and the inputs come + // back for the next build. + self.abandon_unsent_build( + in_broadcast_pin, + &funding_accounts, + &tx, + reservation_token, ) + .await; + return Err(PlatformWalletError::Persistence(format!( + "shielded recipient was not made durable before dispatch: {error}" + ))); } - })); + (_, Err(removal)) => removal, + }; + // The row may be on disk, or reach it with a later flush, + // where a restart resumes and sends it, so whether this + // funding is made is not known here. It is reported as that, + // not as a failure, and the inputs stay reserved and fenced + // as after a send that may have gone out: a row is never + // resumable while its inputs are re-spendable (see the + // broadcast below), and a caller told the funding failed + // would build it again from other inputs, a second lock + // beside the one the restart sends. The fence ends on an + // observed spend, which only a restart that finds the row + // can bring; one that does not finds the inputs free. + tracing::error!( + %out_point, + %error, + %removal, + "an unsent shielded funding's row may be on disk, and its removal is \ + not known to be; keeping its inputs reserved" + ); + in_broadcast_pin.settle_pending_spend(); + return Err(PlatformWalletError::TransactionBroadcastUnconfirmed( + format!( + "asset lock {out_point} was not sent, but its row may be on disk and \ + the removal of that row is not known to be, so a restart may still \ + resume it; its inputs stay reserved. Shielded recipient was not \ + made durable before dispatch: {error}; removal: {removal}" + ), + )); } self.track_asset_lock(tracked).await; } else { @@ -3750,12 +3776,13 @@ mod tests { } /// An abandoned shielded funding whose row may be on disk, and whose - /// removal is not known to be, is not reported as a plain failure: the - /// error says a restart may still resume it. Nothing was sent, so its - /// inputs come back for the next funding all the same. + /// removal is not known to be, is not reported as a failure: a restart + /// may still resume and send it, so the outcome is unknown and the error + /// says so. Its inputs stay reserved: released, the next funding would + /// spend them, or be built from others beside a lock the restart sends. #[tokio::test] - async fn should_say_a_restart_may_resume_an_abandoned_shielded_funding_whose_removal_is_not_durable( - ) { + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_removal_is_not_durable() + { let persistence = Arc::new(CapturingPersistence { records_shielded_recipients: true, overtaking_puts_back: std::sync::atomic::AtomicBool::new(true), @@ -3786,9 +3813,8 @@ mod tests { let refused = fund().await; assert!( - matches!(&refused, Err(PlatformWalletError::Persistence(message)) - if message.contains("shielded recipient was not made durable before dispatch") - && message.contains("a restart may still resume it")), + matches!(&refused, Err(PlatformWalletError::TransactionBroadcastUnconfirmed(message)) + if message.contains("a restart may still resume it")), "{refused:?}" ); assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); @@ -3799,10 +3825,100 @@ mod tests { "the row is on disk, its removal is not" ); - fund() - .await - .expect("the abandoned build released its inputs"); - assert_eq!(broadcaster.calls(), 1); + // The unsent transaction's inputs stay fenced as after a send that + // may have gone out, so a sweep of the reservation does not free them. + let unsent = persistence + .stored + .lock() + .expect("capturing persistence mutex") + .iter() + .filter_map(|cs| cs.asset_locks.as_ref()) + .flat_map(|locks| locks.asset_locks.values()) + .map(|row| row.transaction.clone()) + .next() + .expect("the row was stored"); + { + let wm = manager.wallet_manager.read().await; + let (_, info) = wm + .get_wallet_and_info(&manager.wallet_id) + .expect("wallet still present"); + assert!(info.generation.in_broadcast_conflict(&unsent).is_some()); + } + + // The one coin the wallet holds stays reserved too, so the next + // funding finds nothing to select. + let rebuild = fund().await; + assert!( + matches!( + rebuild, + Err(PlatformWalletError::AssetLockInsufficientFunds { available: 0, .. }) + ), + "{rebuild:?}" + ); + assert_eq!(broadcaster.calls(), 0); + } + + /// The same where the row waits in the buffer, its own flush having + /// failed transiently, and the backend then refuses its removal: the next + /// flush writes the row with nothing behind it, and a restart resumes it. + /// So this outcome is unknown too, and the inputs stay reserved. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_removal_was_refused() { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + flush_keeps_buffer: std::sync::atomic::AtomicBool::new(true), + fail_remove_store: true, + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + let fund = || { + manager.broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + &signer, + None, + None, + ) + }; + + let refused = fund().await; + assert!( + matches!(&refused, Err(PlatformWalletError::TransactionBroadcastUnconfirmed(message)) + if message.contains("a restart may still resume it")), + "{refused:?}" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + assert!(persistence.removed_outpoints().is_empty()); + + persistence + .flush_keeps_buffer + .store(false, std::sync::atomic::Ordering::SeqCst); + manager.persister.flush().expect("the backend recovered"); + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the later flush wrote the row, and nothing removes it" + ); + + let rebuild = fund().await; + assert!( + matches!( + rebuild, + Err(PlatformWalletError::AssetLockInsufficientFunds { available: 0, .. }) + ), + "{rebuild:?}" + ); + assert_eq!(broadcaster.calls(), 0); } /// The broadcast half returns when the broadcaster accepts the transaction: From 366ef664f8f6a0eccc4a755b8d6b521d37b292b2 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 14 Nov 2026 04:35:11 +0700 Subject: [PATCH 5/7] refactor(platform-wallet): report a payment round that is not durable through the shared mapping `DashPayView::store_durably` wrote out how each `NotDurable` answer becomes a wallet error. The base branch now has that mapping as `WalletPersister::classify_not_durable`, so the view calls it and the mapping lives in one place. What a caller gets back is unchanged. Co-Authored-By: Claude Fable 5.2 --- .../identity/network/prepared_payment.rs | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs b/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs index 6f0faf1b725..fb3f4a6a83b 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs @@ -74,14 +74,11 @@ use super::payments::{ }; use super::DashPayView; use crate::broadcaster::TransactionBroadcaster; -use crate::changeset::{ - account_address_pool_entries, PersistenceError, PersistenceErrorKind, PlatformWalletChangeSet, -}; +use crate::changeset::{account_address_pool_entries, PlatformWalletChangeSet}; use crate::error::PlatformWalletError; use crate::util::now_secs; use crate::wallet::core::{CoreWallet, SignedCoreTransaction, WalletGeneration}; use crate::wallet::identity::types::dashpay::payment::PaymentEntry; -use crate::wallet::persister::NotDurable; use crate::wallet::platform_wallet::PlatformWalletInfo; use crate::wallet::reservations::reservation_expired; use crate::SEND_FUNDING_SOURCES; @@ -745,19 +742,7 @@ impl DashPayView<'_, B> { ) -> Result<(), PlatformWalletError> { self.persister .store_durably(changeset) - .map_err(|not_durable| match not_durable { - NotDurable::Store(e) => self.persister.classify_store_failure(e), - NotDurable::Flush(e) => PlatformWalletError::from_flush_failure(e), - NotDurable::Lost => PlatformWalletError::from_flush_failure( - PersistenceError::backend_with_kind(PersistenceErrorKind::Fatal, not_durable), - ), - NotDurable::OtherFlushUnsettled => { - PlatformWalletError::from_flush_failure(PersistenceError::backend_with_kind( - PersistenceErrorKind::Transient, - not_durable, - )) - } - }) + .map_err(|not_durable| self.persister.classify_not_durable(not_durable)) } /// The wallet info of `generation`, or `WalletNotFound` when the wallet From 88b8cba500f78b6ac5bd5742360535d6ff99d52f Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 14 Nov 2026 13:35:08 +0700 Subject: [PATCH 6/7] fix(platform-wallet): release an abandoned shielded funding's inputs only once its row's removal is on disk An abandoned shielded funding still handed its inputs back, with the removal of its row not known to be on disk, where the backend had refused the row, and where the row's own flush and the removal's had both failed. Neither proves the row is gone. A host that does not roll a failed round back may have written the row before a later part of the store failed (the FFI persister calls the asset-lock callback before the recipient sidecar's), a backend that keeps a failed store or flush for later writes the row then, and a flush that failed may have run beside another that wrote it. A restart that finds the row resumes and sends it. The inputs are now released only when the removal is known to be on disk. Every other failed gate keeps the reservation, leaves the fence pending and returns TransactionBroadcastUnconfirmed. should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_durable pinned the hand-back for a backend that failed the removal too. It now fails the row's store or flush once and lets the removal through, which still hands the inputs back. Co-Authored-By: Claude Fable 5.2 --- .../src/wallet/asset_lock/build.rs | 165 +++++++++++++++--- .../src/wallet/shielded/wallet_drain_tests.rs | 70 +++++++- 2 files changed, 205 insertions(+), 30 deletions(-) diff --git a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs index a7ec7014c13..e1cf9bd5f2b 100644 --- a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs +++ b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs @@ -30,7 +30,6 @@ use crate::changeset::{ }; use crate::error::PlatformWalletError; use crate::wallet::core::InBroadcastPin; -use crate::wallet::persister::NotDurable; use crate::wallet::platform_wallet::PlatformWalletInfo; use crate::ASSET_LOCK_FUNDING_SOURCES; @@ -1546,19 +1545,8 @@ impl AssetLockManager { ..Default::default() }); drop(build_persist_guard); - // Whether the row can outlive this call without its removal. - // Not once the removal is on disk. Not where the backend - // refused the row, which leaves nothing to remove. And not - // where the row's own flush failed and the removal's did - // too: the removal was then accepted behind the row, and the - // backend keeps, writes or drops the two together. Anything - // else leaves it open: the row's own flush succeeded beside - // another one, so the row may be on disk, or the removal was - // refused or met another flush while the row waited. - let removal = match (&error, removed) { - (_, Ok(())) - | (NotDurable::Store(_), Err(_)) - | (NotDurable::Flush(_), Err(NotDurable::Flush(_))) => { + let removal = match removed { + Ok(()) => { // Nothing was sent and no row is left to resume: an // abort like the pool gate's, and the inputs come // back for the next build. @@ -1573,14 +1561,20 @@ impl AssetLockManager { "shielded recipient was not made durable before dispatch: {error}" ))); } - (_, Err(removal)) => removal, + Err(removal) => removal, }; - // The row may be on disk, or reach it with a later flush, - // where a restart resumes and sends it, so whether this - // funding is made is not known here. It is reported as that, - // not as a failure, and the inputs stay reserved and fenced - // as after a send that may have gone out: a row is never - // resumable while its inputs are re-spendable (see the + // Only the removal on disk proves the row is gone. No way the + // row's own store or flush failed does: a backend that does + // not roll a failed round back may have written the row + // before a later part of the store failed, one that keeps a + // failed store or flush for later writes the row then, and a + // flush that failed may have run beside another that wrote + // it. So the row may be on disk, or reach it with a later + // flush, where a restart resumes and sends it, and whether + // this funding is made is not known here. It is reported as + // that, not as a failure, and the inputs stay reserved and + // fenced as after a send that may have gone out: a row is + // never resumable while its inputs are re-spendable (see the // broadcast below), and a caller told the funding failed // would build it again from other inputs, a second lock // beside the one the restart sends. The fence ends on an @@ -2057,6 +2051,13 @@ mod tests { /// While set, a flush fails transiently: the backend keeps what it /// buffered. flush_keeps_buffer: std::sync::atomic::AtomicBool, + /// Whether the overtaking flush succeeds instead, writing what was + /// buffered: the overtaken flush then fails. + overtaking_commits: std::sync::atomic::AtomicBool, + /// A store of asset-lock rows writes them and then fails, as on a + /// host that does not roll a failed round back and whose later + /// callback failed. + lock_store_fails_after_writing: bool, } impl CapturingPersistence { @@ -2174,12 +2175,22 @@ mod tests { hook(); } } + let fails_after_writing = self.lock_store_fails_after_writing + && changeset + .asset_locks + .as_ref() + .is_some_and(|locks| !locks.asset_locks.is_empty()); let mut stored = self.stored.lock().expect("capturing persistence mutex"); stored.push(changeset); - if self.commits_inline { + if self.commits_inline || fails_after_writing { self.committed_stores .store(stored.len(), std::sync::atomic::Ordering::SeqCst); } + if fails_after_writing { + return Err(PersistenceError::backend( + "simulated failure after the row was written", + )); + } Ok(()) } @@ -2192,6 +2203,12 @@ mod tests { let overtaking = self.overtaken_by.lock().unwrap().take(); if let Some(persister) = overtaking { if self + .overtaking_commits + .load(std::sync::atomic::Ordering::SeqCst) + { + persister.flush().expect("the overtaking flush commits"); + return Err(PersistenceError::backend("simulated flush failure")); + } else if self .overtaking_puts_back .load(std::sync::atomic::Ordering::SeqCst) { @@ -3921,6 +3938,110 @@ mod tests { assert_eq!(broadcaster.calls(), 0); } + /// Funds a shielded recipient over a backend set up to fail, expecting + /// the build to be abandoned with its outcome unknown and its inputs + /// kept: nothing is sent or tracked, and the next funding finds the one + /// coin the wallet holds still reserved. + async fn expect_an_abandoned_shielded_funding_to_keep_its_inputs( + manager: &AssetLockManager, + signer: &WalletSigner, + broadcaster: &CountingOkBroadcaster, + ) { + let fund = || { + manager.broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + signer, + None, + None, + ) + }; + let refused = fund().await; + assert!( + matches!(&refused, Err(PlatformWalletError::TransactionBroadcastUnconfirmed(message)) + if message.contains("a restart may still resume it")), + "{refused:?}" + ); + assert!(manager.list_tracked_locks().await.is_empty()); + let rebuild = fund().await; + assert!( + matches!( + rebuild, + Err(PlatformWalletError::AssetLockInsufficientFunds { available: 0, .. }) + ), + "{rebuild:?}" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + } + + /// The same on a host that does not roll a failed round back: the row is + /// written before a later part of its store fails, so the backend + /// refuses the store with the row on disk, and refuses the removal too. + /// A refused store is no proof that nothing was written. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_refused_store_wrote_its_row( + ) { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + commits_inline: true, + lock_store_fails_after_writing: true, + fail_remove_store: true, + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + + expect_an_abandoned_shielded_funding_to_keep_its_inputs(&manager, &signer, &broadcaster) + .await; + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the refused store left its row on disk" + ); + } + + /// The same where another flush of the wallet's records wrote the row + /// just before the row's own flush failed, and the removal's flush fails + /// too: neither answer says the row is on disk, and it is. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_row_another_flush_wrote() + { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + overtaking_commits: std::sync::atomic::AtomicBool::new(true), + // The row's flush, the one that overtakes it, then the removal's. + fail_on_flush: Some(3), + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + expect_an_abandoned_shielded_funding_to_keep_its_inputs(&manager, &signer, &broadcaster) + .await; + assert!( + persistence.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the other flush wrote the row, and nothing wrote its removal" + ); + } + /// The broadcast half returns when the broadcaster accepts the transaction: /// the tracked row is `Broadcast` (recoverable/resumable) and the invitation /// funding pool was persisted AND flushed — all BEFORE any proof wait (the diff --git a/packages/rs-platform-wallet/src/wallet/shielded/wallet_drain_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/wallet_drain_tests.rs index b5ca65d2429..83577b60b77 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/wallet_drain_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/wallet_drain_tests.rs @@ -92,6 +92,10 @@ struct RecoveryPersistence { staged: Mutex>, fail_store: AtomicBool, fail_flush: AtomicBool, + /// Fails the next store that carries an asset-lock row, and only it. + fail_next_lock_store: AtomicBool, + /// Fails the next flush only. + fail_next_flush: AtomicBool, commits_inline: AtomicBool, fail_load: AtomicBool, records_recipients: AtomicBool, @@ -114,7 +118,13 @@ impl PlatformWalletPersistence for RecoveryPersistence { _wallet_id: [u8; 32], cs: PlatformWalletChangeSet, ) -> Result<(), PersistenceError> { - if self.fail_store.load(Ordering::SeqCst) { + if self.fail_store.load(Ordering::SeqCst) + || (cs + .asset_locks + .as_ref() + .is_some_and(|locks| !locks.asset_locks.is_empty()) + && self.fail_next_lock_store.swap(false, Ordering::SeqCst)) + { return Err(PersistenceError::backend("store failed")); } if let Some(cs) = cs.asset_locks { @@ -132,7 +142,9 @@ impl PlatformWalletPersistence for RecoveryPersistence { Ok(()) } fn flush(&self, _wallet_id: [u8; 32]) -> Result<(), PersistenceError> { - if self.fail_flush.load(Ordering::SeqCst) { + if self.fail_flush.load(Ordering::SeqCst) + || self.fail_next_flush.swap(false, Ordering::SeqCst) + { return Err(PersistenceError::backend("flush failed")); } self.commit_staged(); @@ -241,6 +253,8 @@ async fn signed_wallet_fixture( staged: Mutex::new(BTreeMap::new()), fail_store: AtomicBool::new(false), fail_flush: AtomicBool::new(false), + fail_next_lock_store: AtomicBool::new(false), + fail_next_flush: AtomicBool::new(false), // Drains admit inline-committing backends only; the buffered mode // exists for `should_refuse_drain_on_a_buffered_persistence_backend`. commits_inline: AtomicBool::new(true), @@ -1172,9 +1186,10 @@ async fn fund_typed_amount( /// A typed-amount funding whose recipient cannot be made durable stops before /// the broadcaster: a rejected store and a rejected flush of a buffered store /// both fail the call with a persistence error, leave the lock untracked and -/// uncommitted, and hand the inputs back so the next attempt builds again. -/// The abandoned row of a failed flush is withdrawn from the buffer, so the -/// next successful flush does not commit it. +/// uncommitted, and, once the abandoned row's removal is on disk, hand the +/// inputs back so the next attempt builds again. The abandoned row of a +/// failed flush is withdrawn from the buffer, so the flush that writes its +/// removal does not commit it. #[tokio::test] async fn should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_durable() { for fail_flush in [false, true] { @@ -1185,10 +1200,11 @@ async fn should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_dur signed_wallet_fixture(&lock, keys).await; let coordinator = Arc::new(coordinator); persistence.commits_inline.store(false, Ordering::SeqCst); + // The row's own store or flush fails; its removal's goes through. let switch = if fail_flush { - &persistence.fail_flush + &persistence.fail_next_flush } else { - &persistence.fail_store + &persistence.fail_next_lock_store }; switch.store(true, Ordering::SeqCst); @@ -1209,7 +1225,6 @@ async fn should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_dur // The inputs were handed back: the retry builds a lock again and gets // as far as the broadcaster. - switch.store(false, Ordering::SeqCst); let (error, retried) = fund_typed_amount(&wallet, &coordinator, &signer, recipient).await; assert!( !matches!(error, PlatformWalletError::Persistence(_)), @@ -1228,6 +1243,45 @@ async fn should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_dur } } +/// Where the backend keeps failing, so the abandoned row's removal cannot be +/// confirmed either, the funding is not reported as failed: a store or flush +/// that failed is no proof the row is on no disk, and a restart that finds +/// it resumes and sends it. The outcome is unknown, for a rejected store and +/// for a rejected flush alike, and nothing is sent or tracked. +#[tokio::test] +async fn should_report_a_typed_amount_funding_as_unconfirmed_when_its_row_removal_fails_too() { + for fail_flush in [false, true] { + let (mut lock, keys) = drain_lock_fixture(); + lock.operation_id = None; + let recipient = crate::test_support::orchard_address_fixture(0x5a); + let (wallet, persistence, coordinator, _dir, signer) = + signed_wallet_fixture(&lock, keys).await; + let coordinator = Arc::new(coordinator); + persistence.commits_inline.store(false, Ordering::SeqCst); + let switch = if fail_flush { + &persistence.fail_flush + } else { + &persistence.fail_store + }; + + switch.store(true, Ordering::SeqCst); + let (error, seen) = fund_typed_amount(&wallet, &coordinator, &signer, recipient).await; + assert!( + matches!(&error, PlatformWalletError::TransactionBroadcastUnconfirmed(message) + if message.contains("a restart may still resume it")), + "fail_flush={fail_flush}: {error}" + ); + assert_eq!(seen.len(), 1, "fail_flush={fail_flush}"); + assert!(!wallet + .asset_locks + .list_tracked_locks() + .await + .iter() + .any(|tracked| tracked.out_point == seen[0])); + assert!(!persistence.committed.lock().unwrap().contains_key(&seen[0])); + } +} + /// A backend that cannot record a recipient keeps the best-effort row every /// other funding has: a rejected store does not stop the dispatch. #[tokio::test] From c76073b1c724ae54945bf9b09949689db94f12aa Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Sat, 14 Nov 2026 16:43:31 +0700 Subject: [PATCH 7/7] fix(platform-wallet): do not take a row's removal as proof while a flush that may hold the row is in progress An abandoned shielded funding releases its inputs once the removal of its row is known to be on disk. That removal went through `store_durably`, which answers for the removal alone. A flush still in progress may have taken the row before the removal was stored, and nothing says a backend writes what that flush took before what a later one takes: it could write the row after the removal and return before the ledger was read, and a flush that succeeded is counted nowhere. The inputs were then released with the row on disk for a restart to resume. The removal now goes through `WalletPersister::withdraw_durably`, which also refuses to answer, and flushes nothing, while a flush is in progress once the removal is stored. With none in progress then, every flush that could hold the row alone has returned, and one that begins later takes the removal with whatever of the row is still buffered. The `AssetLockBuiltFn` comment said every failure but a rejection leaves the lock tracked and resumable in the session. A shielded funding abandoned before its broadcast is not tracked, and the comment now says so. Co-Authored-By: Claude Fable 5.2 --- .../src/asset_lock/built_callback.rs | 11 +- .../src/wallet/asset_lock/build.rs | 108 +++++++++++++- .../src/wallet/persister.rs | 134 +++++++++++++++++- 3 files changed, 246 insertions(+), 7 deletions(-) diff --git a/packages/rs-platform-wallet-ffi/src/asset_lock/built_callback.rs b/packages/rs-platform-wallet-ffi/src/asset_lock/built_callback.rs index 07414555005..4eb6df4220d 100644 --- a/packages/rs-platform-wallet-ffi/src/asset_lock/built_callback.rs +++ b/packages/rs-platform-wallet-ffi/src/asset_lock/built_callback.rs @@ -22,8 +22,15 @@ use crate::asset_lock_persistence::outpoint_to_bytes; /// result then tells what happened to the lock: /// `ErrorTransactionBroadcastRejected` (26) means it was removed again; any /// other failure leaves it tracked and resumable by this outpoint (a resume -/// may find it already consumed). Both hold for this session; the row store -/// and removal behind them are best-effort, so after a restart a failed +/// may find it already consumed), with one exception. A shielded funding +/// whose recipient cannot be made durable is abandoned before it is tracked +/// or sent, so a resume in this session finds no lock +/// (`ErrorAssetLockNotTracked`, 23). With its row removed again the call +/// fails with a persistence error. Where that removal is not known to be on +/// disk it fails with `ErrorTransactionBroadcastUnconfirmed` (20): the +/// inputs stay reserved, and a restart may still find the row and resume +/// the lock. All of this holds for this session; the row store +/// and removal behind it are best-effort, so after a restart a failed /// store can leave no row for a broadcast lock, or the row of a removed one. /// That holds for the exports that take this callback today: shielded /// funding and platform-address funding from the wallet balance or an diff --git a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs index e1cf9bd5f2b..a27217b5722 100644 --- a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs +++ b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs @@ -1535,9 +1535,12 @@ impl AssetLockManager { // queued behind the row, it would be lost with the process // where the row was not. A backend writes what it buffered in // the order it was stored, and applies removals after - // upserts, so once the removal is on disk the row is not. The + // upserts, so once the removal is on disk the row is not: + // unless a flush still in progress took the row without it + // and writes it afterwards, which the removal's store does + // not answer for (`WalletPersister::withdraw_durably`). The // funding index stays consumed, as for the pool gate above. - let removed = self.persister.store_durably(PlatformWalletChangeSet { + let removed = self.persister.withdraw_durably(PlatformWalletChangeSet { asset_locks: Some(AssetLockChangeSet { removed: [out_point].into(), ..Default::default() @@ -1563,7 +1566,8 @@ impl AssetLockManager { } Err(removal) => removal, }; - // Only the removal on disk proves the row is gone. No way the + // Only the removal on disk, with no flush left that could + // write the row after it, proves the row is gone. No way the // row's own store or flush failed does: a backend that does // not roll a failed round back may have written the row // before a later part of the store failed, one that keeps a @@ -1898,7 +1902,7 @@ fn map_builder_error(e: AssetLockError, requested: u64) -> PlatformWalletError { #[cfg(test)] mod tests { - use std::sync::{Arc, Mutex}; + use std::sync::{mpsc, Arc, Mutex}; use std::time::Duration; use dashcore::OutPoint; @@ -2058,6 +2062,16 @@ mod tests { /// host that does not roll a failed round back and whose later /// callback failed. lock_store_fails_after_writing: bool, + /// Once set, the next flush finds what was buffered taken by a flush + /// through this persister, begun on another thread, that has not + /// returned: it succeeds over nothing. The other flush writes what + /// it took, behind anything written meanwhile, when it is let go + /// (`let_stalled_flush_write`, or the next flush that writes). + stalled_by: Mutex>, + /// Where the flush begun on the other thread stalls. + stall_point: Mutex, mpsc::Receiver<()>)>>, + /// The stalled flush: how to let it go, and how to hear it returned. + stalled_flush: Mutex, mpsc::Receiver<()>)>>, } impl CapturingPersistence { @@ -2112,6 +2126,18 @@ mod tests { rows.into_iter().collect() } + /// Lets a flush stalled on another thread write what it took, and + /// waits until it has returned. Does nothing when none is stalled. + fn let_stalled_flush_write(&self) { + let stalled = self.stalled_flush.lock().unwrap().take(); + if let Some((let_go, has_returned)) = stalled { + let_go.send(()).expect("the stalled flush waits"); + has_returned + .recv_timeout(Duration::from_secs(10)) + .expect("the stalled flush returns"); + } + } + /// Outpoints of asset-lock rows stored across all changesets. fn stored_outpoints(&self) -> Vec { self.stored @@ -2199,6 +2225,38 @@ mod tests { .flushes .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + // The flush begun on another thread: it took what was buffered, + // and writes it only once it is let go. + let stall_point = self.stall_point.lock().unwrap().take(); + if let Some((entered, let_go)) = stall_point { + let taken = self.stored.lock().unwrap().clone(); + entered.send(()).expect("the flush it overtook waits"); + let_go + .recv_timeout(Duration::from_secs(10)) + .expect("let go"); + let mut stored = self.stored.lock().unwrap(); + stored.extend(taken); + self.committed_stores + .store(stored.len(), std::sync::atomic::Ordering::SeqCst); + return Ok(()); + } + let stalled_by = self.stalled_by.lock().unwrap().take(); + if let Some(persister) = stalled_by { + let (entered, has_entered) = mpsc::channel(); + let (let_go, released) = mpsc::channel(); + let (returned, has_returned) = mpsc::channel(); + *self.stall_point.lock().unwrap() = Some((entered, released)); + std::thread::spawn(move || { + let flushed = persister.flush(); + assert!(flushed.is_ok(), "{flushed:?}"); + let _ = returned.send(()); + }); + has_entered + .recv_timeout(Duration::from_secs(10)) + .expect("the other flush is inside the backend"); + *self.stalled_flush.lock().unwrap() = Some((let_go, has_returned)); + return Ok(()); + } // Taken in its own statement: the flush below comes back here. let overtaking = self.overtaken_by.lock().unwrap().take(); if let Some(persister) = overtaking { @@ -2251,6 +2309,9 @@ mod tests { self.stored.lock().unwrap().len(), std::sync::atomic::Ordering::SeqCst, ); + // A flush still stalled writes what it took behind this one, and + // has returned by the time this one does. + self.let_stalled_flush_write(); Ok(()) } @@ -4042,6 +4103,45 @@ mod tests { ); } + /// The same where a flush still in progress took the row: the removal + /// stored behind it would be written first, and the row after it, by a + /// flush that then returns without a failure for anything to see. The + /// removal is not answered for while that flush is in progress, so the + /// inputs stay reserved, and the flush writes the row in the end. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_row_a_flush_in_progress_took( + ) { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + *persistence.stalled_by.lock().unwrap() = Some(manager.persister.clone()); + + expect_an_abandoned_shielded_funding_to_keep_its_inputs(&manager, &signer, &broadcaster) + .await; + assert!( + persistence.stalled_by.lock().unwrap().is_none(), + "a flush took the row and stalled" + ); + assert!( + persistence.rows_on_disk().is_empty(), + "nothing is on disk while that flush is in progress" + ); + + persistence.let_stalled_flush_write(); + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the flush that took the row wrote it in the end" + ); + } + /// The broadcast half returns when the broadcaster accepts the transaction: /// the tracked row is `Broadcast` (recoverable/resumable) and the invitation /// funding pool was persisted AND flushed — all BEFORE any proof wait (the diff --git a/packages/rs-platform-wallet/src/wallet/persister.rs b/packages/rs-platform-wallet/src/wallet/persister.rs index e30ca11ce18..8f7f236f3ec 100644 --- a/packages/rs-platform-wallet/src/wallet/persister.rs +++ b/packages/rs-platform-wallet/src/wallet/persister.rs @@ -720,8 +720,44 @@ impl WalletPersister { &self, changeset: PlatformWalletChangeSet, ) -> Result<(), NotDurable> { - let (lost_before, failed_before) = (self.flush_ledger.lost(), self.flush_ledger.failed()); + let failures_before = (self.flush_ledger.lost(), self.flush_ledger.failed()); self.store(changeset).map_err(NotDurable::Store)?; + self.flush_what_was_stored(failures_before) + } + + /// [`Self::store_durably`] for a changeset that withdraws a record stored + /// before it (a row's removal), where `Ok` has to mean more than that + /// the withdrawal is on disk: that nothing writes the record after it. + /// + /// A flush in progress when the withdrawal is stored may have taken the + /// record without it, and nothing says a backend writes what that flush + /// took before what a later one takes. Should it write the record after + /// the withdrawal and return before the ledger is read, it leaves no + /// trace there: a flush that succeeded is counted nowhere. So a flush in + /// progress once the withdrawal is stored answers + /// [`NotDurable::OtherFlushUnsettled`], and nothing is flushed. With + /// none in progress then, every flush that could hold the record alone + /// has returned, and one that begins later takes the withdrawal with + /// whatever of the record is still buffered, the withdrawal behind it. + pub(crate) fn withdraw_durably( + &self, + changeset: PlatformWalletChangeSet, + ) -> Result<(), NotDurable> { + let failures_before = (self.flush_ledger.lost(), self.flush_ledger.failed()); + self.store(changeset).map_err(NotDurable::Store)?; + if !self.store_commits_inline() && self.flush_ledger.in_flight() != 0 { + return Err(NotDurable::OtherFlushUnsettled); + } + self.flush_what_was_stored(failures_before) + } + + /// The flush half of [`Self::store_durably`], for a changeset just + /// stored: `lost_before` and `failed_before` are the ledger's counts + /// read before that store. + fn flush_what_was_stored( + &self, + (lost_before, failed_before): (u64, u64), + ) -> Result<(), NotDurable> { if self.store_commits_inline() { return Ok(()); } @@ -1378,6 +1414,102 @@ mod tests { assert!(after.is_ok(), "{after:?}"); } + /// A withdrawal is not answered for while another flush of the wallet's + /// records is inside the backend when it is stored: that flush may have + /// taken the record being withdrawn, and may write it after the + /// withdrawal. Nothing is flushed for it. A plain store made durable in + /// the same place is answered for, rightly for the store itself: its own + /// flush wrote it, and the other one returned without failing. + #[test] + fn should_not_answer_for_a_withdrawal_stored_while_another_flush_has_not_returned() { + /// The first flush stalls inside the backend. The second lets it + /// go once it has written, and returns only after the first has: so + /// no flush is in progress, and none failed, when the second returns. + struct LateFirstFlush { + flushes: AtomicU64, + entered: std::sync::Mutex>, + release: std::sync::Mutex>, + let_go: std::sync::Mutex>, + first_returned: std::sync::Mutex>, + } + impl PlatformWalletPersistence for LateFirstFlush { + fn store( + &self, + _wallet_id: WalletId, + _changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + Ok(()) + } + fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { + if self.flushes.fetch_add(1, Ordering::SeqCst) != 0 { + self.let_go + .lock() + .expect("let go") + .send(()) + .expect("the first flush waits"); + self.first_returned + .lock() + .expect("first returned") + .recv_timeout(Duration::from_secs(10)) + .expect("the first flush returns"); + return Ok(()); + } + self.entered + .lock() + .expect("entered") + .send(()) + .expect("the test waits"); + self.release + .lock() + .expect("release") + .recv_timeout(Duration::from_secs(10)) + .expect("released"); + Ok(()) + } + fn load(&self) -> Result { + Ok(ClientStartState::default()) + } + } + + let (entered, has_entered) = mpsc::channel(); + let (let_go, release) = mpsc::channel(); + let (returned, first_returned) = mpsc::channel(); + let backend = Arc::new(LateFirstFlush { + flushes: AtomicU64::new(0), + entered: std::sync::Mutex::new(entered), + release: std::sync::Mutex::new(release), + let_go: std::sync::Mutex::new(let_go), + first_returned: std::sync::Mutex::new(first_returned), + }); + let persister = WalletPersister::new([0; 32], Arc::clone(&backend) as _); + let stalled = { + let persister = persister.clone(); + std::thread::spawn(move || { + let flushed = persister.flush(); + returned.send(()).expect("the second flush waits"); + flushed + }) + }; + has_entered + .recv_timeout(Duration::from_secs(10)) + .expect("the first flush is inside the backend"); + + let withdrawn = persister.withdraw_durably(PlatformWalletChangeSet::default()); + assert!( + matches!(withdrawn, Err(NotDurable::OtherFlushUnsettled)), + "{withdrawn:?}" + ); + assert_eq!( + backend.flushes.load(Ordering::SeqCst), + 1, + "nothing was flushed for the withdrawal" + ); + + let stored = persister.store_durably(PlatformWalletChangeSet::default()); + assert!(stalled.join().expect("the first flush returns").is_ok()); + assert!(stored.is_ok(), "{stored:?}"); + } + /// The host's flush runs with no lock of the persister held: a flush that /// notes another read-back commitment through the same persister, as a /// reentrant backend or another thread would, completes, and nothing a