diff --git a/packages/rs-platform-wallet-ffi/src/asset_lock/built_callback.rs b/packages/rs-platform-wallet-ffi/src/asset_lock/built_callback.rs index 07414555005..4eb6df4220d 100644 --- a/packages/rs-platform-wallet-ffi/src/asset_lock/built_callback.rs +++ b/packages/rs-platform-wallet-ffi/src/asset_lock/built_callback.rs @@ -22,8 +22,15 @@ use crate::asset_lock_persistence::outpoint_to_bytes; /// result then tells what happened to the lock: /// `ErrorTransactionBroadcastRejected` (26) means it was removed again; any /// other failure leaves it tracked and resumable by this outpoint (a resume -/// may find it already consumed). Both hold for this session; the row store -/// and removal behind them are best-effort, so after a restart a failed +/// may find it already consumed), with one exception. A shielded funding +/// whose recipient cannot be made durable is abandoned before it is tracked +/// or sent, so a resume in this session finds no lock +/// (`ErrorAssetLockNotTracked`, 23). With its row removed again the call +/// fails with a persistence error. Where that removal is not known to be on +/// disk it fails with `ErrorTransactionBroadcastUnconfirmed` (20): the +/// inputs stay reserved, and a restart may still find the row and resume +/// the lock. All of this holds for this session; the row store +/// and removal behind it are best-effort, so after a restart a failed /// store can leave no row for a broadcast lock, or the row of a removed one. /// That holds for the exports that take this callback today: shielded /// funding and platform-address funding from the wallet balance or an diff --git a/packages/rs-platform-wallet/src/error.rs b/packages/rs-platform-wallet/src/error.rs index eac4c960f44..ca0976bc8d4 100644 --- a/packages/rs-platform-wallet/src/error.rs +++ b/packages/rs-platform-wallet/src/error.rs @@ -194,8 +194,10 @@ pub enum PlatformWalletError { /// For Core transactions, the /// transaction may already have reached the network (transport timeout /// after delivery, partial peer send, or an internal multi-node retry - /// whose earlier attempt may have succeeded). The spent Core inputs are - /// intentionally kept out of the selectable set, so an immediate retry + /// whose earlier attempt may have succeeded). One case was not sent but + /// may still be: a shielded funding abandoned before its broadcast whose + /// row may be on disk, where a restart resumes it. The spent Core inputs + /// are intentionally kept out of the selectable set, so an immediate retry /// fails at input selection instead of double-spending. /// /// A Platform state transition may instead have been accepted but its diff --git a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs index 57a6ae3b46a..a27217b5722 100644 --- a/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs +++ b/packages/rs-platform-wallet/src/wallet/asset_lock/build.rs @@ -1258,8 +1258,8 @@ impl AssetLockManager { // competing build can sweep and re-reserve them across the durability // gate and the broadcast await below. Every // exit from here on settles it: released on the aborts that never - // reach the broadcaster and on a definitive rejection, left pending - // otherwise. + // reach the broadcaster and leave no row a restart could resume, + // and on a definitive rejection; left pending otherwise. let (tx, path, reservation_token, funding_accounts, in_broadcast_pin, funding_pools) = self .build_asset_lock_transaction_fenced( amount, @@ -1383,17 +1383,30 @@ impl AssetLockManager { // `store()` alone is only a buffer hint under the persistence contract // (backends may defer I/O until `flush`), so the invitation gate also // drives `flush()` — the contract's durability boundary — before - // anything hits the wire. + // anything hits the wire. That flush succeeding is no proof by itself: + // another flush of the wallet's records may have taken the buffered + // pool and failed for good, dropping it, so the gate trusts only a + // store no other flush can have emptied + // (`WalletPersister::store_durably`). // Aborting BEFORE broadcast is harmless (no tx on the wire); the other // asset-lock accounts keep their keys on-device, so they stay best-effort. - let pool_durability = match self.persist_asset_lock_account_pools(funding_pools) { - Ok(()) - if funding_type == AssetLockFundingType::IdentityInvitation - || operation.is_some() => - { - self.persister.flush() - } - other => other, + let pool_durability = if operation.is_some() { + // A drain runs only where a store is the commit + // (`require_wallet_drain_persistence`), so no flush can drop its + // pools: the flush here only notifies the host. + self.persist_asset_lock_account_pools(funding_pools) + .and_then(|()| self.persister.flush()) + .map_err(|e| e.to_string()) + } else if funding_type == AssetLockFundingType::IdentityInvitation { + self.persister + .store_durably(PlatformWalletChangeSet { + account_address_pools: funding_pools, + ..Default::default() + }) + .map_err(|e| e.to_string()) + } else { + self.persist_asset_lock_account_pools(funding_pools) + .map_err(|e| e.to_string()) }; if let Err(e) = pool_durability { tracing::error!(error = %e, "failed to persist asset-lock funding index"); @@ -1503,37 +1516,90 @@ impl AssetLockManager { } let mut changeset = AssetLockChangeSet::default(); changeset.asset_locks.insert(out_point, (&tracked).into()); - let durable = self - .persister - .store(PlatformWalletChangeSet { - asset_locks: Some(changeset), - ..Default::default() - }) - .and_then(|()| self.persister.flush()); + // Durable only once no other flush of the wallet's records can + // have dropped the row (`WalletPersister::store_durably`): a flush + // that succeeded over a buffer another one emptied would send a + // lock whose recipient is on no disk. + let durable = self.persister.store_durably(PlatformWalletChangeSet { + asset_locks: Some(changeset), + ..Default::default() + }); if let Err(error) = durable { // The row may be buffered (a backend that restores a failed - // flush to its buffer) or committed (a store that succeeded - // before a failed flush), and a later flush or a restart would - // then surface a resumable row for inputs released below. Its - // removal is queued behind it — stores apply removals after - // upserts — so the abandoned row does not outlive this call - // unless the backend refuses that write too. The funding index - // stays consumed, as for the pool gate above. - self.queue_asset_lock_changeset(AssetLockChangeSet { - removed: [out_point].into(), + // flush to its buffer) or on disk (a store that succeeded + // before a failed flush, the row's own flush where another + // one failed or was in progress beside it, or a flush still + // in progress that took the row), and a restart would then + // resume a funding reported here as not made. So its removal + // is made durable too, before the inputs are released: only + // queued behind the row, it would be lost with the process + // where the row was not. A backend writes what it buffered in + // the order it was stored, and applies removals after + // upserts, so once the removal is on disk the row is not: + // unless a flush still in progress took the row without it + // and writes it afterwards, which the removal's store does + // not answer for (`WalletPersister::withdraw_durably`). The + // funding index stays consumed, as for the pool gate above. + let removed = self.persister.withdraw_durably(PlatformWalletChangeSet { + asset_locks: Some(AssetLockChangeSet { + removed: [out_point].into(), + ..Default::default() + }), ..Default::default() }); drop(build_persist_guard); - self.abandon_unsent_build( - in_broadcast_pin, - &funding_accounts, - &tx, - reservation_token, - ) - .await; - return Err(PlatformWalletError::Persistence(format!( - "shielded recipient was not made durable before dispatch: {error}" - ))); + let removal = match removed { + Ok(()) => { + // Nothing was sent and no row is left to resume: an + // abort like the pool gate's, and the inputs come + // back for the next build. + self.abandon_unsent_build( + in_broadcast_pin, + &funding_accounts, + &tx, + reservation_token, + ) + .await; + return Err(PlatformWalletError::Persistence(format!( + "shielded recipient was not made durable before dispatch: {error}" + ))); + } + Err(removal) => removal, + }; + // Only the removal on disk, with no flush left that could + // write the row after it, proves the row is gone. No way the + // row's own store or flush failed does: a backend that does + // not roll a failed round back may have written the row + // before a later part of the store failed, one that keeps a + // failed store or flush for later writes the row then, and a + // flush that failed may have run beside another that wrote + // it. So the row may be on disk, or reach it with a later + // flush, where a restart resumes and sends it, and whether + // this funding is made is not known here. It is reported as + // that, not as a failure, and the inputs stay reserved and + // fenced as after a send that may have gone out: a row is + // never resumable while its inputs are re-spendable (see the + // broadcast below), and a caller told the funding failed + // would build it again from other inputs, a second lock + // beside the one the restart sends. The fence ends on an + // observed spend, which only a restart that finds the row + // can bring; one that does not finds the inputs free. + tracing::error!( + %out_point, + %error, + %removal, + "an unsent shielded funding's row may be on disk, and its removal is \ + not known to be; keeping its inputs reserved" + ); + in_broadcast_pin.settle_pending_spend(); + return Err(PlatformWalletError::TransactionBroadcastUnconfirmed( + format!( + "asset lock {out_point} was not sent, but its row may be on disk and \ + the removal of that row is not known to be, so a restart may still \ + resume it; its inputs stay reserved. Shielded recipient was not \ + made durable before dispatch: {error}; removal: {removal}" + ), + )); } self.track_asset_lock(tracked).await; } else { @@ -1836,7 +1902,7 @@ fn map_builder_error(e: AssetLockError, requested: u64) -> PlatformWalletError { #[cfg(test)] mod tests { - use std::sync::{Arc, Mutex}; + use std::sync::{mpsc, Arc, Mutex}; use std::time::Duration; use dashcore::OutPoint; @@ -1861,8 +1927,8 @@ mod tests { use crate::broadcaster::{BroadcastError, TransactionBroadcaster}; use crate::changeset::{ - ClientStartState, ClientWalletStartState, PersistenceError, PlatformWalletChangeSet, - PlatformWalletPersistence, + ClientStartState, ClientWalletStartState, PersistenceError, PersistenceErrorKind, + PlatformWalletChangeSet, PlatformWalletPersistence, }; use crate::test_support::{ funded_wallet_manager, funded_wallet_manager_dual_standard, @@ -1973,6 +2039,39 @@ mod tests { committed_stores: std::sync::atomic::AtomicUsize, fail_lock_store: std::sync::atomic::AtomicBool, failed_lock_stores: Mutex>, + /// Whether the backend also attests `ASSET_LOCK_SHIELDED_RECIPIENTS`. + records_shielded_recipients: bool, + /// Once set, the next flush is overtaken: before it commits, a flush + /// through this persister fails for good, as one from another thread + /// would, dropping what was buffered; the overtaken flush then + /// succeeds over nothing. + overtaken_by: Mutex>, + /// Whether the overtaking flush fails transiently instead, putting + /// back what it took: the overtaken flush then writes it. + overtaking_puts_back: std::sync::atomic::AtomicBool, + /// While set, a flush fails for good: the backend drops what it + /// buffered. + flush_drops_buffer: std::sync::atomic::AtomicBool, + /// While set, a flush fails transiently: the backend keeps what it + /// buffered. + flush_keeps_buffer: std::sync::atomic::AtomicBool, + /// Whether the overtaking flush succeeds instead, writing what was + /// buffered: the overtaken flush then fails. + overtaking_commits: std::sync::atomic::AtomicBool, + /// A store of asset-lock rows writes them and then fails, as on a + /// host that does not roll a failed round back and whose later + /// callback failed. + lock_store_fails_after_writing: bool, + /// Once set, the next flush finds what was buffered taken by a flush + /// through this persister, begun on another thread, that has not + /// returned: it succeeds over nothing. The other flush writes what + /// it took, behind anything written meanwhile, when it is let go + /// (`let_stalled_flush_write`, or the next flush that writes). + stalled_by: Mutex>, + /// Where the flush begun on the other thread stalls. + stall_point: Mutex, mpsc::Receiver<()>)>>, + /// The stalled flush: how to let it go, and how to hear it returned. + stalled_flush: Mutex, mpsc::Receiver<()>)>>, } impl CapturingPersistence { @@ -2003,6 +2102,42 @@ mod tests { .collect() } + /// Outpoints of the asset-lock rows on disk, which a restart would + /// load: what the committed changesets leave, each one's removals + /// applied after its rows. + fn rows_on_disk(&self) -> Vec { + let mut rows = std::collections::BTreeSet::new(); + for locks in self + .stored + .lock() + .expect("capturing persistence mutex") + .iter() + .take( + self.committed_stores + .load(std::sync::atomic::Ordering::SeqCst), + ) + .filter_map(|cs| cs.asset_locks.as_ref()) + { + rows.extend(locks.asset_locks.keys().copied()); + for removed in &locks.removed { + rows.remove(removed); + } + } + rows.into_iter().collect() + } + + /// Lets a flush stalled on another thread write what it took, and + /// waits until it has returned. Does nothing when none is stalled. + fn let_stalled_flush_write(&self) { + let stalled = self.stalled_flush.lock().unwrap().take(); + if let Some((let_go, has_returned)) = stalled { + let_go.send(()).expect("the stalled flush waits"); + has_returned + .recv_timeout(Duration::from_secs(10)) + .expect("the stalled flush returns"); + } + } + /// Outpoints of asset-lock rows stored across all changesets. fn stored_outpoints(&self) -> Vec { self.stored @@ -2021,7 +2156,14 @@ mod tests { } fn persistence_capabilities(&self) -> crate::changeset::PersistenceCapabilities { - crate::changeset::PersistenceCapabilities::WALLET_DRAIN + let drain = crate::changeset::PersistenceCapabilities::WALLET_DRAIN; + if self.records_shielded_recipients { + drain.union( + crate::changeset::PersistenceCapabilities::ASSET_LOCK_SHIELDED_RECIPIENTS, + ) + } else { + drain + } } fn store( @@ -2059,12 +2201,22 @@ mod tests { hook(); } } + let fails_after_writing = self.lock_store_fails_after_writing + && changeset + .asset_locks + .as_ref() + .is_some_and(|locks| !locks.asset_locks.is_empty()); let mut stored = self.stored.lock().expect("capturing persistence mutex"); stored.push(changeset); - if self.commits_inline { + if self.commits_inline || fails_after_writing { self.committed_stores .store(stored.len(), std::sync::atomic::Ordering::SeqCst); } + if fails_after_writing { + return Err(PersistenceError::backend( + "simulated failure after the row was written", + )); + } Ok(()) } @@ -2073,6 +2225,83 @@ mod tests { .flushes .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + // The flush begun on another thread: it took what was buffered, + // and writes it only once it is let go. + let stall_point = self.stall_point.lock().unwrap().take(); + if let Some((entered, let_go)) = stall_point { + let taken = self.stored.lock().unwrap().clone(); + entered.send(()).expect("the flush it overtook waits"); + let_go + .recv_timeout(Duration::from_secs(10)) + .expect("let go"); + let mut stored = self.stored.lock().unwrap(); + stored.extend(taken); + self.committed_stores + .store(stored.len(), std::sync::atomic::Ordering::SeqCst); + return Ok(()); + } + let stalled_by = self.stalled_by.lock().unwrap().take(); + if let Some(persister) = stalled_by { + let (entered, has_entered) = mpsc::channel(); + let (let_go, released) = mpsc::channel(); + let (returned, has_returned) = mpsc::channel(); + *self.stall_point.lock().unwrap() = Some((entered, released)); + std::thread::spawn(move || { + let flushed = persister.flush(); + assert!(flushed.is_ok(), "{flushed:?}"); + let _ = returned.send(()); + }); + has_entered + .recv_timeout(Duration::from_secs(10)) + .expect("the other flush is inside the backend"); + *self.stalled_flush.lock().unwrap() = Some((let_go, has_returned)); + return Ok(()); + } + // Taken in its own statement: the flush below comes back here. + let overtaking = self.overtaken_by.lock().unwrap().take(); + if let Some(persister) = overtaking { + if self + .overtaking_commits + .load(std::sync::atomic::Ordering::SeqCst) + { + persister.flush().expect("the overtaking flush commits"); + return Err(PersistenceError::backend("simulated flush failure")); + } else if self + .overtaking_puts_back + .load(std::sync::atomic::Ordering::SeqCst) + { + self.flush_keeps_buffer + .store(true, std::sync::atomic::Ordering::SeqCst); + let overtook = persister.flush(); + self.flush_keeps_buffer + .store(false, std::sync::atomic::Ordering::SeqCst); + assert!(overtook.is_err_and(|error| error.is_transient())); + // What it put back is written by this flush, below. + } else { + self.flush_drops_buffer + .store(true, std::sync::atomic::Ordering::SeqCst); + let overtook = persister.flush(); + self.flush_drops_buffer + .store(false, std::sync::atomic::Ordering::SeqCst); + assert!(overtook.is_err_and(|error| !error.is_transient())); + return Ok(()); + } + } + if self + .flush_drops_buffer + .load(std::sync::atomic::Ordering::SeqCst) + { + return Err(PersistenceError::backend("the database is corrupt")); + } + if self + .flush_keeps_buffer + .load(std::sync::atomic::Ordering::SeqCst) + { + return Err(PersistenceError::backend_with_kind( + PersistenceErrorKind::Transient, + "the database is busy", + )); + } if self.fail_flush || self.fail_on_flush == Some(count) { return Err(PersistenceError::backend("simulated flush failure")); } @@ -2080,6 +2309,9 @@ mod tests { self.stored.lock().unwrap().len(), std::sync::atomic::Ordering::SeqCst, ); + // A flush still stalled writes what it took behind this one, and + // has returned by the time this one does. + self.let_stalled_flush_write(); Ok(()) } @@ -3443,6 +3675,473 @@ mod tests { ); } + /// The invitation gate does not trust a flush that succeeded over + /// nothing: another flush of the wallet's records took the buffered + /// funding pool first and failed for good, dropping it, so the funding + /// index is on no disk and a restart would export the same voucher key + /// again. The build is abandoned before anything is broadcast, and its + /// inputs come back for the next one. + #[tokio::test] + async fn should_abort_an_invitation_whose_funding_index_another_flush_dropped() { + let persistence = Arc::new(CapturingPersistence::default()); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + let refused = manager + .broadcast_funded_asset_lock( + 1_000_000, + 0, + AssetLockFundingType::IdentityInvitation, + 0, + &signer, + ) + .await; + assert!( + matches!(&refused, Err(PlatformWalletError::AssetLockTransaction(message)) + if message.contains("aborted before broadcast")), + "{refused:?}" + ); + assert!( + persistence.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + + manager + .broadcast_funded_asset_lock( + 1_000_000, + 0, + AssetLockFundingType::IdentityInvitation, + 0, + &signer, + ) + .await + .expect("the abandoned build released its inputs, and this pool is durable"); + assert_eq!(broadcaster.calls(), 1); + } + + /// A funding that pays a shielded recipient is not sent when another + /// flush of the wallet's records took its row first and failed for good, + /// though the row's own flush then succeeded, over nothing: the recipient + /// is recorded nowhere else, so a lock recovered after a kill would pay + /// whatever address a resume names. The build is abandoned before + /// anything is broadcast, its row's removal stored behind it, and its + /// inputs come back for the next one. + #[tokio::test] + async fn should_not_send_a_shielded_funding_whose_recipient_row_another_flush_dropped() { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + let fund = || { + manager.broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + &signer, + None, + None, + ) + }; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + let refused = fund().await; + assert!( + matches!(&refused, Err(PlatformWalletError::Persistence(message)) + if message.contains("shielded recipient was not made durable before dispatch")), + "{refused:?}" + ); + assert!( + persistence.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + assert_eq!( + persistence.removed_outpoints(), + persistence.stored_outpoints(), + "the abandoned row's removal is stored behind it" + ); + assert!(persistence.rows_on_disk().is_empty()); + + let (_, out_point) = fund() + .await + .expect("the abandoned build released its inputs, and this row is durable"); + assert_eq!(broadcaster.calls(), 1); + assert_eq!( + manager + .list_tracked_locks() + .await + .iter() + .map(|lock| (lock.out_point, lock.shielded_recipient)) + .collect::>(), + vec![(out_point, Some([7; 43]))] + ); + } + + /// A funding that pays a shielded recipient and is abandoned leaves no + /// row a restart would resume. Another flush of the wallet's records took + /// the row, failed transiently and put it back, and the row's own flush + /// then wrote it: the row is on disk, though not known to be, so the + /// build is abandoned. Its removal is written before the inputs are + /// released: only queued, it would be lost with the process, and the + /// restart would send a funding reported as not made. + #[tokio::test] + async fn should_leave_no_row_on_disk_for_a_shielded_funding_abandoned_after_its_row_was_written( + ) { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + overtaking_puts_back: std::sync::atomic::AtomicBool::new(true), + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + let refused = manager + .broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + &signer, + None, + None, + ) + .await; + assert!( + matches!(&refused, Err(PlatformWalletError::Persistence(message)) + if message.contains("shielded recipient was not made durable before dispatch") + && !message.contains("restart")), + "{refused:?}" + ); + assert!( + persistence.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + assert_eq!( + persistence.stored_outpoints().len(), + 1, + "the row was stored, and its own flush wrote it" + ); + assert!( + persistence.rows_on_disk().is_empty(), + "a restart finds no row to resume" + ); + } + + /// An abandoned shielded funding whose row may be on disk, and whose + /// removal is not known to be, is not reported as a failure: a restart + /// may still resume and send it, so the outcome is unknown and the error + /// says so. Its inputs stay reserved: released, the next funding would + /// spend them, or be built from others beside a lock the restart sends. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_removal_is_not_durable() + { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + overtaking_puts_back: std::sync::atomic::AtomicBool::new(true), + // The row's flush, the one that overtakes it, then the removal's. + fail_on_flush: Some(3), + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + let fund = || { + manager.broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + &signer, + None, + None, + ) + }; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + let refused = fund().await; + assert!( + matches!(&refused, Err(PlatformWalletError::TransactionBroadcastUnconfirmed(message)) + if message.contains("a restart may still resume it")), + "{refused:?}" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the row is on disk, its removal is not" + ); + + // The unsent transaction's inputs stay fenced as after a send that + // may have gone out, so a sweep of the reservation does not free them. + let unsent = persistence + .stored + .lock() + .expect("capturing persistence mutex") + .iter() + .filter_map(|cs| cs.asset_locks.as_ref()) + .flat_map(|locks| locks.asset_locks.values()) + .map(|row| row.transaction.clone()) + .next() + .expect("the row was stored"); + { + let wm = manager.wallet_manager.read().await; + let (_, info) = wm + .get_wallet_and_info(&manager.wallet_id) + .expect("wallet still present"); + assert!(info.generation.in_broadcast_conflict(&unsent).is_some()); + } + + // The one coin the wallet holds stays reserved too, so the next + // funding finds nothing to select. + let rebuild = fund().await; + assert!( + matches!( + rebuild, + Err(PlatformWalletError::AssetLockInsufficientFunds { available: 0, .. }) + ), + "{rebuild:?}" + ); + assert_eq!(broadcaster.calls(), 0); + } + + /// The same where the row waits in the buffer, its own flush having + /// failed transiently, and the backend then refuses its removal: the next + /// flush writes the row with nothing behind it, and a restart resumes it. + /// So this outcome is unknown too, and the inputs stay reserved. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_removal_was_refused() { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + flush_keeps_buffer: std::sync::atomic::AtomicBool::new(true), + fail_remove_store: true, + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + let fund = || { + manager.broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + &signer, + None, + None, + ) + }; + + let refused = fund().await; + assert!( + matches!(&refused, Err(PlatformWalletError::TransactionBroadcastUnconfirmed(message)) + if message.contains("a restart may still resume it")), + "{refused:?}" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + assert!(manager.list_tracked_locks().await.is_empty()); + assert!(persistence.removed_outpoints().is_empty()); + + persistence + .flush_keeps_buffer + .store(false, std::sync::atomic::Ordering::SeqCst); + manager.persister.flush().expect("the backend recovered"); + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the later flush wrote the row, and nothing removes it" + ); + + let rebuild = fund().await; + assert!( + matches!( + rebuild, + Err(PlatformWalletError::AssetLockInsufficientFunds { available: 0, .. }) + ), + "{rebuild:?}" + ); + assert_eq!(broadcaster.calls(), 0); + } + + /// Funds a shielded recipient over a backend set up to fail, expecting + /// the build to be abandoned with its outcome unknown and its inputs + /// kept: nothing is sent or tracked, and the next funding finds the one + /// coin the wallet holds still reserved. + async fn expect_an_abandoned_shielded_funding_to_keep_its_inputs( + manager: &AssetLockManager, + signer: &WalletSigner, + broadcaster: &CountingOkBroadcaster, + ) { + let fund = || { + manager.broadcast_funded_asset_lock_tracked( + super::AssetLockBuildAmount::Exact(1_000_000), + &ASSET_LOCK_FUNDING_SOURCES, + 0, + AssetLockFundingType::AssetLockShieldedAddressTopUp, + 0, + Some([7; 43]), + signer, + None, + None, + ) + }; + let refused = fund().await; + assert!( + matches!(&refused, Err(PlatformWalletError::TransactionBroadcastUnconfirmed(message)) + if message.contains("a restart may still resume it")), + "{refused:?}" + ); + assert!(manager.list_tracked_locks().await.is_empty()); + let rebuild = fund().await; + assert!( + matches!( + rebuild, + Err(PlatformWalletError::AssetLockInsufficientFunds { available: 0, .. }) + ), + "{rebuild:?}" + ); + assert_eq!(broadcaster.calls(), 0, "nothing reached the wire"); + } + + /// The same on a host that does not roll a failed round back: the row is + /// written before a later part of its store fails, so the backend + /// refuses the store with the row on disk, and refuses the removal too. + /// A refused store is no proof that nothing was written. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_refused_store_wrote_its_row( + ) { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + commits_inline: true, + lock_store_fails_after_writing: true, + fail_remove_store: true, + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + + expect_an_abandoned_shielded_funding_to_keep_its_inputs(&manager, &signer, &broadcaster) + .await; + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the refused store left its row on disk" + ); + } + + /// The same where another flush of the wallet's records wrote the row + /// just before the row's own flush failed, and the removal's flush fails + /// too: neither answer says the row is on disk, and it is. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_row_another_flush_wrote() + { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + overtaking_commits: std::sync::atomic::AtomicBool::new(true), + // The row's flush, the one that overtakes it, then the removal's. + fail_on_flush: Some(3), + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + *persistence.overtaken_by.lock().unwrap() = Some(manager.persister.clone()); + + expect_an_abandoned_shielded_funding_to_keep_its_inputs(&manager, &signer, &broadcaster) + .await; + assert!( + persistence.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the other flush wrote the row, and nothing wrote its removal" + ); + } + + /// The same where a flush still in progress took the row: the removal + /// stored behind it would be written first, and the row after it, by a + /// flush that then returns without a failure for anything to see. The + /// removal is not answered for while that flush is in progress, so the + /// inputs stay reserved, and the flush writes the row in the end. + #[tokio::test] + async fn should_keep_the_inputs_of_an_abandoned_shielded_funding_whose_row_a_flush_in_progress_took( + ) { + let persistence = Arc::new(CapturingPersistence { + records_shielded_recipients: true, + ..Default::default() + }); + let broadcaster = Arc::new(CountingOkBroadcaster::default()); + let (manager, signer) = funded_asset_lock_manager_with_persistence( + Arc::clone(&broadcaster), + Arc::clone(&persistence), + ) + .await; + *persistence.stalled_by.lock().unwrap() = Some(manager.persister.clone()); + + expect_an_abandoned_shielded_funding_to_keep_its_inputs(&manager, &signer, &broadcaster) + .await; + assert!( + persistence.stalled_by.lock().unwrap().is_none(), + "a flush took the row and stalled" + ); + assert!( + persistence.rows_on_disk().is_empty(), + "nothing is on disk while that flush is in progress" + ); + + persistence.let_stalled_flush_write(); + assert_eq!( + persistence.rows_on_disk().len(), + 1, + "the flush that took the row wrote it in the end" + ); + } + /// The broadcast half returns when the broadcaster accepts the transaction: /// the tracked row is `Broadcast` (recoverable/resumable) and the invitation /// funding pool was persisted AND flushed — all BEFORE any proof wait (the diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs index 1c158cf7248..6a609296c9a 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs @@ -24,6 +24,7 @@ use key_wallet::wallet::managed_wallet_info::asset_lock_builder::AssetLockFundin use crate::changeset::{ InvitationChangeSet, InvitationEntry, InvitationStatus, PersistenceCapabilities, + PlatformWalletChangeSet, }; use dash_sdk::platform::transition::put_settings::PutSettings; @@ -35,6 +36,7 @@ pub use dash_sdk::platform::dashpay::{ use crate::error::PlatformWalletError; use crate::wallet::identity::crypto::{encode_invitation_uri, InviterInfo, ParsedInvitation}; use crate::wallet::identity::network::contact_requests::ContactCryptoProvider; +use crate::wallet::persister::NotDurable; use super::*; @@ -83,6 +85,52 @@ fn funding_index_from_path(path: &DerivationPath) -> Option { } impl IdentityWallet { + /// Record the inviter-side row of a voucher that is already funded, and + /// answer `Ok` only once it is on disk. + /// + /// A flush that succeeded is not trusted by itself + /// (`WalletPersister::store_durably`): another flush of the wallet's + /// records may have taken the record and failed for good, dropping it, + /// and the link handed out afterwards would then be for a voucher no + /// restart could list or reclaim. The voucher is funded by now, so a + /// record another flush dropped, or still holds or put back, is stored + /// once more rather than given up: the row is keyed by outpoint, so + /// storing it again changes nothing where the first one did land, and + /// the flush that follows writes what was put back. Only when that + /// fails too is the failure reported, as a store or flush that failed + /// is. + /// + /// A store or flush of this call's own that the backend failed is not + /// tried again, as it never was here. There the outcome is known: a + /// flush that failed transiently kept the record buffered for a later + /// one to write, and any other failure is the backend refusing or + /// dropping what it was given, which its contract says not to retry + /// (`PlatformWalletPersistence::flush`). The two answers stored again + /// are the ones that leave it unknown whether a record this call stored + /// and flushed without an error is on disk, and the second store is + /// what settles that. + fn record_funded_invitation(&self, entry: InvitationEntry) -> Result<(), PlatformWalletError> { + let mut inv_cs = InvitationChangeSet::default(); + inv_cs.invitations.insert(entry.out_point, entry); + let record = || PlatformWalletChangeSet { + invitations: Some(inv_cs.clone()), + ..Default::default() + }; + let recorded = match self.persister.store_durably(record()) { + Err(NotDurable::Lost | NotDurable::OtherFlushUnsettled) => { + self.persister.store_durably(record()) + } + recorded => recorded, + }; + recorded.map_err(|e| { + PlatformWalletError::AssetLockTransaction(format!( + "the invitation voucher is funded but its record could not be \ + persisted (it would be missing from Sent invitations and \ + unreclaimable): {e}" + )) + }) + } + /// Create a DashPay invitation: fund a one-time asset-lock voucher at the /// DIP-13 invitation path and return a shareable `dashpay://invite` link. /// @@ -198,32 +246,15 @@ impl IdentityWallet { // for the invitation account), warn-skip: an untrackable row can't be // reclaimed either way. if let Some(funding_index) = funding_index_from_path(&path) { - let mut inv_cs = InvitationChangeSet::default(); - inv_cs.invitations.insert( + self.record_funded_invitation(InvitationEntry { out_point, - InvitationEntry { - out_point, - funding_index, - amount_duffs, - expiry_unix, - created_at_secs, - has_inviter: inviter.is_some(), - status: InvitationStatus::Created, - }, - ); - self.persister - .store(crate::changeset::PlatformWalletChangeSet { - invitations: Some(inv_cs), - ..Default::default() - }) - .and_then(|_| self.persister.flush()) - .map_err(|e| { - PlatformWalletError::AssetLockTransaction(format!( - "the invitation voucher is funded but its record could not be \ - persisted (it would be missing from Sent invitations and \ - unreclaimable): {e}" - )) - })?; + funding_index, + amount_duffs, + expiry_unix, + created_at_secs, + has_inviter: inviter.is_some(), + status: InvitationStatus::Created, + })?; } else { tracing::warn!( "invitation funding path has no u32 index tail; \ @@ -560,4 +591,250 @@ mod tests { ); } } + + // --- create_invitation: the funded voucher's record --- + + mod funded_record { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + use dashcore::{OutPoint, Txid}; + use key_wallet::mnemonic::Mnemonic; + use key_wallet::wallet::initialization::WalletAccountCreationOptions; + use key_wallet::Network; + + use crate::changeset::{ + ClientStartState, InvitationEntry, InvitationStatus, PersistenceError, + PersistenceErrorKind, PlatformWalletChangeSet, PlatformWalletPersistence, + }; + use crate::events::{EventHandler, PlatformEventHandler}; + use crate::wallet::persister::WalletPersister; + use crate::wallet::platform_wallet::WalletId; + use crate::{PlatformWallet, PlatformWalletError}; + + const TEST_MNEMONIC: &str = "abandon abandon abandon abandon abandon abandon \ + abandon abandon abandon abandon abandon about"; + + struct NoopEventHandler; + impl EventHandler for NoopEventHandler {} + impl PlatformEventHandler for NoopEventHandler {} + + /// A backend that buffers what is stored until a flush writes it to + /// `durable`; a flush that fails for good drops the buffer, and one + /// that fails transiently keeps it. + #[derive(Default)] + struct BufferingPersistence { + buffered: Mutex>, + durable: Mutex>, + /// One handle per flush still to be overtaken: a flush through + /// the handle, as one from another thread would, takes what was + /// buffered before such a flush writes, and fails; the overtaken + /// flush then succeeds over nothing. + overtaken_by: Mutex>, + /// Whether the overtaking flush fails transiently, putting back + /// what it took once the overtaken one is done, or for good, + /// dropping it. + overtaking_puts_back: AtomicBool, + /// While set, a flush is the overtaking one, and fails. + overtaking: AtomicBool, + } + + impl BufferingPersistence { + /// The outpoints of the invitation records on disk, in order. + fn durable_invitations(&self) -> Vec { + self.durable + .lock() + .expect("durable") + .iter() + .filter_map(|changeset| changeset.invitations.as_ref()) + .flat_map(|invitations| invitations.invitations.keys().copied()) + .collect() + } + } + + impl PlatformWalletPersistence for BufferingPersistence { + fn store( + &self, + _wallet_id: WalletId, + changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + self.buffered.lock().expect("buffered").push(changeset); + Ok(()) + } + + fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { + let puts_back = self.overtaking_puts_back.load(Ordering::SeqCst); + if self.overtaking.load(Ordering::SeqCst) { + return Err(if puts_back { + PersistenceError::backend_with_kind( + PersistenceErrorKind::Transient, + "the disk is busy", + ) + } else { + PersistenceError::backend("the database is corrupt") + }); + } + // Taken in its own statement: the flush below comes back here. + let overtaking = self.overtaken_by.lock().expect("overtaken").pop(); + if let Some(persister) = overtaking { + // The other flush holds what was buffered while this one + // runs, over nothing. + let taken = std::mem::take(&mut *self.buffered.lock().expect("buffered")); + self.overtaking.store(true, Ordering::SeqCst); + let overtook = persister.flush(); + self.overtaking.store(false, Ordering::SeqCst); + assert!(overtook.is_err_and(|error| error.is_transient() == puts_back)); + if puts_back { + self.buffered.lock().expect("buffered").extend(taken); + } + return Ok(()); + } + let flushed = std::mem::take(&mut *self.buffered.lock().expect("buffered")); + self.durable.lock().expect("durable").extend(flushed); + Ok(()) + } + + fn load(&self) -> Result { + Ok(ClientStartState::default()) + } + } + + async fn wallet_on(persistence: Arc) -> Arc { + let sdk = Arc::new(dash_sdk::SdkBuilder::new_mock().build().expect("mock sdk")); + let handler: Arc = Arc::new(NoopEventHandler); + let manager = Arc::new(crate::PlatformWalletManager::new(sdk, persistence, handler)); + let seed = Mnemonic::from_phrase(TEST_MNEMONIC) + .expect("valid mnemonic") + .to_seed(""); + manager + .create_wallet_from_seed_bytes( + Network::Testnet, + &seed, + WalletAccountCreationOptions::None, + Some(0), + ) + .await + .expect("wallet creation") + } + + fn funded_voucher() -> InvitationEntry { + InvitationEntry { + out_point: OutPoint::new(Txid::from_byte_array([7; 32]), 0), + funding_index: 0, + amount_duffs: 1_000_000, + expiry_unix: 1, + created_at_secs: 1, + has_inviter: false, + status: InvitationStatus::Created, + } + } + + /// A funded voucher's record is not taken as recorded on the strength + /// of a flush that succeeded over nothing, another flush of the + /// wallet's records having taken it first and failed for good: it is + /// stored once more, and is on disk when the call answers. + #[tokio::test] + async fn should_record_a_funded_invitation_again_when_another_flush_dropped_its_record() { + let persistence = Arc::new(BufferingPersistence::default()); + let wallet = wallet_on(Arc::clone(&persistence)).await; + let iw = wallet.identity(); + let voucher = funded_voucher(); + persistence + .overtaken_by + .lock() + .expect("overtaken") + .push(iw.persister.clone()); + + iw.record_funded_invitation(voucher.clone()) + .expect("stored once more, and durable"); + + assert!( + persistence + .overtaken_by + .lock() + .expect("overtaken") + .is_empty(), + "a flush was overtaken" + ); + assert_eq!( + persistence.durable_invitations(), + vec![voucher.out_point], + "the record is on disk" + ); + } + + /// A funded voucher's record is not taken as recorded while another + /// flush of the wallet's records holds it, a flush that then fails + /// transiently and puts it back: the record is stored once more, and + /// the flush that follows writes it. + #[tokio::test] + async fn should_record_a_funded_invitation_another_flush_took_and_put_back() { + let persistence = Arc::new(BufferingPersistence::default()); + let wallet = wallet_on(Arc::clone(&persistence)).await; + let iw = wallet.identity(); + let voucher = funded_voucher(); + persistence + .overtaking_puts_back + .store(true, Ordering::SeqCst); + persistence + .overtaken_by + .lock() + .expect("overtaken") + .push(iw.persister.clone()); + + iw.record_funded_invitation(voucher.clone()) + .expect("stored once more, and durable"); + + assert!( + persistence + .overtaken_by + .lock() + .expect("overtaken") + .is_empty(), + "a flush was overtaken" + ); + let on_disk = persistence.durable_invitations(); + assert!( + !on_disk.is_empty() + && on_disk + .iter() + .all(|out_point| *out_point == voucher.out_point), + "the record is on disk: {on_disk:?}" + ); + } + + /// When the record stored once more is dropped by another flush as + /// well, the voucher is reported funded and unrecorded, as a flush + /// that failed reports it, and no link follows: it is not reported + /// recorded while on no disk. + #[tokio::test] + async fn should_report_a_funded_invitation_unrecorded_when_another_flush_dropped_it_twice() + { + let persistence = Arc::new(BufferingPersistence::default()); + let wallet = wallet_on(Arc::clone(&persistence)).await; + let iw = wallet.identity(); + persistence + .overtaken_by + .lock() + .expect("overtaken") + .extend([iw.persister.clone(), iw.persister.clone()]); + + let unrecorded = iw.record_funded_invitation(funded_voucher()); + + assert!( + matches!(&unrecorded, Err(PlatformWalletError::AssetLockTransaction(message)) + if message.contains("funded but its record could not be persisted")), + "{unrecorded:?}" + ); + assert!( + persistence + .overtaken_by + .lock() + .expect("overtaken") + .is_empty(), + "both flushes were overtaken" + ); + assert!(persistence.durable_invitations().is_empty(), "on no disk"); + } + } } diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs b/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs index 5060aa6295d..50139d56f7d 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/payments.rs @@ -22,6 +22,7 @@ use crate::broadcaster::TransactionBroadcaster; use crate::changeset::PlatformWalletChangeSet; use crate::error::PlatformWalletError; use crate::wallet::core::WalletGeneration; +use crate::wallet::persister::NotDurable; use crate::wallet::platform_wallet::{PlatformWalletInfo, WalletId}; use crate::wallet::reservations::release_reservation_after_rejected_broadcast; @@ -1562,15 +1563,19 @@ impl DashPayView<'_, B> { )? }; // Host persistence may re-enter wallet APIs, so never invoke it under - // the manager guard. Failure must not expose an unpersisted address. + // the manager guard. Failure must not expose an unpersisted address, + // and a flush that succeeded is no proof by itself: another flush of + // the wallet's records may have taken the reservation and failed for + // good, dropping it (`WalletPersister::store_durably`). The address + // then stays used in memory and is not handed out. self.persister - .store(changeset) - .map_err(|e| self.persister.classify_store_failure(e))?; - self.persister.flush().map_err(|e| { - PlatformWalletError::Persistence(format!( - "failed to flush payment-address reservation: {e}" - )) - })?; + .store_durably(changeset) + .map_err(|not_durable| match not_durable { + NotDurable::Store(e) => self.persister.classify_store_failure(e), + not_flushed => PlatformWalletError::Persistence(format!( + "failed to flush payment-address reservation: {not_flushed}" + )), + })?; Ok(address) } @@ -2196,6 +2201,16 @@ mod tests { /// When set, every round recording a DashPay payment fails /// transiently (and is not recorded). fail_payment_records: Mutex, + /// Once set, the next flush is overtaken: a flush through this + /// persister, as one from another thread would, takes what was + /// buffered before it writes, and fails; the overtaken flush then + /// succeeds over nothing. + overtaken_by: Mutex>, + /// Whether the overtaking flush fails transiently, putting back what + /// it took, or for good, dropping it. + overtaking_puts_back: std::sync::atomic::AtomicBool, + /// While set, a flush is the overtaking one, and fails. + overtaking: std::sync::atomic::AtomicBool, } impl PlatformWalletPersistence for RecordingPersister { @@ -2283,6 +2298,30 @@ mod tests { fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { self.flushes .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + // Taken in its own statement: the flush below comes back here. + let overtaking = self.overtaken_by.lock().unwrap().take(); + let puts_back = self + .overtaking_puts_back + .load(std::sync::atomic::Ordering::SeqCst); + if let Some(persister) = overtaking { + self.overtaking + .store(true, std::sync::atomic::Ordering::SeqCst); + let overtook = persister.flush(); + self.overtaking + .store(false, std::sync::atomic::Ordering::SeqCst); + assert!(overtook.is_err_and(|error| error.is_transient() == puts_back)); + return Ok(()); + } + if self.overtaking.load(std::sync::atomic::Ordering::SeqCst) { + return Err(if puts_back { + PersistenceError::backend_with_kind( + PersistenceErrorKind::Transient, + "the disk is busy", + ) + } else { + PersistenceError::backend("the database is corrupt") + }); + } if *self.fail_flushes.lock().unwrap() { return Err(PersistenceError::backend_with_kind( PersistenceErrorKind::Transient, @@ -12125,6 +12164,60 @@ mod tests { ); } + /// A withdrawal address is not handed out when another flush of the + /// wallet's records took its reservation first and failed for good, + /// though the reservation's own flush then succeeded, over nothing: a + /// restart would hand the same address to the next payment. The address + /// stays used in memory, so the next call hands out another one. + #[tokio::test] + async fn should_not_hand_out_a_payment_address_whose_reservation_another_flush_dropped() { + use crate::wallet::identity::network::contact_requests::SeedCryptoProvider; + let (manager, persister, wallet_id, owner, contact) = + register_sender_and_external_account().await; + let wallet = manager.get_wallet(&wallet_id).await.unwrap(); + let iw = wallet.identity(); + let seed = Mnemonic::from_phrase(TEST_MNEMONIC).unwrap().to_seed(""); + let provider = SeedCryptoProvider::from_seed(*seed, Network::Testnet); + *persister.buffered.lock().unwrap() = true; + *persister.overtaken_by.lock().unwrap() = Some(wallet.persister().clone()); + let refused = iw + .dashpay() + .reserve_payment_address(&owner, &contact, &provider) + .await; + assert!( + matches!(&refused, Err(PlatformWalletError::Persistence(message)) + if message.contains("failed to flush payment-address reservation")), + "{refused:?}" + ); + assert!( + persister.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert_eq!( + external_used_indices(&manager, wallet_id, owner, contact).await, + vec![0], + "the address that was not handed out stays used in memory" + ); + let next = iw + .dashpay() + .reserve_payment_address(&owner, &contact, &provider) + .await + .expect("a later reservation is durable"); + assert_eq!( + external_used_indices(&manager, wallet_id, owner, contact).await, + vec![0, 1] + ); + assert!(persister + .stores + .lock() + .unwrap() + .iter() + .rev() + .flat_map(|(_, cs)| &cs.account_address_pools) + .flat_map(|pool| &pool.addresses) + .any(|info| info.address == next && info.is_used())); + } + /// Persister that parks the FIRST external-pool store inside `store`, /// after the caller has captured its whole-pool snapshot, until the /// test releases it. Every store is recorded in arrival order so the @@ -14882,6 +14975,272 @@ mod tests { iw.dashpay().abandon_prepared_payment(retry).await; } + /// A prepared payment's reservation is not on disk when another flush of + /// the wallet's records took it first and failed for good, though its own + /// flush then succeeded, over nothing: nothing is sent, the address and + /// the inputs come back, and the error is a flush whose buffered round + /// was discarded. + #[tokio::test] + async fn should_send_nothing_when_another_flush_dropped_a_prepared_reservation() { + let (manager, persister, wallet_id, owner_id, contact_id) = + register_sender_and_external_account().await; + let wallet_arc = manager.get_wallet(&wallet_id).await.expect("wallet"); + let iw = wallet_arc.identity(); + // One coin, so the retry proves the inputs were released. + fund_bip44_account_0(&manager, wallet_id, 0xD1, 120_000).await; + let (signer, provider) = test_signer_and_provider(); + let probe = Arc::new(DurabilityProbeBroadcaster { + persister: Arc::clone(&persister), + seen: Mutex::new(None), + }); + let core = crate::CoreWallet::new( + Arc::clone(&wallet_arc.core().sdk), + Arc::clone(&wallet_arc.core().wallet_manager), + wallet_arc.core().wallet_id, + Arc::clone(&probe), + Arc::clone(wallet_arc.core().generation()), + ); + + let prepared = iw + .dashpay() + .prepare_payment( + &core, + &owner_id, + &contact_id, + 100_000, + None, + &signer, + &provider, + ) + .await + .expect("prepare"); + let first_output = payment_script(prepared.transaction(), 100_000); + *persister.buffered.lock().unwrap() = true; + *persister.overtaken_by.lock().unwrap() = Some(wallet_arc.persister().clone()); + let err = iw + .dashpay() + .send_prepared_payment(prepared) + .await + .expect_err("a reservation another flush dropped must not be broadcast"); + assert!( + matches!( + &err, + PlatformWalletError::PersisterFlush(source) + if source.kind() == Some(PersistenceErrorKind::Fatal) + ), + "got {err:?}" + ); + assert!( + persister.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert!( + probe.seen.lock().unwrap().is_none(), + "the broadcaster never ran" + ); + assert!( + external_used_indices(&manager, wallet_id, owner_id, contact_id) + .await + .is_empty() + ); + let retry = iw + .dashpay() + .prepare_payment( + &core, + &owner_id, + &contact_id, + 100_000, + None, + &signer, + &provider, + ) + .await + .expect("the failed send released the only coin"); + assert_eq!( + payment_script(retry.transaction(), 100_000), + first_output, + "the address was released, so the retry pays it" + ); + iw.dashpay() + .send_prepared_payment(retry) + .await + .expect("the retry's reservation is durable"); + assert!( + probe + .seen + .lock() + .unwrap() + .is_some_and(|(_, reserved)| reserved), + "the retry reached the network with its reservation stored" + ); + } + + /// A prepared payment's reservation is not on disk either while another + /// flush of the wallet's records holds it, one that then fails + /// transiently and puts it back: nothing is sent, the address and the + /// inputs come back, and the error is a flush to be tried again, not a + /// lost round. + #[tokio::test] + async fn should_send_nothing_when_another_flush_took_a_prepared_reservation_and_put_it_back() { + let (manager, persister, wallet_id, owner_id, contact_id) = + register_sender_and_external_account().await; + let wallet_arc = manager.get_wallet(&wallet_id).await.expect("wallet"); + let iw = wallet_arc.identity(); + // One coin, so the retry proves the inputs were released. + fund_bip44_account_0(&manager, wallet_id, 0xD3, 120_000).await; + let (signer, provider) = test_signer_and_provider(); + let probe = Arc::new(DurabilityProbeBroadcaster { + persister: Arc::clone(&persister), + seen: Mutex::new(None), + }); + let core = crate::CoreWallet::new( + Arc::clone(&wallet_arc.core().sdk), + Arc::clone(&wallet_arc.core().wallet_manager), + wallet_arc.core().wallet_id, + Arc::clone(&probe), + Arc::clone(wallet_arc.core().generation()), + ); + + let prepared = iw + .dashpay() + .prepare_payment( + &core, + &owner_id, + &contact_id, + 100_000, + None, + &signer, + &provider, + ) + .await + .expect("prepare"); + *persister.buffered.lock().unwrap() = true; + persister + .overtaking_puts_back + .store(true, std::sync::atomic::Ordering::SeqCst); + *persister.overtaken_by.lock().unwrap() = Some(wallet_arc.persister().clone()); + let err = iw + .dashpay() + .send_prepared_payment(prepared) + .await + .expect_err("a reservation another flush holds must not be broadcast"); + assert!( + matches!( + &err, + PlatformWalletError::PersisterFlush(source) + if source.kind() == Some(PersistenceErrorKind::Transient) + ), + "got {err:?}" + ); + assert!( + persister.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert!( + probe.seen.lock().unwrap().is_none(), + "the broadcaster never ran" + ); + assert!( + external_used_indices(&manager, wallet_id, owner_id, contact_id) + .await + .is_empty() + ); + let retry = iw + .dashpay() + .prepare_payment( + &core, + &owner_id, + &contact_id, + 100_000, + None, + &signer, + &provider, + ) + .await + .expect("the failed send released the only coin"); + iw.dashpay() + .send_prepared_payment(retry) + .await + .expect("the retry's reservation is durable"); + assert!( + probe + .seen + .lock() + .unwrap() + .is_some_and(|(_, reserved)| reserved), + "the retry reached the network with its reservation stored" + ); + } + + /// A direct send's reservation is not on disk when another flush of the + /// wallet's records took it first and failed for good, though its own + /// flush then succeeded, over nothing: nothing is sent, and the address + /// and the inputs come back for the retry. + #[tokio::test] + async fn should_send_nothing_when_another_flush_dropped_a_send_s_reservation() { + let (manager, persister, wallet_id, owner_id, contact_id) = + register_sender_and_external_account().await; + let wallet_arc = manager.get_wallet(&wallet_id).await.expect("wallet"); + let iw = wallet_arc.identity(); + // One coin, so the retry proves the inputs were released. + fund_bip44_account_0(&manager, wallet_id, 0xD2, 120_000).await; + let (signer, provider) = test_signer_and_provider(); + let sending = with_identity_broadcaster( + iw, + DurabilityProbeBroadcaster { + persister: Arc::clone(&persister), + seen: Mutex::new(None), + }, + ); + *persister.buffered.lock().unwrap() = true; + *persister.overtaken_by.lock().unwrap() = Some(wallet_arc.persister().clone()); + let err = sending + .dashpay() + .send_payment(&owner_id, &contact_id, 100_000, None, &signer, &provider) + .await + .expect_err("a reservation another flush dropped must not be broadcast"); + assert!( + matches!( + &err, + PlatformWalletError::PersisterFlush(source) + if source.kind() == Some(PersistenceErrorKind::Fatal) + ), + "got {err:?}" + ); + assert!( + persister.overtaken_by.lock().unwrap().is_none(), + "a flush was overtaken" + ); + assert!( + sending.broadcaster.seen.lock().unwrap().is_none(), + "the broadcaster never ran" + ); + assert!( + external_used_indices(&manager, wallet_id, owner_id, contact_id) + .await + .is_empty() + ); + sending + .dashpay() + .send_payment(&owner_id, &contact_id, 100_000, None, &signer, &provider) + .await + .expect("the failed send released the only coin"); + assert!( + sending + .broadcaster + .seen + .lock() + .unwrap() + .is_some_and(|(_, reserved)| reserved), + "the retry reached the network with its reservation stored" + ); + assert_eq!( + external_used_indices(&manager, wallet_id, owner_id, contact_id).await, + vec![0], + "the address was released, so the retry pays it" + ); + } + /// Broadcaster that, while it holds the transaction, checks whether a /// teardown of the funding generation could start, then accepts it. struct TeardownProbeBroadcaster { diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs b/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs index 83d0af0f00b..fb3f4a6a83b 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/prepared_payment.rs @@ -722,23 +722,27 @@ impl DashPayView<'_, B> { } } - /// Store `changeset` and, on a backend that buffers, flush it, so it is - /// on disk when this returns. A failed flush is `PersisterFlush` with the - /// backend's own kind: the round may still be buffered and commit at a - /// later flush, so it is not proof that nothing was stored. + /// Store `changeset` and answer `Ok` only once it is on disk + /// (`WalletPersister::store_durably`): at once on a backend that commits + /// a store inline, and otherwise after a flush that no other flush of the + /// wallet's records can have emptied. A failed flush is `PersisterFlush` + /// with the backend's own kind: the round may still be buffered and + /// commit at a later flush, so it is not proof that nothing was stored. + /// + /// A flush that succeeded is not trusted by itself: another flush may + /// have taken what the store buffered and failed for good, dropping it. + /// That is reported as the flush failure it is, `PersisterFlush` of kind + /// `Fatal`, the buffered round discarded. Another flush that is still in + /// progress, or that failed and put back what it took, leaves the round + /// buffered for a later flush: `PersisterFlush` of kind `Transient`, to + /// be tried again. pub(super) fn store_durably( &self, changeset: PlatformWalletChangeSet, ) -> Result<(), PlatformWalletError> { self.persister - .store(changeset) - .map_err(|e| self.persister.classify_store_failure(e))?; - if !self.persister.store_commits_inline() { - self.persister - .flush() - .map_err(PlatformWalletError::from_flush_failure)?; - } - Ok(()) + .store_durably(changeset) + .map_err(|not_durable| self.persister.classify_not_durable(not_durable)) } /// The wallet info of `generation`, or `WalletNotFound` when the wallet diff --git a/packages/rs-platform-wallet/src/wallet/persister.rs b/packages/rs-platform-wallet/src/wallet/persister.rs index e30ca11ce18..8f7f236f3ec 100644 --- a/packages/rs-platform-wallet/src/wallet/persister.rs +++ b/packages/rs-platform-wallet/src/wallet/persister.rs @@ -720,8 +720,44 @@ impl WalletPersister { &self, changeset: PlatformWalletChangeSet, ) -> Result<(), NotDurable> { - let (lost_before, failed_before) = (self.flush_ledger.lost(), self.flush_ledger.failed()); + let failures_before = (self.flush_ledger.lost(), self.flush_ledger.failed()); self.store(changeset).map_err(NotDurable::Store)?; + self.flush_what_was_stored(failures_before) + } + + /// [`Self::store_durably`] for a changeset that withdraws a record stored + /// before it (a row's removal), where `Ok` has to mean more than that + /// the withdrawal is on disk: that nothing writes the record after it. + /// + /// A flush in progress when the withdrawal is stored may have taken the + /// record without it, and nothing says a backend writes what that flush + /// took before what a later one takes. Should it write the record after + /// the withdrawal and return before the ledger is read, it leaves no + /// trace there: a flush that succeeded is counted nowhere. So a flush in + /// progress once the withdrawal is stored answers + /// [`NotDurable::OtherFlushUnsettled`], and nothing is flushed. With + /// none in progress then, every flush that could hold the record alone + /// has returned, and one that begins later takes the withdrawal with + /// whatever of the record is still buffered, the withdrawal behind it. + pub(crate) fn withdraw_durably( + &self, + changeset: PlatformWalletChangeSet, + ) -> Result<(), NotDurable> { + let failures_before = (self.flush_ledger.lost(), self.flush_ledger.failed()); + self.store(changeset).map_err(NotDurable::Store)?; + if !self.store_commits_inline() && self.flush_ledger.in_flight() != 0 { + return Err(NotDurable::OtherFlushUnsettled); + } + self.flush_what_was_stored(failures_before) + } + + /// The flush half of [`Self::store_durably`], for a changeset just + /// stored: `lost_before` and `failed_before` are the ledger's counts + /// read before that store. + fn flush_what_was_stored( + &self, + (lost_before, failed_before): (u64, u64), + ) -> Result<(), NotDurable> { if self.store_commits_inline() { return Ok(()); } @@ -1378,6 +1414,102 @@ mod tests { assert!(after.is_ok(), "{after:?}"); } + /// A withdrawal is not answered for while another flush of the wallet's + /// records is inside the backend when it is stored: that flush may have + /// taken the record being withdrawn, and may write it after the + /// withdrawal. Nothing is flushed for it. A plain store made durable in + /// the same place is answered for, rightly for the store itself: its own + /// flush wrote it, and the other one returned without failing. + #[test] + fn should_not_answer_for_a_withdrawal_stored_while_another_flush_has_not_returned() { + /// The first flush stalls inside the backend. The second lets it + /// go once it has written, and returns only after the first has: so + /// no flush is in progress, and none failed, when the second returns. + struct LateFirstFlush { + flushes: AtomicU64, + entered: std::sync::Mutex>, + release: std::sync::Mutex>, + let_go: std::sync::Mutex>, + first_returned: std::sync::Mutex>, + } + impl PlatformWalletPersistence for LateFirstFlush { + fn store( + &self, + _wallet_id: WalletId, + _changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + Ok(()) + } + fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { + if self.flushes.fetch_add(1, Ordering::SeqCst) != 0 { + self.let_go + .lock() + .expect("let go") + .send(()) + .expect("the first flush waits"); + self.first_returned + .lock() + .expect("first returned") + .recv_timeout(Duration::from_secs(10)) + .expect("the first flush returns"); + return Ok(()); + } + self.entered + .lock() + .expect("entered") + .send(()) + .expect("the test waits"); + self.release + .lock() + .expect("release") + .recv_timeout(Duration::from_secs(10)) + .expect("released"); + Ok(()) + } + fn load(&self) -> Result { + Ok(ClientStartState::default()) + } + } + + let (entered, has_entered) = mpsc::channel(); + let (let_go, release) = mpsc::channel(); + let (returned, first_returned) = mpsc::channel(); + let backend = Arc::new(LateFirstFlush { + flushes: AtomicU64::new(0), + entered: std::sync::Mutex::new(entered), + release: std::sync::Mutex::new(release), + let_go: std::sync::Mutex::new(let_go), + first_returned: std::sync::Mutex::new(first_returned), + }); + let persister = WalletPersister::new([0; 32], Arc::clone(&backend) as _); + let stalled = { + let persister = persister.clone(); + std::thread::spawn(move || { + let flushed = persister.flush(); + returned.send(()).expect("the second flush waits"); + flushed + }) + }; + has_entered + .recv_timeout(Duration::from_secs(10)) + .expect("the first flush is inside the backend"); + + let withdrawn = persister.withdraw_durably(PlatformWalletChangeSet::default()); + assert!( + matches!(withdrawn, Err(NotDurable::OtherFlushUnsettled)), + "{withdrawn:?}" + ); + assert_eq!( + backend.flushes.load(Ordering::SeqCst), + 1, + "nothing was flushed for the withdrawal" + ); + + let stored = persister.store_durably(PlatformWalletChangeSet::default()); + assert!(stalled.join().expect("the first flush returns").is_ok()); + assert!(stored.is_ok(), "{stored:?}"); + } + /// The host's flush runs with no lock of the persister held: a flush that /// notes another read-back commitment through the same persister, as a /// reentrant backend or another thread would, completes, and nothing a diff --git a/packages/rs-platform-wallet/src/wallet/shielded/wallet_drain_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/wallet_drain_tests.rs index b5ca65d2429..83577b60b77 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/wallet_drain_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/wallet_drain_tests.rs @@ -92,6 +92,10 @@ struct RecoveryPersistence { staged: Mutex>, fail_store: AtomicBool, fail_flush: AtomicBool, + /// Fails the next store that carries an asset-lock row, and only it. + fail_next_lock_store: AtomicBool, + /// Fails the next flush only. + fail_next_flush: AtomicBool, commits_inline: AtomicBool, fail_load: AtomicBool, records_recipients: AtomicBool, @@ -114,7 +118,13 @@ impl PlatformWalletPersistence for RecoveryPersistence { _wallet_id: [u8; 32], cs: PlatformWalletChangeSet, ) -> Result<(), PersistenceError> { - if self.fail_store.load(Ordering::SeqCst) { + if self.fail_store.load(Ordering::SeqCst) + || (cs + .asset_locks + .as_ref() + .is_some_and(|locks| !locks.asset_locks.is_empty()) + && self.fail_next_lock_store.swap(false, Ordering::SeqCst)) + { return Err(PersistenceError::backend("store failed")); } if let Some(cs) = cs.asset_locks { @@ -132,7 +142,9 @@ impl PlatformWalletPersistence for RecoveryPersistence { Ok(()) } fn flush(&self, _wallet_id: [u8; 32]) -> Result<(), PersistenceError> { - if self.fail_flush.load(Ordering::SeqCst) { + if self.fail_flush.load(Ordering::SeqCst) + || self.fail_next_flush.swap(false, Ordering::SeqCst) + { return Err(PersistenceError::backend("flush failed")); } self.commit_staged(); @@ -241,6 +253,8 @@ async fn signed_wallet_fixture( staged: Mutex::new(BTreeMap::new()), fail_store: AtomicBool::new(false), fail_flush: AtomicBool::new(false), + fail_next_lock_store: AtomicBool::new(false), + fail_next_flush: AtomicBool::new(false), // Drains admit inline-committing backends only; the buffered mode // exists for `should_refuse_drain_on_a_buffered_persistence_backend`. commits_inline: AtomicBool::new(true), @@ -1172,9 +1186,10 @@ async fn fund_typed_amount( /// A typed-amount funding whose recipient cannot be made durable stops before /// the broadcaster: a rejected store and a rejected flush of a buffered store /// both fail the call with a persistence error, leave the lock untracked and -/// uncommitted, and hand the inputs back so the next attempt builds again. -/// The abandoned row of a failed flush is withdrawn from the buffer, so the -/// next successful flush does not commit it. +/// uncommitted, and, once the abandoned row's removal is on disk, hand the +/// inputs back so the next attempt builds again. The abandoned row of a +/// failed flush is withdrawn from the buffer, so the flush that writes its +/// removal does not commit it. #[tokio::test] async fn should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_durable() { for fail_flush in [false, true] { @@ -1185,10 +1200,11 @@ async fn should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_dur signed_wallet_fixture(&lock, keys).await; let coordinator = Arc::new(coordinator); persistence.commits_inline.store(false, Ordering::SeqCst); + // The row's own store or flush fails; its removal's goes through. let switch = if fail_flush { - &persistence.fail_flush + &persistence.fail_next_flush } else { - &persistence.fail_store + &persistence.fail_next_lock_store }; switch.store(true, Ordering::SeqCst); @@ -1209,7 +1225,6 @@ async fn should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_dur // The inputs were handed back: the retry builds a lock again and gets // as far as the broadcaster. - switch.store(false, Ordering::SeqCst); let (error, retried) = fund_typed_amount(&wallet, &coordinator, &signer, recipient).await; assert!( !matches!(error, PlatformWalletError::Persistence(_)), @@ -1228,6 +1243,45 @@ async fn should_abort_typed_amount_funding_when_the_recipient_cannot_be_made_dur } } +/// Where the backend keeps failing, so the abandoned row's removal cannot be +/// confirmed either, the funding is not reported as failed: a store or flush +/// that failed is no proof the row is on no disk, and a restart that finds +/// it resumes and sends it. The outcome is unknown, for a rejected store and +/// for a rejected flush alike, and nothing is sent or tracked. +#[tokio::test] +async fn should_report_a_typed_amount_funding_as_unconfirmed_when_its_row_removal_fails_too() { + for fail_flush in [false, true] { + let (mut lock, keys) = drain_lock_fixture(); + lock.operation_id = None; + let recipient = crate::test_support::orchard_address_fixture(0x5a); + let (wallet, persistence, coordinator, _dir, signer) = + signed_wallet_fixture(&lock, keys).await; + let coordinator = Arc::new(coordinator); + persistence.commits_inline.store(false, Ordering::SeqCst); + let switch = if fail_flush { + &persistence.fail_flush + } else { + &persistence.fail_store + }; + + switch.store(true, Ordering::SeqCst); + let (error, seen) = fund_typed_amount(&wallet, &coordinator, &signer, recipient).await; + assert!( + matches!(&error, PlatformWalletError::TransactionBroadcastUnconfirmed(message) + if message.contains("a restart may still resume it")), + "fail_flush={fail_flush}: {error}" + ); + assert_eq!(seen.len(), 1, "fail_flush={fail_flush}"); + assert!(!wallet + .asset_locks + .list_tracked_locks() + .await + .iter() + .any(|tracked| tracked.out_point == seen[0])); + assert!(!persistence.committed.lock().unwrap().contains_key(&seen[0])); + } +} + /// A backend that cannot record a recipient keeps the best-effort row every /// other funding has: a rejected store does not stop the dispatch. #[tokio::test]