diff --git a/packages/rs-platform-wallet-ffi/src/event_handler.rs b/packages/rs-platform-wallet-ffi/src/event_handler.rs index 0db7649a7c6..672a0a6d08d 100644 --- a/packages/rs-platform-wallet-ffi/src/event_handler.rs +++ b/packages/rs-platform-wallet-ffi/src/event_handler.rs @@ -4,11 +4,14 @@ use crate::platform_address_sync::{ PlatformAddressSyncMetricsFFI, PlatformAddressSyncWalletResultFFI, }; use crate::shielded_types::ShieldedSyncWalletResultFFI; +use dashcore::Txid; +use platform_wallet::broadcast_probe::ProbeVerdict; use platform_wallet::events::{EventHandler, PlatformEventHandler, WalletEvent}; use platform_wallet::manager::dpns_sync::{DpnsSyncPassSummary, WalletDpnsSyncOutcome}; #[cfg(feature = "shielded")] use platform_wallet::manager::shielded_sync::{ShieldedSyncPassSummary, WalletShieldedOutcome}; use platform_wallet::{PlatformAddressSyncSummary, WalletSyncOutcome}; +use std::ffi::CString; use std::os::raw::{c_char, c_void}; /// Current layout version of [`EventHandlerCallbacksExtension`]. @@ -49,6 +52,32 @@ pub type DpnsMarketplaceSyncCompletedFn = unsafe extern "C" fn( sync_unix_seconds: u64, ); +/// Verdict codes for [`OutgoingTransactionProbedFn`]. +/// A node holds the send, in its mempool or in a block. +pub const OUTGOING_PROBE_VERDICT_ACCEPTED: u8 = 0; +/// Reserved; not emitted by this version. +pub const OUTGOING_PROBE_VERDICT_DEAD: u8 = 1; +pub const OUTGOING_PROBE_VERDICT_UNRESOLVED: u8 = 2; +/// Not a verdict: drop whatever was kept for the send's earlier verdict. Sent +/// when the send settled or left the wallet, when its wallet was removed, and +/// for every send when probing is turned off — so it does not mean the send +/// settled. `reason` is null. +pub const OUTGOING_PROBE_VERDICT_CLEARED: u8 = 3; + +/// A verdict on an unconfirmed send whose broadcast outcome was unknown. +/// `wallet_id` and `txid` point to 32 bytes each; `txid` is in wire +/// (internal) byte order — reverse it for the display hex. `verdict` is one of +/// the `OUTGOING_PROBE_VERDICT_*` codes; `reason` is null for `ACCEPTED` and +/// `CLEARED`, otherwise a NUL-terminated diagnostic string (never user-facing +/// copy). All pointers are valid only for the duration of the call. +pub type OutgoingTransactionProbedFn = unsafe extern "C" fn( + context: *mut c_void, + wallet_id: *const u8, + txid: *const u8, + verdict: u8, + reason: *const c_char, +); + /// Size/version-tagged event extension. It shares the legacy event /// vtable's context and destructor; only callback pointers are copied. /// This avoids growing [`EventHandlerCallbacks`] and over-reading callers @@ -70,6 +99,17 @@ pub struct EventHandlerCallbacksExtension { sync_unix_seconds: u64, ), >, + /// Appended within version 1: a host whose `struct_size` stops before + /// this slot simply never receives it. See [`OutgoingTransactionProbedFn`]. + pub on_outgoing_transaction_probed_fn: Option< + unsafe extern "C" fn( + context: *mut c_void, + wallet_id: *const u8, + txid: *const u8, + verdict: u8, + reason: *const c_char, + ), + >, } impl Default for EventHandlerCallbacksExtension { @@ -79,6 +119,7 @@ impl Default for EventHandlerCallbacksExtension { version: PLATFORM_WALLET_EVENT_CALLBACKS_EXTENSION_VERSION, reserved: 0, on_dpns_marketplace_sync_completed_fn: None, + on_outgoing_transaction_probed_fn: None, } } } @@ -189,16 +230,19 @@ unsafe impl Sync for EventHandlerCallbacks {} pub(crate) struct FFIEventHandler { callbacks: EventHandlerCallbacks, dpns_sync_callback: Option, + outgoing_probe_callback: Option, } impl FFIEventHandler { pub fn new( callbacks: EventHandlerCallbacks, dpns_sync_callback: Option, + outgoing_probe_callback: Option, ) -> Self { Self { callbacks, dpns_sync_callback, + outgoing_probe_callback, } } } @@ -250,6 +294,53 @@ impl EventHandler for FFIEventHandler { } impl PlatformEventHandler for FFIEventHandler { + fn on_outgoing_transaction_cleared(&self, wallet_id: &[u8; 32], txid: &Txid) { + let Some(callback) = self.outgoing_probe_callback else { + return; + }; + let txid_bytes: &[u8] = txid.as_ref(); + unsafe { + callback( + self.callbacks.context, + wallet_id.as_ptr(), + txid_bytes.as_ptr(), + OUTGOING_PROBE_VERDICT_CLEARED, + std::ptr::null(), + ); + } + } + + fn on_outgoing_transaction_probed( + &self, + wallet_id: &[u8; 32], + txid: &Txid, + verdict: &ProbeVerdict, + ) { + let Some(callback) = self.outgoing_probe_callback else { + return; + }; + let (code, reason) = match verdict { + // The host does not need to tell a mempool from a block: both mean + // the payment is going through. + ProbeVerdict::Accepted => (OUTGOING_PROBE_VERDICT_ACCEPTED, None), + ProbeVerdict::Unresolved { reason } => { + (OUTGOING_PROBE_VERDICT_UNRESOLVED, Some(reason)) + } + }; + // A reason is a node's text: strip interior NULs rather than lose it. + let reason = reason.and_then(|reason| CString::new(reason.replace('\0', "")).ok()); + let txid_bytes: &[u8] = txid.as_ref(); + unsafe { + callback( + self.callbacks.context, + wallet_id.as_ptr(), + txid_bytes.as_ptr(), + code, + reason.as_ref().map_or(std::ptr::null(), |r| r.as_ptr()), + ); + } + } + fn on_dpns_marketplace_sync_completed(&self, summary: &DpnsSyncPassSummary) { let Some(callback) = self.dpns_sync_callback else { return; @@ -496,6 +587,7 @@ mod release_tests { release_fn: None, }, None, + None, ); handler.on_shielded_sync_completed(&ShieldedSyncPassSummary { wallet_results: BTreeMap::from([ @@ -567,6 +659,7 @@ mod release_tests { release_fn: Some(count_release), }, None, + None, )); let straggler = Arc::clone(&handler); @@ -577,3 +670,106 @@ mod release_tests { assert_eq!(releases.load(Ordering::SeqCst), 1); } } + +#[cfg(test)] +mod outgoing_probe_callback_tests { + use std::ffi::CStr; + use std::sync::Mutex; + + use dashcore::hashes::Hash; + use platform_wallet::broadcast_probe::ProbeVerdict; + + use super::*; + + /// (wallet_id, txid, verdict, reason) of one call. + type Call = ([u8; 32], [u8; 32], u8, Option); + + static CALLS: Mutex> = Mutex::new(Vec::new()); + + unsafe extern "C" fn record( + _context: *mut c_void, + wallet_id: *const u8, + txid: *const u8, + verdict: u8, + reason: *const c_char, + ) { + let wallet_id = std::ptr::read(wallet_id as *const [u8; 32]); + let txid = std::ptr::read(txid as *const [u8; 32]); + let reason = + (!reason.is_null()).then(|| CStr::from_ptr(reason).to_string_lossy().into_owned()); + CALLS + .lock() + .expect("calls") + .push((wallet_id, txid, verdict, reason)); + } + + fn handler(probe: Option) -> FFIEventHandler { + FFIEventHandler::new( + EventHandlerCallbacks { + context: std::ptr::null_mut(), + on_wallet_event_fn: None, + on_error_fn: None, + on_platform_address_sync_completed_fn: None, + on_shielded_sync_completed_fn: None, + ..unsafe { std::mem::zeroed() } + }, + None, + probe, + ) + } + + /// One test drives every case so the shared capture is never raced by a + /// parallel test. + #[test] + fn should_deliver_each_verdict_with_its_code_wire_txid_and_reason() { + CALLS.lock().expect("calls").clear(); + let wallet = [7u8; 32]; + let txid = dashcore::Txid::from_byte_array([ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, + ]); + let with_callback = handler(Some(record)); + + with_callback.on_outgoing_transaction_probed(&wallet, &txid, &ProbeVerdict::Accepted); + with_callback.on_outgoing_transaction_probed( + &wallet, + &txid, + &ProbeVerdict::Unresolved { + reason: "no quorum".to_string(), + }, + ); + with_callback.on_outgoing_transaction_probed( + &wallet, + &txid, + &ProbeVerdict::Unresolved { + reason: "odd\0node".to_string(), + }, + ); + with_callback.on_outgoing_transaction_cleared(&wallet, &txid); + // A host without the slot receives nothing. + handler(None).on_outgoing_transaction_probed(&wallet, &txid, &ProbeVerdict::Accepted); + handler(None).on_outgoing_transaction_cleared(&wallet, &txid); + + let calls = CALLS.lock().expect("calls").clone(); + let wire: [u8; 32] = *txid.as_byte_array(); + assert_eq!( + calls, + vec![ + (wallet, wire, OUTGOING_PROBE_VERDICT_ACCEPTED, None), + ( + wallet, + wire, + OUTGOING_PROBE_VERDICT_UNRESOLVED, + Some("no quorum".to_string()) + ), + ( + wallet, + wire, + OUTGOING_PROBE_VERDICT_UNRESOLVED, + Some("oddnode".to_string()) + ), + (wallet, wire, OUTGOING_PROBE_VERDICT_CLEARED, None), + ] + ); + } +} diff --git a/packages/rs-platform-wallet-ffi/src/manager.rs b/packages/rs-platform-wallet-ffi/src/manager.rs index e1ccdaad053..25e7af5e395 100644 --- a/packages/rs-platform-wallet-ffi/src/manager.rs +++ b/packages/rs-platform-wallet-ffi/src/manager.rs @@ -4,7 +4,8 @@ use crate::check_ptr; use crate::error::*; use crate::event_handler::{ DpnsMarketplaceSyncCompletedFn, EventHandlerCallbacks, EventHandlerCallbacksExtension, - FFIEventHandler, PLATFORM_WALLET_EVENT_CALLBACKS_EXTENSION_VERSION, + FFIEventHandler, OutgoingTransactionProbedFn, + PLATFORM_WALLET_EVENT_CALLBACKS_EXTENSION_VERSION, }; use crate::handle::*; use crate::persistence::{ @@ -81,6 +82,7 @@ pub unsafe extern "C" fn platform_wallet_manager_create( PersistenceCapabilities::NONE, PersistenceExtensionCallbacks::default(), None, + None, out_handle, ) } @@ -107,6 +109,7 @@ pub unsafe extern "C" fn platform_wallet_manager_create_with_persistence_capabil declaration, PersistenceExtensionCallbacks::default(), None, + None, out_handle, ) } @@ -140,6 +143,7 @@ pub unsafe extern "C" fn platform_wallet_manager_create_with_persistence_extensi declaration, extensions, None, + None, out_handle, ) } @@ -163,7 +167,9 @@ pub unsafe extern "C" fn platform_wallet_manager_create_with_extensions( check_ptr!(event_extension); let declaration = persistence_capabilities_declaration(&*persistence_capabilities); let persistence_extensions = persistence_extension_callbacks(persistence_extension); - let dpns_event_callback = event_extension_dpns_callback(event_extension); + let event_callbacks = event_extension_callbacks(event_extension); + let dpns_event_callback = event_callbacks.dpns; + let outgoing_probe_callback = event_callbacks.outgoing_probe; platform_wallet_manager_create_impl( sdk_ptr, persistence, @@ -171,6 +177,7 @@ pub unsafe extern "C" fn platform_wallet_manager_create_with_extensions( declaration, persistence_extensions, dpns_event_callback, + outgoing_probe_callback, out_handle, ) } @@ -285,23 +292,42 @@ unsafe fn persistence_extension_callbacks( } } -unsafe fn event_extension_dpns_callback( +/// The additive event callbacks a host supplied, negotiated from ONE snapshot +/// of the extension header — `struct_size` and `version` are read once, so +/// every slot is judged against the same declared layout even if host memory +/// changes under us (see [`persistence_extension_callbacks`]). +struct EventExtensionCallbacks { + dpns: Option, + outgoing_probe: Option, +} + +unsafe fn event_extension_callbacks( extension: *const EventHandlerCallbacksExtension, -) -> Option { +) -> EventExtensionCallbacks { let supplied_size = std::ptr::addr_of!((*extension).struct_size).read(); let version_end = std::mem::offset_of!(EventHandlerCallbacksExtension, version) + std::mem::size_of::(); let version_ok = supplied_size >= version_end && std::ptr::addr_of!((*extension).version).read() == PLATFORM_WALLET_EVENT_CALLBACKS_EXTENSION_VERSION; - negotiated_extension_slot!( - extension, - EventHandlerCallbacksExtension, - supplied_size, - version_ok, - on_dpns_marketplace_sync_completed_fn, - DpnsMarketplaceSyncCompletedFn - ) + EventExtensionCallbacks { + dpns: negotiated_extension_slot!( + extension, + EventHandlerCallbacksExtension, + supplied_size, + version_ok, + on_dpns_marketplace_sync_completed_fn, + DpnsMarketplaceSyncCompletedFn + ), + outgoing_probe: negotiated_extension_slot!( + extension, + EventHandlerCallbacksExtension, + supplied_size, + version_ok, + on_outgoing_transaction_probed_fn, + OutgoingTransactionProbedFn + ), + } } // The C entry point's own shape: every callback table and out-param the @@ -315,6 +341,7 @@ unsafe fn platform_wallet_manager_create_impl( declared_capabilities: PersistenceCapabilities, persistence_extensions: PersistenceExtensionCallbacks, dpns_event_callback: Option, + outgoing_probe_callback: Option, out_handle: *mut Handle, ) -> PlatformWalletFFIResult { check_ptr!(sdk_ptr); @@ -360,6 +387,7 @@ unsafe fn platform_wallet_manager_create_impl( let handler: Arc = Arc::new(FFIEventHandler::new( std::ptr::read(event_handler), dpns_event_callback, + outgoing_probe_callback, )); // `PlatformWalletManager::new` spawns the wallet-event adapter @@ -2164,3 +2192,101 @@ mod remove_wallet_lifecycle_tests { }); } } + +/// Turn automatic probing of unconfirmed sends on or off (off by default). +/// +/// While on, the roots of every unconfirmed chain are resubmitted to evonodes +/// over DAPI — right after SPV reports a send uncertain (each report: the +/// send itself when it is a root, and of the roots its chain reaches at +/// most 8 per pass, shared by the reports handled together, the rest in +/// their turn), then, +/// while the wallet follows the chain tip, every block for 24 blocks from when +/// it first did so for that root, and every 10 blocks after (a return from the +/// background does not restart the 24), 55 times at most (about a day of +/// following the tip, counted within one session); after that only a new uncertain report probes the +/// root, and its last verdict stays as delivered — and each change of a send's verdict, and +/// each clear, reaches the host through +/// `EventHandlerCallbacksExtension::on_outgoing_transaction_probed_fn`. +/// Nothing in the wallet changes; turning it off clears every verdict. +/// +/// # Safety +/// `handle` must be a live manager handle. +#[no_mangle] +pub unsafe extern "C" fn platform_wallet_manager_set_broadcast_probe_enabled( + handle: Handle, + enabled: bool, +) -> PlatformWalletFFIResult { + // Only queues the switch: the clears that turning probing off causes reach + // the host later, from the resolver's task, not under the storage lock. + let option = PLATFORM_WALLET_MANAGER_STORAGE.with_item(handle, |manager| { + manager.set_broadcast_probe_enabled(enabled); + }); + unwrap_option_or_return!(option); + PlatformWalletFFIResult::ok() +} + +#[cfg(test)] +mod outgoing_probe_extension_tests { + use std::os::raw::{c_char, c_void}; + + use super::*; + use crate::event_handler::DpnsSyncWalletResultFFI; + + unsafe extern "C" fn probed( + _context: *mut c_void, + _wallet_id: *const u8, + _txid: *const u8, + _verdict: u8, + _reason: *const c_char, + ) { + } + + unsafe extern "C" fn dpns( + _context: *mut c_void, + _results: *const DpnsSyncWalletResultFFI, + _count: usize, + _sync_unix_seconds: u64, + ) { + } + + fn both_slots() -> EventHandlerCallbacksExtension { + EventHandlerCallbacksExtension { + on_dpns_marketplace_sync_completed_fn: Some(dpns), + on_outgoing_transaction_probed_fn: Some(probed), + ..Default::default() + } + } + + /// A host built before the probe slot existed declares a `struct_size` + /// that ends at the DPNS slot. Its bytes past that point are not its own, + /// so the probe slot must be refused while DPNS keeps working. + #[test] + fn should_not_read_the_probe_slot_from_a_host_whose_extension_ends_before_it() { + let mut ext = both_slots(); + ext.struct_size = std::mem::offset_of!( + EventHandlerCallbacksExtension, + on_outgoing_transaction_probed_fn + ); + unsafe { + assert!(event_extension_callbacks(&ext).dpns.is_some()); + assert!(event_extension_callbacks(&ext).outgoing_probe.is_none()); + } + } + + #[test] + fn should_read_the_probe_slot_from_a_current_host() { + let ext = both_slots(); + unsafe { + assert!(event_extension_callbacks(&ext).outgoing_probe.is_some()); + } + } + + #[test] + fn should_refuse_the_probe_slot_under_an_unknown_version() { + let mut ext = both_slots(); + ext.version = PLATFORM_WALLET_EVENT_CALLBACKS_EXTENSION_VERSION + 1; + unsafe { + assert!(event_extension_callbacks(&ext).outgoing_probe.is_none()); + } + } +} diff --git a/packages/rs-platform-wallet-ffi/src/persistence.rs b/packages/rs-platform-wallet-ffi/src/persistence.rs index d163cb203c7..4694c44f544 100644 --- a/packages/rs-platform-wallet-ffi/src/persistence.rs +++ b/packages/rs-platform-wallet-ffi/src/persistence.rs @@ -8932,6 +8932,7 @@ mod tests { release_fn: None, }, None, + None, ); let _runtime_guard = runtime().enter(); PlatformWalletManager::new( diff --git a/packages/rs-platform-wallet-ffi/src/shielded_sync.rs b/packages/rs-platform-wallet-ffi/src/shielded_sync.rs index dfda7eeb4c1..c8412c1b5fc 100644 --- a/packages/rs-platform-wallet-ffi/src/shielded_sync.rs +++ b/packages/rs-platform-wallet-ffi/src/shielded_sync.rs @@ -813,6 +813,7 @@ mod local_balance_tests { release_fn: None, }, None, + None, ); let _runtime_guard = runtime().enter(); PlatformWalletManager::new( diff --git a/packages/rs-platform-wallet/README.md b/packages/rs-platform-wallet/README.md index 1fc475bed45..ba2d499b542 100644 --- a/packages/rs-platform-wallet/README.md +++ b/packages/rs-platform-wallet/README.md @@ -146,6 +146,16 @@ The local shielded balance API introduces three Rust source compatibility change index is zero. An absent row is different from a recorded empty scan; do not infer presence merely from `last_synced_index > 0`. +## Broadcast-probe API migration + +Probing sends whose broadcast outcome is unknown brings these Rust source +changes: + +- `WalletWorker` gains `BroadcastProbes`, reported in the shutdown report. An + exhaustive `match` needs an arm for it. +- `WalletGeneration::teardown_guard` returns a `TeardownGuard` in place of the + bare `OwnedRwLockWriteGuard<()>`. + ## Dependencies - `key-wallet`: Core wallet functionality diff --git a/packages/rs-platform-wallet/examples/broadcast_probe_testnet.rs b/packages/rs-platform-wallet/examples/broadcast_probe_testnet.rs new file mode 100644 index 00000000000..0369e69b7e0 --- /dev/null +++ b/packages/rs-platform-wallet/examples/broadcast_probe_testnet.rs @@ -0,0 +1,52 @@ +//! Manual testnet check of the broadcast probe against real evonodes. +//! +//! Hands each transaction (raw hex, one per argument) to +//! [`DapiAcceptanceProbe`] and prints the verdict. Run with node-level detail: +//! +//! ```text +//! RUST_LOG=platform_wallet::broadcast_probe=debug \ +//! cargo run -p platform-wallet --example broadcast_probe_testnet -- [ ...] +//! ``` +//! +//! Resubmitting a transaction is harmless — a node either already has it, +//! takes it, or says why it will not — so any transaction may be probed. + +use std::sync::Arc; + +use dash_sdk::SdkBuilder; +use dashcore::{consensus, Network, Transaction}; +use platform_wallet::broadcast_probe::{AcceptanceProbe, DapiAcceptanceProbe}; +use rs_sdk_trusted_context_provider::TrustedHttpContextProvider; + +#[tokio::main(flavor = "multi_thread")] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + + let transactions: Vec = std::env::args() + .skip(1) + .map(|raw| consensus::deserialize(&hex::decode(raw.trim()).expect("hex")).expect("tx")) + .collect(); + if transactions.is_empty() { + return Err("pass at least one raw transaction as hex".into()); + } + + let provider = TrustedHttpContextProvider::new( + Network::Testnet, + None, + std::num::NonZeroUsize::new(100).expect("non-zero"), + )?; + let sdk = Arc::new( + SdkBuilder::new_testnet() + .with_context_provider(provider) + .build()?, + ); + let probe = DapiAcceptanceProbe::new(sdk); + + for transaction in &transactions { + let verdict = probe.probe(transaction).await; + println!("{} {verdict:?}", transaction.txid()); + } + Ok(()) +} diff --git a/packages/rs-platform-wallet/src/broadcast_probe.rs b/packages/rs-platform-wallet/src/broadcast_probe.rs new file mode 100644 index 00000000000..0c604fe13ee --- /dev/null +++ b/packages/rs-platform-wallet/src/broadcast_probe.rs @@ -0,0 +1,1210 @@ +//! Asking Core nodes directly whether a signed transaction has landed. +//! +//! The P2P network has no negative signal — modern Dash Core removed BIP61 +//! `reject` — so a transaction whose SPV broadcast saw no acceptance signal +//! (`BroadcastResult::Uncertain` → [`BroadcastError::MaybeSent`]) stays +//! ambiguous for as long as nobody asks. +//! +//! Core's `sendrawtransaction`, which DAPI's `broadcastTransaction` wraps, is +//! not silent. Measured against Dash Core 22.1.3 (testnet and mainnet, +//! 2026-09-24): +//! +//! | transaction | Core answer | +//! |-----------------------------------------------|-------------------------------------------| +//! | already in the mempool | success, the txid | +//! | already in a block, an output still unspent | `-27 Transaction already in block chain` | +//! | already in a block, **every output spent** | `-25 bad-txns-inputs-missingorspent` | +//! | spends an input that does not exist | `-25 bad-txns-inputs-missingorspent` | +//! | spends an input an IS-locked tx already spent | `-26 tx-txlock-conflict` | +//! +//! The probe uses the positive answers: a node holding the transaction, in its +//! mempool or in a block, makes it accepted. A refusal never +//! settles anything. Proving that a transaction can never land would need the +//! finality of whatever spent its input, which these answers do not show — a +//! node lagging behind a parent's block, an ancestor still propagating, or a +//! competing spend in a block that can still be reorged all look the same. So +//! a refusal only sends the probe to ask whether the txid is known at all +//! (DAPI `getTransaction`, served from the evonode's transaction index): the +//! third row is a *mined* transaction whose outputs are all spent (measured +//! on testnet 2026-09-29: `e758f06a…`, block 1560020, both outputs spent → +//! `-25`), and the lookup is what finds it. Otherwise the transaction stays +//! unresolved, with the refusal in the reason. +//! +//! The probe does not tell "in a mempool" from "in a block": no host and no +//! schedule reads the difference, and a node's word on a block is not +//! settlement — the block may be on a stale fork or about to be reorged out. +//! The wallet's own block processing settles a send; until then the resolver +//! keeps probing an accepted one like any other. +//! +//! The two DAPI implementations surface those codes differently — the JS +//! server maps `-25` to `InvalidArgument`, rs-dapi to `FailedPrecondition` — +//! so the classification keys on Core's reason text, which both pass through +//! in the status message, and on `AlreadyExists` for `-27`, which both agree +//! on. +//! +//! [`BroadcastError::MaybeSent`]: crate::broadcaster::BroadcastError::MaybeSent + +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use dash_sdk::core::transaction_from_reply; +use dash_sdk::dapi_client::transport::TransportError; +use dash_sdk::dapi_client::{ + DapiClientError, DapiRequestExecutor, ExecutionResult, RequestSettings, +}; +use dash_sdk::dapi_grpc::core::v0::{ + BroadcastTransactionRequest, BroadcastTransactionResponse, GetTransactionRequest, + GetTransactionResponse, +}; +use dash_sdk::dapi_grpc::tonic::Code; +use dashcore::consensus; +use dashcore::{Transaction, Txid}; +use tokio::time::timeout; + +/// Core reject reasons about the transaction's inputs. +/// +/// - `bad-txns-inputs-missingorspent`: an input is neither in the UTXO set +/// nor created by anything in this node's mempool. `Missing inputs` is the +/// older spelling of the same check. Also what a mined transaction whose +/// outputs are all spent gets. +/// - `tx-txlock-conflict`: an input is already spent by an InstantSend-locked +/// transaction. +/// +/// Neither settles the transaction (see the module docs); [`REFUSAL_QUORUM`] +/// of them send the probe to txid lookups. +/// +/// `txn-mempool-conflict` is deliberately absent: another, unconfirmed and +/// not IS-locked, transaction spends the same input — nothing to look up. +const REFUSAL_REASONS: &[&str] = &[ + "bad-txns-inputs-missingorspent", + "missing inputs", + "tx-txlock-conflict", +]; + +/// Phrases of a gRPC status that a broken connection, not a node, produced. +const TRANSPORT_MARKERS: &[&str] = &[ + "h2 protocol error", + "connection closed", + "connection reset", + "broken pipe", + "transport error", + // A gateway answered in plain HTTP (a 500 from a load balancer whose + // upstream is down): no node's gRPC status. + "grpc-status header missing", +]; + +/// What a single Core node said when handed a signed transaction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum NodeVerdict { + /// The node holds the transaction: it accepted it now, already had it in + /// its mempool, or has it in a block (`-27`). + Accepted, + /// The node refused it for its inputs ([`REFUSAL_REASONS`]). + Refused { reason: String }, + /// The node answered, but with no verdict: another rejection (such as + /// `txn-mempool-conflict`). + Unknown { reason: String }, + /// No node answered: unreachable, timed out, or no address left to try. + Unreachable { reason: String }, +} + +/// Classify a failed `broadcastTransaction` gRPC response. +pub(crate) fn classify_failed_submission(code: Code, message: &str) -> NodeVerdict { + // Transport-level: no node's answer reached us — including a stream the + // connection broke mid-response, which tonic reports as Unknown or + // Internal (Core's own rejections arrive with their reason instead). + let lowered = message.to_ascii_lowercase(); + let broken_stream = matches!(code, Code::Unknown | Code::Internal) + && TRANSPORT_MARKERS + .iter() + .any(|marker| lowered.contains(marker)); + // rs-dapi-client's own report that it could not open a channel to the + // node (TLS, URI): the request never left. + let never_opened = lowered.contains("channel creation failed"); + if broken_stream + || never_opened + || matches!( + code, + Code::Unavailable | Code::DeadlineExceeded | Code::Cancelled + ) + { + return NodeVerdict::Unreachable { + reason: format!("{code:?}: {message}"), + }; + } + // `-27`: already in a block, by this node's word. + if code == Code::AlreadyExists { + return NodeVerdict::Accepted; + } + if REFUSAL_REASONS + .iter() + .any(|reason| lowered.contains(reason)) + { + return NodeVerdict::Refused { + reason: message.to_string(), + }; + } + NodeVerdict::Unknown { + reason: format!("{code:?}: {message}"), + } +} + +/// The answer a probe gives about one transaction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProbeVerdict { + /// A node holds the transaction, in its mempool or in a block. Not + /// settlement: it can still expire, lose to a conflict or sit in a block + /// a reorg drops. The resolver keeps probing the send until the wallet + /// settles it, which hands the nodes its bytes again; the verdict itself + /// does not go back to Unresolved. + Accepted, + /// Not enough evidence that it landed; the transaction stays ambiguous. + /// The reason says what the nodes answered, refusals included. + Unresolved { reason: String }, +} + +/// Whether one node knows a txid (DAPI `getTransaction`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum LookupAnswer { + /// The node has the transaction, in its mempool or in a block. + Known, + /// The node answered and does not have it. + NotFound, + /// No answer: transport failure, timeout, anything else. + Unknown { reason: String }, +} + +/// Talks to one Core node per call and reports which node answered and what +/// it said. A seam so the probe's evidence rules can be tested without a +/// network. +#[async_trait] +pub(crate) trait NodeSubmitter: Send + Sync { + async fn submit(&self, transaction: &Transaction) -> (Option, NodeVerdict); + async fn lookup(&self, txid: &Txid) -> (Option, LookupAnswer); +} + +/// How many submissions one probe may spend looking for a verdict. +const MAX_SUBMISSIONS_PER_PROBE: usize = 4; + +/// How many txid lookups one probe may spend after refusals. +const MAX_LOOKUPS_PER_PROBE: usize = 4; + +/// How long a probe's request to one node may take, whatever timeout the SDK +/// it goes through is configured with (an SDK may run with none): set on the +/// request itself (see [`single_node`]), so a node that ran out is still +/// named. A probe is at most [`MAX_SUBMISSIONS_PER_PROBE`] plus +/// [`MAX_LOOKUPS_PER_PROBE`] requests, so this bounds how long one holds its +/// wallet's pass. A request that runs out is no answer. +const REQUEST_DEADLINE: Duration = Duration::from_secs(15); + +/// The probe's own limit on one request, a little past [`REQUEST_DEADLINE`]: +/// it holds whatever the transport underneath does with its deadline. +const REQUEST_BACKSTOP: Duration = Duration::from_secs(20); + +/// The reason of a request the backstop ended. +fn out_of_time() -> String { + format!("no answer within {}s", REQUEST_BACKSTOP.as_secs()) +} + +/// How many *distinct* nodes must refuse a transaction for its inputs before +/// the probe looks the txid up. One refusal may be a node lagging behind the +/// tip; two make it worth asking whether it is a mined transaction whose +/// outputs are all spent. +const REFUSAL_QUORUM: usize = 2; + +/// Resubmits a transaction whose broadcast outcome is unknown and turns the +/// nodes' answers into a verdict. +/// +/// Resubmitting is safe: it is the same signed bytes, so a node either takes +/// a transaction it did not have — an extra delivery route — or tells us why +/// it will not. It can never create a second payment. +#[async_trait] +pub trait AcceptanceProbe: Send + Sync + 'static { + /// What the nodes answered, judged within this one probe. + async fn probe(&self, transaction: &Transaction) -> ProbeOutcome; +} + +/// A probe's verdict, and whether any node answered at all. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProbeOutcome { + pub verdict: ProbeVerdict, + /// Some node answered a submission — with a refusal, or any rejection a + /// node gives (a rate limit included). `false` when none did: no node + /// reachable, timeouts, no address left to try — the probe learned + /// nothing and is worth repeating soon. Not part of the verdict the host + /// is told: it says how the probe went, not where the send stands. + pub answered: bool, +} + +impl ProbeOutcome { + /// A verdict some node gave. An outcome no node answered is built + /// explicitly (`answered: false`): never by default. + pub fn answered(verdict: ProbeVerdict) -> Self { + Self { + verdict, + answered: true, + } + } +} + +/// Evidence rules shared by every probe: a node holding the transaction, in +/// its mempool or in a block, makes it accepted; [`REFUSAL_QUORUM`] distinct +/// nodes refusing it for its inputs send the probe to lookups, which find a +/// mined transaction whose outputs are all spent and otherwise leave it +/// unresolved; give up after +/// [`MAX_SUBMISSIONS_PER_PROBE`] submissions and [`MAX_LOOKUPS_PER_PROBE`] +/// lookups. +pub(crate) async fn probe_with( + submitter: &dyn NodeSubmitter, + transaction: &Transaction, +) -> ProbeOutcome { + let mut refusing_nodes: HashSet = HashSet::new(); + let mut first_refusal: Option = None; + let mut last_other = String::from("no submission was answered"); + let mut answered = false; + + for _ in 0..MAX_SUBMISSIONS_PER_PROBE { + let (node, verdict) = timeout(REQUEST_BACKSTOP, submitter.submit(transaction)) + .await + .unwrap_or_else(|_| { + let reason = out_of_time(); + (None, NodeVerdict::Unreachable { reason }) + }); + answered |= !matches!(verdict, NodeVerdict::Unreachable { .. }); + tracing::debug!( + txid = %transaction.txid(), + node = node.as_deref().unwrap_or("unidentified"), + ?verdict, + "broadcast probe: node answered" + ); + match verdict { + NodeVerdict::Accepted => return ProbeOutcome::answered(ProbeVerdict::Accepted), + NodeVerdict::Refused { reason } => { + // An unidentified node cannot count towards the quorum — it + // might be the node we already heard. + if let Some(node) = node { + refusing_nodes.insert(node); + } + let first = first_refusal.get_or_insert(reason); + if refusing_nodes.len() >= REFUSAL_QUORUM { + let reason = first.clone(); + return ProbeOutcome::answered( + look_up_refused(submitter, transaction, reason).await, + ); + } + } + NodeVerdict::Unknown { reason } => last_other = reason, + // A node's answer stays the reason over a later "no node + // reached". + NodeVerdict::Unreachable { reason } => { + if !answered { + last_other = reason; + } + } + } + } + + let verdict = ProbeVerdict::Unresolved { + reason: match first_refusal { + None => last_other, + Some(reason) => format!( + "{} of {REFUSAL_QUORUM} nodes needed refused it as {reason}; last other answer: {last_other}", + refusing_nodes.len() + ), + }, + }; + ProbeOutcome { verdict, answered } +} + +/// [`REFUSAL_QUORUM`] nodes refused the transaction as `reason`: ask by +/// `getTransaction` whether the txid is known at all. A node that has it — a +/// mined transaction whose outputs are all spent is refused the same way — +/// makes it accepted; nothing better than unresolved comes of nodes not +/// knowing it. +async fn look_up_refused( + submitter: &dyn NodeSubmitter, + transaction: &Transaction, + reason: String, +) -> ProbeVerdict { + let txid = &transaction.txid(); + let mut not_found: HashSet = HashSet::new(); + let mut last_unknown = String::from("no lookup was answered"); + for _ in 0..MAX_LOOKUPS_PER_PROBE { + let (node, answer) = timeout(REQUEST_BACKSTOP, submitter.lookup(txid)) + .await + .unwrap_or_else(|_| { + let reason = out_of_time(); + (None, LookupAnswer::Unknown { reason }) + }); + tracing::debug!( + txid = %txid, + node = node.as_deref().unwrap_or("unidentified"), + ?answer, + "broadcast probe: txid lookup answered" + ); + match answer { + LookupAnswer::Known => return ProbeVerdict::Accepted, + LookupAnswer::NotFound => { + if let Some(node) = node { + not_found.insert(node); + } + // Enough nodes say they do not have it: asking on learns + // nothing more. + if not_found.len() >= REFUSAL_QUORUM { + break; + } + } + LookupAnswer::Unknown { reason } => last_unknown = reason, + } + } + ProbeVerdict::Unresolved { + reason: format!( + "refused as {reason}; {} node(s) do not know the txid; last other lookup answer: {last_unknown}", + not_found.len() + ), + } +} + +/// Which node answered a `broadcastTransaction` submission and what it said. +/// A failure that is not a node's gRPC status (no address left to try, a +/// connection that never opened) is no answer at all. +fn submit_reply( + result: ExecutionResult, +) -> (Option, NodeVerdict) { + match result { + Ok(response) => (Some(response.address.to_string()), NodeVerdict::Accepted), + Err(error) => { + let node = error.address.as_ref().map(ToString::to_string); + let verdict = match &error.inner { + // tonic attaches the transport error (h2, hyper, io) as the + // source of a status it made itself; a node's own status has + // none. The message phrases are only a fallback. + DapiClientError::Transport(TransportError::Grpc(status)) + if matches!(status.code(), Code::Unknown | Code::Internal) + && std::error::Error::source(status).is_some() => + { + NodeVerdict::Unreachable { + reason: format!("{:?}: {}", status.code(), status.message()), + } + } + DapiClientError::Transport(TransportError::Grpc(status)) => { + classify_failed_submission(status.code(), status.message()) + } + other => NodeVerdict::Unreachable { + reason: other.to_string(), + }, + }; + (node, verdict) + } + } +} + +/// Which node answered a `getTransaction` lookup and what the answer says. A +/// reply is read the way the SDK reads one ([`transaction_from_reply`]): an +/// empty one, or one carrying another transaction, is the node not showing the +/// txid, like gRPC `NOT_FOUND`; bytes that do not decode are no answer — +/// neither is evidence that it exists. +fn lookup_reply( + txid: &Txid, + result: ExecutionResult, +) -> (Option, LookupAnswer) { + match result { + Ok(response) => { + let node = Some(response.address.to_string()); + let answer = match transaction_from_reply(&response.inner, txid) { + Ok(None) => LookupAnswer::NotFound, + Ok(Some(_)) => lookup_answer(&response.inner), + Err(error) => LookupAnswer::Unknown { + reason: format!("unusable reply: {error}"), + }, + }; + (node, answer) + } + Err(error) => { + let node = error.address.as_ref().map(ToString::to_string); + let answer = match &error.inner { + DapiClientError::Transport(TransportError::Grpc(status)) + if status.code() == Code::NotFound => + { + LookupAnswer::NotFound + } + other => LookupAnswer::Unknown { + reason: other.to_string(), + }, + }; + (node, answer) + } + } +} + +/// What a node's `getTransaction` answer says about a transaction it holds. +/// A block hash with no height and +/// no confirmations is a block off the node's active chain (a stale fork; both +/// DAPI servers send height 0 for Core's -1): neither mined nor in a mempool. +fn lookup_answer(tx: &GetTransactionResponse) -> LookupAnswer { + let in_block = tx.height > 0 || tx.confirmations > 0; + if !in_block && !tx.block_hash.is_empty() { + return LookupAnswer::Unknown { + reason: "known only from a block off the active chain".to_string(), + }; + } + LookupAnswer::Known +} + +/// One node per request: the probe decides whether to ask another, and it has +/// to know which node each answer came from. A probe never bans the address +/// it reached: it runs exactly when the network is flaky, and the SDK's +/// shared address list would carry its bans to every other caller — and to +/// the probe's own retries. +fn single_node() -> RequestSettings { + RequestSettings { + retries: Some(0), + ban_failed_address: Some(false), + timeout: Some(REQUEST_DEADLINE), + ..RequestSettings::default() + } +} + +/// Talks to evonodes through DAPI: `broadcastTransaction` (a Core +/// `sendrawtransaction`) and `getTransaction`, a random node per request. +pub(crate) struct DapiNodeSubmitter { + sdk: Arc, +} + +#[async_trait] +impl NodeSubmitter for DapiNodeSubmitter { + async fn submit(&self, transaction: &Transaction) -> (Option, NodeVerdict) { + let request = BroadcastTransactionRequest { + transaction: consensus::serialize(transaction), + allow_high_fees: false, + bypass_limits: false, + }; + submit_reply(self.sdk.execute(request, single_node()).await) + } + + async fn lookup(&self, txid: &Txid) -> (Option, LookupAnswer) { + // `id` is the display hex, as Core's getrawtransaction takes it. + let request = GetTransactionRequest { + id: txid.to_string(), + }; + lookup_reply(txid, self.sdk.execute(request, single_node()).await) + } +} + +/// The production probe: resubmits through DAPI. +pub struct DapiAcceptanceProbe { + submitter: DapiNodeSubmitter, +} + +impl DapiAcceptanceProbe { + pub fn new(sdk: Arc) -> Self { + Self { + submitter: DapiNodeSubmitter { sdk }, + } + } +} + +#[async_trait] +impl AcceptanceProbe for DapiAcceptanceProbe { + async fn probe(&self, transaction: &Transaction) -> ProbeOutcome { + probe_with(&self.submitter, transaction).await + } +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + use std::sync::Mutex; + + use dash_sdk::dapi_client::{ExecutionError, ExecutionResponse}; + use dash_sdk::dapi_grpc::tonic::{Code, Status}; + + use super::*; + + fn transaction() -> Transaction { + Transaction { + version: 1, + lock_time: 0, + input: Vec::new(), + output: Vec::new(), + special_transaction_payload: None, + } + } + + /// Replays scripted node answers in order and counts calls. Lookups + /// default to two distinct nodes not knowing the txid, unless a test + /// scripts otherwise. + struct ScriptedNodes { + answers: Mutex, NodeVerdict)>>, + lookups: Mutex, LookupAnswer)>>, + submissions: Mutex, + lookups_made: Mutex, + } + + impl ScriptedNodes { + fn new(answers: Vec<(Option<&'static str>, NodeVerdict)>) -> Self { + Self::with_lookups( + answers, + vec![ + (Some("x"), LookupAnswer::NotFound), + (Some("y"), LookupAnswer::NotFound), + ], + ) + } + + fn with_lookups( + answers: Vec<(Option<&'static str>, NodeVerdict)>, + lookups: Vec<(Option<&'static str>, LookupAnswer)>, + ) -> Self { + Self { + answers: Mutex::new(answers.into()), + lookups: Mutex::new(lookups.into()), + submissions: Mutex::new(0), + lookups_made: Mutex::new(0), + } + } + + fn submissions(&self) -> usize { + *self.submissions.lock().expect("submissions") + } + + fn lookups_made(&self) -> usize { + *self.lookups_made.lock().expect("lookups") + } + } + + #[async_trait] + impl NodeSubmitter for ScriptedNodes { + async fn submit(&self, _transaction: &Transaction) -> (Option, NodeVerdict) { + *self.submissions.lock().expect("submissions") += 1; + let (node, verdict) = self + .answers + .lock() + .expect("answers") + .pop_front() + .unwrap_or(( + None, + NodeVerdict::Unknown { + reason: "script exhausted".to_string(), + }, + )); + (node.map(str::to_string), verdict) + } + + async fn lookup(&self, _txid: &Txid) -> (Option, LookupAnswer) { + *self.lookups_made.lock().expect("lookups") += 1; + let (node, answer) = self + .lookups + .lock() + .expect("lookups") + .pop_front() + .unwrap_or(( + None, + LookupAnswer::Unknown { + reason: "script exhausted".to_string(), + }, + )); + (node.map(str::to_string), answer) + } + } + + fn known() -> LookupAnswer { + LookupAnswer::Known + } + + fn conflict() -> NodeVerdict { + NodeVerdict::Refused { + reason: "Transaction is rejected: tx-txlock-conflict".to_string(), + } + } + + fn refused() -> NodeVerdict { + NodeVerdict::Refused { + reason: "bad-txns-inputs-missingorspent".to_string(), + } + } + + fn unknown() -> NodeVerdict { + NodeVerdict::Unknown { + reason: "Unavailable: connection refused".to_string(), + } + } + + fn is_unresolved(verdict: &ProbeVerdict) -> bool { + matches!(verdict, ProbeVerdict::Unresolved { .. }) + } + + #[test] + fn should_classify_core_answers_the_way_both_dapi_servers_surface_them() { + let cases = [ + // -27, both servers. + ( + Code::AlreadyExists, + "Transaction already in chain: Transaction already in block chain", + NodeVerdict::Accepted, + ), + // -25, JS dapi. + ( + Code::InvalidArgument, + "invalid transaction: bad-txns-inputs-missingorspent", + NodeVerdict::Refused { + reason: "invalid transaction: bad-txns-inputs-missingorspent".to_string(), + }, + ), + // -25, rs-dapi. + ( + Code::FailedPrecondition, + "Transaction is rejected: bad-txns-inputs-missingorspent", + NodeVerdict::Refused { + reason: "Transaction is rejected: bad-txns-inputs-missingorspent".to_string(), + }, + ), + // -26 against an IS-locked spend. + ( + Code::FailedPrecondition, + "Transaction is rejected: tx-txlock-conflict", + conflict(), + ), + // -26 against an unconfirmed spend. + ( + Code::FailedPrecondition, + "Transaction is rejected: txn-mempool-conflict", + NodeVerdict::Unknown { + reason: "FailedPrecondition: Transaction is rejected: txn-mempool-conflict" + .to_string(), + }, + ), + ]; + + for (code, message, expected) in cases { + assert_eq!( + classify_failed_submission(code, message), + expected, + "{code:?} {message}" + ); + } + } + + /// Neither a transport failure nor an unknown rejection is a verdict; + /// only the second is a node's answer. + #[test] + fn should_leave_transport_failures_and_unknown_rejections_without_a_verdict() { + for (code, message) in [ + (Code::Unavailable, "connection refused"), + (Code::DeadlineExceeded, "timeout"), + ] { + assert!( + matches!( + classify_failed_submission(code, message), + NodeVerdict::Unreachable { .. } + ), + "{code:?} {message}" + ); + } + for (code, message) in [ + ( + Code::InvalidArgument, + "invalid transaction: TX decode failed", + ), + (Code::Internal, "bad-txns-in-belowout"), + // A gateway's rate limit is a node's answer: back off, not an + // outage. + (Code::ResourceExhausted, "rate limited"), + ] { + assert!( + matches!( + classify_failed_submission(code, message), + NodeVerdict::Unknown { .. } + ), + "{code:?} {message}" + ); + } + } + + #[tokio::test] + async fn should_accept_as_soon_as_any_node_holds_the_transaction() { + let nodes = ScriptedNodes::new(vec![ + (Some("a"), refused()), + (Some("b"), NodeVerdict::Accepted), + ]); + + assert_eq!( + probe_with(&nodes, &transaction()).await.verdict, + ProbeVerdict::Accepted + ); + assert_eq!(nodes.submissions(), 2); + } + + /// Refusals never settle a transaction: a lagging node, an ancestor + /// still propagating and a reorgable competing spend all look alike. + #[tokio::test] + async fn should_never_settle_a_transaction_on_refusals_and_unknown_txids() { + for answers in [ + vec![(Some("a"), refused()), (Some("b"), refused())], + vec![(Some("a"), conflict()), (Some("b"), conflict())], + vec![(Some("a"), conflict()), (Some("b"), refused())], + ] { + let nodes = ScriptedNodes::new(answers); + let verdict = probe_with(&nodes, &transaction()).await.verdict; + + assert!(is_unresolved(&verdict), "{verdict:?}"); + assert_eq!(nodes.lookups_made(), REFUSAL_QUORUM); + } + } + + #[tokio::test] + async fn should_keep_the_refusal_in_the_unresolved_reason() { + let nodes = ScriptedNodes::new(vec![(Some("a"), conflict()), (Some("b"), conflict())]); + + let ProbeVerdict::Unresolved { reason } = probe_with(&nodes, &transaction()).await.verdict + else { + panic!("expected unresolved"); + }; + assert!(reason.contains("tx-txlock-conflict"), "{reason}"); + } + + /// A lagging node answers `missingorspent` for a valid transaction. Hearing + /// the same node twice must not look like two opinions. + #[tokio::test] + async fn should_not_count_the_same_node_twice_towards_the_refusal_quorum() { + let nodes = ScriptedNodes::new(vec![ + (Some("a"), refused()), + (Some("a"), refused()), + (Some("a"), refused()), + (Some("a"), refused()), + ]); + + assert!(is_unresolved( + &probe_with(&nodes, &transaction()).await.verdict + )); + assert_eq!(nodes.submissions(), MAX_SUBMISSIONS_PER_PROBE); + assert_eq!(nodes.lookups_made(), 0); + } + + #[tokio::test] + async fn should_not_count_an_unidentified_node_towards_the_refusal_quorum() { + let nodes = ScriptedNodes::new(vec![(Some("a"), refused()), (None, refused())]); + + assert!(is_unresolved( + &probe_with(&nodes, &transaction()).await.verdict + )); + assert_eq!(nodes.lookups_made(), 0); + } + + #[tokio::test] + async fn should_stay_unresolved_when_nodes_only_fail() { + let nodes = ScriptedNodes::new(vec![ + ( + Some("a"), + NodeVerdict::Unknown { + reason: "txn-mempool-conflict".to_string(), + }, + ), + (Some("b"), unknown()), + (Some("c"), unknown()), + (Some("d"), unknown()), + ]); + + assert!(is_unresolved( + &probe_with(&nodes, &transaction()).await.verdict + )); + assert_eq!(nodes.submissions(), MAX_SUBMISSIONS_PER_PROBE); + } + + #[tokio::test] + async fn should_reach_the_refusal_quorum_across_intervening_failures() { + let nodes = ScriptedNodes::new(vec![ + (Some("a"), refused()), + (Some("b"), unknown()), + (Some("c"), refused()), + ]); + + assert!(is_unresolved( + &probe_with(&nodes, &transaction()).await.verdict + )); + assert_eq!(nodes.submissions(), 3); + assert_eq!(nodes.lookups_made(), REFUSAL_QUORUM); + } + + /// Core answers `missingorspent` for a mined transaction whose outputs + /// are all spent (testnet `e758f06a…`, 2026-09-29), and for one still in + /// a mempool that lost a double-spend race on the refusing nodes. A node + /// that knows the txid turns the refusals into Accepted, at the first + /// lookup. + #[tokio::test] + async fn should_accept_a_refused_transaction_a_node_still_knows() { + let nodes = ScriptedNodes::with_lookups( + vec![(Some("a"), refused()), (Some("b"), refused())], + vec![(Some("c"), known())], + ); + + assert_eq!( + probe_with(&nodes, &transaction()).await.verdict, + ProbeVerdict::Accepted + ); + assert_eq!(nodes.lookups_made(), 1); + } + + /// A node's "already in a block" (`-27`) is acceptance, with no lookup + /// for a second opinion: the probe does not tell a block from a mempool. + #[tokio::test] + async fn should_accept_on_one_nodes_word_that_it_is_in_a_block() { + let nodes = ScriptedNodes::with_lookups( + vec![( + Some("a"), + classify_failed_submission(Code::AlreadyExists, "Transaction already in chain"), + )], + vec![(Some("b"), known())], + ); + + assert_eq!( + probe_with(&nodes, &transaction()).await.verdict, + ProbeVerdict::Accepted + ); + assert_eq!(nodes.submissions(), 1); + assert_eq!(nodes.lookups_made(), 0); + } + + #[test] + fn should_read_a_stale_fork_lookup_as_neither_mined_nor_mempool() { + let answer = |block_hash: Vec, height, confirmations| { + lookup_answer(&GetTransactionResponse { + block_hash, + height, + confirmations, + ..GetTransactionResponse::default() + }) + }; + + assert_eq!(answer(Vec::new(), 0, 0), known()); + assert_eq!(answer(vec![1; 32], 100, 3), known()); + assert!(matches!( + answer(vec![1; 32], 0, 0), + LookupAnswer::Unknown { .. } + )); + } + + /// The production lookup adapter reads a reply as the SDK does: empty + /// bytes or another transaction are the node not showing the txid, + /// malformed bytes are no answer — neither is evidence that it exists — + /// and the node that answered is kept either way. + #[test] + fn should_not_take_an_empty_or_malformed_lookup_reply_as_known() { + let held = Transaction { + lock_time: 7, + ..transaction() + }; + let reply = |bytes: Vec| { + Ok(ExecutionResponse { + inner: GetTransactionResponse { + transaction: bytes, + ..GetTransactionResponse::default() + }, + retries: 0, + address: "http://127.0.0.1:1443".parse().expect("address"), + }) + }; + let txid = held.txid(); + + let (node, empty) = lookup_reply(&txid, reply(Vec::new())); + assert_eq!(empty, LookupAnswer::NotFound); + assert!(node.is_some(), "the serving node is kept"); + + let (node, malformed) = lookup_reply(&txid, reply(vec![0xff, 0x01])); + assert!( + matches!(malformed, LookupAnswer::Unknown { .. }), + "{malformed:?}" + ); + assert!(node.is_some()); + + let other = consensus::serialize(&transaction()); + let (_, mismatched) = lookup_reply(&txid, reply(other)); + assert_eq!(mismatched, LookupAnswer::NotFound); + + let (_, answer) = lookup_reply(&txid, reply(consensus::serialize(&held))); + assert_eq!(answer, known()); + } + + /// A node's answer stays the reason when later submissions reach no + /// node: the conflict is what the log should show, not the outage. + #[tokio::test] + async fn should_keep_a_nodes_answer_as_the_reason_over_later_unreachable_ones() { + let unreachable = || NodeVerdict::Unreachable { + reason: "no available addresses".to_string(), + }; + let nodes = ScriptedNodes::new(vec![ + ( + Some("a"), + NodeVerdict::Unknown { + reason: "txn-mempool-conflict".to_string(), + }, + ), + (None, unreachable()), + (None, unreachable()), + (None, unreachable()), + ]); + + let outcome = probe_with(&nodes, &transaction()).await; + + assert!(outcome.answered); + let ProbeVerdict::Unresolved { reason } = outcome.verdict else { + panic!("unresolved"); + }; + assert!(reason.contains("txn-mempool-conflict"), "{reason}"); + } + + /// A rate-limited submission is a node's answer: no fast retry. + #[tokio::test] + async fn should_count_a_rate_limit_as_an_answer() { + let limited = || NodeVerdict::Unknown { + reason: "ResourceExhausted: rate limited".to_string(), + }; + let nodes = ScriptedNodes::new(vec![ + (Some("a"), limited()), + (Some("b"), limited()), + (Some("c"), limited()), + (Some("d"), limited()), + ]); + assert!(probe_with(&nodes, &transaction()).await.answered); + assert!(matches!( + classify_failed_submission(Code::ResourceExhausted, "rate limited"), + NodeVerdict::Unknown { .. } + )); + } + + /// The live log's case: every address banned, so the SDK fails with + /// `NoAvailableAddresses` before any node is asked — no answer, which + /// arms the resolver's retry. A node's status is an answer. + #[test] + fn should_read_a_submission_no_node_received_as_unreachable() { + let failed = |inner: DapiClientError| { + Err(ExecutionError { + inner, + retries: 0, + address: None, + }) + }; + let (node, verdict) = submit_reply(failed(DapiClientError::NoAvailableAddresses)); + assert!(node.is_none()); + assert!( + matches!(verdict, NodeVerdict::Unreachable { .. }), + "{verdict:?}" + ); + + let (_, verdict) = submit_reply(failed(DapiClientError::Transport(TransportError::Grpc( + Status::failed_precondition("txn-mempool-conflict"), + )))); + assert!( + matches!(verdict, NodeVerdict::Unknown { .. }), + "{verdict:?}" + ); + + // A stream that broke mid-response, whatever its wording: tonic's + // status carries the transport error as its source. + let broken = Status::from_error(Box::new(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "unexpected end of file", + ))); + let (_, verdict) = submit_reply(failed(DapiClientError::Transport(TransportError::Grpc( + broken, + )))); + assert!( + matches!(verdict, NodeVerdict::Unreachable { .. }), + "{verdict:?}" + ); + // A node's Unknown/Internal with no transport source is an answer. + let (_, verdict) = submit_reply(failed(DapiClientError::Transport(TransportError::Grpc( + Status::internal("bad-txns-in-belowout"), + )))); + assert!( + matches!(verdict, NodeVerdict::Unknown { .. }), + "{verdict:?}" + ); + let (_, verdict) = submit_reply(failed(DapiClientError::Transport(TransportError::Grpc( + Status::cancelled("cancelled"), + )))); + assert!( + matches!(verdict, NodeVerdict::Unreachable { .. }), + "{verdict:?}" + ); + } + + /// A probe never bans the address it reached, asks one node per request, + /// and gives each request its own deadline, inside the probe's backstop, + /// so the SDK's timeout (or its absence) does not decide it. + #[test] + fn should_probe_without_banning_without_retries_and_with_its_own_deadline() { + let settings = single_node(); + assert_eq!(settings.ban_failed_address, Some(false)); + assert_eq!(settings.retries, Some(0)); + assert_eq!(settings.timeout, Some(REQUEST_DEADLINE)); + assert!(REQUEST_DEADLINE < REQUEST_BACKSTOP); + } + + /// A failed lookup: gRPC `NOT_FOUND` is the node not knowing the txid, + /// anything else no answer — the node, when known, is kept either way + /// (the refusal quorum counts distinct nodes). + #[test] + fn should_read_a_failed_lookup_by_its_status_and_keep_the_node() { + let failed = |status: Status| { + Err(ExecutionError { + inner: DapiClientError::Transport(TransportError::Grpc(status)), + retries: 0, + address: Some("http://127.0.0.1:1443".parse().expect("address")), + }) + }; + let txid = transaction().txid(); + + let (node, missing) = lookup_reply(&txid, failed(Status::not_found("no such tx"))); + assert_eq!(missing, LookupAnswer::NotFound); + assert!(node.is_some(), "the node that answered is kept"); + + let (node, other) = lookup_reply(&txid, failed(Status::unavailable("down"))); + assert!(matches!(other, LookupAnswer::Unknown { .. }), "{other:?}"); + assert!(node.is_some()); + } + + #[tokio::test] + async fn should_stop_looking_up_once_two_distinct_nodes_do_not_know_the_txid() { + let nodes = ScriptedNodes::with_lookups( + vec![(Some("a"), refused()), (Some("b"), refused())], + vec![ + (Some("c"), LookupAnswer::NotFound), + (Some("c"), LookupAnswer::NotFound), + (None, LookupAnswer::NotFound), + (Some("d"), LookupAnswer::NotFound), + ], + ); + + assert!(is_unresolved( + &probe_with(&nodes, &transaction()).await.verdict + )); + assert_eq!(nodes.lookups_made(), MAX_LOOKUPS_PER_PROBE); + } + + /// A probe tells whether any node answered: every submission failing + /// without an answer (no reachable node) is `answered: false` — worth + /// repeating soon — while a refusal short of the quorum, or any other + /// rejection, is an answer. + #[tokio::test] + async fn should_tell_a_probe_no_node_answered_from_one_a_node_refused() { + let unreachable = || NodeVerdict::Unreachable { + reason: "no available addresses".to_string(), + }; + let nodes = ScriptedNodes::new(vec![ + (None, unreachable()), + (None, unreachable()), + (None, unreachable()), + (None, unreachable()), + ]); + assert!(!probe_with(&nodes, &transaction()).await.answered); + + for answer in [ + refused(), + NodeVerdict::Unknown { + reason: "txn-mempool-conflict".to_string(), + }, + ] { + let nodes = ScriptedNodes::new(vec![ + (Some("a"), answer), + (None, unreachable()), + (None, unreachable()), + (None, unreachable()), + ]); + assert!(probe_with(&nodes, &transaction()).await.answered); + } + for (code, message) in [ + (Code::DeadlineExceeded, "no complete response"), + ( + Code::Internal, + "h2 protocol error: error reading a body from connection", + ), + ( + Code::Unknown, + "grpc-status header missing, mapped from HTTP status code 500", + ), + ( + Code::InvalidArgument, + "Channel creation failed: invalid uri", + ), + (Code::Unknown, "connection closed before message completed"), + ] { + assert!( + matches!( + classify_failed_submission(code, message), + NodeVerdict::Unreachable { .. } + ), + "{code:?} {message}" + ); + } + } + + #[tokio::test] + async fn should_not_look_up_a_transaction_nobody_refused() { + let nodes = ScriptedNodes::new(vec![(Some("a"), NodeVerdict::Accepted)]); + + probe_with(&nodes, &transaction()).await; + + assert_eq!(nodes.lookups_made(), 0); + } + + /// Nodes that never answer, as behind an SDK configured with no request + /// timeout. + struct SilentNodes; + + #[async_trait] + impl NodeSubmitter for SilentNodes { + async fn submit(&self, _transaction: &Transaction) -> (Option, NodeVerdict) { + std::future::pending().await + } + + async fn lookup(&self, _txid: &Txid) -> (Option, LookupAnswer) { + std::future::pending().await + } + } + + /// A probe ends by its own deadline when no node ever answers: it does + /// not hold its wallet's pass for as long as the transport waits. + #[tokio::test(start_paused = true)] + async fn should_end_a_probe_whose_requests_are_never_answered() { + let started = tokio::time::Instant::now(); + + let outcome = probe_with(&SilentNodes, &transaction()).await; + + assert!(!outcome.answered); + assert!(is_unresolved(&outcome.verdict)); + assert_eq!( + started.elapsed(), + REQUEST_BACKSTOP * MAX_SUBMISSIONS_PER_PROBE as u32 + ); + } + + /// A lookup that is never answered does not take back the refusals the + /// probe already heard: it ends unresolved, as an answered probe. + #[tokio::test(start_paused = true)] + async fn should_end_a_lookup_that_is_never_answered() { + struct RefusingThenSilent(ScriptedNodes); + + #[async_trait] + impl NodeSubmitter for RefusingThenSilent { + async fn submit(&self, transaction: &Transaction) -> (Option, NodeVerdict) { + self.0.submit(transaction).await + } + + async fn lookup(&self, _txid: &Txid) -> (Option, LookupAnswer) { + std::future::pending().await + } + } + let nodes = RefusingThenSilent(ScriptedNodes::new(vec![ + (Some("a"), refused()), + (Some("b"), refused()), + ])); + + let outcome = probe_with(&nodes, &transaction()).await; + + assert!(outcome.answered); + assert!(is_unresolved(&outcome.verdict)); + } +} diff --git a/packages/rs-platform-wallet/src/broadcaster.rs b/packages/rs-platform-wallet/src/broadcaster.rs index dbbbf6ead04..c88d4b0bf1a 100644 --- a/packages/rs-platform-wallet/src/broadcaster.rs +++ b/packages/rs-platform-wallet/src/broadcaster.rs @@ -9,15 +9,21 @@ //! announcing the txid back, an InstantSend lock, or a confirmation proves //! the network accepted it. Trustless; no DAPI involvement. //! - [`DapiBroadcaster`] (fallback for wallets without an SPV runtime): -//! submission via DAPI's gRPC endpoint, with every failure conservatively -//! classified as [`BroadcastError::MaybeSent`]. +//! submission via DAPI's gRPC endpoint. A node that already has the +//! transaction on chain counts as success; every other failure is +//! conservatively [`BroadcastError::MaybeSent`] (see [`crate::broadcast_probe`] +//! for why one node's rejection proves nothing). use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; +use dash_sdk::dapi_client::transport::TransportError; +use dash_sdk::dapi_client::{DapiClientError, DapiRequestExecutor, IntoInner, RequestSettings}; +use dash_sdk::dapi_grpc::core::v0::{BroadcastTransactionRequest, BroadcastTransactionResponse}; +use dash_sdk::dapi_grpc::tonic::Code; use dash_spv::BroadcastResult; -use dashcore::{Transaction, Txid}; +use dashcore::{consensus, Transaction, Txid}; use crate::error::PlatformWalletError; use crate::spv::SpvRuntime; @@ -117,10 +123,6 @@ impl DapiBroadcaster { #[async_trait] impl TransactionBroadcaster for DapiBroadcaster { async fn broadcast(&self, transaction: &Transaction) -> Result { - use dash_sdk::dapi_client::{DapiRequestExecutor, IntoInner, RequestSettings}; - use dash_sdk::dapi_grpc::core::v0::BroadcastTransactionRequest; - use dashcore::consensus; - let tx_bytes = consensus::serialize(transaction); let request = BroadcastTransactionRequest { @@ -129,27 +131,42 @@ impl TransactionBroadcaster for DapiBroadcaster { bypass_limits: false, }; - // Every DAPI failure is classified `MaybeSent`: `sdk.execute` retries - // across nodes internally (RequestSettings::default()), so the error - // surfaced here is only the *last* attempt's — an earlier attempt may - // have delivered the transaction even though the response was lost - // (the classic shape being a node that accepts the tx while its gRPC - // response times out, followed by a retry that fails differently). - // Distinguishing a genuinely pre-send rejection would require - // disabling the internal retries and inspecting transport errors; - // until then the conservative classification keeps reserved inputs - // safe from double-spends at the cost of holding them for the - // reservation TTL. - let _response = self + let result = self .sdk .execute(request, RequestSettings::default()) .await - .into_inner() - .map_err(|e| BroadcastError::MaybeSent { - reason: format!("DAPI broadcast failed: {}", e), - })?; + .into_inner(); + dapi_broadcast_outcome(transaction.txid(), result) + } +} - Ok(transaction.txid()) +/// What a DAPI broadcast answer means for the send. +/// +/// A node that answers `AlreadyExists` (Core's -27: the transaction is +/// mined and at least one of its outputs is still unspent) has it, so that +/// is success, whatever the message says. Every other +/// failure stays `MaybeSent`, including Core's own rejections: one node's +/// `missingorspent` is also what a lagging node, a node that has not seen +/// our unconfirmed parent, and a mined transaction whose outputs are all +/// spent produce (see `broadcast_probe`), and the `Rejected` contract +/// demands proof that the transaction never entered the network, which +/// one answer cannot give. `sdk.execute` also retries transport failures +/// across nodes, so an earlier attempt may have delivered it before this +/// error. +fn dapi_broadcast_outcome( + txid: Txid, + result: Result, +) -> Result { + match result { + Ok(_response) => Ok(txid), + Err(DapiClientError::Transport(TransportError::Grpc(status))) + if status.code() == Code::AlreadyExists => + { + Ok(txid) + } + Err(error) => Err(BroadcastError::MaybeSent { + reason: format!("DAPI broadcast failed: {error}"), + }), } } @@ -283,6 +300,8 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Mutex; + use dash_sdk::dapi_grpc::tonic::Status; + use super::*; struct AcceptanceSpy { @@ -379,6 +398,87 @@ mod tests { } } + /// A DAPI node that already has the transaction (`AlreadyExists`) is a + /// delivered send. A Core refusal stays `MaybeSent`, never `Rejected`, + /// whichever code DAPI gives it (-25: `InvalidArgument` from JS-DAPI, + /// `FailedPrecondition` from rs-dapi; -26: `FailedPrecondition` from + /// both; any other Core error: `Internal` from JS-DAPI, `Unavailable` + /// from rs-dapi): a mined + /// transaction whose outputs are all spent is refused the same way. So + /// does a transport failure, and running out of nodes after an attempt + /// that may have delivered it. + #[test] + fn should_take_already_exists_as_sent_and_every_other_dapi_failure_as_maybe_sent() { + let txid = transaction().txid(); + let grpc = |code, message: &str| { + Err(DapiClientError::Transport(TransportError::Grpc( + Status::new(code, message), + ))) + }; + + let sent = [ + ("accepted", Ok(BroadcastTransactionResponse::default())), + ( + "already in chain", + grpc(Code::AlreadyExists, "Transaction already in chain: "), + ), + ]; + for (label, answer) in sent { + let outcome = dapi_broadcast_outcome(txid, answer); + assert!( + matches!(outcome, Ok(sent) if sent == txid), + "{label}: got {outcome:?}" + ); + } + + let maybe_sent = [ + ( + "JS-DAPI refusal", + grpc( + Code::InvalidArgument, + "invalid transaction: bad-txns-inputs-missingorspent", + ), + ), + ( + "rs-dapi refusal", + grpc( + Code::FailedPrecondition, + "Transaction is rejected: bad-txns-inputs-missingorspent", + ), + ), + ( + "lock conflict", + grpc( + Code::FailedPrecondition, + "Transaction is rejected: tx-txlock-conflict", + ), + ), + ( + "JS-DAPI rethrown Core error", + grpc(Code::Internal, "Internal error"), + ), + ( + "rs-dapi other Core error", + grpc(Code::Unavailable, "still warming up"), + ), + ("transport", grpc(Code::Unavailable, "connection refused")), + ("deadline", grpc(Code::DeadlineExceeded, "timeout")), + ( + "out of nodes after an attempt", + Err(DapiClientError::NoAvailableAddressesToRetry(Box::new( + TransportError::Grpc(Status::new(Code::Unavailable, "connection refused")), + ))), + ), + ]; + for (label, answer) in maybe_sent { + let outcome = dapi_broadcast_outcome(txid, answer); + assert!( + matches!(outcome, Err(BroadcastError::MaybeSent { .. })), + "{label}: got {outcome:?}" + ); + } + } + /// The readiness gate the resume paths depend on has to reach the SPV /// channel, budget intact. Callers can only observe readiness through /// `TransactionBroadcaster`, so a `SpvBroadcaster` that silently kept diff --git a/packages/rs-platform-wallet/src/changeset/core_bridge.rs b/packages/rs-platform-wallet/src/changeset/core_bridge.rs index b752f465d88..a6ce3fa90e6 100644 --- a/packages/rs-platform-wallet/src/changeset/core_bridge.rs +++ b/packages/rs-platform-wallet/src/changeset/core_bridge.rs @@ -40,6 +40,7 @@ //! manager's lifetime; on shutdown, fire the [`CancellationToken`] to //! make the task exit cleanly. +use crate::wallet::core::{is_contact_watch_only, record_spends_own_coins}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, Weak}; @@ -514,6 +515,41 @@ pub fn spawn_wallet_event_adapter_with_durable_cursors

( durable_cursors: DurableCursors, cancel: CancellationToken, ) -> JoinHandle<()> +where + P: PlatformWalletPersistence + 'static, +{ + spawn_observed_wallet_event_adapter( + wallet_manager, + persister, + receiver, + sync_fault, + durable_cursors, + cancel, + None, + ) +} + +/// Sees every wallet event the adapter takes off the lossless channel, in +/// order, before it is projected. It runs on the adapter task: it must not +/// block, and a panic in it would end persistence for the session. +pub(crate) type WalletEventObserver = Arc; + +/// [`spawn_wallet_event_adapter_with_durable_cursors`] with an observer of the +/// events it consumes. +/// +/// The persistence channel is the one stream of wallet events that cannot +/// lose any (the broadcast fan-out ends on a lagged receiver), and it has a +/// single receiver, so a second consumer that must not miss an event listens +/// here rather than on a channel of its own. +pub(crate) fn spawn_observed_wallet_event_adapter

( + wallet_manager: Arc>>, + persister: Weak

, + receiver: mpsc::UnboundedReceiver, + sync_fault: Arc, + durable_cursors: DurableCursors, + cancel: CancellationToken, + observer: Option, +) -> JoinHandle<()> where P: PlatformWalletPersistence + 'static, { @@ -524,6 +560,7 @@ where sync_fault, durable_cursors, cancel, + observer, )) } @@ -712,6 +749,7 @@ async fn run_wallet_event_adapter

( sync_fault: Arc, durable_cursors: DurableCursors, cancel: CancellationToken, + observer: Option, ) where P: PlatformWalletPersistence + 'static, { @@ -766,6 +804,11 @@ async fn run_wallet_event_adapter

( } break; }; + // Before the persister is claimed: the observer sees an event even + // when its drain exits uncommitted. + if let Some(observer) = &observer { + observer(&event); + } // Claim the persister before folding anything else off the channel, so // everything this drain consumes is guaranteed a commit: an owner @@ -809,6 +852,9 @@ async fn run_wallet_event_adapter

( while folded < ADAPTER_STORE_BATCH_LIMIT { match receiver.try_recv() { Ok(event) => { + if let Some(observer) = &observer { + observer(&event); + } project_event_into_batch(&wallet_manager, &durable_cursors, &event, &mut batch) .await; folded += 1; @@ -2516,61 +2562,6 @@ fn slices_and_verdicts_for_txid( (slices, verdicts) } -/// Is this record owned by a contact's watch-only DashPay chain? -/// -/// A `DashpayExternalAccount` derives its addresses from the -/// **contact's** xpub, so this wallet can observe those outputs but can -/// never sign for them. They are the contact's coins; this wallet only -/// ever pays into them. -/// -/// dashpay/rust-dashcore#926 established exactly that policy at the -/// balance layer, dropping `dashpay_external_accounts` from -/// `ManagedAccountCollection::all_funding_accounts` (and `_mut`), which -/// covers `balance`, `account_balances`, `utxos` and -/// `get_spendable_utxos` in one place. `dashpay_receival_accounts` were -/// deliberately kept: those derive from *our* xpub, so a contact paying -/// into them really is money arriving. -/// -/// The persistence seam is the same rule's second home. Upstream -/// `check_core_transaction` emits **one `TransactionRecord` per matched -/// account** (`key_wallet::transaction_checking::wallet_checker`), so a -/// payment to a contact produces two records sharing one txid: -/// -/// | record's account | `direction` | `net_amount` | -/// |---------------------------|-------------|------------------| -/// | funding (BIP44/BIP32/…) | `Outgoing` | `change - spent` | -/// | `DashpayExternalAccount` | `Incoming` | `+paid` | -/// -/// The external account's record is not *wrong* about its own account — -/// that chain did receive an output. It is wrong as a description of the -/// **wallet**, and the persisted `transactions` row is keyed by txid -/// alone, with no per-account dimension to disambiguate (the -/// `transaction_account_involvements` table is only written for -/// provider-key accounts). Whichever record is stored last therefore -/// defines the row, and the watch-only one — emitted last, because -/// `all_accounts` visits the DashPay accounts after the standard ones — -/// wins: a payment *away* is persisted as an incoming credit, and its -/// output is written into `txos` as a wallet-owned coin no key of ours -/// can spend. -/// -/// So external-account records are excluded from the persist-time -/// projection entirely, exactly as #926 excluded the accounts from -/// balance aggregation. What survives from the same event is everything -/// that is genuinely ours to remember: the address-used flips and -/// highest-used watermarks (so contact address rotation keeps working), -/// the derived-address rows, and `derive_spent_utxos` (so a contact -/// spending an output persisted *before* this fix still clears the -/// stale row). -/// -/// The predicate itself is canonical upstream -/// ([`AccountType::is_contact_owned`], dashpay/rust-dashcore#952): its -/// exhaustive match forces any future account type to declare whether -/// its coins are the wallet's or a contact's, so this seam and #926's -/// cannot drift apart. -fn is_contact_watch_only(record: &TransactionRecord) -> bool { - record.account_type.is_contact_owned() -} - /// Derive the "ours" UTXOs created by a transaction's outputs. /// /// Walks `record.output_details`, keeps entries with role `Received` or @@ -2592,9 +2583,8 @@ fn derive_new_utxos(record: &TransactionRecord) -> Vec { ); let is_instant = matches!(record.context, TransactionContext::InstantSend(_)); let is_coinbase = record.transaction.is_coin_base(); - // We own at least one input iff the wallet recorded any input details - // (those entries are keyed to inputs that spent our outpoints). - let owns_any_input = !record.input_details.is_empty(); + // Contact-owned records returned above, so this is the input rule alone. + let owns_any_input = record_spends_own_coins(record); record .output_details @@ -5474,7 +5464,8 @@ mod tests { // per-wallet batch folding. use super::{ - run_wallet_event_adapter, AdapterFaultState, PaymentOverlay, ADAPTER_STORE_BATCH_LIMIT, + run_wallet_event_adapter, spawn_observed_wallet_event_adapter, AdapterFaultState, + DurableCursors, PaymentOverlay, WalletEventObserver, ADAPTER_STORE_BATCH_LIMIT, }; use crate::changeset::changeset::PlatformWalletChangeSet; use crate::changeset::client_start_state::ClientStartState; @@ -6246,6 +6237,42 @@ mod tests { } } + /// The observer sees every event of a burst once and in order, whether + /// the adapter took it as the first of a batch or folded it in. + #[tokio::test] + async fn should_show_the_observer_every_event_in_order() { + const BURST: u32 = 3000; + let wallet_id = [7u8; 32]; + let (tx, rx) = unbounded_channel::(); + for h in 1..=BURST { + tx.send(sync_height_event(wallet_id, h)).unwrap(); + } + let seen = Arc::new(Mutex::new(Vec::new())); + let observer: WalletEventObserver = { + let seen = Arc::clone(&seen); + Arc::new(move |event: &WalletEvent| { + if let WalletEvent::SyncHeightAdvanced { height, .. } = event { + seen.lock().unwrap().push(*height); + } + }) + }; + let (obs_tx, _obs_rx) = unbounded_channel(); + let persister = Arc::new(ProbePersister::new(obs_tx)); + let handle = spawn_observed_wallet_event_adapter( + test_manager(), + Arc::downgrade(&persister), + rx, + Arc::new(AtomicBool::new(false)), + DurableCursors::default(), + CancellationToken::new(), + Some(observer), + ); + drop(tx); + handle.await.unwrap(); + + assert_eq!(*seen.lock().unwrap(), (1..=BURST).collect::>()); + } + /// (b) THE ROOT-CAUSE FIX: a burst far larger than the old bounded ring /// (`DEFAULT_WALLET_EVENT_CAPACITY` == 1000) is delivered losslessly over /// the unbounded persistence channel, so the fault latch never trips and @@ -6274,6 +6301,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); // Close the channel so the adapter drains the entire burst (across as @@ -6336,6 +6364,7 @@ mod tests { Arc::new(AtomicBool::new(false)), crate::changeset::DurableCursors::default(), cancel, + None, ) .await; @@ -6392,6 +6421,7 @@ mod tests { Arc::new(AtomicBool::new(false)), crate::changeset::DurableCursors::default(), cancel, + None, )); // Park inside the first chunk's `store()`, then release the only @@ -6454,6 +6484,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); // 1) Record-bearing event whose store() is rejected → faults wallet. @@ -6509,6 +6540,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); // Trip the guard. @@ -6558,6 +6590,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); let observed = obs_rx @@ -6607,6 +6640,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); let observed = obs_rx.recv().await.expect("merged store must arrive"); @@ -6661,6 +6695,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); let mut heights: BTreeMap> = BTreeMap::new(); @@ -6709,6 +6744,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); // The store for this batch panics: no observation is emitted, and the @@ -6797,6 +6833,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); // Park the commit inside a synchronous `store()`. @@ -6859,6 +6896,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); assert_eq!( @@ -6951,6 +6989,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); // All three in one drain: `healthy` stores, `doomed` panics, and @@ -7053,6 +7092,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); // Trip the fault and wait until it has latched. @@ -7142,6 +7182,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); tx.send(swept_event(wallet_id, 0x51, 0x52)).unwrap(); @@ -7217,6 +7258,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); // Bounded like the neighbouring capability tests below: both the @@ -7268,6 +7310,7 @@ mod tests { Arc::clone(&sync_fault), crate::changeset::DurableCursors::default(), cancel.clone(), + None, )); tx.send(swept_event(wallet_id, 0x61, 0x62)).unwrap(); @@ -8596,6 +8639,7 @@ mod tests { Arc::new(AtomicBool::new(false)), crate::changeset::DurableCursors::default(), CancellationToken::new(), + None, )); handle.await.expect("the adapter exits on a closed channel"); diff --git a/packages/rs-platform-wallet/src/events.rs b/packages/rs-platform-wallet/src/events.rs index c329c5a32f5..7a2ead39d6d 100644 --- a/packages/rs-platform-wallet/src/events.rs +++ b/packages/rs-platform-wallet/src/events.rs @@ -12,10 +12,13 @@ use std::sync::Arc; use arc_swap::ArcSwap; +use dashcore::Txid; +use key_wallet_manager::WalletId; pub use dash_spv::EventHandler; pub use key_wallet_manager::WalletEvent; +use crate::broadcast_probe::ProbeVerdict; use crate::manager::dpns_sync::DpnsSyncPassSummary; use crate::manager::platform_address_sync::PlatformAddressSyncSummary; #[cfg(feature = "shielded")] @@ -45,6 +48,32 @@ pub trait PlatformEventHandler: EventHandler { /// [`DpnsSyncManager`]: crate::manager::dpns_sync::DpnsSyncManager fn on_dpns_marketplace_sync_completed(&self, _summary: &DpnsSyncPassSummary) {} + /// Fired when the verdict on an unconfirmed send of this wallet changes — + /// the network was asked about the root of its chain. `Accepted` means a + /// node holds the transaction, in its mempool or in a block — advisory: + /// the resolver keeps asking until the wallet settles the send, and it + /// stays published until then; `Unresolved` that there is no verdict + /// yet — refusals included, since no node's answer proves a transaction + /// can never land; the resolver asks again on a later block. Nothing in + /// the wallet changes either way. + /// + /// Default impl is a no-op so existing handlers don't have to care. + fn on_outgoing_transaction_probed( + &self, + _wallet_id: &WalletId, + _txid: &Txid, + _verdict: &ProbeVerdict, + ) { + } + + /// Fired when the host must drop a send's published verdict: the send + /// settled (block or InstantSend lock) or left the wallet, its wallet was + /// removed, or probing was turned off (then for every send). It does not + /// mean the send settled. Hosts drop any state they kept for the verdict. + /// + /// Default impl is a no-op so existing handlers don't have to care. + fn on_outgoing_transaction_cleared(&self, _wallet_id: &WalletId, _txid: &Txid) {} + /// Fired after each [`ShieldedSyncManager`] pass completes, /// including passes that produced no updates or skipped every /// wallet because none had a bound shielded sub-wallet yet. @@ -153,6 +182,28 @@ impl PlatformEventManager { } } + /// Dispatch a broadcast-probe verdict to every handler. Rare: at most one + /// per unconfirmed send per block. + pub fn on_outgoing_transaction_probed( + &self, + wallet_id: &WalletId, + txid: &Txid, + verdict: &ProbeVerdict, + ) { + let handlers = self.handlers.load(); + for h in handlers.iter() { + h.on_outgoing_transaction_probed(wallet_id, txid, verdict); + } + } + + /// Dispatch a cleared broadcast-probe verdict to every handler. + pub fn on_outgoing_transaction_cleared(&self, wallet_id: &WalletId, txid: &Txid) { + let handlers = self.handlers.load(); + for h in handlers.iter() { + h.on_outgoing_transaction_cleared(wallet_id, txid); + } + } + /// Dispatch a shielded sync completion to every handler. /// /// Not on the SPV hot path — called once per shielded sync pass diff --git a/packages/rs-platform-wallet/src/lib.rs b/packages/rs-platform-wallet/src/lib.rs index 55a9695f34c..8eca79a447b 100644 --- a/packages/rs-platform-wallet/src/lib.rs +++ b/packages/rs-platform-wallet/src/lib.rs @@ -14,6 +14,7 @@ pub mod address_paths; pub(crate) mod broadcast_outcome; +pub mod broadcast_probe; pub mod broadcaster; pub mod changeset; pub mod error; diff --git a/packages/rs-platform-wallet/src/manager/accessors.rs b/packages/rs-platform-wallet/src/manager/accessors.rs index 6fea92fd06b..fb6f908e15c 100644 --- a/packages/rs-platform-wallet/src/manager/accessors.rs +++ b/packages/rs-platform-wallet/src/manager/accessors.rs @@ -305,7 +305,7 @@ impl OutpointClass { /// (`DashpayExternalAccount`): coins there belong to the contact, so the /// inventory omits them and the classifier has no verdict for them. pub fn is_watch_only_contact(account_type: &AccountType) -> bool { - matches!(account_type, AccountType::DashpayExternalAccount { .. }) + account_type.is_contact_owned() } /// One page of `wallet_id`'s UTXO inventory across every funds account @@ -523,6 +523,34 @@ impl PlatformWalletManager

{ self.persister.persistence_capabilities() } + /// Turn automatic probing of unconfirmed sends on or off. + /// + /// When on, the roots of a registered wallet's unconfirmed chains are + /// resubmitted to evonodes over DAPI — right after dash-spv reports a send + /// `Uncertain` (each report: the send itself when it is a root, and of + /// the roots its chain reaches at most 8 per pass, shared by the reports + /// handled together, the rest in their turn), then, + /// while the wallet follows the chain tip, + /// every block for 24 blocks from when it first did so for that root, and + /// every 10 blocks after, a return from the background included, 55 + /// times at most (about a day of following the tip, counted within one + /// session), after which only a + /// new `Uncertain` report probes the root — and + /// each change of a send's verdict is published as + /// `on_outgoing_transaction_probed`, each send that no longer needs one + /// as `on_outgoing_transaction_cleared`. Nothing in the wallet changes. + /// Off by default; turning it off clears every published verdict. Returns + /// at once: verdicts and clears reach the host from the resolver's own + /// task, never on the calling thread. + pub fn set_broadcast_probe_enabled(&self, enabled: bool) { + self.broadcast_resolver.set_enabled(enabled); + } + + /// Whether automatic probing of unconfirmed sends is on. + pub fn is_broadcast_probe_enabled(&self) -> bool { + self.broadcast_resolver.is_enabled() + } + /// The SDK instance. pub fn sdk(&self) -> &dash_sdk::Sdk { &self.sdk diff --git a/packages/rs-platform-wallet/src/manager/load.rs b/packages/rs-platform-wallet/src/manager/load.rs index 1b612536087..d1b62f24e84 100644 --- a/packages/rs-platform-wallet/src/manager/load.rs +++ b/packages/rs-platform-wallet/src/manager/load.rs @@ -431,7 +431,7 @@ impl PlatformWalletManager

{ } } let mut wm = self.wallet_manager.write().await; - if let Err(e) = wm.insert_wallet(wallet, platform_info) { + if let Err(e) = self.insert_into_inner(&mut wm, wallet, platform_info) { load_error = Some(PlatformWalletError::WalletCreation(format!( "Failed to register persisted wallet in WalletManager: {}", e @@ -782,11 +782,7 @@ pub(super) async fn resend_one( tx: &dashcore::Transaction, ) -> ResendOutcome { let _payment = generation.payment_guard().await; - let still_live = { - let wm = wallet_manager.read().await; - wm.get_wallet_info(wallet_id) - .is_some_and(|info| Arc::ptr_eq(&info.generation, generation)) - }; + let still_live = generation.is_registered_in(&*wallet_manager.read().await, wallet_id); if !still_live { tracing::info!( wallet_id = %hex::encode(wallet_id), diff --git a/packages/rs-platform-wallet/src/manager/mod.rs b/packages/rs-platform-wallet/src/manager/mod.rs index 6ea8ba9b5fd..22a8e2827bf 100644 --- a/packages/rs-platform-wallet/src/manager/mod.rs +++ b/packages/rs-platform-wallet/src/manager/mod.rs @@ -27,12 +27,13 @@ use tokio_util::sync::CancellationToken; use dpp::prelude::Identifier; use key_wallet::wallet::ManagedWalletInfo; -use key_wallet_manager::WalletManager; +use key_wallet_manager::{WalletEvent, WalletManager}; +use crate::broadcast_probe::{AcceptanceProbe, DapiAcceptanceProbe}; +use crate::changeset::core_bridge::spawn_observed_wallet_event_adapter; use crate::changeset::{ - persists_backfill_coverage, spawn_wallet_event_adapter_with_durable_cursors, CoreChangeSet, - DashPayBackfillRecord, DurableCursor, DurableCursors, PlatformWalletChangeSet, - PlatformWalletPersistence, + persists_backfill_coverage, CoreChangeSet, DashPayBackfillRecord, DurableCursor, + DurableCursors, PlatformWalletChangeSet, PlatformWalletPersistence, }; use crate::error::PlatformWalletError; use crate::events::{PlatformEventHandler, PlatformEventManager}; @@ -44,6 +45,7 @@ use crate::manager::platform_address_sync::PlatformAddressSyncManager; use crate::manager::shielded_sync::ShieldedSyncManager; use crate::spv::SpvRuntime; use crate::wallet::asset_lock::LockNotifyHandler; +use crate::wallet::core::broadcast_resolver::BroadcastResolver; use crate::wallet::core::{BalanceUpdateHandler, SpendObservationHandler}; use crate::wallet::identity::network::DashPayPaymentHandler; use crate::wallet::platform_wallet::{PlatformWalletInfo, WalletId}; @@ -82,6 +84,11 @@ pub enum WalletWorker { /// tasks; its drain outcome is folded into the report because those /// tasks clone the FFI persister and can fire host callbacks. DashPayPayments, + /// The `BroadcastResolver` task and the reads and probes it runs. Not a + /// registry worker: shutdown stops the task, which aborts and awaits its + /// jobs; the outcome is folded into the report because the task publishes + /// through the host's event callbacks. + BroadcastProbes, /// The wallet-event adapter task — the sink coordinator stores feed /// into. Not a registry worker: joined by /// [`shutdown`](PlatformWalletManager::shutdown) under a bounded @@ -390,6 +397,10 @@ pub struct PlatformWalletManager { /// Tracks asynchronous payment hooks so manager shutdown can close /// admission and drain every task before host callback contexts are freed. pub(super) dashpay_payment_handler: Arc, + /// Asks the network about this manager's unconfirmed sends whose + /// broadcast outcome was unknown and publishes the verdicts. Off until + /// the host calls [`set_broadcast_probe_enabled`](Self::set_broadcast_probe_enabled). + pub(super) broadcast_resolver: Arc, /// Periodic shielded (Orchard) note sync coordinator (spends are /// detected during the note scan, no separate nullifier pass). /// Iterates every wallet that has been bound via @@ -523,6 +534,18 @@ impl PlatformWalletManager

{ sdk: Arc, persister: Arc

, app_handler: Arc, + ) -> Self { + let probe = Arc::new(DapiAcceptanceProbe::new(Arc::clone(&sdk))); + Self::with_acceptance_probe(sdk, persister, app_handler, probe) + } + + /// [`Self::new`] with the probe the broadcast resolver asks the network + /// through. + pub(crate) fn with_acceptance_probe( + sdk: Arc, + persister: Arc

, + app_handler: Arc, + acceptance_probe: Arc, ) -> Self { // Take the manager's lossless, unbounded persistence receiver BEFORE // the manager is wrapped in the shared `Arc` and handed to any @@ -549,17 +572,34 @@ impl PlatformWalletManager

{ let sync_fault = Arc::new(std::sync::atomic::AtomicBool::new(false)); let durable_cursors = DurableCursors::default(); + // BroadcastResolver probes unconfirmed sends whose broadcast outcome + // was unknown (read-only; see its module docs). It publishes through + // the event manager it is registered with, so it gets a weak handle + // to it once that exists. Built before the adapter, which also feeds + // it wallet events from the lossless channel. + let broadcast_resolver = Arc::new(BroadcastResolver::new( + acceptance_probe, + Arc::clone(&wallet_manager), + )); // Spawn the wallet-event adapter that translates upstream // `WalletEvent`s into `CoreChangeSet`s and forwards them to // the persister. let event_adapter_cancel = CancellationToken::new(); - let event_adapter_join = spawn_wallet_event_adapter_with_durable_cursors( + // Weak: the resolver's lifetime stays with the manager, not with + // the adapter task, which may outlive it draining its backlog. + let resolver_observer = Arc::downgrade(&broadcast_resolver); + let event_adapter_join = spawn_observed_wallet_event_adapter( Arc::clone(&wallet_manager), Arc::downgrade(&persister), event_receiver, Arc::clone(&sync_fault), Arc::clone(&durable_cursors), event_adapter_cancel.clone(), + Some(Arc::new(move |event: &WalletEvent| { + if let Some(resolver) = resolver_observer.upgrade() { + resolver.on_recorded_wallet_event(event); + } + })), ); // Build handler list: app handler + internal handlers. @@ -594,7 +634,9 @@ impl PlatformWalletManager

{ balance_handler, spend_observation_handler, Arc::clone(&dashpay_payment_handler) as Arc, + Arc::clone(&broadcast_resolver) as Arc, ])); + broadcast_resolver.set_event_manager(Arc::downgrade(&event_manager)); let spv = Arc::new(SpvRuntime::new( Arc::clone(&wallet_manager), @@ -644,6 +686,7 @@ impl PlatformWalletManager

{ dashpay_sync_manager: dashpay_sync, dpns_sync_manager: dpns_sync, dashpay_payment_handler, + broadcast_resolver, #[cfg(feature = "shielded")] shielded_sync_manager: shielded_sync, #[cfg(feature = "shielded")] @@ -667,14 +710,20 @@ impl PlatformWalletManager

{ /// [`Self::inherit_rewind_barrier`]) — and for the adapter, which drops /// the removed wallet's advances while nothing holds the id and counts /// them off this state as it does. Called under the manager write lock, - /// which is what makes the hand-off atomic against the adapter. + /// which is what makes the hand-off atomic against the adapter. The + /// broadcast resolver is told too, under the same guard (see + /// [`insert_into_inner`](Self::insert_into_inner)). pub(super) fn remove_from_wallet_manager( &self, wm: &mut WalletManager, wallet_id: &WalletId, ) -> Result<(), key_wallet_manager::WalletError> { use key_wallet::wallet::managed_wallet_info::wallet_info_interface::WalletInfoInterface; - let (_, info) = wm.remove_wallet(wallet_id)?; + let removed = wm.remove_wallet(wallet_id); + // Whatever the inner manager said — it can only fail for an id it no + // longer holds — the wallet is gone: the resolver forgets it. + self.broadcast_resolver.wallet_removed(wallet_id); + let (_, info) = removed?; let retiring = info .rewind_barrier .retire(WalletInfoInterface::account_generation(&info)); @@ -1125,7 +1174,7 @@ impl PlatformWalletManager

{ /// /// Returns a [`ShutdownReport`] keyed by [`WalletWorker`] — including /// the non-registry workers [`WalletWorker::Spv`], - /// [`WalletWorker::DashPayPayments`], and + /// [`WalletWorker::DashPayPayments`], [`WalletWorker::BroadcastProbes`], and /// [`WalletWorker::EventAdapter`], so no callback-capable background /// work is excluded from the verdict. Inspect /// [`ShutdownReport::all_clean`] before freeing the host callback @@ -1148,10 +1197,17 @@ impl PlatformWalletManager

{ // Close payment-hook admission and join the admitted tasks — // they clone the FFI persister, so a straggler is exactly the // callback-after-destroy hazard the report exists to catch. - let payments_drained = self - .dashpay_payment_handler - .quiesce_within(PAYMENT_DRAIN_BUDGET) - .await; + // + // The broadcast resolver's task publishes through the host's event + // callbacks too, so it must be gone before destroy returns: it is told + // to stop, aborts and awaits its probes, and ends. It is not aborted + // itself — it may be inside a host callback — so a Timeout here means + // it may still call the host. Both drains share one budget. + let (payments_drained, probes_status) = tokio::join!( + self.dashpay_payment_handler + .quiesce_within(PAYMENT_DRAIN_BUDGET), + self.broadcast_resolver.stop_within(PAYMENT_DRAIN_BUDGET), + ); // Drain the coordinators concurrently against one shared budget so // the drain phase as a whole is bounded (a wedged pass surfaces as @@ -1228,6 +1284,9 @@ impl PlatformWalletManager

{ WorkerStatus::Timeout }, ); + report + .per_worker + .insert(WalletWorker::BroadcastProbes, probes_status); // The wallet-event adapter is the sink the coordinators' stores // feed into, so it drains AFTER them. It is a plain tokio task, @@ -1298,12 +1357,16 @@ impl Drop for PlatformWalletManager

{ mod tests { use super::*; + use crate::broadcast_probe::{ProbeOutcome, ProbeVerdict}; + use dashcore::hashes::Hash as _; + use dash_async::WorkerStatus; use crate::changeset::{ ClientStartState, PersistenceError, PlatformWalletChangeSet, PlatformWalletPersistence, }; use crate::events::{EventHandler, PlatformEventHandler}; + use crate::test_support::{test_platform_wallet_manager_with_probe, NoopTestPersister}; struct NoopPersister; impl PlatformWalletPersistence for NoopPersister { @@ -1474,6 +1537,7 @@ mod tests { for worker in [ WalletWorker::Spv, WalletWorker::DashPayPayments, + WalletWorker::BroadcastProbes, WalletWorker::EventAdapter, ] { assert!( @@ -1559,6 +1623,263 @@ mod tests { ); } + /// Counts the probes it is asked for and decides nothing. + #[derive(Default)] + struct CountingProbe(std::sync::atomic::AtomicUsize); + + impl CountingProbe { + fn probes(&self) -> usize { + self.0.load(std::sync::atomic::Ordering::SeqCst) + } + } + + #[async_trait::async_trait] + impl AcceptanceProbe for CountingProbe { + async fn probe(&self, _transaction: &dashcore::Transaction) -> ProbeOutcome { + self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + ProbeOutcome { + verdict: ProbeVerdict::Unresolved { + reason: "counted".to_string(), + }, + answered: true, + } + } + } + + /// How long a test waits for something another task does. + const TEST_WAIT: Duration = Duration::from_secs(10); + + /// Wait for `holds`, asked every few milliseconds, or fail naming `what`. + async fn eventually(what: &str, mut holds: F) + where + F: FnMut() -> Fut, + Fut: std::future::Future, + { + let waited = tokio::time::timeout(TEST_WAIT, async { + while !holds().await { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await; + assert!(waited.is_ok(), "never happened: {what}"); + } + + /// An InstantSend-locked payment to the manager's wallet and an + /// unsettled send of the wallet's that spends it, as the mempool hands + /// them to the wallet: the send is a root of its own, so its probe's + /// verdict is published. No SPV client runs in these tests, so their + /// events reach the resolver through the lossless channel alone. + async fn receive_and_spend( + manager: &PlatformWalletManager

, + ) { + use dashcore::ephemerealdata::instant_lock::InstantLock; + use dashcore::{OutPoint, ScriptBuf, Transaction, TxIn, TxOut, Witness}; + use key_wallet_manager::WalletInterface; + + let spending = |txid, output| Transaction { + version: 2, + lock_time: 0, + input: vec![TxIn { + previous_output: OutPoint { txid, vout: 0 }, + script_sig: ScriptBuf::new(), + sequence: 0xffffffff, + witness: Witness::new(), + }], + output: vec![output], + special_transaction_payload: None, + }; + let mut wallet_manager = manager.wallet_manager.write().await; + let address = wallet_manager + .monitored_addresses() + .into_iter() + .next() + .expect("a monitored address"); + let funding = spending( + dashcore::Txid::from_byte_array([7; 32]), + TxOut { + value: 100_000_000, + script_pubkey: address.script_pubkey(), + }, + ); + let send = spending( + funding.txid(), + TxOut { + value: 99_999_000, + script_pubkey: ScriptBuf::new_op_return(&[0u8; 4]), + }, + ); + let lock = InstantLock { + txid: funding.txid(), + ..Default::default() + }; + wallet_manager + .process_mempool_transaction(&funding, Some(lock)) + .await; + wallet_manager + .process_mempool_transaction(&send, None) + .await; + } + + /// A manager over `probe` and `handler` with one wallet registered and + /// probing on. + async fn probing_manager( + probe: Arc, + handler: Arc, + ) -> (Arc>, WalletId) { + let (manager, wallet_id) = + test_platform_wallet_manager_with_probe(handler, Some(probe)).await; + manager.broadcast_resolver.set_enabled(true); + (manager, wallet_id) + } + + /// Move the wallet's synced height to `height` and tell the resolver, as + /// the SPV fan-out tells it. + async fn step_to( + manager: &PlatformWalletManager

, + wallet_id: WalletId, + height: u32, + ) { + use key_wallet_manager::{WalletEvent, WalletInterface}; + + manager + .wallet_manager + .write() + .await + .update_wallet_synced_height(&wallet_id, height); + manager + .broadcast_resolver + .on_wallet_event(&WalletEvent::SyncHeightAdvanced { wallet_id, height }); + } + + /// The constructor must connect the lossless wallet-event channel to the + /// broadcast resolver: a record change that reaches the resolver only + /// through that channel wakes an idle wallet, and the next block's pass + /// reads it and probes the new send. No SPV client runs here, so nothing + /// forwards the fan-out's copy of the events; without the constructor's + /// observer the wallet stays idle and the first wait below fails. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn should_wire_the_lossless_event_channel_into_the_broadcast_resolver() { + let probe = Arc::new(CountingProbe::default()); + let (manager, wallet_id) = probing_manager( + Arc::clone(&probe) as Arc, + Arc::new(NoopEventHandler), + ) + .await; + let resolver = Arc::clone(&manager.broadcast_resolver); + + // Following the tip with nothing unsettled: the wallet goes idle. + step_to(&manager, wallet_id, 100).await; + step_to(&manager, wallet_id, 101).await; + eventually("the wallet goes idle", || resolver.is_idle(&wallet_id)).await; + assert_eq!(probe.probes(), 0); + + receive_and_spend(&manager).await; + eventually("the lossless channel wakes the idle wallet", || async { + !resolver.is_idle(&wallet_id).await + }) + .await; + assert_eq!(probe.probes(), 0, "a wake reads nothing by itself"); + + step_to(&manager, wallet_id, 102).await; + eventually("the next block's pass probes the send's chain", || async { + probe.probes() > 0 + }) + .await; + + let report = manager.shutdown().await; + assert_eq!( + report.per_worker.get(&WalletWorker::EventAdapter), + Some(&WorkerStatus::Ok), + "{report:?}" + ); + } + + /// Holds the resolver's task inside a verdict callback until released, + /// and counts the verdicts it was handed. + struct HeldVerdictHandler { + entered: tokio::sync::Notify, + release: std::sync::Mutex>, + verdicts: std::sync::atomic::AtomicUsize, + } + + impl EventHandler for HeldVerdictHandler {} + impl PlatformEventHandler for HeldVerdictHandler { + fn on_outgoing_transaction_probed( + &self, + _wallet_id: &WalletId, + _txid: &dashcore::Txid, + _verdict: &ProbeVerdict, + ) { + self.verdicts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.entered.notify_one(); + // Released once; later verdicts pass straight through. + let _ = self.release.lock().expect("release").recv(); + } + } + + /// A removal returns only once the resolver's task is past it: a verdict + /// the task was publishing for the wallet when it was removed is + /// delivered before `remove_wallet` returns, and none follows. A host + /// that fences its own queue at that return (the Swift manager bumps a + /// per-wallet generation there) then sees no verdict of the removed + /// wallet stamped as a successor's. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn should_finish_a_removal_only_after_a_verdict_the_resolver_is_publishing() { + let (release, held) = std::sync::mpsc::channel(); + let handler = Arc::new(HeldVerdictHandler { + entered: tokio::sync::Notify::new(), + release: std::sync::Mutex::new(held), + verdicts: std::sync::atomic::AtomicUsize::new(0), + }); + let (manager, wallet_id) = probing_manager( + Arc::new(CountingProbe::default()), + Arc::clone(&handler) as Arc, + ) + .await; + + // A send to probe, then two steps that follow the tip: the pass + // probes it and publishes its verdict, which the handler holds. + receive_and_spend(&manager).await; + step_to(&manager, wallet_id, 100).await; + step_to(&manager, wallet_id, 101).await; + tokio::time::timeout(TEST_WAIT, handler.entered.notified()) + .await + .expect("the send's verdict is published"); + + let mut removal = tokio::spawn({ + let manager = Arc::clone(&manager); + async move { manager.remove_wallet(&wallet_id).await } + }); + // The removal has done all it changes (the wallet is out of the + // public map, its last mutation) and still does not return: what is + // left of it is the wait for the resolver's task. + eventually("the wallet leaves the manager", || async { + manager.get_wallet(&wallet_id).await.is_none() + }) + .await; + assert!( + tokio::time::timeout(Duration::from_millis(200), &mut removal) + .await + .is_err(), + "the removal returned while a verdict of the wallet was still being delivered" + ); + + drop(release); // the held callback returns; later ones do not wait + removal + .await + .expect("removal task") + .expect("the wallet is removed"); + let delivered = handler.verdicts.load(std::sync::atomic::Ordering::SeqCst); + + // Nothing of the removed wallet's is published from here on. + assert!(manager.broadcast_resolver.caught_up(TEST_WAIT).await); + assert_eq!( + handler.verdicts.load(std::sync::atomic::Ordering::SeqCst), + delivered + ); + } + /// `reset_platform_address_sync_state` must fail closed when the /// in-flight pass does not drain: resetting watermarks and balances /// under a live pass would let that pass's tail re-persist the state diff --git a/packages/rs-platform-wallet/src/manager/wallet_lifecycle.rs b/packages/rs-platform-wallet/src/manager/wallet_lifecycle.rs index 9366ca45d7d..1c679f949e6 100644 --- a/packages/rs-platform-wallet/src/manager/wallet_lifecycle.rs +++ b/packages/rs-platform-wallet/src/manager/wallet_lifecycle.rs @@ -1,6 +1,7 @@ //! Wallet creation, registration, and removal on [`PlatformWalletManager`]. use std::sync::Arc; +use std::time::Duration; use dash_spv::chain::CheckpointManager; use key_wallet::mnemonic::Mnemonic; @@ -8,6 +9,7 @@ use key_wallet::wallet::initialization::WalletAccountCreationOptions; use key_wallet::wallet::managed_wallet_info::ManagedWalletInfo; use key_wallet::wallet::Wallet; use key_wallet::Network; +use key_wallet_manager::{WalletError, WalletManager}; #[cfg(any(feature = "bls", feature = "eddsa"))] use crate::changeset::ProviderKeyExtendedPubKey; @@ -44,6 +46,11 @@ fn parse_mnemonic_any_language(phrase: &str) -> Result { Mnemonic::from_phrase(phrase).map_err(|_| "phrase does not match any supported BIP-39 wordlist") } +/// How long a removal waits for the broadcast resolver's task to get past +/// it (see `remove_wallet_with_teardown`). The task only has to finish the +/// one command or result it is handling, a host callback at most. +const RESOLVER_REMOVAL_BUDGET: Duration = Duration::from_secs(5); + /// Test-only rendezvous fired inside [`PlatformWalletManager::remove_wallet_with_teardown`], /// between the inner-manager removal and the public-map removal. /// @@ -71,6 +78,24 @@ pub(crate) static REMOVE_WALLET_MIDPOINT_HOOK: std::sync::Mutex PlatformWalletManager

{ + /// Insert into the inner wallet manager and tell the broadcast resolver, + /// under the caller's write guard — the way this manager adds wallets, so + /// the resolver hears of every wallet before its events and in order with + /// removals ([`remove_from_wallet_manager`](Self::remove_from_wallet_manager)). + /// (An embedder writing through `wallet_manager_arc` bypasses + /// it; the resolver then ignores that wallet, or fails safe on its + /// removal.) + pub(super) fn insert_into_inner( + &self, + wm: &mut WalletManager, + wallet: Wallet, + info: PlatformWalletInfo, + ) -> Result { + let wallet_id = wm.insert_wallet(wallet, info)?; + self.broadcast_resolver.wallet_added(&wallet_id); + Ok(wallet_id) + } + /// Create a PlatformWallet from a BIP39 mnemonic phrase. /// /// The mnemonic's language is auto-detected by trying each @@ -542,10 +567,10 @@ impl PlatformWalletManager

{ snapshot_cursor.unwrap_or(created_cursor), )?; let mut wm = self.wallet_manager.write().await; - let wallet_id = wm - .insert_wallet(wallet, platform_info) + let wallet_id = self + .insert_into_inner(&mut wm, wallet, platform_info) .map_err(|e| match e { - key_wallet_manager::WalletError::WalletExists(id) => already_registered(id), + WalletError::WalletExists(id) => already_registered(id), other => PlatformWalletError::WalletCreation(format!( "Failed to register wallet in WalletManager: {}", other @@ -890,15 +915,16 @@ impl PlatformWalletManager

{ // The validated handle is carried out of the loop: it is both what this // call returns and tears down, and the identity every mutation below is // matched against. - let (removed, _teardown) = loop { - let candidate = { - let wallets = self.wallets.load(); - match wallets.get(wallet_id) { - None => { - return Err(PlatformWalletError::WalletNotFound(hex::encode(wallet_id))) - } - Some(wallet) => Arc::clone(wallet), - } + let (removed, teardown) = loop { + let registered = self.wallets.load().get(wallet_id).cloned(); + let Some(candidate) = registered else { + // Another removal of this id may be in its last step, + // waiting for the resolver: a caller told "not found" takes + // the wallet for gone, so this return keeps the same promise. + // An id that was never registered waits too, only while a + // host handler holds the resolver's task. + self.wait_for_resolver(wallet_id).await; + return Err(PlatformWalletError::WalletNotFound(hex::encode(wallet_id))); }; let guard = candidate.generation().teardown_guard().await; let still_current = { @@ -1001,6 +1027,8 @@ impl PlatformWalletManager

{ their identity-sync rows are left for the next launch to reconcile" ); } + // The resolver drops the wallet's broadcast-probe state too, and + // the host is told to drop the verdicts it shows for its sends. if let Err(e) = self.remove_from_wallet_manager(&mut wm, wallet_id) { tracing::warn!( wallet_id = %hex::encode(wallet_id), @@ -1067,9 +1095,41 @@ impl PlatformWalletManager

{ // itself, so no payment operation can observe the wallet as live and then // act on it after this returns. tear_down(&removed); + drop(teardown); + + // The removal only queued the broadcast resolver's part. Its task + // could be publishing a verdict of this wallet's right now, and a + // host cannot tell one that arrives after this call returned from a + // verdict of a wallet registered under the same id since. Wait for + // the task to get past the removal. Last, with the wallet out of both + // maps and the gate released, so nothing else waits with it; and + // outside the wallet-manager lock, which the task's reads take. + // Bounded: a host handler that never returns must not hold a removal + // for ever. Past the budget the removal still succeeds and a verdict + // of this wallet's may follow it; so may the first verdicts of a + // wallet registered under the same id while this waits. The removal + // is complete before this point: a caller that drops this future + // here has removed the wallet without getting it back. + self.wait_for_resolver(wallet_id).await; Ok(removed) } + + /// Wait, for at most [`RESOLVER_REMOVAL_BUDGET`], until the broadcast + /// resolver's task is past everything sent to it so far. + async fn wait_for_resolver(&self, wallet_id: &WalletId) { + if !self + .broadcast_resolver + .caught_up(RESOLVER_REMOVAL_BUDGET) + .await + { + tracing::warn!( + wallet_id = %hex::encode(wallet_id), + "remove_wallet: the broadcast resolver's task was still busy after the wait; \ + if this wallet was registered, a verdict for it may still reach the host" + ); + } + } } #[cfg(test)] diff --git a/packages/rs-platform-wallet/src/masternode/update_service.rs b/packages/rs-platform-wallet/src/masternode/update_service.rs index 397ad8abf05..ce729248bfe 100644 --- a/packages/rs-platform-wallet/src/masternode/update_service.rs +++ b/packages/rs-platform-wallet/src/masternode/update_service.rs @@ -153,16 +153,18 @@ async fn fetch_operator_reward( })? .ok_or_else(|| { PlatformWalletError::InvalidParameter(format!( - "registration transaction {display} was not found; cannot determine the \ - operator reward" + "registration transaction {display} was not found (or the node answered \ + with another transaction); cannot determine the operator reward" )) })?; operator_reward_from_registration(pro_tx_hash, &fetched.transaction) } /// Read `operatorReward` out of a fetched registration transaction, -/// binding the response to the request first: DAPI's get-transaction reply -/// is not authenticated, so the decoded transaction must hash to the +/// binding the response to the request first — defense in depth: the SDK's +/// getter already reads a reply carrying another transaction as a miss, but +/// this function trusts no caller to have used it. DAPI's get-transaction +/// reply is not authenticated, so the decoded transaction must hash to the /// requested proTxHash before its payload is trusted. Only the operator /// reward is read from it; the service values come from the synced /// masternode list's entry for that proTxHash. Without the hash check a diff --git a/packages/rs-platform-wallet/src/test_support.rs b/packages/rs-platform-wallet/src/test_support.rs index 9879ceb1041..7da156eab9d 100644 --- a/packages/rs-platform-wallet/src/test_support.rs +++ b/packages/rs-platform-wallet/src/test_support.rs @@ -30,8 +30,12 @@ use key_wallet::{DerivationPath, Wallet}; use key_wallet_manager::WalletManager; use tokio::sync::RwLock; +use crate::broadcast_probe::AcceptanceProbe; #[cfg(test)] use crate::broadcaster::{BroadcastError, TransactionBroadcaster}; +use crate::changeset::PlatformWalletPersistence; +use crate::events::PlatformEventHandler; +use crate::manager::PlatformWalletManager; use crate::wallet::core::WalletGeneration; use crate::wallet::identity::IdentityManager; use crate::wallet::platform_wallet::{PlatformWalletInfo, WalletId}; @@ -633,7 +637,7 @@ where /// that need a full [`PlatformWallet`] but no real persistence pipeline. pub struct NoopTestPersister; -impl crate::changeset::PlatformWalletPersistence for NoopTestPersister { +impl PlatformWalletPersistence for NoopTestPersister { fn store( &self, _wallet_id: WalletId, @@ -680,7 +684,30 @@ pub async fn test_platform_wallet_manager_with

( persister: Arc

, ) -> (Arc>, WalletId) where - P: crate::changeset::PlatformWalletPersistence + 'static, + P: PlatformWalletPersistence + 'static, +{ + build_test_platform_wallet_manager(persister, Arc::new(NoopTestEventHandler), None).await +} + +/// [`test_platform_wallet_manager`] with the host's `event_handler` and, +/// when given, the probe the broadcast resolver asks the network through +/// (the DAPI one otherwise). +#[cfg(test)] +pub(crate) async fn test_platform_wallet_manager_with_probe( + event_handler: Arc, + acceptance_probe: Option>, +) -> (Arc>, WalletId) { + build_test_platform_wallet_manager(Arc::new(NoopTestPersister), event_handler, acceptance_probe) + .await +} + +async fn build_test_platform_wallet_manager

( + persister: Arc

, + event_handler: Arc, + acceptance_probe: Option>, +) -> (Arc>, WalletId) +where + P: PlatformWalletPersistence + 'static, { use key_wallet::mnemonic::Mnemonic; use key_wallet::wallet::initialization::WalletAccountCreationOptions; @@ -690,13 +717,12 @@ where abandon abandon abandon abandon abandon about"; let sdk = Arc::new(dash_sdk::SdkBuilder::new_mock().build().expect("mock sdk")); - let event_handler: Arc = - Arc::new(NoopTestEventHandler); - let manager = Arc::new(crate::PlatformWalletManager::new( - sdk, - persister, - event_handler, - )); + let manager = Arc::new(match acceptance_probe { + Some(probe) => { + PlatformWalletManager::with_acceptance_probe(sdk, persister, event_handler, probe) + } + None => PlatformWalletManager::new(sdk, persister, event_handler), + }); let mnemonic = Mnemonic::from_phrase(TEST_MNEMONIC).expect("valid test mnemonic"); let seed_bytes = mnemonic.to_seed(""); diff --git a/packages/rs-platform-wallet/src/wallet/core/broadcast_resolver.rs b/packages/rs-platform-wallet/src/wallet/core/broadcast_resolver.rs new file mode 100644 index 00000000000..53d7758ea41 --- /dev/null +++ b/packages/rs-platform-wallet/src/wallet/core/broadcast_resolver.rs @@ -0,0 +1,8189 @@ +//! Automatically asking the network about sends whose broadcast outcome is +//! unknown. +//! +//! A send the SPV broadcaster could not confirm ends as `MaybeSent` +//! (`BroadcastResult::Uncertain`): the wallet keeps its record as an +//! unconfirmed spend and nothing ever tells the user whether the payment +//! landed. This module re-asks the network through +//! [`AcceptanceProbe`](crate::broadcast_probe::AcceptanceProbe) and publishes +//! the verdict; it **changes nothing** in the wallet. A probe resubmits the +//! signed transaction, so it goes out only under the lifecycle gate of the +//! registration it was read from: a wallet's removal stops the one in +//! flight and none is started after it (see [`gated_probe`]). Verdicts are positive +//! only — accepted — or unresolved: no answer the probe gets proves +//! that a transaction can never land (see +//! [`broadcast_probe`](crate::broadcast_probe)). +//! +//! What is probed: the *roots* of the chains the wallet signed into — its +//! unsettled sends (not in a block, not InstantSend-locked) plus the unsettled +//! ancestors they depend on, including someone else's incoming payment — +//! taking only transactions that spend no output of an unsettled transaction. +//! Probing a child is pointless and misleading: a node that has not seen the +//! parent answers `missingorspent` for a perfectly valid child. A healthy send +//! is IS-locked within seconds, so in practice only sends that went quiet are +//! ever probed. +//! +//! What is published: verdicts per **own send**, which is what the host +//! shows. A root's verdict goes to the root when it is the wallet's send. +//! Every verdict is advisory: only the wallet itself (a block, an +//! InstantSend lock) settles a send, so a root the nodes accepted, in a +//! mempool or in a block, stays a root — probed on its schedule like any +//! other — until the wallet settles it; those probes hand the signed bytes +//! to the nodes again and tell the host nothing new. Only changes are +//! published: Unresolved never replaces Accepted, so an Accepted stays +//! published until the wallet settles the send, also for a send that was +//! evicted or reorged out since. The host is told to drop a send's verdict +//! (cleared) when the send settles or leaves the wallet, when its wallet is +//! removed and, for every send, when probing is turned off; cleared says +//! only "forget the verdict", not that the send settled. A +//! wallet event that reports a send settled (in a block or InstantSend- +//! locked) or swept clears its verdict at once, even mid-scan; a pass +//! clears whatever else left the wallet when it next reads it. +//! +//! When: once as soon as dash-spv reports an `Uncertain` broadcast — the +//! current root of that transaction's chain; a send built on it becomes due +//! by schedule once the root settles, not at once — then on each advance of +//! the wallet's synced height that follows the tip. Steps of more than +//! [`CATCH_UP_STEP`] blocks, steps back (a rescan) and the first step after +//! launch are catch-up and skipped (passes queued before them probe only +//! what an `Uncertain` result forced). Each dash-spv sync completion (after +//! every block, and when a launch's or a reconnect's catch-up ends) queues +//! a followed pass for every wallet within [`CATCH_UP_STEP`] of its tip — +//! unless one already read the wallet at that height — so the pass a +//! catch-up skipped runs at the tip without waiting for the next block (a +//! completion while probing is off is ignored: the next one covers it). A +//! completion can be handled before the wallet's own step to that tip (the +//! two come from separate tasks): that step, when it follows within +//! [`COMPLETED_AHEAD_TTL`], is the wallet's arrival at the tip and runs the +//! pass, however large it is. A step that never comes (the fan-out can +//! drop one) is not made up for: the pass then runs at the next block. A +//! root is probed every block for [`EVERY_BLOCK_WINDOW`] blocks, counted +//! from its first probe by a height-driven pass while the wallet follows the +//! tip (a forced pass does not start it), and every [`SLOW_INTERVAL`] blocks +//! after, [`MAX_SLOW_PROBES`] times at most: from then on only a new +//! `Uncertain` report probes it, and its last verdict stays published. A +//! probe no node answered (DAPI unreachable — its addresses +//! still banned just after an outage) does not count: its root is due again +//! at the next block, whatever its phase, and at the same height a retry +//! probes it again (as does any `Uncertain` report, which forces a probe) — after [`NO_ANSWER_RETRY`] if no block comes +//! first, then after twice and four times that again (at most +//! [`MAX_NO_ANSWER_RETRIES`] per height). Probes never ban a DAPI address +//! themselves. Each Uncertain report forces a probe of its chain's current +//! roots in the next pass (reports queued together share it): the send +//! itself when it is a root, and at most [`MAX_FORCED_ROOTS_PER_PASS`] of +//! the roots its chain reaches, the wallet's own sends first; the others +//! take their turn among the scheduled roots. A pass under +//! way in a wallet whose own send it is gives way to it after its probe in +//! flight — once per height; a later request waits behind it. A pass probes at most +//! [`MAX_ROOTS_PER_PASS`] scheduled roots, those probed longest ago first — +//! a wallet's records can be fed by a peer, so one pass's network work is +//! bounded and the rest come round at the next heights. The roots of sends +//! reported `Uncertain` and not settled yet go first, with at most half of +//! the slots while other roots wait; a no-answer retry also makes a pass +//! under way give way (within the same once-per-height allowance). +//! Between passes a wallet waits for the height at which its next root is due; +//! one whose last pass had nothing left to probe is idle and not scanned +//! again until a wallet event that touched its records (a new or +//! InstantSend-locked transaction, a block that changed records, a sweep) or +//! an `Uncertain` result for one of its transactions wakes it. Those wallet +//! events wake it twice: from dash-spv's fan-out, which is timely but ends +//! when a receiver lags, and from the lossless persistence channel +//! ([`BroadcastResolver::on_recorded_wallet_event`]), so a lost one cannot +//! leave a wallet idle over an unsettled send. No verdict +//! pauses the probing of a root: neither a mempool nor the nodes' word on a +//! block is settlement. +//! Probes run only while the SPV client delivers blocks, i.e. while the app is +//! in the foreground. +//! +//! How: one task (the actor) owns all of this state and handles commands — +//! from wallet and SPV events, the switch and wallet removal — and the +//! results of its reads and probes one at a time, in order, so none of them +//! race. At most one pass per wallet runs; requests that arrive meanwhile +//! are merged into the next. Every host callback comes from that task, never +//! from a caller that may hold its own locks. + +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; +use std::mem; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError, Weak}; +use std::time::Duration; + +use async_trait::async_trait; +use dash_async::WorkerStatus; +use dash_spv::sync::SyncEvent; +use dash_spv::{BroadcastResult, EventHandler}; +use dashcore::{Transaction, Txid}; +use key_wallet::managed_account::transaction_record::TransactionRecord; +use key_wallet::transaction_checking::TransactionContext; +use key_wallet::wallet::managed_wallet_info::wallet_info_interface::WalletInfoInterface; +use key_wallet::wallet::Wallet; +use key_wallet_manager::{WalletEvent, WalletId, WalletManager}; +use tokio::runtime::Handle; +use tokio::sync::{mpsc, oneshot, Mutex as AsyncMutex, RwLock}; +use tokio::task::{self, AbortHandle, JoinError, JoinHandle, JoinSet}; +use tokio::time::{timeout_at, Instant}; + +use crate::broadcast_probe::{AcceptanceProbe, ProbeOutcome, ProbeVerdict}; +use crate::events::{PlatformEventHandler, PlatformEventManager}; +use crate::wallet::core::{record_spends_own_coins, WalletGeneration}; +use crate::wallet::platform_wallet::PlatformWalletInfo; + +/// Blocks from the start of a root's window (see [`ProbeSchedule`]) during +/// which it is probed on every block. +pub(crate) const EVERY_BLOCK_WINDOW: u32 = 24; + +/// Blocks between probes once [`EVERY_BLOCK_WINDOW`] has passed. +pub(crate) const SLOW_INTERVAL: u32 = 10; + +/// Probes of a root after its every-block window, beyond which the schedule +/// stops probing it: about a day of following the tip, as long as dash-spv +/// itself keeps rebroadcasting a send. A root that never settles would +/// otherwise be resubmitted for as long as the wallet holds it — and a +/// wallet's records can be fed by a peer (see [`MAX_ROOTS_PER_PASS`]), so +/// such roots would hold probe slots for good. Counted in probes a node +/// answered, not in blocks, so the blocks an app slept through do not use it +/// up; a forced probe (an `Uncertain` report, a retry) after the window +/// counts like a scheduled one. A new `Uncertain` report still probes a +/// retired root (within [`MAX_FORCED_ROOTS_PER_PASS`] when the report only +/// reaches it), and so does a probe of it that no node answered. Counted +/// within one session: the schedule lives in memory, so a relaunch, a +/// rewind, switching probing off and on, or registering the wallet again +/// starts a root's count anew. +pub(crate) const MAX_SLOW_PROBES: u32 = 55; + +/// Scheduled roots a pass probes at most; forced ones (an `Uncertain` +/// report's, a no-answer retry's) come on top: those forced by name, and at +/// most [`MAX_FORCED_ROOTS_PER_PASS`] their chains reach. A wallet's records are not +/// all its own doing — a peer can feed it transactions that make many roots — +/// so the work of one pass is bounded, and within it the roots probed +/// longest ago (or never) go first: the rest stay due and come round at the +/// next heights. The roots of sends reported `Uncertain` and not settled yet +/// go first, but take at most half of the slots while other roots wait. +pub(crate) const MAX_ROOTS_PER_PASS: usize = 8; + +/// Roots a pass probes at most because a forced send's chain reaches them. +/// One report can reach many: a send's chain goes back to every unsettled +/// ancestor, and a peer can feed the wallet incoming parents for it to +/// spend, so the roots of a single reported send are not bounded by what the +/// wallet did. The wallet's own sends go first, then those probed longest +/// ago (or never). A root beyond the allowance is not forced: it takes its +/// turn among the scheduled roots, where a reported send's roots go first, +/// in the passes that follow the tip. A pass confined to forced roots (a +/// report's own, or any pass of a wallet behind the tip) leaves it for the +/// first of those. +/// +/// A root that is itself the forced transaction is outside the allowance: a +/// reported send that is its own root (one per report the wallet's own +/// broadcast produced), and each root a no-answer retry repeats (no more +/// than the passes at that height probed). +pub(crate) const MAX_FORCED_ROOTS_PER_PASS: usize = MAX_ROOTS_PER_PASS; + +/// How long after a probe that no node answered (all of DAPI unreachable — +/// the network just coming back, its addresses still banned for at least a +/// minute) the wallet's roots are probed again instead of waiting for the +/// next block; doubled for each further retry at the same height, up to +/// [`MAX_NO_ANSWER_RETRIES`]. +pub(crate) const NO_ANSWER_RETRY: Duration = Duration::from_secs(60); + +/// Retries per wallet and height after probes no node answered: 1, 2 and 4 +/// times [`NO_ANSWER_RETRY`] — then the next block decides. +pub(crate) const MAX_NO_ANSWER_RETRIES: u32 = 3; + +/// Followed blocks an `Uncertain` send waits for its peer echo: as long as +/// the every-block probe window. Past that the probes decide alone. +pub(crate) const UNCERTAIN_EXPIRY: u32 = EVERY_BLOCK_WINDOW; + +/// How far a wallet's synced height may advance in one step and still count +/// as following the tip. Larger steps, and steps back (a rescan after a +/// rewind), are catch-up: the scan itself settles most roots, so a +/// height-driven pass waits for it. +pub(crate) const CATCH_UP_STEP: u32 = 3; + +/// One unsettled transaction of the wallet, merged across the accounts that +/// record it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct OutgoingView { + pub txid: Txid, + /// The transaction spends at least one of this wallet's coins, i.e. the + /// wallet signed it. + pub spends_own_coins: bool, + /// Shared with the passes and probe jobs that send it; never changed. + pub transaction: Arc, +} + +impl OutgoingView { + fn spends_output_of(&self, txids: &impl Fn(&Txid) -> bool) -> bool { + self.transaction + .input + .iter() + .any(|input| txids(&input.previous_output.txid)) + } +} + +/// One account's record of a transaction, as far as the resolver cares. +pub(crate) struct RecordFacts<'a> { + pub txid: Txid, + /// In a block, InstantSend-locked, or finalized in this account. + pub settled: bool, + /// This account records the transaction spending the wallet's coins. + pub own: bool, + pub transaction: &'a Transaction, +} + +/// The one place the multi-account rule lives: a transaction is one +/// transaction however many accounts record it; it is settled if any account +/// holds it settled, and the wallet's own send if any account records it +/// spending the wallet's coins (a transfer between accounts is "incoming" in +/// the receiving one). Returns the unsettled ones. +pub(crate) fn merge_records<'a>( + records: impl IntoIterator>, +) -> Vec { + let mut settled: HashSet = HashSet::new(); + let mut unsettled: HashMap = HashMap::new(); + for record in records { + if record.settled { + settled.insert(record.txid); + continue; + } + unsettled + .entry(record.txid) + .and_modify(|(own, _)| *own |= record.own) + .or_insert((record.own, record.transaction)); + } + unsettled + .into_iter() + .filter(|(txid, _)| !settled.contains(txid)) + .map(|(txid, (own, transaction))| OutgoingView { + txid, + spends_own_coins: own, + transaction: Arc::new(transaction.clone()), + }) + .collect() +} + +/// This wallet's unsettled transactions. One walk over the records; only +/// unsettled ones are cloned — on a healthy wallet there are none or a +/// handful. +/// +/// `signs`: the wallet can sign ([`Wallet::can_sign`]); a watch-only wallet sends +/// nothing, so none of its transactions is an own send. +pub(crate) fn collect_views(info: &PlatformWalletInfo, signs: bool) -> Vec { + let accounts = info.core_wallet.accounts.all_accounts(); + merge_records(accounts.iter().flat_map(|account| { + account.transactions().values().map(move |record| { + record_facts( + record, + || account.transaction_is_finalized(&record.txid), + signs, + ) + }) + })) +} + +/// One account's record as the resolver sees it; `finalized`: whether the +/// account holds the transaction finalized, asked only of a record not +/// already settled; `signs`: the wallet can sign ([`Wallet::can_sign`]). +fn record_facts( + record: &TransactionRecord, + finalized: impl FnOnce() -> bool, + signs: bool, +) -> RecordFacts<'_> { + RecordFacts { + txid: record.txid, + settled: is_settled(record) || finalized(), + // The wallet signed it: it spends coins the wallet owns + // ([`record_spends_own_coins`]; a contact's watch-only chain records + // the contact's spends). Other records still take part in the chain + // walk as ancestors of a send that is ours. + own: signs && record_spends_own_coins(record), + transaction: &record.transaction, + } +} + +/// Whether the accounts' records of one transaction make it an unsettled own +/// send — the same rule as [`merge_records`]: settled in any account means +/// settled, and own in any account means own. +fn is_unsettled_own<'a>(records: impl IntoIterator>) -> bool { + // No allocation: this runs per holding wallet on every holder lookup. + records + .into_iter() + .try_fold(false, |own, record| { + (!record.settled).then_some(own || record.own) + }) + .unwrap_or(false) +} + +/// This wallet's unsettled transactions as chains. `views` hold one entry per +/// transaction (see [`merge_records`]). +pub(crate) struct ChainGraph<'a> { + by_txid: HashMap, + /// Transactions an ancestry traversal expanded, for the tests that pin + /// its cost. + #[cfg(test)] + expanded: std::cell::Cell, +} + +impl<'a> ChainGraph<'a> { + /// Only the wallet settles a transaction: a probe's verdict changes + /// nothing here. + pub(crate) fn new(views: &'a [OutgoingView]) -> Self { + Self { + by_txid: views.iter().map(|view| (view.txid, view)).collect(), + #[cfg(test)] + expanded: std::cell::Cell::new(0), + } + } + + fn is_own(&self, txid: &Txid) -> bool { + self.by_txid + .get(txid) + .is_some_and(|view| view.spends_own_coins) + } + + fn is_unsettled(&self, txid: &Txid) -> bool { + self.by_txid.contains_key(txid) + } + + /// The wallet's unsettled sends. + pub(crate) fn own_unsettled(&self) -> BTreeSet { + self.by_txid + .keys() + .filter(|txid| self.is_own(txid)) + .copied() + .collect() + } + + /// The unsettled transactions any of `sources` depends on, transitively, + /// sources included — one traversal with one visited set, so each + /// transaction is expanded once however many sources share its ancestry + /// (a long chain of sends costs its length, not its square). + fn with_ancestors(&self, sources: impl IntoIterator) -> BTreeSet { + let mut found = BTreeSet::new(); + let mut frontier: Vec = sources.into_iter().collect(); + while let Some(txid) = frontier.pop() { + if !self.is_unsettled(&txid) || !found.insert(txid) { + continue; + } + #[cfg(test)] + self.expanded.set(self.expanded.get() + 1); + frontier.extend( + self.by_txid[&txid] + .transaction + .input + .iter() + .map(|input| input.previous_output.txid), + ); + } + found + } + + fn is_root(&self, txid: &Txid) -> bool { + self.is_unsettled(txid) + && !self.by_txid[txid].spends_output_of(&|parent| self.is_unsettled(parent)) + } + + /// The roots to probe: members of the chains the wallet signed into — its + /// unsettled sends plus every unsettled ancestor they depend on, including + /// someone else's incoming payment — that spend no output of an unsettled + /// transaction. A node that has not seen a parent refuses a valid child. + pub(crate) fn roots(&self) -> Vec<&'a OutgoingView> { + self.with_ancestors(self.own_unsettled()) + .iter() + .filter(|txid| self.is_root(txid)) + .map(|txid| self.by_txid[txid]) + .collect() + } + + /// The roots of the chains of `txids` — what to probe when dash-spv + /// reports them uncertain but they spend an unsettled parent. One + /// traversal for all of them. + pub(crate) fn roots_of(&self, txids: impl IntoIterator) -> BTreeSet { + self.with_ancestors(txids) + .into_iter() + .filter(|txid| self.is_root(txid)) + .collect() + } +} + +/// The roots of `views` — see [`ChainGraph::roots`]. +#[cfg(test)] +fn ambiguous_roots(views: &[OutgoingView]) -> Vec<&OutgoingView> { + ChainGraph::new(views).roots() +} + +/// When a root is next due for a probe. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct ProbeSchedule { + /// Where the every-block window starts: the root's first probe by a + /// height-driven pass while the wallet followed the tip. `None` until + /// then — waiting in a backlog, or probed only by forced passes, whose + /// height may be a stale stored tip; `None` counting as in-window matters + /// for `next_due`. + window_start: Option, + /// The first height a pass that followed the tip saw the root at: its + /// place in the queue until its first probe (see `MAX_ROOTS_PER_PASS`). + seen_at: Option, + last_probe: Option, + /// The last probe reached no node: due again at the next height, in any + /// phase — but not at the same height, where only a retry + /// ([`NO_ANSWER_RETRY`]) probes it again. + unanswered: bool, + /// Its last probe job panicked: it waits its turn behind the other roots, + /// even a reported send's, until a probe of it completes. + panicked: bool, + /// Probes a node answered after the every-block window, up to + /// [`MAX_SLOW_PROBES`]. + slow_probes: u32, +} + +impl ProbeSchedule { + /// Start the window here if it has not started — once. + fn anchor(&mut self, height: u32) { + self.window_start.get_or_insert(height); + } + + fn in_window(&self, height: u32) -> bool { + self.window_start + .is_none_or(|first| height.saturating_sub(first) < EVERY_BLOCK_WINDOW) + } + + /// Out of slow probes ([`MAX_SLOW_PROBES`]): the schedule probes the + /// root no more. + fn retired(&self) -> bool { + self.slow_probes >= MAX_SLOW_PROBES + } + + pub(crate) fn is_due(&self, height: u32) -> bool { + match self.last_probe { + None => true, + Some(last) if height <= last => false, + // A probe no node answered is owed whatever the root's phase. + Some(_) if self.unanswered => true, + Some(_) if self.retired() => false, + Some(_) if self.in_window(height) => true, + Some(last) => height - last >= SLOW_INTERVAL, + } + } + + /// Never moves back: a probe marked at a stale height (a pass that ran + /// across a catch-up) must not make the root due again below its last. + fn probed_at(&mut self, height: u32) { + self.last_probe = Some(self.last_probe.map_or(height, |last| last.max(height))); + // A new probe is out: its own answer decides the mark. + self.unanswered = false; + } + + /// Whether the probe sent at `height` reached a node. One that reached + /// none is due again; one that did counts towards [`MAX_SLOW_PROBES`] + /// once the every-block window is over. + fn answered(&mut self, answered: bool, height: u32) { + self.unanswered = !answered; + if answered + && self.window_start.is_some() + && !self.in_window(height) + && self.slow_probes < MAX_SLOW_PROBES + { + self.slow_probes += 1; + } + } + + /// The last probe produced nothing: due again at once. + fn withdraw(&mut self) { + self.last_probe = None; + } + + /// The lowest height at which [`is_due`](Self::is_due) holds. + /// `u32::MAX` once the root is retired: no height makes it due again. + pub(crate) fn next_due(&self) -> u32 { + match self.last_probe { + None => 0, + Some(last) if self.unanswered => last.saturating_add(1), + Some(_) if self.retired() => u32::MAX, + Some(last) if self.in_window(last.saturating_add(1)) => last.saturating_add(1), + Some(last) => last.saturating_add(SLOW_INTERVAL), + } + } +} + +/// What the host must hear, per own send. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ResolverEvent { + /// A send's verdict changed. + Verdict(Txid, ProbeVerdict), + /// The host must drop the verdict it holds for a send: the send settled + /// or left the wallet, its wallet was removed, or probing was turned off. + /// Not a statement that the send settled. + Cleared(Txid), +} + +/// One event for the host, with what the log line needs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Outgoing { + wallet_id: WalletId, + event: ResolverEvent, + height: Option, + trigger: &'static str, +} + +/// What started a pass: dash-spv's `Uncertain` result for one of the +/// wallet's transactions, or an advance of its synced height. +fn trigger_of(forced: bool) -> &'static str { + if !forced { + "height" + } else { + "uncertain" + } +} + +/// Bookkeeping for every wallet. Schedules are per probed root; published +/// verdicts are per own send, which is what the host shows. +/// Everything for a transaction is kept while it stays unsettled in the +/// wallet, and dropped once it settles or leaves. Owned by the resolver's +/// actor, so every change happens in one sequence. +#[derive(Debug, Default)] +pub(crate) struct ResolverState { + schedules: HashMap<(WalletId, Txid), ProbeSchedule>, + /// What the host holds per send. + published: HashMap<(WalletId, Txid), ProbeVerdict>, +} + +impl ResolverState { + /// Drop everything kept for `wallet_id`; a clear for every send the host + /// had a verdict for. + fn forget_wallet(&mut self, wallet_id: &WalletId, trigger: &'static str) -> Vec { + self.schedules.retain(|(wallet, _), _| wallet != wallet_id); + self.clear_where(|wallet, _| wallet == wallet_id, None, trigger) + } + + /// Drop everything; a clear for every published send. + fn forget_all(&mut self) -> Vec { + self.schedules.clear(); + self.clear_where(|_, _| true, None, "disabled") + } + + /// A wallet event reported `txids` settled or gone: forget them and clear + /// their verdicts. + fn settle(&mut self, wallet_id: WalletId, txids: &HashSet) -> Vec { + for txid in txids { + self.schedules.remove(&(wallet_id, *txid)); + } + self.clear_where( + |wallet, txid| *wallet == wallet_id && txids.contains(txid), + None, + "settled-or-left", + ) + } + + /// Unpublish every send `drop` matches; a clear for each, in a stable order. + fn clear_where( + &mut self, + drop: impl Fn(&WalletId, &Txid) -> bool, + height: Option, + trigger: &'static str, + ) -> Vec { + let mut cleared: Vec<(WalletId, Txid)> = self + .published + .keys() + .filter(|(wallet, txid)| drop(wallet, txid)) + .copied() + .collect(); + cleared.sort(); + self.published + .retain(|(wallet, txid), _| !drop(wallet, txid)); + cleared + .into_iter() + .map(|(wallet_id, txid)| Outgoing { + wallet_id, + event: ResolverEvent::Cleared(txid), + height, + trigger, + }) + .collect() + } + + /// Record `verdict` for `send`; returns whether the host must hear it: + /// a first verdict, or Accepted after Unresolved. Accepted stays until + /// the send is cleared — Unresolved never replaces it, since one probe + /// that only met transport errors says nothing new — and every + /// Unresolved counts as one, its reason varying from probe to probe. + fn publish(&mut self, key: (WalletId, Txid), verdict: &ProbeVerdict) -> bool { + let changed = matches!( + (self.published.get(&key), verdict), + (None, _) + | ( + Some(ProbeVerdict::Unresolved { .. }), + ProbeVerdict::Accepted + ) + ); + if changed { + self.published.insert(key, verdict.clone()); + } + changed + } + + /// Publish `verdict` for each of `sends`, collecting what changed. + fn publish_all( + &mut self, + wallet_id: WalletId, + sends: impl IntoIterator, + verdict: &ProbeVerdict, + height: u32, + trigger: &'static str, + out: &mut Vec, + ) { + for send in sends { + if self.publish((wallet_id, send), verdict) { + out.push(Outgoing { + wallet_id, + event: ResolverEvent::Verdict(send, verdict.clone()), + height: Some(height), + trigger, + }); + } + } + } +} + +/// The first advance after launch has no previous height to compare with, and +/// it is usually the stored tip from before the scan: treat it as catch-up +/// too. So is a step back — a rescan after a rewind (a new account) replays +/// old heights. Probing starts from the next step that follows the tip. +pub(crate) fn is_catch_up_step(previous: Option, height: u32) -> bool { + previous.is_none_or(|previous| height < previous || height - previous > CATCH_UP_STEP) +} + +/// How long a completion that overtook a wallet's height step stands (see +/// `WalletEntry::completed_ahead`). The step it waits for was already on its +/// way, behind another task's queue, so it comes within moments. Without a +/// limit, a completion whose step was lost would turn a step of some later +/// scan that happens to land at that old tip into an arrival, and start +/// windows there. A step that takes longer than this is a catch-up step +/// again, and its pass waits for the next completion. Measured on the +/// monotonic clock, which stands still while the device sleeps. +const COMPLETED_AHEAD_TTL: Duration = Duration::from_secs(30); + +/// A wallet at `height` is at a sync's completed `tip`, as far as following +/// goes. +fn near_tip(height: u32, tip: u32) -> bool { + height.abs_diff(tip) <= CATCH_UP_STEP +} + +/// A forced request (a report's, or a no-answer retry's) waits for the +/// wallet's run: the run gives way, keeping only its own forced roots, so the +/// forced pass starts after the probe in flight, not after a queue of +/// scheduled roots. The scheduled roots it had not sent stay due and, probed +/// longest ago, go first at the next height. A run gives way at most once +/// per height (its pass's): reports that keep coming cannot drop the +/// scheduled roots again and again — a later request waits behind the run. +fn yield_to_forced(entry: &mut WalletEntry) { + let Some(run) = entry.run.as_mut() else { + return; + }; + // Only for a request the run does not already carry (a repeated report + // of a send it forces itself changes nothing). + let Some(pending) = entry.pending.as_ref() else { + return; + }; + let carried = |txid: &Txid| run.forced.contains(txid) || run.retried.contains(txid); + if pending.forced.iter().chain(&pending.retried).all(carried) { + return; + } + // Still reading: nothing to drop yet — the read's end asks again. + let Some(probing) = run.probing.as_mut() else { + return; + }; + if entry.yielded_at == Some(probing.height) { + return; + } + // The allowance is used only by a yield that dropped something: one of a + // pass with only forced roots left costs the backlog nothing. + if probing.drop_scheduled() { + entry.yielded_at = Some(probing.height); + } +} + +/// `wallet_id` no longer holds the reported sends `leaves` picks: take it out +/// of their holders, and drop a send no wallet holds any more. +fn forget_reported( + reported: &mut HashMap>, + wallet_id: &WalletId, + leaves: impl Fn(&Txid) -> bool, +) { + reported.retain(|txid, holders| { + if leaves(txid) { + holders.remove(wallet_id); + } + !holders.is_empty() + }); +} + +/// A root chosen for a probe. +#[derive(Debug, Clone)] +pub(crate) struct DueRoot { + pub txid: Txid, + /// Shared with the probe job: sent as is, never changed. + pub transaction: Arc, + /// The root is the wallet's own send. + pub own: bool, + /// An `Uncertain` result forced it (it may also be due by its schedule). + /// Labels its verdict in the log, and is the only kind of root a pass + /// without an anchor probes. + pub forced: bool, + /// A no-answer retry repeats it (see [`NO_ANSWER_RETRY`]): labels its + /// verdict. + pub retry: bool, +} + +impl DueRoot { + /// What made this probe happen, for the log. + fn trigger(&self) -> &'static str { + if self.retry { + "no-answer-retry" + } else { + trigger_of(self.forced) + } + } +} + +/// The start of a pass: what the host must hear now, and the roots to probe. +#[derive(Debug)] +pub(crate) struct PassStart { + pub events: Vec, + pub due: VecDeque, +} + +/// Start a pass over one wallet at `height`: forget what settled or left +/// (clearing the sends the host had a verdict for) and pick the roots due for +/// a probe. The +/// roots of every `forced` transaction's chain (dash-spv has just reported +/// it `Uncertain`) are due regardless of their schedule — even if one went +/// out at this height already: a repeat probe is cheaper than bookkeeping +/// that tells the two apart. A root named in `forced` always goes; of the +/// roots a forced send's chain only reaches, up to +/// [`MAX_FORCED_ROOTS_PER_PASS`]. +/// `anchor`: the tip height +/// a height-driven pass knows the wallet follows — a root it takes starts +/// its every-block window when its probe goes out (see [`mark_sent`]). +/// `priority`: the wallet's sends reported `Uncertain` and not settled yet — +/// their roots go first among the scheduled ones (see +/// [`MAX_ROOTS_PER_PASS`]). +pub(crate) fn begin_pass( + state: &mut ResolverState, + wallet_id: WalletId, + height: u32, + views: &[OutgoingView], + forced: &HashSet, + priority: &HashSet, + anchor: Option, +) -> PassStart { + let mut events = Vec::new(); + // What the wallet itself still holds unsettled — what state is kept for. + let present: HashSet = views.iter().map(|view| view.txid).collect(); + let own_present: HashSet = views + .iter() + .filter(|view| view.spends_own_coins) + .map(|view| view.txid) + .collect(); + let is_gone = |wallet: &WalletId, txid: &Txid| *wallet == wallet_id && !present.contains(txid); + state + .schedules + .retain(|(wallet, txid), _| !is_gone(wallet, txid)); + // Cleared because the send is no longer an unsettled own send: it + // settled, or left the wallet — not a statement that it settled. + events.extend(state.clear_where( + |wallet, txid| *wallet == wallet_id && !own_present.contains(txid), + Some(height), + "settled-or-left", + )); + + let graph = ChainGraph::new(views); + let roots = graph.roots(); + let forced_roots = graph.roots_of(forced.iter().copied()); + + // Forced roots first. A root named in `forced` itself always goes: a + // reported send that is its own root is what the report is about, and a + // retry names the roots it repeats. The roots a forced send's chain + // only reaches (its unsettled ancestors, which can be a peer's doing) + // are at most `MAX_FORCED_ROOTS_PER_PASS`: own sends before the rest, + // then in turn. Then at most `MAX_ROOTS_PER_PASS` scheduled ones + // in turn: those probed longest ago first, a never-probed one by the + // height it was first seen at — a queue, so fresh roots (a peer can feed + // them at will) wait behind older ones and cannot starve them. Reported + // sends' roots go first, in the same order among themselves, but leave + // at least half of the slots to the rest: a peer whose outputs a reported + // send spends makes its parents such roots too. + let priority = graph.roots_of(priority.iter().copied()); + let due_root = |view: &OutgoingView, forced| DueRoot { + txid: view.txid, + transaction: Arc::clone(&view.transaction), + own: graph.is_own(&view.txid), + forced, + retry: false, + }; + type Turn = (u32, bool, Txid); + let mut named: Vec<(Turn, &OutgoingView)> = Vec::new(); + let mut forced_queue: Vec<((bool, Turn), &OutgoingView)> = Vec::new(); + let mut scheduled_queue: Vec<(Turn, bool, &OutgoingView)> = Vec::new(); + for view in &roots { + let schedule = state.schedules.entry((wallet_id, view.txid)).or_default(); + // A forced pass sees the roots its chains reach too, so the ones it + // leaves beyond the allowance go ahead of the ones it probed. + let reached = forced_roots.contains(&view.txid); + if let Some(seen) = anchor.or(reached.then_some(height)) { + schedule.seen_at.get_or_insert(seen); + } + // At the same height, one still waiting for its first probe goes + // before one probed there. + let turn = ( + schedule.last_probe.or(schedule.seen_at).unwrap_or(0), + schedule.last_probe.is_some(), + view.txid, + ); + if forced.contains(&view.txid) { + named.push((turn, view)); + } else if reached { + forced_queue.push(((!graph.is_own(&view.txid), turn), view)); + } + // Without an anchor the pass is confined to forced roots. + if anchor.is_some() && schedule.is_due(height) { + let first = priority.contains(&view.txid) && !schedule.panicked; + scheduled_queue.push((turn, first, view)); + } + } + named.sort_unstable_by_key(|(turn, _)| *turn); + let reached_due = take_in_turn(forced_queue, MAX_FORCED_ROOTS_PER_PASS); + let forced_due: Vec<&OutgoingView> = named + .into_iter() + .map(|(_, view)| view) + .chain(reached_due.into_iter().map(|(_, view)| view)) + .collect(); + let taken: HashSet = forced_due.iter().map(|view| view.txid).collect(); + let mut due: VecDeque = forced_due + .into_iter() + .map(|view| due_root(view, true)) + .collect(); + let mut first: Vec<(Turn, &OutgoingView)> = Vec::new(); + let mut rest: Vec<(Turn, &OutgoingView)> = Vec::new(); + for (turn, goes_first, view) in scheduled_queue { + if taken.contains(&view.txid) { + continue; + } + if goes_first { + first.push((turn, view)); + } else { + rest.push((turn, view)); + } + } + let first_slots = first + .len() + .min((MAX_ROOTS_PER_PASS / 2).max(MAX_ROOTS_PER_PASS.saturating_sub(rest.len()))); + let rest_slots = rest.len().min(MAX_ROOTS_PER_PASS - first_slots); + let mut scheduled = take_in_turn(first, first_slots); + scheduled.extend(take_in_turn(rest, rest_slots)); + due.extend(scheduled.into_iter().map(|(_, view)| due_root(view, false))); + PassStart { events, due } +} + +/// The first `count` of `queue` by turn, in turn. +fn take_in_turn(mut queue: Vec<(K, V)>, count: usize) -> Vec<(K, V)> { + if queue.len() > count { + queue.select_nth_unstable_by_key(count, |(turn, _)| *turn); + queue.truncate(count); + } + queue.sort_unstable_by_key(|(turn, _)| *turn); + queue +} + +/// A root is being sent to the network at `height`: its schedule counts from +/// here. Marked when the probe goes out, not when the pass is planned, so a +/// root the pass never reached stays due. +pub(crate) fn mark_sent( + state: &mut ResolverState, + wallet_id: WalletId, + root: Txid, + height: u32, + anchor: Option, +) { + if let Some(schedule) = state.schedules.get_mut(&(wallet_id, root)) { + // A root's every-block window starts at its first probe by a pass + // that follows the tip — not while it waits in a backlog, nor when a + // pass that planned it gave way before sending it. + if let Some(anchor) = anchor { + schedule.anchor(anchor); + } + schedule.probed_at(height); + } +} + +/// Whether a root's last probe reached a node (see +/// [`ProbeSchedule::unanswered`]). +fn mark_answered( + state: &mut ResolverState, + wallet_id: WalletId, + root: Txid, + answered: bool, + height: u32, +) { + if let Some(schedule) = state.schedules.get_mut(&(wallet_id, root)) { + schedule.answered(answered, height); + schedule.panicked = false; + } +} + +/// The roots of `wallet_id` no node answered at `height`, made due again — +/// a retry's, the only probe again at that height. A root whose probe is out +/// was marked sent, which cleared its mark. +fn withdraw_unanswered( + state: &mut ResolverState, + wallet_id: WalletId, + height: u32, +) -> HashSet { + let mut roots = HashSet::new(); + for ((wallet, txid), schedule) in state.schedules.iter_mut() { + if *wallet == wallet_id && schedule.unanswered && schedule.last_probe == Some(height) { + schedule.withdraw(); + roots.insert(*txid); + } + } + roots +} + +/// A root's probe never produced a verdict (its job panicked): nothing was +/// learned, so it is due again at once. +pub(crate) fn withdraw_send(state: &mut ResolverState, wallet_id: WalletId, root: Txid) { + if let Some(schedule) = state.schedules.get_mut(&(wallet_id, root)) { + schedule.withdraw(); + } +} + +/// Record one root's probe result: the verdict goes to the root when it is +/// the wallet's own send. `height` is the pass's. +pub(crate) fn record_probe( + state: &mut ResolverState, + wallet_id: WalletId, + height: u32, + root: &DueRoot, + verdict: &ProbeVerdict, +) -> Vec { + let mut events = Vec::new(); + if root.own { + state.publish_all( + wallet_id, + [root.txid], + verdict, + height, + root.trigger(), + &mut events, + ); + } + events +} + +/// The lowest height at which a root of the wallet can be due, if nothing new +/// happens to it: a root without a schedule yet is due at once; `u32::MAX` +/// when there is nothing to probe. +pub(crate) fn next_pass_height( + state: &ResolverState, + wallet_id: WalletId, + views: &[OutgoingView], +) -> u32 { + ChainGraph::new(views) + .roots() + .iter() + .map(|view| { + state + .schedules + .get(&(wallet_id, view.txid)) + .map_or(0, ProbeSchedule::next_due) + }) + .min() + .unwrap_or(u32::MAX) +} + +fn log_outgoing(item: &Outgoing) { + match &item.event { + ResolverEvent::Verdict(txid, verdict) => tracing::info!( + wallet_id = %hex::encode(item.wallet_id), + txid = %txid, + height = ?item.height, + trigger = item.trigger, + ?verdict, + "broadcast probe: verdict for an unconfirmed send" + ), + ResolverEvent::Cleared(txid) => tracing::info!( + wallet_id = %hex::encode(item.wallet_id), + txid = %txid, + height = ?item.height, + trigger = item.trigger, + "broadcast probe: verdict cleared" + ), + } +} + +// ---- The actor ------------------------------------------------------------- + +/// What a pass reads from a wallet. +pub(crate) struct WalletRead { + pub views: Vec, + pub synced_height: u32, + /// The requested `Uncertain` txids the wallet actually holds. + pub forced: HashSet, + /// The registration the views were read from: its transactions are + /// probed only while it is still the wallet's (see [`gated_probe`]). + pub generation: Arc, +} + +/// An `Uncertain` result is queued for every wallet; only one holding the +/// transaction acts on it. `None`: nothing to do for this wallet — the pass +/// need not `walk` the records (it is confined to forced roots) and the +/// wallet holds none of `forced`. +fn forced_held( + walk: bool, + forced: HashSet, + holds: impl Fn(&Txid) -> bool, +) -> Option> { + let forced: HashSet = forced.into_iter().filter(|txid| holds(txid)).collect(); + (walk || !forced.is_empty()).then_some(forced) +} + +/// Which wallets hold a transaction (see [`WalletSource::holders`]). +#[derive(Debug, Default)] +pub(crate) struct Holders { + /// Every wallet whose records hold it: where it is probed. + pub wallets: Vec, + /// Those where it is an unsettled own send: the ones its echo could + /// still be news to. + pub own: Vec, +} + +/// Where passes read wallets from. +#[async_trait] +pub(crate) trait WalletSource: Send + Sync { + /// The wallets whose records hold `txid`, and among them those where it + /// is an unsettled own send — one look under one lock. + async fn holders(&self, txid: Txid) -> Holders; + + /// `None`: no such wallet. `Some(None)`: nothing to do (see + /// [`forced_held`]) — decided without walking the records. `walk` is read + /// once the wallet is at hand: a catch-up may clear it meanwhile. + async fn read( + &self, + wallet_id: WalletId, + walk: &AtomicBool, + forced: HashSet, + ) -> Option>; + + /// Whether `generation` is still the registration under `wallet_id` — a + /// removal, or a re-registration under the same id, ends that. + async fn is_registered(&self, wallet_id: WalletId, generation: &Arc) -> bool; +} + +/// Probe `transaction` for the registration it was read from, or not at all: +/// `None` when `generation` is no longer the wallet's, with nothing sent. +/// +/// A probe resubmits the transaction, so it follows the contract of +/// [`WalletGeneration::payment_guard`]: the gate is held across both the +/// liveness check and the network step. A removal therefore either gets in +/// first, and the probe waits it out and then sees the registration gone, or +/// finds the probe inside — which then stops where it is and lets go of the +/// gate, so the removal waits neither for the wallet-manager lock nor for +/// any node's answer (a probe is up to eight requests, each with its own +/// timeout). A request already handed to the transport is not recalled, but +/// no further one is made. Stopping the job from the actor is not that +/// barrier: the actor learns of a removal only after the manager has let the +/// wallet go. +/// +/// A removal that is given up leaves the wallet registered: the probe then +/// starts over — the same signed bytes again. +/// +/// "Removal" is the manager's `remove_wallet`. The rollback of a +/// registration or load that failed half-way takes no gate: a probe already +/// past the check may finish for a wallet rolled back meanwhile — the same +/// signed bytes, for a wallet nobody asked to remove. +/// +/// Lock order is the gate's: gate first, wallet-manager lock (inside +/// [`WalletSource::is_registered`]) second. +async fn gated_probe( + source: &dyn WalletSource, + probe: &dyn AcceptanceProbe, + wallet_id: WalletId, + generation: &Arc, + transaction: &Transaction, +) -> Option { + loop { + tokio::select! { + biased; + // First: a probe that has finished keeps its verdict. + probed = async { + let _payment = generation.payment_guard().await; + if !source.is_registered(wallet_id, generation).await { + return None; + } + Some(probe.probe(transaction).await) + } => return probed, + _ = generation.teardown_requested() => {} + } + generation.teardown_over().await; + } +} + +/// Whether any account of the wallet records `txid` — a map lookup per +/// account, not a walk over the records. +fn holds(info: &PlatformWalletInfo, txid: &Txid) -> bool { + info.core_wallet + .accounts + .all_accounts() + .iter() + .any(|account| account.transactions().contains_key(txid)) +} + +/// Whether the wallet records `txid` (`None` if not) and, if it does, whether +/// it is an own send no account has settled (in a block, InstantSend-locked, +/// or finalized) — the same rule as [`merge_records`]: settled in any account +/// means settled. One pass over the accounts. +fn holding(wallet: &Wallet, info: &PlatformWalletInfo, txid: &Txid) -> Option { + let signs = wallet.can_sign(); + let accounts = info.core_wallet.accounts.all_accounts(); + // No allocation: `is_unsettled_own` stops at the first settled record, + // which has been seen by then. + let mut held = false; + let own = is_unsettled_own( + accounts + .iter() + .filter_map(|account| { + account.transactions().get(txid).map(|record| { + record_facts(record, || account.transaction_is_finalized(txid), signs) + }) + }) + .inspect(|_| held = true), + ); + held.then_some(own) +} + +struct ManagerSource(Arc>>); + +#[async_trait] +impl WalletSource for ManagerSource { + async fn holders(&self, txid: Txid) -> Holders { + let manager = self.0.read().await; + let mut holders = Holders::default(); + for wallet_id in manager.list_wallets() { + let Some((wallet, info)) = manager.get_wallet_and_info(wallet_id) else { + continue; + }; + if let Some(own) = holding(wallet, info, &txid) { + holders.wallets.push(*wallet_id); + if own { + holders.own.push(*wallet_id); + } + } + } + holders + } + + async fn read( + &self, + wallet_id: WalletId, + walk: &AtomicBool, + forced: HashSet, + ) -> Option> { + let manager = self.0.read().await; + let (wallet, info) = manager.get_wallet_and_info(&wallet_id)?; + // Checked again under this lock: the record may have left since the + // holders lookup (a sweep, a removal). + let forced = forced_held(walk.load(Ordering::SeqCst), forced, |txid| { + holds(info, txid) + }); + Some(forced.map(|forced| WalletRead { + views: collect_views(info, wallet.can_sign()), + synced_height: info.core_wallet.synced_height(), + forced, + generation: Arc::clone(&info.generation), + })) + } + + async fn is_registered(&self, wallet_id: WalletId, generation: &Arc) -> bool { + generation.is_registered_in(&*self.0.read().await, &wallet_id) + } +} + +/// Where events for the host go. +pub(crate) trait VerdictSink: Send + Sync { + fn deliver(&self, item: &Outgoing); +} + +/// The host's event callbacks, reached through the event manager this +/// resolver is registered with (weakly: the manager holds the resolver). +#[derive(Default)] +struct EventSink(Mutex>); + +impl VerdictSink for EventSink { + fn deliver(&self, item: &Outgoing) { + log_outgoing(item); + let events = self + .0 + .lock() + .unwrap_or_else(PoisonError::into_inner) + .upgrade(); + let Some(events) = events else { + return; + }; + match &item.event { + ResolverEvent::Verdict(txid, verdict) => { + events.on_outgoing_transaction_probed(&item.wallet_id, txid, verdict) + } + ResolverEvent::Cleared(txid) => { + events.on_outgoing_transaction_cleared(&item.wallet_id, txid) + } + } + } +} + +/// Hand `events` to the host. The one production handler — the FFI bridge — +/// cannot panic, and the release profiles abort on a panic anyway; for a Rust +/// embedder whose handler panics under unwinding, the event it panicked on is +/// logged and dropped rather than taking the resolver task down. +fn deliver(sink: &Arc, events: Vec) { + for item in &events { + if catch_unwind(AssertUnwindSafe(|| sink.deliver(item))).is_err() { + tracing::error!( + wallet_id = %hex::encode(item.wallet_id), + event = ?item.event, + "broadcast probe: a host handler panicked; its event is dropped" + ); + } + } +} + +/// What the resolver is told. Sent from event handlers and the manager; the +/// actor handles them one at a time, in order. +#[derive(Debug)] +enum Command { + Height { + wallet_id: WalletId, + height: u32, + }, + /// Any other wallet event; `touched` when it may have left something new + /// to probe (see [`may_leave_work`]). + Seen { + wallet_id: WalletId, + touched: bool, + }, + /// Transactions a wallet event reports settled (in a block or + /// InstantSend-locked) or gone (swept): their verdicts are cleared at once, + /// without a pass — during a catch-up too. + Settled { + wallet_id: WalletId, + txids: HashSet, + }, + Uncertain(Txid), + /// dash-spv saw a peer echo a send it had reported `Uncertain`. + Echoed(Txid), + SetEnabled(bool), + /// The wallet was registered or loaded into the manager: only this + /// creates a wallet's entry, so events queued behind a removal cannot + /// bring one back. + WalletAdded(WalletId), + WalletRemoved(WalletId), + /// dash-spv finished a sync cycle with its headers at `tip` — after + /// every new block, and when a launch's or a reconnect's catch-up ends. + /// A wallet within [`CATCH_UP_STEP`] of that tip follows it now: it gets + /// a followed pass, which the per-height schedule bounds (one probe per + /// root per height; a root no node answered is due again). + SyncComplete { + tip: u32, + }, + /// Answered once every command sent before it has been handled, the + /// host callbacks those made included (see + /// [`BroadcastResolver::caught_up`]). + Barrier(oneshot::Sender<()>), + /// Whether the wallet is idle: no pass running or queued, and none due + /// at any height (see [`WalletEntry::wait_until`]). Answered in order + /// with the other commands. + #[cfg(test)] + IsIdle { + wallet_id: WalletId, + reply: oneshot::Sender, + }, +} + +/// What a finished job reports back to the actor. +enum JobDone { + /// The wallets holding a transaction reported `Uncertain` (`wallets`, + /// where it is probed), and those where it is an unsettled own send + /// (`own`, the ones a later echo could still be news to). + /// `report` names the `Uncertain` report this lookup answers: a newer + /// report of the same txid, or a switch-off, makes it obsolete. + Listed { + txid: Txid, + report: u64, + wallets: Vec, + own: Vec, + }, + /// The wallets holding a send that was echoed after its `Uncertain`. + /// `token` names the echo this lookup answers: a send that settled since + /// then has withdrawn it, and a wallet that left or registered had it + /// made again under a new token. + EchoListed { + txid: Txid, + wallets: Vec, + token: u64, + }, + Read { + wallet_id: WalletId, + run: u64, + height: u32, + read: Option>, + }, + Probed { + wallet_id: WalletId, + run: u64, + /// The root it probed: a result for a root no longer in flight (its + /// probe was stopped, but had already finished) is not this one's. + root: Txid, + /// `None`: nothing was sent — the registration the pass read is no + /// longer the wallet's (see [`gated_probe`]). + outcome: Option, + }, + /// [`NO_ANSWER_RETRY`] passed since a probe at `height` that no node + /// answered. + Retry { + wallet_id: WalletId, + series: u64, + height: u32, + }, +} + +#[derive(Default)] +struct PendingPass { + height: u32, + forced: HashSet, + /// The latest height that joined this pass while the wallet followed the + /// tip; a catch-up step clears it. A pass without one — only `Uncertain` + /// results asked for it, or a catch-up overtook it — probes only the + /// roots those results forced and starts no windows. + anchor_at: Option, + /// Roots a retry forces (see [`NO_ANSWER_RETRY`]): probed only if they + /// are still roots — one whose parent has since shown up unsettled is + /// not resolved to that parent. + retried: HashSet, +} + +/// A pass in flight: reading the wallet, then probing its due roots one at +/// a time. Results carry the run id; a result for a run that was dropped +/// (probing off, wallet removed) is ignored. +struct Run { + id: u64, + /// Txids a wallet event settled or removed while the run was reading: the + /// read may still list them, and they are left out of it. Once the run + /// probes, a settle prunes its views and due roots instead (a root whose + /// probe is out is moot once gone from the views). Gone with the run — a + /// later read is current, a send a reorg brings back included. + settled: HashSet, + /// Something touched the wallet during the run: its read may be stale, so + /// the next height must not be skipped. + woken: bool, + /// Where this pass starts windows; `None` confines it to forced roots + /// (see [`PendingPass::anchor_at`]). + anchor_at: Option, + /// What `Uncertain` results forced it to probe: carried over to the next + /// pass when a rewind stops it. + forced: HashSet, + /// What a retry forced it to probe (see [`PendingPass::retried`]). + retried: HashSet, + /// Whether the read walks the records even when none of the forced txids + /// is held (`anchor_at.is_some()`): shared with the read job, cleared by a + /// catch-up. + walk: Arc, + abort: AbortHandle, + probing: Option, +} + +struct Probing { + views: Vec, + /// The registration `views` were read from (see [`WalletRead::generation`]). + generation: Arc, + height: u32, + due: VecDeque, + /// The root whose probe is out now. + in_flight: Option, +} + +impl Probing { + /// Keep only the forced roots still to send: the scheduled ones stay due + /// for a later pass. Whether any was dropped. + fn drop_scheduled(&mut self) -> bool { + let planned = self.due.len(); + self.due.retain(|root| root.forced); + self.due.len() < planned + } +} + +#[derive(Default)] +struct WalletEntry { + /// Last synced height reported, for telling catch-up from following. + height: Option, + /// Height advances below this are skipped — no walk over the wallet's + /// records while no root can be due. `u32::MAX` when nothing is left to + /// probe (idle). Reset by anything that may have left new work — which + /// assumes every change to the wallet's records arrives as a touching + /// wallet event (see [`may_leave_work`]); a path that changes records + /// without emitting one must send the resolver such an event too. + wait_until: Option, + pending: Option, + run: Option, + /// The retries at one height after probes no node answered (see + /// [`NO_ANSWER_RETRY`]): how many were armed, and whether one waits. The + /// roots carry the unanswered mark in their schedules; the retry + /// withdraws them when it fires. A retry that merges into a followed + /// pass already queued at that height rides along with it. + retry_series: Option, + /// The height of the last anchored pass that read the wallet: a sync + /// completion at that height (after every block) adds no second walk + /// over the records. + followed_pass_at: Option, + /// The height at which a run last gave way to a forced request (see + /// [`yield_to_forced`]). + yielded_at: Option, + /// The tip of a sync that completed while the wallet's last reported + /// height was still well below it. Sync events and wallet events come + /// from separate tasks, so a completion can be handled before the height + /// step it follows: the catch-up step that then brings the wallet to that + /// tip is its arrival there, and runs the pass the completion could not. + /// Replaced by every later completion, so a wallet that really is + /// rescanning gets its pass when it reaches the latest tip, not a stale + /// one. Kept with the time it was handled: it stands for + /// [`COMPLETED_AHEAD_TTL`] only (see there). + completed_ahead: Option<(u32, Instant)>, +} + +impl WalletEntry { + /// Whether a pass that follows the tip ran, runs or is queued for the + /// wallet at `height` or above. + fn anchored_at_or_above(&self, height: u32) -> bool { + [ + self.followed_pass_at, + self.run.as_ref().and_then(|run| run.anchor_at), + self.pending.as_ref().and_then(|pending| pending.anchor_at), + ] + .iter() + .any(|at| at.is_some_and(|at| at >= height)) + } +} + +#[derive(Debug, Clone, Copy)] +struct NoAnswerRetries { + /// Names the series: a timer from before a reset (a rewind, a switch) + /// matches no later series at the same height. + series: u64, + height: u32, + armed: u32, + waiting: bool, +} + +/// Owns all of the resolver's state. One task; commands and job results are +/// handled one at a time, so no two of them race, and every host callback +/// comes from this task — never from a caller holding its own locks. +struct Actor { + enabled: bool, + state: ResolverState, + wallets: HashMap, + next_run: u64, + probe: Arc, + source: Arc, + sink: Arc, + jobs: JoinSet, + /// Which run each read or probe job belongs to, so a job that panicked + /// can end its run instead of leaving the wallet stuck behind it. + /// A probe job also names its root. + job_runs: HashMap)>, + /// Sends dash-spv reported `Uncertain`, with the followed height their + /// echo window starts at (`None` until the next followed block after the + /// latest report — a report while the wallets catch up starts nothing): + /// a later peer echo of one of them is its acceptance. Removed by its + /// echo, its settling, the latest report's holder lookup if it finds no + /// unsettled own send, + /// [`UNCERTAIN_EXPIRY`] followed blocks into its window, and all at + /// switch-off. One no wallet resolves (its wallet removed, say) lingers + /// until it expires. + uncertain: HashMap>, + /// The token of the latest `Uncertain` report per txid, until its holder + /// lookup completes: only that completion counts (see `JobDone::Listed`). + /// A newer report or a switch-off withdraws it; a wallet leaving or + /// re-registering makes the lookup again (see `drop_wallet`). A lookup + /// that panicked gives its report up. + /// The job's handle: a superseded lookup is stopped, not left to scan. + report_lookups: HashMap, + next_report_token: u64, + /// Every send dash-spv reported `Uncertain` this session, with the + /// wallets that hold it as an unsettled own send: this process's own + /// broadcasts. Their roots go first in a pass's queue for as long as they + /// stay unsettled — past their echo window too — so a backlog fed by a + /// peer never holds them back. A wallet leaves an entry when the send + /// settles or leaves it (seen by its next read), or when it is removed. + reported: HashMap>, + /// The highest height any wallet has followed the tip at: the one clock + /// the echo windows run on, so a wallet behind the others (rescanning, + /// rewound) neither stamps nor expires them. + followed_tip: Option, + /// Names each retry series (see [`NoAnswerRetries`]), so a stale timer + /// matches no later series. + next_retry_series: u64, + /// Echoes whose holder lookup is out, by the token their result carries + /// and the handle that stops a superseded one. A send that settles + /// withdraws its lookup: an acceptance landing after that must not reach + /// the host. A wallet that leaves or registers has every lookup made + /// again (see `drop_wallet`), so the acceptance goes to the wallets that + /// hold the send then. + echo_lookups: HashMap, + next_echo_token: u64, +} + +impl Actor { + fn new( + probe: Arc, + source: Arc, + sink: Arc, + ) -> Self { + Self { + enabled: false, + state: ResolverState::default(), + wallets: HashMap::new(), + next_run: 0, + probe, + source, + sink, + jobs: JoinSet::new(), + job_runs: HashMap::new(), + uncertain: HashMap::new(), + report_lookups: HashMap::new(), + next_report_token: 0, + reported: HashMap::new(), + followed_tip: None, + next_retry_series: 0, + echo_lookups: HashMap::new(), + next_echo_token: 0, + } + } + + async fn run( + mut self, + mut commands: mpsc::UnboundedReceiver, + mut stop: oneshot::Receiver<()>, + ) { + loop { + tokio::select! { + biased; + _ = &mut stop => break, + command = commands.recv() => match command { + Some(command) => self.handle(command), + None => break, + }, + Some(joined) = self.jobs.join_next_with_id(), if !self.jobs.is_empty() => { + self.joined(joined); + } + } + } + // Abort every read and probe and wait for them, so nothing the + // resolver started outlives it. + self.jobs.shutdown().await; + } + + fn joined(&mut self, joined: Result<(task::Id, JobDone), JoinError>) { + match joined { + Ok((id, done)) => { + self.job_runs.remove(&id); + self.job_done(done); + } + Err(error) => { + let owner = self.job_runs.remove(&error.id()); + if !error.is_panic() { + return; // aborted: its run was dropped on purpose + } + tracing::error!(?owner, %error, "broadcast probe: a job panicked"); + // End the run it belonged to and let the next height probe the + // wallet again — what the run carried (a forced txid) is lost, + // but its roots are due: the panicked one is withdrawn, the + // ones it had not sent yet were never marked. A panicked lookup + // for an `Uncertain` result belongs to no run: wake every + // wallet instead. + match owner { + Some((wallet_id, run, probed)) => { + // Only the wallet's current run: a stale panic (its run + // was dropped) must not undo a newer run's send. The + // root whose probe panicked is the one in flight. + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + let Some(current) = entry.run.as_mut().filter(|current| current.id == run) + else { + return; + }; + // A probe job's panic belongs to its own root only: the + // stopped probe of a settled root is not the one out now. + let in_flight_txid = current + .probing + .as_ref() + .and_then(|probing| probing.in_flight.as_ref()) + .map(|root| root.txid); + if probed.is_some() && probed != in_flight_txid { + return; + } + let in_flight = current.probing.as_mut().and_then(|probing| { + probing.in_flight.take().map(|root| (root, probing.height)) + }); + entry.wait_until = None; + if let Some((root, height)) = in_flight { + withdraw_send(&mut self.state, wallet_id, root.txid); + // To the back of the queue, as if first seen after + // this height: a probe that panics every time must + // not head every pass and end it. + if let Some(schedule) = + self.state.schedules.get_mut(&(wallet_id, root.txid)) + { + schedule.seen_at = Some(height.saturating_add(1)); + schedule.panicked = true; + } + } + self.end_run(wallet_id); + } + None => { + // A panicked report lookup gives its report up: no + // later wallet change brings it back. + self.report_lookups + .retain(|_, (_, lookup)| lookup.id() != error.id()); + // So does a panicked echo lookup its echo: a wallet + // change must not make it again. + self.echo_lookups + .retain(|_, (_, lookup)| lookup.id() != error.id()); + for entry in self.wallets.values_mut() { + entry.wait_until = None; + } + } + } + } + } + } + + fn handle(&mut self, command: Command) { + match command { + Command::Height { wallet_id, height } => { + // Only a wallet the actor was told about (see `WalletAdded`). + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + let previous = entry.height.replace(height); + let rewound = previous.is_some_and(|previous| height < previous); + if rewound { + // A rewind replays the chain: whatever the resolver kept + // for the wallet may rest on blocks the replay drops, so + // it starts over: every verdict cleared, every root due + // at once, its window from the next following pass. A run + // under way rests on that state too: it is stopped, and what + // `Uncertain` results forced it to probe goes to the next pass. + entry.wait_until = None; + entry.retry_series = None; + entry.followed_pass_at = None; + entry.yielded_at = None; + if let Some(run) = entry.run.take() { + run.abort.abort(); + if !run.forced.is_empty() { + entry + .pending + .get_or_insert_with(PendingPass::default) + .forced + .extend(run.forced); + } + } + let events = self.state.forget_wallet(&wallet_id, "rewound"); + deliver(&self.sink, events); + } + // A step up to the tip of a sync that already completed is + // the wallet arriving at the tip, however large the step: its + // completion was handled first. A rewind is a new scan, which + // the completion did not cover. + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + let completed = entry + .completed_ahead + .filter(|(_, at)| at.elapsed() <= COMPLETED_AHEAD_TTL) + .map(|(tip, _)| tip); + let arrived = !rewound && completed.is_some_and(|tip| near_tip(height, tip)); + if rewound || arrived || completed.is_none_or(|tip| height >= tip) { + entry.completed_ahead = None; + } + let gap = is_catch_up_step(previous, height); + let catch_up = gap && !arrived; + if catch_up { + // The tip moved on, unfollowed: no lower wallet's steps + // may run the echo windows now. + self.followed_tip = self.followed_tip.max(Some(height)); + } else { + self.expire_uncertain(height); + } + if gap { + self.confine_to_forced(wallet_id, height, arrived); + } + if catch_up { + return; + } + // The completion that came first may have run this height's + // pass already (a wallet with no step since launch reads its + // own height): no second walk over the records then. A pass + // still running is not taken for that: its read may have + // found the wallet lower and dropped its anchor, so the + // height is queued behind it, and dropped at its end if no + // root is due. + if arrived + && self + .wallets + .get(&wallet_id) + .is_some_and(|entry| entry.followed_pass_at.is_some_and(|at| at >= height)) + { + return; + } + self.queue_followed_pass(wallet_id, height); + } + Command::Settled { wallet_id, txids } => { + let mut next_probe = false; + // Only a wallet already known: an event queued behind its + // removal must not bring back an entry nothing would drop. + if let Some(entry) = self.wallets.get_mut(&wallet_id) { + // A pass under way read the wallet before this: it neither + // probes these any more nor publishes anything for them. + if let Some(run) = entry.run.as_mut() { + match run.probing.as_mut() { + None => run.settled.extend(txids.iter().copied()), + Some(probing) => { + probing.due.retain(|root| !txids.contains(&root.txid)); + probing.views.retain(|view| !txids.contains(&view.txid)); + // The root whose probe is out settled: stop + // that probe and go on with the next root. + if probing + .in_flight + .as_ref() + .is_some_and(|root| txids.contains(&root.txid)) + { + probing.in_flight = None; + run.abort.abort(); + next_probe = true; + } + } + } + } + } + // Settled sends have nothing left to accept: an echo, or a + // lookup for one already out, must not publish after this. + for txid in &txids { + self.uncertain.remove(txid); + if let Some((_, lookup)) = self.echo_lookups.remove(txid) { + lookup.abort(); + } + } + forget_reported(&mut self.reported, &wallet_id, |txid| txids.contains(txid)); + let events = self.state.settle(wallet_id, &txids); + deliver(&self.sink, events); + if next_probe { + self.probe_next(wallet_id); + } + } + Command::Seen { wallet_id, touched } => { + // Only a wallet already known (see `Settled`). + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + if touched { + entry.wait_until = None; + if let Some(run) = entry.run.as_mut() { + run.woken = true; + } + } + } + Command::Uncertain(txid) => { + // The switch is decided here, in order with SetEnabled — not + // by the sender, which may see the flag before the actor does. + if !self.enabled { + return; + } + // A new report restarts the window, from the next followed + // block. + self.uncertain.insert(txid, None); + self.look_up_report(txid); + } + Command::Echoed(txid) => { + // Every healthy send is echoed: only one that was `Uncertain` + // has a verdict to change. + if !self.enabled || self.uncertain.remove(&txid).is_none() { + return; + } + self.look_up_echo(txid); + } + Command::SetEnabled(enabled) => { + if enabled == self.enabled { + return; + } + self.enabled = enabled; + // Either way every wallet waits for nothing, so turning + // probing back on re-publishes what turning it off cleared. + for entry in self.wallets.values_mut() { + entry.wait_until = None; + entry.retry_series = None; + entry.followed_pass_at = None; + entry.yielded_at = None; + if !enabled { + entry.pending = None; + if let Some(run) = entry.run.take() { + run.abort.abort(); + } + } + } + if !enabled { + self.uncertain.clear(); + for (_, (_, lookup)) in self.report_lookups.drain() { + lookup.abort(); + } + self.reported.clear(); + for (_, (_, lookup)) in self.echo_lookups.drain() { + lookup.abort(); + } + let events = self.state.forget_all(); + deliver(&self.sink, events); + } + } + Command::SyncComplete { tip } => { + // Every wallet at the tip follows it now: a launch's or a + // reconnect's catch-up skipped its pass, and the next block + // can be minutes away. A wallet still far below (rescanning) + // waits: its windows must not start at a stale height. One + // without a step since launch reads its own height. While + // probing is off this does nothing (queue_followed_pass): the + // next completion, after the next block, covers it. + // + // A wallet below the tip may also be one whose height step + // has not been handled yet: remember the tip for it (see + // `WalletEntry::completed_ahead`). So may one without a step + // since launch: its first step, when it lands at this tip, is + // then its arrival and keeps the pass started here. + let now = Instant::now(); + for entry in self.wallets.values_mut() { + let below = entry + .height + .is_none_or(|height| height < tip && !near_tip(height, tip)); + entry.completed_ahead = below.then_some((tip, now)); + } + let at_tip: Vec<(WalletId, u32)> = self + .wallets + .iter() + .map(|(wallet_id, entry)| (wallet_id, entry, entry.height.unwrap_or(tip))) + .filter(|(_, entry, height)| { + near_tip(*height, tip) && !entry.anchored_at_or_above(*height) + }) + .map(|(wallet_id, _, height)| (*wallet_id, height)) + .collect(); + for (wallet_id, height) in at_tip { + self.queue_followed_pass(wallet_id, height); + } + } + Command::WalletAdded(wallet_id) => { + // A fresh start: whatever was kept under this id (a same-id + // wallet whose removal never reached the resolver) is stale. + self.drop_wallet(&wallet_id, "re-registered"); + self.wallets.insert(wallet_id, WalletEntry::default()); + } + Command::WalletRemoved(wallet_id) => self.drop_wallet(&wallet_id, "removed"), + Command::Barrier(reply) => { + // The waiter may have given up: nothing to do then. + let _ = reply.send(()); + } + #[cfg(test)] + Command::IsIdle { wallet_id, reply } => { + let idle = self.wallets.get(&wallet_id).is_some_and(|entry| { + entry.run.is_none() + && entry.pending.is_none() + && entry.wait_until == Some(u32::MAX) + }); + let _ = reply.send(idle); + } + } + } + + /// Forget everything kept for a wallet that is gone, `reported` included + /// (a rewind keeps it: the wallet still holds its reported sends). + fn forget_wallet(&mut self, wallet_id: &WalletId, trigger: &'static str) -> Vec { + forget_reported(&mut self.reported, wallet_id, |_| true); + self.state.forget_wallet(wallet_id, trigger) + } + + /// Route a report to the registered wallets holding it, including ones + /// no height step has reached yet; only the latest lookup per txid counts. + fn look_up_report(&mut self, txid: Txid) { + let report = self.next_report_token; + self.next_report_token += 1; + let source = Arc::clone(&self.source); + let job = self.jobs.spawn(async move { + let Holders { wallets, own } = source.holders(txid).await; + JobDone::Listed { + txid, + report, + wallets, + own, + } + }); + if let Some((_, superseded)) = self.report_lookups.insert(txid, (report, job)) { + superseded.abort(); + } + } + + /// Find the wallets an echoed send is an unsettled own send of: its + /// acceptance is theirs. A lookup already out for `txid` is superseded — + /// its result no longer carries the current token. + fn look_up_echo(&mut self, txid: Txid) { + let token = self.next_echo_token; + self.next_echo_token += 1; + let source = Arc::clone(&self.source); + let job = self.jobs.spawn(async move { + JobDone::EchoListed { + txid, + wallets: source.holders(txid).await.own, + token, + } + }); + if let Some((_, superseded)) = self.echo_lookups.insert(txid, (token, job)) { + superseded.abort(); + } + } + + /// The registration a pass read, or was to read, is not in the wallet + /// manager. This manager's own removals queue `WalletRemoved` under the + /// same guard, and commands are handled before job results, so in + /// practice only a removal or replacement through `wallet_manager_arc` + /// gets here. Fail safe: forget its state, the host told to clear what it showed, and + /// end the run. + fn registration_gone(&mut self, wallet_id: WalletId) { + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + // Idle, not re-read every block: a same-id wallet's events or an + // Uncertain wake it. + entry.wait_until = Some(u32::MAX); + let events = self.forget_wallet(&wallet_id, "gone"); + deliver(&self.sink, events); + // An echo lookup from before may still name this registration. + self.relook_echoes(|_| true); + self.end_run(wallet_id); + } + + /// Make again the echo lookups still out whose txid `affected` names: + /// what they found may no longer hold. + fn relook_echoes(&mut self, affected: impl Fn(&Txid) -> bool) { + let echoes: Vec = self + .echo_lookups + .keys() + .filter(|txid| affected(txid)) + .copied() + .collect(); + for txid in echoes { + self.look_up_echo(txid); + } + } + + /// Drop the wallet's entry, stop its pass and forget its state, the host + /// told to clear what it showed (logged with `trigger`). Report and echo + /// lookups still out are made again, and the wallet leaves `reported`. + fn drop_wallet(&mut self, wallet_id: &WalletId, trigger: &'static str) { + if let Some(run) = self.wallets.remove(wallet_id).and_then(|entry| entry.run) { + run.abort.abort(); + } + // An echo's lookup may name the wallet gone, or a registration this + // one replaces: look again, so the acceptance reaches the wallets + // that hold the send now. The echo was already taken out of + // `uncertain`, so dropping its lookup would lose it for the others. + self.relook_echoes(|_| true); + // A report's lookup may name the wallet gone: look again, so the + // other holders still get their forced probe and priority. + let reports: Vec = self.report_lookups.keys().copied().collect(); + for txid in reports { + self.look_up_report(txid); + } + let events = self.forget_wallet(wallet_id, trigger); + deliver(&self.sink, events); + } + + /// A wallet followed the tip to `height`: advance the one clock, start + /// the echo windows of the sends reported since the last followed block, + /// and forget those [`UNCERTAIN_EXPIRY`] blocks into their window. + fn expire_uncertain(&mut self, height: u32) { + if self.followed_tip.is_some_and(|tip| tip >= height) { + return; // a wallet behind another: not the clock + } + self.followed_tip = Some(height); + self.uncertain.retain(|_, start| { + let at = *start.get_or_insert(height); + height < at.saturating_add(UNCERTAIN_EXPIRY) + }); + } + + /// Start the wallet's pending pass unless one is already running. + fn start(&mut self, wallet_id: WalletId) { + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + if entry.run.is_some() { + return; // the running pass takes it up when it ends + } + let Some(pending) = entry.pending.take() else { + return; + }; + let anchor_at = pending.anchor_at; + let forced = pending.forced.clone(); + let retried = pending.retried.clone(); + let to_read: HashSet = pending.forced.union(&pending.retried).copied().collect(); + // Walk the records if the pass may probe by schedule; one confined to + // forced roots with none of them held has nothing to read for. + let walk = Arc::new(AtomicBool::new(anchor_at.is_some())); + self.next_run += 1; + let run = self.next_run; + let source = Arc::clone(&self.source); + let job_walk = Arc::clone(&walk); + let abort = self.jobs.spawn(async move { + let read = source.read(wallet_id, &job_walk, to_read).await; + JobDone::Read { + wallet_id, + run, + height: pending.height, + read, + } + }); + self.job_runs.insert(abort.id(), (wallet_id, run, None)); + entry.run = Some(Run { + id: run, + settled: HashSet::new(), + woken: false, + anchor_at, + forced, + retried, + walk, + abort, + probing: None, + }); + } + + fn job_done(&mut self, done: JobDone) { + match done { + JobDone::Listed { + txid, + report, + wallets, + own, + } => { + // Only the latest report's lookup: a newer report's own + // lookup reads the wallets no earlier than that report, so it + // decides; one from before a switch-off or a wallet's + // departure belongs to nothing. + if !self.enabled + || self + .report_lookups + .get(&txid) + .is_none_or(|(token, _)| *token != report) + { + return; + } + self.report_lookups.remove(&txid); + // No wallet holds it as an unsettled own send: its echo + // would be news to no one. + let own_holders: HashSet = own.iter().copied().collect(); + if own.is_empty() { + self.uncertain.remove(&txid); + self.reported.remove(&txid); + } else { + let known: HashSet = own + .into_iter() + .filter(|id| self.wallets.contains_key(id)) + .collect(); + if !known.is_empty() { + self.reported.entry(txid).or_default().extend(known); + } + } + for wallet_id in wallets { + // Only a wallet the actor was told about (see + // `WalletAdded`): one removed since the lookup is gone. + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + continue; + }; + let pending = entry.pending.get_or_insert_with(PendingPass::default); + pending.height = pending.height.max(entry.height.unwrap_or(0)); + pending.forced.insert(txid); + // Only for a wallet whose own send it is: one that merely + // receives it has no root to hurry. + if own_holders.contains(&wallet_id) { + yield_to_forced(entry); + } + self.start(wallet_id); + } + } + JobDone::EchoListed { + txid, + wallets, + token, + } => { + if !self.enabled + || self.echo_lookups.get(&txid).map(|(current, _)| *current) != Some(token) + { + return; + } + self.echo_lookups.remove(&txid); + let mut events = Vec::new(); + for wallet_id in wallets { + let Some(entry) = self.wallets.get(&wallet_id) else { + continue; + }; + // An echo says only that the send is out there: for one + // already accepted this publishes nothing. + let height = entry.height.unwrap_or(0); + self.state.publish_all( + wallet_id, + [txid], + &ProbeVerdict::Accepted, + height, + "echo", + &mut events, + ); + } + deliver(&self.sink, events); + } + JobDone::Read { + wallet_id, + run, + height, + read, + } => { + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + if !entry.run.as_ref().is_some_and(|current| current.id == run) { + return; + } + match read { + None => { + tracing::error!( + wallet_id = %hex::encode(wallet_id), + "broadcast probe: a registered wallet read as gone" + ); + self.registration_gone(wallet_id); + } + Some(None) => { + // The wallet has no record of any txid this pass + // asked for: they left it with no event naming them. + // Forget them here as `Settled` would, but look their + // echoes up again rather than drop them: another + // wallet may still hold the send. + let Some(run) = entry.run.take() else { + return; // matched by id above + }; + let asked: HashSet = + run.forced.union(&run.retried).copied().collect(); + forget_reported(&mut self.reported, &wallet_id, |txid| { + asked.contains(txid) + }); + for txid in &asked { + self.state.schedules.remove(&(wallet_id, *txid)); + } + let events = self.state.clear_where( + |wallet, txid| *wallet == wallet_id && asked.contains(txid), + Some(height), + "settled-or-left", + ); + deliver(&self.sink, events); + self.relook_echoes(|txid| asked.contains(txid)); + self.end_run(wallet_id); + } + Some(Some(read)) => { + // A request queued before any height was reported + // carries 0; the wallet's own synced height is better. + let height = height.max(read.synced_height); + let Some(current) = entry.run.as_mut() else { + return; // matched by id above + }; + // The wallet is not where this pass was queued: far + // ahead (a catch-up whose height step has not arrived + // yet) or behind (a rewind for a rescan). Either way a + // catch-up — confine the pass to forced roots. + if current.anchor_at.is_some_and(|at| { + read.synced_height < at + || read.synced_height > at.saturating_add(CATCH_UP_STEP) + }) { + current.anchor_at = None; + } + // What an event settled since the read began is not + // the pass's any more. + let views: Vec = read + .views + .into_iter() + .filter(|view| !current.settled.contains(&view.txid)) + .collect(); + // A retried txid that is no longer a root is not the + // retry's business: forcing it would resolve to a parent + // already probed at this height. + let graph = ChainGraph::new(&views); + let roots_now: HashSet = + graph.roots().iter().map(|view| view.txid).collect(); + // A root a report forced (directly or through a child) + // is labelled by the report, not by the retry. + let reported_roots = graph.roots_of(current.forced.iter().copied()); + let forced: HashSet = read + .forced + .iter() + .filter(|txid| { + current.forced.contains(*txid) + || !current.retried.contains(*txid) + || roots_now.contains(*txid) + }) + .copied() + .collect(); + let anchor = current.anchor_at; + // Only the sends this wallet still holds unsettled: one + // that settled or left it is no longer its priority. + let present: HashSet = views.iter().map(|view| view.txid).collect(); + forget_reported(&mut self.reported, &wallet_id, |txid| { + !present.contains(txid) + }); + let own_present: HashSet = views + .iter() + .filter(|view| view.spends_own_coins) + .map(|view| view.txid) + .collect(); + let priority: HashSet = self + .reported + .iter() + .filter(|(_, holders)| holders.contains(&wallet_id)) + .map(|(txid, _)| *txid) + .collect(); + let mut start = begin_pass( + &mut self.state, + wallet_id, + height, + &views, + &forced, + &priority, + anchor, + ); + for root in start.due.iter_mut() { + root.retry = current.retried.contains(&root.txid) + && !reported_roots.contains(&root.txid); + } + if anchor.is_some() { + entry.followed_pass_at = Some(height); + } + let Some(current) = entry.run.as_mut() else { + return; + }; + current.probing = Some(Probing { + views, + generation: read.generation, + height, + due: start.due, + in_flight: None, + }); + // A forced request that came while this pass read. + yield_to_forced(entry); + // Clears reach the host before the pass goes out to + // the network. + deliver(&self.sink, start.events); + self.probe_next(wallet_id); + // This read may have found a send settled or gone + // that no `Settled` named (the lossless channel only + // says a wallet's records changed). An echo lookup + // from before could still name this wallet for such + // a send: look those up again. + self.relook_echoes(|txid| !own_present.contains(txid)); + } + } + } + JobDone::Retry { + wallet_id, + series, + height, + } => { + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + match entry.retry_series.as_mut() { + Some(retries) if retries.series == series => retries.waiting = false, + // Reset since (a rewind, a switch-off, a new height). + _ => return, + } + // Still at that height: a new block's pass probes the + // unanswered roots anyway (due at the next height). + if entry.height.is_some_and(|now| now != height) { + return; + } + tracing::info!( + wallet_id = %hex::encode(wallet_id), + height, + "broadcast probe: retrying probes no node answered" + ); + let roots = withdraw_unanswered(&mut self.state, wallet_id, height); + // Nothing left to retry (settled since): no pass, and the + // wait stands. + if roots.is_empty() { + return; + } + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + // Due now, though the wait was computed before. + entry.wait_until = None; + if !self.enabled { + return; + } + // A pass of exactly those roots, ahead of the per-pass cap and + // of nothing else: it takes no scheduled roots (they would + // fail too while no node answers) and starts no window. + let pending = entry.pending.get_or_insert_with(PendingPass::default); + pending.height = pending.height.max(height); + pending.retried.extend(roots); + yield_to_forced(entry); + self.start(wallet_id); + } + JobDone::Probed { + wallet_id, + run, + root: probed, + outcome, + } => { + let Some(current) = self + .wallets + .get_mut(&wallet_id) + .and_then(|entry| entry.run.as_mut()) + .filter(|current| current.id == run) + else { + return; // a dropped run's result + }; + // Cannot happen: a probe job is spawned only by a probing run + // with its root in flight. Fail safe — end the run rather than + // leave it blocking every later pass for the wallet. + let Some(probing) = current.probing.as_mut() else { + tracing::error!( + wallet_id = %hex::encode(wallet_id), + "broadcast probe: a probe result for a run that is not probing" + ); + current.abort.abort(); + self.end_run(wallet_id); + return; + }; + if probing + .in_flight + .as_ref() + .is_some_and(|root| root.txid != probed) + { + // The stopped probe of a settled root, finished before it + // could be stopped: not the root in flight now. + return; + } + let Some(root) = probing.in_flight.take() else { + // Cannot happen: a settle that clears the root in flight + // re-arms the run or ends it at once. Log it and carry on + // with the roots still due, forced ones included. + tracing::error!( + wallet_id = %hex::encode(wallet_id), + "broadcast probe: a probe result with no root in flight" + ); + self.probe_next(wallet_id); + return; + }; + let Some(ProbeOutcome { verdict, answered }) = outcome else { + // A removal through this manager queues `WalletRemoved` + // before it lets the probe see this, and that drops the + // run: all but a registration removed or replaced behind + // the manager (`wallet_manager_arc`) end there. + tracing::warn!( + wallet_id = %hex::encode(wallet_id), + txid = %root.txid, + "broadcast probe: the wallet's registration changed; nothing sent" + ); + self.registration_gone(wallet_id); + return; + }; + if !probing.views.iter().any(|view| view.txid == root.txid) { + // Settled or gone while its probe was out: moot. + self.probe_next(wallet_id); + return; + } + let height = probing.height; + let events = record_probe(&mut self.state, wallet_id, height, &root, &verdict); + deliver(&self.sink, events); + mark_answered(&mut self.state, wallet_id, root.txid, answered, height); + if !answered { + self.arm_no_answer_retry(wallet_id, root.txid, height); + } + self.probe_next(wallet_id); + } + } + } + + /// The wallet stepped to `height` over a gap: passes queued or running + /// from before are confined to what an `Uncertain` result forced, and + /// start no windows. The scan settles most roots, and its events clear + /// their verdicts (see `Command::Settled`); a pass at the tip follows — + /// the sync's completion runs it, or this step does when the wallet + /// `arrived` at a tip whose completion came first. On such an arrival a + /// pass already anchored at this tip stays as it is: only one anchored + /// before the gap would start its roots' windows at a stale height. + fn confine_to_forced(&mut self, wallet_id: WalletId, height: u32, arrived: bool) { + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + let stays = |anchor: Option| arrived && anchor.is_some_and(|at| near_tip(at, height)); + if let Some(pending) = entry.pending.as_mut() { + if !stays(pending.anchor_at) { + pending.anchor_at = None; + // A rewind replays lower heights: never run above them. + pending.height = height; + if pending.forced.is_empty() { + entry.pending = None; + } + } + } + let Some(run) = entry.run.as_mut() else { + // A rewind stopped the run: start what it carried. An arrival + // queues its own pass next. + if !arrived { + self.start(wallet_id); + } + return; + }; + if stays(run.anchor_at) { + return; + } + run.anchor_at = None; + run.walk.store(false, Ordering::SeqCst); + if let Some(probing) = run.probing.as_mut() { + probing.drop_scheduled(); + } + // Still reading, with nothing forced: nothing it could probe — stop + // its walk over the records now. + if run.probing.is_none() && run.forced.is_empty() { + run.abort.abort(); + self.end_run(wallet_id); + } + } + + /// A height-driven pass of a wallet that follows the tip at `height`: + /// queued unless a pass already runs (its end decides) or no root can be + /// due yet. + fn queue_followed_pass(&mut self, wallet_id: WalletId, height: u32) { + let enabled = self.enabled; + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + // While a pass runs, its end decides the wait: queue the height and + // let `probe_next` drop it if no root turns out due. + if !enabled || (entry.run.is_none() && entry.wait_until.is_some_and(|until| height < until)) + { + return; + } + let pending = entry.pending.get_or_insert_with(PendingPass::default); + pending.height = pending.height.max(height); + pending.anchor_at = Some(height); + self.start(wallet_id); + } + + /// A probe of `root` at `height` reached no node: nothing was learned. + /// The root is due again at the next height, in any phase of its + /// schedule (see [`ProbeSchedule::unanswered`]) — the next block probes + /// it — and at this height only a retry does: after [`NO_ANSWER_RETRY`], + /// then twice and four times that again (about 1, 3 and 7 minutes after + /// the first probe), at most [`MAX_NO_ANSWER_RETRIES`] per height, while + /// no block has come. A retry is a pass of exactly those roots (still + /// roots), ahead of the per-pass cap: it takes no scheduled roots and + /// starts no window. While DAPI + /// stays down, an unanswered root is due once at every block (in its turn + /// under [`MAX_ROOTS_PER_PASS`] when a backlog is larger) plus the + /// retries, which go ahead of the cap, until the next block comes. One info line per such root and + /// probe — an unchanged Unresolved publishes nothing otherwise. + fn arm_no_answer_retry(&mut self, wallet_id: WalletId, root: Txid, height: u32) { + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + let retries = match entry.retry_series { + Some(retries) if retries.height == height => retries, + _ => { + self.next_retry_series += 1; + NoAnswerRetries { + series: self.next_retry_series, + height, + armed: 0, + waiting: false, + } + } + }; + entry.retry_series = Some(retries); + if retries.waiting { + tracing::info!( + wallet_id = %hex::encode(wallet_id), + txid = %root, + height, + "broadcast probe: no node answered; the waiting retry covers it" + ); + return; + } + if retries.armed >= MAX_NO_ANSWER_RETRIES { + tracing::info!( + wallet_id = %hex::encode(wallet_id), + txid = %root, + height, + "broadcast probe: no node answered; retries at this height used up, \ + waiting for the next block" + ); + return; + } + let delay = NO_ANSWER_RETRY * 2u32.pow(retries.armed); + entry.retry_series = Some(NoAnswerRetries { + armed: retries.armed + 1, + waiting: true, + ..retries + }); + let series = retries.series; + tracing::info!( + wallet_id = %hex::encode(wallet_id), + txid = %root, + height, + "broadcast probe: no node answered; probing again in {}s", + delay.as_secs() + ); + self.jobs.spawn(async move { + tokio::time::sleep(delay).await; + JobDone::Retry { + wallet_id, + series, + height, + } + }); + } + + /// Probe the run's next due root, or end the run. + fn probe_next(&mut self, wallet_id: WalletId) { + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + let Some(run) = entry.run.as_mut() else { + return; + }; + let Some(probing) = run.probing.as_mut() else { + return; + }; + if let Some(root) = probing.due.pop_front() { + let probe = Arc::clone(&self.probe); + let source = Arc::clone(&self.source); + let generation = Arc::clone(&probing.generation); + let id = run.id; + let transaction = Arc::clone(&root.transaction); + let root_txid = root.txid; + run.abort = self.jobs.spawn(async move { + let outcome = gated_probe( + source.as_ref(), + probe.as_ref(), + wallet_id, + &generation, + &transaction, + ) + .await; + JobDone::Probed { + wallet_id, + run: id, + root: root_txid, + outcome, + } + }); + self.job_runs + .insert(run.abort.id(), (wallet_id, id, Some(root_txid))); + mark_sent( + &mut self.state, + wallet_id, + root.txid, + probing.height, + run.anchor_at, + ); + probing.in_flight = Some(root); + return; + } + entry.wait_until = + (!run.woken).then(|| next_pass_height(&self.state, wallet_id, &probing.views)); + self.end_run(wallet_id); + } + + /// End the wallet's run and start what is queued — except a height + /// queued during the run that no root is due at; a forced request always + /// runs. + fn end_run(&mut self, wallet_id: WalletId) { + let Some(entry) = self.wallets.get_mut(&wallet_id) else { + return; + }; + entry.run = None; + if let (Some(pending), Some(until)) = (&entry.pending, entry.wait_until) { + if pending.forced.is_empty() && pending.height < until { + entry.pending = None; + } + } + self.start(wallet_id); + } +} + +enum ActorSlot { + /// Waiting for the first command sent from inside a Tokio runtime. + NotStarted(Box, mpsc::UnboundedReceiver), + Running(JoinHandle<()>, oneshot::Sender<()>), + /// Told to stop but not yet ended within a budget: a retried stop joins + /// it again rather than reporting it gone. + Stopping(JoinHandle<()>), + Closed, +} + +/// Probes unconfirmed sends and delivers each change of a send's verdict as +/// [`PlatformEventHandler::on_outgoing_transaction_probed`], and each verdict +/// the host must drop as +/// [`PlatformEventHandler::on_outgoing_transaction_cleared`] — always from +/// the resolver's own task, asynchronously to whoever caused it. +/// +/// Off until the host turns it on with [`set_enabled`](Self::set_enabled): a +/// probe sends the signed transaction to evonodes over DAPI, which is a +/// product decision, not something a wallet should start doing on upgrade. +pub(crate) struct BroadcastResolver { + enabled: AtomicBool, + /// Held while the flag is stored and the switch is queued, so the flag + /// and the actor end in the same state under concurrent calls — event + /// handlers gate on the flag. + switching: Mutex<()>, + commands: mpsc::UnboundedSender, + /// The actor has been started (or the resolver closed): `send` no longer + /// needs the slot. + started: AtomicBool, + actor: Mutex, + /// Serialises [`stop_within`](Self::stop_within) calls. + stopping: AsyncMutex<()>, + sink: Arc, +} + +impl BroadcastResolver { + pub(crate) fn new( + probe: Arc, + wallet_manager: Arc>>, + ) -> Self { + let sink = Arc::new(EventSink::default()); + let actor = Actor::new( + probe, + Arc::new(ManagerSource(wallet_manager)), + Arc::clone(&sink) as Arc, + ); + let (commands, receiver) = mpsc::unbounded_channel(); + Self { + enabled: AtomicBool::new(false), + commands, + started: AtomicBool::new(false), + actor: Mutex::new(ActorSlot::NotStarted(Box::new(actor), receiver)), + stopping: AsyncMutex::new(()), + switching: Mutex::new(()), + sink, + } + } + + /// Where verdicts are delivered. Weak: the event manager holds this + /// resolver as one of its handlers. + pub(crate) fn set_event_manager(&self, events: Weak) { + *self.sink.0.lock().unwrap_or_else(PoisonError::into_inner) = events; + } + + fn send(&self, command: Command) { + // Fails only once the actor is gone (shutdown); nothing to do then. + let _ = self.commands.send(command); + if self.started.load(Ordering::Acquire) { + return; + } + let Ok(runtime) = Handle::try_current() else { + return; // queued; started by the first command sent from a runtime + }; + let mut slot = self.actor.lock().unwrap_or_else(PoisonError::into_inner); + if matches!(*slot, ActorSlot::NotStarted(..)) { + if let ActorSlot::NotStarted(actor, receiver) = + mem::replace(&mut *slot, ActorSlot::Closed) + { + let (stop, stopped) = oneshot::channel(); + *slot = ActorSlot::Running(runtime.spawn(actor.run(receiver, stopped)), stop); + } + } + self.started.store(true, Ordering::Release); + } + + /// Turn probing on or off. Turning it off forgets every send and has the + /// host told to drop every verdict it holds — with no passes running, + /// nothing would ever clear them — and drops the passes in flight. Either + /// way every wallet is woken, so turning it back on re-publishes what was + /// cleared. Returns at once; the clears follow from the resolver's task. + pub(crate) fn set_enabled(&self, enabled: bool) { + let _switching = self + .switching + .lock() + .unwrap_or_else(PoisonError::into_inner); + self.enabled.store(enabled, Ordering::SeqCst); + self.send(Command::SetEnabled(enabled)); + } + + /// The last value passed to [`set_enabled`](Self::set_enabled). + pub(crate) fn is_enabled(&self) -> bool { + self.enabled.load(Ordering::SeqCst) + } + + /// Wait until the resolver's task has handled everything sent to it + /// before this call, the host callbacks that made included, for at most + /// `budget`. Whether it did; `true` also when the task is gone, since + /// then nothing is published any more. + /// + /// After a removal ([`wallet_removed`](Self::wallet_removed)) this is + /// what makes "no verdict for the removed wallet from now on" true: the + /// removal is only queued, and the task could be publishing a verdict of + /// that wallet's at that moment. The task handles one thing at a time and + /// calls the host from itself, so once it has reached this barrier the + /// removal is behind it and so is every callback from before. Not to be + /// awaited under the wallet-manager lock: the task's reads take it. + /// + /// What it does not cover: a handler that holds the task longer than + /// `budget` (`false`), and a wallet registered under the same id while + /// this waits, whose first verdicts can be delivered before the answer. + pub(crate) async fn caught_up(&self, budget: Duration) -> bool { + let (reply, answer) = oneshot::channel(); + self.send(Command::Barrier(reply)); + // Not timed out: answered, or the task is gone. + tokio::time::timeout(budget, answer).await.is_ok() + } + + /// Whether the actor holds the wallet idle (see [`Command::IsIdle`]). + /// `false` once the actor is gone. + #[cfg(test)] + pub(crate) async fn is_idle(&self, wallet_id: &WalletId) -> bool { + let (reply, answer) = oneshot::channel(); + self.send(Command::IsIdle { + wallet_id: *wallet_id, + reply, + }); + answer.await.unwrap_or(false) + } + + /// Forget a removed wallet — call after it has left the wallet manager: + /// its pass in flight, its entry and every verdict, each of which the host + /// is told to drop (from the resolver's task, not the caller's). + pub(crate) fn wallet_removed(&self, wallet_id: &WalletId) { + self.send(Command::WalletRemoved(*wallet_id)); + } + + /// A wallet was registered or loaded — call after it is in the manager. + pub(crate) fn wallet_added(&self, wallet_id: &WalletId) { + self.send(Command::WalletAdded(*wallet_id)); + } + + /// Stop the resolver's task: it aborts every read and probe it started and + /// waits for them, all within `budget`; after this no host callback comes + /// from the resolver. `NotRunning` if it never started or was already + /// stopped; `Timeout` if it did not end in time (a later call waits for + /// it again); `Panicked` if it had died. Concurrent calls take turns. + pub(crate) async fn stop_within(&self, budget: Duration) -> WorkerStatus { + self.started.store(true, Ordering::Release); + stop_slot(&self.actor, &self.stopping, budget).await + } +} + +/// See [`BroadcastResolver::stop_within`]. +async fn stop_slot( + actor: &Mutex, + stopping: &AsyncMutex<()>, + budget: Duration, +) -> WorkerStatus { + let deadline = Instant::now() + budget; + // One stop at a time: a second caller waits for the first instead of + // finding the slot emptied and reporting a live task as not running. + let Ok(_one_at_a_time) = timeout_at(deadline, stopping.lock()).await else { + return WorkerStatus::Timeout; + }; + let slot = mem::replace( + &mut *actor.lock().unwrap_or_else(PoisonError::into_inner), + ActorSlot::Closed, + ); + let handle = match slot { + ActorSlot::Running(handle, stop) => { + let _ = stop.send(()); + handle + } + ActorSlot::Stopping(handle) => handle, + ActorSlot::NotStarted(..) | ActorSlot::Closed => return WorkerStatus::NotRunning, + }; + // Put back unless the task is joined here: on a timeout, and also when the + // caller drops this future mid-wait — otherwise the only handle goes with + // it and a retried stop would report a live task as not running. + let mut held = HeldActor { + actor, + handle: Some(handle), + }; + let Some(handle) = held.handle.as_mut() else { + return WorkerStatus::NotRunning; + }; + let joined = timeout_at(deadline, handle).await; + match joined { + Ok(result) => { + held.handle = None; + match result { + Ok(()) => WorkerStatus::Ok, + Err(error) if error.is_panic() => WorkerStatus::Panicked(error.to_string()), + Err(error) => WorkerStatus::Stopped(Some(error.to_string())), + } + } + // Told to stop, still going — likely inside a host callback. Not + // aborted: `held` puts it back, and a retried stop waits for it to end + // on its own and can then report it clean. + Err(_) => WorkerStatus::Timeout, + } +} + +/// The resolver task's handle while a stop waits for it. Dropped with the +/// handle still in it — the wait timed out, or the stopping future was +/// cancelled — it goes back into the slot as [`ActorSlot::Stopping`]. +struct HeldActor<'a> { + actor: &'a Mutex, + handle: Option>, +} + +impl Drop for HeldActor<'_> { + fn drop(&mut self) { + if let Some(handle) = self.handle.take() { + *self.actor.lock().unwrap_or_else(PoisonError::into_inner) = + ActorSlot::Stopping(handle); + } + } +} + +/// Whether a wallet event may have left something new to probe. A chain lock +/// (it only promotes records their block already reported), or a block that +/// only matured coins or moved balances, does not. +fn may_leave_work(event: &WalletEvent) -> bool { + match event { + WalletEvent::TransactionDetected { .. } + | WalletEvent::TransactionInstantLocked { .. } + | WalletEvent::TransactionsSwept { .. } => true, + // A fork removed records from the blocks above its height, or moved + // them back to the mempool: a send built on one of them is unsettled + // again. Only the lossless channel delivers it (`recorded_work`): + // dash-spv stops the client before truncating and subscribes again + // after, so the fan-out never sees it. The restarted client then + // usually resumes with a forward step, and verdicts stand. + WalletEvent::ChainTruncated { + txids, + unconfirmed_txids, + .. + } => !txids.is_empty() || !unconfirmed_txids.is_empty(), + WalletEvent::BlockProcessed { + inserted, updated, .. + } => !inserted.is_empty() || !updated.is_empty(), + // A chain lock only promotes records their block already reported. + WalletEvent::ChainLockProcessed { .. } | WalletEvent::SyncHeightAdvanced { .. } => false, + } +} + +/// The transactions a wallet event reports settled — in a block or +/// InstantSend-locked — or gone from the wallet (swept). A chain lock adds +/// nothing: it only promotes records their block already reported. +fn settled_or_gone(event: &WalletEvent) -> HashSet { + match event { + WalletEvent::TransactionDetected { record, .. } => is_settled(record) + .then_some(record.txid) + .into_iter() + .collect(), + WalletEvent::TransactionInstantLocked { txid, .. } => HashSet::from([*txid]), + WalletEvent::TransactionsSwept { txids, .. } => txids.iter().copied().collect(), + // Never on the fan-out (see `may_leave_work`), and nothing to clear if + // it were: `txids` were in blocks, so already settled, and + // `unconfirmed_txids` are back in the mempool, unsettled. + WalletEvent::ChainTruncated { .. } => HashSet::new(), + WalletEvent::BlockProcessed { + inserted, updated, .. + } => inserted + .iter() + .chain(updated) + .filter(|record| is_settled(record)) + .map(|record| record.txid) + .collect(), + // A chain lock only promotes records already in a block — reported + // settled when their block was processed. + WalletEvent::ChainLockProcessed { .. } | WalletEvent::SyncHeightAdvanced { .. } => { + HashSet::new() + } + } +} + +fn is_settled(record: &TransactionRecord) -> bool { + record.is_confirmed() || matches!(record.context, TransactionContext::InstantSend(_)) +} + +impl BroadcastResolver { + /// Tell the actor which sends `event` settled or removed. + fn send_settled(&self, event: &WalletEvent) { + // While probing is off nothing is published, so there is nothing to + // clear (turning it off cleared everything). + if !self.is_enabled() { + return; + } + let settled = settled_or_gone(event); + if !settled.is_empty() { + self.send(Command::Settled { + wallet_id: event.wallet_id(), + txids: settled, + }); + } + } + + /// A wallet event as the lossless persistence channel delivered it (see + /// `spawn_observed_wallet_event_adapter`), next to the same event from + /// dash-spv's fan-out ([`EventHandler::on_wallet_event`]). + /// + /// The fan-out is timely but can lose events: it ends when a receiver + /// lags, and the client stops until the host restarts it. An idle wallet + /// learns of a new unsettled send only from a wallet event, so one lost + /// there would leave the send unread until it settles. This channel loses + /// none, at the price of running behind the persister's commits, so it + /// repeats only what is harmless to hear late or twice: that the wallet's + /// records changed, which makes its next pass read it — and a read also + /// clears what settled. Heard late, it costs one more read of the wallet + /// at the next block, and no probe. What settled is not repeated by name: named late, + /// after a reorg or a reinstatement brought the send back, it would clear + /// a send the wallet still holds. Heights stay with the fan-out: a + /// missed one is made up by the next. + pub(crate) fn on_recorded_wallet_event(&self, event: &WalletEvent) { + if let Some(command) = recorded_work(event, self.is_enabled()) { + self.send(command); + } + } +} + +/// What a wallet event from the lossless channel tells the actor: that the +/// wallet's records changed, for the events that change them, so its next +/// pass reads it. Nothing for the rest, heights included, and nothing while +/// probing is off (turning it on reads every wallet). +fn recorded_work(event: &WalletEvent, enabled: bool) -> Option { + (enabled && may_leave_work(event)).then(|| Command::Seen { + wallet_id: event.wallet_id(), + touched: true, + }) +} + +impl EventHandler for BroadcastResolver { + fn on_wallet_event(&self, event: &WalletEvent) { + let wallet_id = event.wallet_id(); + self.send_settled(event); + let command = match event { + WalletEvent::SyncHeightAdvanced { height, .. } => Command::Height { + wallet_id, + height: *height, + }, + other => Command::Seen { + wallet_id, + touched: may_leave_work(other), + }, + }; + self.send(command); + } + + fn on_sync_event(&self, event: &SyncEvent) { + if let Some(command) = sync_command(event) { + self.send(command); + } + } +} + +/// The command an SPV sync event means for the resolver, if any. +fn sync_command(event: &SyncEvent) -> Option { + match event { + SyncEvent::TransactionBroadcastResult { + txid, + result: BroadcastResult::Uncertain, + } => Some(Command::Uncertain(*txid)), + // dash-spv lets a send it reported Uncertain still turn Accepted when + // a peer echoes it later. + SyncEvent::TransactionBroadcastResult { + txid, + result: BroadcastResult::Accepted { .. }, + } => Some(Command::Echoed(*txid)), + // A tip of 0 is dash-spv's "unknown": no height to follow. + SyncEvent::SyncComplete { header_tip, .. } if *header_tip > 0 => { + Some(Command::SyncComplete { tip: *header_tip }) + } + _ => None, + } +} + +impl PlatformEventHandler for BroadcastResolver {} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::Mutex as StdMutex; + + use async_trait::async_trait; + use dashcore::bls_sig_utils::BLSSignature; + use dashcore::ephemerealdata::chain_lock::ChainLock; + use dashcore::hashes::Hash; + use dashcore::Address; + use dashcore::{BlockHash, OutPoint, TxIn}; + use key_wallet::account::{AccountType, StandardAccountType}; + use key_wallet::managed_account::managed_account_trait::ManagedAccountTrait; + use key_wallet::managed_account::transaction_record::{ + InputDetail, TransactionDirection, TransactionRecord, + }; + use key_wallet::transaction_checking::transaction_router::TransactionType; + use key_wallet::transaction_checking::BlockInfo; + use key_wallet::Network; + use key_wallet::WalletCoreBalance; + use tokio::sync::Notify; + use tokio::time::timeout; + + use super::*; + use crate::test_support::test_platform_wallet_manager; + + fn txid(n: u8) -> Txid { + Txid::from_byte_array([n; 32]) + } + + fn outpoint(n: u8, vout: u32) -> OutPoint { + OutPoint { + txid: txid(n), + vout, + } + } + + /// Fixtures key transactions by `lock_time` == txid byte, so a probe can + /// tell which one it was handed. + fn tx(n: u8, inputs: &[OutPoint]) -> Transaction { + Transaction { + version: 2, + lock_time: n as u32, + input: inputs + .iter() + .map(|previous_output| TxIn { + previous_output: *previous_output, + ..Default::default() + }) + .collect(), + output: Vec::new(), + special_transaction_payload: None, + } + } + + fn send(n: u8, inputs: &[OutPoint]) -> OutgoingView { + OutgoingView { + txid: txid(n), + spends_own_coins: true, + transaction: Arc::new(tx(n, inputs)), + } + } + + fn incoming(n: u8, inputs: &[OutPoint]) -> OutgoingView { + OutgoingView { + spends_own_coins: false, + ..send(n, inputs) + } + } + + fn roots(views: &[OutgoingView]) -> BTreeSet { + ambiguous_roots(views).into_iter().map(|v| v.txid).collect() + } + + fn none() -> HashSet { + HashSet::new() + } + + // ---- merge_records ------------------------------------------------------ + + fn facts(n: u8, settled: bool, own: bool, transaction: &Transaction) -> RecordFacts<'_> { + RecordFacts { + txid: txid(n), + settled, + own, + transaction, + } + } + + /// A transfer between the wallet's own accounts is recorded twice: as + /// incoming in the receiving account (listed first) and as spending in the + /// sending one. It is the wallet's own send either way. + #[test] + fn should_treat_a_transfer_between_own_accounts_as_the_wallets_send() { + let t = tx(1, &[outpoint(90, 0)]); + let views = merge_records([facts(1, false, false, &t), facts(1, false, true, &t)]); + assert_eq!(views.len(), 1); + assert!(views[0].spends_own_coins); + } + + /// A transaction settled in any account is settled. + #[test] + fn should_drop_a_transaction_settled_in_any_account() { + let t = tx(1, &[outpoint(90, 0)]); + assert!(merge_records([facts(1, false, true, &t), facts(1, true, false, &t)]).is_empty()); + assert!(merge_records([facts(1, true, false, &t), facts(1, false, true, &t)]).is_empty()); + } + + // ---- ownership of a record --------------------------------------------- + + fn standard_account() -> AccountType { + AccountType::Standard { + index: 0, + standard_account_type: StandardAccountType::BIP44Account, + } + } + + /// A contact's watch-only chain: we see its coins but hold no key. + fn contact_account() -> AccountType { + AccountType::DashpayExternalAccount { + index: 0, + user_identity_id: [1u8; 32], + friend_identity_id: [2u8; 32], + } + } + + /// An unconfirmed record of `transaction` in `account` that spends one + /// coin the account tracked (upstream fills `input_details` from the + /// account's own UTXOs, watch-only ones included). + fn spending_record(account: AccountType, transaction: Transaction) -> TransactionRecord { + TransactionRecord::new( + transaction, + account, + TransactionContext::Mempool, + TransactionType::Standard, + TransactionDirection::Outgoing, + vec![InputDetail { + index: 0, + value: 100_000, + address: Address::dummy(Network::Testnet, 3), + }], + Vec::new(), + -100_000, + ) + } + + /// The production holder lookup, on a real wallet: every record holding + /// the txid makes the wallet a holder, and only an unsettled own send + /// (spends the wallet's coins, signed by it, not settled) makes it one an + /// echo could still be news to. + #[tokio::test] + async fn should_tell_holders_from_unsettled_own_holders_in_a_real_wallet() { + let (manager, wallet_id) = test_platform_wallet_manager().await; + let platform_wallet = manager.get_wallet(&wallet_id).await.expect("wallet"); + let wallet_manager = Arc::clone(platform_wallet.wallet_manager()); + { + let mut guard = wallet_manager.write().await; + let (_, info) = guard.get_wallet_and_info_mut(&wallet_id).expect("wallet"); + let records = info + .core_wallet + .accounts + .funds_account_mut(&standard_account()) + .expect("bip44 account") + .transactions_mut(); + let own = spending_record(standard_account(), tx(1, &[outpoint(90, 0)])); + let incoming = record(2, TransactionContext::Mempool); + let mut settled = spending_record(standard_account(), tx(3, &[outpoint(91, 0)])); + settled.context = in_block(); + for record in [own, incoming, settled] { + records.insert(record.txid, record); + } + } + // Records are keyed by the real txid. + let id = |n: u8| tx(n, &[]).txid(); + let (own_txid, incoming_txid, settled_txid) = ( + tx(1, &[outpoint(90, 0)]).txid(), + record(2, TransactionContext::Mempool).txid, + tx(3, &[outpoint(91, 0)]).txid(), + ); + let source = ManagerSource(wallet_manager); + + let own = source.holders(own_txid).await; + assert_eq!((own.wallets, own.own), (vec![wallet_id], vec![wallet_id])); + for not_own in [incoming_txid, settled_txid] { + let holders = source.holders(not_own).await; + assert_eq!(holders.wallets, vec![wallet_id], "held"); + assert!(holders.own.is_empty(), "but not an unsettled own send"); + } + assert!(source.holders(id(4)).await.wallets.is_empty()); + } + + /// The contact spending a payment it received from us: our watch-only + /// copy of its chain records the spend, but it is the contact's send. + /// It gets no own-send verdict and no probe of its own. + #[test] + fn should_not_treat_a_contact_spend_as_the_wallets_send() { + let record = spending_record(contact_account(), tx(1, &[outpoint(90, 0)])); + let facts = record_facts(&record, || false, true); + assert!(!facts.own, "a contact's spend is not the wallet's send"); + + let views = merge_records([record_facts(&record, || false, true)]); + assert_eq!(views.len(), 1); + assert!(!views[0].spends_own_coins); + let graph = ChainGraph::new(&views); + assert!(graph.own_unsettled().is_empty()); + assert!(roots(&views).is_empty(), "nothing of ours depends on it"); + // Neither a scheduled pass nor an `Uncertain` report of it probes it. + let start = begin_pass( + &mut ResolverState::default(), + wallet(), + 100, + &views, + &HashSet::from([record.txid]), + &HashSet::new(), + Some(100), + ); + assert!(start.due.is_empty(), "no probe for a contact's own spend"); + // A late acceptance is news to no wallet. + assert!(!is_unsettled_own([record_facts(&record, || false, true)])); + } + + /// The same spend recorded by a signing account is the wallet's send. + #[test] + fn should_treat_a_spend_from_a_signing_account_as_the_wallets_send() { + let record = spending_record(standard_account(), tx(1, &[outpoint(90, 0)])); + assert!(record_facts(&record, || false, true).own); + assert!(is_unsettled_own([record_facts(&record, || false, true)])); + assert!( + !is_unsettled_own([record_facts(&record, || true, true)]), + "finalized" + ); + let views = merge_records([record_facts(&record, || false, true)]); + assert_eq!(roots(&views), BTreeSet::from([record.txid])); + } + + /// A watch-only wallet signs nothing: a spend of a coin it tracks is + /// someone else's, even in a standard account. + #[test] + fn should_not_treat_a_spend_seen_by_a_watch_only_wallet_as_its_send() { + let record = spending_record(standard_account(), tx(1, &[outpoint(90, 0)])); + assert!(!record_facts(&record, || false, false).own); + assert!(!is_unsettled_own([record_facts(&record, || false, false)])); + assert!(roots(&merge_records([record_facts(&record, || false, false)])).is_empty()); + } + + /// A contact's unsettled spend that one of our sends builds on is still + /// walked as its ancestor: the contact's transaction is the root probed, + /// while only our send can receive a verdict. + #[test] + fn should_walk_a_contact_spend_as_the_ancestor_of_an_own_send() { + let parent = spending_record(contact_account(), tx(5, &[outpoint(80, 0)])); + let child = spending_record( + standard_account(), + tx( + 6, + &[OutPoint { + txid: parent.txid, + vout: 0, + }], + ), + ); + let views = merge_records([ + record_facts(&parent, || false, true), + record_facts(&child, || false, true), + ]); + let graph = ChainGraph::new(&views); + assert_eq!(graph.own_unsettled(), BTreeSet::from([child.txid])); + assert_eq!(roots(&views), BTreeSet::from([parent.txid])); + assert!(!graph.is_own(&parent.txid)); + let start = begin_pass( + &mut ResolverState::default(), + wallet(), + 100, + &views, + &HashSet::new(), + &HashSet::new(), + Some(100), + ); + let due: Vec<(Txid, bool)> = start.due.iter().map(|root| (root.txid, root.own)).collect(); + assert_eq!( + due, + vec![(parent.txid, false)], + "probed, but no verdict of its own" + ); + assert!(!is_unsettled_own([record_facts(&parent, || false, true)])); + assert!(is_unsettled_own([record_facts(&child, || false, true)])); + } + + // ---- ChainGraph --------------------------------------------------------- + + #[test] + fn should_pick_a_lone_unsettled_send() { + assert_eq!( + roots(&[send(1, &[outpoint(90, 0)])]), + BTreeSet::from([txid(1)]) + ); + } + + /// The customer's shape: sends built on the change of an unsettled send. + /// Only the first is probed — a node that has not seen it answers + /// `missingorspent` for the others. + #[test] + fn should_pick_only_the_root_of_a_chain() { + let views = [ + send(1, &[outpoint(90, 0)]), + send(2, &[outpoint(1, 1)]), + send(3, &[outpoint(2, 1)]), + ]; + assert_eq!(roots(&views), BTreeSet::from([txid(1)])); + } + + /// Once the parent settles it is gone from the views, and the child has no + /// unsettled parent: it is a root in its own right. + #[test] + fn should_treat_the_child_of_a_settled_parent_as_a_root() { + assert_eq!( + roots(&[send(2, &[outpoint(1, 1)])]), + BTreeSet::from([txid(2)]) + ); + } + + /// Nodes that have not seen an unconfirmed incoming payment refuse a send + /// that spends it, so the send is not probed itself — the incoming parent + /// is, since its fate decides the send's. + #[test] + fn should_probe_the_unconfirmed_incoming_parent_of_a_send_instead_of_the_send() { + let views = [incoming(5, &[outpoint(80, 0)]), send(6, &[outpoint(5, 0)])]; + assert_eq!(roots(&views), BTreeSet::from([txid(5)])); + } + + /// An unconfirmed incoming payment nobody built on is not this wallet's + /// business to probe. + #[test] + fn should_not_probe_an_incoming_payment_the_wallet_did_not_spend() { + assert!(roots(&[incoming(5, &[outpoint(80, 0)])]).is_empty()); + } + + /// dash-spv reports the uncertain txid, which may be a child: the root of + /// its chain is what must be probed at once. + #[test] + fn should_find_the_root_of_a_forced_child() { + let views = [ + send(1, &[outpoint(90, 0)]), + send(2, &[outpoint(1, 1)]), + send(3, &[outpoint(2, 1)]), + ]; + let graph = ChainGraph::new(&views); + assert_eq!(graph.roots_of([txid(3)]), BTreeSet::from([txid(1)])); + assert!(graph.roots_of([txid(9)]).is_empty(), "unknown txid"); + } + + /// A long chain of the wallet's own unsettled sends — each spending the + /// last one's change — is walked once: every send is expanded exactly one + /// time, not once per send that depends on it. + #[test] + fn should_expand_each_transaction_of_a_long_chain_once() { + let views: Vec = (1..=200u8) + .map(|n| { + if n == 1 { + send(n, &[outpoint(250, 0)]) + } else { + send(n, &[outpoint(n - 1, 1)]) + } + }) + .collect(); + let graph = ChainGraph::new(&views); + let roots: Vec = graph.roots().iter().map(|view| view.txid).collect(); + assert_eq!(roots, vec![txid(1)]); + assert_eq!(graph.expanded.get(), 200); + } + + /// Sends that share ancestry share the walk: 100 chained sends plus 50 + /// sends each built on the last of them expand 150 transactions, and the + /// forced lookup for all of them is one more walk of the same size. + #[test] + fn should_walk_shared_ancestry_once_for_many_sends() { + let mut views: Vec = (1..=100u8) + .map(|n| { + if n == 1 { + send(n, &[outpoint(250, 0)]) + } else { + send(n, &[outpoint(n - 1, 1)]) + } + }) + .collect(); + views.extend((101..=150u8).map(|n| send(n, &[outpoint(100, u32::from(n))]))); + let graph = ChainGraph::new(&views); + let found: BTreeSet = graph.roots().iter().map(|view| view.txid).collect(); + assert_eq!(found, BTreeSet::from([txid(1)])); + assert_eq!(graph.expanded.get(), 150, "roots(): one walk for all sends"); + let before = graph.expanded.get(); + assert_eq!( + graph.roots_of((101..=150u8).map(txid)), + BTreeSet::from([txid(1)]) + ); + assert_eq!(graph.expanded.get() - before, 150); + } + + // ---- ProbeSchedule / catch-up ------------------------------------------ + + fn anchored(height: u32) -> ProbeSchedule { + let mut schedule = ProbeSchedule::default(); + schedule.anchor(height); + schedule + } + + #[test] + fn should_be_due_immediately_when_never_probed() { + assert!(anchored(100).is_due(100)); + } + + #[test] + fn should_probe_at_most_once_per_block() { + let mut s = anchored(100); + s.probed_at(100); + assert!(!s.is_due(100)); + assert!(s.is_due(101)); + } + + #[test] + fn should_slow_down_after_the_every_block_window() { + let first = 100; + let mut s = anchored(first); + let late = first + EVERY_BLOCK_WINDOW; + s.probed_at(late); + for height in late + 1..late + SLOW_INTERVAL { + assert!(!s.is_due(height), "not due at {height}"); + } + assert!(s.is_due(late + SLOW_INTERVAL)); + } + + /// `next_due` is exactly the first height `is_due` accepts. + #[test] + fn should_agree_on_the_next_due_height_with_is_due() { + let first = 100; + let mut schedule = anchored(first); + for probed in [ + first, + first + 5, + first + EVERY_BLOCK_WINDOW - 1, + first + EVERY_BLOCK_WINDOW + 3, + ] { + schedule.probed_at(probed); + let next = schedule.next_due(); + assert!(schedule.is_due(next), "due at {next}"); + for height in probed + 1..next { + assert!(!schedule.is_due(height), "not due at {height}"); + } + } + } + + #[test] + fn should_treat_only_large_height_steps_as_catch_up() { + assert!( + is_catch_up_step(None, 1_000), + "the first step after launch is catch-up" + ); + assert!(!is_catch_up_step(Some(999), 1_000)); + assert!(!is_catch_up_step(Some(1_000 - CATCH_UP_STEP), 1_000)); + assert!(is_catch_up_step(Some(1_000 - CATCH_UP_STEP - 1), 1_000)); + assert!( + is_catch_up_step(Some(1_000), 999), + "a step back is a rescan, not the tip" + ); + assert!(!is_catch_up_step(Some(1_000), 1_000)); + } + + // ---- begin_pass / record_probe ------------------------------------------ + + /// Answers per txid and records every probe it received. + struct ScriptedProbe { + answers: BTreeMap, + probed: StdMutex>, + } + + impl ScriptedProbe { + fn new(answers: &[(Txid, ProbeVerdict)]) -> Self { + Self { + answers: answers.iter().cloned().collect(), + probed: StdMutex::new(Vec::new()), + } + } + + fn probed(&self) -> Vec { + self.probed.lock().expect("probed").clone() + } + } + + #[async_trait] + impl AcceptanceProbe for ScriptedProbe { + async fn probe(&self, transaction: &Transaction) -> ProbeOutcome { + let id = txid(transaction.lock_time as u8); + self.probed.lock().expect("probed").push(id); + outcome( + self.answers + .get(&id) + .cloned() + .unwrap_or(ProbeVerdict::Unresolved { + reason: "scripted".to_string(), + }), + ) + } + } + + fn wallet() -> WalletId { + [7u8; 32] + } + + fn unresolved() -> ProbeVerdict { + ProbeVerdict::Unresolved { + reason: "no quorum".to_string(), + } + } + + /// The reason the fakes give a probe no node answered. + const NO_ANSWER: &str = "no available addresses"; + + /// No node answered at all (all of DAPI unreachable). + fn unreachable_verdict() -> ProbeVerdict { + ProbeVerdict::Unresolved { + reason: NO_ANSWER.to_string(), + } + } + + /// What a fake probe reports for `verdict`: answered, except for + /// [`unreachable_verdict`]. + fn outcome(verdict: ProbeVerdict) -> ProbeOutcome { + let answered = + !matches!(&verdict, ProbeVerdict::Unresolved { reason } if reason == NO_ANSWER); + ProbeOutcome { verdict, answered } + } + + /// The resolver's bookkeeping plus what the host was sent, driven the way + /// the actor drives it but without the task. + #[derive(Default)] + struct Harness { + state: ResolverState, + sent: Vec, + } + + impl Harness { + fn push(&mut self, events: Vec) { + self.sent.extend(events.into_iter().map(|item| item.event)); + } + + fn forget_wallet(&mut self, wallet_id: &WalletId) { + let events = self.state.forget_wallet(wallet_id, "removed"); + self.push(events); + } + + fn forget_all(&mut self) { + let events = self.state.forget_all(); + self.push(events); + } + } + + async fn pass( + state: &Mutex, + probe: &dyn AcceptanceProbe, + height: u32, + views: &[OutgoingView], + ) -> Vec { + run(state, probe, wallet(), height, views, &none()).await; + outbox(state) + } + + /// One pass: begin, probe each due root with no lock held, record. + /// Returns the height the wallet may wait for. + async fn run( + state: &Mutex, + probe: &dyn AcceptanceProbe, + wallet_id: WalletId, + height: u32, + views: &[OutgoingView], + forced: &HashSet, + ) -> u32 { + let due = { + let mut guard = state.lock().expect("state"); + let start = begin_pass( + &mut guard.state, + wallet_id, + height, + views, + forced, + &HashSet::new(), + Some(height), + ); + guard.push(start.events); + start.due + }; + // The same per-root steps as the actor's: mark sent, probe, record. + for root in due { + mark_sent( + &mut state.lock().expect("state").state, + wallet_id, + root.txid, + height, + Some(height), + ); + let verdict = probe.probe(&root.transaction).await.verdict; + let mut guard = state.lock().expect("state"); + let events = record_probe(&mut guard.state, wallet_id, height, &root, &verdict); + guard.push(events); + } + let guard = state.lock().expect("state"); + next_pass_height(&guard.state, wallet_id, views) + } + + fn outbox(state: &Mutex) -> Vec { + std::mem::take(&mut state.lock().expect("state").sent) + } + + #[tokio::test] + async fn should_probe_only_roots_and_report_each_verdict() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), unresolved())]); + let views = [send(1, &[outpoint(90, 0)]), send(2, &[outpoint(1, 1)])]; + + let events = pass(&state, &probe, 100, &views).await; + + assert_eq!(probe.probed(), vec![txid(1)]); + assert_eq!(events, vec![ResolverEvent::Verdict(txid(1), unresolved())]); + } + + /// Acceptance by one node's mempool is not settlement: the root keeps + /// being asked until it settles, but the host hears about it only once. + #[tokio::test] + async fn should_keep_probing_an_accepted_root_but_publish_it_once() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)]); + let views = [send(1, &[outpoint(90, 0)])]; + + let first = pass(&state, &probe, 100, &views).await; + let second = pass(&state, &probe, 101, &views).await; + + assert_eq!( + first, + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + assert!( + second.is_empty(), + "an unchanged verdict is not published again" + ); + assert_eq!(probe.probed(), vec![txid(1), txid(1)]); + } + + #[tokio::test] + async fn should_keep_probing_an_unresolved_root_on_each_new_block() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), unresolved())]); + let views = [send(1, &[outpoint(90, 0)])]; + + let first = pass(&state, &probe, 100, &views).await; + let same_block = pass(&state, &probe, 100, &views).await; + let next_block = pass(&state, &probe, 101, &views).await; + + assert_eq!(first, vec![ResolverEvent::Verdict(txid(1), unresolved())]); + assert!(same_block.is_empty(), "one probe per block"); + assert!( + next_block.is_empty(), + "still unresolved: nothing new to publish" + ); + assert_eq!(probe.probed(), vec![txid(1), txid(1)]); + } + + /// dash-spv's `Uncertain` result makes the root due at once, even inside + /// the same block as an earlier probe. + #[tokio::test] + async fn should_probe_a_forced_root_regardless_of_its_schedule() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), unresolved())]); + let views = [send(1, &[outpoint(90, 0)])]; + + pass(&state, &probe, 100, &views).await; + run( + &state, + &probe, + wallet(), + 100, + &views, + &HashSet::from([txid(1)]), + ) + .await; + + assert_eq!(probe.probed(), vec![txid(1), txid(1)]); + } + + /// The customer's shape: the uncertain send spends the change of an + /// earlier unsettled send. Forcing the child probes its root at once. + #[tokio::test] + async fn should_probe_the_root_when_a_child_is_forced() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), unresolved())]); + let views = [send(1, &[outpoint(90, 0)]), send(2, &[outpoint(1, 1)])]; + + pass(&state, &probe, 100, &views).await; + run( + &state, + &probe, + wallet(), + 100, + &views, + &HashSet::from([txid(2)]), + ) + .await; + + assert_eq!(probe.probed(), vec![txid(1), txid(1)]); + } + + /// A send that settled or left the wallet is forgotten, and the host is + /// told to drop the verdict it was shown. + #[tokio::test] + async fn should_clear_a_published_send_that_is_no_longer_unconfirmed() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)]); + let views = [send(1, &[outpoint(90, 0)])]; + + pass(&state, &probe, 100, &views).await; + let gone = pass(&state, &probe, 101, &[]).await; + + assert_eq!(gone, vec![ResolverEvent::Cleared(txid(1))]); + let guard = state.lock().expect("state"); + assert!(guard.state.schedules.is_empty()); + assert!(guard.state.published.is_empty()); + } + + #[tokio::test] + async fn should_not_let_one_wallets_pass_forget_another_wallets_roots() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), unresolved())]); + let views = [send(1, &[outpoint(90, 0)])]; + + pass(&state, &probe, 100, &views).await; + run(&state, &probe, [9u8; 32], 100, &[], &none()).await; + + assert_eq!(state.lock().expect("state").state.schedules.len(), 1); + } + + /// Answers in turn, then Unresolved; counts the probes. + struct SequenceProbe { + answers: StdMutex>, + calls: StdMutex, + } + + impl SequenceProbe { + fn new(answers: &[ProbeVerdict]) -> Self { + Self { + answers: StdMutex::new(answers.iter().cloned().collect()), + calls: StdMutex::new(0), + } + } + + fn calls(&self) -> usize { + *self.calls.lock().expect("calls") + } + } + + #[async_trait] + impl AcceptanceProbe for SequenceProbe { + async fn probe(&self, _transaction: &Transaction) -> ProbeOutcome { + *self.calls.lock().expect("calls") += 1; + outcome( + self.answers + .lock() + .expect("answers") + .pop_front() + .unwrap_or_else(unresolved), + ) + } + } + + /// Nodes' word that a send is accepted, in a mempool or a block, is not + /// settlement: the root is still probed, and only the wallet settling + /// the send clears it and makes its child the root. + #[tokio::test] + async fn should_clear_an_accepted_root_only_when_the_wallet_settles_it() { + let state = Mutex::new(Harness::default()); + let probe = SequenceProbe::new(&[ProbeVerdict::Accepted, ProbeVerdict::Accepted]); + let views = [send(1, &[outpoint(90, 0)]), send(2, &[outpoint(1, 0)])]; + + assert_eq!( + pass(&state, &probe, 100, &views).await, + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + assert!( + pass(&state, &probe, 101, &views).await.is_empty(), + "a repeat is no news" + ); + assert!(pass(&state, &probe, 102, &views).await.is_empty()); + assert_eq!(probe.calls(), 3, "only the root, on every block"); + + // The wallet settles the root: it is cleared and its child is the + // root now. + let settled = [send(2, &[outpoint(1, 0)])]; + assert_eq!( + pass(&state, &probe, 103, &settled).await, + vec![ + ResolverEvent::Cleared(txid(1)), + ResolverEvent::Verdict(txid(2), unresolved()), + ] + ); + } + + #[tokio::test] + async fn should_publish_nothing_while_an_incoming_parent_is_only_accepted() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(5), ProbeVerdict::Accepted)]); + let views = [incoming(5, &[outpoint(80, 0)]), send(6, &[outpoint(5, 0)])]; + + assert!(pass(&state, &probe, 100, &views).await.is_empty()); + } + + // ---- ResolverState ------------------------------------------------------ + + #[tokio::test] + async fn should_forget_one_wallet_and_queue_a_clear_for_its_published_sends() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), unresolved()), (txid(2), unresolved())]); + pass(&state, &probe, 100, &[send(1, &[outpoint(90, 0)])]).await; + run( + &state, + &probe, + [9u8; 32], + 100, + &[send(2, &[outpoint(91, 0)])], + &none(), + ) + .await; + outbox(&state); + + state.lock().expect("state").forget_wallet(&wallet()); + + assert_eq!(outbox(&state), vec![ResolverEvent::Cleared(txid(1))]); + let guard = state.lock().expect("state"); + assert_eq!( + guard.state.published.len(), + 1, + "the other wallet is untouched" + ); + assert_eq!(guard.state.schedules.len(), 1); + } + + #[tokio::test] + async fn should_forget_everything_and_queue_a_clear_for_every_published_send() { + let state = Mutex::new(Harness::default()); + let probe = ScriptedProbe::new(&[(txid(1), unresolved()), (txid(2), unresolved())]); + pass(&state, &probe, 100, &[send(1, &[outpoint(90, 0)])]).await; + run( + &state, + &probe, + [9u8; 32], + 100, + &[send(2, &[outpoint(91, 0)])], + &none(), + ) + .await; + outbox(&state); + + state.lock().expect("state").forget_all(); + + assert_eq!(outbox(&state).len(), 2); + let guard = state.lock().expect("state"); + assert!(guard.state.published.is_empty() && guard.state.schedules.is_empty()); + } + + /// One probe that met only transport errors says nothing new about a send + /// already seen accepted; the UI must not flicker back to "unknown". A + /// repeat is no news. + #[test] + fn should_publish_only_a_change_of_verdict() { + let mut state = ResolverState::default(); + let key = (wallet(), txid(1)); + + assert!(state.publish(key, &unresolved())); + let other_reason = ProbeVerdict::Unresolved { + reason: "another reason".to_string(), + }; + assert!(!state.publish(key, &other_reason), "still unresolved"); + assert!(state.publish(key, &ProbeVerdict::Accepted)); + assert!(!state.publish(key, &unresolved())); + assert!(!state.publish(key, &ProbeVerdict::Accepted)); + } + + // ---- the actor ---------------------------------------------------------- + + /// Unsettled views per wallet; records every walk over a wallet's records. + #[derive(Default)] + struct FakeSource { + views: StdMutex>>, + walks: StdMutex>, + /// The synced height each wallet's reads report. + synced: StdMutex>, + /// Each wallet's current registration, made at its first use. + generations: StdMutex>>, + /// Reads wait while set, so a test can hold a pass in its read. + paused: StdMutex, + resume: Notify, + /// Holder lookups panic while set. + holders_panic: StdMutex, + } + + impl FakeSource { + async fn wait_if_paused(&self) { + loop { + let resumed = self.resume.notified(); + if !*self.paused.lock().expect("paused") { + return; + } + resumed.await; + } + } + + fn pause(&self, paused: bool) { + *self.paused.lock().expect("paused") = paused; + if !paused { + self.resume.notify_waiters(); + } + } + + /// The wallet's current registration. + fn generation(&self, wallet_id: WalletId) -> Arc { + Arc::clone( + self.generations + .lock() + .expect("generations") + .entry(wallet_id) + .or_insert_with(|| Arc::new(WalletGeneration::new())), + ) + } + + /// Register the wallet anew under the same id, its records kept. + fn reregister(&self, wallet_id: WalletId) { + self.generations + .lock() + .expect("generations") + .insert(wallet_id, Arc::new(WalletGeneration::new())); + } + } + + #[async_trait] + impl WalletSource for FakeSource { + async fn holders(&self, txid: Txid) -> Holders { + if *self.holders_panic.lock().expect("holders_panic") { + panic!("a holder lookup panics"); + } + let views = self.views.lock().expect("views"); + let held = |own: bool| { + views + .iter() + .filter(|(_, views)| { + views + .iter() + .any(|view| view.txid == txid && (!own || view.spends_own_coins)) + }) + .map(|(wallet_id, _)| *wallet_id) + .collect() + }; + Holders { + wallets: held(false), + own: held(true), + } + } + + async fn read( + &self, + wallet_id: WalletId, + walk: &AtomicBool, + forced: HashSet, + ) -> Option> { + self.wait_if_paused().await; + let views = self.views.lock().expect("views").get(&wallet_id)?.clone(); + let forced = forced_held(walk.load(Ordering::SeqCst), forced, |txid| { + views.iter().any(|view| view.txid == *txid) + }); + Some(forced.map(|forced| { + self.walks.lock().expect("walks").push(wallet_id); + WalletRead { + views, + synced_height: self + .synced + .lock() + .expect("synced") + .get(&wallet_id) + .copied() + .unwrap_or(0), + forced, + generation: self.generation(wallet_id), + } + })) + } + + async fn is_registered( + &self, + wallet_id: WalletId, + generation: &Arc, + ) -> bool { + self.views.lock().expect("views").contains_key(&wallet_id) + && Arc::ptr_eq(&self.generation(wallet_id), generation) + } + } + + #[derive(Default)] + struct CollectSink(StdMutex>); + + impl VerdictSink for CollectSink { + fn deliver(&self, item: &Outgoing) { + self.0.lock().expect("sink").push(item.event.clone()); + } + } + + struct Rig { + actor: Actor, + source: Arc, + sink: Arc, + } + + impl Rig { + fn new(probe: Arc) -> Self { + Self::with_sink(probe, Arc::new(CollectSink::default())) + } + + fn with_sink(probe: Arc, sink: Arc) -> Self { + let source = Arc::new(FakeSource::default()); + let actor = Actor::new( + probe, + Arc::clone(&source) as Arc, + Arc::clone(&sink) as Arc, + ); + Self { + actor, + source, + sink, + } + } + + /// The wallet's unsettled records; registers the wallet if the actor + /// does not know it, as the manager does before any of its events. + fn views(&mut self, wallet_id: WalletId, views: Vec) { + self.source + .views + .lock() + .expect("views") + .insert(wallet_id, views); + if !self.actor.wallets.contains_key(&wallet_id) { + self.actor.handle(Command::WalletAdded(wallet_id)); + } + } + + /// Handle `command`, then every job it caused, as the task would. + async fn send(&mut self, command: Command) { + self.handle(command); + self.settle().await; + } + + /// Handle `command` without running its jobs. A height step moves the + /// wallet's synced height along, as the SPV client does before it + /// reports the step. + fn handle(&mut self, command: Command) { + if let Command::Height { wallet_id, height } = command { + let mut synced = self.source.synced.lock().expect("synced"); + let at = synced.entry(wallet_id).or_insert(0); + *at = (*at).max(height); + } + self.actor.handle(command); + } + + async fn settle(&mut self) { + while let Some(joined) = self.actor.jobs.join_next_with_id().await { + self.actor.joined(joined); + } + } + + /// Two height steps: the first after launch is catch-up. + async fn follow(&mut self, wallet_id: WalletId, from: u32) { + self.send(Command::Height { + wallet_id, + height: from, + }) + .await; + self.send(Command::Height { + wallet_id, + height: from + 1, + }) + .await; + } + + async fn height(&mut self, wallet_id: WalletId, height: u32) { + self.send(Command::Height { wallet_id, height }).await; + } + + fn sent(&self) -> Vec { + std::mem::take(&mut *self.sink.0.lock().expect("sink")) + } + + fn walks(&self) -> usize { + self.source.walks.lock().expect("walks").len() + } + } + + async fn enabled(probe: Arc) -> Rig { + let mut rig = Rig::new(probe); + rig.send(Command::SetEnabled(true)).await; + rig + } + + #[tokio::test] + async fn should_ignore_heights_and_uncertain_results_while_off() { + let mut rig = Rig::new(Arc::new(ScriptedProbe::new(&[]))); + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.follow(wallet(), 100).await; + rig.send(Command::Uncertain(txid(1))).await; + + assert_eq!(rig.walks(), 0); + assert!(rig.sent().is_empty()); + } + + /// An `Uncertain` result carries no wallet id: every wallet hears it, but + /// only the one holding the transaction walks its records. + #[tokio::test] + async fn should_act_on_an_uncertain_result_only_in_the_wallet_that_holds_it() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + let other = [9u8; 32]; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.views(other, vec![send(2, &[outpoint(91, 0)])]); + for wallet_id in [wallet(), other] { + rig.send(Command::Seen { + wallet_id, + touched: false, + }) + .await; + } + + rig.send(Command::Uncertain(txid(1))).await; + + assert_eq!(*rig.source.walks.lock().expect("walks"), vec![wallet()]); + assert_eq!(probe.probed(), vec![txid(1)]); + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), unresolved())] + ); + } + + /// An `Uncertain` result no registered wallet holds (an unrelated + /// broadcast) is forgotten once its holder lookup comes back empty. + #[tokio::test] + async fn should_forget_an_uncertain_result_no_wallet_holds() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.send(Command::Uncertain(txid(7))).await; + + assert!(rig.actor.uncertain.is_empty()); + } + + /// An echo within [`UNCERTAIN_EXPIRY`] followed blocks of the window an + /// `Uncertain` result starts (at the next followed block) still makes an + /// own send Accepted; one after it is ignored — the probes decide by then. + #[tokio::test] + async fn should_accept_an_echo_only_within_the_expiry() { + for (blocks, accepted) in [(UNCERTAIN_EXPIRY - 1, true), (UNCERTAIN_EXPIRY, false)] { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; // at 101 + rig.send(Command::Uncertain(txid(1))).await; + // The window starts at 102. + for height in 102..=102 + blocks { + rig.height(wallet(), height).await; + } + rig.sent(); + + rig.send(Command::Echoed(txid(1))).await; + + assert_eq!( + rig.sent() + .contains(&ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)), + accepted, + "{blocks} blocks into the window" + ); + } + } + + /// The window starts only once a wallet follows the tip, and runs on the + /// highest followed height: catch-up steps and a wallet behind the + /// others neither start nor end it. + #[tokio::test] + async fn should_run_the_echo_window_on_the_followed_tip_only() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + let behind = [9u8; 32]; + rig.send(Command::WalletAdded(behind)).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + // Reported while nothing follows: no window yet. + rig.send(Command::Uncertain(txid(1))).await; + assert_eq!(rig.actor.uncertain.get(&txid(1)), Some(&None)); + + // Catch-up steps (the first step, then a jump) start nothing. + rig.height(wallet(), 500).await; + rig.height(wallet(), 1000).await; + assert_eq!(rig.actor.uncertain.get(&txid(1)), Some(&None)); + + // The first followed block starts it. + rig.height(wallet(), 1001).await; + assert_eq!(rig.actor.uncertain.get(&txid(1)), Some(&Some(1001))); + + // A wallet far behind, following its own (lower) steps, changes + // nothing; nor does a catch-up jump far ahead. + rig.follow(behind, 600).await; + rig.height(wallet(), 1001 + UNCERTAIN_EXPIRY + 50).await; + assert!(rig.actor.uncertain.contains_key(&txid(1))); + + // The tip followed past the window ends it. + rig.height(wallet(), 1001 + UNCERTAIN_EXPIRY + 51).await; + assert!(rig.actor.uncertain.is_empty()); + } + + /// A new `Uncertain` report of the same send restarts its echo window + /// from the next followed block. + #[tokio::test] + async fn should_restart_the_echo_window_on_a_new_report() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.send(Command::Uncertain(txid(1))).await; + rig.height(wallet(), 102).await; + assert_eq!(rig.actor.uncertain.get(&txid(1)), Some(&Some(102))); + + rig.send(Command::Uncertain(txid(1))).await; + rig.height(wallet(), 103).await; + + assert_eq!(rig.actor.uncertain.get(&txid(1)), Some(&Some(103))); + } + + /// A wallet that jumped ahead (catch-up) holds the clock there: another + /// wallet still scanning far behind, in small steps, neither starts nor + /// ends an echo window. + #[tokio::test] + async fn should_not_let_a_wallet_behind_a_catch_up_run_the_echo_window() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + let behind = [9u8; 32]; + rig.send(Command::WalletAdded(behind)).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 1000).await; // follows at 1001 + rig.height(wallet(), 5000).await; // catch-up jump + rig.send(Command::Uncertain(txid(1))).await; + + rig.follow(behind, 1500).await; + for height in 1502..1502 + 2 * UNCERTAIN_EXPIRY { + rig.height(behind, height).await; + } + + assert_eq!(rig.actor.uncertain.get(&txid(1)), Some(&None)); + rig.height(wallet(), 5001).await; + assert_eq!(rig.actor.uncertain.get(&txid(1)), Some(&Some(5001))); + } + + /// A send a wallet holds but not as an unsettled own send (an incoming + /// payment): its echo would be news to no one, so the entry goes — and + /// the holder still probes its chain. + #[tokio::test] + async fn should_drop_an_uncertain_result_held_only_as_incoming_but_still_probe_it() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![incoming(1, &[outpoint(90, 0)]), send(2, &[outpoint(1, 0)])], + ); + + rig.send(Command::Uncertain(txid(1))).await; + + assert!(rig.actor.uncertain.is_empty()); + assert_eq!(probe.probed(), vec![txid(1)]); + } + + fn sync_complete(tip: u32) -> Command { + Command::SyncComplete { tip } + } + + /// The log's first shape: the app launches on a stored tip, the network + /// comes back with one short step (the first after launch, so catch-up) + /// and then stays quiet for minutes. The sync's completion runs the pass + /// the catch-up skipped — once: a later completion at the same height, + /// or after a followed step, probes nothing more. + #[tokio::test] + async fn should_probe_at_the_end_of_a_catch_up_then_wait_for_the_next_block() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.height(wallet(), 1_566_577).await; // first step after launch + assert!(probe.probed().is_empty(), "catch-up: no pass"); + rig.send(sync_complete(1_566_577)).await; + assert_eq!( + probe.probed(), + vec![txid(1)], + "the skipped pass, at the tip" + ); + + rig.send(sync_complete(1_566_577)).await; + rig.height(wallet(), 1_566_578).await; // followed + rig.send(sync_complete(1_566_578)).await; + assert_eq!(probe.probed(), vec![txid(1), txid(1)], "one per block"); + } + + /// A long catch-up (a reconnect after an outage) ends the same way. + #[tokio::test] + async fn should_probe_at_the_end_of_a_long_catch_up() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + let before = probe.probed().len(); + + rig.height(wallet(), 150).await; // catch-up + assert_eq!(probe.probed().len(), before); + rig.send(sync_complete(150)).await; + + assert_eq!(probe.probed().len(), before + 1); + } + + /// A launch with no block since the app last ran gives the wallet no + /// height step at all; the first completion probes it at the tip. + #[tokio::test] + async fn should_probe_a_wallet_with_no_step_since_launch_at_the_first_completion() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + // The wallet reads its stored tip, already the network's. + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 1_566_577); + + rig.send(sync_complete(1_566_577)).await; + + assert_eq!(probe.probed(), vec![txid(1)]); + } + + /// A wallet still rescanning far below the tip gets no pass from a + /// completion: its windows must not start at a stale height. + #[tokio::test] + async fn should_not_probe_a_wallet_far_below_a_completed_tip() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.height(wallet(), 500).await; // catch-up, far below + + rig.send(sync_complete(5_000)).await; + + assert!(probe.probed().is_empty()); + } + + /// The log's second shape: one new block arrives as the network comes + /// back, but DAPI's addresses are still banned, so its probe reaches no + /// node. It is repeated after [`NO_ANSWER_RETRY`] rather than at the next + /// block, minutes away. + #[tokio::test(start_paused = true)] + async fn should_retry_a_probe_no_node_answered_without_waiting_for_a_block() { + let probe = Arc::new(SequenceProbe::new(&[ + unreachable_verdict(), + ProbeVerdict::Accepted, + ])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.follow(wallet(), 100).await; // probes at 101; the retry follows + + assert_eq!(probe.calls(), 2); + assert!(rig + .sent() + .contains(&ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted))); + } + + /// Every root no node answered in a pass is retried, by one timer. + #[tokio::test(start_paused = true)] + async fn should_retry_every_root_no_node_answered() { + let probe = Arc::new(ScriptedProbe::new(&[ + (txid(1), unreachable_verdict()), + (txid(2), unreachable_verdict()), + ])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + + rig.follow(wallet(), 100).await; + + // Each retry probes both: the probe and its retries, for each root. + let probed = probe.probed(); + let each = 1 + MAX_NO_ANSWER_RETRIES as usize; + assert_eq!( + probed.iter().filter(|probed| **probed == txid(1)).count(), + each + ); + assert_eq!( + probed.iter().filter(|probed| **probed == txid(2)).count(), + each + ); + } + + /// The retries are bounded: [`MAX_NO_ANSWER_RETRIES`] per height, with + /// growing delays, however long no node answers. + #[tokio::test(start_paused = true)] + async fn should_retry_probes_no_node_answered_a_bounded_number_of_times_per_height() { + let probe = Arc::new(SequenceProbe::new(&vec![unreachable_verdict(); 10])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + let per_height = 1 + MAX_NO_ANSWER_RETRIES as usize; + + let started = Instant::now(); + rig.follow(wallet(), 100).await; + assert_eq!( + probe.calls(), + per_height, + "the probe and its retries at 101" + ); + assert!( + started.elapsed() >= NO_ANSWER_RETRY * 7, + "1, 2 and 4 times the delay" + ); + + rig.height(wallet(), 102).await; + assert_eq!(probe.calls(), 2 * per_height, "and again at the next block"); + } + + /// A block arriving while a retry waits takes over: the earlier height's + /// retry does not cancel the new height's. + #[tokio::test(start_paused = true)] + async fn should_keep_the_retry_of_a_new_block_when_an_old_retry_fires() { + let probe = Arc::new(SequenceProbe::new(&[ + unreachable_verdict(), + unreachable_verdict(), + ProbeVerdict::Accepted, + ])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 100, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + // Run the pass at 101 (read, then its probe), leaving its retry + // asleep. + for _ in 0..2 { + let job = rig.actor.jobs.join_next_with_id().await.expect("job"); + rig.actor.joined(job); + } + assert_eq!(probe.calls(), 1); + + rig.send(Command::Height { + wallet_id: wallet(), + height: 102, + }) + .await; + + assert_eq!(probe.calls(), 3, "101, 102, then 102's retry"); + assert!(rig + .sent() + .contains(&ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted))); + } + + /// Completions that repeat a tip (peer flaps, no new block) probe a + /// root once per height: the schedule bounds them. + #[tokio::test] + async fn should_probe_once_per_height_however_many_completions() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; // probes at 101 + + for _ in 0..3 { + rig.send(sync_complete(101)).await; + } + + assert_eq!(probe.probed(), vec![txid(1)]); + } + + /// A forced pass (an `Uncertain` report's, here at the stored tip before + /// any step) that no node answered is retried as a forced pass: probed + /// again without waiting for a block, and no window starts at a height + /// the wallet may not follow. + #[tokio::test(start_paused = true)] + async fn should_retry_a_forced_pass_without_starting_a_window() { + let probe = Arc::new(SequenceProbe::new(&[ + unreachable_verdict(), + ProbeVerdict::Accepted, + ])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.send(Command::Uncertain(txid(1))).await; + + assert_eq!(probe.calls(), 2, "the forced probe and its retry"); + assert_eq!( + rig.actor.state.schedules[&(wallet(), txid(1))].window_start, + None + ); + assert!(rig + .sent() + .contains(&ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted))); + } + + /// A retry probes only the roots no node answered: the others were + /// probed at that height. + #[tokio::test(start_paused = true)] + async fn should_retry_only_the_roots_no_node_answered() { + let probe = Arc::new(ScriptedProbe::new(&[ + (txid(1), unreachable_verdict()), + (txid(2), unresolved()), + ])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + + rig.follow(wallet(), 100).await; + + let probed = probe.probed(); + assert_eq!( + probed.iter().filter(|probed| **probed == txid(2)).count(), + 1, + "answered: not retried" + ); + assert_eq!( + probed.iter().filter(|probed| **probed == txid(1)).count(), + 1 + MAX_NO_ANSWER_RETRIES as usize + ); + } + + /// A completion at a height whose probe reached no node adds no probe of + /// its own: the waiting retry is the next try. + #[tokio::test(start_paused = true)] + async fn should_leave_an_unanswered_height_to_its_retry_at_completion() { + let probe = Arc::new(SequenceProbe::new(&[ + unreachable_verdict(), + ProbeVerdict::Accepted, + ])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 100, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + for _ in 0..2 { + let job = rig + .actor + .jobs + .join_next_with_id() + .await + .expect("read, then probe"); + rig.actor.joined(job); + } + assert_eq!(probe.calls(), 1); + + rig.handle(sync_complete(101)); + let entry = &rig.actor.wallets[&wallet()]; + assert!(entry.run.is_none() && entry.pending.is_none(), "no pass"); + + rig.settle().await; // the retry + assert_eq!(probe.calls(), 2); + } + + /// While probing is off a completion does nothing; the next one, after + /// it is on again, runs the pass. + #[tokio::test] + async fn should_ignore_a_completion_while_probing_is_off() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.height(wallet(), 500).await; // catch-up + + rig.send(Command::SetEnabled(false)).await; + rig.send(sync_complete(500)).await; + assert!(probe.probed().is_empty()); + + rig.send(Command::SetEnabled(true)).await; + rig.send(sync_complete(500)).await; + assert_eq!(probe.probed(), vec![txid(1)]); + } + + /// A wallet with no step since launch that reads far below the tip (a + /// new wallet scanning from its birth height) is not anchored at the tip. + #[tokio::test] + async fn should_not_probe_a_wallet_with_no_step_reading_far_below_the_tip() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 1_000); + + rig.send(sync_complete(1_566_577)).await; + + assert!(probe.probed().is_empty()); + } + + /// dash-spv's completion with an unknown tip (0) means nothing to follow. + #[test] + fn should_ignore_a_completion_without_a_tip() { + let complete = |header_tip| SyncEvent::SyncComplete { + header_tip, + cycle: 1, + }; + assert!(sync_command(&complete(0)).is_none()); + assert!(matches!( + sync_command(&complete(7)), + Some(Command::SyncComplete { tip: 7 }) + )); + } + + /// A probe that reached no node does not count in the schedule's pace: + /// the root is due at the next height even in its slow phase — but not + /// at the same height, where only a retry probes it again. + #[test] + fn should_make_an_unanswered_root_due_at_the_next_height_in_any_phase() { + let mut schedule = anchored(100); + schedule.probed_at(130); // past the every-block window + assert!(!schedule.is_due(131)); + + schedule.unanswered = true; + + assert!(!schedule.is_due(130), "not again at the same height"); + assert!(schedule.is_due(131)); + assert_eq!(schedule.next_due(), 131); + } + + /// Once the retries at a height are used up, completions at that height + /// probe nothing more; the next block does. + #[tokio::test(start_paused = true)] + async fn should_probe_an_unanswered_root_at_the_next_block_after_its_retries() { + let probe = Arc::new(SequenceProbe::new(&vec![unreachable_verdict(); 10])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + let at_101 = 1 + MAX_NO_ANSWER_RETRIES as usize; + assert_eq!(probe.calls(), at_101); + + rig.send(sync_complete(101)).await; + assert_eq!(probe.calls(), at_101, "the retries were the tries at 101"); + + rig.height(wallet(), 102).await; + assert!(probe.calls() > at_101, "the next block probes it"); + } + + /// The production order: the block's completion arrives while that + /// block's probe is still out. The probe reaches no node; the queued + /// completion pass probes nothing more at that height — the retry does. + #[tokio::test(start_paused = true)] + async fn should_not_probe_twice_at_a_height_when_its_completion_comes_mid_probe() { + let probe = Arc::new(SequenceProbe::new(&[ + unreachable_verdict(), + ProbeVerdict::Accepted, + ])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 100, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // the probe is out + rig.handle(sync_complete(101)); // queued behind the running pass + let probed = rig.actor.jobs.join_next_with_id().await.expect("probe"); + // No answer: the root is not due again at 101, so the run's end + // drops the queued pass (its height is below the wait). + rig.actor.joined(probed); + assert!(rig.actor.wallets[&wallet()].pending.is_none()); + assert_eq!(probe.calls(), 1, "nothing due again at 101"); + + rig.settle().await; // the retry + assert_eq!(probe.calls(), 2); + } + + /// A new probe of a root clears its unanswered mark: only that probe's + /// own answer decides it (a retry must not withdraw a root whose probe is + /// out). + #[test] + fn should_clear_the_unanswered_mark_when_a_probe_goes_out() { + let mut schedule = anchored(100); + schedule.probed_at(100); + schedule.unanswered = true; + + schedule.probed_at(101); + + assert!(!schedule.unanswered); + } + + /// A rewind or a switch while a retry waits resets the series: the old + /// timer then does nothing. + #[tokio::test(start_paused = true)] + async fn should_drop_a_waiting_retry_on_a_rewind_or_a_switch() { + for reset in [ + vec![Command::SetEnabled(false), Command::SetEnabled(true)], + vec![Command::Height { + wallet_id: wallet(), + height: 50, + }], + ] { + let probe = Arc::new(SequenceProbe::new(&[unreachable_verdict()])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 100, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + for _ in 0..2 { + let job = rig + .actor + .jobs + .join_next_with_id() + .await + .expect("read, then probe"); + rig.actor.joined(job); + } + assert!(rig.actor.wallets[&wallet()].retry_series.is_some()); + + for command in reset { + rig.handle(command); + } + assert!(rig.actor.wallets[&wallet()].retry_series.is_none()); + let calls = probe.calls(); + rig.settle().await; // the old timer fires + assert_eq!(probe.calls(), calls, "the old retry does nothing"); + } + } + + /// Sync events and wallet events come from separate tasks: a completion + /// can be handled before the height step it follows. The step that then + /// brings the wallet to that tip runs the pass the completion could not, + /// instead of waiting for the next block as a catch-up step does. + #[tokio::test] + async fn should_probe_at_the_tip_when_the_completion_overtook_the_height_step() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.height(wallet(), 100).await; // the first step after launch + assert_eq!(probe.probed().len(), 0); + + // The wallet is at 150 already; the actor has not heard yet. + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 150); + rig.send(sync_complete(150)).await; + assert_eq!( + probe.probed().len(), + 0, + "the actor still has the wallet at 100" + ); + + rig.height(wallet(), 150).await; + + assert_eq!(probe.probed().len(), 1); + assert_eq!( + rig.actor.state.schedules[&(wallet(), txid(1))].window_start, + Some(150) + ); + } + + /// A completion a wallet is well below is remembered only until the + /// wallet reaches that tip: steps on the way stay catch-up, a later + /// completion replaces the tip, and a rewind forgets it. + #[tokio::test] + async fn should_keep_a_scan_below_a_completed_tip_as_catch_up() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.height(wallet(), 100).await; + rig.send(sync_complete(150)).await; + + rig.height(wallet(), 120).await; // on the way + assert_eq!(probe.probed().len(), 0); + rig.send(sync_complete(200)).await; // the tip moved on + rig.height(wallet(), 150).await; // the old tip: still below + assert_eq!(probe.probed().len(), 0); + rig.height(wallet(), 110).await; // a rewind: a new scan + rig.height(wallet(), 200).await; + assert_eq!( + probe.probed().len(), + 0, + "the completion did not cover the new scan" + ); + } + + /// The completion after a block whose pass already read the wallet adds + /// no second walk over its records. + #[tokio::test] + async fn should_not_walk_the_records_again_for_the_blocks_completion() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + let walks = rig.walks(); + + rig.send(sync_complete(101)).await; + + assert_eq!(rig.walks(), walks); + } + + /// A retried root that has since become a child (its parent showed up + /// unsettled) is not forced: the retry would resolve it to the parent, + /// already probed at that height. + #[tokio::test(start_paused = true)] + async fn should_not_force_a_retried_root_that_is_no_longer_a_root() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(5), unresolved())])); + let mut rig = enabled(probe.clone()).await; + // Send 1 was the root no node answered; its parent 5 has shown up + // unsettled since, so 1 is a child now. + rig.views( + wallet(), + vec![incoming(5, &[outpoint(80, 0)]), send(1, &[outpoint(5, 0)])], + ); + let mut pending = PendingPass::default(); + pending.retried.insert(txid(1)); + rig.actor + .wallets + .get_mut(&wallet()) + .expect("wallet") + .pending = Some(pending); + + rig.actor.start(wallet()); + rig.settle().await; + + assert!( + probe.probed().is_empty(), + "neither the child nor its parent" + ); + } + + /// A retry forces a root that still is one. + #[tokio::test] + async fn should_force_a_retried_root_that_still_is_one() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + let mut pending = PendingPass::default(); + pending.retried.insert(txid(1)); + rig.actor + .wallets + .get_mut(&wallet()) + .expect("wallet") + .pending = Some(pending); + + rig.actor.start(wallet()); + rig.settle().await; + + assert_eq!(probe.probed(), vec![txid(1)]); + } + + /// The launch case end to end: a wallet with no step yet, its first + /// completion's probe reaches no node, and the retry probes it again + /// without waiting for a block. + #[tokio::test(start_paused = true)] + async fn should_retry_a_completion_probe_of_a_wallet_with_no_step() { + let probe = Arc::new(SequenceProbe::new(&[ + unreachable_verdict(), + ProbeVerdict::Accepted, + ])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 1_566_577); + + rig.send(sync_complete(1_566_577)).await; + + assert_eq!(probe.calls(), 2); + assert!(rig + .sent() + .contains(&ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted))); + } + + /// An older report's holder lookup that completes after a newer report + /// of the same txid is obsolete: it found no holder (the record was not + /// there yet), but must not delete the newer report's echo window. + #[tokio::test] + async fn should_not_let_an_obsolete_lookup_delete_a_newer_reports_window() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())]))).await; + rig.views(wallet(), Vec::new()); + rig.handle(Command::Uncertain(txid(1))); + // Its lookup finds no holder: held back. + let old = rig + .actor + .jobs + .join_next_with_id() + .await + .expect("old lookup"); + assert!(matches!( + &old, + Ok((_, JobDone::Listed { own, .. })) if own.is_empty() + )); + + // The owning record arrives, and dash-spv reports the send again. + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.handle(Command::Uncertain(txid(1))); + rig.actor.joined(old); // the old, empty completion lands now + assert!( + rig.actor.uncertain.contains_key(&txid(1)), + "the window stays" + ); + rig.settle().await; + rig.sent(); + + rig.send(Command::Echoed(txid(1))).await; + + assert!(rig + .sent() + .contains(&ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted))); + } + + /// The reverse order: the newer report's lookup completes first, then + /// the older, empty one — still obsolete. + #[tokio::test] + async fn should_drop_an_older_lookup_that_completes_after_the_newer_one() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())]))).await; + rig.views(wallet(), Vec::new()); + rig.handle(Command::Uncertain(txid(1))); + let old = rig + .actor + .jobs + .join_next_with_id() + .await + .expect("old lookup"); + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.handle(Command::Uncertain(txid(1))); + let newer = rig + .actor + .jobs + .join_next_with_id() + .await + .expect("newer lookup"); + rig.actor.joined(newer); + + rig.actor.joined(old); + + assert!( + rig.actor.uncertain.contains_key(&txid(1)), + "the window stays" + ); + } + + /// A lookup from before a switch-off is obsolete after it. + #[tokio::test] + async fn should_drop_a_lookup_from_before_a_switch_off() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.handle(Command::Uncertain(txid(1))); + let old = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + + rig.handle(Command::SetEnabled(false)); + rig.handle(Command::SetEnabled(true)); + rig.actor.joined(old); + rig.settle().await; + + assert!( + probe.probed().is_empty(), + "no forced pass from the old report" + ); + } + + /// Records a peer can feed the wallet: unsettled "incoming" parents, + /// each with a child that spends the wallet's coins as matched — every + /// parent a root to probe. + fn peer_injected(parents: std::ops::Range) -> Vec { + parents + .flat_map(|parent| { + [ + incoming(parent, &[outpoint(parent.wrapping_add(100), 0)]), + send(parent.wrapping_add(60), &[outpoint(parent, 0)]), + ] + }) + .collect() + } + + /// Reported sends' roots go first, in turn among themselves, and leave + /// half of the slots to the rest: a peer whose outputs a reported send + /// spends makes its parents such roots, and they must not fill every pass. + #[test] + fn should_share_a_pass_between_reported_and_other_roots() { + let views = peer_injected(10..30); + // Ten reported sends, each the child of one peer parent (10..20). + let reported: HashSet = (70..80).map(txid).collect(); + let reported_roots: HashSet = (10..20).map(txid).collect(); + let mut state = ResolverState::default(); + let mut passes = Vec::new(); + for height in 100..103 { + let start = begin_pass( + &mut state, + wallet(), + height, + &views, + &HashSet::new(), + &reported, + Some(height), + ); + let due: Vec = start.due.iter().map(|root| root.txid).collect(); + for root in &due { + mark_sent(&mut state, wallet(), *root, height, Some(height)); + } + passes.push(due); + } + + for due in &passes { + assert_eq!(due.len(), MAX_ROOTS_PER_PASS); + let first = due.iter().filter(|t| reported_roots.contains(*t)).count(); + assert_eq!(first, MAX_ROOTS_PER_PASS / 2, "half of the slots each"); + assert!( + due[..first].iter().all(|t| reported_roots.contains(t)), + "reported roots go first" + ); + } + let probed: HashSet = passes.iter().flatten().copied().collect(); + assert!( + reported_roots.iter().all(|root| probed.contains(root)), + "every reported root came round in turn" + ); + } + + /// Peer-injected records make many roots: a pass probes at most + /// `MAX_ROOTS_PER_PASS` of them, the ones probed longest ago first, so + /// each comes round in turn at the following heights. + #[tokio::test] + async fn should_bound_the_roots_a_pass_probes() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), peer_injected(10..40)); // 30 roots + + rig.follow(wallet(), 100).await; + assert_eq!(probe.probed().len(), MAX_ROOTS_PER_PASS); + + rig.height(wallet(), 102).await; + let probed = probe.probed(); + assert_eq!(probed.len(), 2 * MAX_ROOTS_PER_PASS); + let distinct: HashSet = probed.iter().copied().collect(); + assert_eq!( + distinct.len(), + 2 * MAX_ROOTS_PER_PASS, + "the next ones in turn" + ); + } + + /// A genuine send reported `Uncertain` while a pass works through + /// peer-injected roots does not wait behind them: the run gives way + /// after its probe in flight, and the send is probed next. + #[tokio::test] + async fn should_probe_a_genuine_uncertain_send_before_a_backlog() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), peer_injected(10..40)); + rig.follow(wallet(), 100).await; + let before = probe.probed().len(); + // The genuine send, broadcast now — sorting after the backlog, so + // only the yield can bring it forward. + let mut views = peer_injected(10..40); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // a scheduled probe is out + + rig.handle(Command::Uncertain(txid(200))); + while !probe.probed().contains(&txid(200)) { + let job = rig.actor.jobs.join_next_with_id().await.expect("job"); + rig.actor.joined(job); + } + + let since = &probe.probed()[before..]; + let genuine_at = since + .iter() + .position(|probed| *probed == txid(200)) + .expect("probed"); + assert!( + genuine_at <= 2, + "only the probe in flight (and at most one sent before the report \ + was routed) goes first: {since:?}" + ); + rig.settle().await; + // Two passes at that height — the one that gave way, and the forced + // one that takes the roots it dropped — each bounded by the cap. + assert!( + probe.probed().len() - before <= 2 * MAX_ROOTS_PER_PASS + 1, + "bounded work at that height" + ); + } + + /// A peer feeding fresh roots every block cannot starve older ones: the + /// queue serves roots in the order they were first seen. + #[tokio::test] + async fn should_not_let_fresh_roots_starve_older_ones() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..30); // 20 roots, seen at 101 + rig.views(wallet(), views.clone()); + rig.follow(wallet(), 100).await; + + for (height, fresh) in [(102, 30..38), (103, 38..46), (104, 46..54)] { + views.extend(peer_injected(fresh)); // 8 fresh roots each block + rig.views(wallet(), views.clone()); + rig.height(wallet(), height).await; + } + + let probed: HashSet = probe.probed().into_iter().collect(); + assert!( + (10..30).all(|n| probed.contains(&txid(n))), + "every older root came round before the fresh ones" + ); + } + + /// A root waiting in a backlog keeps its every-block window: it starts at + /// its first probe, not when the root was first seen. + #[tokio::test] + async fn should_start_a_window_at_the_first_probe_not_while_waiting() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), peer_injected(10..30)); // 20 roots + rig.follow(wallet(), 100).await; // 8 probed at 101 + + let waiting = &rig.actor.state.schedules[&(wallet(), txid(29))]; + assert_eq!(waiting.window_start, None); + assert_eq!(waiting.seen_at, Some(101)); + rig.height(wallet(), 102).await; + rig.height(wallet(), 103).await; // 29 is in the third batch + + assert_eq!( + rig.actor.state.schedules[&(wallet(), txid(29))].window_start, + Some(103) + ); + } + + /// A forced request routed while the run is still reading makes it give + /// way as soon as the read is in. + #[tokio::test] + async fn should_give_way_to_a_forced_request_routed_during_the_read() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..30); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.follow(wallet(), 100).await; + let before = probe.probed().len(); + rig.source.pause(true); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + rig.actor.handle(Command::Uncertain(txid(200))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.actor.joined(listed); // routed while the read waits + rig.source.pause(false); + + rig.settle().await; + + let since = &probe.probed()[before..]; + assert_eq!(since.first(), Some(&txid(200)), "{since:?}"); + } + + /// A send that spends twenty unsettled incoming parents, with those + /// parents: one report of it forces twenty roots. + fn send_over_many_parents(send_n: u8) -> Vec { + let parents = 10..30u8; + let spent: Vec = parents.clone().map(|parent| outpoint(parent, 0)).collect(); + let mut views: Vec = parents + .map(|parent| incoming(parent, &[outpoint(parent.wrapping_add(100), 0)])) + .collect(); + views.push(send(send_n, &spent)); + views + } + + /// One reported send can force more roots than a pass may probe (a peer + /// can feed the parents it spends): the pass takes + /// `MAX_FORCED_ROOTS_PER_PASS` of them and no more, anchored or not, and + /// an anchored pass adds its scheduled slots on top. + #[test] + fn should_bound_the_roots_one_report_forces() { + let views = send_over_many_parents(200); + let forced: HashSet = [txid(200)].into(); + + let mut state = ResolverState::default(); + let unanchored = begin_pass(&mut state, wallet(), 100, &views, &forced, &forced, None); + assert_eq!(unanchored.due.len(), MAX_FORCED_ROOTS_PER_PASS); + assert!(unanchored.due.iter().all(|root| root.forced)); + + let mut state = ResolverState::default(); + let anchored = begin_pass( + &mut state, + wallet(), + 100, + &views, + &forced, + &forced, + Some(100), + ); + let forced_roots = anchored.due.iter().filter(|root| root.forced).count(); + assert_eq!(forced_roots, MAX_FORCED_ROOTS_PER_PASS); + assert_eq!( + anchored.due.len(), + MAX_FORCED_ROOTS_PER_PASS + MAX_ROOTS_PER_PASS + ); + let distinct: HashSet = anchored.due.iter().map(|root| root.txid).collect(); + assert_eq!(distinct.len(), anchored.due.len(), "no root twice"); + } + + /// Among forced roots the wallet's own sends go first: a reported send + /// that is its own root is not left behind the peer-fed parents of + /// another reported send. + #[test] + fn should_put_own_sends_first_among_forced_roots() { + let mut views = send_over_many_parents(200); + views.push(send(201, &[outpoint(251, 0)])); + let forced: HashSet = [txid(200), txid(201)].into(); + let mut state = ResolverState::default(); + + let start = begin_pass(&mut state, wallet(), 100, &views, &forced, &forced, None); + + // The reported root itself, then the allowance of the other's parents. + assert_eq!(start.due.len(), 1 + MAX_FORCED_ROOTS_PER_PASS); + assert_eq!(start.due.front().map(|root| root.txid), Some(txid(201))); + } + + /// The allowance is for the roots a forced send's chain reaches, not for + /// a root forced by name: every reported send that is its own root is + /// probed, and a retry repeats every root it names, however many. + #[test] + fn should_probe_every_root_forced_by_name() { + let named = 1..=(MAX_FORCED_ROOTS_PER_PASS as u8 + 4); + let views: Vec = named + .clone() + .map(|n| send(n, &[outpoint(n.wrapping_add(100), 0)])) + .collect(); + let forced: HashSet = named.clone().map(txid).collect(); + let mut state = ResolverState::default(); + + let start = begin_pass(&mut state, wallet(), 100, &views, &forced, &forced, None); + + let due: HashSet = start.due.iter().map(|root| root.txid).collect(); + assert_eq!(due, forced); + assert!(start.due.iter().all(|root| root.forced)); + } + + /// A completion can also overtake a wallet's first step since launch. + /// The completion reads the wallet at the tip and starts its pass; the + /// step that follows is the wallet's arrival there, not a catch-up step + /// that would stop that pass. + #[tokio::test] + async fn should_keep_the_pass_of_a_completion_that_overtook_the_first_step() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 150); + + // Both are queued before the pass's read comes back. + rig.handle(sync_complete(150)); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 150, + }); + rig.settle().await; + + assert_eq!(probe.probed(), [txid(1)], "one pass, not stopped"); + assert_eq!( + rig.actor.state.schedules[&(wallet(), txid(1))].window_start, + Some(150) + ); + } + + /// A completion anchors a wallet it has no height for at its own tip. + /// When that wallet's first step lands a block or two below, the step + /// queues the pass at its own height: the completion's pass loses its + /// anchor once its read finds the wallet below it. + #[tokio::test] + async fn should_run_the_pass_when_the_first_step_lands_just_below_the_completion() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 148); + + rig.handle(sync_complete(150)); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 148, + }); + rig.settle().await; + + assert_eq!(probe.probed(), [txid(1)]); + assert_eq!( + rig.actor.state.schedules[&(wallet(), txid(1))].window_start, + Some(148) + ); + } + + /// An arrival at a completed tip still confines a pass anchored before + /// the gap: a root that pass had not sent yet is left to the arrival's + /// own pass, so its window starts at the tip, not at the height the + /// wallet left. + #[tokio::test] + async fn should_not_start_windows_before_the_gap_on_an_arrival() { + let probe = Arc::new(ScriptedProbe::new(&[ + (txid(1), unresolved()), + (txid(2), unresolved()), + ])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + rig.height(wallet(), 100).await; + + // A followed step starts a pass. Its read comes back and its first + // probe goes out; then the wallet is suspended, catches up, and the + // completion comes before the step. + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("the read"); + rig.actor.joined(read); + rig.handle(sync_complete(150)); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 150, + }); + rig.settle().await; + + let mut windows: Vec> = [1, 2] + .into_iter() + .map(|n| rig.actor.state.schedules[&(wallet(), txid(n))].window_start) + .collect(); + windows.sort(); + assert_eq!( + windows, + [Some(101), Some(150)], + "the probe already out counts from 101, the other from the tip" + ); + } + + /// A completion whose height step never came is forgotten after a + /// while: a step of a later scan that lands at that old tip is catch-up. + #[tokio::test(start_paused = true)] + async fn should_forget_a_completion_its_step_did_not_follow() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.height(wallet(), 100).await; + rig.send(sync_complete(150)).await; + + tokio::time::advance(COMPLETED_AHEAD_TTL + Duration::from_secs(1)).await; + rig.height(wallet(), 150).await; + + assert_eq!(probe.probed().len(), 0, "a scan's step, not an arrival"); + } + + /// A report whose send has more ancestor roots than a pass may force + /// costs a bounded pass, and a send reported after it is probed at once, + /// not behind the rest of those roots. Following the tip then brings + /// every one of them round within a few blocks. + #[tokio::test] + async fn should_probe_a_later_report_without_waiting_for_an_oversized_one() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = send_over_many_parents(200); + views.push(send(201, &[outpoint(251, 0)])); + rig.views(wallet(), views); + + rig.send(Command::Uncertain(txid(200))).await; + let oversized = probe.probed(); + assert_eq!(oversized.len(), MAX_FORCED_ROOTS_PER_PASS, "{oversized:?}"); + + rig.send(Command::Uncertain(txid(201))).await; + assert_eq!(probe.probed()[oversized.len()..], [txid(201)]); + + // The twelve the forced pass left go first: eight in the first + // followed pass, the rest in the next. + rig.follow(wallet(), 100).await; + rig.height(wallet(), 102).await; + let probed: HashSet = probe.probed().into_iter().collect(); + let missed: Vec = (10..30) + .filter(|parent| !probed.contains(&txid(*parent))) + .collect(); + assert!(missed.is_empty(), "left over and never probed: {missed:?}"); + } + + /// A forced root no node answered stays ahead of the cap at its retry, + /// whatever the backlog. + #[tokio::test(start_paused = true)] + async fn should_retry_a_forced_root_ahead_of_a_backlog() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(200), unreachable_verdict())])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..40); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 100, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + rig.handle(Command::Uncertain(txid(200))); + + rig.settle().await; // the anchored pass with the forced root, then retries + + let genuine = probe + .probed() + .iter() + .filter(|probed| **probed == txid(200)) + .count(); + assert!( + genuine > MAX_NO_ANSWER_RETRIES as usize, + "its probe and every retry, whatever the backlog: {genuine}" + ); + } + + /// A genuine send reported `Uncertain` keeps its place ahead of a + /// peer-fed backlog after its first probe: it is probed at every block + /// of its window, not once per trip round the queue. + #[tokio::test] + async fn should_keep_an_uncertain_send_ahead_of_a_backlog_after_its_first_probe() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..40); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.follow(wallet(), 100).await; + rig.send(Command::Uncertain(txid(200))).await; + let after_report = probe.probed().iter().filter(|p| **p == txid(200)).count(); + + for height in 102..=104 { + rig.height(wallet(), height).await; + } + + let probed = probe.probed().iter().filter(|p| **p == txid(200)).count(); + assert_eq!(probed, after_report + 3, "once at each block"); + } + + /// While no node answers, a retry probes only the roots that went + /// unanswered: the work at one height does not grow with each retry. + #[tokio::test(start_paused = true)] + async fn should_not_widen_a_retry_with_more_scheduled_roots() { + let probe = Arc::new(SequenceProbe::new(&vec![unreachable_verdict(); 200])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), peer_injected(10..40)); + + rig.follow(wallet(), 100).await; + + assert_eq!( + probe.calls(), + MAX_ROOTS_PER_PASS * (1 + MAX_NO_ANSWER_RETRIES as usize) + ); + } + + /// Roots a run planned but dropped when it gave way are not lost: they + /// stay due and, probed longest ago, go first at the next height. + #[tokio::test] + async fn should_take_the_roots_a_run_dropped_for_a_forced_request_next() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..40); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 100, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // the first of 8 scheduled roots is out + let planned: Vec = rig.actor.wallets[&wallet()] + .run + .as_ref() + .and_then(|run| run.probing.as_ref()) + .map(|probing| probing.due.iter().map(|root| root.txid).collect()) + .expect("probing"); + let before = probe.probed().len(); + + rig.send(Command::Uncertain(txid(200))).await; + + let since = probe.probed()[before..].to_vec(); + let genuine = since + .iter() + .position(|probed| *probed == txid(200)) + .expect("probed"); + // The probe in flight finishes and the next planned root goes out + // before the report is routed. + assert!(genuine <= 2, "{since:?}"); + let before = probe.probed().len(); + + rig.height(wallet(), 102).await; + + let next = probe.probed()[before..].to_vec(); + let dropped: Vec<&Txid> = planned + .iter() + .filter(|txid| !since.contains(txid)) + .collect(); + assert!(!dropped.is_empty(), "{since:?}"); + assert!( + dropped + .iter() + .all(|txid| next[..dropped.len() + 1].contains(txid)), + "first at the next height: {next:?}" + ); + } + + /// A genuine send keeps its place ahead of a backlog for as long as it + /// stays unsettled — past its echo window, which ends after + /// `UNCERTAIN_EXPIRY` blocks. + #[tokio::test] + async fn should_keep_an_uncertain_send_first_past_its_echo_window() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..40); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.follow(wallet(), 100).await; + rig.send(Command::Uncertain(txid(200))).await; + let last = 102 + UNCERTAIN_EXPIRY + SLOW_INTERVAL; + for height in 102..=last { + rig.height(wallet(), height).await; + } + assert!(rig.actor.uncertain.is_empty(), "the echo window is over"); + let before = probe.probed().iter().filter(|p| **p == txid(200)).count(); + + for height in last + 1..=last + SLOW_INTERVAL { + rig.height(wallet(), height).await; + } + + let after = probe.probed().iter().filter(|p| **p == txid(200)).count(); + assert_eq!(after, before + 1, "probed when due, ahead of the backlog"); + } + + /// A reported send stops being a priority once it leaves the wallet: the + /// set does not grow for the session. + #[tokio::test] + async fn should_forget_a_reported_send_that_left_the_wallet() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..12); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.follow(wallet(), 100).await; + rig.send(Command::Uncertain(txid(200))).await; + rig.height(wallet(), 101).await; + assert!(rig.actor.reported.contains_key(&txid(200))); + + rig.views(wallet(), peer_injected(10..12)); + rig.height(wallet(), 102).await; + + assert!(rig.actor.reported.is_empty(), "left with the send"); + } + + /// A repeated report of a send the run already forces changes nothing: + /// the run keeps its scheduled roots. + #[tokio::test] + async fn should_not_give_way_to_a_report_the_run_already_carries() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..40); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 100, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); + let run = rig + .actor + .wallets + .get_mut(&wallet()) + .and_then(|e| e.run.as_mut()) + .expect("run"); + run.forced.insert(txid(200)); // the run carries this report already + let planned = run.probing.as_ref().map(|p| p.due.len()).expect("probing"); + + rig.handle(Command::Uncertain(txid(200))); + loop { + let job = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + let listed = matches!(&job, Ok((_, JobDone::Listed { .. }))); + rig.actor.joined(job); + if listed { + break; + } + } + + let due = rig.actor.wallets[&wallet()] + .run + .as_ref() + .and_then(|run| run.probing.as_ref()) + .map(|p| p.due.len()) + .expect("probing"); + assert!( + due + 1 >= planned, + "kept its scheduled roots: {due} of {planned}" + ); + } + + /// A run gives way once per height: a further report at that height + /// waits behind it, so reports that keep coming cannot drop the + /// scheduled roots again and again. + #[tokio::test] + async fn should_give_way_once_per_height() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + let mut views = peer_injected(10..40); + views.push(send(200, &[outpoint(250, 0)])); + rig.views(wallet(), views); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 100, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); + let entry = rig.actor.wallets.get_mut(&wallet()).expect("entry"); + entry.yielded_at = Some(101); // gave way at this height already + let planned = entry + .run + .as_ref() + .and_then(|run| run.probing.as_ref()) + .map(|p| p.due.len()) + .expect("probing"); + + rig.handle(Command::Uncertain(txid(200))); + loop { + let job = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + let listed = matches!(&job, Ok((_, JobDone::Listed { .. }))); + rig.actor.joined(job); + if listed { + break; + } + } + + let due = rig.actor.wallets[&wallet()] + .run + .as_ref() + .and_then(|run| run.probing.as_ref()) + .map(|p| p.due.len()) + .expect("probing"); + assert!( + due + 1 >= planned, + "kept its scheduled roots: {due} of {planned}" + ); + assert!( + rig.actor.wallets[&wallet()] + .pending + .as_ref() + .is_some_and(|pending| pending.forced.contains(&txid(200))), + "the report waits for the run" + ); + } + + /// A wallet added while a report's holder lookup is out does not cost + /// the other holders the report: the lookup is made again. + #[tokio::test] + async fn should_keep_a_report_when_a_wallet_is_added_during_its_lookup() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(200, &[outpoint(250, 0)])]); + rig.follow(wallet(), 100).await; + let before = probe.probed().len(); + + rig.handle(Command::Uncertain(txid(200))); + // The lookup has finished, its result not delivered yet, when a + // wallet is added. + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + let stale = match &listed { + Ok((_, JobDone::Listed { report, own, .. })) => { + assert_eq!(own, &vec![wallet()]); + *report + } + _ => panic!("expected the holder lookup"), + }; + // The added wallet holds the send too: only a lookup made after it + // was added can name it. + let added = [9u8; 32]; + rig.views(added, vec![send(200, &[outpoint(250, 0)])]); + rig.handle(Command::WalletAdded(added)); + rig.actor.joined(listed); + + // The stale result routed nothing; a newer lookup is out. + assert!( + rig.actor + .report_lookups + .get(&txid(200)) + .is_some_and(|(token, _)| *token > stale), + "looked up again" + ); + assert!(!rig.actor.reported.contains_key(&txid(200))); + let entry = &rig.actor.wallets[&wallet()]; + assert!(entry + .pending + .as_ref() + .is_none_or(|pending| !pending.forced.contains(&txid(200)))); + assert!(entry + .run + .as_ref() + .is_none_or(|run| !run.forced.contains(&txid(200)))); + + rig.settle().await; + + let holders = rig.actor.reported.get(&txid(200)).expect("reported"); + assert!( + holders.contains(&wallet()), + "still a priority of its wallet" + ); + assert!(holders.contains(&added), "named by the new lookup only"); + assert!( + probe.probed()[before..].contains(&txid(200)), + "forced probe went out" + ); + } + + /// A yield that drops nothing — the run still reading — does not use up + /// the height's allowance: once the read has planned scheduled roots, the + /// run still gives way to the request. + #[tokio::test] + async fn should_keep_the_allowance_when_a_yield_drops_nothing() { + let views = peer_injected(10..20); + let mut state = ResolverState::default(); + let start = begin_pass( + &mut state, + wallet(), + 101, + &views, + &HashSet::new(), + &HashSet::new(), + Some(101), + ); + assert_eq!(start.due.len(), MAX_ROOTS_PER_PASS); + let mut entry = WalletEntry { + height: Some(101), + run: Some(Run { + id: 1, + settled: HashSet::new(), + woken: false, + anchor_at: Some(101), + forced: HashSet::new(), + retried: HashSet::new(), + walk: Arc::new(AtomicBool::new(false)), + abort: tokio::spawn(async {}).abort_handle(), + probing: None, // reading + }), + pending: Some(PendingPass { + height: 101, + forced: HashSet::from([txid(200)]), + ..PendingPass::default() + }), + ..WalletEntry::default() + }; + + yield_to_forced(&mut entry); + assert_eq!(entry.yielded_at, None, "nothing dropped yet"); + + entry.run.as_mut().expect("run").probing = Some(Probing { + views, + generation: Arc::new(WalletGeneration::new()), + height: 101, + due: start.due, + in_flight: None, + }); + yield_to_forced(&mut entry); + + let run = entry.run.as_ref().expect("run"); + assert!( + run.probing.as_ref().is_some_and(|p| p.due.is_empty()), + "gave way once the read planned its roots" + ); + assert_eq!(entry.yielded_at, Some(101)); + } + + /// A refusal is an answer: no early retry. + #[tokio::test(start_paused = true)] + async fn should_not_retry_a_probe_a_node_answered() { + let probe = Arc::new(SequenceProbe::new(&[unresolved(), unresolved()])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.follow(wallet(), 100).await; + + assert_eq!(probe.calls(), 1); + } + + /// Turning probing off forgets every `Uncertain` send. + #[tokio::test] + async fn should_forget_uncertain_results_when_probing_goes_off() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Uncertain(txid(1))).await; + assert!(rig.actor.uncertain.contains_key(&txid(1))); + + rig.send(Command::SetEnabled(false)).await; + + assert!(rig.actor.uncertain.is_empty()); + } + + /// dash-spv may still see a send it reported `Uncertain` echoed by a peer + /// later: with no block, no InstantSend lock and no probe answer, that + /// echo alone turns its verdict into Accepted. + #[tokio::test] + async fn should_accept_a_send_echoed_after_its_uncertain_result() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + rig.send(Command::Uncertain(txid(1))).await; + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), unresolved())] + ); + + rig.send(Command::Echoed(txid(1))).await; + + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + } + + /// An echo whose holder lookup panicked is given up, as a report whose + /// lookup panicked is: a wallet registered later does not have it looked + /// up again and accepted at that unrelated moment. + #[tokio::test] + async fn should_give_up_an_echo_whose_lookup_panicked() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + rig.send(Command::Uncertain(txid(1))).await; + rig.sent(); + + *rig.source.holders_panic.lock().expect("holders_panic") = true; + rig.send(Command::Echoed(txid(1))).await; + *rig.source.holders_panic.lock().expect("holders_panic") = false; + assert!(rig.actor.echo_lookups.is_empty()); + + rig.send(Command::WalletAdded([0xEE; 32])).await; + + assert!( + !rig.sent() + .contains(&ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)), + "the given-up echo came back at a wallet change" + ); + } + + /// An echo adds nothing to a send a probe already found accepted: no + /// second Accepted is published. + #[tokio::test] + async fn should_not_publish_an_accepted_send_again_on_its_echo() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + rig.send(Command::Uncertain(txid(1))).await; + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + + rig.send(Command::Echoed(txid(1))).await; + + assert!(rig.sent().is_empty()); + assert_eq!( + rig.actor.state.published.get(&(wallet(), txid(1))), + Some(&ProbeVerdict::Accepted) + ); + } + + /// An acceptance whose lookup was out when the send settled must not + /// reach the host after the send's verdict was cleared. + #[tokio::test] + async fn should_drop_an_acceptance_that_lands_after_the_send_settled() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + rig.send(Command::Uncertain(txid(1))).await; + rig.sent(); + + // The echo's lookup finds the wallet holding the send; the send + // settles before that result is handled. + rig.handle(Command::Echoed(txid(1))); + let landed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + assert!(matches!( + &landed, + Ok((_, JobDone::EchoListed { wallets, .. })) if wallets == &[wallet()] + )); + rig.handle(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(1)]), + }); + rig.actor.joined(landed); + rig.settle().await; + + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + } + + /// The same when no event named the settlement and a pass's read found + /// it: the lookup from before that read must not accept the send. + #[tokio::test] + async fn should_drop_an_acceptance_that_lands_after_a_read_found_the_send_settled() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.send(Command::Uncertain(txid(1))).await; + rig.sent(); + + rig.handle(Command::Echoed(txid(1))); + let landed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + assert!(matches!( + &landed, + Ok((_, JobDone::EchoListed { wallets, .. })) if wallets == &[wallet()] + )); + // The send settles; only the next pass's read learns it. + rig.views(wallet(), Vec::new()); + rig.height(wallet(), 102).await; + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + + rig.actor.joined(landed); + rig.settle().await; + + assert!(rig.sent().is_empty(), "accepted after it was cleared"); + } + + /// The same when the read that finds the send gone is a forced one that + /// reads nothing (`Some(None)`): its verdict is cleared, and the lookup + /// from before accepts it only for a wallet that still holds it. + #[tokio::test] + async fn should_clear_a_send_a_forced_read_found_gone_and_keep_its_echo_for_the_others() { + let other: WalletId = [8u8; 32]; + for other_holds in [false, true] { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + if other_holds { + rig.views(other, vec![send(1, &[outpoint(90, 0)])]); + rig.follow(other, 100).await; + } + rig.send(Command::Uncertain(txid(1))).await; + rig.sent(); + + rig.handle(Command::Echoed(txid(1))); + let echo = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + assert!(matches!(&echo, Ok((_, JobDone::EchoListed { .. })))); + // A new report forces a pass for the wallet still holding the + // send; the send leaves it before that pass reads. + rig.handle(Command::Uncertain(txid(1))); + let report = rig.actor.jobs.join_next_with_id().await.expect("report"); + rig.views(wallet(), Vec::new()); + rig.actor.joined(report); + rig.settle().await; + assert!( + rig.actor.wallets[&wallet()].run.is_none(), + "{other_holds}: the forced pass ended" + ); + + rig.actor.joined(echo); + rig.settle().await; + + let published = &rig.actor.state.published; + assert_eq!( + published.get(&(wallet(), txid(1))), + None, + "{other_holds}: cleared for the wallet it left" + ); + assert!( + !rig.actor + .reported + .get(&txid(1)) + .is_some_and(|holders| holders.contains(&wallet())), + "{other_holds}: no priority for a send the wallet lost" + ); + assert_eq!( + published.get(&(other, txid(1))), + other_holds.then_some(&ProbeVerdict::Accepted), + "{other_holds}: the echo still reaches the wallet holding it" + ); + let events = rig.sent(); + assert!( + events.contains(&ResolverEvent::Cleared(txid(1))), + "{other_holds}: got {events:?}" + ); + } + } + + /// An echo's acceptance survives another wallet's registration or removal + /// while its lookup is out: the lookup is made again, not dropped. + #[tokio::test] + async fn should_keep_an_echo_whose_lookup_another_wallets_change_overtook() { + let other: WalletId = [8u8; 32]; + for change in [Command::WalletAdded(other), Command::WalletRemoved(other)] { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.views(other, Vec::new()); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + rig.send(Command::Uncertain(txid(1))).await; + rig.sent(); + + // The echo's lookup is out when the other wallet changes. + rig.handle(Command::Echoed(txid(1))); + rig.handle(change); + rig.settle().await; + + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + } + } + + /// The wallet an echo was for is removed while the lookup is out: the + /// lookup made again names no holder, and nothing is published for it. + #[tokio::test] + async fn should_drop_an_echo_whose_wallet_was_removed_before_its_lookup_landed() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + rig.send(Command::Uncertain(txid(1))).await; + rig.sent(); + + rig.handle(Command::Echoed(txid(1))); + rig.source.views.lock().expect("views").remove(&wallet()); + rig.handle(Command::WalletRemoved(wallet())); + rig.settle().await; + + assert!(!rig + .sent() + .iter() + .any(|event| matches!(event, ResolverEvent::Verdict(..)))); + } + + /// A late acceptance is news only to a wallet where the send is still an + /// unsettled own send: one that only receives it hears nothing. + #[tokio::test] + async fn should_publish_a_late_acceptance_only_to_an_own_unsettled_send() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe).await; + rig.views(wallet(), vec![incoming(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + rig.send(Command::Uncertain(txid(1))).await; + rig.sent(); + + rig.send(Command::Echoed(txid(1))).await; + + assert!(rig.sent().is_empty()); + } + + /// Every healthy send is echoed: only one reported `Uncertain` first gets + /// a verdict from it. + #[tokio::test] + async fn should_publish_nothing_for_the_echo_of_a_send_never_uncertain() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + + rig.send(Command::Echoed(txid(1))).await; + + assert!(rig.sent().is_empty()); + } + + /// With no root left to probe the wallet is idle: height advances do not + /// walk its records again until an event that touched them. + #[tokio::test] + async fn should_go_idle_when_no_root_is_left_until_a_touch() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + rig.views(wallet(), Vec::new()); + + rig.follow(wallet(), 100).await; + assert!(rig.sent().is_empty()); + assert_eq!(rig.walks(), 1); + + rig.height(wallet(), 102).await; + rig.send(Command::Seen { + wallet_id: wallet(), + touched: false, + }) + .await; + rig.height(wallet(), 103).await; + assert_eq!( + rig.walks(), + 1, + "idle: no walk, even after an untouching event" + ); + + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + rig.height(wallet(), 104).await; + assert_eq!(rig.walks(), 2); + } + + /// Something touched the wallet while its pass ran: the pass may have read + /// it before, so it must not leave the wallet idle. + #[tokio::test] + async fn should_not_leave_idle_a_wallet_touched_during_its_pass() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[( + txid(1), + ProbeVerdict::Accepted, + )]))) + .await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.height(wallet(), 100).await; + + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + rig.actor.handle(Command::Seen { + wallet_id: wallet(), + touched: true, + }); + rig.settle().await; + rig.height(wallet(), 102).await; + + assert_eq!(rig.walks(), 2); + } + + /// Off clears every verdict and wakes every wallet; back on, the next + /// height re-publishes what was cleared. + #[tokio::test] + async fn should_republish_after_probing_goes_off_and_on() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.sent(); + + rig.send(Command::SetEnabled(false)).await; + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + + rig.send(Command::SetEnabled(true)).await; + rig.height(wallet(), 102).await; + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + assert_eq!(probe.probed(), vec![txid(1), txid(1)]); + } + + /// Probing turned off while a pass is out on the network: its result is + /// dropped, nothing is published after the clears. + #[tokio::test] + async fn should_drop_a_pass_in_flight_when_probing_goes_off() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[( + txid(1), + ProbeVerdict::Accepted, + )]))) + .await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.height(wallet(), 100).await; + + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + rig.actor.handle(Command::SetEnabled(false)); + rig.settle().await; + + assert!(rig.sent().is_empty()); + assert!(rig.actor.state.published.is_empty()); + } + + #[tokio::test] + async fn should_clear_a_removed_wallets_verdicts_and_forget_it() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.sent(); + + rig.send(Command::WalletRemoved(wallet())).await; + + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + assert!(!rig.actor.wallets.contains_key(&wallet())); + } + + /// An `Uncertain` result right after launch, before any height step + /// reached the wallet, still gets its immediate probe. + #[tokio::test] + async fn should_route_an_uncertain_result_to_a_wallet_no_event_introduced() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.send(Command::Uncertain(txid(1))).await; + + assert_eq!(probe.probed(), vec![txid(1)]); + } + + /// The fail-safe for a registered wallet read as gone (removed by an + /// embedder through `wallet_manager_arc`, bypassing the notification): its + /// verdicts are cleared and it goes idle rather than being re-read every + /// block. + #[tokio::test] + async fn should_clear_the_verdicts_of_a_wallet_gone_without_a_removal() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.sent(); + + rig.source.views.lock().expect("views").remove(&wallet()); + rig.height(wallet(), 102).await; + + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + assert!(rig.actor.state.schedules.is_empty()); + assert_eq!( + rig.actor.wallets[&wallet()].wait_until, + Some(u32::MAX), + "idle: not re-read on the next block" + ); + } + + /// How long a test waits to see that a job stays blocked. + const STAYS_BLOCKED: Duration = Duration::from_millis(50); + + /// Start a followed pass and hand its read to the actor: the probe of + /// its first root is spawned, not yet run. + async fn spawn_first_probe(rig: &mut Rig, wallet_id: WalletId, from: u32) { + for height in [from, from + 1] { + rig.handle(Command::Height { wallet_id, height }); + } + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); + } + + /// The race a removal must win: the probe job exists when the wallet's + /// removal takes the gate, and the actor has not handled `WalletRemoved` + /// yet. The probe waits the removal out and sends nothing — not while + /// it runs, not after it returns. + #[tokio::test(start_paused = true)] + async fn should_not_send_a_probe_for_a_wallet_being_removed() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + let teardown = rig.source.generation(wallet()).teardown_guard().await; + spawn_first_probe(&mut rig, wallet(), 100).await; + assert!( + timeout(STAYS_BLOCKED, rig.actor.jobs.join_next_with_id()) + .await + .is_err(), + "the probe waits for the removal" + ); + assert!(probe.probed().is_empty(), "nothing sent meanwhile"); + + rig.source.views.lock().expect("views").remove(&wallet()); + drop(teardown); + rig.settle().await; + + assert!(probe.probed().is_empty(), "nothing sent"); + assert!(rig.sent().is_empty(), "and no verdict"); + rig.send(Command::WalletRemoved(wallet())).await; + assert!(probe.probed().is_empty()); + assert!(!rig.actor.wallets.contains_key(&wallet())); + } + + /// A probe job that gets to the gate only after the removal is over — + /// and the id registered anew — finds the registration it was read from + /// replaced: the transaction the old pass captured is not sent for it; + /// the new registration's own pass probes it. + #[tokio::test] + async fn should_not_send_a_removed_registrations_probe_for_its_successor() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + spawn_first_probe(&mut rig, wallet(), 100).await; + + // The whole removal and the new registration, before the job runs. + rig.source.reregister(wallet()); + let probed = rig.actor.jobs.join_next_with_id().await.expect("probe"); + rig.actor.joined(probed); + assert!(probe.probed().is_empty(), "nothing sent for the old one"); + + rig.send(Command::WalletRemoved(wallet())).await; + rig.send(Command::WalletAdded(wallet())).await; + rig.follow(wallet(), 102).await; + assert_eq!(probe.probed(), vec![txid(1)], "the new one's own pass"); + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + } + + /// A registration replaced without the resolver being told (a removal + /// that did not go through the manager): the pass that read the old one + /// sends nothing, what was kept for the old one is dropped — the host + /// told to clear its verdict — and the wallet now there starts afresh at + /// its first event. + #[tokio::test] + async fn should_start_afresh_after_a_registration_changed_unannounced() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); + rig.source.reregister(wallet()); + + rig.settle().await; + assert_eq!(probe.probed().len(), 1, "nothing sent for the old one"); + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + assert!(rig.actor.state.schedules.is_empty()); + + // Idle until the wallet now there shows up in an event. + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + rig.height(wallet(), 103).await; + assert_eq!(probe.probed().len(), 2, "the registration now there"); + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + } + + /// A removal that was asked for and then abandoned leaves the wallet + /// registered: the probe it held back goes out after all, and the pass + /// carries on as if nothing had happened. + #[tokio::test(start_paused = true)] + async fn should_probe_after_an_abandoned_removal() { + let probe = Arc::new(ScriptedProbe::new(&[ + (txid(1), ProbeVerdict::Accepted), + (txid(2), ProbeVerdict::Accepted), + ])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + let teardown = rig.source.generation(wallet()).teardown_guard().await; + spawn_first_probe(&mut rig, wallet(), 100).await; + assert!( + timeout(STAYS_BLOCKED, rig.actor.jobs.join_next_with_id()) + .await + .is_err(), + "held back while the removal is asked for" + ); + + drop(teardown); + rig.settle().await; + + assert_eq!(probe.probed().len(), 2, "both roots of the pass"); + assert_eq!(rig.sent().len(), 2); + } + + /// The other order, against the real manager: a probe is out — no node + /// answering — when its wallet is removed. The removal does not wait for + /// the nodes: the probe stops there, and only then does the wallet go. + #[tokio::test] + async fn should_stop_a_probe_already_out_when_its_wallet_is_removed() { + let (manager, wallet_id) = test_platform_wallet_manager().await; + let platform_wallet = manager.get_wallet(&wallet_id).await.expect("wallet"); + let source = ManagerSource(Arc::clone(platform_wallet.wallet_manager())); + let generation = Arc::clone(platform_wallet.generation()); + let probe = Arc::new(HangingProbe::default()); + let job = { + let probe = Arc::clone(&probe); + let generation = Arc::clone(&generation); + tokio::spawn(async move { + let transaction = tx(1, &[outpoint(90, 0)]); + gated_probe(&source, &*probe, wallet_id, &generation, &transaction).await + }) + }; + timeout(Duration::from_secs(5), probe.started.notified()) + .await + .expect("the probe is out"); + assert!( + generation.lifecycle_is_held(), + "and inside the wallet's gate" + ); + + timeout(Duration::from_secs(5), manager.remove_wallet(&wallet_id)) + .await + .expect("the removal does not wait for the nodes") + .expect("remove"); + + assert!(probe.dropped.load(Ordering::SeqCst), "the probe stopped"); + let outcome = timeout(Duration::from_secs(5), job) + .await + .expect("the job ends once the removal is over") + .expect("job"); + assert!(outcome.is_none(), "with no verdict, and not started over"); + } + + /// Against the real manager: the registration a read names is the + /// wallet's until the manager lets it go; another under the same id is + /// not it; and once removed, its transactions are not probed. + #[tokio::test] + async fn should_not_probe_for_a_registration_the_manager_removed() { + let (manager, wallet_id) = test_platform_wallet_manager().await; + let platform_wallet = manager.get_wallet(&wallet_id).await.expect("wallet"); + let source = ManagerSource(Arc::clone(platform_wallet.wallet_manager())); + let read = source + .read(wallet_id, &AtomicBool::new(true), HashSet::new()) + .await + .expect("registered") + .expect("walked"); + assert!(Arc::ptr_eq(&read.generation, platform_wallet.generation())); + assert!(source.is_registered(wallet_id, &read.generation).await); + let another = Arc::new(WalletGeneration::new()); + assert!(!source.is_registered(wallet_id, &another).await); + let probe = ScriptedProbe::new(&[]); + let transaction = tx(1, &[outpoint(90, 0)]); + + let sent = gated_probe(&source, &probe, wallet_id, &read.generation, &transaction).await; + assert!(sent.is_some(), "probed while registered"); + manager.remove_wallet(&wallet_id).await.expect("remove"); + let refused = gated_probe(&source, &probe, wallet_id, &read.generation, &transaction).await; + + assert!(refused.is_none()); + assert_eq!(probe.probed().len(), 1, "nothing sent after the removal"); + } + + /// An `Uncertain` result forces its root even if a height pass sent it at + /// this height already: one repeat probe is the accepted price of not + /// tracking which report came first. + #[tokio::test] + async fn should_force_a_reported_root_even_if_probed_at_this_height() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + + rig.send(Command::Uncertain(txid(1))).await; + + assert_eq!(probe.probed(), vec![txid(1), txid(1)]); + } + + /// A run whose read finds nothing to do still drops a height queued + /// during it that no root is due at. + #[tokio::test] + async fn should_drop_a_queued_height_when_a_forced_read_finds_nothing() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + // Idle with no verdicts out: nothing to probe, nothing to clear. + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + let walks = rig.walks(); + + // An `Uncertain` for a txid the wallet no longer holds by read time. + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.actor.handle(Command::Uncertain(txid(1))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.views(wallet(), Vec::new()); + rig.actor.joined(listed); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + rig.settle().await; + + assert_eq!(rig.walks(), walks, "idle: neither pass walked the records"); + } + + /// A wallet waiting for nothing (idle) is forced by an `Uncertain` result; + /// a height that arrives while that pass runs is kept, so the new root + /// gets its next-block probe. + #[tokio::test] + async fn should_keep_a_height_that_arrives_during_a_forced_pass() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(2), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.height(wallet(), 102).await; + assert!(probe.probed().is_empty(), "idle: nothing to probe"); + rig.views(wallet(), vec![send(2, &[outpoint(91, 0)])]); + + rig.actor.handle(Command::Uncertain(txid(2))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.actor.joined(listed); + // The forced pass has read the wallet; its probe of root 2 is out. + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 103, + }); + rig.settle().await; + + assert_eq!(probe.probed(), vec![txid(2), txid(2)]); + } + + /// A run that ends early (a probe panicked) leaves the roots it never sent + /// due at once, so the next pass probes them. + #[test] + fn should_keep_due_the_roots_a_run_never_sent() { + let mut state = ResolverState::default(); + let views = [send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])]; + let start = begin_pass( + &mut state, + wallet(), + 100, + &views, + &HashSet::new(), + &HashSet::new(), + Some(100), + ); + let planned: Vec = start.due.iter().map(|root| root.txid).collect(); + assert_eq!(planned.len(), 2); + + // Only the first root goes out before the run ends. + mark_sent(&mut state, wallet(), planned[0], 100, Some(100)); + let again = begin_pass( + &mut state, + wallet(), + 100, + &views, + &HashSet::new(), + &HashSet::new(), + Some(100), + ); + + assert_eq!( + again.due.iter().map(|root| root.txid).collect::>(), + vec![planned[1]] + ); + } + + /// A probe that never produced a verdict (its job panicked) is withdrawn: + /// the root is due again at once, not after the slow interval. + #[test] + fn should_make_a_root_due_at_once_when_its_probe_is_withdrawn() { + let mut state = ResolverState::default(); + let views = [send(1, &[outpoint(90, 0)])]; + begin_pass( + &mut state, + wallet(), + 100, + &views, + &HashSet::new(), + &HashSet::new(), + Some(100), + ); + mark_sent(&mut state, wallet(), txid(1), 100, Some(100)); + assert!(!state.schedules[&(wallet(), txid(1))].is_due(100)); + + withdraw_send(&mut state, wallet(), txid(1)); + + assert!(state.schedules[&(wallet(), txid(1))].is_due(100)); + } + + /// A task that ends only when the test releases it: no stop can finish + /// before then, whatever the executor's timing. + fn held_task() -> (JoinHandle<()>, oneshot::Sender<()>) { + let (release, released) = oneshot::channel::<()>(); + let handle = tokio::spawn(async move { + let _ = released.await; + }); + (handle, release) + } + + fn is_stopping(slot: &Mutex) -> bool { + matches!(*slot.lock().expect("slot"), ActorSlot::Stopping(_)) + } + + fn is_taken(slot: &Mutex) -> bool { + matches!(*slot.lock().expect("slot"), ActorSlot::Closed) + } + + /// Poll `future` exactly once: its output if it is ready then. + async fn poll_once(future: &mut F) -> Option { + tokio::select! { + biased; + output = future => Some(output), + () = std::future::ready(()) => None, + } + } + + /// A safety bound for a stop the test expects to return: a regression + /// that waits on the held task forever fails instead of hanging. + async fn bounded(future: F) -> F::Output { + tokio::time::timeout(Duration::from_secs(5), future) + .await + .expect("the stop returned within its bound") + } + + /// Two stops at once: the second waits for the first and never reports + /// the task gone while it still runs. + #[tokio::test] + async fn should_not_report_a_running_task_as_not_running_to_a_concurrent_stop() { + let (stop, _stopped) = oneshot::channel(); + let (handle, release) = held_task(); + let slot = Mutex::new(ActorSlot::Running(handle, stop)); + let stopping = AsyncMutex::new(()); + + let mut first = Box::pin(stop_slot(&slot, &stopping, Duration::from_secs(5))); + let mut second = Box::pin(stop_slot(&slot, &stopping, Duration::from_secs(5))); + assert!( + poll_once(&mut first).await.is_none(), + "waits on the held task" + ); + assert!( + poll_once(&mut second).await.is_none(), + "waits for the first stop, not reporting the live task gone" + ); + + release.send(()).expect("task alive"); + assert_eq!(bounded(first).await, WorkerStatus::Ok); + assert_eq!(bounded(second).await, WorkerStatus::NotRunning); + } + + /// A stop that runs out of budget keeps the task without aborting it: a + /// retried stop waits for it to end and reports it clean (an aborted task + /// would join as stopped, not Ok). + #[tokio::test] + async fn should_report_a_task_that_outlived_a_stop_clean_on_retry() { + let (stop, _stopped) = oneshot::channel(); + let (handle, release) = held_task(); + let slot = Mutex::new(ActorSlot::Running(handle, stop)); + let stopping = AsyncMutex::new(()); + + // No budget at all: the held task cannot end within it. + let first = bounded(stop_slot(&slot, &stopping, Duration::ZERO)).await; + assert_eq!(first, WorkerStatus::Timeout, "the task is held"); + assert!(is_stopping(&slot), "put back as Stopping"); + + release.send(()).expect("task alive"); + let second = bounded(stop_slot(&slot, &stopping, Duration::from_secs(5))).await; + let third = bounded(stop_slot(&slot, &stopping, Duration::from_secs(5))).await; + + assert_eq!(second, WorkerStatus::Ok); + assert_eq!(third, WorkerStatus::NotRunning); + } + + /// A stop whose caller gives up mid-wait (its future dropped) must not take + /// the task with it: a retried stop still finds it and waits for it. + #[tokio::test] + async fn should_keep_the_task_joinable_when_a_stop_is_cancelled() { + let (stop, _stopped) = oneshot::channel(); + let (handle, release) = held_task(); + let slot = Mutex::new(ActorSlot::Running(handle, stop)); + let stopping = AsyncMutex::new(()); + + // Poll the stop once — it takes the task and waits on it — then drop + // it mid-wait. + let mut cancelled = Box::pin(stop_slot(&slot, &stopping, Duration::from_secs(5))); + assert!( + poll_once(&mut cancelled).await.is_none(), + "the task is held" + ); + assert!( + is_taken(&slot), + "the stop took the task and is waiting on it" + ); + drop(cancelled); + assert!(is_stopping(&slot), "the cancelled stop put the task back"); + + release.send(()).expect("task alive"); + let retried = bounded(stop_slot(&slot, &stopping, Duration::from_secs(5))).await; + + assert_eq!(retried, WorkerStatus::Ok, "the retry joined the live task"); + } + + /// An `Uncertain` result right after launch runs a pass at the stale stored + /// tip; once the wallet follows the tip again, the root still gets its + /// every-block window. + #[tokio::test] + async fn should_give_a_root_forced_before_the_first_height_its_every_block_window() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Uncertain(txid(1))).await; + assert_eq!(probe.probed().len(), 1, "forced at the stored tip (0 here)"); + + rig.follow(wallet(), 500).await; + for height in 502..=505 { + rig.height(wallet(), height).await; + } + + assert_eq!(probe.probed().len(), 6, "every block from 501 to 505"); + } + + /// A return to following the tip does not restart the every-block window + /// of a root already past it: exactly one probe, when the slow interval + /// is up. + #[tokio::test] + async fn should_not_restart_the_window_of_a_root_past_it_after_a_catch_up() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + // Window 101..=124, then 134. + let past = 101 + EVERY_BLOCK_WINDOW + SLOW_INTERVAL; + for height in 102..=past { + rig.height(wallet(), height).await; + } + assert_eq!(probe.probed().len(), EVERY_BLOCK_WINDOW as usize + 1); + + // Away for a while (a catch-up step), then following again. + rig.height(wallet(), past + 10).await; + for height in past + 11..=past + 14 { + rig.height(wallet(), height).await; + } + + assert_eq!( + probe.probed().len(), + EVERY_BLOCK_WINDOW as usize + 2, + "one probe at the first following height past the slow interval" + ); + } + + /// A forced read whose result is joined only after the wallet follows the + /// tip again does not start the window at the stale stored height: the + /// next height-driven pass starts it at the tip, for exactly + /// EVERY_BLOCK_WINDOW blocks. + #[tokio::test] + async fn should_anchor_a_root_read_before_the_wallet_followed_at_the_tip() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.actor.handle(Command::Uncertain(txid(1))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.actor.joined(listed); + for height in [500, 501] { + rig.handle(Command::Height { + wallet_id: wallet(), + height, + }); + } + rig.settle().await; + for height in 502..=530 { + rig.height(wallet(), height).await; + } + + // The forced probe, then 501..=524 every block, nothing up to 530. + assert_eq!(probe.probed().len(), 1 + EVERY_BLOCK_WINDOW as usize); + } + + /// Back from the background with no height event yet, a send reported + /// `Uncertain` is probed at once but its window waits for the wallet to + /// follow the tip again — it does not start at the height from before. + #[tokio::test] + async fn should_not_start_a_window_at_the_height_from_before_the_background() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.send(Command::Uncertain(txid(1))).await; + assert_eq!(probe.probed().len(), 1, "forced at once"); + // 30 blocks later: a catch-up step, then following. + rig.height(wallet(), 131).await; + for height in 132..=136 { + rig.height(wallet(), height).await; + } + + assert_eq!(probe.probed().len(), 1 + 5, "every block from 132"); + } + + /// A height pass queued before the app went to the background, whose read + /// is handled only after the wallet follows the tip again, does not start + /// a window at its old height; the next pass starts it at the tip. + #[tokio::test] + async fn should_not_anchor_a_pass_overtaken_by_a_catch_up() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + + for height in [102, 132, 133] { + rig.handle(Command::Height { + wallet_id: wallet(), + height, + }); + } + rig.settle().await; + for height in 134..=137 { + rig.height(wallet(), height).await; + } + + // Nothing at 102 (stale: catch-up skips it), then every block from 133. + assert_eq!(probe.probed().len(), 5); + } + + /// A height pass whose read finds the wallet far ahead of the height it was + /// queued at — a catch-up whose height step has not arrived yet — is + /// stale: it neither probes by schedule nor starts a window. + #[tokio::test] + async fn should_treat_a_pass_whose_read_is_far_ahead_as_catch_up() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 1_101); + + rig.height(wallet(), 102).await; + assert!( + probe.probed().is_empty(), + "catch-up: no probe during the scan" + ); + rig.height(wallet(), 1_101).await; + for height in 1_102..=1_105 { + rig.height(wallet(), height).await; + } + + assert_eq!(probe.probed().len(), 4, "every block from 1102"); + } + + /// A pass without an anchor — here a forced one during a catch-up — + /// probes only the root the `Uncertain` result forced, not every root + /// due by schedule. + #[tokio::test] + async fn should_probe_only_forced_roots_in_a_pass_without_an_anchor() { + let probe = Arc::new(ScriptedProbe::new(&[ + (txid(1), ProbeVerdict::Accepted), + (txid(2), ProbeVerdict::Accepted), + ])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + rig.follow(wallet(), 100).await; + assert_eq!(probe.probed().len(), 2); + + rig.height(wallet(), 140).await; // catch-up + rig.send(Command::Uncertain(txid(1))).await; + + assert_eq!(probe.probed()[2..], [txid(1)]); + } + + /// A pass whose read finds the wallet behind the height it was queued at — + /// a rewind for a rescan — is confined to forced roots too. + #[tokio::test] + async fn should_treat_a_pass_whose_read_is_behind_as_catch_up() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), ProbeVerdict::Accepted)])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + let before = probe.probed().len(); + + rig.actor.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 50); // rewound + rig.settle().await; + + assert_eq!(probe.probed().len(), before); + } + + /// A catch-up step stops a pass that is still reading and carries nothing + /// forced: its walk over the records could only be thrown away. + #[tokio::test] + async fn should_stop_a_reading_pass_with_nothing_forced_on_a_catch_up() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + // No verdict published: nothing for the pass to clear. + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + let walks = rig.walks(); + + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 140, + }); + rig.settle().await; + + assert_eq!(rig.walks(), walks); + assert!(rig.actor.wallets[&wallet()].run.is_none()); + } + + /// A catch-up takes the anchor from a queued pass that carries a forced + /// txid; the pass still runs, confined to what was forced, and — with the + /// txid no longer held — without walking the records. + #[tokio::test] + async fn should_run_a_forced_pass_overtaken_by_a_catch_up_without_an_anchor() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + // No verdict published: nothing for the pass to clear. + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(9, &[outpoint(99, 0)])], + ); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + let walks = rig.walks(); + + // A height pass holds in its read; an `Uncertain` result and the next + // height queue behind it. + rig.source.pause(true); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + rig.actor.handle(Command::Uncertain(txid(9))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.actor.joined(listed); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 103, + }); + // The forced txid leaves the wallet; then a catch-up. + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 140, + }); + + let run = rig.actor.wallets[&wallet()] + .run + .as_ref() + .expect("the forced pass"); + assert_eq!(run.anchor_at, None); + assert!(!run.walk.load(Ordering::SeqCst)); + rig.source.pause(false); + rig.settle().await; + assert_eq!(rig.walks(), walks, "txid 9 is not held: no walk"); + } + + /// A wallet event that settles a send clears its verdict at once — during + /// a catch-up too — without a pass or a walk over the records, and only + /// that send's. + #[tokio::test] + async fn should_clear_a_verdict_as_soon_as_an_event_settles_its_send() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[ + (txid(1), ProbeVerdict::Accepted), + (txid(2), unresolved()), + ]))) + .await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + rig.follow(wallet(), 100).await; + rig.sent(); + rig.height(wallet(), 140).await; // catch-up + let walks = rig.walks(); + + rig.send(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(1)]), + }) + .await; + + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + assert_eq!(rig.walks(), walks); + assert!(rig.actor.state.published.contains_key(&(wallet(), txid(2)))); + } + + /// Which transactions a wallet event reports settled or gone. + #[test] + fn should_read_settled_and_gone_txids_from_wallet_events() { + let lock = WalletEvent::ChainLockProcessed { + wallet_id: wallet(), + chain_lock: chain_lock(), + locked_transactions: BTreeMap::from([( + AccountType::CoinJoin { index: 0 }, + vec![txid(1)], + )]), + }; + let swept = WalletEvent::TransactionsSwept { + wallet_id: wallet(), + txids: vec![txid(2)], + superseded_by: txid(3), + winner_mined_height: None, + released_outpoints: Vec::new(), + balance: WalletCoreBalance::default(), + account_balances: BTreeMap::new(), + }; + let height = WalletEvent::SyncHeightAdvanced { + wallet_id: wallet(), + height: 100, + }; + + assert!( + settled_or_gone(&lock).is_empty(), + "a chain lock settles nothing new: its block did" + ); + assert_eq!(settled_or_gone(&swept), HashSet::from([txid(2)])); + assert!(settled_or_gone(&height).is_empty()); + assert!( + settled_or_gone(&truncated()).is_empty(), + "a fork's dropped records were settled; its re-queued ones are not" + ); + } + + /// A fork that dropped records above `height = 99`: `txid(4)` removed, + /// `txid(5)` back in the mempool InstantSend-locked. + fn truncated() -> WalletEvent { + WalletEvent::ChainTruncated { + wallet_id: wallet(), + height: 99, + txids: vec![txid(4)], + unconfirmed_txids: vec![txid(5)], + restored_outpoints: Vec::new(), + balance: WalletCoreBalance::default(), + account_balances: BTreeMap::new(), + } + } + + /// A step back (a rewind for a rescan) lowers a queued forced pass to the + /// replayed height: a probe marked above it would keep the root from + /// being due until the rescan passed that height again. + #[tokio::test] + async fn should_lower_a_queued_pass_to_the_height_of_a_rewind() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.source.pause(true); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + rig.actor.handle(Command::Uncertain(txid(1))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.actor.joined(listed); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 103, + }); + + // The rewind: synced height goes back, then the step is reported. + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 50); + rig.actor.handle(Command::Height { + wallet_id: wallet(), + height: 50, + }); + rig.source.pause(false); + rig.settle().await; + + // The forced probe went out at the replayed height, not at 103. + assert_eq!( + rig.actor.state.schedules[&(wallet(), txid(1))].last_probe, + Some(50) + ); + } + + /// A send settled while its probe was out: the event clears it, and the + /// probe's late answer publishes nothing for it again. + #[tokio::test] + async fn should_not_republish_a_send_settled_while_its_probe_was_out() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.sent(); + + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // its probe is out now + rig.handle(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(1)]), + }); + rig.settle().await; + + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + } + + /// A step back (a rewind for a rescan) starts the wallet over: every + /// verdict cleared, every schedule gone, so every root is due at once. + #[tokio::test] + async fn should_start_over_on_a_rewind() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[( + txid(1), + ProbeVerdict::Accepted, + )]))) + .await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(1, 1)])], + ); + rig.follow(wallet(), 100).await; + rig.sent(); + + rig.height(wallet(), 50).await; + + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + assert!(rig.actor.state.schedules.is_empty()); + assert_eq!(rig.actor.wallets[&wallet()].wait_until, None); + } + + fn record(n: u8, context: TransactionContext) -> TransactionRecord { + TransactionRecord::new( + tx(n, &[outpoint(90, 0)]), + AccountType::CoinJoin { index: 0 }, + context, + TransactionType::Standard, + TransactionDirection::Outgoing, + Vec::new(), + Vec::new(), + 0, + ) + } + + fn in_block() -> TransactionContext { + TransactionContext::InBlock(BlockInfo::new(100, BlockHash::all_zeros(), 0)) + } + + /// Block and new-transaction events name only the records that are + /// settled; a mempool record is not. + #[test] + fn should_read_settled_records_from_block_and_transaction_events() { + let block = WalletEvent::BlockProcessed { + wallet_id: wallet(), + height: 100, + chain_lock: None, + inserted: vec![ + record(1, in_block()), + record(2, TransactionContext::Mempool), + ], + updated: vec![record(3, in_block())], + matured: Vec::new(), + balance: WalletCoreBalance::default(), + account_balances: BTreeMap::new(), + addresses_derived: Vec::new(), + }; + let detected = |context| WalletEvent::TransactionDetected { + wallet_id: wallet(), + record: Box::new(record(4, context)), + balance: WalletCoreBalance::default(), + account_balances: BTreeMap::new(), + addresses_derived: Vec::new(), + }; + let txid_of = |n: u8| tx(n, &[outpoint(90, 0)]).txid(); + + assert_eq!( + settled_or_gone(&block), + HashSet::from([txid_of(1), txid_of(3)]) + ); + assert_eq!( + settled_or_gone(&detected(in_block())), + HashSet::from([txid_of(4)]) + ); + assert!(settled_or_gone(&detected(TransactionContext::Mempool)).is_empty()); + } + + /// The lossless channel repeats only that a wallet's records changed: a + /// new transaction wakes it, a height or a chain lock says nothing, and + /// nothing is said while probing is off. + #[test] + fn should_repeat_only_record_changes_from_the_recorded_events() { + let detected = WalletEvent::TransactionDetected { + wallet_id: wallet(), + record: Box::new(record(4, TransactionContext::Mempool)), + balance: WalletCoreBalance::default(), + account_balances: BTreeMap::new(), + addresses_derived: Vec::new(), + }; + assert!(matches!( + recorded_work(&detected, true), + Some(Command::Seen { wallet_id, touched: true }) if wallet_id == wallet() + )); + assert!(recorded_work(&detected, false).is_none()); + let height = WalletEvent::SyncHeightAdvanced { + wallet_id: wallet(), + height: 101, + }; + assert!(recorded_work(&height, true).is_none()); + let chain_lock = WalletEvent::ChainLockProcessed { + wallet_id: wallet(), + chain_lock: ChainLock::dummy(100), + locked_transactions: BTreeMap::new(), + }; + assert!(recorded_work(&chain_lock, true).is_none()); + } + + /// An idle wallet whose new send's event was lost on dash-spv's fan-out + /// is woken by what the lossless channel repeats, and reads the send at + /// the next block. + #[tokio::test] + async fn should_wake_an_idle_wallet_by_what_a_recorded_event_repeats() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![]); + rig.follow(wallet(), 100).await; + assert_eq!(rig.actor.wallets[&wallet()].wait_until, Some(u32::MAX)); + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.height(wallet(), 102).await; + assert!(probe.probed().is_empty(), "idle: the wallet is not read"); + + let detected = WalletEvent::TransactionDetected { + wallet_id: wallet(), + record: Box::new(record(1, TransactionContext::Mempool)), + balance: WalletCoreBalance::default(), + account_balances: BTreeMap::new(), + addresses_derived: Vec::new(), + }; + rig.send(recorded_work(&detected, true).expect("a new transaction is work")) + .await; + rig.height(wallet(), 103).await; + + assert_eq!(probe.probed(), vec![txid(1)]); + } + + /// A reorg brings a settled send back unsettled: a run that starts after + /// the settle event reads it as current, and the send is probed again. + #[tokio::test] + async fn should_probe_a_send_again_after_a_reorg_unsettles_it() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.send(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(1)]), + }) + .await; + rig.sent(); + + // Reorged back to the mempool: the next read lists it again. + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + rig.height(wallet(), 102).await; + + assert_eq!(probe.probed(), vec![txid(1), txid(1)]); + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), unresolved())] + ); + } + + /// A rewind stops a forced pass still reading; what was forced runs in + /// the pass after it, at the rewound height. + #[tokio::test] + async fn should_carry_a_forced_probe_over_a_rewind() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + let before = probe.probed().len(); + rig.source.pause(true); + rig.actor.handle(Command::Uncertain(txid(1))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.actor.joined(listed); // forced pass reading, queued at 101 + + rig.source + .synced + .lock() + .expect("synced") + .insert(wallet(), 50); + rig.actor.handle(Command::Height { + wallet_id: wallet(), + height: 50, + }); + rig.source.pause(false); + rig.settle().await; + + assert_eq!( + probe.probed()[before..], + [txid(1)], + "probed once, after the rewind" + ); + assert_eq!( + rig.actor.state.schedules[&(wallet(), txid(1))].last_probe, + Some(50) + ); + } + + /// A rewind stops the pass probing: the probe out is dropped and the + /// roots still due are not sent. + #[tokio::test] + async fn should_stop_probing_on_a_rewind() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + rig.height(wallet(), 100).await; + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // root 1's probe is out, root 2 due next + + rig.handle(Command::Height { + wallet_id: wallet(), + height: 50, + }); + rig.settle().await; + + assert!(probe.probed().is_empty()); + assert!(rig.actor.wallets[&wallet()].run.is_none()); + assert!(rig.actor.state.schedules.is_empty()); + } + + /// A send settled while the pass is still reading is left out of the read + /// that may still list it: not probed, nothing published. + #[tokio::test] + async fn should_leave_a_send_settled_during_the_read_out_of_the_pass() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(2), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.views(wallet(), vec![send(2, &[outpoint(91, 0)])]); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + + rig.source.pause(true); + rig.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + rig.handle(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(2)]), + }); + rig.source.pause(false); + rig.settle().await; + + assert!(probe.probed().is_empty()); + assert!(rig.sent().is_empty()); + } + + /// Two roots due; while the first is out, the second settles: it is not + /// sent to the network. + #[tokio::test] + async fn should_not_probe_a_queued_root_settled_while_probing() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + rig.height(wallet(), 100).await; + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // root 1 out, root 2 queued + + rig.handle(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(2)]), + }); + rig.settle().await; + + assert_eq!(probe.probed(), vec![txid(1)]); + } + + /// The root whose probe is out settles: that probe is stopped and the + /// pass goes on with the next root. + #[tokio::test] + async fn should_stop_the_probe_of_a_root_that_settles_while_out() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + rig.height(wallet(), 100).await; + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // root 1's probe spawned, not yet run + + rig.handle(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(1)]), + }); + rig.settle().await; + + assert_eq!(probe.probed(), vec![txid(2)]); + } + + /// The first root's probe finishes Accepted, then the root settles before + /// the result is handled: the leftover answer is not recorded against + /// the next root in flight. + #[tokio::test] + async fn should_not_record_a_stopped_probe_answer_against_the_next_root() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[( + txid(1), + ProbeVerdict::Accepted, + )]))) + .await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(91, 0)])], + ); + rig.height(wallet(), 100).await; + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // root 1's probe spawned + + // Its finished answer, held back: completed, not yet handled. + let probed = rig.actor.jobs.join_next_with_id().await.expect("probe"); + assert!(matches!( + &probed, + Ok((_, JobDone::Probed { root, outcome, .. })) + if *root == txid(1) + && outcome.as_ref().map(|outcome| &outcome.verdict) + == Some(&ProbeVerdict::Accepted) + )); + + rig.handle(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(1)]), + }); + rig.actor.joined(probed); + rig.settle().await; + + assert!(!rig + .sent() + .contains(&ResolverEvent::Verdict(txid(2), ProbeVerdict::Accepted))); + assert_ne!( + rig.actor.state.published.get(&(wallet(), txid(2))), + Some(&ProbeVerdict::Accepted) + ); + } + + /// A settle unrelated to a forced send's chain does not probe that send + /// again. + #[tokio::test] + async fn should_not_requeue_a_forced_send_for_an_unrelated_settle() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.views( + wallet(), + vec![ + incoming(5, &[outpoint(80, 0)]), + send(6, &[outpoint(5, 0)]), + incoming(8, &[outpoint(88, 0)]), + ], + ); + rig.actor.handle(Command::Uncertain(txid(6))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.actor.joined(listed); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // root 5 out + + rig.handle(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(8)]), + }); + rig.settle().await; + + assert_eq!(probe.probed(), vec![txid(5)]); + } + + /// A height step queued behind a wallet's removal does not leave an entry + /// behind. + #[tokio::test] + async fn should_not_keep_an_entry_for_a_height_after_removal() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.source.views.lock().expect("views").remove(&wallet()); + + rig.send(Command::WalletRemoved(wallet())).await; + // Straight to the actor: a step queued behind the removal. + rig.actor.handle(Command::Height { + wallet_id: wallet(), + height: 102, + }); + rig.actor.handle(Command::Seen { + wallet_id: wallet(), + touched: true, + }); + rig.settle().await; + + assert!(!rig.actor.wallets.contains_key(&wallet())); + } + + /// A forced send built on two unsettled roots: one of them settling + /// leaves the other as its only root — no new root, no requeue. + #[tokio::test] + async fn should_not_requeue_when_a_settle_leaves_no_new_root() { + let probe = Arc::new(ScriptedProbe::new(&[])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), Vec::new()); + rig.follow(wallet(), 100).await; + rig.views( + wallet(), + vec![ + incoming(1, &[outpoint(80, 0)]), + incoming(2, &[outpoint(81, 0)]), + send(3, &[outpoint(1, 0), outpoint(2, 0)]), + ], + ); + rig.actor.handle(Command::Uncertain(txid(3))); + let listed = rig.actor.jobs.join_next_with_id().await.expect("lookup"); + rig.actor.joined(listed); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // root 1 out, root 2 queued + + rig.handle(Command::Settled { + wallet_id: wallet(), + txids: HashSet::from([txid(1)]), + }); + rig.settle().await; + + assert_eq!(probe.probed(), vec![txid(2)], "root 1 stopped, root 2 once"); + } + + /// An `Uncertain` lookup that lands after its wallet was removed does not + /// bring the wallet's entry back. + #[tokio::test] + async fn should_not_bring_back_a_removed_wallet_from_a_lookup() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.actor.handle(Command::Uncertain(txid(1))); + rig.actor.handle(Command::WalletRemoved(wallet())); + + rig.settle().await; + + assert!(!rig.actor.wallets.contains_key(&wallet())); + } + + /// A same-id wallet registered again starts fresh: a stale idle wait and + /// stale verdicts from before do not carry over. + #[tokio::test] + async fn should_start_a_re_registered_wallet_fresh() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[( + txid(1), + ProbeVerdict::Accepted, + )]))) + .await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + assert_eq!( + rig.sent(), + vec![ResolverEvent::Verdict(txid(1), ProbeVerdict::Accepted)] + ); + assert!(rig.actor.wallets[&wallet()].wait_until.is_some()); + + rig.send(Command::WalletAdded(wallet())).await; + + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + let entry = &rig.actor.wallets[&wallet()]; + assert_eq!(entry.wait_until, None); + assert_eq!(entry.height, None); + } + + /// An echoed send that leaves the wallet is forgotten by the next pass, + /// its echo's verdict cleared. + #[tokio::test] + async fn should_forget_the_echo_of_a_send_that_left() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())]))).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.send(Command::Uncertain(txid(1))).await; + rig.send(Command::Echoed(txid(1))).await; + assert_eq!( + rig.actor.state.published.get(&(wallet(), txid(1))), + Some(&ProbeVerdict::Accepted) + ); + + rig.sent(); + + rig.views(wallet(), Vec::new()); + rig.send(Command::Seen { + wallet_id: wallet(), + touched: true, + }) + .await; + rig.height(wallet(), 102).await; + + assert_eq!(rig.sent(), vec![ResolverEvent::Cleared(txid(1))]); + assert!(rig.actor.state.published.is_empty()); + } + + /// A probe sent before a rewind would answer Accepted: the answer belongs to + /// the state the rewind dropped — the rewind stops it, and nothing is told + /// or recorded. + #[tokio::test] + async fn should_neither_tell_nor_record_a_pre_rewind_answer() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[( + txid(1), + ProbeVerdict::Accepted, + )]))) + .await; + rig.views( + wallet(), + vec![send(1, &[outpoint(90, 0)]), send(2, &[outpoint(1, 1)])], + ); + rig.height(wallet(), 100).await; + rig.handle(Command::Height { + wallet_id: wallet(), + height: 101, + }); + let read = rig.actor.jobs.join_next_with_id().await.expect("read"); + rig.actor.joined(read); // root 1's probe is out + + rig.handle(Command::Height { + wallet_id: wallet(), + height: 50, + }); + rig.settle().await; + + assert!(rig.sent().is_empty()); + assert!(rig.actor.state.published.is_empty()); + } + + /// A probe that panics on its `nth` call and accepts otherwise. + struct PanicOnCall { + nth: usize, + calls: StdMutex, + } + + #[async_trait] + impl AcceptanceProbe for PanicOnCall { + async fn probe(&self, _transaction: &Transaction) -> ProbeOutcome { + let call = { + let mut calls = self.calls.lock().expect("calls"); + *calls += 1; + *calls + }; + if call == self.nth { + panic!("probe bug"); + } + ProbeOutcome::answered(ProbeVerdict::Accepted) + } + } + + /// A root in its slow interval whose probe panicked is probed again on + /// the next block, not SLOW_INTERVAL blocks later. + #[tokio::test] + async fn should_probe_again_on_the_next_block_after_a_panicked_slow_probe() { + let first = 101; + let slow = first + EVERY_BLOCK_WINDOW - 1 + SLOW_INTERVAL; + let probe = Arc::new(PanicOnCall { + nth: EVERY_BLOCK_WINDOW as usize + 1, + calls: StdMutex::new(0), + }); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), first - 1).await; + for height in first + 1..=slow { + rig.height(wallet(), height).await; + } + assert_eq!( + *probe.calls.lock().expect("calls"), + EVERY_BLOCK_WINDOW as usize + 1, + "the slow probe went out and panicked" + ); + + rig.height(wallet(), slow + 1).await; + + assert_eq!( + *probe.calls.lock().expect("calls"), + EVERY_BLOCK_WINDOW as usize + 2 + ); + } + + /// A probe that panics. + struct PanickingProbe; + + #[async_trait] + impl AcceptanceProbe for PanickingProbe { + async fn probe(&self, _transaction: &Transaction) -> ProbeOutcome { + panic!("probe bug"); + } + } + + /// A probe that panics for one transaction and records the others. + struct PanicOn { + txid: Txid, + probed: StdMutex>, + } + + #[async_trait] + impl AcceptanceProbe for PanicOn { + async fn probe(&self, transaction: &Transaction) -> ProbeOutcome { + let txid = transaction.txid(); + if txid == self.txid { + panic!("probe bug"); + } + self.probed.lock().expect("probed").push(txid); + ProbeOutcome::answered(unresolved()) + } + } + + /// A root whose probe panics goes to the back of the queue: a probe that + /// panics every time does not head every pass and end it before the + /// wallet's other roots. + #[tokio::test] + async fn should_not_let_a_panicking_probe_head_every_pass() { + let views = peer_injected(10..12); + let probe = Arc::new(PanicOn { + txid: views[0].transaction.txid(), + probed: StdMutex::new(Vec::new()), + }); + let other = views[2].transaction.txid(); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), views); + rig.follow(wallet(), 100).await; + rig.height(wallet(), 102).await; + + assert!( + probe.probed.lock().expect("probed").contains(&other), + "the other root came round" + ); + } + + /// The same for a reported send's root: its priority does not keep a + /// panicking probe at the head of every pass. + #[tokio::test] + async fn should_not_let_a_panicking_reported_root_head_every_pass() { + let views = peer_injected(10..12); + let probe = Arc::new(PanicOn { + txid: views[0].transaction.txid(), + probed: StdMutex::new(Vec::new()), + }); + let other = views[2].transaction.txid(); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), views); + rig.follow(wallet(), 100).await; + rig.send(Command::Uncertain(txid(70))).await; // the child of root 10 + assert!(rig.actor.reported.contains_key(&txid(70))); + probe.probed.lock().expect("probed").clear(); + rig.height(wallet(), 102).await; + rig.height(wallet(), 103).await; + + assert!( + probe.probed.lock().expect("probed").contains(&other), + "the other root came round" + ); + } + + /// A job that panics ends its run; the wallet's next pass still starts. + #[tokio::test] + async fn should_not_leave_a_wallet_stuck_behind_a_panicked_job() { + let mut rig = enabled(Arc::new(PanickingProbe)).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + + rig.follow(wallet(), 100).await; + rig.height(wallet(), 102).await; + + assert_eq!(rig.walks(), 2); + assert!(rig.actor.wallets[&wallet()].run.is_none()); + } + + /// A root that never settles is probed `MAX_SLOW_PROBES` times after its + /// every-block window and no further: the wallet goes idle, its last + /// verdict stays published, and a new report of the send still probes it. + #[tokio::test] + async fn should_stop_probing_a_root_out_of_slow_probes() { + let probe = Arc::new(ScriptedProbe::new(&[(txid(1), unresolved())])); + let mut rig = enabled(probe.clone()).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + let first = rig.actor.state.schedules[&(wallet(), txid(1))] + .window_start + .expect("the first followed probe starts the window"); + rig.sent(); + + // Through the every-block window, then well past the last slow probe. + let window_end = first + EVERY_BLOCK_WINDOW - 1; + for height in 102..=window_end { + rig.height(wallet(), height).await; + } + let in_window = probe.probed().len(); + for height in window_end + 1..=window_end + (MAX_SLOW_PROBES + 3) * SLOW_INTERVAL { + rig.height(wallet(), height).await; + } + let retired = probe.probed().len(); + assert_eq!(retired, in_window + MAX_SLOW_PROBES as usize); + assert_eq!( + rig.actor.state.schedules[&(wallet(), txid(1))].slow_probes, + MAX_SLOW_PROBES + ); + assert_eq!( + rig.actor.wallets[&wallet()].wait_until, + Some(u32::MAX), + "nothing left to wait for" + ); + assert!(rig.sent().is_empty(), "the verdict stays as published"); + + rig.send(Command::Uncertain(txid(1))).await; + assert_eq!( + probe.probed().len(), + retired + 1, + "a report probes it again" + ); + } + + /// The count is of probes a node answered, after the window: the blocks + /// an app slept through do not use it up, and neither does a probe no + /// node answered or one that was withdrawn — which stay due even for a + /// retired root. It stops at the cap. + #[test] + fn should_retire_a_schedule_by_its_answered_slow_probes() { + let mut schedule = ProbeSchedule::default(); + schedule.anchor(100); + schedule.probed_at(100); + schedule.answered(true, 100); + assert_eq!(schedule.slow_probes, 0, "inside the window"); + // A day of blocks later, never followed: still due, not retired. + assert!(schedule.is_due(100 + 600)); + + let mut height = 100 + 600; + for _ in 0..MAX_SLOW_PROBES { + schedule.probed_at(height); + schedule.answered(true, height); + height += SLOW_INTERVAL; + } + assert_eq!(schedule.next_due(), u32::MAX); + assert!(!schedule.is_due(height)); + assert!(!schedule.is_due(u32::MAX)); + + // A forced probe of the retired root that reached no node is owed. + schedule.probed_at(height); + schedule.answered(false, height); + assert!(schedule.is_due(height + 1)); + assert_eq!(schedule.next_due(), height + 1); + + // Withdrawn (its probe produced nothing): due once more. + schedule.probed_at(height + 1); + schedule.withdraw(); + assert!(schedule.is_due(height + 1)); + + // Answered at last: retired again, the count still at the cap. + schedule.probed_at(height + 1); + schedule.answered(true, height + 1); + assert_eq!(schedule.slow_probes, MAX_SLOW_PROBES); + assert_eq!(schedule.next_due(), u32::MAX); + } + + /// A root probed only by forced passes has no window, so nothing counts. + #[test] + fn should_not_count_probes_of_a_root_without_a_window() { + let mut schedule = ProbeSchedule::default(); + schedule.probed_at(700); + schedule.answered(true, 700); + assert_eq!(schedule.slow_probes, 0); + } + + /// After the every-block window a root is due only every SLOW_INTERVAL + /// blocks: the heights in between do not walk the wallet's records. + #[tokio::test] + async fn should_not_read_the_wallet_on_heights_where_no_root_is_due() { + let mut rig = enabled(Arc::new(ScriptedProbe::new(&[( + txid(1), + ProbeVerdict::Accepted, + )]))) + .await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + for height in 102..=101 + EVERY_BLOCK_WINDOW { + rig.height(wallet(), height).await; + } + let walks = rig.walks(); + + for height in 102 + EVERY_BLOCK_WINDOW..=101 + EVERY_BLOCK_WINDOW + SLOW_INTERVAL { + rig.height(wallet(), height).await; + } + + assert_eq!(rig.walks(), walks + 1, "one walk in SLOW_INTERVAL blocks"); + } + + /// A probe that never answers. + /// A probe that never answers, and reports when it started and when its + /// future was torn down. + #[derive(Default)] + struct HangingProbe { + started: tokio::sync::Notify, + dropped: Arc, + } + + /// Sets its flag when the probe future holding it is dropped. + struct DropFlag(Arc); + + impl Drop for DropFlag { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + + #[async_trait] + impl AcceptanceProbe for HangingProbe { + async fn probe(&self, _transaction: &Transaction) -> ProbeOutcome { + let _torn_down = DropFlag(Arc::clone(&self.dropped)); + self.started.notify_one(); + std::future::pending().await + } + } + + /// Stopping the task aborts and awaits every job it started: by the time + /// the task has ended, the hanging probe's future is gone too. + #[tokio::test] + async fn should_end_with_every_job_when_stopped() { + let probe = Arc::new(HangingProbe::default()); + let mut rig = Rig::new(probe.clone()); + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + let (commands, receiver) = mpsc::unbounded_channel(); + let (stop, stopped) = oneshot::channel(); + let task = tokio::spawn(rig.actor.run(receiver, stopped)); + for command in [Command::SetEnabled(true), Command::Uncertain(txid(1))] { + commands.send(command).expect("send"); + } + timeout(Duration::from_secs(5), probe.started.notified()) + .await + .expect("the probe started"); + assert!( + !probe.dropped.load(Ordering::SeqCst), + "still running before the stop" + ); + + stop.send(()).expect("stop"); + + timeout(Duration::from_secs(5), task) + .await + .expect("stopped in time") + .expect("no panic"); + assert!( + probe.dropped.load(Ordering::SeqCst), + "the probe was torn down before the task ended" + ); + } + + /// A probe that records what the host had been sent by the time it ran. + struct SnapshotProbe { + sink: Arc, + seen: StdMutex>, + } + + #[async_trait] + impl AcceptanceProbe for SnapshotProbe { + async fn probe(&self, _transaction: &Transaction) -> ProbeOutcome { + *self.seen.lock().expect("seen") = self.sink.0.lock().expect("sink").clone(); + ProbeOutcome::answered(ProbeVerdict::Accepted) + } + } + + /// A clear reaches the host before the pass goes out to the network. + #[tokio::test] + async fn should_deliver_clears_before_probing() { + let sink = Arc::new(CollectSink::default()); + let probe = Arc::new(SnapshotProbe { + sink: Arc::clone(&sink), + seen: StdMutex::new(Vec::new()), + }); + let mut rig = Rig::with_sink(probe.clone(), sink); + rig.send(Command::SetEnabled(true)).await; + rig.views(wallet(), vec![send(1, &[outpoint(90, 0)])]); + rig.follow(wallet(), 100).await; + rig.sent(); + + rig.views(wallet(), vec![send(2, &[outpoint(91, 0)])]); + rig.height(wallet(), 102).await; + + assert_eq!( + *probe.seen.lock().expect("seen"), + vec![ResolverEvent::Cleared(txid(1))] + ); + } + + fn chain_lock() -> ChainLock { + ChainLock { + block_height: 100, + block_hash: BlockHash::all_zeros(), + signature: BLSSignature::from([0u8; 96]), + } + } + + /// Only a block that changed records (or a new, locked or swept + /// transaction, or a fork's truncation) may leave work; a chain lock — about one per block for + /// every wallet — only promotes what its block reported. + #[test] + fn should_wake_only_on_events_that_touched_records() { + let block = |inserted: Vec| WalletEvent::BlockProcessed { + wallet_id: wallet(), + height: 100, + chain_lock: None, + inserted, + updated: Vec::new(), + matured: Vec::new(), + balance: WalletCoreBalance::default(), + account_balances: BTreeMap::new(), + addresses_derived: Vec::new(), + }; + let lock = |txids: Vec| WalletEvent::ChainLockProcessed { + wallet_id: wallet(), + chain_lock: chain_lock(), + locked_transactions: BTreeMap::from([(AccountType::CoinJoin { index: 0 }, txids)]), + }; + + assert!(!may_leave_work(&block(Vec::new()))); + assert!(!may_leave_work(&lock(Vec::new()))); + assert!( + !may_leave_work(&lock(vec![txid(1)])), + "a chain lock promotes what its block reported" + ); + assert!(!may_leave_work(&WalletEvent::SyncHeightAdvanced { + wallet_id: wallet(), + height: 100, + })); + assert!( + may_leave_work(&truncated()), + "a send built on a record the fork dropped is unsettled again" + ); + let mut untouched = truncated(); + if let WalletEvent::ChainTruncated { + txids, + unconfirmed_txids, + .. + } = &mut untouched + { + txids.clear(); + unconfirmed_txids.clear(); + } + assert!( + !may_leave_work(&untouched), + "a fork above no record of the wallet leaves nothing to probe" + ); + } + + #[test] + fn should_act_on_forced_txids_only_when_held() { + let held = |txid: &Txid| *txid == self::txid(1); + let forced = HashSet::from([txid(1), txid(2)]); + + assert_eq!( + forced_held(false, forced.clone(), held), + Some(HashSet::from([txid(1)])) + ); + assert_eq!(forced_held(false, HashSet::from([txid(2)]), held), None); + assert_eq!( + forced_held(true, HashSet::from([txid(2)]), held), + Some(HashSet::new()) + ); + } +} diff --git a/packages/rs-platform-wallet/src/wallet/core/generation.rs b/packages/rs-platform-wallet/src/wallet/core/generation.rs index 5224985f838..335b46fe7f4 100644 --- a/packages/rs-platform-wallet/src/wallet/core/generation.rs +++ b/packages/rs-platform-wallet/src/wallet/core/generation.rs @@ -8,9 +8,41 @@ use std::ops::Deref; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use dashcore::{OutPoint, Transaction}; -use tokio::sync::{OwnedRwLockWriteGuard, RwLock, RwLockReadGuard}; +use key_wallet_manager::{WalletId, WalletManager}; +use tokio::sync::{watch, OwnedRwLockWriteGuard, RwLock, RwLockReadGuard}; use super::balance::WalletBalance; +use crate::wallet::platform_wallet::PlatformWalletInfo; + +/// One teardown's place in [`WalletGeneration::teardown_requested`]'s count, +/// given up when dropped — with the guard, or with a +/// [`teardown_guard`](WalletGeneration::teardown_guard) call abandoned while +/// it waits. +#[derive(Debug)] +struct TeardownRequest(watch::Sender); + +impl TeardownRequest { + fn new(leaving: &watch::Sender) -> Self { + leaving.send_modify(|asking| *asking += 1); + Self(leaving.clone()) + } +} + +impl Drop for TeardownRequest { + fn drop(&mut self) { + self.0.send_modify(|asking| *asking -= 1); + } +} + +/// The exclusive side of a generation's lifecycle gate (see +/// [`WalletGeneration::teardown_guard`]). Owned: it may outlive the +/// `Arc` binding it was taken from. +#[derive(Debug)] +#[must_use = "the gate is released as soon as the guard is dropped"] +pub struct TeardownGuard { + _gate: OwnedRwLockWriteGuard<()>, + _request: TeardownRequest, +} /// The state one wallet *generation* shares across every handle that names it. /// @@ -63,6 +95,10 @@ pub struct WalletGeneration { /// a retry loop, and the guard must outlive the loop iteration that produced /// the `Arc` it came from. lifecycle: Arc>, + /// How many teardowns are asking for the gate or hold it (see + /// [`teardown_requested`](Self::teardown_requested)). Shared with their + /// guards, which take themselves off it when dropped. + leaving: watch::Sender, /// Outpoints currently fenced against re-selection because a broadcast /// dispatch owns them ([`pin_in_broadcast`](Self::pin_in_broadcast)). /// @@ -391,6 +427,7 @@ impl WalletGeneration { Self { balance: WalletBalance::new(), lifecycle: Arc::new(RwLock::new(())), + leaving: watch::Sender::new(0), in_broadcast: fences, #[cfg(test)] settle_boundary_hook: SettleBoundaryHook::default(), @@ -444,8 +481,59 @@ impl WalletGeneration { /// Returns an *owned* guard so it can outlive the `Arc` /// binding it was taken from — the remover resolves the current generation in /// a retry loop and must carry the guard out of the iteration that found it. - pub async fn teardown_guard(&self) -> OwnedRwLockWriteGuard<()> { - Arc::clone(&self.lifecycle).write_owned().await + /// + /// From this future's first poll until the guard is dropped — or until + /// the future is, if it is abandoned while still waiting — + /// [`teardown_requested`](Self::teardown_requested) resolves. + pub async fn teardown_guard(&self) -> TeardownGuard { + let request = TeardownRequest::new(&self.leaving); + let gate = Arc::clone(&self.lifecycle).write_owned().await; + TeardownGuard { + _gate: gate, + _request: request, + } + } + + /// Resolves while a teardown is asking for this generation's gate or + /// holds it; pending otherwise. + /// + /// For an operation that holds [`payment_guard`](Self::payment_guard) + /// across network work it can give up without harm: raced against this, it + /// stops and lets go of the gate instead of making the removal wait for + /// its timeouts. Not a liveness check — a teardown that was abandoned + /// leaves the generation registered and this pending again. + pub(crate) async fn teardown_requested(&self) { + let mut leaving = self.leaving.subscribe(); + // Fails only without a sender, which `self` holds: stop then, too. + let _ = leaving.wait_for(|asking| *asking > 0).await; + } + + /// Resolves once no teardown is asking for this generation's gate or + /// holding it: the removal is done — the generation is then no longer + /// registered — or was given up. + pub(crate) async fn teardown_over(&self) { + let mut leaving = self.leaving.subscribe(); + let _ = leaving.wait_for(|asking| *asking == 0).await; + } + + /// Whether some operation is inside the gate right now. + #[cfg(test)] + pub(crate) fn lifecycle_is_held(&self) -> bool { + self.lifecycle.try_write().is_err() + } + + /// Whether this generation is the one registered under `wallet_id` — a + /// removal, or a re-registration under the same id, ends that. A + /// point-in-time observation: to act on it, hold + /// [`payment_guard`](Self::payment_guard) across the check and the action. + pub(crate) fn is_registered_in( + self: &Arc, + wallet_manager: &WalletManager, + wallet_id: &WalletId, + ) -> bool { + wallet_manager + .get_wallet_info(wallet_id) + .is_some_and(|info| Arc::ptr_eq(&info.generation, self)) } /// Recovers from a poisoned mutex rather than panicking: the guarded data @@ -1483,3 +1571,64 @@ mod tests { assert_eq!(fired.load(Ordering::SeqCst), 1); } } + +#[cfg(test)] +mod teardown_request_tests { + use std::time::Duration; + + use tokio::time::timeout; + + use super::*; + + /// Long enough for anything that was going to resolve to have done so. + const SETTLED: Duration = Duration::from_millis(50); + + async fn is_requested(generation: &WalletGeneration) -> bool { + timeout(SETTLED, generation.teardown_requested()) + .await + .is_ok() + } + + /// A teardown is "requested" from the moment it asks for the gate — while + /// it still waits behind a payment operation — until its guard is dropped. + #[tokio::test(start_paused = true)] + async fn should_report_a_teardown_from_its_request_until_its_guard_is_dropped() { + let generation = WalletGeneration::new(); + assert!(!is_requested(&generation).await); + let payment = generation.payment_guard().await; + let mut teardown = Box::pin(generation.teardown_guard()); + assert!( + timeout(SETTLED, &mut teardown).await.is_err(), + "waits behind the payment operation" + ); + + assert!(is_requested(&generation).await, "asked, not yet inside"); + drop(payment); + let guard = teardown.await; + assert!(is_requested(&generation).await, "inside"); + let mut over = Box::pin(generation.teardown_over()); + assert!(timeout(SETTLED, &mut over).await.is_err(), "not over yet"); + drop(guard); + assert!(!is_requested(&generation).await); + over.await; + } + + /// A teardown given up while it waits for the gate leaves no trace: the + /// generation is still registered, and its operations must go on. + #[tokio::test(start_paused = true)] + async fn should_withdraw_the_request_of_an_abandoned_teardown() { + let generation = WalletGeneration::new(); + let _payment = generation.payment_guard().await; + let mut outlasting = Box::pin(generation.teardown_guard()); + assert!(timeout(SETTLED, &mut outlasting).await.is_err()); + { + let mut abandoned = Box::pin(generation.teardown_guard()); + assert!(timeout(SETTLED, &mut abandoned).await.is_err()); + } + assert!(is_requested(&generation).await, "one still asks"); + + drop(outlasting); + + assert!(!is_requested(&generation).await); + } +} diff --git a/packages/rs-platform-wallet/src/wallet/core/mod.rs b/packages/rs-platform-wallet/src/wallet/core/mod.rs index e03363948f2..183c943be97 100644 --- a/packages/rs-platform-wallet/src/wallet/core/mod.rs +++ b/packages/rs-platform-wallet/src/wallet/core/mod.rs @@ -1,6 +1,7 @@ pub mod balance; pub mod balance_handler; mod broadcast; +pub(crate) mod broadcast_resolver; pub mod generation; // Inherent `CoreWallet::sign_message` only — no types to re-export. mod sign_message; @@ -11,9 +12,75 @@ pub mod wallet; pub use balance::WalletBalance; pub use balance_handler::BalanceUpdateHandler; -pub use generation::WalletGeneration; pub(crate) use generation::{InBroadcastFences, InBroadcastPin}; +pub use generation::{TeardownGuard, WalletGeneration}; pub use spend_observer::SpendObservationHandler; pub(crate) use transaction::resolve_source_accounts; pub use transaction::{SignedCoreTransaction, ASSET_LOCK_FUNDING_SOURCES, SEND_FUNDING_SOURCES}; pub use wallet::CoreWallet; + +use key_wallet::managed_account::transaction_record::TransactionRecord; + +/// Is this record owned by a contact's watch-only DashPay chain? +/// +/// A `DashpayExternalAccount` derives its addresses from the +/// **contact's** xpub, so this wallet can observe those outputs but can +/// never sign for them. They are the contact's coins; this wallet only +/// ever pays into them. +/// +/// dashpay/rust-dashcore#926 established exactly that policy at the +/// balance layer, dropping `dashpay_external_accounts` from +/// `ManagedAccountCollection::all_funding_accounts` (and `_mut`), which +/// covers `balance`, `account_balances`, `utxos` and +/// `get_spendable_utxos` in one place. `dashpay_receival_accounts` were +/// deliberately kept: those derive from *our* xpub, so a contact paying +/// into them really is money arriving. +/// +/// The persistence seam is the same rule's second home. Upstream +/// `check_core_transaction` emits **one `TransactionRecord` per matched +/// account** (`key_wallet::transaction_checking::wallet_checker`), so a +/// payment to a contact produces two records sharing one txid: +/// +/// | record's account | `direction` | `net_amount` | +/// |---------------------------|-------------|------------------| +/// | funding (BIP44/BIP32/…) | `Outgoing` | `change - spent` | +/// | `DashpayExternalAccount` | `Incoming` | `+paid` | +/// +/// The external account's record is not *wrong* about its own account — +/// that chain did receive an output. It is wrong as a description of the +/// **wallet**, and the persisted `transactions` row is keyed by txid +/// alone, with no per-account dimension to disambiguate (the +/// `transaction_account_involvements` table is only written for +/// provider-key accounts). Whichever record is stored last therefore +/// defines the row, and the watch-only one — emitted last, because +/// `all_accounts` visits the DashPay accounts after the standard ones — +/// wins: a payment *away* is persisted as an incoming credit, and its +/// output is written into `txos` as a wallet-owned coin no key of ours +/// can spend. +/// +/// So external-account records are excluded from the persist-time +/// projection entirely, exactly as #926 excluded the accounts from +/// balance aggregation. What survives from the same event is everything +/// that is genuinely ours to remember: the address-used flips and +/// highest-used watermarks (so contact address rotation keeps working), +/// the derived-address rows, and `derive_spent_utxos` (so a contact +/// spending an output persisted *before* this fix still clears the +/// stale row). +/// +/// The predicate itself is canonical upstream +/// (`AccountType::is_contact_owned`, dashpay/rust-dashcore#952): its +/// exhaustive match forces any future account type to declare whether +/// its coins are the wallet's or a contact's, so this seam and #926's +/// cannot drift apart. +pub(crate) fn is_contact_watch_only(record: &TransactionRecord) -> bool { + record.account_type.is_contact_owned() +} + +/// Whether `record` spends coins the wallet owns: it recorded input details +/// (entries keyed to inputs that spent the account's outpoints) and the +/// account is not a contact's watch-only chain ([`is_contact_watch_only`]), +/// whose input details record the *contact* spending. The persisted projection's input rule and the +/// broadcast resolver's "own send". +pub(crate) fn record_spends_own_coins(record: &TransactionRecord) -> bool { + !record.input_details.is_empty() && !is_contact_watch_only(record) +} diff --git a/packages/rs-platform-wallet/src/wallet/core/wallet.rs b/packages/rs-platform-wallet/src/wallet/core/wallet.rs index bdc0470445e..b0783cf6b27 100644 --- a/packages/rs-platform-wallet/src/wallet/core/wallet.rs +++ b/packages/rs-platform-wallet/src/wallet/core/wallet.rs @@ -397,8 +397,7 @@ impl CoreWallet { /// observation. pub async fn is_current_generation(&self) -> bool { let wm = self.wallet_manager.read().await; - wm.get_wallet_info(&self.wallet_id) - .is_some_and(|info| Arc::ptr_eq(&info.generation, self.generation())) + self.generation().is_registered_in(&wm, &self.wallet_id) } } diff --git a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs index b92ca970f19..e9040ca0880 100644 --- a/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs +++ b/packages/rs-platform-wallet/src/wallet/identity/network/invitation.rs @@ -739,8 +739,8 @@ impl IdentityWallet { .ok_or_else(|| { PlatformWalletError::InvalidIdentityData( "invitation funding transaction not found (tried both byte orders across \ - repeated attempts); it may not have propagated to the queried DAPI node yet — \ - retry shortly" + repeated attempts); it may not have propagated to the queried DAPI node yet, \ + or the node answered with another transaction — retry shortly" .to_string(), ) }) @@ -970,8 +970,9 @@ fn inspect_funding_tx( invitation: &ParsedInvitation, ) -> Result { // Fail-fast: the fetched tx must actually be the funding tx (either byte - // order). DAPI returns whatever tx matches the id we asked for, so this - // guards a backend that answers with an unrelated tx. + // order). The SDK's getter already reads a reply carrying another + // transaction as a miss; this stays as defense in depth for any other + // source of the transaction. let fetched_txid = transaction.txid().to_string(); let reversed_txid = reverse_txid_hex(&invitation.funding_txid).ok(); if fetched_txid != invitation.funding_txid diff --git a/packages/rs-sdk/src/core/mod.rs b/packages/rs-sdk/src/core/mod.rs index b17ed4f01bf..027ccebb81f 100644 --- a/packages/rs-sdk/src/core/mod.rs +++ b/packages/rs-sdk/src/core/mod.rs @@ -6,4 +6,4 @@ mod dash_core_client; mod transaction; #[cfg(feature = "mocks")] pub use dash_core_client::LowLevelDashCoreClient; -pub use transaction::{CoreTransactionPlacement, FetchedCoreTransaction}; +pub use transaction::{transaction_from_reply, CoreTransactionPlacement, FetchedCoreTransaction}; diff --git a/packages/rs-sdk/src/core/transaction.rs b/packages/rs-sdk/src/core/transaction.rs index 99f9862aa5d..e20debabd46 100644 --- a/packages/rs-sdk/src/core/transaction.rs +++ b/packages/rs-sdk/src/core/transaction.rs @@ -92,28 +92,68 @@ fn decode_block(bytes: &[u8]) -> Result { Block::consensus_decode(&mut &bytes[..]).map_err(|e| Error::CoreError(e.into())) } +/// The transaction a successful `getTransaction` reply for `expected` +/// carries. `Ok(None)` when the reply does not carry it: empty (the node does +/// not know it — the same as gRPC `NOT_FOUND`) or another transaction (a +/// faulty node; never returned in its place — a miss, so a caller retries as +/// it would for one). `Err` for bytes that do not decode. Takes the reply +/// alone, so a caller that executes the request itself keeps the serving +/// node's address. +pub fn transaction_from_reply( + response: &GetTransactionResponse, + expected: &Txid, +) -> Result, Error> { + if response.transaction.is_empty() { + return Ok(None); + } + let transaction = decode_transaction(&response.transaction)?; + if transaction.txid() != *expected { + tracing::warn!( + requested = %expected, + answered = %transaction.txid(), + "getTransaction answered with another transaction; treated as a miss" + ); + return Ok(None); + } + Ok(Some(transaction)) +} + +/// The txid `getTransaction` is asked for, as a reply is checked against. +fn requested_txid(txid: &str) -> Result { + txid.parse() + .map_err(|e| Error::Generic(format!("invalid txid {txid}: {e}"))) +} + impl Sdk { /// Fetch a Core transaction by its id via DAPI `getTransaction`. /// /// `txid` is the transaction id as a hex string (big-endian display form). /// Returns `Ok(Some(..))` with the decoded transaction plus its - /// confirmation/lock metadata; `Ok(None)` when the node does not know the tx - /// (empty response or gRPC `NOT_FOUND`) so the caller can retry with the id - /// byte-reversed; and `Err` for a transient/transport failure that must not - /// be masked by a doomed reversed-id retry. + /// confirmation/lock metadata; `Ok(None)` when the node does not give the + /// tx — it does not know it (empty response or gRPC `NOT_FOUND`), or it + /// answered with another transaction (a faulty node; see + /// [`transaction_from_reply`]) — so the caller can retry, with the id + /// byte-reversed or later on another node; and `Err` for a + /// transient/transport failure that must not be masked by a doomed + /// reversed-id retry, for bytes that do not decode, or for a `txid` that + /// is not a txid (refused before any request). pub async fn get_transaction( &self, txid: &str, ) -> Result, Error> { + let expected = requested_txid(txid)?; let Some(response) = self .fetch_core_transaction(txid, RequestSettings::default()) .await? else { return Ok(None); }; + let Some(transaction) = transaction_from_reply(&response, &expected)? else { + return Ok(None); + }; Ok(Some(FetchedCoreTransaction { - transaction: decode_transaction(&response.transaction)?, + transaction, height: response.height, is_chain_locked: response.is_chain_locked, is_instant_locked: response.is_instant_locked, @@ -132,12 +172,16 @@ impl Sdk { txid: &str, settings: RequestSettings, ) -> Result, Error> { + let expected = requested_txid(txid)?; let Some(response) = self.fetch_core_transaction(txid, settings).await? else { return Ok(None); }; + let Some(transaction) = transaction_from_reply(&response, &expected)? else { + return Ok(None); + }; Ok(Some(CoreTransactionPlacement { - transaction: decode_transaction(&response.transaction)?, + transaction, height: response.height, block_hash: block_hash_from_display_bytes(&response.block_hash), is_chain_locked: response.is_chain_locked, @@ -185,8 +229,9 @@ impl Sdk { decode_block(&response.block).map(Some) } - /// Run `getTransaction`, mapping an unknown transaction (gRPC `NOT_FOUND` - /// or an empty reply) to `Ok(None)` and every other failure to `Err`. + /// Run `getTransaction`, mapping gRPC `NOT_FOUND` to `Ok(None)` and every + /// other failure to `Err`; an empty reply is read by + /// [`transaction_from_reply`]. async fn fetch_core_transaction( &self, txid: &str, @@ -213,9 +258,7 @@ impl Sdk { } }; - if response.transaction.is_empty() { - return Ok(None); - } + // An empty reply is decided by `transaction_from_reply`. Ok(Some(response)) } @@ -456,6 +499,112 @@ impl Sdk { #[cfg(test)] mod tests { use super::*; + use dpp::dashcore::consensus::serialize; + + fn reply(transaction: Vec) -> GetTransactionResponse { + GetTransactionResponse { + transaction, + ..GetTransactionResponse::default() + } + } + + /// Script one `getTransaction` reply for `txid` on a mock SDK. + #[cfg(feature = "mocks")] + async fn sdk_answering(txid: &str, transaction: Vec) -> Sdk { + let sdk = Sdk::new_mock(); + sdk.mock_dapi_client() + .lock() + .await + .expect( + &GetTransactionRequest { + id: txid.to_string(), + }, + &Ok(rs_dapi_client::ExecutionResponse { + inner: reply(transaction), + retries: 0, + address: "http://127.0.0.1:1443".parse().expect("address"), + }), + ) + .expect("expectation"); + sdk + } + + #[cfg(feature = "mocks")] + fn tx(lock_time: u32) -> Transaction { + Transaction { + version: 1, + lock_time, + input: Vec::new(), + output: Vec::new(), + special_transaction_payload: None, + } + } + + /// The getters' contract: the requested transaction is returned; an empty + /// reply or another transaction is a miss; an unparsable txid is refused + /// before any request. + #[cfg(feature = "mocks")] + #[tokio::test] + async fn should_return_only_the_requested_transaction_from_the_getters() { + let wanted = tx(1); + let id = wanted.txid().to_string(); + + let sdk = sdk_answering(&id, serialize(&wanted)).await; + let fetched = sdk.get_transaction(&id).await.expect("ok").expect("found"); + assert_eq!(fetched.transaction, wanted); + let placed = sdk + .get_transaction_placement(&id, RequestSettings::default()) + .await + .expect("ok") + .expect("found"); + assert_eq!(placed.transaction, wanted); + + for answer in [Vec::new(), serialize(&tx(2))] { + let sdk = sdk_answering(&id, answer).await; + assert!(sdk.get_transaction(&id).await.expect("ok").is_none()); + assert!(sdk + .get_transaction_placement(&id, RequestSettings::default()) + .await + .expect("ok") + .is_none()); + } + + let refused = Sdk::new_mock().get_transaction("not a txid").await; + assert!( + matches!(refused, Err(Error::Generic(ref message)) if message.contains("invalid txid")), + "{refused:?}" + ); + } + + /// A reply is read, never trusted: empty or another transaction is a + /// miss, undecodable bytes are an error, only the requested one is + /// returned. + #[test] + fn should_return_only_the_requested_transaction_from_a_reply() { + let tx = |lock_time| Transaction { + version: 1, + lock_time, + input: Vec::new(), + output: Vec::new(), + special_transaction_payload: None, + }; + let wanted = tx(1); + let txid = wanted.txid(); + + assert!(matches!( + transaction_from_reply(&reply(Vec::new()), &txid), + Ok(None) + )); + assert!(transaction_from_reply(&reply(vec![0xff, 0x01]), &txid).is_err()); + assert!(matches!( + transaction_from_reply(&reply(serialize(&tx(2))), &txid), + Ok(None) + )); + assert_eq!( + transaction_from_reply(&reply(serialize(&wanted)), &txid).expect("decodes"), + Some(wanted) + ); + } /// DAPI's display-order bytes come back as the hash's internal order. #[test] diff --git a/packages/rs-sdk/src/sdk.rs b/packages/rs-sdk/src/sdk.rs index 79861fe736f..588e9a60cf1 100644 --- a/packages/rs-sdk/src/sdk.rs +++ b/packages/rs-sdk/src/sdk.rs @@ -396,6 +396,16 @@ impl Sdk { .expect("mock should be created") } + /// The mock DAPI client, for unit tests that script raw transport + /// requests (no proof involved). + #[cfg(all(test, feature = "mocks"))] + pub(crate) fn mock_dapi_client(&self) -> Arc> { + match &self.inner { + SdkInstance::Mock { dapi, .. } => Arc::clone(dapi), + _ => panic!("not a mock sdk"), + } + } + /// Return freshness criteria (height tolerance and time tolerance) for given request method. /// /// Note that if self.metadata_height_tolerance or self.metadata_time_tolerance_ms is None, diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift index 433250e87b8..32cb4006fb3 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManager.swift @@ -455,6 +455,23 @@ public class PlatformWalletManager: ObservableObject { /// Rust. Every native pointer has been copied before publication. @Published public internal(set) var lastDpnsSyncEvent: DpnsSyncEvent? + /// The latest broadcast-probe verdict per unconfirmed send (wallet + txid). + /// Filled only while `setBroadcastProbeEnabled(true)` is on; an entry is + /// removed when its send settles or leaves the wallet, when its wallet is + /// deleted, and all entries when probing is turned off — a removal does + /// not mean the send settled. See + /// `OutgoingTransactionProbeEvent`. + @Published public internal(set) var outgoingTransactionVerdicts: [OutgoingTransactionKey: OutgoingTransactionProbeEvent] = [:] + + /// The most recent broadcast-probe verdict, for observers that react to + /// each one as it arrives. + @Published public internal(set) var lastOutgoingTransactionProbe: OutgoingTransactionProbeEvent? + + /// Per-wallet generation that fences broadcast-probe callbacks queued for + /// the main actor against `deleteWallet`. `nonisolated` so the C + /// trampoline can read it on the Rust thread. + nonisolated let outgoingProbeWalletGeneration = WalletGenerationCounter() + /// Cumulative number of encrypted notes scanned in the **current** /// in-flight shielded sync pass, published once per chunk (~every /// 2048 notes) via the Rust-side progress callback. Nil between @@ -919,6 +936,10 @@ public class PlatformWalletManager: ObservableObject { drainingNativeHandle = h handle = NULL_HANDLE isConfigured = false + // The native side forgets every probe verdict on teardown, and its + // clears are dropped once `shutdownRequested` is set: drop them here. + outgoingTransactionVerdicts = [:] + lastOutgoingTransactionProbe = nil // Poll/reconcile cancellation is independent of admitted local-read // generations. Stop remaining reads and reconcile persistence steps // before the blocking shielded stop. @@ -2647,6 +2668,9 @@ public class PlatformWalletManager: ObservableObject { // with the same deterministic id doesn't inherit a stale banner (the // poller would also prune it, but not until the next tick). dashPayUnlockStatus.removeValue(forKey: walletId) + // Same for broadcast-probe verdicts, including any still queued for + // the main actor. + forgetOutgoingTransactionProbes(walletId: walletId) // A store reconcile in flight for any wallet stops between pages: // its next step would read rows `deleteWalletData` is about to // remove. Coarse on purpose — the cadence re-runs the others. diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerAddressSync.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerAddressSync.swift index 47b17f2bc94..1983f7c7484 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerAddressSync.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerAddressSync.swift @@ -86,6 +86,7 @@ final class PlatformWalletEventHandler: @unchecked Sendable { callbacks.version = UInt32(PLATFORM_WALLET_EVENT_CALLBACKS_EXTENSION_VERSION) callbacks.reserved = 0 callbacks.on_dpns_marketplace_sync_completed_fn = dpnsMarketplaceSyncCompletedCallback + callbacks.on_outgoing_transaction_probed_fn = outgoingTransactionProbedCallback return callbacks } } diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerBroadcastProbe.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerBroadcastProbe.swift new file mode 100644 index 00000000000..efaf888bac9 --- /dev/null +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerBroadcastProbe.swift @@ -0,0 +1,205 @@ +import Foundation +import DashSDKFFI + +/// What the network said about an unconfirmed send whose broadcast outcome +/// was unknown ("Transaction status unknown"). +public enum OutgoingTransactionVerdict: Sendable, Equatable { + /// A node holds the transaction: it is in a mempool or a block. + case accepted + /// No verdict yet; the SDK asks again on later blocks, up to a limit per + /// launch (see `setBroadcastProbeEnabled`). Nothing is published when it + /// stops asking: the verdict stays `unresolved`. A refusal by the nodes asked lands + /// here too: no answer proves a send can never land. + /// `reason` is diagnostic, never user-facing copy. + case unresolved(reason: String) +} + +/// Identifies one unconfirmed send across the wallets of a manager. +public struct OutgoingTransactionKey: Hashable, Sendable { + public let walletId: Data + /// 32 bytes, wire (internal) byte order. + public let txidWire: Data + + public init(walletId: Data, txidWire: Data) { + self.walletId = walletId + self.txidWire = txidWire + } +} + +/// One broadcast-probe verdict. Nothing in the wallet changed because of it. +public struct OutgoingTransactionProbeEvent: Sendable, Equatable { + public let walletId: Data + /// 32 bytes, wire (internal) byte order. + public let txidWire: Data + public let verdict: OutgoingTransactionVerdict + + /// The txid as explorers and the UI display it (byte-reversed hex). + public var txidDisplayHex: String { + txidWire.reversed().map { String(format: "%02x", $0) }.joined() + } + + public var key: OutgoingTransactionKey { + OutgoingTransactionKey(walletId: walletId, txidWire: txidWire) + } + + public init(walletId: Data, txidWire: Data, verdict: OutgoingTransactionVerdict) { + self.walletId = walletId + self.txidWire = txidWire + self.verdict = verdict + } +} + +/// Lock-guarded generation counter per wallet id, safe to read and bump from +/// any thread: `SyncGenerationCounter` for state that is one wallet's. A +/// broadcast-probe callback reads its wallet's generation on the Rust thread +/// that delivers it; `deleteWallet` bumps it once the native wallet is gone, +/// so a verdict or clear still queued for the main actor is dropped there. +/// The native removal returns only after the resolver's task is past it +/// (it waits up to five seconds for a handler the task is held in), so no +/// verdict of the deleted wallet is handed to the callback after the bump +/// and stamped with the new generation. +/// A wallet restored under the same deterministic id keeps the bumped +/// generation: the deleted wallet's queued callbacks stay stale, its own +/// are current. For that reason an entry is never removed: one counter stays +/// per wallet id ever deleted. +final class WalletGenerationCounter: @unchecked Sendable { + private let lock = NSLock() + private var values: [Data: UInt64] = [:] + func current(_ walletId: Data) -> UInt64 { lock.withLock { values[walletId] ?? 0 } } + func bump(_ walletId: Data) { lock.withLock { values[walletId, default: 0] &+= 1 } } +} + +/// C trampoline for +/// `EventHandlerCallbacksExtension.on_outgoing_transaction_probed_fn`. Rust +/// owns every pointer only for this call, so everything is copied before the +/// hop to the main queue. `DispatchQueue.main` is FIFO, so a verdict and a +/// later clear for the same send are applied in the order Rust sent them — +/// separate `Task`s give no such guarantee. The FIFO does not order them +/// against a synchronous `deleteWallet` on the main actor, so the wallet's +/// generation is read here, before the hop, and checked on delivery. +func outgoingTransactionProbedCallback( + context: UnsafeMutableRawPointer?, + walletIdPtr: UnsafePointer?, + txidPtr: UnsafePointer?, + verdictCode: UInt8, + reasonPtr: UnsafePointer? +) { + guard let context, let walletIdPtr, let txidPtr else { return } + let handler = Unmanaged + .fromOpaque(context) + .takeUnretainedValue() + + let walletId = Data(bytes: walletIdPtr, count: 32) + let txidWire = Data(bytes: txidPtr, count: 32) + let generation = handler.manager?.outgoingProbeWalletGeneration.current(walletId) ?? 0 + if verdictCode == UInt8(OUTGOING_PROBE_VERDICT_CLEARED) { + let key = OutgoingTransactionKey(walletId: walletId, txidWire: txidWire) + DispatchQueue.main.async { [weak manager = handler.manager] in + MainActor.assumeIsolated { + manager?.handleOutgoingTransactionCleared(key, generation: generation) + } + } + return + } + let reason = reasonPtr.map { String(cString: $0) } ?? "" + let verdict: OutgoingTransactionVerdict + switch verdictCode { + case UInt8(OUTGOING_PROBE_VERDICT_ACCEPTED): + verdict = .accepted + default: + // UNRESOLVED, and the reserved OUTGOING_PROBE_VERDICT_DEAD, which + // this version never emits. + verdict = .unresolved(reason: reason) + } + let event = OutgoingTransactionProbeEvent( + walletId: walletId, + txidWire: txidWire, + verdict: verdict + ) + + DispatchQueue.main.async { [weak manager = handler.manager] in + MainActor.assumeIsolated { + manager?.handleOutgoingTransactionProbed(event, generation: generation) + } + } +} + +extension PlatformWalletManager { + /// Turn automatic probing of unconfirmed sends on or off (off by default). + /// + /// While on, the SDK resubmits to evonodes over DAPI the root of every + /// chain of unconfirmed sends — the first unsettled transaction the chain + /// depends on, which may be an incoming payment the wallet spent — right + /// after SPV reports a broadcast uncertain (each report: the send itself + /// when it is a root, and of the roots its chain reaches at most 8 per + /// pass, shared by the reports handled together, the rest in their + /// turn), then, while the + /// wallet follows the chain tip, every block for 24 blocks from when it + /// first did so for that root, and every 10 blocks after — a return from + /// the background does not restart the 24 — 55 times at most (about a + /// day of following the tip, counted within one launch), after which only a new uncertain report + /// probes the root and its last verdict stays published; plus at the end of a sync + /// that was catching up (a launch, a reconnect). A root no evonode + /// answered (DAPI unreachable) is due again at every block, and retried a + /// minute later if no block has come (then after 2 and 4 more minutes; + /// at most three retries per block). Each pass probes a bounded number of + /// scheduled roots, in the order they were last probed (or first seen), + /// so with a large backlog a root waits its turn. The roots of sends + /// reported uncertain go ahead of them until those sends settle, taking + /// at most half of a pass while other roots wait. A report, or a retry + /// of probes no evonode answered, makes a pass under way stop after its + /// probe in flight once per block; a later one waits for that pass. + /// It publishes each *change* of + /// verdict for the wallet's own sends in `outgoingTransactionVerdicts` / + /// `lastOutgoingTransactionProbe` (an accepted send stays accepted until + /// its entry is removed). Every verdict + /// is advisory: only the wallet's own sync settles a send. An entry is removed when its send + /// settles or leaves the wallet, when its wallet is deleted, and every + /// entry when probing is turned off; a removal does not mean the send + /// settled. + /// Resubmitting sends the same signed bytes: it can deliver a payment, it + /// can never create a second one. Nothing in the wallet changes. + public func setBroadcastProbeEnabled(_ enabled: Bool) throws { + guard isConfigured, handle != NULL_HANDLE else { + throw PlatformWalletError.invalidHandle("PlatformWalletManager not configured") + } + try platform_wallet_manager_set_broadcast_probe_enabled(handle, enabled).check() + } + + /// The native wallet is gone: drop its verdicts and make every callback + /// still queued for it stale. Rust forgets the wallet's sends on removal, + /// and a restored wallet (same id) must start clean. + @MainActor + func forgetOutgoingTransactionProbes(walletId: Data) { + outgoingProbeWalletGeneration.bump(walletId) + outgoingTransactionVerdicts = outgoingTransactionVerdicts.filter { $0.key.walletId != walletId } + if lastOutgoingTransactionProbe?.walletId == walletId { + lastOutgoingTransactionProbe = nil + } + } + + /// `generation` is the wallet's `outgoingProbeWalletGeneration` as read + /// when the callback arrived; a verdict queued before its wallet was + /// deleted no longer matches and is dropped. + @MainActor + func handleOutgoingTransactionProbed(_ event: OutgoingTransactionProbeEvent, generation: UInt64) { + guard !shutdownRequested, isConfigured, + generation == outgoingProbeWalletGeneration.current(event.walletId) else { return } + outgoingTransactionVerdicts[event.key] = event + lastOutgoingTransactionProbe = event + } + + /// Drop the send's verdict: the send settled or left the wallet, its wallet + /// was removed, or probing was turned off. Not a statement that it settled. + /// A clear queued before its wallet was deleted is dropped like a verdict; + /// the deletion already removed the entry. + @MainActor + func handleOutgoingTransactionCleared(_ key: OutgoingTransactionKey, generation: UInt64) { + guard !shutdownRequested, isConfigured, + generation == outgoingProbeWalletGeneration.current(key.walletId) else { return } + outgoingTransactionVerdicts.removeValue(forKey: key) + if lastOutgoingTransactionProbe?.key == key { + lastOutgoingTransactionProbe = nil + } + } +} diff --git a/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/BroadcastProbeWalletGenerationTests.swift b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/BroadcastProbeWalletGenerationTests.swift new file mode 100644 index 00000000000..b31f07e804c --- /dev/null +++ b/packages/swift-sdk/SwiftTests/SwiftDashSDKTests/BroadcastProbeWalletGenerationTests.swift @@ -0,0 +1,138 @@ +import XCTest +import DashSDKFFI +@testable import SwiftDashSDK + +/// Regression coverage for broadcast-probe callbacks that are queued for the +/// main actor while `deleteWallet` runs on it. +@MainActor +final class BroadcastProbeWalletGenerationTests: XCTestCase { + + private nonisolated static func noOpCalls() -> PlatformWalletNativeTeardownCalls { + let success: PlatformWalletNativeTeardownCalls.Call = { _ in + PlatformWalletFFIResult( + code: PLATFORM_WALLET_FFI_RESULT_CODE_SUCCESS, + message: nil + ) + } + return PlatformWalletNativeTeardownCalls( + spvStop: success, + platformAddressSyncStop: success, + shieldedSyncStop: success, + dashPaySyncStop: success, + dpnsSyncStop: success, + destroy: success + ) + } + + private func makeManager() -> PlatformWalletManager { + PlatformWalletManager.makeForTesting(handle: 72, calls: Self.noOpCalls()) + } + + private func id(_ byte: UInt8) -> Data { Data(repeating: byte, count: 32) } + + /// Deliver a callback the way Rust does, through the C trampoline. It + /// reads the wallet's generation now and queues the rest for the main + /// queue, which cannot run before this actor turn yields. + private func deliver( + _ handler: PlatformWalletEventHandler, + walletId: Data, + txid: Data, + verdictCode: Int + ) { + walletId.withUnsafeBytes { wallet in + txid.withUnsafeBytes { tx in + outgoingTransactionProbedCallback( + context: Unmanaged.passUnretained(handler).toOpaque(), + walletIdPtr: wallet.bindMemory(to: UInt8.self).baseAddress, + txidPtr: tx.bindMemory(to: UInt8.self).baseAddress, + verdictCode: UInt8(verdictCode), + reasonPtr: nil + ) + } + } + } + + /// Returns once everything queued on the main queue so far has run. + private func drainMainQueue() async { + await withCheckedContinuation { continuation in + DispatchQueue.main.async { continuation.resume() } + } + } + + func testShouldPublishAQueuedVerdictAndApplyItsLaterClear() async { + let manager = makeManager() + let handler = PlatformWalletEventHandler(manager: manager) + let key = OutgoingTransactionKey(walletId: id(1), txidWire: id(9)) + + deliver(handler, walletId: id(1), txid: id(9), verdictCode: Int(OUTGOING_PROBE_VERDICT_ACCEPTED)) + await drainMainQueue() + XCTAssertEqual(manager.outgoingTransactionVerdicts[key]?.verdict, .accepted) + XCTAssertEqual(manager.lastOutgoingTransactionProbe?.key, key) + + deliver(handler, walletId: id(1), txid: id(9), verdictCode: Int(OUTGOING_PROBE_VERDICT_CLEARED)) + await drainMainQueue() + XCTAssertNil(manager.outgoingTransactionVerdicts[key]) + XCTAssertNil(manager.lastOutgoingTransactionProbe) + await manager.shutdown() + } + + func testShouldDropAQueuedVerdictWhenItsWalletIsDeletedBeforeDelivery() async { + let manager = makeManager() + let handler = PlatformWalletEventHandler(manager: manager) + + deliver(handler, walletId: id(1), txid: id(9), verdictCode: Int(OUTGOING_PROBE_VERDICT_ACCEPTED)) + manager.forgetOutgoingTransactionProbes(walletId: id(1)) + await drainMainQueue() + + XCTAssertTrue(manager.outgoingTransactionVerdicts.isEmpty) + XCTAssertNil(manager.lastOutgoingTransactionProbe) + await manager.shutdown() + } + + /// A wallet restored from the same mnemonic has the same id, so wallet + /// membership cannot tell its verdicts from the deleted wallet's. + func testShouldNotPublishADeletedWalletsQueuedVerdictForItsRestoredSuccessor() async { + let manager = makeManager() + let handler = PlatformWalletEventHandler(manager: manager) + + deliver(handler, walletId: id(1), txid: id(9), verdictCode: Int(OUTGOING_PROBE_VERDICT_ACCEPTED)) + manager.forgetOutgoingTransactionProbes(walletId: id(1)) + // The successor's own verdict, for another send. + deliver(handler, walletId: id(1), txid: id(8), verdictCode: Int(OUTGOING_PROBE_VERDICT_UNRESOLVED)) + await drainMainQueue() + + let successorKey = OutgoingTransactionKey(walletId: id(1), txidWire: id(8)) + XCTAssertEqual(Array(manager.outgoingTransactionVerdicts.keys), [successorKey]) + XCTAssertEqual(manager.lastOutgoingTransactionProbe?.key, successorKey) + await manager.shutdown() + } + + func testShouldNotLetADeletedWalletsClearRemoveItsSuccessorsVerdict() async { + let manager = makeManager() + let key = OutgoingTransactionKey(walletId: id(1), txidWire: id(9)) + let deletedWalletsGeneration = manager.outgoingProbeWalletGeneration.current(id(1)) + manager.forgetOutgoingTransactionProbes(walletId: id(1)) + let event = OutgoingTransactionProbeEvent(walletId: id(1), txidWire: id(9), verdict: .accepted) + manager.handleOutgoingTransactionProbed( + event, generation: manager.outgoingProbeWalletGeneration.current(id(1))) + + manager.handleOutgoingTransactionCleared(key, generation: deletedWalletsGeneration) + + XCTAssertEqual(manager.outgoingTransactionVerdicts[key], event) + XCTAssertEqual(manager.lastOutgoingTransactionProbe, event) + await manager.shutdown() + } + + func testShouldKeepAnotherWalletsQueuedVerdictWhenAWalletIsDeleted() async { + let manager = makeManager() + let handler = PlatformWalletEventHandler(manager: manager) + let otherKey = OutgoingTransactionKey(walletId: id(2), txidWire: id(9)) + + deliver(handler, walletId: id(2), txid: id(9), verdictCode: Int(OUTGOING_PROBE_VERDICT_ACCEPTED)) + manager.forgetOutgoingTransactionProbes(walletId: id(1)) + await drainMainQueue() + + XCTAssertEqual(manager.outgoingTransactionVerdicts[otherKey]?.verdict, .accepted) + await manager.shutdown() + } +}