From 016b70e76fc75ac19e7ff1964c24ed703ff46058 Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Thu, 29 Oct 2026 12:05:06 +0100 Subject: [PATCH 01/12] fix(platform-wallet): reopen a latched quiesce gate when a sync loop starts A `quiesce()` that times out latches the coordinator's `QuiesceGate` closed so the wedged pass cannot be followed by a fresh one. Only a later successful drain cleared that latch, and `start()` never drains: a restart after a timed-out stop spawned a loop whose every pass bailed on the closed gate, while the host saw `is_running() == true` and no completion event. `QuiesceGate::reopen_if_drained` clears the latch once the wedged pass has released `is_syncing`, and never earlier. Each of the five coordinators calls it in `start()` and before every loop pass, so a loop started behind a still-running wedged pass resumes on its own the moment that pass is gone. Live holds and the seal still close the gate. Closes #5222. Co-Authored-By: Claude Fable 5.1 --- .../src/manager/dashpay_sync.rs | 11 +- .../src/manager/dpns_sync.rs | 8 + .../src/manager/identity_sync.rs | 12 +- .../rs-platform-wallet/src/manager/mod.rs | 84 +++++++++- .../src/manager/platform_address_sync.rs | 88 +++++++++- .../src/manager/shielded_sync.rs | 158 +++++++++++++++++- 6 files changed, 356 insertions(+), 5 deletions(-) diff --git a/packages/rs-platform-wallet/src/manager/dashpay_sync.rs b/packages/rs-platform-wallet/src/manager/dashpay_sync.rs index 88fe1421fab..467a0aec24e 100644 --- a/packages/rs-platform-wallet/src/manager/dashpay_sync.rs +++ b/packages/rs-platform-wallet/src/manager/dashpay_sync.rs @@ -230,6 +230,13 @@ impl DashPaySyncManager { stack_size: NonZeroUsize::new(DASHPAY_SYNC_STACK_BYTES), ..coordinator_worker_config() }; + // A timed-out `quiesce` latched the gate; reopen it once the wedged + // pass has drained (now, or before each pass below). + if !this.quiescing.reopen_if_drained(&this.is_syncing) { + tracing::debug!( + "dashpay sync loop started behind a wedged pass; passes resume once it drains" + ); + } registry.start_thread(WalletWorker::DashPaySync, cfg, move |cancel| { handle.block_on(async move { loop { @@ -237,6 +244,7 @@ impl DashPaySyncManager { break; } + this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; let interval = this.interval(); @@ -298,7 +306,8 @@ impl DashPaySyncManager { /// wedged pass cannot be followed by a fresh one; the caller must /// treat the coordinator as non-clean (shutdown reports it, clear / /// reset paths abort fail-closed). A later successful `quiesce` - /// reopens the gate. + /// reopens the gate, as does [`start`](Self::start) (or the running + /// loop) once the wedged pass has released `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { // The guard drops here, reopening the gate — this is the // "drain only" flavor. diff --git a/packages/rs-platform-wallet/src/manager/dpns_sync.rs b/packages/rs-platform-wallet/src/manager/dpns_sync.rs index 6344c425c53..1ab18b5abd5 100644 --- a/packages/rs-platform-wallet/src/manager/dpns_sync.rs +++ b/packages/rs-platform-wallet/src/manager/dpns_sync.rs @@ -202,6 +202,13 @@ impl DpnsSyncManager { stack_size: NonZeroUsize::new(DPNS_SYNC_STACK_BYTES), ..coordinator_worker_config() }; + // A timed-out `quiesce` latched the gate; reopen it once the wedged + // pass has drained (now, or before each pass below). + if !this.quiescing.reopen_if_drained(&this.is_syncing) { + tracing::debug!( + "dpns sync loop started behind a wedged pass; passes resume once it drains" + ); + } registry.start_thread(WalletWorker::DpnsSync, cfg, move |cancel| { handle.block_on(async move { loop { @@ -209,6 +216,7 @@ impl DpnsSyncManager { break; } + this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; let interval = this.interval(); diff --git a/packages/rs-platform-wallet/src/manager/identity_sync.rs b/packages/rs-platform-wallet/src/manager/identity_sync.rs index 855cefa055b..286a0d8a94d 100644 --- a/packages/rs-platform-wallet/src/manager/identity_sync.rs +++ b/packages/rs-platform-wallet/src/manager/identity_sync.rs @@ -426,6 +426,14 @@ where // thread, and parks any still-draining prior generation — all under // one slot lock, so there is no check-then-spawn gap to race. A // no-op if a worker is already live, or if teardown has begun. + // + // A timed-out `quiesce` latched the gate; reopen it once the wedged + // pass has drained (now, or before each pass below). + if !this.quiescing.reopen_if_drained(&this.is_syncing) { + tracing::debug!( + "identity sync loop started behind a wedged pass; passes resume once it drains" + ); + } registry.start_thread( WalletWorker::IdentitySync, coordinator_worker_config(), @@ -436,6 +444,7 @@ where break; } + this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; let interval = this.interval(); @@ -495,7 +504,8 @@ where /// that path the `quiescing` gate is deliberately **left closed** so the /// wedged pass cannot be followed by a fresh one; the caller must /// treat the coordinator as non-clean. A later successful `quiesce` - /// reopens the gate. + /// reopens the gate, as does [`start`](Self::start) (or the running + /// loop) once the wedged pass has released `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { // The guard drops here, reopening the gate — this is the // "drain only" flavor. diff --git a/packages/rs-platform-wallet/src/manager/mod.rs b/packages/rs-platform-wallet/src/manager/mod.rs index 6ea8ba9b5fd..3f6c172cc5d 100644 --- a/packages/rs-platform-wallet/src/manager/mod.rs +++ b/packages/rs-platform-wallet/src/manager/mod.rs @@ -153,7 +153,9 @@ impl Drop for SyncSlotGuard<'_> { /// mutating; the gate reopens when the last hold drops, /// - the latch — a drain that timed out leaves the gate stuck closed with /// no holder, so the wedged pass cannot be followed by a fresh one; the -/// next *successful* drain clears it, +/// next *successful* drain clears it, as does a loop (re)start once the +/// wedged pass has released `is_syncing` +/// ([`reopen_if_drained`](Self::reopen_if_drained)), /// - [`seal`](Self::seal) — terminal, set by /// [`shutdown`](PlatformWalletManager::shutdown); never reopens, so a /// direct `sync_now` that was already dispatched on a host thread @@ -246,6 +248,30 @@ impl QuiesceGate { self.publish_locked(&bookkeeping); } + /// Clear the timeout latch if the pass it guarded has since drained. + /// + /// Called by a coordinator's `start()` and by its loop before each + /// pass. Without it, a `quiesce` that timed out (the host saw + /// `ErrorShutdownIncomplete`) leaves the latch set until *another* + /// drain succeeds — and a plain restart never drains, so the new loop + /// reports running while every pass bails on the closed gate. + /// + /// Keeps the latch's guarantee: while `is_syncing` is still held, the + /// wedged pass is the only thing that can hold it (a latched gate makes + /// every new pass bail right after its CAS), so the gate stays closed + /// until that pass is gone. A live hold or the seal still closes the + /// gate regardless of the latch. + /// + /// Returns `true` when no latch remains after the call. + fn reopen_if_drained(&self, is_syncing: &std::sync::atomic::AtomicBool) -> bool { + let mut bookkeeping = self.bookkeeping(); + if bookkeeping.latched && !is_syncing.load(std::sync::atomic::Ordering::Acquire) { + bookkeeping.latched = false; + } + self.publish_locked(&bookkeeping); + !bookkeeping.latched + } + /// Drop a hold, reopening the gate only if nothing else closes it. fn release(&self) { let mut bookkeeping = self.bookkeeping(); @@ -1693,6 +1719,62 @@ mod tests { assert!(!gate.is_closed(), "successful drain clears the latch"); } + /// `reopen_if_drained` is what a loop (re)start uses to clear a + /// timeout latch: it must keep the gate closed while the wedged pass + /// still holds `is_syncing`, clear the latch once that pass is gone, + /// and never override a live hold or the seal. + #[tokio::test(start_paused = true)] + async fn reopen_if_drained_keeps_latch_while_pass_in_flight_and_clears_it_after() { + use std::sync::atomic::{AtomicBool, Ordering}; + + let gate = QuiesceGate::default(); + let is_syncing = AtomicBool::new(true); + + assert!( + drain_pass(&gate, &is_syncing, || {}, Duration::from_millis(50)) + .await + .is_none(), + "a wedged pass must time the drain out" + ); + assert!(gate.is_closed(), "timed-out drain leaves the gate latched"); + + // The wedged pass is still running: the latch must hold. + assert!( + !gate.reopen_if_drained(&is_syncing), + "latch must survive while the wedged pass holds is_syncing" + ); + assert!(gate.is_closed()); + + // The wedged pass finished: a restart may clear the latch itself. + is_syncing.store(false, Ordering::Release); + assert!(gate.reopen_if_drained(&is_syncing)); + assert!(!gate.is_closed(), "cleared latch reopens admission"); + + // Idempotent on an open gate. + assert!(gate.reopen_if_drained(&is_syncing)); + assert!(!gate.is_closed()); + + // A live hold still closes the gate even though the latch is clear. + is_syncing.store(true, Ordering::Release); + assert!( + drain_pass(&gate, &is_syncing, || {}, Duration::from_millis(50)) + .await + .is_none(), + "a wedged pass must time the drain out" + ); + is_syncing.store(false, Ordering::Release); + let hold = gate.hold(); + assert!(gate.reopen_if_drained(&is_syncing), "latch is cleared"); + assert!(gate.is_closed(), "but the live hold keeps the gate closed"); + drop(hold); + assert!(!gate.is_closed()); + + // The seal wins over everything. + gate.seal(); + assert!(gate.reopen_if_drained(&is_syncing)); + assert!(gate.is_closed(), "a sealed gate never reopens"); + } + /// `SyncSlotGuard` must clear the `is_syncing` slot on panic unwind, /// not just on normal fall-through. Without this, a pass that panics /// leaves the flag latched and every subsequent `quiesce()` drain diff --git a/packages/rs-platform-wallet/src/manager/platform_address_sync.rs b/packages/rs-platform-wallet/src/manager/platform_address_sync.rs index d0a5bb2eabb..ee5cab6cec7 100644 --- a/packages/rs-platform-wallet/src/manager/platform_address_sync.rs +++ b/packages/rs-platform-wallet/src/manager/platform_address_sync.rs @@ -211,6 +211,14 @@ impl PlatformAddressSyncManager { // The registry owns the whole lifecycle (see `IdentitySyncManager::start`): // it takes the teardown latch, installs the cancellation token, spawns // the thread, and reaps any prior generation under one slot lock. + // + // A timed-out `quiesce` latched the gate; reopen it once the wedged + // pass has drained (now, or before each pass below). + if !this.quiescing.reopen_if_drained(&this.is_syncing) { + tracing::debug!( + "platform-address sync loop started behind a wedged pass; passes resume once it drains" + ); + } registry.start_thread( WalletWorker::PlatformAddressSync, coordinator_worker_config(), @@ -221,6 +229,7 @@ impl PlatformAddressSyncManager { break; } + this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; let interval = this.interval(); @@ -282,7 +291,8 @@ impl PlatformAddressSyncManager { /// that path the `quiescing` gate is deliberately **left closed** so the /// wedged pass cannot be followed by a fresh one; the caller must /// treat the coordinator as non-clean. A later successful `quiesce` - /// reopens the gate. + /// reopens the gate, as does [`start`](Self::start) (or the running + /// loop) once the wedged pass has released `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { // The guard drops here, reopening the gate — this is the // "drain only" flavor. @@ -737,4 +747,80 @@ mod tests { ); assert!(!mgr.is_syncing(), "the slot must be released on the bail"); } + + /// Regression for dashpay/platform#5222: a `quiesce` that timed out + /// latches the gate, and before this fix a later `start()` spawned a + /// loop whose every pass bailed on the closed gate — the host saw a + /// running loop that never synced and never emitted a completion. + /// Now the loop reopens the gate once the wedged pass has drained, + /// and not a moment earlier. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn start_resumes_passes_once_a_timed_out_drains_pass_clears() { + let (mgr, counter) = make_manager(); + + // Wedge a pass, then let a bounded quiesce time out on it. + assert!(mgr.wedge_sync_slot_for_test()); + assert!(!mgr.quiesce_within(Duration::from_millis(50)).await); + assert!( + mgr.quiescing.is_closed(), + "timed-out drain latches the gate" + ); + + // A restart behind the still-wedged pass must not run a pass. The + // loop's first attempt loses the slot CAS to the wedge and sleeps + // an interval; keep that short (the minimum is 1 s). + mgr.set_interval(Duration::from_secs(1)); + Arc::clone(&mgr).start(); + assert!(mgr.is_running()); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(mgr.quiescing.is_closed(), "gate stays latched while wedged"); + assert_eq!( + counter.completions.load(AtomicOrdering::SeqCst), + 0, + "no pass may run behind a wedged one" + ); + + // The wedged pass drains: the running loop reopens the gate and + // its next pass completes. + mgr.is_syncing.store(false, Ordering::Release); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while counter.completions.load(AtomicOrdering::SeqCst) == 0 { + assert!( + tokio::time::Instant::now() < deadline, + "loop never resumed passes after the wedged pass drained" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!mgr.quiescing.is_closed(), "latch is cleared"); + + mgr.stop(); + assert!(mgr.quiesce().await); + } + + /// Same as above, but the wedged pass has already drained by the time + /// `start()` runs: the start itself clears the latch. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn start_reopens_a_latched_gate_when_the_wedged_pass_is_gone() { + let (mgr, counter) = make_manager(); + + assert!(mgr.wedge_sync_slot_for_test()); + assert!(!mgr.quiesce_within(Duration::from_millis(50)).await); + mgr.is_syncing.store(false, Ordering::Release); + assert!(mgr.quiescing.is_closed(), "latch outlives the wedged pass"); + + Arc::clone(&mgr).start(); + assert!(!mgr.quiescing.is_closed(), "start clears a stale latch"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while counter.completions.load(AtomicOrdering::SeqCst) == 0 { + assert!( + tokio::time::Instant::now() < deadline, + "loop never ran a pass after start cleared the latch" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + mgr.stop(); + assert!(mgr.quiesce().await); + } } diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index dac831cb809..be4ece5856a 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -235,6 +235,16 @@ impl ShieldedSyncManager { // can re-persist notes into the store the clear is about to wipe), // installs the cancellation token, spawns the thread, and reaps any // prior generation. No check-then-spawn gap to race. + // + // A `quiesce` that timed out latched the gate closed. Clear it now + // if the wedged pass has since drained; otherwise the loop below + // re-checks before each pass, so admission resumes the moment the + // pass is gone without the host having to stop again. + if !this.quiescing.reopen_if_drained(&this.is_syncing) { + tracing::debug!( + "shielded sync loop started behind a wedged pass; passes resume once it drains" + ); + } registry.start_thread( WalletWorker::ShieldedSync, coordinator_worker_config(), @@ -245,6 +255,8 @@ impl ShieldedSyncManager { break; } + this.quiescing.reopen_if_drained(&this.is_syncing); + // Background-loop cadence — honor the // per-wallet caught-up cooldown so a // sleepy network doesn't refetch + @@ -313,7 +325,8 @@ impl ShieldedSyncManager { /// the wedged pass cannot be followed by a fresh one; the caller must /// treat the coordinator as non-clean (Clear aborts fail-closed, the /// FFI stop surfaces `ErrorShutdownIncomplete`). A later successful - /// `quiesce` reopens the gate. + /// `quiesce` reopens the gate, as does [`start`](Self::start) (or the + /// running loop) once the wedged pass has released `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { // The guard drops here, reopening the gate — this is the // "drain only" flavor. @@ -515,3 +528,146 @@ impl std::fmt::Debug for ShieldedSyncManager { .finish() } } + +#[cfg(test)] +mod tests { + use super::*; + + use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; + + use dash_spv::EventHandler; + + use crate::events::PlatformEventHandler; + + /// Counts `on_shielded_sync_completed` dispatches, standing in for the + /// host's FFI handler. + struct CompletionCounter { + completions: AtomicUsize, + } + + impl EventHandler for CompletionCounter {} + + impl PlatformEventHandler for CompletionCounter { + fn on_shielded_sync_completed(&self, _summary: &ShieldedSyncPassSummary) { + self.completions.fetch_add(1, AtomicOrdering::SeqCst); + } + } + + /// A manager with an empty coordinator slot: `sync_now` still runs the + /// slot → gate → event protocol, which is all these tests exercise. + fn make_manager() -> (Arc, Arc) { + let counter = Arc::new(CompletionCounter { + completions: AtomicUsize::new(0), + }); + let event_manager = Arc::new(PlatformEventManager::new(vec![ + Arc::clone(&counter) as Arc + ])); + ( + Arc::new(ShieldedSyncManager::new( + event_manager, + Arc::new(RwLock::new(None)), + ThreadRegistry::::new(), + )), + counter, + ) + } + + fn wedge_sync_slot(mgr: &ShieldedSyncManager) { + assert!( + mgr.is_syncing + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_ok(), + "test should own the is_syncing slot" + ); + } + + /// Regression for dashpay/platform#5222, traced on this coordinator in + /// Dash Wallet iOS: Sync Info → Stop during a pass longer than the + /// drain budget (`ErrorShutdownIncomplete`), then Sync Now / restart. + /// The loop reported running while every pass bailed on the latched + /// gate. Now the loop reopens the gate as soon as the wedged pass has + /// drained, and not before. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn start_resumes_passes_once_a_timed_out_drains_pass_clears() { + let (mgr, counter) = make_manager(); + + wedge_sync_slot(&mgr); + assert!(!mgr.quiesce_within(Duration::from_millis(50)).await); + assert!( + mgr.quiescing.is_closed(), + "timed-out drain latches the gate" + ); + + // The loop's first attempt loses the slot CAS to the wedge and + // sleeps an interval; keep that short (the minimum is 1 s). + mgr.set_interval(Duration::from_secs(1)); + Arc::clone(&mgr).start(); + assert!(mgr.is_running()); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(mgr.quiescing.is_closed(), "gate stays latched while wedged"); + assert_eq!( + counter.completions.load(AtomicOrdering::SeqCst), + 0, + "no pass may run behind a wedged one" + ); + + mgr.is_syncing.store(false, Ordering::Release); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while counter.completions.load(AtomicOrdering::SeqCst) == 0 { + assert!( + tokio::time::Instant::now() < deadline, + "loop never resumed passes after the wedged pass drained" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!mgr.quiescing.is_closed(), "latch is cleared"); + + mgr.stop(); + assert!(mgr.quiesce().await); + } + + /// The wedged pass is already gone when `start()` runs: the start + /// itself clears the latch. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn start_reopens_a_latched_gate_when_the_wedged_pass_is_gone() { + let (mgr, counter) = make_manager(); + + wedge_sync_slot(&mgr); + assert!(!mgr.quiesce_within(Duration::from_millis(50)).await); + mgr.is_syncing.store(false, Ordering::Release); + assert!(mgr.quiescing.is_closed(), "latch outlives the wedged pass"); + + Arc::clone(&mgr).start(); + assert!(!mgr.quiescing.is_closed(), "start clears a stale latch"); + + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while counter.completions.load(AtomicOrdering::SeqCst) == 0 { + assert!( + tokio::time::Instant::now() < deadline, + "loop never ran a pass after start cleared the latch" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + mgr.stop(); + assert!(mgr.quiesce().await); + } + + /// A manual `sync_now` cannot clear the latch itself: it only ever + /// runs once a `start()` or the loop has observed the drained pass. + #[tokio::test] + async fn sync_now_still_bails_while_latched() { + let (mgr, counter) = make_manager(); + + wedge_sync_slot(&mgr); + assert!(!mgr.quiesce_within(Duration::from_millis(50)).await); + mgr.is_syncing.store(false, Ordering::Release); + + assert!(mgr.sync_now(true).await.is_empty()); + assert_eq!(counter.completions.load(AtomicOrdering::SeqCst), 0); + assert!( + mgr.quiescing.is_closed(), + "a direct pass leaves the latch alone" + ); + } +} From 07ae5566b5fd0f8648e081d85141e986ff56ebfa Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Thu, 29 Oct 2026 12:16:43 +0100 Subject: [PATCH 02/12] fix(platform-wallet): let a shielded sync pass observe cancellation A shielded sync pass had no cancellation point: only the loop's sleep between passes watched the registry token, so `stop()` returned at once but `quiesce()` waited for the whole pass (10 s, then code 27) and `destroy` for up to 37 s on measured iOS teardowns. The manager now owns a pass-cancellation token that `stop()` cancels and rotates, so passes dispatched from host threads (`sync_now(true)`, `sync_wallet`) are cancellable too. The coordinator checks it before every network step, and the note scan selects on it between batches. A cancelled scan drops its in-flight fetches and then finalizes the prefix it consumed exactly as a completed pass would (checkpoint, note saves, spend replay, watermark) and flushes that to the persisters, so the store and the host mirror stay consistent; `quiesce` keeps its barrier by waiting for that finalize. The caught-up cooldown is not stamped for a cancelled pass. Closes #5200. Co-Authored-By: Claude Fable 5.1 --- .../examples/shielded_sync.rs | 4 +- .../examples/shielded_sync_paloma.rs | 4 +- .../src/manager/shielded_sync.rs | 115 +++++- .../src/wallet/shielded/balance_tests.rs | 12 +- .../src/wallet/shielded/coordinator.rs | 82 +++- .../src/wallet/shielded/sync.rs | 386 +++++++++++------- .../src/wallet/shielded/sync/cancel_tests.rs | 217 ++++++++++ 7 files changed, 645 insertions(+), 175 deletions(-) create mode 100644 packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs diff --git a/packages/rs-platform-wallet/examples/shielded_sync.rs b/packages/rs-platform-wallet/examples/shielded_sync.rs index d4e84f45234..5abfc7d8706 100644 --- a/packages/rs-platform-wallet/examples/shielded_sync.rs +++ b/packages/rs-platform-wallet/examples/shielded_sync.rs @@ -267,7 +267,9 @@ async fn run_wallet_sync_test(wallet: WalletIndex) { // --- 6. Run a single sync pass through the coordinator. `force = true` // skips the cooldown gate so the example runs immediately after // the chain comes up. --- - let summary = coordinator.sync(true).await; + let summary = coordinator + .sync(true, &tokio_util::sync::CancellationToken::new()) + .await; eprintln!("{:?}: sync summary: {:?}", wallet, summary); // --- 7. Assert the pass completed cleanly for our wallet: a genuine diff --git a/packages/rs-platform-wallet/examples/shielded_sync_paloma.rs b/packages/rs-platform-wallet/examples/shielded_sync_paloma.rs index 762280b0eb0..364569a58fa 100644 --- a/packages/rs-platform-wallet/examples/shielded_sync_paloma.rs +++ b/packages/rs-platform-wallet/examples/shielded_sync_paloma.rs @@ -258,7 +258,9 @@ async fn main() { .expect("bind_shielded"); // --- 5. Run one sync pass with cooldown bypassed. --- - let summary = coordinator.sync(true).await; + let summary = coordinator + .sync(true, &tokio_util::sync::CancellationToken::new()) + .await; eprintln!("sync summary: {:?}", summary); // --- 6. Assert the pass ran cleanly and actually walked the diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index be4ece5856a..1a998a06070 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -33,6 +33,7 @@ use std::sync::{ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; use dash_async::ThreadRegistry; @@ -160,6 +161,14 @@ pub struct ShieldedSyncManager { /// giving Clear / stop a real "no more host-visible mutations" /// barrier that cancel-only [`stop`](Self::stop) does not provide. quiescing: QuiesceGate, + /// Parent of the cancellation token every pass runs under. The + /// manager owns it rather than reusing the loop's registry token so a + /// pass dispatched directly from a host thread (`sync_now(true)`, + /// `sync_wallet`) is cancellable too: [`stop`](Self::stop) cancels the + /// current parent and rotates in a fresh one, so passes started after + /// the stop are not born cancelled. Held only for a handful of + /// instructions, never across an `.await`. + pass_cancel: std::sync::Mutex, /// Unix seconds of the last completed pass. `0` = never. last_sync_unix: AtomicU64, } @@ -177,10 +186,20 @@ impl ShieldedSyncManager { interval_secs: AtomicU64::new(DEFAULT_SYNC_INTERVAL_SECS), is_syncing: AtomicBool::new(false), quiescing: QuiesceGate::default(), + pass_cancel: std::sync::Mutex::new(CancellationToken::new()), last_sync_unix: AtomicU64::new(0), } } + /// A child of the current pass-cancellation parent. Fires when + /// [`stop`](Self::stop) runs before the pass that took it finishes. + pub(crate) fn pass_token(&self) -> CancellationToken { + self.pass_cancel + .lock() + .expect("shielded pass-cancel mutex poisoned") + .child_token() + } + /// Set the polling interval. Clamped to a minimum of 1s. /// /// The running loop picks this up on its next sleep. @@ -277,15 +296,28 @@ impl ShieldedSyncManager { ); } - /// Stop the background sync loop. No-op if not running. + /// Stop the background sync loop and cancel the in-flight pass, if + /// any. Returns immediately. /// - /// **Cancel-only**: this requests cancellation and returns - /// immediately. A pass already inside `sync_now` / - /// `coordinator.sync()` keeps running to completion (including its - /// persister-callback fan-out). For a real "nothing is running and - /// nothing more will be persisted" barrier — required by Clear, - /// unregister, and rebind — use [`quiesce`](Self::quiesce). + /// A pass already inside `coordinator.sync()` observes the + /// cancellation at its next batch or network boundary, finalizes the + /// prefix it consumed and flushes that to the persisters, then + /// releases `is_syncing`. So this is **not** a barrier: for a real + /// "nothing is running and nothing more will be persisted" guarantee + /// — required by Clear, unregister, and rebind — use + /// [`quiesce`](Self::quiesce), which calls this first and then waits + /// for that finalize. pub fn stop(&self) { + // Cancel the current pass parent and rotate in a fresh one, so a + // pass that starts after this stop runs under an uncancelled token. + let previous = { + let mut parent = self + .pass_cancel + .lock() + .expect("shielded pass-cancel mutex poisoned"); + std::mem::replace(&mut *parent, CancellationToken::new()) + }; + previous.cancel(); self.registry.cancel(WalletWorker::ShieldedSync); } @@ -396,6 +428,13 @@ impl ShieldedSyncManager { // so a failed pass can never wedge `quiesce()`'s drain. let _slot = SyncSlotGuard(&self.is_syncing); + // Take the pass token BEFORE the gate check. `drain_pass` takes its + // hold (closing the gate) and only then calls `stop()` (rotating + // the token), so a pass that took its token before the rotation + // is on the cancelled lineage, and one that took it after sees the + // closed gate below. Either way no pass runs past a drain. + let pass_cancel = self.pass_token(); + // A `quiesce()` may have raised the gate between our CAS and // here; if so, release the slot and bail without running a pass // so the drain can complete and Clear/stop get a true barrier. @@ -419,11 +458,20 @@ impl ShieldedSyncManager { }; let mut summary = if let Some(coordinator) = coordinator_snapshot { - coordinator.sync(force).await + coordinator.sync(force, &pass_cancel).await } else { ShieldedSyncPassSummary::default() }; + // A pass cancelled before it did anything is not a completed pass: + // no timestamp, no completion event (same as losing the slot CAS). + // A cancelled pass that did consume batches has flushed them to the + // persisters, so it is reported like any other so the host applies + // the deltas. + if pass_cancel.is_cancelled() && summary.is_empty() { + return summary; + } + let now = SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) @@ -492,13 +540,16 @@ impl ShieldedSyncManager { // so a failed pass can never wedge `quiesce()`'s drain. let _slot = SyncSlotGuard(&self.is_syncing); + // Token before the gate check — see `sync_now` for the ordering. + let pass_cancel = self.pass_token(); + // Bail if a `quiesce()` raised the gate after our CAS (see // `sync_now`) so the drain barrier holds. if self.quiescing.is_closed() { return Ok(None); } - let pass = coordinator.sync(force).await; + let pass = coordinator.sync(force, &pass_cancel).await; // Extract this wallet's slice from the network-wide pass // summary. If the wallet is registered, we'll get back an @@ -670,4 +721,50 @@ mod tests { "a direct pass leaves the latch alone" ); } + + /// `stop()` cancels the token the in-flight pass runs under and + /// rotates in a fresh one, so a pass started after the stop is not + /// born cancelled (dashpay/platform#5200). + #[tokio::test] + async fn should_cancel_current_pass_token_on_stop_and_rotate() { + let (mgr, _counter) = make_manager(); + + let in_flight = mgr.pass_token(); + assert!(!in_flight.is_cancelled()); + + mgr.stop(); + assert!(in_flight.is_cancelled(), "stop cancels the running pass"); + assert!( + !mgr.pass_token().is_cancelled(), + "a pass after the stop runs under a fresh token" + ); + } + + /// A pass that observes its token lets `quiesce()` drain right away, + /// instead of after the whole pass or the drain budget: the stop + /// inside the drain cancels the pass, the pass releases the slot, the + /// drain sees the falling edge. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn should_quiesce_promptly_when_pass_observes_cancellation() { + let (mgr, _counter) = make_manager(); + + // Stand-in for a pass parked in a network await that checks its + // token: holds the slot until the manager cancels it. + let holder = Arc::clone(&mgr); + let pass = tokio::spawn(async move { + wedge_sync_slot(&holder); + let _slot = SyncSlotGuard(&holder.is_syncing); + holder.pass_token().cancelled().await; + }); + while !mgr.is_syncing() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + + let drained = tokio::time::timeout(Duration::from_secs(2), mgr.quiesce()) + .await + .expect("quiesce must not wait out the drain budget"); + assert!(drained, "the cancelled pass must drain"); + assert!(!mgr.is_syncing()); + pass.await.expect("pass task"); + } } diff --git a/packages/rs-platform-wallet/src/wallet/shielded/balance_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/balance_tests.rs index 30a459ce40f..b39ec66673e 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/balance_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/balance_tests.rs @@ -222,7 +222,9 @@ async fn local_balance_successful_empty_scan_marks_known_zero() { ready(&coordinator, wallet).await.accounts[&0].source, ShieldedBalanceSource::NoHistory ); - let outcome = coordinator.sync(true).await; + let outcome = coordinator + .sync(true, &tokio_util::sync::CancellationToken::new()) + .await; assert_eq!( outcome.success_count(), 1, @@ -249,7 +251,9 @@ async fn local_balance_successful_scan_covers_unhydrated_registration() { coordinator.local_balance_snapshot(wallet).await.unwrap(), ShieldedLocalBalanceState::RestoreIncomplete ); - let outcome = coordinator.sync(true).await; + let outcome = coordinator + .sync(true, &tokio_util::sync::CancellationToken::new()) + .await; assert_eq!(outcome.success_count(), 1, "{outcome:?}"); let account = ready(&coordinator, wallet).await.accounts[&0]; assert_eq!(account.spendable_credits, 0); @@ -300,7 +304,9 @@ async fn local_balance_recovers_after_failed_clear_and_completed_scan() { coordinator.local_balance_snapshot(wallet).await.unwrap(), ShieldedLocalBalanceState::RestoreIncomplete ); - let outcome = coordinator.sync(true).await; + let outcome = coordinator + .sync(true, &tokio_util::sync::CancellationToken::new()) + .await; assert_eq!(outcome.success_count(), 1, "{outcome:?}"); let account = ready(&coordinator, wallet).await.accounts[&0]; assert_eq!(account.spendable_credits, 900); diff --git a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs index 731b35e51ad..90c6d0f7219 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs @@ -55,6 +55,8 @@ use std::collections::{BTreeMap, HashSet}; use std::path::PathBuf; use std::sync::Arc; +use tokio_util::sync::CancellationToken; + /// Callback fired once per chunk during a coordinator sync pass — /// the **"downloaded"** progress signal. /// Arguments: `(cumulative_scanned, latest_block_height)`. Forwarded @@ -1311,10 +1313,20 @@ impl NetworkShieldedCoordinator { /// per-[`WalletId`] and queued through each registered /// [`WalletPersister`]. /// + /// Cancellation: `cancel` is checked before every network step and + /// between note batches. Once it fires, the pass skips what is left + /// of the network work (the rest of the scan, spend re-drives and + /// stranded-reservation releases) but still finalizes and flushes to + /// the host persisters whatever the scan had consumed: a note saved + /// in the store under an advanced watermark is never re-emitted, so + /// an unflushed changeset would leave the host mirror permanently + /// behind. The caught-up cooldown is not stamped for a cancelled + /// pass, so the next pass walks Platform again right away. + /// /// [`sync_notes_across`]: super::sync::sync_notes_across /// [`PlatformEventManager::on_shielded_sync_completed`]: /// crate::events::PlatformEventManager::on_shielded_sync_completed - pub async fn sync(&self, force: bool) -> ShieldedSyncPassSummary { + pub async fn sync(&self, force: bool, cancel: &CancellationToken) -> ShieldedSyncPassSummary { // Network-wide cooldown gate. Snapshot the remaining // window into a local before any await — `std::sync::Mutex` // is `!Send` across await points (clippy's @@ -1353,6 +1365,14 @@ impl NetworkShieldedCoordinator { return summary; } + // A stop that landed before any network work: nothing to finalize, + // so return the empty summary without touching Platform or the + // store (the manager then skips the completion event). + if cancel.is_cancelled() { + tracing::debug!("Coordinator sync cancelled before it started"); + return summary; + } + // Snapshot armed reservations and prefetch Platform's recorded // anchor set BEFORE the note scan. The release pass after the scan // may only act on this pre-scan pair: an anchor already absent from @@ -1384,12 +1404,17 @@ impl NetworkShieldedCoordinator { &subwallets, on_progress.as_ref(), on_tree_progress.as_ref(), + cancel, ) .await { Ok(r) => r, Err(e) => return self.fail_all_wallets(&subwallets, &e), }; + // Everything below that talks to the network is skipped once the + // pass is cancelled; the store-only steps and the host flush of + // what the scan consumed still run (see the method docs). + let cancelled = cancel.is_cancelled(); // Scan-based spend detection now happens INSIDE // `sync_notes_across`: every scanned action's nullifier is // replayed against each subwallet's store as part of the note @@ -1413,8 +1438,9 @@ impl NetworkShieldedCoordinator { // never execute). // // Runs before the balance read so freed notes are reflected in - // this pass's balances. - if let Some((snapshot, recorded)) = stranded_release { + // this pass's balances. Skipped on a cancelled pass: both halves + // broadcast, and a skipped release simply waits for the next pass. + if let Some((snapshot, recorded)) = stranded_release.filter(|_| !cancelled) { // Snapshot the per-wallet persisters BEFORE the loop and drop // the read guard: `redrive_pending_spends` performs network // broadcasts, and holding the persisters lock across those @@ -1470,6 +1496,10 @@ impl NetworkShieldedCoordinator { // enumeration includes guards rehydrated from the durable store. let wallet_ids: HashSet = subwallets.iter().map(|(id, _)| id.wallet_id).collect(); for wallet_id in wallet_ids { + // Network broadcasts: stop between wallets once cancelled. + if cancel.is_cancelled() { + break; + } super::operations::redrive_pending_identity_shields(&self.sdk, &self.store, wallet_id) .await; } @@ -1507,15 +1537,24 @@ impl NetworkShieldedCoordinator { // Cooldown decision based on aggregate activity across // every subwallet — any new commitment scanned or // newly-spent nullifier anywhere on the network clears - // the stamp so the next pass runs immediately. + // the stamp so the next pass runs immediately. A cancelled + // pass proves nothing about being caught up (it may have + // stopped before its first batch), so it leaves the stamp alone. let any_activity = notes.total_scanned > 0 || newly_spent_per_sub.values().any(|&n| n > 0); - if let Ok(mut guard) = self.last_caught_up_at.lock() { - if any_activity { - *guard = None; - } else { - *guard = Some(Instant::now()); + if !cancelled { + if let Ok(mut guard) = self.last_caught_up_at.lock() { + if any_activity { + *guard = None; + } else { + *guard = Some(Instant::now()); + } } } + tracing::debug!( + cancelled, + total_scanned = notes.total_scanned, + "Coordinator sync pass finished" + ); // Demux multi-subwallet results into the per-wallet // `ShieldedSyncSummary` shape that @@ -3703,4 +3742,29 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + + /// A token cancelled before the pass starts: no network call (the + /// mock SDK would otherwise fail every wallet), no store change and + /// no caught-up cooldown stamp (dashpay/platform#5200). + #[tokio::test] + async fn should_skip_network_and_cooldown_when_cancelled_before_scan() { + let dir = temp_dir("cancel_before_scan"); + let coordinator = coordinator_with_one_wallet(&dir).await; + let cancel = CancellationToken::new(); + cancel.cancel(); + + let summary = coordinator.sync(true, &cancel).await; + + assert!( + summary.is_empty(), + "a pass cancelled before it started reports nothing" + ); + assert!( + coordinator.last_caught_up_at.lock().unwrap().is_none(), + "a cancelled pass must not stamp the cooldown" + ); + assert_eq!(coordinator.store().read().await.tree_size().unwrap(), 0); + + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index 4f01ea876ef..a984b319335 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -29,12 +29,14 @@ use std::collections::BTreeMap; use std::sync::Arc; +use dash_sdk::platform::shielded::notes_sync::types::ShieldedChunkBatch; use dash_sdk::platform::shielded::{ sync_shielded_notes_stream, try_decrypt_note, try_recover_outgoing_note, }; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use grovedb_commitment_tree::{ExtractedNoteCommitment, Note as OrchardNote, PaymentAddress}; use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; use tracing::{debug, info}; use super::keys::AccountViewingKeys; @@ -190,17 +192,97 @@ impl MultiSyncNotesResult { /// (FVK for nullifier derivation, IVK for trial decryption). /// No `SpendAuthorizingKey` is needed by sync — the spend /// surface re-attaches it at call time. +/// +/// Cancellation: `cancel` is observed before the store is locked and +/// between batches. A cancelled pass stops pulling batches (dropping +/// the in-flight network fetches) and then finalizes the prefix it +/// consumed exactly as a completed pass would — checkpoint, note saves, +/// spend replay, watermark — so the store and the returned changeset +/// stay consistent with each other. It is a clean early stop, not an +/// error. pub(super) async fn sync_notes_across( sdk: &Arc, store: &Arc>, subwallets: &[(SubwalletId, AccountViewingKeys)], on_progress: Option<&super::coordinator::ShieldedProgressCallback>, on_tree_progress: Option<&super::coordinator::ShieldedTreeProgressCallback>, + cancel: &CancellationToken, ) -> Result { if subwallets.is_empty() { return Ok(MultiSyncNotesResult::default()); } + // Drive the FIRST subwallet's IVK as the streaming driver. Its hits + // come back per batch as `batch.decrypted`; every other subwallet's + // are produced by local trial-decryption against `batch.notes`. + let driver_ivk = subwallets[0].1.prepared_ivk.clone(); + + // Builds the SDK stream once the consumer has snapshotted its start + // position and the pre-stream tree size (both feed the config). + let make_stream = |aligned_start: u64, tree_size: u64| { + // Network-only config carrying the caller's "downloaded" progress + // callback; the SDK fires it once per completed network chunk inside + // the stream. The tree-progress ("checked") callback is owned by + // the consumer and fired there — it never travels through the SDK + // config (the SDK doesn't append to a tree). + // Fetch up to 16 chunks concurrently (default is 4) to parallelize + // across the network's ~13 nodes. The per-chunk cost is dominated by + // server-side proof generation, so more in-flight requests is the main + // client-side lever; the pull-based stream still caps in-flight fetches + // at this bound and keeps memory bounded. + let mut sync_config = dash_sdk::platform::shielded::notes_sync::types::ShieldedSyncConfig { + max_concurrent: 16, + ..Default::default() + }; + if let Some(cb) = on_progress { + // Clamp the SDK's "downloaded" value up to the pre-stream tree leaf + // count before forwarding it to the host. The SDK reports downloaded + // as `aligned_start + scanned`, where `aligned_start` is the rewound + // MIN watermark across subwallets and can sit far below `tree_size` + // (a second subwallet binding at watermark 0 collapses it to 0; a + // partial-chunk-tail resume rewinds it below the tail). The "checked" + // signal is the absolute tree size, so over the gate-skipped re-scan + // region (`global_pos < tree_size`, no new appends) the unclamped + // download value reads *below* checked and breaks the advertised + // `Checked ≤ Downloaded ≤ total` invariant. Everything already in the + // tree was necessarily downloaded on a prior pass, so counting it as + // downloaded is accurate; the clamp is a no-op once the stream + // advances past `tree_size`, and the cold-sync case (tree_size == 0) + // is entirely unaffected. + let cb = cb.clone(); + let tree_baseline = tree_size; + sync_config.on_chunk_completed = Some(Arc::new(move |downloaded: u64, height: u64| { + cb(downloaded.max(tree_baseline), height); + })); + } + sync_shielded_notes_stream(sdk, &driver_ivk, aligned_start, Some(sync_config)) + }; + + sync_notes_from_stream(store, subwallets, on_tree_progress, cancel, make_stream).await +} + +/// The consumer half of [`sync_notes_across`]: snapshots the watermarks, +/// takes the store write lock, drains the batch stream `make_stream` +/// builds for `(aligned_start, tree_size)`, and finalizes the store. +/// +/// Split out so a test can feed a synthetic stream and exercise the +/// cancellation and finalize paths without a network. +pub(super) async fn sync_notes_from_stream( + store: &Arc>, + subwallets: &[(SubwalletId, AccountViewingKeys)], + on_tree_progress: Option<&super::coordinator::ShieldedTreeProgressCallback>, + cancel: &CancellationToken, + make_stream: F, +) -> Result +where + S: ShieldedStore, + St: Stream>, + F: FnOnce(u64, u64) -> St, +{ + if subwallets.is_empty() { + return Ok(MultiSyncNotesResult::default()); + } + // Snapshot each subwallet's watermark and the shared tree's // current leaf count. Two distinct quantities drive the rest // of this pass: @@ -261,45 +343,15 @@ pub(super) async fn sync_notes_across( // (total 0) until the first batch lands. let mut total_target: u64 = tree_size; - // Drive the FIRST subwallet's IVK as the streaming driver. Its hits - // come back per batch as `batch.decrypted`; every other subwallet's - // are produced by local trial-decryption against `batch.notes`. - let (driver_id, driver_views) = &subwallets[0]; - let driver_ivk = driver_views.prepared_ivk.clone(); - // Network-only config carrying the caller's "downloaded" progress - // callback; the SDK fires it once per completed network chunk inside - // the stream. The tree-progress ("checked") callback is owned by - // this consumer and fired below — it never travels through the SDK - // config (the SDK doesn't append to a tree). - // Fetch up to 16 chunks concurrently (default is 4) to parallelize - // across the network's ~13 nodes. The per-chunk cost is dominated by - // server-side proof generation, so more in-flight requests is the main - // client-side lever; the pull-based stream still caps in-flight fetches - // at this bound and keeps memory bounded. - let mut sync_config = dash_sdk::platform::shielded::notes_sync::types::ShieldedSyncConfig { - max_concurrent: 16, - ..Default::default() - }; - if let Some(cb) = on_progress { - // Clamp the SDK's "downloaded" value up to the pre-stream tree leaf - // count before forwarding it to the host. The SDK reports downloaded - // as `aligned_start + scanned`, where `aligned_start` is the rewound - // MIN watermark across subwallets and can sit far below `tree_size` - // (a second subwallet binding at watermark 0 collapses it to 0; a - // partial-chunk-tail resume rewinds it below the tail). The "checked" - // signal is the absolute tree size, so over the gate-skipped re-scan - // region (`global_pos < tree_size`, no new appends) the unclamped - // download value reads *below* checked and breaks the advertised - // `Checked ≤ Downloaded ≤ total` invariant. Everything already in the - // tree was necessarily downloaded on a prior pass, so counting it as - // downloaded is accurate; the clamp is a no-op once the stream - // advances past `tree_size`, and the cold-sync case (tree_size == 0) - // is entirely unaffected. - let cb = cb.clone(); - let tree_baseline = tree_size; - sync_config.on_chunk_completed = Some(Arc::new(move |downloaded: u64, height: u64| { - cb(downloaded.max(tree_baseline), height); - })); + // The FIRST subwallet drives the stream (see `sync_notes_across`); + // its hits arrive pre-decrypted in `batch.decrypted`. + let (driver_id, _) = &subwallets[0]; + + // Nothing has been touched yet, so a stop that landed before the scan + // costs nothing: skip the lock and the fetch entirely. + if cancel.is_cancelled() { + debug!("Shielded note sync cancelled before the scan started"); + return Ok(MultiSyncNotesResult::default()); } // Acquire the store write lock for the whole interleaved consume. @@ -311,9 +363,6 @@ pub(super) async fn sync_notes_across( // network fetches at `max_concurrent`. let mut store = store.write().await; - let stream = sync_shielded_notes_stream(sdk, &driver_ivk, aligned_start, Some(sync_config)); - futures::pin_mut!(stream); - // Route decryptions to the subwallet that owns the IVK, accumulated // across every batch. let mut decrypted_by_subwallet: BTreeMap> = BTreeMap::new(); @@ -403,125 +452,152 @@ pub(super) async fn sync_notes_across( // so `mark_spent`'s by-nullifier lookup still resolves. let mut scanned_nullifiers: Vec<[u8; 32]> = Vec::new(); - while let Some(item) = stream.next().await { - let batch = item.map_err(|e| PlatformWalletError::ShieldedSyncFailed(e.to_string()))?; - // The denominator arrives with the batch (extracted from the - // note-fetch proof). It is stable across a sync; take the max-seen - // so a late chunk proven at a slightly higher block never lowers - // it. Stays 0 (indeterminate) only if no batch is ever produced. - total_target = total_target.max(batch.total_count); - total_notes_scanned += batch.notes.len() as u64; - if !batch.notes.is_empty() { - last_nonempty = Some((batch.start_index, batch.is_partial)); - } + // The stream lives in this block so it — and the up-to-`max_concurrent` + // network fetches it holds in flight — is dropped before the finalize + // below runs, whether the loop ran to the end or was cancelled. + let cancelled = { + let stream = make_stream(aligned_start, tree_size); + futures::pin_mut!(stream); + + loop { + // Batches arrive in tree order as a contiguous prefix, so + // stopping between two of them leaves a shorter but valid scan: + // the finalize checkpoints and watermarks exactly what was + // consumed. `biased` so a pending stop wins over a ready batch. + let item = tokio::select! { + biased; + _ = cancel.cancelled() => { + debug!( + total_scanned = total_notes_scanned, + "Shielded note sync cancelled; finalizing the consumed prefix" + ); + break true; + } + next = stream.next() => match next { + Some(item) => item, + None => break false, + }, + }; + let batch = item.map_err(|e| PlatformWalletError::ShieldedSyncFailed(e.to_string()))?; + // The denominator arrives with the batch (extracted from the + // note-fetch proof). It is stable across a sync; take the max-seen + // so a late chunk proven at a slightly higher block never lowers + // it. Stays 0 (indeterminate) only if no batch is ever produced. + total_target = total_target.max(batch.total_count); + total_notes_scanned += batch.notes.len() as u64; + if !batch.notes.is_empty() { + last_nonempty = Some((batch.start_index, batch.is_partial)); + } - // 1. Append commitments for THIS batch, applying the same - // idempotency gate against the pre-stream `tree_size` - // snapshot. `batch.start_index + i` is the global tree - // position. Each action's `nullifier` is the nullifier of - // the note that action SPENT — collect every one across the - // whole scan for the spend-match replay after receipts are - // persisted. These are collected for ALL positions (even - // those gate-skipped for tree append: a re-fetched chunk can - // still surface a spend whose receipt only persisted on a - // later pass), since `mark_spent` is an idempotent no-op for - // nullifiers the wallet doesn't own or already marked spent. - for (i, raw_note) in batch.notes.iter().enumerate() { - // A malformed nullifier length means the proven note item is corrupt; - // fail fast (consistent with the cmx check below) rather than silently - // dropping it — a dropped nullifier would leave a spent note marked - // unspent. - let nf_bytes = <[u8; 32]>::try_from(raw_note.nullifier.as_slice()).map_err(|_| { - PlatformWalletError::ShieldedSyncFailed("Invalid nullifier length".into()) - })?; - scanned_nullifiers.push(nf_bytes); - let global_pos = batch.start_index + i as u64; - if global_pos < tree_size { - continue; + // 1. Append commitments for THIS batch, applying the same + // idempotency gate against the pre-stream `tree_size` + // snapshot. `batch.start_index + i` is the global tree + // position. Each action's `nullifier` is the nullifier of + // the note that action SPENT — collect every one across the + // whole scan for the spend-match replay after receipts are + // persisted. These are collected for ALL positions (even + // those gate-skipped for tree append: a re-fetched chunk can + // still surface a spend whose receipt only persisted on a + // later pass), since `mark_spent` is an idempotent no-op for + // nullifiers the wallet doesn't own or already marked spent. + for (i, raw_note) in batch.notes.iter().enumerate() { + // A malformed nullifier length means the proven note item is corrupt; + // fail fast (consistent with the cmx check below) rather than silently + // dropping it — a dropped nullifier would leave a spent note marked + // unspent. + let nf_bytes = + <[u8; 32]>::try_from(raw_note.nullifier.as_slice()).map_err(|_| { + PlatformWalletError::ShieldedSyncFailed("Invalid nullifier length".into()) + })?; + scanned_nullifiers.push(nf_bytes); + let global_pos = batch.start_index + i as u64; + if global_pos < tree_size { + continue; + } + let cmx_bytes: [u8; 32] = raw_note.cmx.as_slice().try_into().map_err(|_| { + PlatformWalletError::ShieldedSyncFailed("Invalid cmx length".into()) + })?; + store + .append_commitment(&cmx_bytes, true) + .map_err(|e| PlatformWalletError::ShieldedTreeUpdateFailed(e.to_string()))?; + appended += 1; + leaves_committed += 1; } - let cmx_bytes: [u8; 32] = raw_note.cmx.as_slice().try_into().map_err(|_| { - PlatformWalletError::ShieldedSyncFailed("Invalid cmx length".into()) - })?; - store - .append_commitment(&cmx_bytes, true) - .map_err(|e| PlatformWalletError::ShieldedTreeUpdateFailed(e.to_string()))?; - appended += 1; - leaves_committed += 1; - } - // 2. Fire the tree-progress ("checked") callback once per batch - // (already coarse at ~8192-note batches). `total_target` is - // sourced from this batch's proof above; it is set by the first - // batch and stable thereafter. It is 0 (indeterminate on the - // host) only before any batch lands — which can't happen inside - // this loop since we're holding a batch. - if let Some(cb) = on_tree_progress { - cb(leaves_committed, total_target); - } + // 2. Fire the tree-progress ("checked") callback once per batch + // (already coarse at ~8192-note batches). `total_target` is + // sourced from this batch's proof above; it is set by the first + // batch and stable thereafter. It is 0 (indeterminate on the + // host) only before any batch lands — which can't happen inside + // this loop since we're holding a batch. + if let Some(cb) = on_tree_progress { + cb(leaves_committed, total_target); + } - // 3. Trial-decrypt THIS batch. Driver hits come pre-decrypted - // in `batch.decrypted`; other subwallets via local - // trial-decryption over `batch.notes`. - for dn in batch.decrypted { - decrypted_by_subwallet - .entry(*driver_id) - .or_default() - .push(DiscoveredNote { - position: dn.position, - cmx: dn.cmx, - note: dn.note, - block_height: batch.block_height, - }); - } - for (id, views) in subwallets.iter().skip(1) { - for (i, raw_note) in batch.notes.iter().enumerate() { - let position = batch.start_index + i as u64; - if let Some((note, _addr)) = try_decrypt_note(&views.prepared_ivk, raw_note) { - let cmx_bytes: [u8; 32] = match raw_note.cmx.as_slice().try_into() { - Ok(b) => b, - Err(_) => continue, - }; - decrypted_by_subwallet - .entry(*id) - .or_default() - .push(DiscoveredNote { - position, - cmx: cmx_bytes, - note, - block_height: batch.block_height, - }); + // 3. Trial-decrypt THIS batch. Driver hits come pre-decrypted + // in `batch.decrypted`; other subwallets via local + // trial-decryption over `batch.notes`. + for dn in batch.decrypted { + decrypted_by_subwallet + .entry(*driver_id) + .or_default() + .push(DiscoveredNote { + position: dn.position, + cmx: dn.cmx, + note: dn.note, + block_height: batch.block_height, + }); + } + for (id, views) in subwallets.iter().skip(1) { + for (i, raw_note) in batch.notes.iter().enumerate() { + let position = batch.start_index + i as u64; + if let Some((note, _addr)) = try_decrypt_note(&views.prepared_ivk, raw_note) { + let cmx_bytes: [u8; 32] = match raw_note.cmx.as_slice().try_into() { + Ok(b) => b, + Err(_) => continue, + }; + decrypted_by_subwallet + .entry(*id) + .or_default() + .push(DiscoveredNote { + position, + cmx: cmx_bytes, + note, + block_height: batch.block_height, + }); + } } } - } - // 4. OVK recovery (outgoing/SENT notes). Attempt to open each - // scanned note's `out_ciphertext` with EVERY subwallet's OVK - // (driver included — unlike IVK trial-decrypt, the driver's - // sent notes are NOT pre-recovered by the SDK). A note only - // recovers under the OVK of the wallet that sent it, so this - // is precisely each subwallet's own send history; a received - // note (or another wallet's send, or a dummy output) yields - // `None` and is skipped. Recovered notes are accumulated and - // recorded after this pass's incoming receipts persist (the - // record is `cmx`-idempotent at the store). - for (id, views) in subwallets.iter() { - for raw_note in batch.notes.iter() { - if let Some((note, recipient, memo)) = - try_recover_outgoing_note(&views.outgoing_viewing_key, raw_note) - { - recovered_outgoing_by_subwallet - .entry(*id) - .or_default() - .push(RecoveredOutgoing { - note, - recipient, - memo, - block_height: batch.block_height, - }); + // 4. OVK recovery (outgoing/SENT notes). Attempt to open each + // scanned note's `out_ciphertext` with EVERY subwallet's OVK + // (driver included — unlike IVK trial-decrypt, the driver's + // sent notes are NOT pre-recovered by the SDK). A note only + // recovers under the OVK of the wallet that sent it, so this + // is precisely each subwallet's own send history; a received + // note (or another wallet's send, or a dummy output) yields + // `None` and is skipped. Recovered notes are accumulated and + // recorded after this pass's incoming receipts persist (the + // record is `cmx`-idempotent at the store). + for (id, views) in subwallets.iter() { + for raw_note in batch.notes.iter() { + if let Some((note, recipient, memo)) = + try_recover_outgoing_note(&views.outgoing_viewing_key, raw_note) + { + recovered_outgoing_by_subwallet + .entry(*id) + .or_default() + .push(RecoveredOutgoing { + note, + recipient, + memo, + block_height: batch.block_height, + }); + } } } } - } + }; // Diagnostic mirror of the SDK's one-shot `next_start_index` // (rewinds to the last partial chunk's start). NOT the watermark: @@ -544,6 +620,7 @@ pub(super) async fn sync_notes_across( .map(|v| v.len()) .unwrap_or(0), next_start_index, + cancelled, "SDK stream consumed" ); @@ -1074,3 +1151,8 @@ mod ovk_builder_roundtrip_tests; /// full-decryption and the OVK send-history recovery primitives. #[cfg(test)] mod memo_roundtrip_tests; + +/// Cancellation of a note-sync pass: the consumed prefix is finalized, +/// a pre-scan cancel touches nothing, and a later pass resumes cleanly. +#[cfg(test)] +mod cancel_tests; diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs new file mode 100644 index 00000000000..72958029a46 --- /dev/null +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs @@ -0,0 +1,217 @@ +//! Cancellation of a note-sync pass (dashpay/platform#5200). +//! +//! Feeds `sync_notes_from_stream` a synthetic batch stream so the +//! cancel and finalize paths run without a network. The stream yields +//! real-shaped batches, then a tail future that cancels the token and +//! pends forever — exactly the moment a stop would land between two +//! chunk fetches. + +use std::sync::Arc; + +use dash_sdk::platform::shielded::notes_sync::types::ShieldedChunkBatch; +use drive_proof_verifier::types::ShieldedEncryptedNote; +use futures::{Stream, StreamExt}; +use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; + +use super::{sync_notes_from_stream, MultiSyncNotesResult}; +use crate::wallet::shielded::file_store::FileBackedShieldedStore; +use crate::wallet::shielded::keys::{AccountViewingKeys, OrchardKeySet}; +use crate::wallet::shielded::store::{ShieldedStore, SubwalletId}; + +type BatchItem = Result; + +fn temp_tree_path(tag: &str) -> std::path::PathBuf { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + std::env::temp_dir().join(format!("shielded_sync_cancel_{tag}_{nanos}.sqlite")) +} + +fn one_subwallet() -> Vec<(SubwalletId, AccountViewingKeys)> { + let views = OrchardKeySet::from_seed(&[0x42u8; 64], dashcore::Network::Testnet, 0) + .expect("derive viewing keys") + .viewing_keys(); + vec![(SubwalletId::new([0x11; 32], 0), views)] +} + +/// A batch of `len` notes at `start_index`. The notes are opaque to the +/// scanner: an empty ciphertext never OVK-recovers, the single subwallet +/// is the stream driver so no IVK trial-decryption runs, and a nullifier +/// nobody owns is a no-op in the spend replay. Only the commitment is +/// load-bearing — a distinct small value is a valid Pallas element. +fn batch(start_index: u64, len: u64, total_count: u64) -> ShieldedChunkBatch { + let notes = (0..len) + .map(|i| { + let mut cmx = [0u8; 32]; + let pos = start_index + i; + cmx[0] = (pos as u8) + 1; + cmx[1] = (pos >> 8) as u8; + ShieldedEncryptedNote { + cmx: cmx.to_vec(), + nullifier: vec![0xAA; 32], + cv_net: vec![], + encrypted_note: vec![], + } + }) + .collect(); + ShieldedChunkBatch { + start_index, + notes, + decrypted: vec![], + block_height: 1_000 + start_index, + is_partial: false, + total_count, + } +} + +/// `batches`, then a tail that fires `cancel` when first polled and never +/// yields: the consumer sees the cancellation exactly between batches. +fn batches_then_cancel( + batches: Vec, + cancel: CancellationToken, +) -> impl Stream { + futures::stream::iter(batches.into_iter().map(Ok)).chain(futures::stream::once(async move { + cancel.cancel(); + futures::future::pending::().await + })) +} + +async fn run( + store: &Arc>, + subwallets: &[(SubwalletId, AccountViewingKeys)], + cancel: &CancellationToken, + stream: impl Stream, +) -> MultiSyncNotesResult { + tokio::time::timeout( + std::time::Duration::from_secs(5), + sync_notes_from_stream(store, subwallets, None, cancel, move |_, _| stream), + ) + .await + .expect("a cancelled pass must return promptly") + .expect("pass must succeed") +} + +#[tokio::test] +async fn should_finalize_consumed_prefix_when_cancelled_mid_stream() { + let path = temp_tree_path("prefix"); + let store = Arc::new(RwLock::new( + FileBackedShieldedStore::open_path(&path, 100).expect("open store"), + )); + let subwallets = one_subwallet(); + let sub = subwallets[0].0; + let cancel = CancellationToken::new(); + + let result = run( + &store, + &subwallets, + &cancel, + batches_then_cancel(vec![batch(0, 4, 12), batch(4, 4, 12)], cancel.clone()), + ) + .await; + + // The prefix the pass consumed is finalized like a completed pass. + assert_eq!(result.total_scanned, 8); + let store = store.read().await; + assert_eq!(store.tree_size().unwrap(), 8, "both batches appended"); + assert_eq!( + store.last_synced_note_index(sub).unwrap(), + 8, + "watermark covers exactly the consumed batches" + ); + assert!( + store.witness(7).unwrap().is_some(), + "a checkpoint covers the appended leaves" + ); + + let _ = std::fs::remove_file(&path); +} + +#[tokio::test] +async fn should_return_empty_without_touching_store_when_cancelled_before_scan() { + let path = temp_tree_path("before"); + let store = Arc::new(RwLock::new( + FileBackedShieldedStore::open_path(&path, 100).expect("open store"), + )); + let subwallets = one_subwallet(); + let sub = subwallets[0].0; + let cancel = CancellationToken::new(); + cancel.cancel(); + + let result = sync_notes_from_stream( + &store, + &subwallets, + None, + &cancel, + |_, _| -> futures::stream::Pending { + panic!("a pass cancelled before the scan must not build the stream") + }, + ) + .await + .expect("pass must succeed"); + + assert_eq!(result.total_scanned, 0); + assert!(result.per_subwallet_new_notes.is_empty()); + let store = store.read().await; + assert_eq!(store.tree_size().unwrap(), 0); + assert_eq!(store.last_synced_note_index(sub).unwrap(), 0); + + let _ = std::fs::remove_file(&path); +} + +#[tokio::test] +async fn should_resume_from_partial_watermark_on_next_pass() { + let path = temp_tree_path("resume"); + let store = Arc::new(RwLock::new( + FileBackedShieldedStore::open_path(&path, 100).expect("open store"), + )); + let subwallets = one_subwallet(); + let sub = subwallets[0].0; + + // Pass 1: cancelled after two batches. + let cancel = CancellationToken::new(); + run( + &store, + &subwallets, + &cancel, + batches_then_cancel(vec![batch(0, 4, 12), batch(4, 4, 12)], cancel.clone()), + ) + .await; + assert_eq!(store.read().await.last_synced_note_index(sub).unwrap(), 8); + + // Pass 2: the SDK re-fetches from the chunk-aligned start (0 here), so + // the stream re-delivers the first two batches — gate-skipped by the + // tree-size snapshot — and then the new one. Runs to completion. + let cancel = CancellationToken::new(); + let result = run( + &store, + &subwallets, + &cancel, + futures::stream::iter( + vec![batch(0, 4, 12), batch(4, 4, 12), batch(8, 4, 12)] + .into_iter() + .map(Ok), + ), + ) + .await; + + assert_eq!( + result.total_scanned, 4, + "only the new notes count as scan volume" + ); + let store = store.read().await; + assert_eq!( + store.tree_size().unwrap(), + 12, + "re-delivered positions are not re-appended" + ); + assert_eq!(store.last_synced_note_index(sub).unwrap(), 12); + assert!( + store.witness(11).unwrap().is_some(), + "the second pass checkpoints past the partial one" + ); + assert!(store.witness(7).unwrap().is_some()); + + let _ = std::fs::remove_file(&path); +} From 7e1fac2ae567ff17bd5338e78853f299afaa45e0 Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Thu, 29 Oct 2026 14:40:36 +0100 Subject: [PATCH 03/12] fix(platform-wallet): report nothing for a cancelled pass that scanned nothing `build_per_wallet_summary` mints an all-zero `Ok` for every registered wallet, so the manager's `is_empty()` check could not tell a pass cancelled before its scan from a completed one and still stamped the timestamp and emitted a completion event. The coordinator now hands back the empty summary when a cancelled pass consumed nothing, and the stranded-release prefetch (a read-only RPC) is selected against the token so a stop that lands during it does not wait the request out. Co-Authored-By: Claude Fable 5.1 --- .../src/manager/shielded_sync.rs | 7 ++--- .../src/wallet/shielded/coordinator.rs | 27 ++++++++++++++++++- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index 1a998a06070..25e925cbda2 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -465,9 +465,10 @@ impl ShieldedSyncManager { // A pass cancelled before it did anything is not a completed pass: // no timestamp, no completion event (same as losing the slot CAS). - // A cancelled pass that did consume batches has flushed them to the - // persisters, so it is reported like any other so the host applies - // the deltas. + // The coordinator returns the empty summary for exactly that case + // (a cancelled scan that consumed nothing); one that did consume + // batches has flushed them to the persisters, so it is reported + // like any other pass and the host applies the deltas. if pass_cancel.is_cancelled() && summary.is_empty() { return summary; } diff --git a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs index 90c6d0f7219..430b9f02b1e 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs @@ -1386,7 +1386,14 @@ impl NetworkShieldedCoordinator { // scan are deliberately absent from the snapshot — their anchors // may be newer than this set — and get checked next pass. Skips the // network call entirely when nothing is armed (the common case). - let stranded_release = self.prefetch_stranded_release(&subwallets).await; + // The prefetch is a read-only RPC, so a stop that lands while it is + // in flight simply drops it: the release pass is skipped on a + // cancelled pass anyway, and the scan below bails before locking. + let stranded_release = tokio::select! { + biased; + _ = cancel.cancelled() => None, + prefetched = self.prefetch_stranded_release(&subwallets) => prefetched, + }; // ONE SDK call covers every registered IVK on the network. // Snapshot the optional progress handler installed by the @@ -1414,7 +1421,25 @@ impl NetworkShieldedCoordinator { // Everything below that talks to the network is skipped once the // pass is cancelled; the store-only steps and the host flush of // what the scan consumed still run (see the method docs). + // + // Snapshotted here on purpose: the scan is the step that was or + // was not cut short, and the cooldown decision below is about + // whether that scan reached the tip. A stop that lands during the + // store-only steps after a complete scan does not change that. let cancelled = cancel.is_cancelled(); + // A pass cut short before the scan consumed anything (the stop + // landed in the prefetch, or before the store lock) has nothing to + // flush and nothing to report: hand back the empty summary so the + // manager treats it as "no pass ran" — `build_per_wallet_summary` + // would otherwise mint an all-zero `Ok` for every registered + // wallet, and the host would see a completion event for no work. + if cancelled + && notes.total_scanned == 0 + && crate::changeset::merge::Merge::is_empty(¬es.changeset) + { + tracing::debug!("Coordinator sync cancelled before the scan consumed anything"); + return summary; + } // Scan-based spend detection now happens INSIDE // `sync_notes_across`: every scanned action's nullifier is // replayed against each subwallet's store as part of the note From 26593f7bf5f91235b7b5bbc808f5e89cab73042b Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Thu, 29 Oct 2026 16:54:41 +0100 Subject: [PATCH 04/12] fix(platform-wallet): never rewind a subwallet watermark on a cancelled pass A cancelled pass finalizes `aligned_start + scanned`, and `aligned_start` is the union MIN watermark, so a subwallet that was already further along could be written backwards. Finalize `max(prior, new)` and leave an uncovered subwallet's row alone on a cancelled pass; a completed pass keeps the unconditional write (its `ScannedThisSession` stamp feeds the restore gate). A pass cancelled before its first batch now records nothing, so the coordinator's empty-pass gate fires. Take the pass token before the `is_syncing` CAS in both entry points: a `stop()` between the CAS and `pass_token()` rotated the parent and handed the admitted pass an uncancelled child. Check cancellation ahead of the cooldown fast path, whose skip summary reported one caught-up wallet for a pass that never ran. Co-Authored-By: Claude Fable 5.1 --- .../src/manager/shielded_sync.rs | 33 ++++--- .../src/wallet/shielded/coordinator.rs | 40 ++++++-- .../src/wallet/shielded/sync.rs | 19 +++- .../src/wallet/shielded/sync/cancel_tests.rs | 94 ++++++++++++++++++- 4 files changed, 160 insertions(+), 26 deletions(-) diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index 25e925cbda2..ae1cd2cfe53 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -139,7 +139,8 @@ pub struct ShieldedSyncManager { event_manager: Arc, /// Network-scoped shielded coordinator slot, shared with the /// owning `PlatformWalletManager`. Sync passes route through - /// `coordinator.sync(force)` whenever the slot is populated; + /// `coordinator.sync(force, &pass_cancel)` — under the + /// manager-owned pass token — whenever the slot is populated; /// an empty slot returns an empty pass summary (no wallets /// can be shielded-bound without `configure_shielded` having /// run first, so an empty slot guarantees no shielded state @@ -417,6 +418,17 @@ impl ShieldedSyncManager { /// If a pass is already in flight, returns an empty summary and /// skips — the caller can inspect [`is_syncing`] to distinguish. pub async fn sync_now(&self, force: bool) -> ShieldedSyncPassSummary { + // Take the pass token BEFORE claiming the slot. A `stop()` that + // lands after the token is taken cancels this lineage, whichever + // side of the CAS it falls on; taken after the CAS, a stop that + // slipped in between would rotate the parent first and hand this + // pass a fresh, uncancelled child. Losing the CAS just drops the + // token. It is also before the gate check: `drain_pass` takes its + // hold (closing the gate) and only then calls `stop()`, so a pass + // that took its token before the rotation is on the cancelled + // lineage, and one that took it after sees the closed gate below. + // Either way no pass runs past a drain. + let pass_cancel = self.pass_token(); if self .is_syncing .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -428,13 +440,6 @@ impl ShieldedSyncManager { // so a failed pass can never wedge `quiesce()`'s drain. let _slot = SyncSlotGuard(&self.is_syncing); - // Take the pass token BEFORE the gate check. `drain_pass` takes its - // hold (closing the gate) and only then calls `stop()` (rotating - // the token), so a pass that took its token before the rotation - // is on the cancelled lineage, and one that took it after sees the - // closed gate below. Either way no pass runs past a drain. - let pass_cancel = self.pass_token(); - // A `quiesce()` may have raised the gate between our CAS and // here; if so, release the slot and bail without running a pass // so the drain can complete and Clear/stop get a true barrier. @@ -510,8 +515,10 @@ impl ShieldedSyncManager { /// /// Returns `Ok(None)` if `wallet_id` isn't registered on the /// coordinator (e.g. shielded support hasn't been configured, - /// or the wallet has never called `bind_shielded`), or if - /// another sync pass was already in flight. + /// or the wallet has never called `bind_shielded`), if + /// another sync pass was already in flight, or if the pass was + /// cancelled by [`stop`](Self::stop) before it consumed any + /// scan data (the coordinator then reports no wallets). pub async fn sync_wallet( &self, wallet_id: &WalletId, @@ -525,6 +532,9 @@ impl ShieldedSyncManager { return Ok(None); }; + // Token before the slot CAS and the gate check — see `sync_now` + // for the ordering. + let pass_cancel = self.pass_token(); // Reuse the manager-wide `is_syncing` flag so a per-wallet // `sync_wallet()` can't race the periodic `sync_now()` // against the same store — both go through @@ -541,9 +551,6 @@ impl ShieldedSyncManager { // so a failed pass can never wedge `quiesce()`'s drain. let _slot = SyncSlotGuard(&self.is_syncing); - // Token before the gate check — see `sync_now` for the ordering. - let pass_cancel = self.pass_token(); - // Bail if a `quiesce()` raised the gate after our CAS (see // `sync_now`) so the drain barrier holds. if self.quiescing.is_closed() { diff --git a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs index 430b9f02b1e..ac00f9158a7 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs @@ -1327,6 +1327,16 @@ impl NetworkShieldedCoordinator { /// [`PlatformEventManager::on_shielded_sync_completed`]: /// crate::events::PlatformEventManager::on_shielded_sync_completed pub async fn sync(&self, force: bool, cancel: &CancellationToken) -> ShieldedSyncPassSummary { + // A stop that landed before any work: nothing to finalize, so + // return the empty summary without touching Platform or the + // store (the manager then skips the completion event). Checked + // ahead of the cooldown gate, whose skip summary would otherwise + // report one caught-up wallet each for a pass that never ran. + if cancel.is_cancelled() { + tracing::debug!("Coordinator sync cancelled before it started"); + return ShieldedSyncPassSummary::default(); + } + // Network-wide cooldown gate. Snapshot the remaining // window into a local before any await — `std::sync::Mutex` // is `!Send` across await points (clippy's @@ -1365,14 +1375,6 @@ impl NetworkShieldedCoordinator { return summary; } - // A stop that landed before any network work: nothing to finalize, - // so return the empty summary without touching Platform or the - // store (the manager then skips the completion event). - if cancel.is_cancelled() { - tracing::debug!("Coordinator sync cancelled before it started"); - return summary; - } - // Snapshot armed reservations and prefetch Platform's recorded // anchor set BEFORE the note scan. The release pass after the scan // may only act on this pre-scan pair: an anchor already absent from @@ -3792,4 +3794,26 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + + /// Cancelled while inside the caught-up cooldown: the cooldown's + /// skip summary would report one caught-up wallet, which the + /// manager would stamp and announce as a completed pass. The + /// cancellation check runs first, so the pass reports nothing. + #[tokio::test] + async fn should_report_nothing_when_cancelled_inside_cooldown() { + let dir = temp_dir("cancel_inside_cooldown"); + let coordinator = coordinator_with_one_wallet(&dir).await; + *coordinator.last_caught_up_at.lock().unwrap() = Some(std::time::Instant::now()); + let cancel = CancellationToken::new(); + cancel.cancel(); + + let summary = coordinator.sync(false, &cancel).await; + + assert!( + summary.is_empty(), + "a cancelled pass must not borrow the cooldown's caught-up summary" + ); + + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index a984b319335..a84763a3fc7 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -767,12 +767,27 @@ where // the union. `total_notes_scanned` is accumulated from every // streamed batch's note count — identical to the one-shot path's // `result.total_notes_scanned`. + // + // Never rewind. `aligned_start` is the MIN watermark across the + // union, so a pass cut short after a few batches can end below a + // subwallet that was already further along; a completed pass + // always reaches the tip, which is at or past every watermark. + // A cancelled pass leaves such a subwallet's row alone entirely: + // its consumed prefix never covered that subwallet, so neither + // the watermark nor the `ScannedThisSession` stamp the setter + // applies belongs to it. A completed pass keeps the unconditional + // write, so a caught-up subwallet still gets its stamp. let new_index = aligned_start + total_notes_scanned; for (id, _) in subwallets { + let prior = watermarks.get(id).copied().unwrap_or(0); + let advanced = prior.max(new_index); + if cancelled && advanced == prior { + continue; + } store - .set_last_synced_note_index(*id, new_index) + .set_last_synced_note_index(*id, advanced) .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; - changeset.record_synced_index(*id, new_index); + changeset.record_synced_index(*id, advanced); } // Drop the write lock before returning so the caller's // persister queue (which may take its own synchronous diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs index 72958029a46..4492256eb12 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs @@ -29,11 +29,15 @@ fn temp_tree_path(tag: &str) -> std::path::PathBuf { std::env::temp_dir().join(format!("shielded_sync_cancel_{tag}_{nanos}.sqlite")) } -fn one_subwallet() -> Vec<(SubwalletId, AccountViewingKeys)> { - let views = OrchardKeySet::from_seed(&[0x42u8; 64], dashcore::Network::Testnet, 0) +fn subwallet(account: u32) -> (SubwalletId, AccountViewingKeys) { + let views = OrchardKeySet::from_seed(&[0x42u8; 64], dashcore::Network::Testnet, account) .expect("derive viewing keys") .viewing_keys(); - vec![(SubwalletId::new([0x11; 32], 0), views)] + (SubwalletId::new([0x11; 32], account), views) +} + +fn one_subwallet() -> Vec<(SubwalletId, AccountViewingKeys)> { + vec![subwallet(0)] } /// A batch of `len` notes at `start_index`. The notes are opaque to the @@ -215,3 +219,87 @@ async fn should_resume_from_partial_watermark_on_next_pass() { let _ = std::fs::remove_file(&path); } + +/// Two subwallets, one already past where the cancelled prefix ends. +/// `aligned_start` is the union MIN, so the pass walks 0..8 while the +/// second subwallet sits at 12: its watermark must not move backwards, +/// and the changeset must not carry a rewound value to the host. +#[tokio::test] +async fn should_not_rewind_a_more_advanced_subwallet_when_cancelled() { + let path = temp_tree_path("no_rewind"); + let store = Arc::new(RwLock::new( + FileBackedShieldedStore::open_path(&path, 100).expect("open store"), + )); + let subwallets = vec![subwallet(0), subwallet(1)]; + let (lagging, advanced) = (subwallets[0].0, subwallets[1].0); + store + .write() + .await + .set_last_synced_note_index(advanced, 12) + .expect("seed the advanced watermark"); + let cancel = CancellationToken::new(); + + let result = run( + &store, + &subwallets, + &cancel, + batches_then_cancel(vec![batch(0, 4, 12), batch(4, 4, 12)], cancel.clone()), + ) + .await; + + let store = store.read().await; + assert_eq!(store.last_synced_note_index(lagging).unwrap(), 8); + assert_eq!( + store.last_synced_note_index(advanced).unwrap(), + 12, + "a partial prefix below a subwallet's watermark must not rewind it" + ); + assert_eq!( + result.changeset.synced_indices.get(&lagging).copied(), + Some(8) + ); + assert!( + !result.changeset.synced_indices.contains_key(&advanced), + "no rewound watermark may reach the host persister" + ); + + let _ = std::fs::remove_file(&path); +} + +/// Cancelled after the store lock but before the first batch: nothing +/// was consumed, so the store and the changeset stay untouched and the +/// coordinator's empty-pass gate can fire. +#[tokio::test] +async fn should_leave_store_and_changeset_empty_when_cancelled_before_first_batch() { + let path = temp_tree_path("first_batch"); + let store = Arc::new(RwLock::new( + FileBackedShieldedStore::open_path(&path, 100).expect("open store"), + )); + let subwallets = one_subwallet(); + let sub = subwallets[0].0; + store + .write() + .await + .set_last_synced_note_index(sub, 4) + .expect("seed the watermark"); + let cancel = CancellationToken::new(); + + let result = run( + &store, + &subwallets, + &cancel, + batches_then_cancel(vec![], cancel.clone()), + ) + .await; + + assert_eq!(result.total_scanned, 0); + assert!( + crate::changeset::merge::Merge::is_empty(&result.changeset), + "a pass that consumed nothing records nothing, not even its own watermark" + ); + let store = store.read().await; + assert_eq!(store.tree_size().unwrap(), 0); + assert_eq!(store.last_synced_note_index(sub).unwrap(), 4); + + let _ = std::fs::remove_file(&path); +} From f9e2a0ad9289146c0cb1f415e16b4da809b4f8bf Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Thu, 29 Oct 2026 17:31:15 +0100 Subject: [PATCH 05/12] fix(platform-wallet): clear a drained latch from one-shot passes and poll behind a wedge A one-shot `sync_now` (or per-wallet sync) after a timed-out drain won the slot and then bailed on the stale latch, with no loop running to clear it. Reopen a drained latch before the slot CAS in every entry point: a false `is_syncing` means no pass is in flight, and the gate check after the CAS still honors a live hold or the seal. A loop started behind a still-wedged pass slept its full interval (up to 60 s) before noticing the pass had drained. While the latch holds, poll the slot at `LATCHED_RETRY_INTERVAL` (1 s) instead. Co-Authored-By: Claude Fable 5.1 --- .../src/manager/dashpay_sync.rs | 25 ++++++-- .../src/manager/dpns_sync.rs | 20 +++++- .../src/manager/identity_sync.rs | 25 ++++++-- .../rs-platform-wallet/src/manager/mod.rs | 12 +++- .../src/manager/platform_address_sync.rs | 62 +++++++++++++++--- .../src/manager/shielded_sync.rs | 63 ++++++++++++++----- 6 files changed, 169 insertions(+), 38 deletions(-) diff --git a/packages/rs-platform-wallet/src/manager/dashpay_sync.rs b/packages/rs-platform-wallet/src/manager/dashpay_sync.rs index 467a0aec24e..2b4fee9f417 100644 --- a/packages/rs-platform-wallet/src/manager/dashpay_sync.rs +++ b/packages/rs-platform-wallet/src/manager/dashpay_sync.rs @@ -58,7 +58,7 @@ use dash_async::{ThreadRegistry, WorkerConfig}; use crate::error::PlatformWalletError; use crate::manager::{ coordinator_worker_config, drain_pass, QuiesceGate, QuiesceGuard, SyncSlotGuard, WalletWorker, - COORDINATOR_DRAIN_BUDGET, + COORDINATOR_DRAIN_BUDGET, LATCHED_RETRY_INTERVAL, }; use crate::wallet::platform_wallet::WalletId; use crate::wallet::PlatformWallet; @@ -244,10 +244,17 @@ impl DashPaySyncManager { break; } - this.quiescing.reopen_if_drained(&this.is_syncing); + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; - let interval = this.interval(); + // Behind a still-wedged pass the pass above bailed; poll the + // slot briskly instead of sleeping a full interval so passes + // resume promptly once that pass drains. + let interval = if gate_open { + this.interval() + } else { + this.interval().min(LATCHED_RETRY_INTERVAL) + }; tokio::select! { _ = tokio::time::sleep(interval) => {} _ = cancel.cancelled() => break, @@ -306,8 +313,9 @@ impl DashPaySyncManager { /// wedged pass cannot be followed by a fresh one; the caller must /// treat the coordinator as non-clean (shutdown reports it, clear / /// reset paths abort fail-closed). A later successful `quiesce` - /// reopens the gate, as does [`start`](Self::start) (or the running - /// loop) once the wedged pass has released `is_syncing`. + /// reopens the gate, as does [`start`](Self::start), the running + /// loop, or a one-shot `sync_now`, once the wedged pass has released + /// `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { // The guard drops here, reopening the gate — this is the // "drain only" flavor. @@ -352,6 +360,13 @@ impl DashPaySyncManager { /// consulted), calling `dashpay_sync()` per wallet. Errors are /// logged and recorded in the summary but never abort the sweep. pub async fn sync_now(&self) -> DashPaySyncSummary { + // A drained timeout latch is cleared here as well as in `start()`: + // with no loop running, a one-shot pass after a timed-out drain + // would otherwise bail on the stale latch until something + // restarted the loop. Safe before the CAS — a false `is_syncing` + // means no pass is in flight — and the gate check after the CAS + // still honors a live hold or the seal. + self.quiescing.reopen_if_drained(&self.is_syncing); if self .is_syncing .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) diff --git a/packages/rs-platform-wallet/src/manager/dpns_sync.rs b/packages/rs-platform-wallet/src/manager/dpns_sync.rs index 1ab18b5abd5..8a37fe8dca7 100644 --- a/packages/rs-platform-wallet/src/manager/dpns_sync.rs +++ b/packages/rs-platform-wallet/src/manager/dpns_sync.rs @@ -49,7 +49,7 @@ use dash_async::{ThreadRegistry, WorkerConfig}; use crate::events::PlatformEventManager; use crate::manager::{ coordinator_worker_config, drain_pass, QuiesceGate, QuiesceGuard, SyncSlotGuard, WalletWorker, - COORDINATOR_DRAIN_BUDGET, + COORDINATOR_DRAIN_BUDGET, LATCHED_RETRY_INTERVAL, }; use crate::wallet::identity::network::DpnsMarketplaceSyncSummary; use crate::wallet::platform_wallet::WalletId; @@ -216,10 +216,17 @@ impl DpnsSyncManager { break; } - this.quiescing.reopen_if_drained(&this.is_syncing); + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; - let interval = this.interval(); + // Behind a still-wedged pass the pass above bailed; poll the + // slot briskly instead of sleeping a full interval so passes + // resume promptly once that pass drains. + let interval = if gate_open { + this.interval() + } else { + this.interval().min(LATCHED_RETRY_INTERVAL) + }; tokio::select! { _ = tokio::time::sleep(interval) => {} _ = cancel.cancelled() => break, @@ -278,6 +285,13 @@ impl DpnsSyncManager { /// summary but never abort the sweep. Dispatches the completion /// event before returning. pub async fn sync_now(&self) -> DpnsSyncPassSummary { + // A drained timeout latch is cleared here as well as in `start()`: + // with no loop running, a one-shot pass after a timed-out drain + // would otherwise bail on the stale latch until something + // restarted the loop. Safe before the CAS — a false `is_syncing` + // means no pass is in flight — and the gate check after the CAS + // still honors a live hold or the seal. + self.quiescing.reopen_if_drained(&self.is_syncing); if self .is_syncing .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) diff --git a/packages/rs-platform-wallet/src/manager/identity_sync.rs b/packages/rs-platform-wallet/src/manager/identity_sync.rs index 286a0d8a94d..687d19b3be1 100644 --- a/packages/rs-platform-wallet/src/manager/identity_sync.rs +++ b/packages/rs-platform-wallet/src/manager/identity_sync.rs @@ -71,7 +71,7 @@ use dash_async::ThreadRegistry; use crate::changeset::{PlatformWalletPersistence, TokenBalanceChangeSet}; use crate::manager::{ coordinator_worker_config, drain_pass, QuiesceGate, QuiesceGuard, SyncSlotGuard, WalletWorker, - COORDINATOR_DRAIN_BUDGET, + COORDINATOR_DRAIN_BUDGET, LATCHED_RETRY_INTERVAL, }; use crate::wallet::platform_wallet::WalletId; @@ -444,10 +444,17 @@ where break; } - this.quiescing.reopen_if_drained(&this.is_syncing); + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; - let interval = this.interval(); + // Behind a still-wedged pass the pass above bailed; poll the + // slot briskly instead of sleeping a full interval so passes + // resume promptly once that pass drains. + let interval = if gate_open { + this.interval() + } else { + this.interval().min(LATCHED_RETRY_INTERVAL) + }; tokio::select! { _ = tokio::time::sleep(interval) => {} _ = cancel.cancelled() => break, @@ -504,8 +511,9 @@ where /// that path the `quiescing` gate is deliberately **left closed** so the /// wedged pass cannot be followed by a fresh one; the caller must /// treat the coordinator as non-clean. A later successful `quiesce` - /// reopens the gate, as does [`start`](Self::start) (or the running - /// loop) once the wedged pass has released `is_syncing`. + /// reopens the gate, as does [`start`](Self::start), the running + /// loop, or a one-shot `sync_now`, once the wedged pass has released + /// `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { // The guard drops here, reopening the gate — this is the // "drain only" flavor. @@ -559,6 +567,13 @@ where /// `!Send` (no `tokio::spawn`) and because the design brief /// explicitly forbids it. pub async fn sync_now(&self) { + // A drained timeout latch is cleared here as well as in `start()`: + // with no loop running, a one-shot pass after a timed-out drain + // would otherwise bail on the stale latch until something + // restarted the loop. Safe before the CAS — a false `is_syncing` + // means no pass is in flight — and the gate check after the CAS + // still honors a live hold or the seal. + self.quiescing.reopen_if_drained(&self.is_syncing); if self .is_syncing .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) diff --git a/packages/rs-platform-wallet/src/manager/mod.rs b/packages/rs-platform-wallet/src/manager/mod.rs index 3f6c172cc5d..956434bd767 100644 --- a/packages/rs-platform-wallet/src/manager/mod.rs +++ b/packages/rs-platform-wallet/src/manager/mod.rs @@ -104,6 +104,13 @@ pub enum WalletWorker { /// so `all_clean()` fails and the host keeps its callback context alive. pub(crate) const COORDINATOR_DRAIN_BUDGET: Duration = Duration::from_secs(10); +/// How often a running sync loop re-checks a latched gate while the pass +/// a timed-out drain gave up on still holds `is_syncing`. The loop cannot +/// run a pass behind that one anyway, so instead of sleeping its normal +/// interval (up to 60 s) it polls the slot at this cadence and resumes +/// within about a second of the wedged pass draining. +pub(crate) const LATCHED_RETRY_INTERVAL: Duration = Duration::from_secs(1); + /// Deadline for draining the DashPay payment-hook tasks at shutdown. /// After it lapses each straggler is aborted and given /// [`PAYMENT_ABORT_GRACE`] to confirm termination; anything still alive @@ -254,7 +261,10 @@ impl QuiesceGate { /// pass. Without it, a `quiesce` that timed out (the host saw /// `ErrorShutdownIncomplete`) leaves the latch set until *another* /// drain succeeds — and a plain restart never drains, so the new loop - /// reports running while every pass bails on the closed gate. + /// reports running while every pass bails on the closed gate. Called + /// by `start()`, by the running loop before each pass, and by the + /// one-shot `sync_now` / per-wallet entry points before they claim + /// the slot, so a stale latch never outlives the pass it guarded. /// /// Keeps the latch's guarantee: while `is_syncing` is still held, the /// wedged pass is the only thing that can hold it (a latched gate makes diff --git a/packages/rs-platform-wallet/src/manager/platform_address_sync.rs b/packages/rs-platform-wallet/src/manager/platform_address_sync.rs index ee5cab6cec7..86aa8abcaf0 100644 --- a/packages/rs-platform-wallet/src/manager/platform_address_sync.rs +++ b/packages/rs-platform-wallet/src/manager/platform_address_sync.rs @@ -26,7 +26,7 @@ use crate::error::PlatformWalletError; use crate::events::PlatformEventManager; use crate::manager::{ coordinator_worker_config, drain_pass, QuiesceGate, QuiesceGuard, SyncSlotGuard, WalletWorker, - COORDINATOR_DRAIN_BUDGET, + COORDINATOR_DRAIN_BUDGET, LATCHED_RETRY_INTERVAL, }; use crate::wallet::platform_wallet::WalletId; use crate::wallet::PlatformWallet; @@ -229,10 +229,17 @@ impl PlatformAddressSyncManager { break; } - this.quiescing.reopen_if_drained(&this.is_syncing); + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; - let interval = this.interval(); + // Behind a still-wedged pass the pass above bailed; poll the + // slot briskly instead of sleeping a full interval so passes + // resume promptly once that pass drains. + let interval = if gate_open { + this.interval() + } else { + this.interval().min(LATCHED_RETRY_INTERVAL) + }; tokio::select! { _ = tokio::time::sleep(interval) => {} _ = cancel.cancelled() => break, @@ -291,8 +298,9 @@ impl PlatformAddressSyncManager { /// that path the `quiescing` gate is deliberately **left closed** so the /// wedged pass cannot be followed by a fresh one; the caller must /// treat the coordinator as non-clean. A later successful `quiesce` - /// reopens the gate, as does [`start`](Self::start) (or the running - /// loop) once the wedged pass has released `is_syncing`. + /// reopens the gate, as does [`start`](Self::start), the running + /// loop, or a one-shot `sync_now`, once the wedged pass has released + /// `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { // The guard drops here, reopening the gate — this is the // "drain only" flavor. @@ -344,6 +352,13 @@ impl PlatformAddressSyncManager { /// If a pass is already in flight, returns an empty summary and /// skips — the caller can inspect [`is_syncing`] to distinguish. pub async fn sync_now(&self) -> PlatformAddressSyncSummary { + // A drained timeout latch is cleared here as well as in `start()`: + // with no loop running, a one-shot pass after a timed-out drain + // would otherwise bail on the stale latch until something + // restarted the loop. Safe before the CAS — a false `is_syncing` + // means no pass is in flight — and the gate check after the CAS + // still honors a live hold or the seal. + self.quiescing.reopen_if_drained(&self.is_syncing); if self .is_syncing .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -438,6 +453,13 @@ impl PlatformAddressSyncManager { wallet_id: &WalletId, ) -> Result, PlatformWalletError> { + // A drained timeout latch is cleared here as well as in `start()`: + // with no loop running, a one-shot pass after a timed-out drain + // would otherwise bail on the stale latch until something + // restarted the loop. Safe before the CAS — a false `is_syncing` + // means no pass is in flight — and the gate check after the CAS + // still honors a live hold or the seal. + self.quiescing.reopen_if_drained(&self.is_syncing); if self .is_syncing .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -767,9 +789,8 @@ mod tests { ); // A restart behind the still-wedged pass must not run a pass. The - // loop's first attempt loses the slot CAS to the wedge and sleeps - // an interval; keep that short (the minimum is 1 s). - mgr.set_interval(Duration::from_secs(1)); + // loop's first attempt loses the slot CAS to the wedge; it then + // polls at `LATCHED_RETRY_INTERVAL`, not the 15 s default. Arc::clone(&mgr).start(); assert!(mgr.is_running()); tokio::time::sleep(Duration::from_millis(100)).await; @@ -823,4 +844,29 @@ mod tests { mgr.stop(); assert!(mgr.quiesce().await); } + + /// A one-shot `sync_now` with no loop running clears a latch whose + /// wedged pass has drained, instead of bailing until a restart. + #[tokio::test] + async fn sync_now_reopens_a_drained_latch() { + let (mgr, counter) = make_manager(); + + assert!(mgr.wedge_sync_slot_for_test()); + assert!(!mgr.quiesce_within(Duration::from_millis(50)).await); + mgr.sync_now().await; + assert_eq!(counter.completions.load(AtomicOrdering::SeqCst), 0); + assert!(mgr.quiescing.is_closed(), "latch stays while wedged"); + + mgr.is_syncing.store(false, Ordering::Release); + mgr.sync_now().await; + assert_eq!( + counter.completions.load(AtomicOrdering::SeqCst), + 1, + "a direct pass runs once the wedged pass is gone" + ); + assert!( + !mgr.quiescing.is_closed(), + "the direct pass cleared the latch" + ); + } } diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index be4ece5856a..50734d884c9 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -39,7 +39,7 @@ use dash_async::ThreadRegistry; use crate::events::PlatformEventManager; use crate::manager::{ coordinator_worker_config, drain_pass, QuiesceGate, QuiesceGuard, SyncSlotGuard, WalletWorker, - COORDINATOR_DRAIN_BUDGET, + COORDINATOR_DRAIN_BUDGET, LATCHED_RETRY_INTERVAL, }; use crate::wallet::platform_wallet::WalletId; use crate::wallet::shielded::{NetworkShieldedCoordinator, ShieldedSyncSummary}; @@ -255,7 +255,7 @@ impl ShieldedSyncManager { break; } - this.quiescing.reopen_if_drained(&this.is_syncing); + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); // Background-loop cadence — honor the // per-wallet caught-up cooldown so a @@ -266,7 +266,14 @@ impl ShieldedSyncManager { // entry point below and bypass this. this.sync_now(false).await; - let interval = this.interval(); + // Behind a still-wedged pass the pass above bailed; poll the + // slot briskly instead of sleeping a full interval so passes + // resume promptly once that pass drains. + let interval = if gate_open { + this.interval() + } else { + this.interval().min(LATCHED_RETRY_INTERVAL) + }; tokio::select! { _ = tokio::time::sleep(interval) => {} _ = cancel.cancelled() => break, @@ -325,8 +332,9 @@ impl ShieldedSyncManager { /// the wedged pass cannot be followed by a fresh one; the caller must /// treat the coordinator as non-clean (Clear aborts fail-closed, the /// FFI stop surfaces `ErrorShutdownIncomplete`). A later successful - /// `quiesce` reopens the gate, as does [`start`](Self::start) (or the - /// running loop) once the wedged pass has released `is_syncing`. + /// `quiesce` reopens the gate, as does [`start`](Self::start), the + /// running loop, or a one-shot `sync_now`, once the wedged pass has + /// released `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { // The guard drops here, reopening the gate — this is the // "drain only" flavor. @@ -385,6 +393,13 @@ impl ShieldedSyncManager { /// If a pass is already in flight, returns an empty summary and /// skips — the caller can inspect [`is_syncing`] to distinguish. pub async fn sync_now(&self, force: bool) -> ShieldedSyncPassSummary { + // A drained timeout latch is cleared here as well as in `start()`: + // with no loop running, a one-shot pass after a timed-out drain + // would otherwise bail on the stale latch until something + // restarted the loop. Safe before the CAS — a false `is_syncing` + // means no pass is in flight — and the gate check after the CAS + // still honors a live hold or the seal. + self.quiescing.reopen_if_drained(&self.is_syncing); if self .is_syncing .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -481,6 +496,13 @@ impl ShieldedSyncManager { // against the same store — both go through // `coordinator.sync()`, which serializes per-coordinator // but the manager flag is what the host UI watches. + // A drained timeout latch is cleared here as well as in `start()`: + // with no loop running, a one-shot pass after a timed-out drain + // would otherwise bail on the stale latch until something + // restarted the loop. Safe before the CAS — a false `is_syncing` + // means no pass is in flight — and the gate check after the CAS + // still honors a live hold or the seal. + self.quiescing.reopen_if_drained(&self.is_syncing); if self .is_syncing .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) @@ -598,9 +620,8 @@ mod tests { "timed-out drain latches the gate" ); - // The loop's first attempt loses the slot CAS to the wedge and - // sleeps an interval; keep that short (the minimum is 1 s). - mgr.set_interval(Duration::from_secs(1)); + // The loop's first attempt loses the slot CAS to the wedge; it then + // polls at `LATCHED_RETRY_INTERVAL`, not the 60 s default. Arc::clone(&mgr).start(); assert!(mgr.is_running()); tokio::time::sleep(Duration::from_millis(100)).await; @@ -653,21 +674,31 @@ mod tests { assert!(mgr.quiesce().await); } - /// A manual `sync_now` cannot clear the latch itself: it only ever - /// runs once a `start()` or the loop has observed the drained pass. + /// A one-shot `sync_now` with no loop running: the latch outlives the + /// wedged pass, and the direct call clears it itself (a restart is + /// not required). #[tokio::test] - async fn sync_now_still_bails_while_latched() { + async fn sync_now_reopens_a_drained_latch() { let (mgr, counter) = make_manager(); wedge_sync_slot(&mgr); assert!(!mgr.quiesce_within(Duration::from_millis(50)).await); - mgr.is_syncing.store(false, Ordering::Release); + assert!( + mgr.sync_now(true).await.is_empty(), + "bails behind the wedge" + ); + assert!(mgr.quiescing.is_closed(), "latch stays while wedged"); - assert!(mgr.sync_now(true).await.is_empty()); - assert_eq!(counter.completions.load(AtomicOrdering::SeqCst), 0); + mgr.is_syncing.store(false, Ordering::Release); + mgr.sync_now(true).await; + assert_eq!( + counter.completions.load(AtomicOrdering::SeqCst), + 1, + "a direct pass runs once the wedged pass is gone" + ); assert!( - mgr.quiescing.is_closed(), - "a direct pass leaves the latch alone" + !mgr.quiescing.is_closed(), + "the direct pass cleared the latch" ); } } From d8093c502ca23dcb9619f3deb722408817560b40 Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Thu, 29 Oct 2026 19:42:43 +0100 Subject: [PATCH 06/12] fix(platform-wallet): pick the loop cadence from the latch state after the pass The loop sampled the latch before awaiting `sync_now`, so a latch raised or cleared during the pass could sleep a full interval instead of the 1 s retry, or retry needlessly. `sync_now` already clears a drained latch before its slot CAS, so drop the pre-pass sample and read the latch once after the pass. Document the recovery path on the DPNS `quiesce_within` like its siblings. Co-Authored-By: Claude Fable 5.1 --- .../src/manager/dashpay_sync.rs | 11 +++++++---- .../rs-platform-wallet/src/manager/dpns_sync.rs | 15 ++++++++++----- .../src/manager/identity_sync.rs | 11 +++++++---- packages/rs-platform-wallet/src/manager/mod.rs | 7 ++++--- .../src/manager/platform_address_sync.rs | 11 +++++++---- .../src/manager/shielded_sync.rs | 12 +++++++----- 6 files changed, 42 insertions(+), 25 deletions(-) diff --git a/packages/rs-platform-wallet/src/manager/dashpay_sync.rs b/packages/rs-platform-wallet/src/manager/dashpay_sync.rs index 2b4fee9f417..fcc41dc9350 100644 --- a/packages/rs-platform-wallet/src/manager/dashpay_sync.rs +++ b/packages/rs-platform-wallet/src/manager/dashpay_sync.rs @@ -244,12 +244,15 @@ impl DashPaySyncManager { break; } - let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; - // Behind a still-wedged pass the pass above bailed; poll the - // slot briskly instead of sleeping a full interval so passes - // resume promptly once that pass drains. + // `sync_now` clears a drained latch itself before its slot CAS. + // Sample the latch AFTER the pass, so a latch raised or cleared + // while it ran is what picks the cadence: behind a still-wedged + // pass the call above bailed, so poll the slot briskly instead of + // sleeping a full interval and passes resume promptly once that + // pass drains. + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); let interval = if gate_open { this.interval() } else { diff --git a/packages/rs-platform-wallet/src/manager/dpns_sync.rs b/packages/rs-platform-wallet/src/manager/dpns_sync.rs index 8a37fe8dca7..35e4e6a1501 100644 --- a/packages/rs-platform-wallet/src/manager/dpns_sync.rs +++ b/packages/rs-platform-wallet/src/manager/dpns_sync.rs @@ -216,12 +216,15 @@ impl DpnsSyncManager { break; } - let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; - // Behind a still-wedged pass the pass above bailed; poll the - // slot briskly instead of sleeping a full interval so passes - // resume promptly once that pass drains. + // `sync_now` clears a drained latch itself before its slot CAS. + // Sample the latch AFTER the pass, so a latch raised or cleared + // while it ran is what picks the cadence: behind a still-wedged + // pass the call above bailed, so poll the slot briskly instead of + // sleeping a full interval and passes resume promptly once that + // pass drains. + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); let interval = if gate_open { this.interval() } else { @@ -252,7 +255,9 @@ impl DpnsSyncManager { /// [`quiesce`](Self::quiesce) with an explicit drain budget. On /// timeout the admission gate is left closed and the caller must - /// fail closed. + /// fail closed. A later successful `quiesce` reopens the gate, as + /// does [`start`](Self::start), the running loop, or a one-shot + /// `sync_now`, once the wedged pass has released `is_syncing`. pub(crate) async fn quiesce_within(&self, budget: Duration) -> bool { self.quiesce_held_within(budget).await.is_some() } diff --git a/packages/rs-platform-wallet/src/manager/identity_sync.rs b/packages/rs-platform-wallet/src/manager/identity_sync.rs index 687d19b3be1..8def2baa913 100644 --- a/packages/rs-platform-wallet/src/manager/identity_sync.rs +++ b/packages/rs-platform-wallet/src/manager/identity_sync.rs @@ -444,12 +444,15 @@ where break; } - let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; - // Behind a still-wedged pass the pass above bailed; poll the - // slot briskly instead of sleeping a full interval so passes - // resume promptly once that pass drains. + // `sync_now` clears a drained latch itself before its slot CAS. + // Sample the latch AFTER the pass, so a latch raised or cleared + // while it ran is what picks the cadence: behind a still-wedged + // pass the call above bailed, so poll the slot briskly instead of + // sleeping a full interval and passes resume promptly once that + // pass drains. + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); let interval = if gate_open { this.interval() } else { diff --git a/packages/rs-platform-wallet/src/manager/mod.rs b/packages/rs-platform-wallet/src/manager/mod.rs index 956434bd767..d0f5e21222a 100644 --- a/packages/rs-platform-wallet/src/manager/mod.rs +++ b/packages/rs-platform-wallet/src/manager/mod.rs @@ -262,9 +262,10 @@ impl QuiesceGate { /// `ErrorShutdownIncomplete`) leaves the latch set until *another* /// drain succeeds — and a plain restart never drains, so the new loop /// reports running while every pass bails on the closed gate. Called - /// by `start()`, by the running loop before each pass, and by the - /// one-shot `sync_now` / per-wallet entry points before they claim - /// the slot, so a stale latch never outlives the pass it guarded. + /// by `start()`, by the one-shot `sync_now` / per-wallet entry points + /// before they claim the slot, and by the running loop after each + /// pass to pick its cadence, so a stale latch never outlives the pass + /// it guarded. /// /// Keeps the latch's guarantee: while `is_syncing` is still held, the /// wedged pass is the only thing that can hold it (a latched gate makes diff --git a/packages/rs-platform-wallet/src/manager/platform_address_sync.rs b/packages/rs-platform-wallet/src/manager/platform_address_sync.rs index 86aa8abcaf0..295427f9fb1 100644 --- a/packages/rs-platform-wallet/src/manager/platform_address_sync.rs +++ b/packages/rs-platform-wallet/src/manager/platform_address_sync.rs @@ -229,12 +229,15 @@ impl PlatformAddressSyncManager { break; } - let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); this.sync_now().await; - // Behind a still-wedged pass the pass above bailed; poll the - // slot briskly instead of sleeping a full interval so passes - // resume promptly once that pass drains. + // `sync_now` clears a drained latch itself before its slot CAS. + // Sample the latch AFTER the pass, so a latch raised or cleared + // while it ran is what picks the cadence: behind a still-wedged + // pass the call above bailed, so poll the slot briskly instead of + // sleeping a full interval and passes resume promptly once that + // pass drains. + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); let interval = if gate_open { this.interval() } else { diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index 50734d884c9..c337fdd3426 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -255,8 +255,6 @@ impl ShieldedSyncManager { break; } - let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); - // Background-loop cadence — honor the // per-wallet caught-up cooldown so a // sleepy network doesn't refetch + @@ -266,9 +264,13 @@ impl ShieldedSyncManager { // entry point below and bypass this. this.sync_now(false).await; - // Behind a still-wedged pass the pass above bailed; poll the - // slot briskly instead of sleeping a full interval so passes - // resume promptly once that pass drains. + // `sync_now` clears a drained latch itself before its slot CAS. + // Sample the latch AFTER the pass, so a latch raised or cleared + // while it ran is what picks the cadence: behind a still-wedged + // pass the call above bailed, so poll the slot briskly instead of + // sleeping a full interval and passes resume promptly once that + // pass drains. + let gate_open = this.quiescing.reopen_if_drained(&this.is_syncing); let interval = if gate_open { this.interval() } else { From 7fd0e14183c58d034a0ee918c496f60c83a236ee Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Tue, 3 Nov 2026 13:40:50 +0100 Subject: [PATCH 07/12] fix(platform-wallet): keep a cancelled shielded pass from certifying an incomplete scan A cancelled pass flushed its consumed prefix through the completed-pass path, which overstated what it had covered in three ways: - the watermark setter also stamped the account as scanned this session, so an unhydrated account could read as a ready zero balance off a prefix that held none of its notes. The store now has a resume-only cursor, used for cancelled passes; the scan coverage and balance source move only when a pass reaches the tip. - activity derivation ran over the prefix and froze a proof-height cluster that the resumed pass could then only confirm, never extend, permanently dropping a later note's value from history. Derivation is deferred to the next completed pass, which derives over all notes at once, exactly as an uninterrupted scan would. - the post-scan re-drive helpers kept starting network requests after a stop. They now take the pass token, stop between records and before each new request, and drop the read-only nonce fetch on cancel; a broadcast already on the wire still finishes and is classified as before. The coordinator regression drives the real coordinator against a mock pool spanning two fetch chunks with one owned receipt each, cancels from the tree-progress callback after the first chunk, and checks that the resumed pass yields the same history and readiness as one uninterrupted scan. The quiesce test takes its pass token before occupying the slot, as the entry points do, so a preempted spawn cannot wait on a rotated lineage. Co-Authored-By: Claude Fable 5.1 --- .../src/manager/shielded_sync.rs | 7 +- .../wallet/shielded/cancelled_scan_tests.rs | 307 ++++++++++++++++++ .../src/wallet/shielded/coordinator.rs | 51 ++- .../src/wallet/shielded/file_store.rs | 5 + .../src/wallet/shielded/mod.rs | 2 + .../src/wallet/shielded/operations.rs | 72 +++- .../src/wallet/shielded/store.rs | 18 +- .../src/wallet/shielded/sync.rs | 27 +- .../src/wallet/shielded/sync/cancel_tests.rs | 13 + 9 files changed, 471 insertions(+), 31 deletions(-) create mode 100644 packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index 0665d5671eb..858ca698139 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -793,9 +793,14 @@ mod tests { // token: holds the slot until the manager cancels it. let holder = Arc::clone(&mgr); let pass = tokio::spawn(async move { + // Token before the slot, as the production entry points do: + // taken after, a quiesce that saw the occupied slot first + // would have rotated the parent and this task would wait on + // a fresh, uncancelled lineage. + let pass_cancel = holder.pass_token(); wedge_sync_slot(&holder); let _slot = SyncSlotGuard(&holder.is_syncing); - holder.pass_token().cancelled().await; + pass_cancel.cancelled().await; }); while !mgr.is_syncing() { tokio::time::sleep(Duration::from_millis(5)).await; diff --git a/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs new file mode 100644 index 00000000000..d68865a588b --- /dev/null +++ b/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs @@ -0,0 +1,307 @@ +//! A cancelled scan pass must not certify what it did not cover +//! (dashpay/platform#5200 review). +//! +//! Drives the real coordinator against a mock Platform whose pool spans +//! two fetch chunks, each holding one owned note, and cancels the pass +//! from the tree-progress callback after the first chunk lands. The +//! consumed prefix is flushed, but it is a resume point, not a complete +//! scan: no activity entry is derived from it and the ledger stays +//! unavailable. The resumed pass then yields exactly what an +//! uninterrupted scan yields — one entry carrying both notes. + +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use dash_sdk::Sdk; +use drive_proof_verifier::types::{ + ShieldedEncryptedNote, ShieldedEncryptedNotes, ShieldedEncryptedNotesQuery, +}; +use grovedb_commitment_tree::{ + DashMemo, Domain, ExtractedNoteCommitment, Note, NoteValue, Nullifier, OrchardDomain, + OrchardNoteEncryption, RandomSeed, Rho, ValueCommitTrapdoor, ValueCommitment, +}; +use rand::{rngs::OsRng, RngCore}; +use tokio_util::sync::CancellationToken; + +use super::activity::ShieldedActivityEntry; +use super::balance::{ShieldedBalanceSource, ShieldedLocalBalanceState}; +use super::keys::AccountViewingKeys; +use super::{ + FileBackedShieldedStore, NetworkShieldedCoordinator, OrchardKeySet, ShieldedStore, SubwalletId, +}; +use crate::wallet::persister::{NoPlatformPersistence, WalletPersister}; + +const WALLET: [u8; 32] = [7; 32]; +const FIRST_VALUE: u64 = 100; +const SECOND_VALUE: u64 = 200; + +fn tree_path() -> PathBuf { + static NEXT_PATH: AtomicU64 = AtomicU64::new(0); + let sequence = NEXT_PATH.fetch_add(1, Ordering::Relaxed); + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos(); + std::env::temp_dir().join(format!( + "shielded_cancelled_scan_{}_{nonce}_{sequence}.sqlite", + std::process::id() + )) +} + +fn viewing_keys() -> AccountViewingKeys { + OrchardKeySet::from_seed(&[42; 64], dashcore::Network::Testnet, 0) + .expect("test viewing key") + .viewing_keys() +} + +/// A note of `value` sent to the test account by someone else, as the +/// network serves it. Real IVK encryption so the SDK's driver decryption +/// finds it; no OVK, so it is a plain receipt and not a self-send; no +/// proof, since nothing here is admitted to consensus. +fn owned_note(keys: &AccountViewingKeys, value: u64) -> ShieldedEncryptedNote { + let mut rng = OsRng; + let (nf, rho) = loop { + let mut b = [0u8; 32]; + rng.fill_bytes(&mut b); + if let (Some(nf), Some(rho)) = ( + Nullifier::from_bytes(&b).into_option(), + Rho::from_bytes(&b).into_option(), + ) { + break (nf, rho); + } + }; + let rseed = loop { + let mut b = [0u8; 32]; + rng.fill_bytes(&mut b); + if let Some(rseed) = RandomSeed::from_bytes(b, &rho).into_option() { + break rseed; + } + }; + let value = NoteValue::from_raw(value); + let note = Note::from_parts(keys.default_address, value, rho, rseed) + .into_option() + .expect("valid note parts"); + let cmx = ExtractedNoteCommitment::from(note.commitment()); + let rcv = loop { + let mut b = [0u8; 32]; + rng.fill_bytes(&mut b); + if let Some(rcv) = ValueCommitTrapdoor::from_bytes(b).into_option() { + break rcv; + } + }; + let cv = ValueCommitment::derive(value - NoteValue::from_raw(0), rcv); + let ne = OrchardNoteEncryption::::new(None, note, [0; 36]); + let epk = OrchardDomain::::epk_bytes(ne.epk()); + let enc = ne.encrypt_note_plaintext(); + let out = ne.encrypt_outgoing_plaintext(&cv, &cmx, &mut rng); + let mut encrypted_note = Vec::with_capacity(216); + encrypted_note.extend_from_slice(&epk.0); + encrypted_note.extend_from_slice(enc.as_ref()); + encrypted_note.extend_from_slice(&out); + ShieldedEncryptedNote { + cmx: cmx.to_bytes().to_vec(), + nullifier: nf.to_bytes().to_vec(), + cv_net: cv.to_bytes().to_vec(), + encrypted_note, + } +} + +/// A note nobody owns at tree position `pos`: an empty ciphertext never +/// decrypts or OVK-recovers, and a distinct small commitment is a valid +/// Pallas element the tree can append. +fn opaque_note(pos: u64) -> ShieldedEncryptedNote { + let mut cmx = [0u8; 32]; + cmx[0] = pos as u8; + cmx[1] = (pos >> 8) as u8; + cmx[2] = 1; + ShieldedEncryptedNote { + cmx: cmx.to_vec(), + nullifier: vec![0xAA; 32], + cv_net: vec![], + encrypted_note: vec![], + } +} + +/// A pool that spans two fetch chunks: the first is full and opens with +/// the first owned note, the second holds only the second owned note. +/// Both chunks are proven at the mock's one block height, so the two +/// receipts fall into the same activity cluster. +async fn two_chunk_pool( + first: &ShieldedEncryptedNote, + second: &ShieldedEncryptedNote, +) -> (Arc, u64) { + let mut sdk = Sdk::new_mock(); + let count = 2048 + * u64::from( + sdk.version() + .drive_abci + .query + .shielded_queries + .max_query_chunks, + ); + let total_count = count + 1; + // The first chunk is full, so the stream's 16-wide speculative window + // slides past it before the short second chunk ends the pool: cover + // every chunk it may still request. + for chunk in 0..48u64 { + let notes = match chunk { + 0 => std::iter::once(first.clone()) + .chain((1..count).map(opaque_note)) + .collect(), + 1 => vec![second.clone()], + _ => Vec::new(), + }; + sdk.mock() + .expect_fetch( + ShieldedEncryptedNotesQuery { + start_index: count * chunk, + count: count as u32, + }, + Some(ShieldedEncryptedNotes { notes, total_count }), + ) + .await + .expect("mock pool chunk"); + } + (Arc::new(sdk), count) +} + +async fn coordinator_over(sdk: Arc, keys: &AccountViewingKeys) -> NetworkShieldedCoordinator { + let path = tree_path(); + let coordinator = NetworkShieldedCoordinator::new( + sdk, + dashcore::Network::Testnet, + path.clone(), + FileBackedShieldedStore::open_path(path, 100).expect("open test store"), + ); + coordinator + .register_wallet( + WALLET, + [(0, keys.clone())].into_iter().collect(), + WalletPersister::new(WALLET, Arc::new(NoPlatformPersistence)), + ) + .await + .expect("register test wallet"); + coordinator +} + +async fn activity(coordinator: &NetworkShieldedCoordinator) -> Vec { + let mut entries = coordinator + .store() + .read() + .await + .get_activity(SubwalletId::new(WALLET, 0), 0, usize::MAX) + .expect("read activity"); + for entry in &mut entries { + entry.note_cmxs.sort(); + } + entries.sort_by_key(|entry| entry.id); + entries +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn should_defer_activity_and_readiness_until_a_pass_completes() { + let keys = viewing_keys(); + let sub = SubwalletId::new(WALLET, 0); + let first = owned_note(&keys, FIRST_VALUE); + let second = owned_note(&keys, SECOND_VALUE); + + // The same pool, scanned in one go: the reference history. + let (baseline_sdk, _) = two_chunk_pool(&first, &second).await; + let baseline = coordinator_over(baseline_sdk, &keys).await; + let summary = baseline.sync(true, &CancellationToken::new()).await; + assert_eq!( + baseline + .store() + .read() + .await + .get_all_notes(sub) + .expect("notes") + .len(), + 2, + "the uninterrupted scan finds both receipts: {summary:?}" + ); + let expected = activity(&baseline).await; + assert_eq!(expected.len(), 1, "both receipts share one cluster"); + assert_eq!(expected[0].amount, FIRST_VALUE + SECOND_VALUE); + assert_eq!(expected[0].note_cmxs.len(), 2); + + // Pass 1: the stop lands after the first chunk is committed. + let (sdk, count) = two_chunk_pool(&first, &second).await; + let coordinator = coordinator_over(sdk, &keys).await; + let cancel = CancellationToken::new(); + let stop = cancel.clone(); + coordinator.install_tree_progress_handler(Some(Arc::new(move |_, _| stop.cancel()))); + let summary = tokio::time::timeout( + std::time::Duration::from_secs(60), + coordinator.sync(true, &cancel), + ) + .await + .expect("a cancelled pass returns"); + coordinator.install_tree_progress_handler(None); + assert_eq!(summary.success_count(), 1, "the consumed prefix is flushed"); + { + let store = coordinator.store().read().await; + let notes = store.get_all_notes(sub).expect("notes"); + assert_eq!( + notes.len(), + 1, + "only the first chunk's receipt was consumed" + ); + assert_eq!(notes[0].value, FIRST_VALUE); + assert_eq!( + store.last_synced_note_index(sub).expect("watermark"), + count, + "the resume cursor covers the consumed chunk" + ); + let balance = store.local_account_balance(sub).expect("balance"); + assert_eq!( + balance.source, + ShieldedBalanceSource::NoHistory, + "a consumed prefix is not a completed scan" + ); + assert_eq!(balance.last_scanned_index, None); + } + assert!( + activity(&coordinator).await.is_empty(), + "no activity entry may be derived from a partial scan" + ); + assert_eq!( + coordinator + .local_balance_snapshot(WALLET) + .await + .expect("snapshot"), + ShieldedLocalBalanceState::RestoreIncomplete, + "an unhydrated account must not read as ready on a partial scan" + ); + + // Pass 2: resumes from the cursor and reaches the tip. + let summary = coordinator.sync(true, &CancellationToken::new()).await; + assert_eq!(summary.success_count(), 1); + { + let store = coordinator.store().read().await; + assert_eq!(store.get_all_notes(sub).expect("notes").len(), 2); + let balance = store.local_account_balance(sub).expect("balance"); + assert_eq!(balance.source, ShieldedBalanceSource::ScannedThisSession); + assert_eq!(balance.last_scanned_index, Some(count + 1)); + assert_eq!(balance.spendable_credits, FIRST_VALUE + SECOND_VALUE); + } + assert_eq!( + activity(&coordinator).await, + expected, + "cancel + resume must derive the same history as one uninterrupted scan" + ); + match coordinator + .local_balance_snapshot(WALLET) + .await + .expect("snapshot") + { + ShieldedLocalBalanceState::Ready(snapshot) => { + assert_eq!( + snapshot.accounts[&0].spendable_credits, + FIRST_VALUE + SECOND_VALUE + ); + } + other => panic!("expected a ready ledger, got {other:?}"), + } +} diff --git a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs index ac00f9158a7..0572fdee041 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs @@ -1481,6 +1481,12 @@ impl NetworkShieldedCoordinator { .collect() }; for (id, persister) in &subwallet_persisters { + // Network broadcasts: a stop that lands after the scan's + // snapshot above stops the loop here, and the helper + // checks the token before each request it starts. + if cancel.is_cancelled() { + break; + } super::operations::redrive_pending_spends( &self.sdk, &self.store, @@ -1488,10 +1494,13 @@ impl NetworkShieldedCoordinator { id.wallet_id, *id, &recorded, + cancel, ) .await; } - self.release_stranded_spends(snapshot, &recorded).await; + if !cancel.is_cancelled() { + self.release_stranded_spends(snapshot, &recorded).await; + } } let balances_per_sub = match super::sync::balances_across(&self.store, &subwallets).await { @@ -1501,14 +1510,27 @@ impl NetworkShieldedCoordinator { // Restore-path activity derivation: reconstruct per-operation // activity entries best-effort from the notes / outgoing notes - // this pass (and earlier passes) persisted. Runs every pass so it - // doubles as the one-time backfill over an already-populated - // store — `derive_activity_from_scan_data` skips clusters whose - // id a live entry already owns, and `save_activity` upserts by - // id, so re-running is idempotent. Failures are logged and - // swallowed: a derivation miss must never fail a sync pass. + // this pass (and earlier passes) persisted. Runs every completed + // pass so it doubles as the one-time backfill over an + // already-populated store — `derive_activity_from_scan_data` + // skips clusters whose id a live entry already owns, and + // `save_activity` upserts by id, so re-running is idempotent. + // Failures are logged and swallowed: a derivation miss must never + // fail a sync pass. + // + // Deferred on a cancelled pass. Derived entries are immutable + // clusters keyed by the discovering chunk's proof height: an + // entry derived from a consumed prefix would freeze the cluster, + // and a note the resumed pass finds at the same height could only + // confirm that entry, never extend it — the amount and cmx set + // would permanently omit the later note. The raw notes, spent + // marks and cursor are flushed regardless; the next pass that + // reaches the tip derives over all of them at once, exactly as an + // uninterrupted scan would have. let mut notes = notes; - if let Err(e) = self + if cancelled { + tracing::debug!("Shielded activity derivation deferred: pass cancelled"); + } else if let Err(e) = self .derive_activity_into_changeset(&subwallets, &mut notes.changeset) .await { @@ -1523,12 +1545,19 @@ impl NetworkShieldedCoordinator { // enumeration includes guards rehydrated from the durable store. let wallet_ids: HashSet = subwallets.iter().map(|(id, _)| id.wallet_id).collect(); for wallet_id in wallet_ids { - // Network broadcasts: stop between wallets once cancelled. + // Network requests: stop between wallets once cancelled; the + // helper also checks the token between records and before + // each request it starts. if cancel.is_cancelled() { break; } - super::operations::redrive_pending_identity_shields(&self.sdk, &self.store, wallet_id) - .await; + super::operations::redrive_pending_identity_shields( + &self.sdk, + &self.store, + wallet_id, + cancel, + ) + .await; } // The note-side changeset already carries saves, synced diff --git a/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs b/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs index 0970e75d48a..123136afefa 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs @@ -877,6 +877,11 @@ impl ShieldedStore for FileBackedShieldedStore { Ok(()) } + fn set_resume_note_index(&mut self, id: SubwalletId, index: u64) -> Result<(), Self::Error> { + self.subwallets.entry(id).or_default().last_synced_index = index; + Ok(()) + } + fn purge_wallet(&mut self, wallet_id: WalletId) -> Result<(), Self::Error> { // The redrive table IS durable (unlike the rest of subwallet // state), so purging the in-memory map alone would leave this diff --git a/packages/rs-platform-wallet/src/wallet/shielded/mod.rs b/packages/rs-platform-wallet/src/wallet/shielded/mod.rs index 3b986f0b770..e5c92c28be6 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/mod.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/mod.rs @@ -38,6 +38,8 @@ pub mod activity_recorder; pub mod balance; #[cfg(test)] mod balance_tests; +#[cfg(test)] +mod cancelled_scan_tests; pub mod coordinator; pub mod file_store; pub mod fund_from_asset_lock; diff --git a/packages/rs-platform-wallet/src/wallet/shielded/operations.rs b/packages/rs-platform-wallet/src/wallet/shielded/operations.rs index ea864889a3f..419219f4235 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/operations.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/operations.rs @@ -73,6 +73,7 @@ use dpp::withdrawal::Pooling; use dpp::ProtocolError; use grovedb_commitment_tree::{Anchor, PaymentAddress}; use tokio::sync::RwLock; +use tokio_util::sync::CancellationToken; use tracing::{debug, info, trace, warn}; /// Number of Orchard actions in a `Shield` (Type 15) bundle. @@ -3118,6 +3119,7 @@ pub(super) async fn redrive_pending_identity_shields( sdk: &Arc, store: &Arc>, wallet_id: WalletId, + cancel: &CancellationToken, ) { use dpp::serialization::PlatformDeserializableTrusted; use dpp::state_transition::shield_from_identity_transition::accessors::ShieldFromIdentityTransitionAccessorsV0; @@ -3130,6 +3132,13 @@ pub(super) async fn redrive_pending_identity_shields( } }; for (id, redrive) in redrives { + // A stop ends the pass between records: no new request starts + // once cancelled. A request already on the wire is left to + // finish on its own (its outcome is handled conservatively). + if cancel.is_cancelled() { + debug!("identity redrive: pass cancelled; remaining guards wait for the next pass"); + return; + } if !redrive.nullifiers.is_empty() || redrive.identity_nonce_finalized || redrive.identity_user_abandoned @@ -3147,9 +3156,17 @@ pub(super) async fn redrive_pending_identity_shields( let StateTransition::ShieldFromIdentity(identity_transition) = &state_transition else { continue; }; - if let Ok(fetched) = - IdentityNonceFetcher::fetch(sdk, identity_transition.identity_id()).await - { + // The nonce read is the one wait here a stop can cut short: it + // is read-only, so dropping it mid-flight loses nothing. + let fetched = tokio::select! { + biased; + _ = cancel.cancelled() => { + debug!("identity redrive: pass cancelled during the nonce read"); + return; + } + fetched = IdentityNonceFetcher::fetch(sdk, identity_transition.identity_id()) => fetched, + }; + if let Ok(fetched) = fetched { let result = { let mut store = store.write().await; reconcile_identity_shield_nonce( @@ -3188,6 +3205,12 @@ pub(super) async fn redrive_pending_identity_shields( if !still_active { continue; } + // Last check before a broadcast starts; a broadcast already + // submitted cannot be cancelled by this check. + if cancel.is_cancelled() { + debug!("identity redrive: pass cancelled before the re-broadcast"); + return; + } // Unused or unverified state permits only the original signed bytes. // Even an apparently definitive rejection cannot release the guard: // this request, or an earlier relay, may deliver the debit afterward. @@ -3223,6 +3246,7 @@ pub(super) async fn redrive_pending_spends( wallet_id: WalletId, id: SubwalletId, recorded: &std::collections::HashSet<[u8; 32]>, + cancel: &CancellationToken, ) { use dpp::serialization::PlatformDeserializableTrusted; @@ -3237,6 +3261,13 @@ pub(super) async fn redrive_pending_spends( } }; for redrive in redrives { + // A stop ends the pass between records: no new re-broadcast starts + // once cancelled. One already submitted is left to finish and is + // classified as usual — its outcome handling stays conservative. + if cancel.is_cancelled() { + debug!("redrive: pass cancelled; remaining records wait for the next pass"); + return; + } // Empty-nullifier identity debits are wallet-wide and are driven by // `redrive_pending_identity_shields`, independently of spending anchors. if redrive.nullifiers.is_empty() { @@ -3646,8 +3677,13 @@ mod redrive_tests { assert!(!has_unresolved_identity_shield(&store, wallet_id, identity_id).unwrap()); let store = Arc::new(RwLock::new(store)); // No mock nonce responses: accidentally retrying would reach an unconfigured SDK. - redrive_pending_identity_shields(&Arc::new(dash_sdk::Sdk::new_mock()), &store, wallet_id) - .await; + redrive_pending_identity_shields( + &Arc::new(dash_sdk::Sdk::new_mock()), + &store, + wallet_id, + &CancellationToken::new(), + ) + .await; assert_eq!( store.read().await.pending_redrives(id).unwrap(), vec![archived.clone()] @@ -3711,7 +3747,7 @@ mod redrive_tests { assert!(store.pending_redrives(id).unwrap()[0].identity_nonce_finalized); let store = Arc::new(RwLock::new(store)); let sdk = Arc::new(dash_sdk::Sdk::new_mock()); - redrive_pending_identity_shields(&sdk, &store, wallet_id).await; + redrive_pending_identity_shields(&sdk, &store, wallet_id, &CancellationToken::new()).await; assert_eq!( store.read().await.pending_redrives(id).unwrap()[0].attempts, 0, @@ -3900,7 +3936,7 @@ mod redrive_tests { ) .expect("arm identity guard"); - redrive_pending_identity_shields(&sdk, &store, wallet_id).await; + redrive_pending_identity_shields(&sdk, &store, wallet_id, &CancellationToken::new()).await; let retained = store .read() @@ -3947,7 +3983,16 @@ mod redrive_tests { .expect("arm identity guard"); let recorded = [anchor].into_iter().collect(); - redrive_pending_spends(&sdk, &store, None, wallet_id, id, &recorded).await; + redrive_pending_spends( + &sdk, + &store, + None, + wallet_id, + id, + &recorded, + &CancellationToken::new(), + ) + .await; let retained = store.read().await.pending_redrives(id).expect("guard"); assert_eq!(retained.len(), 1); @@ -3993,7 +4038,16 @@ mod redrive_tests { let recorded: std::collections::HashSet<[u8; 32]> = [[11u8; 32], [12u8; 32]].into_iter().collect(); - redrive_pending_spends(&sdk, &store, None, wallet_id, id, &recorded).await; + redrive_pending_spends( + &sdk, + &store, + None, + wallet_id, + id, + &recorded, + &CancellationToken::new(), + ) + .await; let left = store.read().await.pending_redrives(id).unwrap(); let ids: Vec<[u8; 32]> = left.iter().map(|r| r.activity_id).collect(); diff --git a/packages/rs-platform-wallet/src/wallet/shielded/store.rs b/packages/rs-platform-wallet/src/wallet/shielded/store.rs index 6e20a0ae1fc..62476079df6 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/store.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/store.rs @@ -469,13 +469,24 @@ pub trait ShieldedStore: Send + Sync { /// (scanning through position `N-1` sets this to `N`). fn last_synced_note_index(&self, id: SubwalletId) -> Result; - /// Persist the sync watermark (next index to scan) for `id`. + /// Persist the sync watermark (next index to scan) for `id` as the + /// end of a COMPLETE scan: also records it as the scan coverage + /// behind [`local_account_balance`](Self::local_account_balance) + /// and stamps the balance source `ScannedThisSession`. fn set_last_synced_note_index( &mut self, id: SubwalletId, index: u64, ) -> Result<(), Self::Error>; + /// Persist the sync watermark (next index to scan) for `id` as a + /// RESUME cursor only: the pass that reached `index` was cut short, + /// so the next pass resumes from here, but the scan coverage and + /// balance source behind + /// [`local_account_balance`](Self::local_account_balance) stay as + /// they were — a consumed prefix is not evidence of a complete scan. + fn set_resume_note_index(&mut self, id: SubwalletId, index: u64) -> Result<(), Self::Error>; + // ── Per-subwallet lifecycle ──────────────────────────────────────── /// Drop ALL in-memory per-subwallet state (decrypted notes, @@ -1111,6 +1122,11 @@ impl ShieldedStore for InMemoryShieldedStore { Ok(()) } + fn set_resume_note_index(&mut self, id: SubwalletId, index: u64) -> Result<(), Self::Error> { + self.subwallets.entry(id).or_default().last_synced_index = index; + Ok(()) + } + fn purge_wallet(&mut self, wallet_id: WalletId) -> Result<(), Self::Error> { self.subwallets.retain(|id, _| id.wallet_id != wallet_id); Ok(()) diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index 079e91504e1..b2bded1e607 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -776,20 +776,29 @@ where // subwallet that was already further along; a completed pass // always reaches the tip, which is at or past every watermark. // A cancelled pass leaves such a subwallet's row alone entirely: - // its consumed prefix never covered that subwallet, so neither - // the watermark nor the `ScannedThisSession` stamp the setter - // applies belongs to it. A completed pass keeps the unconditional - // write, so a caught-up subwallet still gets its stamp. + // its consumed prefix never covered that subwallet. Where it did + // advance a subwallet, only the RESUME cursor moves: the prefix is + // not a complete scan, so the scan coverage and balance source the + // completed-pass setter stamps (`ScannedThisSession`, which lets + // `local_balance_snapshot` report an unhydrated account as ready) + // stay untouched until a pass runs to the tip. The host still gets + // the cursor in the changeset so a restart resumes from it. let new_index = aligned_start + total_notes_scanned; for (id, _) in subwallets { let prior = watermarks.get(id).copied().unwrap_or(0); let advanced = prior.max(new_index); - if cancelled && advanced == prior { - continue; + if cancelled { + if advanced == prior { + continue; + } + store + .set_resume_note_index(*id, advanced) + .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; + } else { + store + .set_last_synced_note_index(*id, advanced) + .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; } - store - .set_last_synced_note_index(*id, advanced) - .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; changeset.record_synced_index(*id, advanced); } // Drop the write lock before returning so the caller's diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs index 4492256eb12..0092c191164 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs @@ -15,6 +15,7 @@ use tokio::sync::RwLock; use tokio_util::sync::CancellationToken; use super::{sync_notes_from_stream, MultiSyncNotesResult}; +use crate::wallet::shielded::balance::ShieldedBalanceSource; use crate::wallet::shielded::file_store::FileBackedShieldedStore; use crate::wallet::shielded::keys::{AccountViewingKeys, OrchardKeySet}; use crate::wallet::shielded::store::{ShieldedStore, SubwalletId}; @@ -128,6 +129,11 @@ async fn should_finalize_consumed_prefix_when_cancelled_mid_stream() { store.witness(7).unwrap().is_some(), "a checkpoint covers the appended leaves" ); + // A resume cursor, not a completed scan: the ledger's coverage and + // provenance are untouched, so an unhydrated account stays unavailable. + let balance = store.local_account_balance(sub).unwrap(); + assert_eq!(balance.last_scanned_index, None); + assert_eq!(balance.source, ShieldedBalanceSource::NoHistory); let _ = std::fs::remove_file(&path); } @@ -216,6 +222,13 @@ async fn should_resume_from_partial_watermark_on_next_pass() { "the second pass checkpoints past the partial one" ); assert!(store.witness(7).unwrap().is_some()); + let balance = store.local_account_balance(sub).unwrap(); + assert_eq!(balance.last_scanned_index, Some(12)); + assert_eq!( + balance.source, + ShieldedBalanceSource::ScannedThisSession, + "the completed pass certifies the scan" + ); let _ = std::fs::remove_file(&path); } From 4f9d039c5dad1a66095f5d938caa23bf3a2cecf7 Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Tue, 3 Nov 2026 17:04:46 +0100 Subject: [PATCH 08/12] fix(platform-wallet): keep a cancelled pass's cursor out of the host watermark The changeset watermark is what a host restores as completed-scan coverage, and the persistence contract has no separate resume field, so a partial cursor sent there made a restart report an account as covered up to a position no pass ever completed. A cancelled pass now keeps its cursor in memory only; a restart resumes from the last completed watermark and re-walks the prefix, gate-skipped by the persisted tree. The consumed notes and spent marks still reach the host. Also: a cancelled pass that consumed anything clears a stale caught-up stamp so the next background pass is not parked behind the cooldown, and sync_wallet takes its pass token before the coordinator read, as sync_now does. Regressions: the coordinator test records the host changesets and checks that the receipt arrives on the cancelled pass and no watermark does; a second test cancels after a chunk with no owned note, persists, reopens the tree in a new coordinator, restores from the recorded host state and reads Ready with no coverage, then finds the receipt on the first pass after the restart. Co-Authored-By: Claude Fable 5.1 --- .../src/manager/shielded_sync.rs | 10 +- .../wallet/shielded/cancelled_scan_tests.rs | 228 +++++++++++++++++- .../src/wallet/shielded/coordinator.rs | 21 +- .../src/wallet/shielded/sync.rs | 39 +-- .../src/wallet/shielded/sync/cancel_tests.rs | 12 +- 5 files changed, 266 insertions(+), 44 deletions(-) diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index 858ca698139..fd85477dc1b 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -541,6 +541,12 @@ impl ShieldedSyncManager { wallet_id: &WalletId, force: bool, ) -> Result, crate::error::PlatformWalletError> { + // Token first — before the coordinator read below as well as the + // slot CAS and the gate check (see `sync_now` for the ordering): + // a `stop()` that lands while this waits for the coordinator lock + // must still cancel this pass, not hand it a child of the rotated, + // uncancelled parent. + let pass_cancel = self.pass_token(); let coordinator_snapshot: Option> = { let slot = self.coordinator_slot.read().await; slot.as_ref().map(Arc::clone) @@ -548,10 +554,6 @@ impl ShieldedSyncManager { let Some(coordinator) = coordinator_snapshot else { return Ok(None); }; - - // Token before the slot CAS and the gate check — see `sync_now` - // for the ordering. - let pass_cancel = self.pass_token(); // Reuse the manager-wide `is_syncing` flag so a per-wallet // `sync_wallet()` can't race the periodic `sync_now()` // against the same store — both go through diff --git a/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs index d68865a588b..ddce1191eec 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs @@ -8,10 +8,16 @@ //! scan: no activity entry is derived from it and the ledger stays //! unavailable. The resumed pass then yields exactly what an //! uninterrupted scan yields — one entry carrying both notes. +//! +//! The host side is checked through a recording persister: the consumed +//! receipt reaches the host, the partial cursor does not, and a restart +//! restored from what the host recorded carries no scan coverage until +//! a pass completes. +use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use dash_sdk::Sdk; use drive_proof_verifier::types::{ @@ -30,12 +36,79 @@ use super::keys::AccountViewingKeys; use super::{ FileBackedShieldedStore, NetworkShieldedCoordinator, OrchardKeySet, ShieldedStore, SubwalletId, }; -use crate::wallet::persister::{NoPlatformPersistence, WalletPersister}; +use crate::changeset::{ + ClientStartState, PersistenceError, PlatformWalletChangeSet, PlatformWalletPersistence, + ShieldedChangeSet, ShieldedSubwalletStartState, ShieldedSyncStartState, +}; +use crate::wallet::persister::WalletPersister; +use crate::wallet::platform_wallet::WalletId; const WALLET: [u8; 32] = [7; 32]; const FIRST_VALUE: u64 = 100; const SECOND_VALUE: u64 = 200; +/// A host persister that keeps every shielded changeset it was handed, +/// and can hand them back as the start state a cold bind would restore. +#[derive(Default)] +struct RecordingPersistence { + changesets: Mutex>, +} + +impl PlatformWalletPersistence for RecordingPersistence { + fn store( + &self, + _wallet_id: WalletId, + changeset: PlatformWalletChangeSet, + ) -> Result<(), PersistenceError> { + if let Some(shielded) = changeset.shielded { + self.changesets.lock().expect("recorder").push(shielded); + } + Ok(()) + } + + fn flush(&self, _wallet_id: WalletId) -> Result<(), PersistenceError> { + Ok(()) + } + + fn load(&self) -> Result { + Ok(ClientStartState::default()) + } +} + +impl RecordingPersistence { + fn recorded(&self) -> Vec { + self.changesets.lock().expect("recorder").clone() + } + + /// Fold the recorded changesets the way a host snapshot does: notes + /// accumulate, the watermark is the highest one received, and sync + /// state exists only if a watermark was ever received. + fn start_state(&self) -> ShieldedSyncStartState { + let mut per_subwallet: BTreeMap = BTreeMap::new(); + for changeset in self.recorded() { + for (id, notes) in changeset.notes_saved { + per_subwallet.entry(id).or_default().notes.extend(notes); + } + for (id, index) in changeset.synced_indices { + let sub = per_subwallet.entry(id).or_default(); + sub.has_sync_state = true; + sub.last_synced_index = sub.last_synced_index.max(index); + } + for (id, entries) in changeset.activity_entries { + per_subwallet + .entry(id) + .or_default() + .activity + .extend(entries); + } + } + ShieldedSyncStartState { + per_subwallet, + ..Default::default() + } + } +} + fn tree_path() -> PathBuf { static NEXT_PATH: AtomicU64 = AtomicU64::new(0); let sequence = NEXT_PATH.fetch_add(1, Ordering::Relaxed); @@ -166,25 +239,56 @@ async fn two_chunk_pool( (Arc::new(sdk), count) } -async fn coordinator_over(sdk: Arc, keys: &AccountViewingKeys) -> NetworkShieldedCoordinator { - let path = tree_path(); +async fn coordinator_at( + sdk: Arc, + path: &std::path::Path, + keys: &AccountViewingKeys, + host: Arc, +) -> NetworkShieldedCoordinator { let coordinator = NetworkShieldedCoordinator::new( sdk, dashcore::Network::Testnet, - path.clone(), + path.to_path_buf(), FileBackedShieldedStore::open_path(path, 100).expect("open test store"), ); coordinator .register_wallet( WALLET, [(0, keys.clone())].into_iter().collect(), - WalletPersister::new(WALLET, Arc::new(NoPlatformPersistence)), + WalletPersister::new(WALLET, host), ) .await .expect("register test wallet"); coordinator } +async fn coordinator_over( + sdk: Arc, + keys: &AccountViewingKeys, +) -> (NetworkShieldedCoordinator, Arc) { + let host = Arc::new(RecordingPersistence::default()); + let coordinator = coordinator_at(sdk, &tree_path(), keys, Arc::clone(&host)).await; + (coordinator, host) +} + +fn host_synced_index(host: &RecordingPersistence, sub: SubwalletId) -> Option { + host.recorded() + .iter() + .filter_map(|changeset| changeset.synced_indices.get(&sub).copied()) + .max() +} + +fn host_note_values(host: &RecordingPersistence, sub: SubwalletId) -> Vec { + let mut values: Vec = host + .recorded() + .iter() + .flat_map(|changeset| changeset.notes_saved.get(&sub).cloned().unwrap_or_default()) + .map(|note| note.value) + .collect(); + values.sort_unstable(); + values +} + async fn activity(coordinator: &NetworkShieldedCoordinator) -> Vec { let mut entries = coordinator .store() @@ -208,7 +312,7 @@ async fn should_defer_activity_and_readiness_until_a_pass_completes() { // The same pool, scanned in one go: the reference history. let (baseline_sdk, _) = two_chunk_pool(&first, &second).await; - let baseline = coordinator_over(baseline_sdk, &keys).await; + let (baseline, _) = coordinator_over(baseline_sdk, &keys).await; let summary = baseline.sync(true, &CancellationToken::new()).await; assert_eq!( baseline @@ -228,7 +332,7 @@ async fn should_defer_activity_and_readiness_until_a_pass_completes() { // Pass 1: the stop lands after the first chunk is committed. let (sdk, count) = two_chunk_pool(&first, &second).await; - let coordinator = coordinator_over(sdk, &keys).await; + let (coordinator, host) = coordinator_over(sdk, &keys).await; let cancel = CancellationToken::new(); let stop = cancel.clone(); coordinator.install_tree_progress_handler(Some(Arc::new(move |_, _| stop.cancel()))); @@ -266,6 +370,15 @@ async fn should_defer_activity_and_readiness_until_a_pass_completes() { activity(&coordinator).await.is_empty(), "no activity entry may be derived from a partial scan" ); + // The host got the consumed receipt (the advanced in-memory cursor + // means no later pass re-emits it), but not the cursor itself, which + // it would restore as completed-scan coverage. + assert_eq!(host_note_values(&host, sub), vec![FIRST_VALUE]); + assert_eq!(host_synced_index(&host, sub), None); + assert!(host + .recorded() + .iter() + .all(|changeset| changeset.activity_entries.is_empty())); assert_eq!( coordinator .local_balance_snapshot(WALLET) @@ -291,6 +404,15 @@ async fn should_defer_activity_and_readiness_until_a_pass_completes() { expected, "cancel + resume must derive the same history as one uninterrupted scan" ); + assert_eq!( + host_note_values(&host, sub), + vec![FIRST_VALUE, SECOND_VALUE] + ); + assert_eq!( + host_synced_index(&host, sub), + Some(count + 1), + "the completed pass is the first watermark the host sees" + ); match coordinator .local_balance_snapshot(WALLET) .await @@ -305,3 +427,93 @@ async fn should_defer_activity_and_readiness_until_a_pass_completes() { other => panic!("expected a ready ledger, got {other:?}"), } } + +/// Cancel → persist → restart → restore, with no owned note in the +/// cancelled prefix (the worst case: a restored account that reads as +/// covered would read as an empty, finished scan). The host restored +/// nothing that claims coverage, and the first pass after the restart +/// finds the receipt the cancelled pass never reached. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn should_not_restore_scan_coverage_from_a_cancelled_prefix() { + let keys = viewing_keys(); + let sub = SubwalletId::new(WALLET, 0); + // No receipt in the first chunk: make the "first" note someone else's + // by encrypting it to a different account. + let foreign = OrchardKeySet::from_seed(&[43; 64], dashcore::Network::Testnet, 0) + .expect("foreign viewing key") + .viewing_keys(); + let first = owned_note(&foreign, FIRST_VALUE); + let second = owned_note(&keys, SECOND_VALUE); + let path = tree_path(); + + // Session 1: the pass is stopped after the first (foreign-only) chunk. + let host = Arc::new(RecordingPersistence::default()); + { + let (sdk, count) = two_chunk_pool(&first, &second).await; + let coordinator = coordinator_at(sdk, &path, &keys, Arc::clone(&host)).await; + let cancel = CancellationToken::new(); + let stop = cancel.clone(); + coordinator.install_tree_progress_handler(Some(Arc::new(move |_, _| stop.cancel()))); + tokio::time::timeout( + std::time::Duration::from_secs(60), + coordinator.sync(true, &cancel), + ) + .await + .expect("a cancelled pass returns"); + let store = coordinator.store().read().await; + assert_eq!(store.last_synced_note_index(sub).expect("cursor"), count); + assert!(store.get_all_notes(sub).expect("notes").is_empty()); + } + assert!(host_note_values(&host, sub).is_empty()); + assert_eq!(host_synced_index(&host, sub), None); + + // Session 2: reopen the same tree file, restore what the host kept, + // and read the ledger offline as a cold bind does. + let (sdk, count) = two_chunk_pool(&first, &second).await; + let coordinator = coordinator_at(sdk, &path, &keys, Arc::clone(&host)).await; + coordinator + .restore_for_wallet(WALLET, &host.start_state()) + .await + .expect("restore"); + coordinator.mark_hydrated(WALLET, true).await; + match coordinator + .local_balance_snapshot(WALLET) + .await + .expect("snapshot") + { + ShieldedLocalBalanceState::Ready(snapshot) => { + let account = snapshot.accounts[&0]; + assert_eq!(account.spendable_credits, 0); + assert_eq!( + account.last_scanned_index, None, + "a cancelled prefix must not come back as scan coverage" + ); + assert_eq!(account.source, ShieldedBalanceSource::NoHistory); + } + other => panic!("expected the hydrated ledger, got {other:?}"), + } + assert_eq!( + coordinator + .store() + .read() + .await + .last_synced_note_index(sub) + .expect("cursor"), + 0, + "the restart resumes from the last completed watermark" + ); + + // The first pass after the restart re-walks the prefix (gate-skipped + // by the persisted tree) and reaches the receipt. + let summary = coordinator.sync(true, &CancellationToken::new()).await; + assert_eq!(summary.success_count(), 1); + let store = coordinator.store().read().await; + let balance = store.local_account_balance(sub).expect("balance"); + assert_eq!(balance.spendable_credits, SECOND_VALUE); + assert_eq!(balance.last_scanned_index, Some(count + 1)); + assert_eq!(balance.source, ShieldedBalanceSource::ScannedThisSession); + drop(store); + assert_eq!(host_note_values(&host, sub), vec![SECOND_VALUE]); + assert_eq!(host_synced_index(&host, sub), Some(count + 1)); + let _ = std::fs::remove_file(&path); +} diff --git a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs index 0572fdee041..02910a730f5 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs @@ -1593,17 +1593,18 @@ impl NetworkShieldedCoordinator { // Cooldown decision based on aggregate activity across // every subwallet — any new commitment scanned or // newly-spent nullifier anywhere on the network clears - // the stamp so the next pass runs immediately. A cancelled - // pass proves nothing about being caught up (it may have - // stopped before its first batch), so it leaves the stamp alone. + // the stamp so the next pass runs immediately — a cancelled pass + // included: what it consumed is real activity, and leaving a + // stamp from before it would park the next background pass + // behind the cooldown. Only a NEW stamp needs an uncancelled + // pass: a cancelled one proves nothing about being caught up (it + // may have stopped before its first batch). let any_activity = notes.total_scanned > 0 || newly_spent_per_sub.values().any(|&n| n > 0); - if !cancelled { - if let Ok(mut guard) = self.last_caught_up_at.lock() { - if any_activity { - *guard = None; - } else { - *guard = Some(Instant::now()); - } + if let Ok(mut guard) = self.last_caught_up_at.lock() { + if any_activity { + *guard = None; + } else if !cancelled { + *guard = Some(Instant::now()); } } tracing::debug!( diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index b2bded1e607..4b47aa98cc7 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -777,28 +777,37 @@ where // always reaches the tip, which is at or past every watermark. // A cancelled pass leaves such a subwallet's row alone entirely: // its consumed prefix never covered that subwallet. Where it did - // advance a subwallet, only the RESUME cursor moves: the prefix is - // not a complete scan, so the scan coverage and balance source the - // completed-pass setter stamps (`ScannedThisSession`, which lets - // `local_balance_snapshot` report an unhydrated account as ready) - // stay untouched until a pass runs to the tip. The host still gets - // the cursor in the changeset so a restart resumes from it. + // advance a subwallet, only the in-memory RESUME cursor moves: the + // prefix is not a complete scan, so the scan coverage and balance + // source the completed-pass setter stamps (`ScannedThisSession`, + // which lets `local_balance_snapshot` report an unhydrated account + // as ready) stay untouched until a pass runs to the tip. + // + // The cursor is NOT sent to the host. The changeset's watermark is + // the one the host restores as completed-scan coverage (`Restored` + // provenance with `last_scanned_index` set), and the persistence + // contract has no separate resume field; emitting a partial cursor + // there would make a restart report an account as covered up to a + // position no pass ever completed. So a restart resumes from the + // last COMPLETED watermark and re-walks the prefix: cheap, since + // the appended leaves are gate-skipped by the tree size and the + // flushed notes are re-saved idempotently. The consumed notes and + // spent marks still reach the host below, so none of them is lost. let new_index = aligned_start + total_notes_scanned; for (id, _) in subwallets { let prior = watermarks.get(id).copied().unwrap_or(0); let advanced = prior.max(new_index); if cancelled { - if advanced == prior { - continue; + if advanced > prior { + store + .set_resume_note_index(*id, advanced) + .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; } - store - .set_resume_note_index(*id, advanced) - .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; - } else { - store - .set_last_synced_note_index(*id, advanced) - .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; + continue; } + store + .set_last_synced_note_index(*id, advanced) + .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; changeset.record_synced_index(*id, advanced); } // Drop the write lock before returning so the caller's diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs index 0092c191164..a50b041a40b 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs @@ -267,13 +267,11 @@ async fn should_not_rewind_a_more_advanced_subwallet_when_cancelled() { 12, "a partial prefix below a subwallet's watermark must not rewind it" ); - assert_eq!( - result.changeset.synced_indices.get(&lagging).copied(), - Some(8) - ); assert!( - !result.changeset.synced_indices.contains_key(&advanced), - "no rewound watermark may reach the host persister" + result.changeset.synced_indices.is_empty(), + "a cancelled pass sends no watermark to the host: neither the partial \ + cursor (the host would restore it as completed coverage) nor a \ + rewound one" ); let _ = std::fs::remove_file(&path); @@ -308,7 +306,7 @@ async fn should_leave_store_and_changeset_empty_when_cancelled_before_first_batc assert_eq!(result.total_scanned, 0); assert!( crate::changeset::merge::Merge::is_empty(&result.changeset), - "a pass that consumed nothing records nothing, not even its own watermark" + "a pass that consumed nothing records nothing" ); let store = store.read().await; assert_eq!(store.tree_size().unwrap(), 0); From 193d446829f3ec90cf6eec60e9a2d05fd6740013 Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Tue, 3 Nov 2026 17:37:39 +0100 Subject: [PATCH 09/12] fix(platform-wallet): leave receipts the store already holds alone on a re-walked prefix A restart after a cancelled pass resumes from the last completed watermark and re-walks a prefix whose receipts the host restored, possibly already marked spent by that same pass. save_note replaces by nullifier, so re-saving one as a fresh is_spent: false receipt resurrected a spent note until the replay reached its nullifier, which a second stop could prevent. The scan now skips notes it knows by nullifier and does not re-emit them. Regression: receipt in chunk 0, its spend in chunk 1, cancel after chunk 1; restart, restore (receipt spent), cancel the replay after chunk 0. The receipt stays spent and is not re-emitted; the pass that reaches the tip keeps it spent. The recording persister now applies spent marks when it folds a start state, as a host snapshot does. Co-Authored-By: Claude Fable 5.1 --- .../wallet/shielded/cancelled_scan_tests.rs | 179 ++++++++++++++++-- .../src/wallet/shielded/sync.rs | 37 +++- 2 files changed, 191 insertions(+), 25 deletions(-) diff --git a/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs index ddce1191eec..1ef50639d27 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs @@ -81,14 +81,22 @@ impl RecordingPersistence { } /// Fold the recorded changesets the way a host snapshot does: notes - /// accumulate, the watermark is the highest one received, and sync - /// state exists only if a watermark was ever received. + /// accumulate, a spent nullifier marks its note spent, the watermark + /// is the highest one received, and sync state exists only if a + /// watermark was ever received. fn start_state(&self) -> ShieldedSyncStartState { let mut per_subwallet: BTreeMap = BTreeMap::new(); for changeset in self.recorded() { for (id, notes) in changeset.notes_saved { per_subwallet.entry(id).or_default().notes.extend(notes); } + for (id, nullifiers) in changeset.nullifiers_spent { + for note in &mut per_subwallet.entry(id).or_default().notes { + if nullifiers.contains(¬e.nullifier) { + note.is_spent = true; + } + } + } for (id, index) in changeset.synced_indices { let sub = per_subwallet.entry(id).or_default(); sub.has_sync_state = true; @@ -133,6 +141,15 @@ fn viewing_keys() -> AccountViewingKeys { /// finds it; no OVK, so it is a plain receipt and not a self-send; no /// proof, since nothing here is admitted to consensus. fn owned_note(keys: &AccountViewingKeys, value: u64) -> ShieldedEncryptedNote { + owned_note_with_nullifier(keys, value).0 +} + +/// [`owned_note`] plus the nullifier the account derives for it: the +/// value a later action carries when it spends this note. +fn owned_note_with_nullifier( + keys: &AccountViewingKeys, + value: u64, +) -> (ShieldedEncryptedNote, [u8; 32]) { let mut rng = OsRng; let (nf, rho) = loop { let mut b = [0u8; 32]; @@ -172,12 +189,16 @@ fn owned_note(keys: &AccountViewingKeys, value: u64) -> ShieldedEncryptedNote { encrypted_note.extend_from_slice(&epk.0); encrypted_note.extend_from_slice(enc.as_ref()); encrypted_note.extend_from_slice(&out); - ShieldedEncryptedNote { - cmx: cmx.to_bytes().to_vec(), - nullifier: nf.to_bytes().to_vec(), - cv_net: cv.to_bytes().to_vec(), - encrypted_note, - } + let own_nullifier = note.nullifier(&keys.full_viewing_key).to_bytes(); + ( + ShieldedEncryptedNote { + cmx: cmx.to_bytes().to_vec(), + nullifier: nf.to_bytes().to_vec(), + cv_net: cv.to_bytes().to_vec(), + encrypted_note, + }, + own_nullifier, + ) } /// A note nobody owns at tree position `pos`: an empty ciphertext never @@ -204,6 +225,13 @@ async fn two_chunk_pool( first: &ShieldedEncryptedNote, second: &ShieldedEncryptedNote, ) -> (Arc, u64) { + chunked_pool(&[vec![first.clone()], vec![second.clone()]]).await +} + +/// A pool of full chunks, each opening with the given notes and padded +/// with opaque ones, except the last chunk, which holds its notes alone +/// and so ends the pool. +async fn chunked_pool(leading: &[Vec]) -> (Arc, u64) { let mut sdk = Sdk::new_mock(); let count = 2048 * u64::from( @@ -213,17 +241,20 @@ async fn two_chunk_pool( .shielded_queries .max_query_chunks, ); - let total_count = count + 1; - // The first chunk is full, so the stream's 16-wide speculative window - // slides past it before the short second chunk ends the pool: cover - // every chunk it may still request. - for chunk in 0..48u64 { - let notes = match chunk { - 0 => std::iter::once(first.clone()) - .chain((1..count).map(opaque_note)) + let last = leading.len() as u64 - 1; + let total_count = count * last + leading[last as usize].len() as u64; + // Full chunks let the stream's 16-wide speculative window slide past + // them before the short last chunk ends the pool: cover every chunk + // it may still request. + for chunk in 0..(48 + 16 * last) { + let notes: Vec = match leading.get(chunk as usize) { + Some(notes) if chunk == last => notes.clone(), + Some(notes) => notes + .iter() + .cloned() + .chain((notes.len() as u64..count).map(|i| opaque_note(count * chunk + i))) .collect(), - 1 => vec![second.clone()], - _ => Vec::new(), + None => Vec::new(), }; sdk.mock() .expect_fetch( @@ -517,3 +548,115 @@ async fn should_not_restore_scan_coverage_from_a_cancelled_prefix() { assert_eq!(host_synced_index(&host, sub), Some(count + 1)); let _ = std::fs::remove_file(&path); } + +/// The receipt is in chunk 0 and the action that spends it in chunk 1. +/// Session 1 is cancelled after chunk 1, so the host holds the receipt +/// marked spent and no watermark. Session 2 restores it, re-walks from +/// the completed watermark and is cancelled after chunk 0 — before the +/// nullifier replay could mark it spent again. The replayed receipt +/// must not come back unspent, in the store or towards the host. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn should_keep_a_replayed_receipt_spent_across_a_cancelled_restart() { + let keys = viewing_keys(); + let sub = SubwalletId::new(WALLET, 0); + let (receipt, own_nullifier) = owned_note_with_nullifier(&keys, FIRST_VALUE); + let mut spend = opaque_note(u64::MAX); + spend.nullifier = own_nullifier.to_vec(); + let tail = owned_note(&keys, SECOND_VALUE); + let layout = [vec![receipt], vec![spend], vec![tail]]; + let path = tree_path(); + + /// Cancel once the `after`-th chunk has been committed. + fn cancel_after(cancel: &CancellationToken, after: u64) -> Arc { + let stop = cancel.clone(); + let committed = Arc::new(std::sync::atomic::AtomicU64::new(0)); + Arc::new(move |_, _| { + if committed.fetch_add(1, Ordering::SeqCst) + 1 >= after { + stop.cancel(); + } + }) + } + + // Session 1: receipt and its spend consumed, then stopped. + let host = Arc::new(RecordingPersistence::default()); + { + let (sdk, _) = chunked_pool(&layout).await; + let coordinator = coordinator_at(sdk, &path, &keys, Arc::clone(&host)).await; + let cancel = CancellationToken::new(); + coordinator.install_tree_progress_handler(Some(cancel_after(&cancel, 2))); + tokio::time::timeout( + std::time::Duration::from_secs(60), + coordinator.sync(true, &cancel), + ) + .await + .expect("a cancelled pass returns"); + let store = coordinator.store().read().await; + let notes = store.get_all_notes(sub).expect("notes"); + assert_eq!(notes.len(), 1); + assert!(notes[0].is_spent, "the spend in chunk 1 was replayed"); + assert_eq!(store.spendable_balance(sub).expect("balance"), 0); + } + assert_eq!(host_note_values(&host, sub), vec![FIRST_VALUE]); + assert!(host.recorded().iter().any(|changeset| changeset + .nullifiers_spent + .get(&sub) + .is_some_and(|nfs| nfs.contains(&own_nullifier)))); + assert_eq!(host_synced_index(&host, sub), None); + + // Session 2: restore the spent receipt, re-walk chunk 0, stop again. + let (sdk, count) = chunked_pool(&layout).await; + let coordinator = coordinator_at(sdk, &path, &keys, Arc::clone(&host)).await; + coordinator + .restore_for_wallet(WALLET, &host.start_state()) + .await + .expect("restore"); + coordinator.mark_hydrated(WALLET, true).await; + { + let store = coordinator.store().read().await; + let notes = store.get_all_notes(sub).expect("notes"); + assert_eq!(notes.len(), 1); + assert!(notes[0].is_spent, "restore installs the receipt as spent"); + } + let recorded_before = host.recorded().len(); + let cancel = CancellationToken::new(); + coordinator.install_tree_progress_handler(Some(cancel_after(&cancel, 1))); + tokio::time::timeout( + std::time::Duration::from_secs(60), + coordinator.sync(true, &cancel), + ) + .await + .expect("a cancelled pass returns"); + coordinator.install_tree_progress_handler(None); + { + let store = coordinator.store().read().await; + let notes = store.get_all_notes(sub).expect("notes"); + assert_eq!(notes.len(), 1); + assert!( + notes[0].is_spent, + "a replayed receipt must keep its spent state" + ); + assert_eq!(store.spendable_balance(sub).expect("balance"), 0); + assert_eq!(store.last_synced_note_index(sub).expect("cursor"), count); + } + assert!( + host.recorded()[recorded_before..] + .iter() + .all(|changeset| !changeset.notes_saved.contains_key(&sub)), + "a replayed receipt is not re-emitted to the host" + ); + + // The pass that reaches the tip keeps it spent and finds the tail. + coordinator.sync(true, &CancellationToken::new()).await; + let store = coordinator.store().read().await; + let mut values: Vec<(u64, bool)> = store + .get_all_notes(sub) + .expect("notes") + .iter() + .map(|note| (note.value, note.is_spent)) + .collect(); + values.sort_unstable(); + assert_eq!(values, vec![(FIRST_VALUE, true), (SECOND_VALUE, false)]); + assert_eq!(store.spendable_balance(sub).expect("balance"), SECOND_VALUE); + drop(store); + let _ = std::fs::remove_file(&path); +} diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index 4b47aa98cc7..c533f680ac4 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -26,7 +26,7 @@ //! [`NetworkShieldedCoordinator::sync`]: //! super::coordinator::NetworkShieldedCoordinator::sync -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use dash_sdk::platform::shielded::notes_sync::types::ShieldedChunkBatch; @@ -672,16 +672,38 @@ where // Gate on THIS subwallet's own watermark (not the network // min): a caught-up subwallet skips re-deriving nullifiers // for notes it already stored, while a lagging one still - // saves everything from its own start. `save_note` is an - // idempotent overwrite-by-nullifier, so a stray re-save is - // harmless — but gating per-subwallet keeps the - // `per_subwallet_new_notes` count honest. + // saves everything from its own start. let sub_watermark = watermarks.get(id).copied().unwrap_or(0); + // Notes the store already holds, by nullifier. A position above + // the watermark can still be a known note: a restart after a + // cancelled pass resumes from the last COMPLETED watermark (the + // cancelled cursor is not persisted) and re-walks a prefix whose + // receipts the host restored, possibly already marked spent by + // that same pass. `save_note` replaces by nullifier, so re-saving + // one as a fresh `is_spent: false` receipt would resurrect a + // spent note until the replay reaches its nullifier — which a + // second stop can prevent. A known note is left alone and not + // re-emitted: the host has it. + let known_nullifiers: BTreeSet<[u8; 32]> = store + .get_all_notes(*id) + .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))? + .into_iter() + .map(|note| note.nullifier) + .collect(); for d in discovered { if d.position < sub_watermark { continue; } let nullifier = d.note.nullifier(&views.full_viewing_key); + if known_nullifiers.contains(&nullifier.to_bytes()) { + debug!( + wallet_id = %hex::encode(id.wallet_id), + account = id.account_index, + position = d.position, + "Note already on file; replayed receipt left as is" + ); + continue; + } let value = d.note.value().inner(); debug!( wallet_id = %hex::encode(id.wallet_id), @@ -791,8 +813,9 @@ where // position no pass ever completed. So a restart resumes from the // last COMPLETED watermark and re-walks the prefix: cheap, since // the appended leaves are gate-skipped by the tree size and the - // flushed notes are re-saved idempotently. The consumed notes and - // spent marks still reach the host below, so none of them is lost. + // receipts the host restored are recognised by nullifier and left + // alone (spent state included). The consumed notes and spent marks + // still reach the host below, so none of them is lost. let new_index = aligned_start + total_notes_scanned; for (id, _) in subwallets { let prior = watermarks.get(id).copied().unwrap_or(0); From bfde07795659ef1a243381ed78f0da4c7850dc4d Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Tue, 3 Nov 2026 18:22:35 +0100 Subject: [PATCH 10/12] fix(platform-wallet): publish nothing from a shielded scan that was cut short A consumed prefix is not a complete scan: a receipt in it may already be spent by an action in a chunk the scan never reached, and the ledger has no way to hold such a note without offering it for selection. Finalizing and flushing the prefix could not be made correct without new ledger state; the earlier rounds fixed frozen activity clusters, certified coverage and resurrected spent notes in turn, and each fix surfaced the next. A scan cut short now checkpoints the commitments it appended to the local tree and returns the empty result: no notes, no spent marks, no watermark, nothing to the host, no completion event. The next pass re-walks from the last completed watermark; the appended leaves make it skip every append and checkpoint already made, so a stop costs the download and decryption of the chunks it had consumed, and is never worse than the status quo. The resume-only store setter and the known-note skip are gone with the prefix they served; the pass token in the re-drive helpers and the cooldown handling stay. Regressions (real coordinator, mock pool, recording host persister): cancel after one of two chunks publishes nothing and the next pass derives history byte-identical to an uninterrupted scan; cancel, reopen the tree, restore and hydrate reads Ready with no coverage and the first pass finds the receipt; a receipt whose spend sits in the next chunk, cancelled after that chunk and again after a restart before it, never appears spendable. Co-Authored-By: Claude Fable 5.1 --- .../src/manager/shielded_sync.rs | 33 +-- .../wallet/shielded/cancelled_scan_tests.rs | 260 ++++++++---------- .../src/wallet/shielded/coordinator.rs | 80 +++--- .../src/wallet/shielded/file_store.rs | 5 - .../src/wallet/shielded/store.rs | 21 +- .../src/wallet/shielded/sync.rs | 106 +++---- .../src/wallet/shielded/sync/cancel_tests.rs | 61 ++-- 7 files changed, 246 insertions(+), 320 deletions(-) diff --git a/packages/rs-platform-wallet/src/manager/shielded_sync.rs b/packages/rs-platform-wallet/src/manager/shielded_sync.rs index fd85477dc1b..e8fd82a64a4 100644 --- a/packages/rs-platform-wallet/src/manager/shielded_sync.rs +++ b/packages/rs-platform-wallet/src/manager/shielded_sync.rs @@ -310,13 +310,14 @@ impl ShieldedSyncManager { /// any. Returns immediately. /// /// A pass already inside `coordinator.sync()` observes the - /// cancellation at its next batch or network boundary, finalizes the - /// prefix it consumed and flushes that to the persisters, then - /// releases `is_syncing`. So this is **not** a barrier: for a real - /// "nothing is running and nothing more will be persisted" guarantee - /// — required by Clear, unregister, and rebind — use - /// [`quiesce`](Self::quiesce), which calls this first and then waits - /// for that finalize. + /// cancellation at its next batch or network boundary: a scan cut + /// short keeps the commitments it appended to the local tree and + /// publishes nothing else, a scan that had completed still flushes + /// its result, then the pass releases `is_syncing`. So this is + /// **not** a barrier: for a real "nothing is running and nothing + /// more will be persisted" guarantee — required by Clear, + /// unregister, and rebind — use [`quiesce`](Self::quiesce), which + /// calls this first and then waits for that release. pub fn stop(&self) { // Cancel the current pass parent and rotate in a fresh one, so a // pass that starts after this stop runs under an uncancelled token. @@ -485,12 +486,12 @@ impl ShieldedSyncManager { ShieldedSyncPassSummary::default() }; - // A pass cancelled before it did anything is not a completed pass: - // no timestamp, no completion event (same as losing the slot CAS). - // The coordinator returns the empty summary for exactly that case - // (a cancelled scan that consumed nothing); one that did consume - // batches has flushed them to the persisters, so it is reported - // like any other pass and the host applies the deltas. + // A pass whose scan was cut short is not a completed pass: no + // timestamp, no completion event (same as losing the slot CAS). + // The coordinator returns the empty summary for exactly that case; + // a pass stopped only after its scan completed has flushed the + // scan to the persisters, so it is reported like any other pass + // and the host applies the deltas. if pass_cancel.is_cancelled() && summary.is_empty() { return summary; } @@ -533,9 +534,9 @@ impl ShieldedSyncManager { /// Returns `Ok(None)` if `wallet_id` isn't registered on the /// coordinator (e.g. shielded support hasn't been configured, /// or the wallet has never called `bind_shielded`), if - /// another sync pass was already in flight, or if the pass was - /// cancelled by [`stop`](Self::stop) before it consumed any - /// scan data (the coordinator then reports no wallets). + /// another sync pass was already in flight, or if the pass's scan + /// was cut short by [`stop`](Self::stop) (the coordinator then + /// reports no wallets). pub async fn sync_wallet( &self, wallet_id: &WalletId, diff --git a/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs index 1ef50639d27..e81c06b7bfa 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/cancelled_scan_tests.rs @@ -1,18 +1,18 @@ -//! A cancelled scan pass must not certify what it did not cover -//! (dashpay/platform#5200 review). +//! A cancelled scan pass publishes nothing (dashpay/platform#5200 review). //! //! Drives the real coordinator against a mock Platform whose pool spans //! two fetch chunks, each holding one owned note, and cancels the pass //! from the tree-progress callback after the first chunk lands. The -//! consumed prefix is flushed, but it is a resume point, not a complete -//! scan: no activity entry is derived from it and the ledger stays -//! unavailable. The resumed pass then yields exactly what an -//! uninterrupted scan yields — one entry carrying both notes. +//! appended commitments stay in the local tree; no note, spent mark, +//! watermark, activity entry or completion leaves the scan, and the +//! ledger stays unavailable. The next pass re-walks from the completed +//! watermark and yields exactly what an uninterrupted scan yields — one +//! entry carrying both notes. //! -//! The host side is checked through a recording persister: the consumed -//! receipt reaches the host, the partial cursor does not, and a restart -//! restored from what the host recorded carries no scan coverage until -//! a pass completes. +//! The host side is checked through a recording persister: nothing +//! reaches the host on a cancelled pass, and a restart restored from +//! what the host recorded carries no scan coverage until a pass +//! completes. use std::collections::BTreeMap; use std::path::PathBuf; @@ -230,7 +230,7 @@ async fn two_chunk_pool( /// A pool of full chunks, each opening with the given notes and padded /// with opaque ones, except the last chunk, which holds its notes alone -/// and so ends the pool. +/// and so ends the pool. Returns the SDK and the chunk size. async fn chunked_pool(leading: &[Vec]) -> (Arc, u64) { let mut sdk = Sdk::new_mock(); let count = 2048 @@ -302,6 +302,34 @@ async fn coordinator_over( (coordinator, host) } +/// Cancel once the `after`-th chunk has been committed to the tree. +fn cancel_after(cancel: &CancellationToken, after: u64) -> Arc { + let stop = cancel.clone(); + let committed = Arc::new(AtomicU64::new(0)); + Arc::new(move |_, _| { + if committed.fetch_add(1, Ordering::SeqCst) + 1 >= after { + stop.cancel(); + } + }) +} + +/// Run one pass that is stopped after `after` chunks, bounded in time. +async fn cancelled_pass( + coordinator: &NetworkShieldedCoordinator, + after: u64, +) -> crate::manager::shielded_sync::ShieldedSyncPassSummary { + let cancel = CancellationToken::new(); + coordinator.install_tree_progress_handler(Some(cancel_after(&cancel, after))); + let summary = tokio::time::timeout( + std::time::Duration::from_secs(60), + coordinator.sync(true, &cancel), + ) + .await + .expect("a cancelled pass returns"); + coordinator.install_tree_progress_handler(None); + summary +} + fn host_synced_index(host: &RecordingPersistence, sub: SubwalletId) -> Option { host.recorded() .iter() @@ -335,7 +363,7 @@ async fn activity(coordinator: &NetworkShieldedCoordinator) -> Vec panic!("expected the hydrated ledger, got {other:?}"), } - assert_eq!( - coordinator - .store() - .read() - .await - .last_synced_note_index(sub) - .expect("cursor"), - 0, - "the restart resumes from the last completed watermark" - ); + { + let store = coordinator.store().read().await; + assert_eq!( + store.tree_size().expect("tree"), + count, + "the reopened tree keeps the appended prefix" + ); + assert_eq!(store.last_synced_note_index(sub).expect("cursor"), 0); + } // The first pass after the restart re-walks the prefix (gate-skipped // by the persisted tree) and reaches the receipt. @@ -550,13 +553,14 @@ async fn should_not_restore_scan_coverage_from_a_cancelled_prefix() { } /// The receipt is in chunk 0 and the action that spends it in chunk 1. -/// Session 1 is cancelled after chunk 1, so the host holds the receipt -/// marked spent and no watermark. Session 2 restores it, re-walks from -/// the completed watermark and is cancelled after chunk 0 — before the -/// nullifier replay could mark it spent again. The replayed receipt -/// must not come back unspent, in the store or towards the host. +/// Session 1 is cancelled after chunk 1 and session 2, after a restart, +/// after chunk 0 — the one order in which a published prefix would have +/// shown the receipt as spendable (receipt in, spend not yet replayed). +/// Neither stop publishes anything, so the receipt never appears +/// unspent in the ledger, in a balance or at the host; the pass that +/// reaches the tip records it spent from the start. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn should_keep_a_replayed_receipt_spent_across_a_cancelled_restart() { +async fn should_never_show_a_spent_receipt_as_spendable_across_cancelled_restarts() { let keys = viewing_keys(); let sub = SubwalletId::new(WALLET, 0); let (receipt, own_nullifier) = owned_note_with_nullifier(&keys, FIRST_VALUE); @@ -566,44 +570,20 @@ async fn should_keep_a_replayed_receipt_spent_across_a_cancelled_restart() { let layout = [vec![receipt], vec![spend], vec![tail]]; let path = tree_path(); - /// Cancel once the `after`-th chunk has been committed. - fn cancel_after(cancel: &CancellationToken, after: u64) -> Arc { - let stop = cancel.clone(); - let committed = Arc::new(std::sync::atomic::AtomicU64::new(0)); - Arc::new(move |_, _| { - if committed.fetch_add(1, Ordering::SeqCst) + 1 >= after { - stop.cancel(); - } - }) - } - // Session 1: receipt and its spend consumed, then stopped. let host = Arc::new(RecordingPersistence::default()); { - let (sdk, _) = chunked_pool(&layout).await; + let (sdk, count) = chunked_pool(&layout).await; let coordinator = coordinator_at(sdk, &path, &keys, Arc::clone(&host)).await; - let cancel = CancellationToken::new(); - coordinator.install_tree_progress_handler(Some(cancel_after(&cancel, 2))); - tokio::time::timeout( - std::time::Duration::from_secs(60), - coordinator.sync(true, &cancel), - ) - .await - .expect("a cancelled pass returns"); + cancelled_pass(&coordinator, 2).await; let store = coordinator.store().read().await; - let notes = store.get_all_notes(sub).expect("notes"); - assert_eq!(notes.len(), 1); - assert!(notes[0].is_spent, "the spend in chunk 1 was replayed"); + assert_eq!(store.tree_size().expect("tree"), 2 * count); + assert!(store.get_all_notes(sub).expect("notes").is_empty()); assert_eq!(store.spendable_balance(sub).expect("balance"), 0); } - assert_eq!(host_note_values(&host, sub), vec![FIRST_VALUE]); - assert!(host.recorded().iter().any(|changeset| changeset - .nullifiers_spent - .get(&sub) - .is_some_and(|nfs| nfs.contains(&own_nullifier)))); - assert_eq!(host_synced_index(&host, sub), None); + assert!(host.recorded().is_empty()); - // Session 2: restore the spent receipt, re-walk chunk 0, stop again. + // Session 2: restart, re-walk chunk 0 only, stop again. let (sdk, count) = chunked_pool(&layout).await; let coordinator = coordinator_at(sdk, &path, &keys, Arc::clone(&host)).await; coordinator @@ -611,41 +591,31 @@ async fn should_keep_a_replayed_receipt_spent_across_a_cancelled_restart() { .await .expect("restore"); coordinator.mark_hydrated(WALLET, true).await; + cancelled_pass(&coordinator, 1).await; { let store = coordinator.store().read().await; - let notes = store.get_all_notes(sub).expect("notes"); - assert_eq!(notes.len(), 1); - assert!(notes[0].is_spent, "restore installs the receipt as spent"); - } - let recorded_before = host.recorded().len(); - let cancel = CancellationToken::new(); - coordinator.install_tree_progress_handler(Some(cancel_after(&cancel, 1))); - tokio::time::timeout( - std::time::Duration::from_secs(60), - coordinator.sync(true, &cancel), - ) - .await - .expect("a cancelled pass returns"); - coordinator.install_tree_progress_handler(None); - { - let store = coordinator.store().read().await; - let notes = store.get_all_notes(sub).expect("notes"); - assert_eq!(notes.len(), 1); assert!( - notes[0].is_spent, - "a replayed receipt must keep its spent state" + store.get_all_notes(sub).expect("notes").is_empty(), + "a receipt whose spend was not replayed must not enter the ledger" ); assert_eq!(store.spendable_balance(sub).expect("balance"), 0); - assert_eq!(store.last_synced_note_index(sub).expect("cursor"), count); + assert_eq!(store.last_synced_note_index(sub).expect("cursor"), 0); + assert_eq!(store.tree_size().expect("tree"), 2 * count); + } + assert!(host.recorded().is_empty(), "nothing reaches the host"); + match coordinator + .local_balance_snapshot(WALLET) + .await + .expect("snapshot") + { + ShieldedLocalBalanceState::Ready(snapshot) => { + assert_eq!(snapshot.accounts[&0].spendable_credits, 0); + } + other => panic!("expected the hydrated ledger, got {other:?}"), } - assert!( - host.recorded()[recorded_before..] - .iter() - .all(|changeset| !changeset.notes_saved.contains_key(&sub)), - "a replayed receipt is not re-emitted to the host" - ); - // The pass that reaches the tip keeps it spent and finds the tail. + // The pass that reaches the tip records the receipt spent from the + // start and finds the tail. coordinator.sync(true, &CancellationToken::new()).await; let store = coordinator.store().read().await; let mut values: Vec<(u64, bool)> = store @@ -658,5 +628,13 @@ async fn should_keep_a_replayed_receipt_spent_across_a_cancelled_restart() { assert_eq!(values, vec![(FIRST_VALUE, true), (SECOND_VALUE, false)]); assert_eq!(store.spendable_balance(sub).expect("balance"), SECOND_VALUE); drop(store); + assert_eq!( + host_note_values(&host, sub), + vec![FIRST_VALUE, SECOND_VALUE] + ); + assert!(host.recorded().iter().any(|changeset| changeset + .nullifiers_spent + .get(&sub) + .is_some_and(|nfs| nfs.contains(&own_nullifier)))); let _ = std::fs::remove_file(&path); } diff --git a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs index 02910a730f5..5b929b5d2fa 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs @@ -1314,14 +1314,15 @@ impl NetworkShieldedCoordinator { /// [`WalletPersister`]. /// /// Cancellation: `cancel` is checked before every network step and - /// between note batches. Once it fires, the pass skips what is left - /// of the network work (the rest of the scan, spend re-drives and - /// stranded-reservation releases) but still finalizes and flushes to - /// the host persisters whatever the scan had consumed: a note saved - /// in the store under an advanced watermark is never re-emitted, so - /// an unflushed changeset would leave the host mirror permanently - /// behind. The caught-up cooldown is not stamped for a cancelled - /// pass, so the next pass walks Platform again right away. + /// between note batches. A scan cut short keeps the commitments it + /// appended to the local tree and publishes nothing else (see + /// [`sync_notes_across`]): the pass then returns the empty summary, + /// so the manager reports no completion and the host sees no + /// partial ledger. A stop that lands after the scan completed skips + /// the remaining network work (spend re-drives, stranded releases, + /// identity-shield re-drives) but still runs the store-only steps + /// and flushes the completed scan to the host persisters. The + /// caught-up cooldown is never stamped by a cancelled pass. /// /// [`sync_notes_across`]: super::sync::sync_notes_across /// [`PlatformEventManager::on_shielded_sync_completed`]: @@ -1421,25 +1422,24 @@ impl NetworkShieldedCoordinator { Err(e) => return self.fail_all_wallets(&subwallets, &e), }; // Everything below that talks to the network is skipped once the - // pass is cancelled; the store-only steps and the host flush of - // what the scan consumed still run (see the method docs). + // pass is cancelled; the store-only steps and the host flush of a + // COMPLETED scan still run (see the method docs). // - // Snapshotted here on purpose: the scan is the step that was or - // was not cut short, and the cooldown decision below is about - // whether that scan reached the tip. A stop that lands during the - // store-only steps after a complete scan does not change that. + // Snapshotted here on purpose: the cooldown decision below is + // about whether the scan reached the tip. A stop that lands during + // the store-only steps after a complete scan does not change that. let cancelled = cancel.is_cancelled(); - // A pass cut short before the scan consumed anything (the stop - // landed in the prefetch, or before the store lock) has nothing to - // flush and nothing to report: hand back the empty summary so the - // manager treats it as "no pass ran" — `build_per_wallet_summary` + // A scan cut short publishes nothing (`sync_notes_across` returns + // the empty result whether the stop landed in the prefetch, before + // the store lock or between batches): hand back the empty summary + // so the manager treats it as "no pass ran" — `build_per_wallet_summary` // would otherwise mint an all-zero `Ok` for every registered // wallet, and the host would see a completion event for no work. if cancelled && notes.total_scanned == 0 && crate::changeset::merge::Merge::is_empty(¬es.changeset) { - tracing::debug!("Coordinator sync cancelled before the scan consumed anything"); + tracing::debug!("Coordinator sync cancelled; the scan published nothing"); return summary; } // Scan-based spend detection now happens INSIDE @@ -1510,27 +1510,21 @@ impl NetworkShieldedCoordinator { // Restore-path activity derivation: reconstruct per-operation // activity entries best-effort from the notes / outgoing notes - // this pass (and earlier passes) persisted. Runs every completed - // pass so it doubles as the one-time backfill over an - // already-populated store — `derive_activity_from_scan_data` - // skips clusters whose id a live entry already owns, and - // `save_activity` upserts by id, so re-running is idempotent. - // Failures are logged and swallowed: a derivation miss must never - // fail a sync pass. + // this pass (and earlier passes) persisted. Runs every pass so it + // doubles as the one-time backfill over an already-populated + // store — `derive_activity_from_scan_data` skips clusters whose + // id a live entry already owns, and `save_activity` upserts by + // id, so re-running is idempotent. Failures are logged and + // swallowed: a derivation miss must never fail a sync pass. // - // Deferred on a cancelled pass. Derived entries are immutable - // clusters keyed by the discovering chunk's proof height: an - // entry derived from a consumed prefix would freeze the cluster, - // and a note the resumed pass finds at the same height could only - // confirm that entry, never extend it — the amount and cmx set - // would permanently omit the later note. The raw notes, spent - // marks and cursor are flushed regardless; the next pass that - // reaches the tip derives over all of them at once, exactly as an - // uninterrupted scan would have. + // Only a completed scan reaches this point (a cut-short one + // returned above), which matters here: derived entries are + // immutable clusters keyed by the discovering chunk's proof + // height, so deriving over a consumed prefix would freeze a + // cluster that a later note at the same height could only + // confirm, never extend. let mut notes = notes; - if cancelled { - tracing::debug!("Shielded activity derivation deferred: pass cancelled"); - } else if let Err(e) = self + if let Err(e) = self .derive_activity_into_changeset(&subwallets, &mut notes.changeset) .await { @@ -1593,12 +1587,10 @@ impl NetworkShieldedCoordinator { // Cooldown decision based on aggregate activity across // every subwallet — any new commitment scanned or // newly-spent nullifier anywhere on the network clears - // the stamp so the next pass runs immediately — a cancelled pass - // included: what it consumed is real activity, and leaving a - // stamp from before it would park the next background pass - // behind the cooldown. Only a NEW stamp needs an uncancelled - // pass: a cancelled one proves nothing about being caught up (it - // may have stopped before its first batch). + // the stamp so the next pass runs immediately. Only a NEW stamp + // needs an uncancelled pass: a stop after a complete scan that + // found nothing proves nothing about staying caught up, so the + // next pass walks Platform again right away. let any_activity = notes.total_scanned > 0 || newly_spent_per_sub.values().any(|&n| n > 0); if let Ok(mut guard) = self.last_caught_up_at.lock() { if any_activity { diff --git a/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs b/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs index 123136afefa..0970e75d48a 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/file_store.rs @@ -877,11 +877,6 @@ impl ShieldedStore for FileBackedShieldedStore { Ok(()) } - fn set_resume_note_index(&mut self, id: SubwalletId, index: u64) -> Result<(), Self::Error> { - self.subwallets.entry(id).or_default().last_synced_index = index; - Ok(()) - } - fn purge_wallet(&mut self, wallet_id: WalletId) -> Result<(), Self::Error> { // The redrive table IS durable (unlike the rest of subwallet // state), so purging the in-memory map alone would leave this diff --git a/packages/rs-platform-wallet/src/wallet/shielded/store.rs b/packages/rs-platform-wallet/src/wallet/shielded/store.rs index 62476079df6..8efd0acfdcb 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/store.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/store.rs @@ -469,24 +469,16 @@ pub trait ShieldedStore: Send + Sync { /// (scanning through position `N-1` sets this to `N`). fn last_synced_note_index(&self, id: SubwalletId) -> Result; - /// Persist the sync watermark (next index to scan) for `id` as the - /// end of a COMPLETE scan: also records it as the scan coverage - /// behind [`local_account_balance`](Self::local_account_balance) - /// and stamps the balance source `ScannedThisSession`. + /// Persist the sync watermark (next index to scan) for `id`. Only a + /// pass that reached the tip calls this: the setter also records the + /// scan coverage and the `ScannedThisSession` balance source behind + /// [`local_account_balance`](Self::local_account_balance). fn set_last_synced_note_index( &mut self, id: SubwalletId, index: u64, ) -> Result<(), Self::Error>; - /// Persist the sync watermark (next index to scan) for `id` as a - /// RESUME cursor only: the pass that reached `index` was cut short, - /// so the next pass resumes from here, but the scan coverage and - /// balance source behind - /// [`local_account_balance`](Self::local_account_balance) stay as - /// they were — a consumed prefix is not evidence of a complete scan. - fn set_resume_note_index(&mut self, id: SubwalletId, index: u64) -> Result<(), Self::Error>; - // ── Per-subwallet lifecycle ──────────────────────────────────────── /// Drop ALL in-memory per-subwallet state (decrypted notes, @@ -1122,11 +1114,6 @@ impl ShieldedStore for InMemoryShieldedStore { Ok(()) } - fn set_resume_note_index(&mut self, id: SubwalletId, index: u64) -> Result<(), Self::Error> { - self.subwallets.entry(id).or_default().last_synced_index = index; - Ok(()) - } - fn purge_wallet(&mut self, wallet_id: WalletId) -> Result<(), Self::Error> { self.subwallets.retain(|id, _| id.wallet_id != wallet_id); Ok(()) diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index c533f680ac4..e137f50c2f6 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -26,7 +26,7 @@ //! [`NetworkShieldedCoordinator::sync`]: //! super::coordinator::NetworkShieldedCoordinator::sync -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::sync::Arc; use dash_sdk::platform::shielded::notes_sync::types::ShieldedChunkBatch; @@ -198,11 +198,16 @@ impl MultiSyncNotesResult { /// /// Cancellation: `cancel` is observed before the store is locked and /// between batches. A cancelled pass stops pulling batches (dropping -/// the in-flight network fetches) and then finalizes the prefix it -/// consumed exactly as a completed pass would — checkpoint, note saves, -/// spend replay, watermark — so the store and the returned changeset -/// stay consistent with each other. It is a clean early stop, not an -/// error. +/// the in-flight network fetches), checkpoints the commitments it +/// appended to the local tree, and publishes NOTHING else: no notes, no +/// spent marks, no watermark, an empty result. A consumed prefix is not +/// a complete scan — a receipt in it may already be spent by an action +/// in a chunk it never reached, and the ledger has no way to hold such a +/// note without offering it for selection. The next pass re-walks from +/// the last completed watermark; the appended leaves make that re-walk +/// skip every append and checkpoint it already made, so what a stop +/// costs is the download and decryption of the chunks it had consumed. +/// It is a clean early stop, not an error. pub(super) async fn sync_notes_across( sdk: &Arc, store: &Arc>, @@ -464,15 +469,15 @@ where loop { // Batches arrive in tree order as a contiguous prefix, so - // stopping between two of them leaves a shorter but valid scan: - // the finalize checkpoints and watermarks exactly what was - // consumed. `biased` so a pending stop wins over a ready batch. + // stopping between two of them leaves the tree one checkpoint + // further along and nothing else to publish. `biased` so a + // pending stop wins over a ready batch. let item = tokio::select! { biased; _ = cancel.cancelled() => { debug!( total_scanned = total_notes_scanned, - "Shielded note sync cancelled; finalizing the consumed prefix" + "Shielded note sync cancelled; committing tree progress only" ); break true; } @@ -657,6 +662,20 @@ where .map_err(|e| PlatformWalletError::ShieldedTreeUpdateFailed(e.to_string()))?; } + // A cancelled pass ends here: the appended leaves are checkpointed + // (the next pass skips them), and nothing the prefix decrypted is + // saved, replayed, watermarked or handed to the caller. See the + // cancellation note on `sync_notes_across` for why a consumed prefix + // must not reach the ledger. + if cancelled { + drop(store); + info!( + total_scanned = total_notes_scanned, + appended, "Shielded note sync stopped; tree progress kept, nothing published" + ); + return Ok(MultiSyncNotesResult::default()); + } + // Save decrypted notes per subwallet; count new notes per // subwallet; build a single consolidated changeset. let mut per_subwallet_new_notes: BTreeMap = BTreeMap::new(); @@ -672,38 +691,16 @@ where // Gate on THIS subwallet's own watermark (not the network // min): a caught-up subwallet skips re-deriving nullifiers // for notes it already stored, while a lagging one still - // saves everything from its own start. + // saves everything from its own start. `save_note` is an + // idempotent overwrite-by-nullifier, so a stray re-save is + // harmless — but gating per-subwallet keeps the + // `per_subwallet_new_notes` count honest. let sub_watermark = watermarks.get(id).copied().unwrap_or(0); - // Notes the store already holds, by nullifier. A position above - // the watermark can still be a known note: a restart after a - // cancelled pass resumes from the last COMPLETED watermark (the - // cancelled cursor is not persisted) and re-walks a prefix whose - // receipts the host restored, possibly already marked spent by - // that same pass. `save_note` replaces by nullifier, so re-saving - // one as a fresh `is_spent: false` receipt would resurrect a - // spent note until the replay reaches its nullifier — which a - // second stop can prevent. A known note is left alone and not - // re-emitted: the host has it. - let known_nullifiers: BTreeSet<[u8; 32]> = store - .get_all_notes(*id) - .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))? - .into_iter() - .map(|note| note.nullifier) - .collect(); for d in discovered { if d.position < sub_watermark { continue; } let nullifier = d.note.nullifier(&views.full_viewing_key); - if known_nullifiers.contains(&nullifier.to_bytes()) { - debug!( - wallet_id = %hex::encode(id.wallet_id), - account = id.account_index, - position = d.position, - "Note already on file; replayed receipt left as is" - ); - continue; - } let value = d.note.value().inner(); debug!( wallet_id = %hex::encode(id.wallet_id), @@ -794,40 +791,17 @@ where // `result.total_notes_scanned`. // // Never rewind. `aligned_start` is the MIN watermark across the - // union, so a pass cut short after a few batches can end below a - // subwallet that was already further along; a completed pass - // always reaches the tip, which is at or past every watermark. - // A cancelled pass leaves such a subwallet's row alone entirely: - // its consumed prefix never covered that subwallet. Where it did - // advance a subwallet, only the in-memory RESUME cursor moves: the - // prefix is not a complete scan, so the scan coverage and balance - // source the completed-pass setter stamps (`ScannedThisSession`, - // which lets `local_balance_snapshot` report an unhydrated account - // as ready) stay untouched until a pass runs to the tip. - // - // The cursor is NOT sent to the host. The changeset's watermark is - // the one the host restores as completed-scan coverage (`Restored` - // provenance with `last_scanned_index` set), and the persistence - // contract has no separate resume field; emitting a partial cursor - // there would make a restart report an account as covered up to a - // position no pass ever completed. So a restart resumes from the - // last COMPLETED watermark and re-walks the prefix: cheap, since - // the appended leaves are gate-skipped by the tree size and the - // receipts the host restored are recognised by nullifier and left - // alone (spent state included). The consumed notes and spent marks - // still reach the host below, so none of them is lost. + // union, so a pass that re-walks a lagging subwallet's range can + // end below a subwallet that was already further along; a completed + // pass reaches the tip, which is at or past every watermark. Only a + // completed pass gets here (a cancelled one returned above), so the + // write is unconditional: the setter also stamps the scan coverage + // and the `ScannedThisSession` balance source, which is exactly what + // reaching the tip means. let new_index = aligned_start + total_notes_scanned; for (id, _) in subwallets { let prior = watermarks.get(id).copied().unwrap_or(0); let advanced = prior.max(new_index); - if cancelled { - if advanced > prior { - store - .set_resume_note_index(*id, advanced) - .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; - } - continue; - } store .set_last_synced_note_index(*id, advanced) .map_err(|e| PlatformWalletError::ShieldedStoreError(e.to_string()))?; diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs index a50b041a40b..7aca62ea957 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs @@ -1,10 +1,11 @@ //! Cancellation of a note-sync pass (dashpay/platform#5200). //! //! Feeds `sync_notes_from_stream` a synthetic batch stream so the -//! cancel and finalize paths run without a network. The stream yields -//! real-shaped batches, then a tail future that cancels the token and -//! pends forever — exactly the moment a stop would land between two -//! chunk fetches. +//! cancel path runs without a network. The stream yields real-shaped +//! batches, then a tail future that cancels the token and pends forever +//! — exactly the moment a stop would land between two chunk fetches. +//! A cut-short pass keeps the tree progress it made and publishes +//! nothing else; the next pass re-walks from the completed watermark. use std::sync::Arc; @@ -99,7 +100,7 @@ async fn run( } #[tokio::test] -async fn should_finalize_consumed_prefix_when_cancelled_mid_stream() { +async fn should_keep_tree_progress_and_publish_nothing_when_cancelled_mid_stream() { let path = temp_tree_path("prefix"); let store = Arc::new(RwLock::new( FileBackedShieldedStore::open_path(&path, 100).expect("open store"), @@ -116,21 +117,20 @@ async fn should_finalize_consumed_prefix_when_cancelled_mid_stream() { ) .await; - // The prefix the pass consumed is finalized like a completed pass. - assert_eq!(result.total_scanned, 8); + // The appended leaves stay, checkpointed; nothing else is published. + assert_eq!(result.total_scanned, 0, "a cut-short pass reports no scan"); + assert!(crate::changeset::merge::Merge::is_empty(&result.changeset)); let store = store.read().await; assert_eq!(store.tree_size().unwrap(), 8, "both batches appended"); - assert_eq!( - store.last_synced_note_index(sub).unwrap(), - 8, - "watermark covers exactly the consumed batches" - ); assert!( store.witness(7).unwrap().is_some(), "a checkpoint covers the appended leaves" ); - // A resume cursor, not a completed scan: the ledger's coverage and - // provenance are untouched, so an unhydrated account stays unavailable. + assert_eq!( + store.last_synced_note_index(sub).unwrap(), + 0, + "the watermark only moves when a pass reaches the tip" + ); let balance = store.local_account_balance(sub).unwrap(); assert_eq!(balance.last_scanned_index, None); assert_eq!(balance.source, ShieldedBalanceSource::NoHistory); @@ -171,7 +171,7 @@ async fn should_return_empty_without_touching_store_when_cancelled_before_scan() } #[tokio::test] -async fn should_resume_from_partial_watermark_on_next_pass() { +async fn should_rewalk_the_appended_prefix_without_reappending_on_next_pass() { let path = temp_tree_path("resume"); let store = Arc::new(RwLock::new( FileBackedShieldedStore::open_path(&path, 100).expect("open store"), @@ -188,11 +188,12 @@ async fn should_resume_from_partial_watermark_on_next_pass() { batches_then_cancel(vec![batch(0, 4, 12), batch(4, 4, 12)], cancel.clone()), ) .await; - assert_eq!(store.read().await.last_synced_note_index(sub).unwrap(), 8); + assert_eq!(store.read().await.last_synced_note_index(sub).unwrap(), 0); + assert_eq!(store.read().await.tree_size().unwrap(), 8); - // Pass 2: the SDK re-fetches from the chunk-aligned start (0 here), so - // the stream re-delivers the first two batches — gate-skipped by the - // tree-size snapshot — and then the new one. Runs to completion. + // Pass 2: re-walks from the completed watermark (0), so the stream + // re-delivers the first two batches — gate-skipped by the tree-size + // snapshot — and then the new one. Runs to completion. let cancel = CancellationToken::new(); let result = run( &store, @@ -207,8 +208,8 @@ async fn should_resume_from_partial_watermark_on_next_pass() { .await; assert_eq!( - result.total_scanned, 4, - "only the new notes count as scan volume" + result.total_scanned, 12, + "the whole range from the completed watermark is scanned" ); let store = store.read().await; assert_eq!( @@ -235,10 +236,10 @@ async fn should_resume_from_partial_watermark_on_next_pass() { /// Two subwallets, one already past where the cancelled prefix ends. /// `aligned_start` is the union MIN, so the pass walks 0..8 while the -/// second subwallet sits at 12: its watermark must not move backwards, -/// and the changeset must not carry a rewound value to the host. +/// second subwallet sits at 12: neither watermark moves, and the +/// changeset carries nothing to the host. #[tokio::test] -async fn should_not_rewind_a_more_advanced_subwallet_when_cancelled() { +async fn should_leave_every_watermark_alone_when_cancelled() { let path = temp_tree_path("no_rewind"); let store = Arc::new(RwLock::new( FileBackedShieldedStore::open_path(&path, 100).expect("open store"), @@ -261,25 +262,23 @@ async fn should_not_rewind_a_more_advanced_subwallet_when_cancelled() { .await; let store = store.read().await; - assert_eq!(store.last_synced_note_index(lagging).unwrap(), 8); + assert_eq!(store.tree_size().unwrap(), 8); + assert_eq!(store.last_synced_note_index(lagging).unwrap(), 0); assert_eq!( store.last_synced_note_index(advanced).unwrap(), 12, "a partial prefix below a subwallet's watermark must not rewind it" ); assert!( - result.changeset.synced_indices.is_empty(), - "a cancelled pass sends no watermark to the host: neither the partial \ - cursor (the host would restore it as completed coverage) nor a \ - rewound one" + crate::changeset::merge::Merge::is_empty(&result.changeset), + "a cancelled pass sends nothing to the host" ); let _ = std::fs::remove_file(&path); } /// Cancelled after the store lock but before the first batch: nothing -/// was consumed, so the store and the changeset stay untouched and the -/// coordinator's empty-pass gate can fire. +/// was consumed, so the store and the changeset stay untouched. #[tokio::test] async fn should_leave_store_and_changeset_empty_when_cancelled_before_first_batch() { let path = temp_tree_path("first_batch"); From 6dc6815736f0aff888857936c7e05461c893b54e Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Tue, 3 Nov 2026 19:00:27 +0100 Subject: [PATCH 11/12] fix(platform-wallet): let a shielded scan observe cancellation while waiting for the store lock A pass queued behind another store writer has made no mutation yet, so a stop that lands during the lock wait returns at once instead of after the unrelated holder is done. Regression: hold the store write guard, poll a scan into the wait, cancel it, and check it returns before the guard is released with nothing appended. Co-Authored-By: Claude Fable 5.1 --- .../src/wallet/shielded/sync.rs | 14 ++++++- .../src/wallet/shielded/sync/cancel_tests.rs | 38 +++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index e137f50c2f6..41cb3020dae 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -306,7 +306,19 @@ where // is no lock-ordering deadlock. Backpressure is pull-based — a slow // append simply polls the stream less often, capping in-flight // network fetches at `max_concurrent`. - let mut store = store.write().await; + // + // The wait for the guard observes the token too: a pass queued behind + // another writer (a spend flow, a concurrent scan in tests) has made + // no mutation yet, so a stop that lands here returns at once instead + // of after the unrelated holder is done. + let mut store = tokio::select! { + biased; + _ = cancel.cancelled() => { + debug!("Shielded note sync cancelled while waiting for the store lock"); + return Ok(MultiSyncNotesResult::default()); + } + guard = store.write() => guard, + }; // Snapshot each subwallet's watermark and the shared tree's // current leaf count. Two distinct quantities drive the rest diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs index 7aca62ea957..a683da2d53c 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync/cancel_tests.rs @@ -313,3 +313,41 @@ async fn should_leave_store_and_changeset_empty_when_cancelled_before_first_batc let _ = std::fs::remove_file(&path); } + +/// Cancelled while another writer holds the store: the pass has touched +/// nothing yet, so it must return without waiting for that writer. +#[tokio::test] +async fn should_return_while_waiting_for_the_store_lock_when_cancelled() { + let path = temp_tree_path("lock_wait"); + let store = Arc::new(RwLock::new( + FileBackedShieldedStore::open_path(&path, 100).expect("open store"), + )); + let subwallets = one_subwallet(); + let sub = subwallets[0].0; + let cancel = CancellationToken::new(); + + let held = store.write().await; + let pass = sync_notes_from_stream(&store, &subwallets, None, &cancel, |_, _| { + futures::stream::iter(vec![batch(0, 4, 4)].into_iter().map(Ok)) + }); + futures::pin_mut!(pass); + assert!( + futures::poll!(pass.as_mut()).is_pending(), + "the pass is queued behind the held guard" + ); + + cancel.cancel(); + let result = tokio::time::timeout(std::time::Duration::from_secs(2), pass.as_mut()) + .await + .expect("a cancelled pass returns without the guard") + .expect("pass must succeed"); + assert_eq!(result.total_scanned, 0); + assert!(crate::changeset::merge::Merge::is_empty(&result.changeset)); + + drop(held); + let store = store.read().await; + assert_eq!(store.tree_size().unwrap(), 0, "nothing was appended"); + assert_eq!(store.last_synced_note_index(sub).unwrap(), 0); + + let _ = std::fs::remove_file(&path); +} From 680d1619f4f8fdb3010ad69fe501863c7ceecf7e Mon Sep 17 00:00:00 2001 From: Bartosz Rozwarski Date: Wed, 4 Nov 2026 14:04:35 +0100 Subject: [PATCH 12/12] fix(platform-wallet): clear the caught-up stamp when a shielded scan is cut short A scan stopped mid-stream may have appended leaves the last completed pass never saw, so leaving its stamp in place let a `sync(false)` within the cooldown announce a "caught up" pass over a chain that had moved. The cancelled early return now drops the stamp. Also from the same review: the three read-only waits that observed the token through a hand-written `biased` `select!` use `run_until_cancelled`; the `Merge::is_empty` and `CancellationToken` paths move to imports; the `cancel_tests` module doc and the stranded-release comment say what the code does; the FFI `sync_wallet` doc and the Swift stop docs describe the cancellable pass. Co-Authored-By: Claude Fable 5.1 --- .../src/shielded_sync.rs | 5 +++- .../src/wallet/shielded/balance_tests.rs | 13 +++------ .../src/wallet/shielded/coordinator.rs | 29 +++++++++++-------- .../src/wallet/shielded/operations.rs | 16 +++++----- .../src/wallet/shielded/sync.rs | 15 ++++------ .../PlatformWalletManagerShieldedSync.swift | 26 ++++++++++------- 6 files changed, 56 insertions(+), 48 deletions(-) diff --git a/packages/rs-platform-wallet-ffi/src/shielded_sync.rs b/packages/rs-platform-wallet-ffi/src/shielded_sync.rs index 95f4b94f037..1371dfaabd1 100644 --- a/packages/rs-platform-wallet-ffi/src/shielded_sync.rs +++ b/packages/rs-platform-wallet-ffi/src/shielded_sync.rs @@ -697,7 +697,10 @@ pub unsafe extern "C" fn platform_wallet_manager_shielded_default_address( /// you want to avoid concurrent passes. Returns an error if the /// wallet doesn't exist or the sync itself fails. Wallets with no /// bound shielded sub-wallet succeed silently with no observable -/// state change. +/// state change, and so does a pass whose scan a concurrent +/// [`platform_wallet_manager_shielded_sync_stop`] cut short: it +/// published nothing, and the next pass re-walks from the last +/// completed watermark. /// /// # Safety /// - `wallet_id_bytes` must point at 32 readable bytes. diff --git a/packages/rs-platform-wallet/src/wallet/shielded/balance_tests.rs b/packages/rs-platform-wallet/src/wallet/shielded/balance_tests.rs index b39ec66673e..03adbe56175 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/balance_tests.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/balance_tests.rs @@ -15,6 +15,7 @@ use crate::changeset::{ShieldedSubwalletStartState, ShieldedSyncStartState}; use crate::error::PlatformWalletError; use crate::wallet::persister::{NoPlatformPersistence, WalletPersister}; use drive_proof_verifier::types::{ShieldedEncryptedNotes, ShieldedEncryptedNotesQuery}; +use tokio_util::sync::CancellationToken; fn tree_path() -> PathBuf { // Clock resolution alone does not distinguish parallel test invocations. @@ -222,9 +223,7 @@ async fn local_balance_successful_empty_scan_marks_known_zero() { ready(&coordinator, wallet).await.accounts[&0].source, ShieldedBalanceSource::NoHistory ); - let outcome = coordinator - .sync(true, &tokio_util::sync::CancellationToken::new()) - .await; + let outcome = coordinator.sync(true, &CancellationToken::new()).await; assert_eq!( outcome.success_count(), 1, @@ -251,9 +250,7 @@ async fn local_balance_successful_scan_covers_unhydrated_registration() { coordinator.local_balance_snapshot(wallet).await.unwrap(), ShieldedLocalBalanceState::RestoreIncomplete ); - let outcome = coordinator - .sync(true, &tokio_util::sync::CancellationToken::new()) - .await; + let outcome = coordinator.sync(true, &CancellationToken::new()).await; assert_eq!(outcome.success_count(), 1, "{outcome:?}"); let account = ready(&coordinator, wallet).await.accounts[&0]; assert_eq!(account.spendable_credits, 0); @@ -304,9 +301,7 @@ async fn local_balance_recovers_after_failed_clear_and_completed_scan() { coordinator.local_balance_snapshot(wallet).await.unwrap(), ShieldedLocalBalanceState::RestoreIncomplete ); - let outcome = coordinator - .sync(true, &tokio_util::sync::CancellationToken::new()) - .await; + let outcome = coordinator.sync(true, &CancellationToken::new()).await; assert_eq!(outcome.success_count(), 1, "{outcome:?}"); let account = ready(&coordinator, wallet).await.accounts[&0]; assert_eq!(account.spendable_credits, 900); diff --git a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs index 5b929b5d2fa..884f44d8dda 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/coordinator.rs @@ -57,6 +57,8 @@ use std::sync::Arc; use tokio_util::sync::CancellationToken; +use crate::changeset::merge::Merge; + /// Callback fired once per chunk during a coordinator sync pass — /// the **"downloaded"** progress signal. /// Arguments: `(cumulative_scanned, latest_block_height)`. Forwarded @@ -1392,11 +1394,10 @@ impl NetworkShieldedCoordinator { // The prefetch is a read-only RPC, so a stop that lands while it is // in flight simply drops it: the release pass is skipped on a // cancelled pass anyway, and the scan below bails before locking. - let stranded_release = tokio::select! { - biased; - _ = cancel.cancelled() => None, - prefetched = self.prefetch_stranded_release(&subwallets) => prefetched, - }; + let stranded_release = cancel + .run_until_cancelled(self.prefetch_stranded_release(&subwallets)) + .await + .flatten(); // ONE SDK call covers every registered IVK on the network. // Snapshot the optional progress handler installed by the @@ -1435,10 +1436,14 @@ impl NetworkShieldedCoordinator { // so the manager treats it as "no pass ran" — `build_per_wallet_summary` // would otherwise mint an all-zero `Ok` for every registered // wallet, and the host would see a completion event for no work. - if cancelled - && notes.total_scanned == 0 - && crate::changeset::merge::Merge::is_empty(¬es.changeset) - { + if cancelled && notes.total_scanned == 0 && Merge::is_empty(¬es.changeset) { + // The scan may have appended new leaves before the stop, so the + // chain has moved past whatever the last completed pass saw: + // drop the caught-up stamp so the next pass walks Platform at + // once instead of taking the cooldown skip. + if let Ok(mut guard) = self.last_caught_up_at.lock() { + *guard = None; + } tracing::debug!("Coordinator sync cancelled; the scan published nothing"); return summary; } @@ -1465,8 +1470,8 @@ impl NetworkShieldedCoordinator { // never execute). // // Runs before the balance read so freed notes are reflected in - // this pass's balances. Skipped on a cancelled pass: both halves - // broadcast, and a skipped release simply waits for the next pass. + // this pass's balances. Skipped on a cancelled pass: the re-drive + // broadcasts, and a skipped release simply waits for the next pass. if let Some((snapshot, recorded)) = stranded_release.filter(|_| !cancelled) { // Snapshot the per-wallet persisters BEFORE the loop and drop // the read guard: `redrive_pending_spends` performs network @@ -1559,7 +1564,7 @@ impl NetworkShieldedCoordinator { // WalletId directly — each per-wallet `WalletPersister.store` // only sees its own wallet's deltas. let consolidated = notes.changeset.clone(); - if !crate::changeset::merge::Merge::is_empty(&consolidated) { + if !Merge::is_empty(&consolidated) { let per_wallet = consolidated.split_by_wallet_id(); let persisters = self.persisters.read().await; for (wallet_id, cs) in per_wallet { diff --git a/packages/rs-platform-wallet/src/wallet/shielded/operations.rs b/packages/rs-platform-wallet/src/wallet/shielded/operations.rs index 419219f4235..8a8f8a4b39b 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/operations.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/operations.rs @@ -3158,13 +3158,15 @@ pub(super) async fn redrive_pending_identity_shields( }; // The nonce read is the one wait here a stop can cut short: it // is read-only, so dropping it mid-flight loses nothing. - let fetched = tokio::select! { - biased; - _ = cancel.cancelled() => { - debug!("identity redrive: pass cancelled during the nonce read"); - return; - } - fetched = IdentityNonceFetcher::fetch(sdk, identity_transition.identity_id()) => fetched, + let Some(fetched) = cancel + .run_until_cancelled(IdentityNonceFetcher::fetch( + sdk, + identity_transition.identity_id(), + )) + .await + else { + debug!("identity redrive: pass cancelled during the nonce read"); + return; }; if let Ok(fetched) = fetched { let result = { diff --git a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs index 41cb3020dae..7d4cb149951 100644 --- a/packages/rs-platform-wallet/src/wallet/shielded/sync.rs +++ b/packages/rs-platform-wallet/src/wallet/shielded/sync.rs @@ -311,13 +311,9 @@ where // another writer (a spend flow, a concurrent scan in tests) has made // no mutation yet, so a stop that lands here returns at once instead // of after the unrelated holder is done. - let mut store = tokio::select! { - biased; - _ = cancel.cancelled() => { - debug!("Shielded note sync cancelled while waiting for the store lock"); - return Ok(MultiSyncNotesResult::default()); - } - guard = store.write() => guard, + let Some(mut store) = cancel.run_until_cancelled(store.write()).await else { + debug!("Shielded note sync cancelled while waiting for the store lock"); + return Ok(MultiSyncNotesResult::default()); }; // Snapshot each subwallet's watermark and the shared tree's @@ -1197,8 +1193,9 @@ mod ovk_builder_roundtrip_tests; #[cfg(test)] mod memo_roundtrip_tests; -/// Cancellation of a note-sync pass: the consumed prefix is finalized, -/// a pre-scan cancel touches nothing, and a later pass resumes cleanly. +/// Cancellation of a note-sync pass: a scan cut short keeps only its tree +/// progress and publishes nothing, a pre-scan cancel touches nothing, and +/// a later pass re-walks the prefix without re-appending it. #[cfg(test)] mod cancel_tests; diff --git a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedSync.swift b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedSync.swift index 97d86c8dfb9..617c00d1160 100644 --- a/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedSync.swift +++ b/packages/swift-sdk/Sources/SwiftDashSDK/PlatformWallet/PlatformWalletManagerShieldedSync.swift @@ -321,11 +321,15 @@ extension PlatformWalletManager { /// Stop the shielded sync loop, waiting on the calling thread. /// - /// The native stop cancels the loop, then waits up to 10 s (the Rust - /// coordinator's drain budget) for a pass in flight to finish. On the - /// main actor that freezes the UI for as long; the async overload runs - /// the same stop off the main thread. If the pass does not drain in time - /// this throws `shutdownIncomplete`, before bumping the generation. + /// The native stop cancels the loop and the pass in flight, then waits + /// up to 10 s (the Rust coordinator's drain budget) for that pass to + /// stop at its next checkpoint; a scan cut short publishes nothing and + /// the next pass re-walks from the last completed watermark. On the + /// main actor the wait freezes the UI for as long; the async overload + /// runs the same stop off the main thread. If the pass does not drain + /// in time (it is wedged in a host persister, or in a broadcast the + /// SDK is still retrying) this throws `shutdownIncomplete`, before + /// bumping the generation. /// /// Throws `walletOperation` while an async [`stopShieldedSync()`] is in /// flight: this stop would wait on the same drain on the calling thread. @@ -348,11 +352,13 @@ extension PlatformWalletManager { /// context overload resolution prefers this variant; sync contexts keep /// the blocking one. /// - /// The native stop cancels the loop at once, so no new pass starts, then - /// waits up to 10 s for a pass in flight to finish; a running pass cannot - /// be interrupted. If it does not drain in time this throws - /// `shutdownIncomplete`, and the pass keeps running: it can still persist - /// what it finds and deliver its events after this returns. + /// The native stop cancels the loop at once, so no new pass starts, and + /// cancels the pass in flight, which stops at its next checkpoint and + /// publishes nothing; the stop waits up to 10 s for it to do so. If it + /// does not drain in time (wedged in a host persister, or in a broadcast + /// the SDK is still retrying) this throws `shutdownIncomplete`, and the + /// pass keeps running: it can still persist what it finds and deliver + /// its events after this returns. /// /// Events: while the stop is in flight, progress events that reach the /// main actor are dropped, and a completion is held