From 4beeb529033653df840fdb7183dff3331be1054d Mon Sep 17 00:00:00 2001 From: Jeon1691 <16333003+Jeon1691@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:31:14 +0900 Subject: [PATCH 1/2] fix: preserve spk_cache in custom persistence IndexerChangeSet only carried last_revealed, so converting from the upstream keychain_txout::ChangeSet dropped spk_cache and converting back replaced it with an empty default. A wallet using custom persistence with use_spk_cache never stored its script pubkey cache and derived every script pubkey again on load. Add spk_cache to IndexerChangeSet and convert it in both directions. The field defaults to an empty map in the generated constructors, so existing IndexerChangeSet(lastRevealed) calls keep compiling. Fixes #1099 --- bdk-ffi/src/tests/wallet.rs | 193 +++++++++++++++++++++++++++++++++++- bdk-ffi/src/types.rs | 30 +++++- 2 files changed, 217 insertions(+), 6 deletions(-) diff --git a/bdk-ffi/src/tests/wallet.rs b/bdk-ffi/src/tests/wallet.rs index 970ae5a3..622f14fe 100644 --- a/bdk-ffi/src/tests/wallet.rs +++ b/bdk-ffi/src/tests/wallet.rs @@ -1,18 +1,21 @@ use crate::bitcoin::{Amount, BlockHash, FeeRate, Network, NetworkKind}; use crate::descriptor::Descriptor; -use crate::error::{CreateTxError, LoadWithPersistError}; +use crate::error::{CreateTxError, LoadWithPersistError, PersistenceError}; use crate::signer::SignersContainer; -use crate::store::Persister; +use crate::store::{Persistence, Persister}; use crate::tx_builder::{BumpFeeTxBuilder, TxBuilder}; -use crate::types::{UnconfirmedTx, Update}; +use crate::types::{ChangeSet, IndexerChangeSet, UnconfirmedTx, Update}; use crate::wallet::{CreateParams, LoadParams, Wallet}; use bdk_wallet::bitcoin::Amount as BdkAmount; +use bdk_wallet::bitcoin::ScriptBuf as BdkScriptBuf; use bdk_wallet::bitcoin::Transaction as BdkTransaction; use bdk_wallet::bitcoin::{absolute, transaction, TxOut as BdkTxOut}; +use bdk_wallet::chain::DescriptorId as BdkDescriptorId; use bdk_wallet::KeychainKind; -use std::sync::Arc; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; const EXTERNAL_DESCRIPTOR: &str = "wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/1h/0/*)"; const INTERNAL_DESCRIPTOR: &str = "wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/1h/1/*)"; @@ -499,3 +502,185 @@ fn test_load_from_two_path_descriptor_with_params() { error => panic!("expected InvalidChangeSet error, got {:?}", error), } } + +/// Number of script pubkeys derived ahead of the last revealed index in the spk cache tests. +const SPK_CACHE_LOOKAHEAD: u32 = 5; + +/// A `Persistence` implementation that keeps the aggregate `ChangeSet` in memory, the same way a +/// foreign-language custom persister would. +#[derive(Default)] +struct InMemoryCustomPersistence { + aggregate: Mutex>>, + last_persisted: Mutex>>, +} + +type BdkSpkCache = BTreeMap>; + +impl InMemoryCustomPersistence { + fn spk_cache_of(changeset: &Mutex>>) -> BdkSpkCache { + let changeset: bdk_wallet::ChangeSet = changeset + .lock() + .unwrap() + .as_ref() + .expect("nothing was persisted") + .as_ref() + .clone() + .into(); + changeset.indexer.spk_cache + } + + /// The spk cache in the aggregate of everything persisted so far. + fn aggregate_spk_cache(&self) -> BdkSpkCache { + Self::spk_cache_of(&self.aggregate) + } + + /// The spk cache in the most recent `persist` call only. + fn last_persisted_spk_cache(&self) -> BdkSpkCache { + Self::spk_cache_of(&self.last_persisted) + } +} + +impl Persistence for InMemoryCustomPersistence { + fn initialize(&self) -> Result, PersistenceError> { + Ok(self + .aggregate + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| Arc::new(ChangeSet::new()))) + } + + fn persist(&self, changeset: Arc) -> Result<(), PersistenceError> { + let mut aggregate = self.aggregate.lock().unwrap(); + let merged = match aggregate.take() { + Some(existing) => Arc::new(ChangeSet::from_merge(existing, Arc::clone(&changeset))), + None => Arc::clone(&changeset), + }; + *aggregate = Some(merged); + *self.last_persisted.lock().unwrap() = Some(changeset); + Ok(()) + } +} + +fn spk_cache_load_params() -> LoadParams { + LoadParams { + check_network: None, + check_genesis_hash: None, + lookahead: SPK_CACHE_LOOKAHEAD, + use_spk_cache: true, + } +} + +#[test] +fn test_custom_persistence_persists_spk_cache() { + let persistence = Arc::new(InMemoryCustomPersistence::default()); + let persister = Arc::new(Persister::custom(persistence.clone())); + let wallet = Wallet::create_with_params( + external_descriptor(), + internal_descriptor(), + Network::Signet, + Arc::clone(&persister), + CreateParams { + genesis_hash: None, + lookahead: SPK_CACHE_LOOKAHEAD, + use_spk_cache: true, + }, + ) + .unwrap(); + + // The index stages the script pubkeys it derives and hands them over on the next reveal. + wallet.reveal_next_address(KeychainKind::External); + assert!(wallet.persist(Arc::clone(&persister)).unwrap()); + + let spk_cache = persistence.aggregate_spk_cache(); + let external_id = BdkDescriptorId(external_descriptor().descriptor_id().0); + let internal_id = BdkDescriptorId(internal_descriptor().descriptor_id().0); + // Revealing index 0 keeps `SPK_CACHE_LOOKAHEAD` more script pubkeys derived past it. + let expected_lengths = [ + (KeychainKind::External, external_id, SPK_CACHE_LOOKAHEAD + 1), + (KeychainKind::Internal, internal_id, SPK_CACHE_LOOKAHEAD), + ]; + for (keychain, descriptor_id, expected_len) in expected_lengths { + let cached = spk_cache + .get(&descriptor_id) + .unwrap_or_else(|| panic!("no spk cache persisted for {:?}", keychain)); + assert_eq!(cached.len() as u32, expected_len); + for (index, script) in cached { + let expected = wallet + .peek_address(keychain, *index) + .address + .script_pubkey(); + assert_eq!(script.as_bytes(), expected.to_bytes().as_slice()); + } + } +} + +#[test] +fn test_custom_persistence_reloads_spk_cache_without_rederiving() { + let persistence = Arc::new(InMemoryCustomPersistence::default()); + let persister = Arc::new(Persister::custom(persistence.clone())); + let wallet = Wallet::create_with_params( + external_descriptor(), + internal_descriptor(), + Network::Signet, + Arc::clone(&persister), + CreateParams { + genesis_hash: None, + lookahead: SPK_CACHE_LOOKAHEAD, + use_spk_cache: true, + }, + ) + .unwrap(); + wallet.reveal_next_address(KeychainKind::External); + assert!(wallet.persist(Arc::clone(&persister)).unwrap()); + drop(wallet); + + let loaded = Wallet::load_with_params( + external_descriptor(), + internal_descriptor(), + Arc::clone(&persister), + spk_cache_load_params(), + ) + .unwrap(); + let revealed = loaded.reveal_next_address(KeychainKind::External); + assert!(loaded.persist(Arc::clone(&persister)).unwrap()); + + // With the cache restored, only the one script pubkey newly derived past the lookahead window + // is staged. Without it, every script pubkey would be derived again on load and staged here. + let newly_derived_index = revealed.index + SPK_CACHE_LOOKAHEAD; + let external_id = BdkDescriptorId(external_descriptor().descriptor_id().0); + let staged: Vec<(BdkDescriptorId, u32)> = persistence + .last_persisted_spk_cache() + .into_iter() + .flat_map(|(id, spks)| spks.into_keys().map(move |index| (id, index))) + .collect(); + assert_eq!(staged, vec![(external_id, newly_derived_index)]); +} + +/// Arbitrary derivation indices for the `IndexerChangeSet` round-trip test. +const ROUND_TRIP_LAST_REVEALED_INDEX: u32 = 3; +const ROUND_TRIP_CACHED_SPK_INDEX: u32 = 7; + +#[test] +fn test_indexer_changeset_round_trips_spk_cache() { + let descriptor_id = BdkDescriptorId(external_descriptor().descriptor_id().0); + let script = build_wallet() + .peek_address(KeychainKind::External, ROUND_TRIP_CACHED_SPK_INDEX) + .address + .script_pubkey() + .0 + .clone(); + let original = bdk_wallet::chain::indexer::keychain_txout::ChangeSet { + last_revealed: [(descriptor_id, ROUND_TRIP_LAST_REVEALED_INDEX)].into(), + spk_cache: [( + descriptor_id, + [(ROUND_TRIP_CACHED_SPK_INDEX, script)].into(), + )] + .into(), + }; + + let ffi: IndexerChangeSet = original.clone().into(); + let round_tripped: bdk_wallet::chain::indexer::keychain_txout::ChangeSet = ffi.into(); + + assert_eq!(round_tripped, original); +} diff --git a/bdk-ffi/src/types.rs b/bdk-ffi/src/types.rs index 68642fe7..c8d177d9 100644 --- a/bdk-ffi/src/types.rs +++ b/bdk-ffi/src/types.rs @@ -1014,10 +1014,19 @@ pub struct EvictedTx { pub evicted_at: u64, } -/// Mapping of descriptors to their last revealed index. +/// Changes to the keychain indexer: the last revealed index of each descriptor and the cache of +/// derived script pubkeys. #[derive(Debug, Clone, uniffi::Record)] pub struct IndexerChangeSet { + /// Maps each descriptor to its last revealed derivation index. pub last_revealed: HashMap, u32>, + /// Cache of derived script pubkeys, keyed by descriptor and derivation index. + /// + /// Only populated when the wallet is created or loaded with `use_spk_cache`. Custom + /// persistence should store it so the script pubkeys are not derived again when the wallet + /// is loaded with `use_spk_cache`. + #[uniffi(default)] + pub spk_cache: HashMap, HashMap>>, } impl From for IndexerChangeSet { @@ -1026,8 +1035,17 @@ impl From for IndexerChan for (id, index) in core::mem::take(&mut value.last_revealed) { changes.insert(Arc::new(DescriptorId(id.0)), index); } + let mut spk_cache = HashMap::new(); + for (id, spks) in core::mem::take(&mut value.spk_cache) { + let spks = spks + .into_iter() + .map(|(index, spk)| (index, Arc::new(Script(spk)))) + .collect(); + spk_cache.insert(Arc::new(DescriptorId(id.0)), spks); + } Self { last_revealed: changes, + spk_cache, } } } @@ -1039,9 +1057,17 @@ impl From for bdk_wallet::chain::indexer::keychain_txout::Chan let descriptor_id = bdk_wallet::chain::DescriptorId(id.0); changes.insert(descriptor_id, index); } + let mut spk_cache = BTreeMap::new(); + for (id, spks) in core::mem::take(&mut value.spk_cache) { + let spks = spks + .into_iter() + .map(|(index, spk)| (index, spk.0.clone())) + .collect(); + spk_cache.insert(bdk_wallet::chain::DescriptorId(id.0), spks); + } Self { last_revealed: changes, - spk_cache: Default::default(), + spk_cache, } } } From fde9430b0e6d45d71abd3a8cd146a57c7435d95c Mon Sep 17 00:00:00 2001 From: Jeon1691 <16333003+Jeon1691@users.noreply.github.com> Date: Sun, 15 Nov 2026 21:31:14 +0900 Subject: [PATCH 2/2] test: add kotlin and swift tests for custom persistence spk cache --- .../bitcoindevkit/CustomPersistenceTest.kt | 116 +++++++++++++++++ .../CustomPersistenceTests.swift | 117 ++++++++++++++++++ 2 files changed, 233 insertions(+) create mode 100644 bdk-android/lib/src/androidTest/kotlin/org/bitcoindevkit/CustomPersistenceTest.kt create mode 100644 bdk-swift/Tests/BitcoinDevKitTests/CustomPersistenceTests.swift diff --git a/bdk-android/lib/src/androidTest/kotlin/org/bitcoindevkit/CustomPersistenceTest.kt b/bdk-android/lib/src/androidTest/kotlin/org/bitcoindevkit/CustomPersistenceTest.kt new file mode 100644 index 00000000..575b5567 --- /dev/null +++ b/bdk-android/lib/src/androidTest/kotlin/org/bitcoindevkit/CustomPersistenceTest.kt @@ -0,0 +1,116 @@ +package org.bitcoindevkit + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import org.junit.runner.RunWith +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue + +/** Keeps the aggregate `ChangeSet` in memory, the way an app-provided persister would store it. */ +private class InMemoryPersistence : Persistence { + private var aggregate: ChangeSet? = null + var lastPersisted: ChangeSet? = null + private set + + @Synchronized + override fun initialize(): ChangeSet = aggregate ?: ChangeSet() + + @Synchronized + override fun persist(changeset: ChangeSet) { + aggregate = aggregate?.let { ChangeSet.fromMerge(it, changeset) } ?: changeset + lastPersisted = changeset + } + + @Synchronized + fun aggregateSpkCache(): Map> = + aggregate?.indexerChangeset()?.spkCache ?: emptyMap() +} + +@RunWith(AndroidJUnit4::class) +class CustomPersistenceTest { + /** Number of script pubkeys the wallet derives past the last revealed index. */ + private val lookahead: UInt = 5u + + private val descriptor: Descriptor = Descriptor( + "wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/1h/0/*)", + NetworkKind.TEST + ) + private val changeDescriptor: Descriptor = Descriptor( + "wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/1h/1/*)", + NetworkKind.TEST + ) + + private fun createWallet(persister: Persister): Wallet = Wallet.createWithParams( + descriptor = descriptor, + changeDescriptor = changeDescriptor, + network = Network.SIGNET, + persister = persister, + params = CreateParams(genesisHash = null, lookahead = lookahead, useSpkCache = true), + ) + + // Code written before `spkCache` existed still compiles, and gets an empty cache. + @Test + fun indexerChangeSetDefaultsToEmptySpkCache() { + val changeset = IndexerChangeSet(lastRevealed = mapOf(descriptor.descriptorId() to 3u)) + + assertTrue(changeset.spkCache.isEmpty()) + } + + // The script pubkeys the wallet derives reach a custom persister, so it can store them. + @Test + fun customPersistenceReceivesSpkCache() { + val persistence = InMemoryPersistence() + val persister = Persister.custom(persistence) + val wallet = createWallet(persister) + + // The derived script pubkeys are handed over together with the next reveal. + wallet.revealNextAddress(KeychainKind.EXTERNAL) + assertTrue(wallet.persist(persister)) + + val spkCache = persistence.aggregateSpkCache() + val expected = listOf( + // Revealing index 0 keeps `lookahead` more script pubkeys derived past it. + Triple(KeychainKind.EXTERNAL, descriptor.descriptorId(), lookahead + 1u), + Triple(KeychainKind.INTERNAL, changeDescriptor.descriptorId(), lookahead), + ) + for ((keychain, descriptorId, count) in expected) { + val cached = assertNotNull(spkCache[descriptorId], "no spk cache for $keychain") + assertEquals(count, cached.size.toUInt()) + for ((index, script) in cached) { + val address = wallet.peekAddress(keychain, index).address + assertContentEquals(address.scriptPubkey().toBytes(), script.toBytes()) + } + } + } + + // A wallet loaded from a custom persister reuses the stored script pubkeys instead of deriving + // them again, so the next reveal only stages the one new script pubkey. + @Test + fun loadingFromCustomPersistenceReusesSpkCache() { + val persistence = InMemoryPersistence() + val persister = Persister.custom(persistence) + val wallet = createWallet(persister) + wallet.revealNextAddress(KeychainKind.EXTERNAL) + assertTrue(wallet.persist(persister)) + + val loaded = Wallet.loadWithParams( + descriptor = descriptor, + changeDescriptor = changeDescriptor, + persister = persister, + params = LoadParams( + checkNetwork = null, + checkGenesisHash = null, + lookahead = lookahead, + useSpkCache = true, + ), + ) + val revealed = loaded.revealNextAddress(KeychainKind.EXTERNAL) + assertTrue(loaded.persist(persister)) + + val staged = assertNotNull(persistence.lastPersisted).indexerChangeset().spkCache + .flatMap { (descriptorId, spks) -> spks.keys.map { descriptorId to it } } + assertEquals(listOf(descriptor.descriptorId() to revealed.index + lookahead), staged) + } +} diff --git a/bdk-swift/Tests/BitcoinDevKitTests/CustomPersistenceTests.swift b/bdk-swift/Tests/BitcoinDevKitTests/CustomPersistenceTests.swift new file mode 100644 index 00000000..8f21d437 --- /dev/null +++ b/bdk-swift/Tests/BitcoinDevKitTests/CustomPersistenceTests.swift @@ -0,0 +1,117 @@ +import XCTest +@testable import BitcoinDevKit + +/// Keeps the aggregate `ChangeSet` in memory, the way an app-provided persister would store it. +private final class InMemoryPersistence: Persistence, @unchecked Sendable { + private let lock = NSLock() + private var aggregate: ChangeSet? + private(set) var lastPersisted: ChangeSet? + + func initialize() throws -> ChangeSet { + lock.lock() + defer { lock.unlock() } + return aggregate ?? ChangeSet() + } + + func persist(changeset: ChangeSet) throws { + lock.lock() + defer { lock.unlock() } + aggregate = aggregate.map { ChangeSet.fromMerge(left: $0, right: changeset) } ?? changeset + lastPersisted = changeset + } + + func aggregateSpkCache() -> [DescriptorId: [UInt32: Script]] { + lock.lock() + defer { lock.unlock() } + return aggregate?.indexerChangeset().spkCache ?? [:] + } +} + +final class CustomPersistenceTests: XCTestCase { + /// Number of script pubkeys the wallet derives past the last revealed index. + private let lookahead: UInt32 = 5 + + private let descriptor = try! Descriptor( + descriptor: "wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/1h/0/*)", + networkKind: NetworkKind.test + ) + private let changeDescriptor = try! Descriptor( + descriptor: "wpkh(tprv8ZgxMBicQKsPf2qfrEygW6fdYseJDDrVnDv26PH5BHdvSuG6ecCbHqLVof9yZcMoM31z9ur3tTYbSnr1WBqbGX97CbXcmp5H6qeMpyvx35B/84h/1h/1h/1/*)", + networkKind: NetworkKind.test + ) + + private func createWallet(persister: Persister) throws -> Wallet { + try Wallet.createWithParams( + descriptor: descriptor, + changeDescriptor: changeDescriptor, + network: .signet, + persister: persister, + params: CreateParams(genesisHash: nil, lookahead: lookahead, useSpkCache: true) + ) + } + + // Code written before `spkCache` existed still compiles, and gets an empty cache. + func testIndexerChangeSetDefaultsToEmptySpkCache() { + let changeset = IndexerChangeSet(lastRevealed: [descriptor.descriptorId(): 3]) + + XCTAssertTrue(changeset.spkCache.isEmpty) + } + + // The script pubkeys the wallet derives reach a custom persister, so it can store them. + func testCustomPersistenceReceivesSpkCache() throws { + let persistence = InMemoryPersistence() + let persister = Persister.custom(persistence: persistence) + let wallet = try createWallet(persister: persister) + + // The derived script pubkeys are handed over together with the next reveal. + _ = wallet.revealNextAddress(keychain: .external) + XCTAssertTrue(try wallet.persist(persister: persister)) + + let spkCache = persistence.aggregateSpkCache() + let expected: [(KeychainKind, DescriptorId, UInt32)] = [ + // Revealing index 0 keeps `lookahead` more script pubkeys derived past it. + (.external, descriptor.descriptorId(), lookahead + 1), + (.internal, changeDescriptor.descriptorId(), lookahead), + ] + for (keychain, descriptorId, count) in expected { + let cached = try XCTUnwrap(spkCache[descriptorId], "no spk cache for \(keychain)") + XCTAssertEqual(UInt32(cached.count), count) + for (index, script) in cached { + let address = wallet.peekAddress(keychain: keychain, index: index).address + XCTAssertEqual(script.toBytes(), address.scriptPubkey().toBytes()) + } + } + } + + // A wallet loaded from a custom persister reuses the stored script pubkeys instead of deriving + // them again, so the next reveal only stages the one new script pubkey. + func testLoadingFromCustomPersistenceReusesSpkCache() throws { + let persistence = InMemoryPersistence() + let persister = Persister.custom(persistence: persistence) + let wallet = try createWallet(persister: persister) + _ = wallet.revealNextAddress(keychain: .external) + XCTAssertTrue(try wallet.persist(persister: persister)) + + let loaded = try Wallet.loadWithParams( + descriptor: descriptor, + changeDescriptor: changeDescriptor, + persister: persister, + params: LoadParams( + checkNetwork: nil, + checkGenesisHash: nil, + lookahead: lookahead, + useSpkCache: true + ) + ) + let revealed = loaded.revealNextAddress(keychain: .external) + XCTAssertTrue(try loaded.persist(persister: persister)) + + let staged = try XCTUnwrap(persistence.lastPersisted).indexerChangeset().spkCache + let stagedIndices = staged.flatMap { descriptorId, spks in + spks.keys.map { (descriptorId, $0) } + } + XCTAssertEqual(stagedIndices.count, 1) + XCTAssertEqual(stagedIndices.first?.0, descriptor.descriptorId()) + XCTAssertEqual(stagedIndices.first?.1, revealed.index + lookahead) + } +}