From 4ea2e06650b14fe069c6065d3e1c11b059e0d29e Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 24 Sep 2026 10:26:43 +0000 Subject: [PATCH 01/10] test: drive the BIP-352 vectors through the live short receive path (SNB-541) The official vectors only reached Scanner::scan_transaction_full, which has no production caller. The daemon receives through scan_short_block_data -> scan_transaction_short (k = 0, labels enumerated by hand, 8-byte x-only prefixes) and, on a probable match, apply_block_relevant on the external indexer. A false negative in the short step means the block is never fetched: silent loss. - Serve every receiving vector the way the oracle does (taproot outputs' 8-byte x-only prefixes, the 33-byte tweak, the display-order txid) and assert a probable match wherever the vector expects >= 1 output. Pins which k = 0 branch did the work: unlabelled P_0, labels of both output parities, and the change label m = 0. False positives are counted and reported (zero on these vectors). - Drive the real post-match step for the same vectors: the vector tx in a full block through apply_block_relevant, asserting the stored outputs and priv_key_tweaks equal expected.outputs. - Extract the post-match step (txid mapping + apply_block_relevant + stage merge) from scan_block_range into Scanner::apply_matched_block, unchanged, so the test exercises the production code rather than a copy. - K_max through the live path is SNB-540: an ignored test records that it stores 2324 outputs where BIP-352 allows 2323. Co-Authored-By: Claude Opus 5.5 (1M context) --- blindbit-lib/src/scanner/bip352_vectors.rs | 511 ++++++++++++++++++++- blindbit-lib/src/scanner/scanning.rs | 75 +-- 2 files changed, 552 insertions(+), 34 deletions(-) diff --git a/blindbit-lib/src/scanner/bip352_vectors.rs b/blindbit-lib/src/scanner/bip352_vectors.rs index 40f6125..55638b4 100644 --- a/blindbit-lib/src/scanner/bip352_vectors.rs +++ b/blindbit-lib/src/scanner/bip352_vectors.rs @@ -1,11 +1,16 @@ //! Official BIP-352 receiver vectors exercised at the blindbit-lib boundary. //! -//! The boundary under test is [`Scanner::scan_transaction_full`], which has no -//! production caller today — the daemon receives through -//! `Scanner::scan_transaction_short` plus `apply_block_relevant` on the external -//! indexer. These tests are what exercises it, and they pin its behaviour for -//! whenever full block responses are scanned again; they say nothing directly -//! about the live short-scan path. +//! Two boundaries are under test: +//! +//! * [`Scanner::scan_transaction_full`], which has no production caller today. +//! The first group of tests pins its behaviour for whenever full block +//! responses are scanned again. +//! * The live receive path the daemon actually runs: the oracle's short block +//! data through `Scanner::scan_short_block_data` (and so +//! `scan_transaction_short` / `match_short_pubkey`), then, on a probable match, +//! the full block through `Scanner::apply_matched_block` +//! (`apply_block_relevant` on the external indexer). See the section "The live +//! short receive path" at the end of this file. //! //! Fixture provenance (re-checked at runtime by [`vectors`], so a silent swap or //! a corrupted copy fails loudly instead of quietly changing coverage): @@ -1555,3 +1560,497 @@ fn official_vectors_extra_change_label_is_harmless() { "exactly one non-K_max sub-case (the sender-change vector) registers m=0 itself" ); } + +// --------------------------------------------------------------------------- +// The live short receive path +// --------------------------------------------------------------------------- +// +// Everything above goes through `scan_transaction_full`. The daemon does not: +// per block it asks the oracle for `BlockScanDataShortResponse`, runs +// `scan_short_block_data` -> `probabilistic_match` -> `scan_transaction_short` +// (k = 0 only, labels enumerated by hand, 8-byte x-only prefixes), and only on a +// probable match fetches the full block and hands it to `apply_matched_block`, +// i.e. `apply_block_relevant` on the external indexer. A false negative in the +// short step means the block is never fetched, so the wallet silently never +// sees the payment. The tests below drive every receiving vector through that +// exact sequence of production functions. + +/// Length of one `outputs_short` entry: the oracle stores `Pubkey[:8]` of every +/// taproot output (`blindbit-oracle` `dbpebble/store.go`). Deliberately not +/// shared with `match_short_pubkey`, so a change there cannot silently change +/// what the test builds too. +const SHORT_PREFIX_LEN: usize = 8; + +/// Height the synthetic blocks claim to be at. +const LIVE_BLOCK_HEIGHT: u64 = 840_000; + +/// A txid as the oracle puts it on the wire: display order (see [`full_item`]). +fn wire_txid(tx: &Transaction) -> [u8; 32] { + let mut txid = tx.compute_txid().to_raw_hash().to_byte_array(); + txid.reverse(); + txid +} + +/// The `ComputeIndexTxItem` the oracle serves for `tx`: the 33-byte tweak +/// (`input_hash * A`) and, for every **taproot** output in output order, the +/// first [`SHORT_PREFIX_LEN`] bytes of its x-only key. Non-taproot outputs are +/// not stored by the oracle and so never appear. +fn short_item(tx: &Transaction, tweak: PublicKey) -> crate::oracle_grpc::ComputeIndexTxItem { + let outputs_short = tx + .output + .iter() + .filter(|output| output.script_pubkey.is_p2tr()) + .flat_map(|output| output.script_pubkey.as_bytes()[2..2 + SHORT_PREFIX_LEN].to_vec()) + .collect(); + crate::oracle_grpc::ComputeIndexTxItem { + txid: wire_txid(tx).to_vec(), + tweak: tweak.serialize().to_vec(), + outputs_short, + } +} + +/// A non-coinbase transaction that is not ours, so the block the receive step +/// sees is not just the vector transaction on its own. +fn decoy_transaction() -> Transaction { + let decoy_key = XOnlyPublicKey::from_str( + "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", + ) + .expect("decoy key"); + Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { + txid: Txid::from_byte_array([0x11; 32]), + vout: 7, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(5_000), + script_pubkey: ScriptBuf::new_p2tr_tweaked( + bdk_sp::bitcoin::key::TweakedPublicKey::dangerous_assume_tweaked(decoy_key), + ), + }], + } +} + +/// The full block the daemon would fetch over P2P: a coinbase (which +/// `apply_block_relevant` skips by position), a decoy, then the vector's +/// transaction. +fn live_block(tx: &Transaction) -> bitcoin::Block { + let coinbase = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes(vec![0x03, 0x40, 0xd1, 0x0c]), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(312_500_000), + script_pubkey: ScriptBuf::from_bytes(vec![0x6a]), + }], + }; + let mut block = bitcoin::Block { + header: bitcoin::block::Header { + version: bitcoin::block::Version::TWO, + prev_blockhash: bitcoin::BlockHash::all_zeros(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: 1_713_571_767, + bits: bitcoin::CompactTarget::from_consensus(0x1703_4219), + nonce: 0, + }, + txdata: vec![coinbase, decoy_transaction(), tx.clone()], + }; + block.header.merkle_root = block.compute_merkle_root().expect("non-empty block"); + block +} + +/// The per-block short response the oracle streams, carrying `items`. +fn short_block( + block: &bitcoin::Block, + items: Vec, +) -> crate::oracle_grpc::BlockScanDataShortResponse { + let mut block_hash = block.block_hash().to_raw_hash().to_byte_array(); + block_hash.reverse(); + crate::oracle_grpc::BlockScanDataShortResponse { + block_identifier: Some(crate::oracle_grpc::BlockIdentifier { + block_hash: block_hash.to_vec(), + block_height: LIVE_BLOCK_HEIGHT, + }), + comp_index: items, + spent_outputs: Vec::new(), + } +} + +/// Which k = 0 candidate the vector transaction actually carries, by full +/// 32-byte key: `Some(None)` for the unlabelled `P_0`, `Some(Some(m))` for +/// `P_0 + label_m`, `None` if neither. This is independent of the short path +/// and says which label iteration the short path *must* perform to match. +fn k0_candidates( + tx: &Transaction, + given: &ReceivingGiven, + scan_sk: SecretKey, + spend_pk: PublicKey, + shared_secret: PublicKey, +) -> BTreeSet> { + let keys: HashSet = tx + .output + .iter() + .filter(|output| output.script_pubkey.is_p2tr()) + .map(|output| { + XOnlyPublicKey::from_slice(&output.script_pubkey.as_bytes()[2..]).expect("taproot key") + }) + .collect(); + std::iter::once(None) + .chain(given.labels.iter().map(|&m| Some(m))) + .filter(|label| { + let label_pt = label.map(|m| label_point(scan_sk, m)); + let p_0 = get_silentpayment_pubkey(&spend_pk, &shared_secret, 0, label_pt.as_ref()); + keys.contains(&p_0.x_only_public_key().0) + }) + .collect() +} + +/// No false negatives on the live short path, for every receiving vector. +/// +/// Each vector transaction is served the way the oracle serves it (8-byte +/// prefixes of its taproot outputs, the 33-byte tweak, the display-order txid) +/// and pushed through `scan_short_block_data`, the function the daemon calls per +/// block. Wherever the vector expects the wallet to find at least one output, +/// the block must come back as a probable match carrying that txid and tweak. +/// +/// The k = 0 candidate each matched vector carries is pinned independently (see +/// [`k0_candidates`]), so the test also proves *which* short-path branch did the +/// work: the unlabelled `P_0`, a labelled one of either output parity, and the +/// change label `m = 0`. Dropping the label iteration, or matching on anything +/// other than the 8-byte x-only prefix, turns this red. +/// +/// False positives (a probable match where the vector expects nothing) are only +/// a wasted block fetch; they are counted and reported, and pinned at zero on +/// these vectors because an 8-byte prefix collision among them would be a +/// fixture surprise worth looking at. +#[test] +fn official_vectors_reach_the_live_short_receive_path() { + let mut no_wire_tweak = 0usize; + let mut must_match = 0usize; + let mut must_not_match = 0usize; + let mut false_positives = Vec::new(); + // Matched sub-cases whose only k = 0 candidate is labelled, by label. + let mut label_only: BTreeMap = BTreeMap::new(); + let mut unlabelled_k0 = 0usize; + let mut parity_cases: BTreeMap<&str, bool> = BTreeMap::new(); + + for (case_idx, case) in vectors().iter().enumerate() { + for (sub_idx, receiving) in case.receiving.iter().enumerate() { + let what = format!("case {} sub {sub_idx} ({})", case_idx + 1, case.comment); + let (given, expected) = (&receiving.given, &receiving.expected); + let (tx, prevouts) = build_transaction(given); + + let tweak = match compute_tweak_data(&tx, &prevouts) { + Ok(tweak) => tweak, + Err(err) => { + // The oracle cannot publish a tweak for these, so there is no + // short item to serve; classified as in the boundary test. + let reason = classify_no_tweak(&tx, &prevouts, &err, &what); + assert_eq!(reason, expected_no_tweak_reason(&case.comment), "{what}"); + assert!(expected.outputs.is_empty() && expected.n_outputs.is_none()); + no_wire_tweak += 1; + continue; + } + }; + assert_eq!( + tweak.to_string(), + expected.tweak.as_deref().expect("vector tweak"), + "{what}: served tweak" + ); + let (scan_sk, spend_pk) = scan_keys(given); + let shared_secret = bdk_sp::compute_shared_secret(&scan_sk, &tweak); + + let item = short_item(&tx, tweak); + assert_eq!(item.tweak.len(), 33, "{what}: served tweak length"); + assert_eq!( + item.outputs_short.len(), + SHORT_PREFIX_LEN * given.outputs.len(), + "{what}: every vector output is taproot and must be served" + ); + + let mut scanner = blindbit_scanner( + scan_sk, + spend_pk, + &given.labels, + &format!("short-{case_idx}-{sub_idx}"), + ); + let probable = scanner + .scan_short_block_data(short_block(&live_block(&tx), vec![item])) + .expect("short block scan"); + let matched = match &probable { + Some(probable) => { + assert!(!probable.spent, "{what}: no spent outputs were served"); + assert_eq!( + probable.matched_txs, + vec![(wire_txid(&tx), tweak)], + "{what}: probable match must carry the served txid and tweak" + ); + true + } + None => false, + }; + + let expected_count = expected.n_outputs.unwrap_or(expected.outputs.len()); + if expected_count == 0 { + must_not_match += 1; + if matched { + false_positives.push(what); + } + continue; + } + + must_match += 1; + assert!( + matched, + "{what}: FALSE NEGATIVE on the live short path — the vector expects \ + {expected_count} output(s), but the block would never be fetched" + ); + + let k0 = k0_candidates(&tx, given, scan_sk, spend_pk, shared_secret); + assert!( + !k0.is_empty(), + "{what}: a transaction paying us must carry a k = 0 output" + ); + if k0.contains(&None) { + unlabelled_k0 += 1; + } else { + for m in k0.iter().flatten() { + *label_only.entry(*m).or_default() += 1; + } + } + + for kind in ["even", "odd"] { + if case.comment.contains(&format!("label with {kind} parity")) { + parity_cases.insert(kind, !k0.contains(&None)); + } + } + } + } + + for fp in &false_positives { + eprintln!("short-path false positive: {fp}"); + } + eprintln!( + "short path: {must_match} sub-cases must match (all did), {must_not_match} must \ + not, {} false positive(s), {no_wire_tweak} without a wire tweak", + false_positives.len() + ); + + assert_eq!(no_wire_tweak, NO_TWEAK_SUBCASE_COUNT); + assert_eq!( + must_match + must_not_match + no_wire_tweak, + RECEIVING_SUBCASE_COUNT, + "not every receiving sub-case reached the short path" + ); + assert_eq!( + must_not_match, 1, + "the 'recipient ignores unrelated outputs' vector is no longer covered" + ); + assert!( + false_positives.is_empty(), + "unexpected short-path false positives: {false_positives:?}" + ); + + // Branch coverage of the short path: vectors that can only be found through + // the label iteration, including both parities and the change label m = 0. + assert_eq!( + parity_cases.get("even"), + Some(&true), + "the even-parity label vector must be reachable only through a label" + ); + assert_eq!( + parity_cases.get("odd"), + Some(&true), + "the odd-parity label vector must be reachable only through a label" + ); + assert!( + label_only.contains_key(&0), + "no vector is reachable only through the change label m = 0: {label_only:?}" + ); + let label_only_total: usize = label_only.values().sum(); + assert_eq!( + label_only_total, 7, + "label-only coverage changed: {label_only:?}" + ); + assert!(unlabelled_k0 > 0, "no unlabelled k = 0 vector was matched"); + eprintln!("short path: {unlabelled_k0} matched via P_0, label-only by m: {label_only:?}"); +} + +/// Runs one vector through the whole live receive sequence — short scan, then +/// (only on a probable match, as the daemon does) the full-block step — and +/// returns every output the indexer now owns in the vector transaction, as +/// `x-only key -> priv_key_tweak`, plus its label. +fn live_receive( + given: &ReceivingGiven, + tx: &Transaction, + tweak: PublicKey, + tag: &str, +) -> (Scanner, BTreeMap)>) { + let (scan_sk, spend_pk) = scan_keys(given); + let mut scanner = blindbit_scanner(scan_sk, spend_pk, &given.labels, tag); + let block = live_block(tx); + + let probable = scanner + .scan_short_block_data(short_block(&block, vec![short_item(tx, tweak)])) + .expect("short block scan"); + if let Some(probable) = probable { + scanner.apply_matched_block(&block, &probable, LIVE_BLOCK_HEIGHT as u32); + } + + let txid = tx.compute_txid(); + let index = scanner.internal_indexer.index(); + let labels: BTreeMap> = index + .by_label + .iter() + .map(|&(label, outpoint)| (outpoint, label)) + .collect(); + let mut found = BTreeMap::new(); + for (outpoint, tweak) in &index.by_shared_secret { + assert_eq!( + outpoint.txid, txid, + "{tag}: indexed an output of another tx" + ); + let script = &tx.output[outpoint.vout as usize].script_pubkey; + let xonly = XOnlyPublicKey::from_slice(&script.as_bytes()[2..]).expect("taproot output"); + let label = *labels + .get(outpoint) + .expect("every indexed output has a label entry"); + assert!( + found.insert(xonly, (*tweak, label)).is_none(), + "{tag}: output key indexed twice" + ); + } + (scanner, found) +} + +/// The real receive step after a probable match, for every vector. +/// +/// After [`official_vectors_reach_the_live_short_receive_path`] establishes that +/// the block is fetched, this pins what the daemon then *stores*: the vector +/// transaction is placed in a full block and handed through +/// `Scanner::apply_matched_block` (the production code that maps the oracle's +/// display-order txids to the block's transactions and calls +/// `apply_block_relevant` on the external indexer). The outputs the indexer +/// owns afterwards must be exactly the vector's `expected.outputs`, with the +/// vector's `priv_key_tweak` and a label that reproduces the output key. +/// +/// The K_max vector is excluded here and has its own ignored test (SNB-540). +#[test] +fn official_vectors_live_receive_step_finds_expected_outputs() { + let mut exercised = 0usize; + let mut asserted_outputs = 0usize; + let mut labelled_outputs = 0usize; + + for (case_idx, case) in vectors().iter().enumerate() { + for (sub_idx, receiving) in case.receiving.iter().enumerate() { + let what = format!("case {} sub {sub_idx} ({})", case_idx + 1, case.comment); + let (given, expected) = (&receiving.given, &receiving.expected); + if expected.n_outputs.is_some() { + continue; // K_max: see the ignored SNB-540 test below. + } + let (tx, prevouts) = build_transaction(given); + let Ok(tweak) = compute_tweak_data(&tx, &prevouts) else { + continue; // no wire tweak; classified in the short-path test + }; + let (scan_sk, spend_pk) = scan_keys(given); + let shared_secret = bdk_sp::compute_shared_secret(&scan_sk, &tweak); + + let (scanner, found) = + live_receive(given, &tx, tweak, &format!("live-{case_idx}-{sub_idx}")); + + let found_tweaks: BTreeMap = found + .iter() + .map(|(key, (tweak, _))| (*key, *tweak)) + .collect(); + assert_eq!( + found_tweaks, + expected_outputs(expected), + "{what}: live receive step outputs / priv_key_tweak" + ); + + // Each stored label must reproduce the output key at some k. + for (key, (tweak, label)) in &found { + if let Some(m) = label { + assert!(given.labels.contains(m), "{what}: stray label {m}"); + labelled_outputs += 1; + } + let k = (0..given.outputs.len() as u32) + .find(|&k| tweak_for(shared_secret, scan_sk, k, *label) == *tweak) + .unwrap_or_else(|| panic!("{what}: tweak/label mismatch for {key}")); + let label_pt = label.map(|m| label_point(scan_sk, m)); + let p_k = get_silentpayment_pubkey(&spend_pk, &shared_secret, k, label_pt.as_ref()); + assert_eq!(p_k.x_only_public_key().0, *key, "{what}: stored label"); + } + + // And the transaction is in the wallet graph iff something was found. + let in_graph = scanner + .internal_indexer + .graph() + .get_tx(tx.compute_txid()) + .is_some(); + assert_eq!(in_graph, !found.is_empty(), "{what}: wallet graph"); + let staged = scanner + .stage + .indexer + .txid_to_partial_secret + .contains_key(&tx.compute_txid()); + assert_eq!(staged, !found.is_empty(), "{what}: staged for persistence"); + + asserted_outputs += found.len(); + exercised += 1; + } + } + + assert_eq!( + exercised, + SCANNED_SUBCASE_COUNT - K_MAX_SUBCASE_COUNT, + "not every scannable non-K_max sub-case reached the live receive step" + ); + assert_eq!( + asserted_outputs, EXPECTED_OUTPUT_COUNT, + "the fixture's per-output expectations were not all asserted on the live path" + ); + assert_eq!(labelled_outputs, 10, "labelled-output coverage changed"); +} + +/// The BIP-352 recipient limit on the live receive path. +/// +/// Ignored because it fails today: `apply_block_relevant` enforces no K_max, so +/// the live path stores one output more than BIP-352 allows on this vector. +/// That is SNB-540; this test is the acceptance check for it. It is also slow +/// (the external indexer's scan is quadratic in the 2324 outputs). +#[test] +#[ignore = "SNB-540: the live receive path enforces no K_max (stores 2324 of 2323)"] +fn official_vectors_live_receive_step_enforces_k_max() { + let mut exercised = 0usize; + for case in vectors() { + for receiving in &case.receiving { + let Some(n_outputs) = receiving.expected.n_outputs else { + continue; + }; + let given = &receiving.given; + let (tx, prevouts) = build_transaction(given); + let tweak = compute_tweak_data(&tx, &prevouts).expect("K_max vector inputs are valid"); + let (_, found) = live_receive(given, &tx, tweak, "live-kmax"); + assert_eq!( + found.len(), + n_outputs, + "{}: receiver must stop at K_max", + case.comment + ); + exercised += 1; + } + } + assert_eq!(exercised, K_MAX_SUBCASE_COUNT); +} diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index 8ee15ce..9e2dca5 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -173,38 +173,12 @@ impl Scanner { .fetch_block_with_retry(&mut p2p_conn, block_hash, block_identifier.block_height) .await?; - // build partial secret hashmap, only populate with txids and secrets where we - // suspect matches, skip the rest - let mut partial_secrets = - HashMap::with_capacity(probable_match.matched_txs.len()); - - for tx in &block.txdata { - if probable_match.spent { - // todo: we will need to look at all spent outpoints in this - // block and find the relevant txids for spent - } - - for (txid_arr, tweak) in &probable_match.matched_txs { - // this check should be optimised to a map lookup on all items - let mut item_txid = *txid_arr; - item_txid.reverse(); - - if Txid::from_byte_array(item_txid) != tx.compute_txid() { - continue; - } - - let txid = byte_array_to_txid(txid_arr); - - partial_secrets.insert(txid, *tweak); - } - } // Apply block to indexer and stage the changes - let indexer_changes = self.internal_indexer.apply_block_relevant( + self.apply_matched_block( &block, - partial_secrets, + &probable_match, block_identifier.block_height as u32, ); - self.stage.indexer.merge(indexer_changes); // Update block checkpoints: only store blocks where we found something let block_height_u32 = block_identifier.block_height as u32; @@ -691,6 +665,51 @@ impl Scanner { } } + /// The live receive step after a probable match: hands the full block and + /// the matched transactions' tweaks to the external indexer, which does the + /// real BIP-352 scan, and stages what it found. + /// + /// `probable_match.matched_txs` carries each txid as the oracle serves it + /// (display order) together with the served tweak (`input_hash * A`), which + /// `apply_block_relevant` takes as its "partial secret" and finishes the ECDH + /// on itself. + fn apply_matched_block( + &mut self, + block: &bitcoin::Block, + probable_match: &ProbableMatch, + height: u32, + ) { + // build partial secret hashmap, only populate with txids and secrets where we + // suspect matches, skip the rest + let mut partial_secrets = HashMap::with_capacity(probable_match.matched_txs.len()); + + for tx in &block.txdata { + if probable_match.spent { + // todo: we will need to look at all spent outpoints in this + // block and find the relevant txids for spent + } + + for (txid_arr, tweak) in &probable_match.matched_txs { + // this check should be optimised to a map lookup on all items + let mut item_txid = *txid_arr; + item_txid.reverse(); + + if Txid::from_byte_array(item_txid) != tx.compute_txid() { + continue; + } + + let txid = byte_array_to_txid(txid_arr); + + partial_secrets.insert(txid, *tweak); + } + } + // Apply block to indexer and stage the changes + let indexer_changes = self + .internal_indexer + .apply_block_relevant(block, partial_secrets, height); + self.stage.indexer.merge(indexer_changes); + } + fn probabilistic_match( &mut self, item: &ComputeIndexTxItem, From be4e8bbab416c4dd94283964834cba8e46c9a1f8 Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 24 Sep 2026 11:08:34 +0000 Subject: [PATCH 02/10] fix: record owned outputs so spends are detected; persist labels The scanner's `owned_outputs` set, which the per-block spent-output check matches the oracle's 8-byte `spent_outputs` prefixes against, had a writer (`add_owned_output`) that no call site ever invoked. The set stayed empty, the spent check never fired, and a block whose only wallet-relevant transaction was a spend (no Silent Payments output back to the wallet) was never fetched. The spending transaction never reached the wallet graph, so blindbit-cli and friglet reported a balance that still included spent coins and Electrum clients never saw the spend. Spends were only noticed by accident, when the spending transaction also paid the wallet (e.g. change). - `Scanner::sync_owned_outputs` rebuilds the owned-output records from the indexer after every applied block and on restore: outpoint, full x-only key, amount, height, and the BIP-352 label `m` the output matched. - A record is marked spent only when a confirmed transaction in the wallet graph spends exactly that outpoint. The 8-byte prefixes, matched through a prefix -> unspent-outpoint map, only decide which blocks to fetch; a foreign key sharing the prefix fetches the block but marks nothing. - The state file's `owned_outputs` array now holds these records (label and spend included). Legacy bare-hex entries are accepted and dropped; the records are re-derived from the indexer on load. - Restore registered labels only after re-indexing the stored transactions, and `load_from_file` blanks the persisted label table, so every labelled output (change included, m = 0) silently dropped out of the index, the balance and spend detection after each restart. Labels are now registered before the changeset is applied. - `apply_matched_block` is extracted from `scan_block_range` (same shape as in #17) so the tests drive the real per-block path without P2P. Tests (in-process, `owned_outputs_tests.rs`): receive then spend in a later block marks the output spent and the balance drops; a colliding foreign prefix marks nothing; records, labels and spends survive save/load/restore and spend detection keeps working after it; a labelled receive records the right m. Co-Authored-By: Claude Opus 5.5 (1M context) --- blindbit-lib/src/scanner/changeset.rs | 68 +-- blindbit-lib/src/scanner/mod.rs | 2 +- .../src/scanner/owned_outputs_tests.rs | 483 ++++++++++++++++++ blindbit-lib/src/scanner/scanner.rs | 148 +++++- blindbit-lib/src/scanner/scanning.rs | 115 +++-- blindbit-lib/src/scanner/types.rs | 44 +- 6 files changed, 769 insertions(+), 91 deletions(-) create mode 100644 blindbit-lib/src/scanner/owned_outputs_tests.rs diff --git a/blindbit-lib/src/scanner/changeset.rs b/blindbit-lib/src/scanner/changeset.rs index 66ca190..2468683 100644 --- a/blindbit-lib/src/scanner/changeset.rs +++ b/blindbit-lib/src/scanner/changeset.rs @@ -3,34 +3,38 @@ use indexer::bdk_chain::ConfirmationBlockTime; use indexer::bdk_chain::bdk_core::Merge; use std::collections::BTreeMap; -/// Helper module for hex encoding/decoding byte arrays in serialization +use super::types::OwnedOutputRecord; + +/// Deserialisation of the persisted `owned_outputs` array. #[cfg(feature = "serde")] -mod serde_hex { - use serde::{Deserialize, Deserializer, Serialize, Serializer}; +mod owned_outputs_serde { + use super::OwnedOutputRecord; + use serde::{Deserialize, Deserializer}; - pub fn serialize_vec(vec: &Vec<[u8; 32]>, serializer: S) -> Result - where - S: Serializer, - { - let hex_strings: Vec = vec.iter().map(|bytes| hex::encode(bytes)).collect(); - hex_strings.serialize(serializer) + #[derive(Deserialize)] + #[serde(untagged)] + enum Entry { + Record(OwnedOutputRecord), + /// State files written before owned outputs were recorded stored bare + /// hex keys here. No released writer ever populated that list, and a + /// bare key carries no outpoint, so such entries are dropped; the + /// records are rebuilt from the indexer on load + /// (`Scanner::sync_owned_outputs`). + Legacy(#[allow(dead_code)] String), } - pub fn deserialize_vec<'de, D>(deserializer: D) -> Result, D::Error> + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, { - let hex_strings: Vec = Vec::deserialize(deserializer)?; - let mut result = Vec::new(); - for hex_string in hex_strings { - let bytes = hex::decode(&hex_string) - .map_err(|e| serde::de::Error::custom(format!("Invalid hex string: {}", e)))?; - let bytes_array: [u8; 32] = bytes - .try_into() - .map_err(|_| serde::de::Error::custom("Expected 32 bytes"))?; - result.push(bytes_array); - } - Ok(result) + let entries: Vec = Vec::deserialize(deserializer)?; + Ok(entries + .into_iter() + .filter_map(|entry| match entry { + Entry::Record(record) => Some(record), + Entry::Legacy(_) => None, + }) + .collect()) } } @@ -53,15 +57,13 @@ pub struct ChangeSet { pub last_scanned_block_height: u64, /// The last block height that was scanned on most recent rescan pub last_scanned_block_height_rescan: u64, - /// Owned output pubkeys; used to check for spent outputs (hex encoded in JSON) + /// Wallet-owned outputs (full x-only key, label, spent status); the + /// scanner matches the oracle's spent-output prefixes against these. #[cfg_attr( feature = "serde", - serde( - serialize_with = "serde_hex::serialize_vec", - deserialize_with = "serde_hex::deserialize_vec" - ) + serde(default, deserialize_with = "owned_outputs_serde::deserialize") )] - pub owned_outputs: Vec<[u8; 32]>, + pub owned_outputs: Vec, /// Secret scan key (hex encoded) - needed to reconstruct the indexer pub secret_scan_hex: Option, /// Public spend key (hex encoded) - needed to reconstruct the indexer @@ -85,10 +87,16 @@ impl Merge for ChangeSet { self.last_scanned_block_height_rescan = other.last_scanned_block_height_rescan; } - // Merge owned_outputs (deduplicate) + // Merge owned_outputs by outpoint; the newer record wins (it may carry + // a spend the older one did not). for output in other.owned_outputs { - if !self.owned_outputs.contains(&output) { - self.owned_outputs.push(output); + match self + .owned_outputs + .iter_mut() + .find(|existing| existing.outpoint == output.outpoint) + { + Some(existing) => *existing = output, + None => self.owned_outputs.push(output), } } diff --git a/blindbit-lib/src/scanner/mod.rs b/blindbit-lib/src/scanner/mod.rs index c84fe07..4f3eef1 100644 --- a/blindbit-lib/src/scanner/mod.rs +++ b/blindbit-lib/src/scanner/mod.rs @@ -19,7 +19,7 @@ pub use config::ScannerConfig; pub use electrum_index::{ScriptHashEntry, SpHistoryEntry, WalletElectrumIndex, electrum_scripthash, electrum_status}; pub use p2p::broadcast_tx; pub use scanner::Scanner; -pub use types::{BlockIdentifierDisplay, OwnedOutput}; +pub use types::{BlockIdentifierDisplay, OwnedOutput, OwnedOutputRecord}; // Re-export load_scanner function when serde feature is enabled #[cfg(feature = "serde")] diff --git a/blindbit-lib/src/scanner/owned_outputs_tests.rs b/blindbit-lib/src/scanner/owned_outputs_tests.rs new file mode 100644 index 0000000..282e652 --- /dev/null +++ b/blindbit-lib/src/scanner/owned_outputs_tests.rs @@ -0,0 +1,483 @@ +//! Owned-output bookkeeping through the live short receive path. +//! +//! Each test drives a real [`Scanner`] exactly as `scan_block_range` does for +//! one oracle block: `scan_short_block_data` decides whether the block is worth +//! fetching, and only if it is, the full block goes through +//! `apply_matched_block` + `sync_owned_outputs` and becomes a checkpoint. The +//! P2P fetch is the only step replaced (the block is handed in directly). + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::OnceLock; + +use bdk_sp::bitcoin::key::Secp256k1; +use bdk_sp::hashes::get_label_tweak; +use bdk_sp::receive::get_silentpayment_pubkey; +use bitcoin::absolute::LockTime; +use bitcoin::hashes::Hash; +use bitcoin::key::TweakedPublicKey; +use bitcoin::secp256k1::{PublicKey, SecretKey}; +use bitcoin::transaction::Version; +use bitcoin::{ + Amount, Block, BlockHash, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, + Witness, XOnlyPublicKey, +}; +use bitcoin_rev::Network; +use indexer::bdk_chain::local_chain::LocalChain; +use indexer::bdk_chain::{BlockId, CanonicalizationParams}; +use tonic::transport::Channel; + +use super::Scanner; +use crate::oracle_grpc::{BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem}; +use crate::scanner::OwnedOutputRecord; + +const RECEIVE_HEIGHT: u32 = 100; +const SPEND_HEIGHT: u32 = 200; + +fn oracle_client() -> crate::OracleServiceClient { + static RUNTIME: OnceLock = OnceLock::new(); + let runtime = RUNTIME.get_or_init(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime for the lazy tonic channel") + }); + let _guard = runtime.enter(); + crate::OracleServiceClient::new(Channel::from_static("http://[::1]:50051").connect_lazy()) +} + +fn state_file(tag: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "blindbit-owned-outputs-{tag}-{}.json", + std::process::id() + )) +} + +fn secret(byte: u8) -> SecretKey { + SecretKey::from_slice(&[byte; 32]).expect("valid secret") +} + +fn keys() -> (SecretKey, PublicKey) { + let secp = Secp256k1::new(); + (secret(0x11), secret(0x22).public_key(&secp)) +} + +fn scanner(tag: &str, max_label_num: u32) -> Scanner { + let (scan_sk, spend_pk) = keys(); + let socket: SocketAddr = "127.0.0.1:8333".parse().expect("socket address"); + Scanner::new( + oracle_client(), + socket, + scan_sk, + spend_pk, + max_label_num, + state_file(tag), + Network::Regtest, + ) +} + +/// The tweak (`input_hash * A`) the oracle would serve for a payment. Any +/// point works: the receiver only ever sees this value. +fn served_tweak(byte: u8) -> PublicKey { + secret(byte).public_key(&Secp256k1::new()) +} + +/// `P_0` for the served tweak, plus `label_m * G` when labelled. +fn sp_output_key(tweak: &PublicKey, label: Option) -> XOnlyPublicKey { + let (scan_sk, spend_pk) = keys(); + let shared = bdk_sp::compute_shared_secret(&scan_sk, tweak); + let p0 = get_silentpayment_pubkey(&spend_pk, &shared, 0, None); + let key = match label { + None => p0, + Some(m) => { + let label_sk = SecretKey::from_slice(&get_label_tweak(scan_sk, m).to_be_bytes()) + .expect("label tweak is a valid scalar"); + p0.combine(&label_sk.public_key(&Secp256k1::new())) + .expect("labelled key") + } + }; + key.x_only_public_key().0 +} + +fn foreign_key(byte: u8) -> XOnlyPublicKey { + secret(byte) + .public_key(&Secp256k1::new()) + .x_only_public_key() + .0 +} + +fn p2tr(key: XOnlyPublicKey) -> ScriptBuf { + ScriptBuf::new_p2tr_tweaked(TweakedPublicKey::dangerous_assume_tweaked(key)) +} + +fn tx(inputs: Vec, outputs: Vec<(XOnlyPublicKey, u64)>) -> Transaction { + Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: inputs + .into_iter() + .map(|previous_output| TxIn { + previous_output, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }) + .collect(), + output: outputs + .into_iter() + .map(|(key, sat)| TxOut { + value: Amount::from_sat(sat), + script_pubkey: p2tr(key), + }) + .collect(), + } +} + +fn block(height: u32, txs: Vec) -> Block { + let coinbase = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes(height.to_le_bytes().to_vec()), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(312_500_000), + script_pubkey: ScriptBuf::from_bytes(vec![0x6a]), + }], + }; + let mut txdata = vec![coinbase]; + txdata.extend(txs); + let mut block = Block { + header: bitcoin::block::Header { + version: bitcoin::block::Version::TWO, + prev_blockhash: BlockHash::all_zeros(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: 1_713_571_767 + height, + bits: bitcoin::CompactTarget::from_consensus(0x207f_ffff), + nonce: height, + }, + txdata, + }; + block.header.merkle_root = block.compute_merkle_root().expect("non-empty block"); + block +} + +/// The oracle's `ComputeIndexTxItem`: wire-order txid, tweak, and the 8-byte +/// prefix of every taproot output key. +fn item(tx: &Transaction, tweak: PublicKey) -> ComputeIndexTxItem { + let mut txid = tx.compute_txid().to_byte_array(); + txid.reverse(); + ComputeIndexTxItem { + txid: txid.to_vec(), + tweak: tweak.serialize().to_vec(), + outputs_short: tx + .output + .iter() + .flat_map(|o| o.script_pubkey.as_bytes()[2..10].to_vec()) + .collect(), + } +} + +/// One block of `scan_block_range`, minus the P2P fetch. Returns whether the +/// scanner decided to fetch (and therefore apply) the block. +fn process_block( + scanner: &mut Scanner, + block: &Block, + height: u32, + items: Vec, + spent_prefixes: &[XOnlyPublicKey], +) -> bool { + let mut hash = block.block_hash().to_byte_array(); + hash.reverse(); + let short = BlockScanDataShortResponse { + block_identifier: Some(BlockIdentifier { + block_hash: hash.to_vec(), + block_height: height as u64, + }), + comp_index: items, + spent_outputs: spent_prefixes + .iter() + .flat_map(|key| key.serialize()[..8].to_vec()) + .collect(), + }; + let Some(probable_match) = scanner.scan_short_block_data(short).expect("short scan") else { + return false; + }; + scanner.apply_matched_block(block, &probable_match, height); + scanner.sync_owned_outputs(); + scanner.block_checkpoints.insert(height, block.block_hash()); + scanner + .stage + .block_checkpoints + .insert(height, block.block_hash()); + true +} + +/// The confirmed balance exactly as `scan_block_range` computes it. +fn balance(scanner: &Scanner) -> u64 { + let (&height, &hash) = scanner + .block_checkpoints + .iter() + .next_back() + .expect("genesis checkpoint"); + let outpoints: Vec<(u32, OutPoint)> = scanner + .internal_indexer + .index() + .by_shared_secret + .keys() + .map(|op| (op.vout, *op)) + .collect(); + let chain = LocalChain::from_blocks(scanner.block_checkpoints.clone()).expect("chain"); + scanner + .internal_indexer + .graph() + .balance( + &chain, + BlockId { height, hash }, + CanonicalizationParams::default(), + outpoints, + |_, _| true, + ) + .confirmed + .to_sat() +} + +fn record(scanner: &Scanner, outpoint: OutPoint) -> OwnedOutputRecord { + scanner + .owned_outputs() + .find(|r| r.outpoint == outpoint) + .cloned() + .unwrap_or_else(|| panic!("no owned-output record for {outpoint}")) +} + +/// Receives 50_000 sat to the unlabelled address at `RECEIVE_HEIGHT`. +fn receive(scanner: &mut Scanner) -> (OutPoint, XOnlyPublicKey) { + let tweak = served_tweak(0x33); + let ours = sp_output_key(&tweak, None); + let recv = tx( + vec![OutPoint::new(Txid::from_byte_array([0x44; 32]), 0)], + vec![(foreign_key(0x55), 10_000), (ours, 50_000)], + ); + let fetched = process_block( + scanner, + &block(RECEIVE_HEIGHT, vec![recv.clone()]), + RECEIVE_HEIGHT, + vec![item(&recv, tweak)], + &[], + ); + assert!(fetched, "receive block must be fetched"); + (OutPoint::new(recv.compute_txid(), 1), ours) +} + +/// A spend of `outpoint` paying only a foreign key: no output comes back to +/// the wallet, so only the spent-output prefix can make the scanner fetch it. +fn spend_of(outpoint: OutPoint) -> Transaction { + tx(vec![outpoint], vec![(foreign_key(0x66), 49_000)]) +} + +#[test] +fn spend_in_later_block_marks_output_spent_and_balance_drops() { + let mut scanner = scanner("spend", 0); + let (outpoint, key) = receive(&mut scanner); + + let rec = record(&scanner, outpoint); + assert_eq!(rec.pubkey, key, "full x-only key recorded"); + assert_eq!((rec.amount_sat, rec.height), (50_000, RECEIVE_HEIGHT)); + assert!(!rec.is_spent()); + assert_eq!(balance(&scanner), 50_000); + + // A block that spends nothing of ours is not fetched. + let unrelated = tx( + vec![OutPoint::new(Txid::from_byte_array([0x77; 32]), 3)], + vec![(foreign_key(0x78), 1_000)], + ); + assert!(!process_block( + &mut scanner, + &block(150, vec![unrelated]), + 150, + vec![], + &[foreign_key(0x79)], + )); + + let spend = spend_of(outpoint); + let fetched = process_block( + &mut scanner, + &block(SPEND_HEIGHT, vec![spend.clone()]), + SPEND_HEIGHT, + // The spend has a (foreign) taproot output, so the oracle indexes it + // with some tweak that does not match us. + vec![item(&spend, served_tweak(0x99))], + &[foreign_key(0x7a), key], + ); + assert!( + fetched, + "spent prefix of an owned output must fetch the block" + ); + + let rec = record(&scanner, outpoint); + assert_eq!(rec.spent_by, Some(spend.compute_txid())); + assert_eq!(rec.spent_height, Some(SPEND_HEIGHT)); + assert_eq!(balance(&scanner), 0, "spent coin must leave the balance"); +} + +#[test] +fn foreign_output_with_colliding_prefix_marks_nothing() { + let mut scanner = scanner("collision", 0); + let (outpoint, key) = receive(&mut scanner); + + // Someone else's output shares our key's 8-byte prefix and is spent: the + // oracle serves exactly our prefix, but the spending tx consumes a + // different outpoint. + let foreign_outpoint = OutPoint::new(Txid::from_byte_array([0x88; 32]), 0); + let foreign_spend = spend_of(foreign_outpoint); + let fetched = process_block( + &mut scanner, + &block(SPEND_HEIGHT, vec![foreign_spend]), + SPEND_HEIGHT, + vec![], + &[key], + ); + assert!(fetched, "a prefix hit must still fetch the block to check"); + + let rec = record(&scanner, outpoint); + assert!( + !rec.is_spent(), + "prefix collision alone must not mark: {rec:?}" + ); + assert_eq!(scanner.owned_outputs().count(), 1); + assert_eq!(balance(&scanner), 50_000); +} + +#[test] +fn labelled_receive_records_its_label() { + let mut scanner = scanner("labels", 5); + let tweak = served_tweak(0x34); + let unlabelled = sp_output_key(&tweak, None); + let change = sp_output_key(&tweak, Some(0)); + let labelled = sp_output_key(&tweak, Some(3)); + // Three payments to one wallet in one tx: k = 0 is taken by whichever + // candidate matches first, so use one tx per key to keep k = 0 each. + let txs: Vec = [(unlabelled, 0x01u8), (change, 0x02), (labelled, 0x03)] + .iter() + .map(|&(key, salt)| { + tx( + vec![OutPoint::new(Txid::from_byte_array([salt; 32]), 0)], + vec![(key, 10_000 * salt as u64)], + ) + }) + .collect(); + let items = txs.iter().map(|t| item(t, tweak)).collect(); + assert!(process_block( + &mut scanner, + &block(RECEIVE_HEIGHT, txs.clone()), + RECEIVE_HEIGHT, + items, + &[], + )); + + let labels: Vec<(XOnlyPublicKey, Option)> = txs + .iter() + .map(|t| { + let r = record(&scanner, OutPoint::new(t.compute_txid(), 0)); + (r.pubkey, r.label) + }) + .collect(); + assert_eq!( + labels, + vec![(unlabelled, None), (change, Some(0)), (labelled, Some(3))] + ); +} + +#[cfg(feature = "serde")] +#[test] +fn owned_outputs_labels_and_spends_survive_restart() { + let tag = "restart"; + let path = state_file(tag); + let _ = std::fs::remove_file(&path); + + let mut scanner = scanner(tag, 2); + let tweak = served_tweak(0x35); + let change_key = sp_output_key(&tweak, Some(0)); + let label_key = sp_output_key(&tweak, Some(2)); + let recv_change = tx( + vec![OutPoint::new(Txid::from_byte_array([0x0a; 32]), 0)], + vec![(change_key, 20_000)], + ); + let recv_label = tx( + vec![OutPoint::new(Txid::from_byte_array([0x0b; 32]), 0)], + vec![(label_key, 30_000)], + ); + assert!(process_block( + &mut scanner, + &block( + RECEIVE_HEIGHT, + vec![recv_change.clone(), recv_label.clone()] + ), + RECEIVE_HEIGHT, + vec![item(&recv_change, tweak), item(&recv_label, tweak)], + &[], + )); + let change_op = OutPoint::new(recv_change.compute_txid(), 0); + let label_op = OutPoint::new(recv_label.compute_txid(), 0); + + // Spend the change output before the restart. + let spend = spend_of(change_op); + assert!(process_block( + &mut scanner, + &block(SPEND_HEIGHT, vec![spend.clone()]), + SPEND_HEIGHT, + vec![], + &[change_key], + )); + assert_eq!(balance(&scanner), 30_000); + let before: Vec<_> = scanner.owned_outputs().cloned().collect(); + assert_eq!(before.len(), 2); + + scanner.save_to_file(&path).expect("save state"); + let json: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + let persisted = json["owned_outputs"] + .as_array() + .expect("owned_outputs array"); + assert_eq!(persisted.len(), 2, "state file lists every owned output"); + let label_entry = persisted + .iter() + .find(|e| e["outpoint"] == label_op.to_string()) + .expect("labelled output persisted"); + assert_eq!(label_entry["label"], 2); + assert_eq!(label_entry["pubkey"], label_key.to_string()); + + let changeset = Scanner::load_from_file(&path).expect("load state"); + let socket: SocketAddr = "127.0.0.1:8333".parse().unwrap(); + let mut restored = Scanner::from_changeset( + oracle_client(), + socket, + changeset, + path.clone(), + Network::Regtest, + ) + .expect("restore"); + let _ = std::fs::remove_file(&path); + + let after: Vec<_> = restored.owned_outputs().cloned().collect(); + assert_eq!(after, before, "records survive the restart unchanged"); + assert_eq!(balance(&restored), 30_000, "restored balance"); + + // Spend detection keeps working after the restart. + let spend2 = spend_of(label_op); + assert!(process_block( + &mut restored, + &block(300, vec![spend2.clone()]), + 300, + vec![], + &[label_key], + )); + assert_eq!( + record(&restored, label_op).spent_by, + Some(spend2.compute_txid()) + ); + assert_eq!(balance(&restored), 0); +} diff --git a/blindbit-lib/src/scanner/scanner.rs b/blindbit-lib/src/scanner/scanner.rs index 8e5e65f..f20d892 100644 --- a/blindbit-lib/src/scanner/scanner.rs +++ b/blindbit-lib/src/scanner/scanner.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::net::SocketAddr; #[cfg(feature = "serde")] use std::path::Path; @@ -7,7 +7,7 @@ use std::sync::Arc; use bitcoin::BlockHash; use bitcoin::Network as BTCNetwork; -use bitcoin::Txid; +use bitcoin::{OutPoint, Txid}; use bitcoin::hashes::Hash; use bitcoin::secp256k1::{PublicKey, SecretKey}; use bitcoin_rev::Network; @@ -22,6 +22,7 @@ use indexer::bdk_chain::ConfirmationBlockTime; use super::changeset::ChangeSet; use super::config::ScannerConfig; +use super::types::OwnedOutputRecord; use super::electrum_index::{ScriptHashEntry, SpHistoryEntry, WalletElectrumIndex, electrum_scripthash}; use super::ScannerError; @@ -59,9 +60,14 @@ pub struct Scanner { /// the last block height that was scanned on most recent rescan pub(crate) last_scanned_block_height_rescan: u64, - /// owned output pubkeys; used to check for spent outputs - // todo: should this be a hashmap instead of a vector? - pub(crate) owned_outputs: Vec<[u8; 32]>, + /// Wallet-owned outputs, keyed by outpoint. Maintained by + /// [`Scanner::sync_owned_outputs`] from the indexer after every applied + /// block and on restore; persisted as the state file's `owned_outputs`. + pub(crate) owned_outputs: BTreeMap, + + /// 8-byte x-only key prefix -> *unspent* owned outpoints with that prefix. + /// Lets the per-block spent-output check run in O(spent prefixes). + pub(crate) owned_prefixes: HashMap<[u8; 8], Vec>, /// Staged changes that can be persisted pub(crate) stage: ChangeSet, @@ -143,7 +149,8 @@ impl Scanner { notify_spent_outpoints: broadcast::channel(100).0, last_scanned_block_height: 0, last_scanned_block_height_rescan: 0, - owned_outputs: vec![], + owned_outputs: BTreeMap::new(), + owned_prefixes: HashMap::new(), stage, state_file, network, @@ -394,12 +401,95 @@ impl Scanner { self.notify_probabilistic_matches.subscribe() } - /// add an owned output pubkey - /// if already exists nothing happens - pub fn add_owned_output(&mut self, pubkey: [u8; 32]) { - // todo: should this be a hashmap instead of a vector? - if !self.owned_outputs.contains(&pubkey) { - self.owned_outputs.push(pubkey); + /// All wallet-owned outputs found so far (spent and unspent), in outpoint + /// order. + pub fn owned_outputs(&self) -> impl Iterator { + self.owned_outputs.values() + } + + /// Bring [`Self::owned_outputs`] in line with the indexer. + /// + /// Every output the indexer has matched (`by_script`, full BIP-352 match) + /// gets a record with its full x-only key and label. A record is marked + /// spent only when a *confirmed* transaction in the wallet graph has an + /// input whose `previous_output` is exactly that outpoint; the oracle's + /// 8-byte spent prefixes only decide which blocks get fetched and never + /// mark anything themselves. New or changed records are staged. + pub(crate) fn sync_owned_outputs(&mut self) { + let index = self.internal_indexer.index(); + let graph = self.internal_indexer.graph(); + + let labels: HashMap> = index + .by_label + .iter() + .map(|(label, outpoint)| (*outpoint, *label)) + .collect(); + let anchor_height = |txid: Txid| { + graph + .get_tx_node(txid) + .and_then(|node| node.anchors.iter().map(|a| a.block_id.height).min()) + }; + + let mut changed: Vec = Vec::new(); + + for (xonly, outpoint) in index.by_xonly() { + if self.owned_outputs.contains_key(outpoint) { + continue; + } + let Some(txout) = graph.get_txout(*outpoint) else { + continue; + }; + let Some(height) = anchor_height(outpoint.txid) else { + continue; + }; + self.owned_outputs.insert( + *outpoint, + OwnedOutputRecord { + outpoint: *outpoint, + pubkey: xonly, + amount_sat: txout.value.to_sat(), + height, + label: labels.get(outpoint).copied().flatten(), + spent_by: None, + spent_height: None, + }, + ); + changed.push(*outpoint); + } + + for node in graph.full_txs() { + let Some(spent_height) = node.anchors.iter().map(|a| a.block_id.height).min() else { + continue; + }; + for input in &node.tx.input { + let Some(record) = self.owned_outputs.get_mut(&input.previous_output) else { + continue; + }; + if record.spent_by == Some(node.txid) && record.spent_height == Some(spent_height) { + continue; + } + record.spent_by = Some(node.txid); + record.spent_height = Some(spent_height); + changed.push(input.previous_output); + } + } + + if !changed.is_empty() { + self.stage.merge(ChangeSet { + owned_outputs: changed + .iter() + .filter_map(|outpoint| self.owned_outputs.get(outpoint).cloned()) + .collect(), + ..ChangeSet::default() + }); + } + + self.owned_prefixes.clear(); + for record in self.owned_outputs.values().filter(|r| !r.is_spent()) { + self.owned_prefixes + .entry(record.short_pubkey()) + .or_default() + .push(record.outpoint); } } @@ -481,7 +571,7 @@ impl Scanner { } changeset.last_scanned_block_height = self.last_scanned_block_height; changeset.last_scanned_block_height_rescan = self.last_scanned_block_height_rescan; - changeset.owned_outputs = self.owned_outputs.clone(); + changeset.owned_outputs = self.owned_outputs.values().cloned().collect(); let json = serde_json::to_string_pretty(&changeset)?; std::fs::write(path, json)?; @@ -556,16 +646,25 @@ impl Scanner { let _public_spend = PublicKey::from_slice(&public_spend_bytes)?; // Reconstruct the indexer from the changeset (this restores the graph and transaction data) - let mut indexer = SpIndexerV2::try_from(changeset.indexer.clone()) - .map_err(|e| format!("Failed to reconstruct indexer from changeset: {:?}", e))?; + let (Some(indexer_scan_sk), Some(indexer_spend_pk)) = + (changeset.indexer.scan_sk, changeset.indexer.spend_pk) + else { + return Err("Failed to reconstruct indexer from changeset: keys missing".into()); + }; + let mut indexer = SpIndexerV2::new(indexer_scan_sk, indexer_spend_pk); - // Regenerate labels + // Regenerate labels BEFORE applying the changeset: `apply_changeset` + // re-runs the BIP-352 scan (`index_tx`) over every stored transaction, + // and `load_from_file` blanks the persisted `label_lookup`. Labels added + // afterwards would leave every labelled output (change included, m = 0) + // out of the restored index. // ignore change set and be aligned with max_label_num let max_label_num = changeset.max_label_num; _ = indexer.add_label(0); for i in 1..=max_label_num { _ = indexer.add_label(i); } + indexer.apply_changeset(changeset.indexer.clone()); // Reconstruct block checkpoints from the changeset let mut block_checkpoints = changeset.block_checkpoints.clone(); @@ -583,7 +682,13 @@ impl Scanner { .get_address(convert_network(network)) .to_string(); - Ok(Self { + let owned_outputs = changeset + .owned_outputs + .iter() + .map(|record| (record.outpoint, record.clone())) + .collect(); + + let mut scanner = Self { client, p2p_peer: p2p_socket_addr, internal_indexer: indexer, @@ -593,7 +698,8 @@ impl Scanner { notify_spent_outpoints: broadcast::channel(100).0, last_scanned_block_height: changeset.last_scanned_block_height, last_scanned_block_height_rescan: changeset.last_scanned_block_height_rescan, - owned_outputs: changeset.owned_outputs.clone(), + owned_outputs, + owned_prefixes: HashMap::new(), stage: changeset, state_file: state_file, network: network, @@ -604,7 +710,11 @@ impl Scanner { idx.sp_labels = (0..=max_label_num).collect(); idx })), - }) + }; + // Records older state files never had, spends already in the graph, + // and the prefix lookup, all derived from the restored indexer. + scanner.sync_owned_outputs(); + Ok(scanner) } } diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index 8ee15ce..a083ac9 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -173,38 +173,14 @@ impl Scanner { .fetch_block_with_retry(&mut p2p_conn, block_hash, block_identifier.block_height) .await?; - // build partial secret hashmap, only populate with txids and secrets where we - // suspect matches, skip the rest - let mut partial_secrets = - HashMap::with_capacity(probable_match.matched_txs.len()); - - for tx in &block.txdata { - if probable_match.spent { - // todo: we will need to look at all spent outpoints in this - // block and find the relevant txids for spent - } - - for (txid_arr, tweak) in &probable_match.matched_txs { - // this check should be optimised to a map lookup on all items - let mut item_txid = *txid_arr; - item_txid.reverse(); - - if Txid::from_byte_array(item_txid) != tx.compute_txid() { - continue; - } - - let txid = byte_array_to_txid(txid_arr); - - partial_secrets.insert(txid, *tweak); - } - } // Apply block to indexer and stage the changes - let indexer_changes = self.internal_indexer.apply_block_relevant( + self.apply_matched_block( &block, - partial_secrets, + &probable_match, block_identifier.block_height as u32, ); - self.stage.indexer.merge(indexer_changes); + // Record newly found outputs and confirmed spends of owned ones. + self.sync_owned_outputs(); // Update block checkpoints: only store blocks where we found something let block_height_u32 = block_identifier.block_height as u32; @@ -664,21 +640,32 @@ impl Scanner { } // Spent-output check — only when we have a valid block_hash to report. + // The oracle serves the first 8 bytes of each spent taproot output's + // x-only key. A prefix hit on an unspent owned output only makes the + // block worth fetching; the spend is confirmed (by exact outpoint) when + // the block is applied, so a foreign key sharing the prefix marks + // nothing. if let Some(block_hash) = block_hash_opt { let spent_outputs_count = block_data.spent_outputs.len() / 8; for i in 0..spent_outputs_count { - let spent_output = &block_data.spent_outputs[i * 8..(i + 1) * 8]; - for pubkey in &self.owned_outputs { - if pubkey[..8] == *spent_output { - probable_match.spent = true; - - tracing::info!(pubkey = %hex::encode(pubkey), "spent output detected"); - if self.notify_spent_outpoints.is_empty() { - continue; - } - if let Err(e) = self.notify_spent_outpoints.send(block_hash) { - tracing::warn!(error = ?e, "failed to send spent output notification"); - } + let prefix: [u8; 8] = block_data.spent_outputs[i * 8..(i + 1) * 8] + .try_into() + .expect("slice is 8 bytes"); + let Some(candidates) = self.owned_prefixes.get(&prefix) else { + continue; + }; + probable_match.spent = true; + for outpoint in candidates { + tracing::info!( + outpoint = %outpoint, + short_pubkey = %hex::encode(prefix), + "possible spend of owned output; fetching block to confirm" + ); + if self.notify_spent_outpoints.is_empty() { + continue; + } + if let Err(e) = self.notify_spent_outpoints.send(block_hash) { + tracing::warn!(error = ?e, "failed to send spent output notification"); } } } @@ -691,6 +678,50 @@ impl Scanner { } } + /// The live receive step after a probable match: hands the full block and + /// the matched transactions' tweaks to the external indexer, which does the + /// real BIP-352 scan, and stages what it found. + /// + /// `probable_match.matched_txs` carries each txid as the oracle serves it + /// (display order) together with the served tweak (`input_hash * A`), which + /// `apply_block_relevant` takes as its "partial secret" and finishes the ECDH + /// on itself. + /// + /// Spends need no extra step here: `apply_block_relevant` inserts every + /// transaction whose input spends an indexed outpoint into the wallet graph, + /// which is what the balance and [`Scanner::sync_owned_outputs`] read. + fn apply_matched_block( + &mut self, + block: &bitcoin::Block, + probable_match: &ProbableMatch, + height: u32, + ) { + // build partial secret hashmap, only populate with txids and secrets where we + // suspect matches, skip the rest + let mut partial_secrets = HashMap::with_capacity(probable_match.matched_txs.len()); + + for tx in &block.txdata { + for (txid_arr, tweak) in &probable_match.matched_txs { + // this check should be optimised to a map lookup on all items + let mut item_txid = *txid_arr; + item_txid.reverse(); + + if Txid::from_byte_array(item_txid) != tx.compute_txid() { + continue; + } + + let txid = byte_array_to_txid(txid_arr); + + partial_secrets.insert(txid, *tweak); + } + } + // Apply block to indexer and stage the changes + let indexer_changes = self + .internal_indexer + .apply_block_relevant(block, partial_secrets, height); + self.stage.indexer.merge(indexer_changes); + } + fn probabilistic_match( &mut self, item: &ComputeIndexTxItem, @@ -841,3 +872,7 @@ impl Scanner { #[cfg(test)] #[path = "bip352_vectors.rs"] mod bip352_vectors; + +#[cfg(test)] +#[path = "owned_outputs_tests.rs"] +mod owned_outputs_tests; diff --git a/blindbit-lib/src/scanner/types.rs b/blindbit-lib/src/scanner/types.rs index 50b6660..7ad8c96 100644 --- a/blindbit-lib/src/scanner/types.rs +++ b/blindbit-lib/src/scanner/types.rs @@ -1,7 +1,7 @@ use crate::oracle_grpc::BlockIdentifier; use bitcoin::absolute::Height; use bitcoin::secp256k1::PublicKey; -use bitcoin::{Amount, OutPoint, ScriptBuf}; +use bitcoin::{Amount, OutPoint, ScriptBuf, Txid, XOnlyPublicKey}; use indexer::v2::indexes::Label; /// Wrapper for `BlockIdentifier` that implements Display with hex formatting @@ -40,6 +40,48 @@ pub struct OwnedOutput { pub spent: Option, } +/// One wallet-owned Silent Payments output, as the scanner records and +/// persists it (the `owned_outputs` array of the state file). +/// +/// A record is created when the indexer confirms an output as ours (full +/// BIP-352 match in a fetched block) and is marked spent only when a +/// confirmed transaction in the wallet graph spends exactly this `outpoint`. +/// The oracle's 8-byte `spent_outputs` prefixes are used solely as a trigger to +/// fetch the block; they never mark a record on their own. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct OwnedOutputRecord { + /// The output itself (`txid:vout`). + pub outpoint: OutPoint, + /// Full x-only taproot output key (the key the oracle's spent-output + /// prefixes are taken from). + pub pubkey: XOnlyPublicKey, + /// Output value in satoshis. + pub amount_sat: u64, + /// Height of the block that confirmed the output. + pub height: u32, + /// BIP-352 label `m` the output matched; `None` for the unlabelled + /// address, `Some(0)` for change. + pub label: Option, + /// Confirmed transaction that spends this output, if any. + pub spent_by: Option, + /// Height of the block that confirmed `spent_by`. + pub spent_height: Option, +} + +impl OwnedOutputRecord { + /// The 8-byte prefix the oracle serves for a spent taproot output. + pub fn short_pubkey(&self) -> [u8; 8] { + let mut prefix = [0u8; 8]; + prefix.copy_from_slice(&self.pubkey.serialize()[..8]); + prefix + } + + pub fn is_spent(&self) -> bool { + self.spent_by.is_some() + } +} + /// `ProbableMatch` is a struct that contains a list of txids that are probable matches /// and a boolean indicating if a utxo might be spent pub(crate) struct ProbableMatch { From ad80ea45cdcb7cb45f6364781e73815c1db23631 Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 24 Sep 2026 11:20:40 +0000 Subject: [PATCH 03/10] test(scanner): pin how a range scan handles unscannable oracle stream messages Adds an in-process seam (BlockScanDataStream) between scan_block_range and the scan loop, with no behaviour change, and tests that drive a real Scanner through it. They fail on this commit: an empty-hash block mid-range is treated as an empty block (payments in it are lost and the scan reports success), height gaps and short streams are accepted, and any error status panics the process via unwrap(). Co-Authored-By: Claude Opus 5.5 (1M context) --- blindbit-lib/src/scanner/mod.rs | 2 + blindbit-lib/src/scanner/scanning.rs | 41 +- .../src/scanner/stream_safety_tests.rs | 363 ++++++++++++++++++ 3 files changed, 404 insertions(+), 2 deletions(-) create mode 100644 blindbit-lib/src/scanner/stream_safety_tests.rs diff --git a/blindbit-lib/src/scanner/mod.rs b/blindbit-lib/src/scanner/mod.rs index c84fe07..9c4aaca 100644 --- a/blindbit-lib/src/scanner/mod.rs +++ b/blindbit-lib/src/scanner/mod.rs @@ -7,6 +7,8 @@ mod load; mod p2p; mod scanner; mod scanning; +#[cfg(test)] +mod stream_safety_tests; mod types; mod utils; diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index 8ee15ce..19b2070 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -48,6 +48,26 @@ fn upsert_history_entry(history: &mut Vec, entry: ScriptHashEnt } } +/// The per-block message source of a block-range scan. +/// +/// Production scans read a tonic `Streaming`; tests feed messages and error +/// statuses in-process to exercise how the scanner reacts to what an oracle +/// can send mid-range. +pub(crate) trait BlockScanDataStream { + fn next_message( + &mut self, + ) -> impl Future, tonic::Status>> + Send; +} + +impl BlockScanDataStream for tonic::Streaming { + fn next_message( + &mut self, + ) -> impl Future, tonic::Status>> + Send + { + self.message() + } +} + impl Scanner { /// scan a block range for new utxos and spent outpoints pub async fn scan_block_range( @@ -63,13 +83,24 @@ impl Scanner { dustlimit: 0, cut_through: false, }); - let mut stream = self + let stream = self .client .stream_block_scan_data_short(request) .await .unwrap() .into_inner(); + self.scan_block_stream(start, end, stream).await + } + + /// Scan the per-block messages of one `StreamBlockScanDataShort` response + /// for the range `start..=end`. + pub(crate) async fn scan_block_stream( + &mut self, + start: u64, + end: u64, + mut stream: S, + ) -> Result<(), ScannerError> { // Stamp sp_start_height into the index so the Electrum server's // SP subscription response always uses the correct scan start key. // Only set it on the first call; watch_chain increments start each @@ -92,7 +123,7 @@ impl Scanner { let mut p2p_conn: Option = None; - while let Some(block_scan_data) = stream.message().await.unwrap() { + while let Some(block_scan_data) = stream.next_message().await.unwrap() { let Some(block_identifier) = block_scan_data.block_identifier.clone() else { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -486,6 +517,12 @@ impl Scanner { ) -> Result { const MAX_ATTEMPTS: u32 = 4; + // Tests serve full blocks in-process instead of over P2P. + #[cfg(test)] + if let Some(block) = super::stream_safety_tests::served_block(&block_hash) { + return Ok(block); + } + let mut last_err: Option = None; for attempt in 1..=MAX_ATTEMPTS { if p2p_conn.is_none() { diff --git a/blindbit-lib/src/scanner/stream_safety_tests.rs b/blindbit-lib/src/scanner/stream_safety_tests.rs new file mode 100644 index 0000000..aaa2135 --- /dev/null +++ b/blindbit-lib/src/scanner/stream_safety_tests.rs @@ -0,0 +1,363 @@ +//! How a range scan reacts to what an oracle stream can deliver mid-range. +//! +//! A block the scanner cannot vouch for must never be treated as "no +//! payments": a message with an empty or malformed block hash (what an oracle +//! answers for a height it has not indexed), a height other than the next one +//! expected, an error status, or a stream that ends early each stop the scan +//! with an error, and the scan height stays at the last block that was fully +//! processed so the next scan resumes there. +//! +//! Each test drives a real [`Scanner`] through `scan_block_stream` with +//! in-process messages. The P2P block fetch is the only step replaced: full +//! blocks are served from [`served_block`]. + +use std::collections::{HashMap, VecDeque}; +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Mutex; + +use bdk_sp::bitcoin::key::Secp256k1; +use bdk_sp::receive::get_silentpayment_pubkey; +use bitcoin::absolute::LockTime; +use bitcoin::block::{Header, Version as BlockVersion}; +use bitcoin::hashes::Hash; +use bitcoin::key::TweakedPublicKey; +use bitcoin::secp256k1::{PublicKey, SecretKey}; +use bitcoin::transaction::Version; +use bitcoin::{ + Amount, Block, BlockHash, CompactTarget, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, + TxMerkleNode, TxOut, Txid, Witness, +}; +use bitcoin_rev::Network; +use tonic::transport::Channel; + +use super::Scanner; +use super::scanning::BlockScanDataStream; +use crate::oracle_grpc::{BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem}; + +static SERVED_BLOCKS: Mutex>> = Mutex::new(None); + +/// The full block a test serves for `hash` in place of a P2P fetch. +pub(super) fn served_block(hash: &BlockHash) -> Option { + SERVED_BLOCKS + .lock() + .unwrap() + .as_ref() + .and_then(|blocks| blocks.get(hash).cloned()) +} + +fn serve(block: &Block) { + SERVED_BLOCKS + .lock() + .unwrap() + .get_or_insert_with(HashMap::new) + .insert(block.block_hash(), block.clone()); +} + +/// In-process stand-in for the oracle's `StreamBlockScanDataShort` stream. +struct TestStream(VecDeque>); + +impl TestStream { + fn new(items: Vec>) -> Self { + Self(items.into()) + } +} + +impl BlockScanDataStream for TestStream { + fn next_message( + &mut self, + ) -> impl Future, tonic::Status>> + Send + { + std::future::ready(self.0.pop_front().transpose()) + } +} + +fn run(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime") + .block_on(future) +} + +fn secret(byte: u8) -> SecretKey { + SecretKey::from_slice(&[byte; 32]).expect("valid secret") +} + +fn keys() -> (SecretKey, PublicKey) { + (secret(0x11), secret(0x22).public_key(&Secp256k1::new())) +} + +struct TempState(PathBuf); + +impl Drop for TempState { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// A scanner whose oracle client points at `oracle` (never contacted unless a +/// test calls `scan_block_range`). Must be called inside a Tokio runtime. +fn scanner(tag: &str, oracle: &'static str) -> (Scanner, TempState) { + let (scan_sk, spend_pk) = keys(); + let state = std::env::temp_dir().join(format!( + "blindbit-stream-safety-{tag}-{}.json", + std::process::id() + )); + let client = crate::OracleServiceClient::new(Channel::from_static(oracle).connect_lazy()); + let socket: SocketAddr = "127.0.0.1:1".parse().expect("socket address"); + let scanner = Scanner::new(client, socket, scan_sk, spend_pk, 0, state.clone(), Network::Regtest); + (scanner, TempState(state)) +} + +/// One block at `height` holding a single silent payment to the test wallet, +/// plus the oracle message that describes it. +struct PaymentBlock { + height: u64, + block: Block, + message: BlockScanDataShortResponse, +} + +fn payment_block(height: u64) -> PaymentBlock { + let (scan_sk, spend_pk) = keys(); + // The tweak (`input_hash * A`) the oracle would serve; any point works. + let tweak = secret(0x40u8.wrapping_add(height as u8)).public_key(&Secp256k1::new()); + let shared = bdk_sp::compute_shared_secret(&scan_sk, &tweak); + let output_key = get_silentpayment_pubkey(&spend_pk, &shared, 0, None) + .x_only_public_key() + .0; + + let coinbase = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes(height.to_le_bytes().to_vec()), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![], + }; + let payment = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { + txid: Txid::from_byte_array([height as u8; 32]), + vout: 0, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(10_000 + height), + script_pubkey: ScriptBuf::new_p2tr_tweaked(TweakedPublicKey::dangerous_assume_tweaked( + output_key, + )), + }], + }; + let payment_txid = payment.compute_txid(); + + let mut block = Block { + header: Header { + version: BlockVersion::ONE, + prev_blockhash: BlockHash::all_zeros(), + merkle_root: TxMerkleNode::all_zeros(), + time: height as u32, + bits: CompactTarget::from_consensus(0x207f_ffff), + nonce: 0, + }, + txdata: vec![coinbase, payment], + }; + block.header.merkle_root = block.compute_merkle_root().expect("non-empty block"); + + // The oracle serves hashes and txids in display order. + let mut block_hash = block.block_hash().to_byte_array(); + block_hash.reverse(); + let mut txid = payment_txid.to_byte_array(); + txid.reverse(); + + let message = BlockScanDataShortResponse { + block_identifier: Some(BlockIdentifier { + block_hash: block_hash.to_vec(), + block_height: height, + }), + comp_index: vec![ComputeIndexTxItem { + txid: txid.to_vec(), + tweak: tweak.serialize().to_vec(), + outputs_short: output_key.serialize()[..8].to_vec(), + }], + spent_outputs: vec![], + }; + serve(&block); + PaymentBlock { height, block, message } +} + +/// What an oracle sends today for a height it has not indexed: OK, with an +/// empty block hash and no data. +fn unindexed_message(height: u64) -> BlockScanDataShortResponse { + BlockScanDataShortResponse { + block_identifier: Some(BlockIdentifier { + block_hash: vec![], + block_height: height, + }), + comp_index: vec![], + spent_outputs: vec![], + } +} + +fn owned_outputs(scanner: &Scanner) -> usize { + scanner.internal_indexer.index().by_shared_secret.len() +} + +/// The regression the testkit's `unindexed-ok-empty` mutation check proved: +/// an unindexed height mid-range was taken as an empty block, the scan +/// advanced past it and reported success, and the payments in it were never +/// found. Now the scan stops before that block, and the next scan (once the +/// oracle serves it) finds every payment. +#[test] +fn empty_hash_block_mid_range_stops_scan_and_next_scan_finds_its_outputs() { + run(async { + let (mut scanner, _state) = scanner("empty-hash", "http://127.0.0.1:1"); + let a = payment_block(1100); + let b = payment_block(1101); + let c = payment_block(1102); + assert_eq!(a.block.block_hash(), served_block(&a.block.block_hash()).unwrap().block_hash()); + + let first = TestStream::new(vec![ + Ok(a.message.clone()), + Ok(unindexed_message(b.height)), + Ok(c.message.clone()), + ]); + let err = scanner + .scan_block_stream(a.height, c.height, first) + .await + .expect_err("a block without a valid hash must stop the scan"); + let msg = err.to_string(); + assert!(msg.contains(&b.height.to_string()), "error names the height: {msg}"); + assert_eq!(scanner.get_last_scanned_block_height(), a.height); + assert_eq!(owned_outputs(&scanner), 1, "only the block before the gap is applied"); + + // Resume where the scanner says, now that the oracle serves the block. + let resume = scanner.get_last_scanned_block_height() + 1; + assert_eq!(resume, b.height); + let second = TestStream::new(vec![Ok(b.message.clone()), Ok(c.message.clone())]); + scanner + .scan_block_stream(resume, c.height, second) + .await + .expect("resumed scan succeeds"); + assert_eq!(scanner.get_last_scanned_block_height(), c.height); + assert_eq!(owned_outputs(&scanner), 3, "no payment is lost"); + }); +} + +#[test] +fn malformed_block_hash_stops_scan() { + run(async { + let (mut scanner, _state) = scanner("short-hash", "http://127.0.0.1:1"); + let a = payment_block(1200); + let mut bad = payment_block(1201).message; + bad.block_identifier.as_mut().unwrap().block_hash.truncate(31); + let mut zero = payment_block(1201).message; + zero.block_identifier.as_mut().unwrap().block_hash = vec![0; 32]; + + for bad in [bad, zero] { + let stream = TestStream::new(vec![Ok(a.message.clone()), Ok(bad)]); + scanner + .scan_block_stream(a.height, a.height + 1, stream) + .await + .expect_err("a malformed block hash must stop the scan"); + assert_eq!(scanner.get_last_scanned_block_height(), a.height); + } + }); +} + +#[test] +fn skipped_height_stops_scan() { + run(async { + let (mut scanner, _state) = scanner("skipped", "http://127.0.0.1:1"); + let a = payment_block(1300); + let c = payment_block(1302); + let stream = TestStream::new(vec![Ok(a.message.clone()), Ok(c.message.clone())]); + let err = scanner + .scan_block_stream(a.height, c.height, stream) + .await + .expect_err("a height gap must stop the scan"); + assert!(err.to_string().contains("1301"), "error names the expected height: {err}"); + assert_eq!(scanner.get_last_scanned_block_height(), a.height); + assert_eq!(owned_outputs(&scanner), 1, "the block after the gap is not applied"); + }); +} + +#[test] +fn stream_starting_at_wrong_height_stops_scan() { + run(async { + let (mut scanner, _state) = scanner("wrong-start", "http://127.0.0.1:1"); + scanner.update_last_scanned_block_height(1399); + let b = payment_block(1401); + let stream = TestStream::new(vec![Ok(b.message.clone())]); + scanner + .scan_block_stream(1400, 1401, stream) + .await + .expect_err("the first block must be the requested start"); + assert_eq!(scanner.get_last_scanned_block_height(), 1399); + assert_eq!(owned_outputs(&scanner), 0); + }); +} + +/// A fixed oracle ends the stream with NOT_FOUND at an unindexed height; a +/// transient failure ends it with UNAVAILABLE. Neither may panic the process, +/// and neither may advance the scan height past the last full block. +#[test] +fn error_status_mid_stream_is_a_clean_error() { + run(async { + let statuses = [ + tonic::Status::not_found("height 1501 is not indexed"), + tonic::Status::unavailable("connection reset"), + ]; + for (i, status) in statuses.into_iter().enumerate() { + let code = status.code(); + let (mut scanner, _state) = scanner(&format!("status-{i}"), "http://127.0.0.1:1"); + let a = payment_block(1500); + let stream = TestStream::new(vec![Ok(a.message.clone()), Err(status)]); + let err = scanner + .scan_block_stream(a.height, a.height + 1, stream) + .await + .expect_err("an error status must end the scan with an error"); + assert!(err.to_string().contains("1501"), "{code:?}: error names the height: {err}"); + assert_eq!(scanner.get_last_scanned_block_height(), a.height, "{code:?}"); + assert_eq!(owned_outputs(&scanner), 1, "{code:?}"); + } + }); +} + +#[test] +fn stream_ending_before_requested_end_is_an_error() { + run(async { + let (mut scanner, _state) = scanner("short-stream", "http://127.0.0.1:1"); + let a = payment_block(1600); + let stream = TestStream::new(vec![Ok(a.message.clone())]); + let err = scanner + .scan_block_stream(a.height, a.height + 2, stream) + .await + .expect_err("a stream that stops short of `end` must not report success"); + assert!(err.to_string().contains("1601"), "error names the missing height: {err}"); + assert_eq!(scanner.get_last_scanned_block_height(), a.height); + }); +} + +/// Opening the stream against an unreachable oracle used to `unwrap()`. +#[test] +fn unreachable_oracle_is_a_clean_error() { + run(async { + let (mut scanner, _state) = scanner("unreachable", "http://127.0.0.1:1"); + scanner.update_last_scanned_block_height(1699); + scanner + .scan_block_range(1700, 1710) + .await + .expect_err("an unreachable oracle must be an error, not a panic"); + assert_eq!(scanner.get_last_scanned_block_height(), 1699); + }); +} From 27c1e905c37c67c05d7941640064ce30e22aef69 Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 24 Sep 2026 11:25:54 +0000 Subject: [PATCH 04/10] fix(scanner): stop a range scan at an unscannable block instead of skipping it An oracle answers a height it has not indexed with an OK message carrying an empty block hash and no data. scan_block_range took that as a block with no payments: it warned, advanced the scan height past it and reported success, so any payment in that block was never found (proven by the testkit mutation check unindexed-ok-empty: 80000 sats reported instead of 120600). Separately, every stream message was unwrap()ed, so an error status (NOT_FOUND, UNAVAILABLE) panicked the process, as did failing to open the stream. Now each message must be the next expected height and carry a valid 32-byte, non-zero block hash; otherwise, and on any error status or a stream that ends before the requested end, the scan returns an error naming the height to rescan from. The scan height stays at the last fully processed block, so watch_chain retries from there on its next poll (every 10 s, no spin) and blindbit-cli exits non-zero after saving that height. The example is updated the same way. Co-Authored-By: Claude Opus 5.5 (1M context) --- blindbit-lib/examples/stream_oracle.rs | 16 ++- blindbit-lib/src/scanner/scanning.rs | 103 +++++++++++++++--- .../src/scanner/stream_safety_tests.rs | 61 ++++++++--- 3 files changed, 146 insertions(+), 34 deletions(-) diff --git a/blindbit-lib/examples/stream_oracle.rs b/blindbit-lib/examples/stream_oracle.rs index e293e1f..8e73581 100644 --- a/blindbit-lib/examples/stream_oracle.rs +++ b/blindbit-lib/examples/stream_oracle.rs @@ -17,10 +17,11 @@ async fn main() -> Result<(), Box> { let mut stream = client .stream_block_scan_data_short(request) - .await - .unwrap() + .await? .into_inner(); - while let Some(block_scan_data) = stream.message().await.unwrap() { + // An error status (e.g. NOT_FOUND for a height the oracle has not + // indexed) ends the stream; `?` surfaces it instead of panicking. + while let Some(block_scan_data) = stream.message().await? { let Some(block_identifier) = block_scan_data.block_identifier.clone() else { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -28,6 +29,15 @@ async fn main() -> Result<(), Box> { ) .into()); }; + // Older oracles answer an unindexed height with an empty block hash + // and no data; that is not an empty block. + if block_identifier.block_hash.len() != 32 { + return Err(format!( + "height {} has no valid block hash; the oracle has probably not indexed it", + block_identifier.block_height + ) + .into()); + } println!("height: {}", block_identifier.block_height); } diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index 19b2070..d8096fa 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -10,7 +10,8 @@ use indexer::bdk_chain::{BlockId, CanonicalizationParams}; use tokio::time; use crate::oracle_grpc::{ - BlockScanDataShortResponse, ComputeIndexTxItem, FullTxItem, RangedBlockHeightRequestFiltered, + BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem, FullTxItem, + RangedBlockHeightRequestFiltered, }; use super::electrum_index::{ScriptHashEntry, SpHistoryEntry, electrum_scripthash}; @@ -68,6 +69,61 @@ impl BlockScanDataStream for tonic::Streaming { } } +/// Read the next block message, turning an error status into a scanner error +/// that names the height the scan stopped at. +/// +/// There is no retry here: the scan stops at `height` and the caller rescans +/// from there (`watch_chain` on its next poll, `blindbit-cli` on its next run). +async fn next_block( + stream: &mut S, + height: u64, +) -> Result, ScannerError> { + stream.next_message().await.map_err(|status| { + stream_stopped( + height, + &format!("oracle stream error {:?}: {}", status.code(), status.message()), + ) + }) +} + +/// A block message is only scannable if it is the next height in the range +/// and carries a real block hash. An oracle answers a height it has not +/// indexed with an empty hash and no data; treating that as an empty block +/// would silently skip any payment in it. +fn check_block_identifier( + block_identifier: &BlockIdentifier, + expected_height: u64, +) -> Result<(), ScannerError> { + if block_identifier.block_height != expected_height { + return Err(stream_stopped( + expected_height, + &format!( + "the oracle sent height {} instead", + block_identifier.block_height + ), + )); + } + let hash = &block_identifier.block_hash; + if hash.len() != 32 || hash.iter().all(|b| *b == 0) { + return Err(stream_stopped( + expected_height, + &format!( + "the oracle sent no valid block hash ({} bytes); the height is probably not indexed", + hash.len() + ), + )); + } + Ok(()) +} + +fn stream_stopped(height: u64, reason: &str) -> ScannerError { + format!( + "scan stopped at height {height}: {reason}; nothing from height {height} on was scanned, \ + rescan from height {height}" + ) + .into() +} + impl Scanner { /// scan a block range for new utxos and spent outpoints pub async fn scan_block_range( @@ -87,7 +143,14 @@ impl Scanner { .client .stream_block_scan_data_short(request) .await - .unwrap() + .map_err(|status| -> ScannerError { + format!( + "oracle refused to stream blocks {start}..={end}: {:?}: {}", + status.code(), + status.message() + ) + .into() + })? .into_inner(); self.scan_block_stream(start, end, stream).await @@ -95,6 +158,14 @@ impl Scanner { /// Scan the per-block messages of one `StreamBlockScanDataShort` response /// for the range `start..=end`. + /// + /// Every height in the range must arrive, in order, with a valid block + /// hash. Anything else — a message the oracle sends for a height it has + /// not indexed (empty block hash), a skipped or repeated height, an error + /// status, or a stream that ends before `end` — stops the scan with an + /// error. Such a block is never taken as "no payments": the scan height + /// stays at the last block that was fully processed, so the next scan + /// resumes at the block that could not be scanned. pub(crate) async fn scan_block_stream( &mut self, start: u64, @@ -123,7 +194,9 @@ impl Scanner { let mut p2p_conn: Option = None; - while let Some(block_scan_data) = stream.next_message().await.unwrap() { + let mut expected_height = start; + + while let Some(block_scan_data) = next_block(&mut stream, expected_height).await? { let Some(block_identifier) = block_scan_data.block_identifier.clone() else { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -131,6 +204,8 @@ impl Scanner { ) .into()); }; + check_block_identifier(&block_identifier, expected_height)?; + expected_height += 1; let block_id = BlockIdentifierDisplay(&block_identifier); tracing::debug!(height = block_id.0.block_height, "received block data from oracle"); @@ -172,21 +247,6 @@ impl Scanner { } Some(probable_match) => probable_match, }; - // Guard: if block_hash is malformed (already warned in - // scan_short_block_data) we cannot pull the full block - // via P2P. Advance progress and move on. - if block_identifier.block_hash.len() != 32 { - self.notify_electrum_scan_progress( - block_identifier.block_height, - start, - end, - ) - .await; - self.last_scanned_block_height = block_identifier.block_height; - self.stage.last_scanned_block_height = block_identifier.block_height; - continue; - } - // pull the full block data // Ensure we have exactly 32 bytes @@ -463,6 +523,13 @@ impl Scanner { self.stage.last_scanned_block_height = block_identifier.block_height; } + if expected_height <= end { + return Err(stream_stopped( + expected_height, + "the oracle ended the stream before this height", + )); + } + let outpoints: Vec<(u32, OutPoint)> = self .internal_indexer .index() diff --git a/blindbit-lib/src/scanner/stream_safety_tests.rs b/blindbit-lib/src/scanner/stream_safety_tests.rs index aaa2135..bca50f0 100644 --- a/blindbit-lib/src/scanner/stream_safety_tests.rs +++ b/blindbit-lib/src/scanner/stream_safety_tests.rs @@ -107,15 +107,22 @@ fn scanner(tag: &str, oracle: &'static str) -> (Scanner, TempState) { )); let client = crate::OracleServiceClient::new(Channel::from_static(oracle).connect_lazy()); let socket: SocketAddr = "127.0.0.1:1".parse().expect("socket address"); - let scanner = Scanner::new(client, socket, scan_sk, spend_pk, 0, state.clone(), Network::Regtest); + let scanner = Scanner::new( + client, + socket, + scan_sk, + spend_pk, + 0, + state.clone(), + Network::Regtest, + ); (scanner, TempState(state)) } /// One block at `height` holding a single silent payment to the test wallet, -/// plus the oracle message that describes it. +/// served in place of P2P, and the oracle message that describes it. struct PaymentBlock { height: u64, - block: Block, message: BlockScanDataShortResponse, } @@ -192,7 +199,7 @@ fn payment_block(height: u64) -> PaymentBlock { spent_outputs: vec![], }; serve(&block); - PaymentBlock { height, block, message } + PaymentBlock { height, message } } /// What an oracle sends today for a height it has not indexed: OK, with an @@ -224,7 +231,6 @@ fn empty_hash_block_mid_range_stops_scan_and_next_scan_finds_its_outputs() { let a = payment_block(1100); let b = payment_block(1101); let c = payment_block(1102); - assert_eq!(a.block.block_hash(), served_block(&a.block.block_hash()).unwrap().block_hash()); let first = TestStream::new(vec![ Ok(a.message.clone()), @@ -236,9 +242,16 @@ fn empty_hash_block_mid_range_stops_scan_and_next_scan_finds_its_outputs() { .await .expect_err("a block without a valid hash must stop the scan"); let msg = err.to_string(); - assert!(msg.contains(&b.height.to_string()), "error names the height: {msg}"); + assert!( + msg.contains(&b.height.to_string()), + "error names the height: {msg}" + ); assert_eq!(scanner.get_last_scanned_block_height(), a.height); - assert_eq!(owned_outputs(&scanner), 1, "only the block before the gap is applied"); + assert_eq!( + owned_outputs(&scanner), + 1, + "only the block before the gap is applied" + ); // Resume where the scanner says, now that the oracle serves the block. let resume = scanner.get_last_scanned_block_height() + 1; @@ -259,7 +272,11 @@ fn malformed_block_hash_stops_scan() { let (mut scanner, _state) = scanner("short-hash", "http://127.0.0.1:1"); let a = payment_block(1200); let mut bad = payment_block(1201).message; - bad.block_identifier.as_mut().unwrap().block_hash.truncate(31); + bad.block_identifier + .as_mut() + .unwrap() + .block_hash + .truncate(31); let mut zero = payment_block(1201).message; zero.block_identifier.as_mut().unwrap().block_hash = vec![0; 32]; @@ -285,9 +302,16 @@ fn skipped_height_stops_scan() { .scan_block_stream(a.height, c.height, stream) .await .expect_err("a height gap must stop the scan"); - assert!(err.to_string().contains("1301"), "error names the expected height: {err}"); + assert!( + err.to_string().contains("1301"), + "error names the expected height: {err}" + ); assert_eq!(scanner.get_last_scanned_block_height(), a.height); - assert_eq!(owned_outputs(&scanner), 1, "the block after the gap is not applied"); + assert_eq!( + owned_outputs(&scanner), + 1, + "the block after the gap is not applied" + ); }); } @@ -326,8 +350,16 @@ fn error_status_mid_stream_is_a_clean_error() { .scan_block_stream(a.height, a.height + 1, stream) .await .expect_err("an error status must end the scan with an error"); - assert!(err.to_string().contains("1501"), "{code:?}: error names the height: {err}"); - assert_eq!(scanner.get_last_scanned_block_height(), a.height, "{code:?}"); + let msg = err.to_string(); + assert!( + msg.contains("height 1501") && msg.contains(&format!("{code:?}")), + "error names the height and the status code: {msg}" + ); + assert_eq!( + scanner.get_last_scanned_block_height(), + a.height, + "{code:?}" + ); assert_eq!(owned_outputs(&scanner), 1, "{code:?}"); } }); @@ -343,7 +375,10 @@ fn stream_ending_before_requested_end_is_an_error() { .scan_block_stream(a.height, a.height + 2, stream) .await .expect_err("a stream that stops short of `end` must not report success"); - assert!(err.to_string().contains("1601"), "error names the missing height: {err}"); + assert!( + err.to_string().contains("1601"), + "error names the missing height: {err}" + ); assert_eq!(scanner.get_last_scanned_block_height(), a.height); }); } From 3ce140416301c319042e771017850ca2d7c34252 Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 24 Sep 2026 12:20:19 +0000 Subject: [PATCH 05/10] fix(scanner): detect chain reorganisations and roll back to the fork point The live scanner had no reorg handling: after a reorganisation it kept every output and spend learned from disconnected blocks. An evicted payment stayed in the balance as a phantom output, a payment that re-confirmed at another height kept its old height, and an output whose spend was disconnected stayed marked spent. The scanner now remembers the block hash of each scanned height for the last REORG_LOOKBACK (144) heights, persisted in the state file as `scanned_block_hashes`. A range scan that continues the scanned chain streams one block early and compares it; watch_chain also runs that check when the oracle has no new block, which catches a same-height tip replacement. On a mismatch the scan walks up from the oldest remembered hash to find the fork point. It rolls back everything above it: graph transactions and anchors from disconnected blocks with their tweaks, the owned-output records and spent marks derived from them, checkpoints, remembered hashes, and Electrum history/SP history/headers. It persists that state and scans the new branch from the fork. A fork at or below the oldest remembered hash is a ReorgTooDeep error. That error changes nothing and repeats on every attempt, and it tells the user to rescan from the wallet birthday. friglet drops its snapshotted checkpoint hashes and cached headers above the fork (and rewrites the header sidecar) so it never serves a header of a disconnected block. blindbit-cli goes through scan_block_range and gets the same check. Tests drive the stream seam with an in-process oracle, mirroring the testkit T2 reorg scenario (fork at 104, 2 disconnected, 3 mined): eviction, re-confirmation at a new height, a disconnected spend, a same-height tip replacement, persistence plus restart mid-rescan, and a reorg deeper than the lookback. Co-Authored-By: Claude Opus 5.5 (1M context) --- blindbit-lib/src/scanner/changeset.rs | 9 + blindbit-lib/src/scanner/mod.rs | 4 + blindbit-lib/src/scanner/reorg.rs | 264 +++++++ blindbit-lib/src/scanner/reorg_tests.rs | 694 ++++++++++++++++++ blindbit-lib/src/scanner/scanner.rs | 21 + blindbit-lib/src/scanner/scanning.rs | 200 ++++- .../src/scanner/stream_safety_tests.rs | 2 +- friglet/src/electrum/mod.rs | 121 ++- friglet/src/main.rs | 4 +- 9 files changed, 1287 insertions(+), 32 deletions(-) create mode 100644 blindbit-lib/src/scanner/reorg.rs create mode 100644 blindbit-lib/src/scanner/reorg_tests.rs diff --git a/blindbit-lib/src/scanner/changeset.rs b/blindbit-lib/src/scanner/changeset.rs index 2468683..9c178e3 100644 --- a/blindbit-lib/src/scanner/changeset.rs +++ b/blindbit-lib/src/scanner/changeset.rs @@ -70,6 +70,13 @@ pub struct ChangeSet { pub public_spend_hex: Option, /// Maximum label number used pub max_label_num: u32, + /// Hash of every block scanned within the reorg lookback window + /// (`REORG_LOOKBACK` heights below the highest one). Compared with the + /// oracle's hashes to detect a chain reorganisation; empty in state files + /// written before reorg handling, which start recording from their next + /// scan. + #[cfg_attr(feature = "serde", serde(default))] + pub scanned_block_hashes: BTreeMap, } impl Merge for ChangeSet { @@ -77,6 +84,7 @@ impl Merge for ChangeSet { fn merge(&mut self, other: Self) { // Merge block checkpoints (extend with new ones) self.block_checkpoints.extend(other.block_checkpoints); + self.scanned_block_hashes.extend(other.scanned_block_hashes); Merge::merge(&mut self.indexer, other.indexer); // Update metadata with the latest values @@ -121,5 +129,6 @@ impl Merge for ChangeSet { && self.last_scanned_block_height == 0 && self.last_scanned_block_height_rescan == 0 && self.owned_outputs.is_empty() + && self.scanned_block_hashes.is_empty() } } diff --git a/blindbit-lib/src/scanner/mod.rs b/blindbit-lib/src/scanner/mod.rs index 9c4a458..bff4d9c 100644 --- a/blindbit-lib/src/scanner/mod.rs +++ b/blindbit-lib/src/scanner/mod.rs @@ -5,6 +5,9 @@ mod config; pub mod electrum_index; mod load; mod p2p; +mod reorg; +#[cfg(test)] +mod reorg_tests; mod scanner; mod scanning; #[cfg(test)] @@ -20,6 +23,7 @@ pub use changeset::ChangeSet; pub use config::ScannerConfig; pub use electrum_index::{ScriptHashEntry, SpHistoryEntry, WalletElectrumIndex, electrum_scripthash, electrum_status}; pub use p2p::broadcast_tx; +pub use reorg::{REORG_LOOKBACK, ReorgTooDeep}; pub use scanner::Scanner; pub use types::{BlockIdentifierDisplay, OwnedOutput, OwnedOutputRecord}; diff --git a/blindbit-lib/src/scanner/reorg.rs b/blindbit-lib/src/scanner/reorg.rs new file mode 100644 index 0000000..d801db6 --- /dev/null +++ b/blindbit-lib/src/scanner/reorg.rs @@ -0,0 +1,264 @@ +//! Chain reorganisation handling. +//! +//! The scanner remembers the block hash of every height it scanned within +//! the last [`REORG_LOOKBACK`] heights (`Scanner::scanned_block_hashes`, +//! persisted as the state file's `scanned_block_hashes`). Every block the +//! oracle streams for a height the scanner already knows is compared with +//! that hash: +//! +//! - A range scan that continues the scanned chain starts its stream one +//! height early, at the last scanned height, so the first message proves +//! the chain underneath is unchanged. `watch_chain` does the same when the +//! oracle has no new block, which catches a same-height tip replacement. +//! - When that first message disagrees, the fork is somewhere below it: the +//! scan reopens the stream at the lowest remembered height and walks up. +//! The first height whose hash disagrees is the first disconnected block, +//! the one below it is the fork point. +//! - Everything learned above the fork point is rolled back +//! ([`Scanner::rollback_to`]): transactions and anchors from disconnected +//! blocks, the owned-output records they created, spent marks they set, +//! checkpoints, remembered hashes, and the Electrum history. The state is +//! persisted, and the same stream then scans the new branch as new blocks, +//! so a transaction that re-confirms is found again at its new height and +//! one that did not simply stays gone. +//! - A fork at or below the lowest remembered height cannot be located. The +//! scan stops with [`ReorgTooDeep`] and changes nothing; the wallet must be +//! rescanned from its birthday with a fresh state file. It keeps failing +//! loudly on every attempt rather than continuing on a chain it cannot +//! verify. + +use std::collections::BTreeSet; + +use bitcoin::hashes::Hash; +use bitcoin::{BlockHash, Txid}; +use indexer::bdk_chain::ConfirmationBlockTime; +use indexer::v2::SpIndexerV2; + +use super::ScannerError; +use super::scanner::Scanner; + +/// How many of the most recent scanned heights keep their block hash, and so +/// the deepest reorganisation the scanner can roll back by itself. +/// +/// 144 blocks is one day of mainnet blocks, several times deeper than any +/// mainnet reorganisation since 2013; storing it costs a few kilobytes of +/// state. +pub const REORG_LOOKBACK: u32 = 144; + +/// A chain reorganisation reaches at or below the oldest block hash the +/// scanner still remembers, so the fork point cannot be found. +/// +/// Nothing was rolled back or scanned. The wallet state may hold outputs and +/// spends from blocks that are no longer in the chain; it has to be rebuilt +/// by rescanning from the wallet birthday with a fresh state file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReorgTooDeep { + /// Lowest height with a remembered hash; the oracle serves a different + /// block there. + pub lowest_known_height: u64, + /// Last height the scanner had scanned. + pub last_scanned_height: u64, +} + +impl std::fmt::Display for ReorgTooDeep { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "chain reorganisation deeper than the scanner can roll back: the oracle serves a \ + different block at height {} (the oldest block hash remembered, lookback {} blocks, \ + last scanned height {}); the wallet state may contain outputs from blocks that are no \ + longer in the chain; nothing was changed; move the state file away and rescan from \ + the wallet birthday", + self.lowest_known_height, REORG_LOOKBACK, self.last_scanned_height + ) + } +} + +impl std::error::Error for ReorgTooDeep {} + +/// The first block of a stream disagrees with the remembered hash, so the +/// fork lies below the stream's start; the scan reopens the stream lower. +#[derive(Debug)] +pub(crate) struct ReorgBelowStreamStart { + pub height: u64, +} + +impl std::fmt::Display for ReorgBelowStreamStart { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "chain reorganisation detected at height {}; the fork point lies below it", + self.height + ) + } +} + +impl std::error::Error for ReorgBelowStreamStart {} + +/// The oracle serves block hashes in display order. +pub(crate) fn block_hash_from_oracle(display_order: &[u8]) -> Option { + let mut bytes: [u8; 32] = display_order.try_into().ok()?; + bytes.reverse(); + Some(BlockHash::from_byte_array(bytes)) +} + +/// What the scanner does with one oracle block, judged by its hash. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum BlockCheck { + /// No remembered hash, or the remembered hash matches. + Consistent, + /// Remembered hash differs: the chain was reorganised at this height. + Reorganised, +} + +impl Scanner { + /// Lowest height with a remembered block hash. + pub(crate) fn lowest_known_block_height(&self) -> Option { + self.scanned_block_hashes + .keys() + .next() + .map(|height| u64::from(*height)) + } + + pub(crate) fn knows_block_hash(&self, height: u64) -> bool { + u32::try_from(height).is_ok_and(|height| self.scanned_block_hashes.contains_key(&height)) + } + + pub(crate) fn check_block_hash(&self, height: u64, hash: &BlockHash) -> BlockCheck { + match u32::try_from(height) + .ok() + .and_then(|height| self.scanned_block_hashes.get(&height)) + { + Some(known) if known != hash => BlockCheck::Reorganised, + _ => BlockCheck::Consistent, + } + } + + /// Remember the hash of a block that was just scanned, keeping only the + /// lookback window below the highest remembered height. + pub(crate) fn record_scanned_block_hash(&mut self, height: u64, hash: BlockHash) { + let Ok(height) = u32::try_from(height) else { + return; + }; + self.scanned_block_hashes.insert(height, hash); + let highest = *self + .scanned_block_hashes + .keys() + .next_back() + .expect("just inserted"); + let keep_from = highest.saturating_sub(REORG_LOOKBACK - 1); + self.scanned_block_hashes = self.scanned_block_hashes.split_off(&keep_from); + self.stage.scanned_block_hashes = self.scanned_block_hashes.clone(); + } + + /// Undo everything learned from blocks above `fork_height`, which were + /// disconnected by a chain reorganisation, and persist the result. + /// + /// Transactions confirmed only in disconnected blocks leave the wallet + /// graph (with their tweaks), so their outputs leave the balance and the + /// owned-output records, and outputs they spent are unspent again. The + /// indexer is rebuilt from the remaining changeset, the same way a + /// restart restores it. Whatever the new branch confirms is found by + /// scanning it afterwards. + pub(crate) async fn rollback_to(&mut self, fork_height: u64) -> Result<(), ScannerError> { + let fork = u32::try_from(fork_height) + .map_err(|_| format!("fork height {fork_height} is out of range"))?; + let previous_height = self.last_scanned_block_height; + + // Wallet graph and tweaks. + let removed = drop_blocks_above(&mut self.stage.indexer, fork); + let scan_sk = *self.internal_indexer.scan_sk(); + let spend_pk = *self.internal_indexer.spend_pk(); + self.stage.indexer.scan_sk = Some(scan_sk); + self.stage.indexer.spend_pk = Some(spend_pk); + let mut indexer = SpIndexerV2::::new(scan_sk, spend_pk); + // Labels first: applying the changeset re-runs the BIP-352 scan of + // every stored transaction, which needs them (as on restore). + _ = indexer.add_label(0); + for m in 1..=self.max_label_num { + _ = indexer.add_label(m); + } + indexer.apply_changeset(self.stage.indexer.clone()); + self.internal_indexer = indexer; + + // Owned-output records and spent marks are derived from the graph; + // rebuild them so no record or spend from a disconnected block stays. + self.owned_outputs.clear(); + self.stage.owned_outputs.clear(); + self.sync_owned_outputs(); + + // Chain position. + self.block_checkpoints.split_off(&(fork + 1)); + self.stage.block_checkpoints.split_off(&(fork + 1)); + self.scanned_block_hashes.split_off(&(fork + 1)); + self.stage.scanned_block_hashes = self.scanned_block_hashes.clone(); + self.last_scanned_block_height = self.last_scanned_block_height.min(fork_height); + self.stage.last_scanned_block_height = self.last_scanned_block_height; + self.last_scanned_block_height_rescan = + self.last_scanned_block_height_rescan.min(fork_height); + self.stage.last_scanned_block_height_rescan = self.last_scanned_block_height_rescan; + + // Electrum view: history, SP history, raw txs and headers of the + // disconnected blocks. Unconfirmed (height 0) entries stay. + { + let mut idx = self.electrum_index.lock().await; + for history in idx.scripthash_history.values_mut() { + history.retain(|entry| entry.height <= fork); + } + idx.sp_history.retain(|entry| entry.height <= fork); + idx.headers.retain(|height, _| *height <= fork); + for txid in &removed { + idx.txs.remove(&txid.to_string()); + } + let header = idx.headers.get(&fork).cloned().unwrap_or_default(); + idx.tip = Some((fork, header)); + } + + tracing::warn!( + fork_height, + previous_height, + disconnected_blocks = previous_height.saturating_sub(fork_height), + dropped_transactions = removed.len(), + owned_outputs = self.owned_outputs.len(), + "chain reorganisation: rolled wallet state back to the fork point; rescanning the new branch" + ); + + #[cfg(feature = "serde")] + self.save_to_file(&self.state_file)?; + + let _ = self.notify_reorg.send(fork); + let _ = self + .notify_found_utxos + .send(self.internal_indexer.index().by_shared_secret.len()); + Ok(()) + } +} + +/// Remove from an indexer changeset every anchor above `fork` and every +/// transaction that was confirmed only in those blocks, with its tweak and +/// timestamps. Returns the removed txids. Transactions that never had an +/// anchor (unconfirmed) are kept. +fn drop_blocks_above( + changeset: &mut indexer::v2::ChangeSet, + fork: u32, +) -> BTreeSet { + let graph = &mut changeset.graph; + let anchored_before: BTreeSet = graph.anchors.iter().map(|(_, txid)| *txid).collect(); + graph + .anchors + .retain(|(anchor, _)| anchor.block_id.height <= fork); + let anchored_after: BTreeSet = graph.anchors.iter().map(|(_, txid)| *txid).collect(); + let removed: BTreeSet = anchored_before + .difference(&anchored_after) + .copied() + .collect(); + + graph.txs.retain(|tx| !removed.contains(&tx.compute_txid())); + graph.last_seen.retain(|txid, _| !removed.contains(txid)); + graph.first_seen.retain(|txid, _| !removed.contains(txid)); + graph.last_evicted.retain(|txid, _| !removed.contains(txid)); + changeset + .txid_to_partial_secret + .retain(|txid, _| !removed.contains(txid)); + removed +} diff --git a/blindbit-lib/src/scanner/reorg_tests.rs b/blindbit-lib/src/scanner/reorg_tests.rs new file mode 100644 index 0000000..a9c2b55 --- /dev/null +++ b/blindbit-lib/src/scanner/reorg_tests.rs @@ -0,0 +1,694 @@ +//! Chain reorganisations through the live scan path. +//! +//! The scenario mirrors the testkit's T2 `reorg` scenario: a wallet scans a +//! chain, then the oracle switches to a branch that forks at height 104. +//! Blocks above the fork are disconnected; one payment re-confirms on the new +//! branch at a different height, one is evicted. Each test drives a real +//! [`Scanner`] through `scan_range_from`, the body of `scan_block_range`, +//! against an in-process oracle; only the P2P block fetch is replaced (full +//! blocks are served by `stream_safety_tests::serve`). + +use std::collections::{BTreeMap, VecDeque}; +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; + +use bdk_sp::bitcoin::key::Secp256k1; +use bdk_sp::receive::get_silentpayment_pubkey; +use bitcoin::absolute::LockTime; +use bitcoin::hashes::{Hash, sha256d}; +use bitcoin::key::TweakedPublicKey; +use bitcoin::secp256k1::{PublicKey, SecretKey}; +use bitcoin::transaction::Version; +use bitcoin::{ + Amount, Block, BlockHash, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, + Witness, XOnlyPublicKey, +}; +use bitcoin_rev::Network; +use tonic::transport::Channel; + +use super::reorg::ReorgTooDeep; +use super::scanning::{BlockScanDataStream, BlockStreamSource}; +use super::stream_safety_tests::serve; +use super::{REORG_LOOKBACK, Scanner, ScannerError}; +use crate::oracle_grpc::{BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem}; + +const FORK: u64 = 104; + +// --------------------------------------------------------------------------- +// In-process oracle +// --------------------------------------------------------------------------- + +type Message = BlockScanDataShortResponse; + +/// One chain as the oracle serves it: height -> block message. +#[derive(Clone, Default)] +struct Oracle { + chain: BTreeMap, + /// Ends a stream with an UNAVAILABLE status at this height, if the + /// stream began below it (so the first block of a stream is always + /// served). + fail_at: Option, +} + +struct ChainStream(VecDeque>); + +impl BlockScanDataStream for ChainStream { + fn next_message( + &mut self, + ) -> impl Future, tonic::Status>> + Send { + std::future::ready(self.0.pop_front().transpose()) + } +} + +impl BlockStreamSource for Oracle { + type Stream = ChainStream; + + async fn open(&mut self, start: u64, end: u64) -> Result { + let mut items = VecDeque::new(); + for height in start..=end { + if self.fail_at == Some(height) && start < height { + items.push_back(Err(tonic::Status::unavailable("connection reset"))); + break; + } + match self.chain.get(&height) { + Some(message) => items.push_back(Ok(message.clone())), + None => break, + } + } + Ok(ChainStream(items)) + } +} + +impl Oracle { + fn tip(&self) -> u64 { + *self.chain.keys().next_back().expect("non-empty chain") + } + + /// A copy of this chain up to and including `fork`, as the start of a + /// competing branch. + fn branch_at(&self, fork: u64) -> Oracle { + Oracle { + chain: self + .chain + .range(..=fork) + .map(|(h, m)| (*h, m.clone())) + .collect(), + fail_at: None, + } + } + + fn empty(&mut self, branch: u8, height: u64) { + let hash = sha256d::Hash::hash(&[&[branch][..], &height.to_le_bytes()].concat()); + self.chain.insert( + height, + message(BlockHash::from_raw_hash(hash), height, vec![], &[]), + ); + } + + /// A block holding `txs` (payments carry their served tweak), served in + /// place of P2P. + fn block( + &mut self, + branch: u8, + height: u64, + txs: Vec<(Transaction, Option)>, + spent: &[XOnlyPublicKey], + ) { + let items = txs + .iter() + .filter_map(|(tx, tweak)| tweak.map(|tweak| item(tx, tweak))) + .collect(); + let block = full_block(branch, height, txs.into_iter().map(|(tx, _)| tx).collect()); + serve(&block); + self.chain + .insert(height, message(block.block_hash(), height, items, spent)); + } +} + +fn message( + hash: BlockHash, + height: u64, + comp_index: Vec, + spent: &[XOnlyPublicKey], +) -> Message { + // The oracle serves hashes in display order. + let mut display = hash.to_byte_array(); + display.reverse(); + BlockScanDataShortResponse { + block_identifier: Some(BlockIdentifier { + block_hash: display.to_vec(), + block_height: height, + }), + comp_index, + spent_outputs: spent + .iter() + .flat_map(|key| key.serialize()[..8].to_vec()) + .collect(), + } +} + +fn item(tx: &Transaction, tweak: PublicKey) -> ComputeIndexTxItem { + let mut txid = tx.compute_txid().to_byte_array(); + txid.reverse(); + ComputeIndexTxItem { + txid: txid.to_vec(), + tweak: tweak.serialize().to_vec(), + outputs_short: tx + .output + .iter() + .flat_map(|o| o.script_pubkey.as_bytes()[2..10].to_vec()) + .collect(), + } +} + +fn full_block(branch: u8, height: u64, txs: Vec) -> Block { + let coinbase = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes([&[branch][..], &height.to_le_bytes()].concat()), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(312_500_000), + script_pubkey: ScriptBuf::from_bytes(vec![0x6a]), + }], + }; + let mut txdata = vec![coinbase]; + txdata.extend(txs); + let mut block = Block { + header: bitcoin::block::Header { + version: bitcoin::block::Version::TWO, + prev_blockhash: BlockHash::all_zeros(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: 1_713_571_767 + height as u32, + bits: bitcoin::CompactTarget::from_consensus(0x207f_ffff), + nonce: u32::from(branch), + }, + txdata, + }; + block.header.merkle_root = block.compute_merkle_root().expect("non-empty block"); + block +} + +// --------------------------------------------------------------------------- +// Wallet +// --------------------------------------------------------------------------- + +fn secret(byte: u8) -> SecretKey { + SecretKey::from_slice(&[byte; 32]).expect("valid secret") +} + +fn keys() -> (SecretKey, PublicKey) { + (secret(0x11), secret(0x22).public_key(&Secp256k1::new())) +} + +fn state_file(tag: &str) -> PathBuf { + std::env::temp_dir().join(format!("blindbit-reorg-{tag}-{}.json", std::process::id())) +} + +struct TempState(PathBuf); + +impl Drop for TempState { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +fn client() -> crate::OracleServiceClient { + crate::OracleServiceClient::new(Channel::from_static("http://127.0.0.1:1").connect_lazy()) +} + +fn socket() -> SocketAddr { + "127.0.0.1:1".parse().expect("socket address") +} + +/// A fresh wallet. Must be called inside a Tokio runtime. +fn wallet(tag: &str) -> (Scanner, TempState) { + let (scan_sk, spend_pk) = keys(); + let path = state_file(tag); + let _ = std::fs::remove_file(&path); + let scanner = Scanner::new( + client(), + socket(), + scan_sk, + spend_pk, + 0, + path.clone(), + Network::Regtest, + ); + (scanner, TempState(path)) +} + +/// The wallet as a restarted daemon loads it from its state file. +#[cfg(feature = "serde")] +fn restart(path: &std::path::Path) -> Scanner { + let changeset = Scanner::load_from_file(path).expect("load state"); + Scanner::from_changeset( + client(), + socket(), + changeset, + path.to_path_buf(), + Network::Regtest, + ) + .expect("restore") +} + +/// A payment of `sat` to the wallet's unlabelled address; the tweak the +/// oracle serves for it and the paid outpoint. +fn payment(seed: u8, sat: u64) -> (Transaction, PublicKey, OutPoint, XOnlyPublicKey) { + let (scan_sk, spend_pk) = keys(); + let tweak = secret(seed).public_key(&Secp256k1::new()); + let shared = bdk_sp::compute_shared_secret(&scan_sk, &tweak); + let key = get_silentpayment_pubkey(&spend_pk, &shared, 0, None) + .x_only_public_key() + .0; + let tx = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::new(Txid::from_byte_array([seed; 32]), 0), + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(sat), + script_pubkey: p2tr(key), + }], + }; + let outpoint = OutPoint::new(tx.compute_txid(), 0); + (tx, tweak, outpoint, key) +} + +/// A spend of `outpoint` to a foreign key. +fn spend_of(outpoint: OutPoint) -> Transaction { + let foreign = secret(0x66) + .public_key(&Secp256k1::new()) + .x_only_public_key() + .0; + Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: outpoint, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(1_000), + script_pubkey: p2tr(foreign), + }], + } +} + +fn p2tr(key: XOnlyPublicKey) -> ScriptBuf { + ScriptBuf::new_p2tr_tweaked(TweakedPublicKey::dangerous_assume_tweaked(key)) +} + +fn run(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime") + .block_on(future) +} + +/// One `watch_chain` iteration against `oracle`. +async fn watch_step(scanner: &mut Scanner, oracle: &Oracle) -> Result<(), ScannerError> { + let tip = oracle.tip(); + let last = scanner.get_last_scanned_block_height(); + let (from, to) = if tip > last { + (last + 1, tip) + } else { + (tip + 1, tip) + }; + scanner.scan_range_from(from, to, oracle.clone()).await +} + +/// Sum of the unspent owned outputs: the wallet balance. +fn balance(scanner: &Scanner) -> u64 { + scanner + .owned_outputs() + .filter(|r| !r.is_spent()) + .map(|r| r.amount_sat) + .sum() +} + +fn owned_height(scanner: &Scanner, outpoint: OutPoint) -> Option { + scanner + .owned_outputs() + .find(|r| r.outpoint == outpoint) + .map(|r| r.height) +} + +/// Heights at which the wallet graph holds `txid` confirmed. +fn anchor_heights(scanner: &Scanner, txid: Txid) -> Vec { + scanner + .internal_indexer + .graph() + .get_tx_node(txid) + .map(|node| node.anchors.iter().map(|a| a.block_id.height).collect()) + .unwrap_or_default() +} + +async fn sp_history(scanner: &Scanner) -> Vec<(String, u32)> { + let idx = scanner.electrum_index.lock().await; + idx.sp_history + .iter() + .map(|e| (e.tx_hash.clone(), e.height)) + .collect() +} + +async fn in_scripthash_history(scanner: &Scanner, txid: Txid) -> bool { + let idx = scanner.electrum_index.lock().await; + idx.scripthash_history + .values() + .flatten() + .any(|e| e.tx_hash == txid.to_string()) +} + +// --------------------------------------------------------------------------- +// The T2 scenario: fork at 104, 2 blocks disconnected, 3 mined +// --------------------------------------------------------------------------- + +struct Scenario { + old: Oracle, + new: Oracle, + survivor: OutPoint, + reconfirmed: OutPoint, + evicted: OutPoint, +} + +/// Old chain: 101 pays `survivor`, 105 pays `reconfirmed`, 106 pays +/// `evicted`. New chain forks at 104: 105' is empty, 106' re-confirms +/// `reconfirmed`, 107' is empty; `evicted` never confirms again. +fn scenario() -> Scenario { + let (survivor_tx, survivor_tweak, survivor, _) = payment(0x31, 50_000); + let (keep_tx, keep_tweak, reconfirmed, _) = payment(0x32, 30_000); + let (evict_tx, evict_tweak, evicted, _) = payment(0x33, 20_000); + + let mut old = Oracle::default(); + old.block(0xa, 101, vec![(survivor_tx, Some(survivor_tweak))], &[]); + for h in 102..=FORK { + old.empty(0xa, h); + } + old.block(0xa, 105, vec![(keep_tx.clone(), Some(keep_tweak))], &[]); + old.block(0xa, 106, vec![(evict_tx, Some(evict_tweak))], &[]); + + let mut new = old.branch_at(FORK); + new.empty(0xb, 105); + new.block(0xb, 106, vec![(keep_tx, Some(keep_tweak))], &[]); + new.empty(0xb, 107); + + Scenario { + old, + new, + survivor, + reconfirmed, + evicted, + } +} + +async fn scan_old_chain(scanner: &mut Scanner, s: &Scenario) { + scanner + .scan_range_from(101, s.old.tip(), s.old.clone()) + .await + .expect("old chain scans"); + assert_eq!(balance(scanner), 100_000, "all three payments found"); +} + +#[test] +fn reorg_evicting_a_payment_drops_its_output_and_balance() { + run(async { + let (mut scanner, _state) = wallet("evict"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + + watch_step(&mut scanner, &s.new) + .await + .expect("new branch scans"); + + assert_eq!( + owned_height(&scanner, s.evicted), + None, + "evicted output is gone" + ); + assert!( + !scanner + .internal_indexer + .index() + .by_shared_secret + .contains_key(&s.evicted), + "evicted output left the index" + ); + assert!(anchor_heights(&scanner, s.evicted.txid).is_empty()); + assert_eq!(balance(&scanner), 80_000, "balance no longer counts it"); + assert!( + !sp_history(&scanner) + .await + .iter() + .any(|(txid, _)| *txid == s.evicted.txid.to_string()), + "evicted tx left the SP history" + ); + assert!(!in_scripthash_history(&scanner, s.evicted.txid).await); + assert_eq!(owned_height(&scanner, s.survivor), Some(101)); + assert_eq!(scanner.get_last_scanned_block_height(), 107); + }); +} + +#[test] +fn reorg_reconfirming_a_payment_keeps_it_at_its_new_height() { + run(async { + let (mut scanner, _state) = wallet("reconfirm"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + assert_eq!(owned_height(&scanner, s.reconfirmed), Some(105)); + + watch_step(&mut scanner, &s.new) + .await + .expect("new branch scans"); + + assert_eq!( + owned_height(&scanner, s.reconfirmed), + Some(106), + "found again, at its new height" + ); + assert_eq!(anchor_heights(&scanner, s.reconfirmed.txid), vec![106]); + let history = sp_history(&scanner).await; + assert!(history.contains(&(s.reconfirmed.txid.to_string(), 106))); + assert!(!history.contains(&(s.reconfirmed.txid.to_string(), 105))); + }); +} + +/// The fork replaces the tip without making the chain longer: the watch +/// loop's check with no new block still finds it. +#[test] +fn same_height_tip_replacement_is_detected_without_a_new_block() { + run(async { + let (mut scanner, _state) = wallet("same-height"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + + let mut replaced = s.old.branch_at(105); + replaced.empty(0xc, 106); + assert_eq!(replaced.tip(), scanner.get_last_scanned_block_height()); + watch_step(&mut scanner, &replaced) + .await + .expect("check succeeds"); + + assert_eq!(owned_height(&scanner, s.evicted), None); + assert_eq!(owned_height(&scanner, s.reconfirmed), Some(105)); + assert_eq!(balance(&scanner), 80_000); + assert_eq!(scanner.get_last_scanned_block_height(), 106); + }); +} + +#[test] +fn reorg_disconnecting_a_spend_makes_the_output_unspent_again() { + run(async { + let (mut scanner, _state) = wallet("unspend"); + let (recv_tx, tweak, outpoint, key) = payment(0x41, 50_000); + let mut old = Oracle::default(); + old.block(0xa, 101, vec![(recv_tx, Some(tweak))], &[]); + for h in 102..=FORK { + old.empty(0xa, h); + } + let spend = spend_of(outpoint); + old.block(0xa, 105, vec![(spend.clone(), None)], &[key]); + let mut new = old.branch_at(FORK); + new.empty(0xb, 105); + new.empty(0xb, 106); + + scanner + .scan_range_from(101, 105, old.clone()) + .await + .expect("old chain scans"); + let spent_by = scanner + .owned_outputs() + .find(|r| r.outpoint == outpoint) + .and_then(|r| r.spent_by); + assert_eq!(spent_by, Some(spend.compute_txid())); + assert_eq!(balance(&scanner), 0); + + watch_step(&mut scanner, &new) + .await + .expect("new branch scans"); + + let record = scanner + .owned_outputs() + .find(|r| r.outpoint == outpoint) + .cloned() + .expect("output still owned"); + assert_eq!((record.spent_by, record.spent_height), (None, None)); + assert_eq!(balance(&scanner), 50_000, "the output is spendable again"); + assert!(anchor_heights(&scanner, spend.compute_txid()).is_empty()); + assert!(!in_scripthash_history(&scanner, spend.compute_txid()).await); + // Its prefix is watched again, so a later spend is still detected. + assert!(scanner.owned_prefixes.contains_key(&record.short_pubkey())); + }); +} + +/// Hashes are persisted, so a daemon restarted after the reorg still finds +/// it; the rollback is persisted before the new branch is scanned, so a scan +/// that dies right after it leaves no phantom output on disk; and resuming +/// from that state ends up exactly where an uninterrupted scan does. +#[cfg(feature = "serde")] +#[test] +fn rollback_is_persisted_and_restart_mid_rescan_is_safe() { + run(async { + let (mut scanner, state) = wallet("restart"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + drop(scanner); + + let json: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&state.0).unwrap()).unwrap(); + assert_eq!( + json["scanned_block_hashes"].as_object().map(|m| m.len()), + Some(6), + "every scanned height's hash is persisted" + ); + + // Restart after the reorg. The scan locates the fork, rolls back, + // scans 105' and then its stream dies before 106'. + let mut restarted = restart(&state.0); + let mut flaky = s.new.clone(); + flaky.fail_at = Some(106); + watch_step(&mut restarted, &flaky) + .await + .expect_err("the stream fails mid-rescan"); + drop(restarted); + + // The rollback to the fork point is on disk; 105' was scanned but + // not yet saved, so it is simply scanned again. + let mut resumed = restart(&state.0); + assert_eq!(resumed.get_last_scanned_block_height(), FORK); + assert_eq!( + owned_height(&resumed, s.evicted), + None, + "no phantom on disk" + ); + assert_eq!( + owned_height(&resumed, s.reconfirmed), + None, + "not re-found yet" + ); + assert_eq!(balance(&resumed), 50_000); + + watch_step(&mut resumed, &s.new).await.expect("resume"); + assert_eq!(owned_height(&resumed, s.reconfirmed), Some(106)); + assert_eq!(owned_height(&resumed, s.evicted), None); + assert_eq!(balance(&resumed), 80_000); + assert_eq!(resumed.get_last_scanned_block_height(), 107); + + let resumed_again = restart(&state.0); + assert_eq!(balance(&resumed_again), 80_000, "final state is persisted"); + }); +} + +/// A fork below every remembered hash cannot be located: the scan fails +/// loudly with `ReorgTooDeep`, on every attempt, and leaves the state (in +/// memory and on disk) exactly as it was. +#[test] +fn reorg_deeper_than_the_lookback_is_a_loud_error_and_changes_nothing() { + run(async { + let (mut scanner, state) = wallet("too-deep"); + let (recv_tx, tweak, outpoint, _) = payment(0x51, 50_000); + let tip = 101 + u64::from(REORG_LOOKBACK) + 10; + let mut old = Oracle::default(); + old.block(0xa, 101, vec![(recv_tx, Some(tweak))], &[]); + for h in 102..=tip { + old.empty(0xa, h); + } + let mut new = old.branch_at(101); + for h in 102..=tip + 1 { + new.empty(0xb, h); + } + + scanner + .scan_range_from(101, tip, old.clone()) + .await + .expect("old chain scans"); + assert_eq!( + scanner.scanned_block_hashes.len(), + REORG_LOOKBACK as usize, + "only the lookback window is remembered" + ); + let hashes = scanner.scanned_block_hashes.clone(); + let records: Vec<_> = scanner.owned_outputs().cloned().collect(); + let on_disk = std::fs::read(&state.0).ok(); + + for _ in 0..2 { + let err = watch_step(&mut scanner, &new) + .await + .expect_err("a reorg below the lookback must fail"); + let deep = err + .downcast_ref::() + .unwrap_or_else(|| panic!("expected ReorgTooDeep, got: {err}")); + assert_eq!( + deep.lowest_known_height, + tip - u64::from(REORG_LOOKBACK) + 1 + ); + assert!(err.to_string().contains("rescan"), "says what to do: {err}"); + } + + assert_eq!(scanner.get_last_scanned_block_height(), tip); + assert_eq!(scanner.scanned_block_hashes, hashes); + assert_eq!( + scanner.owned_outputs().cloned().collect::>(), + records + ); + assert_eq!(owned_height(&scanner, outpoint), Some(101)); + assert_eq!( + std::fs::read(&state.0).ok(), + on_disk, + "state file untouched" + ); + }); +} + +/// With no reorg, the watch loop's check reads one block and changes +/// nothing, and continuing the chain still scans new blocks. +#[test] +fn unchanged_chain_passes_the_check() { + run(async { + let (mut scanner, _state) = wallet("unchanged"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + + watch_step(&mut scanner, &s.old).await.expect("check"); + assert_eq!(scanner.get_last_scanned_block_height(), 106); + assert_eq!(balance(&scanner), 100_000); + + let mut longer = s.old.clone(); + longer.empty(0xa, 107); + watch_step(&mut scanner, &longer).await.expect("extend"); + assert_eq!(scanner.get_last_scanned_block_height(), 107); + assert_eq!(balance(&scanner), 100_000); + }); +} diff --git a/blindbit-lib/src/scanner/scanner.rs b/blindbit-lib/src/scanner/scanner.rs index f20d892..d510a92 100644 --- a/blindbit-lib/src/scanner/scanner.rs +++ b/blindbit-lib/src/scanner/scanner.rs @@ -69,6 +69,14 @@ pub struct Scanner { /// Lets the per-block spent-output check run in O(spent prefixes). pub(crate) owned_prefixes: HashMap<[u8; 8], Vec>, + /// Hash of each scanned height within the reorg lookback window; see + /// [`ChangeSet::scanned_block_hashes`] and `scanner/reorg.rs`. + pub(crate) scanned_block_hashes: BTreeMap, + + /// sends the fork height whenever a chain reorganisation rolled the + /// wallet state back + pub(crate) notify_reorg: broadcast::Sender, + /// Staged changes that can be persisted pub(crate) stage: ChangeSet, @@ -127,6 +135,7 @@ impl Scanner { secret_scan_hex: Some(hex::encode(secret_scan.secret_bytes())), public_spend_hex: Some(hex::encode(public_spend.serialize())), max_label_num, + scanned_block_hashes: BTreeMap::new(), }; // Merge the initial label changes @@ -151,6 +160,8 @@ impl Scanner { last_scanned_block_height_rescan: 0, owned_outputs: BTreeMap::new(), owned_prefixes: HashMap::new(), + scanned_block_hashes: BTreeMap::new(), + notify_reorg: broadcast::channel(16).0, stage, state_file, network, @@ -396,6 +407,13 @@ impl Scanner { self.notify_spent_outpoints.subscribe() } + /// subscribe to chain reorganisations: each message is the fork height + /// the wallet state was rolled back to (everything above it was dropped + /// and is rescanned from the oracle's current chain) + pub fn subscribe_to_reorgs(&self) -> broadcast::Receiver { + self.notify_reorg.subscribe() + } + /// subscribe to notifications when a probabilistic match is found pub fn subscribe_to_probabilistic_matches(&self) -> broadcast::Receiver<[u8; 32]> { self.notify_probabilistic_matches.subscribe() @@ -572,6 +590,7 @@ impl Scanner { changeset.last_scanned_block_height = self.last_scanned_block_height; changeset.last_scanned_block_height_rescan = self.last_scanned_block_height_rescan; changeset.owned_outputs = self.owned_outputs.values().cloned().collect(); + changeset.scanned_block_hashes = self.scanned_block_hashes.clone(); let json = serde_json::to_string_pretty(&changeset)?; std::fs::write(path, json)?; @@ -700,6 +719,8 @@ impl Scanner { last_scanned_block_height_rescan: changeset.last_scanned_block_height_rescan, owned_outputs, owned_prefixes: HashMap::new(), + scanned_block_hashes: changeset.scanned_block_hashes.clone(), + notify_reorg: broadcast::channel(16).0, stage: changeset, state_file: state_file, network: network, diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index d105fd8..4f62f13 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -16,6 +16,7 @@ use crate::oracle_grpc::{ use super::electrum_index::{ScriptHashEntry, SpHistoryEntry, electrum_scripthash}; use super::p2p; +use super::reorg::{BlockCheck, ReorgBelowStreamStart, ReorgTooDeep, block_hash_from_oracle}; use super::scanner::Scanner; use super::types::{BlockIdentifierDisplay, ProbableMatch}; use super::utils::{byte_array_to_txid, construct_dummy_tx, match_short_pubkey}; @@ -69,6 +70,47 @@ impl BlockScanDataStream for tonic::Streaming { } } +/// Opens `StreamBlockScanDataShort` streams for a range scan. A scan may +/// open a second stream lower down when it has to locate a fork point. +pub(crate) trait BlockStreamSource { + type Stream: BlockScanDataStream + Send; + + fn open( + &mut self, + start: u64, + end: u64, + ) -> impl Future> + Send; +} + +/// The oracle's gRPC service as a [`BlockStreamSource`]. +struct OracleStreams(crate::oracle_grpc::oracle_service_client::OracleServiceClient); + +impl BlockStreamSource for OracleStreams { + type Stream = tonic::Streaming; + + async fn open(&mut self, start: u64, end: u64) -> Result { + let request = tonic::Request::new(RangedBlockHeightRequestFiltered { + start, + end, + dustlimit: 0, + cut_through: false, + }); + Ok(self + .0 + .stream_block_scan_data_short(request) + .await + .map_err(|status| -> ScannerError { + format!( + "oracle refused to stream blocks {start}..={end}: {:?}: {}", + status.code(), + status.message() + ) + .into() + })? + .into_inner()) + } +} + /// Read the next block message, turning an error status into a scanner error /// that names the height the scan stopped at. /// @@ -125,35 +167,68 @@ fn stream_stopped(height: u64, reason: &str) -> ScannerError { } impl Scanner { - /// scan a block range for new utxos and spent outpoints + /// Scan a block range for new utxos and spent outpoints, after checking + /// that the chain already scanned below it was not reorganised. + /// + /// When `start - 1` is a scanned height, its block is streamed too and + /// compared with the remembered hash; a reorganisation is rolled back to + /// the fork point and the new branch scanned (see `scanner/reorg.rs`). + /// `start = end + 1` only runs that check. A reorganisation deeper than + /// [`super::REORG_LOOKBACK`] is a [`ReorgTooDeep`] error. pub async fn scan_block_range( &mut self, start: u64, end: u64, ) -> Result<(), ScannerError> { - // TODO: Implement sync check if needed - - let request = tonic::Request::new(RangedBlockHeightRequestFiltered { - start, - end, - dustlimit: 0, - cut_through: false, - }); - let stream = self - .client - .stream_block_scan_data_short(request) - .await - .map_err(|status| -> ScannerError { - format!( - "oracle refused to stream blocks {start}..={end}: {:?}: {}", - status.code(), - status.message() - ) - .into() - })? - .into_inner(); + let source = OracleStreams(self.client.clone()); + self.scan_range_from(start, end, source).await + } - self.scan_block_stream(start, end, stream).await + /// [`Self::scan_block_range`] over any [`BlockStreamSource`]. + pub(crate) async fn scan_range_from( + &mut self, + start: u64, + end: u64, + mut source: Src, + ) -> Result<(), ScannerError> { + // Overlap one block with what was already scanned, so the stream + // itself proves the chain below `start` is unchanged. + let mut first = match start.checked_sub(1) { + Some(prev) if prev > 0 && self.knows_block_hash(prev) => prev, + _ => start, + }; + if first > end { + return Ok(()); + } + loop { + let stream = source.open(first, end).await?; + match self.scan_block_stream_from(first, start, end, stream).await { + Err(e) => { + let Some(reorg) = e.downcast_ref::() else { + return Err(e); + }; + // The fork is below the stream's first block: walk up + // from the oldest remembered hash to find it. + match self.lowest_known_block_height() { + Some(lowest) if lowest < reorg.height => { + tracing::warn!( + height = reorg.height, + from = lowest, + "chain reorganisation detected; locating the fork point" + ); + first = lowest; + } + _ => { + return Err(Box::new(ReorgTooDeep { + lowest_known_height: reorg.height, + last_scanned_height: self.last_scanned_block_height, + })); + } + } + } + ok => return ok, + } + } } /// Scan the per-block messages of one `StreamBlockScanDataShort` response @@ -166,10 +241,30 @@ impl Scanner { /// error. Such a block is never taken as "no payments": the scan height /// stays at the last block that was fully processed, so the next scan /// resumes at the block that could not be scanned. + #[cfg(test)] pub(crate) async fn scan_block_stream( &mut self, start: u64, end: u64, + stream: S, + ) -> Result<(), ScannerError> { + self.scan_block_stream_from(start, start, end, stream).await + } + + /// [`Self::scan_block_stream`] for a stream that begins at `first`, at or + /// below the scan's `start`. Blocks below `start` are only checked + /// against the remembered hashes, not scanned again. Any block whose + /// hash differs from the remembered one is a reorganisation: when a + /// block before it in this stream agreed, the one below it is the fork + /// point, the state is rolled back there and scanning goes on from that + /// block; when it is the stream's first block, the scan returns + /// [`ReorgBelowStreamStart`] (or [`ReorgTooDeep`] if nothing older is + /// remembered) without changing anything. + pub(crate) async fn scan_block_stream_from( + &mut self, + first: u64, + start: u64, + end: u64, mut stream: S, ) -> Result<(), ScannerError> { // Stamp sp_start_height into the index so the Electrum server's @@ -194,7 +289,14 @@ impl Scanner { let mut p2p_conn: Option = None; - let mut expected_height = start; + let mut expected_height = first; + // Blocks from here on are scanned; below it they are only checked. + let mut scan_from = start; + // Whether an earlier block of this stream is known to be on the + // oracle's chain (checked or scanned). + let mut have_prior_block = false; + // Whether anything was scanned or rolled back (worth a save). + let mut changed = false; while let Some(block_scan_data) = next_block(&mut stream, expected_height).await? { let Some(block_identifier) = block_scan_data.block_identifier.clone() else { @@ -206,6 +308,32 @@ impl Scanner { }; check_block_identifier(&block_identifier, expected_height)?; expected_height += 1; + let oracle_block_hash = block_hash_from_oracle(&block_identifier.block_hash) + .expect("block hash length checked above"); + let height = block_identifier.block_height; + match self.check_block_hash(height, &oracle_block_hash) { + BlockCheck::Reorganised if !have_prior_block => { + if self.lowest_known_block_height() == Some(height) { + return Err(Box::new(ReorgTooDeep { + lowest_known_height: height, + last_scanned_height: self.last_scanned_block_height, + })); + } + return Err(Box::new(ReorgBelowStreamStart { height })); + } + BlockCheck::Reorganised => { + self.rollback_to(height - 1).await?; + scan_from = scan_from.min(height); + } + BlockCheck::Consistent if height < scan_from => { + // Already scanned, still on the chain. + have_prior_block = true; + continue; + } + BlockCheck::Consistent => {} + } + have_prior_block = true; + changed = true; let block_id = BlockIdentifierDisplay(&block_identifier); tracing::debug!(height = block_id.0.block_height, "received block data from oracle"); @@ -235,6 +363,7 @@ impl Scanner { .await; self.last_scanned_block_height = block_identifier.block_height; self.stage.last_scanned_block_height = block_identifier.block_height; + self.record_scanned_block_hash(height, oracle_block_hash); // Periodically checkpoint progress so a crash/restart during a // long initial catch-up scan doesn't lose everything. #[cfg(feature = "serde")] @@ -497,6 +626,7 @@ impl Scanner { // Update last scanned block height and stage it self.last_scanned_block_height = block_identifier.block_height; self.stage.last_scanned_block_height = block_identifier.block_height; + self.record_scanned_block_hash(height, oracle_block_hash); } if expected_height <= end { @@ -506,6 +636,11 @@ impl Scanner { )); } + if !changed { + // Only checked blocks that were already scanned. + return Ok(()); + } + let outpoints: Vec<(u32, OutPoint)> = self .internal_indexer .index() @@ -665,10 +800,23 @@ impl Scanner { } }; - if oracle_tip > self.last_scanned_block_height { + // With no new block, still check the scanned tip (or the + // oracle's, if it is behind) against the oracle: a reorganisation + // can replace blocks without making the chain longer. + let (from, to) = if oracle_tip > self.last_scanned_block_height { let from = self.last_scanned_block_height + 1; tracing::info!(from, to = oracle_tip, "new blocks available, scanning"); - if let Err(e) = self.scan_block_range(from, oracle_tip).await { + (from, oracle_tip) + } else { + (oracle_tip + 1, oracle_tip) + }; + if let Err(e) = self.scan_block_range(from, to).await { + if e.downcast_ref::().is_some() { + tracing::error!( + error = %e, + "wallet state cannot follow the chain; scanning is halted until it is rescanned" + ); + } else { tracing::error!(error = %e, "scan_block_range failed, will retry on next poll"); } } diff --git a/blindbit-lib/src/scanner/stream_safety_tests.rs b/blindbit-lib/src/scanner/stream_safety_tests.rs index bca50f0..82e8932 100644 --- a/blindbit-lib/src/scanner/stream_safety_tests.rs +++ b/blindbit-lib/src/scanner/stream_safety_tests.rs @@ -47,7 +47,7 @@ pub(super) fn served_block(hash: &BlockHash) -> Option { .and_then(|blocks| blocks.get(hash).cloned()) } -fn serve(block: &Block) { +pub(super) fn serve(block: &Block) { SERVED_BLOCKS .lock() .unwrap() diff --git a/friglet/src/electrum/mod.rs b/friglet/src/electrum/mod.rs index 7297518..108dce0 100644 --- a/friglet/src/electrum/mod.rs +++ b/friglet/src/electrum/mod.rs @@ -104,7 +104,9 @@ struct ElectrumServerState { p2p_peer: SocketAddr, network: Network, oracle_url: String, - block_checkpoints: HashMap, + /// Wallet checkpoints (heights with findings) snapshotted at startup; + /// entries above a reorg's fork point are dropped when it happens. + block_checkpoints: std::sync::Mutex>, header_sidecar: PathBuf, header_fetch_lock: Mutex<()>, /// Broadcast channel: pre-serialised JSON notification lines sent to every @@ -124,6 +126,7 @@ struct ElectrumServerState { pub async fn run( index: Arc>, mut found_rx: broadcast::Receiver, + mut reorg_rx: broadcast::Receiver, addr: &str, p2p_peer: SocketAddr, network: Network, @@ -139,7 +142,7 @@ pub async fn run( p2p_peer, network, oracle_url, - block_checkpoints, + block_checkpoints: std::sync::Mutex::new(block_checkpoints), header_sidecar, header_fetch_lock: Mutex::new(()), push_tx: push_tx.clone(), @@ -180,6 +183,22 @@ pub async fn run( }); } + // Chain reorganisations: the scanner already dropped the disconnected + // blocks from the index; forget their hashes and cached headers here too + // so no header of a block that left the chain is served again. + { + let state = state.clone(); + tokio::spawn(async move { + loop { + match reorg_rx.recv().await { + Ok(fork) => forget_blocks_above(&state, fork).await, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => return, + } + } + }); + } + let listener = TcpListener::bind(addr).await?; tracing::info!(addr = %addr, "Electrum server listening"); @@ -277,9 +296,44 @@ async fn push_notifications(state: &ElectrumServerState) { .send(serde_json::to_string(¬ification).unwrap() + "\n"); } +/// Drop block hashes and headers above a reorg's fork point, and rewrite +/// the header sidecar without them. +async fn forget_blocks_above(state: &ElectrumServerState, fork: u32) { + state + .block_checkpoints + .lock() + .expect("checkpoint lock") + .retain(|height, _| *height <= fork); + let headers = { + let mut index = state.index.lock().await; + index.headers.retain(|height, _| *height <= fork); + index.headers.clone() + }; + tracing::warn!( + fork, + "chain reorganisation: dropped cached headers above the fork point" + ); + let sidecar = state.header_sidecar.clone(); + match tokio::task::spawn_blocking(move || blockheader::save_headers(&sidecar, &headers)).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!( + path = %state.header_sidecar.display(), + error = %error, + "failed to rewrite block-header sidecar after reorg" + ), + Err(error) => tracing::warn!(error = %error, "block-header sidecar write task failed"), + } +} + async fn resolve_block_hash(state: &ElectrumServerState, height: u32) -> Result { - if let Some(hash) = state.block_checkpoints.get(&height) { - return Ok(*hash); + let checkpoint = state + .block_checkpoints + .lock() + .expect("checkpoint lock") + .get(&height) + .copied(); + if let Some(hash) = checkpoint { + return Ok(hash); } let oracle_url = state.oracle_url.clone(); @@ -837,3 +891,62 @@ async fn handle_request(req: &JsonRpcRequest, state: &ElectrumServerState) -> St .into_line(), } } + +#[cfg(test)] +mod tests { + use super::*; + use bitcoin::hashes::Hash; + + #[tokio::test] + async fn reorg_forgets_hashes_and_headers_above_the_fork() { + let dir = + std::env::temp_dir().join(format!("friglet-electrum-reorg-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let sidecar = dir.join("scanner_state.headers.json"); + let hash = |b: u8| BlockHash::from_byte_array([b; 32]); + + let index = Arc::new(Mutex::new(WalletElectrumIndex::new())); + { + let mut idx = index.lock().await; + for height in [100u32, 104, 105, 106] { + idx.headers.insert(height, format!("header-{height}")); + } + } + let state = ElectrumServerState { + index: index.clone(), + p2p_peer: "127.0.0.1:1".parse().unwrap(), + network: Network::Regtest, + oracle_url: "http://127.0.0.1:1".into(), + block_checkpoints: std::sync::Mutex::new(HashMap::from([ + (0, hash(0)), + (104, hash(4)), + (105, hash(5)), + ])), + header_sidecar: sidecar.clone(), + header_fetch_lock: Mutex::new(()), + push_tx: broadcast::channel(1).0, + }; + + forget_blocks_above(&state, 104).await; + + let mut checkpoints: Vec = state + .block_checkpoints + .lock() + .unwrap() + .keys() + .copied() + .collect(); + checkpoints.sort(); + assert_eq!(checkpoints, vec![0, 104]); + let mut cached: Vec = index.lock().await.headers.keys().copied().collect(); + cached.sort(); + assert_eq!(cached, vec![100, 104]); + let mut persisted: Vec = blockheader::load_headers(&sidecar) + .keys() + .copied() + .collect(); + persisted.sort(); + assert_eq!(persisted, vec![100, 104], "sidecar rewritten without them"); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/friglet/src/main.rs b/friglet/src/main.rs index 549017f..5953c88 100644 --- a/friglet/src/main.rs +++ b/friglet/src/main.rs @@ -130,7 +130,7 @@ async fn run(args: ScanArgs) -> Result<(), Box Result<(), Box Result<(), Box Date: Thu, 1 Oct 2026 13:31:11 +0000 Subject: [PATCH 06/10] fix(scanner): start at the oracle's first block when the wallet starts below it; report stalls A wallet whose start height lies below the oracle's first indexed block (the hosted oracle has nothing below 800000 on mainnet and 100000 on signet) stopped at its first height on every poll, forever, with only an error-level log line. Since #20 refuses to scan past an unindexed height, that is the only thing such a wallet ever did. - When `watch_chain`'s scan stops at the wallet's start height because the oracle has not indexed it, the scanner locates the oracle's lowest indexed height with GetBlockHashByHeight lookups and starts there, logs a warning, persists the adjustment (`oracle_floor_start`, optional in the state file) and publishes it in the scan health. - Only heights below the oracle's floor are skipped. A height above the first scanned block, or one with an indexed height below it (a gap in the oracle's index), keeps #20's stop-and-retry. The search is a binary search verified by exponential back-off probes below the boundary, so a gap above the floor does not move the floor up. NOT_FOUND and an empty or zero hash both mean "not indexed". - A scan that cannot get past a height (or cannot reach the oracle) is published as a stall: height, reason, and the time of the first failed attempt at that height. It clears when the scan makes progress. - Stream stops are a typed `ScanStopped` error (same message text) so the loop can tell "not indexed" from other failures. - Health lives in the wallet's Electrum index, which status readers lock without waiting on the scan task. friglet's GetStatus reports it as `StatusInfo::scan_health` (serde default, so old trays and daemons interoperate), and the tray's status window shows a stall line and a note for an adjusted start. Co-Authored-By: Claude Opus 5.5 --- blindbit-lib/src/scanner/changeset.rs | 13 + blindbit-lib/src/scanner/electrum_index.rs | 6 + blindbit-lib/src/scanner/health.rs | 319 ++++++++++++++++ blindbit-lib/src/scanner/mod.rs | 4 + .../src/scanner/oracle_floor_tests.rs | 339 ++++++++++++++++++ blindbit-lib/src/scanner/scanner.rs | 3 + blindbit-lib/src/scanner/scanning.rs | 73 ++-- .../src/scanner/stream_safety_tests.rs | 22 +- friglet-ipc/src/lib.rs | 48 +++ friglet-ipc/tests/roundtrip.rs | 1 + friglet-tray/examples/fake-daemon.rs | 1 + friglet-tray/src/lib.rs | 1 + friglet-tray/tests/lifecycle_e2e.rs | 1 + friglet-tray/ui/index.html | 8 + friglet-tray/ui/main.js | 26 ++ friglet-tray/ui/style.css | 1 + friglet/src/control/mod.rs | 61 ++++ 17 files changed, 896 insertions(+), 31 deletions(-) create mode 100644 blindbit-lib/src/scanner/health.rs create mode 100644 blindbit-lib/src/scanner/oracle_floor_tests.rs diff --git a/blindbit-lib/src/scanner/changeset.rs b/blindbit-lib/src/scanner/changeset.rs index 66ca190..ef2f5ec 100644 --- a/blindbit-lib/src/scanner/changeset.rs +++ b/blindbit-lib/src/scanner/changeset.rs @@ -3,6 +3,8 @@ use indexer::bdk_chain::ConfirmationBlockTime; use indexer::bdk_chain::bdk_core::Merge; use std::collections::BTreeMap; +use super::health::OracleFloorStart; + /// Helper module for hex encoding/decoding byte arrays in serialization #[cfg(feature = "serde")] mod serde_hex { @@ -68,6 +70,13 @@ pub struct ChangeSet { pub public_spend_hex: Option, /// Maximum label number used pub max_label_num: u32, + /// Set when the wallet's start height lay below the oracle's first + /// indexed block and scanning began there instead. + #[cfg_attr( + feature = "serde", + serde(default, skip_serializing_if = "Option::is_none") + )] + pub oracle_floor_start: Option, } impl Merge for ChangeSet { @@ -104,6 +113,10 @@ impl Merge for ChangeSet { if other.max_label_num > self.max_label_num { self.max_label_num = other.max_label_num; } + + if other.oracle_floor_start.is_some() { + self.oracle_floor_start = other.oracle_floor_start; + } } /// Checks if the [`ChangeSet`] is empty (contains no changes). diff --git a/blindbit-lib/src/scanner/electrum_index.rs b/blindbit-lib/src/scanner/electrum_index.rs index dde8d28..43a5a98 100644 --- a/blindbit-lib/src/scanner/electrum_index.rs +++ b/blindbit-lib/src/scanner/electrum_index.rs @@ -7,6 +7,8 @@ use bitcoin::ScriptBuf; use bitcoin::hashes::{Hash, sha256}; use std::collections::HashMap; +use super::health::ScanHealth; + /// Lightweight Electrum-compatible index built from scanner state. #[derive(Default)] pub struct WalletElectrumIndex { @@ -54,6 +56,10 @@ pub struct WalletElectrumIndex { /// Ordered list of confirmed SP receives. pub sp_history: Vec, + + /// Stalls and start-height adjustments of the running scan, published + /// here for status readers (see `scanner/health.rs`). + pub scan_health: ScanHealth, } /// A single entry in a scripthash's transaction history. diff --git a/blindbit-lib/src/scanner/health.rs b/blindbit-lib/src/scanner/health.rs new file mode 100644 index 0000000..f80456d --- /dev/null +++ b/blindbit-lib/src/scanner/health.rs @@ -0,0 +1,319 @@ +//! What a long-running scan (`Scanner::watch_chain`) reports besides its +//! progress: why it is stuck, and where it started when the wallet's start +//! height lay below the oracle's first indexed block. +//! +//! The scan task holds the scanner lock for as long as it runs, so these +//! conditions are published through the wallet's [`WalletElectrumIndex`] +//! (`scan_health`), which status readers can lock without waiting for the +//! scan. +//! +//! [`WalletElectrumIndex`]: super::electrum_index::WalletElectrumIndex + +use std::future::Future; +use std::time::{SystemTime, UNIX_EPOCH}; + +use bitcoin::BlockHash; +use bitcoin::hashes::Hash; +use tonic::transport::Channel; + +use super::ScannerError; +use super::scanner::Scanner; +use crate::oracle_grpc::BlockHeightRequest; +use crate::oracle_grpc::oracle_service_client::OracleServiceClient; + +/// Conditions of a running scan worth showing to the user. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ScanHealth { + /// Set while the scan cannot get past a height. The scan keeps retrying; + /// the entry is cleared as soon as it makes progress again. + pub stall: Option, + /// Set when the wallet's start height lay below the oracle's first + /// indexed block, so scanning began at the oracle's floor instead. + pub oracle_floor_start: Option, +} + +/// The scan cannot get past `height`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScanStall { + /// First height that could not be scanned (the scan resumes here). + pub height: u64, + /// Why, as the scanner reported it. + pub reason: String, + /// Unix time (seconds) of the first failed attempt at this height. + pub since_unix: u64, +} + +/// The wallet's start height had no oracle data, so scanning began at the +/// oracle's lowest indexed height. Blocks `requested_height..floor_height` +/// were not scanned. Persisted in the state file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct OracleFloorStart { + /// The start height the wallet asked for. + pub requested_height: u64, + /// The oracle's lowest indexed height, where scanning began. + pub floor_height: u64, +} + +/// A range scan stopped at `height` without scanning it. +/// +/// Every block below `height` in the range was fully processed; nothing from +/// `height` on was. The caller resumes at `height`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScanStopped { + /// First height that was not scanned. + pub height: u64, + /// What the oracle sent (or failed to send) for `height`. + pub reason: String, + /// The oracle reported `height` as not indexed: an empty or zero block + /// hash, or a `NOT_FOUND` status. + pub not_indexed: bool, +} + +impl std::fmt::Display for ScanStopped { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let height = self.height; + write!( + f, + "scan stopped at height {height}: {}; nothing from height {height} on was scanned, \ + rescan from height {height}", + self.reason + ) + } +} + +impl std::error::Error for ScanStopped {} + +/// The oracle's per-height block hash lookup (`GetBlockHashByHeight`). +pub(crate) trait OracleProbe { + /// The block hash the oracle has indexed at `height`, or `None` when it + /// has not indexed that height (an empty or zero hash, or `NOT_FOUND`). + /// Any other failure is an error. + fn block_hash_at( + &mut self, + height: u64, + ) -> impl Future, ScannerError>> + Send; +} + +impl OracleProbe for OracleServiceClient { + async fn block_hash_at(&mut self, height: u64) -> Result, ScannerError> { + let request = tonic::Request::new(BlockHeightRequest { + block_height: height, + }); + match self.get_block_hash_by_height(request).await { + Ok(response) => Ok(indexed_block_hash(&response.into_inner().block_hash)), + Err(status) if status.code() == tonic::Code::NotFound => Ok(None), + Err(status) => Err(format!( + "oracle GetBlockHashByHeight({height}) failed: {:?}: {}", + status.code(), + status.message() + ) + .into()), + } + } +} + +/// A block hash as the oracle serves it (display order), or `None` when it is +/// not a real hash: what an oracle answers for a height it has not indexed. +pub(crate) fn indexed_block_hash(display_order: &[u8]) -> Option { + let mut bytes: [u8; 32] = display_order.try_into().ok()?; + if bytes.iter().all(|b| *b == 0) { + return None; + } + bytes.reverse(); + Some(BlockHash::from_byte_array(bytes)) +} + +/// Upper bound on `GetBlockHashByHeight` calls one floor search may make. +/// A contiguous index needs about `3 * log2(tip)` (~60 on mainnet). +const FLOOR_SEARCH_MAX_PROBES: u32 = 400; + +/// Find the oracle's lowest indexed height when `from`, the wallet's start +/// height, is not indexed. +/// +/// Returns `Some(floor)` only when `from` lies *below* the oracle's floor: +/// nothing at or below `from` is indexed and `floor` is the lowest indexed +/// height above it. Returns `None` when `from` is a gap *above* the floor +/// (some lower height is indexed), when `from` is indexed after all, or when +/// the oracle has no data even at `tip`; such a height must not be skipped. +/// +/// The index need not be contiguous. A binary search finds a boundary, then +/// probes at exponentially growing distances below it look for indexed +/// heights the search jumped over; any hit restarts the search below it. The +/// same back-off below `from` tells a gap above the floor from the region +/// below it. A gap can only be missed when it is wider than the indexed run +/// below it, which no real oracle produces. +pub(crate) async fn find_oracle_floor( + probe: &mut P, + from: u64, + tip: u64, +) -> Result, ScannerError> { + let mut oracle = CountedProbe { probe, probes: 0 }; + + if from >= tip || oracle.indexed(from).await? { + return Ok(None); + } + // Anything indexed below `from` makes it a gap above the floor. + let mut distance = 1u64; + while distance <= from { + if oracle.indexed(from - distance).await? { + return Ok(None); + } + distance *= 2; + } + + if !oracle.indexed(tip).await? { + return Ok(None); + } + // Invariant: `from` is not indexed, `hi` is. + let mut hi = tip; + loop { + let mut lo = from; + while hi - lo > 1 { + let mid = lo + (hi - lo) / 2; + if oracle.indexed(mid).await? { + hi = mid; + } else { + lo = mid; + } + } + // `hi - 1` is not indexed; look further down for indexed runs the + // search jumped over. + let mut lower = None; + let mut distance = 2u64; + while hi - from > distance { + if oracle.indexed(hi - distance).await? { + lower = Some(hi - distance); + break; + } + distance *= 2; + } + match lower { + Some(height) => hi = height, + None => return Ok(Some(hi)), + } + } +} + +/// An [`OracleProbe`] with a lookup budget. +struct CountedProbe<'a, P> { + probe: &'a mut P, + probes: u32, +} + +impl CountedProbe<'_, P> { + async fn indexed(&mut self, height: u64) -> Result { + self.probes += 1; + if self.probes > FLOOR_SEARCH_MAX_PROBES { + return Err(format!( + "gave up locating the oracle's first indexed block after \ + {FLOOR_SEARCH_MAX_PROBES} lookups" + ) + .into()); + } + Ok(self.probe.block_hash_at(height).await?.is_some()) + } +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +impl Scanner { + /// Current scan health as published to status readers. + pub async fn scan_health(&self) -> ScanHealth { + self.electrum_index.lock().await.scan_health.clone() + } + + /// Publish that the scan cannot get past `height`. Repeated failures at + /// the same height keep the time of the first one. + pub(crate) async fn report_stall(&self, height: u64, reason: String) { + let mut idx = self.electrum_index.lock().await; + let since_unix = match &idx.scan_health.stall { + Some(stall) if stall.height == height => stall.since_unix, + _ => unix_now(), + }; + idx.scan_health.stall = Some(ScanStall { + height, + reason, + since_unix, + }); + } + + pub(crate) async fn clear_stall(&self) { + let mut idx = self.electrum_index.lock().await; + if idx.scan_health.stall.take().is_some() { + tracing::info!( + height = self.last_scanned_block_height, + "scan is making progress again" + ); + } + } + + /// After a failed scan of `from..=tip`: when `from` is the wallet's start + /// height and lies below the oracle's first indexed block, move the + /// start up to that block and return `true` (scan again right away). + /// + /// Only heights below the oracle's floor are ever skipped. A height the + /// oracle has not indexed above its floor (a gap) is never skipped; the + /// scan keeps stopping there until the oracle serves it. + pub(crate) async fn start_at_oracle_floor_if_below( + &mut self, + error: &ScannerError, + from: u64, + tip: u64, + probe: &mut P, + ) -> bool { + let Some(stopped) = error.downcast_ref::() else { + return false; + }; + if !stopped.not_indexed || stopped.height != from { + return false; + } + // Only at the wallet's start: once a block was scanned, the height + // after it is a gap, not the floor. + let wallet_start = self.electrum_index.lock().await.sp_start_height; + if wallet_start == 0 || from > wallet_start { + return false; + } + let floor = match find_oracle_floor(probe, from, tip).await { + Ok(Some(floor)) => floor, + Ok(None) => return false, + Err(e) => { + tracing::warn!( + error = %e, + height = from, + "could not locate the oracle's first indexed block" + ); + return false; + } + }; + + tracing::warn!( + requested_height = from, + oracle_floor = floor, + skipped_blocks = floor - from, + "the wallet's start height is below the oracle's first indexed block; scanning \ + starts at the oracle's first block, and blocks below it are not scanned" + ); + let note = OracleFloorStart { + requested_height: from, + floor_height: floor, + }; + self.update_last_scanned_block_height(floor - 1); + self.stage.oracle_floor_start = Some(note); + { + let mut idx = self.electrum_index.lock().await; + idx.scan_health.oracle_floor_start = Some(note); + idx.scan_health.stall = None; + } + #[cfg(feature = "serde")] + if let Err(e) = self.save_to_file(&self.state_file) { + tracing::warn!(error = %e, "failed to save state"); + } + true + } +} diff --git a/blindbit-lib/src/scanner/mod.rs b/blindbit-lib/src/scanner/mod.rs index 9c4aaca..e548884 100644 --- a/blindbit-lib/src/scanner/mod.rs +++ b/blindbit-lib/src/scanner/mod.rs @@ -3,7 +3,10 @@ mod changeset; mod config; pub mod electrum_index; +mod health; mod load; +#[cfg(test)] +mod oracle_floor_tests; mod p2p; mod scanner; mod scanning; @@ -19,6 +22,7 @@ pub type ScannerError = Box; pub use changeset::ChangeSet; pub use config::ScannerConfig; pub use electrum_index::{ScriptHashEntry, SpHistoryEntry, WalletElectrumIndex, electrum_scripthash, electrum_status}; +pub use health::{OracleFloorStart, ScanHealth, ScanStall, ScanStopped}; pub use p2p::broadcast_tx; pub use scanner::Scanner; pub use types::{BlockIdentifierDisplay, OwnedOutput}; diff --git a/blindbit-lib/src/scanner/oracle_floor_tests.rs b/blindbit-lib/src/scanner/oracle_floor_tests.rs new file mode 100644 index 0000000..ab80713 --- /dev/null +++ b/blindbit-lib/src/scanner/oracle_floor_tests.rs @@ -0,0 +1,339 @@ +//! A wallet whose start height lies below the oracle's first indexed block +//! starts at that block instead of stalling forever; a height the oracle has +//! not indexed *above* its first block is never skipped. A scan that cannot +//! get past a height is published as a stall. + +use std::future::Future; +use std::ops::RangeInclusive; + +use bitcoin::BlockHash; +use bitcoin::hashes::Hash; + +use super::ScannerError; +use super::health::{OracleProbe, ScanStopped, find_oracle_floor}; +use super::scanner::Scanner; +use super::stream_safety_tests::{ + TestStream, owned_outputs, payment_block, run, scanner, unindexed_message, +}; + +/// `GetBlockHashByHeight` over fixed runs of indexed heights. +struct FakeOracle { + indexed: Vec>, + probes: u32, + fail: bool, +} + +impl FakeOracle { + fn contiguous(floor: u64, tip: u64) -> Self { + Self { + indexed: std::iter::once(floor..=tip).collect(), + probes: 0, + fail: false, + } + } + + fn empty() -> Self { + Self { + indexed: vec![], + probes: 0, + fail: false, + } + } + + fn without(mut self, gap: RangeInclusive) -> Self { + self.indexed = self + .indexed + .into_iter() + .flat_map(|run| { + let below = *run.start()..=(*gap.start()).min(run.end() + 1).saturating_sub(1); + let above = (*gap.end() + 1).max(*run.start())..=*run.end(); + [below, above] + }) + .filter(|run| !run.is_empty()) + .collect(); + self + } + + fn is_indexed(&self, height: u64) -> bool { + self.indexed.iter().any(|run| run.contains(&height)) + } +} + +impl OracleProbe for FakeOracle { + fn block_hash_at( + &mut self, + height: u64, + ) -> impl Future, ScannerError>> + Send { + self.probes += 1; + let result = if self.fail { + Err("oracle unavailable".into()) + } else { + Ok(self + .is_indexed(height) + .then(|| BlockHash::from_byte_array([1; 32]))) + }; + std::future::ready(result) + } +} + +fn floor(oracle: &mut FakeOracle, from: u64, tip: u64) -> Option { + run(find_oracle_floor(oracle, from, tip)).expect("probe succeeds") +} + +/// The hosted signet oracle has nothing below 100 000; mainnet nothing below +/// 800 000. A search over the whole chain stays cheap. +#[test] +fn start_below_a_contiguous_index_finds_its_first_block() { + let mut signet = FakeOracle::contiguous(100_000, 324_494); + assert_eq!(floor(&mut signet, 1, 324_494), Some(100_000)); + assert!(signet.probes <= 60, "{} lookups", signet.probes); + + let mut mainnet = FakeOracle::contiguous(800_000, 965_000); + assert_eq!(floor(&mut mainnet, 700_000, 965_000), Some(800_000)); + assert_eq!(floor(&mut mainnet, 799_999, 965_000), Some(800_000)); + assert!(mainnet.probes <= 140, "{} lookups", mainnet.probes); +} + +/// Before blindbit-oracle#66 a fresh oracle never indexed its configured +/// start height: nothing at or below it, everything above it. +#[test] +fn oracle_missing_its_own_start_height_starts_one_above() { + let mut oracle = FakeOracle::contiguous(101, 300); + assert_eq!(floor(&mut oracle, 100, 300), Some(101)); + assert_eq!(floor(&mut oracle, 1, 300), Some(101)); +} + +/// A binary search alone would land in the gap at 2000..=2010 (the midpoint +/// of 10..4000 is 2005) and report 2011, skipping 1000 indexed blocks. +#[test] +fn gap_above_the_floor_does_not_move_the_floor_up() { + let mut oracle = FakeOracle::contiguous(1_000, 4_000).without(2_000..=2_010); + assert_eq!(floor(&mut oracle, 10, 4_000), Some(1_000)); +} + +#[test] +fn start_inside_a_gap_above_the_floor_is_never_skipped() { + let mut oracle = FakeOracle::contiguous(1_000, 4_000).without(2_000..=2_010); + for from in [2_000, 2_005, 2_010] { + assert_eq!(floor(&mut oracle, from, 4_000), None, "from {from}"); + } + // One missing block right above a long indexed run. + let mut oracle = FakeOracle::contiguous(1_000, 4_000).without(3_000..=3_000); + assert_eq!(floor(&mut oracle, 3_000, 4_000), None); +} + +/// Randomised layouts: a floor, then a handful of gaps above it, each +/// narrower than the run below it. The search must always return the +/// floor, and never skip from inside a gap. +#[test] +fn randomised_gappy_indexes() { + let mut seed = 0x9e37_79b9_7f4a_7c15_u64; + let mut next = move |bound: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % bound + }; + for _ in 0..2_000 { + let tip = 5_000 + next(1_000_000); + let floor_height = 1 + next(tip - 100); + let mut oracle = FakeOracle::contiguous(floor_height, tip); + let mut gaps = Vec::new(); + let mut cursor = floor_height; + for _ in 0..next(6) { + let run = 50 + next(5_000); + let start = cursor + run; + let width = 1 + next(run.min(40)); + if start + width >= tip { + break; + } + oracle = oracle.without(start..=start + width - 1); + gaps.push(start..=start + width - 1); + cursor = start + width; + } + let from = next(floor_height); + assert_eq!( + floor(&mut oracle, from, tip), + Some(floor_height), + "floor {floor_height} tip {tip} gaps {gaps:?} from {from}" + ); + for gap in &gaps { + assert_eq!(floor(&mut oracle, *gap.start(), tip), None, "gap {gap:?}"); + } + } +} + +#[test] +fn indexed_start_or_empty_oracle_is_not_a_floor() { + let mut oracle = FakeOracle::contiguous(100, 300); + assert_eq!(floor(&mut oracle, 150, 300), None, "start is indexed"); + let mut empty = FakeOracle::empty(); + assert_eq!( + floor(&mut empty, 1, 300), + None, + "nothing indexed at the tip" + ); +} + +#[test] +fn oracle_error_is_an_error_not_a_floor() { + let mut oracle = FakeOracle::contiguous(100, 300); + oracle.fail = true; + assert!(run(find_oracle_floor(&mut oracle, 1, 300)).is_err()); +} + +fn not_indexed_at(height: u64) -> ScannerError { + Box::new(ScanStopped { + height, + reason: "the oracle sent no valid block hash (0 bytes)".into(), + not_indexed: true, + }) +} + +async fn set_wallet_start(scanner: &mut Scanner, start: u64) { + scanner.update_last_scanned_block_height(start - 1); + scanner.electrum_index.lock().await.sp_start_height = start; +} + +/// End to end through the scan loop's pieces: the first stream stops at the +/// unindexed start height, the wallet moves to the oracle's first block, the +/// next stream from there finds the payment in it, and the adjustment is in +/// the status and survives a restart. +#[test] +fn wallet_start_below_the_floor_starts_at_the_floor_and_says_so() { + run(async { + let (mut scanner, state) = scanner("floor-start", "http://127.0.0.1:1"); + set_wallet_start(&mut scanner, 50).await; + let paid = payment_block(1_000); + let mut oracle = FakeOracle::contiguous(1_000, 1_005); + + let err = scanner + .scan_block_stream(50, 1_005, TestStream::new(vec![Ok(unindexed_message(50))])) + .await + .expect_err("the unindexed start height stops the scan"); + assert!( + scanner + .start_at_oracle_floor_if_below(&err, 50, 1_005, &mut oracle) + .await + ); + assert_eq!(scanner.get_last_scanned_block_height(), 999); + let health = scanner.scan_health().await; + let note = health.oracle_floor_start.expect("adjustment is published"); + assert_eq!((note.requested_height, note.floor_height), (50, 1_000)); + assert_eq!(health.stall, None); + + scanner + .scan_block_stream( + 1_000, + 1_000, + TestStream::new(vec![Ok(paid.message.clone())]), + ) + .await + .expect("scan from the floor"); + assert_eq!( + owned_outputs(&scanner), + 1, + "the payment at the floor is found" + ); + + #[cfg(feature = "serde")] + { + scanner.save_to_file(&state.0).expect("save"); + let restored = Scanner::from_changeset( + scanner.client.clone(), + scanner.p2p_peer(), + Scanner::load_from_file(&state.0).expect("load"), + state.0.clone(), + scanner.network(), + ) + .expect("restore"); + assert_eq!(restored.scan_health().await.oracle_floor_start, Some(note)); + } + let _ = &state; + }); +} + +/// Once a block was scanned, an unindexed height after it is a gap in the +/// oracle's index, not its floor: the scan keeps stopping there. +#[test] +fn unindexed_height_after_the_wallet_start_is_never_skipped() { + run(async { + let (mut scanner, _state) = scanner("floor-gap", "http://127.0.0.1:1"); + set_wallet_start(&mut scanner, 1_000).await; + scanner.update_last_scanned_block_height(1_499); + // Even an oracle that has nothing below 1 600 does not justify it. + let mut oracle = FakeOracle::contiguous(1_600, 2_000); + assert!( + !scanner + .start_at_oracle_floor_if_below(¬_indexed_at(1_500), 1_500, 2_000, &mut oracle) + .await + ); + assert_eq!(scanner.get_last_scanned_block_height(), 1_499); + assert_eq!(oracle.probes, 0); + assert_eq!(scanner.scan_health().await.oracle_floor_start, None); + }); +} + +#[test] +fn only_a_not_indexed_stop_at_the_start_moves_the_start() { + run(async { + let (mut scanner, _state) = scanner("floor-other", "http://127.0.0.1:1"); + set_wallet_start(&mut scanner, 50).await; + let mut oracle = FakeOracle::contiguous(1_000, 1_005); + let unavailable: ScannerError = Box::new(ScanStopped { + height: 50, + reason: "oracle stream error Unavailable".into(), + not_indexed: false, + }); + let plain: ScannerError = "failed to fetch block".into(); + for err in [unavailable, plain, not_indexed_at(51)] { + assert!( + !scanner + .start_at_oracle_floor_if_below(&err, 50, 1_005, &mut oracle) + .await, + "{err}" + ); + } + assert_eq!(scanner.get_last_scanned_block_height(), 49); + // Start height inside a gap above the floor. + let mut gappy = FakeOracle::contiguous(10, 1_005).without(40..=60); + assert!( + !scanner + .start_at_oracle_floor_if_below(¬_indexed_at(50), 50, 1_005, &mut gappy) + .await + ); + assert_eq!(scanner.get_last_scanned_block_height(), 49); + }); +} + +#[test] +fn a_stall_keeps_its_start_time_until_the_height_changes_and_clears_on_progress() { + run(async { + let (scanner, _state) = scanner("stall", "http://127.0.0.1:1"); + scanner.report_stall(1_500, "first".into()).await; + let first = scanner.scan_health().await.stall.expect("stall"); + { + let mut idx = scanner.electrum_index.lock().await; + idx.scan_health.stall.as_mut().unwrap().since_unix -= 600; + } + scanner.report_stall(1_500, "again".into()).await; + let again = scanner.scan_health().await.stall.expect("stall"); + assert_eq!(again.height, 1_500); + assert_eq!(again.reason, "again"); + assert_eq!( + again.since_unix, + first.since_unix - 600, + "same height keeps its start" + ); + + scanner.report_stall(1_600, "later".into()).await; + let moved = scanner.scan_health().await.stall.expect("stall"); + assert!( + moved.since_unix >= first.since_unix, + "a new height starts a new stall" + ); + + scanner.clear_stall().await; + assert_eq!(scanner.scan_health().await.stall, None); + }); +} diff --git a/blindbit-lib/src/scanner/scanner.rs b/blindbit-lib/src/scanner/scanner.rs index 8e5e65f..c5d7c7e 100644 --- a/blindbit-lib/src/scanner/scanner.rs +++ b/blindbit-lib/src/scanner/scanner.rs @@ -121,6 +121,7 @@ impl Scanner { secret_scan_hex: Some(hex::encode(secret_scan.secret_bytes())), public_spend_hex: Some(hex::encode(public_spend.serialize())), max_label_num, + oracle_floor_start: None, }; // Merge the initial label changes @@ -582,6 +583,7 @@ impl Scanner { let sp_address = indexer .get_address(convert_network(network)) .to_string(); + let oracle_floor_start = changeset.oracle_floor_start; Ok(Self { client, @@ -602,6 +604,7 @@ impl Scanner { let mut idx = WalletElectrumIndex::new(); idx.sp_address = sp_address; idx.sp_labels = (0..=max_label_num).collect(); + idx.scan_health.oracle_floor_start = oracle_floor_start; idx })), }) diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index d8096fa..14fb3cd 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -15,6 +15,7 @@ use crate::oracle_grpc::{ }; use super::electrum_index::{ScriptHashEntry, SpHistoryEntry, electrum_scripthash}; +use super::health::ScanStopped; use super::p2p; use super::scanner::Scanner; use super::types::{BlockIdentifierDisplay, ProbableMatch}; @@ -79,10 +80,11 @@ async fn next_block( height: u64, ) -> Result, ScannerError> { stream.next_message().await.map_err(|status| { - stream_stopped( + Box::new(ScanStopped { height, - &format!("oracle stream error {:?}: {}", status.code(), status.message()), - ) + reason: format!("oracle stream error {:?}: {}", status.code(), status.message()), + not_indexed: status.code() == tonic::Code::NotFound, + }) as ScannerError }) } @@ -105,23 +107,24 @@ fn check_block_identifier( } let hash = &block_identifier.block_hash; if hash.len() != 32 || hash.iter().all(|b| *b == 0) { - return Err(stream_stopped( - expected_height, - &format!( + return Err(Box::new(ScanStopped { + height: expected_height, + reason: format!( "the oracle sent no valid block hash ({} bytes); the height is probably not indexed", hash.len() ), - )); + not_indexed: true, + })); } Ok(()) } fn stream_stopped(height: u64, reason: &str) -> ScannerError { - format!( - "scan stopped at height {height}: {reason}; nothing from height {height} on was scanned, \ - rescan from height {height}" - ) - .into() + Box::new(ScanStopped { + height, + reason: reason.to_string(), + not_indexed: false, + }) } impl Scanner { @@ -144,12 +147,15 @@ impl Scanner { .stream_block_scan_data_short(request) .await .map_err(|status| -> ScannerError { - format!( - "oracle refused to stream blocks {start}..={end}: {:?}: {}", - status.code(), - status.message() - ) - .into() + Box::new(ScanStopped { + height: start, + reason: format!( + "oracle refused to stream blocks {start}..={end}: {:?}: {}", + status.code(), + status.message() + ), + not_indexed: status.code() == tonic::Code::NotFound, + }) })? .into_inner(); @@ -674,6 +680,12 @@ impl Scanner { /// height advances past `last_scanned_block_height`, the missing range is /// scanned with `scan_block_range`. This mirrors `blindbit-desktop`'s /// `Watch()` loop and means callers never need to supply an `end_height`. + /// + /// A scan that cannot get past a height is retried on every poll and + /// published as a stall ([`Scanner::scan_health`]) until it makes + /// progress. When the wallet's start height lies below the oracle's first + /// indexed block, scanning starts at that block instead (see + /// [`Scanner::start_at_oracle_floor_if_below`]). pub async fn watch_chain(&mut self) -> Result<(), ScannerError> { loop { let oracle_tip = match self @@ -684,6 +696,11 @@ impl Scanner { Ok(resp) => resp.into_inner().height, Err(e) => { tracing::error!(error = %e, "failed to reach oracle, will retry"); + self.report_stall( + self.last_scanned_block_height + 1, + format!("cannot reach the oracle: {e}"), + ) + .await; time::sleep(time::Duration::from_secs(10)).await; continue; } @@ -692,9 +709,25 @@ impl Scanner { if oracle_tip > self.last_scanned_block_height { let from = self.last_scanned_block_height + 1; tracing::info!(from, to = oracle_tip, "new blocks available, scanning"); - if let Err(e) = self.scan_block_range(from, oracle_tip).await { - tracing::error!(error = %e, "scan_block_range failed, will retry on next poll"); + match self.scan_block_range(from, oracle_tip).await { + Ok(()) => self.clear_stall().await, + Err(e) => { + let mut probe = self.client.clone(); + if self + .start_at_oracle_floor_if_below(&e, from, oracle_tip, &mut probe) + .await + { + continue; + } + tracing::error!(error = %e, "scan_block_range failed, will retry on next poll"); + let height = e + .downcast_ref::() + .map_or(self.last_scanned_block_height + 1, |s| s.height); + self.report_stall(height, e.to_string()).await; + } } + } else { + self.clear_stall().await; } time::sleep(time::Duration::from_secs(10)).await; diff --git a/blindbit-lib/src/scanner/stream_safety_tests.rs b/blindbit-lib/src/scanner/stream_safety_tests.rs index bca50f0..900f526 100644 --- a/blindbit-lib/src/scanner/stream_safety_tests.rs +++ b/blindbit-lib/src/scanner/stream_safety_tests.rs @@ -56,10 +56,10 @@ fn serve(block: &Block) { } /// In-process stand-in for the oracle's `StreamBlockScanDataShort` stream. -struct TestStream(VecDeque>); +pub(super) struct TestStream(VecDeque>); impl TestStream { - fn new(items: Vec>) -> Self { + pub(super) fn new(items: Vec>) -> Self { Self(items.into()) } } @@ -73,7 +73,7 @@ impl BlockScanDataStream for TestStream { } } -fn run(future: F) -> F::Output { +pub(super) fn run(future: F) -> F::Output { tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -89,7 +89,7 @@ fn keys() -> (SecretKey, PublicKey) { (secret(0x11), secret(0x22).public_key(&Secp256k1::new())) } -struct TempState(PathBuf); +pub(super) struct TempState(pub(super) PathBuf); impl Drop for TempState { fn drop(&mut self) { @@ -99,7 +99,7 @@ impl Drop for TempState { /// A scanner whose oracle client points at `oracle` (never contacted unless a /// test calls `scan_block_range`). Must be called inside a Tokio runtime. -fn scanner(tag: &str, oracle: &'static str) -> (Scanner, TempState) { +pub(super) fn scanner(tag: &str, oracle: &'static str) -> (Scanner, TempState) { let (scan_sk, spend_pk) = keys(); let state = std::env::temp_dir().join(format!( "blindbit-stream-safety-{tag}-{}.json", @@ -121,12 +121,12 @@ fn scanner(tag: &str, oracle: &'static str) -> (Scanner, TempState) { /// One block at `height` holding a single silent payment to the test wallet, /// served in place of P2P, and the oracle message that describes it. -struct PaymentBlock { - height: u64, - message: BlockScanDataShortResponse, +pub(super) struct PaymentBlock { + pub(super) height: u64, + pub(super) message: BlockScanDataShortResponse, } -fn payment_block(height: u64) -> PaymentBlock { +pub(super) fn payment_block(height: u64) -> PaymentBlock { let (scan_sk, spend_pk) = keys(); // The tweak (`input_hash * A`) the oracle would serve; any point works. let tweak = secret(0x40u8.wrapping_add(height as u8)).public_key(&Secp256k1::new()); @@ -204,7 +204,7 @@ fn payment_block(height: u64) -> PaymentBlock { /// What an oracle sends today for a height it has not indexed: OK, with an /// empty block hash and no data. -fn unindexed_message(height: u64) -> BlockScanDataShortResponse { +pub(super) fn unindexed_message(height: u64) -> BlockScanDataShortResponse { BlockScanDataShortResponse { block_identifier: Some(BlockIdentifier { block_hash: vec![], @@ -215,7 +215,7 @@ fn unindexed_message(height: u64) -> BlockScanDataShortResponse { } } -fn owned_outputs(scanner: &Scanner) -> usize { +pub(super) fn owned_outputs(scanner: &Scanner) -> usize { scanner.internal_indexer.index().by_shared_secret.len() } diff --git a/friglet-ipc/src/lib.rs b/friglet-ipc/src/lib.rs index f6ee43c..ab73d8a 100644 --- a/friglet-ipc/src/lib.rs +++ b/friglet-ipc/src/lib.rs @@ -159,6 +159,42 @@ pub struct StatusInfo { /// instead of assuming it did not spawn it. #[serde(default)] pub spawned_by_tray: bool, + /// Conditions of the scan beyond its progress: why it is stuck, and + /// whether it started above the configured start height. + #[serde(default)] + pub scan_health: ScanHealthInfo, +} + +/// Scan conditions the user should see (see [`StatusInfo::scan_health`]). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ScanHealthInfo { + /// Set while the scan cannot get past a height. The daemon keeps + /// retrying; this clears once the scan makes progress again. + pub stall: Option, + /// Set when the configured start height lay below the oracle's first + /// indexed block, so scanning began at that block instead. + pub start_adjusted: Option, +} + +/// The scan cannot get past `height`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ScanStallInfo { + /// First height that could not be scanned. + pub height: u64, + /// Why, as the scanner reported it. + pub reason: String, + /// Unix time (seconds) of the first failed attempt at this height. + pub since_unix: u64, +} + +/// Scanning began at `oracle_floor` instead of the configured +/// `requested_height`: the oracle has no data below it, so blocks +/// `requested_height..oracle_floor` were not scanned. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StartAdjustedInfo { + pub requested_height: u64, + pub oracle_floor: u64, } /// A configured Silent Payments label and its receive address. @@ -458,6 +494,17 @@ mod tests { }], version: "test".to_string(), spawned_by_tray: true, + scan_health: ScanHealthInfo { + stall: Some(ScanStallInfo { + height: 102, + reason: "the oracle sent no valid block hash".to_string(), + since_unix: 1_790_000_000, + }), + start_adjusted: Some(StartAdjustedInfo { + requested_height: 1, + oracle_floor: 100, + }), + }, }; let json = serde_json::to_string(&status).unwrap(); assert_eq!(serde_json::from_str::(&json).unwrap(), status); @@ -482,6 +529,7 @@ mod tests { assert_eq!(status.outputs_found, 0); assert!(status.label_addresses.is_empty()); assert!(!status.spawned_by_tray); + assert_eq!(status.scan_health, ScanHealthInfo::default()); } #[test] diff --git a/friglet-ipc/tests/roundtrip.rs b/friglet-ipc/tests/roundtrip.rs index 61df624..ae608d9 100644 --- a/friglet-ipc/tests/roundtrip.rs +++ b/friglet-ipc/tests/roundtrip.rs @@ -37,6 +37,7 @@ async fn get_status_roundtrip() { }], version: "0.1.0".to_string(), spawned_by_tray: true, + scan_health: Default::default(), }; let expected = status.clone(); diff --git a/friglet-tray/examples/fake-daemon.rs b/friglet-tray/examples/fake-daemon.rs index 4a422c3..11c1faf 100644 --- a/friglet-tray/examples/fake-daemon.rs +++ b/friglet-tray/examples/fake-daemon.rs @@ -83,6 +83,7 @@ async fn main() -> std::io::Result<()> { ], version: "fake-0.0.0".to_string(), spawned_by_tray: false, + scan_health: Default::default(), }), Request::Start => { scanning.store(true, Ordering::Relaxed); diff --git a/friglet-tray/src/lib.rs b/friglet-tray/src/lib.rs index c7be3d4..a066c32 100644 --- a/friglet-tray/src/lib.rs +++ b/friglet-tray/src/lib.rs @@ -690,6 +690,7 @@ mod tests { label_addresses: Vec::new(), version: "test".to_string(), spawned_by_tray, + scan_health: Default::default(), }), _ => Response::Ok, }; diff --git a/friglet-tray/tests/lifecycle_e2e.rs b/friglet-tray/tests/lifecycle_e2e.rs index 017c5da..7237f06 100644 --- a/friglet-tray/tests/lifecycle_e2e.rs +++ b/friglet-tray/tests/lifecycle_e2e.rs @@ -42,6 +42,7 @@ fn dummy_status() -> StatusInfo { label_addresses: Vec::new(), version: "test".to_string(), spawned_by_tray: false, + scan_health: Default::default(), } } diff --git a/friglet-tray/ui/index.html b/friglet-tray/ui/index.html index 06ad500..f58fed2 100644 --- a/friglet-tray/ui/index.html +++ b/friglet-tray/ui/index.html @@ -64,6 +64,14 @@

Friglet

Last error + +
diff --git a/friglet-tray/ui/main.js b/friglet-tray/ui/main.js index efe2517..1bc09d1 100644 --- a/friglet-tray/ui/main.js +++ b/friglet-tray/ui/main.js @@ -42,11 +42,37 @@ function render({ reachable, status }) { $("error-block").hidden = !status.last_error; setText("last-error", status.last_error ?? ""); + renderScanHealth(status.scan_health ?? {}); $("start-btn").disabled = !reachable || status.scanning; $("stop-btn").disabled = !reachable || !status.scanning; } +// Why the scan is stuck, and where it started (the daemon's `scan_health`; +// absent from older daemons). +function renderScanHealth(health) { + const stall = health.stall; + $("stall-block").hidden = !stall; + setText( + "stall", + stall + ? `at height ${stall.height.toLocaleString()} since ` + + `${new Date(stall.since_unix * 1000).toLocaleString()}: ${stall.reason}` + : "", + ); + + const notes = []; + const start = health.start_adjusted; + if (start) { + notes.push( + `Scanning began at block ${start.oracle_floor.toLocaleString()} instead of ` + + `${start.requested_height.toLocaleString()}: the oracle has no data below it.`, + ); + } + $("scan-notes-block").hidden = notes.length === 0; + setText("scan-notes", notes.join("\n")); +} + async function refresh() { try { const status = await invoke("get_status"); diff --git a/friglet-tray/ui/style.css b/friglet-tray/ui/style.css index 3104613..ac43b48 100644 --- a/friglet-tray/ui/style.css +++ b/friglet-tray/ui/style.css @@ -71,6 +71,7 @@ code { color: var(--text); } .error { color: var(--bad); } +#scan-notes { font-size: 13px; white-space: pre-line; } .small { font-size: 12px; min-height: 18px; } .banner { diff --git a/friglet/src/control/mod.rs b/friglet/src/control/mod.rs index 4e238dd..3d3d29d 100644 --- a/friglet/src/control/mod.rs +++ b/friglet/src/control/mod.rs @@ -118,6 +118,26 @@ pub(crate) fn wallet_network(network: bitcoin_rev::Network) -> BitcoinNetwork { } } +/// The scanner's published health as sent over the control socket. +fn scan_health_info(health: &scanner::ScanHealth) -> friglet_ipc::ScanHealthInfo { + friglet_ipc::ScanHealthInfo { + stall: health + .stall + .as_ref() + .map(|stall| friglet_ipc::ScanStallInfo { + height: stall.height, + reason: stall.reason.clone(), + since_unix: stall.since_unix, + }), + start_adjusted: health + .oracle_floor_start + .map(|start| friglet_ipc::StartAdjustedInfo { + requested_height: start.requested_height, + oracle_floor: start.floor_height, + }), + } +} + /// Everything the request handler needs from the daemon. pub struct ControlCtx { pub supervisor: Arc, @@ -234,6 +254,7 @@ impl ControlCtx { let addr = (!idx.sp_address.is_empty()).then(|| idx.sp_address.clone()); (tip, idx.scan_progress, addr, idx.sp_history.len() as u64) }; + let scan_health = scan_health_info(&index.lock().await.scan_health); // While the scan task runs it holds the scanner mutex, so fall back // to the electrum index tip (which the scanner advances per scanned @@ -266,6 +287,7 @@ impl ControlCtx { label_addresses: self.label_addresses.lock().unwrap().clone(), version: env!("CARGO_PKG_VERSION").to_string(), spawned_by_tray: self.spawned_by_tray, + scan_health, } } @@ -773,6 +795,7 @@ mod tests { label_addresses: Vec::new(), version: "test".to_string(), spawned_by_tray: false, + scan_health: Default::default(), } } @@ -801,6 +824,44 @@ mod tests { assert_eq!(ctx.status().await.tx_count, 2); } + #[tokio::test] + async fn status_reports_a_stalled_scan_and_an_adjusted_start() { + let dir = temp_dir("status-scan-health"); + let ctx = test_ctx(&dir); + assert_eq!( + ctx.status().await.scan_health, + friglet_ipc::ScanHealthInfo::default() + ); + + let index = ctx.electrum_index.lock().unwrap().clone(); + { + let mut idx = index.lock().await; + idx.scan_health.stall = Some(scanner::ScanStall { + height: 100_002, + reason: "scan stopped at height 100002: the oracle sent no valid block hash" + .to_string(), + since_unix: 1_790_000_000, + }); + idx.scan_health.oracle_floor_start = Some(scanner::OracleFloorStart { + requested_height: 50_000, + floor_height: 100_000, + }); + } + + let health = ctx.status().await.scan_health; + let stall = health.stall.expect("stall is reported"); + assert_eq!(stall.height, 100_002); + assert_eq!(stall.since_unix, 1_790_000_000); + assert!(stall.reason.contains("no valid block hash")); + assert_eq!( + health.start_adjusted, + Some(friglet_ipc::StartAdjustedInfo { + requested_height: 50_000, + oracle_floor: 100_000, + }) + ); + } + #[tokio::test] async fn get_status_roundtrip_over_control_socket() { #[cfg(windows)] From 04d57ad720fa4edac51afa2c33aba9c924b7fc18 Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 1 Oct 2026 13:38:08 +0000 Subject: [PATCH 07/10] fix(scanner): version the state file; rescan an old state once for spends it missed A state file written before owned outputs were recorded still loads, but its wallet never fetched blocks whose only wallet-relevant transaction was a spend, so those spends stay missing and nothing said so. - The state file now carries `format_version` (STATE_FORMAT_VERSION = 1; absent = 0, written before the owned-output records). Every save writes the current version. - Restoring a format-0 state rewinds the scan to just below its oldest unspent output, the lowest height a missed spend can be at. The next scan revisits those blocks and, with the outputs now recorded, fetches the ones that spend them. Re-applying blocks already in the graph is idempotent. With no unspent output there is nothing to re-check and only the version is bumped. - The rewind is logged and published in the scan health (`StateRescan`), shown in friglet's status and the tray, and cleared once the scan is back where the state had been. While it runs, the Electrum tip stays at that height, so clients never see wallet history above the tip. - Test fixture `testdata/state_before_owned_outputs.json` was written by the pre-fix code (integration 93f6a92): payment + change at 100, scanned to 300, spend at 200 missing. The test loads it as written and with legacy bare-hex `owned_outputs` entries, checks the restored records and labels, the rewind to 99, the held Electrum tip, that the rescan finds the spend, and that the upgraded file restores without another rewind. Co-Authored-By: Claude Opus 5.5 --- blindbit-lib/src/scanner/changeset.rs | 16 ++ blindbit-lib/src/scanner/electrum_index.rs | 11 ++ blindbit-lib/src/scanner/health.rs | 28 +++ blindbit-lib/src/scanner/mod.rs | 4 +- .../src/scanner/owned_outputs_tests.rs | 155 +++++++++++++++++ blindbit-lib/src/scanner/scanner.rs | 63 ++++++- blindbit-lib/src/scanner/scanning.rs | 7 +- .../testdata/state_before_owned_outputs.json | 164 ++++++++++++++++++ friglet-ipc/src/lib.rs | 15 ++ friglet-tray/ui/main.js | 8 + friglet/src/control/mod.rs | 17 ++ 11 files changed, 482 insertions(+), 6 deletions(-) create mode 100644 blindbit-lib/src/scanner/testdata/state_before_owned_outputs.json diff --git a/blindbit-lib/src/scanner/changeset.rs b/blindbit-lib/src/scanner/changeset.rs index c78f963..2b6f73b 100644 --- a/blindbit-lib/src/scanner/changeset.rs +++ b/blindbit-lib/src/scanner/changeset.rs @@ -6,6 +6,16 @@ use std::collections::BTreeMap; use super::health::OracleFloorStart; use super::types::OwnedOutputRecord; +/// Format of the persisted [`ChangeSet`] this build writes. +/// +/// - `0` (field absent): written before owned outputs were recorded. Such a +/// state never fetched blocks whose only wallet-relevant transaction was a +/// spend, so spends of its outputs may be missing; restoring it rescans +/// from the oldest unspent output (see `Scanner::from_changeset`). +/// - `1`: `owned_outputs` holds [`OwnedOutputRecord`]s. Builds older than +/// this format cannot read it (they expect bare hex keys there). +pub const STATE_FORMAT_VERSION: u32 = 1; + /// Deserialisation of the persisted `owned_outputs` array. #[cfg(feature = "serde")] mod owned_outputs_serde { @@ -50,6 +60,10 @@ mod owned_outputs_serde { #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] #[must_use] pub struct ChangeSet { + /// [`STATE_FORMAT_VERSION`] of the build that wrote this state; `0` when + /// absent (written before the field existed). + #[cfg_attr(feature = "serde", serde(default))] + pub format_version: u32, /// Sparse block checkpoints: only blocks where we found something (height -> hash) pub block_checkpoints: BTreeMap, /// Changes related to the Silent Payments indexer data. @@ -124,6 +138,8 @@ impl Merge for ChangeSet { if other.oracle_floor_start.is_some() { self.oracle_floor_start = other.oracle_floor_start; } + + self.format_version = self.format_version.max(other.format_version); } /// Checks if the [`ChangeSet`] is empty (contains no changes). diff --git a/blindbit-lib/src/scanner/electrum_index.rs b/blindbit-lib/src/scanner/electrum_index.rs index 43a5a98..ff7efac 100644 --- a/blindbit-lib/src/scanner/electrum_index.rs +++ b/blindbit-lib/src/scanner/electrum_index.rs @@ -87,6 +87,17 @@ impl WalletElectrumIndex { pub fn new() -> Self { Self::default() } + + /// Height to report as the chain tip once `scanned` is scanned. While a + /// restored state is rescanned for missed spends (`scan_health. + /// state_rescan`), the tip stays at the height the state had reached, so + /// clients never see wallet history above the tip. + pub fn tip_height_for(&self, scanned: u32) -> u32 { + match self.scan_health.state_rescan { + Some(rescan) => scanned.max(u32::try_from(rescan.until_height).unwrap_or(u32::MAX)), + None => scanned, + } + } } /// Compute the Electrum scripthash for a script. diff --git a/blindbit-lib/src/scanner/health.rs b/blindbit-lib/src/scanner/health.rs index f80456d..02df13a 100644 --- a/blindbit-lib/src/scanner/health.rs +++ b/blindbit-lib/src/scanner/health.rs @@ -30,6 +30,21 @@ pub struct ScanHealth { /// Set when the wallet's start height lay below the oracle's first /// indexed block, so scanning began at the oracle's floor instead. pub oracle_floor_start: Option, + /// Set while a state file from before owned outputs were recorded is + /// being rescanned for spends it missed; cleared once the scan is back + /// at `until_height`. + pub state_rescan: Option, +} + +/// The restored state predates owned-output records (state format 0), so +/// blocks `from_height..=until_height` are scanned again to find spends of +/// its outputs that it never fetched. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StateRescan { + /// Height of the oldest output the state still showed unspent. + pub from_height: u64, + /// Height the state had been scanned to. + pub until_height: u64, } /// The scan cannot get past `height`. @@ -243,6 +258,8 @@ impl Scanner { }); } + /// The scan made progress or is caught up: clear any stall, and end a + /// state rescan ([`StateRescan`]) once it is back where it started. pub(crate) async fn clear_stall(&self) { let mut idx = self.electrum_index.lock().await; if idx.scan_health.stall.take().is_some() { @@ -251,6 +268,17 @@ impl Scanner { "scan is making progress again" ); } + if idx + .scan_health + .state_rescan + .is_some_and(|rescan| self.last_scanned_block_height >= rescan.until_height) + { + idx.scan_health.state_rescan = None; + tracing::info!( + height = self.last_scanned_block_height, + "rescan for spends missed by the old state file is complete" + ); + } } /// After a failed scan of `from..=tip`: when `from` is the wallet's start diff --git a/blindbit-lib/src/scanner/mod.rs b/blindbit-lib/src/scanner/mod.rs index cc542c1..be620ca 100644 --- a/blindbit-lib/src/scanner/mod.rs +++ b/blindbit-lib/src/scanner/mod.rs @@ -19,10 +19,10 @@ mod utils; pub type ScannerError = Box; // Re-export public types and the main Scanner struct -pub use changeset::ChangeSet; +pub use changeset::{ChangeSet, STATE_FORMAT_VERSION}; pub use config::ScannerConfig; pub use electrum_index::{ScriptHashEntry, SpHistoryEntry, WalletElectrumIndex, electrum_scripthash, electrum_status}; -pub use health::{OracleFloorStart, ScanHealth, ScanStall, ScanStopped}; +pub use health::{OracleFloorStart, ScanHealth, ScanStall, ScanStopped, StateRescan}; pub use p2p::broadcast_tx; pub use scanner::Scanner; pub use types::{BlockIdentifierDisplay, OwnedOutput, OwnedOutputRecord}; diff --git a/blindbit-lib/src/scanner/owned_outputs_tests.rs b/blindbit-lib/src/scanner/owned_outputs_tests.rs index 282e652..46acf57 100644 --- a/blindbit-lib/src/scanner/owned_outputs_tests.rs +++ b/blindbit-lib/src/scanner/owned_outputs_tests.rs @@ -481,3 +481,158 @@ fn owned_outputs_labels_and_spends_survive_restart() { ); assert_eq!(balance(&restored), 0); } + +/// A state file written by the code before owned outputs were recorded +/// (integration 93f6a92): a payment and a change output received at 100, the +/// wallet scanned on to 300, and the payment spent at 200 in a block that +/// code never fetched, so the spend is not in the file. +#[cfg(feature = "serde")] +const STATE_BEFORE_OWNED_OUTPUTS: &str = include_str!("testdata/state_before_owned_outputs.json"); + +#[cfg(feature = "serde")] +fn restore(tag: &str, json: &str) -> (Scanner, PathBuf) { + let path = state_file(tag); + std::fs::write(&path, json).expect("write state file"); + let changeset = Scanner::load_from_file(&path).expect("load state"); + let socket: SocketAddr = "127.0.0.1:8333".parse().unwrap(); + let scanner = Scanner::from_changeset( + oracle_client(), + socket, + changeset, + path.clone(), + Network::Regtest, + ) + .expect("restore"); + (scanner, path) +} + +#[cfg(feature = "serde")] +fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime") + .block_on(future) +} + +/// An old state file still loads, and because its spends of owned outputs +/// may be missing, the scan rewinds to just below its oldest unspent output. +/// Rescanning from there finds the missed spend; the upgraded file then +/// loads without another rewind. +#[cfg(feature = "serde")] +#[test] +fn state_file_from_before_owned_outputs_rescans_for_missed_spends() { + let old: serde_json::Value = serde_json::from_str(STATE_BEFORE_OWNED_OUTPUTS).unwrap(); + assert!(old.get("format_version").is_none()); + assert_eq!(old["owned_outputs"], serde_json::json!([])); + assert_eq!(old["last_scanned_block_height"], 300); + let tweak = served_tweak(0x33); + let pay_key = sp_output_key(&tweak, None); + // The old field held bare hex keys; no released writer filled it, but such + // entries must not break the load either. + let mut with_legacy_keys = old.clone(); + with_legacy_keys["owned_outputs"] = serde_json::json!([pay_key.to_string()]); + + for (case, json) in [ + ("as written", STATE_BEFORE_OWNED_OUTPUTS.to_string()), + ("legacy keys", with_legacy_keys.to_string()), + ] { + let (mut scanner, path) = restore("pre-records", &json); + + // Both outputs are back, the change output with its label. + let records: Vec<_> = scanner.owned_outputs().cloned().collect(); + assert_eq!(records.len(), 2, "{case}"); + let pay = records + .iter() + .find(|r| r.pubkey == pay_key) + .expect("payment record") + .clone(); + let change = records.iter().find(|r| r.pubkey != pay_key).unwrap(); + assert_eq!((pay.amount_sat, pay.label, pay.height), (50_000, None, 100)); + assert_eq!( + (change.amount_sat, change.label, change.height), + (20_000, Some(0), 100) + ); + assert_eq!(balance(&scanner), 70_000, "{case}: the spend is missing"); + + // Rewound to just below the oldest unspent output, and says so. + assert_eq!(scanner.get_last_scanned_block_height(), 99, "{case}"); + let health = block_on(scanner.scan_health()); + assert_eq!( + health.state_rescan, + Some(crate::scanner::StateRescan { + from_height: 100, + until_height: 300, + }), + "{case}" + ); + // Electrum clients keep seeing the old tip while the rescan runs. + block_on(scanner.rebuild_electrum_index_from_graph(1)); + let tip = block_on(scanner.electrum_index().lock()).tip.clone(); + assert_eq!(tip.map(|(height, _)| height), Some(300), "{case}"); + + // The rescan reaches block 200; with the output recorded, the oracle's + // spent prefix now gets that block fetched and the spend found. + let spend = spend_of(pay.outpoint); + assert!(process_block( + &mut scanner, + &block(SPEND_HEIGHT, vec![spend.clone()]), + SPEND_HEIGHT, + vec![], + &[pay_key], + )); + assert_eq!( + record(&scanner, pay.outpoint).spent_by, + Some(spend.compute_txid()), + "{case}" + ); + assert_eq!(balance(&scanner), 20_000, "{case}"); + scanner.update_last_scanned_block_height(300); + block_on(scanner.clear_stall()); + assert_eq!(block_on(scanner.scan_health()).state_rescan, None); + + // Saved in the current format, it restores without another rewind. + scanner.save_to_file(&path).expect("save"); + let saved: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!( + saved["format_version"], + crate::scanner::STATE_FORMAT_VERSION + ); + let (again, path) = restore("pre-records", &serde_json::to_string(&saved).unwrap()); + let _ = std::fs::remove_file(&path); + assert_eq!(again.get_last_scanned_block_height(), 300, "{case}"); + assert_eq!(block_on(again.scan_health()).state_rescan, None, "{case}"); + assert_eq!(balance(&again), 20_000, "{case}"); + } +} + +/// Without an unspent output there is nothing a missed spend could affect: +/// the old file is upgraded in place, without a rescan. +#[cfg(feature = "serde")] +#[test] +fn state_file_from_before_owned_outputs_without_unspent_outputs_is_not_rescanned() { + let mut old: serde_json::Value = serde_json::from_str(STATE_BEFORE_OWNED_OUTPUTS).unwrap(); + // The same wallet before anything was found. + let fresh = { + let scanner = scanner("pre-records-empty-src", 1); + let mut json = serde_json::to_value(&scanner.stage).unwrap(); + json.as_object_mut().unwrap().remove("format_version"); + json + }; + old["indexer"] = fresh["indexer"].clone(); + old["block_checkpoints"] = fresh["block_checkpoints"].clone(); + + let (scanner, path) = restore("pre-records-empty", &old.to_string()); + assert_eq!(scanner.owned_outputs().count(), 0); + assert_eq!(scanner.get_last_scanned_block_height(), 300); + assert_eq!(block_on(scanner.scan_health()).state_rescan, None); + scanner.save_to_file(&path).expect("save"); + let saved: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + let _ = std::fs::remove_file(&path); + assert_eq!( + saved["format_version"], + crate::scanner::STATE_FORMAT_VERSION + ); +} diff --git a/blindbit-lib/src/scanner/scanner.rs b/blindbit-lib/src/scanner/scanner.rs index 24ac65c..77c81fd 100644 --- a/blindbit-lib/src/scanner/scanner.rs +++ b/blindbit-lib/src/scanner/scanner.rs @@ -20,7 +20,7 @@ use tonic::transport::Channel; use crate::oracle_grpc::oracle_service_client::OracleServiceClient; use indexer::bdk_chain::ConfirmationBlockTime; -use super::changeset::ChangeSet; +use super::changeset::{ChangeSet, STATE_FORMAT_VERSION}; use super::config::ScannerConfig; use super::types::OwnedOutputRecord; use super::electrum_index::{ScriptHashEntry, SpHistoryEntry, WalletElectrumIndex, electrum_scripthash}; @@ -119,6 +119,7 @@ impl Scanner { // Initialize the staged changeset with initial state let mut stage = ChangeSet { + format_version: STATE_FORMAT_VERSION, block_checkpoints: block_checkpoints.clone(), indexer: indexer.initial_changeset(), last_scanned_block_height: 0, @@ -379,7 +380,7 @@ impl Scanner { // Header hex is not recoverable without re-fetching; Sparrow's BlockHeaderTip // handles an empty hex gracefully. if self.last_scanned_block_height > 0 { - let h = self.last_scanned_block_height as u32; + let h = index.tip_height_for(self.last_scanned_block_height as u32); let hex = index.headers.get(&h).cloned().unwrap_or_default(); index.tip = Some((h, hex)); } @@ -494,6 +495,59 @@ impl Scanner { } } + /// Upgrade a restored state written before owned outputs were recorded + /// (format 0). + /// + /// That code fetched a block only when it held a probable *receive*, so a + /// block whose only wallet-relevant transaction spent one of our outputs + /// was never fetched, and the spend is missing from the restored graph. + /// Every output the graph still shows unspent may have been spent that + /// way, at or after its own height. The scan is therefore rewound to just + /// below the oldest such output, so the next scan revisits every block a + /// missed spend can be in; with the owned outputs now recorded, it fetches + /// those blocks. Re-applying blocks already in the graph is idempotent. + /// + /// The rewind is published in the scan health ([`StateRescan`]) and the + /// state is marked current, so it happens once: the next save persists + /// the lower height together with the new format. + /// + /// [`StateRescan`]: super::health::StateRescan + fn rescan_for_missed_spends(&mut self) { + self.stage.format_version = STATE_FORMAT_VERSION; + let previous_height = self.last_scanned_block_height; + let Some(oldest_unspent) = self + .owned_outputs + .values() + .filter(|record| !record.is_spent()) + .map(|record| u64::from(record.height)) + .min() + .filter(|height| *height <= previous_height) + else { + tracing::info!( + "state file predates owned-output records; no unspent output to re-check" + ); + return; + }; + + let resume_height = oldest_unspent - 1; + tracing::warn!( + from_height = oldest_unspent, + previous_height, + unspent_outputs = self.owned_outputs.values().filter(|r| !r.is_spent()).count(), + "state file predates owned-output records, so spends in blocks it never fetched \ + can be missing; rescanning from the oldest unspent output" + ); + self.update_last_scanned_block_height(resume_height); + let rescan = super::health::StateRescan { + from_height: oldest_unspent, + until_height: previous_height, + }; + // `from_changeset` holds the only reference to the new index. + if let Some(index) = Arc::get_mut(&mut self.electrum_index) { + index.get_mut().scan_health.state_rescan = Some(rescan); + } + } + /// Returns an optional reference to the currently staged [`ChangeSet`]. /// /// # Returns @@ -573,6 +627,7 @@ impl Scanner { changeset.last_scanned_block_height = self.last_scanned_block_height; changeset.last_scanned_block_height_rescan = self.last_scanned_block_height_rescan; changeset.owned_outputs = self.owned_outputs.values().cloned().collect(); + changeset.format_version = STATE_FORMAT_VERSION; let json = serde_json::to_string_pretty(&changeset)?; std::fs::write(path, json)?; @@ -683,6 +738,7 @@ impl Scanner { .get_address(convert_network(network)) .to_string(); let oracle_floor_start = changeset.oracle_floor_start; + let format_version = changeset.format_version; let owned_outputs = changeset .owned_outputs @@ -717,6 +773,9 @@ impl Scanner { // Records older state files never had, spends already in the graph, // and the prefix lookup, all derived from the restored indexer. scanner.sync_owned_outputs(); + if format_version < STATE_FORMAT_VERSION { + scanner.rescan_for_missed_spends(); + } Ok(scanner) } } diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index ad588f3..854412d 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -371,7 +371,9 @@ impl Scanner { let mut idx = self.electrum_index.lock().await; idx.headers.insert(block_height_u32, header_hex.clone()); - idx.tip = Some((block_height_u32, header_hex)); + if idx.tip_height_for(block_height_u32) == block_height_u32 { + idx.tip = Some((block_height_u32, header_hex)); + } for tx in &block.txdata { let txid = tx.compute_txid(); @@ -643,7 +645,8 @@ impl Scanner { .as_ref() .map(|(_, hex)| hex.clone()) .unwrap_or_default(); - idx.tip = Some((height_u32, header_hex)); + let tip_height = idx.tip_height_for(height_u32); + idx.tip = Some((tip_height, header_hex)); } let utxo_count = self.internal_indexer.index().by_shared_secret.len(); diff --git a/blindbit-lib/src/scanner/testdata/state_before_owned_outputs.json b/blindbit-lib/src/scanner/testdata/state_before_owned_outputs.json new file mode 100644 index 0000000..9b09442 --- /dev/null +++ b/blindbit-lib/src/scanner/testdata/state_before_owned_outputs.json @@ -0,0 +1,164 @@ +{ + "block_checkpoints": { + "0": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f", + "100": "0d52ac6e67ab550e176ebb2fb29d8fdf9a113ae2549b05d800cde721dde38d29" + }, + "indexer": { + "scan_sk": "1111111111111111111111111111111111111111111111111111111111111111", + "spend_pk": "02466d7fcae563e5cb09a0d1870bb580344804617879a14949cf22285f1bae3f27", + "txid_to_partial_secret": { + "cd8a3efcfeedb7d07d87fce2288ceb2a6bfc3348b035f6bf8c878efad06f7a81": "023c72addb4fdf09af94f0c94d7fe92a386a7e70cf8a1d85916386bb2535c7b1b1", + "1d0e05d33c0c47a86009983a84044e3c59c702d8e08cd5adf8e3c95e0b665dbe": "023c72addb4fdf09af94f0c94d7fe92a386a7e70cf8a1d85916386bb2535c7b1b1" + }, + "label_lookup": [ + [ + 0, + [ + 53, + 81, + 172, + 101, + 113, + 186, + 106, + 173, + 137, + 199, + 228, + 228, + 29, + 126, + 48, + 225, + 160, + 91, + 41, + 98, + 115, + 232, + 204, + 116, + 50, + 67, + 212, + 126, + 61, + 200, + 124, + 148 + ], + "038b2709f0aa4014d750140cbdc445634f3e4b16e3c686806127d53a9c8cf29921" + ], + [ + 1, + [ + 194, + 60, + 36, + 4, + 114, + 227, + 65, + 153, + 163, + 180, + 221, + 100, + 222, + 126, + 135, + 216, + 187, + 62, + 157, + 157, + 102, + 237, + 145, + 164, + 174, + 11, + 209, + 154, + 6, + 161, + 183, + 32 + ], + "035be3a727b6bccdf4bbd4e6ebf27266b41816b3ee373237540942b8fc339c5bf9" + ] + ], + "graph": { + "txs": [ + { + "version": 2, + "lock_time": 0, + "input": [ + { + "previous_output": "0101010101010101010101010101010101010101010101010101010101010101:0", + "script_sig": "", + "sequence": 4294967295, + "witness": [] + } + ], + "output": [ + { + "value": 50000, + "script_pubkey": "5120495feefbace0631dfad36337aeaa066615dc1f34598b21e455599670607858f1" + } + ] + }, + { + "version": 2, + "lock_time": 0, + "input": [ + { + "previous_output": "0202020202020202020202020202020202020202020202020202020202020202:0", + "script_sig": "", + "sequence": 4294967295, + "witness": [] + } + ], + "output": [ + { + "value": 20000, + "script_pubkey": "512090e97c584c5888d5287961b2af2c7b729f60f4e5973a24e394b993d017024f19" + } + ] + } + ], + "txouts": {}, + "anchors": [ + [ + { + "block_id": { + "height": 100, + "hash": "0d52ac6e67ab550e176ebb2fb29d8fdf9a113ae2549b05d800cde721dde38d29" + }, + "confirmation_time": 1713571867 + }, + "cd8a3efcfeedb7d07d87fce2288ceb2a6bfc3348b035f6bf8c878efad06f7a81" + ], + [ + { + "block_id": { + "height": 100, + "hash": "0d52ac6e67ab550e176ebb2fb29d8fdf9a113ae2549b05d800cde721dde38d29" + }, + "confirmation_time": 1713571867 + }, + "1d0e05d33c0c47a86009983a84044e3c59c702d8e08cd5adf8e3c95e0b665dbe" + ] + ], + "last_seen": {}, + "last_evicted": {}, + "first_seen": {} + } + }, + "last_scanned_block_height": 300, + "last_scanned_block_height_rescan": 0, + "owned_outputs": [], + "secret_scan_hex": "1111111111111111111111111111111111111111111111111111111111111111", + "public_spend_hex": "02466d7fcae563e5cb09a0d1870bb580344804617879a14949cf22285f1bae3f27", + "max_label_num": 1 +} \ No newline at end of file diff --git a/friglet-ipc/src/lib.rs b/friglet-ipc/src/lib.rs index ab73d8a..fd843bd 100644 --- a/friglet-ipc/src/lib.rs +++ b/friglet-ipc/src/lib.rs @@ -175,6 +175,17 @@ pub struct ScanHealthInfo { /// Set when the configured start height lay below the oracle's first /// indexed block, so scanning began at that block instead. pub start_adjusted: Option, + /// Set while a state file written before spends were tracked is being + /// rescanned once for the spends it missed. + pub state_rescan: Option, +} + +/// Blocks `from_height..=until_height` are scanned again because the state +/// file predates spend tracking (see blindbit-lib `StateRescan`). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StateRescanInfo { + pub from_height: u64, + pub until_height: u64, } /// The scan cannot get past `height`. @@ -504,6 +515,10 @@ mod tests { requested_height: 1, oracle_floor: 100, }), + state_rescan: Some(StateRescanInfo { + from_height: 100, + until_height: 101, + }), }, }; let json = serde_json::to_string(&status).unwrap(); diff --git a/friglet-tray/ui/main.js b/friglet-tray/ui/main.js index 1bc09d1..8cd0923 100644 --- a/friglet-tray/ui/main.js +++ b/friglet-tray/ui/main.js @@ -69,6 +69,14 @@ function renderScanHealth(health) { `${start.requested_height.toLocaleString()}: the oracle has no data below it.`, ); } + const rescan = health.state_rescan; + if (rescan) { + notes.push( + `Rescanning blocks ${rescan.from_height.toLocaleString()}–` + + `${rescan.until_height.toLocaleString()} once: the state file predates ` + + `spend tracking, so spends it missed are being looked for.`, + ); + } $("scan-notes-block").hidden = notes.length === 0; setText("scan-notes", notes.join("\n")); } diff --git a/friglet/src/control/mod.rs b/friglet/src/control/mod.rs index 3d3d29d..f155840 100644 --- a/friglet/src/control/mod.rs +++ b/friglet/src/control/mod.rs @@ -135,6 +135,12 @@ fn scan_health_info(health: &scanner::ScanHealth) -> friglet_ipc::ScanHealthInfo requested_height: start.requested_height, oracle_floor: start.floor_height, }), + state_rescan: health + .state_rescan + .map(|rescan| friglet_ipc::StateRescanInfo { + from_height: rescan.from_height, + until_height: rescan.until_height, + }), } } @@ -846,6 +852,10 @@ mod tests { requested_height: 50_000, floor_height: 100_000, }); + idx.scan_health.state_rescan = Some(scanner::StateRescan { + from_height: 100_500, + until_height: 101_000, + }); } let health = ctx.status().await.scan_health; @@ -860,6 +870,13 @@ mod tests { oracle_floor: 100_000, }) ); + assert_eq!( + health.state_rescan, + Some(friglet_ipc::StateRescanInfo { + from_height: 100_500, + until_height: 101_000, + }) + ); } #[tokio::test] From 9740d6663e97c8aeb43a72fd5928bb7a7f2f54b1 Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 1 Oct 2026 13:50:37 +0000 Subject: [PATCH 08/10] perf(scanner): check the chain with hash lookups instead of re-streaming blocks; catch mid-stream oracle reorgs The reorg check streamed the tip block again on every idle poll (~45 KB of compute index every 10 s on mainnet, ~385 MB a day per friglet and the same egress on the oracle), streamed the last scanned block again in front of every new block, and located a fork by streaming the whole 144-block lookback window. - The block below a range is checked with one GetBlockHashByHeight lookup against the remembered hash. An idle poll streams nothing; a new block is streamed once. An empty hash or NOT_FOUND means "cannot check yet", not a reorganisation. - A disagreement is located with a binary search over the remembered heights (lookups only), and the rollback stream starts at the fork point. The oracle disagreeing at the oldest remembered height is ReorgTooDeep without streaming anything, so a halted wallet costs two lookups per poll. - The short stream carries no previous-block hash, so an oracle that switches branches while a stream runs could hand over blocks of both and nothing noticed. When a stream ends, its newest blocks (POST_STREAM_CHECK = 6) and the block below them are checked again; a disagreement is rolled back like any other reorganisation (bounded to 3 restarts per scan). - The per-poll body of watch_chain is `watch_step`, generic over the oracle, so the tests drive the real loop: a ReorgTooDeep is published as a stall (reason says to rescan from the wallet birthday), like any other scan that cannot proceed. - record_scanned_block_hash no longer copies the 144-hash window into the staged changeset on every block (save_to_file writes the current window), and trims the window in place. Co-Authored-By: Claude Opus 5.5 --- blindbit-lib/src/scanner/reorg.rs | 129 +++++++++-- blindbit-lib/src/scanner/reorg_tests.rs | 281 +++++++++++++++++++++++- blindbit-lib/src/scanner/scanning.rs | 175 ++++++++++----- 3 files changed, 512 insertions(+), 73 deletions(-) diff --git a/blindbit-lib/src/scanner/reorg.rs b/blindbit-lib/src/scanner/reorg.rs index d801db6..08c34e7 100644 --- a/blindbit-lib/src/scanner/reorg.rs +++ b/blindbit-lib/src/scanner/reorg.rs @@ -6,14 +6,26 @@ //! oracle streams for a height the scanner already knows is compared with //! that hash: //! -//! - A range scan that continues the scanned chain starts its stream one -//! height early, at the last scanned height, so the first message proves -//! the chain underneath is unchanged. `watch_chain` does the same when the -//! oracle has no new block, which catches a same-height tip replacement. -//! - When that first message disagrees, the fork is somewhere below it: the -//! scan reopens the stream at the lowest remembered height and walks up. -//! The first height whose hash disagrees is the first disconnected block, -//! the one below it is the fork point. +//! - Before a range scan continues the scanned chain, the last scanned block +//! is checked with one `GetBlockHashByHeight` lookup (a few dozen bytes) +//! instead of being streamed again. `watch_chain` runs the same check when +//! the oracle has no new block, which catches a same-height tip +//! replacement without downloading anything else. +//! - When that block disagrees, the fork is somewhere below it. A binary +//! search over the remembered heights, again with lookups only, finds the +//! highest one the oracle still agrees with ([`Scanner::locate_fork`]); +//! the stream that rolls back and rescans starts there. If the oracle +//! disagrees even at the oldest remembered height, that is +//! [`ReorgTooDeep`] without streaming anything. +//! - A stream reads the oracle's index height by height, so an oracle that +//! switches branches while a stream runs can hand over blocks of both. When +//! a stream ends, its newest blocks ([`POST_STREAM_CHECK`]) and the block +//! below them are checked again; a disagreement is handled like any other +//! reorganisation. Reorganisations happen at the tip, which is where a +//! stream ends, so that window covers them. +//! - Inside a stream, every block whose height has a remembered hash is still +//! compared with it. The first height whose hash disagrees is the first +//! disconnected block, the one below it is the fork point. //! - Everything learned above the fork point is rolled back //! ([`Scanner::rollback_to`]): transactions and anchors from disconnected //! blocks, the owned-output records they created, spent marks they set, @@ -35,6 +47,7 @@ use indexer::bdk_chain::ConfirmationBlockTime; use indexer::v2::SpIndexerV2; use super::ScannerError; +use super::health::OracleProbe; use super::scanner::Scanner; /// How many of the most recent scanned heights keep their block hash, and so @@ -45,6 +58,10 @@ use super::scanner::Scanner; /// state. pub const REORG_LOOKBACK: u32 = 144; +/// How many of the newest blocks a stream delivered are checked against the +/// oracle again once the stream has ended (see the module docs). +pub(crate) const POST_STREAM_CHECK: u64 = 6; + /// A chain reorganisation reaches at or below the oldest block hash the /// scanner still remembers, so the fork point cannot be found. /// @@ -102,6 +119,20 @@ pub(crate) fn block_hash_from_oracle(display_order: &[u8]) -> Option Some(BlockHash::from_byte_array(bytes)) } +/// A remembered block compared with the block the oracle serves now at the +/// same height. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OracleView { + /// The oracle serves the remembered block. + Same, + /// The oracle serves a different block: the chain was reorganised. + Different, + /// Nothing is remembered at that height, or the oracle serves no block + /// there (an empty hash or `NOT_FOUND`). It cannot be checked yet, which + /// is not a reorganisation. + Unknown, +} + /// What the scanner does with one oracle block, judged by its hash. #[derive(Debug, PartialEq, Eq)] pub(crate) enum BlockCheck { @@ -120,10 +151,6 @@ impl Scanner { .map(|height| u64::from(*height)) } - pub(crate) fn knows_block_hash(&self, height: u64) -> bool { - u32::try_from(height).is_ok_and(|height| self.scanned_block_hashes.contains_key(&height)) - } - pub(crate) fn check_block_hash(&self, height: u64, hash: &BlockHash) -> BlockCheck { match u32::try_from(height) .ok() @@ -147,8 +174,81 @@ impl Scanner { .next_back() .expect("just inserted"); let keep_from = highest.saturating_sub(REORG_LOOKBACK - 1); - self.scanned_block_hashes = self.scanned_block_hashes.split_off(&keep_from); - self.stage.scanned_block_hashes = self.scanned_block_hashes.clone(); + while self + .scanned_block_hashes + .first_key_value() + .is_some_and(|(height, _)| *height < keep_from) + { + self.scanned_block_hashes.pop_first(); + } + // Not copied into `stage`: `save_to_file` writes the current window. + } + + /// Compare the remembered block at `height` with the oracle's, with one + /// `GetBlockHashByHeight` lookup (none when nothing is remembered there). + pub(crate) async fn oracle_view( + &self, + oracle: &mut P, + height: u64, + ) -> Result { + let Some(known) = u32::try_from(height) + .ok() + .and_then(|height| self.scanned_block_hashes.get(&height)) + .copied() + else { + return Ok(OracleView::Unknown); + }; + Ok(match oracle.block_hash_at(height).await? { + Some(hash) if hash == known => OracleView::Same, + Some(_) => OracleView::Different, + None => OracleView::Unknown, + }) + } + + /// The oracle disagrees with the remembered block at `disagreeing`. Find + /// the highest remembered height below it that the oracle still agrees + /// with, the fork point, with a binary search over lookups. A height the + /// oracle cannot answer for counts as disagreeing, which can only move + /// the result lower. + /// + /// When the oracle disagrees at the oldest remembered height the fork + /// cannot be located: [`ReorgTooDeep`], found without streaming anything, + /// so a halted wallet costs two lookups per poll. + pub(crate) async fn locate_fork( + &self, + oracle: &mut P, + disagreeing: u64, + ) -> Result { + let lowest = self + .lowest_known_block_height() + .filter(|lowest| *lowest <= disagreeing) + .ok_or_else(|| format!("no remembered block hash at or below height {disagreeing}"))?; + match self.oracle_view(oracle, lowest).await? { + OracleView::Same => {} + OracleView::Different => { + return Err(Box::new(ReorgTooDeep { + lowest_known_height: lowest, + last_scanned_height: self.last_scanned_block_height, + })); + } + OracleView::Unknown => { + return Err(format!( + "chain reorganisation detected at height {disagreeing}, but the oracle serves \ + no block at height {lowest}, the oldest one remembered; retrying" + ) + .into()); + } + } + let (mut agrees, mut differs) = (lowest, disagreeing); + while differs - agrees > 1 { + let mid = agrees + (differs - agrees) / 2; + if self.oracle_view(oracle, mid).await? == OracleView::Same { + agrees = mid; + } else { + differs = mid; + } + } + Ok(agrees) } /// Undo everything learned from blocks above `fork_height`, which were @@ -191,7 +291,6 @@ impl Scanner { self.block_checkpoints.split_off(&(fork + 1)); self.stage.block_checkpoints.split_off(&(fork + 1)); self.scanned_block_hashes.split_off(&(fork + 1)); - self.stage.scanned_block_hashes = self.scanned_block_hashes.clone(); self.last_scanned_block_height = self.last_scanned_block_height.min(fork_height); self.stage.last_scanned_block_height = self.last_scanned_block_height; self.last_scanned_block_height_rescan = diff --git a/blindbit-lib/src/scanner/reorg_tests.rs b/blindbit-lib/src/scanner/reorg_tests.rs index a9c2b55..136fe80 100644 --- a/blindbit-lib/src/scanner/reorg_tests.rs +++ b/blindbit-lib/src/scanner/reorg_tests.rs @@ -12,6 +12,8 @@ use std::collections::{BTreeMap, VecDeque}; use std::future::Future; use std::net::SocketAddr; use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use bdk_sp::bitcoin::key::Secp256k1; use bdk_sp::receive::get_silentpayment_pubkey; @@ -27,11 +29,15 @@ use bitcoin::{ use bitcoin_rev::Network; use tonic::transport::Channel; +use super::health::{OracleProbe, indexed_block_hash}; use super::reorg::ReorgTooDeep; use super::scanning::{BlockScanDataStream, BlockStreamSource}; use super::stream_safety_tests::serve; use super::{REORG_LOOKBACK, Scanner, ScannerError}; -use crate::oracle_grpc::{BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem}; +use crate::oracle_grpc::{ + BlockHashResponse, BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem, +}; +use prost::Message as _; const FORK: u64 = 104; @@ -49,6 +55,31 @@ struct Oracle { /// stream began below it (so the first block of a stream is always /// served). fail_at: Option, + /// What the scanner asked of this oracle (shared by its clones). + traffic: Arc, +} + +/// Requests and response payload bytes (gRPC message plus its 5-byte frame +/// header), as the oracle would send them. +#[derive(Default)] +struct Traffic { + streams: AtomicUsize, + streamed_blocks: AtomicUsize, + streamed_bytes: AtomicUsize, + lookups: AtomicUsize, + lookup_bytes: AtomicUsize, +} + +impl Traffic { + fn take(&self) -> [usize; 5] { + [ + self.streams.swap(0, Ordering::SeqCst), + self.streamed_blocks.swap(0, Ordering::SeqCst), + self.streamed_bytes.swap(0, Ordering::SeqCst), + self.lookups.swap(0, Ordering::SeqCst), + self.lookup_bytes.swap(0, Ordering::SeqCst), + ] + } } struct ChainStream(VecDeque>); @@ -65,6 +96,7 @@ impl BlockStreamSource for Oracle { type Stream = ChainStream; async fn open(&mut self, start: u64, end: u64) -> Result { + self.traffic.streams.fetch_add(1, Ordering::SeqCst); let mut items = VecDeque::new(); for height in start..=end { if self.fail_at == Some(height) && start < height { @@ -72,7 +104,13 @@ impl BlockStreamSource for Oracle { break; } match self.chain.get(&height) { - Some(message) => items.push_back(Ok(message.clone())), + Some(message) => { + self.traffic.streamed_blocks.fetch_add(1, Ordering::SeqCst); + self.traffic + .streamed_bytes + .fetch_add(message.encoded_len() + 5, Ordering::SeqCst); + items.push_back(Ok(message.clone())) + } None => break, } } @@ -80,6 +118,27 @@ impl BlockStreamSource for Oracle { } } +impl OracleProbe for Oracle { + async fn block_hash_at(&mut self, height: u64) -> Result, ScannerError> { + let block_hash = self + .chain + .get(&height) + .and_then(|m| m.block_identifier.as_ref()) + .map(|id| id.block_hash.clone()) + .unwrap_or_default(); + self.traffic.lookups.fetch_add(1, Ordering::SeqCst); + self.traffic.lookup_bytes.fetch_add( + BlockHashResponse { + block_hash: block_hash.clone(), + } + .encoded_len() + + 5, + Ordering::SeqCst, + ); + Ok(indexed_block_hash(&block_hash)) + } +} + impl Oracle { fn tip(&self) -> u64 { *self.chain.keys().next_back().expect("non-empty chain") @@ -95,6 +154,7 @@ impl Oracle { .map(|(h, m)| (*h, m.clone())) .collect(), fail_at: None, + traffic: Arc::default(), } } @@ -672,8 +732,8 @@ fn reorg_deeper_than_the_lookback_is_a_loud_error_and_changes_nothing() { }); } -/// With no reorg, the watch loop's check reads one block and changes -/// nothing, and continuing the chain still scans new blocks. +/// With no reorg, the watch loop's check changes nothing, and continuing +/// the chain still scans new blocks. #[test] fn unchanged_chain_passes_the_check() { run(async { @@ -692,3 +752,216 @@ fn unchanged_chain_passes_the_check() { assert_eq!(balance(&scanner), 100_000); }); } + +// --------------------------------------------------------------------------- +// What a poll costs, and an oracle that switches branches mid-stream +// --------------------------------------------------------------------------- + +/// An oracle that switches from `before` to `after` once a stream has +/// delivered `switch_after` blocks; lookups answer from the current branch. +#[derive(Clone)] +struct SwitchingOracle { + before: Oracle, + after: Oracle, + switch_after: usize, + switched: Arc, +} + +impl BlockStreamSource for SwitchingOracle { + type Stream = ChainStream; + + async fn open(&mut self, start: u64, end: u64) -> Result { + let mut items = VecDeque::new(); + for height in start..=end { + let switched = self.switched.load(Ordering::SeqCst); + if !switched && items.len() == self.switch_after { + self.switched.store(true, Ordering::SeqCst); + } + let chain = if self.switched.load(Ordering::SeqCst) { + &self.after + } else { + &self.before + }; + match chain.chain.get(&height) { + Some(message) => items.push_back(Ok(message.clone())), + None => break, + } + } + Ok(ChainStream(items)) + } +} + +impl OracleProbe for SwitchingOracle { + async fn block_hash_at(&mut self, height: u64) -> Result, ScannerError> { + if self.switched.load(Ordering::SeqCst) { + self.after.block_hash_at(height).await + } else { + self.before.block_hash_at(height).await + } + } +} + +/// With no new block, a poll checks the scanned tip with one hash lookup and +/// downloads no block; a new block is streamed once, not together with the +/// block below it. +#[test] +fn idle_poll_is_one_hash_lookup_and_a_new_block_is_streamed_once() { + run(async { + let (mut scanner, _state) = wallet("poll-cost"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + let tip_block = s.old.chain[&s.old.tip()].encoded_len() + 5; + s.old.traffic.take(); + + for _ in 0..3 { + assert!(!scanner.watch_step(s.old.tip(), s.old.clone()).await); + let [streams, blocks, _, lookups, lookup_bytes] = s.old.traffic.take(); + assert_eq!((streams, blocks, lookups), (0, 0, 1), "idle poll"); + assert!( + lookup_bytes < tip_block, + "{lookup_bytes} B lookup vs the {tip_block} B block the overlap streamed" + ); + } + + let mut longer = s.old.clone(); + longer.empty(0xa, 107); + assert!(!scanner.watch_step(107, longer.clone()).await); + let [streams, blocks, _, lookups, _] = longer.traffic.take(); + assert_eq!((streams, blocks), (1, 1), "only the new block is streamed"); + // The block below it before, and both again after the stream. + assert_eq!(lookups, 3); + assert_eq!(scanner.get_last_scanned_block_height(), 107); + assert_eq!(scanner.scan_health().await.stall, None); + }); +} + +/// A one-block tip replacement is located with lookups; only the fork point +/// and the new block are streamed, not the whole lookback window. +#[test] +fn tip_reorg_streams_from_the_fork_point_only() { + run(async { + let (mut scanner, _state) = wallet("locate-cheap"); + let (recv_tx, tweak, outpoint, _) = payment(0x61, 40_000); + let tip = 101 + u64::from(REORG_LOOKBACK) + 10; + let mut old = Oracle::default(); + old.block(0xa, 101, vec![(recv_tx, Some(tweak))], &[]); + for h in 102..=tip { + old.empty(0xa, h); + } + scanner + .scan_range_from(101, tip, old.clone()) + .await + .expect("old chain scans"); + + let mut replaced = old.branch_at(tip - 1); + replaced.empty(0xb, tip); + assert!(!scanner.watch_step(tip, replaced.clone()).await); + let [streams, blocks, _, lookups, _] = replaced.traffic.take(); + assert_eq!(streams, 1); + assert_eq!(blocks, 2, "the fork point (checked) and the replaced tip"); + // 1 for the tip, 1 for the oldest remembered height, a binary search + // over the 144-height window, 3 after the stream. + assert!(lookups <= 14, "{lookups} lookups"); + assert_eq!(scanner.get_last_scanned_block_height(), tip); + assert_eq!(owned_height(&scanner, outpoint), Some(101)); + }); +} + +/// The oracle switches branches while a stream runs: it serves 105 from the +/// old branch (a payment) and 106 from the new one, which forks at 104. Each +/// block looks fine on its own, since nothing links 106' to 105. The check +/// after the stream finds the disagreement and rolls the payment back. +#[test] +fn oracle_switching_branches_mid_stream_is_rolled_back() { + run(async { + let (mut scanner, _state) = wallet("mid-stream"); + let s = scenario(); + let mut upto_fork = s.old.branch_at(FORK); + upto_fork.traffic = Arc::default(); + scanner + .scan_range_from(101, FORK, upto_fork) + .await + .expect("chain up to the fork scans"); + let mut new = s.new.clone(); + new.empty(0xb, 108); + let oracle = SwitchingOracle { + before: s.old.clone(), + after: new.clone(), + switch_after: 1, + switched: Arc::default(), + }; + + assert!(!scanner.watch_step(106, oracle).await); + + assert_eq!( + owned_height(&scanner, s.evicted), + None, + "old 106 was never served" + ); + assert_eq!( + owned_height(&scanner, s.reconfirmed), + Some(106), + "the old 105 payment is gone and found again at 106'" + ); + assert_eq!(anchor_heights(&scanner, s.reconfirmed.txid), vec![106]); + assert_eq!(balance(&scanner), 80_000); + assert_eq!(scanner.get_last_scanned_block_height(), 106); + let mut probe = new.clone(); + for height in 101..=106 { + assert_eq!( + scanner.oracle_view(&mut probe, height).await.unwrap(), + super::reorg::OracleView::Same, + "remembered hash at {height} is on the new branch" + ); + } + }); +} + +/// A reorganisation below the lookback halts the wallet loudly: a stall in +/// the status says what to do, every poll fails again, and each poll costs +/// two hash lookups rather than a stream of the lookback window. +#[test] +fn too_deep_reorg_is_a_stall_and_polls_stay_cheap() { + run(async { + let (mut scanner, _state) = wallet("too-deep-stall"); + let tip = 101 + u64::from(REORG_LOOKBACK) + 10; + let mut old = Oracle::default(); + for h in 101..=tip { + old.empty(0xa, h); + } + scanner + .scan_range_from(101, tip, old.clone()) + .await + .expect("old chain scans"); + let mut new = old.branch_at(101); + for h in 102..=tip { + new.empty(0xb, h); + } + + let mut first_since = None; + for _ in 0..3 { + assert!(!scanner.watch_step(tip, new.clone()).await); + let [streams, blocks, _, lookups, _] = new.traffic.take(); + assert_eq!((streams, blocks, lookups), (0, 0, 2)); + let stall = scanner + .scan_health() + .await + .stall + .expect("published as a stall"); + assert!( + stall + .reason + .contains("deeper than the scanner can roll back") + && stall.reason.contains("rescan"), + "{}", + stall.reason + ); + assert_eq!(stall.height, tip + 1); + assert_eq!( + *first_since.get_or_insert(stall.since_unix), + stall.since_unix + ); + } + assert_eq!(scanner.get_last_scanned_block_height(), tip); + }); +} diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index bef0141..2e484e9 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -17,7 +17,11 @@ use crate::oracle_grpc::{ use super::electrum_index::{ScriptHashEntry, SpHistoryEntry, electrum_scripthash}; use super::health::ScanStopped; use super::p2p; -use super::reorg::{BlockCheck, ReorgBelowStreamStart, ReorgTooDeep, block_hash_from_oracle}; +use super::health::OracleProbe; +use super::reorg::{ + BlockCheck, OracleView, POST_STREAM_CHECK, ReorgBelowStreamStart, ReorgTooDeep, + block_hash_from_oracle, +}; use super::scanner::Scanner; use super::types::{BlockIdentifierDisplay, ProbableMatch}; use super::utils::{byte_array_to_txid, construct_dummy_tx, match_short_pubkey}; @@ -83,9 +87,16 @@ pub(crate) trait BlockStreamSource { ) -> impl Future> + Send; } -/// The oracle's gRPC service as a [`BlockStreamSource`]. +/// The oracle's gRPC service as a [`BlockStreamSource`] and [`OracleProbe`]. +#[derive(Clone)] struct OracleStreams(crate::oracle_grpc::oracle_service_client::OracleServiceClient); +impl OracleProbe for OracleStreams { + async fn block_hash_at(&mut self, height: u64) -> Result, ScannerError> { + self.0.block_hash_at(height).await + } +} + impl BlockStreamSource for OracleStreams { type Stream = tonic::Streaming; @@ -176,10 +187,11 @@ impl Scanner { /// Scan a block range for new utxos and spent outpoints, after checking /// that the chain already scanned below it was not reorganised. /// - /// When `start - 1` is a scanned height, its block is streamed too and - /// compared with the remembered hash; a reorganisation is rolled back to - /// the fork point and the new branch scanned (see `scanner/reorg.rs`). - /// `start = end + 1` only runs that check. A reorganisation deeper than + /// When `start - 1` is a scanned height, the oracle's hash for it is + /// looked up and compared with the remembered one; a reorganisation is + /// rolled back to the fork point and the new branch scanned (see + /// `scanner/reorg.rs`). `start = end + 1` only runs that check, which + /// streams nothing unless the chain changed. A reorganisation deeper than /// [`super::REORG_LOOKBACK`] is a [`ReorgTooDeep`] error. pub async fn scan_block_range( &mut self, @@ -190,31 +202,39 @@ impl Scanner { self.scan_range_from(start, end, source).await } - /// [`Self::scan_block_range`] over any [`BlockStreamSource`]. - pub(crate) async fn scan_range_from( + /// [`Self::scan_block_range`] over any oracle. + pub(crate) async fn scan_range_from( &mut self, start: u64, end: u64, mut source: Src, ) -> Result<(), ScannerError> { - // Overlap one block with what was already scanned, so the stream - // itself proves the chain below `start` is unchanged. - let mut first = match start.checked_sub(1) { - Some(prev) if prev > 0 && self.knows_block_hash(prev) => prev, - _ => start, - }; - if first > end { - return Ok(()); + let mut start = start; + // The block below `start` is checked with a hash lookup instead of + // being streamed again. + let mut first = start; + if let Some(prev) = start.checked_sub(1).filter(|prev| *prev > 0) + && self.oracle_view(&mut source, prev).await? == OracleView::Different + { + tracing::warn!( + height = prev, + "chain reorganisation detected; locating the fork point" + ); + first = self.locate_fork(&mut source, prev).await?; } + let mut restarts = 0; loop { + if first > end { + return Ok(()); + } let stream = source.open(first, end).await?; match self.scan_block_stream_from(first, start, end, stream).await { Err(e) => { let Some(reorg) = e.downcast_ref::() else { return Err(e); }; - // The fork is below the stream's first block: walk up - // from the oldest remembered hash to find it. + // The chain changed again between the lookups and the + // stream: walk up from the oldest remembered hash. match self.lowest_known_block_height() { Some(lowest) if lowest < reorg.height => { tracing::warn!( @@ -232,7 +252,39 @@ impl Scanner { } } } - ok => return ok, + Ok(()) => { + // A stream that ran while the oracle switched branches + // can hand over blocks of both: check its newest blocks, + // and the one below them, again. + let check_from = end + .saturating_sub(POST_STREAM_CHECK - 1) + .max(first.saturating_sub(1)) + .max(1); + let mut disagreeing = None; + for height in check_from..=end { + if self.oracle_view(&mut source, height).await? == OracleView::Different { + disagreeing = Some(height); + break; + } + } + let Some(height) = disagreeing else { + return Ok(()); + }; + restarts += 1; + if restarts > 3 { + return Err(format!( + "the oracle's chain kept changing while blocks up to {end} were \ + scanned; will retry" + ) + .into()); + } + tracing::warn!( + height, + "chain reorganisation while scanning; locating the fork point" + ); + first = self.locate_fork(&mut source, height).await?; + start = self.last_scanned_block_height + 1; + } } } } @@ -820,43 +872,58 @@ impl Scanner { } }; - // With no new block, still check the scanned tip (or the - // oracle's, if it is behind) against the oracle: a reorganisation - // can replace blocks without making the chain longer. - let (from, to) = if oracle_tip > self.last_scanned_block_height { - let from = self.last_scanned_block_height + 1; - tracing::info!(from, to = oracle_tip, "new blocks available, scanning"); - (from, oracle_tip) - } else { - (oracle_tip + 1, oracle_tip) - }; - match self.scan_block_range(from, to).await { - Ok(()) => self.clear_stall().await, - Err(e) => { - let mut probe = self.client.clone(); - if from <= to - && self - .start_at_oracle_floor_if_below(&e, from, to, &mut probe) - .await - { - continue; - } - if e.downcast_ref::().is_some() { - tracing::error!( - error = %e, - "wallet state cannot follow the chain; scanning is halted until it is rescanned" - ); - } else { - tracing::error!(error = %e, "scan_block_range failed, will retry on next poll"); - } - let height = e - .downcast_ref::() - .map_or(self.last_scanned_block_height + 1, |s| s.height); - self.report_stall(height, e.to_string()).await; - } + let oracle = OracleStreams(self.client.clone()); + if !self.watch_step(oracle_tip, oracle).await { + time::sleep(time::Duration::from_secs(10)).await; } + } + } - time::sleep(time::Duration::from_secs(10)).await; + /// One poll of [`Self::watch_chain`], once the oracle's tip is known: + /// scan new blocks, or with none, check the scanned tip (or the oracle's, + /// if it is behind) against the oracle, since a reorganisation can + /// replace blocks without making the chain longer. Publishes a stall on + /// failure and clears it on success. Returns `true` when the next poll + /// should run right away. + pub(crate) async fn watch_step( + &mut self, + oracle_tip: u64, + mut oracle: Src, + ) -> bool { + let (from, to) = if oracle_tip > self.last_scanned_block_height { + let from = self.last_scanned_block_height + 1; + tracing::info!(from, to = oracle_tip, "new blocks available, scanning"); + (from, oracle_tip) + } else { + (oracle_tip + 1, oracle_tip) + }; + match self.scan_range_from(from, to, oracle.clone()).await { + Ok(()) => { + self.clear_stall().await; + false + } + Err(e) => { + if from <= to + && self + .start_at_oracle_floor_if_below(&e, from, to, &mut oracle) + .await + { + return true; + } + if e.downcast_ref::().is_some() { + tracing::error!( + error = %e, + "wallet state cannot follow the chain; scanning is halted until it is rescanned" + ); + } else { + tracing::error!(error = %e, "scan_block_range failed, will retry on next poll"); + } + let height = e + .downcast_ref::() + .map_or(self.last_scanned_block_height + 1, |s| s.height); + self.report_stall(height, e.to_string()).await; + false + } } } From 54927656ae5b2342b9ffc095d125b6c5193b841b Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 1 Oct 2026 14:02:08 +0000 Subject: [PATCH 09/10] fix(tray): hide hidden status blocks (.col display overrode the hidden attribute) The status window's error, stall and scan-notes blocks use class="col", whose display: flex beats the user-agent [hidden] rule, so an empty "Last error" label was always shown and the new stall/notes blocks would be too. Seen in a screenshot against a live daemon. Co-Authored-By: Claude Opus 5.5 --- friglet-tray/ui/style.css | 2 ++ 1 file changed, 2 insertions(+) diff --git a/friglet-tray/ui/style.css b/friglet-tray/ui/style.css index ac43b48..ee8b9e8 100644 --- a/friglet-tray/ui/style.css +++ b/friglet-tray/ui/style.css @@ -71,6 +71,8 @@ code { color: var(--text); } .error { color: var(--bad); } +/* `.col` sets display: flex, which would override the `hidden` attribute. */ +.col[hidden] { display: none; } #scan-notes { font-size: 13px; white-space: pre-line; } .small { font-size: 12px; min-height: 18px; } From d989e537f87e22cb74faf91489274fd7697b0842 Mon Sep 17 00:00:00 2001 From: setavenger Date: Thu, 1 Oct 2026 15:03:06 +0000 Subject: [PATCH 10/10] fix(cli): print a failed scan's error with Display, not Debug Since ScanStopped became a struct (22c50bf), blindbit-cli printed a stopped scan as `Error: ScanStopped { height: 104, reason: "...", not_indexed: true }`, the Debug form main uses for a returned error. The message the type renders for the user, "scan stopped at height 104: ...; nothing from height 104 on was scanned, rescan from height 104", no longer appeared. The testkit's mutation checks match that message and went red for stale-view, truncated-stream, close-mid-stream and unindexed-ok-empty. main now runs the command and prints the error with Display. The exit status of a failed command is still 1. Co-Authored-By: Claude Opus 5.5 --- blindbit-cli/src/main.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/blindbit-cli/src/main.rs b/blindbit-cli/src/main.rs index 883443c..79bf73b 100644 --- a/blindbit-cli/src/main.rs +++ b/blindbit-cli/src/main.rs @@ -1,7 +1,7 @@ use bitcoin::secp256k1::{PublicKey, SecretKey}; use blindbit_lib::scanner::{self, ScannerConfig}; use clap::{Parser, Subcommand}; -use std::{net::SocketAddr, path::PathBuf, str::FromStr}; +use std::{net::SocketAddr, path::PathBuf, process::ExitCode, str::FromStr}; use bitcoin_rev::Network; @@ -58,7 +58,20 @@ enum Commands { } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> ExitCode { + match run().await { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + // Display, not the Debug form `main` would print for a returned error: a + // stopped scan (`ScanStopped`) says which height it stopped at and where to + // rescan from, and its Debug form is a struct dump. + eprintln!("Error: {e}"); + ExitCode::FAILURE + } + } +} + +async fn run() -> Result<(), Box> { let cli = Cli::parse(); match cli.command {