diff --git a/blindbit-cli/src/main.rs b/blindbit-cli/src/main.rs index 883443c..79bf73b 100644 --- a/blindbit-cli/src/main.rs +++ b/blindbit-cli/src/main.rs @@ -1,7 +1,7 @@ use bitcoin::secp256k1::{PublicKey, SecretKey}; use blindbit_lib::scanner::{self, ScannerConfig}; use clap::{Parser, Subcommand}; -use std::{net::SocketAddr, path::PathBuf, str::FromStr}; +use std::{net::SocketAddr, path::PathBuf, process::ExitCode, str::FromStr}; use bitcoin_rev::Network; @@ -58,7 +58,20 @@ enum Commands { } #[tokio::main] -async fn main() -> Result<(), Box> { +async fn main() -> ExitCode { + match run().await { + Ok(()) => ExitCode::SUCCESS, + Err(e) => { + // Display, not the Debug form `main` would print for a returned error: a + // stopped scan (`ScanStopped`) says which height it stopped at and where to + // rescan from, and its Debug form is a struct dump. + eprintln!("Error: {e}"); + ExitCode::FAILURE + } + } +} + +async fn run() -> Result<(), Box> { let cli = Cli::parse(); match cli.command { diff --git a/blindbit-lib/examples/stream_oracle.rs b/blindbit-lib/examples/stream_oracle.rs index e293e1f..8e73581 100644 --- a/blindbit-lib/examples/stream_oracle.rs +++ b/blindbit-lib/examples/stream_oracle.rs @@ -17,10 +17,11 @@ async fn main() -> Result<(), Box> { let mut stream = client .stream_block_scan_data_short(request) - .await - .unwrap() + .await? .into_inner(); - while let Some(block_scan_data) = stream.message().await.unwrap() { + // An error status (e.g. NOT_FOUND for a height the oracle has not + // indexed) ends the stream; `?` surfaces it instead of panicking. + while let Some(block_scan_data) = stream.message().await? { let Some(block_identifier) = block_scan_data.block_identifier.clone() else { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -28,6 +29,15 @@ async fn main() -> Result<(), Box> { ) .into()); }; + // Older oracles answer an unindexed height with an empty block hash + // and no data; that is not an empty block. + if block_identifier.block_hash.len() != 32 { + return Err(format!( + "height {} has no valid block hash; the oracle has probably not indexed it", + block_identifier.block_height + ) + .into()); + } println!("height: {}", block_identifier.block_height); } diff --git a/blindbit-lib/src/scanner/bip352_vectors.rs b/blindbit-lib/src/scanner/bip352_vectors.rs index 40f6125..55638b4 100644 --- a/blindbit-lib/src/scanner/bip352_vectors.rs +++ b/blindbit-lib/src/scanner/bip352_vectors.rs @@ -1,11 +1,16 @@ //! Official BIP-352 receiver vectors exercised at the blindbit-lib boundary. //! -//! The boundary under test is [`Scanner::scan_transaction_full`], which has no -//! production caller today — the daemon receives through -//! `Scanner::scan_transaction_short` plus `apply_block_relevant` on the external -//! indexer. These tests are what exercises it, and they pin its behaviour for -//! whenever full block responses are scanned again; they say nothing directly -//! about the live short-scan path. +//! Two boundaries are under test: +//! +//! * [`Scanner::scan_transaction_full`], which has no production caller today. +//! The first group of tests pins its behaviour for whenever full block +//! responses are scanned again. +//! * The live receive path the daemon actually runs: the oracle's short block +//! data through `Scanner::scan_short_block_data` (and so +//! `scan_transaction_short` / `match_short_pubkey`), then, on a probable match, +//! the full block through `Scanner::apply_matched_block` +//! (`apply_block_relevant` on the external indexer). See the section "The live +//! short receive path" at the end of this file. //! //! Fixture provenance (re-checked at runtime by [`vectors`], so a silent swap or //! a corrupted copy fails loudly instead of quietly changing coverage): @@ -1555,3 +1560,497 @@ fn official_vectors_extra_change_label_is_harmless() { "exactly one non-K_max sub-case (the sender-change vector) registers m=0 itself" ); } + +// --------------------------------------------------------------------------- +// The live short receive path +// --------------------------------------------------------------------------- +// +// Everything above goes through `scan_transaction_full`. The daemon does not: +// per block it asks the oracle for `BlockScanDataShortResponse`, runs +// `scan_short_block_data` -> `probabilistic_match` -> `scan_transaction_short` +// (k = 0 only, labels enumerated by hand, 8-byte x-only prefixes), and only on a +// probable match fetches the full block and hands it to `apply_matched_block`, +// i.e. `apply_block_relevant` on the external indexer. A false negative in the +// short step means the block is never fetched, so the wallet silently never +// sees the payment. The tests below drive every receiving vector through that +// exact sequence of production functions. + +/// Length of one `outputs_short` entry: the oracle stores `Pubkey[:8]` of every +/// taproot output (`blindbit-oracle` `dbpebble/store.go`). Deliberately not +/// shared with `match_short_pubkey`, so a change there cannot silently change +/// what the test builds too. +const SHORT_PREFIX_LEN: usize = 8; + +/// Height the synthetic blocks claim to be at. +const LIVE_BLOCK_HEIGHT: u64 = 840_000; + +/// A txid as the oracle puts it on the wire: display order (see [`full_item`]). +fn wire_txid(tx: &Transaction) -> [u8; 32] { + let mut txid = tx.compute_txid().to_raw_hash().to_byte_array(); + txid.reverse(); + txid +} + +/// The `ComputeIndexTxItem` the oracle serves for `tx`: the 33-byte tweak +/// (`input_hash * A`) and, for every **taproot** output in output order, the +/// first [`SHORT_PREFIX_LEN`] bytes of its x-only key. Non-taproot outputs are +/// not stored by the oracle and so never appear. +fn short_item(tx: &Transaction, tweak: PublicKey) -> crate::oracle_grpc::ComputeIndexTxItem { + let outputs_short = tx + .output + .iter() + .filter(|output| output.script_pubkey.is_p2tr()) + .flat_map(|output| output.script_pubkey.as_bytes()[2..2 + SHORT_PREFIX_LEN].to_vec()) + .collect(); + crate::oracle_grpc::ComputeIndexTxItem { + txid: wire_txid(tx).to_vec(), + tweak: tweak.serialize().to_vec(), + outputs_short, + } +} + +/// A non-coinbase transaction that is not ours, so the block the receive step +/// sees is not just the vector transaction on its own. +fn decoy_transaction() -> Transaction { + let decoy_key = XOnlyPublicKey::from_str( + "79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798", + ) + .expect("decoy key"); + Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { + txid: Txid::from_byte_array([0x11; 32]), + vout: 7, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(5_000), + script_pubkey: ScriptBuf::new_p2tr_tweaked( + bdk_sp::bitcoin::key::TweakedPublicKey::dangerous_assume_tweaked(decoy_key), + ), + }], + } +} + +/// The full block the daemon would fetch over P2P: a coinbase (which +/// `apply_block_relevant` skips by position), a decoy, then the vector's +/// transaction. +fn live_block(tx: &Transaction) -> bitcoin::Block { + let coinbase = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes(vec![0x03, 0x40, 0xd1, 0x0c]), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(312_500_000), + script_pubkey: ScriptBuf::from_bytes(vec![0x6a]), + }], + }; + let mut block = bitcoin::Block { + header: bitcoin::block::Header { + version: bitcoin::block::Version::TWO, + prev_blockhash: bitcoin::BlockHash::all_zeros(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: 1_713_571_767, + bits: bitcoin::CompactTarget::from_consensus(0x1703_4219), + nonce: 0, + }, + txdata: vec![coinbase, decoy_transaction(), tx.clone()], + }; + block.header.merkle_root = block.compute_merkle_root().expect("non-empty block"); + block +} + +/// The per-block short response the oracle streams, carrying `items`. +fn short_block( + block: &bitcoin::Block, + items: Vec, +) -> crate::oracle_grpc::BlockScanDataShortResponse { + let mut block_hash = block.block_hash().to_raw_hash().to_byte_array(); + block_hash.reverse(); + crate::oracle_grpc::BlockScanDataShortResponse { + block_identifier: Some(crate::oracle_grpc::BlockIdentifier { + block_hash: block_hash.to_vec(), + block_height: LIVE_BLOCK_HEIGHT, + }), + comp_index: items, + spent_outputs: Vec::new(), + } +} + +/// Which k = 0 candidate the vector transaction actually carries, by full +/// 32-byte key: `Some(None)` for the unlabelled `P_0`, `Some(Some(m))` for +/// `P_0 + label_m`, `None` if neither. This is independent of the short path +/// and says which label iteration the short path *must* perform to match. +fn k0_candidates( + tx: &Transaction, + given: &ReceivingGiven, + scan_sk: SecretKey, + spend_pk: PublicKey, + shared_secret: PublicKey, +) -> BTreeSet> { + let keys: HashSet = tx + .output + .iter() + .filter(|output| output.script_pubkey.is_p2tr()) + .map(|output| { + XOnlyPublicKey::from_slice(&output.script_pubkey.as_bytes()[2..]).expect("taproot key") + }) + .collect(); + std::iter::once(None) + .chain(given.labels.iter().map(|&m| Some(m))) + .filter(|label| { + let label_pt = label.map(|m| label_point(scan_sk, m)); + let p_0 = get_silentpayment_pubkey(&spend_pk, &shared_secret, 0, label_pt.as_ref()); + keys.contains(&p_0.x_only_public_key().0) + }) + .collect() +} + +/// No false negatives on the live short path, for every receiving vector. +/// +/// Each vector transaction is served the way the oracle serves it (8-byte +/// prefixes of its taproot outputs, the 33-byte tweak, the display-order txid) +/// and pushed through `scan_short_block_data`, the function the daemon calls per +/// block. Wherever the vector expects the wallet to find at least one output, +/// the block must come back as a probable match carrying that txid and tweak. +/// +/// The k = 0 candidate each matched vector carries is pinned independently (see +/// [`k0_candidates`]), so the test also proves *which* short-path branch did the +/// work: the unlabelled `P_0`, a labelled one of either output parity, and the +/// change label `m = 0`. Dropping the label iteration, or matching on anything +/// other than the 8-byte x-only prefix, turns this red. +/// +/// False positives (a probable match where the vector expects nothing) are only +/// a wasted block fetch; they are counted and reported, and pinned at zero on +/// these vectors because an 8-byte prefix collision among them would be a +/// fixture surprise worth looking at. +#[test] +fn official_vectors_reach_the_live_short_receive_path() { + let mut no_wire_tweak = 0usize; + let mut must_match = 0usize; + let mut must_not_match = 0usize; + let mut false_positives = Vec::new(); + // Matched sub-cases whose only k = 0 candidate is labelled, by label. + let mut label_only: BTreeMap = BTreeMap::new(); + let mut unlabelled_k0 = 0usize; + let mut parity_cases: BTreeMap<&str, bool> = BTreeMap::new(); + + for (case_idx, case) in vectors().iter().enumerate() { + for (sub_idx, receiving) in case.receiving.iter().enumerate() { + let what = format!("case {} sub {sub_idx} ({})", case_idx + 1, case.comment); + let (given, expected) = (&receiving.given, &receiving.expected); + let (tx, prevouts) = build_transaction(given); + + let tweak = match compute_tweak_data(&tx, &prevouts) { + Ok(tweak) => tweak, + Err(err) => { + // The oracle cannot publish a tweak for these, so there is no + // short item to serve; classified as in the boundary test. + let reason = classify_no_tweak(&tx, &prevouts, &err, &what); + assert_eq!(reason, expected_no_tweak_reason(&case.comment), "{what}"); + assert!(expected.outputs.is_empty() && expected.n_outputs.is_none()); + no_wire_tweak += 1; + continue; + } + }; + assert_eq!( + tweak.to_string(), + expected.tweak.as_deref().expect("vector tweak"), + "{what}: served tweak" + ); + let (scan_sk, spend_pk) = scan_keys(given); + let shared_secret = bdk_sp::compute_shared_secret(&scan_sk, &tweak); + + let item = short_item(&tx, tweak); + assert_eq!(item.tweak.len(), 33, "{what}: served tweak length"); + assert_eq!( + item.outputs_short.len(), + SHORT_PREFIX_LEN * given.outputs.len(), + "{what}: every vector output is taproot and must be served" + ); + + let mut scanner = blindbit_scanner( + scan_sk, + spend_pk, + &given.labels, + &format!("short-{case_idx}-{sub_idx}"), + ); + let probable = scanner + .scan_short_block_data(short_block(&live_block(&tx), vec![item])) + .expect("short block scan"); + let matched = match &probable { + Some(probable) => { + assert!(!probable.spent, "{what}: no spent outputs were served"); + assert_eq!( + probable.matched_txs, + vec![(wire_txid(&tx), tweak)], + "{what}: probable match must carry the served txid and tweak" + ); + true + } + None => false, + }; + + let expected_count = expected.n_outputs.unwrap_or(expected.outputs.len()); + if expected_count == 0 { + must_not_match += 1; + if matched { + false_positives.push(what); + } + continue; + } + + must_match += 1; + assert!( + matched, + "{what}: FALSE NEGATIVE on the live short path — the vector expects \ + {expected_count} output(s), but the block would never be fetched" + ); + + let k0 = k0_candidates(&tx, given, scan_sk, spend_pk, shared_secret); + assert!( + !k0.is_empty(), + "{what}: a transaction paying us must carry a k = 0 output" + ); + if k0.contains(&None) { + unlabelled_k0 += 1; + } else { + for m in k0.iter().flatten() { + *label_only.entry(*m).or_default() += 1; + } + } + + for kind in ["even", "odd"] { + if case.comment.contains(&format!("label with {kind} parity")) { + parity_cases.insert(kind, !k0.contains(&None)); + } + } + } + } + + for fp in &false_positives { + eprintln!("short-path false positive: {fp}"); + } + eprintln!( + "short path: {must_match} sub-cases must match (all did), {must_not_match} must \ + not, {} false positive(s), {no_wire_tweak} without a wire tweak", + false_positives.len() + ); + + assert_eq!(no_wire_tweak, NO_TWEAK_SUBCASE_COUNT); + assert_eq!( + must_match + must_not_match + no_wire_tweak, + RECEIVING_SUBCASE_COUNT, + "not every receiving sub-case reached the short path" + ); + assert_eq!( + must_not_match, 1, + "the 'recipient ignores unrelated outputs' vector is no longer covered" + ); + assert!( + false_positives.is_empty(), + "unexpected short-path false positives: {false_positives:?}" + ); + + // Branch coverage of the short path: vectors that can only be found through + // the label iteration, including both parities and the change label m = 0. + assert_eq!( + parity_cases.get("even"), + Some(&true), + "the even-parity label vector must be reachable only through a label" + ); + assert_eq!( + parity_cases.get("odd"), + Some(&true), + "the odd-parity label vector must be reachable only through a label" + ); + assert!( + label_only.contains_key(&0), + "no vector is reachable only through the change label m = 0: {label_only:?}" + ); + let label_only_total: usize = label_only.values().sum(); + assert_eq!( + label_only_total, 7, + "label-only coverage changed: {label_only:?}" + ); + assert!(unlabelled_k0 > 0, "no unlabelled k = 0 vector was matched"); + eprintln!("short path: {unlabelled_k0} matched via P_0, label-only by m: {label_only:?}"); +} + +/// Runs one vector through the whole live receive sequence — short scan, then +/// (only on a probable match, as the daemon does) the full-block step — and +/// returns every output the indexer now owns in the vector transaction, as +/// `x-only key -> priv_key_tweak`, plus its label. +fn live_receive( + given: &ReceivingGiven, + tx: &Transaction, + tweak: PublicKey, + tag: &str, +) -> (Scanner, BTreeMap)>) { + let (scan_sk, spend_pk) = scan_keys(given); + let mut scanner = blindbit_scanner(scan_sk, spend_pk, &given.labels, tag); + let block = live_block(tx); + + let probable = scanner + .scan_short_block_data(short_block(&block, vec![short_item(tx, tweak)])) + .expect("short block scan"); + if let Some(probable) = probable { + scanner.apply_matched_block(&block, &probable, LIVE_BLOCK_HEIGHT as u32); + } + + let txid = tx.compute_txid(); + let index = scanner.internal_indexer.index(); + let labels: BTreeMap> = index + .by_label + .iter() + .map(|&(label, outpoint)| (outpoint, label)) + .collect(); + let mut found = BTreeMap::new(); + for (outpoint, tweak) in &index.by_shared_secret { + assert_eq!( + outpoint.txid, txid, + "{tag}: indexed an output of another tx" + ); + let script = &tx.output[outpoint.vout as usize].script_pubkey; + let xonly = XOnlyPublicKey::from_slice(&script.as_bytes()[2..]).expect("taproot output"); + let label = *labels + .get(outpoint) + .expect("every indexed output has a label entry"); + assert!( + found.insert(xonly, (*tweak, label)).is_none(), + "{tag}: output key indexed twice" + ); + } + (scanner, found) +} + +/// The real receive step after a probable match, for every vector. +/// +/// After [`official_vectors_reach_the_live_short_receive_path`] establishes that +/// the block is fetched, this pins what the daemon then *stores*: the vector +/// transaction is placed in a full block and handed through +/// `Scanner::apply_matched_block` (the production code that maps the oracle's +/// display-order txids to the block's transactions and calls +/// `apply_block_relevant` on the external indexer). The outputs the indexer +/// owns afterwards must be exactly the vector's `expected.outputs`, with the +/// vector's `priv_key_tweak` and a label that reproduces the output key. +/// +/// The K_max vector is excluded here and has its own ignored test (SNB-540). +#[test] +fn official_vectors_live_receive_step_finds_expected_outputs() { + let mut exercised = 0usize; + let mut asserted_outputs = 0usize; + let mut labelled_outputs = 0usize; + + for (case_idx, case) in vectors().iter().enumerate() { + for (sub_idx, receiving) in case.receiving.iter().enumerate() { + let what = format!("case {} sub {sub_idx} ({})", case_idx + 1, case.comment); + let (given, expected) = (&receiving.given, &receiving.expected); + if expected.n_outputs.is_some() { + continue; // K_max: see the ignored SNB-540 test below. + } + let (tx, prevouts) = build_transaction(given); + let Ok(tweak) = compute_tweak_data(&tx, &prevouts) else { + continue; // no wire tweak; classified in the short-path test + }; + let (scan_sk, spend_pk) = scan_keys(given); + let shared_secret = bdk_sp::compute_shared_secret(&scan_sk, &tweak); + + let (scanner, found) = + live_receive(given, &tx, tweak, &format!("live-{case_idx}-{sub_idx}")); + + let found_tweaks: BTreeMap = found + .iter() + .map(|(key, (tweak, _))| (*key, *tweak)) + .collect(); + assert_eq!( + found_tweaks, + expected_outputs(expected), + "{what}: live receive step outputs / priv_key_tweak" + ); + + // Each stored label must reproduce the output key at some k. + for (key, (tweak, label)) in &found { + if let Some(m) = label { + assert!(given.labels.contains(m), "{what}: stray label {m}"); + labelled_outputs += 1; + } + let k = (0..given.outputs.len() as u32) + .find(|&k| tweak_for(shared_secret, scan_sk, k, *label) == *tweak) + .unwrap_or_else(|| panic!("{what}: tweak/label mismatch for {key}")); + let label_pt = label.map(|m| label_point(scan_sk, m)); + let p_k = get_silentpayment_pubkey(&spend_pk, &shared_secret, k, label_pt.as_ref()); + assert_eq!(p_k.x_only_public_key().0, *key, "{what}: stored label"); + } + + // And the transaction is in the wallet graph iff something was found. + let in_graph = scanner + .internal_indexer + .graph() + .get_tx(tx.compute_txid()) + .is_some(); + assert_eq!(in_graph, !found.is_empty(), "{what}: wallet graph"); + let staged = scanner + .stage + .indexer + .txid_to_partial_secret + .contains_key(&tx.compute_txid()); + assert_eq!(staged, !found.is_empty(), "{what}: staged for persistence"); + + asserted_outputs += found.len(); + exercised += 1; + } + } + + assert_eq!( + exercised, + SCANNED_SUBCASE_COUNT - K_MAX_SUBCASE_COUNT, + "not every scannable non-K_max sub-case reached the live receive step" + ); + assert_eq!( + asserted_outputs, EXPECTED_OUTPUT_COUNT, + "the fixture's per-output expectations were not all asserted on the live path" + ); + assert_eq!(labelled_outputs, 10, "labelled-output coverage changed"); +} + +/// The BIP-352 recipient limit on the live receive path. +/// +/// Ignored because it fails today: `apply_block_relevant` enforces no K_max, so +/// the live path stores one output more than BIP-352 allows on this vector. +/// That is SNB-540; this test is the acceptance check for it. It is also slow +/// (the external indexer's scan is quadratic in the 2324 outputs). +#[test] +#[ignore = "SNB-540: the live receive path enforces no K_max (stores 2324 of 2323)"] +fn official_vectors_live_receive_step_enforces_k_max() { + let mut exercised = 0usize; + for case in vectors() { + for receiving in &case.receiving { + let Some(n_outputs) = receiving.expected.n_outputs else { + continue; + }; + let given = &receiving.given; + let (tx, prevouts) = build_transaction(given); + let tweak = compute_tweak_data(&tx, &prevouts).expect("K_max vector inputs are valid"); + let (_, found) = live_receive(given, &tx, tweak, "live-kmax"); + assert_eq!( + found.len(), + n_outputs, + "{}: receiver must stop at K_max", + case.comment + ); + exercised += 1; + } + } + assert_eq!(exercised, K_MAX_SUBCASE_COUNT); +} diff --git a/blindbit-lib/src/scanner/changeset.rs b/blindbit-lib/src/scanner/changeset.rs index 66ca190..22fcc79 100644 --- a/blindbit-lib/src/scanner/changeset.rs +++ b/blindbit-lib/src/scanner/changeset.rs @@ -3,34 +3,49 @@ use indexer::bdk_chain::ConfirmationBlockTime; use indexer::bdk_chain::bdk_core::Merge; use std::collections::BTreeMap; -/// Helper module for hex encoding/decoding byte arrays in serialization +use super::health::OracleFloorStart; +use super::types::OwnedOutputRecord; + +/// Format of the persisted [`ChangeSet`] this build writes. +/// +/// - `0` (field absent): written before owned outputs were recorded. Such a +/// state never fetched blocks whose only wallet-relevant transaction was a +/// spend, so spends of its outputs may be missing; restoring it rescans +/// from the oldest unspent output (see `Scanner::from_changeset`). +/// - `1`: `owned_outputs` holds [`OwnedOutputRecord`]s. Builds older than +/// this format cannot read it (they expect bare hex keys there). +pub const STATE_FORMAT_VERSION: u32 = 1; + +/// Deserialisation of the persisted `owned_outputs` array. #[cfg(feature = "serde")] -mod serde_hex { - use serde::{Deserialize, Deserializer, Serialize, Serializer}; +mod owned_outputs_serde { + use super::OwnedOutputRecord; + use serde::{Deserialize, Deserializer}; - pub fn serialize_vec(vec: &Vec<[u8; 32]>, serializer: S) -> Result - where - S: Serializer, - { - let hex_strings: Vec = vec.iter().map(|bytes| hex::encode(bytes)).collect(); - hex_strings.serialize(serializer) + #[derive(Deserialize)] + #[serde(untagged)] + enum Entry { + Record(OwnedOutputRecord), + /// State files written before owned outputs were recorded stored bare + /// hex keys here. No released writer ever populated that list, and a + /// bare key carries no outpoint, so such entries are dropped; the + /// records are rebuilt from the indexer on load + /// (`Scanner::sync_owned_outputs`). + Legacy(#[allow(dead_code)] String), } - pub fn deserialize_vec<'de, D>(deserializer: D) -> Result, D::Error> + pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> where D: Deserializer<'de>, { - let hex_strings: Vec = Vec::deserialize(deserializer)?; - let mut result = Vec::new(); - for hex_string in hex_strings { - let bytes = hex::decode(&hex_string) - .map_err(|e| serde::de::Error::custom(format!("Invalid hex string: {}", e)))?; - let bytes_array: [u8; 32] = bytes - .try_into() - .map_err(|_| serde::de::Error::custom("Expected 32 bytes"))?; - result.push(bytes_array); - } - Ok(result) + let entries: Vec = Vec::deserialize(deserializer)?; + Ok(entries + .into_iter() + .filter_map(|entry| match entry { + Entry::Record(record) => Some(record), + Entry::Legacy(_) => None, + }) + .collect()) } } @@ -45,6 +60,10 @@ mod serde_hex { #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] #[must_use] pub struct ChangeSet { + /// [`STATE_FORMAT_VERSION`] of the build that wrote this state; `0` when + /// absent (written before the field existed). + #[cfg_attr(feature = "serde", serde(default))] + pub format_version: u32, /// Sparse block checkpoints: only blocks where we found something (height -> hash) pub block_checkpoints: BTreeMap, /// Changes related to the Silent Payments indexer data. @@ -53,21 +72,33 @@ pub struct ChangeSet { pub last_scanned_block_height: u64, /// The last block height that was scanned on most recent rescan pub last_scanned_block_height_rescan: u64, - /// Owned output pubkeys; used to check for spent outputs (hex encoded in JSON) + /// Wallet-owned outputs (full x-only key, label, spent status); the + /// scanner matches the oracle's spent-output prefixes against these. #[cfg_attr( feature = "serde", - serde( - serialize_with = "serde_hex::serialize_vec", - deserialize_with = "serde_hex::deserialize_vec" - ) + serde(default, deserialize_with = "owned_outputs_serde::deserialize") )] - pub owned_outputs: Vec<[u8; 32]>, + pub owned_outputs: Vec, /// Secret scan key (hex encoded) - needed to reconstruct the indexer pub secret_scan_hex: Option, /// Public spend key (hex encoded) - needed to reconstruct the indexer pub public_spend_hex: Option, /// Maximum label number used pub max_label_num: u32, + /// Hash of every block scanned within the reorg lookback window + /// (`REORG_LOOKBACK` heights below the highest one). Compared with the + /// oracle's hashes to detect a chain reorganisation; empty in state files + /// written before reorg handling, which start recording from their next + /// scan. + #[cfg_attr(feature = "serde", serde(default))] + pub scanned_block_hashes: BTreeMap, + /// Set when the wallet's start height lay below the oracle's first + /// indexed block and scanning began there instead. + #[cfg_attr( + feature = "serde", + serde(default, skip_serializing_if = "Option::is_none") + )] + pub oracle_floor_start: Option, } impl Merge for ChangeSet { @@ -75,6 +106,7 @@ impl Merge for ChangeSet { fn merge(&mut self, other: Self) { // Merge block checkpoints (extend with new ones) self.block_checkpoints.extend(other.block_checkpoints); + self.scanned_block_hashes.extend(other.scanned_block_hashes); Merge::merge(&mut self.indexer, other.indexer); // Update metadata with the latest values @@ -85,10 +117,16 @@ impl Merge for ChangeSet { self.last_scanned_block_height_rescan = other.last_scanned_block_height_rescan; } - // Merge owned_outputs (deduplicate) + // Merge owned_outputs by outpoint; the newer record wins (it may carry + // a spend the older one did not). for output in other.owned_outputs { - if !self.owned_outputs.contains(&output) { - self.owned_outputs.push(output); + match self + .owned_outputs + .iter_mut() + .find(|existing| existing.outpoint == output.outpoint) + { + Some(existing) => *existing = output, + None => self.owned_outputs.push(output), } } @@ -104,6 +142,12 @@ impl Merge for ChangeSet { if other.max_label_num > self.max_label_num { self.max_label_num = other.max_label_num; } + + if other.oracle_floor_start.is_some() { + self.oracle_floor_start = other.oracle_floor_start; + } + + self.format_version = self.format_version.max(other.format_version); } /// Checks if the [`ChangeSet`] is empty (contains no changes). @@ -113,5 +157,6 @@ impl Merge for ChangeSet { && self.last_scanned_block_height == 0 && self.last_scanned_block_height_rescan == 0 && self.owned_outputs.is_empty() + && self.scanned_block_hashes.is_empty() } } diff --git a/blindbit-lib/src/scanner/electrum_index.rs b/blindbit-lib/src/scanner/electrum_index.rs index dde8d28..ff7efac 100644 --- a/blindbit-lib/src/scanner/electrum_index.rs +++ b/blindbit-lib/src/scanner/electrum_index.rs @@ -7,6 +7,8 @@ use bitcoin::ScriptBuf; use bitcoin::hashes::{Hash, sha256}; use std::collections::HashMap; +use super::health::ScanHealth; + /// Lightweight Electrum-compatible index built from scanner state. #[derive(Default)] pub struct WalletElectrumIndex { @@ -54,6 +56,10 @@ pub struct WalletElectrumIndex { /// Ordered list of confirmed SP receives. pub sp_history: Vec, + + /// Stalls and start-height adjustments of the running scan, published + /// here for status readers (see `scanner/health.rs`). + pub scan_health: ScanHealth, } /// A single entry in a scripthash's transaction history. @@ -81,6 +87,17 @@ impl WalletElectrumIndex { pub fn new() -> Self { Self::default() } + + /// Height to report as the chain tip once `scanned` is scanned. While a + /// restored state is rescanned for missed spends (`scan_health. + /// state_rescan`), the tip stays at the height the state had reached, so + /// clients never see wallet history above the tip. + pub fn tip_height_for(&self, scanned: u32) -> u32 { + match self.scan_health.state_rescan { + Some(rescan) => scanned.max(u32::try_from(rescan.until_height).unwrap_or(u32::MAX)), + None => scanned, + } + } } /// Compute the Electrum scripthash for a script. diff --git a/blindbit-lib/src/scanner/health.rs b/blindbit-lib/src/scanner/health.rs new file mode 100644 index 0000000..02df13a --- /dev/null +++ b/blindbit-lib/src/scanner/health.rs @@ -0,0 +1,347 @@ +//! What a long-running scan (`Scanner::watch_chain`) reports besides its +//! progress: why it is stuck, and where it started when the wallet's start +//! height lay below the oracle's first indexed block. +//! +//! The scan task holds the scanner lock for as long as it runs, so these +//! conditions are published through the wallet's [`WalletElectrumIndex`] +//! (`scan_health`), which status readers can lock without waiting for the +//! scan. +//! +//! [`WalletElectrumIndex`]: super::electrum_index::WalletElectrumIndex + +use std::future::Future; +use std::time::{SystemTime, UNIX_EPOCH}; + +use bitcoin::BlockHash; +use bitcoin::hashes::Hash; +use tonic::transport::Channel; + +use super::ScannerError; +use super::scanner::Scanner; +use crate::oracle_grpc::BlockHeightRequest; +use crate::oracle_grpc::oracle_service_client::OracleServiceClient; + +/// Conditions of a running scan worth showing to the user. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ScanHealth { + /// Set while the scan cannot get past a height. The scan keeps retrying; + /// the entry is cleared as soon as it makes progress again. + pub stall: Option, + /// Set when the wallet's start height lay below the oracle's first + /// indexed block, so scanning began at the oracle's floor instead. + pub oracle_floor_start: Option, + /// Set while a state file from before owned outputs were recorded is + /// being rescanned for spends it missed; cleared once the scan is back + /// at `until_height`. + pub state_rescan: Option, +} + +/// The restored state predates owned-output records (state format 0), so +/// blocks `from_height..=until_height` are scanned again to find spends of +/// its outputs that it never fetched. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StateRescan { + /// Height of the oldest output the state still showed unspent. + pub from_height: u64, + /// Height the state had been scanned to. + pub until_height: u64, +} + +/// The scan cannot get past `height`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScanStall { + /// First height that could not be scanned (the scan resumes here). + pub height: u64, + /// Why, as the scanner reported it. + pub reason: String, + /// Unix time (seconds) of the first failed attempt at this height. + pub since_unix: u64, +} + +/// The wallet's start height had no oracle data, so scanning began at the +/// oracle's lowest indexed height. Blocks `requested_height..floor_height` +/// were not scanned. Persisted in the state file. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct OracleFloorStart { + /// The start height the wallet asked for. + pub requested_height: u64, + /// The oracle's lowest indexed height, where scanning began. + pub floor_height: u64, +} + +/// A range scan stopped at `height` without scanning it. +/// +/// Every block below `height` in the range was fully processed; nothing from +/// `height` on was. The caller resumes at `height`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScanStopped { + /// First height that was not scanned. + pub height: u64, + /// What the oracle sent (or failed to send) for `height`. + pub reason: String, + /// The oracle reported `height` as not indexed: an empty or zero block + /// hash, or a `NOT_FOUND` status. + pub not_indexed: bool, +} + +impl std::fmt::Display for ScanStopped { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let height = self.height; + write!( + f, + "scan stopped at height {height}: {}; nothing from height {height} on was scanned, \ + rescan from height {height}", + self.reason + ) + } +} + +impl std::error::Error for ScanStopped {} + +/// The oracle's per-height block hash lookup (`GetBlockHashByHeight`). +pub(crate) trait OracleProbe { + /// The block hash the oracle has indexed at `height`, or `None` when it + /// has not indexed that height (an empty or zero hash, or `NOT_FOUND`). + /// Any other failure is an error. + fn block_hash_at( + &mut self, + height: u64, + ) -> impl Future, ScannerError>> + Send; +} + +impl OracleProbe for OracleServiceClient { + async fn block_hash_at(&mut self, height: u64) -> Result, ScannerError> { + let request = tonic::Request::new(BlockHeightRequest { + block_height: height, + }); + match self.get_block_hash_by_height(request).await { + Ok(response) => Ok(indexed_block_hash(&response.into_inner().block_hash)), + Err(status) if status.code() == tonic::Code::NotFound => Ok(None), + Err(status) => Err(format!( + "oracle GetBlockHashByHeight({height}) failed: {:?}: {}", + status.code(), + status.message() + ) + .into()), + } + } +} + +/// A block hash as the oracle serves it (display order), or `None` when it is +/// not a real hash: what an oracle answers for a height it has not indexed. +pub(crate) fn indexed_block_hash(display_order: &[u8]) -> Option { + let mut bytes: [u8; 32] = display_order.try_into().ok()?; + if bytes.iter().all(|b| *b == 0) { + return None; + } + bytes.reverse(); + Some(BlockHash::from_byte_array(bytes)) +} + +/// Upper bound on `GetBlockHashByHeight` calls one floor search may make. +/// A contiguous index needs about `3 * log2(tip)` (~60 on mainnet). +const FLOOR_SEARCH_MAX_PROBES: u32 = 400; + +/// Find the oracle's lowest indexed height when `from`, the wallet's start +/// height, is not indexed. +/// +/// Returns `Some(floor)` only when `from` lies *below* the oracle's floor: +/// nothing at or below `from` is indexed and `floor` is the lowest indexed +/// height above it. Returns `None` when `from` is a gap *above* the floor +/// (some lower height is indexed), when `from` is indexed after all, or when +/// the oracle has no data even at `tip`; such a height must not be skipped. +/// +/// The index need not be contiguous. A binary search finds a boundary, then +/// probes at exponentially growing distances below it look for indexed +/// heights the search jumped over; any hit restarts the search below it. The +/// same back-off below `from` tells a gap above the floor from the region +/// below it. A gap can only be missed when it is wider than the indexed run +/// below it, which no real oracle produces. +pub(crate) async fn find_oracle_floor( + probe: &mut P, + from: u64, + tip: u64, +) -> Result, ScannerError> { + let mut oracle = CountedProbe { probe, probes: 0 }; + + if from >= tip || oracle.indexed(from).await? { + return Ok(None); + } + // Anything indexed below `from` makes it a gap above the floor. + let mut distance = 1u64; + while distance <= from { + if oracle.indexed(from - distance).await? { + return Ok(None); + } + distance *= 2; + } + + if !oracle.indexed(tip).await? { + return Ok(None); + } + // Invariant: `from` is not indexed, `hi` is. + let mut hi = tip; + loop { + let mut lo = from; + while hi - lo > 1 { + let mid = lo + (hi - lo) / 2; + if oracle.indexed(mid).await? { + hi = mid; + } else { + lo = mid; + } + } + // `hi - 1` is not indexed; look further down for indexed runs the + // search jumped over. + let mut lower = None; + let mut distance = 2u64; + while hi - from > distance { + if oracle.indexed(hi - distance).await? { + lower = Some(hi - distance); + break; + } + distance *= 2; + } + match lower { + Some(height) => hi = height, + None => return Ok(Some(hi)), + } + } +} + +/// An [`OracleProbe`] with a lookup budget. +struct CountedProbe<'a, P> { + probe: &'a mut P, + probes: u32, +} + +impl CountedProbe<'_, P> { + async fn indexed(&mut self, height: u64) -> Result { + self.probes += 1; + if self.probes > FLOOR_SEARCH_MAX_PROBES { + return Err(format!( + "gave up locating the oracle's first indexed block after \ + {FLOOR_SEARCH_MAX_PROBES} lookups" + ) + .into()); + } + Ok(self.probe.block_hash_at(height).await?.is_some()) + } +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +impl Scanner { + /// Current scan health as published to status readers. + pub async fn scan_health(&self) -> ScanHealth { + self.electrum_index.lock().await.scan_health.clone() + } + + /// Publish that the scan cannot get past `height`. Repeated failures at + /// the same height keep the time of the first one. + pub(crate) async fn report_stall(&self, height: u64, reason: String) { + let mut idx = self.electrum_index.lock().await; + let since_unix = match &idx.scan_health.stall { + Some(stall) if stall.height == height => stall.since_unix, + _ => unix_now(), + }; + idx.scan_health.stall = Some(ScanStall { + height, + reason, + since_unix, + }); + } + + /// The scan made progress or is caught up: clear any stall, and end a + /// state rescan ([`StateRescan`]) once it is back where it started. + pub(crate) async fn clear_stall(&self) { + let mut idx = self.electrum_index.lock().await; + if idx.scan_health.stall.take().is_some() { + tracing::info!( + height = self.last_scanned_block_height, + "scan is making progress again" + ); + } + if idx + .scan_health + .state_rescan + .is_some_and(|rescan| self.last_scanned_block_height >= rescan.until_height) + { + idx.scan_health.state_rescan = None; + tracing::info!( + height = self.last_scanned_block_height, + "rescan for spends missed by the old state file is complete" + ); + } + } + + /// After a failed scan of `from..=tip`: when `from` is the wallet's start + /// height and lies below the oracle's first indexed block, move the + /// start up to that block and return `true` (scan again right away). + /// + /// Only heights below the oracle's floor are ever skipped. A height the + /// oracle has not indexed above its floor (a gap) is never skipped; the + /// scan keeps stopping there until the oracle serves it. + pub(crate) async fn start_at_oracle_floor_if_below( + &mut self, + error: &ScannerError, + from: u64, + tip: u64, + probe: &mut P, + ) -> bool { + let Some(stopped) = error.downcast_ref::() else { + return false; + }; + if !stopped.not_indexed || stopped.height != from { + return false; + } + // Only at the wallet's start: once a block was scanned, the height + // after it is a gap, not the floor. + let wallet_start = self.electrum_index.lock().await.sp_start_height; + if wallet_start == 0 || from > wallet_start { + return false; + } + let floor = match find_oracle_floor(probe, from, tip).await { + Ok(Some(floor)) => floor, + Ok(None) => return false, + Err(e) => { + tracing::warn!( + error = %e, + height = from, + "could not locate the oracle's first indexed block" + ); + return false; + } + }; + + tracing::warn!( + requested_height = from, + oracle_floor = floor, + skipped_blocks = floor - from, + "the wallet's start height is below the oracle's first indexed block; scanning \ + starts at the oracle's first block, and blocks below it are not scanned" + ); + let note = OracleFloorStart { + requested_height: from, + floor_height: floor, + }; + self.update_last_scanned_block_height(floor - 1); + self.stage.oracle_floor_start = Some(note); + { + let mut idx = self.electrum_index.lock().await; + idx.scan_health.oracle_floor_start = Some(note); + idx.scan_health.stall = None; + } + #[cfg(feature = "serde")] + if let Err(e) = self.save_to_file(&self.state_file) { + tracing::warn!(error = %e, "failed to save state"); + } + true + } +} diff --git a/blindbit-lib/src/scanner/mod.rs b/blindbit-lib/src/scanner/mod.rs index c84fe07..0c5a2f2 100644 --- a/blindbit-lib/src/scanner/mod.rs +++ b/blindbit-lib/src/scanner/mod.rs @@ -3,10 +3,18 @@ mod changeset; mod config; pub mod electrum_index; +mod health; mod load; +#[cfg(test)] +mod oracle_floor_tests; mod p2p; +mod reorg; +#[cfg(test)] +mod reorg_tests; mod scanner; mod scanning; +#[cfg(test)] +mod stream_safety_tests; mod types; mod utils; @@ -14,12 +22,14 @@ mod utils; pub type ScannerError = Box; // Re-export public types and the main Scanner struct -pub use changeset::ChangeSet; +pub use changeset::{ChangeSet, STATE_FORMAT_VERSION}; pub use config::ScannerConfig; pub use electrum_index::{ScriptHashEntry, SpHistoryEntry, WalletElectrumIndex, electrum_scripthash, electrum_status}; +pub use health::{OracleFloorStart, ScanHealth, ScanStall, ScanStopped, StateRescan}; pub use p2p::broadcast_tx; +pub use reorg::{REORG_LOOKBACK, ReorgTooDeep}; pub use scanner::Scanner; -pub use types::{BlockIdentifierDisplay, OwnedOutput}; +pub use types::{BlockIdentifierDisplay, OwnedOutput, OwnedOutputRecord}; // Re-export load_scanner function when serde feature is enabled #[cfg(feature = "serde")] diff --git a/blindbit-lib/src/scanner/oracle_floor_tests.rs b/blindbit-lib/src/scanner/oracle_floor_tests.rs new file mode 100644 index 0000000..ab80713 --- /dev/null +++ b/blindbit-lib/src/scanner/oracle_floor_tests.rs @@ -0,0 +1,339 @@ +//! A wallet whose start height lies below the oracle's first indexed block +//! starts at that block instead of stalling forever; a height the oracle has +//! not indexed *above* its first block is never skipped. A scan that cannot +//! get past a height is published as a stall. + +use std::future::Future; +use std::ops::RangeInclusive; + +use bitcoin::BlockHash; +use bitcoin::hashes::Hash; + +use super::ScannerError; +use super::health::{OracleProbe, ScanStopped, find_oracle_floor}; +use super::scanner::Scanner; +use super::stream_safety_tests::{ + TestStream, owned_outputs, payment_block, run, scanner, unindexed_message, +}; + +/// `GetBlockHashByHeight` over fixed runs of indexed heights. +struct FakeOracle { + indexed: Vec>, + probes: u32, + fail: bool, +} + +impl FakeOracle { + fn contiguous(floor: u64, tip: u64) -> Self { + Self { + indexed: std::iter::once(floor..=tip).collect(), + probes: 0, + fail: false, + } + } + + fn empty() -> Self { + Self { + indexed: vec![], + probes: 0, + fail: false, + } + } + + fn without(mut self, gap: RangeInclusive) -> Self { + self.indexed = self + .indexed + .into_iter() + .flat_map(|run| { + let below = *run.start()..=(*gap.start()).min(run.end() + 1).saturating_sub(1); + let above = (*gap.end() + 1).max(*run.start())..=*run.end(); + [below, above] + }) + .filter(|run| !run.is_empty()) + .collect(); + self + } + + fn is_indexed(&self, height: u64) -> bool { + self.indexed.iter().any(|run| run.contains(&height)) + } +} + +impl OracleProbe for FakeOracle { + fn block_hash_at( + &mut self, + height: u64, + ) -> impl Future, ScannerError>> + Send { + self.probes += 1; + let result = if self.fail { + Err("oracle unavailable".into()) + } else { + Ok(self + .is_indexed(height) + .then(|| BlockHash::from_byte_array([1; 32]))) + }; + std::future::ready(result) + } +} + +fn floor(oracle: &mut FakeOracle, from: u64, tip: u64) -> Option { + run(find_oracle_floor(oracle, from, tip)).expect("probe succeeds") +} + +/// The hosted signet oracle has nothing below 100 000; mainnet nothing below +/// 800 000. A search over the whole chain stays cheap. +#[test] +fn start_below_a_contiguous_index_finds_its_first_block() { + let mut signet = FakeOracle::contiguous(100_000, 324_494); + assert_eq!(floor(&mut signet, 1, 324_494), Some(100_000)); + assert!(signet.probes <= 60, "{} lookups", signet.probes); + + let mut mainnet = FakeOracle::contiguous(800_000, 965_000); + assert_eq!(floor(&mut mainnet, 700_000, 965_000), Some(800_000)); + assert_eq!(floor(&mut mainnet, 799_999, 965_000), Some(800_000)); + assert!(mainnet.probes <= 140, "{} lookups", mainnet.probes); +} + +/// Before blindbit-oracle#66 a fresh oracle never indexed its configured +/// start height: nothing at or below it, everything above it. +#[test] +fn oracle_missing_its_own_start_height_starts_one_above() { + let mut oracle = FakeOracle::contiguous(101, 300); + assert_eq!(floor(&mut oracle, 100, 300), Some(101)); + assert_eq!(floor(&mut oracle, 1, 300), Some(101)); +} + +/// A binary search alone would land in the gap at 2000..=2010 (the midpoint +/// of 10..4000 is 2005) and report 2011, skipping 1000 indexed blocks. +#[test] +fn gap_above_the_floor_does_not_move_the_floor_up() { + let mut oracle = FakeOracle::contiguous(1_000, 4_000).without(2_000..=2_010); + assert_eq!(floor(&mut oracle, 10, 4_000), Some(1_000)); +} + +#[test] +fn start_inside_a_gap_above_the_floor_is_never_skipped() { + let mut oracle = FakeOracle::contiguous(1_000, 4_000).without(2_000..=2_010); + for from in [2_000, 2_005, 2_010] { + assert_eq!(floor(&mut oracle, from, 4_000), None, "from {from}"); + } + // One missing block right above a long indexed run. + let mut oracle = FakeOracle::contiguous(1_000, 4_000).without(3_000..=3_000); + assert_eq!(floor(&mut oracle, 3_000, 4_000), None); +} + +/// Randomised layouts: a floor, then a handful of gaps above it, each +/// narrower than the run below it. The search must always return the +/// floor, and never skip from inside a gap. +#[test] +fn randomised_gappy_indexes() { + let mut seed = 0x9e37_79b9_7f4a_7c15_u64; + let mut next = move |bound: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % bound + }; + for _ in 0..2_000 { + let tip = 5_000 + next(1_000_000); + let floor_height = 1 + next(tip - 100); + let mut oracle = FakeOracle::contiguous(floor_height, tip); + let mut gaps = Vec::new(); + let mut cursor = floor_height; + for _ in 0..next(6) { + let run = 50 + next(5_000); + let start = cursor + run; + let width = 1 + next(run.min(40)); + if start + width >= tip { + break; + } + oracle = oracle.without(start..=start + width - 1); + gaps.push(start..=start + width - 1); + cursor = start + width; + } + let from = next(floor_height); + assert_eq!( + floor(&mut oracle, from, tip), + Some(floor_height), + "floor {floor_height} tip {tip} gaps {gaps:?} from {from}" + ); + for gap in &gaps { + assert_eq!(floor(&mut oracle, *gap.start(), tip), None, "gap {gap:?}"); + } + } +} + +#[test] +fn indexed_start_or_empty_oracle_is_not_a_floor() { + let mut oracle = FakeOracle::contiguous(100, 300); + assert_eq!(floor(&mut oracle, 150, 300), None, "start is indexed"); + let mut empty = FakeOracle::empty(); + assert_eq!( + floor(&mut empty, 1, 300), + None, + "nothing indexed at the tip" + ); +} + +#[test] +fn oracle_error_is_an_error_not_a_floor() { + let mut oracle = FakeOracle::contiguous(100, 300); + oracle.fail = true; + assert!(run(find_oracle_floor(&mut oracle, 1, 300)).is_err()); +} + +fn not_indexed_at(height: u64) -> ScannerError { + Box::new(ScanStopped { + height, + reason: "the oracle sent no valid block hash (0 bytes)".into(), + not_indexed: true, + }) +} + +async fn set_wallet_start(scanner: &mut Scanner, start: u64) { + scanner.update_last_scanned_block_height(start - 1); + scanner.electrum_index.lock().await.sp_start_height = start; +} + +/// End to end through the scan loop's pieces: the first stream stops at the +/// unindexed start height, the wallet moves to the oracle's first block, the +/// next stream from there finds the payment in it, and the adjustment is in +/// the status and survives a restart. +#[test] +fn wallet_start_below_the_floor_starts_at_the_floor_and_says_so() { + run(async { + let (mut scanner, state) = scanner("floor-start", "http://127.0.0.1:1"); + set_wallet_start(&mut scanner, 50).await; + let paid = payment_block(1_000); + let mut oracle = FakeOracle::contiguous(1_000, 1_005); + + let err = scanner + .scan_block_stream(50, 1_005, TestStream::new(vec![Ok(unindexed_message(50))])) + .await + .expect_err("the unindexed start height stops the scan"); + assert!( + scanner + .start_at_oracle_floor_if_below(&err, 50, 1_005, &mut oracle) + .await + ); + assert_eq!(scanner.get_last_scanned_block_height(), 999); + let health = scanner.scan_health().await; + let note = health.oracle_floor_start.expect("adjustment is published"); + assert_eq!((note.requested_height, note.floor_height), (50, 1_000)); + assert_eq!(health.stall, None); + + scanner + .scan_block_stream( + 1_000, + 1_000, + TestStream::new(vec![Ok(paid.message.clone())]), + ) + .await + .expect("scan from the floor"); + assert_eq!( + owned_outputs(&scanner), + 1, + "the payment at the floor is found" + ); + + #[cfg(feature = "serde")] + { + scanner.save_to_file(&state.0).expect("save"); + let restored = Scanner::from_changeset( + scanner.client.clone(), + scanner.p2p_peer(), + Scanner::load_from_file(&state.0).expect("load"), + state.0.clone(), + scanner.network(), + ) + .expect("restore"); + assert_eq!(restored.scan_health().await.oracle_floor_start, Some(note)); + } + let _ = &state; + }); +} + +/// Once a block was scanned, an unindexed height after it is a gap in the +/// oracle's index, not its floor: the scan keeps stopping there. +#[test] +fn unindexed_height_after_the_wallet_start_is_never_skipped() { + run(async { + let (mut scanner, _state) = scanner("floor-gap", "http://127.0.0.1:1"); + set_wallet_start(&mut scanner, 1_000).await; + scanner.update_last_scanned_block_height(1_499); + // Even an oracle that has nothing below 1 600 does not justify it. + let mut oracle = FakeOracle::contiguous(1_600, 2_000); + assert!( + !scanner + .start_at_oracle_floor_if_below(¬_indexed_at(1_500), 1_500, 2_000, &mut oracle) + .await + ); + assert_eq!(scanner.get_last_scanned_block_height(), 1_499); + assert_eq!(oracle.probes, 0); + assert_eq!(scanner.scan_health().await.oracle_floor_start, None); + }); +} + +#[test] +fn only_a_not_indexed_stop_at_the_start_moves_the_start() { + run(async { + let (mut scanner, _state) = scanner("floor-other", "http://127.0.0.1:1"); + set_wallet_start(&mut scanner, 50).await; + let mut oracle = FakeOracle::contiguous(1_000, 1_005); + let unavailable: ScannerError = Box::new(ScanStopped { + height: 50, + reason: "oracle stream error Unavailable".into(), + not_indexed: false, + }); + let plain: ScannerError = "failed to fetch block".into(); + for err in [unavailable, plain, not_indexed_at(51)] { + assert!( + !scanner + .start_at_oracle_floor_if_below(&err, 50, 1_005, &mut oracle) + .await, + "{err}" + ); + } + assert_eq!(scanner.get_last_scanned_block_height(), 49); + // Start height inside a gap above the floor. + let mut gappy = FakeOracle::contiguous(10, 1_005).without(40..=60); + assert!( + !scanner + .start_at_oracle_floor_if_below(¬_indexed_at(50), 50, 1_005, &mut gappy) + .await + ); + assert_eq!(scanner.get_last_scanned_block_height(), 49); + }); +} + +#[test] +fn a_stall_keeps_its_start_time_until_the_height_changes_and_clears_on_progress() { + run(async { + let (scanner, _state) = scanner("stall", "http://127.0.0.1:1"); + scanner.report_stall(1_500, "first".into()).await; + let first = scanner.scan_health().await.stall.expect("stall"); + { + let mut idx = scanner.electrum_index.lock().await; + idx.scan_health.stall.as_mut().unwrap().since_unix -= 600; + } + scanner.report_stall(1_500, "again".into()).await; + let again = scanner.scan_health().await.stall.expect("stall"); + assert_eq!(again.height, 1_500); + assert_eq!(again.reason, "again"); + assert_eq!( + again.since_unix, + first.since_unix - 600, + "same height keeps its start" + ); + + scanner.report_stall(1_600, "later".into()).await; + let moved = scanner.scan_health().await.stall.expect("stall"); + assert!( + moved.since_unix >= first.since_unix, + "a new height starts a new stall" + ); + + scanner.clear_stall().await; + assert_eq!(scanner.scan_health().await.stall, None); + }); +} diff --git a/blindbit-lib/src/scanner/owned_outputs_tests.rs b/blindbit-lib/src/scanner/owned_outputs_tests.rs new file mode 100644 index 0000000..46acf57 --- /dev/null +++ b/blindbit-lib/src/scanner/owned_outputs_tests.rs @@ -0,0 +1,638 @@ +//! Owned-output bookkeeping through the live short receive path. +//! +//! Each test drives a real [`Scanner`] exactly as `scan_block_range` does for +//! one oracle block: `scan_short_block_data` decides whether the block is worth +//! fetching, and only if it is, the full block goes through +//! `apply_matched_block` + `sync_owned_outputs` and becomes a checkpoint. The +//! P2P fetch is the only step replaced (the block is handed in directly). + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::OnceLock; + +use bdk_sp::bitcoin::key::Secp256k1; +use bdk_sp::hashes::get_label_tweak; +use bdk_sp::receive::get_silentpayment_pubkey; +use bitcoin::absolute::LockTime; +use bitcoin::hashes::Hash; +use bitcoin::key::TweakedPublicKey; +use bitcoin::secp256k1::{PublicKey, SecretKey}; +use bitcoin::transaction::Version; +use bitcoin::{ + Amount, Block, BlockHash, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, + Witness, XOnlyPublicKey, +}; +use bitcoin_rev::Network; +use indexer::bdk_chain::local_chain::LocalChain; +use indexer::bdk_chain::{BlockId, CanonicalizationParams}; +use tonic::transport::Channel; + +use super::Scanner; +use crate::oracle_grpc::{BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem}; +use crate::scanner::OwnedOutputRecord; + +const RECEIVE_HEIGHT: u32 = 100; +const SPEND_HEIGHT: u32 = 200; + +fn oracle_client() -> crate::OracleServiceClient { + static RUNTIME: OnceLock = OnceLock::new(); + let runtime = RUNTIME.get_or_init(|| { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime for the lazy tonic channel") + }); + let _guard = runtime.enter(); + crate::OracleServiceClient::new(Channel::from_static("http://[::1]:50051").connect_lazy()) +} + +fn state_file(tag: &str) -> PathBuf { + std::env::temp_dir().join(format!( + "blindbit-owned-outputs-{tag}-{}.json", + std::process::id() + )) +} + +fn secret(byte: u8) -> SecretKey { + SecretKey::from_slice(&[byte; 32]).expect("valid secret") +} + +fn keys() -> (SecretKey, PublicKey) { + let secp = Secp256k1::new(); + (secret(0x11), secret(0x22).public_key(&secp)) +} + +fn scanner(tag: &str, max_label_num: u32) -> Scanner { + let (scan_sk, spend_pk) = keys(); + let socket: SocketAddr = "127.0.0.1:8333".parse().expect("socket address"); + Scanner::new( + oracle_client(), + socket, + scan_sk, + spend_pk, + max_label_num, + state_file(tag), + Network::Regtest, + ) +} + +/// The tweak (`input_hash * A`) the oracle would serve for a payment. Any +/// point works: the receiver only ever sees this value. +fn served_tweak(byte: u8) -> PublicKey { + secret(byte).public_key(&Secp256k1::new()) +} + +/// `P_0` for the served tweak, plus `label_m * G` when labelled. +fn sp_output_key(tweak: &PublicKey, label: Option) -> XOnlyPublicKey { + let (scan_sk, spend_pk) = keys(); + let shared = bdk_sp::compute_shared_secret(&scan_sk, tweak); + let p0 = get_silentpayment_pubkey(&spend_pk, &shared, 0, None); + let key = match label { + None => p0, + Some(m) => { + let label_sk = SecretKey::from_slice(&get_label_tweak(scan_sk, m).to_be_bytes()) + .expect("label tweak is a valid scalar"); + p0.combine(&label_sk.public_key(&Secp256k1::new())) + .expect("labelled key") + } + }; + key.x_only_public_key().0 +} + +fn foreign_key(byte: u8) -> XOnlyPublicKey { + secret(byte) + .public_key(&Secp256k1::new()) + .x_only_public_key() + .0 +} + +fn p2tr(key: XOnlyPublicKey) -> ScriptBuf { + ScriptBuf::new_p2tr_tweaked(TweakedPublicKey::dangerous_assume_tweaked(key)) +} + +fn tx(inputs: Vec, outputs: Vec<(XOnlyPublicKey, u64)>) -> Transaction { + Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: inputs + .into_iter() + .map(|previous_output| TxIn { + previous_output, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }) + .collect(), + output: outputs + .into_iter() + .map(|(key, sat)| TxOut { + value: Amount::from_sat(sat), + script_pubkey: p2tr(key), + }) + .collect(), + } +} + +fn block(height: u32, txs: Vec) -> Block { + let coinbase = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes(height.to_le_bytes().to_vec()), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(312_500_000), + script_pubkey: ScriptBuf::from_bytes(vec![0x6a]), + }], + }; + let mut txdata = vec![coinbase]; + txdata.extend(txs); + let mut block = Block { + header: bitcoin::block::Header { + version: bitcoin::block::Version::TWO, + prev_blockhash: BlockHash::all_zeros(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: 1_713_571_767 + height, + bits: bitcoin::CompactTarget::from_consensus(0x207f_ffff), + nonce: height, + }, + txdata, + }; + block.header.merkle_root = block.compute_merkle_root().expect("non-empty block"); + block +} + +/// The oracle's `ComputeIndexTxItem`: wire-order txid, tweak, and the 8-byte +/// prefix of every taproot output key. +fn item(tx: &Transaction, tweak: PublicKey) -> ComputeIndexTxItem { + let mut txid = tx.compute_txid().to_byte_array(); + txid.reverse(); + ComputeIndexTxItem { + txid: txid.to_vec(), + tweak: tweak.serialize().to_vec(), + outputs_short: tx + .output + .iter() + .flat_map(|o| o.script_pubkey.as_bytes()[2..10].to_vec()) + .collect(), + } +} + +/// One block of `scan_block_range`, minus the P2P fetch. Returns whether the +/// scanner decided to fetch (and therefore apply) the block. +fn process_block( + scanner: &mut Scanner, + block: &Block, + height: u32, + items: Vec, + spent_prefixes: &[XOnlyPublicKey], +) -> bool { + let mut hash = block.block_hash().to_byte_array(); + hash.reverse(); + let short = BlockScanDataShortResponse { + block_identifier: Some(BlockIdentifier { + block_hash: hash.to_vec(), + block_height: height as u64, + }), + comp_index: items, + spent_outputs: spent_prefixes + .iter() + .flat_map(|key| key.serialize()[..8].to_vec()) + .collect(), + }; + let Some(probable_match) = scanner.scan_short_block_data(short).expect("short scan") else { + return false; + }; + scanner.apply_matched_block(block, &probable_match, height); + scanner.sync_owned_outputs(); + scanner.block_checkpoints.insert(height, block.block_hash()); + scanner + .stage + .block_checkpoints + .insert(height, block.block_hash()); + true +} + +/// The confirmed balance exactly as `scan_block_range` computes it. +fn balance(scanner: &Scanner) -> u64 { + let (&height, &hash) = scanner + .block_checkpoints + .iter() + .next_back() + .expect("genesis checkpoint"); + let outpoints: Vec<(u32, OutPoint)> = scanner + .internal_indexer + .index() + .by_shared_secret + .keys() + .map(|op| (op.vout, *op)) + .collect(); + let chain = LocalChain::from_blocks(scanner.block_checkpoints.clone()).expect("chain"); + scanner + .internal_indexer + .graph() + .balance( + &chain, + BlockId { height, hash }, + CanonicalizationParams::default(), + outpoints, + |_, _| true, + ) + .confirmed + .to_sat() +} + +fn record(scanner: &Scanner, outpoint: OutPoint) -> OwnedOutputRecord { + scanner + .owned_outputs() + .find(|r| r.outpoint == outpoint) + .cloned() + .unwrap_or_else(|| panic!("no owned-output record for {outpoint}")) +} + +/// Receives 50_000 sat to the unlabelled address at `RECEIVE_HEIGHT`. +fn receive(scanner: &mut Scanner) -> (OutPoint, XOnlyPublicKey) { + let tweak = served_tweak(0x33); + let ours = sp_output_key(&tweak, None); + let recv = tx( + vec![OutPoint::new(Txid::from_byte_array([0x44; 32]), 0)], + vec![(foreign_key(0x55), 10_000), (ours, 50_000)], + ); + let fetched = process_block( + scanner, + &block(RECEIVE_HEIGHT, vec![recv.clone()]), + RECEIVE_HEIGHT, + vec![item(&recv, tweak)], + &[], + ); + assert!(fetched, "receive block must be fetched"); + (OutPoint::new(recv.compute_txid(), 1), ours) +} + +/// A spend of `outpoint` paying only a foreign key: no output comes back to +/// the wallet, so only the spent-output prefix can make the scanner fetch it. +fn spend_of(outpoint: OutPoint) -> Transaction { + tx(vec![outpoint], vec![(foreign_key(0x66), 49_000)]) +} + +#[test] +fn spend_in_later_block_marks_output_spent_and_balance_drops() { + let mut scanner = scanner("spend", 0); + let (outpoint, key) = receive(&mut scanner); + + let rec = record(&scanner, outpoint); + assert_eq!(rec.pubkey, key, "full x-only key recorded"); + assert_eq!((rec.amount_sat, rec.height), (50_000, RECEIVE_HEIGHT)); + assert!(!rec.is_spent()); + assert_eq!(balance(&scanner), 50_000); + + // A block that spends nothing of ours is not fetched. + let unrelated = tx( + vec![OutPoint::new(Txid::from_byte_array([0x77; 32]), 3)], + vec![(foreign_key(0x78), 1_000)], + ); + assert!(!process_block( + &mut scanner, + &block(150, vec![unrelated]), + 150, + vec![], + &[foreign_key(0x79)], + )); + + let spend = spend_of(outpoint); + let fetched = process_block( + &mut scanner, + &block(SPEND_HEIGHT, vec![spend.clone()]), + SPEND_HEIGHT, + // The spend has a (foreign) taproot output, so the oracle indexes it + // with some tweak that does not match us. + vec![item(&spend, served_tweak(0x99))], + &[foreign_key(0x7a), key], + ); + assert!( + fetched, + "spent prefix of an owned output must fetch the block" + ); + + let rec = record(&scanner, outpoint); + assert_eq!(rec.spent_by, Some(spend.compute_txid())); + assert_eq!(rec.spent_height, Some(SPEND_HEIGHT)); + assert_eq!(balance(&scanner), 0, "spent coin must leave the balance"); +} + +#[test] +fn foreign_output_with_colliding_prefix_marks_nothing() { + let mut scanner = scanner("collision", 0); + let (outpoint, key) = receive(&mut scanner); + + // Someone else's output shares our key's 8-byte prefix and is spent: the + // oracle serves exactly our prefix, but the spending tx consumes a + // different outpoint. + let foreign_outpoint = OutPoint::new(Txid::from_byte_array([0x88; 32]), 0); + let foreign_spend = spend_of(foreign_outpoint); + let fetched = process_block( + &mut scanner, + &block(SPEND_HEIGHT, vec![foreign_spend]), + SPEND_HEIGHT, + vec![], + &[key], + ); + assert!(fetched, "a prefix hit must still fetch the block to check"); + + let rec = record(&scanner, outpoint); + assert!( + !rec.is_spent(), + "prefix collision alone must not mark: {rec:?}" + ); + assert_eq!(scanner.owned_outputs().count(), 1); + assert_eq!(balance(&scanner), 50_000); +} + +#[test] +fn labelled_receive_records_its_label() { + let mut scanner = scanner("labels", 5); + let tweak = served_tweak(0x34); + let unlabelled = sp_output_key(&tweak, None); + let change = sp_output_key(&tweak, Some(0)); + let labelled = sp_output_key(&tweak, Some(3)); + // Three payments to one wallet in one tx: k = 0 is taken by whichever + // candidate matches first, so use one tx per key to keep k = 0 each. + let txs: Vec = [(unlabelled, 0x01u8), (change, 0x02), (labelled, 0x03)] + .iter() + .map(|&(key, salt)| { + tx( + vec![OutPoint::new(Txid::from_byte_array([salt; 32]), 0)], + vec![(key, 10_000 * salt as u64)], + ) + }) + .collect(); + let items = txs.iter().map(|t| item(t, tweak)).collect(); + assert!(process_block( + &mut scanner, + &block(RECEIVE_HEIGHT, txs.clone()), + RECEIVE_HEIGHT, + items, + &[], + )); + + let labels: Vec<(XOnlyPublicKey, Option)> = txs + .iter() + .map(|t| { + let r = record(&scanner, OutPoint::new(t.compute_txid(), 0)); + (r.pubkey, r.label) + }) + .collect(); + assert_eq!( + labels, + vec![(unlabelled, None), (change, Some(0)), (labelled, Some(3))] + ); +} + +#[cfg(feature = "serde")] +#[test] +fn owned_outputs_labels_and_spends_survive_restart() { + let tag = "restart"; + let path = state_file(tag); + let _ = std::fs::remove_file(&path); + + let mut scanner = scanner(tag, 2); + let tweak = served_tweak(0x35); + let change_key = sp_output_key(&tweak, Some(0)); + let label_key = sp_output_key(&tweak, Some(2)); + let recv_change = tx( + vec![OutPoint::new(Txid::from_byte_array([0x0a; 32]), 0)], + vec![(change_key, 20_000)], + ); + let recv_label = tx( + vec![OutPoint::new(Txid::from_byte_array([0x0b; 32]), 0)], + vec![(label_key, 30_000)], + ); + assert!(process_block( + &mut scanner, + &block( + RECEIVE_HEIGHT, + vec![recv_change.clone(), recv_label.clone()] + ), + RECEIVE_HEIGHT, + vec![item(&recv_change, tweak), item(&recv_label, tweak)], + &[], + )); + let change_op = OutPoint::new(recv_change.compute_txid(), 0); + let label_op = OutPoint::new(recv_label.compute_txid(), 0); + + // Spend the change output before the restart. + let spend = spend_of(change_op); + assert!(process_block( + &mut scanner, + &block(SPEND_HEIGHT, vec![spend.clone()]), + SPEND_HEIGHT, + vec![], + &[change_key], + )); + assert_eq!(balance(&scanner), 30_000); + let before: Vec<_> = scanner.owned_outputs().cloned().collect(); + assert_eq!(before.len(), 2); + + scanner.save_to_file(&path).expect("save state"); + let json: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + let persisted = json["owned_outputs"] + .as_array() + .expect("owned_outputs array"); + assert_eq!(persisted.len(), 2, "state file lists every owned output"); + let label_entry = persisted + .iter() + .find(|e| e["outpoint"] == label_op.to_string()) + .expect("labelled output persisted"); + assert_eq!(label_entry["label"], 2); + assert_eq!(label_entry["pubkey"], label_key.to_string()); + + let changeset = Scanner::load_from_file(&path).expect("load state"); + let socket: SocketAddr = "127.0.0.1:8333".parse().unwrap(); + let mut restored = Scanner::from_changeset( + oracle_client(), + socket, + changeset, + path.clone(), + Network::Regtest, + ) + .expect("restore"); + let _ = std::fs::remove_file(&path); + + let after: Vec<_> = restored.owned_outputs().cloned().collect(); + assert_eq!(after, before, "records survive the restart unchanged"); + assert_eq!(balance(&restored), 30_000, "restored balance"); + + // Spend detection keeps working after the restart. + let spend2 = spend_of(label_op); + assert!(process_block( + &mut restored, + &block(300, vec![spend2.clone()]), + 300, + vec![], + &[label_key], + )); + assert_eq!( + record(&restored, label_op).spent_by, + Some(spend2.compute_txid()) + ); + assert_eq!(balance(&restored), 0); +} + +/// A state file written by the code before owned outputs were recorded +/// (integration 93f6a92): a payment and a change output received at 100, the +/// wallet scanned on to 300, and the payment spent at 200 in a block that +/// code never fetched, so the spend is not in the file. +#[cfg(feature = "serde")] +const STATE_BEFORE_OWNED_OUTPUTS: &str = include_str!("testdata/state_before_owned_outputs.json"); + +#[cfg(feature = "serde")] +fn restore(tag: &str, json: &str) -> (Scanner, PathBuf) { + let path = state_file(tag); + std::fs::write(&path, json).expect("write state file"); + let changeset = Scanner::load_from_file(&path).expect("load state"); + let socket: SocketAddr = "127.0.0.1:8333".parse().unwrap(); + let scanner = Scanner::from_changeset( + oracle_client(), + socket, + changeset, + path.clone(), + Network::Regtest, + ) + .expect("restore"); + (scanner, path) +} + +#[cfg(feature = "serde")] +fn block_on(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime") + .block_on(future) +} + +/// An old state file still loads, and because its spends of owned outputs +/// may be missing, the scan rewinds to just below its oldest unspent output. +/// Rescanning from there finds the missed spend; the upgraded file then +/// loads without another rewind. +#[cfg(feature = "serde")] +#[test] +fn state_file_from_before_owned_outputs_rescans_for_missed_spends() { + let old: serde_json::Value = serde_json::from_str(STATE_BEFORE_OWNED_OUTPUTS).unwrap(); + assert!(old.get("format_version").is_none()); + assert_eq!(old["owned_outputs"], serde_json::json!([])); + assert_eq!(old["last_scanned_block_height"], 300); + let tweak = served_tweak(0x33); + let pay_key = sp_output_key(&tweak, None); + // The old field held bare hex keys; no released writer filled it, but such + // entries must not break the load either. + let mut with_legacy_keys = old.clone(); + with_legacy_keys["owned_outputs"] = serde_json::json!([pay_key.to_string()]); + + for (case, json) in [ + ("as written", STATE_BEFORE_OWNED_OUTPUTS.to_string()), + ("legacy keys", with_legacy_keys.to_string()), + ] { + let (mut scanner, path) = restore("pre-records", &json); + + // Both outputs are back, the change output with its label. + let records: Vec<_> = scanner.owned_outputs().cloned().collect(); + assert_eq!(records.len(), 2, "{case}"); + let pay = records + .iter() + .find(|r| r.pubkey == pay_key) + .expect("payment record") + .clone(); + let change = records.iter().find(|r| r.pubkey != pay_key).unwrap(); + assert_eq!((pay.amount_sat, pay.label, pay.height), (50_000, None, 100)); + assert_eq!( + (change.amount_sat, change.label, change.height), + (20_000, Some(0), 100) + ); + assert_eq!(balance(&scanner), 70_000, "{case}: the spend is missing"); + + // Rewound to just below the oldest unspent output, and says so. + assert_eq!(scanner.get_last_scanned_block_height(), 99, "{case}"); + let health = block_on(scanner.scan_health()); + assert_eq!( + health.state_rescan, + Some(crate::scanner::StateRescan { + from_height: 100, + until_height: 300, + }), + "{case}" + ); + // Electrum clients keep seeing the old tip while the rescan runs. + block_on(scanner.rebuild_electrum_index_from_graph(1)); + let tip = block_on(scanner.electrum_index().lock()).tip.clone(); + assert_eq!(tip.map(|(height, _)| height), Some(300), "{case}"); + + // The rescan reaches block 200; with the output recorded, the oracle's + // spent prefix now gets that block fetched and the spend found. + let spend = spend_of(pay.outpoint); + assert!(process_block( + &mut scanner, + &block(SPEND_HEIGHT, vec![spend.clone()]), + SPEND_HEIGHT, + vec![], + &[pay_key], + )); + assert_eq!( + record(&scanner, pay.outpoint).spent_by, + Some(spend.compute_txid()), + "{case}" + ); + assert_eq!(balance(&scanner), 20_000, "{case}"); + scanner.update_last_scanned_block_height(300); + block_on(scanner.clear_stall()); + assert_eq!(block_on(scanner.scan_health()).state_rescan, None); + + // Saved in the current format, it restores without another rewind. + scanner.save_to_file(&path).expect("save"); + let saved: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!( + saved["format_version"], + crate::scanner::STATE_FORMAT_VERSION + ); + let (again, path) = restore("pre-records", &serde_json::to_string(&saved).unwrap()); + let _ = std::fs::remove_file(&path); + assert_eq!(again.get_last_scanned_block_height(), 300, "{case}"); + assert_eq!(block_on(again.scan_health()).state_rescan, None, "{case}"); + assert_eq!(balance(&again), 20_000, "{case}"); + } +} + +/// Without an unspent output there is nothing a missed spend could affect: +/// the old file is upgraded in place, without a rescan. +#[cfg(feature = "serde")] +#[test] +fn state_file_from_before_owned_outputs_without_unspent_outputs_is_not_rescanned() { + let mut old: serde_json::Value = serde_json::from_str(STATE_BEFORE_OWNED_OUTPUTS).unwrap(); + // The same wallet before anything was found. + let fresh = { + let scanner = scanner("pre-records-empty-src", 1); + let mut json = serde_json::to_value(&scanner.stage).unwrap(); + json.as_object_mut().unwrap().remove("format_version"); + json + }; + old["indexer"] = fresh["indexer"].clone(); + old["block_checkpoints"] = fresh["block_checkpoints"].clone(); + + let (scanner, path) = restore("pre-records-empty", &old.to_string()); + assert_eq!(scanner.owned_outputs().count(), 0); + assert_eq!(scanner.get_last_scanned_block_height(), 300); + assert_eq!(block_on(scanner.scan_health()).state_rescan, None); + scanner.save_to_file(&path).expect("save"); + let saved: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + let _ = std::fs::remove_file(&path); + assert_eq!( + saved["format_version"], + crate::scanner::STATE_FORMAT_VERSION + ); +} diff --git a/blindbit-lib/src/scanner/reorg.rs b/blindbit-lib/src/scanner/reorg.rs new file mode 100644 index 0000000..08c34e7 --- /dev/null +++ b/blindbit-lib/src/scanner/reorg.rs @@ -0,0 +1,363 @@ +//! Chain reorganisation handling. +//! +//! The scanner remembers the block hash of every height it scanned within +//! the last [`REORG_LOOKBACK`] heights (`Scanner::scanned_block_hashes`, +//! persisted as the state file's `scanned_block_hashes`). Every block the +//! oracle streams for a height the scanner already knows is compared with +//! that hash: +//! +//! - Before a range scan continues the scanned chain, the last scanned block +//! is checked with one `GetBlockHashByHeight` lookup (a few dozen bytes) +//! instead of being streamed again. `watch_chain` runs the same check when +//! the oracle has no new block, which catches a same-height tip +//! replacement without downloading anything else. +//! - When that block disagrees, the fork is somewhere below it. A binary +//! search over the remembered heights, again with lookups only, finds the +//! highest one the oracle still agrees with ([`Scanner::locate_fork`]); +//! the stream that rolls back and rescans starts there. If the oracle +//! disagrees even at the oldest remembered height, that is +//! [`ReorgTooDeep`] without streaming anything. +//! - A stream reads the oracle's index height by height, so an oracle that +//! switches branches while a stream runs can hand over blocks of both. When +//! a stream ends, its newest blocks ([`POST_STREAM_CHECK`]) and the block +//! below them are checked again; a disagreement is handled like any other +//! reorganisation. Reorganisations happen at the tip, which is where a +//! stream ends, so that window covers them. +//! - Inside a stream, every block whose height has a remembered hash is still +//! compared with it. The first height whose hash disagrees is the first +//! disconnected block, the one below it is the fork point. +//! - Everything learned above the fork point is rolled back +//! ([`Scanner::rollback_to`]): transactions and anchors from disconnected +//! blocks, the owned-output records they created, spent marks they set, +//! checkpoints, remembered hashes, and the Electrum history. The state is +//! persisted, and the same stream then scans the new branch as new blocks, +//! so a transaction that re-confirms is found again at its new height and +//! one that did not simply stays gone. +//! - A fork at or below the lowest remembered height cannot be located. The +//! scan stops with [`ReorgTooDeep`] and changes nothing; the wallet must be +//! rescanned from its birthday with a fresh state file. It keeps failing +//! loudly on every attempt rather than continuing on a chain it cannot +//! verify. + +use std::collections::BTreeSet; + +use bitcoin::hashes::Hash; +use bitcoin::{BlockHash, Txid}; +use indexer::bdk_chain::ConfirmationBlockTime; +use indexer::v2::SpIndexerV2; + +use super::ScannerError; +use super::health::OracleProbe; +use super::scanner::Scanner; + +/// How many of the most recent scanned heights keep their block hash, and so +/// the deepest reorganisation the scanner can roll back by itself. +/// +/// 144 blocks is one day of mainnet blocks, several times deeper than any +/// mainnet reorganisation since 2013; storing it costs a few kilobytes of +/// state. +pub const REORG_LOOKBACK: u32 = 144; + +/// How many of the newest blocks a stream delivered are checked against the +/// oracle again once the stream has ended (see the module docs). +pub(crate) const POST_STREAM_CHECK: u64 = 6; + +/// A chain reorganisation reaches at or below the oldest block hash the +/// scanner still remembers, so the fork point cannot be found. +/// +/// Nothing was rolled back or scanned. The wallet state may hold outputs and +/// spends from blocks that are no longer in the chain; it has to be rebuilt +/// by rescanning from the wallet birthday with a fresh state file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReorgTooDeep { + /// Lowest height with a remembered hash; the oracle serves a different + /// block there. + pub lowest_known_height: u64, + /// Last height the scanner had scanned. + pub last_scanned_height: u64, +} + +impl std::fmt::Display for ReorgTooDeep { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "chain reorganisation deeper than the scanner can roll back: the oracle serves a \ + different block at height {} (the oldest block hash remembered, lookback {} blocks, \ + last scanned height {}); the wallet state may contain outputs from blocks that are no \ + longer in the chain; nothing was changed; move the state file away and rescan from \ + the wallet birthday", + self.lowest_known_height, REORG_LOOKBACK, self.last_scanned_height + ) + } +} + +impl std::error::Error for ReorgTooDeep {} + +/// The first block of a stream disagrees with the remembered hash, so the +/// fork lies below the stream's start; the scan reopens the stream lower. +#[derive(Debug)] +pub(crate) struct ReorgBelowStreamStart { + pub height: u64, +} + +impl std::fmt::Display for ReorgBelowStreamStart { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "chain reorganisation detected at height {}; the fork point lies below it", + self.height + ) + } +} + +impl std::error::Error for ReorgBelowStreamStart {} + +/// The oracle serves block hashes in display order. +pub(crate) fn block_hash_from_oracle(display_order: &[u8]) -> Option { + let mut bytes: [u8; 32] = display_order.try_into().ok()?; + bytes.reverse(); + Some(BlockHash::from_byte_array(bytes)) +} + +/// A remembered block compared with the block the oracle serves now at the +/// same height. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OracleView { + /// The oracle serves the remembered block. + Same, + /// The oracle serves a different block: the chain was reorganised. + Different, + /// Nothing is remembered at that height, or the oracle serves no block + /// there (an empty hash or `NOT_FOUND`). It cannot be checked yet, which + /// is not a reorganisation. + Unknown, +} + +/// What the scanner does with one oracle block, judged by its hash. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum BlockCheck { + /// No remembered hash, or the remembered hash matches. + Consistent, + /// Remembered hash differs: the chain was reorganised at this height. + Reorganised, +} + +impl Scanner { + /// Lowest height with a remembered block hash. + pub(crate) fn lowest_known_block_height(&self) -> Option { + self.scanned_block_hashes + .keys() + .next() + .map(|height| u64::from(*height)) + } + + pub(crate) fn check_block_hash(&self, height: u64, hash: &BlockHash) -> BlockCheck { + match u32::try_from(height) + .ok() + .and_then(|height| self.scanned_block_hashes.get(&height)) + { + Some(known) if known != hash => BlockCheck::Reorganised, + _ => BlockCheck::Consistent, + } + } + + /// Remember the hash of a block that was just scanned, keeping only the + /// lookback window below the highest remembered height. + pub(crate) fn record_scanned_block_hash(&mut self, height: u64, hash: BlockHash) { + let Ok(height) = u32::try_from(height) else { + return; + }; + self.scanned_block_hashes.insert(height, hash); + let highest = *self + .scanned_block_hashes + .keys() + .next_back() + .expect("just inserted"); + let keep_from = highest.saturating_sub(REORG_LOOKBACK - 1); + while self + .scanned_block_hashes + .first_key_value() + .is_some_and(|(height, _)| *height < keep_from) + { + self.scanned_block_hashes.pop_first(); + } + // Not copied into `stage`: `save_to_file` writes the current window. + } + + /// Compare the remembered block at `height` with the oracle's, with one + /// `GetBlockHashByHeight` lookup (none when nothing is remembered there). + pub(crate) async fn oracle_view( + &self, + oracle: &mut P, + height: u64, + ) -> Result { + let Some(known) = u32::try_from(height) + .ok() + .and_then(|height| self.scanned_block_hashes.get(&height)) + .copied() + else { + return Ok(OracleView::Unknown); + }; + Ok(match oracle.block_hash_at(height).await? { + Some(hash) if hash == known => OracleView::Same, + Some(_) => OracleView::Different, + None => OracleView::Unknown, + }) + } + + /// The oracle disagrees with the remembered block at `disagreeing`. Find + /// the highest remembered height below it that the oracle still agrees + /// with, the fork point, with a binary search over lookups. A height the + /// oracle cannot answer for counts as disagreeing, which can only move + /// the result lower. + /// + /// When the oracle disagrees at the oldest remembered height the fork + /// cannot be located: [`ReorgTooDeep`], found without streaming anything, + /// so a halted wallet costs two lookups per poll. + pub(crate) async fn locate_fork( + &self, + oracle: &mut P, + disagreeing: u64, + ) -> Result { + let lowest = self + .lowest_known_block_height() + .filter(|lowest| *lowest <= disagreeing) + .ok_or_else(|| format!("no remembered block hash at or below height {disagreeing}"))?; + match self.oracle_view(oracle, lowest).await? { + OracleView::Same => {} + OracleView::Different => { + return Err(Box::new(ReorgTooDeep { + lowest_known_height: lowest, + last_scanned_height: self.last_scanned_block_height, + })); + } + OracleView::Unknown => { + return Err(format!( + "chain reorganisation detected at height {disagreeing}, but the oracle serves \ + no block at height {lowest}, the oldest one remembered; retrying" + ) + .into()); + } + } + let (mut agrees, mut differs) = (lowest, disagreeing); + while differs - agrees > 1 { + let mid = agrees + (differs - agrees) / 2; + if self.oracle_view(oracle, mid).await? == OracleView::Same { + agrees = mid; + } else { + differs = mid; + } + } + Ok(agrees) + } + + /// Undo everything learned from blocks above `fork_height`, which were + /// disconnected by a chain reorganisation, and persist the result. + /// + /// Transactions confirmed only in disconnected blocks leave the wallet + /// graph (with their tweaks), so their outputs leave the balance and the + /// owned-output records, and outputs they spent are unspent again. The + /// indexer is rebuilt from the remaining changeset, the same way a + /// restart restores it. Whatever the new branch confirms is found by + /// scanning it afterwards. + pub(crate) async fn rollback_to(&mut self, fork_height: u64) -> Result<(), ScannerError> { + let fork = u32::try_from(fork_height) + .map_err(|_| format!("fork height {fork_height} is out of range"))?; + let previous_height = self.last_scanned_block_height; + + // Wallet graph and tweaks. + let removed = drop_blocks_above(&mut self.stage.indexer, fork); + let scan_sk = *self.internal_indexer.scan_sk(); + let spend_pk = *self.internal_indexer.spend_pk(); + self.stage.indexer.scan_sk = Some(scan_sk); + self.stage.indexer.spend_pk = Some(spend_pk); + let mut indexer = SpIndexerV2::::new(scan_sk, spend_pk); + // Labels first: applying the changeset re-runs the BIP-352 scan of + // every stored transaction, which needs them (as on restore). + _ = indexer.add_label(0); + for m in 1..=self.max_label_num { + _ = indexer.add_label(m); + } + indexer.apply_changeset(self.stage.indexer.clone()); + self.internal_indexer = indexer; + + // Owned-output records and spent marks are derived from the graph; + // rebuild them so no record or spend from a disconnected block stays. + self.owned_outputs.clear(); + self.stage.owned_outputs.clear(); + self.sync_owned_outputs(); + + // Chain position. + self.block_checkpoints.split_off(&(fork + 1)); + self.stage.block_checkpoints.split_off(&(fork + 1)); + self.scanned_block_hashes.split_off(&(fork + 1)); + self.last_scanned_block_height = self.last_scanned_block_height.min(fork_height); + self.stage.last_scanned_block_height = self.last_scanned_block_height; + self.last_scanned_block_height_rescan = + self.last_scanned_block_height_rescan.min(fork_height); + self.stage.last_scanned_block_height_rescan = self.last_scanned_block_height_rescan; + + // Electrum view: history, SP history, raw txs and headers of the + // disconnected blocks. Unconfirmed (height 0) entries stay. + { + let mut idx = self.electrum_index.lock().await; + for history in idx.scripthash_history.values_mut() { + history.retain(|entry| entry.height <= fork); + } + idx.sp_history.retain(|entry| entry.height <= fork); + idx.headers.retain(|height, _| *height <= fork); + for txid in &removed { + idx.txs.remove(&txid.to_string()); + } + let header = idx.headers.get(&fork).cloned().unwrap_or_default(); + idx.tip = Some((fork, header)); + } + + tracing::warn!( + fork_height, + previous_height, + disconnected_blocks = previous_height.saturating_sub(fork_height), + dropped_transactions = removed.len(), + owned_outputs = self.owned_outputs.len(), + "chain reorganisation: rolled wallet state back to the fork point; rescanning the new branch" + ); + + #[cfg(feature = "serde")] + self.save_to_file(&self.state_file)?; + + let _ = self.notify_reorg.send(fork); + let _ = self + .notify_found_utxos + .send(self.internal_indexer.index().by_shared_secret.len()); + Ok(()) + } +} + +/// Remove from an indexer changeset every anchor above `fork` and every +/// transaction that was confirmed only in those blocks, with its tweak and +/// timestamps. Returns the removed txids. Transactions that never had an +/// anchor (unconfirmed) are kept. +fn drop_blocks_above( + changeset: &mut indexer::v2::ChangeSet, + fork: u32, +) -> BTreeSet { + let graph = &mut changeset.graph; + let anchored_before: BTreeSet = graph.anchors.iter().map(|(_, txid)| *txid).collect(); + graph + .anchors + .retain(|(anchor, _)| anchor.block_id.height <= fork); + let anchored_after: BTreeSet = graph.anchors.iter().map(|(_, txid)| *txid).collect(); + let removed: BTreeSet = anchored_before + .difference(&anchored_after) + .copied() + .collect(); + + graph.txs.retain(|tx| !removed.contains(&tx.compute_txid())); + graph.last_seen.retain(|txid, _| !removed.contains(txid)); + graph.first_seen.retain(|txid, _| !removed.contains(txid)); + graph.last_evicted.retain(|txid, _| !removed.contains(txid)); + changeset + .txid_to_partial_secret + .retain(|txid, _| !removed.contains(txid)); + removed +} diff --git a/blindbit-lib/src/scanner/reorg_tests.rs b/blindbit-lib/src/scanner/reorg_tests.rs new file mode 100644 index 0000000..136fe80 --- /dev/null +++ b/blindbit-lib/src/scanner/reorg_tests.rs @@ -0,0 +1,967 @@ +//! Chain reorganisations through the live scan path. +//! +//! The scenario mirrors the testkit's T2 `reorg` scenario: a wallet scans a +//! chain, then the oracle switches to a branch that forks at height 104. +//! Blocks above the fork are disconnected; one payment re-confirms on the new +//! branch at a different height, one is evicted. Each test drives a real +//! [`Scanner`] through `scan_range_from`, the body of `scan_block_range`, +//! against an in-process oracle; only the P2P block fetch is replaced (full +//! blocks are served by `stream_safety_tests::serve`). + +use std::collections::{BTreeMap, VecDeque}; +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use bdk_sp::bitcoin::key::Secp256k1; +use bdk_sp::receive::get_silentpayment_pubkey; +use bitcoin::absolute::LockTime; +use bitcoin::hashes::{Hash, sha256d}; +use bitcoin::key::TweakedPublicKey; +use bitcoin::secp256k1::{PublicKey, SecretKey}; +use bitcoin::transaction::Version; +use bitcoin::{ + Amount, Block, BlockHash, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid, + Witness, XOnlyPublicKey, +}; +use bitcoin_rev::Network; +use tonic::transport::Channel; + +use super::health::{OracleProbe, indexed_block_hash}; +use super::reorg::ReorgTooDeep; +use super::scanning::{BlockScanDataStream, BlockStreamSource}; +use super::stream_safety_tests::serve; +use super::{REORG_LOOKBACK, Scanner, ScannerError}; +use crate::oracle_grpc::{ + BlockHashResponse, BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem, +}; +use prost::Message as _; + +const FORK: u64 = 104; + +// --------------------------------------------------------------------------- +// In-process oracle +// --------------------------------------------------------------------------- + +type Message = BlockScanDataShortResponse; + +/// One chain as the oracle serves it: height -> block message. +#[derive(Clone, Default)] +struct Oracle { + chain: BTreeMap, + /// Ends a stream with an UNAVAILABLE status at this height, if the + /// stream began below it (so the first block of a stream is always + /// served). + fail_at: Option, + /// What the scanner asked of this oracle (shared by its clones). + traffic: Arc, +} + +/// Requests and response payload bytes (gRPC message plus its 5-byte frame +/// header), as the oracle would send them. +#[derive(Default)] +struct Traffic { + streams: AtomicUsize, + streamed_blocks: AtomicUsize, + streamed_bytes: AtomicUsize, + lookups: AtomicUsize, + lookup_bytes: AtomicUsize, +} + +impl Traffic { + fn take(&self) -> [usize; 5] { + [ + self.streams.swap(0, Ordering::SeqCst), + self.streamed_blocks.swap(0, Ordering::SeqCst), + self.streamed_bytes.swap(0, Ordering::SeqCst), + self.lookups.swap(0, Ordering::SeqCst), + self.lookup_bytes.swap(0, Ordering::SeqCst), + ] + } +} + +struct ChainStream(VecDeque>); + +impl BlockScanDataStream for ChainStream { + fn next_message( + &mut self, + ) -> impl Future, tonic::Status>> + Send { + std::future::ready(self.0.pop_front().transpose()) + } +} + +impl BlockStreamSource for Oracle { + type Stream = ChainStream; + + async fn open(&mut self, start: u64, end: u64) -> Result { + self.traffic.streams.fetch_add(1, Ordering::SeqCst); + let mut items = VecDeque::new(); + for height in start..=end { + if self.fail_at == Some(height) && start < height { + items.push_back(Err(tonic::Status::unavailable("connection reset"))); + break; + } + match self.chain.get(&height) { + Some(message) => { + self.traffic.streamed_blocks.fetch_add(1, Ordering::SeqCst); + self.traffic + .streamed_bytes + .fetch_add(message.encoded_len() + 5, Ordering::SeqCst); + items.push_back(Ok(message.clone())) + } + None => break, + } + } + Ok(ChainStream(items)) + } +} + +impl OracleProbe for Oracle { + async fn block_hash_at(&mut self, height: u64) -> Result, ScannerError> { + let block_hash = self + .chain + .get(&height) + .and_then(|m| m.block_identifier.as_ref()) + .map(|id| id.block_hash.clone()) + .unwrap_or_default(); + self.traffic.lookups.fetch_add(1, Ordering::SeqCst); + self.traffic.lookup_bytes.fetch_add( + BlockHashResponse { + block_hash: block_hash.clone(), + } + .encoded_len() + + 5, + Ordering::SeqCst, + ); + Ok(indexed_block_hash(&block_hash)) + } +} + +impl Oracle { + fn tip(&self) -> u64 { + *self.chain.keys().next_back().expect("non-empty chain") + } + + /// A copy of this chain up to and including `fork`, as the start of a + /// competing branch. + fn branch_at(&self, fork: u64) -> Oracle { + Oracle { + chain: self + .chain + .range(..=fork) + .map(|(h, m)| (*h, m.clone())) + .collect(), + fail_at: None, + traffic: Arc::default(), + } + } + + fn empty(&mut self, branch: u8, height: u64) { + let hash = sha256d::Hash::hash(&[&[branch][..], &height.to_le_bytes()].concat()); + self.chain.insert( + height, + message(BlockHash::from_raw_hash(hash), height, vec![], &[]), + ); + } + + /// A block holding `txs` (payments carry their served tweak), served in + /// place of P2P. + fn block( + &mut self, + branch: u8, + height: u64, + txs: Vec<(Transaction, Option)>, + spent: &[XOnlyPublicKey], + ) { + let items = txs + .iter() + .filter_map(|(tx, tweak)| tweak.map(|tweak| item(tx, tweak))) + .collect(); + let block = full_block(branch, height, txs.into_iter().map(|(tx, _)| tx).collect()); + serve(&block); + self.chain + .insert(height, message(block.block_hash(), height, items, spent)); + } +} + +fn message( + hash: BlockHash, + height: u64, + comp_index: Vec, + spent: &[XOnlyPublicKey], +) -> Message { + // The oracle serves hashes in display order. + let mut display = hash.to_byte_array(); + display.reverse(); + BlockScanDataShortResponse { + block_identifier: Some(BlockIdentifier { + block_hash: display.to_vec(), + block_height: height, + }), + comp_index, + spent_outputs: spent + .iter() + .flat_map(|key| key.serialize()[..8].to_vec()) + .collect(), + } +} + +fn item(tx: &Transaction, tweak: PublicKey) -> ComputeIndexTxItem { + let mut txid = tx.compute_txid().to_byte_array(); + txid.reverse(); + ComputeIndexTxItem { + txid: txid.to_vec(), + tweak: tweak.serialize().to_vec(), + outputs_short: tx + .output + .iter() + .flat_map(|o| o.script_pubkey.as_bytes()[2..10].to_vec()) + .collect(), + } +} + +fn full_block(branch: u8, height: u64, txs: Vec) -> Block { + let coinbase = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes([&[branch][..], &height.to_le_bytes()].concat()), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(312_500_000), + script_pubkey: ScriptBuf::from_bytes(vec![0x6a]), + }], + }; + let mut txdata = vec![coinbase]; + txdata.extend(txs); + let mut block = Block { + header: bitcoin::block::Header { + version: bitcoin::block::Version::TWO, + prev_blockhash: BlockHash::all_zeros(), + merkle_root: bitcoin::TxMerkleNode::all_zeros(), + time: 1_713_571_767 + height as u32, + bits: bitcoin::CompactTarget::from_consensus(0x207f_ffff), + nonce: u32::from(branch), + }, + txdata, + }; + block.header.merkle_root = block.compute_merkle_root().expect("non-empty block"); + block +} + +// --------------------------------------------------------------------------- +// Wallet +// --------------------------------------------------------------------------- + +fn secret(byte: u8) -> SecretKey { + SecretKey::from_slice(&[byte; 32]).expect("valid secret") +} + +fn keys() -> (SecretKey, PublicKey) { + (secret(0x11), secret(0x22).public_key(&Secp256k1::new())) +} + +fn state_file(tag: &str) -> PathBuf { + std::env::temp_dir().join(format!("blindbit-reorg-{tag}-{}.json", std::process::id())) +} + +struct TempState(PathBuf); + +impl Drop for TempState { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +fn client() -> crate::OracleServiceClient { + crate::OracleServiceClient::new(Channel::from_static("http://127.0.0.1:1").connect_lazy()) +} + +fn socket() -> SocketAddr { + "127.0.0.1:1".parse().expect("socket address") +} + +/// A fresh wallet. Must be called inside a Tokio runtime. +fn wallet(tag: &str) -> (Scanner, TempState) { + let (scan_sk, spend_pk) = keys(); + let path = state_file(tag); + let _ = std::fs::remove_file(&path); + let scanner = Scanner::new( + client(), + socket(), + scan_sk, + spend_pk, + 0, + path.clone(), + Network::Regtest, + ); + (scanner, TempState(path)) +} + +/// The wallet as a restarted daemon loads it from its state file. +#[cfg(feature = "serde")] +fn restart(path: &std::path::Path) -> Scanner { + let changeset = Scanner::load_from_file(path).expect("load state"); + Scanner::from_changeset( + client(), + socket(), + changeset, + path.to_path_buf(), + Network::Regtest, + ) + .expect("restore") +} + +/// A payment of `sat` to the wallet's unlabelled address; the tweak the +/// oracle serves for it and the paid outpoint. +fn payment(seed: u8, sat: u64) -> (Transaction, PublicKey, OutPoint, XOnlyPublicKey) { + let (scan_sk, spend_pk) = keys(); + let tweak = secret(seed).public_key(&Secp256k1::new()); + let shared = bdk_sp::compute_shared_secret(&scan_sk, &tweak); + let key = get_silentpayment_pubkey(&spend_pk, &shared, 0, None) + .x_only_public_key() + .0; + let tx = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::new(Txid::from_byte_array([seed; 32]), 0), + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(sat), + script_pubkey: p2tr(key), + }], + }; + let outpoint = OutPoint::new(tx.compute_txid(), 0); + (tx, tweak, outpoint, key) +} + +/// A spend of `outpoint` to a foreign key. +fn spend_of(outpoint: OutPoint) -> Transaction { + let foreign = secret(0x66) + .public_key(&Secp256k1::new()) + .x_only_public_key() + .0; + Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: outpoint, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(1_000), + script_pubkey: p2tr(foreign), + }], + } +} + +fn p2tr(key: XOnlyPublicKey) -> ScriptBuf { + ScriptBuf::new_p2tr_tweaked(TweakedPublicKey::dangerous_assume_tweaked(key)) +} + +fn run(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime") + .block_on(future) +} + +/// One `watch_chain` iteration against `oracle`. +async fn watch_step(scanner: &mut Scanner, oracle: &Oracle) -> Result<(), ScannerError> { + let tip = oracle.tip(); + let last = scanner.get_last_scanned_block_height(); + let (from, to) = if tip > last { + (last + 1, tip) + } else { + (tip + 1, tip) + }; + scanner.scan_range_from(from, to, oracle.clone()).await +} + +/// Sum of the unspent owned outputs: the wallet balance. +fn balance(scanner: &Scanner) -> u64 { + scanner + .owned_outputs() + .filter(|r| !r.is_spent()) + .map(|r| r.amount_sat) + .sum() +} + +fn owned_height(scanner: &Scanner, outpoint: OutPoint) -> Option { + scanner + .owned_outputs() + .find(|r| r.outpoint == outpoint) + .map(|r| r.height) +} + +/// Heights at which the wallet graph holds `txid` confirmed. +fn anchor_heights(scanner: &Scanner, txid: Txid) -> Vec { + scanner + .internal_indexer + .graph() + .get_tx_node(txid) + .map(|node| node.anchors.iter().map(|a| a.block_id.height).collect()) + .unwrap_or_default() +} + +async fn sp_history(scanner: &Scanner) -> Vec<(String, u32)> { + let idx = scanner.electrum_index.lock().await; + idx.sp_history + .iter() + .map(|e| (e.tx_hash.clone(), e.height)) + .collect() +} + +async fn in_scripthash_history(scanner: &Scanner, txid: Txid) -> bool { + let idx = scanner.electrum_index.lock().await; + idx.scripthash_history + .values() + .flatten() + .any(|e| e.tx_hash == txid.to_string()) +} + +// --------------------------------------------------------------------------- +// The T2 scenario: fork at 104, 2 blocks disconnected, 3 mined +// --------------------------------------------------------------------------- + +struct Scenario { + old: Oracle, + new: Oracle, + survivor: OutPoint, + reconfirmed: OutPoint, + evicted: OutPoint, +} + +/// Old chain: 101 pays `survivor`, 105 pays `reconfirmed`, 106 pays +/// `evicted`. New chain forks at 104: 105' is empty, 106' re-confirms +/// `reconfirmed`, 107' is empty; `evicted` never confirms again. +fn scenario() -> Scenario { + let (survivor_tx, survivor_tweak, survivor, _) = payment(0x31, 50_000); + let (keep_tx, keep_tweak, reconfirmed, _) = payment(0x32, 30_000); + let (evict_tx, evict_tweak, evicted, _) = payment(0x33, 20_000); + + let mut old = Oracle::default(); + old.block(0xa, 101, vec![(survivor_tx, Some(survivor_tweak))], &[]); + for h in 102..=FORK { + old.empty(0xa, h); + } + old.block(0xa, 105, vec![(keep_tx.clone(), Some(keep_tweak))], &[]); + old.block(0xa, 106, vec![(evict_tx, Some(evict_tweak))], &[]); + + let mut new = old.branch_at(FORK); + new.empty(0xb, 105); + new.block(0xb, 106, vec![(keep_tx, Some(keep_tweak))], &[]); + new.empty(0xb, 107); + + Scenario { + old, + new, + survivor, + reconfirmed, + evicted, + } +} + +async fn scan_old_chain(scanner: &mut Scanner, s: &Scenario) { + scanner + .scan_range_from(101, s.old.tip(), s.old.clone()) + .await + .expect("old chain scans"); + assert_eq!(balance(scanner), 100_000, "all three payments found"); +} + +#[test] +fn reorg_evicting_a_payment_drops_its_output_and_balance() { + run(async { + let (mut scanner, _state) = wallet("evict"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + + watch_step(&mut scanner, &s.new) + .await + .expect("new branch scans"); + + assert_eq!( + owned_height(&scanner, s.evicted), + None, + "evicted output is gone" + ); + assert!( + !scanner + .internal_indexer + .index() + .by_shared_secret + .contains_key(&s.evicted), + "evicted output left the index" + ); + assert!(anchor_heights(&scanner, s.evicted.txid).is_empty()); + assert_eq!(balance(&scanner), 80_000, "balance no longer counts it"); + assert!( + !sp_history(&scanner) + .await + .iter() + .any(|(txid, _)| *txid == s.evicted.txid.to_string()), + "evicted tx left the SP history" + ); + assert!(!in_scripthash_history(&scanner, s.evicted.txid).await); + assert_eq!(owned_height(&scanner, s.survivor), Some(101)); + assert_eq!(scanner.get_last_scanned_block_height(), 107); + }); +} + +#[test] +fn reorg_reconfirming_a_payment_keeps_it_at_its_new_height() { + run(async { + let (mut scanner, _state) = wallet("reconfirm"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + assert_eq!(owned_height(&scanner, s.reconfirmed), Some(105)); + + watch_step(&mut scanner, &s.new) + .await + .expect("new branch scans"); + + assert_eq!( + owned_height(&scanner, s.reconfirmed), + Some(106), + "found again, at its new height" + ); + assert_eq!(anchor_heights(&scanner, s.reconfirmed.txid), vec![106]); + let history = sp_history(&scanner).await; + assert!(history.contains(&(s.reconfirmed.txid.to_string(), 106))); + assert!(!history.contains(&(s.reconfirmed.txid.to_string(), 105))); + }); +} + +/// The fork replaces the tip without making the chain longer: the watch +/// loop's check with no new block still finds it. +#[test] +fn same_height_tip_replacement_is_detected_without_a_new_block() { + run(async { + let (mut scanner, _state) = wallet("same-height"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + + let mut replaced = s.old.branch_at(105); + replaced.empty(0xc, 106); + assert_eq!(replaced.tip(), scanner.get_last_scanned_block_height()); + watch_step(&mut scanner, &replaced) + .await + .expect("check succeeds"); + + assert_eq!(owned_height(&scanner, s.evicted), None); + assert_eq!(owned_height(&scanner, s.reconfirmed), Some(105)); + assert_eq!(balance(&scanner), 80_000); + assert_eq!(scanner.get_last_scanned_block_height(), 106); + }); +} + +#[test] +fn reorg_disconnecting_a_spend_makes_the_output_unspent_again() { + run(async { + let (mut scanner, _state) = wallet("unspend"); + let (recv_tx, tweak, outpoint, key) = payment(0x41, 50_000); + let mut old = Oracle::default(); + old.block(0xa, 101, vec![(recv_tx, Some(tweak))], &[]); + for h in 102..=FORK { + old.empty(0xa, h); + } + let spend = spend_of(outpoint); + old.block(0xa, 105, vec![(spend.clone(), None)], &[key]); + let mut new = old.branch_at(FORK); + new.empty(0xb, 105); + new.empty(0xb, 106); + + scanner + .scan_range_from(101, 105, old.clone()) + .await + .expect("old chain scans"); + let spent_by = scanner + .owned_outputs() + .find(|r| r.outpoint == outpoint) + .and_then(|r| r.spent_by); + assert_eq!(spent_by, Some(spend.compute_txid())); + assert_eq!(balance(&scanner), 0); + + watch_step(&mut scanner, &new) + .await + .expect("new branch scans"); + + let record = scanner + .owned_outputs() + .find(|r| r.outpoint == outpoint) + .cloned() + .expect("output still owned"); + assert_eq!((record.spent_by, record.spent_height), (None, None)); + assert_eq!(balance(&scanner), 50_000, "the output is spendable again"); + assert!(anchor_heights(&scanner, spend.compute_txid()).is_empty()); + assert!(!in_scripthash_history(&scanner, spend.compute_txid()).await); + // Its prefix is watched again, so a later spend is still detected. + assert!(scanner.owned_prefixes.contains_key(&record.short_pubkey())); + }); +} + +/// Hashes are persisted, so a daemon restarted after the reorg still finds +/// it; the rollback is persisted before the new branch is scanned, so a scan +/// that dies right after it leaves no phantom output on disk; and resuming +/// from that state ends up exactly where an uninterrupted scan does. +#[cfg(feature = "serde")] +#[test] +fn rollback_is_persisted_and_restart_mid_rescan_is_safe() { + run(async { + let (mut scanner, state) = wallet("restart"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + drop(scanner); + + let json: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(&state.0).unwrap()).unwrap(); + assert_eq!( + json["scanned_block_hashes"].as_object().map(|m| m.len()), + Some(6), + "every scanned height's hash is persisted" + ); + + // Restart after the reorg. The scan locates the fork, rolls back, + // scans 105' and then its stream dies before 106'. + let mut restarted = restart(&state.0); + let mut flaky = s.new.clone(); + flaky.fail_at = Some(106); + watch_step(&mut restarted, &flaky) + .await + .expect_err("the stream fails mid-rescan"); + drop(restarted); + + // The rollback to the fork point is on disk; 105' was scanned but + // not yet saved, so it is simply scanned again. + let mut resumed = restart(&state.0); + assert_eq!(resumed.get_last_scanned_block_height(), FORK); + assert_eq!( + owned_height(&resumed, s.evicted), + None, + "no phantom on disk" + ); + assert_eq!( + owned_height(&resumed, s.reconfirmed), + None, + "not re-found yet" + ); + assert_eq!(balance(&resumed), 50_000); + + watch_step(&mut resumed, &s.new).await.expect("resume"); + assert_eq!(owned_height(&resumed, s.reconfirmed), Some(106)); + assert_eq!(owned_height(&resumed, s.evicted), None); + assert_eq!(balance(&resumed), 80_000); + assert_eq!(resumed.get_last_scanned_block_height(), 107); + + let resumed_again = restart(&state.0); + assert_eq!(balance(&resumed_again), 80_000, "final state is persisted"); + }); +} + +/// A fork below every remembered hash cannot be located: the scan fails +/// loudly with `ReorgTooDeep`, on every attempt, and leaves the state (in +/// memory and on disk) exactly as it was. +#[test] +fn reorg_deeper_than_the_lookback_is_a_loud_error_and_changes_nothing() { + run(async { + let (mut scanner, state) = wallet("too-deep"); + let (recv_tx, tweak, outpoint, _) = payment(0x51, 50_000); + let tip = 101 + u64::from(REORG_LOOKBACK) + 10; + let mut old = Oracle::default(); + old.block(0xa, 101, vec![(recv_tx, Some(tweak))], &[]); + for h in 102..=tip { + old.empty(0xa, h); + } + let mut new = old.branch_at(101); + for h in 102..=tip + 1 { + new.empty(0xb, h); + } + + scanner + .scan_range_from(101, tip, old.clone()) + .await + .expect("old chain scans"); + assert_eq!( + scanner.scanned_block_hashes.len(), + REORG_LOOKBACK as usize, + "only the lookback window is remembered" + ); + let hashes = scanner.scanned_block_hashes.clone(); + let records: Vec<_> = scanner.owned_outputs().cloned().collect(); + let on_disk = std::fs::read(&state.0).ok(); + + for _ in 0..2 { + let err = watch_step(&mut scanner, &new) + .await + .expect_err("a reorg below the lookback must fail"); + let deep = err + .downcast_ref::() + .unwrap_or_else(|| panic!("expected ReorgTooDeep, got: {err}")); + assert_eq!( + deep.lowest_known_height, + tip - u64::from(REORG_LOOKBACK) + 1 + ); + assert!(err.to_string().contains("rescan"), "says what to do: {err}"); + } + + assert_eq!(scanner.get_last_scanned_block_height(), tip); + assert_eq!(scanner.scanned_block_hashes, hashes); + assert_eq!( + scanner.owned_outputs().cloned().collect::>(), + records + ); + assert_eq!(owned_height(&scanner, outpoint), Some(101)); + assert_eq!( + std::fs::read(&state.0).ok(), + on_disk, + "state file untouched" + ); + }); +} + +/// With no reorg, the watch loop's check changes nothing, and continuing +/// the chain still scans new blocks. +#[test] +fn unchanged_chain_passes_the_check() { + run(async { + let (mut scanner, _state) = wallet("unchanged"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + + watch_step(&mut scanner, &s.old).await.expect("check"); + assert_eq!(scanner.get_last_scanned_block_height(), 106); + assert_eq!(balance(&scanner), 100_000); + + let mut longer = s.old.clone(); + longer.empty(0xa, 107); + watch_step(&mut scanner, &longer).await.expect("extend"); + assert_eq!(scanner.get_last_scanned_block_height(), 107); + assert_eq!(balance(&scanner), 100_000); + }); +} + +// --------------------------------------------------------------------------- +// What a poll costs, and an oracle that switches branches mid-stream +// --------------------------------------------------------------------------- + +/// An oracle that switches from `before` to `after` once a stream has +/// delivered `switch_after` blocks; lookups answer from the current branch. +#[derive(Clone)] +struct SwitchingOracle { + before: Oracle, + after: Oracle, + switch_after: usize, + switched: Arc, +} + +impl BlockStreamSource for SwitchingOracle { + type Stream = ChainStream; + + async fn open(&mut self, start: u64, end: u64) -> Result { + let mut items = VecDeque::new(); + for height in start..=end { + let switched = self.switched.load(Ordering::SeqCst); + if !switched && items.len() == self.switch_after { + self.switched.store(true, Ordering::SeqCst); + } + let chain = if self.switched.load(Ordering::SeqCst) { + &self.after + } else { + &self.before + }; + match chain.chain.get(&height) { + Some(message) => items.push_back(Ok(message.clone())), + None => break, + } + } + Ok(ChainStream(items)) + } +} + +impl OracleProbe for SwitchingOracle { + async fn block_hash_at(&mut self, height: u64) -> Result, ScannerError> { + if self.switched.load(Ordering::SeqCst) { + self.after.block_hash_at(height).await + } else { + self.before.block_hash_at(height).await + } + } +} + +/// With no new block, a poll checks the scanned tip with one hash lookup and +/// downloads no block; a new block is streamed once, not together with the +/// block below it. +#[test] +fn idle_poll_is_one_hash_lookup_and_a_new_block_is_streamed_once() { + run(async { + let (mut scanner, _state) = wallet("poll-cost"); + let s = scenario(); + scan_old_chain(&mut scanner, &s).await; + let tip_block = s.old.chain[&s.old.tip()].encoded_len() + 5; + s.old.traffic.take(); + + for _ in 0..3 { + assert!(!scanner.watch_step(s.old.tip(), s.old.clone()).await); + let [streams, blocks, _, lookups, lookup_bytes] = s.old.traffic.take(); + assert_eq!((streams, blocks, lookups), (0, 0, 1), "idle poll"); + assert!( + lookup_bytes < tip_block, + "{lookup_bytes} B lookup vs the {tip_block} B block the overlap streamed" + ); + } + + let mut longer = s.old.clone(); + longer.empty(0xa, 107); + assert!(!scanner.watch_step(107, longer.clone()).await); + let [streams, blocks, _, lookups, _] = longer.traffic.take(); + assert_eq!((streams, blocks), (1, 1), "only the new block is streamed"); + // The block below it before, and both again after the stream. + assert_eq!(lookups, 3); + assert_eq!(scanner.get_last_scanned_block_height(), 107); + assert_eq!(scanner.scan_health().await.stall, None); + }); +} + +/// A one-block tip replacement is located with lookups; only the fork point +/// and the new block are streamed, not the whole lookback window. +#[test] +fn tip_reorg_streams_from_the_fork_point_only() { + run(async { + let (mut scanner, _state) = wallet("locate-cheap"); + let (recv_tx, tweak, outpoint, _) = payment(0x61, 40_000); + let tip = 101 + u64::from(REORG_LOOKBACK) + 10; + let mut old = Oracle::default(); + old.block(0xa, 101, vec![(recv_tx, Some(tweak))], &[]); + for h in 102..=tip { + old.empty(0xa, h); + } + scanner + .scan_range_from(101, tip, old.clone()) + .await + .expect("old chain scans"); + + let mut replaced = old.branch_at(tip - 1); + replaced.empty(0xb, tip); + assert!(!scanner.watch_step(tip, replaced.clone()).await); + let [streams, blocks, _, lookups, _] = replaced.traffic.take(); + assert_eq!(streams, 1); + assert_eq!(blocks, 2, "the fork point (checked) and the replaced tip"); + // 1 for the tip, 1 for the oldest remembered height, a binary search + // over the 144-height window, 3 after the stream. + assert!(lookups <= 14, "{lookups} lookups"); + assert_eq!(scanner.get_last_scanned_block_height(), tip); + assert_eq!(owned_height(&scanner, outpoint), Some(101)); + }); +} + +/// The oracle switches branches while a stream runs: it serves 105 from the +/// old branch (a payment) and 106 from the new one, which forks at 104. Each +/// block looks fine on its own, since nothing links 106' to 105. The check +/// after the stream finds the disagreement and rolls the payment back. +#[test] +fn oracle_switching_branches_mid_stream_is_rolled_back() { + run(async { + let (mut scanner, _state) = wallet("mid-stream"); + let s = scenario(); + let mut upto_fork = s.old.branch_at(FORK); + upto_fork.traffic = Arc::default(); + scanner + .scan_range_from(101, FORK, upto_fork) + .await + .expect("chain up to the fork scans"); + let mut new = s.new.clone(); + new.empty(0xb, 108); + let oracle = SwitchingOracle { + before: s.old.clone(), + after: new.clone(), + switch_after: 1, + switched: Arc::default(), + }; + + assert!(!scanner.watch_step(106, oracle).await); + + assert_eq!( + owned_height(&scanner, s.evicted), + None, + "old 106 was never served" + ); + assert_eq!( + owned_height(&scanner, s.reconfirmed), + Some(106), + "the old 105 payment is gone and found again at 106'" + ); + assert_eq!(anchor_heights(&scanner, s.reconfirmed.txid), vec![106]); + assert_eq!(balance(&scanner), 80_000); + assert_eq!(scanner.get_last_scanned_block_height(), 106); + let mut probe = new.clone(); + for height in 101..=106 { + assert_eq!( + scanner.oracle_view(&mut probe, height).await.unwrap(), + super::reorg::OracleView::Same, + "remembered hash at {height} is on the new branch" + ); + } + }); +} + +/// A reorganisation below the lookback halts the wallet loudly: a stall in +/// the status says what to do, every poll fails again, and each poll costs +/// two hash lookups rather than a stream of the lookback window. +#[test] +fn too_deep_reorg_is_a_stall_and_polls_stay_cheap() { + run(async { + let (mut scanner, _state) = wallet("too-deep-stall"); + let tip = 101 + u64::from(REORG_LOOKBACK) + 10; + let mut old = Oracle::default(); + for h in 101..=tip { + old.empty(0xa, h); + } + scanner + .scan_range_from(101, tip, old.clone()) + .await + .expect("old chain scans"); + let mut new = old.branch_at(101); + for h in 102..=tip { + new.empty(0xb, h); + } + + let mut first_since = None; + for _ in 0..3 { + assert!(!scanner.watch_step(tip, new.clone()).await); + let [streams, blocks, _, lookups, _] = new.traffic.take(); + assert_eq!((streams, blocks, lookups), (0, 0, 2)); + let stall = scanner + .scan_health() + .await + .stall + .expect("published as a stall"); + assert!( + stall + .reason + .contains("deeper than the scanner can roll back") + && stall.reason.contains("rescan"), + "{}", + stall.reason + ); + assert_eq!(stall.height, tip + 1); + assert_eq!( + *first_since.get_or_insert(stall.since_unix), + stall.since_unix + ); + } + assert_eq!(scanner.get_last_scanned_block_height(), tip); + }); +} diff --git a/blindbit-lib/src/scanner/scanner.rs b/blindbit-lib/src/scanner/scanner.rs index 8e5e65f..c4dd348 100644 --- a/blindbit-lib/src/scanner/scanner.rs +++ b/blindbit-lib/src/scanner/scanner.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::net::SocketAddr; #[cfg(feature = "serde")] use std::path::Path; @@ -7,7 +7,7 @@ use std::sync::Arc; use bitcoin::BlockHash; use bitcoin::Network as BTCNetwork; -use bitcoin::Txid; +use bitcoin::{OutPoint, Txid}; use bitcoin::hashes::Hash; use bitcoin::secp256k1::{PublicKey, SecretKey}; use bitcoin_rev::Network; @@ -20,8 +20,9 @@ use tonic::transport::Channel; use crate::oracle_grpc::oracle_service_client::OracleServiceClient; use indexer::bdk_chain::ConfirmationBlockTime; -use super::changeset::ChangeSet; +use super::changeset::{ChangeSet, STATE_FORMAT_VERSION}; use super::config::ScannerConfig; +use super::types::OwnedOutputRecord; use super::electrum_index::{ScriptHashEntry, SpHistoryEntry, WalletElectrumIndex, electrum_scripthash}; use super::ScannerError; @@ -59,9 +60,22 @@ pub struct Scanner { /// the last block height that was scanned on most recent rescan pub(crate) last_scanned_block_height_rescan: u64, - /// owned output pubkeys; used to check for spent outputs - // todo: should this be a hashmap instead of a vector? - pub(crate) owned_outputs: Vec<[u8; 32]>, + /// Wallet-owned outputs, keyed by outpoint. Maintained by + /// [`Scanner::sync_owned_outputs`] from the indexer after every applied + /// block and on restore; persisted as the state file's `owned_outputs`. + pub(crate) owned_outputs: BTreeMap, + + /// 8-byte x-only key prefix -> *unspent* owned outpoints with that prefix. + /// Lets the per-block spent-output check run in O(spent prefixes). + pub(crate) owned_prefixes: HashMap<[u8; 8], Vec>, + + /// Hash of each scanned height within the reorg lookback window; see + /// [`ChangeSet::scanned_block_hashes`] and `scanner/reorg.rs`. + pub(crate) scanned_block_hashes: BTreeMap, + + /// sends the fork height whenever a chain reorganisation rolled the + /// wallet state back + pub(crate) notify_reorg: broadcast::Sender, /// Staged changes that can be persisted pub(crate) stage: ChangeSet, @@ -113,6 +127,7 @@ impl Scanner { // Initialize the staged changeset with initial state let mut stage = ChangeSet { + format_version: STATE_FORMAT_VERSION, block_checkpoints: block_checkpoints.clone(), indexer: indexer.initial_changeset(), last_scanned_block_height: 0, @@ -121,6 +136,8 @@ impl Scanner { secret_scan_hex: Some(hex::encode(secret_scan.secret_bytes())), public_spend_hex: Some(hex::encode(public_spend.serialize())), max_label_num, + scanned_block_hashes: BTreeMap::new(), + oracle_floor_start: None, }; // Merge the initial label changes @@ -143,7 +160,10 @@ impl Scanner { notify_spent_outpoints: broadcast::channel(100).0, last_scanned_block_height: 0, last_scanned_block_height_rescan: 0, - owned_outputs: vec![], + owned_outputs: BTreeMap::new(), + owned_prefixes: HashMap::new(), + scanned_block_hashes: BTreeMap::new(), + notify_reorg: broadcast::channel(16).0, stage, state_file, network, @@ -371,7 +391,7 @@ impl Scanner { // Header hex is not recoverable without re-fetching; Sparrow's BlockHeaderTip // handles an empty hex gracefully. if self.last_scanned_block_height > 0 { - let h = self.last_scanned_block_height as u32; + let h = index.tip_height_for(self.last_scanned_block_height as u32); let hex = index.headers.get(&h).cloned().unwrap_or_default(); index.tip = Some((h, hex)); } @@ -389,17 +409,160 @@ impl Scanner { self.notify_spent_outpoints.subscribe() } + /// subscribe to chain reorganisations: each message is the fork height + /// the wallet state was rolled back to (everything above it was dropped + /// and is rescanned from the oracle's current chain) + pub fn subscribe_to_reorgs(&self) -> broadcast::Receiver { + self.notify_reorg.subscribe() + } + /// subscribe to notifications when a probabilistic match is found pub fn subscribe_to_probabilistic_matches(&self) -> broadcast::Receiver<[u8; 32]> { self.notify_probabilistic_matches.subscribe() } - /// add an owned output pubkey - /// if already exists nothing happens - pub fn add_owned_output(&mut self, pubkey: [u8; 32]) { - // todo: should this be a hashmap instead of a vector? - if !self.owned_outputs.contains(&pubkey) { - self.owned_outputs.push(pubkey); + /// All wallet-owned outputs found so far (spent and unspent), in outpoint + /// order. + pub fn owned_outputs(&self) -> impl Iterator { + self.owned_outputs.values() + } + + /// Bring [`Self::owned_outputs`] in line with the indexer. + /// + /// Every output the indexer has matched (`by_script`, full BIP-352 match) + /// gets a record with its full x-only key and label. A record is marked + /// spent only when a *confirmed* transaction in the wallet graph has an + /// input whose `previous_output` is exactly that outpoint; the oracle's + /// 8-byte spent prefixes only decide which blocks get fetched and never + /// mark anything themselves. New or changed records are staged. + pub(crate) fn sync_owned_outputs(&mut self) { + let index = self.internal_indexer.index(); + let graph = self.internal_indexer.graph(); + + let labels: HashMap> = index + .by_label + .iter() + .map(|(label, outpoint)| (*outpoint, *label)) + .collect(); + let anchor_height = |txid: Txid| { + graph + .get_tx_node(txid) + .and_then(|node| node.anchors.iter().map(|a| a.block_id.height).min()) + }; + + let mut changed: Vec = Vec::new(); + + for (xonly, outpoint) in index.by_xonly() { + if self.owned_outputs.contains_key(outpoint) { + continue; + } + let Some(txout) = graph.get_txout(*outpoint) else { + continue; + }; + let Some(height) = anchor_height(outpoint.txid) else { + continue; + }; + self.owned_outputs.insert( + *outpoint, + OwnedOutputRecord { + outpoint: *outpoint, + pubkey: xonly, + amount_sat: txout.value.to_sat(), + height, + label: labels.get(outpoint).copied().flatten(), + spent_by: None, + spent_height: None, + }, + ); + changed.push(*outpoint); + } + + for node in graph.full_txs() { + let Some(spent_height) = node.anchors.iter().map(|a| a.block_id.height).min() else { + continue; + }; + for input in &node.tx.input { + let Some(record) = self.owned_outputs.get_mut(&input.previous_output) else { + continue; + }; + if record.spent_by == Some(node.txid) && record.spent_height == Some(spent_height) { + continue; + } + record.spent_by = Some(node.txid); + record.spent_height = Some(spent_height); + changed.push(input.previous_output); + } + } + + if !changed.is_empty() { + self.stage.merge(ChangeSet { + owned_outputs: changed + .iter() + .filter_map(|outpoint| self.owned_outputs.get(outpoint).cloned()) + .collect(), + ..ChangeSet::default() + }); + } + + self.owned_prefixes.clear(); + for record in self.owned_outputs.values().filter(|r| !r.is_spent()) { + self.owned_prefixes + .entry(record.short_pubkey()) + .or_default() + .push(record.outpoint); + } + } + + /// Upgrade a restored state written before owned outputs were recorded + /// (format 0). + /// + /// That code fetched a block only when it held a probable *receive*, so a + /// block whose only wallet-relevant transaction spent one of our outputs + /// was never fetched, and the spend is missing from the restored graph. + /// Every output the graph still shows unspent may have been spent that + /// way, at or after its own height. The scan is therefore rewound to just + /// below the oldest such output, so the next scan revisits every block a + /// missed spend can be in; with the owned outputs now recorded, it fetches + /// those blocks. Re-applying blocks already in the graph is idempotent. + /// + /// The rewind is published in the scan health ([`StateRescan`]) and the + /// state is marked current, so it happens once: the next save persists + /// the lower height together with the new format. + /// + /// [`StateRescan`]: super::health::StateRescan + fn rescan_for_missed_spends(&mut self) { + self.stage.format_version = STATE_FORMAT_VERSION; + let previous_height = self.last_scanned_block_height; + let Some(oldest_unspent) = self + .owned_outputs + .values() + .filter(|record| !record.is_spent()) + .map(|record| u64::from(record.height)) + .min() + .filter(|height| *height <= previous_height) + else { + tracing::info!( + "state file predates owned-output records; no unspent output to re-check" + ); + return; + }; + + let resume_height = oldest_unspent - 1; + tracing::warn!( + from_height = oldest_unspent, + previous_height, + unspent_outputs = self.owned_outputs.values().filter(|r| !r.is_spent()).count(), + "state file predates owned-output records, so spends in blocks it never fetched \ + can be missing; rescanning from the oldest unspent output" + ); + self.update_last_scanned_block_height(resume_height); + let rescan = super::health::StateRescan { + from_height: oldest_unspent, + until_height: previous_height, + }; + // `from_changeset` holds the only reference to the new index. + if let Some(index) = Arc::get_mut(&mut self.electrum_index) { + index.get_mut().scan_health.state_rescan = Some(rescan); } } @@ -481,7 +644,9 @@ impl Scanner { } changeset.last_scanned_block_height = self.last_scanned_block_height; changeset.last_scanned_block_height_rescan = self.last_scanned_block_height_rescan; - changeset.owned_outputs = self.owned_outputs.clone(); + changeset.owned_outputs = self.owned_outputs.values().cloned().collect(); + changeset.scanned_block_hashes = self.scanned_block_hashes.clone(); + changeset.format_version = STATE_FORMAT_VERSION; let json = serde_json::to_string_pretty(&changeset)?; std::fs::write(path, json)?; @@ -556,16 +721,25 @@ impl Scanner { let _public_spend = PublicKey::from_slice(&public_spend_bytes)?; // Reconstruct the indexer from the changeset (this restores the graph and transaction data) - let mut indexer = SpIndexerV2::try_from(changeset.indexer.clone()) - .map_err(|e| format!("Failed to reconstruct indexer from changeset: {:?}", e))?; + let (Some(indexer_scan_sk), Some(indexer_spend_pk)) = + (changeset.indexer.scan_sk, changeset.indexer.spend_pk) + else { + return Err("Failed to reconstruct indexer from changeset: keys missing".into()); + }; + let mut indexer = SpIndexerV2::new(indexer_scan_sk, indexer_spend_pk); - // Regenerate labels + // Regenerate labels BEFORE applying the changeset: `apply_changeset` + // re-runs the BIP-352 scan (`index_tx`) over every stored transaction, + // and `load_from_file` blanks the persisted `label_lookup`. Labels added + // afterwards would leave every labelled output (change included, m = 0) + // out of the restored index. // ignore change set and be aligned with max_label_num let max_label_num = changeset.max_label_num; _ = indexer.add_label(0); for i in 1..=max_label_num { _ = indexer.add_label(i); } + indexer.apply_changeset(changeset.indexer.clone()); // Reconstruct block checkpoints from the changeset let mut block_checkpoints = changeset.block_checkpoints.clone(); @@ -582,8 +756,16 @@ impl Scanner { let sp_address = indexer .get_address(convert_network(network)) .to_string(); + let oracle_floor_start = changeset.oracle_floor_start; + let format_version = changeset.format_version; + + let owned_outputs = changeset + .owned_outputs + .iter() + .map(|record| (record.outpoint, record.clone())) + .collect(); - Ok(Self { + let mut scanner = Self { client, p2p_peer: p2p_socket_addr, internal_indexer: indexer, @@ -593,7 +775,10 @@ impl Scanner { notify_spent_outpoints: broadcast::channel(100).0, last_scanned_block_height: changeset.last_scanned_block_height, last_scanned_block_height_rescan: changeset.last_scanned_block_height_rescan, - owned_outputs: changeset.owned_outputs.clone(), + owned_outputs, + owned_prefixes: HashMap::new(), + scanned_block_hashes: changeset.scanned_block_hashes.clone(), + notify_reorg: broadcast::channel(16).0, stage: changeset, state_file: state_file, network: network, @@ -602,9 +787,17 @@ impl Scanner { let mut idx = WalletElectrumIndex::new(); idx.sp_address = sp_address; idx.sp_labels = (0..=max_label_num).collect(); + idx.scan_health.oracle_floor_start = oracle_floor_start; idx })), - }) + }; + // Records older state files never had, spends already in the graph, + // and the prefix lookup, all derived from the restored indexer. + scanner.sync_owned_outputs(); + if format_version < STATE_FORMAT_VERSION { + scanner.rescan_for_missed_spends(); + } + Ok(scanner) } } diff --git a/blindbit-lib/src/scanner/scanning.rs b/blindbit-lib/src/scanner/scanning.rs index 8ee15ce..2e484e9 100644 --- a/blindbit-lib/src/scanner/scanning.rs +++ b/blindbit-lib/src/scanner/scanning.rs @@ -10,11 +10,18 @@ use indexer::bdk_chain::{BlockId, CanonicalizationParams}; use tokio::time; use crate::oracle_grpc::{ - BlockScanDataShortResponse, ComputeIndexTxItem, FullTxItem, RangedBlockHeightRequestFiltered, + BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem, FullTxItem, + RangedBlockHeightRequestFiltered, }; use super::electrum_index::{ScriptHashEntry, SpHistoryEntry, electrum_scripthash}; +use super::health::ScanStopped; use super::p2p; +use super::health::OracleProbe; +use super::reorg::{ + BlockCheck, OracleView, POST_STREAM_CHECK, ReorgBelowStreamStart, ReorgTooDeep, + block_hash_from_oracle, +}; use super::scanner::Scanner; use super::types::{BlockIdentifierDisplay, ProbableMatch}; use super::utils::{byte_array_to_txid, construct_dummy_tx, match_short_pubkey}; @@ -48,28 +55,276 @@ fn upsert_history_entry(history: &mut Vec, entry: ScriptHashEnt } } -impl Scanner { - /// scan a block range for new utxos and spent outpoints - pub async fn scan_block_range( +/// The per-block message source of a block-range scan. +/// +/// Production scans read a tonic `Streaming`; tests feed messages and error +/// statuses in-process to exercise how the scanner reacts to what an oracle +/// can send mid-range. +pub(crate) trait BlockScanDataStream { + fn next_message( + &mut self, + ) -> impl Future, tonic::Status>> + Send; +} + +impl BlockScanDataStream for tonic::Streaming { + fn next_message( + &mut self, + ) -> impl Future, tonic::Status>> + Send + { + self.message() + } +} + +/// Opens `StreamBlockScanDataShort` streams for a range scan. A scan may +/// open a second stream lower down when it has to locate a fork point. +pub(crate) trait BlockStreamSource { + type Stream: BlockScanDataStream + Send; + + fn open( &mut self, start: u64, end: u64, - ) -> Result<(), ScannerError> { - // TODO: Implement sync check if needed + ) -> impl Future> + Send; +} + +/// The oracle's gRPC service as a [`BlockStreamSource`] and [`OracleProbe`]. +#[derive(Clone)] +struct OracleStreams(crate::oracle_grpc::oracle_service_client::OracleServiceClient); + +impl OracleProbe for OracleStreams { + async fn block_hash_at(&mut self, height: u64) -> Result, ScannerError> { + self.0.block_hash_at(height).await + } +} + +impl BlockStreamSource for OracleStreams { + type Stream = tonic::Streaming; + async fn open(&mut self, start: u64, end: u64) -> Result { let request = tonic::Request::new(RangedBlockHeightRequestFiltered { start, end, dustlimit: 0, cut_through: false, }); - let mut stream = self - .client + Ok(self + .0 .stream_block_scan_data_short(request) .await - .unwrap() - .into_inner(); + .map_err(|status| -> ScannerError { + Box::new(ScanStopped { + height: start, + reason: format!( + "oracle refused to stream blocks {start}..={end}: {:?}: {}", + status.code(), + status.message() + ), + not_indexed: status.code() == tonic::Code::NotFound, + }) + })? + .into_inner()) + } +} + +/// Read the next block message, turning an error status into a scanner error +/// that names the height the scan stopped at. +/// +/// There is no retry here: the scan stops at `height` and the caller rescans +/// from there (`watch_chain` on its next poll, `blindbit-cli` on its next run). +async fn next_block( + stream: &mut S, + height: u64, +) -> Result, ScannerError> { + stream.next_message().await.map_err(|status| { + Box::new(ScanStopped { + height, + reason: format!("oracle stream error {:?}: {}", status.code(), status.message()), + not_indexed: status.code() == tonic::Code::NotFound, + }) as ScannerError + }) +} + +/// A block message is only scannable if it is the next height in the range +/// and carries a real block hash. An oracle answers a height it has not +/// indexed with an empty hash and no data; treating that as an empty block +/// would silently skip any payment in it. +fn check_block_identifier( + block_identifier: &BlockIdentifier, + expected_height: u64, +) -> Result<(), ScannerError> { + if block_identifier.block_height != expected_height { + return Err(stream_stopped( + expected_height, + &format!( + "the oracle sent height {} instead", + block_identifier.block_height + ), + )); + } + let hash = &block_identifier.block_hash; + if hash.len() != 32 || hash.iter().all(|b| *b == 0) { + return Err(Box::new(ScanStopped { + height: expected_height, + reason: format!( + "the oracle sent no valid block hash ({} bytes); the height is probably not indexed", + hash.len() + ), + not_indexed: true, + })); + } + Ok(()) +} +fn stream_stopped(height: u64, reason: &str) -> ScannerError { + Box::new(ScanStopped { + height, + reason: reason.to_string(), + not_indexed: false, + }) +} + +impl Scanner { + /// Scan a block range for new utxos and spent outpoints, after checking + /// that the chain already scanned below it was not reorganised. + /// + /// When `start - 1` is a scanned height, the oracle's hash for it is + /// looked up and compared with the remembered one; a reorganisation is + /// rolled back to the fork point and the new branch scanned (see + /// `scanner/reorg.rs`). `start = end + 1` only runs that check, which + /// streams nothing unless the chain changed. A reorganisation deeper than + /// [`super::REORG_LOOKBACK`] is a [`ReorgTooDeep`] error. + pub async fn scan_block_range( + &mut self, + start: u64, + end: u64, + ) -> Result<(), ScannerError> { + let source = OracleStreams(self.client.clone()); + self.scan_range_from(start, end, source).await + } + + /// [`Self::scan_block_range`] over any oracle. + pub(crate) async fn scan_range_from( + &mut self, + start: u64, + end: u64, + mut source: Src, + ) -> Result<(), ScannerError> { + let mut start = start; + // The block below `start` is checked with a hash lookup instead of + // being streamed again. + let mut first = start; + if let Some(prev) = start.checked_sub(1).filter(|prev| *prev > 0) + && self.oracle_view(&mut source, prev).await? == OracleView::Different + { + tracing::warn!( + height = prev, + "chain reorganisation detected; locating the fork point" + ); + first = self.locate_fork(&mut source, prev).await?; + } + let mut restarts = 0; + loop { + if first > end { + return Ok(()); + } + let stream = source.open(first, end).await?; + match self.scan_block_stream_from(first, start, end, stream).await { + Err(e) => { + let Some(reorg) = e.downcast_ref::() else { + return Err(e); + }; + // The chain changed again between the lookups and the + // stream: walk up from the oldest remembered hash. + match self.lowest_known_block_height() { + Some(lowest) if lowest < reorg.height => { + tracing::warn!( + height = reorg.height, + from = lowest, + "chain reorganisation detected; locating the fork point" + ); + first = lowest; + } + _ => { + return Err(Box::new(ReorgTooDeep { + lowest_known_height: reorg.height, + last_scanned_height: self.last_scanned_block_height, + })); + } + } + } + Ok(()) => { + // A stream that ran while the oracle switched branches + // can hand over blocks of both: check its newest blocks, + // and the one below them, again. + let check_from = end + .saturating_sub(POST_STREAM_CHECK - 1) + .max(first.saturating_sub(1)) + .max(1); + let mut disagreeing = None; + for height in check_from..=end { + if self.oracle_view(&mut source, height).await? == OracleView::Different { + disagreeing = Some(height); + break; + } + } + let Some(height) = disagreeing else { + return Ok(()); + }; + restarts += 1; + if restarts > 3 { + return Err(format!( + "the oracle's chain kept changing while blocks up to {end} were \ + scanned; will retry" + ) + .into()); + } + tracing::warn!( + height, + "chain reorganisation while scanning; locating the fork point" + ); + first = self.locate_fork(&mut source, height).await?; + start = self.last_scanned_block_height + 1; + } + } + } + } + + /// Scan the per-block messages of one `StreamBlockScanDataShort` response + /// for the range `start..=end`. + /// + /// Every height in the range must arrive, in order, with a valid block + /// hash. Anything else — a message the oracle sends for a height it has + /// not indexed (empty block hash), a skipped or repeated height, an error + /// status, or a stream that ends before `end` — stops the scan with an + /// error. Such a block is never taken as "no payments": the scan height + /// stays at the last block that was fully processed, so the next scan + /// resumes at the block that could not be scanned. + #[cfg(test)] + pub(crate) async fn scan_block_stream( + &mut self, + start: u64, + end: u64, + stream: S, + ) -> Result<(), ScannerError> { + self.scan_block_stream_from(start, start, end, stream).await + } + + /// [`Self::scan_block_stream`] for a stream that begins at `first`, at or + /// below the scan's `start`. Blocks below `start` are only checked + /// against the remembered hashes, not scanned again. Any block whose + /// hash differs from the remembered one is a reorganisation: when a + /// block before it in this stream agreed, the one below it is the fork + /// point, the state is rolled back there and scanning goes on from that + /// block; when it is the stream's first block, the scan returns + /// [`ReorgBelowStreamStart`] (or [`ReorgTooDeep`] if nothing older is + /// remembered) without changing anything. + pub(crate) async fn scan_block_stream_from( + &mut self, + first: u64, + start: u64, + end: u64, + mut stream: S, + ) -> Result<(), ScannerError> { // Stamp sp_start_height into the index so the Electrum server's // SP subscription response always uses the correct scan start key. // Only set it on the first call; watch_chain increments start each @@ -92,7 +347,16 @@ impl Scanner { let mut p2p_conn: Option = None; - while let Some(block_scan_data) = stream.message().await.unwrap() { + let mut expected_height = first; + // Blocks from here on are scanned; below it they are only checked. + let mut scan_from = start; + // Whether an earlier block of this stream is known to be on the + // oracle's chain (checked or scanned). + let mut have_prior_block = false; + // Whether anything was scanned or rolled back (worth a save). + let mut changed = false; + + while let Some(block_scan_data) = next_block(&mut stream, expected_height).await? { let Some(block_identifier) = block_scan_data.block_identifier.clone() else { return Err(std::io::Error::new( std::io::ErrorKind::InvalidData, @@ -100,6 +364,34 @@ impl Scanner { ) .into()); }; + check_block_identifier(&block_identifier, expected_height)?; + expected_height += 1; + let oracle_block_hash = block_hash_from_oracle(&block_identifier.block_hash) + .expect("block hash length checked above"); + let height = block_identifier.block_height; + match self.check_block_hash(height, &oracle_block_hash) { + BlockCheck::Reorganised if !have_prior_block => { + if self.lowest_known_block_height() == Some(height) { + return Err(Box::new(ReorgTooDeep { + lowest_known_height: height, + last_scanned_height: self.last_scanned_block_height, + })); + } + return Err(Box::new(ReorgBelowStreamStart { height })); + } + BlockCheck::Reorganised => { + self.rollback_to(height - 1).await?; + scan_from = scan_from.min(height); + } + BlockCheck::Consistent if height < scan_from => { + // Already scanned, still on the chain. + have_prior_block = true; + continue; + } + BlockCheck::Consistent => {} + } + have_prior_block = true; + changed = true; let block_id = BlockIdentifierDisplay(&block_identifier); tracing::debug!(height = block_id.0.block_height, "received block data from oracle"); @@ -129,6 +421,7 @@ impl Scanner { .await; self.last_scanned_block_height = block_identifier.block_height; self.stage.last_scanned_block_height = block_identifier.block_height; + self.record_scanned_block_hash(height, oracle_block_hash); // Periodically checkpoint progress so a crash/restart during a // long initial catch-up scan doesn't lose everything. #[cfg(feature = "serde")] @@ -141,21 +434,6 @@ impl Scanner { } Some(probable_match) => probable_match, }; - // Guard: if block_hash is malformed (already warned in - // scan_short_block_data) we cannot pull the full block - // via P2P. Advance progress and move on. - if block_identifier.block_hash.len() != 32 { - self.notify_electrum_scan_progress( - block_identifier.block_height, - start, - end, - ) - .await; - self.last_scanned_block_height = block_identifier.block_height; - self.stage.last_scanned_block_height = block_identifier.block_height; - continue; - } - // pull the full block data // Ensure we have exactly 32 bytes @@ -173,38 +451,14 @@ impl Scanner { .fetch_block_with_retry(&mut p2p_conn, block_hash, block_identifier.block_height) .await?; - // build partial secret hashmap, only populate with txids and secrets where we - // suspect matches, skip the rest - let mut partial_secrets = - HashMap::with_capacity(probable_match.matched_txs.len()); - - for tx in &block.txdata { - if probable_match.spent { - // todo: we will need to look at all spent outpoints in this - // block and find the relevant txids for spent - } - - for (txid_arr, tweak) in &probable_match.matched_txs { - // this check should be optimised to a map lookup on all items - let mut item_txid = *txid_arr; - item_txid.reverse(); - - if Txid::from_byte_array(item_txid) != tx.compute_txid() { - continue; - } - - let txid = byte_array_to_txid(txid_arr); - - partial_secrets.insert(txid, *tweak); - } - } // Apply block to indexer and stage the changes - let indexer_changes = self.internal_indexer.apply_block_relevant( + self.apply_matched_block( &block, - partial_secrets, + &probable_match, block_identifier.block_height as u32, ); - self.stage.indexer.merge(indexer_changes); + // Record newly found outputs and confirmed spends of owned ones. + self.sync_owned_outputs(); // Update block checkpoints: only store blocks where we found something let block_height_u32 = block_identifier.block_height as u32; @@ -298,7 +552,9 @@ impl Scanner { let mut idx = self.electrum_index.lock().await; idx.headers.insert(block_height_u32, header_hex.clone()); - idx.tip = Some((block_height_u32, header_hex)); + if idx.tip_height_for(block_height_u32) == block_height_u32 { + idx.tip = Some((block_height_u32, header_hex)); + } for tx in &block.txdata { let txid = tx.compute_txid(); @@ -430,6 +686,19 @@ impl Scanner { // Update last scanned block height and stage it self.last_scanned_block_height = block_identifier.block_height; self.stage.last_scanned_block_height = block_identifier.block_height; + self.record_scanned_block_hash(height, oracle_block_hash); + } + + if expected_height <= end { + return Err(stream_stopped( + expected_height, + "the oracle ended the stream before this height", + )); + } + + if !changed { + // Only checked blocks that were already scanned. + return Ok(()); } let outpoints: Vec<(u32, OutPoint)> = self @@ -486,6 +755,12 @@ impl Scanner { ) -> Result { const MAX_ATTEMPTS: u32 = 4; + // Tests serve full blocks in-process instead of over P2P. + #[cfg(test)] + if let Some(block) = super::stream_safety_tests::served_block(&block_hash) { + return Ok(block); + } + let mut last_err: Option = None; for attempt in 1..=MAX_ATTEMPTS { if p2p_conn.is_none() { @@ -557,7 +832,8 @@ impl Scanner { .as_ref() .map(|(_, hex)| hex.clone()) .unwrap_or_default(); - idx.tip = Some((height_u32, header_hex)); + let tip_height = idx.tip_height_for(height_u32); + idx.tip = Some((tip_height, header_hex)); } let utxo_count = self.internal_indexer.index().by_shared_secret.len(); @@ -570,6 +846,12 @@ impl Scanner { /// height advances past `last_scanned_block_height`, the missing range is /// scanned with `scan_block_range`. This mirrors `blindbit-desktop`'s /// `Watch()` loop and means callers never need to supply an `end_height`. + /// + /// A scan that cannot get past a height is retried on every poll and + /// published as a stall ([`Scanner::scan_health`]) until it makes + /// progress. When the wallet's start height lies below the oracle's first + /// indexed block, scanning starts at that block instead (see + /// [`Scanner::start_at_oracle_floor_if_below`]). pub async fn watch_chain(&mut self) -> Result<(), ScannerError> { loop { let oracle_tip = match self @@ -580,20 +862,68 @@ impl Scanner { Ok(resp) => resp.into_inner().height, Err(e) => { tracing::error!(error = %e, "failed to reach oracle, will retry"); + self.report_stall( + self.last_scanned_block_height + 1, + format!("cannot reach the oracle: {e}"), + ) + .await; time::sleep(time::Duration::from_secs(10)).await; continue; } }; - if oracle_tip > self.last_scanned_block_height { - let from = self.last_scanned_block_height + 1; - tracing::info!(from, to = oracle_tip, "new blocks available, scanning"); - if let Err(e) = self.scan_block_range(from, oracle_tip).await { + let oracle = OracleStreams(self.client.clone()); + if !self.watch_step(oracle_tip, oracle).await { + time::sleep(time::Duration::from_secs(10)).await; + } + } + } + + /// One poll of [`Self::watch_chain`], once the oracle's tip is known: + /// scan new blocks, or with none, check the scanned tip (or the oracle's, + /// if it is behind) against the oracle, since a reorganisation can + /// replace blocks without making the chain longer. Publishes a stall on + /// failure and clears it on success. Returns `true` when the next poll + /// should run right away. + pub(crate) async fn watch_step( + &mut self, + oracle_tip: u64, + mut oracle: Src, + ) -> bool { + let (from, to) = if oracle_tip > self.last_scanned_block_height { + let from = self.last_scanned_block_height + 1; + tracing::info!(from, to = oracle_tip, "new blocks available, scanning"); + (from, oracle_tip) + } else { + (oracle_tip + 1, oracle_tip) + }; + match self.scan_range_from(from, to, oracle.clone()).await { + Ok(()) => { + self.clear_stall().await; + false + } + Err(e) => { + if from <= to + && self + .start_at_oracle_floor_if_below(&e, from, to, &mut oracle) + .await + { + return true; + } + if e.downcast_ref::().is_some() { + tracing::error!( + error = %e, + "wallet state cannot follow the chain; scanning is halted until it is rescanned" + ); + } else { tracing::error!(error = %e, "scan_block_range failed, will retry on next poll"); } + let height = e + .downcast_ref::() + .map_or(self.last_scanned_block_height + 1, |s| s.height); + self.report_stall(height, e.to_string()).await; + false } - - time::sleep(time::Duration::from_secs(10)).await; } } @@ -664,21 +994,32 @@ impl Scanner { } // Spent-output check — only when we have a valid block_hash to report. + // The oracle serves the first 8 bytes of each spent taproot output's + // x-only key. A prefix hit on an unspent owned output only makes the + // block worth fetching; the spend is confirmed (by exact outpoint) when + // the block is applied, so a foreign key sharing the prefix marks + // nothing. if let Some(block_hash) = block_hash_opt { let spent_outputs_count = block_data.spent_outputs.len() / 8; for i in 0..spent_outputs_count { - let spent_output = &block_data.spent_outputs[i * 8..(i + 1) * 8]; - for pubkey in &self.owned_outputs { - if pubkey[..8] == *spent_output { - probable_match.spent = true; - - tracing::info!(pubkey = %hex::encode(pubkey), "spent output detected"); - if self.notify_spent_outpoints.is_empty() { - continue; - } - if let Err(e) = self.notify_spent_outpoints.send(block_hash) { - tracing::warn!(error = ?e, "failed to send spent output notification"); - } + let prefix: [u8; 8] = block_data.spent_outputs[i * 8..(i + 1) * 8] + .try_into() + .expect("slice is 8 bytes"); + let Some(candidates) = self.owned_prefixes.get(&prefix) else { + continue; + }; + probable_match.spent = true; + for outpoint in candidates { + tracing::info!( + outpoint = %outpoint, + short_pubkey = %hex::encode(prefix), + "possible spend of owned output; fetching block to confirm" + ); + if self.notify_spent_outpoints.is_empty() { + continue; + } + if let Err(e) = self.notify_spent_outpoints.send(block_hash) { + tracing::warn!(error = ?e, "failed to send spent output notification"); } } } @@ -691,6 +1032,50 @@ impl Scanner { } } + /// The live receive step after a probable match: hands the full block and + /// the matched transactions' tweaks to the external indexer, which does the + /// real BIP-352 scan, and stages what it found. + /// + /// `probable_match.matched_txs` carries each txid as the oracle serves it + /// (display order) together with the served tweak (`input_hash * A`), which + /// `apply_block_relevant` takes as its "partial secret" and finishes the ECDH + /// on itself. + /// + /// Spends need no extra step here: `apply_block_relevant` inserts every + /// transaction whose input spends an indexed outpoint into the wallet graph, + /// which is what the balance and [`Scanner::sync_owned_outputs`] read. + fn apply_matched_block( + &mut self, + block: &bitcoin::Block, + probable_match: &ProbableMatch, + height: u32, + ) { + // build partial secret hashmap, only populate with txids and secrets where we + // suspect matches, skip the rest + let mut partial_secrets = HashMap::with_capacity(probable_match.matched_txs.len()); + + for tx in &block.txdata { + for (txid_arr, tweak) in &probable_match.matched_txs { + // this check should be optimised to a map lookup on all items + let mut item_txid = *txid_arr; + item_txid.reverse(); + + if Txid::from_byte_array(item_txid) != tx.compute_txid() { + continue; + } + + let txid = byte_array_to_txid(txid_arr); + + partial_secrets.insert(txid, *tweak); + } + } + // Apply block to indexer and stage the changes + let indexer_changes = self + .internal_indexer + .apply_block_relevant(block, partial_secrets, height); + self.stage.indexer.merge(indexer_changes); + } + fn probabilistic_match( &mut self, item: &ComputeIndexTxItem, @@ -841,3 +1226,7 @@ impl Scanner { #[cfg(test)] #[path = "bip352_vectors.rs"] mod bip352_vectors; + +#[cfg(test)] +#[path = "owned_outputs_tests.rs"] +mod owned_outputs_tests; diff --git a/blindbit-lib/src/scanner/stream_safety_tests.rs b/blindbit-lib/src/scanner/stream_safety_tests.rs new file mode 100644 index 0000000..4ae7098 --- /dev/null +++ b/blindbit-lib/src/scanner/stream_safety_tests.rs @@ -0,0 +1,398 @@ +//! How a range scan reacts to what an oracle stream can deliver mid-range. +//! +//! A block the scanner cannot vouch for must never be treated as "no +//! payments": a message with an empty or malformed block hash (what an oracle +//! answers for a height it has not indexed), a height other than the next one +//! expected, an error status, or a stream that ends early each stop the scan +//! with an error, and the scan height stays at the last block that was fully +//! processed so the next scan resumes there. +//! +//! Each test drives a real [`Scanner`] through `scan_block_stream` with +//! in-process messages. The P2P block fetch is the only step replaced: full +//! blocks are served from [`served_block`]. + +use std::collections::{HashMap, VecDeque}; +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Mutex; + +use bdk_sp::bitcoin::key::Secp256k1; +use bdk_sp::receive::get_silentpayment_pubkey; +use bitcoin::absolute::LockTime; +use bitcoin::block::{Header, Version as BlockVersion}; +use bitcoin::hashes::Hash; +use bitcoin::key::TweakedPublicKey; +use bitcoin::secp256k1::{PublicKey, SecretKey}; +use bitcoin::transaction::Version; +use bitcoin::{ + Amount, Block, BlockHash, CompactTarget, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, + TxMerkleNode, TxOut, Txid, Witness, +}; +use bitcoin_rev::Network; +use tonic::transport::Channel; + +use super::Scanner; +use super::scanning::BlockScanDataStream; +use crate::oracle_grpc::{BlockIdentifier, BlockScanDataShortResponse, ComputeIndexTxItem}; + +static SERVED_BLOCKS: Mutex>> = Mutex::new(None); + +/// The full block a test serves for `hash` in place of a P2P fetch. +pub(super) fn served_block(hash: &BlockHash) -> Option { + SERVED_BLOCKS + .lock() + .unwrap() + .as_ref() + .and_then(|blocks| blocks.get(hash).cloned()) +} + +pub(super) fn serve(block: &Block) { + SERVED_BLOCKS + .lock() + .unwrap() + .get_or_insert_with(HashMap::new) + .insert(block.block_hash(), block.clone()); +} + +/// In-process stand-in for the oracle's `StreamBlockScanDataShort` stream. +pub(super) struct TestStream(VecDeque>); + +impl TestStream { + pub(super) fn new(items: Vec>) -> Self { + Self(items.into()) + } +} + +impl BlockScanDataStream for TestStream { + fn next_message( + &mut self, + ) -> impl Future, tonic::Status>> + Send + { + std::future::ready(self.0.pop_front().transpose()) + } +} + +pub(super) fn run(future: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("tokio runtime") + .block_on(future) +} + +fn secret(byte: u8) -> SecretKey { + SecretKey::from_slice(&[byte; 32]).expect("valid secret") +} + +fn keys() -> (SecretKey, PublicKey) { + (secret(0x11), secret(0x22).public_key(&Secp256k1::new())) +} + +pub(super) struct TempState(pub(super) PathBuf); + +impl Drop for TempState { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// A scanner whose oracle client points at `oracle` (never contacted unless a +/// test calls `scan_block_range`). Must be called inside a Tokio runtime. +pub(super) fn scanner(tag: &str, oracle: &'static str) -> (Scanner, TempState) { + let (scan_sk, spend_pk) = keys(); + let state = std::env::temp_dir().join(format!( + "blindbit-stream-safety-{tag}-{}.json", + std::process::id() + )); + let client = crate::OracleServiceClient::new(Channel::from_static(oracle).connect_lazy()); + let socket: SocketAddr = "127.0.0.1:1".parse().expect("socket address"); + let scanner = Scanner::new( + client, + socket, + scan_sk, + spend_pk, + 0, + state.clone(), + Network::Regtest, + ); + (scanner, TempState(state)) +} + +/// One block at `height` holding a single silent payment to the test wallet, +/// served in place of P2P, and the oracle message that describes it. +pub(super) struct PaymentBlock { + pub(super) height: u64, + pub(super) message: BlockScanDataShortResponse, +} + +pub(super) fn payment_block(height: u64) -> PaymentBlock { + let (scan_sk, spend_pk) = keys(); + // The tweak (`input_hash * A`) the oracle would serve; any point works. + let tweak = secret(0x40u8.wrapping_add(height as u8)).public_key(&Secp256k1::new()); + let shared = bdk_sp::compute_shared_secret(&scan_sk, &tweak); + let output_key = get_silentpayment_pubkey(&spend_pk, &shared, 0, None) + .x_only_public_key() + .0; + + let coinbase = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint::null(), + script_sig: ScriptBuf::from_bytes(height.to_le_bytes().to_vec()), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![], + }; + let payment = Transaction { + version: Version::TWO, + lock_time: LockTime::ZERO, + input: vec![TxIn { + previous_output: OutPoint { + txid: Txid::from_byte_array([height as u8; 32]), + vout: 0, + }, + script_sig: ScriptBuf::new(), + sequence: Sequence::MAX, + witness: Witness::new(), + }], + output: vec![TxOut { + value: Amount::from_sat(10_000 + height), + script_pubkey: ScriptBuf::new_p2tr_tweaked(TweakedPublicKey::dangerous_assume_tweaked( + output_key, + )), + }], + }; + let payment_txid = payment.compute_txid(); + + let mut block = Block { + header: Header { + version: BlockVersion::ONE, + prev_blockhash: BlockHash::all_zeros(), + merkle_root: TxMerkleNode::all_zeros(), + time: height as u32, + bits: CompactTarget::from_consensus(0x207f_ffff), + nonce: 0, + }, + txdata: vec![coinbase, payment], + }; + block.header.merkle_root = block.compute_merkle_root().expect("non-empty block"); + + // The oracle serves hashes and txids in display order. + let mut block_hash = block.block_hash().to_byte_array(); + block_hash.reverse(); + let mut txid = payment_txid.to_byte_array(); + txid.reverse(); + + let message = BlockScanDataShortResponse { + block_identifier: Some(BlockIdentifier { + block_hash: block_hash.to_vec(), + block_height: height, + }), + comp_index: vec![ComputeIndexTxItem { + txid: txid.to_vec(), + tweak: tweak.serialize().to_vec(), + outputs_short: output_key.serialize()[..8].to_vec(), + }], + spent_outputs: vec![], + }; + serve(&block); + PaymentBlock { height, message } +} + +/// What an oracle sends today for a height it has not indexed: OK, with an +/// empty block hash and no data. +pub(super) fn unindexed_message(height: u64) -> BlockScanDataShortResponse { + BlockScanDataShortResponse { + block_identifier: Some(BlockIdentifier { + block_hash: vec![], + block_height: height, + }), + comp_index: vec![], + spent_outputs: vec![], + } +} + +pub(super) fn owned_outputs(scanner: &Scanner) -> usize { + scanner.internal_indexer.index().by_shared_secret.len() +} + +/// The regression the testkit's `unindexed-ok-empty` mutation check proved: +/// an unindexed height mid-range was taken as an empty block, the scan +/// advanced past it and reported success, and the payments in it were never +/// found. Now the scan stops before that block, and the next scan (once the +/// oracle serves it) finds every payment. +#[test] +fn empty_hash_block_mid_range_stops_scan_and_next_scan_finds_its_outputs() { + run(async { + let (mut scanner, _state) = scanner("empty-hash", "http://127.0.0.1:1"); + let a = payment_block(1100); + let b = payment_block(1101); + let c = payment_block(1102); + + let first = TestStream::new(vec![ + Ok(a.message.clone()), + Ok(unindexed_message(b.height)), + Ok(c.message.clone()), + ]); + let err = scanner + .scan_block_stream(a.height, c.height, first) + .await + .expect_err("a block without a valid hash must stop the scan"); + let msg = err.to_string(); + assert!( + msg.contains(&b.height.to_string()), + "error names the height: {msg}" + ); + assert_eq!(scanner.get_last_scanned_block_height(), a.height); + assert_eq!( + owned_outputs(&scanner), + 1, + "only the block before the gap is applied" + ); + + // Resume where the scanner says, now that the oracle serves the block. + let resume = scanner.get_last_scanned_block_height() + 1; + assert_eq!(resume, b.height); + let second = TestStream::new(vec![Ok(b.message.clone()), Ok(c.message.clone())]); + scanner + .scan_block_stream(resume, c.height, second) + .await + .expect("resumed scan succeeds"); + assert_eq!(scanner.get_last_scanned_block_height(), c.height); + assert_eq!(owned_outputs(&scanner), 3, "no payment is lost"); + }); +} + +#[test] +fn malformed_block_hash_stops_scan() { + run(async { + let (mut scanner, _state) = scanner("short-hash", "http://127.0.0.1:1"); + let a = payment_block(1200); + let mut bad = payment_block(1201).message; + bad.block_identifier + .as_mut() + .unwrap() + .block_hash + .truncate(31); + let mut zero = payment_block(1201).message; + zero.block_identifier.as_mut().unwrap().block_hash = vec![0; 32]; + + for bad in [bad, zero] { + let stream = TestStream::new(vec![Ok(a.message.clone()), Ok(bad)]); + scanner + .scan_block_stream(a.height, a.height + 1, stream) + .await + .expect_err("a malformed block hash must stop the scan"); + assert_eq!(scanner.get_last_scanned_block_height(), a.height); + } + }); +} + +#[test] +fn skipped_height_stops_scan() { + run(async { + let (mut scanner, _state) = scanner("skipped", "http://127.0.0.1:1"); + let a = payment_block(1300); + let c = payment_block(1302); + let stream = TestStream::new(vec![Ok(a.message.clone()), Ok(c.message.clone())]); + let err = scanner + .scan_block_stream(a.height, c.height, stream) + .await + .expect_err("a height gap must stop the scan"); + assert!( + err.to_string().contains("1301"), + "error names the expected height: {err}" + ); + assert_eq!(scanner.get_last_scanned_block_height(), a.height); + assert_eq!( + owned_outputs(&scanner), + 1, + "the block after the gap is not applied" + ); + }); +} + +#[test] +fn stream_starting_at_wrong_height_stops_scan() { + run(async { + let (mut scanner, _state) = scanner("wrong-start", "http://127.0.0.1:1"); + scanner.update_last_scanned_block_height(1399); + let b = payment_block(1401); + let stream = TestStream::new(vec![Ok(b.message.clone())]); + scanner + .scan_block_stream(1400, 1401, stream) + .await + .expect_err("the first block must be the requested start"); + assert_eq!(scanner.get_last_scanned_block_height(), 1399); + assert_eq!(owned_outputs(&scanner), 0); + }); +} + +/// A fixed oracle ends the stream with NOT_FOUND at an unindexed height; a +/// transient failure ends it with UNAVAILABLE. Neither may panic the process, +/// and neither may advance the scan height past the last full block. +#[test] +fn error_status_mid_stream_is_a_clean_error() { + run(async { + let statuses = [ + tonic::Status::not_found("height 1501 is not indexed"), + tonic::Status::unavailable("connection reset"), + ]; + for (i, status) in statuses.into_iter().enumerate() { + let code = status.code(); + let (mut scanner, _state) = scanner(&format!("status-{i}"), "http://127.0.0.1:1"); + let a = payment_block(1500); + let stream = TestStream::new(vec![Ok(a.message.clone()), Err(status)]); + let err = scanner + .scan_block_stream(a.height, a.height + 1, stream) + .await + .expect_err("an error status must end the scan with an error"); + let msg = err.to_string(); + assert!( + msg.contains("height 1501") && msg.contains(&format!("{code:?}")), + "error names the height and the status code: {msg}" + ); + assert_eq!( + scanner.get_last_scanned_block_height(), + a.height, + "{code:?}" + ); + assert_eq!(owned_outputs(&scanner), 1, "{code:?}"); + } + }); +} + +#[test] +fn stream_ending_before_requested_end_is_an_error() { + run(async { + let (mut scanner, _state) = scanner("short-stream", "http://127.0.0.1:1"); + let a = payment_block(1600); + let stream = TestStream::new(vec![Ok(a.message.clone())]); + let err = scanner + .scan_block_stream(a.height, a.height + 2, stream) + .await + .expect_err("a stream that stops short of `end` must not report success"); + assert!( + err.to_string().contains("1601"), + "error names the missing height: {err}" + ); + assert_eq!(scanner.get_last_scanned_block_height(), a.height); + }); +} + +/// Opening the stream against an unreachable oracle used to `unwrap()`. +#[test] +fn unreachable_oracle_is_a_clean_error() { + run(async { + let (mut scanner, _state) = scanner("unreachable", "http://127.0.0.1:1"); + scanner.update_last_scanned_block_height(1699); + scanner + .scan_block_range(1700, 1710) + .await + .expect_err("an unreachable oracle must be an error, not a panic"); + assert_eq!(scanner.get_last_scanned_block_height(), 1699); + }); +} diff --git a/blindbit-lib/src/scanner/testdata/state_before_owned_outputs.json b/blindbit-lib/src/scanner/testdata/state_before_owned_outputs.json new file mode 100644 index 0000000..9b09442 --- /dev/null +++ b/blindbit-lib/src/scanner/testdata/state_before_owned_outputs.json @@ -0,0 +1,164 @@ +{ + "block_checkpoints": { + "0": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f", + "100": "0d52ac6e67ab550e176ebb2fb29d8fdf9a113ae2549b05d800cde721dde38d29" + }, + "indexer": { + "scan_sk": "1111111111111111111111111111111111111111111111111111111111111111", + "spend_pk": "02466d7fcae563e5cb09a0d1870bb580344804617879a14949cf22285f1bae3f27", + "txid_to_partial_secret": { + "cd8a3efcfeedb7d07d87fce2288ceb2a6bfc3348b035f6bf8c878efad06f7a81": "023c72addb4fdf09af94f0c94d7fe92a386a7e70cf8a1d85916386bb2535c7b1b1", + "1d0e05d33c0c47a86009983a84044e3c59c702d8e08cd5adf8e3c95e0b665dbe": "023c72addb4fdf09af94f0c94d7fe92a386a7e70cf8a1d85916386bb2535c7b1b1" + }, + "label_lookup": [ + [ + 0, + [ + 53, + 81, + 172, + 101, + 113, + 186, + 106, + 173, + 137, + 199, + 228, + 228, + 29, + 126, + 48, + 225, + 160, + 91, + 41, + 98, + 115, + 232, + 204, + 116, + 50, + 67, + 212, + 126, + 61, + 200, + 124, + 148 + ], + "038b2709f0aa4014d750140cbdc445634f3e4b16e3c686806127d53a9c8cf29921" + ], + [ + 1, + [ + 194, + 60, + 36, + 4, + 114, + 227, + 65, + 153, + 163, + 180, + 221, + 100, + 222, + 126, + 135, + 216, + 187, + 62, + 157, + 157, + 102, + 237, + 145, + 164, + 174, + 11, + 209, + 154, + 6, + 161, + 183, + 32 + ], + "035be3a727b6bccdf4bbd4e6ebf27266b41816b3ee373237540942b8fc339c5bf9" + ] + ], + "graph": { + "txs": [ + { + "version": 2, + "lock_time": 0, + "input": [ + { + "previous_output": "0101010101010101010101010101010101010101010101010101010101010101:0", + "script_sig": "", + "sequence": 4294967295, + "witness": [] + } + ], + "output": [ + { + "value": 50000, + "script_pubkey": "5120495feefbace0631dfad36337aeaa066615dc1f34598b21e455599670607858f1" + } + ] + }, + { + "version": 2, + "lock_time": 0, + "input": [ + { + "previous_output": "0202020202020202020202020202020202020202020202020202020202020202:0", + "script_sig": "", + "sequence": 4294967295, + "witness": [] + } + ], + "output": [ + { + "value": 20000, + "script_pubkey": "512090e97c584c5888d5287961b2af2c7b729f60f4e5973a24e394b993d017024f19" + } + ] + } + ], + "txouts": {}, + "anchors": [ + [ + { + "block_id": { + "height": 100, + "hash": "0d52ac6e67ab550e176ebb2fb29d8fdf9a113ae2549b05d800cde721dde38d29" + }, + "confirmation_time": 1713571867 + }, + "cd8a3efcfeedb7d07d87fce2288ceb2a6bfc3348b035f6bf8c878efad06f7a81" + ], + [ + { + "block_id": { + "height": 100, + "hash": "0d52ac6e67ab550e176ebb2fb29d8fdf9a113ae2549b05d800cde721dde38d29" + }, + "confirmation_time": 1713571867 + }, + "1d0e05d33c0c47a86009983a84044e3c59c702d8e08cd5adf8e3c95e0b665dbe" + ] + ], + "last_seen": {}, + "last_evicted": {}, + "first_seen": {} + } + }, + "last_scanned_block_height": 300, + "last_scanned_block_height_rescan": 0, + "owned_outputs": [], + "secret_scan_hex": "1111111111111111111111111111111111111111111111111111111111111111", + "public_spend_hex": "02466d7fcae563e5cb09a0d1870bb580344804617879a14949cf22285f1bae3f27", + "max_label_num": 1 +} \ No newline at end of file diff --git a/blindbit-lib/src/scanner/types.rs b/blindbit-lib/src/scanner/types.rs index 50b6660..7ad8c96 100644 --- a/blindbit-lib/src/scanner/types.rs +++ b/blindbit-lib/src/scanner/types.rs @@ -1,7 +1,7 @@ use crate::oracle_grpc::BlockIdentifier; use bitcoin::absolute::Height; use bitcoin::secp256k1::PublicKey; -use bitcoin::{Amount, OutPoint, ScriptBuf}; +use bitcoin::{Amount, OutPoint, ScriptBuf, Txid, XOnlyPublicKey}; use indexer::v2::indexes::Label; /// Wrapper for `BlockIdentifier` that implements Display with hex formatting @@ -40,6 +40,48 @@ pub struct OwnedOutput { pub spent: Option, } +/// One wallet-owned Silent Payments output, as the scanner records and +/// persists it (the `owned_outputs` array of the state file). +/// +/// A record is created when the indexer confirms an output as ours (full +/// BIP-352 match in a fetched block) and is marked spent only when a +/// confirmed transaction in the wallet graph spends exactly this `outpoint`. +/// The oracle's 8-byte `spent_outputs` prefixes are used solely as a trigger to +/// fetch the block; they never mark a record on their own. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct OwnedOutputRecord { + /// The output itself (`txid:vout`). + pub outpoint: OutPoint, + /// Full x-only taproot output key (the key the oracle's spent-output + /// prefixes are taken from). + pub pubkey: XOnlyPublicKey, + /// Output value in satoshis. + pub amount_sat: u64, + /// Height of the block that confirmed the output. + pub height: u32, + /// BIP-352 label `m` the output matched; `None` for the unlabelled + /// address, `Some(0)` for change. + pub label: Option, + /// Confirmed transaction that spends this output, if any. + pub spent_by: Option, + /// Height of the block that confirmed `spent_by`. + pub spent_height: Option, +} + +impl OwnedOutputRecord { + /// The 8-byte prefix the oracle serves for a spent taproot output. + pub fn short_pubkey(&self) -> [u8; 8] { + let mut prefix = [0u8; 8]; + prefix.copy_from_slice(&self.pubkey.serialize()[..8]); + prefix + } + + pub fn is_spent(&self) -> bool { + self.spent_by.is_some() + } +} + /// `ProbableMatch` is a struct that contains a list of txids that are probable matches /// and a boolean indicating if a utxo might be spent pub(crate) struct ProbableMatch { diff --git a/friglet-ipc/src/lib.rs b/friglet-ipc/src/lib.rs index f6ee43c..fd843bd 100644 --- a/friglet-ipc/src/lib.rs +++ b/friglet-ipc/src/lib.rs @@ -159,6 +159,53 @@ pub struct StatusInfo { /// instead of assuming it did not spawn it. #[serde(default)] pub spawned_by_tray: bool, + /// Conditions of the scan beyond its progress: why it is stuck, and + /// whether it started above the configured start height. + #[serde(default)] + pub scan_health: ScanHealthInfo, +} + +/// Scan conditions the user should see (see [`StatusInfo::scan_health`]). +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ScanHealthInfo { + /// Set while the scan cannot get past a height. The daemon keeps + /// retrying; this clears once the scan makes progress again. + pub stall: Option, + /// Set when the configured start height lay below the oracle's first + /// indexed block, so scanning began at that block instead. + pub start_adjusted: Option, + /// Set while a state file written before spends were tracked is being + /// rescanned once for the spends it missed. + pub state_rescan: Option, +} + +/// Blocks `from_height..=until_height` are scanned again because the state +/// file predates spend tracking (see blindbit-lib `StateRescan`). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StateRescanInfo { + pub from_height: u64, + pub until_height: u64, +} + +/// The scan cannot get past `height`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ScanStallInfo { + /// First height that could not be scanned. + pub height: u64, + /// Why, as the scanner reported it. + pub reason: String, + /// Unix time (seconds) of the first failed attempt at this height. + pub since_unix: u64, +} + +/// Scanning began at `oracle_floor` instead of the configured +/// `requested_height`: the oracle has no data below it, so blocks +/// `requested_height..oracle_floor` were not scanned. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct StartAdjustedInfo { + pub requested_height: u64, + pub oracle_floor: u64, } /// A configured Silent Payments label and its receive address. @@ -458,6 +505,21 @@ mod tests { }], version: "test".to_string(), spawned_by_tray: true, + scan_health: ScanHealthInfo { + stall: Some(ScanStallInfo { + height: 102, + reason: "the oracle sent no valid block hash".to_string(), + since_unix: 1_790_000_000, + }), + start_adjusted: Some(StartAdjustedInfo { + requested_height: 1, + oracle_floor: 100, + }), + state_rescan: Some(StateRescanInfo { + from_height: 100, + until_height: 101, + }), + }, }; let json = serde_json::to_string(&status).unwrap(); assert_eq!(serde_json::from_str::(&json).unwrap(), status); @@ -482,6 +544,7 @@ mod tests { assert_eq!(status.outputs_found, 0); assert!(status.label_addresses.is_empty()); assert!(!status.spawned_by_tray); + assert_eq!(status.scan_health, ScanHealthInfo::default()); } #[test] diff --git a/friglet-ipc/tests/roundtrip.rs b/friglet-ipc/tests/roundtrip.rs index 61df624..ae608d9 100644 --- a/friglet-ipc/tests/roundtrip.rs +++ b/friglet-ipc/tests/roundtrip.rs @@ -37,6 +37,7 @@ async fn get_status_roundtrip() { }], version: "0.1.0".to_string(), spawned_by_tray: true, + scan_health: Default::default(), }; let expected = status.clone(); diff --git a/friglet-tray/examples/fake-daemon.rs b/friglet-tray/examples/fake-daemon.rs index 4a422c3..11c1faf 100644 --- a/friglet-tray/examples/fake-daemon.rs +++ b/friglet-tray/examples/fake-daemon.rs @@ -83,6 +83,7 @@ async fn main() -> std::io::Result<()> { ], version: "fake-0.0.0".to_string(), spawned_by_tray: false, + scan_health: Default::default(), }), Request::Start => { scanning.store(true, Ordering::Relaxed); diff --git a/friglet-tray/src/lib.rs b/friglet-tray/src/lib.rs index c7be3d4..a066c32 100644 --- a/friglet-tray/src/lib.rs +++ b/friglet-tray/src/lib.rs @@ -690,6 +690,7 @@ mod tests { label_addresses: Vec::new(), version: "test".to_string(), spawned_by_tray, + scan_health: Default::default(), }), _ => Response::Ok, }; diff --git a/friglet-tray/tests/lifecycle_e2e.rs b/friglet-tray/tests/lifecycle_e2e.rs index 017c5da..7237f06 100644 --- a/friglet-tray/tests/lifecycle_e2e.rs +++ b/friglet-tray/tests/lifecycle_e2e.rs @@ -42,6 +42,7 @@ fn dummy_status() -> StatusInfo { label_addresses: Vec::new(), version: "test".to_string(), spawned_by_tray: false, + scan_health: Default::default(), } } diff --git a/friglet-tray/ui/index.html b/friglet-tray/ui/index.html index 06ad500..f58fed2 100644 --- a/friglet-tray/ui/index.html +++ b/friglet-tray/ui/index.html @@ -64,6 +64,14 @@

Friglet

Last error + +
diff --git a/friglet-tray/ui/main.js b/friglet-tray/ui/main.js index efe2517..8cd0923 100644 --- a/friglet-tray/ui/main.js +++ b/friglet-tray/ui/main.js @@ -42,11 +42,45 @@ function render({ reachable, status }) { $("error-block").hidden = !status.last_error; setText("last-error", status.last_error ?? ""); + renderScanHealth(status.scan_health ?? {}); $("start-btn").disabled = !reachable || status.scanning; $("stop-btn").disabled = !reachable || !status.scanning; } +// Why the scan is stuck, and where it started (the daemon's `scan_health`; +// absent from older daemons). +function renderScanHealth(health) { + const stall = health.stall; + $("stall-block").hidden = !stall; + setText( + "stall", + stall + ? `at height ${stall.height.toLocaleString()} since ` + + `${new Date(stall.since_unix * 1000).toLocaleString()}: ${stall.reason}` + : "", + ); + + const notes = []; + const start = health.start_adjusted; + if (start) { + notes.push( + `Scanning began at block ${start.oracle_floor.toLocaleString()} instead of ` + + `${start.requested_height.toLocaleString()}: the oracle has no data below it.`, + ); + } + const rescan = health.state_rescan; + if (rescan) { + notes.push( + `Rescanning blocks ${rescan.from_height.toLocaleString()}–` + + `${rescan.until_height.toLocaleString()} once: the state file predates ` + + `spend tracking, so spends it missed are being looked for.`, + ); + } + $("scan-notes-block").hidden = notes.length === 0; + setText("scan-notes", notes.join("\n")); +} + async function refresh() { try { const status = await invoke("get_status"); diff --git a/friglet-tray/ui/style.css b/friglet-tray/ui/style.css index 3104613..ee8b9e8 100644 --- a/friglet-tray/ui/style.css +++ b/friglet-tray/ui/style.css @@ -71,6 +71,9 @@ code { color: var(--text); } .error { color: var(--bad); } +/* `.col` sets display: flex, which would override the `hidden` attribute. */ +.col[hidden] { display: none; } +#scan-notes { font-size: 13px; white-space: pre-line; } .small { font-size: 12px; min-height: 18px; } .banner { diff --git a/friglet/src/control/mod.rs b/friglet/src/control/mod.rs index 4e238dd..f155840 100644 --- a/friglet/src/control/mod.rs +++ b/friglet/src/control/mod.rs @@ -118,6 +118,32 @@ pub(crate) fn wallet_network(network: bitcoin_rev::Network) -> BitcoinNetwork { } } +/// The scanner's published health as sent over the control socket. +fn scan_health_info(health: &scanner::ScanHealth) -> friglet_ipc::ScanHealthInfo { + friglet_ipc::ScanHealthInfo { + stall: health + .stall + .as_ref() + .map(|stall| friglet_ipc::ScanStallInfo { + height: stall.height, + reason: stall.reason.clone(), + since_unix: stall.since_unix, + }), + start_adjusted: health + .oracle_floor_start + .map(|start| friglet_ipc::StartAdjustedInfo { + requested_height: start.requested_height, + oracle_floor: start.floor_height, + }), + state_rescan: health + .state_rescan + .map(|rescan| friglet_ipc::StateRescanInfo { + from_height: rescan.from_height, + until_height: rescan.until_height, + }), + } +} + /// Everything the request handler needs from the daemon. pub struct ControlCtx { pub supervisor: Arc, @@ -234,6 +260,7 @@ impl ControlCtx { let addr = (!idx.sp_address.is_empty()).then(|| idx.sp_address.clone()); (tip, idx.scan_progress, addr, idx.sp_history.len() as u64) }; + let scan_health = scan_health_info(&index.lock().await.scan_health); // While the scan task runs it holds the scanner mutex, so fall back // to the electrum index tip (which the scanner advances per scanned @@ -266,6 +293,7 @@ impl ControlCtx { label_addresses: self.label_addresses.lock().unwrap().clone(), version: env!("CARGO_PKG_VERSION").to_string(), spawned_by_tray: self.spawned_by_tray, + scan_health, } } @@ -773,6 +801,7 @@ mod tests { label_addresses: Vec::new(), version: "test".to_string(), spawned_by_tray: false, + scan_health: Default::default(), } } @@ -801,6 +830,55 @@ mod tests { assert_eq!(ctx.status().await.tx_count, 2); } + #[tokio::test] + async fn status_reports_a_stalled_scan_and_an_adjusted_start() { + let dir = temp_dir("status-scan-health"); + let ctx = test_ctx(&dir); + assert_eq!( + ctx.status().await.scan_health, + friglet_ipc::ScanHealthInfo::default() + ); + + let index = ctx.electrum_index.lock().unwrap().clone(); + { + let mut idx = index.lock().await; + idx.scan_health.stall = Some(scanner::ScanStall { + height: 100_002, + reason: "scan stopped at height 100002: the oracle sent no valid block hash" + .to_string(), + since_unix: 1_790_000_000, + }); + idx.scan_health.oracle_floor_start = Some(scanner::OracleFloorStart { + requested_height: 50_000, + floor_height: 100_000, + }); + idx.scan_health.state_rescan = Some(scanner::StateRescan { + from_height: 100_500, + until_height: 101_000, + }); + } + + let health = ctx.status().await.scan_health; + let stall = health.stall.expect("stall is reported"); + assert_eq!(stall.height, 100_002); + assert_eq!(stall.since_unix, 1_790_000_000); + assert!(stall.reason.contains("no valid block hash")); + assert_eq!( + health.start_adjusted, + Some(friglet_ipc::StartAdjustedInfo { + requested_height: 50_000, + oracle_floor: 100_000, + }) + ); + assert_eq!( + health.state_rescan, + Some(friglet_ipc::StateRescanInfo { + from_height: 100_500, + until_height: 101_000, + }) + ); + } + #[tokio::test] async fn get_status_roundtrip_over_control_socket() { #[cfg(windows)] diff --git a/friglet/src/electrum/mod.rs b/friglet/src/electrum/mod.rs index 7297518..108dce0 100644 --- a/friglet/src/electrum/mod.rs +++ b/friglet/src/electrum/mod.rs @@ -104,7 +104,9 @@ struct ElectrumServerState { p2p_peer: SocketAddr, network: Network, oracle_url: String, - block_checkpoints: HashMap, + /// Wallet checkpoints (heights with findings) snapshotted at startup; + /// entries above a reorg's fork point are dropped when it happens. + block_checkpoints: std::sync::Mutex>, header_sidecar: PathBuf, header_fetch_lock: Mutex<()>, /// Broadcast channel: pre-serialised JSON notification lines sent to every @@ -124,6 +126,7 @@ struct ElectrumServerState { pub async fn run( index: Arc>, mut found_rx: broadcast::Receiver, + mut reorg_rx: broadcast::Receiver, addr: &str, p2p_peer: SocketAddr, network: Network, @@ -139,7 +142,7 @@ pub async fn run( p2p_peer, network, oracle_url, - block_checkpoints, + block_checkpoints: std::sync::Mutex::new(block_checkpoints), header_sidecar, header_fetch_lock: Mutex::new(()), push_tx: push_tx.clone(), @@ -180,6 +183,22 @@ pub async fn run( }); } + // Chain reorganisations: the scanner already dropped the disconnected + // blocks from the index; forget their hashes and cached headers here too + // so no header of a block that left the chain is served again. + { + let state = state.clone(); + tokio::spawn(async move { + loop { + match reorg_rx.recv().await { + Ok(fork) => forget_blocks_above(&state, fork).await, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => return, + } + } + }); + } + let listener = TcpListener::bind(addr).await?; tracing::info!(addr = %addr, "Electrum server listening"); @@ -277,9 +296,44 @@ async fn push_notifications(state: &ElectrumServerState) { .send(serde_json::to_string(¬ification).unwrap() + "\n"); } +/// Drop block hashes and headers above a reorg's fork point, and rewrite +/// the header sidecar without them. +async fn forget_blocks_above(state: &ElectrumServerState, fork: u32) { + state + .block_checkpoints + .lock() + .expect("checkpoint lock") + .retain(|height, _| *height <= fork); + let headers = { + let mut index = state.index.lock().await; + index.headers.retain(|height, _| *height <= fork); + index.headers.clone() + }; + tracing::warn!( + fork, + "chain reorganisation: dropped cached headers above the fork point" + ); + let sidecar = state.header_sidecar.clone(); + match tokio::task::spawn_blocking(move || blockheader::save_headers(&sidecar, &headers)).await { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!( + path = %state.header_sidecar.display(), + error = %error, + "failed to rewrite block-header sidecar after reorg" + ), + Err(error) => tracing::warn!(error = %error, "block-header sidecar write task failed"), + } +} + async fn resolve_block_hash(state: &ElectrumServerState, height: u32) -> Result { - if let Some(hash) = state.block_checkpoints.get(&height) { - return Ok(*hash); + let checkpoint = state + .block_checkpoints + .lock() + .expect("checkpoint lock") + .get(&height) + .copied(); + if let Some(hash) = checkpoint { + return Ok(hash); } let oracle_url = state.oracle_url.clone(); @@ -837,3 +891,62 @@ async fn handle_request(req: &JsonRpcRequest, state: &ElectrumServerState) -> St .into_line(), } } + +#[cfg(test)] +mod tests { + use super::*; + use bitcoin::hashes::Hash; + + #[tokio::test] + async fn reorg_forgets_hashes_and_headers_above_the_fork() { + let dir = + std::env::temp_dir().join(format!("friglet-electrum-reorg-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let sidecar = dir.join("scanner_state.headers.json"); + let hash = |b: u8| BlockHash::from_byte_array([b; 32]); + + let index = Arc::new(Mutex::new(WalletElectrumIndex::new())); + { + let mut idx = index.lock().await; + for height in [100u32, 104, 105, 106] { + idx.headers.insert(height, format!("header-{height}")); + } + } + let state = ElectrumServerState { + index: index.clone(), + p2p_peer: "127.0.0.1:1".parse().unwrap(), + network: Network::Regtest, + oracle_url: "http://127.0.0.1:1".into(), + block_checkpoints: std::sync::Mutex::new(HashMap::from([ + (0, hash(0)), + (104, hash(4)), + (105, hash(5)), + ])), + header_sidecar: sidecar.clone(), + header_fetch_lock: Mutex::new(()), + push_tx: broadcast::channel(1).0, + }; + + forget_blocks_above(&state, 104).await; + + let mut checkpoints: Vec = state + .block_checkpoints + .lock() + .unwrap() + .keys() + .copied() + .collect(); + checkpoints.sort(); + assert_eq!(checkpoints, vec![0, 104]); + let mut cached: Vec = index.lock().await.headers.keys().copied().collect(); + cached.sort(); + assert_eq!(cached, vec![100, 104]); + let mut persisted: Vec = blockheader::load_headers(&sidecar) + .keys() + .copied() + .collect(); + persisted.sort(); + assert_eq!(persisted, vec![100, 104], "sidecar rewritten without them"); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/friglet/src/main.rs b/friglet/src/main.rs index 549017f..5953c88 100644 --- a/friglet/src/main.rs +++ b/friglet/src/main.rs @@ -130,7 +130,7 @@ async fn run(args: ScanArgs) -> Result<(), Box Result<(), Box Result<(), Box