diff --git a/server/src/models/SyncState.ts b/server/src/models/SyncState.ts index 215b913..bc82992 100644 --- a/server/src/models/SyncState.ts +++ b/server/src/models/SyncState.ts @@ -12,6 +12,7 @@ export interface SyncStateDocument extends Document { error: string | null; addressIndexVersion?: number; satoshiDataVersion?: number; + addressStatsVersion?: number; } const syncStateSchema = new Schema({ @@ -26,6 +27,8 @@ const syncStateSchema = new Schema({ error: { type: String, default: null }, addressIndexVersion: { type: Number, default: 0 }, satoshiDataVersion: { type: Number, default: 0 }, + // Version of how address statistics are counted; an older one rebuilds the address index once. + addressStatsVersion: { type: Number, default: 0 }, }); export const SyncState = mongoose.model('SyncState', syncStateSchema); diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index b34f738..8488ef9 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -31,6 +31,7 @@ import { collectInvolvedAddresses, mergeAddressDeltas, normalizeSat, + type AddressDelta, } from './sync/addressIndexing'; import { runScheduledSyncAttempt } from './sync/syncScheduling'; @@ -77,6 +78,9 @@ class SyncService { private readonly CURSOR_MAX_TIME_MS = Math.max(30_000, config.rpc.timeout); private readonly ADDRESS_INDEX_VERSION = 3; private readonly SATOSHI_DATA_VERSION = 3; + // 1: txCount counts each transaction an address takes part in (it counted blocks during live + // sync and transactions in a rebuild). An older stored version rebuilds the address index once. + private readonly ADDRESS_STATS_VERSION = 1; private readonly SAT_MULTIPLIER = 10 ** COIN.DECIMALS; private readonly TX_EVENT_MAX_PER_BLOCK = 250; @@ -158,7 +162,8 @@ class SyncService { return before?.addressRebuildRequired === true; } - private async recoverStartupState(): Promise { + /** Repairs what a previous run left behind; returns true when it rebuilt the address index. */ + private async recoverStartupState(): Promise { const syncState = await SyncState.findOne({ key: 'main' }).lean(); const indexedTip = await Block.findOne({}) .sort({ height: -1 }) @@ -179,7 +184,7 @@ class SyncService { }, { upsert: true } ); - return; + return false; } // If sync state points past our indexed tip (possible after crash during rollback), @@ -214,7 +219,7 @@ class SyncService { ); } - if (!recoveryPlan.rebuildAddressIndex) return; + if (!recoveryPlan.rebuildAddressIndex) return false; logger.warn('Detected pending address index rebuild flag. Rebuilding from transactions...'); await this.rebuildAddressIndexFromTransactions(); @@ -228,6 +233,33 @@ class SyncService { { upsert: true } ); logger.info('Pending address rebuild recovered successfully.'); + return true; + } + + /** + * Brings the stored address statistics to ADDRESS_STATS_VERSION: if the stored version is older, + * rebuilds the address index once and records the version. A rebuild that startup recovery has + * just done counts, so a start never rebuilds twice. Interrupted before the version is written, + * the next start simply rebuilds again. + */ + private async ensureAddressStatsVersion(rebuiltThisStart: boolean): Promise { + const syncState = await SyncState.findOne({ key: 'main' }).select({ addressStatsVersion: 1 }).lean<{ addressStatsVersion?: number }>(); + const currentVersion = syncState?.addressStatsVersion ?? 0; + if (currentVersion >= this.ADDRESS_STATS_VERSION) return; + + if (!rebuiltThisStart) { + logger.warn( + `Address statistics version ${currentVersion} -> ${this.ADDRESS_STATS_VERSION}: rebuilding the address index once...` + ); + const startedAt = Date.now(); + await this.rebuildAddressIndexFromTransactions(); + logger.info(`Address statistics rebuild finished in ${Math.round((Date.now() - startedAt) / 1000)} s.`); + } + await SyncState.findOneAndUpdate( + { key: 'main' }, + { addressStatsVersion: this.ADDRESS_STATS_VERSION, heartbeatAt: new Date() }, + { upsert: true } + ); } // Rebuild address balances from stored transactions. @@ -269,28 +301,8 @@ class SyncService { continue; } - const addressMap = new Map(); - - for (const vout of tx.vout || []) { - const addresses = vout.scriptPubKey?.addresses; - if (!addresses?.length) continue; - const valueSat = normalizeSat(vout.valueSat); - for (const addr of addresses) { - const entry = addressMap.get(addr) || { receivedSat: 0n, sentSat: 0n }; - entry.receivedSat += valueSat; - addressMap.set(addr, entry); - } - } - - if (!tx.isCoinbase) { - for (const vin of tx.vin || []) { - const valueSat = normalizeSat(vin.valueSat); - if (!vin.address || valueSat <= 0n) continue; - const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n }; - entry.sentSat += valueSat; - addressMap.set(vin.address, entry); - } - } + // The same per-transaction changes live sync collects (processTransaction). + const addressMap = collectAddressDeltas(tx.vin || [], tx.vout || [], !!tx.isCoinbase); for (const [addr, update] of addressMap) { bulkOps.push({ @@ -614,7 +626,8 @@ class SyncService { try { await this.ensureIndexVersion(); await this.ensureSatoshiDataVersion(); - await this.recoverStartupState(); + const rebuiltThisStart = await this.recoverStartupState(); + await this.ensureAddressStatsVersion(rebuiltThisStart); await this.addressResolver.preInferVersionByte(); // Run initial sync @@ -982,7 +995,7 @@ class SyncService { let totalValueOutSat = 0n; let minedBy: string | undefined; const txids: string[] = []; - const blockAddressDeltas = new Map(); + const blockAddressDeltas = new Map(); const txEntries = (rpcBlock.tx || []) as Array; @@ -1115,7 +1128,7 @@ class SyncService { ): Promise<{ totalValueOutSat: bigint; isCoinbase: boolean; - addressDeltas?: Map; + addressDeltas?: Map; }> { const isCoinbase = rpcTx.vin.length > 0 && !!rpcTx.vin[0].coinbase; @@ -1369,7 +1382,7 @@ class SyncService { // ── Address Balance Updates ───────────────────────────────────── private async applyAddressDeltas( - addressMap: Map, + addressMap: Map, blocktime: number ): Promise { if (addressMap.size === 0) return false; @@ -1455,6 +1468,7 @@ class SyncService { error: null, addressIndexVersion: this.ADDRESS_INDEX_VERSION, satoshiDataVersion: this.SATOSHI_DATA_VERSION, + addressStatsVersion: this.ADDRESS_STATS_VERSION, }, { upsert: true } ); diff --git a/server/src/services/sync/addressIndexing.ts b/server/src/services/sync/addressIndexing.ts index b88a623..4cc3fba 100644 --- a/server/src/services/sync/addressIndexing.ts +++ b/server/src/services/sync/addressIndexing.ts @@ -4,6 +4,8 @@ import { coinToSat, satToBigInt } from '../../utils/amounts'; export interface AddressDelta { receivedSat: bigint; sentSat: bigint; + /** How many transactions the address takes part in: 1 for one transaction, summed over a block. */ + txCount: number; } interface VinLike { @@ -57,7 +59,7 @@ export function collectAddressDeltas( if (!addresses?.length) continue; const valueSat = normalizeSat(vout.valueSat); for (const address of addresses) { - const entry = addressMap.get(address) || { receivedSat: 0n, sentSat: 0n }; + const entry = addressMap.get(address) || { receivedSat: 0n, sentSat: 0n, txCount: 1 }; entry.receivedSat += valueSat; addressMap.set(address, entry); } @@ -66,7 +68,7 @@ export function collectAddressDeltas( for (const vin of vins) { const valueSat = normalizeSat(vin.valueSat); if (!vin.address || valueSat <= 0n) continue; - const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n }; + const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n, txCount: 1 }; entry.sentSat += valueSat; addressMap.set(vin.address, entry); } @@ -79,9 +81,10 @@ export function mergeAddressDeltas( source: Map ): void { for (const [address, delta] of source) { - const current = target.get(address) || { receivedSat: 0n, sentSat: 0n }; + const current = target.get(address) || { receivedSat: 0n, sentSat: 0n, txCount: 0 }; current.receivedSat += delta.receivedSat; current.sentSat += delta.sentSat; + current.txCount += delta.txCount; target.set(address, current); } } @@ -106,7 +109,7 @@ export function buildAddressDeltaPipeline( balanceSat: { $add: [{ $ifNull: ['$balanceSat', zeroDec] }, netSatDec] }, totalReceivedSat: { $add: [{ $ifNull: ['$totalReceivedSat', zeroDec] }, receivedDec] }, totalSentSat: { $add: [{ $ifNull: ['$totalSentSat', zeroDec] }, sentDec] }, - txCount: { $add: [{ $ifNull: ['$txCount', 0] }, 1] }, + txCount: { $add: [{ $ifNull: ['$txCount', 0] }, delta.txCount] }, firstSeen: { $min: [{ $ifNull: ['$firstSeen', blocktime] }, blocktime] }, lastSeen: { $max: [{ $ifNull: ['$lastSeen', 0] }, blocktime] }, }, diff --git a/server/test/addressIndexing.test.ts b/server/test/addressIndexing.test.ts index a2c8e99..abbaf6a 100644 --- a/server/test/addressIndexing.test.ts +++ b/server/test/addressIndexing.test.ts @@ -5,6 +5,7 @@ import { collectInvolvedAddresses, mergeAddressDeltas, normalizeSat, + type AddressDelta, } from '../src/services/sync/addressIndexing'; describe('address indexing domain', () => { @@ -32,34 +33,45 @@ describe('address indexing domain', () => { expect(collectInvolvedAddresses(vins, vouts)).toEqual(['Dsender', 'Dreceiver']); expect(collectAddressDeltas(vins, vouts, false)).toEqual( new Map([ - ['Dreceiver', { receivedSat: 7n, sentSat: 0n }], - ['Dsender', { receivedSat: 4n, sentSat: 7n }], + ['Dreceiver', { receivedSat: 7n, sentSat: 0n, txCount: 1 }], + ['Dsender', { receivedSat: 4n, sentSat: 7n, txCount: 1 }], ]) ); }); it('merges per-transaction deltas for block-level bulk writes', () => { - const target = new Map([['Daddr', { receivedSat: 2n, sentSat: 1n }]]); + const target = new Map([['Daddr', { receivedSat: 2n, sentSat: 1n, txCount: 1 }]]); mergeAddressDeltas( target, new Map([ - ['Daddr', { receivedSat: 3n, sentSat: 4n }], - ['Dnew', { receivedSat: 5n, sentSat: 0n }], + ['Daddr', { receivedSat: 3n, sentSat: 4n, txCount: 1 }], + ['Dnew', { receivedSat: 5n, sentSat: 0n, txCount: 1 }], ]) ); expect(target).toEqual( new Map([ - ['Daddr', { receivedSat: 5n, sentSat: 5n }], - ['Dnew', { receivedSat: 5n, sentSat: 0n }], + ['Daddr', { receivedSat: 5n, sentSat: 5n, txCount: 2 }], + ['Dnew', { receivedSat: 5n, sentSat: 0n, txCount: 1 }], ]) ); }); + it('counts an address once for each transaction of a block it takes part in', () => { + const block = new Map(); + const pay = (address: string, valueSat: string) => ({ valueSat, scriptPubKey: { addresses: [address] } }); + // Two transactions of one block, both spending from and paying back to Dself. + mergeAddressDeltas(block, collectAddressDeltas([{ txid: 'p1', address: 'Dself', valueSat: '10' }], [pay('Dother', '4'), pay('Dself', '6')], false)); + mergeAddressDeltas(block, collectAddressDeltas([{ txid: 'p2', address: 'Dself', valueSat: '6' }], [pay('Dother', '1'), pay('Dself', '5')], false)); + + expect(block.get('Dself')).toEqual({ receivedSat: 11n, sentSat: 16n, txCount: 2 }); + expect(block.get('Dother')).toEqual({ receivedSat: 5n, sentSat: 0n, txCount: 2 }); + }); + it('builds the atomic MongoDB update pipeline used by live sync and rebuilds', () => { const pipeline = buildAddressDeltaPipeline( 'Dabc', - { receivedSat: 11n, sentSat: 3n }, + { receivedSat: 11n, sentSat: 3n, txCount: 2 }, 1700000000 ); const setStage = (pipeline[0] as { $set?: Record }).$set; @@ -67,6 +79,7 @@ describe('address indexing domain', () => { expect(pipeline).toHaveLength(1); expect(setStage).toBeDefined(); expect((setStage?.address as { $ifNull: [string, string] }).$ifNull[1]).toBe('Dabc'); - expect((setStage?.txCount as { $add: [number, number] }).$add[1]).toBe(1); + // The block's transactions with this address, not one per block. + expect((setStage?.txCount as { $add: [number, number] }).$add[1]).toBe(2); }); }); diff --git a/server/test/addressStats.integration.test.ts b/server/test/addressStats.integration.test.ts new file mode 100644 index 0000000..7f6ac96 --- /dev/null +++ b/server/test/addressStats.integration.test.ts @@ -0,0 +1,196 @@ +import mongoose from 'mongoose'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { syncService } from '../src/services/sync.service'; +import { rpcService } from '../src/services/rpc.service'; +import { realtimeService } from '../src/services/realtime.service'; +import { telegramService } from '../src/services/telegram.service'; +import { Address } from '../src/models/Address'; +import { Block } from '../src/models/Block'; +import { SyncState } from '../src/models/SyncState'; +import { Transaction } from '../src/models/Transaction'; + +// This suite requires an explicitly supplied disposable localhost database. +// The normal unit suite must never connect to an operator's configured MongoDB. +const baseUri = process.env.TEST_POSE_MONGO_URI; + +const A = 'DNSSwp3CWfyus5h8TSBvJHFoTUASQCF8Rj'; +const B = 'D8e1nUDayNHvXSYZPv1QECNcSgCxgsXw9d'; +const C = 'DGK3yNxxedc2pvaPPwTbVBey2NmFC8FUGZ'; +const D = 'DKgo69ycaxuut3QxaTSfyqbkFvyRvvq8Hf'; +const E = 'D9NofFTND226dT475V33QaHU9LKHK7UwVe'; +const COIN = 100_000_000; + +type RpcVin = { coinbase?: string; txid?: string; vout?: number; sequence: number; address?: string; value?: number; valueSat?: number }; +type RpcVout = { value: number; valueSat: number; n: number; scriptPubKey: { asm: string; hex: string; type: string; addresses: string[] } }; +type RpcTx = { txid: string; version: number; size: number; locktime: number; vin: RpcVin[]; vout: RpcVout[] }; + +const hashOf = (height: number) => height.toString(16).padStart(64, 'e'); +const txidOf = (label: string) => label.padStart(64, '0'); +const pay = (n: number, coins: number, address: string): RpcVout => ({ + value: coins, valueSat: Math.round(coins * COIN), n, + scriptPubKey: { asm: '', hex: '76a914' + '00'.repeat(20) + '88ac', type: 'pubkeyhash', addresses: [address] }, +}); +const spend = (prev: string, n: number, coins: number, address: string): RpcVin => ({ + txid: txidOf(prev), vout: n, sequence: 0, address, value: coins, valueSat: Math.round(coins * COIN), +}); +const coinbase = (label: string, coins: number, address: string): RpcTx => ({ + txid: txidOf(label), version: 3, size: 100, locktime: 0, vin: [{ coinbase: '03aa0d02', sequence: 0 }], vout: [pay(0, coins, address)], +}); +const transfer = (label: string, vin: RpcVin[], vout: RpcVout[]): RpcTx => ({ txid: txidOf(label), version: 3, size: 225, locktime: 0, vin, vout }); + +// In block 2, A spends twice and D is paid twice: each takes part in two transactions of one block. +const TXS: RpcTx[][] = [ + [coinbase('c0', 1000, A)], + [coinbase('c1', 1000, B)], + [ + coinbase('c2', 50, C), + transfer('x2', [spend('c0', 0, 1000, A)], [pay(0, 400, D), pay(1, 599.99, A)]), + transfer('y2', [spend('x2', 1, 599.99, A)], [pay(0, 100, D), pay(1, 499.98, A)]), + ], + [ + coinbase('c3', 50, C), + transfer('z3', [spend('x2', 0, 400, D)], [pay(0, 399.99, E)]), + ], +]; +const BLOCKS = TXS.map((tx, height) => ({ + hash: hashOf(height), confirmations: 1, size: 500, height, version: 1, merkleroot: 'm'.repeat(64), tx, + time: 1_800_000_000 + height * 150, mediantime: 1_800_000_000 + height * 150, nonce: 0, bits: '1d00ffff', + difficulty: 1, chainwork: '00', nTx: tx.length, + ...(height > 0 ? { previousblockhash: hashOf(height - 1) } : {}), +})); + +/** How many transactions each address takes part in, from the fixture alone. */ +const EXPECTED_TX_COUNT = { [A]: 3, [B]: 1, [C]: 2, [D]: 3, [E]: 1 }; + +function mockDaemon(tip: number) { + vi.spyOn(rpcService, 'getBlockCount').mockResolvedValue(tip); + vi.spyOn(rpcService, 'getBlockHash').mockImplementation(async (height: number) => hashOf(height)); + vi.spyOn(rpcService, 'getBlock').mockImplementation(async (hash: string) => { + const block = BLOCKS.find((candidate) => candidate.hash === hash); + if (!block) throw new Error(`unknown block ${hash}`); + return structuredClone(block); + }); +} + +type StoredAddress = { address: string; balanceSat: unknown; totalReceivedSat: unknown; totalSentSat: unknown; txCount: number; firstSeen: number; lastSeen: number }; +async function storedAddresses(fields: Array) { + const docs = await Address.find({}).lean(); + return Object.fromEntries(docs.sort((a, b) => a.address.localeCompare(b.address)).map((doc) => + [doc.address, Object.fromEntries(fields.map((field) => [field, String(doc[field])]))])); +} +const ALL_FIELDS: Array = ['balanceSat', 'totalReceivedSat', 'totalSentSat', 'txCount', 'firstSeen', 'lastSeen']; + +const internals = syncService as unknown as { + stopRequested: boolean; + ADDRESS_STATS_VERSION: number; + rebuildAddressIndexFromTransactions: () => Promise; +}; +const syncState = () => SyncState.findOne({ key: 'main' }).lean>(); +const freshState = (extra: Record = {}) => + SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3, ...extra }); + +describe.skipIf(!baseUri)('address statistics from live sync and from a rebuild, with disposable MongoDB', () => { + beforeAll(async () => { + if (!baseUri || !/^mongodb:\/\/127\.0\.0\.1:\d+\/deftrack_pose_test_[a-f0-9]+$/.test(baseUri)) { + throw new Error('TEST_POSE_MONGO_URI must name an isolated localhost test database'); + } + // Own database, so it cannot collide with other integration suites that may run at the same time. + await mongoose.connect(`${baseUri}as`, { serverSelectionTimeoutMS: 5000 }); + await Promise.all([Block.init(), Transaction.init(), Address.init(), SyncState.init()]); + }); + beforeEach(async () => { + internals.stopRequested = false; + vi.spyOn(realtimeService, 'publishNewBlock').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishNewTx').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishSyncStatus').mockImplementation(() => undefined); + vi.spyOn(telegramService, 'notifyNewBlock').mockImplementation(() => undefined); + await Promise.all([Block.deleteMany({}), Transaction.deleteMany({}), Address.deleteMany({}), SyncState.deleteMany({})]); + }); + afterAll(async () => { + internals.stopRequested = false; + await syncService.stop(); + if (mongoose.connection.readyState === 1) await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + it('live sync and a rebuild give the same balances (as before this change)', async () => { + mockDaemon(3); + await freshState(); + await syncService.sync(); + const live = await storedAddresses(['balanceSat', 'totalReceivedSat', 'totalSentSat']); + + await internals.rebuildAddressIndexFromTransactions(); + + expect(await storedAddresses(['balanceSat', 'totalReceivedSat', 'totalSentSat'])).toEqual(live); + }); + + it('live sync and a rebuild give the same value in every address field, counting each transaction once', async () => { + mockDaemon(3); + await freshState(); + await syncService.sync(); + const live = await storedAddresses(ALL_FIELDS); + + await internals.rebuildAddressIndexFromTransactions(); + + expect(await storedAddresses(ALL_FIELDS)).toEqual(live); + const counts = Object.fromEntries(Object.entries(live).map(([address, fields]) => [address, Number(fields.txCount)])); + expect(counts).toEqual(EXPECTED_TX_COUNT); + }); + + it('an older address statistics version rebuilds the address index once at start, and only once', async () => { + mockDaemon(3); + await freshState(); + await syncService.sync(); + // Counts as an older release left them: one per block instead of one per transaction. + await Address.updateMany({}, { $set: { txCount: 1 } }); + await SyncState.updateOne({ key: 'main' }, { $unset: { addressStatsVersion: 1 } }); + const rebuild = vi.spyOn(internals, 'rebuildAddressIndexFromTransactions'); + + try { + await syncService.start(); + expect(rebuild).toHaveBeenCalledTimes(1); + expect(await syncState()).toMatchObject({ addressStatsVersion: internals.ADDRESS_STATS_VERSION }); + const counts = Object.fromEntries(Object.entries(await storedAddresses(['txCount'])).map(([address, fields]) => [address, Number(fields.txCount)])); + expect(counts).toEqual(EXPECTED_TX_COUNT); + + // The next start finds the current version and does not rebuild again. + await syncService.stop(); + await syncService.start(); + expect(rebuild).toHaveBeenCalledTimes(1); + } finally { + await syncService.stop(); + } + }); + + it('an empty database starts at the current version without a rebuild', async () => { + mockDaemon(0); + // An index version change on an empty database resets the state (ensureIndexVersion). + await freshState({ addressIndexVersion: 0 }); + const rebuild = vi.spyOn(internals, 'rebuildAddressIndexFromTransactions'); + + try { + await syncService.start(); + expect(rebuild).not.toHaveBeenCalled(); + expect(await syncState()).toMatchObject({ addressStatsVersion: internals.ADDRESS_STATS_VERSION, lastSyncedHeight: 0 }); + } finally { + await syncService.stop(); + } + }); + + it('a start that already rebuilds for another reason settles the version with that one rebuild', async () => { + mockDaemon(3); + await freshState(); + await syncService.sync(); + // A lock left behind by a killed process: startup recovery rebuilds anyway. + await SyncState.updateOne({ key: 'main' }, { $set: { isRunning: true }, $unset: { addressStatsVersion: 1 } }); + const rebuild = vi.spyOn(internals, 'rebuildAddressIndexFromTransactions'); + + try { + await syncService.start(); + expect(rebuild).toHaveBeenCalledTimes(1); + expect(await syncState()).toMatchObject({ addressStatsVersion: internals.ADDRESS_STATS_VERSION, isRunning: false }); + } finally { + await syncService.stop(); + } + }); +});