From 307b293d9ce80ecd2522983c3115f884ca3a105e Mon Sep 17 00:00:00 2001 From: minuszka <101508242+minuszka@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:36:54 +0100 Subject: [PATCH 1/4] Keep address balances correct when indexing a block is interrupted If indexing a block stopped halfway, for example on a database error, the transactions already stored were treated as done when the block was indexed again, and their balance changes were silently lost. - The indexer now marks a block as in progress before its first write and clears the mark after its last. If anything fails in between, the next run first rebuilds the address balances from the stored transactions. - A normal shutdown now waits up to 15 seconds for the block being indexed and stops between blocks, so a restart during a deploy no longer leaves half a block behind. If the block does not finish in time, startup recovery rebuilds as before. - Each new block costs two small extra writes to the sync state. - Checked with database tests that interrupt indexing at each step (a transaction write, the balance write, the block write, a shutdown) and compare every address balance with the expected one; each part of the fix was undone once to confirm that a test fails without it. --- server/src/services/sync.service.ts | 118 +++++-- .../test/syncInterruption.integration.test.ts | 292 ++++++++++++++++++ 2 files changed, 383 insertions(+), 27 deletions(-) create mode 100644 server/test/syncInterruption.integration.test.ts diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index 586368c..95f652e 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -59,6 +59,11 @@ interface ProcessedBlockSummary { class SyncService { private isRunning = false; private intervalId: NodeJS.Timeout | null = null; + // Set by stop(): the running sync ends after its current block, and no new run starts. + private stopRequested = false; + private currentSync: Promise | null = null; + // How long stop() waits for the current block before it leaves the lock to startup recovery. + private stopTimeoutMs = 15_000; private readonly LOG_INTERVAL = 100; private readonly REBUILD_LOG_INTERVAL = 10000; private readonly ADDRESS_BULK_FLUSH_THRESHOLD = 2000; @@ -133,6 +138,21 @@ class SyncService { ); } + /** + * Sets addressRebuildRequired before a block's first write (see processBlock) and says whether it + * was already set, so that the block does not clear a rebuild requested for another reason. + */ + private async markBlockInProgress(): Promise { + const before = await SyncState.findOneAndUpdate( + { key: 'main' }, + { addressRebuildRequired: true, heartbeatAt: new Date() }, + { upsert: true, new: false } + ) + .select({ addressRebuildRequired: 1 }) + .lean<{ addressRebuildRequired?: boolean }>(); + return before?.addressRebuildRequired === true; + } + private async recoverStartupState(): Promise { const syncState = await SyncState.findOne({ key: 'main' }).lean(); const indexedTip = await Block.findOne({}) @@ -581,6 +601,7 @@ class SyncService { } logger.info('Starting blockchain sync service...'); + this.stopRequested = false; await this.ensureIndexVersion(); await this.ensureSatoshiDataVersion(); @@ -607,7 +628,27 @@ class SyncService { clearInterval(this.intervalId); this.intervalId = null; } - this.isRunning = false; + this.stopRequested = true; + + // A running sync stops between blocks and releases the lock itself. If the block in progress + // does not finish in time, the lock stays: the next start sees it and rebuilds the address index. + const running = this.currentSync; + if (running) { + let timer: NodeJS.Timeout | undefined; + const finished = await Promise.race([ + running.then(() => true, () => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), this.stopTimeoutMs); + }), + ]); + clearTimeout(timer); + if (!finished) { + logger.warn(`Sync did not stop within ${this.stopTimeoutMs} ms; leaving its lock for startup recovery.`); + return; + } + logger.info('Sync service stopped'); + return; + } await SyncState.findOneAndUpdate( { key: 'main' }, @@ -623,10 +664,20 @@ class SyncService { } async sync(): Promise { - if (this.isRunning) { + if (this.isRunning || this.currentSync || this.stopRequested) { return; } + const run = this.runSync(); + this.currentSync = run; + try { + await run; + } finally { + if (this.currentSync === run) this.currentSync = null; + } + } + + private async runSync(): Promise { const syncStartedAt = new Date(); // Atomic DB-level lock: only acquire if not already running. @@ -719,6 +770,8 @@ class SyncService { let lastCheckpointAt = Date.now(); while (currentHeight <= daemonHeight) { + // A requested stop takes effect between blocks, never in the middle of one. + if (this.stopRequested) break; const processedBlock = await this.processBlock(currentHeight); lastHash = processedBlock.hash; lastKnownSyncedHeight = currentHeight; @@ -773,11 +826,12 @@ class SyncService { } } - // Final update + // Final update: the last block indexed, which is below the daemon's tip after a stop. + const reachedHeight = currentHeight - 1; await SyncState.findOneAndUpdate( { key: 'main' }, { - lastSyncedHeight: daemonHeight, + lastSyncedHeight: reachedHeight, lastSyncedHash: lastHash, lastSyncedAt: new Date(), isRunning: false, @@ -787,7 +841,12 @@ class SyncService { }, { upsert: true } ); - lastKnownSyncedHeight = daemonHeight; + lastKnownSyncedHeight = reachedHeight; + if (reachedHeight < daemonHeight) { + this.publishSyncStatus('idle', daemonHeight, reachedHeight, { message: 'Sync stopped.' }); + logger.info(`Sync stopped at height ${reachedHeight}.`); + return; + } this.publishSyncStatus('complete', daemonHeight, daemonHeight, { message: 'Sync complete.', }); @@ -864,6 +923,14 @@ class SyncService { rpcBlock = (await rpcService.getBlock(hash, 1)) as RpcBlock; } + // From here on the block is written piece by piece: each transaction (marked as applied), then + // the block's balance changes, then the block. Mark the block as in progress before the first + // write and clear the mark after the last. If anything in between fails, or the process stops, + // the mark stays and the next run rebuilds the address index from the stored transactions; the + // per-transaction marks then apply exactly what is still missing. If the mark cannot be + // written, nothing is. + const rebuildPendingBefore = await this.markBlockInProgress(); + const mintSat = coinToSat(typeof rpcBlock.mint === 'number' ? rpcBlock.mint : 0); let rewardSat = mintSat; let totalValueOutSat = 0n; @@ -894,7 +961,7 @@ class SyncService { rpcTx = entry; } - const result = await this.processTransaction(rpcTx, rpcBlock, { deferAddressUpdates: true }); + const result = await this.processTransaction(rpcTx, rpcBlock); if (result.addressDeltas) { mergeAddressDeltas(blockAddressDeltas, result.addressDeltas); } @@ -973,6 +1040,14 @@ class SyncService { ); } + // Everything of this block is written. A rebuild that was already pending stays pending. + if (!rebuildPendingBefore) { + await SyncState.findOneAndUpdate( + { key: 'main' }, + { addressRebuildRequired: false, heartbeatAt: new Date() } + ); + } + return { hash: rpcBlock.hash, height: rpcBlock.height, @@ -984,10 +1059,13 @@ class SyncService { // ── Transaction Processing ────────────────────────────────────── + /** + * Stores one transaction and returns its balance changes, if they have not been applied before. + * The caller (processBlock) merges them and writes them once for the whole block. + */ private async processTransaction( rpcTx: RpcTransaction, - rpcBlock: RpcBlock, - opts?: { deferAddressUpdates?: boolean } + rpcBlock: RpcBlock ): Promise<{ totalValueOutSat: bigint; isCoinbase: boolean; @@ -1131,25 +1209,11 @@ class SyncService { canApplyAddressDeltasForTx && previousTx?.addressUpdatesApplied !== true; if (shouldApplyAddressDeltas) { const addressDeltas = collectAddressDeltas(vins, vouts, isCoinbase); - - if (opts?.deferAddressUpdates) { - return { - totalValueOutSat, - isCoinbase, - addressDeltas: addressDeltas.size > 0 ? addressDeltas : undefined, - }; - } - - if (addressDeltas.size > 0) { - try { - await this.applyAddressDeltas(addressDeltas, rpcBlock.time); - } catch (error) { - await this.markAddressRebuildRequired( - `Address update failed for tx ${rpcTx.txid}; forcing deterministic rebuild.` - ); - throw error; - } - } + return { + totalValueOutSat, + isCoinbase, + addressDeltas: addressDeltas.size > 0 ? addressDeltas : undefined, + }; } return { totalValueOutSat, isCoinbase }; diff --git a/server/test/syncInterruption.integration.test.ts b/server/test/syncInterruption.integration.test.ts new file mode 100644 index 0000000..d2343a9 --- /dev/null +++ b/server/test/syncInterruption.integration.test.ts @@ -0,0 +1,292 @@ +import mongoose from 'mongoose'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { syncService } from '../src/services/sync.service'; +import { rpcService } from '../src/services/rpc.service'; +import { realtimeService } from '../src/services/realtime.service'; +import { telegramService } from '../src/services/telegram.service'; +import { Address } from '../src/models/Address'; +import { Block } from '../src/models/Block'; +import { SyncState } from '../src/models/SyncState'; +import { Transaction } from '../src/models/Transaction'; + +// This suite requires an explicitly supplied disposable localhost database. +// The normal unit suite must never connect to an operator's configured MongoDB. +const baseUri = process.env.TEST_POSE_MONGO_URI; + +const A = 'DNSSwp3CWfyus5h8TSBvJHFoTUASQCF8Rj'; +const B = 'D8e1nUDayNHvXSYZPv1QECNcSgCxgsXw9d'; +const C = 'DGK3yNxxedc2pvaPPwTbVBey2NmFC8FUGZ'; +const D = 'DKgo69ycaxuut3QxaTSfyqbkFvyRvvq8Hf'; +const E = 'D9NofFTND226dT475V33QaHU9LKHK7UwVe'; +const COIN = 100_000_000; + +type RpcVin = { coinbase?: string; txid?: string; vout?: number; sequence: number; address?: string; value?: number; valueSat?: number }; +type RpcVout = { value: number; valueSat: number; n: number; scriptPubKey: { asm: string; hex: string; type: string; addresses: string[] } }; +type RpcTx = { txid: string; version: number; size: number; locktime: number; vin: RpcVin[]; vout: RpcVout[] }; +type RpcBlock = { hash: string; height: number; time: number; tx: RpcTx[]; previousblockhash?: string; nextblockhash?: string } & Record; + +const hashOf = (height: number) => height.toString(16).padStart(64, 'b'); +const txidOf = (label: string) => label.padStart(64, '0'); +const pay = (n: number, coins: number, address: string): RpcVout => ({ + value: coins, valueSat: Math.round(coins * COIN), n, + scriptPubKey: { asm: '', hex: '76a914' + '00'.repeat(20) + '88ac', type: 'pubkeyhash', addresses: [address] }, +}); +const spend = (prev: string, n: number, coins: number, address: string): RpcVin => ({ + txid: txidOf(prev), vout: n, sequence: 0, address, value: coins, valueSat: Math.round(coins * COIN), +}); +const coinbase = (label: string, coins: number, address: string): RpcTx => ({ + txid: txidOf(label), version: 3, size: 100, locktime: 0, vin: [{ coinbase: '03aa0d02', sequence: 0 }], vout: [pay(0, coins, address)], +}); +const transfer = (label: string, vin: RpcVin[], vout: RpcVout[]): RpcTx => ({ txid: txidOf(label), version: 3, size: 225, locktime: 0, vin, vout }); + +// A small chain shaped like the daemon's verbose blocks. Block 2 has a coinbase and two transfers. +const TXS: RpcTx[][] = [ + [coinbase('c0', 1000, A)], + [coinbase('c1', 1000, B)], + [ + coinbase('c2', 50, C), + transfer('x2', [spend('c0', 0, 1000, A)], [pay(0, 400, D), pay(1, 599.99, A)]), + transfer('y2', [spend('c1', 0, 1000, B)], [pay(0, 100, D), pay(1, 899.99, B)]), + ], + [ + coinbase('c3', 50, C), + transfer('z3', [spend('x2', 0, 400, D)], [pay(0, 399.99, E)]), + ], +]; +const BLOCKS: RpcBlock[] = TXS.map((tx, height) => ({ + hash: hashOf(height), confirmations: 1, size: 500, height, version: 1, merkleroot: 'm'.repeat(64), tx, + time: 1_800_000_000 + height * 150, mediantime: 1_800_000_000 + height * 150, nonce: 0, bits: '1d00ffff', + difficulty: 1, chainwork: '00', nTx: tx.length, + ...(height > 0 ? { previousblockhash: hashOf(height - 1) } : {}), + ...(height < TXS.length - 1 ? { nextblockhash: hashOf(height + 1) } : {}), +})); +const BLOCK2 = BLOCKS[2]; + +/** What every address should hold after the blocks up to `height`, from the fixture alone. */ +function expectedBalances(height: number) { + const totals = new Map(); + const entry = (address: string) => { + const found = totals.get(address) ?? { received: 0n, sent: 0n }; + totals.set(address, found); + return found; + }; + for (const block of BLOCKS.slice(0, height + 1)) { + for (const tx of block.tx) { + for (const output of tx.vout) for (const address of output.scriptPubKey.addresses) entry(address).received += BigInt(output.valueSat); + for (const input of tx.vin) if (input.address) entry(input.address).sent += BigInt(input.valueSat ?? 0); + } + } + return Object.fromEntries([...totals].sort(([a], [b]) => a.localeCompare(b)).map(([address, { received, sent }]) => + [address, { balance: String(received - sent), received: String(received), sent: String(sent) }])); +} + +async function storedBalances() { + const docs = await Address.find({}).lean>(); + return Object.fromEntries(docs.sort((a, b) => a.address.localeCompare(b.address)).map((doc) => + [doc.address, { balance: String(doc.balanceSat), received: String(doc.totalReceivedSat), sent: String(doc.totalSentSat) }])); +} + +const syncState = () => SyncState.findOne({ key: 'main' }).lean>(); + +/** The daemon, answering from the fixture up to `tip`. `hold` keeps a block's answer back until released. */ +function mockDaemon(tip: number) { + vi.spyOn(rpcService, 'getBlockCount').mockResolvedValue(tip); + vi.spyOn(rpcService, 'getBlockHash').mockImplementation(async (height: number) => hashOf(height)); + return vi.spyOn(rpcService, 'getBlock').mockImplementation(async (hash: string) => { + const block = BLOCKS.find((candidate) => candidate.hash === hash); + if (!block) throw new Error(`unknown block ${hash}`); + return structuredClone(block); + }); +} + +type AnyMethods = Record unknown>; + +/** Makes the first call of `method` that `matches` its arguments throw; every other call goes through. */ +function failOnce(model: object, method: string, matches: (...args: unknown[]) => boolean) { + const target = model as AnyMethods; + const original = target[method].bind(model); + const state = { failed: false }; + vi.spyOn(target, method).mockImplementation((...args: unknown[]) => { + if (!state.failed && matches(...args)) { + state.failed = true; + throw new Error(`injected failure in ${method}`); + } + return original(...args); + }); + return state; +} + +/** Holds the first call of `method` that `matches` until `release()`; every other call goes through. */ +function holdOnce(model: object, method: string, matches: (...args: unknown[]) => boolean) { + const target = model as AnyMethods; + const original = target[method].bind(model); + let release = () => {}; + const gate = new Promise((resolve) => { release = resolve; }); + const state = { reached: false, release: () => release() }; + vi.spyOn(target, method).mockImplementation((...args: unknown[]) => { + if (!state.reached && matches(...args)) { + state.reached = true; + // A thenable query stand-in: the real query runs once the gate opens. + const run = gate.then(() => original(...args) as Promise); + const chain = { select: () => chain, lean: () => chain, then: run.then.bind(run), catch: run.catch.bind(run) }; + return chain; + } + return original(...args); + }); + return state; +} + +const internals = syncService as unknown as { stopRequested: boolean; stopTimeoutMs: number }; + +describe.skipIf(!baseUri)('indexing a block that is interrupted, with disposable MongoDB', () => { + beforeAll(async () => { + if (!baseUri || !/^mongodb:\/\/127\.0\.0\.1:\d+\/deftrack_pose_test_[a-f0-9]+$/.test(baseUri)) { + throw new Error('TEST_POSE_MONGO_URI must name an isolated localhost test database'); + } + // Own database, so it cannot collide with other integration suites that may run at the same time. + await mongoose.connect(`${baseUri}si`, { serverSelectionTimeoutMS: 5000 }); + await Promise.all([Block.init(), Transaction.init(), Address.init(), SyncState.init()]); + }); + beforeEach(async () => { + internals.stopRequested = false; + vi.spyOn(realtimeService, 'publishNewBlock').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishNewTx').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishSyncStatus').mockImplementation(() => undefined); + vi.spyOn(telegramService, 'notifyNewBlock').mockImplementation(() => undefined); + await Promise.all([Block.deleteMany({}), Transaction.deleteMany({}), Address.deleteMany({}), SyncState.deleteMany({})]); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + }); + afterAll(async () => { + internals.stopRequested = false; + if (mongoose.connection.readyState === 1) await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + it('the fixture chain indexes to the expected balances without failures', async () => { + mockDaemon(3); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 3, isRunning: false, addressRebuildRequired: false, error: null }); + }); + + it('T1: a transaction write that fails halfway through a block loses no balance change', async () => { + mockDaemon(3); + const failure = failOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('x2')); + + await syncService.sync(); + expect(failure.failed).toBe(true); + expect(await syncState()).toMatchObject({ isRunning: false, error: 'injected failure in findOneAndUpdate' }); + + await syncService.sync(); + expect(await storedBalances()).toEqual(expectedBalances(3)); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 3, addressRebuildRequired: false, error: null }); + }); + + it('T2: a failed balance write counts nothing twice', async () => { + mockDaemon(3); + const failure = failOnce(Address, 'bulkWrite', (ops) => + Array.isArray(ops) && ops.some((op) => (op as { updateOne?: { filter?: { address?: string } } }).updateOne?.filter?.address === D)); + + await syncService.sync(); + expect(failure.failed).toBe(true); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); + + it('T3: a failed block write counts nothing twice', async () => { + mockDaemon(3); + const failure = failOnce(Block, 'findOneAndUpdate', (filter, update) => + (filter as { hash?: string })?.hash === BLOCK2.hash && (update as { height?: number })?.height === 2); + + await syncService.sync(); + expect(failure.failed).toBe(true); + expect(await Block.findOne({ height: 2 }).lean()).toBeNull(); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); + + it('T4: stop() waits for the block being indexed, then stops before the next one', async () => { + mockDaemon(3); + const held = holdOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('y2')); + + const running = syncService.sync(); + let stopped = false; + try { + await vi.waitFor(() => expect(held.reached).toBe(true)); + const stopping = syncService.stop().then(() => { stopped = true; }); + await new Promise((resolve) => setTimeout(resolve, 300)); + // Still waiting: block 2 is half written. + expect(stopped).toBe(false); + held.release(); + await stopping; + } finally { + held.release(); + await running; + } + + expect(await Block.findOne({ height: 2 }).lean()).not.toBeNull(); + expect(await Block.findOne({ height: 3 }).lean()).toBeNull(); + expect(await syncState()).toMatchObject({ + isRunning: false, addressRebuildRequired: false, lastSyncedHeight: 2, lastSyncedHash: BLOCK2.hash, + }); + expect(await storedBalances()).toEqual(expectedBalances(2)); + // A stopped service does not start another run: it neither asks the daemon nor writes the state. + const daemonCalls = vi.mocked(rpcService.getBlockCount).mock.calls.length; + const stateBefore = await syncState(); + await syncService.sync(); + expect(vi.mocked(rpcService.getBlockCount).mock.calls.length).toBe(daemonCalls); + expect(await syncState()).toEqual(stateBefore); + expect(await Block.findOne({ height: 3 }).lean()).toBeNull(); + }); + + it('T5: when the block does not finish in time, stop() leaves the lock for startup recovery', async () => { + mockDaemon(3); + const held = holdOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('y2')); + const timeoutBefore = internals.stopTimeoutMs; + internals.stopTimeoutMs = 200; + + const running = syncService.sync(); + await vi.waitFor(() => expect(held.reached).toBe(true)); + try { + await syncService.stop(); + expect(await syncState()).toMatchObject({ isRunning: true, addressRebuildRequired: true }); + } finally { + internals.stopTimeoutMs = timeoutBefore; + held.release(); + await running; + } + }); + + it('a block does not clear a rebuild that was already requested before it started', async () => { + mockDaemon(0); + await SyncState.updateOne({ key: 'main' }, { addressRebuildRequired: true }); + + await (syncService as unknown as { processBlock: (height: number) => Promise }).processBlock(0); + + expect(await Block.findOne({ height: 0 }).lean()).not.toBeNull(); + expect(await syncState()).toMatchObject({ addressRebuildRequired: true }); + }); + + it('T6: when the in-progress mark cannot be written, the block is not started', async () => { + mockDaemon(1); + await syncService.sync(); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 1, addressRebuildRequired: false }); + + mockDaemon(3); + const failure = failOnce(SyncState, 'findOneAndUpdate', (_filter, update) => { + const fields = update as { addressRebuildRequired?: boolean; $set?: { addressRebuildRequired?: boolean } }; + return fields?.addressRebuildRequired === true || fields?.$set?.addressRebuildRequired === true; + }); + await syncService.sync(); + + expect(failure.failed).toBe(true); + expect(await Transaction.countDocuments({ blockheight: 2 })).toBe(0); + expect(await syncState()).toMatchObject({ isRunning: false, error: 'injected failure in findOneAndUpdate' }); + + await syncService.sync(); + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); +}); From 5fc459d051e754e7f6a812c23c65cf85b79a66d5 Mon Sep 17 00:00:00 2001 From: minuszka <101508242+minuszka@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:13:47 +0100 Subject: [PATCH 2/4] Let indexing recover by itself after a database hiccup If the database failed at the wrong moment, the indexer could stay blocked until the next restart, and a database reconnect started a second copy of the indexer and of the Telegram bot. - The indexing lock now expires: if its holder has shown no sign of life for 10 minutes (each indexed block and each step of an address rebuild counts), the next run takes it over and first rebuilds the address balances. Before, a lock left behind by a failed run made every later run fail until a restart. - When recording an indexing error fails as well, the original error is still logged and the run ends normally. - Starting the indexer or the Telegram bot a second time, as happens after a database reconnect, no longer starts a second copy. - Checked with database tests for an expired lock, a recent lock, a failed error record, a long rebuild and repeated starts; each part of the fix was undone once to confirm that a test fails without it. --- server/src/services/sync.service.ts | 108 ++++++--- server/src/services/telegram.service.ts | 6 + server/test/syncLock.integration.test.ts | 283 +++++++++++++++++++++++ server/test/telegram.service.test.ts | 65 ++++++ 4 files changed, 431 insertions(+), 31 deletions(-) create mode 100644 server/test/syncLock.integration.test.ts create mode 100644 server/test/telegram.service.test.ts diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index 95f652e..b34f738 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -64,6 +64,11 @@ class SyncService { private currentSync: Promise | null = null; // How long stop() waits for the current block before it leaves the lock to startup recovery. private stopTimeoutMs = 15_000; + // Set by start() until stop(): a second start() (for example after a database reconnect) does nothing. + private started = false; + // A lock whose holder has shown no heartbeat for this long may be taken over. Each indexed block + // and each batch of an address rebuild writes a heartbeat, so this is far above any normal gap. + private readonly SYNC_LOCK_EXPIRY_MS = 10 * 60 * 1000; private readonly LOG_INTERVAL = 100; private readonly REBUILD_LOG_INTERVAL = 10000; private readonly ADDRESS_BULK_FLUSH_THRESHOLD = 2000; @@ -254,6 +259,8 @@ class SyncService { if (bulkOps.length === 0) return; await Address.bulkWrite(bulkOps, { ordered: false }); bulkOps.length = 0; + // A long rebuild keeps its sync lock from looking abandoned (see acquireSyncLock). + await SyncState.updateOne({ key: 'main' }, { heartbeatAt: new Date() }); }; for await (const tx of cursor) { @@ -595,21 +602,31 @@ class SyncService { } async start(): Promise { - if (this.isRunning) { + if (this.started || this.isRunning) { logger.warn('Sync service is already running'); return; } + this.started = true; logger.info('Starting blockchain sync service...'); this.stopRequested = false; - await this.ensureIndexVersion(); - await this.ensureSatoshiDataVersion(); - await this.recoverStartupState(); - await this.addressResolver.preInferVersionByte(); + try { + await this.ensureIndexVersion(); + await this.ensureSatoshiDataVersion(); + await this.recoverStartupState(); + await this.addressResolver.preInferVersionByte(); + + // Run initial sync + await this.sync(); + } catch (error) { + // A failed start can be tried again, for example on the next database reconnect. + this.started = false; + throw error; + } - // Run initial sync - await this.sync(); + // Stopped while starting: no periodic sync. + if (this.stopRequested) return; // Set up periodic sync for new blocks. // Guard: skip if previous sync is still running to prevent overlap. @@ -629,6 +646,7 @@ class SyncService { this.intervalId = null; } this.stopRequested = true; + this.started = false; // A running sync stops between blocks and releases the lock itself. If the block in progress // does not finish in time, the lock stays: the next start sees it and rebuilds the address index. @@ -677,23 +695,46 @@ class SyncService { } } - private async runSync(): Promise { - const syncStartedAt = new Date(); + /** + * Atomic database lock, so that only one run indexes at a time. It is taken when free, or when + * its holder has shown no heartbeat for SYNC_LOCK_EXPIRY_MS (or never did): a run that died + * without releasing it, for example because the database error that ended it also stopped its + * final write. Taking over an expired lock requests an address rebuild, as startup recovery does + * for a lock left behind, since the holder may have stopped in the middle of a block. + */ + private async acquireSyncLock(now: Date): Promise { + // The state document normally exists already (ensureIndexVersion writes it). Create it if it + // does not, without touching an existing one, so the lock updates below need no upsert: an + // upsert against a held lock would try to insert a second document with the same key. + await SyncState.updateOne({ key: 'main' }, { $setOnInsert: { key: 'main' } }, { upsert: true }); + + const lock = { isRunning: true, startedAt: now, heartbeatAt: now, error: null }; + const free = await SyncState.findOneAndUpdate({ key: 'main', isRunning: { $ne: true } }, lock, { new: true }) + .select({ _id: 1 }) + .lean(); + if (free) return true; - // Atomic DB-level lock: only acquire if not already running. - // This prevents race conditions when multiple instances or overlapping calls exist. - const lockAcquired = await SyncState.findOneAndUpdate( - { key: 'main', isRunning: { $ne: true } }, - { - isRunning: true, - startedAt: syncStartedAt, - heartbeatAt: syncStartedAt, - error: null, - }, - { upsert: true, new: true } + const expiredBefore = new Date(now.getTime() - this.SYNC_LOCK_EXPIRY_MS); + const takenOver = await SyncState.findOneAndUpdate( + { key: 'main', isRunning: true, $or: [{ heartbeatAt: { $lt: expiredBefore } }, { heartbeatAt: null }] }, + { ...lock, addressRebuildRequired: true }, + { new: true } + ) + .select({ _id: 1 }) + .lean(); + if (!takenOver) return false; + + logger.warn( + `Took over a sync lock without a heartbeat for ${this.SYNC_LOCK_EXPIRY_MS / 60_000} minutes; ` + + 'rebuilding the address index first.' ); + return true; + } + + private async runSync(): Promise { + const syncStartedAt = new Date(); - if (!lockAcquired || lockAcquired.isRunning !== true) { + if (!(await this.acquireSyncLock(syncStartedAt))) { logger.debug('Sync lock not acquired — another instance is running'); return; } @@ -855,16 +896,21 @@ class SyncService { } catch (error) { logger.error('Sync error:', error); const errorMessage = error instanceof Error ? error.message : 'Unknown sync error'; - await SyncState.findOneAndUpdate( - { key: 'main' }, - { - isRunning: false, - startedAt: null, - heartbeatAt: new Date(), - error: errorMessage, - }, - { upsert: true } - ); + try { + await SyncState.findOneAndUpdate( + { key: 'main' }, + { + isRunning: false, + startedAt: null, + heartbeatAt: new Date(), + error: errorMessage, + }, + { upsert: true } + ); + } catch (stateError) { + // Often the same database trouble. The lock stays held until it expires (acquireSyncLock). + logger.error('Could not record the sync error or release the sync lock:', stateError); + } // WebSocket clients are public: broadcast the masked message, not the raw one. this.publishSyncStatus('error', daemonHeight, lastKnownSyncedHeight, { error: mapPublicSyncError(errorMessage, true).error, diff --git a/server/src/services/telegram.service.ts b/server/src/services/telegram.service.ts index 2617238..5cf2386 100644 --- a/server/src/services/telegram.service.ts +++ b/server/src/services/telegram.service.ts @@ -32,6 +32,10 @@ class TelegramService { return; } + // Started already (start runs again after a database reconnect): a second bot would compete + // for the same updates and send every alert twice. + if (this.bot) return; + this.bot = new Telegraf(token); this.enabled = true; this.registerCommands(); @@ -48,6 +52,8 @@ class TelegramService { stop(): void { this.bot?.stop(); + this.bot = null; + this.enabled = false; if (this.checkTimer) { clearInterval(this.checkTimer); this.checkTimer = null; diff --git a/server/test/syncLock.integration.test.ts b/server/test/syncLock.integration.test.ts new file mode 100644 index 0000000..3124ee0 --- /dev/null +++ b/server/test/syncLock.integration.test.ts @@ -0,0 +1,283 @@ +import mongoose from 'mongoose'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { config } from '../src/config'; +import { syncService } from '../src/services/sync.service'; +import { rpcService } from '../src/services/rpc.service'; +import { realtimeService } from '../src/services/realtime.service'; +import { telegramService } from '../src/services/telegram.service'; +import { Address } from '../src/models/Address'; +import { Block } from '../src/models/Block'; +import { SyncState } from '../src/models/SyncState'; +import { Transaction } from '../src/models/Transaction'; + +// This suite requires an explicitly supplied disposable localhost database. +// The normal unit suite must never connect to an operator's configured MongoDB. +const baseUri = process.env.TEST_POSE_MONGO_URI; + +const A = 'DNSSwp3CWfyus5h8TSBvJHFoTUASQCF8Rj'; +const B = 'D8e1nUDayNHvXSYZPv1QECNcSgCxgsXw9d'; +const STALE = 'D9NofFTND226dT475V33QaHU9LKHK7UwVe'; +const MINUTE = 60_000; + +const hashOf = (height: number) => height.toString(16).padStart(64, 'c'); +const pay = (n: number, coins: number, address: string) => ({ + value: coins, valueSat: coins * 100_000_000, n, + scriptPubKey: { asm: '', hex: '76a914' + '00'.repeat(20) + '88ac', type: 'pubkeyhash', addresses: [address] }, +}); +const coinbase = (height: number, address: string) => ({ + txid: `cb${height}`.padStart(64, '0'), version: 3, size: 100, locktime: 0, + vin: [{ coinbase: '03aa0d02', sequence: 0 }], vout: [pay(0, 1000, address)], +}); +// Three blocks, each a single coinbase: A, B, A. +const BLOCKS = [A, B, A].map((address, height) => ({ + hash: hashOf(height), confirmations: 1, size: 300, height, version: 1, merkleroot: 'm'.repeat(64), + tx: [coinbase(height, address)], time: 1_800_000_000 + height * 150, mediantime: 1_800_000_000 + height * 150, + nonce: 0, bits: '1d00ffff', difficulty: 1, chainwork: '00', nTx: 1, + ...(height > 0 ? { previousblockhash: hashOf(height - 1) } : {}), +})); + +/** + * The daemon, answering from the fixture up to `tip`. `failBlock` makes that block's first two fetches + * fail: the indexer retries a failed verbose fetch once with a lighter one. + */ +function mockDaemon(tip: number, options: { failBlock?: number } = {}) { + let failures = 0; + const count = vi.spyOn(rpcService, 'getBlockCount').mockResolvedValue(tip); + vi.spyOn(rpcService, 'getBlockHash').mockImplementation(async (height: number) => hashOf(height)); + vi.spyOn(rpcService, 'getBlock').mockImplementation(async (hash: string) => { + const block = BLOCKS.find((candidate) => candidate.hash === hash); + if (!block) throw new Error(`unknown block ${hash}`); + if (block.height === options.failBlock && failures < 2) { + failures += 1; + throw new Error('daemon unavailable'); + } + return structuredClone(block); + }); + return count; +} + +const syncState = () => SyncState.findOne({ key: 'main' }).lean>(); +const balanceOf = async (address: string) => { + const doc = await Address.findOne({ address }).lean<{ balanceSat?: unknown }>(); + return doc ? String(doc.balanceSat) : null; +}; +const ago = (ms: number) => new Date(Date.now() - ms); + +const internals = syncService as unknown as { + stopRequested: boolean; + ADDRESS_BULK_FLUSH_THRESHOLD: number; + rebuildAddressIndexFromTransactions: () => Promise; +}; + +describe.skipIf(!baseUri)('the sync lock and the service start, with disposable MongoDB', () => { + beforeAll(async () => { + if (!baseUri || !/^mongodb:\/\/127\.0\.0\.1:\d+\/deftrack_pose_test_[a-f0-9]+$/.test(baseUri)) { + throw new Error('TEST_POSE_MONGO_URI must name an isolated localhost test database'); + } + // Own database, so it cannot collide with other integration suites that may run at the same time. + await mongoose.connect(`${baseUri}sl`, { serverSelectionTimeoutMS: 5000 }); + await Promise.all([Block.init(), Transaction.init(), Address.init(), SyncState.init()]); + }); + beforeEach(async () => { + internals.stopRequested = false; + vi.spyOn(realtimeService, 'publishNewBlock').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishNewTx').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishSyncStatus').mockImplementation(() => undefined); + vi.spyOn(telegramService, 'notifyNewBlock').mockImplementation(() => undefined); + await Promise.all([Block.deleteMany({}), Transaction.deleteMany({}), Address.deleteMany({}), SyncState.deleteMany({})]); + }); + afterAll(async () => { + internals.stopRequested = false; + if (mongoose.connection.readyState === 1) await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + /** A lock held by a run that last showed a heartbeat at `heartbeatAt` (null: never). */ + async function heldLock(heartbeatAt: Date | null) { + const startedAt = ago(30 * MINUTE); + await SyncState.create({ + key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3, + isRunning: true, startedAt, heartbeatAt, + }); + // A balance only a rebuild removes: no stored transaction pays this address. + await Address.create({ address: STALE, balanceSat: '123', totalReceivedSat: '123', totalSentSat: '0', txCount: 1 }); + return startedAt; + } + + it('takes over a lock whose holder has shown no heartbeat for over 10 minutes, and rebuilds first', async () => { + mockDaemon(2); + await heldLock(ago(11 * MINUTE)); + + await expect(syncService.sync()).resolves.toBeUndefined(); + + expect(await balanceOf(STALE)).toBeNull(); + expect(await balanceOf(A)).toBe('200000000000'); + expect(await syncState()).toMatchObject({ isRunning: false, addressRebuildRequired: false, lastSyncedHeight: 2, error: null }); + }); + + it('takes over a held lock that never had a heartbeat', async () => { + mockDaemon(2); + await heldLock(null); + + await expect(syncService.sync()).resolves.toBeUndefined(); + + expect(await balanceOf(STALE)).toBeNull(); + expect(await syncState()).toMatchObject({ isRunning: false, lastSyncedHeight: 2 }); + }); + + it('leaves a lock with a recent heartbeat alone, without an error', async () => { + const daemon = mockDaemon(2); + const startedAt = await heldLock(ago(1 * MINUTE)); + + await expect(syncService.sync()).resolves.toBeUndefined(); + + expect(daemon).not.toHaveBeenCalled(); + expect(await Block.countDocuments()).toBe(0); + expect(await balanceOf(STALE)).toBe('123'); + expect(await syncState()).toMatchObject({ isRunning: true, startedAt }); + }); + + it('when the error cannot be recorded, the run still ends quietly and the lock is taken over once it expires', async () => { + mockDaemon(2, { failBlock: 1 }); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + const original = SyncState.findOneAndUpdate.bind(SyncState); + let recordFailed = false; + vi.spyOn(SyncState, 'findOneAndUpdate').mockImplementation(((filter: unknown, update: { error?: unknown }, ...rest: unknown[]) => { + if (!recordFailed && typeof update?.error === 'string') { + recordFailed = true; + throw new Error('state write failed'); + } + return (original as (...args: unknown[]) => unknown)(filter, update, ...rest); + }) as never); + + await expect(syncService.sync()).resolves.toBeUndefined(); + expect(recordFailed).toBe(true); + // The lock could not be released, so it is still held. + expect(await syncState()).toMatchObject({ isRunning: true }); + + // Not yet expired: the next run leaves it alone. + await expect(syncService.sync()).resolves.toBeUndefined(); + expect(await Block.countDocuments({ height: { $gte: 1 } })).toBe(0); + + // Expired: the next run takes it over and finishes the chain. + await SyncState.updateOne({ key: 'main' }, { heartbeatAt: ago(11 * MINUTE) }); + await expect(syncService.sync()).resolves.toBeUndefined(); + expect(await syncState()).toMatchObject({ isRunning: false, lastSyncedHeight: 2, addressRebuildRequired: false }); + expect(await balanceOf(B)).toBe('100000000000'); + }); + + it('a rebuild keeps the lock alive with a heartbeat while it writes', async () => { + mockDaemon(2); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + await syncService.sync(); + await SyncState.updateOne({ key: 'main' }, { heartbeatAt: ago(11 * MINUTE) }); + const flushBefore = internals.ADDRESS_BULK_FLUSH_THRESHOLD; + internals.ADDRESS_BULK_FLUSH_THRESHOLD = 1; + const beats: Date[] = []; + const record = (update: unknown) => { + const fields = update as { heartbeatAt?: Date; $set?: { heartbeatAt?: Date } }; + const at = fields?.heartbeatAt ?? fields?.$set?.heartbeatAt; + if (at) beats.push(at); + }; + const updateOne = SyncState.updateOne.bind(SyncState); + vi.spyOn(SyncState, 'updateOne').mockImplementation(((filter: unknown, update: unknown, ...rest: unknown[]) => { + record(update); + return (updateOne as (...args: unknown[]) => unknown)(filter, update, ...rest); + }) as never); + const findOneAndUpdate = SyncState.findOneAndUpdate.bind(SyncState); + vi.spyOn(SyncState, 'findOneAndUpdate').mockImplementation(((filter: unknown, update: unknown, ...rest: unknown[]) => { + record(update); + return (findOneAndUpdate as (...args: unknown[]) => unknown)(filter, update, ...rest); + }) as never); + + try { + await internals.rebuildAddressIndexFromTransactions(); + } finally { + internals.ADDRESS_BULK_FLUSH_THRESHOLD = flushBefore; + } + + // Three coinbases, one balance write each: at least one heartbeat per write. + expect(beats.length).toBeGreaterThanOrEqual(3); + const state = await syncState(); + expect(Date.now() - new Date(state?.heartbeatAt as Date).getTime()).toBeLessThan(MINUTE); + }); + + it('a start that failed can be tried again', async () => { + mockDaemon(1); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + const findOne = SyncState.findOne.bind(SyncState); + let failed = false; + vi.spyOn(SyncState, 'findOne').mockImplementation(((...args: unknown[]) => { + if (!failed) { + failed = true; + throw new Error('database not ready'); + } + return (findOne as (...a: unknown[]) => unknown)(...args); + }) as never); + const intervals = vi.spyOn(globalThis, 'setInterval'); + + try { + await expect(syncService.start()).rejects.toThrow('database not ready'); + await syncService.start(); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 1 }); + } finally { + await syncService.stop(); + for (const result of intervals.mock.results) clearInterval(result.value as ReturnType); + } + }); + + it('a service stopped while it is starting sets up no periodic sync', async () => { + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + mockDaemon(1); + let release = () => {}; + const gate = new Promise((resolve) => { release = resolve; }); + let reached = false; + vi.spyOn(rpcService, 'getBlockCount').mockImplementation(async () => { + reached = true; + await gate; + return 1; + }); + const intervals = vi.spyOn(globalThis, 'setInterval'); + const periodic = () => intervals.mock.calls.filter((call) => call[1] === config.sync.intervalMs).length; + + const starting = syncService.start(); + try { + await vi.waitFor(() => expect(reached).toBe(true)); + const stopping = syncService.stop(); + release(); + await stopping; + await starting; + expect(periodic()).toBe(0); + } finally { + release(); + await starting.catch(() => undefined); + await syncService.stop(); + for (const result of intervals.mock.results) clearInterval(result.value as ReturnType); + } + }); + + it('start() twice sets up one periodic sync, and a stopped service can start again', async () => { + mockDaemon(1); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + const intervals = vi.spyOn(globalThis, 'setInterval'); + const created: Array> = []; + const periodic = () => intervals.mock.calls.filter((call) => call[1] === config.sync.intervalMs).length; + + try { + await Promise.all([syncService.start(), syncService.start()]); + for (const result of intervals.mock.results) created.push(result.value as ReturnType); + expect(periodic()).toBe(1); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 1 }); + + await syncService.stop(); + mockDaemon(2); + await syncService.start(); + for (const result of intervals.mock.results) created.push(result.value as ReturnType); + expect(periodic()).toBe(2); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 2, isRunning: false }); + } finally { + await syncService.stop(); + for (const id of created) clearInterval(id); + } + }); +}); diff --git a/server/test/telegram.service.test.ts b/server/test/telegram.service.test.ts new file mode 100644 index 0000000..4aa0f26 --- /dev/null +++ b/server/test/telegram.service.test.ts @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ bots: [] as Array<{ launch: ReturnType; stop: ReturnType }> })); + +vi.mock('telegraf', () => ({ + Telegraf: class { + launch = vi.fn(() => Promise.resolve()); + stop = vi.fn(); + command = vi.fn(); + action = vi.fn(); + use = vi.fn(); + constructor() { + mocks.bots.push(this); + } + }, + Markup: { inlineKeyboard: vi.fn(), button: { callback: vi.fn() } }, +})); + +import { config } from '../src/config'; +import { telegramService } from '../src/services/telegram.service'; + +describe('telegram service start', () => { + const telegram = config.telegram as { botToken: string; adminId: number }; + const saved = { ...telegram }; + const signals = ['SIGINT', 'SIGTERM'] as const; + const listenersBefore = Object.fromEntries(signals.map((signal) => [signal, process.listeners(signal)])); + + afterEach(() => { + telegramService.stop(); + Object.assign(telegram, saved); + // Drop the shutdown handlers the service registered, so they do not outlive the test. + for (const signal of signals) { + for (const listener of process.listeners(signal)) { + if (!listenersBefore[signal].includes(listener)) process.removeListener(signal, listener); + } + } + mocks.bots.length = 0; + }); + + it('starts one bot, one stall check and one pair of shutdown handlers when started twice', () => { + Object.assign(telegram, { botToken: 'test-token', adminId: 42 }); + const intervals = vi.spyOn(globalThis, 'setInterval'); + const sigterm = process.listenerCount('SIGTERM'); + + telegramService.start(); + telegramService.start(); + + expect(mocks.bots).toHaveLength(1); + expect(mocks.bots[0].launch).toHaveBeenCalledTimes(1); + expect(intervals).toHaveBeenCalledTimes(1); + expect(process.listenerCount('SIGTERM')).toBe(sigterm + 1); + }); + + it('can start again after a stop', () => { + Object.assign(telegram, { botToken: 'test-token', adminId: 42 }); + + telegramService.start(); + telegramService.stop(); + telegramService.start(); + + expect(mocks.bots).toHaveLength(2); + expect(mocks.bots[0].stop).toHaveBeenCalledTimes(1); + expect(mocks.bots[1].launch).toHaveBeenCalledTimes(1); + }); +}); From 0f9bf4ac0b696fd0326e49c143eebd23925f1277 Mon Sep 17 00:00:00 2001 From: minuszka <101508242+minuszka@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:33:18 +0100 Subject: [PATCH 3/4] Count an address's transactions the same way during sync and rebuild The transaction count at the top of an address page could be lower than the real number. During normal indexing, an address that took part in several transactions of the same block was counted once for that block, while a rebuild of the address data counted every transaction. - Normal indexing now counts each transaction, as a rebuild does; both use the same code. - To correct the counts already stored, the first start after this update rebuilds the address data once and records that it has done so; later starts do not rebuild. On the live database the rebuild takes a while, and address data may be incomplete while it runs. - Checked with database tests that index the same small chain normally and by rebuild and compare every address field, and that the one-time rebuild runs exactly once; each part of the change was undone once to confirm that a test fails without it. --- server/src/models/SyncState.ts | 3 + server/src/services/sync.service.ts | 72 ++++--- server/src/services/sync/addressIndexing.ts | 11 +- server/test/addressIndexing.test.ts | 31 ++- server/test/addressStats.integration.test.ts | 196 +++++++++++++++++++ 5 files changed, 271 insertions(+), 42 deletions(-) create mode 100644 server/test/addressStats.integration.test.ts diff --git a/server/src/models/SyncState.ts b/server/src/models/SyncState.ts index 215b913..bc82992 100644 --- a/server/src/models/SyncState.ts +++ b/server/src/models/SyncState.ts @@ -12,6 +12,7 @@ export interface SyncStateDocument extends Document { error: string | null; addressIndexVersion?: number; satoshiDataVersion?: number; + addressStatsVersion?: number; } const syncStateSchema = new Schema({ @@ -26,6 +27,8 @@ const syncStateSchema = new Schema({ error: { type: String, default: null }, addressIndexVersion: { type: Number, default: 0 }, satoshiDataVersion: { type: Number, default: 0 }, + // Version of how address statistics are counted; an older one rebuilds the address index once. + addressStatsVersion: { type: Number, default: 0 }, }); export const SyncState = mongoose.model('SyncState', syncStateSchema); diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index b34f738..8488ef9 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -31,6 +31,7 @@ import { collectInvolvedAddresses, mergeAddressDeltas, normalizeSat, + type AddressDelta, } from './sync/addressIndexing'; import { runScheduledSyncAttempt } from './sync/syncScheduling'; @@ -77,6 +78,9 @@ class SyncService { private readonly CURSOR_MAX_TIME_MS = Math.max(30_000, config.rpc.timeout); private readonly ADDRESS_INDEX_VERSION = 3; private readonly SATOSHI_DATA_VERSION = 3; + // 1: txCount counts each transaction an address takes part in (it counted blocks during live + // sync and transactions in a rebuild). An older stored version rebuilds the address index once. + private readonly ADDRESS_STATS_VERSION = 1; private readonly SAT_MULTIPLIER = 10 ** COIN.DECIMALS; private readonly TX_EVENT_MAX_PER_BLOCK = 250; @@ -158,7 +162,8 @@ class SyncService { return before?.addressRebuildRequired === true; } - private async recoverStartupState(): Promise { + /** Repairs what a previous run left behind; returns true when it rebuilt the address index. */ + private async recoverStartupState(): Promise { const syncState = await SyncState.findOne({ key: 'main' }).lean(); const indexedTip = await Block.findOne({}) .sort({ height: -1 }) @@ -179,7 +184,7 @@ class SyncService { }, { upsert: true } ); - return; + return false; } // If sync state points past our indexed tip (possible after crash during rollback), @@ -214,7 +219,7 @@ class SyncService { ); } - if (!recoveryPlan.rebuildAddressIndex) return; + if (!recoveryPlan.rebuildAddressIndex) return false; logger.warn('Detected pending address index rebuild flag. Rebuilding from transactions...'); await this.rebuildAddressIndexFromTransactions(); @@ -228,6 +233,33 @@ class SyncService { { upsert: true } ); logger.info('Pending address rebuild recovered successfully.'); + return true; + } + + /** + * Brings the stored address statistics to ADDRESS_STATS_VERSION: if the stored version is older, + * rebuilds the address index once and records the version. A rebuild that startup recovery has + * just done counts, so a start never rebuilds twice. Interrupted before the version is written, + * the next start simply rebuilds again. + */ + private async ensureAddressStatsVersion(rebuiltThisStart: boolean): Promise { + const syncState = await SyncState.findOne({ key: 'main' }).select({ addressStatsVersion: 1 }).lean<{ addressStatsVersion?: number }>(); + const currentVersion = syncState?.addressStatsVersion ?? 0; + if (currentVersion >= this.ADDRESS_STATS_VERSION) return; + + if (!rebuiltThisStart) { + logger.warn( + `Address statistics version ${currentVersion} -> ${this.ADDRESS_STATS_VERSION}: rebuilding the address index once...` + ); + const startedAt = Date.now(); + await this.rebuildAddressIndexFromTransactions(); + logger.info(`Address statistics rebuild finished in ${Math.round((Date.now() - startedAt) / 1000)} s.`); + } + await SyncState.findOneAndUpdate( + { key: 'main' }, + { addressStatsVersion: this.ADDRESS_STATS_VERSION, heartbeatAt: new Date() }, + { upsert: true } + ); } // Rebuild address balances from stored transactions. @@ -269,28 +301,8 @@ class SyncService { continue; } - const addressMap = new Map(); - - for (const vout of tx.vout || []) { - const addresses = vout.scriptPubKey?.addresses; - if (!addresses?.length) continue; - const valueSat = normalizeSat(vout.valueSat); - for (const addr of addresses) { - const entry = addressMap.get(addr) || { receivedSat: 0n, sentSat: 0n }; - entry.receivedSat += valueSat; - addressMap.set(addr, entry); - } - } - - if (!tx.isCoinbase) { - for (const vin of tx.vin || []) { - const valueSat = normalizeSat(vin.valueSat); - if (!vin.address || valueSat <= 0n) continue; - const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n }; - entry.sentSat += valueSat; - addressMap.set(vin.address, entry); - } - } + // The same per-transaction changes live sync collects (processTransaction). + const addressMap = collectAddressDeltas(tx.vin || [], tx.vout || [], !!tx.isCoinbase); for (const [addr, update] of addressMap) { bulkOps.push({ @@ -614,7 +626,8 @@ class SyncService { try { await this.ensureIndexVersion(); await this.ensureSatoshiDataVersion(); - await this.recoverStartupState(); + const rebuiltThisStart = await this.recoverStartupState(); + await this.ensureAddressStatsVersion(rebuiltThisStart); await this.addressResolver.preInferVersionByte(); // Run initial sync @@ -982,7 +995,7 @@ class SyncService { let totalValueOutSat = 0n; let minedBy: string | undefined; const txids: string[] = []; - const blockAddressDeltas = new Map(); + const blockAddressDeltas = new Map(); const txEntries = (rpcBlock.tx || []) as Array; @@ -1115,7 +1128,7 @@ class SyncService { ): Promise<{ totalValueOutSat: bigint; isCoinbase: boolean; - addressDeltas?: Map; + addressDeltas?: Map; }> { const isCoinbase = rpcTx.vin.length > 0 && !!rpcTx.vin[0].coinbase; @@ -1369,7 +1382,7 @@ class SyncService { // ── Address Balance Updates ───────────────────────────────────── private async applyAddressDeltas( - addressMap: Map, + addressMap: Map, blocktime: number ): Promise { if (addressMap.size === 0) return false; @@ -1455,6 +1468,7 @@ class SyncService { error: null, addressIndexVersion: this.ADDRESS_INDEX_VERSION, satoshiDataVersion: this.SATOSHI_DATA_VERSION, + addressStatsVersion: this.ADDRESS_STATS_VERSION, }, { upsert: true } ); diff --git a/server/src/services/sync/addressIndexing.ts b/server/src/services/sync/addressIndexing.ts index b88a623..4cc3fba 100644 --- a/server/src/services/sync/addressIndexing.ts +++ b/server/src/services/sync/addressIndexing.ts @@ -4,6 +4,8 @@ import { coinToSat, satToBigInt } from '../../utils/amounts'; export interface AddressDelta { receivedSat: bigint; sentSat: bigint; + /** How many transactions the address takes part in: 1 for one transaction, summed over a block. */ + txCount: number; } interface VinLike { @@ -57,7 +59,7 @@ export function collectAddressDeltas( if (!addresses?.length) continue; const valueSat = normalizeSat(vout.valueSat); for (const address of addresses) { - const entry = addressMap.get(address) || { receivedSat: 0n, sentSat: 0n }; + const entry = addressMap.get(address) || { receivedSat: 0n, sentSat: 0n, txCount: 1 }; entry.receivedSat += valueSat; addressMap.set(address, entry); } @@ -66,7 +68,7 @@ export function collectAddressDeltas( for (const vin of vins) { const valueSat = normalizeSat(vin.valueSat); if (!vin.address || valueSat <= 0n) continue; - const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n }; + const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n, txCount: 1 }; entry.sentSat += valueSat; addressMap.set(vin.address, entry); } @@ -79,9 +81,10 @@ export function mergeAddressDeltas( source: Map ): void { for (const [address, delta] of source) { - const current = target.get(address) || { receivedSat: 0n, sentSat: 0n }; + const current = target.get(address) || { receivedSat: 0n, sentSat: 0n, txCount: 0 }; current.receivedSat += delta.receivedSat; current.sentSat += delta.sentSat; + current.txCount += delta.txCount; target.set(address, current); } } @@ -106,7 +109,7 @@ export function buildAddressDeltaPipeline( balanceSat: { $add: [{ $ifNull: ['$balanceSat', zeroDec] }, netSatDec] }, totalReceivedSat: { $add: [{ $ifNull: ['$totalReceivedSat', zeroDec] }, receivedDec] }, totalSentSat: { $add: [{ $ifNull: ['$totalSentSat', zeroDec] }, sentDec] }, - txCount: { $add: [{ $ifNull: ['$txCount', 0] }, 1] }, + txCount: { $add: [{ $ifNull: ['$txCount', 0] }, delta.txCount] }, firstSeen: { $min: [{ $ifNull: ['$firstSeen', blocktime] }, blocktime] }, lastSeen: { $max: [{ $ifNull: ['$lastSeen', 0] }, blocktime] }, }, diff --git a/server/test/addressIndexing.test.ts b/server/test/addressIndexing.test.ts index a2c8e99..abbaf6a 100644 --- a/server/test/addressIndexing.test.ts +++ b/server/test/addressIndexing.test.ts @@ -5,6 +5,7 @@ import { collectInvolvedAddresses, mergeAddressDeltas, normalizeSat, + type AddressDelta, } from '../src/services/sync/addressIndexing'; describe('address indexing domain', () => { @@ -32,34 +33,45 @@ describe('address indexing domain', () => { expect(collectInvolvedAddresses(vins, vouts)).toEqual(['Dsender', 'Dreceiver']); expect(collectAddressDeltas(vins, vouts, false)).toEqual( new Map([ - ['Dreceiver', { receivedSat: 7n, sentSat: 0n }], - ['Dsender', { receivedSat: 4n, sentSat: 7n }], + ['Dreceiver', { receivedSat: 7n, sentSat: 0n, txCount: 1 }], + ['Dsender', { receivedSat: 4n, sentSat: 7n, txCount: 1 }], ]) ); }); it('merges per-transaction deltas for block-level bulk writes', () => { - const target = new Map([['Daddr', { receivedSat: 2n, sentSat: 1n }]]); + const target = new Map([['Daddr', { receivedSat: 2n, sentSat: 1n, txCount: 1 }]]); mergeAddressDeltas( target, new Map([ - ['Daddr', { receivedSat: 3n, sentSat: 4n }], - ['Dnew', { receivedSat: 5n, sentSat: 0n }], + ['Daddr', { receivedSat: 3n, sentSat: 4n, txCount: 1 }], + ['Dnew', { receivedSat: 5n, sentSat: 0n, txCount: 1 }], ]) ); expect(target).toEqual( new Map([ - ['Daddr', { receivedSat: 5n, sentSat: 5n }], - ['Dnew', { receivedSat: 5n, sentSat: 0n }], + ['Daddr', { receivedSat: 5n, sentSat: 5n, txCount: 2 }], + ['Dnew', { receivedSat: 5n, sentSat: 0n, txCount: 1 }], ]) ); }); + it('counts an address once for each transaction of a block it takes part in', () => { + const block = new Map(); + const pay = (address: string, valueSat: string) => ({ valueSat, scriptPubKey: { addresses: [address] } }); + // Two transactions of one block, both spending from and paying back to Dself. + mergeAddressDeltas(block, collectAddressDeltas([{ txid: 'p1', address: 'Dself', valueSat: '10' }], [pay('Dother', '4'), pay('Dself', '6')], false)); + mergeAddressDeltas(block, collectAddressDeltas([{ txid: 'p2', address: 'Dself', valueSat: '6' }], [pay('Dother', '1'), pay('Dself', '5')], false)); + + expect(block.get('Dself')).toEqual({ receivedSat: 11n, sentSat: 16n, txCount: 2 }); + expect(block.get('Dother')).toEqual({ receivedSat: 5n, sentSat: 0n, txCount: 2 }); + }); + it('builds the atomic MongoDB update pipeline used by live sync and rebuilds', () => { const pipeline = buildAddressDeltaPipeline( 'Dabc', - { receivedSat: 11n, sentSat: 3n }, + { receivedSat: 11n, sentSat: 3n, txCount: 2 }, 1700000000 ); const setStage = (pipeline[0] as { $set?: Record }).$set; @@ -67,6 +79,7 @@ describe('address indexing domain', () => { expect(pipeline).toHaveLength(1); expect(setStage).toBeDefined(); expect((setStage?.address as { $ifNull: [string, string] }).$ifNull[1]).toBe('Dabc'); - expect((setStage?.txCount as { $add: [number, number] }).$add[1]).toBe(1); + // The block's transactions with this address, not one per block. + expect((setStage?.txCount as { $add: [number, number] }).$add[1]).toBe(2); }); }); diff --git a/server/test/addressStats.integration.test.ts b/server/test/addressStats.integration.test.ts new file mode 100644 index 0000000..7f6ac96 --- /dev/null +++ b/server/test/addressStats.integration.test.ts @@ -0,0 +1,196 @@ +import mongoose from 'mongoose'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { syncService } from '../src/services/sync.service'; +import { rpcService } from '../src/services/rpc.service'; +import { realtimeService } from '../src/services/realtime.service'; +import { telegramService } from '../src/services/telegram.service'; +import { Address } from '../src/models/Address'; +import { Block } from '../src/models/Block'; +import { SyncState } from '../src/models/SyncState'; +import { Transaction } from '../src/models/Transaction'; + +// This suite requires an explicitly supplied disposable localhost database. +// The normal unit suite must never connect to an operator's configured MongoDB. +const baseUri = process.env.TEST_POSE_MONGO_URI; + +const A = 'DNSSwp3CWfyus5h8TSBvJHFoTUASQCF8Rj'; +const B = 'D8e1nUDayNHvXSYZPv1QECNcSgCxgsXw9d'; +const C = 'DGK3yNxxedc2pvaPPwTbVBey2NmFC8FUGZ'; +const D = 'DKgo69ycaxuut3QxaTSfyqbkFvyRvvq8Hf'; +const E = 'D9NofFTND226dT475V33QaHU9LKHK7UwVe'; +const COIN = 100_000_000; + +type RpcVin = { coinbase?: string; txid?: string; vout?: number; sequence: number; address?: string; value?: number; valueSat?: number }; +type RpcVout = { value: number; valueSat: number; n: number; scriptPubKey: { asm: string; hex: string; type: string; addresses: string[] } }; +type RpcTx = { txid: string; version: number; size: number; locktime: number; vin: RpcVin[]; vout: RpcVout[] }; + +const hashOf = (height: number) => height.toString(16).padStart(64, 'e'); +const txidOf = (label: string) => label.padStart(64, '0'); +const pay = (n: number, coins: number, address: string): RpcVout => ({ + value: coins, valueSat: Math.round(coins * COIN), n, + scriptPubKey: { asm: '', hex: '76a914' + '00'.repeat(20) + '88ac', type: 'pubkeyhash', addresses: [address] }, +}); +const spend = (prev: string, n: number, coins: number, address: string): RpcVin => ({ + txid: txidOf(prev), vout: n, sequence: 0, address, value: coins, valueSat: Math.round(coins * COIN), +}); +const coinbase = (label: string, coins: number, address: string): RpcTx => ({ + txid: txidOf(label), version: 3, size: 100, locktime: 0, vin: [{ coinbase: '03aa0d02', sequence: 0 }], vout: [pay(0, coins, address)], +}); +const transfer = (label: string, vin: RpcVin[], vout: RpcVout[]): RpcTx => ({ txid: txidOf(label), version: 3, size: 225, locktime: 0, vin, vout }); + +// In block 2, A spends twice and D is paid twice: each takes part in two transactions of one block. +const TXS: RpcTx[][] = [ + [coinbase('c0', 1000, A)], + [coinbase('c1', 1000, B)], + [ + coinbase('c2', 50, C), + transfer('x2', [spend('c0', 0, 1000, A)], [pay(0, 400, D), pay(1, 599.99, A)]), + transfer('y2', [spend('x2', 1, 599.99, A)], [pay(0, 100, D), pay(1, 499.98, A)]), + ], + [ + coinbase('c3', 50, C), + transfer('z3', [spend('x2', 0, 400, D)], [pay(0, 399.99, E)]), + ], +]; +const BLOCKS = TXS.map((tx, height) => ({ + hash: hashOf(height), confirmations: 1, size: 500, height, version: 1, merkleroot: 'm'.repeat(64), tx, + time: 1_800_000_000 + height * 150, mediantime: 1_800_000_000 + height * 150, nonce: 0, bits: '1d00ffff', + difficulty: 1, chainwork: '00', nTx: tx.length, + ...(height > 0 ? { previousblockhash: hashOf(height - 1) } : {}), +})); + +/** How many transactions each address takes part in, from the fixture alone. */ +const EXPECTED_TX_COUNT = { [A]: 3, [B]: 1, [C]: 2, [D]: 3, [E]: 1 }; + +function mockDaemon(tip: number) { + vi.spyOn(rpcService, 'getBlockCount').mockResolvedValue(tip); + vi.spyOn(rpcService, 'getBlockHash').mockImplementation(async (height: number) => hashOf(height)); + vi.spyOn(rpcService, 'getBlock').mockImplementation(async (hash: string) => { + const block = BLOCKS.find((candidate) => candidate.hash === hash); + if (!block) throw new Error(`unknown block ${hash}`); + return structuredClone(block); + }); +} + +type StoredAddress = { address: string; balanceSat: unknown; totalReceivedSat: unknown; totalSentSat: unknown; txCount: number; firstSeen: number; lastSeen: number }; +async function storedAddresses(fields: Array) { + const docs = await Address.find({}).lean(); + return Object.fromEntries(docs.sort((a, b) => a.address.localeCompare(b.address)).map((doc) => + [doc.address, Object.fromEntries(fields.map((field) => [field, String(doc[field])]))])); +} +const ALL_FIELDS: Array = ['balanceSat', 'totalReceivedSat', 'totalSentSat', 'txCount', 'firstSeen', 'lastSeen']; + +const internals = syncService as unknown as { + stopRequested: boolean; + ADDRESS_STATS_VERSION: number; + rebuildAddressIndexFromTransactions: () => Promise; +}; +const syncState = () => SyncState.findOne({ key: 'main' }).lean>(); +const freshState = (extra: Record = {}) => + SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3, ...extra }); + +describe.skipIf(!baseUri)('address statistics from live sync and from a rebuild, with disposable MongoDB', () => { + beforeAll(async () => { + if (!baseUri || !/^mongodb:\/\/127\.0\.0\.1:\d+\/deftrack_pose_test_[a-f0-9]+$/.test(baseUri)) { + throw new Error('TEST_POSE_MONGO_URI must name an isolated localhost test database'); + } + // Own database, so it cannot collide with other integration suites that may run at the same time. + await mongoose.connect(`${baseUri}as`, { serverSelectionTimeoutMS: 5000 }); + await Promise.all([Block.init(), Transaction.init(), Address.init(), SyncState.init()]); + }); + beforeEach(async () => { + internals.stopRequested = false; + vi.spyOn(realtimeService, 'publishNewBlock').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishNewTx').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishSyncStatus').mockImplementation(() => undefined); + vi.spyOn(telegramService, 'notifyNewBlock').mockImplementation(() => undefined); + await Promise.all([Block.deleteMany({}), Transaction.deleteMany({}), Address.deleteMany({}), SyncState.deleteMany({})]); + }); + afterAll(async () => { + internals.stopRequested = false; + await syncService.stop(); + if (mongoose.connection.readyState === 1) await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + it('live sync and a rebuild give the same balances (as before this change)', async () => { + mockDaemon(3); + await freshState(); + await syncService.sync(); + const live = await storedAddresses(['balanceSat', 'totalReceivedSat', 'totalSentSat']); + + await internals.rebuildAddressIndexFromTransactions(); + + expect(await storedAddresses(['balanceSat', 'totalReceivedSat', 'totalSentSat'])).toEqual(live); + }); + + it('live sync and a rebuild give the same value in every address field, counting each transaction once', async () => { + mockDaemon(3); + await freshState(); + await syncService.sync(); + const live = await storedAddresses(ALL_FIELDS); + + await internals.rebuildAddressIndexFromTransactions(); + + expect(await storedAddresses(ALL_FIELDS)).toEqual(live); + const counts = Object.fromEntries(Object.entries(live).map(([address, fields]) => [address, Number(fields.txCount)])); + expect(counts).toEqual(EXPECTED_TX_COUNT); + }); + + it('an older address statistics version rebuilds the address index once at start, and only once', async () => { + mockDaemon(3); + await freshState(); + await syncService.sync(); + // Counts as an older release left them: one per block instead of one per transaction. + await Address.updateMany({}, { $set: { txCount: 1 } }); + await SyncState.updateOne({ key: 'main' }, { $unset: { addressStatsVersion: 1 } }); + const rebuild = vi.spyOn(internals, 'rebuildAddressIndexFromTransactions'); + + try { + await syncService.start(); + expect(rebuild).toHaveBeenCalledTimes(1); + expect(await syncState()).toMatchObject({ addressStatsVersion: internals.ADDRESS_STATS_VERSION }); + const counts = Object.fromEntries(Object.entries(await storedAddresses(['txCount'])).map(([address, fields]) => [address, Number(fields.txCount)])); + expect(counts).toEqual(EXPECTED_TX_COUNT); + + // The next start finds the current version and does not rebuild again. + await syncService.stop(); + await syncService.start(); + expect(rebuild).toHaveBeenCalledTimes(1); + } finally { + await syncService.stop(); + } + }); + + it('an empty database starts at the current version without a rebuild', async () => { + mockDaemon(0); + // An index version change on an empty database resets the state (ensureIndexVersion). + await freshState({ addressIndexVersion: 0 }); + const rebuild = vi.spyOn(internals, 'rebuildAddressIndexFromTransactions'); + + try { + await syncService.start(); + expect(rebuild).not.toHaveBeenCalled(); + expect(await syncState()).toMatchObject({ addressStatsVersion: internals.ADDRESS_STATS_VERSION, lastSyncedHeight: 0 }); + } finally { + await syncService.stop(); + } + }); + + it('a start that already rebuilds for another reason settles the version with that one rebuild', async () => { + mockDaemon(3); + await freshState(); + await syncService.sync(); + // A lock left behind by a killed process: startup recovery rebuilds anyway. + await SyncState.updateOne({ key: 'main' }, { $set: { isRunning: true }, $unset: { addressStatsVersion: 1 } }); + const rebuild = vi.spyOn(internals, 'rebuildAddressIndexFromTransactions'); + + try { + await syncService.start(); + expect(rebuild).toHaveBeenCalledTimes(1); + expect(await syncState()).toMatchObject({ addressStatsVersion: internals.ADDRESS_STATS_VERSION, isRunning: false }); + } finally { + await syncService.stop(); + } + }); +}); From 4760a6012bc7ef42c2f2db90de02f85b8e1361a3 Mon Sep 17 00:00:00 2001 From: minuszka <101508242+minuszka@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:56:17 +0100 Subject: [PATCH 4/4] Keep address data complete while the address index is rebuilt Rebuilding the address data (after an interrupted indexing run, and once after the previous change) first emptied it, so the rich list, the address pages and the address statistics showed incomplete data until the rebuild finished. - The rebuild now writes into a separate collection with the same indexes and swaps it in only when it is complete, so visitors keep seeing the previous, complete data in the meantime. - If a rebuild fails, the previous data stays as it was, the unfinished copy is removed, and the rebuild is tried again on the next run. - Checked with database tests that read the address data and the rich list in the middle of a rebuild, make a rebuild fail, and check the indexes of the swapped-in data; each part of the change was undone once to confirm that a test fails without it. --- server/src/models/Address.ts | 9 + server/src/services/sync.service.ts | 31 ++- .../test/addressRebuild.integration.test.ts | 184 ++++++++++++++++++ 3 files changed, 221 insertions(+), 3 deletions(-) create mode 100644 server/test/addressRebuild.integration.test.ts diff --git a/server/src/models/Address.ts b/server/src/models/Address.ts index e48944a..981118a 100644 --- a/server/src/models/Address.ts +++ b/server/src/models/Address.ts @@ -27,3 +27,12 @@ addressSchema.index({ balanceSat: -1 }); addressSchema.index({ firstSeen: -1 }); export const Address = mongoose.model('Address', addressSchema); + +// A rebuild of the address index writes into this collection and then renames it over the live +// one (sync.service), so readers never see a half-built index. It exists only during a rebuild: +// the model neither creates the collection nor its indexes on its own. +export const ADDRESS_REBUILD_COLLECTION = 'addresses_rebuild'; +const addressRebuildSchema = addressSchema.clone(); +addressRebuildSchema.set('autoCreate', false); +addressRebuildSchema.set('autoIndex', false); +export const AddressRebuild = mongoose.model('AddressRebuild', addressRebuildSchema, ADDRESS_REBUILD_COLLECTION); diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index 8488ef9..79eb3a2 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -6,7 +6,7 @@ import { invalidateDashboardCache } from '../routes/dashboard.routes'; import { invalidateRichlistCache } from '../routes/richlist.routes'; import { Block } from '../models/Block'; import { Transaction } from '../models/Transaction'; -import { Address } from '../models/Address'; +import { Address, AddressRebuild, ADDRESS_REBUILD_COLLECTION } from '../models/Address'; import { SyncState } from '../models/SyncState'; import mongoose from 'mongoose'; import { COIN } from '@defcon/shared'; @@ -262,12 +262,37 @@ class SyncService { ); } + /** Drops the rebuild collection if it exists (left by an interrupted rebuild, or after a failure). */ + private async dropAddressRebuildCollection(): Promise { + const existing = await mongoose.connection.db! + .listCollections({ name: ADDRESS_REBUILD_COLLECTION }, { nameOnly: true }) + .toArray(); + if (existing.length > 0) await AddressRebuild.collection.drop(); + } + // Rebuild address balances from stored transactions. // This gives a deterministic recovery path after rollback/restart on MongoDB tiers // where multi-collection transactions are unavailable. + // The new index is built in a separate collection with the same indexes and renamed over the + // live one at the end, so readers keep the complete old data while it runs, and a rebuild that + // fails leaves the old data as it was (the caller keeps addressRebuildRequired set). private async rebuildAddressIndexFromTransactions(): Promise { - await Address.deleteMany({}); + await this.dropAddressRebuildCollection(); + await AddressRebuild.createCollection(); + await AddressRebuild.createIndexes(); + try { + await this.writeAddressIndexInto(AddressRebuild); + await AddressRebuild.collection.rename(Address.collection.collectionName, { dropTarget: true }); + } catch (error) { + await this.dropAddressRebuildCollection().catch((dropError) => + logger.error('Could not drop the unfinished address rebuild collection:', dropError) + ); + throw error; + } + } + /** Computes every address from the stored transactions into `target`, which starts empty. */ + private async writeAddressIndexInto(target: typeof AddressRebuild): Promise { const cursor = Transaction.find({}) // Sort by blockheight to guarantee correct chronological order for balance deltas. // The compound index { blockheight: -1, _id: -1 } exists; ascending scan is fine. @@ -289,7 +314,7 @@ class SyncService { const flushBulk = async () => { if (bulkOps.length === 0) return; - await Address.bulkWrite(bulkOps, { ordered: false }); + await target.bulkWrite(bulkOps, { ordered: false }); bulkOps.length = 0; // A long rebuild keeps its sync lock from looking abandoned (see acquireSyncLock). await SyncState.updateOne({ key: 'main' }, { heartbeatAt: new Date() }); diff --git a/server/test/addressRebuild.integration.test.ts b/server/test/addressRebuild.integration.test.ts new file mode 100644 index 0000000..f0aaebb --- /dev/null +++ b/server/test/addressRebuild.integration.test.ts @@ -0,0 +1,184 @@ +import express from 'express'; +import mongoose from 'mongoose'; +import request from 'supertest'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import richlistRoutes, { invalidateRichlistCache } from '../src/routes/richlist.routes'; +import { syncService } from '../src/services/sync.service'; +import { rpcService } from '../src/services/rpc.service'; +import { realtimeService } from '../src/services/realtime.service'; +import { telegramService } from '../src/services/telegram.service'; +import { Address } from '../src/models/Address'; +import { Block } from '../src/models/Block'; +import { SyncState } from '../src/models/SyncState'; +import { Transaction } from '../src/models/Transaction'; + +// This suite requires an explicitly supplied disposable localhost database. +// The normal unit suite must never connect to an operator's configured MongoDB. +const baseUri = process.env.TEST_POSE_MONGO_URI; + +const A = 'DNSSwp3CWfyus5h8TSBvJHFoTUASQCF8Rj'; +const B = 'D8e1nUDayNHvXSYZPv1QECNcSgCxgsXw9d'; +const C = 'DGK3yNxxedc2pvaPPwTbVBey2NmFC8FUGZ'; +const REBUILD_COLLECTION = 'addresses_rebuild'; + +const hashOf = (height: number) => height.toString(16).padStart(64, 'f'); +const pay = (n: number, coins: number, address: string) => ({ + value: coins, valueSat: coins * 100_000_000, n, + scriptPubKey: { asm: '', hex: '76a914' + '00'.repeat(20) + '88ac', type: 'pubkeyhash', addresses: [address] }, +}); +// Three blocks, each a single coinbase: A, B, C. +const BLOCKS = [A, B, C].map((address, height) => ({ + hash: hashOf(height), confirmations: 1, size: 300, height, version: 1, merkleroot: 'm'.repeat(64), + tx: [{ txid: `cb${height}`.padStart(64, '0'), version: 3, size: 100, locktime: 0, vin: [{ coinbase: '03aa0d02', sequence: 0 }], vout: [pay(0, 1000 * (height + 1), address)] }], + time: 1_800_000_000 + height * 150, mediantime: 1_800_000_000 + height * 150, nonce: 0, bits: '1d00ffff', + difficulty: 1, chainwork: '00', nTx: 1, + ...(height > 0 ? { previousblockhash: hashOf(height - 1) } : {}), +})); + +function mockDaemon(tip: number) { + vi.spyOn(rpcService, 'getBlockCount').mockResolvedValue(tip); + vi.spyOn(rpcService, 'getBlockHash').mockImplementation(async (height: number) => hashOf(height)); + vi.spyOn(rpcService, 'getBlock').mockImplementation(async (hash: string) => { + const block = BLOCKS.find((candidate) => candidate.hash === hash); + if (!block) throw new Error(`unknown block ${hash}`); + return structuredClone(block); + }); +} + +type Stored = { address: string; balanceSat: unknown; totalReceivedSat: unknown; totalSentSat: unknown; txCount: number }; +async function storedAddresses() { + const docs = await Address.find({}).lean(); + return Object.fromEntries(docs.sort((a, b) => a.address.localeCompare(b.address)).map((doc) => + [doc.address, { balance: String(doc.balanceSat), received: String(doc.totalReceivedSat), sent: String(doc.totalSentSat), txCount: doc.txCount }])); +} +const collectionExists = async (name: string) => + (await mongoose.connection.db!.listCollections({ name }, { nameOnly: true }).toArray()).length > 0; + +/** + * Takes over every model's bulkWrite while the address index is rebuilt. `hold` keeps the first + * write back until `release()`; `fail` makes the first write throw. + */ +function interceptRebuildWrites(mode: 'hold' | 'fail') { + const original = mongoose.Model.bulkWrite; + let release = () => {}; + const gate = new Promise((resolve) => { release = resolve; }); + const state = { reached: false, release: () => release() }; + vi.spyOn(mongoose.Model, 'bulkWrite').mockImplementation(async function (this: typeof mongoose.Model, ...args: unknown[]) { + if (!state.reached) { + state.reached = true; + if (mode === 'fail') throw new Error('injected rebuild write failure'); + await gate; + } + return (original as (...a: unknown[]) => Promise).apply(this, args); + } as never); + return state; +} + +const internals = syncService as unknown as { + stopRequested: boolean; + rebuildAddressIndexFromTransactions: () => Promise; +}; +const syncState = () => SyncState.findOne({ key: 'main' }).lean>(); + +describe.skipIf(!baseUri)('rebuilding the address index while it is being read, with disposable MongoDB', () => { + const app = express(); + app.use('/api/richlist', richlistRoutes); + + beforeAll(async () => { + if (!baseUri || !/^mongodb:\/\/127\.0\.0\.1:\d+\/deftrack_pose_test_[a-f0-9]+$/.test(baseUri)) { + throw new Error('TEST_POSE_MONGO_URI must name an isolated localhost test database'); + } + // Own database, so it cannot collide with other integration suites that may run at the same time. + await mongoose.connect(`${baseUri}ar`, { serverSelectionTimeoutMS: 5000 }); + await Promise.all([Block.init(), Transaction.init(), Address.init(), SyncState.init()]); + }); + beforeEach(async () => { + internals.stopRequested = false; + vi.spyOn(realtimeService, 'publishNewBlock').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishNewTx').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishSyncStatus').mockImplementation(() => undefined); + vi.spyOn(telegramService, 'notifyNewBlock').mockImplementation(() => undefined); + await Promise.all([Block.deleteMany({}), Transaction.deleteMany({}), Address.deleteMany({}), SyncState.deleteMany({})]); + if (await collectionExists(REBUILD_COLLECTION)) await mongoose.connection.db!.dropCollection(REBUILD_COLLECTION); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3, addressStatsVersion: 1 }); + mockDaemon(2); + await syncService.sync(); + invalidateRichlistCache(); + }); + afterAll(async () => { + internals.stopRequested = false; + if (mongoose.connection.readyState === 1) await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + it('readers keep the complete previous address data while the rebuild runs', async () => { + const before = await storedAddresses(); + const richlistBefore = (await request(app).get('/api/richlist')).body; + expect(Object.keys(before)).toHaveLength(3); + const writes = interceptRebuildWrites('hold'); + + const rebuilding = internals.rebuildAddressIndexFromTransactions(); + try { + await vi.waitFor(() => expect(writes.reached).toBe(true)); + // Mid-rebuild: the live collection is untouched, and the rich list answers from it. + expect(await Address.countDocuments()).toBe(3); + expect(await storedAddresses()).toEqual(before); + invalidateRichlistCache(); + const richlistDuring = await request(app).get('/api/richlist'); + expect(richlistDuring.status).toBe(200); + expect(richlistDuring.body.data).toEqual(richlistBefore.data); + } finally { + writes.release(); + await rebuilding; + } + + expect(await storedAddresses()).toEqual(before); + expect(await collectionExists(REBUILD_COLLECTION)).toBe(false); + }); + + it('a rebuild that fails leaves the previous address data as it was and no unfinished collection behind', async () => { + const before = await storedAddresses(); + interceptRebuildWrites('fail'); + + await expect(internals.rebuildAddressIndexFromTransactions()).rejects.toThrow('injected rebuild write failure'); + + expect(await storedAddresses()).toEqual(before); + expect(await collectionExists(REBUILD_COLLECTION)).toBe(false); + }); + + it('a requested rebuild that fails stays requested, and the next run completes it', async () => { + const before = await storedAddresses(); + await SyncState.updateOne({ key: 'main' }, { addressRebuildRequired: true }); + interceptRebuildWrites('fail'); + + await syncService.sync(); + expect(await syncState()).toMatchObject({ addressRebuildRequired: true, isRunning: false }); + expect(await storedAddresses()).toEqual(before); + + vi.mocked(mongoose.Model.bulkWrite).mockRestore(); + await syncService.sync(); + expect(await syncState()).toMatchObject({ addressRebuildRequired: false, error: null }); + expect(await storedAddresses()).toEqual(before); + }); + + it('a collection left by an interrupted earlier rebuild does not leak into the next one', async () => { + const before = await storedAddresses(); + await mongoose.connection.db!.collection(REBUILD_COLLECTION).insertOne({ address: 'Dleftover', balanceSat: '5', txCount: 9 }); + + await internals.rebuildAddressIndexFromTransactions(); + + expect(await storedAddresses()).toEqual(before); + expect(await collectionExists(REBUILD_COLLECTION)).toBe(false); + }); + + it('the swapped-in collection has the address indexes, including the unique address key', async () => { + await internals.rebuildAddressIndexFromTransactions(); + + const indexes = await Address.collection.indexes(); + const byKey = (key: Record) => indexes.find((index) => JSON.stringify(index.key) === JSON.stringify(key)); + expect(byKey({ address: 1 })).toMatchObject({ unique: true }); + expect(byKey({ balanceSat: -1 })).toBeDefined(); + expect(byKey({ firstSeen: -1 })).toBeDefined(); + await expect(Address.collection.insertOne({ address: A })).rejects.toThrow(/E11000/); + }); +});