From 307b293d9ce80ecd2522983c3115f884ca3a105e Mon Sep 17 00:00:00 2001 From: minuszka <101508242+minuszka@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:36:54 +0100 Subject: [PATCH] Keep address balances correct when indexing a block is interrupted If indexing a block stopped halfway, for example on a database error, the transactions already stored were treated as done when the block was indexed again, and their balance changes were silently lost. - The indexer now marks a block as in progress before its first write and clears the mark after its last. If anything fails in between, the next run first rebuilds the address balances from the stored transactions. - A normal shutdown now waits up to 15 seconds for the block being indexed and stops between blocks, so a restart during a deploy no longer leaves half a block behind. If the block does not finish in time, startup recovery rebuilds as before. - Each new block costs two small extra writes to the sync state. - Checked with database tests that interrupt indexing at each step (a transaction write, the balance write, the block write, a shutdown) and compare every address balance with the expected one; each part of the fix was undone once to confirm that a test fails without it. --- server/src/services/sync.service.ts | 118 +++++-- .../test/syncInterruption.integration.test.ts | 292 ++++++++++++++++++ 2 files changed, 383 insertions(+), 27 deletions(-) create mode 100644 server/test/syncInterruption.integration.test.ts diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index 586368c..95f652e 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -59,6 +59,11 @@ interface ProcessedBlockSummary { class SyncService { private isRunning = false; private intervalId: NodeJS.Timeout | null = null; + // Set by stop(): the running sync ends after its current block, and no new run starts. + private stopRequested = false; + private currentSync: Promise | null = null; + // How long stop() waits for the current block before it leaves the lock to startup recovery. + private stopTimeoutMs = 15_000; private readonly LOG_INTERVAL = 100; private readonly REBUILD_LOG_INTERVAL = 10000; private readonly ADDRESS_BULK_FLUSH_THRESHOLD = 2000; @@ -133,6 +138,21 @@ class SyncService { ); } + /** + * Sets addressRebuildRequired before a block's first write (see processBlock) and says whether it + * was already set, so that the block does not clear a rebuild requested for another reason. + */ + private async markBlockInProgress(): Promise { + const before = await SyncState.findOneAndUpdate( + { key: 'main' }, + { addressRebuildRequired: true, heartbeatAt: new Date() }, + { upsert: true, new: false } + ) + .select({ addressRebuildRequired: 1 }) + .lean<{ addressRebuildRequired?: boolean }>(); + return before?.addressRebuildRequired === true; + } + private async recoverStartupState(): Promise { const syncState = await SyncState.findOne({ key: 'main' }).lean(); const indexedTip = await Block.findOne({}) @@ -581,6 +601,7 @@ class SyncService { } logger.info('Starting blockchain sync service...'); + this.stopRequested = false; await this.ensureIndexVersion(); await this.ensureSatoshiDataVersion(); @@ -607,7 +628,27 @@ class SyncService { clearInterval(this.intervalId); this.intervalId = null; } - this.isRunning = false; + this.stopRequested = true; + + // A running sync stops between blocks and releases the lock itself. If the block in progress + // does not finish in time, the lock stays: the next start sees it and rebuilds the address index. + const running = this.currentSync; + if (running) { + let timer: NodeJS.Timeout | undefined; + const finished = await Promise.race([ + running.then(() => true, () => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), this.stopTimeoutMs); + }), + ]); + clearTimeout(timer); + if (!finished) { + logger.warn(`Sync did not stop within ${this.stopTimeoutMs} ms; leaving its lock for startup recovery.`); + return; + } + logger.info('Sync service stopped'); + return; + } await SyncState.findOneAndUpdate( { key: 'main' }, @@ -623,10 +664,20 @@ class SyncService { } async sync(): Promise { - if (this.isRunning) { + if (this.isRunning || this.currentSync || this.stopRequested) { return; } + const run = this.runSync(); + this.currentSync = run; + try { + await run; + } finally { + if (this.currentSync === run) this.currentSync = null; + } + } + + private async runSync(): Promise { const syncStartedAt = new Date(); // Atomic DB-level lock: only acquire if not already running. @@ -719,6 +770,8 @@ class SyncService { let lastCheckpointAt = Date.now(); while (currentHeight <= daemonHeight) { + // A requested stop takes effect between blocks, never in the middle of one. + if (this.stopRequested) break; const processedBlock = await this.processBlock(currentHeight); lastHash = processedBlock.hash; lastKnownSyncedHeight = currentHeight; @@ -773,11 +826,12 @@ class SyncService { } } - // Final update + // Final update: the last block indexed, which is below the daemon's tip after a stop. + const reachedHeight = currentHeight - 1; await SyncState.findOneAndUpdate( { key: 'main' }, { - lastSyncedHeight: daemonHeight, + lastSyncedHeight: reachedHeight, lastSyncedHash: lastHash, lastSyncedAt: new Date(), isRunning: false, @@ -787,7 +841,12 @@ class SyncService { }, { upsert: true } ); - lastKnownSyncedHeight = daemonHeight; + lastKnownSyncedHeight = reachedHeight; + if (reachedHeight < daemonHeight) { + this.publishSyncStatus('idle', daemonHeight, reachedHeight, { message: 'Sync stopped.' }); + logger.info(`Sync stopped at height ${reachedHeight}.`); + return; + } this.publishSyncStatus('complete', daemonHeight, daemonHeight, { message: 'Sync complete.', }); @@ -864,6 +923,14 @@ class SyncService { rpcBlock = (await rpcService.getBlock(hash, 1)) as RpcBlock; } + // From here on the block is written piece by piece: each transaction (marked as applied), then + // the block's balance changes, then the block. Mark the block as in progress before the first + // write and clear the mark after the last. If anything in between fails, or the process stops, + // the mark stays and the next run rebuilds the address index from the stored transactions; the + // per-transaction marks then apply exactly what is still missing. If the mark cannot be + // written, nothing is. + const rebuildPendingBefore = await this.markBlockInProgress(); + const mintSat = coinToSat(typeof rpcBlock.mint === 'number' ? rpcBlock.mint : 0); let rewardSat = mintSat; let totalValueOutSat = 0n; @@ -894,7 +961,7 @@ class SyncService { rpcTx = entry; } - const result = await this.processTransaction(rpcTx, rpcBlock, { deferAddressUpdates: true }); + const result = await this.processTransaction(rpcTx, rpcBlock); if (result.addressDeltas) { mergeAddressDeltas(blockAddressDeltas, result.addressDeltas); } @@ -973,6 +1040,14 @@ class SyncService { ); } + // Everything of this block is written. A rebuild that was already pending stays pending. + if (!rebuildPendingBefore) { + await SyncState.findOneAndUpdate( + { key: 'main' }, + { addressRebuildRequired: false, heartbeatAt: new Date() } + ); + } + return { hash: rpcBlock.hash, height: rpcBlock.height, @@ -984,10 +1059,13 @@ class SyncService { // ── Transaction Processing ────────────────────────────────────── + /** + * Stores one transaction and returns its balance changes, if they have not been applied before. + * The caller (processBlock) merges them and writes them once for the whole block. + */ private async processTransaction( rpcTx: RpcTransaction, - rpcBlock: RpcBlock, - opts?: { deferAddressUpdates?: boolean } + rpcBlock: RpcBlock ): Promise<{ totalValueOutSat: bigint; isCoinbase: boolean; @@ -1131,25 +1209,11 @@ class SyncService { canApplyAddressDeltasForTx && previousTx?.addressUpdatesApplied !== true; if (shouldApplyAddressDeltas) { const addressDeltas = collectAddressDeltas(vins, vouts, isCoinbase); - - if (opts?.deferAddressUpdates) { - return { - totalValueOutSat, - isCoinbase, - addressDeltas: addressDeltas.size > 0 ? addressDeltas : undefined, - }; - } - - if (addressDeltas.size > 0) { - try { - await this.applyAddressDeltas(addressDeltas, rpcBlock.time); - } catch (error) { - await this.markAddressRebuildRequired( - `Address update failed for tx ${rpcTx.txid}; forcing deterministic rebuild.` - ); - throw error; - } - } + return { + totalValueOutSat, + isCoinbase, + addressDeltas: addressDeltas.size > 0 ? addressDeltas : undefined, + }; } return { totalValueOutSat, isCoinbase }; diff --git a/server/test/syncInterruption.integration.test.ts b/server/test/syncInterruption.integration.test.ts new file mode 100644 index 0000000..d2343a9 --- /dev/null +++ b/server/test/syncInterruption.integration.test.ts @@ -0,0 +1,292 @@ +import mongoose from 'mongoose'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { syncService } from '../src/services/sync.service'; +import { rpcService } from '../src/services/rpc.service'; +import { realtimeService } from '../src/services/realtime.service'; +import { telegramService } from '../src/services/telegram.service'; +import { Address } from '../src/models/Address'; +import { Block } from '../src/models/Block'; +import { SyncState } from '../src/models/SyncState'; +import { Transaction } from '../src/models/Transaction'; + +// This suite requires an explicitly supplied disposable localhost database. +// The normal unit suite must never connect to an operator's configured MongoDB. +const baseUri = process.env.TEST_POSE_MONGO_URI; + +const A = 'DNSSwp3CWfyus5h8TSBvJHFoTUASQCF8Rj'; +const B = 'D8e1nUDayNHvXSYZPv1QECNcSgCxgsXw9d'; +const C = 'DGK3yNxxedc2pvaPPwTbVBey2NmFC8FUGZ'; +const D = 'DKgo69ycaxuut3QxaTSfyqbkFvyRvvq8Hf'; +const E = 'D9NofFTND226dT475V33QaHU9LKHK7UwVe'; +const COIN = 100_000_000; + +type RpcVin = { coinbase?: string; txid?: string; vout?: number; sequence: number; address?: string; value?: number; valueSat?: number }; +type RpcVout = { value: number; valueSat: number; n: number; scriptPubKey: { asm: string; hex: string; type: string; addresses: string[] } }; +type RpcTx = { txid: string; version: number; size: number; locktime: number; vin: RpcVin[]; vout: RpcVout[] }; +type RpcBlock = { hash: string; height: number; time: number; tx: RpcTx[]; previousblockhash?: string; nextblockhash?: string } & Record; + +const hashOf = (height: number) => height.toString(16).padStart(64, 'b'); +const txidOf = (label: string) => label.padStart(64, '0'); +const pay = (n: number, coins: number, address: string): RpcVout => ({ + value: coins, valueSat: Math.round(coins * COIN), n, + scriptPubKey: { asm: '', hex: '76a914' + '00'.repeat(20) + '88ac', type: 'pubkeyhash', addresses: [address] }, +}); +const spend = (prev: string, n: number, coins: number, address: string): RpcVin => ({ + txid: txidOf(prev), vout: n, sequence: 0, address, value: coins, valueSat: Math.round(coins * COIN), +}); +const coinbase = (label: string, coins: number, address: string): RpcTx => ({ + txid: txidOf(label), version: 3, size: 100, locktime: 0, vin: [{ coinbase: '03aa0d02', sequence: 0 }], vout: [pay(0, coins, address)], +}); +const transfer = (label: string, vin: RpcVin[], vout: RpcVout[]): RpcTx => ({ txid: txidOf(label), version: 3, size: 225, locktime: 0, vin, vout }); + +// A small chain shaped like the daemon's verbose blocks. Block 2 has a coinbase and two transfers. +const TXS: RpcTx[][] = [ + [coinbase('c0', 1000, A)], + [coinbase('c1', 1000, B)], + [ + coinbase('c2', 50, C), + transfer('x2', [spend('c0', 0, 1000, A)], [pay(0, 400, D), pay(1, 599.99, A)]), + transfer('y2', [spend('c1', 0, 1000, B)], [pay(0, 100, D), pay(1, 899.99, B)]), + ], + [ + coinbase('c3', 50, C), + transfer('z3', [spend('x2', 0, 400, D)], [pay(0, 399.99, E)]), + ], +]; +const BLOCKS: RpcBlock[] = TXS.map((tx, height) => ({ + hash: hashOf(height), confirmations: 1, size: 500, height, version: 1, merkleroot: 'm'.repeat(64), tx, + time: 1_800_000_000 + height * 150, mediantime: 1_800_000_000 + height * 150, nonce: 0, bits: '1d00ffff', + difficulty: 1, chainwork: '00', nTx: tx.length, + ...(height > 0 ? { previousblockhash: hashOf(height - 1) } : {}), + ...(height < TXS.length - 1 ? { nextblockhash: hashOf(height + 1) } : {}), +})); +const BLOCK2 = BLOCKS[2]; + +/** What every address should hold after the blocks up to `height`, from the fixture alone. */ +function expectedBalances(height: number) { + const totals = new Map(); + const entry = (address: string) => { + const found = totals.get(address) ?? { received: 0n, sent: 0n }; + totals.set(address, found); + return found; + }; + for (const block of BLOCKS.slice(0, height + 1)) { + for (const tx of block.tx) { + for (const output of tx.vout) for (const address of output.scriptPubKey.addresses) entry(address).received += BigInt(output.valueSat); + for (const input of tx.vin) if (input.address) entry(input.address).sent += BigInt(input.valueSat ?? 0); + } + } + return Object.fromEntries([...totals].sort(([a], [b]) => a.localeCompare(b)).map(([address, { received, sent }]) => + [address, { balance: String(received - sent), received: String(received), sent: String(sent) }])); +} + +async function storedBalances() { + const docs = await Address.find({}).lean>(); + return Object.fromEntries(docs.sort((a, b) => a.address.localeCompare(b.address)).map((doc) => + [doc.address, { balance: String(doc.balanceSat), received: String(doc.totalReceivedSat), sent: String(doc.totalSentSat) }])); +} + +const syncState = () => SyncState.findOne({ key: 'main' }).lean>(); + +/** The daemon, answering from the fixture up to `tip`. `hold` keeps a block's answer back until released. */ +function mockDaemon(tip: number) { + vi.spyOn(rpcService, 'getBlockCount').mockResolvedValue(tip); + vi.spyOn(rpcService, 'getBlockHash').mockImplementation(async (height: number) => hashOf(height)); + return vi.spyOn(rpcService, 'getBlock').mockImplementation(async (hash: string) => { + const block = BLOCKS.find((candidate) => candidate.hash === hash); + if (!block) throw new Error(`unknown block ${hash}`); + return structuredClone(block); + }); +} + +type AnyMethods = Record unknown>; + +/** Makes the first call of `method` that `matches` its arguments throw; every other call goes through. */ +function failOnce(model: object, method: string, matches: (...args: unknown[]) => boolean) { + const target = model as AnyMethods; + const original = target[method].bind(model); + const state = { failed: false }; + vi.spyOn(target, method).mockImplementation((...args: unknown[]) => { + if (!state.failed && matches(...args)) { + state.failed = true; + throw new Error(`injected failure in ${method}`); + } + return original(...args); + }); + return state; +} + +/** Holds the first call of `method` that `matches` until `release()`; every other call goes through. */ +function holdOnce(model: object, method: string, matches: (...args: unknown[]) => boolean) { + const target = model as AnyMethods; + const original = target[method].bind(model); + let release = () => {}; + const gate = new Promise((resolve) => { release = resolve; }); + const state = { reached: false, release: () => release() }; + vi.spyOn(target, method).mockImplementation((...args: unknown[]) => { + if (!state.reached && matches(...args)) { + state.reached = true; + // A thenable query stand-in: the real query runs once the gate opens. + const run = gate.then(() => original(...args) as Promise); + const chain = { select: () => chain, lean: () => chain, then: run.then.bind(run), catch: run.catch.bind(run) }; + return chain; + } + return original(...args); + }); + return state; +} + +const internals = syncService as unknown as { stopRequested: boolean; stopTimeoutMs: number }; + +describe.skipIf(!baseUri)('indexing a block that is interrupted, with disposable MongoDB', () => { + beforeAll(async () => { + if (!baseUri || !/^mongodb:\/\/127\.0\.0\.1:\d+\/deftrack_pose_test_[a-f0-9]+$/.test(baseUri)) { + throw new Error('TEST_POSE_MONGO_URI must name an isolated localhost test database'); + } + // Own database, so it cannot collide with other integration suites that may run at the same time. + await mongoose.connect(`${baseUri}si`, { serverSelectionTimeoutMS: 5000 }); + await Promise.all([Block.init(), Transaction.init(), Address.init(), SyncState.init()]); + }); + beforeEach(async () => { + internals.stopRequested = false; + vi.spyOn(realtimeService, 'publishNewBlock').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishNewTx').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishSyncStatus').mockImplementation(() => undefined); + vi.spyOn(telegramService, 'notifyNewBlock').mockImplementation(() => undefined); + await Promise.all([Block.deleteMany({}), Transaction.deleteMany({}), Address.deleteMany({}), SyncState.deleteMany({})]); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + }); + afterAll(async () => { + internals.stopRequested = false; + if (mongoose.connection.readyState === 1) await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + it('the fixture chain indexes to the expected balances without failures', async () => { + mockDaemon(3); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 3, isRunning: false, addressRebuildRequired: false, error: null }); + }); + + it('T1: a transaction write that fails halfway through a block loses no balance change', async () => { + mockDaemon(3); + const failure = failOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('x2')); + + await syncService.sync(); + expect(failure.failed).toBe(true); + expect(await syncState()).toMatchObject({ isRunning: false, error: 'injected failure in findOneAndUpdate' }); + + await syncService.sync(); + expect(await storedBalances()).toEqual(expectedBalances(3)); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 3, addressRebuildRequired: false, error: null }); + }); + + it('T2: a failed balance write counts nothing twice', async () => { + mockDaemon(3); + const failure = failOnce(Address, 'bulkWrite', (ops) => + Array.isArray(ops) && ops.some((op) => (op as { updateOne?: { filter?: { address?: string } } }).updateOne?.filter?.address === D)); + + await syncService.sync(); + expect(failure.failed).toBe(true); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); + + it('T3: a failed block write counts nothing twice', async () => { + mockDaemon(3); + const failure = failOnce(Block, 'findOneAndUpdate', (filter, update) => + (filter as { hash?: string })?.hash === BLOCK2.hash && (update as { height?: number })?.height === 2); + + await syncService.sync(); + expect(failure.failed).toBe(true); + expect(await Block.findOne({ height: 2 }).lean()).toBeNull(); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); + + it('T4: stop() waits for the block being indexed, then stops before the next one', async () => { + mockDaemon(3); + const held = holdOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('y2')); + + const running = syncService.sync(); + let stopped = false; + try { + await vi.waitFor(() => expect(held.reached).toBe(true)); + const stopping = syncService.stop().then(() => { stopped = true; }); + await new Promise((resolve) => setTimeout(resolve, 300)); + // Still waiting: block 2 is half written. + expect(stopped).toBe(false); + held.release(); + await stopping; + } finally { + held.release(); + await running; + } + + expect(await Block.findOne({ height: 2 }).lean()).not.toBeNull(); + expect(await Block.findOne({ height: 3 }).lean()).toBeNull(); + expect(await syncState()).toMatchObject({ + isRunning: false, addressRebuildRequired: false, lastSyncedHeight: 2, lastSyncedHash: BLOCK2.hash, + }); + expect(await storedBalances()).toEqual(expectedBalances(2)); + // A stopped service does not start another run: it neither asks the daemon nor writes the state. + const daemonCalls = vi.mocked(rpcService.getBlockCount).mock.calls.length; + const stateBefore = await syncState(); + await syncService.sync(); + expect(vi.mocked(rpcService.getBlockCount).mock.calls.length).toBe(daemonCalls); + expect(await syncState()).toEqual(stateBefore); + expect(await Block.findOne({ height: 3 }).lean()).toBeNull(); + }); + + it('T5: when the block does not finish in time, stop() leaves the lock for startup recovery', async () => { + mockDaemon(3); + const held = holdOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('y2')); + const timeoutBefore = internals.stopTimeoutMs; + internals.stopTimeoutMs = 200; + + const running = syncService.sync(); + await vi.waitFor(() => expect(held.reached).toBe(true)); + try { + await syncService.stop(); + expect(await syncState()).toMatchObject({ isRunning: true, addressRebuildRequired: true }); + } finally { + internals.stopTimeoutMs = timeoutBefore; + held.release(); + await running; + } + }); + + it('a block does not clear a rebuild that was already requested before it started', async () => { + mockDaemon(0); + await SyncState.updateOne({ key: 'main' }, { addressRebuildRequired: true }); + + await (syncService as unknown as { processBlock: (height: number) => Promise }).processBlock(0); + + expect(await Block.findOne({ height: 0 }).lean()).not.toBeNull(); + expect(await syncState()).toMatchObject({ addressRebuildRequired: true }); + }); + + it('T6: when the in-progress mark cannot be written, the block is not started', async () => { + mockDaemon(1); + await syncService.sync(); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 1, addressRebuildRequired: false }); + + mockDaemon(3); + const failure = failOnce(SyncState, 'findOneAndUpdate', (_filter, update) => { + const fields = update as { addressRebuildRequired?: boolean; $set?: { addressRebuildRequired?: boolean } }; + return fields?.addressRebuildRequired === true || fields?.$set?.addressRebuildRequired === true; + }); + await syncService.sync(); + + expect(failure.failed).toBe(true); + expect(await Transaction.countDocuments({ blockheight: 2 })).toBe(0); + expect(await syncState()).toMatchObject({ isRunning: false, error: 'injected failure in findOneAndUpdate' }); + + await syncService.sync(); + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); +});