From 307b293d9ce80ecd2522983c3115f884ca3a105e Mon Sep 17 00:00:00 2001 From: minuszka <101508242+minuszka@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:36:54 +0100 Subject: [PATCH 1/2] Keep address balances correct when indexing a block is interrupted If indexing a block stopped halfway, for example on a database error, the transactions already stored were treated as done when the block was indexed again, and their balance changes were silently lost. - The indexer now marks a block as in progress before its first write and clears the mark after its last. If anything fails in between, the next run first rebuilds the address balances from the stored transactions. - A normal shutdown now waits up to 15 seconds for the block being indexed and stops between blocks, so a restart during a deploy no longer leaves half a block behind. If the block does not finish in time, startup recovery rebuilds as before. - Each new block costs two small extra writes to the sync state. - Checked with database tests that interrupt indexing at each step (a transaction write, the balance write, the block write, a shutdown) and compare every address balance with the expected one; each part of the fix was undone once to confirm that a test fails without it. --- server/src/services/sync.service.ts | 118 +++++-- .../test/syncInterruption.integration.test.ts | 292 ++++++++++++++++++ 2 files changed, 383 insertions(+), 27 deletions(-) create mode 100644 server/test/syncInterruption.integration.test.ts diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index 586368c..95f652e 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -59,6 +59,11 @@ interface ProcessedBlockSummary { class SyncService { private isRunning = false; private intervalId: NodeJS.Timeout | null = null; + // Set by stop(): the running sync ends after its current block, and no new run starts. + private stopRequested = false; + private currentSync: Promise | null = null; + // How long stop() waits for the current block before it leaves the lock to startup recovery. + private stopTimeoutMs = 15_000; private readonly LOG_INTERVAL = 100; private readonly REBUILD_LOG_INTERVAL = 10000; private readonly ADDRESS_BULK_FLUSH_THRESHOLD = 2000; @@ -133,6 +138,21 @@ class SyncService { ); } + /** + * Sets addressRebuildRequired before a block's first write (see processBlock) and says whether it + * was already set, so that the block does not clear a rebuild requested for another reason. + */ + private async markBlockInProgress(): Promise { + const before = await SyncState.findOneAndUpdate( + { key: 'main' }, + { addressRebuildRequired: true, heartbeatAt: new Date() }, + { upsert: true, new: false } + ) + .select({ addressRebuildRequired: 1 }) + .lean<{ addressRebuildRequired?: boolean }>(); + return before?.addressRebuildRequired === true; + } + private async recoverStartupState(): Promise { const syncState = await SyncState.findOne({ key: 'main' }).lean(); const indexedTip = await Block.findOne({}) @@ -581,6 +601,7 @@ class SyncService { } logger.info('Starting blockchain sync service...'); + this.stopRequested = false; await this.ensureIndexVersion(); await this.ensureSatoshiDataVersion(); @@ -607,7 +628,27 @@ class SyncService { clearInterval(this.intervalId); this.intervalId = null; } - this.isRunning = false; + this.stopRequested = true; + + // A running sync stops between blocks and releases the lock itself. If the block in progress + // does not finish in time, the lock stays: the next start sees it and rebuilds the address index. + const running = this.currentSync; + if (running) { + let timer: NodeJS.Timeout | undefined; + const finished = await Promise.race([ + running.then(() => true, () => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), this.stopTimeoutMs); + }), + ]); + clearTimeout(timer); + if (!finished) { + logger.warn(`Sync did not stop within ${this.stopTimeoutMs} ms; leaving its lock for startup recovery.`); + return; + } + logger.info('Sync service stopped'); + return; + } await SyncState.findOneAndUpdate( { key: 'main' }, @@ -623,10 +664,20 @@ class SyncService { } async sync(): Promise { - if (this.isRunning) { + if (this.isRunning || this.currentSync || this.stopRequested) { return; } + const run = this.runSync(); + this.currentSync = run; + try { + await run; + } finally { + if (this.currentSync === run) this.currentSync = null; + } + } + + private async runSync(): Promise { const syncStartedAt = new Date(); // Atomic DB-level lock: only acquire if not already running. @@ -719,6 +770,8 @@ class SyncService { let lastCheckpointAt = Date.now(); while (currentHeight <= daemonHeight) { + // A requested stop takes effect between blocks, never in the middle of one. + if (this.stopRequested) break; const processedBlock = await this.processBlock(currentHeight); lastHash = processedBlock.hash; lastKnownSyncedHeight = currentHeight; @@ -773,11 +826,12 @@ class SyncService { } } - // Final update + // Final update: the last block indexed, which is below the daemon's tip after a stop. + const reachedHeight = currentHeight - 1; await SyncState.findOneAndUpdate( { key: 'main' }, { - lastSyncedHeight: daemonHeight, + lastSyncedHeight: reachedHeight, lastSyncedHash: lastHash, lastSyncedAt: new Date(), isRunning: false, @@ -787,7 +841,12 @@ class SyncService { }, { upsert: true } ); - lastKnownSyncedHeight = daemonHeight; + lastKnownSyncedHeight = reachedHeight; + if (reachedHeight < daemonHeight) { + this.publishSyncStatus('idle', daemonHeight, reachedHeight, { message: 'Sync stopped.' }); + logger.info(`Sync stopped at height ${reachedHeight}.`); + return; + } this.publishSyncStatus('complete', daemonHeight, daemonHeight, { message: 'Sync complete.', }); @@ -864,6 +923,14 @@ class SyncService { rpcBlock = (await rpcService.getBlock(hash, 1)) as RpcBlock; } + // From here on the block is written piece by piece: each transaction (marked as applied), then + // the block's balance changes, then the block. Mark the block as in progress before the first + // write and clear the mark after the last. If anything in between fails, or the process stops, + // the mark stays and the next run rebuilds the address index from the stored transactions; the + // per-transaction marks then apply exactly what is still missing. If the mark cannot be + // written, nothing is. + const rebuildPendingBefore = await this.markBlockInProgress(); + const mintSat = coinToSat(typeof rpcBlock.mint === 'number' ? rpcBlock.mint : 0); let rewardSat = mintSat; let totalValueOutSat = 0n; @@ -894,7 +961,7 @@ class SyncService { rpcTx = entry; } - const result = await this.processTransaction(rpcTx, rpcBlock, { deferAddressUpdates: true }); + const result = await this.processTransaction(rpcTx, rpcBlock); if (result.addressDeltas) { mergeAddressDeltas(blockAddressDeltas, result.addressDeltas); } @@ -973,6 +1040,14 @@ class SyncService { ); } + // Everything of this block is written. A rebuild that was already pending stays pending. + if (!rebuildPendingBefore) { + await SyncState.findOneAndUpdate( + { key: 'main' }, + { addressRebuildRequired: false, heartbeatAt: new Date() } + ); + } + return { hash: rpcBlock.hash, height: rpcBlock.height, @@ -984,10 +1059,13 @@ class SyncService { // ── Transaction Processing ────────────────────────────────────── + /** + * Stores one transaction and returns its balance changes, if they have not been applied before. + * The caller (processBlock) merges them and writes them once for the whole block. + */ private async processTransaction( rpcTx: RpcTransaction, - rpcBlock: RpcBlock, - opts?: { deferAddressUpdates?: boolean } + rpcBlock: RpcBlock ): Promise<{ totalValueOutSat: bigint; isCoinbase: boolean; @@ -1131,25 +1209,11 @@ class SyncService { canApplyAddressDeltasForTx && previousTx?.addressUpdatesApplied !== true; if (shouldApplyAddressDeltas) { const addressDeltas = collectAddressDeltas(vins, vouts, isCoinbase); - - if (opts?.deferAddressUpdates) { - return { - totalValueOutSat, - isCoinbase, - addressDeltas: addressDeltas.size > 0 ? addressDeltas : undefined, - }; - } - - if (addressDeltas.size > 0) { - try { - await this.applyAddressDeltas(addressDeltas, rpcBlock.time); - } catch (error) { - await this.markAddressRebuildRequired( - `Address update failed for tx ${rpcTx.txid}; forcing deterministic rebuild.` - ); - throw error; - } - } + return { + totalValueOutSat, + isCoinbase, + addressDeltas: addressDeltas.size > 0 ? addressDeltas : undefined, + }; } return { totalValueOutSat, isCoinbase }; diff --git a/server/test/syncInterruption.integration.test.ts b/server/test/syncInterruption.integration.test.ts new file mode 100644 index 0000000..d2343a9 --- /dev/null +++ b/server/test/syncInterruption.integration.test.ts @@ -0,0 +1,292 @@ +import mongoose from 'mongoose'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { syncService } from '../src/services/sync.service'; +import { rpcService } from '../src/services/rpc.service'; +import { realtimeService } from '../src/services/realtime.service'; +import { telegramService } from '../src/services/telegram.service'; +import { Address } from '../src/models/Address'; +import { Block } from '../src/models/Block'; +import { SyncState } from '../src/models/SyncState'; +import { Transaction } from '../src/models/Transaction'; + +// This suite requires an explicitly supplied disposable localhost database. +// The normal unit suite must never connect to an operator's configured MongoDB. +const baseUri = process.env.TEST_POSE_MONGO_URI; + +const A = 'DNSSwp3CWfyus5h8TSBvJHFoTUASQCF8Rj'; +const B = 'D8e1nUDayNHvXSYZPv1QECNcSgCxgsXw9d'; +const C = 'DGK3yNxxedc2pvaPPwTbVBey2NmFC8FUGZ'; +const D = 'DKgo69ycaxuut3QxaTSfyqbkFvyRvvq8Hf'; +const E = 'D9NofFTND226dT475V33QaHU9LKHK7UwVe'; +const COIN = 100_000_000; + +type RpcVin = { coinbase?: string; txid?: string; vout?: number; sequence: number; address?: string; value?: number; valueSat?: number }; +type RpcVout = { value: number; valueSat: number; n: number; scriptPubKey: { asm: string; hex: string; type: string; addresses: string[] } }; +type RpcTx = { txid: string; version: number; size: number; locktime: number; vin: RpcVin[]; vout: RpcVout[] }; +type RpcBlock = { hash: string; height: number; time: number; tx: RpcTx[]; previousblockhash?: string; nextblockhash?: string } & Record; + +const hashOf = (height: number) => height.toString(16).padStart(64, 'b'); +const txidOf = (label: string) => label.padStart(64, '0'); +const pay = (n: number, coins: number, address: string): RpcVout => ({ + value: coins, valueSat: Math.round(coins * COIN), n, + scriptPubKey: { asm: '', hex: '76a914' + '00'.repeat(20) + '88ac', type: 'pubkeyhash', addresses: [address] }, +}); +const spend = (prev: string, n: number, coins: number, address: string): RpcVin => ({ + txid: txidOf(prev), vout: n, sequence: 0, address, value: coins, valueSat: Math.round(coins * COIN), +}); +const coinbase = (label: string, coins: number, address: string): RpcTx => ({ + txid: txidOf(label), version: 3, size: 100, locktime: 0, vin: [{ coinbase: '03aa0d02', sequence: 0 }], vout: [pay(0, coins, address)], +}); +const transfer = (label: string, vin: RpcVin[], vout: RpcVout[]): RpcTx => ({ txid: txidOf(label), version: 3, size: 225, locktime: 0, vin, vout }); + +// A small chain shaped like the daemon's verbose blocks. Block 2 has a coinbase and two transfers. +const TXS: RpcTx[][] = [ + [coinbase('c0', 1000, A)], + [coinbase('c1', 1000, B)], + [ + coinbase('c2', 50, C), + transfer('x2', [spend('c0', 0, 1000, A)], [pay(0, 400, D), pay(1, 599.99, A)]), + transfer('y2', [spend('c1', 0, 1000, B)], [pay(0, 100, D), pay(1, 899.99, B)]), + ], + [ + coinbase('c3', 50, C), + transfer('z3', [spend('x2', 0, 400, D)], [pay(0, 399.99, E)]), + ], +]; +const BLOCKS: RpcBlock[] = TXS.map((tx, height) => ({ + hash: hashOf(height), confirmations: 1, size: 500, height, version: 1, merkleroot: 'm'.repeat(64), tx, + time: 1_800_000_000 + height * 150, mediantime: 1_800_000_000 + height * 150, nonce: 0, bits: '1d00ffff', + difficulty: 1, chainwork: '00', nTx: tx.length, + ...(height > 0 ? { previousblockhash: hashOf(height - 1) } : {}), + ...(height < TXS.length - 1 ? { nextblockhash: hashOf(height + 1) } : {}), +})); +const BLOCK2 = BLOCKS[2]; + +/** What every address should hold after the blocks up to `height`, from the fixture alone. */ +function expectedBalances(height: number) { + const totals = new Map(); + const entry = (address: string) => { + const found = totals.get(address) ?? { received: 0n, sent: 0n }; + totals.set(address, found); + return found; + }; + for (const block of BLOCKS.slice(0, height + 1)) { + for (const tx of block.tx) { + for (const output of tx.vout) for (const address of output.scriptPubKey.addresses) entry(address).received += BigInt(output.valueSat); + for (const input of tx.vin) if (input.address) entry(input.address).sent += BigInt(input.valueSat ?? 0); + } + } + return Object.fromEntries([...totals].sort(([a], [b]) => a.localeCompare(b)).map(([address, { received, sent }]) => + [address, { balance: String(received - sent), received: String(received), sent: String(sent) }])); +} + +async function storedBalances() { + const docs = await Address.find({}).lean>(); + return Object.fromEntries(docs.sort((a, b) => a.address.localeCompare(b.address)).map((doc) => + [doc.address, { balance: String(doc.balanceSat), received: String(doc.totalReceivedSat), sent: String(doc.totalSentSat) }])); +} + +const syncState = () => SyncState.findOne({ key: 'main' }).lean>(); + +/** The daemon, answering from the fixture up to `tip`. `hold` keeps a block's answer back until released. */ +function mockDaemon(tip: number) { + vi.spyOn(rpcService, 'getBlockCount').mockResolvedValue(tip); + vi.spyOn(rpcService, 'getBlockHash').mockImplementation(async (height: number) => hashOf(height)); + return vi.spyOn(rpcService, 'getBlock').mockImplementation(async (hash: string) => { + const block = BLOCKS.find((candidate) => candidate.hash === hash); + if (!block) throw new Error(`unknown block ${hash}`); + return structuredClone(block); + }); +} + +type AnyMethods = Record unknown>; + +/** Makes the first call of `method` that `matches` its arguments throw; every other call goes through. */ +function failOnce(model: object, method: string, matches: (...args: unknown[]) => boolean) { + const target = model as AnyMethods; + const original = target[method].bind(model); + const state = { failed: false }; + vi.spyOn(target, method).mockImplementation((...args: unknown[]) => { + if (!state.failed && matches(...args)) { + state.failed = true; + throw new Error(`injected failure in ${method}`); + } + return original(...args); + }); + return state; +} + +/** Holds the first call of `method` that `matches` until `release()`; every other call goes through. */ +function holdOnce(model: object, method: string, matches: (...args: unknown[]) => boolean) { + const target = model as AnyMethods; + const original = target[method].bind(model); + let release = () => {}; + const gate = new Promise((resolve) => { release = resolve; }); + const state = { reached: false, release: () => release() }; + vi.spyOn(target, method).mockImplementation((...args: unknown[]) => { + if (!state.reached && matches(...args)) { + state.reached = true; + // A thenable query stand-in: the real query runs once the gate opens. + const run = gate.then(() => original(...args) as Promise); + const chain = { select: () => chain, lean: () => chain, then: run.then.bind(run), catch: run.catch.bind(run) }; + return chain; + } + return original(...args); + }); + return state; +} + +const internals = syncService as unknown as { stopRequested: boolean; stopTimeoutMs: number }; + +describe.skipIf(!baseUri)('indexing a block that is interrupted, with disposable MongoDB', () => { + beforeAll(async () => { + if (!baseUri || !/^mongodb:\/\/127\.0\.0\.1:\d+\/deftrack_pose_test_[a-f0-9]+$/.test(baseUri)) { + throw new Error('TEST_POSE_MONGO_URI must name an isolated localhost test database'); + } + // Own database, so it cannot collide with other integration suites that may run at the same time. + await mongoose.connect(`${baseUri}si`, { serverSelectionTimeoutMS: 5000 }); + await Promise.all([Block.init(), Transaction.init(), Address.init(), SyncState.init()]); + }); + beforeEach(async () => { + internals.stopRequested = false; + vi.spyOn(realtimeService, 'publishNewBlock').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishNewTx').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishSyncStatus').mockImplementation(() => undefined); + vi.spyOn(telegramService, 'notifyNewBlock').mockImplementation(() => undefined); + await Promise.all([Block.deleteMany({}), Transaction.deleteMany({}), Address.deleteMany({}), SyncState.deleteMany({})]); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + }); + afterAll(async () => { + internals.stopRequested = false; + if (mongoose.connection.readyState === 1) await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + it('the fixture chain indexes to the expected balances without failures', async () => { + mockDaemon(3); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 3, isRunning: false, addressRebuildRequired: false, error: null }); + }); + + it('T1: a transaction write that fails halfway through a block loses no balance change', async () => { + mockDaemon(3); + const failure = failOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('x2')); + + await syncService.sync(); + expect(failure.failed).toBe(true); + expect(await syncState()).toMatchObject({ isRunning: false, error: 'injected failure in findOneAndUpdate' }); + + await syncService.sync(); + expect(await storedBalances()).toEqual(expectedBalances(3)); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 3, addressRebuildRequired: false, error: null }); + }); + + it('T2: a failed balance write counts nothing twice', async () => { + mockDaemon(3); + const failure = failOnce(Address, 'bulkWrite', (ops) => + Array.isArray(ops) && ops.some((op) => (op as { updateOne?: { filter?: { address?: string } } }).updateOne?.filter?.address === D)); + + await syncService.sync(); + expect(failure.failed).toBe(true); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); + + it('T3: a failed block write counts nothing twice', async () => { + mockDaemon(3); + const failure = failOnce(Block, 'findOneAndUpdate', (filter, update) => + (filter as { hash?: string })?.hash === BLOCK2.hash && (update as { height?: number })?.height === 2); + + await syncService.sync(); + expect(failure.failed).toBe(true); + expect(await Block.findOne({ height: 2 }).lean()).toBeNull(); + await syncService.sync(); + + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); + + it('T4: stop() waits for the block being indexed, then stops before the next one', async () => { + mockDaemon(3); + const held = holdOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('y2')); + + const running = syncService.sync(); + let stopped = false; + try { + await vi.waitFor(() => expect(held.reached).toBe(true)); + const stopping = syncService.stop().then(() => { stopped = true; }); + await new Promise((resolve) => setTimeout(resolve, 300)); + // Still waiting: block 2 is half written. + expect(stopped).toBe(false); + held.release(); + await stopping; + } finally { + held.release(); + await running; + } + + expect(await Block.findOne({ height: 2 }).lean()).not.toBeNull(); + expect(await Block.findOne({ height: 3 }).lean()).toBeNull(); + expect(await syncState()).toMatchObject({ + isRunning: false, addressRebuildRequired: false, lastSyncedHeight: 2, lastSyncedHash: BLOCK2.hash, + }); + expect(await storedBalances()).toEqual(expectedBalances(2)); + // A stopped service does not start another run: it neither asks the daemon nor writes the state. + const daemonCalls = vi.mocked(rpcService.getBlockCount).mock.calls.length; + const stateBefore = await syncState(); + await syncService.sync(); + expect(vi.mocked(rpcService.getBlockCount).mock.calls.length).toBe(daemonCalls); + expect(await syncState()).toEqual(stateBefore); + expect(await Block.findOne({ height: 3 }).lean()).toBeNull(); + }); + + it('T5: when the block does not finish in time, stop() leaves the lock for startup recovery', async () => { + mockDaemon(3); + const held = holdOnce(Transaction, 'findOneAndUpdate', (filter) => (filter as { txid?: string })?.txid === txidOf('y2')); + const timeoutBefore = internals.stopTimeoutMs; + internals.stopTimeoutMs = 200; + + const running = syncService.sync(); + await vi.waitFor(() => expect(held.reached).toBe(true)); + try { + await syncService.stop(); + expect(await syncState()).toMatchObject({ isRunning: true, addressRebuildRequired: true }); + } finally { + internals.stopTimeoutMs = timeoutBefore; + held.release(); + await running; + } + }); + + it('a block does not clear a rebuild that was already requested before it started', async () => { + mockDaemon(0); + await SyncState.updateOne({ key: 'main' }, { addressRebuildRequired: true }); + + await (syncService as unknown as { processBlock: (height: number) => Promise }).processBlock(0); + + expect(await Block.findOne({ height: 0 }).lean()).not.toBeNull(); + expect(await syncState()).toMatchObject({ addressRebuildRequired: true }); + }); + + it('T6: when the in-progress mark cannot be written, the block is not started', async () => { + mockDaemon(1); + await syncService.sync(); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 1, addressRebuildRequired: false }); + + mockDaemon(3); + const failure = failOnce(SyncState, 'findOneAndUpdate', (_filter, update) => { + const fields = update as { addressRebuildRequired?: boolean; $set?: { addressRebuildRequired?: boolean } }; + return fields?.addressRebuildRequired === true || fields?.$set?.addressRebuildRequired === true; + }); + await syncService.sync(); + + expect(failure.failed).toBe(true); + expect(await Transaction.countDocuments({ blockheight: 2 })).toBe(0); + expect(await syncState()).toMatchObject({ isRunning: false, error: 'injected failure in findOneAndUpdate' }); + + await syncService.sync(); + expect(await storedBalances()).toEqual(expectedBalances(3)); + }); +}); From 5fc459d051e754e7f6a812c23c65cf85b79a66d5 Mon Sep 17 00:00:00 2001 From: minuszka <101508242+minuszka@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:13:47 +0100 Subject: [PATCH 2/2] Let indexing recover by itself after a database hiccup If the database failed at the wrong moment, the indexer could stay blocked until the next restart, and a database reconnect started a second copy of the indexer and of the Telegram bot. - The indexing lock now expires: if its holder has shown no sign of life for 10 minutes (each indexed block and each step of an address rebuild counts), the next run takes it over and first rebuilds the address balances. Before, a lock left behind by a failed run made every later run fail until a restart. - When recording an indexing error fails as well, the original error is still logged and the run ends normally. - Starting the indexer or the Telegram bot a second time, as happens after a database reconnect, no longer starts a second copy. - Checked with database tests for an expired lock, a recent lock, a failed error record, a long rebuild and repeated starts; each part of the fix was undone once to confirm that a test fails without it. --- server/src/services/sync.service.ts | 108 ++++++--- server/src/services/telegram.service.ts | 6 + server/test/syncLock.integration.test.ts | 283 +++++++++++++++++++++++ server/test/telegram.service.test.ts | 65 ++++++ 4 files changed, 431 insertions(+), 31 deletions(-) create mode 100644 server/test/syncLock.integration.test.ts create mode 100644 server/test/telegram.service.test.ts diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index 95f652e..b34f738 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -64,6 +64,11 @@ class SyncService { private currentSync: Promise | null = null; // How long stop() waits for the current block before it leaves the lock to startup recovery. private stopTimeoutMs = 15_000; + // Set by start() until stop(): a second start() (for example after a database reconnect) does nothing. + private started = false; + // A lock whose holder has shown no heartbeat for this long may be taken over. Each indexed block + // and each batch of an address rebuild writes a heartbeat, so this is far above any normal gap. + private readonly SYNC_LOCK_EXPIRY_MS = 10 * 60 * 1000; private readonly LOG_INTERVAL = 100; private readonly REBUILD_LOG_INTERVAL = 10000; private readonly ADDRESS_BULK_FLUSH_THRESHOLD = 2000; @@ -254,6 +259,8 @@ class SyncService { if (bulkOps.length === 0) return; await Address.bulkWrite(bulkOps, { ordered: false }); bulkOps.length = 0; + // A long rebuild keeps its sync lock from looking abandoned (see acquireSyncLock). + await SyncState.updateOne({ key: 'main' }, { heartbeatAt: new Date() }); }; for await (const tx of cursor) { @@ -595,21 +602,31 @@ class SyncService { } async start(): Promise { - if (this.isRunning) { + if (this.started || this.isRunning) { logger.warn('Sync service is already running'); return; } + this.started = true; logger.info('Starting blockchain sync service...'); this.stopRequested = false; - await this.ensureIndexVersion(); - await this.ensureSatoshiDataVersion(); - await this.recoverStartupState(); - await this.addressResolver.preInferVersionByte(); + try { + await this.ensureIndexVersion(); + await this.ensureSatoshiDataVersion(); + await this.recoverStartupState(); + await this.addressResolver.preInferVersionByte(); + + // Run initial sync + await this.sync(); + } catch (error) { + // A failed start can be tried again, for example on the next database reconnect. + this.started = false; + throw error; + } - // Run initial sync - await this.sync(); + // Stopped while starting: no periodic sync. + if (this.stopRequested) return; // Set up periodic sync for new blocks. // Guard: skip if previous sync is still running to prevent overlap. @@ -629,6 +646,7 @@ class SyncService { this.intervalId = null; } this.stopRequested = true; + this.started = false; // A running sync stops between blocks and releases the lock itself. If the block in progress // does not finish in time, the lock stays: the next start sees it and rebuilds the address index. @@ -677,23 +695,46 @@ class SyncService { } } - private async runSync(): Promise { - const syncStartedAt = new Date(); + /** + * Atomic database lock, so that only one run indexes at a time. It is taken when free, or when + * its holder has shown no heartbeat for SYNC_LOCK_EXPIRY_MS (or never did): a run that died + * without releasing it, for example because the database error that ended it also stopped its + * final write. Taking over an expired lock requests an address rebuild, as startup recovery does + * for a lock left behind, since the holder may have stopped in the middle of a block. + */ + private async acquireSyncLock(now: Date): Promise { + // The state document normally exists already (ensureIndexVersion writes it). Create it if it + // does not, without touching an existing one, so the lock updates below need no upsert: an + // upsert against a held lock would try to insert a second document with the same key. + await SyncState.updateOne({ key: 'main' }, { $setOnInsert: { key: 'main' } }, { upsert: true }); + + const lock = { isRunning: true, startedAt: now, heartbeatAt: now, error: null }; + const free = await SyncState.findOneAndUpdate({ key: 'main', isRunning: { $ne: true } }, lock, { new: true }) + .select({ _id: 1 }) + .lean(); + if (free) return true; - // Atomic DB-level lock: only acquire if not already running. - // This prevents race conditions when multiple instances or overlapping calls exist. - const lockAcquired = await SyncState.findOneAndUpdate( - { key: 'main', isRunning: { $ne: true } }, - { - isRunning: true, - startedAt: syncStartedAt, - heartbeatAt: syncStartedAt, - error: null, - }, - { upsert: true, new: true } + const expiredBefore = new Date(now.getTime() - this.SYNC_LOCK_EXPIRY_MS); + const takenOver = await SyncState.findOneAndUpdate( + { key: 'main', isRunning: true, $or: [{ heartbeatAt: { $lt: expiredBefore } }, { heartbeatAt: null }] }, + { ...lock, addressRebuildRequired: true }, + { new: true } + ) + .select({ _id: 1 }) + .lean(); + if (!takenOver) return false; + + logger.warn( + `Took over a sync lock without a heartbeat for ${this.SYNC_LOCK_EXPIRY_MS / 60_000} minutes; ` + + 'rebuilding the address index first.' ); + return true; + } + + private async runSync(): Promise { + const syncStartedAt = new Date(); - if (!lockAcquired || lockAcquired.isRunning !== true) { + if (!(await this.acquireSyncLock(syncStartedAt))) { logger.debug('Sync lock not acquired — another instance is running'); return; } @@ -855,16 +896,21 @@ class SyncService { } catch (error) { logger.error('Sync error:', error); const errorMessage = error instanceof Error ? error.message : 'Unknown sync error'; - await SyncState.findOneAndUpdate( - { key: 'main' }, - { - isRunning: false, - startedAt: null, - heartbeatAt: new Date(), - error: errorMessage, - }, - { upsert: true } - ); + try { + await SyncState.findOneAndUpdate( + { key: 'main' }, + { + isRunning: false, + startedAt: null, + heartbeatAt: new Date(), + error: errorMessage, + }, + { upsert: true } + ); + } catch (stateError) { + // Often the same database trouble. The lock stays held until it expires (acquireSyncLock). + logger.error('Could not record the sync error or release the sync lock:', stateError); + } // WebSocket clients are public: broadcast the masked message, not the raw one. this.publishSyncStatus('error', daemonHeight, lastKnownSyncedHeight, { error: mapPublicSyncError(errorMessage, true).error, diff --git a/server/src/services/telegram.service.ts b/server/src/services/telegram.service.ts index 2617238..5cf2386 100644 --- a/server/src/services/telegram.service.ts +++ b/server/src/services/telegram.service.ts @@ -32,6 +32,10 @@ class TelegramService { return; } + // Started already (start runs again after a database reconnect): a second bot would compete + // for the same updates and send every alert twice. + if (this.bot) return; + this.bot = new Telegraf(token); this.enabled = true; this.registerCommands(); @@ -48,6 +52,8 @@ class TelegramService { stop(): void { this.bot?.stop(); + this.bot = null; + this.enabled = false; if (this.checkTimer) { clearInterval(this.checkTimer); this.checkTimer = null; diff --git a/server/test/syncLock.integration.test.ts b/server/test/syncLock.integration.test.ts new file mode 100644 index 0000000..3124ee0 --- /dev/null +++ b/server/test/syncLock.integration.test.ts @@ -0,0 +1,283 @@ +import mongoose from 'mongoose'; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { config } from '../src/config'; +import { syncService } from '../src/services/sync.service'; +import { rpcService } from '../src/services/rpc.service'; +import { realtimeService } from '../src/services/realtime.service'; +import { telegramService } from '../src/services/telegram.service'; +import { Address } from '../src/models/Address'; +import { Block } from '../src/models/Block'; +import { SyncState } from '../src/models/SyncState'; +import { Transaction } from '../src/models/Transaction'; + +// This suite requires an explicitly supplied disposable localhost database. +// The normal unit suite must never connect to an operator's configured MongoDB. +const baseUri = process.env.TEST_POSE_MONGO_URI; + +const A = 'DNSSwp3CWfyus5h8TSBvJHFoTUASQCF8Rj'; +const B = 'D8e1nUDayNHvXSYZPv1QECNcSgCxgsXw9d'; +const STALE = 'D9NofFTND226dT475V33QaHU9LKHK7UwVe'; +const MINUTE = 60_000; + +const hashOf = (height: number) => height.toString(16).padStart(64, 'c'); +const pay = (n: number, coins: number, address: string) => ({ + value: coins, valueSat: coins * 100_000_000, n, + scriptPubKey: { asm: '', hex: '76a914' + '00'.repeat(20) + '88ac', type: 'pubkeyhash', addresses: [address] }, +}); +const coinbase = (height: number, address: string) => ({ + txid: `cb${height}`.padStart(64, '0'), version: 3, size: 100, locktime: 0, + vin: [{ coinbase: '03aa0d02', sequence: 0 }], vout: [pay(0, 1000, address)], +}); +// Three blocks, each a single coinbase: A, B, A. +const BLOCKS = [A, B, A].map((address, height) => ({ + hash: hashOf(height), confirmations: 1, size: 300, height, version: 1, merkleroot: 'm'.repeat(64), + tx: [coinbase(height, address)], time: 1_800_000_000 + height * 150, mediantime: 1_800_000_000 + height * 150, + nonce: 0, bits: '1d00ffff', difficulty: 1, chainwork: '00', nTx: 1, + ...(height > 0 ? { previousblockhash: hashOf(height - 1) } : {}), +})); + +/** + * The daemon, answering from the fixture up to `tip`. `failBlock` makes that block's first two fetches + * fail: the indexer retries a failed verbose fetch once with a lighter one. + */ +function mockDaemon(tip: number, options: { failBlock?: number } = {}) { + let failures = 0; + const count = vi.spyOn(rpcService, 'getBlockCount').mockResolvedValue(tip); + vi.spyOn(rpcService, 'getBlockHash').mockImplementation(async (height: number) => hashOf(height)); + vi.spyOn(rpcService, 'getBlock').mockImplementation(async (hash: string) => { + const block = BLOCKS.find((candidate) => candidate.hash === hash); + if (!block) throw new Error(`unknown block ${hash}`); + if (block.height === options.failBlock && failures < 2) { + failures += 1; + throw new Error('daemon unavailable'); + } + return structuredClone(block); + }); + return count; +} + +const syncState = () => SyncState.findOne({ key: 'main' }).lean>(); +const balanceOf = async (address: string) => { + const doc = await Address.findOne({ address }).lean<{ balanceSat?: unknown }>(); + return doc ? String(doc.balanceSat) : null; +}; +const ago = (ms: number) => new Date(Date.now() - ms); + +const internals = syncService as unknown as { + stopRequested: boolean; + ADDRESS_BULK_FLUSH_THRESHOLD: number; + rebuildAddressIndexFromTransactions: () => Promise; +}; + +describe.skipIf(!baseUri)('the sync lock and the service start, with disposable MongoDB', () => { + beforeAll(async () => { + if (!baseUri || !/^mongodb:\/\/127\.0\.0\.1:\d+\/deftrack_pose_test_[a-f0-9]+$/.test(baseUri)) { + throw new Error('TEST_POSE_MONGO_URI must name an isolated localhost test database'); + } + // Own database, so it cannot collide with other integration suites that may run at the same time. + await mongoose.connect(`${baseUri}sl`, { serverSelectionTimeoutMS: 5000 }); + await Promise.all([Block.init(), Transaction.init(), Address.init(), SyncState.init()]); + }); + beforeEach(async () => { + internals.stopRequested = false; + vi.spyOn(realtimeService, 'publishNewBlock').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishNewTx').mockImplementation(() => undefined); + vi.spyOn(realtimeService, 'publishSyncStatus').mockImplementation(() => undefined); + vi.spyOn(telegramService, 'notifyNewBlock').mockImplementation(() => undefined); + await Promise.all([Block.deleteMany({}), Transaction.deleteMany({}), Address.deleteMany({}), SyncState.deleteMany({})]); + }); + afterAll(async () => { + internals.stopRequested = false; + if (mongoose.connection.readyState === 1) await mongoose.connection.dropDatabase(); + await mongoose.disconnect(); + }); + + /** A lock held by a run that last showed a heartbeat at `heartbeatAt` (null: never). */ + async function heldLock(heartbeatAt: Date | null) { + const startedAt = ago(30 * MINUTE); + await SyncState.create({ + key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3, + isRunning: true, startedAt, heartbeatAt, + }); + // A balance only a rebuild removes: no stored transaction pays this address. + await Address.create({ address: STALE, balanceSat: '123', totalReceivedSat: '123', totalSentSat: '0', txCount: 1 }); + return startedAt; + } + + it('takes over a lock whose holder has shown no heartbeat for over 10 minutes, and rebuilds first', async () => { + mockDaemon(2); + await heldLock(ago(11 * MINUTE)); + + await expect(syncService.sync()).resolves.toBeUndefined(); + + expect(await balanceOf(STALE)).toBeNull(); + expect(await balanceOf(A)).toBe('200000000000'); + expect(await syncState()).toMatchObject({ isRunning: false, addressRebuildRequired: false, lastSyncedHeight: 2, error: null }); + }); + + it('takes over a held lock that never had a heartbeat', async () => { + mockDaemon(2); + await heldLock(null); + + await expect(syncService.sync()).resolves.toBeUndefined(); + + expect(await balanceOf(STALE)).toBeNull(); + expect(await syncState()).toMatchObject({ isRunning: false, lastSyncedHeight: 2 }); + }); + + it('leaves a lock with a recent heartbeat alone, without an error', async () => { + const daemon = mockDaemon(2); + const startedAt = await heldLock(ago(1 * MINUTE)); + + await expect(syncService.sync()).resolves.toBeUndefined(); + + expect(daemon).not.toHaveBeenCalled(); + expect(await Block.countDocuments()).toBe(0); + expect(await balanceOf(STALE)).toBe('123'); + expect(await syncState()).toMatchObject({ isRunning: true, startedAt }); + }); + + it('when the error cannot be recorded, the run still ends quietly and the lock is taken over once it expires', async () => { + mockDaemon(2, { failBlock: 1 }); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + const original = SyncState.findOneAndUpdate.bind(SyncState); + let recordFailed = false; + vi.spyOn(SyncState, 'findOneAndUpdate').mockImplementation(((filter: unknown, update: { error?: unknown }, ...rest: unknown[]) => { + if (!recordFailed && typeof update?.error === 'string') { + recordFailed = true; + throw new Error('state write failed'); + } + return (original as (...args: unknown[]) => unknown)(filter, update, ...rest); + }) as never); + + await expect(syncService.sync()).resolves.toBeUndefined(); + expect(recordFailed).toBe(true); + // The lock could not be released, so it is still held. + expect(await syncState()).toMatchObject({ isRunning: true }); + + // Not yet expired: the next run leaves it alone. + await expect(syncService.sync()).resolves.toBeUndefined(); + expect(await Block.countDocuments({ height: { $gte: 1 } })).toBe(0); + + // Expired: the next run takes it over and finishes the chain. + await SyncState.updateOne({ key: 'main' }, { heartbeatAt: ago(11 * MINUTE) }); + await expect(syncService.sync()).resolves.toBeUndefined(); + expect(await syncState()).toMatchObject({ isRunning: false, lastSyncedHeight: 2, addressRebuildRequired: false }); + expect(await balanceOf(B)).toBe('100000000000'); + }); + + it('a rebuild keeps the lock alive with a heartbeat while it writes', async () => { + mockDaemon(2); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + await syncService.sync(); + await SyncState.updateOne({ key: 'main' }, { heartbeatAt: ago(11 * MINUTE) }); + const flushBefore = internals.ADDRESS_BULK_FLUSH_THRESHOLD; + internals.ADDRESS_BULK_FLUSH_THRESHOLD = 1; + const beats: Date[] = []; + const record = (update: unknown) => { + const fields = update as { heartbeatAt?: Date; $set?: { heartbeatAt?: Date } }; + const at = fields?.heartbeatAt ?? fields?.$set?.heartbeatAt; + if (at) beats.push(at); + }; + const updateOne = SyncState.updateOne.bind(SyncState); + vi.spyOn(SyncState, 'updateOne').mockImplementation(((filter: unknown, update: unknown, ...rest: unknown[]) => { + record(update); + return (updateOne as (...args: unknown[]) => unknown)(filter, update, ...rest); + }) as never); + const findOneAndUpdate = SyncState.findOneAndUpdate.bind(SyncState); + vi.spyOn(SyncState, 'findOneAndUpdate').mockImplementation(((filter: unknown, update: unknown, ...rest: unknown[]) => { + record(update); + return (findOneAndUpdate as (...args: unknown[]) => unknown)(filter, update, ...rest); + }) as never); + + try { + await internals.rebuildAddressIndexFromTransactions(); + } finally { + internals.ADDRESS_BULK_FLUSH_THRESHOLD = flushBefore; + } + + // Three coinbases, one balance write each: at least one heartbeat per write. + expect(beats.length).toBeGreaterThanOrEqual(3); + const state = await syncState(); + expect(Date.now() - new Date(state?.heartbeatAt as Date).getTime()).toBeLessThan(MINUTE); + }); + + it('a start that failed can be tried again', async () => { + mockDaemon(1); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + const findOne = SyncState.findOne.bind(SyncState); + let failed = false; + vi.spyOn(SyncState, 'findOne').mockImplementation(((...args: unknown[]) => { + if (!failed) { + failed = true; + throw new Error('database not ready'); + } + return (findOne as (...a: unknown[]) => unknown)(...args); + }) as never); + const intervals = vi.spyOn(globalThis, 'setInterval'); + + try { + await expect(syncService.start()).rejects.toThrow('database not ready'); + await syncService.start(); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 1 }); + } finally { + await syncService.stop(); + for (const result of intervals.mock.results) clearInterval(result.value as ReturnType); + } + }); + + it('a service stopped while it is starting sets up no periodic sync', async () => { + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + mockDaemon(1); + let release = () => {}; + const gate = new Promise((resolve) => { release = resolve; }); + let reached = false; + vi.spyOn(rpcService, 'getBlockCount').mockImplementation(async () => { + reached = true; + await gate; + return 1; + }); + const intervals = vi.spyOn(globalThis, 'setInterval'); + const periodic = () => intervals.mock.calls.filter((call) => call[1] === config.sync.intervalMs).length; + + const starting = syncService.start(); + try { + await vi.waitFor(() => expect(reached).toBe(true)); + const stopping = syncService.stop(); + release(); + await stopping; + await starting; + expect(periodic()).toBe(0); + } finally { + release(); + await starting.catch(() => undefined); + await syncService.stop(); + for (const result of intervals.mock.results) clearInterval(result.value as ReturnType); + } + }); + + it('start() twice sets up one periodic sync, and a stopped service can start again', async () => { + mockDaemon(1); + await SyncState.create({ key: 'main', lastSyncedHeight: -1, lastSyncedHash: '', addressIndexVersion: 3, satoshiDataVersion: 3 }); + const intervals = vi.spyOn(globalThis, 'setInterval'); + const created: Array> = []; + const periodic = () => intervals.mock.calls.filter((call) => call[1] === config.sync.intervalMs).length; + + try { + await Promise.all([syncService.start(), syncService.start()]); + for (const result of intervals.mock.results) created.push(result.value as ReturnType); + expect(periodic()).toBe(1); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 1 }); + + await syncService.stop(); + mockDaemon(2); + await syncService.start(); + for (const result of intervals.mock.results) created.push(result.value as ReturnType); + expect(periodic()).toBe(2); + expect(await syncState()).toMatchObject({ lastSyncedHeight: 2, isRunning: false }); + } finally { + await syncService.stop(); + for (const id of created) clearInterval(id); + } + }); +}); diff --git a/server/test/telegram.service.test.ts b/server/test/telegram.service.test.ts new file mode 100644 index 0000000..4aa0f26 --- /dev/null +++ b/server/test/telegram.service.test.ts @@ -0,0 +1,65 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ bots: [] as Array<{ launch: ReturnType; stop: ReturnType }> })); + +vi.mock('telegraf', () => ({ + Telegraf: class { + launch = vi.fn(() => Promise.resolve()); + stop = vi.fn(); + command = vi.fn(); + action = vi.fn(); + use = vi.fn(); + constructor() { + mocks.bots.push(this); + } + }, + Markup: { inlineKeyboard: vi.fn(), button: { callback: vi.fn() } }, +})); + +import { config } from '../src/config'; +import { telegramService } from '../src/services/telegram.service'; + +describe('telegram service start', () => { + const telegram = config.telegram as { botToken: string; adminId: number }; + const saved = { ...telegram }; + const signals = ['SIGINT', 'SIGTERM'] as const; + const listenersBefore = Object.fromEntries(signals.map((signal) => [signal, process.listeners(signal)])); + + afterEach(() => { + telegramService.stop(); + Object.assign(telegram, saved); + // Drop the shutdown handlers the service registered, so they do not outlive the test. + for (const signal of signals) { + for (const listener of process.listeners(signal)) { + if (!listenersBefore[signal].includes(listener)) process.removeListener(signal, listener); + } + } + mocks.bots.length = 0; + }); + + it('starts one bot, one stall check and one pair of shutdown handlers when started twice', () => { + Object.assign(telegram, { botToken: 'test-token', adminId: 42 }); + const intervals = vi.spyOn(globalThis, 'setInterval'); + const sigterm = process.listenerCount('SIGTERM'); + + telegramService.start(); + telegramService.start(); + + expect(mocks.bots).toHaveLength(1); + expect(mocks.bots[0].launch).toHaveBeenCalledTimes(1); + expect(intervals).toHaveBeenCalledTimes(1); + expect(process.listenerCount('SIGTERM')).toBe(sigterm + 1); + }); + + it('can start again after a stop', () => { + Object.assign(telegram, { botToken: 'test-token', adminId: 42 }); + + telegramService.start(); + telegramService.stop(); + telegramService.start(); + + expect(mocks.bots).toHaveLength(2); + expect(mocks.bots[0].stop).toHaveBeenCalledTimes(1); + expect(mocks.bots[1].launch).toHaveBeenCalledTimes(1); + }); +});