Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions server/src/models/SyncState.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export interface SyncStateDocument extends Document {
error: string | null;
addressIndexVersion?: number;
satoshiDataVersion?: number;
addressStatsVersion?: number;
}

const syncStateSchema = new Schema<SyncStateDocument>({
Expand All @@ -26,6 +27,8 @@ const syncStateSchema = new Schema<SyncStateDocument>({
error: { type: String, default: null },
addressIndexVersion: { type: Number, default: 0 },
satoshiDataVersion: { type: Number, default: 0 },
// Version of how address statistics are counted; an older one rebuilds the address index once.
addressStatsVersion: { type: Number, default: 0 },
});

export const SyncState = mongoose.model<SyncStateDocument>('SyncState', syncStateSchema);
72 changes: 43 additions & 29 deletions server/src/services/sync.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import {
collectInvolvedAddresses,
mergeAddressDeltas,
normalizeSat,
type AddressDelta,
} from './sync/addressIndexing';
import { runScheduledSyncAttempt } from './sync/syncScheduling';

Expand Down Expand Up @@ -77,6 +78,9 @@ class SyncService {
private readonly CURSOR_MAX_TIME_MS = Math.max(30_000, config.rpc.timeout);
private readonly ADDRESS_INDEX_VERSION = 3;
private readonly SATOSHI_DATA_VERSION = 3;
// 1: txCount counts each transaction an address takes part in (it counted blocks during live
// sync and transactions in a rebuild). An older stored version rebuilds the address index once.
private readonly ADDRESS_STATS_VERSION = 1;
private readonly SAT_MULTIPLIER = 10 ** COIN.DECIMALS;
private readonly TX_EVENT_MAX_PER_BLOCK = 250;

Expand Down Expand Up @@ -158,7 +162,8 @@ class SyncService {
return before?.addressRebuildRequired === true;
}

private async recoverStartupState(): Promise<void> {
/** Repairs what a previous run left behind; returns true when it rebuilt the address index. */
private async recoverStartupState(): Promise<boolean> {
const syncState = await SyncState.findOne({ key: 'main' }).lean();
const indexedTip = await Block.findOne({})
.sort({ height: -1 })
Expand All @@ -179,7 +184,7 @@ class SyncService {
},
{ upsert: true }
);
return;
return false;
}

// If sync state points past our indexed tip (possible after crash during rollback),
Expand Down Expand Up @@ -214,7 +219,7 @@ class SyncService {
);
}

if (!recoveryPlan.rebuildAddressIndex) return;
if (!recoveryPlan.rebuildAddressIndex) return false;

logger.warn('Detected pending address index rebuild flag. Rebuilding from transactions...');
await this.rebuildAddressIndexFromTransactions();
Expand All @@ -228,6 +233,33 @@ class SyncService {
{ upsert: true }
);
logger.info('Pending address rebuild recovered successfully.');
return true;
}

/**
* Brings the stored address statistics to ADDRESS_STATS_VERSION: if the stored version is older,
* rebuilds the address index once and records the version. A rebuild that startup recovery has
* just done counts, so a start never rebuilds twice. Interrupted before the version is written,
* the next start simply rebuilds again.
*/
private async ensureAddressStatsVersion(rebuiltThisStart: boolean): Promise<void> {
const syncState = await SyncState.findOne({ key: 'main' }).select({ addressStatsVersion: 1 }).lean<{ addressStatsVersion?: number }>();
const currentVersion = syncState?.addressStatsVersion ?? 0;
if (currentVersion >= this.ADDRESS_STATS_VERSION) return;

if (!rebuiltThisStart) {
logger.warn(
`Address statistics version ${currentVersion} -> ${this.ADDRESS_STATS_VERSION}: rebuilding the address index once...`
);
const startedAt = Date.now();
await this.rebuildAddressIndexFromTransactions();
logger.info(`Address statistics rebuild finished in ${Math.round((Date.now() - startedAt) / 1000)} s.`);
}
await SyncState.findOneAndUpdate(
{ key: 'main' },
{ addressStatsVersion: this.ADDRESS_STATS_VERSION, heartbeatAt: new Date() },
{ upsert: true }
);
}

// Rebuild address balances from stored transactions.
Expand Down Expand Up @@ -269,28 +301,8 @@ class SyncService {
continue;
}

const addressMap = new Map<string, { receivedSat: bigint; sentSat: bigint }>();

for (const vout of tx.vout || []) {
const addresses = vout.scriptPubKey?.addresses;
if (!addresses?.length) continue;
const valueSat = normalizeSat(vout.valueSat);
for (const addr of addresses) {
const entry = addressMap.get(addr) || { receivedSat: 0n, sentSat: 0n };
entry.receivedSat += valueSat;
addressMap.set(addr, entry);
}
}

if (!tx.isCoinbase) {
for (const vin of tx.vin || []) {
const valueSat = normalizeSat(vin.valueSat);
if (!vin.address || valueSat <= 0n) continue;
const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n };
entry.sentSat += valueSat;
addressMap.set(vin.address, entry);
}
}
// The same per-transaction changes live sync collects (processTransaction).
const addressMap = collectAddressDeltas(tx.vin || [], tx.vout || [], !!tx.isCoinbase);

for (const [addr, update] of addressMap) {
bulkOps.push({
Expand Down Expand Up @@ -614,7 +626,8 @@ class SyncService {
try {
await this.ensureIndexVersion();
await this.ensureSatoshiDataVersion();
await this.recoverStartupState();
const rebuiltThisStart = await this.recoverStartupState();
await this.ensureAddressStatsVersion(rebuiltThisStart);
await this.addressResolver.preInferVersionByte();

// Run initial sync
Expand Down Expand Up @@ -982,7 +995,7 @@ class SyncService {
let totalValueOutSat = 0n;
let minedBy: string | undefined;
const txids: string[] = [];
const blockAddressDeltas = new Map<string, { receivedSat: bigint; sentSat: bigint }>();
const blockAddressDeltas = new Map<string, AddressDelta>();

const txEntries = (rpcBlock.tx || []) as Array<string | RpcTransaction>;

Expand Down Expand Up @@ -1115,7 +1128,7 @@ class SyncService {
): Promise<{
totalValueOutSat: bigint;
isCoinbase: boolean;
addressDeltas?: Map<string, { receivedSat: bigint; sentSat: bigint }>;
addressDeltas?: Map<string, AddressDelta>;
}> {
const isCoinbase = rpcTx.vin.length > 0 && !!rpcTx.vin[0].coinbase;

Expand Down Expand Up @@ -1369,7 +1382,7 @@ class SyncService {
// ── Address Balance Updates ─────────────────────────────────────

private async applyAddressDeltas(
addressMap: Map<string, { receivedSat: bigint; sentSat: bigint }>,
addressMap: Map<string, AddressDelta>,
blocktime: number
): Promise<boolean> {
if (addressMap.size === 0) return false;
Expand Down Expand Up @@ -1455,6 +1468,7 @@ class SyncService {
error: null,
addressIndexVersion: this.ADDRESS_INDEX_VERSION,
satoshiDataVersion: this.SATOSHI_DATA_VERSION,
addressStatsVersion: this.ADDRESS_STATS_VERSION,
},
{ upsert: true }
);
Expand Down
11 changes: 7 additions & 4 deletions server/src/services/sync/addressIndexing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import { coinToSat, satToBigInt } from '../../utils/amounts';
export interface AddressDelta {
receivedSat: bigint;
sentSat: bigint;
/** How many transactions the address takes part in: 1 for one transaction, summed over a block. */
txCount: number;
}

interface VinLike {
Expand Down Expand Up @@ -57,7 +59,7 @@ export function collectAddressDeltas(
if (!addresses?.length) continue;
const valueSat = normalizeSat(vout.valueSat);
for (const address of addresses) {
const entry = addressMap.get(address) || { receivedSat: 0n, sentSat: 0n };
const entry = addressMap.get(address) || { receivedSat: 0n, sentSat: 0n, txCount: 1 };
entry.receivedSat += valueSat;
addressMap.set(address, entry);
}
Expand All @@ -66,7 +68,7 @@ export function collectAddressDeltas(
for (const vin of vins) {
const valueSat = normalizeSat(vin.valueSat);
if (!vin.address || valueSat <= 0n) continue;
const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n };
const entry = addressMap.get(vin.address) || { receivedSat: 0n, sentSat: 0n, txCount: 1 };
entry.sentSat += valueSat;
addressMap.set(vin.address, entry);
}
Expand All @@ -79,9 +81,10 @@ export function mergeAddressDeltas(
source: Map<string, AddressDelta>
): void {
for (const [address, delta] of source) {
const current = target.get(address) || { receivedSat: 0n, sentSat: 0n };
const current = target.get(address) || { receivedSat: 0n, sentSat: 0n, txCount: 0 };
current.receivedSat += delta.receivedSat;
current.sentSat += delta.sentSat;
current.txCount += delta.txCount;
target.set(address, current);
}
}
Expand All @@ -106,7 +109,7 @@ export function buildAddressDeltaPipeline(
balanceSat: { $add: [{ $ifNull: ['$balanceSat', zeroDec] }, netSatDec] },
totalReceivedSat: { $add: [{ $ifNull: ['$totalReceivedSat', zeroDec] }, receivedDec] },
totalSentSat: { $add: [{ $ifNull: ['$totalSentSat', zeroDec] }, sentDec] },
txCount: { $add: [{ $ifNull: ['$txCount', 0] }, 1] },
txCount: { $add: [{ $ifNull: ['$txCount', 0] }, delta.txCount] },
firstSeen: { $min: [{ $ifNull: ['$firstSeen', blocktime] }, blocktime] },
lastSeen: { $max: [{ $ifNull: ['$lastSeen', 0] }, blocktime] },
},
Expand Down
31 changes: 22 additions & 9 deletions server/test/addressIndexing.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import {
collectInvolvedAddresses,
mergeAddressDeltas,
normalizeSat,
type AddressDelta,
} from '../src/services/sync/addressIndexing';

describe('address indexing domain', () => {
Expand Down Expand Up @@ -32,41 +33,53 @@ describe('address indexing domain', () => {
expect(collectInvolvedAddresses(vins, vouts)).toEqual(['Dsender', 'Dreceiver']);
expect(collectAddressDeltas(vins, vouts, false)).toEqual(
new Map([
['Dreceiver', { receivedSat: 7n, sentSat: 0n }],
['Dsender', { receivedSat: 4n, sentSat: 7n }],
['Dreceiver', { receivedSat: 7n, sentSat: 0n, txCount: 1 }],
['Dsender', { receivedSat: 4n, sentSat: 7n, txCount: 1 }],
])
);
});

it('merges per-transaction deltas for block-level bulk writes', () => {
const target = new Map([['Daddr', { receivedSat: 2n, sentSat: 1n }]]);
const target = new Map([['Daddr', { receivedSat: 2n, sentSat: 1n, txCount: 1 }]]);
mergeAddressDeltas(
target,
new Map([
['Daddr', { receivedSat: 3n, sentSat: 4n }],
['Dnew', { receivedSat: 5n, sentSat: 0n }],
['Daddr', { receivedSat: 3n, sentSat: 4n, txCount: 1 }],
['Dnew', { receivedSat: 5n, sentSat: 0n, txCount: 1 }],
])
);

expect(target).toEqual(
new Map([
['Daddr', { receivedSat: 5n, sentSat: 5n }],
['Dnew', { receivedSat: 5n, sentSat: 0n }],
['Daddr', { receivedSat: 5n, sentSat: 5n, txCount: 2 }],
['Dnew', { receivedSat: 5n, sentSat: 0n, txCount: 1 }],
])
);
});

it('counts an address once for each transaction of a block it takes part in', () => {
const block = new Map<string, AddressDelta>();
const pay = (address: string, valueSat: string) => ({ valueSat, scriptPubKey: { addresses: [address] } });
// Two transactions of one block, both spending from and paying back to Dself.
mergeAddressDeltas(block, collectAddressDeltas([{ txid: 'p1', address: 'Dself', valueSat: '10' }], [pay('Dother', '4'), pay('Dself', '6')], false));
mergeAddressDeltas(block, collectAddressDeltas([{ txid: 'p2', address: 'Dself', valueSat: '6' }], [pay('Dother', '1'), pay('Dself', '5')], false));

expect(block.get('Dself')).toEqual({ receivedSat: 11n, sentSat: 16n, txCount: 2 });
expect(block.get('Dother')).toEqual({ receivedSat: 5n, sentSat: 0n, txCount: 2 });
});

it('builds the atomic MongoDB update pipeline used by live sync and rebuilds', () => {
const pipeline = buildAddressDeltaPipeline(
'Dabc',
{ receivedSat: 11n, sentSat: 3n },
{ receivedSat: 11n, sentSat: 3n, txCount: 2 },
1700000000
);
const setStage = (pipeline[0] as { $set?: Record<string, unknown> }).$set;

expect(pipeline).toHaveLength(1);
expect(setStage).toBeDefined();
expect((setStage?.address as { $ifNull: [string, string] }).$ifNull[1]).toBe('Dabc');
expect((setStage?.txCount as { $add: [number, number] }).$add[1]).toBe(1);
// The block's transactions with this address, not one per block.
expect((setStage?.txCount as { $add: [number, number] }).$add[1]).toBe(2);
});
});
Loading
Loading