From dda10a0b5cf2d590577d06c61e7c1c8048eb3c94 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 10 Nov 2026 19:39:41 +0700 Subject: [PATCH 01/11] fix(dashmate): stop masternodes safely around DKGs on Core 24 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `dashmate stop`, `restart` and `--safe` decide when stopping a masternode cannot cost it a PoSe penalty for missing DKG messages. On Core 24.0 the rule never let most masternodes stop: it treats any non-zero `quorum dkginfo` `active_dkgs` as a running session once Core reports `upcoming_dkgs`, but Core 24.0 still counts the DKG worker's session records, which linger until the next round of the same quorum type and index. Only Core 24.1 derives the count from membership. With Core 24.0 plain `stop` refused about 94% of the time on mainnet and `--safe` waited hours to days. One rule now serves Core 23, 24.0 and 24.1: - A DKG this node is a member of (or whose membership Core cannot determine) blocks from 6 blocks before its start. Core 24 reports membership in `upcoming_dkgs`; without it, any DKG within 6 blocks blocks, as before. - `quorum dkgstatus` records block from a session's start to the end of its Commit phase (5 * dkgPhaseBlocks), measured from each rotated quorum index's own start. Rotated `llmq_60_75` sessions blocked for 42 blocks from their index's start (the cycle's shared mining window, counted from the wrong base); they now block for 10, as Core 24.1 counts them. - A record that has reached its next round but that Core has not yet replaced blocks as that new round. - Records are read on every Core version: on 24.1 a mid-session ProUpServTx clears the proTxHash, so `active_dkgs` drops to 0 while the session runs. - Core 24.1 and later: a non-zero `active_dkgs` blocks. Core older than 24.1 records a session only after the start block connects, so a "safe" result taken right at a DKG start could be wrong. Each check now reads the chain tip before and after the DKG RPCs and is discarded if it moved. On Core older than 24.1 a safe result is confirmed 15 s after the first check finished, provided the chain moved by at most 6 blocks and did not replace the first check's tip. `--safe` also remembers member DKGs it saw announced, which covers the gap where Core has erased last round's record but not yet created the new one. Core in initial block download takes no part in DKGs and may stop. Plain stop retries an unsettled check up to 3 times, and reports unreadable Core data with its own error instead of suggesting --safe. `group restart --safe` declared its flag as a plain object, which oclif parsed as false, so the restart never waited. Cost: on Core older than 24.1 every plain stop or restart of a masternode that is safe to stop takes about 15 s longer. Tests would have caught this in CI. Run against the previous code, these fail: ✖ before the fix, ✔ after: - should stop waiting on Core 24.0 when active_dkgs counts only finished sessions - should end a rotated DKG session once its own Commit phase is over - should keep waiting at a DKG start block while Core still shows the previous round - should not stop at a DKG start that Core has not recorded yet - should not trust DKG reads that saw a newer tip than the height read - should keep waiting on Core 24.1 while a session runs after the proTxHash was cleared - should parse --safe so the restart waits for a safe DKG window Co-Authored-By: Claude Opus 5.5 --- .../dashmate/src/commands/group/restart.js | 5 +- .../core/quorum/checkMasternodeSafeToStop.js | 164 +++++- .../src/core/quorum/getDkgStopBlockers.js | 302 ++++++++++ .../quorum/isMasternodeSafeToStopDuringDkg.js | 140 ----- .../src/core/quorum/waitForDKGWindowPass.js | 38 +- .../src/listr/tasks/stopNodeTaskFactory.js | 37 +- .../dashmate/src/test/mock/getCoreRpcMock.js | 57 ++ .../test/unit/commands/group/restart.spec.js | 19 + .../quorum/checkMasternodeSafeToStop.spec.js | 240 ++++++-- .../core/quorum/getDkgStopBlockers.spec.js | 496 +++++++++++++++++ .../isMasternodeSafeToStopDuringDkg.spec.js | 427 --------------- .../core/quorum/waitForDKGWindowPass.spec.js | 517 ++++++++++++++---- .../listr/tasks/stopNodeTaskFactory.spec.js | 133 +++++ 13 files changed, 1814 insertions(+), 761 deletions(-) create mode 100644 packages/dashmate/src/core/quorum/getDkgStopBlockers.js delete mode 100644 packages/dashmate/src/core/quorum/isMasternodeSafeToStopDuringDkg.js create mode 100644 packages/dashmate/src/test/mock/getCoreRpcMock.js create mode 100644 packages/dashmate/test/unit/commands/group/restart.spec.js create mode 100644 packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js delete mode 100644 packages/dashmate/test/unit/core/quorum/isMasternodeSafeToStopDuringDkg.spec.js create mode 100644 packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js diff --git a/packages/dashmate/src/commands/group/restart.js b/packages/dashmate/src/commands/group/restart.js index 0ba25f5f8c9..79d7ab097d7 100644 --- a/packages/dashmate/src/commands/group/restart.js +++ b/packages/dashmate/src/commands/group/restart.js @@ -1,3 +1,4 @@ +import { Flags } from '@oclif/core'; import { Listr } from 'listr2'; import GroupBaseCommand from '../../oclif/command/GroupBaseCommand.js'; import MuteOneLineError from '../../oclif/errors/MuteOneLineError.js'; @@ -7,11 +8,11 @@ export default class GroupRestartCommand extends GroupBaseCommand { static flags = { ...GroupBaseCommand.flags, - safe: { + safe: Flags.boolean({ char: 's', description: 'wait for dkg before stop', default: false, - }, + }), }; /** diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index 73d110b53f8..921620162e7 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -1,28 +1,156 @@ -import isMasternodeSafeToStopDuringDkg, { - shouldInspectDkgStatusForSafeStop, -} from './isMasternodeSafeToStopDuringDkg.js'; +import wait from '../../util/wait.js'; +import { MIN_BLOCKS_BEFORE_DKG } from '../../constants.js'; +import getDkgStopBlockers, { + getBlockersAt, + hasMembershipActiveDkgs, +} from './getDkgStopBlockers.js'; /** - * Query Core and decide whether stopping would disrupt a DKG session. - * Older Core responses require the existing per-session status checks. + * How long a safe result from Core older than 24.1 must hold before it is + * trusted. Core's DKG worker records a session it is a member of shortly + * after the session's start block connects; this is comfortably longer. + */ +export const SAFE_STOP_CONFIRMATION_DELAY_MS = 15000; + +/** + * @param {RpcClient} rpcClient + * @return {Promise} Core's numeric `version`, or `null` if + * Core does not report one + */ +export async function readCoreVersion(rpcClient) { + const { result } = await rpcClient.getNetworkInfo(); + + return Number.isInteger(result && result.version) ? result.version : null; +} + +/** + * Read Core's DKG state at one chain tip. Each RPC reads the tip on its own, + * so the tip is read before and after the DKG calls; if a block connected in + * between, they may describe different tips and are discarded. * * @param {RpcClient} rpcClient - * @return {Promise} + * @return {Promise} */ -export default async function checkMasternodeSafeToStop(rpcClient) { +async function readDkgSnapshot(rpcClient) { + const { result: before } = await rpcClient.getBlockchainInfo(); const { result: dkgInfo } = await rpcClient.quorum('dkginfo'); + const { result: dkgStatus } = await rpcClient.quorum('dkgstatus'); + const { result: after } = await rpcClient.getBlockchainInfo(); + + if (before.bestblockhash !== after.bestblockhash) { + return null; + } + + return { + height: after.blocks, + hash: after.bestblockhash, + isSyncing: before.initialblockdownload === true && after.initialblockdownload === true, + dkgInfo, + dkgStatus, + }; +} + +/** + * Core older than 24.1 reports `active_dkgs` as the number of the DKG + * worker's session records, which `quorum dkgstatus` lists. Different counts + * mean the worker changed its records between the two reads, as it does + * when a session starts. + * + * @param {Object} snapshot + * @param {number|null} coreVersion + * @return {boolean} + */ +function haveSessionRecordsChanged({ dkgInfo, dkgStatus }, coreVersion) { + return !hasMembershipActiveDkgs(coreVersion) + && Number.isInteger(dkgInfo && dkgInfo.active_dkgs) + && Array.isArray(dkgStatus && dkgStatus.session) + && dkgInfo.active_dkgs !== dkgStatus.session.length; +} + +/** + * @typedef {Object} SafeStopCheck + * @property {string} status `safe`, `blocked`, or `unsettled` when Core's + * state changed too fast to read a consistent or confirmed one + * @property {number|null} height + * @property {string} [hash] chain tip the check read + * @property {DkgStopBlocker[]} blockers see {@link getDkgStopBlockers} + * @property {boolean} [isSyncing] Core is in initial block download + */ + +/** + * @param {RpcClient} rpcClient + * @param {number|null} coreVersion + * @return {Promise} + */ +async function evaluate(rpcClient, coreVersion) { + const snapshot = await readDkgSnapshot(rpcClient); + + if (!snapshot) { + return { status: 'unsettled', height: null, blockers: [] }; + } + + const { height, hash, isSyncing } = snapshot; + + // Core's DKG worker does nothing during initial block download, and Core + // never re-enters it without a restart, so a node in it has had no DKG + // session since it started. + if (isSyncing) { + return { + status: 'safe', height, hash, blockers: [], isSyncing, + }; + } + + if (haveSessionRecordsChanged(snapshot, coreVersion)) { + return { + status: 'unsettled', height, hash, blockers: [], + }; + } + + const blockers = getDkgStopBlockers({ ...snapshot, coreVersion }); + + return { + status: getBlockersAt(blockers, height).length > 0 ? 'blocked' : 'safe', + height, + hash, + blockers, + }; +} + +/** + * Query Core and decide whether stopping the masternode now would disrupt a + * DKG it takes part in. See {@link getDkgStopBlockers} for the rule. + * + * Core older than 24.1 reports a session only once its DKG worker has + * recorded it, shortly after the start block connects, so a safe result is + * checked again after {@link SAFE_STOP_CONFIRMATION_DELAY_MS}. By then every + * block the first check saw is old enough to have its sessions recorded. + * The first check already ruled out a DKG of this node starting within + * `MIN_BLOCKS_BEFORE_DKG` blocks, so the second may see up to that many new + * blocks; more than that, a lower tip, or a different block at the first + * check's height is `unsettled`. + * + * @param {RpcClient} rpcClient + * @param {number|null} coreVersion from {@link readCoreVersion} + * @return {Promise} + */ +export default async function checkMasternodeSafeToStop(rpcClient, coreVersion) { + const first = await evaluate(rpcClient, coreVersion); + + if (first.status !== 'safe' || first.isSyncing || hasMembershipActiveDkgs(coreVersion)) { + return first; + } + + await wait(SAFE_STOP_CONFIRMATION_DELAY_MS); + + const second = await evaluate(rpcClient, coreVersion); + + const hasChainMoved = second.height < first.height + || (second.height === first.height && second.hash !== first.hash) + || second.height > first.height + MIN_BLOCKS_BEFORE_DKG; - let dkgStatus; - let currentHeight; - if (shouldInspectDkgStatusForSafeStop(dkgInfo)) { - [ - { result: dkgStatus }, - { result: currentHeight }, - ] = await Promise.all([ - rpcClient.quorum('dkgstatus'), - rpcClient.getBlockCount(), - ]); + if (second.status === 'safe' && !second.isSyncing && hasChainMoved) { + return { ...second, status: 'unsettled' }; } - return isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, currentHeight); + return second; } diff --git a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js new file mode 100644 index 00000000000..746865976de --- /dev/null +++ b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js @@ -0,0 +1,302 @@ +import { MIN_BLOCKS_BEFORE_DKG } from '../../constants.js'; + +/** + * First Dash Core release whose `quorum dkginfo` derives `active_dkgs` from + * this masternode's membership, counting a session from its start block + * (Core 24.1.0). Older Core counts the DKG worker's local session records, + * which appear only after the worker has started a session and linger long + * after it ends. Core reports `version` as 10000 * major + 100 * minor + patch. + */ +export const MEMBERSHIP_ACTIVE_DKGS_CORE_VERSION = 240100; + +/** + * DKG parameters from Dash Core `src/llmq/params.h`. `quorum dkgstatus` names + * types by `name`, `quorum dkginfo` `upcoming_dkgs` by numeric `id`. + * + * Keep in sync with `src/llmq/params.h` in Dash Core. + */ +export const LLMQ_TYPES = [ + { + id: 1, name: 'llmq_50_60', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 2, name: 'llmq_400_60', dkgInterval: 288, dkgPhaseBlocks: 4, + }, + { + id: 3, name: 'llmq_400_85', dkgInterval: 576, dkgPhaseBlocks: 4, + }, + { + id: 4, name: 'llmq_100_67', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 5, name: 'llmq_60_75', dkgInterval: 288, dkgPhaseBlocks: 2, + }, + { + id: 6, name: 'llmq_25_67', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 100, name: 'llmq_test', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 101, name: 'llmq_devnet', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 102, name: 'llmq_test_v17', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 103, name: 'llmq_test_dip0024', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 104, name: 'llmq_test_instantsend', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 105, name: 'llmq_devnet_dip0024', dkgInterval: 48, dkgPhaseBlocks: 2, + }, + { + id: 106, name: 'llmq_test_platform', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 107, name: 'llmq_devnet_platform', dkgInterval: 24, dkgPhaseBlocks: 2, + }, +]; + +/** + * Blocks from a session's start to the end of its Commit phase, Core's + * `QuorumPhase::Commit * dkgPhaseBlocks`. By then a member has sent every + * message that decides whether the final commitment counts it as valid, + * which is what PoSe penalizes; finalizing and mining the commitment does + * not need it. Each rotated quorum index starts on its own block and is + * measured from its own `quorumHeight`, not from the cycle's shared mining + * window. + * + * @param {{ dkgPhaseBlocks: number }} llmq + * @return {number} + */ +export function getActiveDkgBlocks(llmq) { + return 5 * llmq.dkgPhaseBlocks; +} + +/** + * @param {number|null} coreVersion + * @return {boolean} + */ +export function hasMembershipActiveDkgs(coreVersion) { + return Number.isInteger(coreVersion) && coreVersion >= MEMBERSHIP_ACTIVE_DKGS_CORE_VERSION; +} + +/** + * @typedef {Object} DkgStopBlocker + * @property {string} reason `session`, `new-round`, `upcoming`, `imminent`, + * `active-count` or `unreadable` + * @property {string|number|null} [llmqType] LLMQ name, or Core's numeric id + * when dashmate does not know the type + * @property {number} [quorumIndex] + * @property {boolean} [membershipKnown] for `upcoming`: `false` when Core + * could not determine whether this node is a member + * @property {number} [count] for `active-count` + * @property {string} [source] for `unreadable`: which Core data was unreadable + * @property {number|null} from first blocked height; `null` blocks from now + * @property {number|null} until first height no longer blocked; `null` when + * the end is unknown + */ + +/** + * @param {DkgStopBlocker} blocker + * @param {number} height + * @return {boolean} + */ +function isBlockedAt(blocker, height) { + return (blocker.from === null || blocker.from <= height) + && (blocker.until === null || height < blocker.until); +} + +/** + * @param {DkgStopBlocker[]} blockers + * @param {number} height + * @return {DkgStopBlocker[]} + */ +export function getBlockersAt(blockers, height) { + return blockers.filter((blocker) => isBlockedAt(blocker, height)); +} + +function isCount(value) { + return Number.isInteger(value) && value >= 0; +} + +function hasValidDkgInfoShape(dkgInfo) { + return !!dkgInfo + && typeof dkgInfo === 'object' + && isCount(dkgInfo.active_dkgs) + && isCount(dkgInfo.next_dkg); +} + +function isValidUpcomingDkg(dkg) { + return !!dkg + && typeof dkg === 'object' + && isCount(dkg.blocksUntilStart) + && typeof dkg.known === 'boolean' + && (!dkg.known || typeof dkg.isMember === 'boolean'); +} + +function unreadable(source) { + return { + reason: 'unreadable', source, from: null, until: null, + }; +} + +/** + * Without membership data (Core < 24, or Core does not know this node's + * proTxHash), any DKG of any type may include this node, so the restart + * margin before the next one blocks with no known end. + */ +function getImminentDkgBlocker(dkgInfo, height) { + return { + reason: 'imminent', + from: height + dkgInfo.next_dkg - MIN_BLOCKS_BEFORE_DKG, + until: null, + }; +} + +/** + * Core lists a DKG in `upcoming_dkgs` from its work block, 8 blocks before + * the cycle starts, so every DKG starting within `MIN_BLOCKS_BEFORE_DKG` + * is listed. A DKG this node is a member of, or whose membership Core could + * not determine, blocks from the restart margin before its start until its + * Commit phase ends. + */ +function getUpcomingDkgBlockers(upcomingDkgs, height) { + if (!Array.isArray(upcomingDkgs)) { + return [unreadable('quorum dkginfo upcoming_dkgs')]; + } + + return upcomingDkgs.flatMap((dkg) => { + if (!isValidUpcomingDkg(dkg)) { + return [unreadable('quorum dkginfo upcoming_dkgs')]; + } + + if (dkg.known && !dkg.isMember) { + return []; + } + + const llmq = LLMQ_TYPES.find(({ id }) => id === dkg.llmqType); + const start = height + dkg.blocksUntilStart; + + return [{ + reason: 'upcoming', + llmqType: llmq ? llmq.name : dkg.llmqType, + quorumIndex: dkg.quorumIndex, + membershipKnown: dkg.known, + from: start - MIN_BLOCKS_BEFORE_DKG, + until: llmq ? start + getActiveDkgBlocks(llmq) : null, + }]; + }); +} + +/** + * The DKG worker keeps a record for the latest session of each LLMQ type + * and quorum index this node was a member of. A record inside its active + * window blocks. A record at least one interval old, but less than one + * interval plus the active window, means the next session of that type and + * index has started and the worker, which runs shortly after the block + * connects, has not yet replaced or removed the record. This node may be a + * member again, so the record blocks until that session's Commit phase ends. + */ +function getSessionRecordBlockers(dkgStatus, height) { + const sessions = dkgStatus && dkgStatus.session; + + if (!Array.isArray(sessions)) { + return [unreadable('quorum dkgstatus')]; + } + + return sessions.flatMap((entry) => { + const llmq = entry && LLMQ_TYPES.find(({ name }) => name === entry.llmqType); + const quorumHeight = entry && entry.status && entry.status.quorumHeight; + + if (!llmq || !isCount(quorumHeight) || quorumHeight > height) { + return [unreadable(`quorum dkgstatus session ${entry && entry.llmqType}`)]; + } + + const activeBlocks = getActiveDkgBlocks(llmq); + const offset = height - quorumHeight; + const session = { llmqType: llmq.name, quorumIndex: entry.quorumIndex }; + + if (offset < activeBlocks) { + return [{ + ...session, + reason: 'session', + from: quorumHeight, + until: quorumHeight + activeBlocks, + }]; + } + + const nextQuorumHeight = quorumHeight + llmq.dkgInterval; + if (height >= nextQuorumHeight && height < nextQuorumHeight + activeBlocks) { + return [{ + ...session, + reason: 'new-round', + from: nextQuorumHeight, + until: nextQuorumHeight + activeBlocks, + }]; + } + + return []; + }); +} + +/** + * Everything that makes stopping this masternode risk a PoSe penalty, as + * height ranges. Stopping is safe at a height no blocker covers (see + * {@link getBlockersAt}). Some blockers start above the current height; they + * do not block yet and only help estimate when stopping becomes safe. + * + * All inputs must come from the same chain tip, `height`. Core older than + * {@link MEMBERSHIP_ACTIVE_DKGS_CORE_VERSION} reports a session only after + * its DKG worker has recorded it, shortly after the start block connects; + * callers must confirm a safe result once that delay has passed. + * + * Unreadable Core data blocks with no known end. + * + * @param {Object} snapshot + * @param {Object} snapshot.dkgInfo Result of `quorum dkginfo`. + * @param {Object} snapshot.dkgStatus Result of `quorum dkgstatus`. + * @param {number} snapshot.height Chain tip both results describe. + * @param {number|null} snapshot.coreVersion Core's numeric `version`. + * @return {DkgStopBlocker[]} + */ +export default function getDkgStopBlockers({ + dkgInfo, + dkgStatus, + height, + coreVersion, +}) { + if (!isCount(height)) { + return [unreadable('block height')]; + } + + if (!hasValidDkgInfoShape(dkgInfo)) { + return [unreadable('quorum dkginfo')]; + } + + const blockers = [ + ...(dkgInfo.upcoming_dkgs === undefined + ? [getImminentDkgBlocker(dkgInfo, height)] + : getUpcomingDkgBlockers(dkgInfo.upcoming_dkgs, height)), + // Read on every Core version: Core 24.1 stops counting a running session + // in active_dkgs when a mid-session ProUpServTx clears its proTxHash, but + // the session keeps going and its record stays. + ...getSessionRecordBlockers(dkgStatus, height), + ]; + + // Core 24.1+ counts every session this node is a member of, or whose + // membership is unknown, from its start block through its Commit phase. + if (hasMembershipActiveDkgs(coreVersion) && dkgInfo.active_dkgs > 0) { + blockers.push({ + reason: 'active-count', + count: dkgInfo.active_dkgs, + from: null, + until: null, + }); + } + + return blockers; +} diff --git a/packages/dashmate/src/core/quorum/isMasternodeSafeToStopDuringDkg.js b/packages/dashmate/src/core/quorum/isMasternodeSafeToStopDuringDkg.js deleted file mode 100644 index de5b8840d97..00000000000 --- a/packages/dashmate/src/core/quorum/isMasternodeSafeToStopDuringDkg.js +++ /dev/null @@ -1,140 +0,0 @@ -import { MIN_BLOCKS_BEFORE_DKG } from '../../constants.js'; - -/** - * `dkgMiningWindowStart` values from Dash Core `src/llmq/params.h`, - * indexed by llmqType string as reported in `quorum dkgstatus`. The - * number is how many blocks after a session's `quorumHeight` the - * active DKG window lasts. Once `currentHeight - quorumHeight >= - * window`, the session is past its active phase and a restart no - * longer risks a PoSe penalty for that session. - * - * Keep in sync with `src/llmq/params.h` in Dash Core. - */ -export const DKG_MINING_WINDOW_START_BY_LLMQ_TYPE = { - llmq_test: 10, - llmq_test_instantsend: 10, - llmq_test_v17: 10, - llmq_test_dip0024: 12, - llmq_test_platform: 10, - llmq_devnet: 10, - llmq_devnet_dip0024: 12, - llmq_devnet_platform: 10, - llmq_50_60: 10, - llmq_60_75: 42, - llmq_400_60: 20, - llmq_400_85: 20, - llmq_100_67: 10, - llmq_25_67: 10, -}; - -function isValidDkgCounter(value) { - return typeof value === 'number' && Number.isFinite(value) && value >= 0; -} - -function hasValidDkgInfoShape(dkgInfo) { - return !!dkgInfo - && typeof dkgInfo === 'object' - && isValidDkgCounter(dkgInfo.active_dkgs) - && isValidDkgCounter(dkgInfo.next_dkg); -} - -/** - * Malformed entries always block; member and unknown-membership entries - * block when they start within the restart margin. - */ -function isUpcomingDkgBlockingStop(dkg) { - if (!dkg - || typeof dkg !== 'object' - || !isValidDkgCounter(dkg.blocksUntilStart) - || typeof dkg.known !== 'boolean' - || (dkg.known && typeof dkg.isMember !== 'boolean')) { - return true; - } - - return dkg.blocksUntilStart <= MIN_BLOCKS_BEFORE_DKG && (!dkg.known || dkg.isMember); -} - -/** - * @param {Object} dkgInfo - * @return {boolean} - */ -export function shouldInspectDkgStatusForSafeStop(dkgInfo) { - if (!hasValidDkgInfoShape(dkgInfo)) { - return false; - } - - return dkgInfo.upcoming_dkgs === undefined - && dkgInfo.active_dkgs > 0 - && dkgInfo.next_dkg > MIN_BLOCKS_BEFORE_DKG; -} - -/** - * With upcoming_dkgs, stopping is safe when no DKG counts toward active_dkgs - * and no member or unknown-membership DKG starts within the restart margin. - * - * Without upcoming_dkgs (older Core, or no proTxHash), use the legacy next_dkg - * guard and resolve tracked sessions against dkgstatus and getblockcount. - * Unknown quorum types or malformed status fail safe; sessions past - * dkgMiningWindowStart are ignored. - * - * @param {Object} dkgInfo Result of quorum dkginfo. - * @param {Object} [dkgStatus] Result of quorum dkgstatus for older Core. - * @param {number} [currentHeight] Chain tip height for older Core. - * @return {boolean} - */ -export default function isMasternodeSafeToStopDuringDkg( - dkgInfo, - dkgStatus, - currentHeight, -) { - if (!hasValidDkgInfoShape(dkgInfo)) { - return false; - } - - const { upcoming_dkgs: upcomingDkgs } = dkgInfo; - - // Relies on active_dkgs counting member and unknown-membership sessions - // (dashpay/dash#7812). - if (upcomingDkgs !== undefined) { - return dkgInfo.active_dkgs === 0 - && Array.isArray(upcomingDkgs) - && !upcomingDkgs.some(isUpcomingDkgBlockingStop); - } - - if (dkgInfo.next_dkg <= MIN_BLOCKS_BEFORE_DKG) { - return false; - } - - if (dkgInfo.active_dkgs === 0) { - return true; - } - - if (!dkgStatus - || !Array.isArray(dkgStatus.session) - || dkgStatus.session.length === 0 - || typeof currentHeight !== 'number' - || !Number.isFinite(currentHeight)) { - return false; - } - - for (const sessionEntry of dkgStatus.session) { - const llmqType = sessionEntry && sessionEntry.llmqType; - const quorumHeight = sessionEntry && sessionEntry.status - && sessionEntry.status.quorumHeight; - - const windowLength = DKG_MINING_WINDOW_START_BY_LLMQ_TYPE[llmqType]; - - if (windowLength === undefined - || typeof quorumHeight !== 'number' - || !Number.isFinite(quorumHeight)) { - return false; - } - - const offset = currentHeight - quorumHeight; - if (offset < 0 || offset < windowLength) { - return false; - } - } - - return true; -} diff --git a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js index 1bd52daa0ab..8f1c4c911e4 100644 --- a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js +++ b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js @@ -1,7 +1,8 @@ import wait from '../../util/wait.js'; -import checkMasternodeSafeToStop from './checkMasternodeSafeToStop.js'; +import checkMasternodeSafeToStop, { readCoreVersion } from './checkMasternodeSafeToStop.js'; +import { getBlockersAt } from './getDkgStopBlockers.js'; -const CHECK_INTERVAL_MS = 10000; +export const CHECK_INTERVAL_MS = 10000; /** * Poll Core until the masternode is safe to stop without disrupting a @@ -9,11 +10,42 @@ const CHECK_INTERVAL_MS = 10000; * safety rule. The only acceptable exit is reaching a safe state, so * that `--safe` cannot silently fall back to an unsafe restart. * + * DKGs this node takes part in are remembered from the polls that saw them + * announced. Once a session starts Core no longer announces it, and Core + * older than 24.1 reports it only after its DKG worker has computed the + * session's members and recorded it, which can take a while at the start + * of a rotation cycle. + * * @param {RpcClient} rpcClient * @return {Promise} */ export default async function waitForDKGWindowPass(rpcClient) { - while (!await checkMasternodeSafeToStop(rpcClient)) { + const coreVersion = await readCoreVersion(rpcClient); + + const announcedDkgs = new Map(); + + for (;;) { + const check = await checkMasternodeSafeToStop(rpcClient, coreVersion); + + if (check.height !== null) { + check.blockers + .filter(({ reason, until }) => reason === 'upcoming' && until !== null) + .forEach((dkg) => { + announcedDkgs.set(`${dkg.llmqType}:${dkg.quorumIndex}:${dkg.from}`, dkg); + }); + + announcedDkgs.forEach((dkg, key) => { + if (dkg.until <= check.height) { + announcedDkgs.delete(key); + } + }); + } + + if (check.status === 'safe' + && (check.isSyncing || getBlockersAt([...announcedDkgs.values()], check.height).length === 0)) { + return; + } + await wait(CHECK_INTERVAL_MS); } } diff --git a/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js b/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js index d03a7dbbe1d..6d8817b5174 100644 --- a/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js @@ -1,7 +1,14 @@ /* eslint-disable no-console */ import { Listr } from 'listr2'; import waitForDKGWindowPass from '../../core/quorum/waitForDKGWindowPass.js'; -import checkMasternodeSafeToStop from '../../core/quorum/checkMasternodeSafeToStop.js'; +import checkMasternodeSafeToStop, { + readCoreVersion, +} from '../../core/quorum/checkMasternodeSafeToStop.js'; +import { getBlockersAt } from '../../core/quorum/getDkgStopBlockers.js'; + +// A check comes back unsettled while the chain moves too fast to read a +// consistent DKG state, such as while Core catches up after a restart. +const SAFE_STOP_CHECK_ATTEMPTS = 3; /** * @param {DockerCompose} dockerCompose @@ -54,11 +61,37 @@ export default function stopNodeTaskFactory( host: await getConnectionHost(config, 'core', 'core.rpc.host'), }); - if (!await checkMasternodeSafeToStop(rpcClient)) { + const coreVersion = await readCoreVersion(rpcClient); + + let check; + for (let attempt = 0; attempt < SAFE_STOP_CHECK_ATTEMPTS; attempt += 1) { + check = await checkMasternodeSafeToStop(rpcClient, coreVersion); + + if (check.status !== 'unsettled') { + break; + } + } + + const unreadable = check.status === 'blocked' + && getBlockersAt(check.blockers, check.height) + .find(({ reason }) => reason === 'unreadable'); + + if (unreadable) { + throw new Error(`dashmate cannot read Core's DKG state (${unreadable.source}). ` + + 'Update dashmate, or continue with --force'); + } + + if (check.status === 'blocked') { throw new Error('Your node is currently participating in a DKG exchange session ' + '(or one is about to start) and stopping it right now may result in a PoSe ban. ' + 'Try again later, or continue with --force or --safe flags'); } + + if (check.status === 'unsettled') { + throw new Error('The chain kept advancing while checking for DKG sessions ' + + '(is Core still syncing?). ' + + 'Try again later, or continue with --force or --safe flags'); + } }, }, { diff --git a/packages/dashmate/src/test/mock/getCoreRpcMock.js b/packages/dashmate/src/test/mock/getCoreRpcMock.js new file mode 100644 index 00000000000..e0b1174e112 --- /dev/null +++ b/packages/dashmate/src/test/mock/getCoreRpcMock.js @@ -0,0 +1,57 @@ +/** + * Core RPC client whose stubs answer from `core`, a plain object a test + * mutates to move the chain or change DKG state between reads. + * `core.beforeDkgInfo` and `core.afterDkgStatus`, when set, run right + * before `quorum dkginfo` is answered and right after `quorum dkgstatus` is. + * The tip hash is derived from the height unless `core.hash` is set. + * + * @param {Sinon} sinon + * @param {{ + * version: number, + * height: number, + * hash?: string, + * dkgInfo: Object, + * dkgStatus: Object, + * isSyncing?: boolean, + * dkgStatusDelayMs?: number, + * beforeDkgInfo?: Function, + * afterDkgStatus?: Function, + * }} core + */ +export default function getCoreRpcMock(sinon, core) { + return { + getNetworkInfo: sinon.stub().callsFake(async () => ({ result: { version: core.version } })), + getBlockchainInfo: sinon.stub().callsFake(async () => ({ + result: { + blocks: core.height, + bestblockhash: core.hash || `hash-${core.height}`, + initialblockdownload: core.isSyncing === true, + }, + })), + quorum: sinon.stub().callsFake(async (command) => { + if (command === 'dkginfo') { + if (core.beforeDkgInfo) { + core.beforeDkgInfo(); + } + + return { result: core.dkgInfo }; + } + + if (command === 'dkgstatus') { + if (core.dkgStatusDelayMs) { + await new Promise((resolve) => { setTimeout(resolve, core.dkgStatusDelayMs); }); + } + + const result = core.dkgStatus; + + if (core.afterDkgStatus) { + core.afterDkgStatus(); + } + + return { result }; + } + + throw new Error(`unexpected quorum ${command}`); + }), + }; +} diff --git a/packages/dashmate/test/unit/commands/group/restart.spec.js b/packages/dashmate/test/unit/commands/group/restart.spec.js new file mode 100644 index 00000000000..060acf69ba3 --- /dev/null +++ b/packages/dashmate/test/unit/commands/group/restart.spec.js @@ -0,0 +1,19 @@ +import { Parser } from '@oclif/core'; +import GroupRestartCommand from '../../../../src/commands/group/restart.js'; + +describe('GroupRestartCommand', () => { + async function parseSafe(argv) { + const { flags } = await Parser.parse(argv, { flags: GroupRestartCommand.flags }); + + return flags.safe; + } + + it('should parse --safe so the restart waits for a safe DKG window', async () => { + expect(await parseSafe(['--safe'])).to.equal(true); + expect(await parseSafe(['-s'])).to.equal(true); + }); + + it('should not wait for a DKG window by default', async () => { + expect(await parseSafe([])).to.equal(false); + }); +}); diff --git a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js index 475b8bfb499..1ae4730fd8e 100644 --- a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js +++ b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js @@ -1,70 +1,204 @@ -import checkMasternodeSafeToStop from '../../../../src/core/quorum/checkMasternodeSafeToStop.js'; +import checkMasternodeSafeToStop, { + readCoreVersion, + SAFE_STOP_CONFIRMATION_DELAY_MS, +} from '../../../../src/core/quorum/checkMasternodeSafeToStop.js'; +import { MIN_BLOCKS_BEFORE_DKG } from '../../../../src/constants.js'; +import getCoreRpcMock from '../../../../src/test/mock/getCoreRpcMock.js'; + +const CORE_23 = 230110; +const CORE_24_0_PATCH = 240099; +const CORE_24_1 = 240100; describe('checkMasternodeSafeToStop', () => { + let core; let rpcClient; + let clock; beforeEach(function beforeEach() { - rpcClient = { quorum: this.sinon.stub(), getBlockCount: this.sinon.stub() }; + clock = this.sinon.useFakeTimers(); + core = { + version: CORE_23, + height: 1012, + dkgInfo: { active_dkgs: 0, next_dkg: 12 }, + dkgStatus: { session: [] }, + }; + rpcClient = getCoreRpcMock(this.sinon, core); }); - it('should allow a non-member immediately using one membership response', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ - result: { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [{ blocksUntilStart: 1, known: true, isMember: false }], - }, - }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(true); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); - expect(rpcClient.getBlockCount).to.not.have.been.called(); - }); - - it('should block a positive active_dkgs without inspecting dkgstatus', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ - result: { - active_dkgs: 1, - next_dkg: 24, - upcoming_dkgs: [], - }, + /** + * Start a check and run the fake clock until the first check has finished + * and, on Core older than 24.1, is waiting to be confirmed. + */ + async function startCheck() { + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version).then((result) => { state.result = result; }); + await clock.tickAsync(0); + + return state; + } + + describe('readCoreVersion', () => { + it('should return the numeric version Core reports', async () => { + expect(await readCoreVersion(rpcClient)).to.equal(CORE_23); }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(false); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); - expect(rpcClient.getBlockCount).to.not.have.been.called(); - }); - - it('should block a member of an imminent DKG', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ - result: { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [{ blocksUntilStart: 1, known: true, isMember: true }], - }, + + it('should return null when Core reports no integer version', async () => { + core.version = '23.1.10'; + + expect(await readCoreVersion(rpcClient)).to.be.null(); }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(false); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); }); - it('should retain the imminent-DKG guard without upcoming_dkgs', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 0, next_dkg: 1 } }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(false); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); + it('should read the chain tip before and after the DKG state', async () => { + core.version = CORE_24_1; + + await startCheck(); + + const [tipBefore, tipAfter] = rpcClient.getBlockchainInfo.getCalls(); + const dkgInfo = rpcClient.quorum.withArgs('dkginfo').getCall(0); + const dkgStatus = rpcClient.quorum.withArgs('dkgstatus').getCall(0); + + expect(tipBefore.calledBefore(dkgInfo)).to.be.true(); + expect(tipBefore.calledBefore(dkgStatus)).to.be.true(); + expect(dkgInfo.calledBefore(tipAfter)).to.be.true(); + expect(dkgStatus.calledBefore(tipAfter)).to.be.true(); }); - it('should allow a legacy response with no active or imminent DKG', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 0, next_dkg: 24 } }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(true); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); + it('should report unsettled when a block connects during the reads', async () => { + core.version = CORE_24_1; + core.beforeDkgInfo = () => { core.height += 1; }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('unsettled'); }); - it('should inspect active sessions for older Core', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 1, next_dkg: 24 } }); - rpcClient.quorum.withArgs('dkgstatus').resolves({ - result: { session: [{ llmqType: 'llmq_60_75', status: { quorumHeight: 1000 } }] }, - }); - rpcClient.getBlockCount.resolves({ result: 1001 }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(false); - expect(rpcClient.quorum.withArgs('dkgstatus')).to.have.been.calledOnce(); - expect(rpcClient.getBlockCount).to.have.been.calledOnce(); + it('should report unsettled when Core changes its session records between the reads', async () => { + // Core older than 24.1 counts the records dkgstatus lists; it erases a + // record when the next session of its type starts. + core.dkgInfo = { active_dkgs: 1, next_dkg: 12 }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('unsettled'); + }); + + it('should report blocked at once without waiting for confirmation', async () => { + core.dkgInfo = { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('blocked'); + expect(result.height).to.equal(1012); + expect(result.blockers[0].reason).to.equal('imminent'); + expect(rpcClient.getBlockchainInfo).to.have.been.calledTwice(); + }); + + it('should confirm a safe result after the confirmation delay on Core older than 24.1', async () => { + const state = await startCheck(); + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS - 1); + + expect(state.result).to.be.undefined(); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + + await clock.tickAsync(1); + + expect(state.result.status).to.equal('safe'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + }); + + it('should confirm a safe result on a Core 24.0 patch release', async () => { + core.version = CORE_24_0_PATCH; + + const state = await startCheck(); + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('safe'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + }); + + it('should start the confirmation delay once the first check has finished', async () => { + // Every block the first check saw must be old enough for Core to have + // recorded its sessions. + const dkgStatusDelayMs = 10000; + core.dkgStatusDelayMs = dkgStatusDelayMs; + + await startCheck(); + await clock.tickAsync(dkgStatusDelayMs + SAFE_STOP_CONFIRMATION_DELAY_MS - 1); + + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + + await clock.tickAsync(1); + + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + }); + + it('should return the confirming check when it finds a session', async () => { + const state = await startCheck(); + + core.dkgInfo = { active_dkgs: 1, next_dkg: 12 }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, status: { quorumHeight: 1012 } }], + }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('blocked'); + expect(state.result.blockers.find(({ reason }) => reason === 'session')) + .to.include({ from: 1012, until: 1022 }); + }); + + it('should accept up to MIN_BLOCKS_BEFORE_DKG new blocks during the confirmation', async () => { + const state = await startCheck(); + core.height += MIN_BLOCKS_BEFORE_DKG; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('safe'); + }); + + it('should report unsettled when more blocks arrive during the confirmation than the first check covered', async () => { + // The first check only ruled out DKGs of this node within + // MIN_BLOCKS_BEFORE_DKG blocks of its tip. + const state = await startCheck(); + core.height += MIN_BLOCKS_BEFORE_DKG + 1; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('unsettled'); + }); + + it('should report unsettled when the confirming check finds another block at the same height', async () => { + // A reorg replaced the first check's tip, so that block may be new. + const state = await startCheck(); + core.hash = 'reorganized'; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('unsettled'); + }); + + it('should report unsettled when the confirming check finds a lower tip', async () => { + const state = await startCheck(); + core.height -= 1; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('unsettled'); + }); + + it('should not wait for confirmation on Core 24.1', async () => { + core.version = CORE_24_1; + + const { result } = await startCheck(); + + expect(result.status).to.equal('safe'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + }); + + it('should treat Core in initial block download as safe without confirmation', async () => { + // Core takes no part in DKGs until it has synced. + core.isSyncing = true; + core.dkgInfo = { active_dkgs: 0, next_dkg: 1 }; + + const { result } = await startCheck(); + + expect(result).to.include({ status: 'safe', isSyncing: true }); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); }); }); diff --git a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js new file mode 100644 index 00000000000..9e5fbc2e310 --- /dev/null +++ b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js @@ -0,0 +1,496 @@ +import getDkgStopBlockers, { + getBlockersAt, + hasMembershipActiveDkgs, + LLMQ_TYPES, +} from '../../../../src/core/quorum/getDkgStopBlockers.js'; +import { MIN_BLOCKS_BEFORE_DKG } from '../../../../src/constants.js'; + +const CORE_23 = 230110; +const CORE_24_0 = 240000; +const CORE_24_1 = 240100; +const HEIGHT = 1000; + +// A value of next_dkg that is safely above MIN_BLOCKS_BEFORE_DKG so the +// imminent-DKG guard never trips in tests that focus on session records. +const NEXT_DKG_NOT_IMMINENT = MIN_BLOCKS_BEFORE_DKG + 5; +const PLATFORM_ACTIVE_BLOCKS = 10; // llmq_test_platform: 5 phases * 2 blocks +const PLATFORM_INTERVAL = 24; + +function blockersAt({ + dkgInfo, + dkgStatus = { session: [] }, + height = HEIGHT, + coreVersion = CORE_23, +}) { + return getBlockersAt(getDkgStopBlockers({ + dkgInfo, dkgStatus, height, coreVersion, + }), height); +} + +function record(llmqType, quorumHeight, quorumIndex = 0) { + return { llmqType, quorumIndex, status: { quorumHeight } }; +} + +// Shape of a `quorum dkginfo` upcoming_dkgs entry once membership is known. +function upcomingDkg(blocksUntilStart, isMember, llmqType = 106) { + return { + llmqType, + quorumIndex: 0, + quorumHeight: HEIGHT + blocksUntilStart, + blocksUntilStart, + known: true, + isMember, + workBlockHeight: HEIGHT + blocksUntilStart - 8, + workBlockHash: '00'.repeat(32), + }; +} + +function unknownUpcomingDkg(blocksUntilStart) { + return { + llmqType: 103, + quorumIndex: 0, + quorumHeight: HEIGHT + blocksUntilStart, + blocksUntilStart, + known: false, + reason: 'rotated quorum snapshots are not available yet', + }; +} + +describe('getDkgStopBlockers', () => { + describe('imminent DKG guard from dkginfo.next_dkg (no membership data)', () => { + it('should block when next_dkg <= MIN_BLOCKS_BEFORE_DKG even with no sessions', () => { + const blockers = blockersAt({ dkgInfo: { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG } }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'imminent', until: null }); + }); + + it('should allow the stop when next_dkg is past the restart margin', () => { + expect(blockersAt({ dkgInfo: { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG + 1 } })) + .to.be.empty(); + }); + + it('should report when the next DKG starts blocking', () => { + const [blocker] = getDkgStopBlockers({ + dkgInfo: { active_dkgs: 0, next_dkg: 20 }, + dkgStatus: { session: [] }, + height: HEIGHT, + coreVersion: CORE_23, + }); + + expect(blocker).to.include({ + reason: 'imminent', from: HEIGHT + 20 - MIN_BLOCKS_BEFORE_DKG, until: null, + }); + }); + + it('should ignore active_dkgs on Core older than 24.1, which counts lingering records', () => { + expect(blockersAt({ dkgInfo: { active_dkgs: 3, next_dkg: NEXT_DKG_NOT_IMMINENT } })) + .to.be.empty(); + }); + }); + + describe('membership from dkginfo.upcoming_dkgs', () => { + it('should allow the stop when an imminent DKG does not include this node', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [upcomingDkg(1, false), upcomingDkg(MIN_BLOCKS_BEFORE_DKG, false)], + }; + + expect(blockersAt({ dkgInfo, coreVersion: CORE_24_0 })).to.be.empty(); + }); + + it('should allow the stop when next_dkg is imminent but no upcoming DKG is listed', () => { + // next_dkg reports 1 for the whole rotation signing window, even + // after the last rotated index has started. + const dkgInfo = { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] }; + + expect(blockersAt({ dkgInfo, coreVersion: CORE_24_0 })).to.be.empty(); + }); + + it('should block when this node is a member of an imminent DKG until its Commit phase ends', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [upcomingDkg(1, false), upcomingDkg(MIN_BLOCKS_BEFORE_DKG, true)], + }; + const start = HEIGHT + MIN_BLOCKS_BEFORE_DKG; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'upcoming', + llmqType: 'llmq_test_platform', + membershipKnown: true, + from: start - MIN_BLOCKS_BEFORE_DKG, + until: start + PLATFORM_ACTIVE_BLOCKS, + }); + }); + + it('should not block yet when this node is a member of a DKG past the restart margin', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: MIN_BLOCKS_BEFORE_DKG + 1, + upcoming_dkgs: [upcomingDkg(MIN_BLOCKS_BEFORE_DKG + 1, true)], + }; + + const blockers = getDkgStopBlockers({ + dkgInfo, dkgStatus: { session: [] }, height: HEIGHT, coreVersion: CORE_24_0, + }); + + expect(getBlockersAt(blockers, HEIGHT)).to.be.empty(); + expect(getBlockersAt(blockers, HEIGHT + 1)).to.have.lengthOf(1); + }); + + it('should block when membership of an imminent DKG is unknown', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [unknownUpcomingDkg(1)], + }; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'upcoming', membershipKnown: false }); + }); + + it('should allow the stop when membership is unknown only for a DKG past the restart margin', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: MIN_BLOCKS_BEFORE_DKG + 1, + upcoming_dkgs: [unknownUpcomingDkg(MIN_BLOCKS_BEFORE_DKG + 1)], + }; + + expect(blockersAt({ dkgInfo, coreVersion: CORE_24_0 })).to.be.empty(); + }); + + it('should block a member DKG of a type dashmate does not know, with no known end', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [upcomingDkg(1, true, 250)], + }; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'upcoming', llmqType: 250, until: null }); + }); + + describe('fail-safe on malformed upcoming_dkgs', () => { + const malformed = { + 'upcoming_dkgs is an object': {}, + 'upcoming_dkgs is null': null, + 'an entry is null': [null], + 'an entry is missing known': [{ blocksUntilStart: 1, isMember: false }], + 'a known entry is missing isMember': [{ blocksUntilStart: 1, known: true }], + 'an entry is missing blocksUntilStart': [{ known: true, isMember: false }], + 'an entry has a negative blocksUntilStart': [ + { blocksUntilStart: -1, known: true, isMember: false }, + ], + 'an entry outside the margin is malformed': [ + { blocksUntilStart: MIN_BLOCKS_BEFORE_DKG + 1, known: true }, + ], + }; + + Object.entries(malformed).forEach(([description, upcomingDkgs]) => { + it(`should block when ${description}`, () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: NEXT_DKG_NOT_IMMINENT, + upcoming_dkgs: upcomingDkgs, + }; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', until: null }); + }); + }); + }); + }); + + describe('session records from dkgstatus', () => { + const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; + + it('should block a session at its start block', () => { + const blockers = blockersAt({ + dkgInfo, + dkgStatus: { session: [record('llmq_test_platform', HEIGHT)] }, + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'session', + llmqType: 'llmq_test_platform', + from: HEIGHT, + until: HEIGHT + PLATFORM_ACTIVE_BLOCKS, + }); + }); + + it('should block a session at the last block of its Commit phase', () => { + const dkgStatus = { session: [record('llmq_test_platform', HEIGHT - PLATFORM_ACTIVE_BLOCKS + 1)] }; + + expect(blockersAt({ dkgInfo, dkgStatus })).to.have.lengthOf(1); + }); + + it('should allow the stop once a session has finished its Commit phase', () => { + const dkgStatus = { session: [record('llmq_test_platform', HEIGHT - PLATFORM_ACTIVE_BLOCKS)] }; + + expect(blockersAt({ dkgInfo, dkgStatus })).to.be.empty(); + }); + + it('should block a 400-member session for its 20 blocks', () => { + expect(blockersAt({ dkgInfo, dkgStatus: { session: [record('llmq_400_60', HEIGHT - 19)] } })) + .to.have.lengthOf(1); + expect(blockersAt({ dkgInfo, dkgStatus: { session: [record('llmq_400_60', HEIGHT - 20)] } })) + .to.be.empty(); + }); + + it('should measure a rotated session from its own quorum index start', () => { + // llmq_60_75 index 20 starts at cycle base + 20, not at the cycle's + // shared mining window (cycle base + 42). + const quorumHeight = 288000 + 20; + + expect(blockersAt({ + dkgInfo, + height: quorumHeight + 9, + dkgStatus: { session: [record('llmq_60_75', quorumHeight, 20)] }, + })).to.have.lengthOf(1); + expect(blockersAt({ + dkgInfo, + height: quorumHeight + 10, + dkgStatus: { session: [record('llmq_60_75', quorumHeight, 20)] }, + })).to.be.empty(); + }); + + it('should allow the stop when every session has finished', () => { + const dkgStatus = { + session: [ + record('llmq_test_platform', HEIGHT - 20), + record('llmq_400_60', HEIGHT - 100), + ], + }; + + expect(blockersAt({ dkgInfo: { active_dkgs: 2, next_dkg: NEXT_DKG_NOT_IMMINENT }, dkgStatus })) + .to.be.empty(); + }); + + it('should block when any single session is still active, even if others have finished', () => { + const dkgStatus = { + session: [ + record('llmq_test_platform', HEIGHT - 50), + record('llmq_400_60', HEIGHT - 5), + ], + }; + + const blockers = blockersAt({ dkgInfo: { active_dkgs: 2, next_dkg: NEXT_DKG_NOT_IMMINENT }, dkgStatus }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ llmqType: 'llmq_400_60' }); + }); + + it('should not need session records when Core reports none', () => { + expect(blockersAt({ dkgInfo: { active_dkgs: 0, next_dkg: NEXT_DKG_NOT_IMMINENT } })) + .to.be.empty(); + }); + + describe('a record from the previous round at the next round', () => { + // Core replaces or removes the record shortly after the next round's + // start block connects; until then this node may be a member again. + const previousRound = HEIGHT - PLATFORM_INTERVAL; + + it('should block from the next round start block', () => { + const blockers = blockersAt({ + dkgInfo, + dkgStatus: { session: [record('llmq_test_platform', previousRound)] }, + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'new-round', from: HEIGHT, until: HEIGHT + PLATFORM_ACTIVE_BLOCKS, + }); + }); + + it('should not block just before the next round starts', () => { + expect(blockersAt({ + dkgInfo, + dkgStatus: { session: [record('llmq_test_platform', previousRound + 1)] }, + })).to.be.empty(); + }); + + it('should stop blocking once the next round has finished its Commit phase', () => { + expect(blockersAt({ + dkgInfo, + dkgStatus: { session: [record('llmq_test_platform', previousRound - PLATFORM_ACTIVE_BLOCKS)] }, + })).to.be.empty(); + }); + }); + }); + + describe('active_dkgs on Core 24.1+', () => { + it('should block while active_dkgs is positive even if no upcoming DKG blocks', () => { + const dkgInfo = { + active_dkgs: 1, + next_dkg: NEXT_DKG_NOT_IMMINENT, + upcoming_dkgs: [upcomingDkg(1, false)], + }; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_1 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'active-count', count: 1, until: null }); + }); + + it('should block a recorded session after Core stopped counting it', () => { + // A ProUpServTx mined mid-session clears Core's proTxHash, so it + // reports active_dkgs 0 and no upcoming_dkgs while the session runs. + const blockers = blockersAt({ + dkgInfo: { active_dkgs: 0, next_dkg: NEXT_DKG_NOT_IMMINENT }, + dkgStatus: { session: [record('llmq_test_platform', HEIGHT - 3)] }, + coreVersion: CORE_24_1, + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'session' }); + }); + + it('should block an unrecorded session even when a later member DKG is announced', () => { + // At a start block Core 24.1 already counts the session, before its + // DKG worker has recorded it; a DKG announced for later does not + // account for it. + const blockers = blockersAt({ + dkgInfo: { + active_dkgs: 1, + next_dkg: NEXT_DKG_NOT_IMMINENT, + upcoming_dkgs: [upcomingDkg(10, true)], + }, + coreVersion: CORE_24_1, + }); + + expect(blockers.map(({ reason }) => reason)).to.deep.equal(['active-count']); + }); + + it('should use Core\'s count from Core 24.1.0 only', () => { + expect(hasMembershipActiveDkgs(240000)).to.be.false(); + expect(hasMembershipActiveDkgs(240099)).to.be.false(); + expect(hasMembershipActiveDkgs(240100)).to.be.true(); + expect(hasMembershipActiveDkgs(250000)).to.be.true(); + expect(hasMembershipActiveDkgs('240100')).to.be.false(); + expect(hasMembershipActiveDkgs(null)).to.be.false(); + }); + }); + + describe('fail-safe on malformed dkginfo', () => { + const malformed = { + 'dkgInfo is undefined': undefined, + 'dkgInfo is null': null, + 'next_dkg is missing': { active_dkgs: 0 }, + 'next_dkg is non-numeric': { active_dkgs: 0, next_dkg: '24' }, + 'next_dkg is NaN': { active_dkgs: 0, next_dkg: NaN }, + 'next_dkg is negative': { active_dkgs: 0, next_dkg: -1 }, + 'active_dkgs is missing': { next_dkg: NEXT_DKG_NOT_IMMINENT }, + 'active_dkgs is non-numeric': { active_dkgs: '1', next_dkg: NEXT_DKG_NOT_IMMINENT }, + 'active_dkgs is NaN': { active_dkgs: NaN, next_dkg: NEXT_DKG_NOT_IMMINENT }, + 'active_dkgs is negative': { active_dkgs: -1, next_dkg: NEXT_DKG_NOT_IMMINENT }, + }; + + Object.entries(malformed).forEach(([description, dkgInfo]) => { + it(`should block when ${description}`, () => { + const blockers = blockersAt({ dkgInfo }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', source: 'quorum dkginfo' }); + }); + }); + + it('should block when the block height is not a height', () => { + const blockers = getDkgStopBlockers({ + dkgInfo: { active_dkgs: 0, next_dkg: NEXT_DKG_NOT_IMMINENT }, + dkgStatus: { session: [] }, + height: undefined, + coreVersion: CORE_23, + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', from: null, until: null }); + }); + }); + + describe('fail-safe on malformed dkgstatus', () => { + const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; + const malformed = { + 'a session has an unknown llmqType': { session: [record('llmq_future_unknown', HEIGHT)] }, + 'a session is missing quorumHeight': { session: [{ llmqType: 'llmq_test_platform', status: {} }] }, + 'a session has a NaN quorumHeight': { session: [record('llmq_test_platform', NaN)] }, + 'a session is missing status entirely': { session: [{ llmqType: 'llmq_test_platform' }] }, + 'a session starts above the block height': { session: [record('llmq_test_platform', HEIGHT + 5)] }, + 'a session entry is null': { session: [null] }, + 'dkgStatus is omitted': undefined, + 'dkgStatus.session is not an array': {}, + }; + + Object.entries(malformed).forEach(([description, dkgStatus]) => { + it(`should block when ${description}`, () => { + const blockers = getBlockersAt(getDkgStopBlockers({ + dkgInfo, dkgStatus, height: HEIGHT, coreVersion: CORE_23, + }), HEIGHT); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', until: null }); + }); + }); + }); + + describe('LLMQ table vs Dash Core src/llmq/params.h', () => { + it('should match every LLMQ type Core defines', () => { + expect(LLMQ_TYPES.map(({ + id, name, dkgInterval, dkgPhaseBlocks, + }) => [id, name, dkgInterval, dkgPhaseBlocks])).to.deep.equal([ + [1, 'llmq_50_60', 24, 2], + [2, 'llmq_400_60', 288, 4], + [3, 'llmq_400_85', 576, 4], + [4, 'llmq_100_67', 24, 2], + [5, 'llmq_60_75', 288, 2], + [6, 'llmq_25_67', 24, 2], + [100, 'llmq_test', 24, 2], + [101, 'llmq_devnet', 24, 2], + [102, 'llmq_test_v17', 24, 2], + [103, 'llmq_test_dip0024', 24, 2], + [104, 'llmq_test_instantsend', 24, 2], + [105, 'llmq_devnet_dip0024', 48, 2], + [106, 'llmq_test_platform', 24, 2], + [107, 'llmq_devnet_platform', 24, 2], + ]); + }); + + // [type, DKG interval, blocks from a session's start to the end of its + // Commit phase, quorum index], as Dash Core schedules them. + [ + ['llmq_60_75', 288, 10, 20], + ['llmq_400_60', 288, 20, 0], + ['llmq_400_85', 576, 20, 0], + ['llmq_100_67', 24, 10, 0], + ['llmq_50_60', 24, 10, 0], + ['llmq_25_67', 24, 10, 0], + ].forEach(([llmqType, interval, activeBlocks, quorumIndex]) => { + it(`should block a ${llmqType} member for ${activeBlocks} blocks of each session, every ${interval} blocks`, () => { + const quorumHeight = 288 * 24 * 10 + quorumIndex; + const isBlockedAtOffset = (offset) => blockersAt({ + dkgInfo: { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }, + height: quorumHeight + offset, + dkgStatus: { session: [record(llmqType, quorumHeight, quorumIndex)] }, + }).length > 0; + + expect(isBlockedAtOffset(activeBlocks - 1), 'last block of the session').to.be.true(); + expect(isBlockedAtOffset(activeBlocks), 'after the Commit phase').to.be.false(); + expect(isBlockedAtOffset(interval - 1), 'before the next session').to.be.false(); + expect(isBlockedAtOffset(interval), 'next session start').to.be.true(); + expect(isBlockedAtOffset(interval + activeBlocks - 1), 'next session end').to.be.true(); + expect(isBlockedAtOffset(interval + activeBlocks), 'after the next session').to.be.false(); + }); + }); + }); +}); diff --git a/packages/dashmate/test/unit/core/quorum/isMasternodeSafeToStopDuringDkg.spec.js b/packages/dashmate/test/unit/core/quorum/isMasternodeSafeToStopDuringDkg.spec.js deleted file mode 100644 index 1088c994022..00000000000 --- a/packages/dashmate/test/unit/core/quorum/isMasternodeSafeToStopDuringDkg.spec.js +++ /dev/null @@ -1,427 +0,0 @@ -import isMasternodeSafeToStopDuringDkg, { - DKG_MINING_WINDOW_START_BY_LLMQ_TYPE, - shouldInspectDkgStatusForSafeStop, -} from '../../../../src/core/quorum/isMasternodeSafeToStopDuringDkg.js'; -import { MIN_BLOCKS_BEFORE_DKG } from '../../../../src/constants.js'; - -// A value of next_dkg that is safely above MIN_BLOCKS_BEFORE_DKG so the -// imminent-DKG guard never trips in tests that focus on the per-session -// active-window logic. -const NEXT_DKG_NOT_IMMINENT = MIN_BLOCKS_BEFORE_DKG + 5; -const PLATFORM_WINDOW = DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_test_platform; // 10 -const LARGE_QUORUM_WINDOW = DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_400_60; // 20 - -describe('isMasternodeSafeToStopDuringDkg', () => { - describe('imminent DKG guard from dkginfo.next_dkg (Dash Core < v24)', () => { - it('blocks when next_dkg <= MIN_BLOCKS_BEFORE_DKG even when active_dkgs is 0', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('allows the stop when active_dkgs is 0 and next_dkg is past the imminent threshold', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG + 1 }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(true); - }); - - it('inspects dkgstatus only when active_dkgs > 0 and no DKG is imminent', () => { - expect(shouldInspectDkgStatusForSafeStop({ active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT })) - .to.equal(true); - expect(shouldInspectDkgStatusForSafeStop({ active_dkgs: 1, next_dkg: MIN_BLOCKS_BEFORE_DKG })) - .to.equal(false); - expect(shouldInspectDkgStatusForSafeStop({ active_dkgs: 0, next_dkg: NEXT_DKG_NOT_IMMINENT })) - .to.equal(false); - }); - }); - - describe('membership from dkginfo.upcoming_dkgs (Core with a proTxHash)', () => { - // Shape of a `quorum dkginfo` upcoming_dkgs entry once membership is known. - function upcomingDkg(blocksUntilStart, isMember) { - return { - llmqType: 106, - quorumIndex: 0, - quorumHeight: 1000 + blocksUntilStart, - blocksUntilStart, - known: true, - isMember, - workBlockHeight: 1000 + blocksUntilStart - 8, - workBlockHash: '00'.repeat(32), - }; - } - - function unknownUpcomingDkg(blocksUntilStart) { - return { - llmqType: 103, - quorumIndex: 0, - quorumHeight: 1000 + blocksUntilStart, - blocksUntilStart, - known: false, - reason: 'rotated quorum snapshots are not available yet', - }; - } - - it('should allow the stop when an imminent DKG does not include this node', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [upcomingDkg(1, false), upcomingDkg(MIN_BLOCKS_BEFORE_DKG, false)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(true); - }); - - it('should allow the stop when next_dkg is imminent but no upcoming DKG is listed', () => { - // next_dkg reports 1 for the whole rotation signing window, even - // after the last rotated index has started. - const dkgInfo = { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(true); - }); - - it('should block when active_dkgs is positive even if no upcoming DKG blocks', () => { - const dkgInfo = { - active_dkgs: 1, - next_dkg: NEXT_DKG_NOT_IMMINENT, - upcoming_dkgs: [upcomingDkg(1, false)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(false); - }); - - it('should not use dkgstatus to override a positive active_dkgs', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT, upcoming_dkgs: [] }; - // Every session is past its window, which would allow the legacy path. - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(shouldInspectDkgStatusForSafeStop(dkgInfo)).to.equal(false); - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000 + PLATFORM_WINDOW)) - .to.equal(false); - }); - - it('should block when this node is a member of an imminent DKG', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [upcomingDkg(1, false), upcomingDkg(MIN_BLOCKS_BEFORE_DKG, true)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(false); - }); - - it('should allow the stop when this node is a member of a DKG that is not yet imminent', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: MIN_BLOCKS_BEFORE_DKG + 1, - upcoming_dkgs: [upcomingDkg(MIN_BLOCKS_BEFORE_DKG + 1, true)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(true); - }); - - it('should block when membership of an imminent DKG is unknown', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [unknownUpcomingDkg(1)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(false); - }); - - it('should allow the stop when membership is unknown only for a DKG that is not yet imminent', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: MIN_BLOCKS_BEFORE_DKG + 1, - upcoming_dkgs: [unknownUpcomingDkg(MIN_BLOCKS_BEFORE_DKG + 1)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(true); - }); - - it('should not inspect dkgstatus when upcoming_dkgs is present', () => { - const dkgInfo = { - active_dkgs: 1, - next_dkg: NEXT_DKG_NOT_IMMINENT, - upcoming_dkgs: [upcomingDkg(1, true)], - }; - - expect(shouldInspectDkgStatusForSafeStop(dkgInfo)).to.equal(false); - }); - - describe('fail-safe on malformed upcoming_dkgs', () => { - const malformed = { - 'upcoming_dkgs is an object': {}, - 'upcoming_dkgs is null': null, - 'an entry is null': [null], - 'an entry is missing known': [{ blocksUntilStart: 1, isMember: false }], - 'a known entry is missing isMember': [{ blocksUntilStart: 1, known: true }], - 'an entry is missing blocksUntilStart': [{ known: true, isMember: false }], - 'an entry has a negative blocksUntilStart': [ - { blocksUntilStart: -1, known: true, isMember: false }, - ], - 'an entry outside the margin is malformed': [ - { blocksUntilStart: MIN_BLOCKS_BEFORE_DKG + 1, known: true }, - ], - }; - - Object.entries(malformed).forEach(([description, upcomingDkgs]) => { - it(`should block when ${description}`, () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: NEXT_DKG_NOT_IMMINENT, - upcoming_dkgs: upcomingDkgs, - }; - - expect(shouldInspectDkgStatusForSafeStop(dkgInfo)).to.equal(false); - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(false); - }); - }); - }); - }); - - describe('per-session active window from dkgstatus + getblockcount', () => { - it('blocks a platform session at offset 0 (quorumHeight == currentHeight)', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks a platform session at the last block of its active window (offset == window - 1)', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg( - dkgInfo, - dkgStatus, - 1000 + (PLATFORM_WINDOW - 1), - )).to.equal(false); - }); - - it('allows the stop once a platform session reaches offset == window (window closed) even with active_dkgs > 0', () => { - // The regression case this PR cares about: aggregate active_dkgs - // is still > 0 (a non-platform LLMQ in its window elsewhere on - // the node, say), but the platform session at offset 10 is past - // its active phase and should not block. - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000 + PLATFORM_WINDOW)) - .to.equal(true); - }); - - it('blocks a non-platform long-window session (llmq_400_60, offset 15) that is still in its 20-block window', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_400_60', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1015)).to.equal(false); - // Sanity: confirm 15 is inside the 20-block llmq_400_60 window. - expect(LARGE_QUORUM_WINDOW).to.be.greaterThan(15); - }); - - it('allows the stop when every session is past its window even with active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 2, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - { llmqType: 'llmq_400_60', status: { quorumHeight: 900 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1020)).to.equal(true); - }); - - it('blocks when any single session is still in its active window, even if others are stale', () => { - const dkgInfo = { active_dkgs: 2, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - // Stale platform session (offset 50). - { llmqType: 'llmq_test_platform', status: { quorumHeight: 950 } }, - // Active llmq_400_60 session (offset 5, window 20). - { llmqType: 'llmq_400_60', status: { quorumHeight: 995 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks a negative offset (quorumHeight ahead of currentHeight) as inconsistent status', () => { - // Core should only report locally tracked sessions whose quorum - // height has already arrived. If the chain tip is behind the - // reported quorum height while active_dkgs > 0, fail safe. - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1005 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - }); - - describe('fail-safe on malformed dkgInfo', () => { - it('blocks when dkgInfo is undefined', () => { - expect(isMasternodeSafeToStopDuringDkg(undefined, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when dkgInfo is null', () => { - expect(isMasternodeSafeToStopDuringDkg(null, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when next_dkg is missing', () => { - const dkgInfo = { active_dkgs: 0 }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when next_dkg is non-numeric', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: '24' }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when next_dkg is NaN', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: NaN }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when next_dkg is negative', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: -1 }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when active_dkgs is missing', () => { - const dkgInfo = { next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when active_dkgs is non-numeric', () => { - const dkgInfo = { active_dkgs: '1', next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when active_dkgs is NaN', () => { - const dkgInfo = { active_dkgs: NaN, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when active_dkgs is negative', () => { - const dkgInfo = { active_dkgs: -1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - }); - - describe('fail-safe on malformed inputs while active_dkgs > 0', () => { - it('blocks when a session has an unknown llmqType', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_future_unknown', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks when a session is missing quorumHeight', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: {} }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks when a session is missing status entirely', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform' }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks when dkgStatus is omitted while active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, undefined, 1000)).to.equal(false); - }); - - it('blocks when dkgStatus.session is not an array while active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, {}, 1000)).to.equal(false); - }); - - it('blocks when dkgStatus.session is empty while active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)).to.equal(false); - }); - - it('blocks when currentHeight is omitted while active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus)).to.equal(false); - }); - }); - - describe('window table sanity vs Dash Core src/llmq/params.h', () => { - it('exposes the expected dkgMiningWindowStart values', () => { - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_test_platform).to.equal(10); - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_test_dip0024).to.equal(12); - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_400_60).to.equal(20); - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_400_85).to.equal(20); - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_60_75).to.equal(42); - }); - }); -}); diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index 872d5db2a49..23061eb8446 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -1,170 +1,455 @@ -import waitForDKGWindowPass from '../../../../src/core/quorum/waitForDKGWindowPass.js'; - -const CHECK_INTERVAL_MS = 10000; +import waitForDKGWindowPass, { + CHECK_INTERVAL_MS, +} from '../../../../src/core/quorum/waitForDKGWindowPass.js'; +import { SAFE_STOP_CONFIRMATION_DELAY_MS } from '../../../../src/core/quorum/checkMasternodeSafeToStop.js'; +import getCoreRpcMock from '../../../../src/test/mock/getCoreRpcMock.js'; + +const CORE_23 = 230110; +const CORE_24_0 = 240000; +const CORE_24_1 = 240100; + +// Long enough for at least one more poll, confirmation included. Ticks of +// this length land between polls, so state a test changes afterwards is +// read by the next poll. +const NEXT_POLL_MS = CHECK_INTERVAL_MS + SAFE_STOP_CONFIRMATION_DELAY_MS + 1; + +function track(promise) { + const state = { resolved: false, error: undefined }; + promise.then(() => { state.resolved = true; }, (e) => { state.error = e; }); + return state; +} describe('waitForDKGWindowPass', () => { - let rpcClient; + let clock; beforeEach(function beforeEach() { - rpcClient = { - quorum: this.sinon.stub(), - getBlockCount: this.sinon.stub(), - }; + clock = this.sinon.useFakeTimers(); }); - it('resolves immediately when there are no active sessions and no imminent cycle', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 0, next_dkg: 24 } }); + async function expectStillPolling(waiting, rpcClient) { + const polls = rpcClient.quorum.withArgs('dkginfo').callCount; - await waitForDKGWindowPass(rpcClient); - - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); - expect(rpcClient.getBlockCount).to.not.have.been.called(); - }); + await clock.tickAsync(NEXT_POLL_MS); - it('should resolve immediately when current and imminent DKGs do not include this node', async function it() { - const clock = this.sinon.useFakeTimers(); + expect(waiting.error).to.be.undefined(); + expect(waiting.resolved).to.be.false(); + expect(rpcClient.quorum.withArgs('dkginfo').callCount).to.be.greaterThan(polls); + } - rpcClient.quorum.withArgs('dkginfo').resolves({ - result: { + it('should resolve at once on Core 24.1 when no DKG includes this node', async function it() { + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_24_1, + height: 1000, + dkgInfo: { active_dkgs: 0, next_dkg: 1, - upcoming_dkgs: [{ blocksUntilStart: 1, known: true, isMember: false }], + upcoming_dkgs: [{ + llmqType: 106, blocksUntilStart: 1, known: true, isMember: false, + }], }, + dkgStatus: { session: [] }, }); - const promise = waitForDKGWindowPass(rpcClient); - + const waiting = track(waitForDKGWindowPass(rpcClient)); await clock.tickAsync(0); - await promise; + expect(waiting.resolved).to.be.true(); expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); - expect(rpcClient.getBlockCount).to.not.have.been.called(); }); - it('should keep waiting while this node is a member of an imminent DKG', async function it() { - const clock = this.sinon.useFakeTimers(); - - rpcClient.quorum.withArgs('dkginfo') - .onFirstCall() - .resolves({ - result: { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [{ blocksUntilStart: 1, known: true, isMember: true }], - }, - }) - .onSecondCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] } }); - - const promise = waitForDKGWindowPass(rpcClient); + it('should resolve once a safe result is confirmed on Core 24.0', async function it() { + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_24_0, + height: 1000, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 106, blocksUntilStart: 1, known: true, isMember: false, + }], + }, + dkgStatus: { session: [] }, + }); - await clock.tickAsync(0); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS - 1); + + expect(waiting.resolved).to.be.false(); - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + await clock.tickAsync(1); + expect(waiting.resolved).to.be.true(); expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); }); - it('should wait while active_dkgs is positive without inspecting dkgstatus', async function it() { - const clock = this.sinon.useFakeTimers(); - rpcClient.quorum.withArgs('dkginfo') - .onFirstCall() - .resolves({ - result: { - active_dkgs: 1, - next_dkg: 1, - upcoming_dkgs: [], - }, - }) - .onSecondCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] } }); - - const promise = waitForDKGWindowPass(rpcClient); - await clock.tickAsync(0); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + it('should keep waiting while this node is a member of an imminent DKG', async function it() { + const core = { + version: CORE_24_1, + height: 1000, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 106, blocksUntilStart: 1, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); - expect(rpcClient.quorum.withArgs('dkgstatus')).to.not.have.been.called(); - expect(rpcClient.getBlockCount).to.not.have.been.called(); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + // The session started at 1001 and its Commit phase has ended. + core.height = 1011; + core.dkgInfo = { active_dkgs: 0, next_dkg: 13, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, status: { quorumHeight: 1001 } }], + }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting on Core 24.1 while active_dkgs is positive', async function it() { + const core = { + version: CORE_24_1, + height: 1000, + dkgInfo: { active_dkgs: 1, next_dkg: 1, upcoming_dkgs: [] }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + core.dkgInfo = { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should wait through an active session and resolve once its Commit phase has ended', async function it() { + const core = { + version: CORE_23, + height: 1005, + dkgInfo: { active_dkgs: 1, next_dkg: 19 }, + dkgStatus: { + session: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, status: { quorumHeight: 1000 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + core.height = 1010; + core.dkgInfo = { active_dkgs: 1, next_dkg: 14 }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting while a DKG is imminent and Core reports no membership', async function it() { + const core = { + version: CORE_23, + height: 1000, + dkgInfo: { active_dkgs: 0, next_dkg: 3 }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + // The DKG passed without including this node. + core.height = 1013; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14 }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting while Core reports a session dashmate cannot read', async function it() { + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_23, + height: 1000, + dkgInfo: { active_dkgs: 1, next_dkg: 20 }, + dkgStatus: { + session: [{ llmqType: 'llmq_future_unknown', quorumIndex: 0, status: { quorumHeight: 900 } }], + }, + }); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(CHECK_INTERVAL_MS * 10); + + await expectStillPolling(waiting, rpcClient); }); - it('waits through an active platform session and resolves once the window has passed', async function it() { - // Block height advances from 1005 (offset 5, in window) to 1010 - // (offset 10, window closed) between polls; active_dkgs stays > 0 - // to model Dash Core's lingering aggregate counter. - const clock = this.sinon.useFakeTimers(); + it('should stop waiting on Core 24.0 when active_dkgs counts only finished sessions', async function it() { + // Core 24.0 reports upcoming_dkgs but counts lingering debug records in + // active_dkgs, so a masternode that was in any recent DKG never sees 0. + const height = 288000 + 12; - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 1, next_dkg: 20 } }); - rpcClient.quorum.withArgs('dkgstatus').resolves({ - result: { + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_24_0, + height, + dkgInfo: { + active_dkgs: 2, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 5, + quorumIndex: 20, + quorumHeight: 288000 + 20, + blocksUntilStart: 8, + known: true, + isMember: false, + }, { + llmqType: 5, + quorumIndex: 14, + quorumHeight: 288000 + 14, + blocksUntilStart: 2, + known: true, + isMember: false, + }], + }, + // This cycle's index 25 has not started, so last cycle's record stays; + // this hour's llmq_100_67 session ended two blocks ago. + dkgStatus: { session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, + { llmqType: 'llmq_60_75', quorumIndex: 25, status: { quorumHeight: 288000 - 288 + 25 } }, + { llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: 288000 } }, ], }, }); - rpcClient.getBlockCount - .onFirstCall().resolves({ result: 1005 }) - .onSecondCall().resolves({ result: 1010 }); - const promise = waitForDKGWindowPass(rpcClient); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); - // Drain microtasks so the first iteration completes its three RPCs - // and parks on `wait(CHECK_INTERVAL_MS)`. - await clock.tickAsync(0); - expect(rpcClient.getBlockCount).to.have.been.calledOnce(); + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting at a rotation cycle start on Core 24.0 until this node\'s index has passed', async function it() { + // Last cycle this node was in llmq_60_75 index 5. The new cycle's index 5 + // has started and Core has not replaced that record yet; Core announces + // only the indices still to come, none of which include this node. + const cycleBase = 288000; + const core = { + version: CORE_24_0, + height: cycleBase + 5, + dkgInfo: { + active_dkgs: 1, + next_dkg: 1, + upcoming_dkgs: [6, 7, 8].map((quorumIndex) => ({ + llmqType: 5, + quorumIndex, + quorumHeight: cycleBase + quorumIndex, + blocksUntilStart: quorumIndex - 5, + known: true, + isMember: false, + })), + }, + dkgStatus: { + session: [{ llmqType: 'llmq_60_75', quorumIndex: 5, status: { quorumHeight: cycleBase - 288 + 5 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); - // Advance past the wait so the second iteration runs and returns. - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); - expect(rpcClient.getBlockCount).to.have.been.calledTwice(); + // Ten blocks later Core has removed the record: this node is not in the + // new index 5. + core.height = cycleBase + 15; + core.dkgInfo = { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] }; + core.dkgStatus = { session: [] }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); }); - it('keeps waiting while next_dkg is imminent even when there are no sessions', async function it() { - const clock = this.sinon.useFakeTimers(); + it('should keep waiting through an announced DKG while Core has not recorded its session', async function it() { + // At the start block Core erases last round's record, computes the new + // session's members and only then records it. On a busy node that can + // outlast the confirmation delay. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 1, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 1, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); - rpcClient.quorum.withArgs('dkginfo') - .onFirstCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 3 } }) - .onSecondCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 24 } }); + core.height = start; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + await expectStillPolling(waiting, rpcClient); + await expectStillPolling(waiting, rpcClient); - const promise = waitForDKGWindowPass(rpcClient); + // The session has passed its Commit phase. + core.height = start + 10; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14, upcoming_dkgs: [] }; + await clock.tickAsync(NEXT_POLL_MS); - await clock.tickAsync(0); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + expect(waiting.resolved).to.be.true(); + }); - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + it('should end a rotated DKG session once its own Commit phase is over', async function it() { + // llmq_60_75 index 31 starts at cycle base + 31; ten blocks later this + // member has sent everything its DKG needs, although the cycle's shared + // mining window only opens at cycle base + 42. + const quorumHeight = 288000 + 31; + + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_23, + height: quorumHeight + 10, + dkgInfo: { active_dkgs: 1, next_dkg: 7 }, + dkgStatus: { + session: [{ llmqType: 'llmq_60_75', quorumIndex: 31, status: { quorumHeight } }], + }, + }); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting at a DKG start block while Core still shows the previous round', async function it() { + // The new llmq_100_67 round starts at `height`. Core has not replaced + // last round's record yet, so it looks 24 blocks old, past its window. + const height = 24 * 5000; + const core = { + version: CORE_23, + height, + dkgInfo: { active_dkgs: 1, next_dkg: 24 }, + dkgStatus: { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: height - 24 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + await expectStillPolling(waiting, rpcClient); + + // Core records the new session; it ends ten blocks after it started. + core.dkgStatus = { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: height } }], + }; + core.height = height + 10; + core.dkgInfo = { active_dkgs: 1, next_dkg: 14 }; + await clock.tickAsync(60000); + + expect(waiting.resolved).to.be.true(); }); - it('fails safe (keeps waiting) when active_dkgs > 0 and a session has an unknown llmqType', async function it() { - const clock = this.sinon.useFakeTimers(); + it('should not stop at a DKG start that Core has not recorded yet', async function it() { + // At the start block Core answers before its DKG worker has created the + // session record; a moment later the record appears. + const height = 24 * 5000; + const core = { + version: CORE_23, + height, + dkgInfo: { active_dkgs: 0, next_dkg: 24 }, + dkgStatus: { session: [] }, + }; + core.afterDkgStatus = () => { + core.dkgInfo = { active_dkgs: 1, next_dkg: 24 }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: height } }], + }; + core.afterDkgStatus = undefined; + }; + const rpcClient = getCoreRpcMock(this.sinon, core); - rpcClient.quorum.withArgs('dkginfo') - .onFirstCall() - .resolves({ result: { active_dkgs: 1, next_dkg: 20 } }) - .onSecondCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 20 } }); - rpcClient.quorum.withArgs('dkgstatus').resolves({ - result: { session: [{ llmqType: 'llmq_future_unknown', status: { quorumHeight: 1000 } }] }, - }); - rpcClient.getBlockCount.resolves({ result: 1000 }); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); - const promise = waitForDKGWindowPass(rpcClient); + await expectStillPolling(waiting, rpcClient); + }); + + it('should not trust DKG reads that saw a newer tip than the height read', async function it() { + // The first check is safe: the member DKG is still 7 blocks away. Before + // the confirming check reads Core's DKG state, the chain reaches that + // DKG's start block, which Core has not recorded yet. The confirming check + // also sees more new blocks than the first covered; checkMasternodeSafeToStop + // tests the torn read on its own. + const height = 24 * 5000 - 7; + const memberDkgStart = height + 7; + const core = { + version: CORE_24_0, + height, + dkgInfo: { + active_dkgs: 0, + next_dkg: 7, + upcoming_dkgs: [{ + llmqType: 4, + quorumIndex: 0, + quorumHeight: memberDkgStart, + blocksUntilStart: 7, + known: true, + isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + const waiting = track(waitForDKGWindowPass(rpcClient)); await clock.tickAsync(0); - expect(rpcClient.quorum.withArgs('dkgstatus')).to.have.been.calledOnce(); - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + // Six blocks arrive while the first verdict waits to be confirmed. + core.height = height + 6; + core.beforeDkgInfo = () => { + core.height = memberDkgStart; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + core.beforeDkgInfo = undefined; + }; + await clock.tickAsync(20000); + + // Core has now recorded the session. + core.dkgInfo = { active_dkgs: 1, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: memberDkgStart } }], + }; + await clock.tickAsync(60000); + + await expectStillPolling(waiting, rpcClient); + }); + + it('should keep waiting on Core 24.1 while a session runs after the proTxHash was cleared', async function it() { + // A ProUpServTx mined mid-session clears Core's proTxHash: dkginfo then + // reports no active DKG and no upcoming_dkgs, but the session continues. + const height = 24 * 5000 + 3; + const core = { + version: CORE_24_1, + height, + dkgInfo: { active_dkgs: 0, next_dkg: 21 }, + dkgStatus: { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: height - 3 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + await expectStillPolling(waiting, rpcClient); + + // The session has passed its Commit phase. + core.height = height + 7; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14 }; + await clock.tickAsync(NEXT_POLL_MS); - expect(rpcClient.quorum.withArgs('dkgstatus')).to.have.been.calledOnce(); + expect(waiting.resolved).to.be.true(); }); }); diff --git a/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js b/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js new file mode 100644 index 00000000000..f62ebc7f82f --- /dev/null +++ b/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js @@ -0,0 +1,133 @@ +import { Listr } from 'listr2'; +import stopNodeTaskFactory from '../../../../src/listr/tasks/stopNodeTaskFactory.js'; +import { SAFE_STOP_CONFIRMATION_DELAY_MS } from '../../../../src/core/quorum/checkMasternodeSafeToStop.js'; +import getConfigMock from '../../../../src/test/mock/getConfigMock.js'; +import getCoreRpcMock from '../../../../src/test/mock/getCoreRpcMock.js'; + +const CORE_24_0 = 240000; +const CORE_24_1 = 240100; + +describe('stopNodeTaskFactory', () => { + let clock; + let core; + let rpcClient; + let config; + let dockerCompose; + let stopNodeTask; + + beforeEach(function beforeEach() { + clock = this.sinon.useFakeTimers(); + + core = { + version: CORE_24_1, + height: 1012, + dkgInfo: { active_dkgs: 0, next_dkg: 12, upcoming_dkgs: [] }, + dkgStatus: { session: [] }, + }; + + config = getConfigMock(this.sinon); + config.get.withArgs('core.masternode.enable').returns(true); + config.getName.returns('mainnet'); + + dockerCompose = { + isNodeRunning: this.sinon.stub().resolves(true), + stop: this.sinon.stub().resolves(), + }; + + stopNodeTask = stopNodeTaskFactory( + dockerCompose, + () => { + rpcClient = getCoreRpcMock(this.sinon, core); + return rpcClient; + }, + async () => '127.0.0.1', + () => [], + ); + }); + + /** + * Run a plain stop, advancing the fake clock past any confirmation delays. + */ + async function stop() { + const state = {}; + new Listr([{ task: () => stopNodeTask(config) }], { renderer: 'silent' }) + .run({ isForce: false, isSafe: false, platformOnly: false }) + .then(() => { state.stopped = true; }, (e) => { state.error = e; }); + + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS * 4); + + return state; + } + + it('should stop a masternode that no DKG includes', async () => { + const { stopped } = await stop(); + + expect(stopped).to.be.true(); + expect(dockerCompose.stop).to.have.been.calledOnce(); + }); + + it('should refuse to stop a masternode during its DKG session', async () => { + core.dkgInfo = { active_dkgs: 1, next_dkg: 12, upcoming_dkgs: [] }; + + const { error } = await stop(); + + expect(error.message).to.include('may result in a PoSe ban'); + expect(dockerCompose.stop).to.not.have.been.called(); + }); + + it('should refuse a stop on Core 24.0 when a session appears during the confirmation', async () => { + // At a DKG start block Core 24.0 reports the session only once its DKG + // worker has recorded it. + core.version = CORE_24_0; + core.afterDkgStatus = () => { + core.dkgInfo = { active_dkgs: 1, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: 1012 } }], + }; + core.afterDkgStatus = undefined; + }; + + const { error } = await stop(); + + expect(error.message).to.include('may result in a PoSe ban'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + expect(dockerCompose.stop).to.not.have.been.called(); + }); + + it('should refuse to stop when Core reports DKG data dashmate cannot read', async () => { + // --safe would wait for this forever, so it is not offered. + core.dkgInfo = { active_dkgs: 1, next_dkg: 12, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_future_unknown', quorumIndex: 0, status: { quorumHeight: 1000 } }], + }; + + const { error } = await stop(); + + expect(error.message).to.include('dashmate cannot read Core\'s DKG state'); + expect(error.message).to.not.include('--safe'); + expect(dockerCompose.stop).to.not.have.been.called(); + }); + + it('should check again when a block connects during a check', async () => { + core.beforeDkgInfo = () => { + core.height += 1; + core.beforeDkgInfo = undefined; + }; + + const { stopped } = await stop(); + + expect(stopped).to.be.true(); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + }); + + it('should give up after three checks while the chain keeps advancing', async () => { + // Core catching up connects a block during every check. + core.beforeDkgInfo = () => { core.height += 1; }; + + const { error } = await stop(); + + expect(error.message).to.include('The chain kept advancing'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledThrice(); + expect(dockerCompose.stop).to.not.have.been.called(); + }); +}); From 630e62412d83ab7f677437a25f708c300fb579b7 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Tue, 10 Nov 2026 22:24:04 +0700 Subject: [PATCH 02/11] fix(dashmate): close remaining unsafe-stop gaps around DKG starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Core older than 24.1, a check that begins at a DKG's start block can fall into the gap where Core has erased the previous round's record but not yet recorded the new session, and that gap has no upper bound. `quorum dkgstatus` lists every enabled quorum with the start of its current session, so a session starting at the tip without a record now blocks that block. `--safe` remembers DKGs Core announced without this node, and a confirming check reuses the first check's announcements, so their start blocks do not wait for a record that never comes. A `dkgstatus` answer without quorum connections is unreadable. The confirming check now reads the active chain's block at the first check's height inside its consistent snapshot. A reorg that replaced the first tip and ended up to 6 blocks higher passed the old same-height hash comparison. Group commands stop a local network's miner before its masternodes. With `--safe`, now effective for `group restart`, the masternodes then waited for blocks that could never come. Once a local network's miner is stopped, later masternodes in the run skip the DKG check: with every masternode going down together, their DKG produces no commitment to penalize them in. Core in initial block download stays safe to stop when blocks connect between the reads, and a Core version that is not an integer is now an error instead of being taken for an old Core. Tests would have caught these: ✖ before the fix, ✔ after: - should not stop at a DKG start reached in Core's record gap, however long it lasts - should refuse a plain stop at a DKG start that Core has not recorded yet - should report unsettled when a reorg replaced the first check's tip and the chain grew - should not wait for DKGs once the local network's miner has stopped - should treat Core in initial block download as safe even while blocks keep connecting - should refuse to guess when Core reports no integer version - should pass on what it knows about DKGs this node is not part of Co-Authored-By: Claude Opus 5.5 --- .../core/quorum/checkMasternodeSafeToStop.js | 102 ++++++++++++------ .../src/core/quorum/getDkgStopBlockers.js | 89 ++++++++++++++- .../src/core/quorum/waitForDKGWindowPass.js | 15 ++- .../src/listr/tasks/stopNodeTaskFactory.js | 15 ++- .../dashmate/src/test/mock/getCoreRpcMock.js | 20 +++- .../quorum/checkMasternodeSafeToStop.spec.js | 48 ++++++++- .../core/quorum/getDkgStopBlockers.spec.js | 102 ++++++++++++++++-- .../core/quorum/waitForDKGWindowPass.spec.js | 65 +++++++++++ .../listr/tasks/stopNodeTaskFactory.spec.js | 40 +++++++ 9 files changed, 445 insertions(+), 51 deletions(-) diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index 921620162e7..5fe42aab6f8 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -1,6 +1,7 @@ import wait from '../../util/wait.js'; import { MIN_BLOCKS_BEFORE_DKG } from '../../constants.js'; import getDkgStopBlockers, { + getAnnouncedNonMemberDkgs, getBlockersAt, hasMembershipActiveDkgs, } from './getDkgStopBlockers.js'; @@ -13,38 +14,59 @@ import getDkgStopBlockers, { export const SAFE_STOP_CONFIRMATION_DELAY_MS = 15000; /** + * Which stop rule applies depends on Core's version, so a version Core does + * not report is an error rather than a guess. + * * @param {RpcClient} rpcClient - * @return {Promise} Core's numeric `version`, or `null` if - * Core does not report one + * @return {Promise} Core's numeric `version` */ export async function readCoreVersion(rpcClient) { const { result } = await rpcClient.getNetworkInfo(); - return Number.isInteger(result && result.version) ? result.version : null; + if (!Number.isInteger(result && result.version)) { + throw new Error('Core did not report its version'); + } + + return result.version; } /** * Read Core's DKG state at one chain tip. Each RPC reads the tip on its own, * so the tip is read before and after the DKG calls; if a block connected in - * between, they may describe different tips and are discarded. + * between, they may describe different tips and are discarded. Core in + * initial block download takes no part in DKGs, so that state holds however + * the tip moves. + * + * With `ancestorHeight`, also reads the hash of the active chain's block at + * that height, at the same tip. * * @param {RpcClient} rpcClient + * @param {number} [ancestorHeight] * @return {Promise} */ -async function readDkgSnapshot(rpcClient) { +async function readDkgSnapshot(rpcClient, ancestorHeight) { const { result: before } = await rpcClient.getBlockchainInfo(); + + let ancestorHash; + if (ancestorHeight !== undefined && before.blocks >= ancestorHeight) { + ({ result: ancestorHash } = await rpcClient.getBlockHash(ancestorHeight)); + } + const { result: dkgInfo } = await rpcClient.quorum('dkginfo'); const { result: dkgStatus } = await rpcClient.quorum('dkgstatus'); const { result: after } = await rpcClient.getBlockchainInfo(); - if (before.bestblockhash !== after.bestblockhash) { + const isSyncing = before.initialblockdownload === true && after.initialblockdownload === true; + + if (!isSyncing && before.bestblockhash !== after.bestblockhash) { return null; } return { height: after.blocks, hash: after.bestblockhash, - isSyncing: before.initialblockdownload === true && after.initialblockdownload === true, + ancestorHash, + isSyncing, dkgInfo, dkgStatus, }; @@ -57,7 +79,7 @@ async function readDkgSnapshot(rpcClient) { * when a session starts. * * @param {Object} snapshot - * @param {number|null} coreVersion + * @param {number} coreVersion * @return {boolean} */ function haveSessionRecordsChanged({ dkgInfo, dkgStatus }, coreVersion) { @@ -74,44 +96,53 @@ function haveSessionRecordsChanged({ dkgInfo, dkgStatus }, coreVersion) { * @property {number|null} height * @property {string} [hash] chain tip the check read * @property {DkgStopBlocker[]} blockers see {@link getDkgStopBlockers} + * @property {Set} nonMemberDkgs DKGs Core announced without this + * node, see {@link getAnnouncedNonMemberDkgs} * @property {boolean} [isSyncing] Core is in initial block download */ /** * @param {RpcClient} rpcClient - * @param {number|null} coreVersion + * @param {number} coreVersion + * @param {Set} knownNonMemberDkgs + * @param {number} [ancestorHeight] * @return {Promise} */ -async function evaluate(rpcClient, coreVersion) { - const snapshot = await readDkgSnapshot(rpcClient); +async function evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestorHeight) { + const snapshot = await readDkgSnapshot(rpcClient, ancestorHeight); if (!snapshot) { - return { status: 'unsettled', height: null, blockers: [] }; + return { + status: 'unsettled', height: null, blockers: [], nonMemberDkgs: new Set(), + }; } const { height, hash, isSyncing } = snapshot; + const result = { + height, + hash, + ancestorHash: snapshot.ancestorHash, + nonMemberDkgs: getAnnouncedNonMemberDkgs(snapshot.dkgInfo, height), + }; // Core's DKG worker does nothing during initial block download, and Core // never re-enters it without a restart, so a node in it has had no DKG // session since it started. if (isSyncing) { return { - status: 'safe', height, hash, blockers: [], isSyncing, + ...result, status: 'safe', blockers: [], isSyncing, }; } if (haveSessionRecordsChanged(snapshot, coreVersion)) { - return { - status: 'unsettled', height, hash, blockers: [], - }; + return { ...result, status: 'unsettled', blockers: [] }; } - const blockers = getDkgStopBlockers({ ...snapshot, coreVersion }); + const blockers = getDkgStopBlockers({ ...snapshot, coreVersion, knownNonMemberDkgs }); return { + ...result, status: getBlockersAt(blockers, height).length > 0 ? 'blocked' : 'safe', - height, - hash, blockers, }; } @@ -121,20 +152,27 @@ async function evaluate(rpcClient, coreVersion) { * DKG it takes part in. See {@link getDkgStopBlockers} for the rule. * * Core older than 24.1 reports a session only once its DKG worker has - * recorded it, shortly after the start block connects, so a safe result is - * checked again after {@link SAFE_STOP_CONFIRMATION_DELAY_MS}. By then every - * block the first check saw is old enough to have its sessions recorded. - * The first check already ruled out a DKG of this node starting within - * `MIN_BLOCKS_BEFORE_DKG` blocks, so the second may see up to that many new - * blocks; more than that, a lower tip, or a different block at the first - * check's height is `unsettled`. + * recorded it, after the start block connects, so a safe result is checked + * again after {@link SAFE_STOP_CONFIRMATION_DELAY_MS}. By then every block the + * first check saw is old enough for Core to have replaced records of previous + * rounds. The first check already ruled out a DKG of this node starting + * within `MIN_BLOCKS_BEFORE_DKG` blocks, so the second may see up to that + * many new blocks on top of the first check's tip; more than that, a lower + * tip, or a chain without the first check's tip is `unsettled`. * * @param {RpcClient} rpcClient - * @param {number|null} coreVersion from {@link readCoreVersion} + * @param {number} coreVersion from {@link readCoreVersion} + * @param {Object} [options] + * @param {Set} [options.knownNonMemberDkgs] DKGs Core announced + * without this node at earlier checks * @return {Promise} */ -export default async function checkMasternodeSafeToStop(rpcClient, coreVersion) { - const first = await evaluate(rpcClient, coreVersion); +export default async function checkMasternodeSafeToStop( + rpcClient, + coreVersion, + { knownNonMemberDkgs = new Set() } = {}, +) { + const first = await evaluate(rpcClient, coreVersion, knownNonMemberDkgs); if (first.status !== 'safe' || first.isSyncing || hasMembershipActiveDkgs(coreVersion)) { return first; @@ -142,10 +180,12 @@ export default async function checkMasternodeSafeToStop(rpcClient, coreVersion) await wait(SAFE_STOP_CONFIRMATION_DELAY_MS); - const second = await evaluate(rpcClient, coreVersion); + const nonMemberDkgs = new Set([...knownNonMemberDkgs, ...first.nonMemberDkgs]); + const second = await evaluate(rpcClient, coreVersion, nonMemberDkgs, first.height); + second.nonMemberDkgs = new Set([...nonMemberDkgs, ...second.nonMemberDkgs]); const hasChainMoved = second.height < first.height - || (second.height === first.height && second.hash !== first.hash) + || second.ancestorHash !== first.hash || second.height > first.height + MIN_BLOCKS_BEFORE_DKG; if (second.status === 'safe' && !second.isSyncing && hasChainMoved) { diff --git a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js index 746865976de..3befec90586 100644 --- a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js +++ b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js @@ -86,8 +86,8 @@ export function hasMembershipActiveDkgs(coreVersion) { /** * @typedef {Object} DkgStopBlocker - * @property {string} reason `session`, `new-round`, `upcoming`, `imminent`, - * `active-count` or `unreadable` + * @property {string} reason `session`, `new-round`, `starting`, `upcoming`, + * `imminent`, `active-count` or `unreadable` * @property {string|number|null} [llmqType] LLMQ name, or Core's numeric id * when dashmate does not know the type * @property {number} [quorumIndex] @@ -243,6 +243,77 @@ function getSessionRecordBlockers(dkgStatus, height) { }); } +/** + * Key naming a DKG session: its quorum and its start block. + * + * @param {string} llmqType + * @param {number} quorumIndex + * @param {number} start + * @return {string} + */ +function getDkgKey(llmqType, quorumIndex, start) { + return `${llmqType}:${quorumIndex}:${start}`; +} + +/** + * The DKGs Core announces this node is not a member of, as + * `llmqType:quorumIndex:start` keys. Once such a DKG starts Core no longer + * announces it, so a caller polling Core can remember these to tell a + * session it is not part of from one Core has not recorded yet. + * + * @param {Object} dkgInfo Result of `quorum dkginfo`. + * @param {number} height Chain tip `dkgInfo` describes. + * @return {Set} + */ +export function getAnnouncedNonMemberDkgs(dkgInfo, height) { + const upcomingDkgs = dkgInfo && dkgInfo.upcoming_dkgs; + + if (!Array.isArray(upcomingDkgs) || !isCount(height)) { + return new Set(); + } + + return new Set(upcomingDkgs + .filter((dkg) => isValidUpcomingDkg(dkg) && dkg.known && !dkg.isMember) + .map((dkg) => [LLMQ_TYPES.find(({ id }) => id === dkg.llmqType), dkg]) + .filter(([llmq]) => llmq) + .map(([llmq, dkg]) => getDkgKey(llmq.name, dkg.quorumIndex, height + dkg.blocksUntilStart))); +} + +/** + * At its start block Core older than 24.1 erases the previous round's record, + * computes the new session's members and only then records the session if + * this node is one of them; nothing tells whether it is until then, however + * long that takes. `quorum dkgstatus` lists every enabled quorum with the + * start of its current session, so a session starting at the tip without a + * record blocks that block, unless Core announced it without this node. + */ +function getStartingDkgBlockers(dkgStatus, height, knownNonMemberDkgs) { + const connections = dkgStatus && dkgStatus.quorumConnections; + + if (!Array.isArray(connections)) { + return [unreadable('quorum dkgstatus quorumConnections')]; + } + + const sessions = Array.isArray(dkgStatus.session) ? dkgStatus.session : []; + + return connections + .filter((quorum) => quorum && quorum.pQuorumBaseBlockIndex === height) + .filter(({ llmqType, quorumIndex }) => !knownNonMemberDkgs + .has(getDkgKey(llmqType, quorumIndex, height))) + .filter(({ llmqType, quorumIndex }) => !sessions.some((entry) => entry + && entry.llmqType === llmqType + && entry.quorumIndex === quorumIndex + && entry.status + && entry.status.quorumHeight === height)) + .map(({ llmqType, quorumIndex }) => ({ + reason: 'starting', + llmqType, + quorumIndex, + from: height, + until: height + 1, + })); +} + /** * Everything that makes stopping this masternode risk a PoSe penalty, as * height ranges. Stopping is safe at a height no blocker covers (see @@ -251,8 +322,9 @@ function getSessionRecordBlockers(dkgStatus, height) { * * All inputs must come from the same chain tip, `height`. Core older than * {@link MEMBERSHIP_ACTIVE_DKGS_CORE_VERSION} reports a session only after - * its DKG worker has recorded it, shortly after the start block connects; - * callers must confirm a safe result once that delay has passed. + * its DKG worker has recorded it, after the start block connects. The start + * block blocks until then; callers must also confirm a safe result a while + * later, once Core has had time to replace records of previous rounds. * * Unreadable Core data blocks with no known end. * @@ -260,7 +332,9 @@ function getSessionRecordBlockers(dkgStatus, height) { * @param {Object} snapshot.dkgInfo Result of `quorum dkginfo`. * @param {Object} snapshot.dkgStatus Result of `quorum dkgstatus`. * @param {number} snapshot.height Chain tip both results describe. - * @param {number|null} snapshot.coreVersion Core's numeric `version`. + * @param {number} snapshot.coreVersion Core's numeric `version`. + * @param {Set} [snapshot.knownNonMemberDkgs] DKGs Core announced + * without this node, from {@link getAnnouncedNonMemberDkgs} * @return {DkgStopBlocker[]} */ export default function getDkgStopBlockers({ @@ -268,6 +342,7 @@ export default function getDkgStopBlockers({ dkgStatus, height, coreVersion, + knownNonMemberDkgs = new Set(), }) { if (!isCount(height)) { return [unreadable('block height')]; @@ -287,6 +362,10 @@ export default function getDkgStopBlockers({ ...getSessionRecordBlockers(dkgStatus, height), ]; + if (!hasMembershipActiveDkgs(coreVersion)) { + blockers.push(...getStartingDkgBlockers(dkgStatus, height, knownNonMemberDkgs)); + } + // Core 24.1+ counts every session this node is a member of, or whose // membership is unknown, from its start block through its Commit phase. if (hasMembershipActiveDkgs(coreVersion) && dkgInfo.active_dkgs > 0) { diff --git a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js index 8f1c4c911e4..140183341cf 100644 --- a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js +++ b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js @@ -14,7 +14,8 @@ export const CHECK_INTERVAL_MS = 10000; * announced. Once a session starts Core no longer announces it, and Core * older than 24.1 reports it only after its DKG worker has computed the * session's members and recorded it, which can take a while at the start - * of a rotation cycle. + * of a rotation cycle. DKGs announced without this node are remembered too, + * so their start blocks need not wait for a record that never comes. * * @param {RpcClient} rpcClient * @return {Promise} @@ -23,11 +24,21 @@ export default async function waitForDKGWindowPass(rpcClient) { const coreVersion = await readCoreVersion(rpcClient); const announcedDkgs = new Map(); + let nonMemberDkgs = new Set(); for (;;) { - const check = await checkMasternodeSafeToStop(rpcClient, coreVersion); + const check = await checkMasternodeSafeToStop(rpcClient, coreVersion, { + knownNonMemberDkgs: nonMemberDkgs, + }); + + nonMemberDkgs = new Set([...nonMemberDkgs, ...check.nonMemberDkgs]); if (check.height !== null) { + // A key ends with the DKG's start block; one that has passed no longer + // matters. + nonMemberDkgs = new Set([...nonMemberDkgs] + .filter((key) => Number(key.split(':').at(-1)) >= check.height)); + check.blockers .filter(({ reason, until }) => reason === 'upcoming' && until !== null) .forEach((dkg) => { diff --git a/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js b/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js index 6d8817b5174..e55431996ab 100644 --- a/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js @@ -5,6 +5,7 @@ import checkMasternodeSafeToStop, { readCoreVersion, } from '../../core/quorum/checkMasternodeSafeToStop.js'; import { getBlockersAt } from '../../core/quorum/getDkgStopBlockers.js'; +import { NETWORK_LOCAL } from '../../constants.js'; // A check comes back unsettled while the chain moves too fast to read a // consistent DKG state, such as while Core catches up after a restart. @@ -52,7 +53,8 @@ export default function stopNodeTaskFactory( }, { title: 'Check node is participating in DKG', - enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && !ctx.isSafe && !ctx.platformOnly, + enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && !ctx.isSafe + && !ctx.platformOnly && !ctx.isLocalMinerStopped, task: async () => { const rpcClient = createRpcClient({ port: config.get('core.rpc.port'), @@ -96,7 +98,8 @@ export default function stopNodeTaskFactory( }, { title: 'Wait for DKG window to pass', - enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && ctx.isSafe && !ctx.platformOnly, + enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && ctx.isSafe + && !ctx.platformOnly && !ctx.isLocalMinerStopped, task: async () => waitForDKGWindowPass(createRpcClient({ port: config.get('core.rpc.port'), user: 'dashmate', @@ -112,6 +115,14 @@ export default function stopNodeTaskFactory( : []; await dockerCompose.stop(config, { profiles }); + + // Group commands stop a local network's miner before its masternodes. + // Without it no block comes, so a DKG wait could never end, and with + // every masternode of the network going down together their DKG + // produces no commitment to penalize them in. + if (config.get('network') === NETWORK_LOCAL && config.get('core.miner.enable')) { + ctx.isLocalMinerStopped = true; + } }, }, ]); diff --git a/packages/dashmate/src/test/mock/getCoreRpcMock.js b/packages/dashmate/src/test/mock/getCoreRpcMock.js index e0b1174e112..a0972ea053a 100644 --- a/packages/dashmate/src/test/mock/getCoreRpcMock.js +++ b/packages/dashmate/src/test/mock/getCoreRpcMock.js @@ -3,13 +3,16 @@ * mutates to move the chain or change DKG state between reads. * `core.beforeDkgInfo` and `core.afterDkgStatus`, when set, run right * before `quorum dkginfo` is answered and right after `quorum dkgstatus` is. - * The tip hash is derived from the height unless `core.hash` is set. + * Block hashes are derived from their height; `core.hash` overrides the tip's, + * and `core.hashAt(height)`, when set, any block's. `quorum dkgstatus` lists + * no quorum connections unless the test's `dkgStatus` does. * * @param {Sinon} sinon * @param {{ * version: number, * height: number, * hash?: string, + * hashAt?: Function, * dkgInfo: Object, * dkgStatus: Object, * isSyncing?: boolean, @@ -19,15 +22,24 @@ * }} core */ export default function getCoreRpcMock(sinon, core) { + const getHash = (height) => { + if (core.hashAt) { + return core.hashAt(height); + } + + return height === core.height && core.hash ? core.hash : `hash-${height}`; + }; + return { getNetworkInfo: sinon.stub().callsFake(async () => ({ result: { version: core.version } })), getBlockchainInfo: sinon.stub().callsFake(async () => ({ result: { blocks: core.height, - bestblockhash: core.hash || `hash-${core.height}`, + bestblockhash: getHash(core.height), initialblockdownload: core.isSyncing === true, }, })), + getBlockHash: sinon.stub().callsFake(async (height) => ({ result: getHash(height) })), quorum: sinon.stub().callsFake(async (command) => { if (command === 'dkginfo') { if (core.beforeDkgInfo) { @@ -42,7 +54,9 @@ export default function getCoreRpcMock(sinon, core) { await new Promise((resolve) => { setTimeout(resolve, core.dkgStatusDelayMs); }); } - const result = core.dkgStatus; + const result = core.dkgStatus && typeof core.dkgStatus === 'object' + ? { quorumConnections: [], ...core.dkgStatus } + : core.dkgStatus; if (core.afterDkgStatus) { core.afterDkgStatus(); diff --git a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js index 1ae4730fd8e..ff991c1d684 100644 --- a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js +++ b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js @@ -42,10 +42,12 @@ describe('checkMasternodeSafeToStop', () => { expect(await readCoreVersion(rpcClient)).to.equal(CORE_23); }); - it('should return null when Core reports no integer version', async () => { + it('should refuse to guess when Core reports no integer version', async () => { + // Which rule applies depends on the version; reading an older one for + // Core 24.1 would ignore its session count. core.version = '23.1.10'; - expect(await readCoreVersion(rpcClient)).to.be.null(); + await expect(readCoreVersion(rpcClient)).to.be.rejectedWith('Core did not report its version'); }); }); @@ -201,4 +203,46 @@ describe('checkMasternodeSafeToStop', () => { expect(result).to.include({ status: 'safe', isSyncing: true }); expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); }); + + it('should report unsettled when a reorg replaced the first check\'s tip and the chain grew', async () => { + // The replacing branch can put this node in a DKG starting on it that + // Core has not recorded yet. + const state = await startCheck(); + const firstTip = core.height; + core.height += 2; + core.hashAt = (height) => `other-branch-${height}`; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('unsettled'); + expect(rpcClient.getBlockHash).to.have.been.calledWith(firstTip); + }); + + it('should treat Core in initial block download as safe even while blocks keep connecting', async () => { + core.isSyncing = true; + core.beforeDkgInfo = () => { core.height += 1; }; + + const { result } = await startCheck(); + + expect(result).to.include({ status: 'safe', isSyncing: true }); + }); + + it('should pass on what it knows about DKGs this node is not part of', async () => { + // A DKG starting at the tip blocks unless announced without this node. + core.dkgInfo = { active_dkgs: 0, next_dkg: 24 }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: 1012 }], + }; + + const blocked = await startCheck(); + expect(blocked.result.status).to.equal('blocked'); + + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version, { + knownNonMemberDkgs: new Set(['llmq_100_67:0:1012']), + }).then((result) => { state.result = result; }); + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('safe'); + }); }); diff --git a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js index 9e5fbc2e310..6441ca819a5 100644 --- a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js +++ b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js @@ -1,4 +1,5 @@ import getDkgStopBlockers, { + getAnnouncedNonMemberDkgs, getBlockersAt, hasMembershipActiveDkgs, LLMQ_TYPES, @@ -16,6 +17,13 @@ const NEXT_DKG_NOT_IMMINENT = MIN_BLOCKS_BEFORE_DKG + 5; const PLATFORM_ACTIVE_BLOCKS = 10; // llmq_test_platform: 5 phases * 2 blocks const PLATFORM_INTERVAL = 24; +// Core lists the quorum connections in every `quorum dkgstatus` answer. +function withConnections(dkgStatus) { + return dkgStatus && typeof dkgStatus === 'object' + ? { quorumConnections: [], ...dkgStatus } + : dkgStatus; +} + function blockersAt({ dkgInfo, dkgStatus = { session: [] }, @@ -23,7 +31,7 @@ function blockersAt({ coreVersion = CORE_23, }) { return getBlockersAt(getDkgStopBlockers({ - dkgInfo, dkgStatus, height, coreVersion, + dkgInfo, dkgStatus: withConnections(dkgStatus), height, coreVersion, }), height); } @@ -73,7 +81,7 @@ describe('getDkgStopBlockers', () => { it('should report when the next DKG starts blocking', () => { const [blocker] = getDkgStopBlockers({ dkgInfo: { active_dkgs: 0, next_dkg: 20 }, - dkgStatus: { session: [] }, + dkgStatus: withConnections({ session: [] }), height: HEIGHT, coreVersion: CORE_23, }); @@ -136,7 +144,7 @@ describe('getDkgStopBlockers', () => { }; const blockers = getDkgStopBlockers({ - dkgInfo, dkgStatus: { session: [] }, height: HEIGHT, coreVersion: CORE_24_0, + dkgInfo, dkgStatus: withConnections({ session: [] }), height: HEIGHT, coreVersion: CORE_24_0, }); expect(getBlockersAt(blockers, HEIGHT)).to.be.empty(); @@ -435,11 +443,13 @@ describe('getDkgStopBlockers', () => { Object.entries(malformed).forEach(([description, dkgStatus]) => { it(`should block when ${description}`, () => { const blockers = getBlockersAt(getDkgStopBlockers({ - dkgInfo, dkgStatus, height: HEIGHT, coreVersion: CORE_23, + dkgInfo, dkgStatus: withConnections(dkgStatus), height: HEIGHT, coreVersion: CORE_23, }), HEIGHT); - expect(blockers).to.have.lengthOf(1); - expect(blockers[0]).to.include({ reason: 'unreadable', until: null }); + expect(blockers).to.not.be.empty(); + blockers.forEach((blocker) => { + expect(blocker).to.include({ reason: 'unreadable', until: null }); + }); }); }); }); @@ -493,4 +503,84 @@ describe('getDkgStopBlockers', () => { }); }); }); + + describe('a DKG that starts at the tip, before Core records it (Core older than 24.1)', () => { + // `quorum dkgstatus` lists every enabled quorum with the start of its + // current session. At a start block Core's DKG worker records the session + // only after it has computed the members, and nothing says whether this + // node is one of them until then. + const startsHere = { + llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: HEIGHT, + }; + const dkgInfo = { active_dkgs: 0, next_dkg: 24 }; + + function blockersWith(dkgStatus, options = {}) { + return getBlockersAt(getDkgStopBlockers({ + dkgInfo, dkgStatus, height: HEIGHT, coreVersion: CORE_23, ...options, + }), HEIGHT); + } + + it('should block the start block while Core has no record of the session', () => { + const blockers = blockersWith({ session: [], quorumConnections: [startsHere] }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'starting', llmqType: 'llmq_100_67', quorumIndex: 0, from: HEIGHT, until: HEIGHT + 1, + }); + }); + + it('should leave the session to its record once Core has one', () => { + const blockers = blockersWith({ + session: [record('llmq_100_67', HEIGHT)], + quorumConnections: [startsHere], + }); + + expect(blockers.map(({ reason }) => reason)).to.deep.equal(['session']); + }); + + it('should not block a session that started before the tip', () => { + expect(blockersWith({ + session: [], + quorumConnections: [{ ...startsHere, pQuorumBaseBlockIndex: HEIGHT - 1 }], + })).to.be.empty(); + }); + + it('should not block a DKG this node was announced to be no member of', () => { + expect(blockersWith( + { session: [], quorumConnections: [startsHere] }, + { knownNonMemberDkgs: new Set([`llmq_100_67:0:${HEIGHT}`]) }, + )).to.be.empty(); + }); + + it('should rely on Core\'s count on Core 24.1, which includes such a session', () => { + expect(blockersWith( + { session: [], quorumConnections: [startsHere] }, + { coreVersion: CORE_24_1 }, + )).to.be.empty(); + }); + + it('should block when quorum connections are missing', () => { + const blockers = blockersWith({ session: [] }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', source: 'quorum dkgstatus quorumConnections' }); + }); + }); + + describe('getAnnouncedNonMemberDkgs', () => { + it('should name the announced DKGs this node is known to be no member of', () => { + expect([...getAnnouncedNonMemberDkgs({ + upcoming_dkgs: [ + upcomingDkg(2, false, 4), + upcomingDkg(3, true, 4), + unknownUpcomingDkg(4), + upcomingDkg(5, false, 250), + ], + }, HEIGHT)]).to.deep.equal([`llmq_100_67:0:${HEIGHT + 2}`]); + }); + + it('should name none without membership data', () => { + expect([...getAnnouncedNonMemberDkgs({ active_dkgs: 0, next_dkg: 3 }, HEIGHT)]).to.be.empty(); + }); + }); }); diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index 23061eb8446..bece6b4497c 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -452,4 +452,69 @@ describe('waitForDKGWindowPass', () => { expect(waiting.resolved).to.be.true(); }); + + it('should not stop at a DKG start reached in Core\'s record gap, however long it lasts', async function it() { + // Waiting starts right at the start block: no earlier poll saw the DKG + // announced, and Core takes longer than the confirmation delay to record + // the session, or never does because this node is not a member. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start, + dkgInfo: { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }, + dkgStatus: { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + await expectStillPolling(waiting, rpcClient); + + // The next block arrives without a record: this node is not a member. + core.height = start + 1; + core.dkgInfo = { active_dkgs: 0, next_dkg: 23, upcoming_dkgs: [] }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should not hold a DKG start it saw announced without this node', async function it() { + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 1, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 1, known: true, isMember: false, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + // Interrupt the first confirmation, so the wait checks again at the start. + core.height = start; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }; + core.beforeDkgInfo = () => { + core.beforeDkgInfo = undefined; + core.height = start + 7; + }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + core.height = start; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); }); diff --git a/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js b/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js index f62ebc7f82f..321fbfb2090 100644 --- a/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js +++ b/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js @@ -130,4 +130,44 @@ describe('stopNodeTaskFactory', () => { expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledThrice(); expect(dockerCompose.stop).to.not.have.been.called(); }); + + it('should not wait for DKGs once the local network\'s miner has stopped', async function it() { + // Group commands stop a local network's miner first. Without it no block + // comes, so a DKG wait could never end, and with every masternode of the + // network going down together their DKG produces no commitment to + // penalize them in. + const minerConfig = getConfigMock(this.sinon); + minerConfig.get.withArgs('core.masternode.enable').returns(false); + minerConfig.get.withArgs('core.miner.enable').returns(true); + minerConfig.get.withArgs('network').returns('local'); + minerConfig.getName.returns('local_seed'); + config.get.withArgs('network').returns('local'); + core.dkgInfo = { active_dkgs: 1, next_dkg: 3, upcoming_dkgs: [] }; + + const state = {}; + new Listr([ + { task: () => stopNodeTask(minerConfig) }, + { task: () => stopNodeTask(config) }, + ], { renderer: 'silent' }) + .run({ isForce: false, isSafe: true, platformOnly: false }) + .then(() => { state.stopped = true; }, (e) => { state.error = e; }); + await clock.tickAsync(1); + + expect(state.stopped).to.be.true(); + expect(dockerCompose.stop).to.have.been.calledTwice(); + }); + + it('should refuse a plain stop at a DKG start that Core has not recorded yet', async () => { + core.version = CORE_24_0; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: 1012 }], + }; + + const { error } = await stop(); + + expect(error.message).to.include('may result in a PoSe ban'); + expect(dockerCompose.stop).to.not.have.been.called(); + }); }); From 9267e83e720f93dea6395552fde1b6edd2443b68 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 11 Nov 2026 00:07:20 +0700 Subject: [PATCH 03/11] fix(dashmate): block DKG sessions Core has not recorded for their whole window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Core older than 24.1, a session's start only blocked its start block. Core records a session after computing its members, which has no time bound, so a check at the next block without a record could not tell a non-member from a member Core had not recorded yet. A current session without a record, from `quorum dkgstatus` quorum connections, now blocks until its Commit phase ends, unless Core announced it without this node. A plain stop is refused for the first blocks of such a session; `--safe` does not wait for sessions it saw announced without this node. Remembered non-memberships held across a reorg. A replacing branch has another work block and may make this node a member, so each poll checks, inside its consistent snapshot, that the previous poll's tip is still on the chain, and a check that finds it replaced, or a confirmation that finds the first check's tip replaced, reports a reorg and drops them. The stopped-miner exemption applied to every config in the run. A group may hold configs of several networks, so it now covers only local-network configs in the stopped miner's group. Tests would have caught these: ✖ before the fix, ✔ after: - should keep blocking after the start block while Core still has no record - should not stop in a session Core has not recorded when waiting starts after its start block - should forget DKGs announced without this node when a reorg replaces their chain - should report a replaced chain when the previous check's tip is gone - should still check a masternode of another network after a local miner stopped Co-Authored-By: Claude Opus 5.5 --- .../core/quorum/checkMasternodeSafeToStop.js | 41 ++++++++--- .../src/core/quorum/getDkgStopBlockers.js | 67 ++++++++++------- .../src/core/quorum/waitForDKGWindowPass.js | 14 +++- .../src/listr/tasks/stopNodeTaskFactory.js | 30 ++++++-- .../quorum/checkMasternodeSafeToStop.spec.js | 25 +++++++ .../core/quorum/getDkgStopBlockers.spec.js | 71 ++++++++++++++----- .../core/quorum/waitForDKGWindowPass.spec.js | 60 +++++++++++++++- .../listr/tasks/stopNodeTaskFactory.spec.js | 27 +++++++ 8 files changed, 272 insertions(+), 63 deletions(-) diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index 5fe42aab6f8..0e7b2a49947 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -99,6 +99,11 @@ function haveSessionRecordsChanged({ dkgInfo, dkgStatus }, coreVersion) { * @property {Set} nonMemberDkgs DKGs Core announced without this * node, see {@link getAnnouncedNonMemberDkgs} * @property {boolean} [isSyncing] Core is in initial block download + * @property {boolean} [isReorg] the chain no longer has the tip of an earlier + * check; what was learned on that branch, such as `nonMemberDkgs`, no + * longer holds + * @property {{ height: number, hash: string }|null} tip chain tip + * `nonMemberDkgs` were read on, or `null` when no consistent read was made */ /** @@ -113,7 +118,7 @@ async function evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestorHeig if (!snapshot) { return { - status: 'unsettled', height: null, blockers: [], nonMemberDkgs: new Set(), + status: 'unsettled', height: null, blockers: [], nonMemberDkgs: new Set(), tip: null, }; } @@ -123,6 +128,7 @@ async function evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestorHeig hash, ancestorHash: snapshot.ancestorHash, nonMemberDkgs: getAnnouncedNonMemberDkgs(snapshot.dkgInfo, height), + tip: { height, hash }, }; // Core's DKG worker does nothing during initial block download, and Core @@ -160,19 +166,30 @@ async function evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestorHeig * many new blocks on top of the first check's tip; more than that, a lower * tip, or a chain without the first check's tip is `unsettled`. * + * `knownNonMemberDkgs` may only hold what earlier checks learned on the + * current chain: pass the tip of the latest of them as `ancestor`, and a + * check that finds it replaced reports `isReorg`. + * * @param {RpcClient} rpcClient * @param {number} coreVersion from {@link readCoreVersion} * @param {Object} [options] * @param {Set} [options.knownNonMemberDkgs] DKGs Core announced * without this node at earlier checks + * @param {{ height: number, hash: string }} [options.ancestor] tip of the + * latest earlier check * @return {Promise} */ export default async function checkMasternodeSafeToStop( rpcClient, coreVersion, - { knownNonMemberDkgs = new Set() } = {}, + { knownNonMemberDkgs = new Set(), ancestor } = {}, ) { - const first = await evaluate(rpcClient, coreVersion, knownNonMemberDkgs); + const first = await evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestor && ancestor.height); + + if (ancestor && first.height !== null + && (first.height < ancestor.height || first.ancestorHash !== ancestor.hash)) { + return { ...first, status: 'unsettled', isReorg: true }; + } if (first.status !== 'safe' || first.isSyncing || hasMembershipActiveDkgs(coreVersion)) { return first; @@ -182,13 +199,21 @@ export default async function checkMasternodeSafeToStop( const nonMemberDkgs = new Set([...knownNonMemberDkgs, ...first.nonMemberDkgs]); const second = await evaluate(rpcClient, coreVersion, nonMemberDkgs, first.height); - second.nonMemberDkgs = new Set([...nonMemberDkgs, ...second.nonMemberDkgs]); - const hasChainMoved = second.height < first.height - || second.ancestorHash !== first.hash - || second.height > first.height + MIN_BLOCKS_BEFORE_DKG; + // The confirming read was discarded: keep what the first check verified. + if (second.height === null) { + return { ...second, nonMemberDkgs, tip: first.tip }; + } + + // The first check's tip was replaced: none of it holds on this chain. + if (second.height < first.height || second.ancestorHash !== first.hash) { + return { ...second, status: 'unsettled', isReorg: true }; + } + + second.nonMemberDkgs = new Set([...nonMemberDkgs, ...second.nonMemberDkgs]); - if (second.status === 'safe' && !second.isSyncing && hasChainMoved) { + if (second.status === 'safe' && !second.isSyncing + && second.height > first.height + MIN_BLOCKS_BEFORE_DKG) { return { ...second, status: 'unsettled' }; } diff --git a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js index 3befec90586..a52009e69b3 100644 --- a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js +++ b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js @@ -86,7 +86,7 @@ export function hasMembershipActiveDkgs(coreVersion) { /** * @typedef {Object} DkgStopBlocker - * @property {string} reason `session`, `new-round`, `starting`, `upcoming`, + * @property {string} reason `session`, `new-round`, `unrecorded`, `upcoming`, * `imminent`, `active-count` or `unreadable` * @property {string|number|null} [llmqType] LLMQ name, or Core's numeric id * when dashmate does not know the type @@ -280,14 +280,15 @@ export function getAnnouncedNonMemberDkgs(dkgInfo, height) { } /** - * At its start block Core older than 24.1 erases the previous round's record, - * computes the new session's members and only then records the session if - * this node is one of them; nothing tells whether it is until then, however - * long that takes. `quorum dkgstatus` lists every enabled quorum with the - * start of its current session, so a session starting at the tip without a - * record blocks that block, unless Core announced it without this node. + * At a session's start block Core older than 24.1 erases the previous round's + * record, computes the new session's members and only then records the + * session if this node is one of them; until then nothing tells whether it + * is, however long that takes. `quorum dkgstatus` lists every enabled quorum + * with the start of its current session, so a current session without a + * record blocks until its Commit phase ends, unless Core announced it + * without this node. */ -function getStartingDkgBlockers(dkgStatus, height, knownNonMemberDkgs) { +function getUnrecordedSessionBlockers(dkgStatus, height, knownNonMemberDkgs) { const connections = dkgStatus && dkgStatus.quorumConnections; if (!Array.isArray(connections)) { @@ -295,23 +296,36 @@ function getStartingDkgBlockers(dkgStatus, height, knownNonMemberDkgs) { } const sessions = Array.isArray(dkgStatus.session) ? dkgStatus.session : []; - - return connections - .filter((quorum) => quorum && quorum.pQuorumBaseBlockIndex === height) - .filter(({ llmqType, quorumIndex }) => !knownNonMemberDkgs - .has(getDkgKey(llmqType, quorumIndex, height))) - .filter(({ llmqType, quorumIndex }) => !sessions.some((entry) => entry + const isRecorded = ({ llmqType, quorumIndex, pQuorumBaseBlockIndex: start }) => sessions + .some((entry) => entry && entry.llmqType === llmqType && entry.quorumIndex === quorumIndex && entry.status - && entry.status.quorumHeight === height)) - .map(({ llmqType, quorumIndex }) => ({ - reason: 'starting', - llmqType, - quorumIndex, - from: height, - until: height + 1, - })); + && entry.status.quorumHeight === start); + + return connections + // A quorum whose current session has not started yet has no start block. + .filter((quorum) => quorum && isCount(quorum.pQuorumBaseBlockIndex) + && quorum.pQuorumBaseBlockIndex <= height) + .flatMap((quorum) => { + const { llmqType, quorumIndex, pQuorumBaseBlockIndex: start } = quorum; + const llmq = LLMQ_TYPES.find(({ name }) => name === llmqType); + + if (!llmq) { + return [unreadable(`quorum dkgstatus quorumConnections ${llmqType}`)]; + } + + const until = start + getActiveDkgBlocks(llmq); + + if (height >= until || isRecorded(quorum) + || knownNonMemberDkgs.has(getDkgKey(llmqType, quorumIndex, start))) { + return []; + } + + return [{ + reason: 'unrecorded', llmqType, quorumIndex, from: start, until, + }]; + }); } /** @@ -322,9 +336,10 @@ function getStartingDkgBlockers(dkgStatus, height, knownNonMemberDkgs) { * * All inputs must come from the same chain tip, `height`. Core older than * {@link MEMBERSHIP_ACTIVE_DKGS_CORE_VERSION} reports a session only after - * its DKG worker has recorded it, after the start block connects. The start - * block blocks until then; callers must also confirm a safe result a while - * later, once Core has had time to replace records of previous rounds. + * its DKG worker has recorded it, after the start block connects; until then + * the session blocks unless Core announced it without this node. Callers + * must also confirm a safe result a while later, once Core has had time to + * replace records of previous rounds. * * Unreadable Core data blocks with no known end. * @@ -363,7 +378,7 @@ export default function getDkgStopBlockers({ ]; if (!hasMembershipActiveDkgs(coreVersion)) { - blockers.push(...getStartingDkgBlockers(dkgStatus, height, knownNonMemberDkgs)); + blockers.push(...getUnrecordedSessionBlockers(dkgStatus, height, knownNonMemberDkgs)); } // Core 24.1+ counts every session this node is a member of, or whose diff --git a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js index 140183341cf..cbe3276d688 100644 --- a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js +++ b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js @@ -15,7 +15,9 @@ export const CHECK_INTERVAL_MS = 10000; * older than 24.1 reports it only after its DKG worker has computed the * session's members and recorded it, which can take a while at the start * of a rotation cycle. DKGs announced without this node are remembered too, - * so their start blocks need not wait for a record that never comes. + * so their sessions need not wait for a record that never comes, as long as + * the chain they were announced on stays: a reorg replacing it may make this + * node a member after all. * * @param {RpcClient} rpcClient * @return {Promise} @@ -25,13 +27,21 @@ export default async function waitForDKGWindowPass(rpcClient) { const announcedDkgs = new Map(); let nonMemberDkgs = new Set(); + let lastTip; for (;;) { const check = await checkMasternodeSafeToStop(rpcClient, coreVersion, { knownNonMemberDkgs: nonMemberDkgs, + ancestor: lastTip, }); - nonMemberDkgs = new Set([...nonMemberDkgs, ...check.nonMemberDkgs]); + nonMemberDkgs = check.isReorg + ? check.nonMemberDkgs + : new Set([...nonMemberDkgs, ...check.nonMemberDkgs]); + + if (check.tip) { + lastTip = check.tip; + } if (check.height !== null) { // A key ends with the DKG's start block; one that has passed no longer diff --git a/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js b/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js index e55431996ab..686417d09db 100644 --- a/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js @@ -7,6 +7,23 @@ import checkMasternodeSafeToStop, { import { getBlockersAt } from '../../core/quorum/getDkgStopBlockers.js'; import { NETWORK_LOCAL } from '../../constants.js'; +/** + * Group commands stop a local network's miner before its masternodes. Without + * it no block comes, so a DKG wait could never end, and with every masternode + * of the network going down together their DKG produces no commitment to + * penalize them in. Only masternodes of that local network, the miner's group, + * are exempt: a group may hold configs of other networks. + * + * @param {Config} config + * @param {Object} ctx + * @return {boolean} + */ +function isLocalNetworkHalted(config, ctx) { + return config.get('network') === NETWORK_LOCAL + && !!ctx.stoppedLocalMinerGroups + && ctx.stoppedLocalMinerGroups.has(config.get('group')); +} + // A check comes back unsettled while the chain moves too fast to read a // consistent DKG state, such as while Core catches up after a restart. const SAFE_STOP_CHECK_ATTEMPTS = 3; @@ -54,7 +71,7 @@ export default function stopNodeTaskFactory( { title: 'Check node is participating in DKG', enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && !ctx.isSafe - && !ctx.platformOnly && !ctx.isLocalMinerStopped, + && !ctx.platformOnly && !isLocalNetworkHalted(config, ctx), task: async () => { const rpcClient = createRpcClient({ port: config.get('core.rpc.port'), @@ -99,7 +116,7 @@ export default function stopNodeTaskFactory( { title: 'Wait for DKG window to pass', enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && ctx.isSafe - && !ctx.platformOnly && !ctx.isLocalMinerStopped, + && !ctx.platformOnly && !isLocalNetworkHalted(config, ctx), task: async () => waitForDKGWindowPass(createRpcClient({ port: config.get('core.rpc.port'), user: 'dashmate', @@ -116,12 +133,11 @@ export default function stopNodeTaskFactory( await dockerCompose.stop(config, { profiles }); - // Group commands stop a local network's miner before its masternodes. - // Without it no block comes, so a DKG wait could never end, and with - // every masternode of the network going down together their DKG - // produces no commitment to penalize them in. if (config.get('network') === NETWORK_LOCAL && config.get('core.miner.enable')) { - ctx.isLocalMinerStopped = true; + ctx.stoppedLocalMinerGroups = new Set([ + ...(ctx.stoppedLocalMinerGroups || []), + config.get('group'), + ]); } }, }, diff --git a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js index ff991c1d684..674701c53b3 100644 --- a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js +++ b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js @@ -245,4 +245,29 @@ describe('checkMasternodeSafeToStop', () => { expect(state.result.status).to.equal('safe'); }); + + it('should report a replaced chain when the previous check\'s tip is gone', async () => { + // Anything learned on the replaced branch, such as which DKGs leave this + // node out, may not hold on the new one. + core.version = CORE_24_1; + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version, { + ancestor: { height: 1010, hash: 'replaced-1010' }, + }).then((result) => { state.result = result; }); + await clock.tickAsync(0); + + expect(state.result).to.include({ status: 'unsettled', isReorg: true }); + }); + + it('should not report a replaced chain when the previous check\'s tip is still there', async () => { + core.version = CORE_24_1; + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version, { + ancestor: { height: 1010, hash: 'hash-1010' }, + }).then((result) => { state.result = result; }); + await clock.tickAsync(0); + + expect(state.result.status).to.equal('safe'); + expect(state.result.isReorg).to.not.equal(true); + }); }); diff --git a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js index 6441ca819a5..2813111b875 100644 --- a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js +++ b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js @@ -504,57 +504,78 @@ describe('getDkgStopBlockers', () => { }); }); - describe('a DKG that starts at the tip, before Core records it (Core older than 24.1)', () => { + describe('a session Core has not recorded (Core older than 24.1)', () => { // `quorum dkgstatus` lists every enabled quorum with the start of its - // current session. At a start block Core's DKG worker records the session - // only after it has computed the members, and nothing says whether this - // node is one of them until then. - const startsHere = { - llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: HEIGHT, + // current session. Core records a session only once it has computed its + // members, which has no time bound, and until then nothing says whether + // this node is one of them. + const start = HEIGHT; + const current = { + llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start, }; const dkgInfo = { active_dkgs: 0, next_dkg: 24 }; function blockersWith(dkgStatus, options = {}) { + const height = options.height ?? start; + return getBlockersAt(getDkgStopBlockers({ - dkgInfo, dkgStatus, height: HEIGHT, coreVersion: CORE_23, ...options, - }), HEIGHT); + dkgInfo, dkgStatus, coreVersion: CORE_23, ...options, height, + }), height); } - it('should block the start block while Core has no record of the session', () => { - const blockers = blockersWith({ session: [], quorumConnections: [startsHere] }); + it('should block the session from its start until its Commit phase ends', () => { + const blockers = blockersWith({ session: [], quorumConnections: [current] }); expect(blockers).to.have.lengthOf(1); expect(blockers[0]).to.include({ - reason: 'starting', llmqType: 'llmq_100_67', quorumIndex: 0, from: HEIGHT, until: HEIGHT + 1, + reason: 'unrecorded', llmqType: 'llmq_100_67', quorumIndex: 0, from: start, until: start + 10, }); }); + it('should keep blocking after the start block while Core still has no record', () => { + expect(blockersWith({ session: [], quorumConnections: [current] }, { height: start + 1 })) + .to.have.lengthOf(1); + expect(blockersWith({ session: [], quorumConnections: [current] }, { height: start + 9 })) + .to.have.lengthOf(1); + expect(blockersWith({ session: [], quorumConnections: [current] }, { height: start + 10 })) + .to.be.empty(); + }); + + it('should block a 400-member session for its 20 blocks', () => { + const large = { llmqType: 'llmq_400_60', quorumIndex: 0, pQuorumBaseBlockIndex: start }; + + expect(blockersWith({ session: [], quorumConnections: [large] }, { height: start + 19 })) + .to.have.lengthOf(1); + expect(blockersWith({ session: [], quorumConnections: [large] }, { height: start + 20 })) + .to.be.empty(); + }); + it('should leave the session to its record once Core has one', () => { const blockers = blockersWith({ - session: [record('llmq_100_67', HEIGHT)], - quorumConnections: [startsHere], - }); + session: [record('llmq_100_67', start)], + quorumConnections: [current], + }, { height: start + 2 }); expect(blockers.map(({ reason }) => reason)).to.deep.equal(['session']); }); - it('should not block a session that started before the tip', () => { + it('should not block a session that has not started', () => { expect(blockersWith({ session: [], - quorumConnections: [{ ...startsHere, pQuorumBaseBlockIndex: HEIGHT - 1 }], + quorumConnections: [{ llmqType: 'llmq_60_75', quorumIndex: 5 }], })).to.be.empty(); }); it('should not block a DKG this node was announced to be no member of', () => { expect(blockersWith( - { session: [], quorumConnections: [startsHere] }, - { knownNonMemberDkgs: new Set([`llmq_100_67:0:${HEIGHT}`]) }, + { session: [], quorumConnections: [current] }, + { height: start + 3, knownNonMemberDkgs: new Set([`llmq_100_67:0:${start}`]) }, )).to.be.empty(); }); it('should rely on Core\'s count on Core 24.1, which includes such a session', () => { expect(blockersWith( - { session: [], quorumConnections: [startsHere] }, + { session: [], quorumConnections: [current] }, { coreVersion: CORE_24_1 }, )).to.be.empty(); }); @@ -565,6 +586,18 @@ describe('getDkgStopBlockers', () => { expect(blockers).to.have.lengthOf(1); expect(blockers[0]).to.include({ reason: 'unreadable', source: 'quorum dkgstatus quorumConnections' }); }); + + it('should block on a quorum type dashmate does not know', () => { + const blockers = blockersWith({ + session: [], + quorumConnections: [{ llmqType: 'llmq_future', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'unreadable', source: 'quorum dkgstatus quorumConnections llmq_future', + }); + }); }); describe('getAnnouncedNonMemberDkgs', () => { diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index bece6b4497c..86ec2e8fd6e 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -473,14 +473,37 @@ describe('waitForDKGWindowPass', () => { await clock.tickAsync(60000); await expectStillPolling(waiting, rpcClient); - // The next block arrives without a record: this node is not a member. + // Without a record, nothing shows this node is not a member, so the + // session blocks until its Commit phase ends. core.height = start + 1; core.dkgInfo = { active_dkgs: 0, next_dkg: 23, upcoming_dkgs: [] }; + await expectStillPolling(waiting, rpcClient); + + core.height = start + 10; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14, upcoming_dkgs: [] }; await clock.tickAsync(NEXT_POLL_MS); expect(waiting.resolved).to.be.true(); }); + it('should not stop in a session Core has not recorded when waiting starts after its start block', async function it() { + const start = 24 * 5000; + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_24_0, + height: start + 1, + dkgInfo: { active_dkgs: 0, next_dkg: 23, upcoming_dkgs: [] }, + dkgStatus: { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }, + }); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + + await expectStillPolling(waiting, rpcClient); + }); + it('should not hold a DKG start it saw announced without this node', async function it() { const start = 24 * 5000; const core = { @@ -517,4 +540,39 @@ describe('waitForDKGWindowPass', () => { expect(waiting.resolved).to.be.true(); }); + + it('should forget DKGs announced without this node when a reorg replaces their chain', async function it() { + // On the replacing branch the work block differs, so this node may be a + // member after all. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 1, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 1, known: true, isMember: false, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + // While the first result waits to be confirmed, a reorg replaces the tip + // and the replacing branch reaches the DKG's start. + core.hashAt = (height) => `other-branch-${height}`; + core.height = start; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }; + await clock.tickAsync(NEXT_POLL_MS * 2); + + await expectStillPolling(waiting, rpcClient); + }); }); diff --git a/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js b/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js index 321fbfb2090..a8c2fe02924 100644 --- a/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js +++ b/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js @@ -140,8 +140,10 @@ describe('stopNodeTaskFactory', () => { minerConfig.get.withArgs('core.masternode.enable').returns(false); minerConfig.get.withArgs('core.miner.enable').returns(true); minerConfig.get.withArgs('network').returns('local'); + minerConfig.get.withArgs('group').returns('local'); minerConfig.getName.returns('local_seed'); config.get.withArgs('network').returns('local'); + config.get.withArgs('group').returns('local'); core.dkgInfo = { active_dkgs: 1, next_dkg: 3, upcoming_dkgs: [] }; const state = {}; @@ -170,4 +172,29 @@ describe('stopNodeTaskFactory', () => { expect(error.message).to.include('may result in a PoSe ban'); expect(dockerCompose.stop).to.not.have.been.called(); }); + + it('should still check a masternode of another network after a local miner stopped', async function it() { + // A group may hold configs of several networks; stopping a local miner + // does not stop blocks on any other. + const minerConfig = getConfigMock(this.sinon); + minerConfig.get.withArgs('core.masternode.enable').returns(false); + minerConfig.get.withArgs('core.miner.enable').returns(true); + minerConfig.get.withArgs('network').returns('local'); + minerConfig.get.withArgs('group').returns('mixed'); + minerConfig.getName.returns('local_seed'); + config.get.withArgs('group').returns('mixed'); + core.dkgInfo = { active_dkgs: 1, next_dkg: 3, upcoming_dkgs: [] }; + + const state = {}; + new Listr([ + { task: () => stopNodeTask(minerConfig) }, + { task: () => stopNodeTask(config) }, + ], { renderer: 'silent' }) + .run({ isForce: false, isSafe: false, platformOnly: false }) + .then(() => { state.stopped = true; }, (e) => { state.error = e; }); + await clock.tickAsync(1); + + expect(state.error).to.be.an('error'); + expect(dockerCompose.stop).to.have.been.calledOnce(); + }); }); From af48c442d86eacb2360f16d00ac962d0ea8c17e8 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 11 Nov 2026 00:10:38 +0700 Subject: [PATCH 04/11] fix(dashmate): remember DKGs announced without this node until their session ends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now that a session Core has not recorded blocks for its whole window, the wait must keep a DKG it saw announced without this node until that window ends, not only until its start block, or a non-member waits the session out anyway. Test would have caught this: ✖ before, ✔ after ("should not hold a session it saw announced without this node after its start block"). Co-Authored-By: Claude Opus 5.5 --- .../src/core/quorum/getDkgStopBlockers.js | 16 +++++++++ .../src/core/quorum/waitForDKGWindowPass.js | 6 ++-- .../core/quorum/waitForDKGWindowPass.spec.js | 36 +++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js index a52009e69b3..4f0f8edc47a 100644 --- a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js +++ b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js @@ -255,6 +255,22 @@ function getDkgKey(llmqType, quorumIndex, start) { return `${llmqType}:${quorumIndex}:${start}`; } +/** + * Whether the DKG a key from {@link getAnnouncedNonMemberDkgs} names can + * still block at `height`: it has not started yet or has not finished its + * Commit phase. + * + * @param {string} key + * @param {number} height + * @return {boolean} + */ +export function isDkgKeyCurrent(key, height) { + const [llmqType, , start] = key.split(':'); + const llmq = LLMQ_TYPES.find(({ name }) => name === llmqType); + + return !llmq || Number(start) + getActiveDkgBlocks(llmq) > height; +} + /** * The DKGs Core announces this node is not a member of, as * `llmqType:quorumIndex:start` keys. Once such a DKG starts Core no longer diff --git a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js index cbe3276d688..e913109642b 100644 --- a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js +++ b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js @@ -1,6 +1,6 @@ import wait from '../../util/wait.js'; import checkMasternodeSafeToStop, { readCoreVersion } from './checkMasternodeSafeToStop.js'; -import { getBlockersAt } from './getDkgStopBlockers.js'; +import { getBlockersAt, isDkgKeyCurrent } from './getDkgStopBlockers.js'; export const CHECK_INTERVAL_MS = 10000; @@ -44,10 +44,8 @@ export default async function waitForDKGWindowPass(rpcClient) { } if (check.height !== null) { - // A key ends with the DKG's start block; one that has passed no longer - // matters. nonMemberDkgs = new Set([...nonMemberDkgs] - .filter((key) => Number(key.split(':').at(-1)) >= check.height)); + .filter((key) => isDkgKeyCurrent(key, check.height))); check.blockers .filter(({ reason, until }) => reason === 'upcoming' && until !== null) diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index 86ec2e8fd6e..f6c03ee3653 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -575,4 +575,40 @@ describe('waitForDKGWindowPass', () => { await expectStillPolling(waiting, rpcClient); }); + + it('should not hold a session it saw announced without this node after its start block', async function it() { + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 1, + dkgInfo: { + active_dkgs: 1, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 1, known: true, isMember: false, + }], + }, + dkgStatus: { + // Another session keeps the node busy until the announced one has started. + session: [{ llmqType: 'llmq_400_60', quorumIndex: 0, status: { quorumHeight: start - 12 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + const connections = [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }]; + core.height = start + 1; + core.dkgInfo = { active_dkgs: 1, next_dkg: 23, upcoming_dkgs: [] }; + core.dkgStatus = { ...core.dkgStatus, quorumConnections: connections }; + await clock.tickAsync(CHECK_INTERVAL_MS); + + // The other session has ended; the announced one is still running. + core.height = start + 8; + core.dkgInfo = { active_dkgs: 1, next_dkg: 16, upcoming_dkgs: [] }; + await clock.tickAsync(NEXT_POLL_MS * 2); + + expect(waiting.resolved).to.be.true(); + }); }); From 1e0ff52352ca8dbe06087939eb017e672fde075b Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 11 Nov 2026 01:39:58 +0700 Subject: [PATCH 05/11] fix(dashmate): keep remembered DKGs blocking while Core syncs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--safe` stopped a node as soon as Core reported initial block download, even when it had seen a DKG this node is a member of announced. A Core restarted into initial block download may finish syncing in time for that DKG, so a remembered DKG now blocks while Core syncs too. An announced DKG of a quorum type dashmate does not know, that may include this node, is now unreadable rather than a blocker with no end: without its Commit phase nothing says when it stops blocking, and a remembered one would hold a `--safe` wait forever. Unknown types in session records and quorum connections were already unreadable. Tests would have caught these: ✖ before, ✔ after: - should keep blocking a DKG it saw announced when Core restarts into initial block download - should not read a DKG of a type dashmate does not know that may include this node Co-Authored-By: Claude Opus 5.5 --- .../src/core/quorum/getDkgStopBlockers.js | 15 ++++++--- .../src/core/quorum/waitForDKGWindowPass.js | 6 ++-- .../core/quorum/getDkgStopBlockers.spec.js | 11 ++++--- .../core/quorum/waitForDKGWindowPass.spec.js | 32 +++++++++++++++++++ 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js index 4f0f8edc47a..47d1c9dd2f8 100644 --- a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js +++ b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js @@ -88,8 +88,7 @@ export function hasMembershipActiveDkgs(coreVersion) { * @typedef {Object} DkgStopBlocker * @property {string} reason `session`, `new-round`, `unrecorded`, `upcoming`, * `imminent`, `active-count` or `unreadable` - * @property {string|number|null} [llmqType] LLMQ name, or Core's numeric id - * when dashmate does not know the type + * @property {string} [llmqType] LLMQ name * @property {number} [quorumIndex] * @property {boolean} [membershipKnown] for `upcoming`: `false` when Core * could not determine whether this node is a member @@ -162,7 +161,7 @@ function getImminentDkgBlocker(dkgInfo, height) { * the cycle starts, so every DKG starting within `MIN_BLOCKS_BEFORE_DKG` * is listed. A DKG this node is a member of, or whose membership Core could * not determine, blocks from the restart margin before its start until its - * Commit phase ends. + * Commit phase ends; one of a type dashmate does not know is unreadable. */ function getUpcomingDkgBlockers(upcomingDkgs, height) { if (!Array.isArray(upcomingDkgs)) { @@ -179,15 +178,21 @@ function getUpcomingDkgBlockers(upcomingDkgs, height) { } const llmq = LLMQ_TYPES.find(({ id }) => id === dkg.llmqType); + + // Without its Commit phase nothing says when such a DKG stops blocking. + if (!llmq) { + return [unreadable(`quorum dkginfo upcoming_dkgs llmqType ${dkg.llmqType}`)]; + } + const start = height + dkg.blocksUntilStart; return [{ reason: 'upcoming', - llmqType: llmq ? llmq.name : dkg.llmqType, + llmqType: llmq.name, quorumIndex: dkg.quorumIndex, membershipKnown: dkg.known, from: start - MIN_BLOCKS_BEFORE_DKG, - until: llmq ? start + getActiveDkgBlocks(llmq) : null, + until: start + getActiveDkgBlocks(llmq), }]; }); } diff --git a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js index e913109642b..1db4757c19b 100644 --- a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js +++ b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js @@ -48,7 +48,7 @@ export default async function waitForDKGWindowPass(rpcClient) { .filter((key) => isDkgKeyCurrent(key, check.height))); check.blockers - .filter(({ reason, until }) => reason === 'upcoming' && until !== null) + .filter(({ reason }) => reason === 'upcoming') .forEach((dkg) => { announcedDkgs.set(`${dkg.llmqType}:${dkg.quorumIndex}:${dkg.from}`, dkg); }); @@ -60,8 +60,10 @@ export default async function waitForDKGWindowPass(rpcClient) { }); } + // A remembered DKG blocks even while Core syncs: Core restarted into initial + // block download may finish in time for it. if (check.status === 'safe' - && (check.isSyncing || getBlockersAt([...announcedDkgs.values()], check.height).length === 0)) { + && getBlockersAt([...announcedDkgs.values()], check.height).length === 0) { return; } diff --git a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js index 2813111b875..0cc46b05877 100644 --- a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js +++ b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js @@ -174,17 +174,20 @@ describe('getDkgStopBlockers', () => { expect(blockersAt({ dkgInfo, coreVersion: CORE_24_0 })).to.be.empty(); }); - it('should block a member DKG of a type dashmate does not know, with no known end', () => { + it('should not read a DKG of a type dashmate does not know that may include this node', () => { + // With no known Commit phase, nothing says when it stops blocking. const dkgInfo = { active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [upcomingDkg(1, true, 250)], + next_dkg: 20, + upcoming_dkgs: [upcomingDkg(20, true, 250), upcomingDkg(3, false, 251)], }; const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); expect(blockers).to.have.lengthOf(1); - expect(blockers[0]).to.include({ reason: 'upcoming', llmqType: 250, until: null }); + expect(blockers[0]).to.include({ + reason: 'unreadable', source: 'quorum dkginfo upcoming_dkgs llmqType 250', from: null, + }); }); describe('fail-safe on malformed upcoming_dkgs', () => { diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index f6c03ee3653..cfa62d295fd 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -611,4 +611,36 @@ describe('waitForDKGWindowPass', () => { expect(waiting.resolved).to.be.true(); }); + + it('should keep blocking a DKG it saw announced when Core restarts into initial block download', async function it() { + // A restarted Core reports initial block download, yet it may finish + // syncing in time for the DKG this node is a member of. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 2, + dkgInfo: { + active_dkgs: 0, + next_dkg: 2, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 2, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.isSyncing = true; + core.dkgInfo = { active_dkgs: 0, next_dkg: 2, upcoming_dkgs: [] }; + await expectStillPolling(waiting, rpcClient); + + // Once the DKG's Commit phase has ended, a syncing Core may stop. + core.height = start + 10; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); }); From 00a8bb726bed62eb5ca90065aa6ff93f9daa9a91 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 11 Nov 2026 03:44:22 +0700 Subject: [PATCH 06/11] fix(dashmate): hold a syncing Core to the DKGs it announces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Core in initial block download was safe to stop without looking at its DKG state. It runs no DKG session until synced, so session records do not apply, but a Core only a little behind can finish syncing in time for the DKGs it announces; announced, imminent and unreadable DKGs now still block while it syncs. When Core restarts into initial block download during a confirmation, the first check's DKGs are kept. The `--safe` wait read Core's version once. Core replaced between polls without a failed call kept the old version's rule, so a Core 24.0 that counts records was read as Core 24.1 that counts members. The version is now read on every poll. Tests would have caught these: ✖ before, ✔ after: - should still block a DKG this node is in while Core syncs - should keep the first check's DKGs when Core restarts into initial block download during confirmation - should judge each poll by the Core version answering it Co-Authored-By: Claude Opus 5.5 --- .../core/quorum/checkMasternodeSafeToStop.js | 23 ++++++++- .../src/core/quorum/waitForDKGWindowPass.js | 6 ++- .../quorum/checkMasternodeSafeToStop.spec.js | 51 +++++++++++++++++-- .../core/quorum/waitForDKGWindowPass.spec.js | 25 +++++++++ 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index 0e7b2a49947..9eb81c7973c 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -133,10 +133,17 @@ async function evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestorHeig // Core's DKG worker does nothing during initial block download, and Core // never re-enters it without a restart, so a node in it has had no DKG - // session since it started. + // session since it started: session records do not apply. DKGs it + // announces still do, as Core may finish syncing in time for them. if (isSyncing) { + const blockers = getDkgStopBlockers({ ...snapshot, coreVersion, knownNonMemberDkgs }) + .filter(({ reason }) => ['upcoming', 'imminent', 'unreadable'].includes(reason)); + return { - ...result, status: 'safe', blockers: [], isSyncing, + ...result, + status: getBlockersAt(blockers, height).length > 0 ? 'blocked' : 'safe', + blockers, + isSyncing, }; } @@ -212,6 +219,18 @@ export default async function checkMasternodeSafeToStop( second.nonMemberDkgs = new Set([...nonMemberDkgs, ...second.nonMemberDkgs]); + // A Core restarted into initial block download may no longer announce the + // DKGs the first check saw. + if (second.isSyncing) { + const blockers = [...second.blockers, ...first.blockers]; + + return { + ...second, + status: getBlockersAt(blockers, second.height).length > 0 ? 'blocked' : 'safe', + blockers, + }; + } + if (second.status === 'safe' && !second.isSyncing && second.height > first.height + MIN_BLOCKS_BEFORE_DKG) { return { ...second, status: 'unsettled' }; diff --git a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js index 1db4757c19b..3c83af538f1 100644 --- a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js +++ b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js @@ -23,13 +23,15 @@ export const CHECK_INTERVAL_MS = 10000; * @return {Promise} */ export default async function waitForDKGWindowPass(rpcClient) { - const coreVersion = await readCoreVersion(rpcClient); - const announcedDkgs = new Map(); let nonMemberDkgs = new Set(); let lastTip; for (;;) { + // Which rule applies depends on the version, and Core may have been + // replaced since the last poll. + const coreVersion = await readCoreVersion(rpcClient); + const check = await checkMasternodeSafeToStop(rpcClient, coreVersion, { knownNonMemberDkgs: nonMemberDkgs, ancestor: lastTip, diff --git a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js index 674701c53b3..5209eb2b6e3 100644 --- a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js +++ b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js @@ -193,10 +193,15 @@ describe('checkMasternodeSafeToStop', () => { expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); }); - it('should treat Core in initial block download as safe without confirmation', async () => { - // Core takes no part in DKGs until it has synced. + it('should not hold Core in initial block download to sessions, without confirmation', async () => { + // Core runs no DKG session until it has synced, so session records and + // unrecorded sessions do not apply; announced DKGs still do. core.isSyncing = true; - core.dkgInfo = { active_dkgs: 0, next_dkg: 1 }; + core.dkgInfo = { active_dkgs: 1, next_dkg: 12 }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, status: { quorumHeight: 1010 } }], + quorumConnections: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, pQuorumBaseBlockIndex: 1010 }], + }; const { result } = await startCheck(); @@ -270,4 +275,44 @@ describe('checkMasternodeSafeToStop', () => { expect(state.result.status).to.equal('safe'); expect(state.result.isReorg).to.not.equal(true); }); + + it('should still block a DKG this node is in while Core syncs', async () => { + // A Core only a little behind leaves initial block download in time for + // the DKGs it announces. + core.isSyncing = true; + core.version = CORE_24_0_PATCH; + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 2, + upcoming_dkgs: [{ + llmqType: 106, quorumIndex: 0, quorumHeight: 1014, blocksUntilStart: 2, known: true, isMember: true, + }], + }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('blocked'); + expect(result.blockers.map(({ reason }) => reason)).to.deep.equal(['upcoming']); + }); + + it('should keep the first check\'s DKGs when Core restarts into initial block download during confirmation', async () => { + // The first check sees a DKG of this node 8 blocks away; the restarted + // Core no longer reports it. + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 8, + upcoming_dkgs: [{ + llmqType: 106, quorumIndex: 0, quorumHeight: 1020, blocksUntilStart: 8, known: true, isMember: true, + }], + }; + core.version = CORE_24_0_PATCH; + + const state = await startCheck(); + core.isSyncing = true; + core.height = 1015; + core.dkgInfo = { active_dkgs: 0, next_dkg: 5 }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('blocked'); + }); }); diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index cfa62d295fd..bd4df363e69 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -643,4 +643,29 @@ describe('waitForDKGWindowPass', () => { expect(waiting.resolved).to.be.true(); }); + + it('should judge each poll by the Core version answering it', async function it() { + // Core is replaced between polls, with no failed call to tell. + const start = 24 * 5000; + const core = { + version: CORE_24_1, + height: start + 1, + dkgInfo: { active_dkgs: 1, next_dkg: 23, upcoming_dkgs: [] }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + // Core 24.0 counts records, and has none yet for the running session. + core.version = CORE_24_0; + core.dkgInfo = { active_dkgs: 0, next_dkg: 23, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }; + + await expectStillPolling(waiting, rpcClient); + }); }); From d80a3ac86104cf2fd9a3c03f059abcc087406092 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 11 Nov 2026 17:48:03 +0700 Subject: [PATCH 07/11] fix(dashmate): anchor DKG announcements and remember running sessions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An announced DKG's start was rebuilt as the tip plus Core's `blocksUntilStart`, and `next_dkg` was counted from the tip read after `dkginfo`. While Core syncs, a snapshot is kept although the tip moves, so a block connecting after `dkginfo` put the start one block late and could let a stop through right at the 6-block margin. Starts now come from Core's absolute `quorumHeight`, as do the keys of DKGs announced without this node, and `next_dkg` counts from the lowest tip `dkginfo` may have seen. `--safe` remembered only announced DKGs. A Core restarted between polls keeps no record of a running session, and Core 24.1 without its proTxHash yet reports no count for it either, so a session the wait had already seen could be forgotten mid-way. Every blocker with a known range is now remembered until it ends. Tests would have caught these: ✖ before, ✔ after: - should place an announced DKG at the start Core gives - should count next_dkg from the earliest tip dkginfo may have seen - should keep blocking a running session after Core restarts and forgets it - should block when an entry is missing quorumHeight Co-Authored-By: Claude Opus 5.5 --- .../core/quorum/checkMasternodeSafeToStop.js | 5 ++- .../src/core/quorum/getDkgStopBlockers.js | 24 ++++++----- .../src/core/quorum/waitForDKGWindowPass.js | 23 ++++++----- .../quorum/checkMasternodeSafeToStop.spec.js | 40 +++++++++++++++++++ .../core/quorum/getDkgStopBlockers.spec.js | 8 ++-- .../core/quorum/waitForDKGWindowPass.spec.js | 35 ++++++++++++++-- 6 files changed, 106 insertions(+), 29 deletions(-) diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index 9eb81c7973c..aa11291e8ec 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -64,6 +64,9 @@ async function readDkgSnapshot(rpcClient, ancestorHeight) { return { height: after.blocks, + // Core counts next_dkg from the tip dkginfo saw, which is no lower than + // this; it is below `height` only while syncing lets the tip move. + dkgInfoHeight: before.blocks, hash: after.bestblockhash, ancestorHash, isSyncing, @@ -127,7 +130,7 @@ async function evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestorHeig height, hash, ancestorHash: snapshot.ancestorHash, - nonMemberDkgs: getAnnouncedNonMemberDkgs(snapshot.dkgInfo, height), + nonMemberDkgs: getAnnouncedNonMemberDkgs(snapshot.dkgInfo), tip: { height, hash }, }; diff --git a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js index 47d1c9dd2f8..140942265e0 100644 --- a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js +++ b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js @@ -132,7 +132,7 @@ function hasValidDkgInfoShape(dkgInfo) { function isValidUpcomingDkg(dkg) { return !!dkg && typeof dkg === 'object' - && isCount(dkg.blocksUntilStart) + && isCount(dkg.quorumHeight) && typeof dkg.known === 'boolean' && (!dkg.known || typeof dkg.isMember === 'boolean'); } @@ -148,10 +148,10 @@ function unreadable(source) { * proTxHash), any DKG of any type may include this node, so the restart * margin before the next one blocks with no known end. */ -function getImminentDkgBlocker(dkgInfo, height) { +function getImminentDkgBlocker(dkgInfo, dkgInfoHeight) { return { reason: 'imminent', - from: height + dkgInfo.next_dkg - MIN_BLOCKS_BEFORE_DKG, + from: dkgInfoHeight + dkgInfo.next_dkg - MIN_BLOCKS_BEFORE_DKG, until: null, }; } @@ -163,7 +163,7 @@ function getImminentDkgBlocker(dkgInfo, height) { * not determine, blocks from the restart margin before its start until its * Commit phase ends; one of a type dashmate does not know is unreadable. */ -function getUpcomingDkgBlockers(upcomingDkgs, height) { +function getUpcomingDkgBlockers(upcomingDkgs) { if (!Array.isArray(upcomingDkgs)) { return [unreadable('quorum dkginfo upcoming_dkgs')]; } @@ -184,7 +184,7 @@ function getUpcomingDkgBlockers(upcomingDkgs, height) { return [unreadable(`quorum dkginfo upcoming_dkgs llmqType ${dkg.llmqType}`)]; } - const start = height + dkg.blocksUntilStart; + const start = dkg.quorumHeight; return [{ reason: 'upcoming', @@ -283,13 +283,12 @@ export function isDkgKeyCurrent(key, height) { * session it is not part of from one Core has not recorded yet. * * @param {Object} dkgInfo Result of `quorum dkginfo`. - * @param {number} height Chain tip `dkgInfo` describes. * @return {Set} */ -export function getAnnouncedNonMemberDkgs(dkgInfo, height) { +export function getAnnouncedNonMemberDkgs(dkgInfo) { const upcomingDkgs = dkgInfo && dkgInfo.upcoming_dkgs; - if (!Array.isArray(upcomingDkgs) || !isCount(height)) { + if (!Array.isArray(upcomingDkgs)) { return new Set(); } @@ -297,7 +296,7 @@ export function getAnnouncedNonMemberDkgs(dkgInfo, height) { .filter((dkg) => isValidUpcomingDkg(dkg) && dkg.known && !dkg.isMember) .map((dkg) => [LLMQ_TYPES.find(({ id }) => id === dkg.llmqType), dkg]) .filter(([llmq]) => llmq) - .map(([llmq, dkg]) => getDkgKey(llmq.name, dkg.quorumIndex, height + dkg.blocksUntilStart))); + .map(([llmq, dkg]) => getDkgKey(llmq.name, dkg.quorumIndex, dkg.quorumHeight))); } /** @@ -368,6 +367,8 @@ function getUnrecordedSessionBlockers(dkgStatus, height, knownNonMemberDkgs) { * @param {Object} snapshot.dkgInfo Result of `quorum dkginfo`. * @param {Object} snapshot.dkgStatus Result of `quorum dkgstatus`. * @param {number} snapshot.height Chain tip both results describe. + * @param {number} [snapshot.dkgInfoHeight] The lowest tip `dkgInfo` may + * describe, when it may be below `height`: Core counts `next_dkg` from it. * @param {number} snapshot.coreVersion Core's numeric `version`. * @param {Set} [snapshot.knownNonMemberDkgs] DKGs Core announced * without this node, from {@link getAnnouncedNonMemberDkgs} @@ -377,6 +378,7 @@ export default function getDkgStopBlockers({ dkgInfo, dkgStatus, height, + dkgInfoHeight = height, coreVersion, knownNonMemberDkgs = new Set(), }) { @@ -390,8 +392,8 @@ export default function getDkgStopBlockers({ const blockers = [ ...(dkgInfo.upcoming_dkgs === undefined - ? [getImminentDkgBlocker(dkgInfo, height)] - : getUpcomingDkgBlockers(dkgInfo.upcoming_dkgs, height)), + ? [getImminentDkgBlocker(dkgInfo, dkgInfoHeight)] + : getUpcomingDkgBlockers(dkgInfo.upcoming_dkgs)), // Read on every Core version: Core 24.1 stops counting a running session // in active_dkgs when a mid-session ProUpServTx clears its proTxHash, but // the session keeps going and its record stays. diff --git a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js index 3c83af538f1..5e006f8f4aa 100644 --- a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js +++ b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js @@ -10,11 +10,12 @@ export const CHECK_INTERVAL_MS = 10000; * safety rule. The only acceptable exit is reaching a safe state, so * that `--safe` cannot silently fall back to an unsafe restart. * - * DKGs this node takes part in are remembered from the polls that saw them - * announced. Once a session starts Core no longer announces it, and Core - * older than 24.1 reports it only after its DKG worker has computed the - * session's members and recorded it, which can take a while at the start - * of a rotation cycle. DKGs announced without this node are remembered too, + * DKGs this node takes part in, announced or running, are remembered until + * their Commit phase ends. Once a session starts Core no longer announces + * it, Core older than 24.1 reports it only after its DKG worker has computed + * the session's members and recorded it, which can take a while at the + * start of a rotation cycle, and a restarted Core keeps no record of it at + * all. DKGs announced without this node are remembered too, * so their sessions need not wait for a record that never comes, as long as * the chain they were announced on stays: a reorg replacing it may make this * node a member after all. @@ -23,7 +24,7 @@ export const CHECK_INTERVAL_MS = 10000; * @return {Promise} */ export default async function waitForDKGWindowPass(rpcClient) { - const announcedDkgs = new Map(); + const knownDkgs = new Map(); let nonMemberDkgs = new Set(); let lastTip; @@ -50,14 +51,14 @@ export default async function waitForDKGWindowPass(rpcClient) { .filter((key) => isDkgKeyCurrent(key, check.height))); check.blockers - .filter(({ reason }) => reason === 'upcoming') + .filter(({ from, until }) => from !== null && until !== null) .forEach((dkg) => { - announcedDkgs.set(`${dkg.llmqType}:${dkg.quorumIndex}:${dkg.from}`, dkg); + knownDkgs.set(`${dkg.reason}:${dkg.llmqType}:${dkg.quorumIndex}:${dkg.from}`, dkg); }); - announcedDkgs.forEach((dkg, key) => { + knownDkgs.forEach((dkg, key) => { if (dkg.until <= check.height) { - announcedDkgs.delete(key); + knownDkgs.delete(key); } }); } @@ -65,7 +66,7 @@ export default async function waitForDKGWindowPass(rpcClient) { // A remembered DKG blocks even while Core syncs: Core restarted into initial // block download may finish in time for it. if (check.status === 'safe' - && getBlockersAt([...announcedDkgs.values()], check.height).length === 0) { + && getBlockersAt([...knownDkgs.values()], check.height).length === 0) { return; } diff --git a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js index 5209eb2b6e3..299c5570860 100644 --- a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js +++ b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js @@ -315,4 +315,44 @@ describe('checkMasternodeSafeToStop', () => { expect(state.result.status).to.equal('blocked'); }); + + describe('Core syncing while a block connects during the reads', () => { + // Core counts next_dkg and blocksUntilStart from the tip dkginfo saw, + // which can be lower than the tip read after it. + const start = 1020; + + beforeEach(() => { + core.isSyncing = true; + core.height = start - 7; + core.afterDkgStatus = () => { + core.height += 1; + core.afterDkgStatus = undefined; + }; + }); + + it('should place an announced DKG at the start Core gives', async () => { + core.version = CORE_24_0_PATCH; + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 7, + upcoming_dkgs: [{ + llmqType: 106, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 7, known: true, isMember: true, + }], + }; + + const { result } = await startCheck(); + + expect(result.height).to.equal(start - 6); + expect(result.status).to.equal('blocked'); + }); + + it('should count next_dkg from the earliest tip dkginfo may have seen', async () => { + core.dkgInfo = { active_dkgs: 0, next_dkg: 7 }; + + const { result } = await startCheck(); + + expect(result.height).to.equal(start - 6); + expect(result.status).to.equal('blocked'); + }); + }); }); diff --git a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js index 0cc46b05877..1886d11713a 100644 --- a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js +++ b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js @@ -197,9 +197,11 @@ describe('getDkgStopBlockers', () => { 'an entry is null': [null], 'an entry is missing known': [{ blocksUntilStart: 1, isMember: false }], 'a known entry is missing isMember': [{ blocksUntilStart: 1, known: true }], - 'an entry is missing blocksUntilStart': [{ known: true, isMember: false }], - 'an entry has a negative blocksUntilStart': [ - { blocksUntilStart: -1, known: true, isMember: false }, + 'an entry is missing quorumHeight': [{ blocksUntilStart: 1, known: true, isMember: false }], + 'an entry has a negative quorumHeight': [ + { + quorumHeight: -1, blocksUntilStart: 1, known: true, isMember: false, + }, ], 'an entry outside the margin is malformed': [ { blocksUntilStart: MIN_BLOCKS_BEFORE_DKG + 1, known: true }, diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index bd4df363e69..9ec9cd6147f 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -44,7 +44,7 @@ describe('waitForDKGWindowPass', () => { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [{ - llmqType: 106, blocksUntilStart: 1, known: true, isMember: false, + llmqType: 106, quorumHeight: 1001, blocksUntilStart: 1, known: true, isMember: false, }], }, dkgStatus: { session: [] }, @@ -65,7 +65,7 @@ describe('waitForDKGWindowPass', () => { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [{ - llmqType: 106, blocksUntilStart: 1, known: true, isMember: false, + llmqType: 106, quorumHeight: 1001, blocksUntilStart: 1, known: true, isMember: false, }], }, dkgStatus: { session: [] }, @@ -90,7 +90,7 @@ describe('waitForDKGWindowPass', () => { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [{ - llmqType: 106, blocksUntilStart: 1, known: true, isMember: true, + llmqType: 106, quorumHeight: 1001, blocksUntilStart: 1, known: true, isMember: true, }], }, dkgStatus: { session: [] }, @@ -668,4 +668,33 @@ describe('waitForDKGWindowPass', () => { await expectStillPolling(waiting, rpcClient); }); + + it('should keep blocking a running session after Core restarts and forgets it', async function it() { + // Restarted Core keeps no session record, and Core 24.1 without its + // proTxHash yet reports no count or announcement for the session. + const start = 24 * 5000; + const core = { + version: CORE_24_1, + height: start + 3, + dkgInfo: { active_dkgs: 1, next_dkg: 21 }, + dkgStatus: { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: start } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.height = start + 4; + core.dkgInfo = { active_dkgs: 0, next_dkg: 20 }; + core.dkgStatus = { session: [] }; + await expectStillPolling(waiting, rpcClient); + + core.height = start + 10; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14 }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); }); From d1a2eb5e433176dc6894418de9532fdaba1cdad6 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Wed, 11 Nov 2026 22:41:53 +0700 Subject: [PATCH 08/11] fix(dashmate): do not stretch an older DKG announcement list over a newer tip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While Core syncs, a snapshot is kept although the tip moves during the reads. Core lists a DKG in `quorum dkginfo` once its work block, 8 blocks before its start, is mined, so a list read at a lower tip lacks DKGs that the 6-block restart margin of the later tip may need. A snapshot whose tip moved by more than the 2 blocks between the two is now unsettled. Test would have caught this: ✖ before, ✔ after ("should not read Core's announcements as covering a tip that moved too far past them"). Co-Authored-By: Claude Opus 5.5 --- .../core/quorum/checkMasternodeSafeToStop.js | 10 ++++++++++ .../quorum/checkMasternodeSafeToStop.spec.js | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index aa11291e8ec..14ba64db67b 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -13,6 +13,10 @@ import getDkgStopBlockers, { */ export const SAFE_STOP_CONFIRMATION_DELAY_MS = 15000; +// Core lists a DKG in `quorum dkginfo` once its work block, this many blocks +// before the DKG starts, is mined (`llmq::WORK_DIFF_DEPTH`). +const DKG_ANNOUNCEMENT_BLOCKS = 8; + /** * Which stop rule applies depends on Core's version, so a version Core does * not report is an error rather than a guess. @@ -62,6 +66,12 @@ async function readDkgSnapshot(rpcClient, ancestorHeight) { return null; } + // A list read at a lower tip lacks the DKGs starting past that tip's + // announcement horizon, which the restart margin of the later tip may need. + if (after.blocks - before.blocks > DKG_ANNOUNCEMENT_BLOCKS - MIN_BLOCKS_BEFORE_DKG) { + return null; + } + return { height: after.blocks, // Core counts next_dkg from the tip dkginfo saw, which is no lower than diff --git a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js index 299c5570860..d23ab9b2ea1 100644 --- a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js +++ b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js @@ -346,6 +346,23 @@ describe('checkMasternodeSafeToStop', () => { expect(result.status).to.equal('blocked'); }); + it('should not read Core\'s announcements as covering a tip that moved too far past them', async () => { + // Core lists a DKG once its work block, 8 blocks before its start, is + // mined. Read at start - 9, the list cannot hold a DKG starting at + // `start`, which the restart margin of start - 6 has to see. + core.version = CORE_24_0_PATCH; + core.height = start - 9; + core.dkgInfo = { active_dkgs: 0, next_dkg: 15, upcoming_dkgs: [] }; + core.afterDkgStatus = () => { + core.height = start - 6; + core.afterDkgStatus = undefined; + }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('unsettled'); + }); + it('should count next_dkg from the earliest tip dkginfo may have seen', async () => { core.dkgInfo = { active_dkgs: 0, next_dkg: 7 }; From ff137eb228c899383f0519b22e93043d2af9b3c3 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 12 Nov 2026 01:02:47 +0700 Subject: [PATCH 09/11] fix(dashmate): keep a syncing Core's moving reads, checked against its chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While Core syncs, its tip can move several blocks during one read, which made every plain-stop attempt unsettled, so a syncing masternode could not be stopped without --force. Core in initial block download takes no part in DKGs and stays in it only while connecting blocks over 6 hours old, so how far its tip moved during the reads no longer discards them. What does matter is whether the chain read last still has the work blocks Core worked out membership from: a branch switched during the reads may put this node in a DKG Core said it was not in. A read whose tip changed is now unsettled unless every work block `quorum dkginfo` names is on that chain. The previous check's tip is also looked up after the DKG reads, on the chain the check reports. Test would have caught this: ✖ before, ✔ after ("should not refuse a Core that connects several blocks during the reads", "should report unsettled when the block Core worked out membership from is replaced", "should report a replaced chain when the previous check's tip is replaced during the reads"). Co-Authored-By: Claude Opus 5.5 --- .../core/quorum/checkMasternodeSafeToStop.js | 66 ++++++++++---- .../quorum/checkMasternodeSafeToStop.spec.js | 89 +++++++++++++++++-- 2 files changed, 131 insertions(+), 24 deletions(-) diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index 14ba64db67b..754227ac054 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -13,10 +13,6 @@ import getDkgStopBlockers, { */ export const SAFE_STOP_CONFIRMATION_DELAY_MS = 15000; -// Core lists a DKG in `quorum dkginfo` once its work block, this many blocks -// before the DKG starts, is mined (`llmq::WORK_DIFF_DEPTH`). -const DKG_ANNOUNCEMENT_BLOCKS = 8; - /** * Which stop rule applies depends on Core's version, so a version Core does * not report is an error rather than a guess. @@ -34,15 +30,49 @@ export async function readCoreVersion(rpcClient) { return result.version; } +/** + * Core works out whether this node is in an upcoming DKG from the DKG's work + * block, so what `quorum dkginfo` says about membership holds on a chain only + * if that chain has the work blocks it names. + * + * @param {RpcClient} rpcClient + * @param {Object} dkgInfo + * @param {number} height chain tip + * @return {Promise} + */ +async function hasWorkBlocksOf(rpcClient, dkgInfo, height) { + const upcomingDkgs = dkgInfo && Array.isArray(dkgInfo.upcoming_dkgs) ? dkgInfo.upcoming_dkgs : []; + const knownDkgs = upcomingDkgs.filter((dkg) => dkg && dkg.known === true); + const workHeights = [...new Set(knownDkgs.map(({ workBlockHeight }) => workBlockHeight))]; + + if (!workHeights.every((workHeight) => Number.isInteger(workHeight) && workHeight <= height)) { + return false; + } + + const workHashes = new Map(await Promise.all(workHeights.map(async (workHeight) => { + const { result } = await rpcClient.getBlockHash(workHeight); + + return [workHeight, result]; + }))); + + return knownDkgs.every(({ workBlockHeight, workBlockHash }) => ( + workHashes.get(workBlockHeight) === workBlockHash + )); +} + /** * Read Core's DKG state at one chain tip. Each RPC reads the tip on its own, * so the tip is read before and after the DKG calls; if a block connected in - * between, they may describe different tips and are discarded. Core in - * initial block download takes no part in DKGs, so that state holds however - * the tip moves. + * between, they may describe different tips and are discarded. + * + * Core in initial block download takes no part in DKGs, and stays in it only + * while it connects blocks older than its maximum tip age, 6 hours: catching + * up on DKGs the network ran hours ago. Its state is then kept however far the + * tip moved, as long as the chain read last still has the work blocks Core's + * answers came from. * - * With `ancestorHeight`, also reads the hash of the active chain's block at - * that height, at the same tip. + * With `ancestorHeight`, also reads the hash of the chain's block at that + * height, after the tip, so that it describes the chain the snapshot reports. * * @param {RpcClient} rpcClient * @param {number} [ancestorHeight] @@ -50,28 +80,26 @@ export async function readCoreVersion(rpcClient) { */ async function readDkgSnapshot(rpcClient, ancestorHeight) { const { result: before } = await rpcClient.getBlockchainInfo(); - - let ancestorHash; - if (ancestorHeight !== undefined && before.blocks >= ancestorHeight) { - ({ result: ancestorHash } = await rpcClient.getBlockHash(ancestorHeight)); - } - const { result: dkgInfo } = await rpcClient.quorum('dkginfo'); const { result: dkgStatus } = await rpcClient.quorum('dkgstatus'); const { result: after } = await rpcClient.getBlockchainInfo(); const isSyncing = before.initialblockdownload === true && after.initialblockdownload === true; + const hasTipChanged = before.bestblockhash !== after.bestblockhash; - if (!isSyncing && before.bestblockhash !== after.bestblockhash) { + if (!isSyncing && hasTipChanged) { return null; } - // A list read at a lower tip lacks the DKGs starting past that tip's - // announcement horizon, which the restart margin of the later tip may need. - if (after.blocks - before.blocks > DKG_ANNOUNCEMENT_BLOCKS - MIN_BLOCKS_BEFORE_DKG) { + if (hasTipChanged && !await hasWorkBlocksOf(rpcClient, dkgInfo, after.blocks)) { return null; } + let ancestorHash; + if (ancestorHeight !== undefined && after.blocks >= ancestorHeight) { + ({ result: ancestorHash } = await rpcClient.getBlockHash(ancestorHeight)); + } + return { height: after.blocks, // Core counts next_dkg from the tip dkginfo saw, which is no lower than diff --git a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js index d23ab9b2ea1..8376867488e 100644 --- a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js +++ b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js @@ -276,6 +276,24 @@ describe('checkMasternodeSafeToStop', () => { expect(state.result.isReorg).to.not.equal(true); }); + it('should report a replaced chain when the previous check\'s tip is replaced during the reads', async () => { + // A syncing Core's tip may move during the reads, so the previous tip is + // looked up on the chain the check reports. + core.isSyncing = true; + core.beforeDkgInfo = () => { + core.height += 1; + core.hashAt = (height) => `other-branch-${height}`; + core.beforeDkgInfo = undefined; + }; + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version, { + ancestor: { height: 1010, hash: 'hash-1010' }, + }).then((result) => { state.result = result; }); + await clock.tickAsync(0); + + expect(state.result).to.include({ status: 'unsettled', isReorg: true }); + }); + it('should still block a DKG this node is in while Core syncs', async () => { // A Core only a little behind leaves initial block download in time for // the DKGs it announces. @@ -336,7 +354,14 @@ describe('checkMasternodeSafeToStop', () => { active_dkgs: 0, next_dkg: 7, upcoming_dkgs: [{ - llmqType: 106, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 7, known: true, isMember: true, + llmqType: 106, + quorumIndex: 0, + quorumHeight: start, + blocksUntilStart: 7, + known: true, + isMember: true, + workBlockHeight: start - 8, + workBlockHash: `hash-${start - 8}`, }], }; @@ -346,10 +371,11 @@ describe('checkMasternodeSafeToStop', () => { expect(result.status).to.equal('blocked'); }); - it('should not read Core\'s announcements as covering a tip that moved too far past them', async () => { - // Core lists a DKG once its work block, 8 blocks before its start, is - // mined. Read at start - 9, the list cannot hold a DKG starting at - // `start`, which the restart margin of start - 6 has to see. + it('should not refuse a Core that connects several blocks during the reads', async () => { + // A syncing Core connects only blocks older than its 6-hour maximum tip + // age, so it moves this fast only while catching up on DKGs the network + // ran hours ago, which Core skips. Its tip moves on until it stops + // whichever tip the check reads. core.version = CORE_24_0_PATCH; core.height = start - 9; core.dkgInfo = { active_dkgs: 0, next_dkg: 15, upcoming_dkgs: [] }; @@ -360,9 +386,62 @@ describe('checkMasternodeSafeToStop', () => { const { result } = await startCheck(); + expect(result).to.include({ status: 'safe', height: start - 6 }); + }); + + it('should report unsettled when the block Core worked out membership from is replaced', async () => { + // Core leaves this node out of the DKG at `start` on the branch it read; + // the branch that replaces it, at the same height, may not. + core.version = CORE_24_0_PATCH; + core.height = start - 1; + core.hashAt = (height) => (height >= start - 8 ? `branch-a-${height}` : `hash-${height}`); + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 106, + quorumIndex: 0, + quorumHeight: start, + blocksUntilStart: 1, + known: true, + isMember: false, + workBlockHeight: start - 8, + workBlockHash: `branch-a-${start - 8}`, + }], + }; + core.afterDkgStatus = () => { + core.hashAt = (height) => (height >= start - 8 ? `branch-b-${height}` : `hash-${height}`); + core.afterDkgStatus = undefined; + }; + + const { result } = await startCheck(); + expect(result.status).to.equal('unsettled'); }); + it('should take Core\'s word on membership while the block it worked it out from stays on the chain', async () => { + core.version = CORE_24_0_PATCH; + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 7, + upcoming_dkgs: [{ + llmqType: 106, + quorumIndex: 0, + quorumHeight: start, + blocksUntilStart: 7, + known: true, + isMember: false, + workBlockHeight: start - 8, + workBlockHash: `hash-${start - 8}`, + }], + }; + + const { result } = await startCheck(); + + expect(result).to.include({ status: 'safe', height: start - 6 }); + expect([...result.nonMemberDkgs]).to.deep.equal([`llmq_test_platform:0:${start}`]); + }); + it('should count next_dkg from the earliest tip dkginfo may have seen', async () => { core.dkgInfo = { active_dkgs: 0, next_dkg: 7 }; From 148c936bf148981079651e9f7f039d94ac506184 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 12 Nov 2026 03:02:36 +0700 Subject: [PATCH 10/11] fix(dashmate): keep a check's DKGs when its confirming read is discarded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the read confirming a safe result was discarded, as when a block connects during it, the check came back with no height and no blockers. `--safe` remembers DKGs only from a check with a height, so it forgot the DKG of this node the first read announced, and could stop the node once Core no longer showed it. Such a check now reports the first read, unsettled, so its announced DKGs are remembered. Test would have caught this: ✖ before, ✔ after ("should keep what a check announced when its confirming read is discarded"). Co-Authored-By: Claude Opus 5.5 --- .../core/quorum/checkMasternodeSafeToStop.js | 5 ++- .../core/quorum/waitForDKGWindowPass.spec.js | 37 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index 754227ac054..b8dd46ce992 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -248,9 +248,10 @@ export default async function checkMasternodeSafeToStop( const nonMemberDkgs = new Set([...knownNonMemberDkgs, ...first.nonMemberDkgs]); const second = await evaluate(rpcClient, coreVersion, nonMemberDkgs, first.height); - // The confirming read was discarded: keep what the first check verified. + // The confirming read was discarded: report the first check, unconfirmed, + // so that a caller polling again keeps the DKGs it announced. if (second.height === null) { - return { ...second, nonMemberDkgs, tip: first.tip }; + return { ...first, status: 'unsettled', nonMemberDkgs }; } // The first check's tip was replaced: none of it holds on this chain. diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index 9ec9cd6147f..0cd2e79b8dc 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -644,6 +644,43 @@ describe('waitForDKGWindowPass', () => { expect(waiting.resolved).to.be.true(); }); + it('should keep what a check announced when its confirming read is discarded', async function it() { + // The first check announces a member DKG for later; a block connects + // during the read confirming it, so that read is discarded. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 8, + dkgInfo: { + active_dkgs: 0, + next_dkg: 8, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 8, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.beforeDkgInfo = () => { + core.height += 1; + core.beforeDkgInfo = undefined; + }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + // After the DKG started, Core reports nothing that shows it, as Core 24.1 + // that has not worked out its proTxHash yet. + core.version = CORE_24_1; + core.height = start + 2; + core.dkgInfo = { active_dkgs: 0, next_dkg: 22 }; + await clock.tickAsync(CHECK_INTERVAL_MS); + + await expectStillPolling(waiting, rpcClient); + }); + it('should judge each poll by the Core version answering it', async function it() { // Core is replaced between polls, with no failed call to tell. const start = 24 * 5000; From 140b3107c5a53f9d51d6ab9b66cf1a9dbaa81192 Mon Sep 17 00:00:00 2001 From: Ivan Shumkov Date: Thu, 12 Nov 2026 03:59:50 +0700 Subject: [PATCH 11/11] fix(dashmate): keep a check's DKGs when Core changes its records while confirming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Core older than 24.1 changed its session records during the read confirming a safe result, the check came back unsettled with no blockers. `--safe` then forgot the DKG of this node the first read announced, and could stop the node once Core no longer showed it. Such a check now carries the first read's blockers. Test would have caught this: ✖ before, ✔ after ("should keep what a check announced when Core changes its session records during the confirming read"). Co-Authored-By: Claude Opus 5.5 --- .../core/quorum/checkMasternodeSafeToStop.js | 6 ++++ .../core/quorum/waitForDKGWindowPass.spec.js | 34 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index b8dd46ce992..aeb626958f2 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -261,6 +261,12 @@ export default async function checkMasternodeSafeToStop( second.nonMemberDkgs = new Set([...nonMemberDkgs, ...second.nonMemberDkgs]); + // Core changed its session records during the confirming read, which then + // shows no DKGs: keep the first check's, unconfirmed. + if (second.status === 'unsettled') { + return { ...second, blockers: first.blockers }; + } + // A Core restarted into initial block download may no longer announce the // DKGs the first check saw. if (second.isSyncing) { diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index 0cd2e79b8dc..d82c7dcd451 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -681,6 +681,40 @@ describe('waitForDKGWindowPass', () => { await expectStillPolling(waiting, rpcClient); }); + it('should keep what a check announced when Core changes its session records during the confirming read', async function it() { + // The first check announces a member DKG for later; the read confirming + // it finds Core 24.0 counting a session record dkgstatus does not list. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 8, + dkgInfo: { + active_dkgs: 0, + next_dkg: 8, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 8, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.dkgInfo = { ...core.dkgInfo, active_dkgs: 1 }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + // After the DKG started, Core reports nothing that shows it, as Core 24.1 + // that has not worked out its proTxHash yet. + core.version = CORE_24_1; + core.height = start + 2; + core.dkgInfo = { active_dkgs: 0, next_dkg: 22 }; + await clock.tickAsync(CHECK_INTERVAL_MS); + + await expectStillPolling(waiting, rpcClient); + }); + it('should judge each poll by the Core version answering it', async function it() { // Core is replaced between polls, with no failed call to tell. const start = 24 * 5000;