diff --git a/packages/dashmate/src/commands/group/restart.js b/packages/dashmate/src/commands/group/restart.js index 0ba25f5f8c9..79d7ab097d7 100644 --- a/packages/dashmate/src/commands/group/restart.js +++ b/packages/dashmate/src/commands/group/restart.js @@ -1,3 +1,4 @@ +import { Flags } from '@oclif/core'; import { Listr } from 'listr2'; import GroupBaseCommand from '../../oclif/command/GroupBaseCommand.js'; import MuteOneLineError from '../../oclif/errors/MuteOneLineError.js'; @@ -7,11 +8,11 @@ export default class GroupRestartCommand extends GroupBaseCommand { static flags = { ...GroupBaseCommand.flags, - safe: { + safe: Flags.boolean({ char: 's', description: 'wait for dkg before stop', default: false, - }, + }), }; /** diff --git a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js index 73d110b53f8..aeb626958f2 100644 --- a/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js +++ b/packages/dashmate/src/core/quorum/checkMasternodeSafeToStop.js @@ -1,28 +1,288 @@ -import isMasternodeSafeToStopDuringDkg, { - shouldInspectDkgStatusForSafeStop, -} from './isMasternodeSafeToStopDuringDkg.js'; +import wait from '../../util/wait.js'; +import { MIN_BLOCKS_BEFORE_DKG } from '../../constants.js'; +import getDkgStopBlockers, { + getAnnouncedNonMemberDkgs, + getBlockersAt, + hasMembershipActiveDkgs, +} from './getDkgStopBlockers.js'; /** - * Query Core and decide whether stopping would disrupt a DKG session. - * Older Core responses require the existing per-session status checks. + * How long a safe result from Core older than 24.1 must hold before it is + * trusted. Core's DKG worker records a session it is a member of shortly + * after the session's start block connects; this is comfortably longer. + */ +export const SAFE_STOP_CONFIRMATION_DELAY_MS = 15000; + +/** + * Which stop rule applies depends on Core's version, so a version Core does + * not report is an error rather than a guess. + * + * @param {RpcClient} rpcClient + * @return {Promise} Core's numeric `version` + */ +export async function readCoreVersion(rpcClient) { + const { result } = await rpcClient.getNetworkInfo(); + + if (!Number.isInteger(result && result.version)) { + throw new Error('Core did not report its version'); + } + + return result.version; +} + +/** + * Core works out whether this node is in an upcoming DKG from the DKG's work + * block, so what `quorum dkginfo` says about membership holds on a chain only + * if that chain has the work blocks it names. * * @param {RpcClient} rpcClient + * @param {Object} dkgInfo + * @param {number} height chain tip * @return {Promise} */ -export default async function checkMasternodeSafeToStop(rpcClient) { +async function hasWorkBlocksOf(rpcClient, dkgInfo, height) { + const upcomingDkgs = dkgInfo && Array.isArray(dkgInfo.upcoming_dkgs) ? dkgInfo.upcoming_dkgs : []; + const knownDkgs = upcomingDkgs.filter((dkg) => dkg && dkg.known === true); + const workHeights = [...new Set(knownDkgs.map(({ workBlockHeight }) => workBlockHeight))]; + + if (!workHeights.every((workHeight) => Number.isInteger(workHeight) && workHeight <= height)) { + return false; + } + + const workHashes = new Map(await Promise.all(workHeights.map(async (workHeight) => { + const { result } = await rpcClient.getBlockHash(workHeight); + + return [workHeight, result]; + }))); + + return knownDkgs.every(({ workBlockHeight, workBlockHash }) => ( + workHashes.get(workBlockHeight) === workBlockHash + )); +} + +/** + * Read Core's DKG state at one chain tip. Each RPC reads the tip on its own, + * so the tip is read before and after the DKG calls; if a block connected in + * between, they may describe different tips and are discarded. + * + * Core in initial block download takes no part in DKGs, and stays in it only + * while it connects blocks older than its maximum tip age, 6 hours: catching + * up on DKGs the network ran hours ago. Its state is then kept however far the + * tip moved, as long as the chain read last still has the work blocks Core's + * answers came from. + * + * With `ancestorHeight`, also reads the hash of the chain's block at that + * height, after the tip, so that it describes the chain the snapshot reports. + * + * @param {RpcClient} rpcClient + * @param {number} [ancestorHeight] + * @return {Promise} + */ +async function readDkgSnapshot(rpcClient, ancestorHeight) { + const { result: before } = await rpcClient.getBlockchainInfo(); const { result: dkgInfo } = await rpcClient.quorum('dkginfo'); + const { result: dkgStatus } = await rpcClient.quorum('dkgstatus'); + const { result: after } = await rpcClient.getBlockchainInfo(); + + const isSyncing = before.initialblockdownload === true && after.initialblockdownload === true; + const hasTipChanged = before.bestblockhash !== after.bestblockhash; + + if (!isSyncing && hasTipChanged) { + return null; + } + + if (hasTipChanged && !await hasWorkBlocksOf(rpcClient, dkgInfo, after.blocks)) { + return null; + } + + let ancestorHash; + if (ancestorHeight !== undefined && after.blocks >= ancestorHeight) { + ({ result: ancestorHash } = await rpcClient.getBlockHash(ancestorHeight)); + } + + return { + height: after.blocks, + // Core counts next_dkg from the tip dkginfo saw, which is no lower than + // this; it is below `height` only while syncing lets the tip move. + dkgInfoHeight: before.blocks, + hash: after.bestblockhash, + ancestorHash, + isSyncing, + dkgInfo, + dkgStatus, + }; +} + +/** + * Core older than 24.1 reports `active_dkgs` as the number of the DKG + * worker's session records, which `quorum dkgstatus` lists. Different counts + * mean the worker changed its records between the two reads, as it does + * when a session starts. + * + * @param {Object} snapshot + * @param {number} coreVersion + * @return {boolean} + */ +function haveSessionRecordsChanged({ dkgInfo, dkgStatus }, coreVersion) { + return !hasMembershipActiveDkgs(coreVersion) + && Number.isInteger(dkgInfo && dkgInfo.active_dkgs) + && Array.isArray(dkgStatus && dkgStatus.session) + && dkgInfo.active_dkgs !== dkgStatus.session.length; +} + +/** + * @typedef {Object} SafeStopCheck + * @property {string} status `safe`, `blocked`, or `unsettled` when Core's + * state changed too fast to read a consistent or confirmed one + * @property {number|null} height + * @property {string} [hash] chain tip the check read + * @property {DkgStopBlocker[]} blockers see {@link getDkgStopBlockers} + * @property {Set} nonMemberDkgs DKGs Core announced without this + * node, see {@link getAnnouncedNonMemberDkgs} + * @property {boolean} [isSyncing] Core is in initial block download + * @property {boolean} [isReorg] the chain no longer has the tip of an earlier + * check; what was learned on that branch, such as `nonMemberDkgs`, no + * longer holds + * @property {{ height: number, hash: string }|null} tip chain tip + * `nonMemberDkgs` were read on, or `null` when no consistent read was made + */ + +/** + * @param {RpcClient} rpcClient + * @param {number} coreVersion + * @param {Set} knownNonMemberDkgs + * @param {number} [ancestorHeight] + * @return {Promise} + */ +async function evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestorHeight) { + const snapshot = await readDkgSnapshot(rpcClient, ancestorHeight); + + if (!snapshot) { + return { + status: 'unsettled', height: null, blockers: [], nonMemberDkgs: new Set(), tip: null, + }; + } + + const { height, hash, isSyncing } = snapshot; + const result = { + height, + hash, + ancestorHash: snapshot.ancestorHash, + nonMemberDkgs: getAnnouncedNonMemberDkgs(snapshot.dkgInfo), + tip: { height, hash }, + }; + + // Core's DKG worker does nothing during initial block download, and Core + // never re-enters it without a restart, so a node in it has had no DKG + // session since it started: session records do not apply. DKGs it + // announces still do, as Core may finish syncing in time for them. + if (isSyncing) { + const blockers = getDkgStopBlockers({ ...snapshot, coreVersion, knownNonMemberDkgs }) + .filter(({ reason }) => ['upcoming', 'imminent', 'unreadable'].includes(reason)); + + return { + ...result, + status: getBlockersAt(blockers, height).length > 0 ? 'blocked' : 'safe', + blockers, + isSyncing, + }; + } + + if (haveSessionRecordsChanged(snapshot, coreVersion)) { + return { ...result, status: 'unsettled', blockers: [] }; + } + + const blockers = getDkgStopBlockers({ ...snapshot, coreVersion, knownNonMemberDkgs }); + + return { + ...result, + status: getBlockersAt(blockers, height).length > 0 ? 'blocked' : 'safe', + blockers, + }; +} + +/** + * Query Core and decide whether stopping the masternode now would disrupt a + * DKG it takes part in. See {@link getDkgStopBlockers} for the rule. + * + * Core older than 24.1 reports a session only once its DKG worker has + * recorded it, after the start block connects, so a safe result is checked + * again after {@link SAFE_STOP_CONFIRMATION_DELAY_MS}. By then every block the + * first check saw is old enough for Core to have replaced records of previous + * rounds. The first check already ruled out a DKG of this node starting + * within `MIN_BLOCKS_BEFORE_DKG` blocks, so the second may see up to that + * many new blocks on top of the first check's tip; more than that, a lower + * tip, or a chain without the first check's tip is `unsettled`. + * + * `knownNonMemberDkgs` may only hold what earlier checks learned on the + * current chain: pass the tip of the latest of them as `ancestor`, and a + * check that finds it replaced reports `isReorg`. + * + * @param {RpcClient} rpcClient + * @param {number} coreVersion from {@link readCoreVersion} + * @param {Object} [options] + * @param {Set} [options.knownNonMemberDkgs] DKGs Core announced + * without this node at earlier checks + * @param {{ height: number, hash: string }} [options.ancestor] tip of the + * latest earlier check + * @return {Promise} + */ +export default async function checkMasternodeSafeToStop( + rpcClient, + coreVersion, + { knownNonMemberDkgs = new Set(), ancestor } = {}, +) { + const first = await evaluate(rpcClient, coreVersion, knownNonMemberDkgs, ancestor && ancestor.height); + + if (ancestor && first.height !== null + && (first.height < ancestor.height || first.ancestorHash !== ancestor.hash)) { + return { ...first, status: 'unsettled', isReorg: true }; + } + + if (first.status !== 'safe' || first.isSyncing || hasMembershipActiveDkgs(coreVersion)) { + return first; + } + + await wait(SAFE_STOP_CONFIRMATION_DELAY_MS); + + const nonMemberDkgs = new Set([...knownNonMemberDkgs, ...first.nonMemberDkgs]); + const second = await evaluate(rpcClient, coreVersion, nonMemberDkgs, first.height); + + // The confirming read was discarded: report the first check, unconfirmed, + // so that a caller polling again keeps the DKGs it announced. + if (second.height === null) { + return { ...first, status: 'unsettled', nonMemberDkgs }; + } + + // The first check's tip was replaced: none of it holds on this chain. + if (second.height < first.height || second.ancestorHash !== first.hash) { + return { ...second, status: 'unsettled', isReorg: true }; + } + + second.nonMemberDkgs = new Set([...nonMemberDkgs, ...second.nonMemberDkgs]); + + // Core changed its session records during the confirming read, which then + // shows no DKGs: keep the first check's, unconfirmed. + if (second.status === 'unsettled') { + return { ...second, blockers: first.blockers }; + } + + // A Core restarted into initial block download may no longer announce the + // DKGs the first check saw. + if (second.isSyncing) { + const blockers = [...second.blockers, ...first.blockers]; + + return { + ...second, + status: getBlockersAt(blockers, second.height).length > 0 ? 'blocked' : 'safe', + blockers, + }; + } - let dkgStatus; - let currentHeight; - if (shouldInspectDkgStatusForSafeStop(dkgInfo)) { - [ - { result: dkgStatus }, - { result: currentHeight }, - ] = await Promise.all([ - rpcClient.quorum('dkgstatus'), - rpcClient.getBlockCount(), - ]); + if (second.status === 'safe' && !second.isSyncing + && second.height > first.height + MIN_BLOCKS_BEFORE_DKG) { + return { ...second, status: 'unsettled' }; } - return isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, currentHeight); + return second; } diff --git a/packages/dashmate/src/core/quorum/getDkgStopBlockers.js b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js new file mode 100644 index 00000000000..140942265e0 --- /dev/null +++ b/packages/dashmate/src/core/quorum/getDkgStopBlockers.js @@ -0,0 +1,419 @@ +import { MIN_BLOCKS_BEFORE_DKG } from '../../constants.js'; + +/** + * First Dash Core release whose `quorum dkginfo` derives `active_dkgs` from + * this masternode's membership, counting a session from its start block + * (Core 24.1.0). Older Core counts the DKG worker's local session records, + * which appear only after the worker has started a session and linger long + * after it ends. Core reports `version` as 10000 * major + 100 * minor + patch. + */ +export const MEMBERSHIP_ACTIVE_DKGS_CORE_VERSION = 240100; + +/** + * DKG parameters from Dash Core `src/llmq/params.h`. `quorum dkgstatus` names + * types by `name`, `quorum dkginfo` `upcoming_dkgs` by numeric `id`. + * + * Keep in sync with `src/llmq/params.h` in Dash Core. + */ +export const LLMQ_TYPES = [ + { + id: 1, name: 'llmq_50_60', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 2, name: 'llmq_400_60', dkgInterval: 288, dkgPhaseBlocks: 4, + }, + { + id: 3, name: 'llmq_400_85', dkgInterval: 576, dkgPhaseBlocks: 4, + }, + { + id: 4, name: 'llmq_100_67', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 5, name: 'llmq_60_75', dkgInterval: 288, dkgPhaseBlocks: 2, + }, + { + id: 6, name: 'llmq_25_67', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 100, name: 'llmq_test', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 101, name: 'llmq_devnet', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 102, name: 'llmq_test_v17', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 103, name: 'llmq_test_dip0024', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 104, name: 'llmq_test_instantsend', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 105, name: 'llmq_devnet_dip0024', dkgInterval: 48, dkgPhaseBlocks: 2, + }, + { + id: 106, name: 'llmq_test_platform', dkgInterval: 24, dkgPhaseBlocks: 2, + }, + { + id: 107, name: 'llmq_devnet_platform', dkgInterval: 24, dkgPhaseBlocks: 2, + }, +]; + +/** + * Blocks from a session's start to the end of its Commit phase, Core's + * `QuorumPhase::Commit * dkgPhaseBlocks`. By then a member has sent every + * message that decides whether the final commitment counts it as valid, + * which is what PoSe penalizes; finalizing and mining the commitment does + * not need it. Each rotated quorum index starts on its own block and is + * measured from its own `quorumHeight`, not from the cycle's shared mining + * window. + * + * @param {{ dkgPhaseBlocks: number }} llmq + * @return {number} + */ +export function getActiveDkgBlocks(llmq) { + return 5 * llmq.dkgPhaseBlocks; +} + +/** + * @param {number|null} coreVersion + * @return {boolean} + */ +export function hasMembershipActiveDkgs(coreVersion) { + return Number.isInteger(coreVersion) && coreVersion >= MEMBERSHIP_ACTIVE_DKGS_CORE_VERSION; +} + +/** + * @typedef {Object} DkgStopBlocker + * @property {string} reason `session`, `new-round`, `unrecorded`, `upcoming`, + * `imminent`, `active-count` or `unreadable` + * @property {string} [llmqType] LLMQ name + * @property {number} [quorumIndex] + * @property {boolean} [membershipKnown] for `upcoming`: `false` when Core + * could not determine whether this node is a member + * @property {number} [count] for `active-count` + * @property {string} [source] for `unreadable`: which Core data was unreadable + * @property {number|null} from first blocked height; `null` blocks from now + * @property {number|null} until first height no longer blocked; `null` when + * the end is unknown + */ + +/** + * @param {DkgStopBlocker} blocker + * @param {number} height + * @return {boolean} + */ +function isBlockedAt(blocker, height) { + return (blocker.from === null || blocker.from <= height) + && (blocker.until === null || height < blocker.until); +} + +/** + * @param {DkgStopBlocker[]} blockers + * @param {number} height + * @return {DkgStopBlocker[]} + */ +export function getBlockersAt(blockers, height) { + return blockers.filter((blocker) => isBlockedAt(blocker, height)); +} + +function isCount(value) { + return Number.isInteger(value) && value >= 0; +} + +function hasValidDkgInfoShape(dkgInfo) { + return !!dkgInfo + && typeof dkgInfo === 'object' + && isCount(dkgInfo.active_dkgs) + && isCount(dkgInfo.next_dkg); +} + +function isValidUpcomingDkg(dkg) { + return !!dkg + && typeof dkg === 'object' + && isCount(dkg.quorumHeight) + && typeof dkg.known === 'boolean' + && (!dkg.known || typeof dkg.isMember === 'boolean'); +} + +function unreadable(source) { + return { + reason: 'unreadable', source, from: null, until: null, + }; +} + +/** + * Without membership data (Core < 24, or Core does not know this node's + * proTxHash), any DKG of any type may include this node, so the restart + * margin before the next one blocks with no known end. + */ +function getImminentDkgBlocker(dkgInfo, dkgInfoHeight) { + return { + reason: 'imminent', + from: dkgInfoHeight + dkgInfo.next_dkg - MIN_BLOCKS_BEFORE_DKG, + until: null, + }; +} + +/** + * Core lists a DKG in `upcoming_dkgs` from its work block, 8 blocks before + * the cycle starts, so every DKG starting within `MIN_BLOCKS_BEFORE_DKG` + * is listed. A DKG this node is a member of, or whose membership Core could + * not determine, blocks from the restart margin before its start until its + * Commit phase ends; one of a type dashmate does not know is unreadable. + */ +function getUpcomingDkgBlockers(upcomingDkgs) { + if (!Array.isArray(upcomingDkgs)) { + return [unreadable('quorum dkginfo upcoming_dkgs')]; + } + + return upcomingDkgs.flatMap((dkg) => { + if (!isValidUpcomingDkg(dkg)) { + return [unreadable('quorum dkginfo upcoming_dkgs')]; + } + + if (dkg.known && !dkg.isMember) { + return []; + } + + const llmq = LLMQ_TYPES.find(({ id }) => id === dkg.llmqType); + + // Without its Commit phase nothing says when such a DKG stops blocking. + if (!llmq) { + return [unreadable(`quorum dkginfo upcoming_dkgs llmqType ${dkg.llmqType}`)]; + } + + const start = dkg.quorumHeight; + + return [{ + reason: 'upcoming', + llmqType: llmq.name, + quorumIndex: dkg.quorumIndex, + membershipKnown: dkg.known, + from: start - MIN_BLOCKS_BEFORE_DKG, + until: start + getActiveDkgBlocks(llmq), + }]; + }); +} + +/** + * The DKG worker keeps a record for the latest session of each LLMQ type + * and quorum index this node was a member of. A record inside its active + * window blocks. A record at least one interval old, but less than one + * interval plus the active window, means the next session of that type and + * index has started and the worker, which runs shortly after the block + * connects, has not yet replaced or removed the record. This node may be a + * member again, so the record blocks until that session's Commit phase ends. + */ +function getSessionRecordBlockers(dkgStatus, height) { + const sessions = dkgStatus && dkgStatus.session; + + if (!Array.isArray(sessions)) { + return [unreadable('quorum dkgstatus')]; + } + + return sessions.flatMap((entry) => { + const llmq = entry && LLMQ_TYPES.find(({ name }) => name === entry.llmqType); + const quorumHeight = entry && entry.status && entry.status.quorumHeight; + + if (!llmq || !isCount(quorumHeight) || quorumHeight > height) { + return [unreadable(`quorum dkgstatus session ${entry && entry.llmqType}`)]; + } + + const activeBlocks = getActiveDkgBlocks(llmq); + const offset = height - quorumHeight; + const session = { llmqType: llmq.name, quorumIndex: entry.quorumIndex }; + + if (offset < activeBlocks) { + return [{ + ...session, + reason: 'session', + from: quorumHeight, + until: quorumHeight + activeBlocks, + }]; + } + + const nextQuorumHeight = quorumHeight + llmq.dkgInterval; + if (height >= nextQuorumHeight && height < nextQuorumHeight + activeBlocks) { + return [{ + ...session, + reason: 'new-round', + from: nextQuorumHeight, + until: nextQuorumHeight + activeBlocks, + }]; + } + + return []; + }); +} + +/** + * Key naming a DKG session: its quorum and its start block. + * + * @param {string} llmqType + * @param {number} quorumIndex + * @param {number} start + * @return {string} + */ +function getDkgKey(llmqType, quorumIndex, start) { + return `${llmqType}:${quorumIndex}:${start}`; +} + +/** + * Whether the DKG a key from {@link getAnnouncedNonMemberDkgs} names can + * still block at `height`: it has not started yet or has not finished its + * Commit phase. + * + * @param {string} key + * @param {number} height + * @return {boolean} + */ +export function isDkgKeyCurrent(key, height) { + const [llmqType, , start] = key.split(':'); + const llmq = LLMQ_TYPES.find(({ name }) => name === llmqType); + + return !llmq || Number(start) + getActiveDkgBlocks(llmq) > height; +} + +/** + * The DKGs Core announces this node is not a member of, as + * `llmqType:quorumIndex:start` keys. Once such a DKG starts Core no longer + * announces it, so a caller polling Core can remember these to tell a + * session it is not part of from one Core has not recorded yet. + * + * @param {Object} dkgInfo Result of `quorum dkginfo`. + * @return {Set} + */ +export function getAnnouncedNonMemberDkgs(dkgInfo) { + const upcomingDkgs = dkgInfo && dkgInfo.upcoming_dkgs; + + if (!Array.isArray(upcomingDkgs)) { + return new Set(); + } + + return new Set(upcomingDkgs + .filter((dkg) => isValidUpcomingDkg(dkg) && dkg.known && !dkg.isMember) + .map((dkg) => [LLMQ_TYPES.find(({ id }) => id === dkg.llmqType), dkg]) + .filter(([llmq]) => llmq) + .map(([llmq, dkg]) => getDkgKey(llmq.name, dkg.quorumIndex, dkg.quorumHeight))); +} + +/** + * At a session's start block Core older than 24.1 erases the previous round's + * record, computes the new session's members and only then records the + * session if this node is one of them; until then nothing tells whether it + * is, however long that takes. `quorum dkgstatus` lists every enabled quorum + * with the start of its current session, so a current session without a + * record blocks until its Commit phase ends, unless Core announced it + * without this node. + */ +function getUnrecordedSessionBlockers(dkgStatus, height, knownNonMemberDkgs) { + const connections = dkgStatus && dkgStatus.quorumConnections; + + if (!Array.isArray(connections)) { + return [unreadable('quorum dkgstatus quorumConnections')]; + } + + const sessions = Array.isArray(dkgStatus.session) ? dkgStatus.session : []; + const isRecorded = ({ llmqType, quorumIndex, pQuorumBaseBlockIndex: start }) => sessions + .some((entry) => entry + && entry.llmqType === llmqType + && entry.quorumIndex === quorumIndex + && entry.status + && entry.status.quorumHeight === start); + + return connections + // A quorum whose current session has not started yet has no start block. + .filter((quorum) => quorum && isCount(quorum.pQuorumBaseBlockIndex) + && quorum.pQuorumBaseBlockIndex <= height) + .flatMap((quorum) => { + const { llmqType, quorumIndex, pQuorumBaseBlockIndex: start } = quorum; + const llmq = LLMQ_TYPES.find(({ name }) => name === llmqType); + + if (!llmq) { + return [unreadable(`quorum dkgstatus quorumConnections ${llmqType}`)]; + } + + const until = start + getActiveDkgBlocks(llmq); + + if (height >= until || isRecorded(quorum) + || knownNonMemberDkgs.has(getDkgKey(llmqType, quorumIndex, start))) { + return []; + } + + return [{ + reason: 'unrecorded', llmqType, quorumIndex, from: start, until, + }]; + }); +} + +/** + * Everything that makes stopping this masternode risk a PoSe penalty, as + * height ranges. Stopping is safe at a height no blocker covers (see + * {@link getBlockersAt}). Some blockers start above the current height; they + * do not block yet and only help estimate when stopping becomes safe. + * + * All inputs must come from the same chain tip, `height`. Core older than + * {@link MEMBERSHIP_ACTIVE_DKGS_CORE_VERSION} reports a session only after + * its DKG worker has recorded it, after the start block connects; until then + * the session blocks unless Core announced it without this node. Callers + * must also confirm a safe result a while later, once Core has had time to + * replace records of previous rounds. + * + * Unreadable Core data blocks with no known end. + * + * @param {Object} snapshot + * @param {Object} snapshot.dkgInfo Result of `quorum dkginfo`. + * @param {Object} snapshot.dkgStatus Result of `quorum dkgstatus`. + * @param {number} snapshot.height Chain tip both results describe. + * @param {number} [snapshot.dkgInfoHeight] The lowest tip `dkgInfo` may + * describe, when it may be below `height`: Core counts `next_dkg` from it. + * @param {number} snapshot.coreVersion Core's numeric `version`. + * @param {Set} [snapshot.knownNonMemberDkgs] DKGs Core announced + * without this node, from {@link getAnnouncedNonMemberDkgs} + * @return {DkgStopBlocker[]} + */ +export default function getDkgStopBlockers({ + dkgInfo, + dkgStatus, + height, + dkgInfoHeight = height, + coreVersion, + knownNonMemberDkgs = new Set(), +}) { + if (!isCount(height)) { + return [unreadable('block height')]; + } + + if (!hasValidDkgInfoShape(dkgInfo)) { + return [unreadable('quorum dkginfo')]; + } + + const blockers = [ + ...(dkgInfo.upcoming_dkgs === undefined + ? [getImminentDkgBlocker(dkgInfo, dkgInfoHeight)] + : getUpcomingDkgBlockers(dkgInfo.upcoming_dkgs)), + // Read on every Core version: Core 24.1 stops counting a running session + // in active_dkgs when a mid-session ProUpServTx clears its proTxHash, but + // the session keeps going and its record stays. + ...getSessionRecordBlockers(dkgStatus, height), + ]; + + if (!hasMembershipActiveDkgs(coreVersion)) { + blockers.push(...getUnrecordedSessionBlockers(dkgStatus, height, knownNonMemberDkgs)); + } + + // Core 24.1+ counts every session this node is a member of, or whose + // membership is unknown, from its start block through its Commit phase. + if (hasMembershipActiveDkgs(coreVersion) && dkgInfo.active_dkgs > 0) { + blockers.push({ + reason: 'active-count', + count: dkgInfo.active_dkgs, + from: null, + until: null, + }); + } + + return blockers; +} diff --git a/packages/dashmate/src/core/quorum/isMasternodeSafeToStopDuringDkg.js b/packages/dashmate/src/core/quorum/isMasternodeSafeToStopDuringDkg.js deleted file mode 100644 index de5b8840d97..00000000000 --- a/packages/dashmate/src/core/quorum/isMasternodeSafeToStopDuringDkg.js +++ /dev/null @@ -1,140 +0,0 @@ -import { MIN_BLOCKS_BEFORE_DKG } from '../../constants.js'; - -/** - * `dkgMiningWindowStart` values from Dash Core `src/llmq/params.h`, - * indexed by llmqType string as reported in `quorum dkgstatus`. The - * number is how many blocks after a session's `quorumHeight` the - * active DKG window lasts. Once `currentHeight - quorumHeight >= - * window`, the session is past its active phase and a restart no - * longer risks a PoSe penalty for that session. - * - * Keep in sync with `src/llmq/params.h` in Dash Core. - */ -export const DKG_MINING_WINDOW_START_BY_LLMQ_TYPE = { - llmq_test: 10, - llmq_test_instantsend: 10, - llmq_test_v17: 10, - llmq_test_dip0024: 12, - llmq_test_platform: 10, - llmq_devnet: 10, - llmq_devnet_dip0024: 12, - llmq_devnet_platform: 10, - llmq_50_60: 10, - llmq_60_75: 42, - llmq_400_60: 20, - llmq_400_85: 20, - llmq_100_67: 10, - llmq_25_67: 10, -}; - -function isValidDkgCounter(value) { - return typeof value === 'number' && Number.isFinite(value) && value >= 0; -} - -function hasValidDkgInfoShape(dkgInfo) { - return !!dkgInfo - && typeof dkgInfo === 'object' - && isValidDkgCounter(dkgInfo.active_dkgs) - && isValidDkgCounter(dkgInfo.next_dkg); -} - -/** - * Malformed entries always block; member and unknown-membership entries - * block when they start within the restart margin. - */ -function isUpcomingDkgBlockingStop(dkg) { - if (!dkg - || typeof dkg !== 'object' - || !isValidDkgCounter(dkg.blocksUntilStart) - || typeof dkg.known !== 'boolean' - || (dkg.known && typeof dkg.isMember !== 'boolean')) { - return true; - } - - return dkg.blocksUntilStart <= MIN_BLOCKS_BEFORE_DKG && (!dkg.known || dkg.isMember); -} - -/** - * @param {Object} dkgInfo - * @return {boolean} - */ -export function shouldInspectDkgStatusForSafeStop(dkgInfo) { - if (!hasValidDkgInfoShape(dkgInfo)) { - return false; - } - - return dkgInfo.upcoming_dkgs === undefined - && dkgInfo.active_dkgs > 0 - && dkgInfo.next_dkg > MIN_BLOCKS_BEFORE_DKG; -} - -/** - * With upcoming_dkgs, stopping is safe when no DKG counts toward active_dkgs - * and no member or unknown-membership DKG starts within the restart margin. - * - * Without upcoming_dkgs (older Core, or no proTxHash), use the legacy next_dkg - * guard and resolve tracked sessions against dkgstatus and getblockcount. - * Unknown quorum types or malformed status fail safe; sessions past - * dkgMiningWindowStart are ignored. - * - * @param {Object} dkgInfo Result of quorum dkginfo. - * @param {Object} [dkgStatus] Result of quorum dkgstatus for older Core. - * @param {number} [currentHeight] Chain tip height for older Core. - * @return {boolean} - */ -export default function isMasternodeSafeToStopDuringDkg( - dkgInfo, - dkgStatus, - currentHeight, -) { - if (!hasValidDkgInfoShape(dkgInfo)) { - return false; - } - - const { upcoming_dkgs: upcomingDkgs } = dkgInfo; - - // Relies on active_dkgs counting member and unknown-membership sessions - // (dashpay/dash#7812). - if (upcomingDkgs !== undefined) { - return dkgInfo.active_dkgs === 0 - && Array.isArray(upcomingDkgs) - && !upcomingDkgs.some(isUpcomingDkgBlockingStop); - } - - if (dkgInfo.next_dkg <= MIN_BLOCKS_BEFORE_DKG) { - return false; - } - - if (dkgInfo.active_dkgs === 0) { - return true; - } - - if (!dkgStatus - || !Array.isArray(dkgStatus.session) - || dkgStatus.session.length === 0 - || typeof currentHeight !== 'number' - || !Number.isFinite(currentHeight)) { - return false; - } - - for (const sessionEntry of dkgStatus.session) { - const llmqType = sessionEntry && sessionEntry.llmqType; - const quorumHeight = sessionEntry && sessionEntry.status - && sessionEntry.status.quorumHeight; - - const windowLength = DKG_MINING_WINDOW_START_BY_LLMQ_TYPE[llmqType]; - - if (windowLength === undefined - || typeof quorumHeight !== 'number' - || !Number.isFinite(quorumHeight)) { - return false; - } - - const offset = currentHeight - quorumHeight; - if (offset < 0 || offset < windowLength) { - return false; - } - } - - return true; -} diff --git a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js index 1bd52daa0ab..5e006f8f4aa 100644 --- a/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js +++ b/packages/dashmate/src/core/quorum/waitForDKGWindowPass.js @@ -1,7 +1,8 @@ import wait from '../../util/wait.js'; -import checkMasternodeSafeToStop from './checkMasternodeSafeToStop.js'; +import checkMasternodeSafeToStop, { readCoreVersion } from './checkMasternodeSafeToStop.js'; +import { getBlockersAt, isDkgKeyCurrent } from './getDkgStopBlockers.js'; -const CHECK_INTERVAL_MS = 10000; +export const CHECK_INTERVAL_MS = 10000; /** * Poll Core until the masternode is safe to stop without disrupting a @@ -9,11 +10,66 @@ const CHECK_INTERVAL_MS = 10000; * safety rule. The only acceptable exit is reaching a safe state, so * that `--safe` cannot silently fall back to an unsafe restart. * + * DKGs this node takes part in, announced or running, are remembered until + * their Commit phase ends. Once a session starts Core no longer announces + * it, Core older than 24.1 reports it only after its DKG worker has computed + * the session's members and recorded it, which can take a while at the + * start of a rotation cycle, and a restarted Core keeps no record of it at + * all. DKGs announced without this node are remembered too, + * so their sessions need not wait for a record that never comes, as long as + * the chain they were announced on stays: a reorg replacing it may make this + * node a member after all. + * * @param {RpcClient} rpcClient * @return {Promise} */ export default async function waitForDKGWindowPass(rpcClient) { - while (!await checkMasternodeSafeToStop(rpcClient)) { + const knownDkgs = new Map(); + let nonMemberDkgs = new Set(); + let lastTip; + + for (;;) { + // Which rule applies depends on the version, and Core may have been + // replaced since the last poll. + const coreVersion = await readCoreVersion(rpcClient); + + const check = await checkMasternodeSafeToStop(rpcClient, coreVersion, { + knownNonMemberDkgs: nonMemberDkgs, + ancestor: lastTip, + }); + + nonMemberDkgs = check.isReorg + ? check.nonMemberDkgs + : new Set([...nonMemberDkgs, ...check.nonMemberDkgs]); + + if (check.tip) { + lastTip = check.tip; + } + + if (check.height !== null) { + nonMemberDkgs = new Set([...nonMemberDkgs] + .filter((key) => isDkgKeyCurrent(key, check.height))); + + check.blockers + .filter(({ from, until }) => from !== null && until !== null) + .forEach((dkg) => { + knownDkgs.set(`${dkg.reason}:${dkg.llmqType}:${dkg.quorumIndex}:${dkg.from}`, dkg); + }); + + knownDkgs.forEach((dkg, key) => { + if (dkg.until <= check.height) { + knownDkgs.delete(key); + } + }); + } + + // A remembered DKG blocks even while Core syncs: Core restarted into initial + // block download may finish in time for it. + if (check.status === 'safe' + && getBlockersAt([...knownDkgs.values()], check.height).length === 0) { + return; + } + await wait(CHECK_INTERVAL_MS); } } diff --git a/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js b/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js index d03a7dbbe1d..686417d09db 100644 --- a/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js +++ b/packages/dashmate/src/listr/tasks/stopNodeTaskFactory.js @@ -1,7 +1,32 @@ /* eslint-disable no-console */ import { Listr } from 'listr2'; import waitForDKGWindowPass from '../../core/quorum/waitForDKGWindowPass.js'; -import checkMasternodeSafeToStop from '../../core/quorum/checkMasternodeSafeToStop.js'; +import checkMasternodeSafeToStop, { + readCoreVersion, +} from '../../core/quorum/checkMasternodeSafeToStop.js'; +import { getBlockersAt } from '../../core/quorum/getDkgStopBlockers.js'; +import { NETWORK_LOCAL } from '../../constants.js'; + +/** + * Group commands stop a local network's miner before its masternodes. Without + * it no block comes, so a DKG wait could never end, and with every masternode + * of the network going down together their DKG produces no commitment to + * penalize them in. Only masternodes of that local network, the miner's group, + * are exempt: a group may hold configs of other networks. + * + * @param {Config} config + * @param {Object} ctx + * @return {boolean} + */ +function isLocalNetworkHalted(config, ctx) { + return config.get('network') === NETWORK_LOCAL + && !!ctx.stoppedLocalMinerGroups + && ctx.stoppedLocalMinerGroups.has(config.get('group')); +} + +// A check comes back unsettled while the chain moves too fast to read a +// consistent DKG state, such as while Core catches up after a restart. +const SAFE_STOP_CHECK_ATTEMPTS = 3; /** * @param {DockerCompose} dockerCompose @@ -45,7 +70,8 @@ export default function stopNodeTaskFactory( }, { title: 'Check node is participating in DKG', - enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && !ctx.isSafe && !ctx.platformOnly, + enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && !ctx.isSafe + && !ctx.platformOnly && !isLocalNetworkHalted(config, ctx), task: async () => { const rpcClient = createRpcClient({ port: config.get('core.rpc.port'), @@ -54,16 +80,43 @@ export default function stopNodeTaskFactory( host: await getConnectionHost(config, 'core', 'core.rpc.host'), }); - if (!await checkMasternodeSafeToStop(rpcClient)) { + const coreVersion = await readCoreVersion(rpcClient); + + let check; + for (let attempt = 0; attempt < SAFE_STOP_CHECK_ATTEMPTS; attempt += 1) { + check = await checkMasternodeSafeToStop(rpcClient, coreVersion); + + if (check.status !== 'unsettled') { + break; + } + } + + const unreadable = check.status === 'blocked' + && getBlockersAt(check.blockers, check.height) + .find(({ reason }) => reason === 'unreadable'); + + if (unreadable) { + throw new Error(`dashmate cannot read Core's DKG state (${unreadable.source}). ` + + 'Update dashmate, or continue with --force'); + } + + if (check.status === 'blocked') { throw new Error('Your node is currently participating in a DKG exchange session ' + '(or one is about to start) and stopping it right now may result in a PoSe ban. ' + 'Try again later, or continue with --force or --safe flags'); } + + if (check.status === 'unsettled') { + throw new Error('The chain kept advancing while checking for DKG sessions ' + + '(is Core still syncing?). ' + + 'Try again later, or continue with --force or --safe flags'); + } }, }, { title: 'Wait for DKG window to pass', - enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && ctx.isSafe && !ctx.platformOnly, + enabled: (ctx) => config.get('core.masternode.enable') && !ctx.isForce && ctx.isSafe + && !ctx.platformOnly && !isLocalNetworkHalted(config, ctx), task: async () => waitForDKGWindowPass(createRpcClient({ port: config.get('core.rpc.port'), user: 'dashmate', @@ -79,6 +132,13 @@ export default function stopNodeTaskFactory( : []; await dockerCompose.stop(config, { profiles }); + + if (config.get('network') === NETWORK_LOCAL && config.get('core.miner.enable')) { + ctx.stoppedLocalMinerGroups = new Set([ + ...(ctx.stoppedLocalMinerGroups || []), + config.get('group'), + ]); + } }, }, ]); diff --git a/packages/dashmate/src/test/mock/getCoreRpcMock.js b/packages/dashmate/src/test/mock/getCoreRpcMock.js new file mode 100644 index 00000000000..a0972ea053a --- /dev/null +++ b/packages/dashmate/src/test/mock/getCoreRpcMock.js @@ -0,0 +1,71 @@ +/** + * Core RPC client whose stubs answer from `core`, a plain object a test + * mutates to move the chain or change DKG state between reads. + * `core.beforeDkgInfo` and `core.afterDkgStatus`, when set, run right + * before `quorum dkginfo` is answered and right after `quorum dkgstatus` is. + * Block hashes are derived from their height; `core.hash` overrides the tip's, + * and `core.hashAt(height)`, when set, any block's. `quorum dkgstatus` lists + * no quorum connections unless the test's `dkgStatus` does. + * + * @param {Sinon} sinon + * @param {{ + * version: number, + * height: number, + * hash?: string, + * hashAt?: Function, + * dkgInfo: Object, + * dkgStatus: Object, + * isSyncing?: boolean, + * dkgStatusDelayMs?: number, + * beforeDkgInfo?: Function, + * afterDkgStatus?: Function, + * }} core + */ +export default function getCoreRpcMock(sinon, core) { + const getHash = (height) => { + if (core.hashAt) { + return core.hashAt(height); + } + + return height === core.height && core.hash ? core.hash : `hash-${height}`; + }; + + return { + getNetworkInfo: sinon.stub().callsFake(async () => ({ result: { version: core.version } })), + getBlockchainInfo: sinon.stub().callsFake(async () => ({ + result: { + blocks: core.height, + bestblockhash: getHash(core.height), + initialblockdownload: core.isSyncing === true, + }, + })), + getBlockHash: sinon.stub().callsFake(async (height) => ({ result: getHash(height) })), + quorum: sinon.stub().callsFake(async (command) => { + if (command === 'dkginfo') { + if (core.beforeDkgInfo) { + core.beforeDkgInfo(); + } + + return { result: core.dkgInfo }; + } + + if (command === 'dkgstatus') { + if (core.dkgStatusDelayMs) { + await new Promise((resolve) => { setTimeout(resolve, core.dkgStatusDelayMs); }); + } + + const result = core.dkgStatus && typeof core.dkgStatus === 'object' + ? { quorumConnections: [], ...core.dkgStatus } + : core.dkgStatus; + + if (core.afterDkgStatus) { + core.afterDkgStatus(); + } + + return { result }; + } + + throw new Error(`unexpected quorum ${command}`); + }), + }; +} diff --git a/packages/dashmate/test/unit/commands/group/restart.spec.js b/packages/dashmate/test/unit/commands/group/restart.spec.js new file mode 100644 index 00000000000..060acf69ba3 --- /dev/null +++ b/packages/dashmate/test/unit/commands/group/restart.spec.js @@ -0,0 +1,19 @@ +import { Parser } from '@oclif/core'; +import GroupRestartCommand from '../../../../src/commands/group/restart.js'; + +describe('GroupRestartCommand', () => { + async function parseSafe(argv) { + const { flags } = await Parser.parse(argv, { flags: GroupRestartCommand.flags }); + + return flags.safe; + } + + it('should parse --safe so the restart waits for a safe DKG window', async () => { + expect(await parseSafe(['--safe'])).to.equal(true); + expect(await parseSafe(['-s'])).to.equal(true); + }); + + it('should not wait for a DKG window by default', async () => { + expect(await parseSafe([])).to.equal(false); + }); +}); diff --git a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js index 475b8bfb499..8376867488e 100644 --- a/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js +++ b/packages/dashmate/test/unit/core/quorum/checkMasternodeSafeToStop.spec.js @@ -1,70 +1,454 @@ -import checkMasternodeSafeToStop from '../../../../src/core/quorum/checkMasternodeSafeToStop.js'; +import checkMasternodeSafeToStop, { + readCoreVersion, + SAFE_STOP_CONFIRMATION_DELAY_MS, +} from '../../../../src/core/quorum/checkMasternodeSafeToStop.js'; +import { MIN_BLOCKS_BEFORE_DKG } from '../../../../src/constants.js'; +import getCoreRpcMock from '../../../../src/test/mock/getCoreRpcMock.js'; + +const CORE_23 = 230110; +const CORE_24_0_PATCH = 240099; +const CORE_24_1 = 240100; describe('checkMasternodeSafeToStop', () => { + let core; let rpcClient; + let clock; beforeEach(function beforeEach() { - rpcClient = { quorum: this.sinon.stub(), getBlockCount: this.sinon.stub() }; + clock = this.sinon.useFakeTimers(); + core = { + version: CORE_23, + height: 1012, + dkgInfo: { active_dkgs: 0, next_dkg: 12 }, + dkgStatus: { session: [] }, + }; + rpcClient = getCoreRpcMock(this.sinon, core); }); - it('should allow a non-member immediately using one membership response', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ - result: { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [{ blocksUntilStart: 1, known: true, isMember: false }], - }, + /** + * Start a check and run the fake clock until the first check has finished + * and, on Core older than 24.1, is waiting to be confirmed. + */ + async function startCheck() { + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version).then((result) => { state.result = result; }); + await clock.tickAsync(0); + + return state; + } + + describe('readCoreVersion', () => { + it('should return the numeric version Core reports', async () => { + expect(await readCoreVersion(rpcClient)).to.equal(CORE_23); }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(true); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); - expect(rpcClient.getBlockCount).to.not.have.been.called(); - }); - - it('should block a positive active_dkgs without inspecting dkgstatus', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ - result: { - active_dkgs: 1, - next_dkg: 24, - upcoming_dkgs: [], - }, + + it('should refuse to guess when Core reports no integer version', async () => { + // Which rule applies depends on the version; reading an older one for + // Core 24.1 would ignore its session count. + core.version = '23.1.10'; + + await expect(readCoreVersion(rpcClient)).to.be.rejectedWith('Core did not report its version'); }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(false); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); - expect(rpcClient.getBlockCount).to.not.have.been.called(); }); - it('should block a member of an imminent DKG', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ - result: { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [{ blocksUntilStart: 1, known: true, isMember: true }], - }, - }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(false); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); + it('should read the chain tip before and after the DKG state', async () => { + core.version = CORE_24_1; + + await startCheck(); + + const [tipBefore, tipAfter] = rpcClient.getBlockchainInfo.getCalls(); + const dkgInfo = rpcClient.quorum.withArgs('dkginfo').getCall(0); + const dkgStatus = rpcClient.quorum.withArgs('dkgstatus').getCall(0); + + expect(tipBefore.calledBefore(dkgInfo)).to.be.true(); + expect(tipBefore.calledBefore(dkgStatus)).to.be.true(); + expect(dkgInfo.calledBefore(tipAfter)).to.be.true(); + expect(dkgStatus.calledBefore(tipAfter)).to.be.true(); + }); + + it('should report unsettled when a block connects during the reads', async () => { + core.version = CORE_24_1; + core.beforeDkgInfo = () => { core.height += 1; }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('unsettled'); + }); + + it('should report unsettled when Core changes its session records between the reads', async () => { + // Core older than 24.1 counts the records dkgstatus lists; it erases a + // record when the next session of its type starts. + core.dkgInfo = { active_dkgs: 1, next_dkg: 12 }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('unsettled'); + }); + + it('should report blocked at once without waiting for confirmation', async () => { + core.dkgInfo = { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('blocked'); + expect(result.height).to.equal(1012); + expect(result.blockers[0].reason).to.equal('imminent'); + expect(rpcClient.getBlockchainInfo).to.have.been.calledTwice(); + }); + + it('should confirm a safe result after the confirmation delay on Core older than 24.1', async () => { + const state = await startCheck(); + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS - 1); + + expect(state.result).to.be.undefined(); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + + await clock.tickAsync(1); + + expect(state.result.status).to.equal('safe'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + }); + + it('should confirm a safe result on a Core 24.0 patch release', async () => { + core.version = CORE_24_0_PATCH; + + const state = await startCheck(); + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('safe'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + }); + + it('should start the confirmation delay once the first check has finished', async () => { + // Every block the first check saw must be old enough for Core to have + // recorded its sessions. + const dkgStatusDelayMs = 10000; + core.dkgStatusDelayMs = dkgStatusDelayMs; + + await startCheck(); + await clock.tickAsync(dkgStatusDelayMs + SAFE_STOP_CONFIRMATION_DELAY_MS - 1); + + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + + await clock.tickAsync(1); + + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + }); + + it('should return the confirming check when it finds a session', async () => { + const state = await startCheck(); + + core.dkgInfo = { active_dkgs: 1, next_dkg: 12 }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, status: { quorumHeight: 1012 } }], + }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('blocked'); + expect(state.result.blockers.find(({ reason }) => reason === 'session')) + .to.include({ from: 1012, until: 1022 }); + }); + + it('should accept up to MIN_BLOCKS_BEFORE_DKG new blocks during the confirmation', async () => { + const state = await startCheck(); + core.height += MIN_BLOCKS_BEFORE_DKG; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('safe'); + }); + + it('should report unsettled when more blocks arrive during the confirmation than the first check covered', async () => { + // The first check only ruled out DKGs of this node within + // MIN_BLOCKS_BEFORE_DKG blocks of its tip. + const state = await startCheck(); + core.height += MIN_BLOCKS_BEFORE_DKG + 1; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('unsettled'); + }); + + it('should report unsettled when the confirming check finds another block at the same height', async () => { + // A reorg replaced the first check's tip, so that block may be new. + const state = await startCheck(); + core.hash = 'reorganized'; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('unsettled'); + }); + + it('should report unsettled when the confirming check finds a lower tip', async () => { + const state = await startCheck(); + core.height -= 1; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('unsettled'); + }); + + it('should not wait for confirmation on Core 24.1', async () => { + core.version = CORE_24_1; + + const { result } = await startCheck(); + + expect(result.status).to.equal('safe'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + }); + + it('should not hold Core in initial block download to sessions, without confirmation', async () => { + // Core runs no DKG session until it has synced, so session records and + // unrecorded sessions do not apply; announced DKGs still do. + core.isSyncing = true; + core.dkgInfo = { active_dkgs: 1, next_dkg: 12 }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, status: { quorumHeight: 1010 } }], + quorumConnections: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, pQuorumBaseBlockIndex: 1010 }], + }; + + const { result } = await startCheck(); + + expect(result).to.include({ status: 'safe', isSyncing: true }); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); }); - it('should retain the imminent-DKG guard without upcoming_dkgs', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 0, next_dkg: 1 } }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(false); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); + it('should report unsettled when a reorg replaced the first check\'s tip and the chain grew', async () => { + // The replacing branch can put this node in a DKG starting on it that + // Core has not recorded yet. + const state = await startCheck(); + const firstTip = core.height; + core.height += 2; + core.hashAt = (height) => `other-branch-${height}`; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('unsettled'); + expect(rpcClient.getBlockHash).to.have.been.calledWith(firstTip); }); - it('should allow a legacy response with no active or imminent DKG', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 0, next_dkg: 24 } }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(true); - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); + it('should treat Core in initial block download as safe even while blocks keep connecting', async () => { + core.isSyncing = true; + core.beforeDkgInfo = () => { core.height += 1; }; + + const { result } = await startCheck(); + + expect(result).to.include({ status: 'safe', isSyncing: true }); + }); + + it('should pass on what it knows about DKGs this node is not part of', async () => { + // A DKG starting at the tip blocks unless announced without this node. + core.dkgInfo = { active_dkgs: 0, next_dkg: 24 }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: 1012 }], + }; + + const blocked = await startCheck(); + expect(blocked.result.status).to.equal('blocked'); + + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version, { + knownNonMemberDkgs: new Set(['llmq_100_67:0:1012']), + }).then((result) => { state.result = result; }); + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('safe'); }); - it('should inspect active sessions for older Core', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 1, next_dkg: 24 } }); - rpcClient.quorum.withArgs('dkgstatus').resolves({ - result: { session: [{ llmqType: 'llmq_60_75', status: { quorumHeight: 1000 } }] }, + it('should report a replaced chain when the previous check\'s tip is gone', async () => { + // Anything learned on the replaced branch, such as which DKGs leave this + // node out, may not hold on the new one. + core.version = CORE_24_1; + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version, { + ancestor: { height: 1010, hash: 'replaced-1010' }, + }).then((result) => { state.result = result; }); + await clock.tickAsync(0); + + expect(state.result).to.include({ status: 'unsettled', isReorg: true }); + }); + + it('should not report a replaced chain when the previous check\'s tip is still there', async () => { + core.version = CORE_24_1; + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version, { + ancestor: { height: 1010, hash: 'hash-1010' }, + }).then((result) => { state.result = result; }); + await clock.tickAsync(0); + + expect(state.result.status).to.equal('safe'); + expect(state.result.isReorg).to.not.equal(true); + }); + + it('should report a replaced chain when the previous check\'s tip is replaced during the reads', async () => { + // A syncing Core's tip may move during the reads, so the previous tip is + // looked up on the chain the check reports. + core.isSyncing = true; + core.beforeDkgInfo = () => { + core.height += 1; + core.hashAt = (height) => `other-branch-${height}`; + core.beforeDkgInfo = undefined; + }; + const state = {}; + checkMasternodeSafeToStop(rpcClient, core.version, { + ancestor: { height: 1010, hash: 'hash-1010' }, + }).then((result) => { state.result = result; }); + await clock.tickAsync(0); + + expect(state.result).to.include({ status: 'unsettled', isReorg: true }); + }); + + it('should still block a DKG this node is in while Core syncs', async () => { + // A Core only a little behind leaves initial block download in time for + // the DKGs it announces. + core.isSyncing = true; + core.version = CORE_24_0_PATCH; + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 2, + upcoming_dkgs: [{ + llmqType: 106, quorumIndex: 0, quorumHeight: 1014, blocksUntilStart: 2, known: true, isMember: true, + }], + }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('blocked'); + expect(result.blockers.map(({ reason }) => reason)).to.deep.equal(['upcoming']); + }); + + it('should keep the first check\'s DKGs when Core restarts into initial block download during confirmation', async () => { + // The first check sees a DKG of this node 8 blocks away; the restarted + // Core no longer reports it. + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 8, + upcoming_dkgs: [{ + llmqType: 106, quorumIndex: 0, quorumHeight: 1020, blocksUntilStart: 8, known: true, isMember: true, + }], + }; + core.version = CORE_24_0_PATCH; + + const state = await startCheck(); + core.isSyncing = true; + core.height = 1015; + core.dkgInfo = { active_dkgs: 0, next_dkg: 5 }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + expect(state.result.status).to.equal('blocked'); + }); + + describe('Core syncing while a block connects during the reads', () => { + // Core counts next_dkg and blocksUntilStart from the tip dkginfo saw, + // which can be lower than the tip read after it. + const start = 1020; + + beforeEach(() => { + core.isSyncing = true; + core.height = start - 7; + core.afterDkgStatus = () => { + core.height += 1; + core.afterDkgStatus = undefined; + }; + }); + + it('should place an announced DKG at the start Core gives', async () => { + core.version = CORE_24_0_PATCH; + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 7, + upcoming_dkgs: [{ + llmqType: 106, + quorumIndex: 0, + quorumHeight: start, + blocksUntilStart: 7, + known: true, + isMember: true, + workBlockHeight: start - 8, + workBlockHash: `hash-${start - 8}`, + }], + }; + + const { result } = await startCheck(); + + expect(result.height).to.equal(start - 6); + expect(result.status).to.equal('blocked'); + }); + + it('should not refuse a Core that connects several blocks during the reads', async () => { + // A syncing Core connects only blocks older than its 6-hour maximum tip + // age, so it moves this fast only while catching up on DKGs the network + // ran hours ago, which Core skips. Its tip moves on until it stops + // whichever tip the check reads. + core.version = CORE_24_0_PATCH; + core.height = start - 9; + core.dkgInfo = { active_dkgs: 0, next_dkg: 15, upcoming_dkgs: [] }; + core.afterDkgStatus = () => { + core.height = start - 6; + core.afterDkgStatus = undefined; + }; + + const { result } = await startCheck(); + + expect(result).to.include({ status: 'safe', height: start - 6 }); + }); + + it('should report unsettled when the block Core worked out membership from is replaced', async () => { + // Core leaves this node out of the DKG at `start` on the branch it read; + // the branch that replaces it, at the same height, may not. + core.version = CORE_24_0_PATCH; + core.height = start - 1; + core.hashAt = (height) => (height >= start - 8 ? `branch-a-${height}` : `hash-${height}`); + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 106, + quorumIndex: 0, + quorumHeight: start, + blocksUntilStart: 1, + known: true, + isMember: false, + workBlockHeight: start - 8, + workBlockHash: `branch-a-${start - 8}`, + }], + }; + core.afterDkgStatus = () => { + core.hashAt = (height) => (height >= start - 8 ? `branch-b-${height}` : `hash-${height}`); + core.afterDkgStatus = undefined; + }; + + const { result } = await startCheck(); + + expect(result.status).to.equal('unsettled'); + }); + + it('should take Core\'s word on membership while the block it worked it out from stays on the chain', async () => { + core.version = CORE_24_0_PATCH; + core.dkgInfo = { + active_dkgs: 0, + next_dkg: 7, + upcoming_dkgs: [{ + llmqType: 106, + quorumIndex: 0, + quorumHeight: start, + blocksUntilStart: 7, + known: true, + isMember: false, + workBlockHeight: start - 8, + workBlockHash: `hash-${start - 8}`, + }], + }; + + const { result } = await startCheck(); + + expect(result).to.include({ status: 'safe', height: start - 6 }); + expect([...result.nonMemberDkgs]).to.deep.equal([`llmq_test_platform:0:${start}`]); + }); + + it('should count next_dkg from the earliest tip dkginfo may have seen', async () => { + core.dkgInfo = { active_dkgs: 0, next_dkg: 7 }; + + const { result } = await startCheck(); + + expect(result.height).to.equal(start - 6); + expect(result.status).to.equal('blocked'); }); - rpcClient.getBlockCount.resolves({ result: 1001 }); - expect(await checkMasternodeSafeToStop(rpcClient)).to.equal(false); - expect(rpcClient.quorum.withArgs('dkgstatus')).to.have.been.calledOnce(); - expect(rpcClient.getBlockCount).to.have.been.calledOnce(); }); }); diff --git a/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js new file mode 100644 index 00000000000..1886d11713a --- /dev/null +++ b/packages/dashmate/test/unit/core/quorum/getDkgStopBlockers.spec.js @@ -0,0 +1,624 @@ +import getDkgStopBlockers, { + getAnnouncedNonMemberDkgs, + getBlockersAt, + hasMembershipActiveDkgs, + LLMQ_TYPES, +} from '../../../../src/core/quorum/getDkgStopBlockers.js'; +import { MIN_BLOCKS_BEFORE_DKG } from '../../../../src/constants.js'; + +const CORE_23 = 230110; +const CORE_24_0 = 240000; +const CORE_24_1 = 240100; +const HEIGHT = 1000; + +// A value of next_dkg that is safely above MIN_BLOCKS_BEFORE_DKG so the +// imminent-DKG guard never trips in tests that focus on session records. +const NEXT_DKG_NOT_IMMINENT = MIN_BLOCKS_BEFORE_DKG + 5; +const PLATFORM_ACTIVE_BLOCKS = 10; // llmq_test_platform: 5 phases * 2 blocks +const PLATFORM_INTERVAL = 24; + +// Core lists the quorum connections in every `quorum dkgstatus` answer. +function withConnections(dkgStatus) { + return dkgStatus && typeof dkgStatus === 'object' + ? { quorumConnections: [], ...dkgStatus } + : dkgStatus; +} + +function blockersAt({ + dkgInfo, + dkgStatus = { session: [] }, + height = HEIGHT, + coreVersion = CORE_23, +}) { + return getBlockersAt(getDkgStopBlockers({ + dkgInfo, dkgStatus: withConnections(dkgStatus), height, coreVersion, + }), height); +} + +function record(llmqType, quorumHeight, quorumIndex = 0) { + return { llmqType, quorumIndex, status: { quorumHeight } }; +} + +// Shape of a `quorum dkginfo` upcoming_dkgs entry once membership is known. +function upcomingDkg(blocksUntilStart, isMember, llmqType = 106) { + return { + llmqType, + quorumIndex: 0, + quorumHeight: HEIGHT + blocksUntilStart, + blocksUntilStart, + known: true, + isMember, + workBlockHeight: HEIGHT + blocksUntilStart - 8, + workBlockHash: '00'.repeat(32), + }; +} + +function unknownUpcomingDkg(blocksUntilStart) { + return { + llmqType: 103, + quorumIndex: 0, + quorumHeight: HEIGHT + blocksUntilStart, + blocksUntilStart, + known: false, + reason: 'rotated quorum snapshots are not available yet', + }; +} + +describe('getDkgStopBlockers', () => { + describe('imminent DKG guard from dkginfo.next_dkg (no membership data)', () => { + it('should block when next_dkg <= MIN_BLOCKS_BEFORE_DKG even with no sessions', () => { + const blockers = blockersAt({ dkgInfo: { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG } }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'imminent', until: null }); + }); + + it('should allow the stop when next_dkg is past the restart margin', () => { + expect(blockersAt({ dkgInfo: { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG + 1 } })) + .to.be.empty(); + }); + + it('should report when the next DKG starts blocking', () => { + const [blocker] = getDkgStopBlockers({ + dkgInfo: { active_dkgs: 0, next_dkg: 20 }, + dkgStatus: withConnections({ session: [] }), + height: HEIGHT, + coreVersion: CORE_23, + }); + + expect(blocker).to.include({ + reason: 'imminent', from: HEIGHT + 20 - MIN_BLOCKS_BEFORE_DKG, until: null, + }); + }); + + it('should ignore active_dkgs on Core older than 24.1, which counts lingering records', () => { + expect(blockersAt({ dkgInfo: { active_dkgs: 3, next_dkg: NEXT_DKG_NOT_IMMINENT } })) + .to.be.empty(); + }); + }); + + describe('membership from dkginfo.upcoming_dkgs', () => { + it('should allow the stop when an imminent DKG does not include this node', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [upcomingDkg(1, false), upcomingDkg(MIN_BLOCKS_BEFORE_DKG, false)], + }; + + expect(blockersAt({ dkgInfo, coreVersion: CORE_24_0 })).to.be.empty(); + }); + + it('should allow the stop when next_dkg is imminent but no upcoming DKG is listed', () => { + // next_dkg reports 1 for the whole rotation signing window, even + // after the last rotated index has started. + const dkgInfo = { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] }; + + expect(blockersAt({ dkgInfo, coreVersion: CORE_24_0 })).to.be.empty(); + }); + + it('should block when this node is a member of an imminent DKG until its Commit phase ends', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [upcomingDkg(1, false), upcomingDkg(MIN_BLOCKS_BEFORE_DKG, true)], + }; + const start = HEIGHT + MIN_BLOCKS_BEFORE_DKG; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'upcoming', + llmqType: 'llmq_test_platform', + membershipKnown: true, + from: start - MIN_BLOCKS_BEFORE_DKG, + until: start + PLATFORM_ACTIVE_BLOCKS, + }); + }); + + it('should not block yet when this node is a member of a DKG past the restart margin', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: MIN_BLOCKS_BEFORE_DKG + 1, + upcoming_dkgs: [upcomingDkg(MIN_BLOCKS_BEFORE_DKG + 1, true)], + }; + + const blockers = getDkgStopBlockers({ + dkgInfo, dkgStatus: withConnections({ session: [] }), height: HEIGHT, coreVersion: CORE_24_0, + }); + + expect(getBlockersAt(blockers, HEIGHT)).to.be.empty(); + expect(getBlockersAt(blockers, HEIGHT + 1)).to.have.lengthOf(1); + }); + + it('should block when membership of an imminent DKG is unknown', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [unknownUpcomingDkg(1)], + }; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'upcoming', membershipKnown: false }); + }); + + it('should allow the stop when membership is unknown only for a DKG past the restart margin', () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: MIN_BLOCKS_BEFORE_DKG + 1, + upcoming_dkgs: [unknownUpcomingDkg(MIN_BLOCKS_BEFORE_DKG + 1)], + }; + + expect(blockersAt({ dkgInfo, coreVersion: CORE_24_0 })).to.be.empty(); + }); + + it('should not read a DKG of a type dashmate does not know that may include this node', () => { + // With no known Commit phase, nothing says when it stops blocking. + const dkgInfo = { + active_dkgs: 0, + next_dkg: 20, + upcoming_dkgs: [upcomingDkg(20, true, 250), upcomingDkg(3, false, 251)], + }; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'unreadable', source: 'quorum dkginfo upcoming_dkgs llmqType 250', from: null, + }); + }); + + describe('fail-safe on malformed upcoming_dkgs', () => { + const malformed = { + 'upcoming_dkgs is an object': {}, + 'upcoming_dkgs is null': null, + 'an entry is null': [null], + 'an entry is missing known': [{ blocksUntilStart: 1, isMember: false }], + 'a known entry is missing isMember': [{ blocksUntilStart: 1, known: true }], + 'an entry is missing quorumHeight': [{ blocksUntilStart: 1, known: true, isMember: false }], + 'an entry has a negative quorumHeight': [ + { + quorumHeight: -1, blocksUntilStart: 1, known: true, isMember: false, + }, + ], + 'an entry outside the margin is malformed': [ + { blocksUntilStart: MIN_BLOCKS_BEFORE_DKG + 1, known: true }, + ], + }; + + Object.entries(malformed).forEach(([description, upcomingDkgs]) => { + it(`should block when ${description}`, () => { + const dkgInfo = { + active_dkgs: 0, + next_dkg: NEXT_DKG_NOT_IMMINENT, + upcoming_dkgs: upcomingDkgs, + }; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_0 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', until: null }); + }); + }); + }); + }); + + describe('session records from dkgstatus', () => { + const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; + + it('should block a session at its start block', () => { + const blockers = blockersAt({ + dkgInfo, + dkgStatus: { session: [record('llmq_test_platform', HEIGHT)] }, + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'session', + llmqType: 'llmq_test_platform', + from: HEIGHT, + until: HEIGHT + PLATFORM_ACTIVE_BLOCKS, + }); + }); + + it('should block a session at the last block of its Commit phase', () => { + const dkgStatus = { session: [record('llmq_test_platform', HEIGHT - PLATFORM_ACTIVE_BLOCKS + 1)] }; + + expect(blockersAt({ dkgInfo, dkgStatus })).to.have.lengthOf(1); + }); + + it('should allow the stop once a session has finished its Commit phase', () => { + const dkgStatus = { session: [record('llmq_test_platform', HEIGHT - PLATFORM_ACTIVE_BLOCKS)] }; + + expect(blockersAt({ dkgInfo, dkgStatus })).to.be.empty(); + }); + + it('should block a 400-member session for its 20 blocks', () => { + expect(blockersAt({ dkgInfo, dkgStatus: { session: [record('llmq_400_60', HEIGHT - 19)] } })) + .to.have.lengthOf(1); + expect(blockersAt({ dkgInfo, dkgStatus: { session: [record('llmq_400_60', HEIGHT - 20)] } })) + .to.be.empty(); + }); + + it('should measure a rotated session from its own quorum index start', () => { + // llmq_60_75 index 20 starts at cycle base + 20, not at the cycle's + // shared mining window (cycle base + 42). + const quorumHeight = 288000 + 20; + + expect(blockersAt({ + dkgInfo, + height: quorumHeight + 9, + dkgStatus: { session: [record('llmq_60_75', quorumHeight, 20)] }, + })).to.have.lengthOf(1); + expect(blockersAt({ + dkgInfo, + height: quorumHeight + 10, + dkgStatus: { session: [record('llmq_60_75', quorumHeight, 20)] }, + })).to.be.empty(); + }); + + it('should allow the stop when every session has finished', () => { + const dkgStatus = { + session: [ + record('llmq_test_platform', HEIGHT - 20), + record('llmq_400_60', HEIGHT - 100), + ], + }; + + expect(blockersAt({ dkgInfo: { active_dkgs: 2, next_dkg: NEXT_DKG_NOT_IMMINENT }, dkgStatus })) + .to.be.empty(); + }); + + it('should block when any single session is still active, even if others have finished', () => { + const dkgStatus = { + session: [ + record('llmq_test_platform', HEIGHT - 50), + record('llmq_400_60', HEIGHT - 5), + ], + }; + + const blockers = blockersAt({ dkgInfo: { active_dkgs: 2, next_dkg: NEXT_DKG_NOT_IMMINENT }, dkgStatus }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ llmqType: 'llmq_400_60' }); + }); + + it('should not need session records when Core reports none', () => { + expect(blockersAt({ dkgInfo: { active_dkgs: 0, next_dkg: NEXT_DKG_NOT_IMMINENT } })) + .to.be.empty(); + }); + + describe('a record from the previous round at the next round', () => { + // Core replaces or removes the record shortly after the next round's + // start block connects; until then this node may be a member again. + const previousRound = HEIGHT - PLATFORM_INTERVAL; + + it('should block from the next round start block', () => { + const blockers = blockersAt({ + dkgInfo, + dkgStatus: { session: [record('llmq_test_platform', previousRound)] }, + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'new-round', from: HEIGHT, until: HEIGHT + PLATFORM_ACTIVE_BLOCKS, + }); + }); + + it('should not block just before the next round starts', () => { + expect(blockersAt({ + dkgInfo, + dkgStatus: { session: [record('llmq_test_platform', previousRound + 1)] }, + })).to.be.empty(); + }); + + it('should stop blocking once the next round has finished its Commit phase', () => { + expect(blockersAt({ + dkgInfo, + dkgStatus: { session: [record('llmq_test_platform', previousRound - PLATFORM_ACTIVE_BLOCKS)] }, + })).to.be.empty(); + }); + }); + }); + + describe('active_dkgs on Core 24.1+', () => { + it('should block while active_dkgs is positive even if no upcoming DKG blocks', () => { + const dkgInfo = { + active_dkgs: 1, + next_dkg: NEXT_DKG_NOT_IMMINENT, + upcoming_dkgs: [upcomingDkg(1, false)], + }; + + const blockers = blockersAt({ dkgInfo, coreVersion: CORE_24_1 }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'active-count', count: 1, until: null }); + }); + + it('should block a recorded session after Core stopped counting it', () => { + // A ProUpServTx mined mid-session clears Core's proTxHash, so it + // reports active_dkgs 0 and no upcoming_dkgs while the session runs. + const blockers = blockersAt({ + dkgInfo: { active_dkgs: 0, next_dkg: NEXT_DKG_NOT_IMMINENT }, + dkgStatus: { session: [record('llmq_test_platform', HEIGHT - 3)] }, + coreVersion: CORE_24_1, + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'session' }); + }); + + it('should block an unrecorded session even when a later member DKG is announced', () => { + // At a start block Core 24.1 already counts the session, before its + // DKG worker has recorded it; a DKG announced for later does not + // account for it. + const blockers = blockersAt({ + dkgInfo: { + active_dkgs: 1, + next_dkg: NEXT_DKG_NOT_IMMINENT, + upcoming_dkgs: [upcomingDkg(10, true)], + }, + coreVersion: CORE_24_1, + }); + + expect(blockers.map(({ reason }) => reason)).to.deep.equal(['active-count']); + }); + + it('should use Core\'s count from Core 24.1.0 only', () => { + expect(hasMembershipActiveDkgs(240000)).to.be.false(); + expect(hasMembershipActiveDkgs(240099)).to.be.false(); + expect(hasMembershipActiveDkgs(240100)).to.be.true(); + expect(hasMembershipActiveDkgs(250000)).to.be.true(); + expect(hasMembershipActiveDkgs('240100')).to.be.false(); + expect(hasMembershipActiveDkgs(null)).to.be.false(); + }); + }); + + describe('fail-safe on malformed dkginfo', () => { + const malformed = { + 'dkgInfo is undefined': undefined, + 'dkgInfo is null': null, + 'next_dkg is missing': { active_dkgs: 0 }, + 'next_dkg is non-numeric': { active_dkgs: 0, next_dkg: '24' }, + 'next_dkg is NaN': { active_dkgs: 0, next_dkg: NaN }, + 'next_dkg is negative': { active_dkgs: 0, next_dkg: -1 }, + 'active_dkgs is missing': { next_dkg: NEXT_DKG_NOT_IMMINENT }, + 'active_dkgs is non-numeric': { active_dkgs: '1', next_dkg: NEXT_DKG_NOT_IMMINENT }, + 'active_dkgs is NaN': { active_dkgs: NaN, next_dkg: NEXT_DKG_NOT_IMMINENT }, + 'active_dkgs is negative': { active_dkgs: -1, next_dkg: NEXT_DKG_NOT_IMMINENT }, + }; + + Object.entries(malformed).forEach(([description, dkgInfo]) => { + it(`should block when ${description}`, () => { + const blockers = blockersAt({ dkgInfo }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', source: 'quorum dkginfo' }); + }); + }); + + it('should block when the block height is not a height', () => { + const blockers = getDkgStopBlockers({ + dkgInfo: { active_dkgs: 0, next_dkg: NEXT_DKG_NOT_IMMINENT }, + dkgStatus: { session: [] }, + height: undefined, + coreVersion: CORE_23, + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', from: null, until: null }); + }); + }); + + describe('fail-safe on malformed dkgstatus', () => { + const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; + const malformed = { + 'a session has an unknown llmqType': { session: [record('llmq_future_unknown', HEIGHT)] }, + 'a session is missing quorumHeight': { session: [{ llmqType: 'llmq_test_platform', status: {} }] }, + 'a session has a NaN quorumHeight': { session: [record('llmq_test_platform', NaN)] }, + 'a session is missing status entirely': { session: [{ llmqType: 'llmq_test_platform' }] }, + 'a session starts above the block height': { session: [record('llmq_test_platform', HEIGHT + 5)] }, + 'a session entry is null': { session: [null] }, + 'dkgStatus is omitted': undefined, + 'dkgStatus.session is not an array': {}, + }; + + Object.entries(malformed).forEach(([description, dkgStatus]) => { + it(`should block when ${description}`, () => { + const blockers = getBlockersAt(getDkgStopBlockers({ + dkgInfo, dkgStatus: withConnections(dkgStatus), height: HEIGHT, coreVersion: CORE_23, + }), HEIGHT); + + expect(blockers).to.not.be.empty(); + blockers.forEach((blocker) => { + expect(blocker).to.include({ reason: 'unreadable', until: null }); + }); + }); + }); + }); + + describe('LLMQ table vs Dash Core src/llmq/params.h', () => { + it('should match every LLMQ type Core defines', () => { + expect(LLMQ_TYPES.map(({ + id, name, dkgInterval, dkgPhaseBlocks, + }) => [id, name, dkgInterval, dkgPhaseBlocks])).to.deep.equal([ + [1, 'llmq_50_60', 24, 2], + [2, 'llmq_400_60', 288, 4], + [3, 'llmq_400_85', 576, 4], + [4, 'llmq_100_67', 24, 2], + [5, 'llmq_60_75', 288, 2], + [6, 'llmq_25_67', 24, 2], + [100, 'llmq_test', 24, 2], + [101, 'llmq_devnet', 24, 2], + [102, 'llmq_test_v17', 24, 2], + [103, 'llmq_test_dip0024', 24, 2], + [104, 'llmq_test_instantsend', 24, 2], + [105, 'llmq_devnet_dip0024', 48, 2], + [106, 'llmq_test_platform', 24, 2], + [107, 'llmq_devnet_platform', 24, 2], + ]); + }); + + // [type, DKG interval, blocks from a session's start to the end of its + // Commit phase, quorum index], as Dash Core schedules them. + [ + ['llmq_60_75', 288, 10, 20], + ['llmq_400_60', 288, 20, 0], + ['llmq_400_85', 576, 20, 0], + ['llmq_100_67', 24, 10, 0], + ['llmq_50_60', 24, 10, 0], + ['llmq_25_67', 24, 10, 0], + ].forEach(([llmqType, interval, activeBlocks, quorumIndex]) => { + it(`should block a ${llmqType} member for ${activeBlocks} blocks of each session, every ${interval} blocks`, () => { + const quorumHeight = 288 * 24 * 10 + quorumIndex; + const isBlockedAtOffset = (offset) => blockersAt({ + dkgInfo: { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }, + height: quorumHeight + offset, + dkgStatus: { session: [record(llmqType, quorumHeight, quorumIndex)] }, + }).length > 0; + + expect(isBlockedAtOffset(activeBlocks - 1), 'last block of the session').to.be.true(); + expect(isBlockedAtOffset(activeBlocks), 'after the Commit phase').to.be.false(); + expect(isBlockedAtOffset(interval - 1), 'before the next session').to.be.false(); + expect(isBlockedAtOffset(interval), 'next session start').to.be.true(); + expect(isBlockedAtOffset(interval + activeBlocks - 1), 'next session end').to.be.true(); + expect(isBlockedAtOffset(interval + activeBlocks), 'after the next session').to.be.false(); + }); + }); + }); + + describe('a session Core has not recorded (Core older than 24.1)', () => { + // `quorum dkgstatus` lists every enabled quorum with the start of its + // current session. Core records a session only once it has computed its + // members, which has no time bound, and until then nothing says whether + // this node is one of them. + const start = HEIGHT; + const current = { + llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start, + }; + const dkgInfo = { active_dkgs: 0, next_dkg: 24 }; + + function blockersWith(dkgStatus, options = {}) { + const height = options.height ?? start; + + return getBlockersAt(getDkgStopBlockers({ + dkgInfo, dkgStatus, coreVersion: CORE_23, ...options, height, + }), height); + } + + it('should block the session from its start until its Commit phase ends', () => { + const blockers = blockersWith({ session: [], quorumConnections: [current] }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'unrecorded', llmqType: 'llmq_100_67', quorumIndex: 0, from: start, until: start + 10, + }); + }); + + it('should keep blocking after the start block while Core still has no record', () => { + expect(blockersWith({ session: [], quorumConnections: [current] }, { height: start + 1 })) + .to.have.lengthOf(1); + expect(blockersWith({ session: [], quorumConnections: [current] }, { height: start + 9 })) + .to.have.lengthOf(1); + expect(blockersWith({ session: [], quorumConnections: [current] }, { height: start + 10 })) + .to.be.empty(); + }); + + it('should block a 400-member session for its 20 blocks', () => { + const large = { llmqType: 'llmq_400_60', quorumIndex: 0, pQuorumBaseBlockIndex: start }; + + expect(blockersWith({ session: [], quorumConnections: [large] }, { height: start + 19 })) + .to.have.lengthOf(1); + expect(blockersWith({ session: [], quorumConnections: [large] }, { height: start + 20 })) + .to.be.empty(); + }); + + it('should leave the session to its record once Core has one', () => { + const blockers = blockersWith({ + session: [record('llmq_100_67', start)], + quorumConnections: [current], + }, { height: start + 2 }); + + expect(blockers.map(({ reason }) => reason)).to.deep.equal(['session']); + }); + + it('should not block a session that has not started', () => { + expect(blockersWith({ + session: [], + quorumConnections: [{ llmqType: 'llmq_60_75', quorumIndex: 5 }], + })).to.be.empty(); + }); + + it('should not block a DKG this node was announced to be no member of', () => { + expect(blockersWith( + { session: [], quorumConnections: [current] }, + { height: start + 3, knownNonMemberDkgs: new Set([`llmq_100_67:0:${start}`]) }, + )).to.be.empty(); + }); + + it('should rely on Core\'s count on Core 24.1, which includes such a session', () => { + expect(blockersWith( + { session: [], quorumConnections: [current] }, + { coreVersion: CORE_24_1 }, + )).to.be.empty(); + }); + + it('should block when quorum connections are missing', () => { + const blockers = blockersWith({ session: [] }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ reason: 'unreadable', source: 'quorum dkgstatus quorumConnections' }); + }); + + it('should block on a quorum type dashmate does not know', () => { + const blockers = blockersWith({ + session: [], + quorumConnections: [{ llmqType: 'llmq_future', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }); + + expect(blockers).to.have.lengthOf(1); + expect(blockers[0]).to.include({ + reason: 'unreadable', source: 'quorum dkgstatus quorumConnections llmq_future', + }); + }); + }); + + describe('getAnnouncedNonMemberDkgs', () => { + it('should name the announced DKGs this node is known to be no member of', () => { + expect([...getAnnouncedNonMemberDkgs({ + upcoming_dkgs: [ + upcomingDkg(2, false, 4), + upcomingDkg(3, true, 4), + unknownUpcomingDkg(4), + upcomingDkg(5, false, 250), + ], + }, HEIGHT)]).to.deep.equal([`llmq_100_67:0:${HEIGHT + 2}`]); + }); + + it('should name none without membership data', () => { + expect([...getAnnouncedNonMemberDkgs({ active_dkgs: 0, next_dkg: 3 }, HEIGHT)]).to.be.empty(); + }); + }); +}); diff --git a/packages/dashmate/test/unit/core/quorum/isMasternodeSafeToStopDuringDkg.spec.js b/packages/dashmate/test/unit/core/quorum/isMasternodeSafeToStopDuringDkg.spec.js deleted file mode 100644 index 1088c994022..00000000000 --- a/packages/dashmate/test/unit/core/quorum/isMasternodeSafeToStopDuringDkg.spec.js +++ /dev/null @@ -1,427 +0,0 @@ -import isMasternodeSafeToStopDuringDkg, { - DKG_MINING_WINDOW_START_BY_LLMQ_TYPE, - shouldInspectDkgStatusForSafeStop, -} from '../../../../src/core/quorum/isMasternodeSafeToStopDuringDkg.js'; -import { MIN_BLOCKS_BEFORE_DKG } from '../../../../src/constants.js'; - -// A value of next_dkg that is safely above MIN_BLOCKS_BEFORE_DKG so the -// imminent-DKG guard never trips in tests that focus on the per-session -// active-window logic. -const NEXT_DKG_NOT_IMMINENT = MIN_BLOCKS_BEFORE_DKG + 5; -const PLATFORM_WINDOW = DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_test_platform; // 10 -const LARGE_QUORUM_WINDOW = DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_400_60; // 20 - -describe('isMasternodeSafeToStopDuringDkg', () => { - describe('imminent DKG guard from dkginfo.next_dkg (Dash Core < v24)', () => { - it('blocks when next_dkg <= MIN_BLOCKS_BEFORE_DKG even when active_dkgs is 0', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('allows the stop when active_dkgs is 0 and next_dkg is past the imminent threshold', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: MIN_BLOCKS_BEFORE_DKG + 1 }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(true); - }); - - it('inspects dkgstatus only when active_dkgs > 0 and no DKG is imminent', () => { - expect(shouldInspectDkgStatusForSafeStop({ active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT })) - .to.equal(true); - expect(shouldInspectDkgStatusForSafeStop({ active_dkgs: 1, next_dkg: MIN_BLOCKS_BEFORE_DKG })) - .to.equal(false); - expect(shouldInspectDkgStatusForSafeStop({ active_dkgs: 0, next_dkg: NEXT_DKG_NOT_IMMINENT })) - .to.equal(false); - }); - }); - - describe('membership from dkginfo.upcoming_dkgs (Core with a proTxHash)', () => { - // Shape of a `quorum dkginfo` upcoming_dkgs entry once membership is known. - function upcomingDkg(blocksUntilStart, isMember) { - return { - llmqType: 106, - quorumIndex: 0, - quorumHeight: 1000 + blocksUntilStart, - blocksUntilStart, - known: true, - isMember, - workBlockHeight: 1000 + blocksUntilStart - 8, - workBlockHash: '00'.repeat(32), - }; - } - - function unknownUpcomingDkg(blocksUntilStart) { - return { - llmqType: 103, - quorumIndex: 0, - quorumHeight: 1000 + blocksUntilStart, - blocksUntilStart, - known: false, - reason: 'rotated quorum snapshots are not available yet', - }; - } - - it('should allow the stop when an imminent DKG does not include this node', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [upcomingDkg(1, false), upcomingDkg(MIN_BLOCKS_BEFORE_DKG, false)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(true); - }); - - it('should allow the stop when next_dkg is imminent but no upcoming DKG is listed', () => { - // next_dkg reports 1 for the whole rotation signing window, even - // after the last rotated index has started. - const dkgInfo = { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(true); - }); - - it('should block when active_dkgs is positive even if no upcoming DKG blocks', () => { - const dkgInfo = { - active_dkgs: 1, - next_dkg: NEXT_DKG_NOT_IMMINENT, - upcoming_dkgs: [upcomingDkg(1, false)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(false); - }); - - it('should not use dkgstatus to override a positive active_dkgs', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT, upcoming_dkgs: [] }; - // Every session is past its window, which would allow the legacy path. - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(shouldInspectDkgStatusForSafeStop(dkgInfo)).to.equal(false); - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000 + PLATFORM_WINDOW)) - .to.equal(false); - }); - - it('should block when this node is a member of an imminent DKG', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [upcomingDkg(1, false), upcomingDkg(MIN_BLOCKS_BEFORE_DKG, true)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(false); - }); - - it('should allow the stop when this node is a member of a DKG that is not yet imminent', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: MIN_BLOCKS_BEFORE_DKG + 1, - upcoming_dkgs: [upcomingDkg(MIN_BLOCKS_BEFORE_DKG + 1, true)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(true); - }); - - it('should block when membership of an imminent DKG is unknown', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [unknownUpcomingDkg(1)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(false); - }); - - it('should allow the stop when membership is unknown only for a DKG that is not yet imminent', () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: MIN_BLOCKS_BEFORE_DKG + 1, - upcoming_dkgs: [unknownUpcomingDkg(MIN_BLOCKS_BEFORE_DKG + 1)], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(true); - }); - - it('should not inspect dkgstatus when upcoming_dkgs is present', () => { - const dkgInfo = { - active_dkgs: 1, - next_dkg: NEXT_DKG_NOT_IMMINENT, - upcoming_dkgs: [upcomingDkg(1, true)], - }; - - expect(shouldInspectDkgStatusForSafeStop(dkgInfo)).to.equal(false); - }); - - describe('fail-safe on malformed upcoming_dkgs', () => { - const malformed = { - 'upcoming_dkgs is an object': {}, - 'upcoming_dkgs is null': null, - 'an entry is null': [null], - 'an entry is missing known': [{ blocksUntilStart: 1, isMember: false }], - 'a known entry is missing isMember': [{ blocksUntilStart: 1, known: true }], - 'an entry is missing blocksUntilStart': [{ known: true, isMember: false }], - 'an entry has a negative blocksUntilStart': [ - { blocksUntilStart: -1, known: true, isMember: false }, - ], - 'an entry outside the margin is malformed': [ - { blocksUntilStart: MIN_BLOCKS_BEFORE_DKG + 1, known: true }, - ], - }; - - Object.entries(malformed).forEach(([description, upcomingDkgs]) => { - it(`should block when ${description}`, () => { - const dkgInfo = { - active_dkgs: 0, - next_dkg: NEXT_DKG_NOT_IMMINENT, - upcoming_dkgs: upcomingDkgs, - }; - - expect(shouldInspectDkgStatusForSafeStop(dkgInfo)).to.equal(false); - expect(isMasternodeSafeToStopDuringDkg(dkgInfo)).to.equal(false); - }); - }); - }); - }); - - describe('per-session active window from dkgstatus + getblockcount', () => { - it('blocks a platform session at offset 0 (quorumHeight == currentHeight)', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks a platform session at the last block of its active window (offset == window - 1)', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg( - dkgInfo, - dkgStatus, - 1000 + (PLATFORM_WINDOW - 1), - )).to.equal(false); - }); - - it('allows the stop once a platform session reaches offset == window (window closed) even with active_dkgs > 0', () => { - // The regression case this PR cares about: aggregate active_dkgs - // is still > 0 (a non-platform LLMQ in its window elsewhere on - // the node, say), but the platform session at offset 10 is past - // its active phase and should not block. - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000 + PLATFORM_WINDOW)) - .to.equal(true); - }); - - it('blocks a non-platform long-window session (llmq_400_60, offset 15) that is still in its 20-block window', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_400_60', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1015)).to.equal(false); - // Sanity: confirm 15 is inside the 20-block llmq_400_60 window. - expect(LARGE_QUORUM_WINDOW).to.be.greaterThan(15); - }); - - it('allows the stop when every session is past its window even with active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 2, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - { llmqType: 'llmq_400_60', status: { quorumHeight: 900 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1020)).to.equal(true); - }); - - it('blocks when any single session is still in its active window, even if others are stale', () => { - const dkgInfo = { active_dkgs: 2, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - // Stale platform session (offset 50). - { llmqType: 'llmq_test_platform', status: { quorumHeight: 950 } }, - // Active llmq_400_60 session (offset 5, window 20). - { llmqType: 'llmq_400_60', status: { quorumHeight: 995 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks a negative offset (quorumHeight ahead of currentHeight) as inconsistent status', () => { - // Core should only report locally tracked sessions whose quorum - // height has already arrived. If the chain tip is behind the - // reported quorum height while active_dkgs > 0, fail safe. - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1005 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - }); - - describe('fail-safe on malformed dkgInfo', () => { - it('blocks when dkgInfo is undefined', () => { - expect(isMasternodeSafeToStopDuringDkg(undefined, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when dkgInfo is null', () => { - expect(isMasternodeSafeToStopDuringDkg(null, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when next_dkg is missing', () => { - const dkgInfo = { active_dkgs: 0 }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when next_dkg is non-numeric', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: '24' }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when next_dkg is NaN', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: NaN }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when next_dkg is negative', () => { - const dkgInfo = { active_dkgs: 0, next_dkg: -1 }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when active_dkgs is missing', () => { - const dkgInfo = { next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when active_dkgs is non-numeric', () => { - const dkgInfo = { active_dkgs: '1', next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when active_dkgs is NaN', () => { - const dkgInfo = { active_dkgs: NaN, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - - it('blocks when active_dkgs is negative', () => { - const dkgInfo = { active_dkgs: -1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)) - .to.equal(false); - }); - }); - - describe('fail-safe on malformed inputs while active_dkgs > 0', () => { - it('blocks when a session has an unknown llmqType', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_future_unknown', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks when a session is missing quorumHeight', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: {} }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks when a session is missing status entirely', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform' }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus, 1000)).to.equal(false); - }); - - it('blocks when dkgStatus is omitted while active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, undefined, 1000)).to.equal(false); - }); - - it('blocks when dkgStatus.session is not an array while active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, {}, 1000)).to.equal(false); - }); - - it('blocks when dkgStatus.session is empty while active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, { session: [] }, 1000)).to.equal(false); - }); - - it('blocks when currentHeight is omitted while active_dkgs > 0', () => { - const dkgInfo = { active_dkgs: 1, next_dkg: NEXT_DKG_NOT_IMMINENT }; - const dkgStatus = { - session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, - ], - }; - - expect(isMasternodeSafeToStopDuringDkg(dkgInfo, dkgStatus)).to.equal(false); - }); - }); - - describe('window table sanity vs Dash Core src/llmq/params.h', () => { - it('exposes the expected dkgMiningWindowStart values', () => { - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_test_platform).to.equal(10); - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_test_dip0024).to.equal(12); - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_400_60).to.equal(20); - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_400_85).to.equal(20); - expect(DKG_MINING_WINDOW_START_BY_LLMQ_TYPE.llmq_60_75).to.equal(42); - }); - }); -}); diff --git a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js index 872d5db2a49..d82c7dcd451 100644 --- a/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js +++ b/packages/dashmate/test/unit/core/quorum/waitForDKGWindowPass.spec.js @@ -1,170 +1,771 @@ -import waitForDKGWindowPass from '../../../../src/core/quorum/waitForDKGWindowPass.js'; - -const CHECK_INTERVAL_MS = 10000; +import waitForDKGWindowPass, { + CHECK_INTERVAL_MS, +} from '../../../../src/core/quorum/waitForDKGWindowPass.js'; +import { SAFE_STOP_CONFIRMATION_DELAY_MS } from '../../../../src/core/quorum/checkMasternodeSafeToStop.js'; +import getCoreRpcMock from '../../../../src/test/mock/getCoreRpcMock.js'; + +const CORE_23 = 230110; +const CORE_24_0 = 240000; +const CORE_24_1 = 240100; + +// Long enough for at least one more poll, confirmation included. Ticks of +// this length land between polls, so state a test changes afterwards is +// read by the next poll. +const NEXT_POLL_MS = CHECK_INTERVAL_MS + SAFE_STOP_CONFIRMATION_DELAY_MS + 1; + +function track(promise) { + const state = { resolved: false, error: undefined }; + promise.then(() => { state.resolved = true; }, (e) => { state.error = e; }); + return state; +} describe('waitForDKGWindowPass', () => { - let rpcClient; + let clock; beforeEach(function beforeEach() { - rpcClient = { - quorum: this.sinon.stub(), - getBlockCount: this.sinon.stub(), - }; + clock = this.sinon.useFakeTimers(); }); - it('resolves immediately when there are no active sessions and no imminent cycle', async () => { - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 0, next_dkg: 24 } }); - - await waitForDKGWindowPass(rpcClient); + async function expectStillPolling(waiting, rpcClient) { + const polls = rpcClient.quorum.withArgs('dkginfo').callCount; - expect(rpcClient.quorum).to.have.been.calledOnceWith('dkginfo'); - expect(rpcClient.getBlockCount).to.not.have.been.called(); - }); + await clock.tickAsync(NEXT_POLL_MS); - it('should resolve immediately when current and imminent DKGs do not include this node', async function it() { - const clock = this.sinon.useFakeTimers(); + expect(waiting.error).to.be.undefined(); + expect(waiting.resolved).to.be.false(); + expect(rpcClient.quorum.withArgs('dkginfo').callCount).to.be.greaterThan(polls); + } - rpcClient.quorum.withArgs('dkginfo').resolves({ - result: { + it('should resolve at once on Core 24.1 when no DKG includes this node', async function it() { + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_24_1, + height: 1000, + dkgInfo: { active_dkgs: 0, next_dkg: 1, - upcoming_dkgs: [{ blocksUntilStart: 1, known: true, isMember: false }], + upcoming_dkgs: [{ + llmqType: 106, quorumHeight: 1001, blocksUntilStart: 1, known: true, isMember: false, + }], }, + dkgStatus: { session: [] }, }); - const promise = waitForDKGWindowPass(rpcClient); - + const waiting = track(waitForDKGWindowPass(rpcClient)); await clock.tickAsync(0); - await promise; + expect(waiting.resolved).to.be.true(); expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); - expect(rpcClient.getBlockCount).to.not.have.been.called(); }); - it('should keep waiting while this node is a member of an imminent DKG', async function it() { - const clock = this.sinon.useFakeTimers(); - - rpcClient.quorum.withArgs('dkginfo') - .onFirstCall() - .resolves({ - result: { - active_dkgs: 0, - next_dkg: 1, - upcoming_dkgs: [{ blocksUntilStart: 1, known: true, isMember: true }], - }, - }) - .onSecondCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] } }); - - const promise = waitForDKGWindowPass(rpcClient); + it('should resolve once a safe result is confirmed on Core 24.0', async function it() { + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_24_0, + height: 1000, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 106, quorumHeight: 1001, blocksUntilStart: 1, known: true, isMember: false, + }], + }, + dkgStatus: { session: [] }, + }); - await clock.tickAsync(0); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS - 1); - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + expect(waiting.resolved).to.be.false(); + + await clock.tickAsync(1); + expect(waiting.resolved).to.be.true(); expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); }); - it('should wait while active_dkgs is positive without inspecting dkgstatus', async function it() { - const clock = this.sinon.useFakeTimers(); - rpcClient.quorum.withArgs('dkginfo') - .onFirstCall() - .resolves({ - result: { - active_dkgs: 1, - next_dkg: 1, - upcoming_dkgs: [], - }, - }) - .onSecondCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] } }); - - const promise = waitForDKGWindowPass(rpcClient); - await clock.tickAsync(0); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + it('should keep waiting while this node is a member of an imminent DKG', async function it() { + const core = { + version: CORE_24_1, + height: 1000, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 106, quorumHeight: 1001, blocksUntilStart: 1, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); - expect(rpcClient.quorum.withArgs('dkgstatus')).to.not.have.been.called(); - expect(rpcClient.getBlockCount).to.not.have.been.called(); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + // The session started at 1001 and its Commit phase has ended. + core.height = 1011; + core.dkgInfo = { active_dkgs: 0, next_dkg: 13, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, status: { quorumHeight: 1001 } }], + }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting on Core 24.1 while active_dkgs is positive', async function it() { + const core = { + version: CORE_24_1, + height: 1000, + dkgInfo: { active_dkgs: 1, next_dkg: 1, upcoming_dkgs: [] }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + core.dkgInfo = { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should wait through an active session and resolve once its Commit phase has ended', async function it() { + const core = { + version: CORE_23, + height: 1005, + dkgInfo: { active_dkgs: 1, next_dkg: 19 }, + dkgStatus: { + session: [{ llmqType: 'llmq_test_platform', quorumIndex: 0, status: { quorumHeight: 1000 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + core.height = 1010; + core.dkgInfo = { active_dkgs: 1, next_dkg: 14 }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting while a DKG is imminent and Core reports no membership', async function it() { + const core = { + version: CORE_23, + height: 1000, + dkgInfo: { active_dkgs: 0, next_dkg: 3 }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + // The DKG passed without including this node. + core.height = 1013; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14 }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting while Core reports a session dashmate cannot read', async function it() { + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_23, + height: 1000, + dkgInfo: { active_dkgs: 1, next_dkg: 20 }, + dkgStatus: { + session: [{ llmqType: 'llmq_future_unknown', quorumIndex: 0, status: { quorumHeight: 900 } }], + }, + }); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(CHECK_INTERVAL_MS * 10); + + await expectStillPolling(waiting, rpcClient); }); - it('waits through an active platform session and resolves once the window has passed', async function it() { - // Block height advances from 1005 (offset 5, in window) to 1010 - // (offset 10, window closed) between polls; active_dkgs stays > 0 - // to model Dash Core's lingering aggregate counter. - const clock = this.sinon.useFakeTimers(); + it('should stop waiting on Core 24.0 when active_dkgs counts only finished sessions', async function it() { + // Core 24.0 reports upcoming_dkgs but counts lingering debug records in + // active_dkgs, so a masternode that was in any recent DKG never sees 0. + const height = 288000 + 12; - rpcClient.quorum.withArgs('dkginfo').resolves({ result: { active_dkgs: 1, next_dkg: 20 } }); - rpcClient.quorum.withArgs('dkgstatus').resolves({ - result: { + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_24_0, + height, + dkgInfo: { + active_dkgs: 2, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 5, + quorumIndex: 20, + quorumHeight: 288000 + 20, + blocksUntilStart: 8, + known: true, + isMember: false, + }, { + llmqType: 5, + quorumIndex: 14, + quorumHeight: 288000 + 14, + blocksUntilStart: 2, + known: true, + isMember: false, + }], + }, + // This cycle's index 25 has not started, so last cycle's record stays; + // this hour's llmq_100_67 session ended two blocks ago. + dkgStatus: { session: [ - { llmqType: 'llmq_test_platform', status: { quorumHeight: 1000 } }, + { llmqType: 'llmq_60_75', quorumIndex: 25, status: { quorumHeight: 288000 - 288 + 25 } }, + { llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: 288000 } }, ], }, }); - rpcClient.getBlockCount - .onFirstCall().resolves({ result: 1005 }) - .onSecondCall().resolves({ result: 1010 }); - const promise = waitForDKGWindowPass(rpcClient); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); - // Drain microtasks so the first iteration completes its three RPCs - // and parks on `wait(CHECK_INTERVAL_MS)`. - await clock.tickAsync(0); - expect(rpcClient.getBlockCount).to.have.been.calledOnce(); + expect(waiting.resolved).to.be.true(); + }); - // Advance past the wait so the second iteration runs and returns. - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + it('should keep waiting at a rotation cycle start on Core 24.0 until this node\'s index has passed', async function it() { + // Last cycle this node was in llmq_60_75 index 5. The new cycle's index 5 + // has started and Core has not replaced that record yet; Core announces + // only the indices still to come, none of which include this node. + const cycleBase = 288000; + const core = { + version: CORE_24_0, + height: cycleBase + 5, + dkgInfo: { + active_dkgs: 1, + next_dkg: 1, + upcoming_dkgs: [6, 7, 8].map((quorumIndex) => ({ + llmqType: 5, + quorumIndex, + quorumHeight: cycleBase + quorumIndex, + blocksUntilStart: quorumIndex - 5, + known: true, + isMember: false, + })), + }, + dkgStatus: { + session: [{ llmqType: 'llmq_60_75', quorumIndex: 5, status: { quorumHeight: cycleBase - 288 + 5 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await expectStillPolling(waiting, rpcClient); + + // Ten blocks later Core has removed the record: this node is not in the + // new index 5. + core.height = cycleBase + 15; + core.dkgInfo = { active_dkgs: 0, next_dkg: 1, upcoming_dkgs: [] }; + core.dkgStatus = { session: [] }; + await clock.tickAsync(NEXT_POLL_MS); - expect(rpcClient.getBlockCount).to.have.been.calledTwice(); + expect(waiting.resolved).to.be.true(); }); - it('keeps waiting while next_dkg is imminent even when there are no sessions', async function it() { - const clock = this.sinon.useFakeTimers(); + it('should keep waiting through an announced DKG while Core has not recorded its session', async function it() { + // At the start block Core erases last round's record, computes the new + // session's members and only then records it. On a busy node that can + // outlast the confirmation delay. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 1, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 1, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.height = start; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + await expectStillPolling(waiting, rpcClient); + await expectStillPolling(waiting, rpcClient); - rpcClient.quorum.withArgs('dkginfo') - .onFirstCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 3 } }) - .onSecondCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 24 } }); + // The session has passed its Commit phase. + core.height = start + 10; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14, upcoming_dkgs: [] }; + await clock.tickAsync(NEXT_POLL_MS); - const promise = waitForDKGWindowPass(rpcClient); + expect(waiting.resolved).to.be.true(); + }); + it('should end a rotated DKG session once its own Commit phase is over', async function it() { + // llmq_60_75 index 31 starts at cycle base + 31; ten blocks later this + // member has sent everything its DKG needs, although the cycle's shared + // mining window only opens at cycle base + 42. + const quorumHeight = 288000 + 31; + + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_23, + height: quorumHeight + 10, + dkgInfo: { active_dkgs: 1, next_dkg: 7 }, + dkgStatus: { + session: [{ llmqType: 'llmq_60_75', quorumIndex: 31, status: { quorumHeight } }], + }, + }); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep waiting at a DKG start block while Core still shows the previous round', async function it() { + // The new llmq_100_67 round starts at `height`. Core has not replaced + // last round's record yet, so it looks 24 blocks old, past its window. + const height = 24 * 5000; + const core = { + version: CORE_23, + height, + dkgInfo: { active_dkgs: 1, next_dkg: 24 }, + dkgStatus: { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: height - 24 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + await expectStillPolling(waiting, rpcClient); + + // Core records the new session; it ends ten blocks after it started. + core.dkgStatus = { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: height } }], + }; + core.height = height + 10; + core.dkgInfo = { active_dkgs: 1, next_dkg: 14 }; + await clock.tickAsync(60000); + + expect(waiting.resolved).to.be.true(); + }); + + it('should not stop at a DKG start that Core has not recorded yet', async function it() { + // At the start block Core answers before its DKG worker has created the + // session record; a moment later the record appears. + const height = 24 * 5000; + const core = { + version: CORE_23, + height, + dkgInfo: { active_dkgs: 0, next_dkg: 24 }, + dkgStatus: { session: [] }, + }; + core.afterDkgStatus = () => { + core.dkgInfo = { active_dkgs: 1, next_dkg: 24 }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: height } }], + }; + core.afterDkgStatus = undefined; + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + + await expectStillPolling(waiting, rpcClient); + }); + + it('should not trust DKG reads that saw a newer tip than the height read', async function it() { + // The first check is safe: the member DKG is still 7 blocks away. Before + // the confirming check reads Core's DKG state, the chain reaches that + // DKG's start block, which Core has not recorded yet. The confirming check + // also sees more new blocks than the first covered; checkMasternodeSafeToStop + // tests the torn read on its own. + const height = 24 * 5000 - 7; + const memberDkgStart = height + 7; + const core = { + version: CORE_24_0, + height, + dkgInfo: { + active_dkgs: 0, + next_dkg: 7, + upcoming_dkgs: [{ + llmqType: 4, + quorumIndex: 0, + quorumHeight: memberDkgStart, + blocksUntilStart: 7, + known: true, + isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); await clock.tickAsync(0); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledOnce(); - await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; + // Six blocks arrive while the first verdict waits to be confirmed. + core.height = height + 6; + core.beforeDkgInfo = () => { + core.height = memberDkgStart; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + core.beforeDkgInfo = undefined; + }; + await clock.tickAsync(20000); - expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + // Core has now recorded the session. + core.dkgInfo = { active_dkgs: 1, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: memberDkgStart } }], + }; + await clock.tickAsync(60000); + + await expectStillPolling(waiting, rpcClient); + }); + + it('should keep waiting on Core 24.1 while a session runs after the proTxHash was cleared', async function it() { + // A ProUpServTx mined mid-session clears Core's proTxHash: dkginfo then + // reports no active DKG and no upcoming_dkgs, but the session continues. + const height = 24 * 5000 + 3; + const core = { + version: CORE_24_1, + height, + dkgInfo: { active_dkgs: 0, next_dkg: 21 }, + dkgStatus: { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: height - 3 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + await expectStillPolling(waiting, rpcClient); + + // The session has passed its Commit phase. + core.height = height + 7; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14 }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); }); - it('fails safe (keeps waiting) when active_dkgs > 0 and a session has an unknown llmqType', async function it() { - const clock = this.sinon.useFakeTimers(); + it('should not stop at a DKG start reached in Core\'s record gap, however long it lasts', async function it() { + // Waiting starts right at the start block: no earlier poll saw the DKG + // announced, and Core takes longer than the confirmation delay to record + // the session, or never does because this node is not a member. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start, + dkgInfo: { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }, + dkgStatus: { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); + await expectStillPolling(waiting, rpcClient); + + // Without a record, nothing shows this node is not a member, so the + // session blocks until its Commit phase ends. + core.height = start + 1; + core.dkgInfo = { active_dkgs: 0, next_dkg: 23, upcoming_dkgs: [] }; + await expectStillPolling(waiting, rpcClient); + + core.height = start + 10; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14, upcoming_dkgs: [] }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); - rpcClient.quorum.withArgs('dkginfo') - .onFirstCall() - .resolves({ result: { active_dkgs: 1, next_dkg: 20 } }) - .onSecondCall() - .resolves({ result: { active_dkgs: 0, next_dkg: 20 } }); - rpcClient.quorum.withArgs('dkgstatus').resolves({ - result: { session: [{ llmqType: 'llmq_future_unknown', status: { quorumHeight: 1000 } }] }, + it('should not stop in a session Core has not recorded when waiting starts after its start block', async function it() { + const start = 24 * 5000; + const rpcClient = getCoreRpcMock(this.sinon, { + version: CORE_24_0, + height: start + 1, + dkgInfo: { active_dkgs: 0, next_dkg: 23, upcoming_dkgs: [] }, + dkgStatus: { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }, }); - rpcClient.getBlockCount.resolves({ result: 1000 }); - const promise = waitForDKGWindowPass(rpcClient); + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(60000); - await clock.tickAsync(0); - expect(rpcClient.quorum.withArgs('dkgstatus')).to.have.been.calledOnce(); + await expectStillPolling(waiting, rpcClient); + }); + + it('should not hold a DKG start it saw announced without this node', async function it() { + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 1, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 1, known: true, isMember: false, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + // Interrupt the first confirmation, so the wait checks again at the start. + core.height = start; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }; + core.beforeDkgInfo = () => { + core.beforeDkgInfo = undefined; + core.height = start + 7; + }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + core.height = start; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should forget DKGs announced without this node when a reorg replaces their chain', async function it() { + // On the replacing branch the work block differs, so this node may be a + // member after all. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 1, + dkgInfo: { + active_dkgs: 0, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 1, known: true, isMember: false, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + // While the first result waits to be confirmed, a reorg replaces the tip + // and the replacing branch reaches the DKG's start. + core.hashAt = (height) => `other-branch-${height}`; + core.height = start; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }; + await clock.tickAsync(NEXT_POLL_MS * 2); + + await expectStillPolling(waiting, rpcClient); + }); + + it('should not hold a session it saw announced without this node after its start block', async function it() { + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 1, + dkgInfo: { + active_dkgs: 1, + next_dkg: 1, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 1, known: true, isMember: false, + }], + }, + dkgStatus: { + // Another session keeps the node busy until the announced one has started. + session: [{ llmqType: 'llmq_400_60', quorumIndex: 0, status: { quorumHeight: start - 12 } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + const connections = [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }]; + core.height = start + 1; + core.dkgInfo = { active_dkgs: 1, next_dkg: 23, upcoming_dkgs: [] }; + core.dkgStatus = { ...core.dkgStatus, quorumConnections: connections }; await clock.tickAsync(CHECK_INTERVAL_MS); - await promise; - expect(rpcClient.quorum.withArgs('dkgstatus')).to.have.been.calledOnce(); + // The other session has ended; the announced one is still running. + core.height = start + 8; + core.dkgInfo = { active_dkgs: 1, next_dkg: 16, upcoming_dkgs: [] }; + await clock.tickAsync(NEXT_POLL_MS * 2); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep blocking a DKG it saw announced when Core restarts into initial block download', async function it() { + // A restarted Core reports initial block download, yet it may finish + // syncing in time for the DKG this node is a member of. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 2, + dkgInfo: { + active_dkgs: 0, + next_dkg: 2, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 2, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.isSyncing = true; + core.dkgInfo = { active_dkgs: 0, next_dkg: 2, upcoming_dkgs: [] }; + await expectStillPolling(waiting, rpcClient); + + // Once the DKG's Commit phase has ended, a syncing Core may stop. + core.height = start + 10; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); + }); + + it('should keep what a check announced when its confirming read is discarded', async function it() { + // The first check announces a member DKG for later; a block connects + // during the read confirming it, so that read is discarded. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 8, + dkgInfo: { + active_dkgs: 0, + next_dkg: 8, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 8, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.beforeDkgInfo = () => { + core.height += 1; + core.beforeDkgInfo = undefined; + }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + // After the DKG started, Core reports nothing that shows it, as Core 24.1 + // that has not worked out its proTxHash yet. + core.version = CORE_24_1; + core.height = start + 2; + core.dkgInfo = { active_dkgs: 0, next_dkg: 22 }; + await clock.tickAsync(CHECK_INTERVAL_MS); + + await expectStillPolling(waiting, rpcClient); + }); + + it('should keep what a check announced when Core changes its session records during the confirming read', async function it() { + // The first check announces a member DKG for later; the read confirming + // it finds Core 24.0 counting a session record dkgstatus does not list. + const start = 24 * 5000; + const core = { + version: CORE_24_0, + height: start - 8, + dkgInfo: { + active_dkgs: 0, + next_dkg: 8, + upcoming_dkgs: [{ + llmqType: 4, quorumIndex: 0, quorumHeight: start, blocksUntilStart: 8, known: true, isMember: true, + }], + }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.dkgInfo = { ...core.dkgInfo, active_dkgs: 1 }; + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS); + + // After the DKG started, Core reports nothing that shows it, as Core 24.1 + // that has not worked out its proTxHash yet. + core.version = CORE_24_1; + core.height = start + 2; + core.dkgInfo = { active_dkgs: 0, next_dkg: 22 }; + await clock.tickAsync(CHECK_INTERVAL_MS); + + await expectStillPolling(waiting, rpcClient); + }); + + it('should judge each poll by the Core version answering it', async function it() { + // Core is replaced between polls, with no failed call to tell. + const start = 24 * 5000; + const core = { + version: CORE_24_1, + height: start + 1, + dkgInfo: { active_dkgs: 1, next_dkg: 23, upcoming_dkgs: [] }, + dkgStatus: { session: [] }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + // Core 24.0 counts records, and has none yet for the running session. + core.version = CORE_24_0; + core.dkgInfo = { active_dkgs: 0, next_dkg: 23, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: start }], + }; + + await expectStillPolling(waiting, rpcClient); + }); + + it('should keep blocking a running session after Core restarts and forgets it', async function it() { + // Restarted Core keeps no session record, and Core 24.1 without its + // proTxHash yet reports no count or announcement for the session. + const start = 24 * 5000; + const core = { + version: CORE_24_1, + height: start + 3, + dkgInfo: { active_dkgs: 1, next_dkg: 21 }, + dkgStatus: { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: start } }], + }, + }; + const rpcClient = getCoreRpcMock(this.sinon, core); + + const waiting = track(waitForDKGWindowPass(rpcClient)); + await clock.tickAsync(1); + + core.height = start + 4; + core.dkgInfo = { active_dkgs: 0, next_dkg: 20 }; + core.dkgStatus = { session: [] }; + await expectStillPolling(waiting, rpcClient); + + core.height = start + 10; + core.dkgInfo = { active_dkgs: 0, next_dkg: 14 }; + await clock.tickAsync(NEXT_POLL_MS); + + expect(waiting.resolved).to.be.true(); }); }); diff --git a/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js b/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js new file mode 100644 index 00000000000..a8c2fe02924 --- /dev/null +++ b/packages/dashmate/test/unit/listr/tasks/stopNodeTaskFactory.spec.js @@ -0,0 +1,200 @@ +import { Listr } from 'listr2'; +import stopNodeTaskFactory from '../../../../src/listr/tasks/stopNodeTaskFactory.js'; +import { SAFE_STOP_CONFIRMATION_DELAY_MS } from '../../../../src/core/quorum/checkMasternodeSafeToStop.js'; +import getConfigMock from '../../../../src/test/mock/getConfigMock.js'; +import getCoreRpcMock from '../../../../src/test/mock/getCoreRpcMock.js'; + +const CORE_24_0 = 240000; +const CORE_24_1 = 240100; + +describe('stopNodeTaskFactory', () => { + let clock; + let core; + let rpcClient; + let config; + let dockerCompose; + let stopNodeTask; + + beforeEach(function beforeEach() { + clock = this.sinon.useFakeTimers(); + + core = { + version: CORE_24_1, + height: 1012, + dkgInfo: { active_dkgs: 0, next_dkg: 12, upcoming_dkgs: [] }, + dkgStatus: { session: [] }, + }; + + config = getConfigMock(this.sinon); + config.get.withArgs('core.masternode.enable').returns(true); + config.getName.returns('mainnet'); + + dockerCompose = { + isNodeRunning: this.sinon.stub().resolves(true), + stop: this.sinon.stub().resolves(), + }; + + stopNodeTask = stopNodeTaskFactory( + dockerCompose, + () => { + rpcClient = getCoreRpcMock(this.sinon, core); + return rpcClient; + }, + async () => '127.0.0.1', + () => [], + ); + }); + + /** + * Run a plain stop, advancing the fake clock past any confirmation delays. + */ + async function stop() { + const state = {}; + new Listr([{ task: () => stopNodeTask(config) }], { renderer: 'silent' }) + .run({ isForce: false, isSafe: false, platformOnly: false }) + .then(() => { state.stopped = true; }, (e) => { state.error = e; }); + + await clock.tickAsync(SAFE_STOP_CONFIRMATION_DELAY_MS * 4); + + return state; + } + + it('should stop a masternode that no DKG includes', async () => { + const { stopped } = await stop(); + + expect(stopped).to.be.true(); + expect(dockerCompose.stop).to.have.been.calledOnce(); + }); + + it('should refuse to stop a masternode during its DKG session', async () => { + core.dkgInfo = { active_dkgs: 1, next_dkg: 12, upcoming_dkgs: [] }; + + const { error } = await stop(); + + expect(error.message).to.include('may result in a PoSe ban'); + expect(dockerCompose.stop).to.not.have.been.called(); + }); + + it('should refuse a stop on Core 24.0 when a session appears during the confirmation', async () => { + // At a DKG start block Core 24.0 reports the session only once its DKG + // worker has recorded it. + core.version = CORE_24_0; + core.afterDkgStatus = () => { + core.dkgInfo = { active_dkgs: 1, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_100_67', quorumIndex: 0, status: { quorumHeight: 1012 } }], + }; + core.afterDkgStatus = undefined; + }; + + const { error } = await stop(); + + expect(error.message).to.include('may result in a PoSe ban'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + expect(dockerCompose.stop).to.not.have.been.called(); + }); + + it('should refuse to stop when Core reports DKG data dashmate cannot read', async () => { + // --safe would wait for this forever, so it is not offered. + core.dkgInfo = { active_dkgs: 1, next_dkg: 12, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [{ llmqType: 'llmq_future_unknown', quorumIndex: 0, status: { quorumHeight: 1000 } }], + }; + + const { error } = await stop(); + + expect(error.message).to.include('dashmate cannot read Core\'s DKG state'); + expect(error.message).to.not.include('--safe'); + expect(dockerCompose.stop).to.not.have.been.called(); + }); + + it('should check again when a block connects during a check', async () => { + core.beforeDkgInfo = () => { + core.height += 1; + core.beforeDkgInfo = undefined; + }; + + const { stopped } = await stop(); + + expect(stopped).to.be.true(); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledTwice(); + }); + + it('should give up after three checks while the chain keeps advancing', async () => { + // Core catching up connects a block during every check. + core.beforeDkgInfo = () => { core.height += 1; }; + + const { error } = await stop(); + + expect(error.message).to.include('The chain kept advancing'); + expect(rpcClient.quorum.withArgs('dkginfo')).to.have.been.calledThrice(); + expect(dockerCompose.stop).to.not.have.been.called(); + }); + + it('should not wait for DKGs once the local network\'s miner has stopped', async function it() { + // Group commands stop a local network's miner first. Without it no block + // comes, so a DKG wait could never end, and with every masternode of the + // network going down together their DKG produces no commitment to + // penalize them in. + const minerConfig = getConfigMock(this.sinon); + minerConfig.get.withArgs('core.masternode.enable').returns(false); + minerConfig.get.withArgs('core.miner.enable').returns(true); + minerConfig.get.withArgs('network').returns('local'); + minerConfig.get.withArgs('group').returns('local'); + minerConfig.getName.returns('local_seed'); + config.get.withArgs('network').returns('local'); + config.get.withArgs('group').returns('local'); + core.dkgInfo = { active_dkgs: 1, next_dkg: 3, upcoming_dkgs: [] }; + + const state = {}; + new Listr([ + { task: () => stopNodeTask(minerConfig) }, + { task: () => stopNodeTask(config) }, + ], { renderer: 'silent' }) + .run({ isForce: false, isSafe: true, platformOnly: false }) + .then(() => { state.stopped = true; }, (e) => { state.error = e; }); + await clock.tickAsync(1); + + expect(state.stopped).to.be.true(); + expect(dockerCompose.stop).to.have.been.calledTwice(); + }); + + it('should refuse a plain stop at a DKG start that Core has not recorded yet', async () => { + core.version = CORE_24_0; + core.dkgInfo = { active_dkgs: 0, next_dkg: 24, upcoming_dkgs: [] }; + core.dkgStatus = { + session: [], + quorumConnections: [{ llmqType: 'llmq_100_67', quorumIndex: 0, pQuorumBaseBlockIndex: 1012 }], + }; + + const { error } = await stop(); + + expect(error.message).to.include('may result in a PoSe ban'); + expect(dockerCompose.stop).to.not.have.been.called(); + }); + + it('should still check a masternode of another network after a local miner stopped', async function it() { + // A group may hold configs of several networks; stopping a local miner + // does not stop blocks on any other. + const minerConfig = getConfigMock(this.sinon); + minerConfig.get.withArgs('core.masternode.enable').returns(false); + minerConfig.get.withArgs('core.miner.enable').returns(true); + minerConfig.get.withArgs('network').returns('local'); + minerConfig.get.withArgs('group').returns('mixed'); + minerConfig.getName.returns('local_seed'); + config.get.withArgs('group').returns('mixed'); + core.dkgInfo = { active_dkgs: 1, next_dkg: 3, upcoming_dkgs: [] }; + + const state = {}; + new Listr([ + { task: () => stopNodeTask(minerConfig) }, + { task: () => stopNodeTask(config) }, + ], { renderer: 'silent' }) + .run({ isForce: false, isSafe: false, platformOnly: false }) + .then(() => { state.stopped = true; }, (e) => { state.error = e; }); + await clock.tickAsync(1); + + expect(state.error).to.be.an('error'); + expect(dockerCompose.stop).to.have.been.calledOnce(); + }); +});