From c3c7513626d5ea18e1a913452a67da8756715d35 Mon Sep 17 00:00:00 2001 From: minuszka Date: Sun, 20 Sep 2026 10:36:13 +0200 Subject: [PATCH 1/2] ops: replay the Sentinel enforcement arithmetic over recorded commitments (G5) The acceptance criteria for switching Sentinel punishment on (H2) end in one measurement nothing else can substitute for: replay the enforcement arithmetic over the commitments already on the chain and require that the suspended and banned sets are exactly the masternodes an operator stopped for long enough, and nobody else. Until now that gate had no tool. ops/dsl-enforcement-replay.py is a line-for-line port of CDeterministicMNList::ApplyServiceCommitment (evo/deterministicmns.cpp, v22.1.x at 0c758792ca): an absent epoch freezes every counter, a mass outage (missed*100 >= size*15) changes nothing, an unobserved member changes nothing, an online member resets and -- when enforcing -- clears the suspension and the DSL ban, a missed member increments and -- when enforcing -- is suspended at 4 and banned at 5. It reads the explorer's epoch rows and the deterministic list at each epoch base (ops/dsl-baselists-dump.sh, a paced read-only protx diff per epoch), because the bit order is that list sorted by proTxHash in the node's INTERNAL byte order. The stored proTxHash lists are cross-checked against the index resolution: sorting the display hex instead names the wrong masternodes, which is exactly what the first real-data run did (right heights, wrong names) and what the cross-check caught. Validated on 2026-09-20 against every window with a known answer: the 2026-09-03 shadow outage (five stopped -> exactly the five suspended and banned), E1b of 2026-09-05 where the chain itself enforced (suspended at 8496, banned at 8520, recovered at 8568, the five host-1 masternodes -- reproduced byte for byte), E2 of 2026-09-06 (the guard opens at 23 of 152, nobody suspended), the 100-epoch baseline (nobody), and the whole history against the node's live missedServiceEpochs (152 of 152 equal). Raising the guard to 16% suspends 21 in E2: the control that the guard branch is live. The tool reports branch coverage, and says so when the unobserved-hold branch guarded no streak -- which is true of every recorded devnet epoch so far. The unit test uses a synthetic fourteen-masternode chain whose display order is the reverse of the internal order, and exercises every branch including the unobserved-hold on a live streak that real data has never produced, plus four negative controls that each flip one rule and must change the answer. Fourteen and not six: with six members one miss is already 16.7% and the guard swallows every epoch, which is what the first draft of the fixture did. --- ops/dsl-baselists-dump.sh | 30 +++ ops/dsl-enforcement-replay.py | 248 ++++++++++++++++++ .../dsl-enforcement-replay-synthetic.json | 51 ++++ ops/tests/test_dsl_enforcement_replay.py | 183 +++++++++++++ 4 files changed, 512 insertions(+) create mode 100644 ops/dsl-baselists-dump.sh create mode 100644 ops/dsl-enforcement-replay.py create mode 100644 ops/tests/fixtures/dsl-enforcement-replay-synthetic.json create mode 100644 ops/tests/test_dsl_enforcement_replay.py diff --git a/ops/dsl-baselists-dump.sh b/ops/dsl-baselists-dump.sh new file mode 100644 index 0000000..a995feb --- /dev/null +++ b/ops/dsl-baselists-dump.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# +# The deterministic masternode list at every Sentinel epoch base, as JSONL, for +# ops/dsl-enforcement-replay.py. +# +# Runs on the explorer VPS against the seed's RPC, read-only and paced: one +# `protx diff 1 ` per epoch, reduced to proRegTxHash, confirmedHash and +# isValid, which is all the replay needs to reproduce the commitment's bit order +# (the list sorted by proTxHash in the node's internal byte order). The explorer +# stores the bit indices and, where it could resolve them, the proTxHashes; it +# does not store the list itself, and its resolver refuses a partial answer. +# +# Usage: ops/dsl-baselists-dump.sh [first-epoch] [last-epoch] > baselists.jsonl +# defaults: from the devnet's first committable epoch (228, base 5472) to the +# tip's epoch. ~0.3 s per epoch. +set -euo pipefail + +CLI="${DEFCON_CLI:-/usr/local/bin/defcon-cli -datadir=/home/defcon/.defcon -conf=/home/defcon/.defcon/defcon.conf}" +EPOCH_INTERVAL=24 +FIRST="${1:-228}" +TIP=$($CLI getblockcount) +LAST="${2:-$(( TIP / EPOCH_INTERVAL ))}" + +for e in $(seq "$FIRST" "$LAST"); do + b=$(( e * EPOCH_INTERVAL )) + hash=$($CLI getblockhash "$b") || continue + $CLI protx diff 1 "$b" | jq -c --argjson e "$e" --argjson b "$b" --arg h "$hash" \ + '{epoch: $e, base: $b, baseHash: $h, mnList: [.mnList[] | {proRegTxHash, confirmedHash, isValid}]}' + sleep 0.2 +done diff --git a/ops/dsl-enforcement-replay.py b/ops/dsl-enforcement-replay.py new file mode 100644 index 0000000..ceaeb2c --- /dev/null +++ b/ops/dsl-enforcement-replay.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Replay the Sentinel layer's enforcement arithmetic over recorded commitments (G5). + +The one question that decides when punishment may be switched on -- "what would +enforcement have done?" -- can be answered from the commitments already on the +chain, with no chain change. This is a line-for-line port of +CDeterministicMNList::ApplyServiceCommitment (evo/deterministicmns.cpp, v22.1.x +at 0c758792ca), applied in epoch order: + + absent epoch the function never runs: every counter is frozen + mass outage missed*100 >= size*nDSLMassOutagePct: record the epoch, change nothing + unobserved member no verdict (format v2): change nothing; v1 observes everyone + online member counter = 0; if enforcing, the suspension and the DSL ban are cleared + missed member counter += 1; if enforcing, >= suspend -> suspended, >= ban -> banned + at the boundary height, once + +Inputs are the explorer's epoch rows (GET /api/v1/dsl/epochs, saved to a file) +and the deterministic masternode list at every epoch base (a JSONL written by +ops/dsl-baselists-dump.sh from `protx diff 1 `). The base list is needed +because the commitment's bit order is that list sorted by proTxHash -- in the +node's INTERNAL byte order, which is the display hex reversed bytewise. Sorting +the display strings names the wrong masternodes; the stored proTxHash lists are +compared against the index resolution so that mistake cannot pass silently. + +Validated 2026-09-20 against every window with a known answer: the shadow +outage of 2026-09-03 (five stopped -> exactly the five suspended and banned), +E1b of 2026-09-05 where the chain itself enforced (suspended at 8496, banned at +8520, recovered at 8568 -- the replay reproduces those heights and names), E2 +of 2026-09-06 (the 15% guard opens at 23 of 152 and nobody is suspended), the +100-epoch baseline of 2026-09-15 (nobody), and the whole history against the +node's own missedServiceEpochs counters (152 of 152 equal). Raising the guard to +16% suspends 21 in E2, which is the control that the guard branch is live. + + ops/dsl-enforcement-replay.py --epochs epochs.json --baselists baselists.jsonl \\ + --from-epoch 581 --to-epoch 680 --enforce-from-epoch 581 [--expect stopped.json] + +Read-only over its input files; it never talks to a node. +""" +import argparse +import collections +import json +import sys + +UNSET_BAN = -1 + + +def internal(h): + """A hash as the node holds it: the display hex reversed bytewise.""" + return bytes.fromhex(h)[::-1] + + +def canonical_order(mn_list): + """The commitment's bit order: the epoch-base list sorted by proTxHash the way + uint256 sorts -- base_blob::Compare is a memcmp over the internal bytes.""" + return sorted((m["proRegTxHash"] for m in mn_list), key=internal) + + +def load_epochs(path): + with open(path, encoding="utf-8") as f: + data = json.load(f) + rows = data["data"]["items"] if isinstance(data, dict) and "data" in data else data + return sorted(rows, key=lambda e: e["epoch"]) + + +def load_baselists(path): + out = {} + with open(path, encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + e = json.loads(line) + out[int(e["epoch"])] = e + return out + + +class Replay: + """State of every masternode's Sentinel counters, advanced one epoch at a time.""" + + def __init__(self, suspend=4, ban=5, mass_outage_pct=15, hold_unobserved=True): + self.suspend, self.ban, self.mass_pct = suspend, ban, mass_outage_pct + self.hold_unobserved = hold_unobserved + self.counter = collections.defaultdict(int) # proTxHash -> nMissedEpochs + self.suspended = set() # fRewardSuspended + self.banned = {} # proTxHash -> nDSLBanHeight + self.last_epoch = {} # nLastServiceEpoch + self.events = [] # (epoch, height, kind, proTxHash) + self.table = [] # per-epoch rows + self.warnings = [] + # Branch coverage: how often a protective branch guarded a member that had a + # streak to protect. A window where these stay at zero has not exercised + # that branch, and a control that flips it cannot discriminate there. + self.hold_protected = 0 # unobserved member with counter > 0 + self.mass_protected = 0 # mass-outage epoch skipping a member with counter > 0 + self.absent_frozen = 0 # absent epoch while any counter was > 0 + + def apply(self, row, base, enforce): + epoch, height = row["epoch"], row["boundaryHeight"] + if row["status"] != "committed": + if any(self.counter.values()): + self.absent_frozen += 1 + self.table.append((epoch, height, "absent", None, None, None, len(self.suspended), len(self.banned))) + return + order = canonical_order(base["mnList"]) + size = len(order) + if row["listSize"] != size: + self.warnings.append("epoch %d: listSize %s != base list %d" % (epoch, row["listSize"], size)) + self.table.append((epoch, height, "size-mismatch", None, None, None, len(self.suspended), len(self.banned))) + return + missed_idx = set(row.get("missedIndices") or []) + unobs_idx = set(row.get("unobservedIndices") or []) + missed = {order[i] for i in missed_idx if i < size} + unobserved = {order[i] for i in unobs_idx if i < size} if self.hold_unobserved else set() + stored = set(row.get("missedProTxHashes") or []) + if stored and stored != missed: + self.warnings.append("epoch %d: stored missedProTxHashes differ from index resolution" % epoch) + missed_count = len(missed_idx) + mass = size > 0 and missed_count * 100 >= size * self.mass_pct + for p in order: + self.last_epoch[p] = epoch + if mass: + if self.counter[p]: + self.mass_protected += 1 + continue + if p in unobserved: + if self.counter[p]: + self.hold_protected += 1 + continue + if p not in missed: + if self.counter[p] != 0 and enforce and (p in self.suspended or p in self.banned): + self.events.append((epoch, height, "recovered", p)) + self.counter[p] = 0 + if enforce: + self.suspended.discard(p) + self.banned.pop(p, None) + else: + self.counter[p] += 1 + if enforce: + if self.counter[p] >= self.suspend and p not in self.suspended: + self.suspended.add(p) + self.events.append((epoch, height, "suspended", p)) + if self.counter[p] >= self.ban and p not in self.banned: + self.banned[p] = height + self.events.append((epoch, height, "banned", p)) + self.table.append((epoch, height, "mass-outage" if mass else "applied", size, missed_count, + len(unobs_idx), len(self.suspended), len(self.banned))) + + +def run(rows, bases, enforce_from=None, **kw): + rp = Replay(**kw) + skipped = 0 + for r in rows: + base = bases.get(r["epoch"]) + if r["status"] == "committed" and base is None: + rp.warnings.append("epoch %d: no base list" % r["epoch"]) + skipped += 1 + continue + rp.apply(r, base, enforce_from is not None and r["epoch"] >= enforce_from) + return rp, skipped + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--epochs", required=True, help="JSON: the /api/v1/dsl/epochs rows (envelope or bare list)") + ap.add_argument("--baselists", required=True, help="JSONL from ops/dsl-baselists-dump.sh") + ap.add_argument("--from-epoch", type=int) + ap.add_argument("--to-epoch", type=int) + ap.add_argument("--enforce-from-epoch", type=int, + help="apply the enforcing branch from this epoch (default: never, i.e. shadow)") + ap.add_argument("--suspend", type=int, default=4) + ap.add_argument("--ban", type=int, default=5) + ap.add_argument("--mass-outage-pct", type=int, default=15) + ap.add_argument("--no-hold-unobserved", action="store_true", + help="CONTROL: treat unobserved members as online, the v1 healing that v2 ended") + ap.add_argument("--expect", help='JSON {"stopped": [proTxHash...], "note": "..."}: the G5 verdict against it') + ap.add_argument("--compare-counters", help="JSON {proTxHash: missedServiceEpochs} from the node") + ap.add_argument("--verbose", action="store_true") + a = ap.parse_args() + + rows = load_epochs(a.epochs) + if a.from_epoch is not None: + rows = [r for r in rows if r["epoch"] >= a.from_epoch] + if a.to_epoch is not None: + rows = [r for r in rows if r["epoch"] <= a.to_epoch] + if not rows: + sys.exit("no epochs in range") + rp, skipped = run(rows, load_baselists(a.baselists), a.enforce_from_epoch, + suspend=a.suspend, ban=a.ban, mass_outage_pct=a.mass_outage_pct, + hold_unobserved=not a.no_hold_unobserved) + + committed = [t for t in rp.table if t[2] in ("applied", "mass-outage")] + absent = [t for t in rp.table if t[2] == "absent"] + mass = [t for t in rp.table if t[2] == "mass-outage"] + print("window: epochs %d..%d (%d rows: %d committed, %d absent, %d mass-outage, %d skipped)" % + (rows[0]["epoch"], rows[-1]["epoch"], len(rows), len(committed), len(absent), len(mass), skipped)) + print("rules: suspend >= %d, ban >= %d, mass-outage >= %d%%, unobserved %s; enforcing from epoch %s" % + (a.suspend, a.ban, a.mass_outage_pct, "held" if not a.no_hold_unobserved else "TREATED AS ONLINE (control)", + a.enforce_from_epoch if a.enforce_from_epoch is not None else "never (shadow)")) + if a.verbose: + print("\nepoch height state size missed unobs susp banned") + for e, h, st, sz, mc, uo, ns, nb in rp.table: + print("%5d %6d %-12s %4s %6s %5s %4d %6d" % ( + e, h, st, "-" if sz is None else sz, "-" if mc is None else mc, "-" if uo is None else uo, ns, nb)) + print() + ev = collections.Counter(k for _, _, k, _ in rp.events) + print("events: %s" % (dict(ev) if ev else "none")) + for e, h, k, p in rp.events: + print(" epoch %d @%d %-9s %s" % (e, h, k, p[:12])) + nonzero = {p: c for p, c in rp.counter.items() if c} + print("final: %d members with a non-zero counter, %d suspended, %d banned" % (len(nonzero), len(rp.suspended), len(rp.banned))) + for p, c in sorted(nonzero.items(), key=lambda kv: -kv[1])[:12]: + print(" %s counter=%d%s%s" % (p[:12], c, " SUSPENDED" if p in rp.suspended else "", + " BANNED@%d" % rp.banned[p] if p in rp.banned else "")) + print("branch coverage (member-epochs where a protective branch guarded a non-zero counter): " + "unobserved-hold %d, mass-outage %d; absent epochs while any counter was non-zero: %d" + % (rp.hold_protected, rp.mass_protected, rp.absent_frozen)) + if rp.hold_protected == 0: + print(" NOTE: the unobserved-hold branch never guarded a streak in this window; a control that flips it cannot discriminate here") + for w in rp.warnings: + print("warning:", w) + + rc = 0 + if a.expect: + with open(a.expect, encoding="utf-8") as f: + exp = json.load(f) + stopped = set(exp["stopped"]) + print("\nG5 verdict against '%s' (%d stopped):" % (exp.get("note", a.expect), len(stopped))) + for name, got in (("suspended", {p for _, _, k, p in rp.events if k == "suspended"}), + ("banned", {p for _, _, k, p in rp.events if k == "banned"})): + extra, missing = got - stopped, stopped - got + ok = not extra and not missing + print(" %-9s %s (got %d, expected %d%s%s)" % ( + name, "PASS" if ok else "FAIL", len(got), len(stopped), + "; EXTRA " + ",".join(p[:8] for p in sorted(extra)) if extra else "", + "; MISSING " + ",".join(p[:8] for p in sorted(missing)) if missing else "")) + rc |= 0 if ok else 1 + if a.compare_counters: + with open(a.compare_counters, encoding="utf-8") as f: + node = {k: int(v) for k, v in json.load(f).items()} + diffs = [(p, rp.counter.get(p, 0), n) for p, n in node.items() if rp.counter.get(p, 0) != n] + print("\ncounter cross-check against the node: %d members compared, %d differ" % (len(node), len(diffs))) + for p, mine, theirs in diffs[:10]: + print(" %s replay=%d node=%d" % (p[:12], mine, theirs)) + rc |= 0 if not diffs else 2 + sys.exit(rc) + + +if __name__ == "__main__": + main() diff --git a/ops/tests/fixtures/dsl-enforcement-replay-synthetic.json b/ops/tests/fixtures/dsl-enforcement-replay-synthetic.json new file mode 100644 index 0000000..787b970 --- /dev/null +++ b/ops/tests/fixtures/dsl-enforcement-replay-synthetic.json @@ -0,0 +1,51 @@ +{ + "_comment": [ + "Synthetic fourteen-masternode chain for ops/tests/test_dsl_enforcement_replay.py.", + "Fourteen and not fewer: with six members one missed masternode is already 16.7% of the", + "list and the 15% mass-outage guard swallows every epoch -- the first draft of this fixture", + "did exactly that and no counter ever moved. With fourteen, one miss is 7.1%, two are 14.3%", + "(below the guard), three are 21.4% (the guard opens).", + "The proTxHashes are chosen so that display order (01.., 02.., ..., 0e..) is the REVERSE of", + "the node's internal byte order (last byte 22 < 33 < ... < ff): canonical index 0 is member", + "N (0e..22) and index 13 is member A (01..ff). A tool that sorts display strings puts A at 0.", + "Members by canonical index: 13=A 12=B 11=C ... 0=N.", + "Story: A (index 13) misses epochs 2-6 -> counter 1..5, suspended at epoch 5 (count 4),", + "banned at epoch 6 (count 5, height 168). B (index 12) misses epoch 3 (counter 1), is", + "UNOBSERVED in epoch 4 while that streak is live (the hold branch no recorded devnet epoch", + "has exercised), is online in epoch 5 (reset). Epoch 2 is format v1 (everyone observed).", + "Epoch 7 is absent (freeze). Epoch 8 is a mass outage (3 of 14 = 21.4%): nothing changes.", + "Epoch 9: A online -> recovered; B missed -> counter 1. Epoch 10 has a wrong listSize and", + "must be skipped, so C (index 11) is never counted.", + "Epoch 3's stored missedProTxHashes is DELIBERATELY the display-order resolution (indices", + "13 and 12 read in display order are N and M), so the cross-check warning must fire there", + "and nowhere else." + ], + "members": { + "A": "01000000000000000000000000000000000000000000000000000000000000ff", + "B": "02000000000000000000000000000000000000000000000000000000000000ee", + "C": "03000000000000000000000000000000000000000000000000000000000000dd", + "D": "04000000000000000000000000000000000000000000000000000000000000cc", + "E": "05000000000000000000000000000000000000000000000000000000000000bb", + "F": "06000000000000000000000000000000000000000000000000000000000000aa", + "G": "0700000000000000000000000000000000000000000000000000000000000099", + "H": "0800000000000000000000000000000000000000000000000000000000000088", + "I": "0900000000000000000000000000000000000000000000000000000000000077", + "J": "0a00000000000000000000000000000000000000000000000000000000000066", + "K": "0b00000000000000000000000000000000000000000000000000000000000055", + "L": "0c00000000000000000000000000000000000000000000000000000000000044", + "M": "0d00000000000000000000000000000000000000000000000000000000000033", + "N": "0e00000000000000000000000000000000000000000000000000000000000022" + }, + "epochs": [ + {"epoch": 1, "boundaryHeight": 48, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 0, "missedIndices": [], "unobservedIndices": [], "missedProTxHashes": []}, + {"epoch": 2, "boundaryHeight": 72, "status": "committed", "commitmentVersion": 1, "listSize": 14, "missedCount": 1, "missedIndices": [13], "missedProTxHashes": ["01000000000000000000000000000000000000000000000000000000000000ff"]}, + {"epoch": 3, "boundaryHeight": 96, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 2, "missedIndices": [13, 12], "unobservedIndices": [], "missedProTxHashes": ["0e00000000000000000000000000000000000000000000000000000000000022", "0d00000000000000000000000000000000000000000000000000000000000033"]}, + {"epoch": 4, "boundaryHeight": 120, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 1, "missedIndices": [13], "unobservedIndices": [12], "missedProTxHashes": ["01000000000000000000000000000000000000000000000000000000000000ff"]}, + {"epoch": 5, "boundaryHeight": 144, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 1, "missedIndices": [13], "unobservedIndices": [], "missedProTxHashes": ["01000000000000000000000000000000000000000000000000000000000000ff"]}, + {"epoch": 6, "boundaryHeight": 168, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 1, "missedIndices": [13], "unobservedIndices": [], "missedProTxHashes": ["01000000000000000000000000000000000000000000000000000000000000ff"]}, + {"epoch": 7, "boundaryHeight": 192, "status": "absent", "commitmentVersion": null, "listSize": null, "missedCount": null, "missedIndices": [], "unobservedIndices": [], "missedProTxHashes": []}, + {"epoch": 8, "boundaryHeight": 216, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 3, "missedIndices": [13, 12, 11], "unobservedIndices": [], "missedProTxHashes": ["01000000000000000000000000000000000000000000000000000000000000ff", "02000000000000000000000000000000000000000000000000000000000000ee", "03000000000000000000000000000000000000000000000000000000000000dd"]}, + {"epoch": 9, "boundaryHeight": 240, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 1, "missedIndices": [12], "unobservedIndices": [], "missedProTxHashes": ["02000000000000000000000000000000000000000000000000000000000000ee"]}, + {"epoch": 10, "boundaryHeight": 264, "status": "committed", "commitmentVersion": 2, "listSize": 15, "missedCount": 1, "missedIndices": [11], "unobservedIndices": [], "missedProTxHashes": []} + ] +} diff --git a/ops/tests/test_dsl_enforcement_replay.py b/ops/tests/test_dsl_enforcement_replay.py new file mode 100644 index 0000000..282dc2e --- /dev/null +++ b/ops/tests/test_dsl_enforcement_replay.py @@ -0,0 +1,183 @@ +""" +ops/dsl-enforcement-replay.py against a synthetic fourteen-masternode chain +(fixtures/dsl-enforcement-replay-synthetic.json) built so that every branch of +CDeterministicMNList::ApplyServiceCommitment runs at least once -- including the +one no recorded devnet epoch has exercised: an UNOBSERVED verdict on a member +whose missed streak is live, which must hold the counter rather than heal it. + +Fourteen members, not fewer: the first draft had six, where one missed +masternode is already 16.7% of the list and the 15% mass-outage guard swallowed +every epoch, so no counter ever moved. That is the guard doing its job on a +fixture that was too small, and it is kept in the fixture's comment as the +reason for the size. + +The fourteen proTxHashes sort in the opposite order by display hex and by the +node's internal byte order, so a tool that sorts the display strings names the +wrong masternode at every index. That is the bug the real-data validation +caught on 2026-09-20 (five right heights, four wrong names); the fixture pins it. + +Real-data validation is not repeated here (the base lists are 13 MB): on +2026-09-20 the tool reproduced the chain's own E1b events (suspended at 8496, +banned at 8520, recovered at 8568, the five host-1 masternodes), the 2026-09-03 +shadow outage, the E2 guard at 23 of 152, the 100-epoch baseline (nobody) and +the node's live counters (152 of 152 equal). + + python3 -m unittest discover -s ops/tests -p 'test_*.py' -v +""" +import importlib.util +import json +import os +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "..", "dsl-enforcement-replay.py") +FIXTURE = os.path.join(HERE, "fixtures", "dsl-enforcement-replay-synthetic.json") + +spec = importlib.util.spec_from_file_location("dsl_enforcement_replay", SCRIPT) +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) + +with open(FIXTURE, encoding="utf-8") as f: + FIX = json.load(f) + +M = FIX["members"] +NAMES = "ABCDEFGHIJKLMN" +A, B, C, N = M["A"], M["B"], M["C"], M["N"] +DISPLAY_ORDER = [M[k] for k in NAMES] +BASE = {"mnList": [{"proRegTxHash": h, "confirmedHash": "00" * 32, "isValid": True} for h in DISPLAY_ORDER]} +BASES = {r["epoch"]: BASE for r in FIX["epochs"]} + + +def replay(enforce_from=1, upto=None, **kw): + rows = [r for r in FIX["epochs"] if upto is None or r["epoch"] <= upto] + rp, skipped = mod.run(rows, BASES, enforce_from, **kw) + return rp, skipped + + +class CanonicalOrder(unittest.TestCase): + def test_internal_byte_order_is_the_reverse_of_display_order_here(self): + order = mod.canonical_order(BASE["mnList"]) + self.assertEqual(order, list(reversed(DISPLAY_ORDER))) + self.assertEqual(order[13], A) + self.assertEqual(order[0], N) + # The display-sorted order is what a naive tool would use; it must differ. + self.assertEqual(sorted(DISPLAY_ORDER), DISPLAY_ORDER) + self.assertNotEqual(sorted(DISPLAY_ORDER), order) + + def test_internal_reverses_bytes(self): + self.assertEqual(mod.internal("00" * 31 + "ff"), b"\xff" + b"\x00" * 31) + + +class Arithmetic(unittest.TestCase): + def test_the_full_story_with_enforcement(self): + rp, skipped = replay(enforce_from=1) + self.assertEqual(skipped, 0) + self.assertEqual([(e, k, p) for e, _, k, p in rp.events], + [(5, "suspended", A), (6, "banned", A), (9, "recovered", A)]) + # The ban height is the boundary block of the epoch that reached the count. + self.assertEqual([h for _, h, k, _ in rp.events if k == "banned"], [168]) + # Final state: A recovered, B missed once at epoch 9, nobody else touched. + self.assertEqual(rp.counter[A], 0) + self.assertEqual(rp.counter[B], 1) + self.assertEqual(rp.suspended, set()) + self.assertEqual(rp.banned, {}) + for k in NAMES[2:]: + self.assertEqual(rp.counter[M[k]], 0, k) + + def test_counters_epoch_by_epoch(self): + # A: 1..5 through epochs 2-6; frozen through 7 (absent) and 8 (mass outage); 0 at 9. + for upto, want in {2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 5, 8: 5, 9: 0, 10: 0}.items(): + rp, _ = replay(enforce_from=1, upto=upto) + self.assertEqual(rp.counter[A], want, "A after epoch %d" % upto) + # B: missed at 3 (1), UNOBSERVED at 4 while the streak is live -> still 1, + # online at 5 -> 0, mass outage at 8 -> still 0, missed at 9 -> 1. + for upto, want in {3: 1, 4: 1, 5: 0, 8: 0, 9: 1}.items(): + rp, _ = replay(enforce_from=1, upto=upto) + self.assertEqual(rp.counter[B], want, "B after epoch %d" % upto) + + def test_suspension_and_ban_land_on_the_right_epochs(self): + four, _ = replay(enforce_from=1, upto=5) + self.assertEqual(four.suspended, {A}) + self.assertEqual(four.banned, {}) + five, _ = replay(enforce_from=1, upto=6) + self.assertEqual(five.banned, {A: 168}) + + def test_v1_epoch_observes_everyone(self): + # Epoch 2 is format v1: no unobservedIndices field at all, and A's miss counts. + rp, _ = replay(enforce_from=1, upto=2) + self.assertEqual(rp.counter[A], 1) + self.assertEqual(rp.hold_protected, 0) + + def test_shadow_counts_but_never_punishes(self): + rp, _ = replay(enforce_from=None) + self.assertEqual(rp.events, []) + self.assertEqual(rp.suspended, set()) + self.assertEqual(rp.banned, {}) + # The counters are the same as under enforcement: shadow mode counts too. + enforced, _ = replay(enforce_from=1) + self.assertEqual(dict(rp.counter), dict(enforced.counter)) + + def test_size_mismatch_is_skipped_with_a_warning(self): + rp, _ = replay(enforce_from=1) + self.assertTrue(any(w.startswith("epoch 10:") and "listSize" in w for w in rp.warnings)) + # Nothing from epoch 10 leaked into the state: C (index 11) was never counted + # (epoch 8, its other miss, was a mass outage). + self.assertEqual(rp.counter[C], 0) + self.assertEqual(rp.table[-1][2], "size-mismatch") + + def test_branch_coverage_is_reported(self): + rp, _ = replay(enforce_from=1) + self.assertEqual(rp.hold_protected, 1) # B at epoch 4 + self.assertEqual(rp.mass_protected, 1) # A at epoch 8 (B and C had no streak there) + self.assertEqual(rp.absent_frozen, 1) # epoch 7 with A at 5 + + +class CrossCheck(unittest.TestCase): + def test_display_order_stored_list_is_flagged(self): + # Epoch 3's stored list was built with display-order resolution on purpose. + rp, _ = replay(enforce_from=1) + flagged = [w for w in rp.warnings if "differ" in w] + self.assertEqual(flagged, ["epoch 3: stored missedProTxHashes differ from index resolution"]) + # And the wrong stored names did not leak into the arithmetic: N and M stay clean. + self.assertEqual(rp.counter[N], 0) + self.assertEqual(rp.counter[M["M"]], 0) + + +class NegativeControls(unittest.TestCase): + """Each control flips one rule and must change the answer; a control that leaves + the answer unchanged would mean that rule is dead in the tool.""" + + def test_unobserved_treated_as_online_heals_the_streak(self): + held, _ = replay(enforce_from=1, upto=4) + healed, _ = replay(enforce_from=1, upto=4, hold_unobserved=False) + self.assertEqual(held.counter[B], 1) + self.assertEqual(healed.counter[B], 0) + self.assertEqual(held.hold_protected, 1) + self.assertEqual(healed.hold_protected, 0) + + def test_guard_raised_lets_the_mass_outage_epoch_apply(self): + guarded, _ = replay(enforce_from=1, upto=8) + unguarded, _ = replay(enforce_from=1, upto=8, mass_outage_pct=40) + # With the guard, epoch 8 changed nothing: A stays at 5, B and C at 0. + self.assertEqual((guarded.counter[A], guarded.counter[B], guarded.counter[C]), (5, 0, 0)) + # Without it, all three misses count. + self.assertEqual((unguarded.counter[A], unguarded.counter[B], unguarded.counter[C]), (6, 1, 1)) + self.assertEqual(guarded.mass_protected, 1) + self.assertEqual(unguarded.mass_protected, 0) + + def test_thresholds_move_the_events(self): + rp, _ = replay(enforce_from=1, suspend=2, ban=3) + self.assertEqual([(e, k) for e, _, k, p in rp.events if p == A][:2], [(3, "suspended"), (4, "banned")]) + + def test_enforcing_later_moves_the_events_but_not_the_counters(self): + late, _ = replay(enforce_from=6) + early, _ = replay(enforce_from=1) + self.assertEqual(dict(late.counter), dict(early.counter)) + # Enforcement starting at epoch 6 finds A already at count 5: suspended and banned + # in the same epoch, recovered at 9. + self.assertEqual([(e, k) for e, _, k, p in late.events if p == A], + [(6, "suspended"), (6, "banned"), (9, "recovered")]) + + +if __name__ == "__main__": + unittest.main() From 34293236bfedef6ece091b2912cc445a71aeddee Mon Sep 17 00:00:00 2001 From: minuszka Date: Sun, 20 Sep 2026 11:48:42 +0200 Subject: [PATCH 2/2] ops(dsl-enforcement-replay): skip removed members, enforce by height, name the limits Follow-up to the independent review of #191 (four notes, none blocking): - A member of the epoch-base list that is gone from the list at the boundary block is skipped, as deterministicmns.cpp:423 skips it. The boundary list is the next epoch's base list (base(k+1) == boundary(k)); without it every base member is counted and the run says so. On the recorded history the count is 0 in every window -- no masternode has ever left this devnet's list. - --enforce-from-height applies the node's own rule (boundary height >= nDSLEnforcementHeight, :419); the epoch form stays for counterfactual windows. E1b replayed from height 8304 gives the same events as from epoch 346 -- on this data, because nobody had a streak at 345. - The counter cross-check reports how many node counters are non-zero and says when the comparison is empty: at the 16627 tip all 152 were 0, so "152/152 equal" carried no weight. The docstring says so instead of implying otherwise. - The docstring narrows the claim to a first-order counterfactual: a real DSL ban sets isValid=false and changes later commitments, so the replay answers the G5 gate and is not a proof that enforcement is safe. --expect is named as window-only with a clean-start assumption; the stored-vs-index cross-check as two ports of one ordering rule, the node's counter as the only oracle. Tests: 19 (was 14) -- the removed member with its negative control (no boundary list -> counted, warned; list at the wrong height -> not used), the height rule at the boundary block and one block later. Ops suite 54/54. The real-data matrix re-run with this version is unchanged (shadow 5/5/5, E1b @8496/@8520/@8568 by height and by epoch, E2 nobody, baseline nobody, control A 21/21 as required). --- ops/dsl-enforcement-replay.py | 127 ++++++++++++++---- .../dsl-enforcement-replay-synthetic.json | 7 +- ops/tests/test_dsl_enforcement_replay.py | 89 ++++++++++-- 3 files changed, 184 insertions(+), 39 deletions(-) diff --git a/ops/dsl-enforcement-replay.py b/ops/dsl-enforcement-replay.py index ceaeb2c..e3ffcfe 100644 --- a/ops/dsl-enforcement-replay.py +++ b/ops/dsl-enforcement-replay.py @@ -1,38 +1,68 @@ #!/usr/bin/env python3 """Replay the Sentinel layer's enforcement arithmetic over recorded commitments (G5). -The one question that decides when punishment may be switched on -- "what would -enforcement have done?" -- can be answered from the commitments already on the -chain, with no chain change. This is a line-for-line port of -CDeterministicMNList::ApplyServiceCommitment (evo/deterministicmns.cpp, v22.1.x -at 0c758792ca), applied in epoch order: +The question that decides when punishment may be switched on -- "what would the +enforcement arithmetic have done to the commitments actually recorded?" -- can +be answered from the chain, with no chain change. This is a line-for-line port +of CDeterministicMNList::ApplyServiceCommitment (evo/deterministicmns.cpp:402-475, +v22.1.x at 0c758792ca), applied in epoch order: absent epoch the function never runs: every counter is frozen + removed member in the base list but gone from the list at the boundary + (:423, "removed from the list since the epoch base"): skipped mass outage missed*100 >= size*nDSLMassOutagePct: record the epoch, change nothing unobserved member no verdict (format v2): change nothing; v1 observes everyone online member counter = 0; if enforcing, the suspension and the DSL ban are cleared missed member counter += 1; if enforcing, >= suspend -> suspended, >= ban -> banned at the boundary height, once +Enforcement is decided the way the node decides it -- the boundary block's height +against nDSLEnforcementHeight (:419) -- with --enforce-from-height. The epoch form, +--enforce-from-epoch, is for counterfactual windows; mapping a height to "the epoch +that begins there" is one epoch late, because the commitment applied AT that height +belongs to the epoch that ends there (8304 on this devnet is epoch 345's boundary). + +WHAT THIS IS, AND IS NOT. A first-order counterfactual: the arithmetic applied to +the commitments as they were recorded. Under real enforcement a DSL ban sets +isValid=false, which changes the sentinel assignment and who can report at all, +so the later commitments would not have been the same ones. That answers the G5 +gate ("would anyone have been punished in this window?"); it is not a proof that +enforcement is safe. + Inputs are the explorer's epoch rows (GET /api/v1/dsl/epochs, saved to a file) and the deterministic masternode list at every epoch base (a JSONL written by ops/dsl-baselists-dump.sh from `protx diff 1 `). The base list is needed because the commitment's bit order is that list sorted by proTxHash -- in the node's INTERNAL byte order, which is the display hex reversed bytewise. Sorting -the display strings names the wrong masternodes; the stored proTxHash lists are -compared against the index resolution so that mistake cannot pass silently. +the display strings names the wrong masternodes. The list at an epoch's boundary +is the next epoch's base list (base(k+1) == boundary(k)); a member of base(k) +missing from it was removed during the epoch and is skipped as the node skips +it. Without that next list every base member is counted, and the run says so. + +Three checks, and what each is worth. The stored proTxHash lists are compared +against the index resolution, so a wrong sort cannot pass silently -- but both +sides are ports of the same ordering rule, not an oracle. --expect compares the +events inside the window with the declared stopped set, and assumes a clean +starting state (no live streak at the window's first epoch; both recorded +outage runs began that way). --compare-counters is the only oracle -- the node's +own missedServiceEpochs -- and it discriminates only while streaks are live: at +a quiet tip every counter is 0 and an all-zero replay passes it too. Capture +that snapshot during an outage run; the tool says when the comparison is empty. Validated 2026-09-20 against every window with a known answer: the shadow outage of 2026-09-03 (five stopped -> exactly the five suspended and banned), -E1b of 2026-09-05 where the chain itself enforced (suspended at 8496, banned at -8520, recovered at 8568 -- the replay reproduces those heights and names), E2 -of 2026-09-06 (the 15% guard opens at 23 of 152 and nobody is suspended), the -100-epoch baseline of 2026-09-15 (nobody), and the whole history against the -node's own missedServiceEpochs counters (152 of 152 equal). Raising the guard to -16% suspends 21 in E2, which is the control that the guard branch is live. +E1b of 2026-09-05 where the chain itself enforced from height 8304 (suspended +at 8496, banned at 8520, recovered at 8568 -- the replay reproduces those +heights and names), E2 of 2026-09-06 (the 15% guard opens at 23 of 152 and +nobody is suspended), the 100-epoch baseline of 2026-09-15 (nobody), and the +whole history against the node's counters (152 of 152 equal -- all zero at that +tip, so that one carried no weight). Raising the guard to 16% suspends 21 in +E2, which is the control that the guard branch is live. ops/dsl-enforcement-replay.py --epochs epochs.json --baselists baselists.jsonl \\ --from-epoch 581 --to-epoch 680 --enforce-from-epoch 581 [--expect stopped.json] + ops/dsl-enforcement-replay.py --epochs epochs.json --baselists baselists.jsonl \\ + --from-epoch 340 --to-epoch 360 --enforce-from-height 8304 --expect e1b.json Read-only over its input files; it never talks to a node. """ @@ -73,6 +103,18 @@ def load_baselists(path): return out +def boundary_list(bases, row): + """The set of proTxHashes on the list at the epoch's boundary block -- the next + epoch's base list, since base(k+1) == boundary(k). None when the dump does not + hold it, or holds it at a different height (then it is not that block's list).""" + nxt = bases.get(row["epoch"] + 1) + if nxt is None: + return None + if nxt.get("base") is not None and nxt["base"] != row["boundaryHeight"]: + return None + return {m["proRegTxHash"] for m in nxt["mnList"]} + + class Replay: """State of every masternode's Sentinel counters, advanced one epoch at a time.""" @@ -92,8 +134,11 @@ def __init__(self, suspend=4, ban=5, mass_outage_pct=15, hold_unobserved=True): self.hold_protected = 0 # unobserved member with counter > 0 self.mass_protected = 0 # mass-outage epoch skipping a member with counter > 0 self.absent_frozen = 0 # absent epoch while any counter was > 0 + self.removed_skipped = 0 # member-epochs skipped because the member had left the list - def apply(self, row, base, enforce): + def apply(self, row, base, enforce, present=None): + """One epoch. `present` is the set of proTxHashes on the list at the boundary + block (None: unknown, every base member is counted).""" epoch, height = row["epoch"], row["boundaryHeight"] if row["status"] != "committed": if any(self.counter.values()): @@ -106,6 +151,9 @@ def apply(self, row, base, enforce): self.warnings.append("epoch %d: listSize %s != base list %d" % (epoch, row["listSize"], size)) self.table.append((epoch, height, "size-mismatch", None, None, None, len(self.suspended), len(self.banned))) return + if present is None: + self.warnings.append("epoch %d: no list at the boundary %d (the next epoch's base): " + "a member removed during the epoch would be counted" % (epoch, height)) missed_idx = set(row.get("missedIndices") or []) unobs_idx = set(row.get("unobservedIndices") or []) missed = {order[i] for i in missed_idx if i < size} @@ -116,6 +164,10 @@ def apply(self, row, base, enforce): missed_count = len(missed_idx) mass = size > 0 and missed_count * 100 >= size * self.mass_pct for p in order: + if present is not None and p not in present: + # deterministicmns.cpp:423 -- GetMN() finds nothing, the node skips it + self.removed_skipped += 1 + continue self.last_epoch[p] = epoch if mass: if self.counter[p]: @@ -145,7 +197,7 @@ def apply(self, row, base, enforce): len(unobs_idx), len(self.suspended), len(self.banned))) -def run(rows, bases, enforce_from=None, **kw): +def run(rows, bases, enforce_from=None, enforce_from_height=None, **kw): rp = Replay(**kw) skipped = 0 for r in rows: @@ -154,7 +206,10 @@ def run(rows, bases, enforce_from=None, **kw): rp.warnings.append("epoch %d: no base list" % r["epoch"]) skipped += 1 continue - rp.apply(r, base, enforce_from is not None and r["epoch"] >= enforce_from) + # The node's rule is by height (:419); the epoch form is for counterfactual windows. + enforce = ((enforce_from is not None and r["epoch"] >= enforce_from) or + (enforce_from_height is not None and r["boundaryHeight"] >= enforce_from_height)) + rp.apply(r, base, enforce, boundary_list(bases, r)) return rp, skipped @@ -164,15 +219,20 @@ def main(): ap.add_argument("--baselists", required=True, help="JSONL from ops/dsl-baselists-dump.sh") ap.add_argument("--from-epoch", type=int) ap.add_argument("--to-epoch", type=int) - ap.add_argument("--enforce-from-epoch", type=int, - help="apply the enforcing branch from this epoch (default: never, i.e. shadow)") + enf = ap.add_mutually_exclusive_group() + enf.add_argument("--enforce-from-epoch", type=int, + help="counterfactual: apply the enforcing branch from this epoch (default: never, i.e. shadow)") + enf.add_argument("--enforce-from-height", type=int, + help="the node's rule: enforce when the boundary block's height >= this (nDSLEnforcementHeight)") ap.add_argument("--suspend", type=int, default=4) ap.add_argument("--ban", type=int, default=5) ap.add_argument("--mass-outage-pct", type=int, default=15) ap.add_argument("--no-hold-unobserved", action="store_true", help="CONTROL: treat unobserved members as online, the v1 healing that v2 ended") - ap.add_argument("--expect", help='JSON {"stopped": [proTxHash...], "note": "..."}: the G5 verdict against it') - ap.add_argument("--compare-counters", help="JSON {proTxHash: missedServiceEpochs} from the node") + ap.add_argument("--expect", help='JSON {"stopped": [proTxHash...], "note": "..."}: the G5 verdict against it ' + '(events inside the window only; assumes no live streak at the first epoch)') + ap.add_argument("--compare-counters", help="JSON {proTxHash: missedServiceEpochs} from the node -- the oracle, " + "but only a snapshot with live streaks can discriminate") ap.add_argument("--verbose", action="store_true") a = ap.parse_args() @@ -183,7 +243,7 @@ def main(): rows = [r for r in rows if r["epoch"] <= a.to_epoch] if not rows: sys.exit("no epochs in range") - rp, skipped = run(rows, load_baselists(a.baselists), a.enforce_from_epoch, + rp, skipped = run(rows, load_baselists(a.baselists), a.enforce_from_epoch, a.enforce_from_height, suspend=a.suspend, ban=a.ban, mass_outage_pct=a.mass_outage_pct, hold_unobserved=not a.no_hold_unobserved) @@ -192,9 +252,15 @@ def main(): mass = [t for t in rp.table if t[2] == "mass-outage"] print("window: epochs %d..%d (%d rows: %d committed, %d absent, %d mass-outage, %d skipped)" % (rows[0]["epoch"], rows[-1]["epoch"], len(rows), len(committed), len(absent), len(mass), skipped)) - print("rules: suspend >= %d, ban >= %d, mass-outage >= %d%%, unobserved %s; enforcing from epoch %s" % + if a.enforce_from_height is not None: + enforcing = "height %d (boundary blocks >= it, the node's rule)" % a.enforce_from_height + elif a.enforce_from_epoch is not None: + enforcing = "epoch %d" % a.enforce_from_epoch + else: + enforcing = "never (shadow)" + print("rules: suspend >= %d, ban >= %d, mass-outage >= %d%%, unobserved %s; enforcing from %s" % (a.suspend, a.ban, a.mass_outage_pct, "held" if not a.no_hold_unobserved else "TREATED AS ONLINE (control)", - a.enforce_from_epoch if a.enforce_from_epoch is not None else "never (shadow)")) + enforcing)) if a.verbose: print("\nepoch height state size missed unobs susp banned") for e, h, st, sz, mc, uo, ns, nb in rp.table: @@ -211,8 +277,9 @@ def main(): print(" %s counter=%d%s%s" % (p[:12], c, " SUSPENDED" if p in rp.suspended else "", " BANNED@%d" % rp.banned[p] if p in rp.banned else "")) print("branch coverage (member-epochs where a protective branch guarded a non-zero counter): " - "unobserved-hold %d, mass-outage %d; absent epochs while any counter was non-zero: %d" - % (rp.hold_protected, rp.mass_protected, rp.absent_frozen)) + "unobserved-hold %d, mass-outage %d; absent epochs while any counter was non-zero: %d; " + "members skipped as removed before the boundary: %d" + % (rp.hold_protected, rp.mass_protected, rp.absent_frozen, rp.removed_skipped)) if rp.hold_protected == 0: print(" NOTE: the unobserved-hold branch never guarded a streak in this window; a control that flips it cannot discriminate here") for w in rp.warnings: @@ -223,7 +290,8 @@ def main(): with open(a.expect, encoding="utf-8") as f: exp = json.load(f) stopped = set(exp["stopped"]) - print("\nG5 verdict against '%s' (%d stopped):" % (exp.get("note", a.expect), len(stopped))) + print("\nG5 verdict against '%s' (%d stopped; events inside the window, clean start assumed):" + % (exp.get("note", a.expect), len(stopped))) for name, got in (("suspended", {p for _, _, k, p in rp.events if k == "suspended"}), ("banned", {p for _, _, k, p in rp.events if k == "banned"})): extra, missing = got - stopped, stopped - got @@ -237,7 +305,12 @@ def main(): with open(a.compare_counters, encoding="utf-8") as f: node = {k: int(v) for k, v in json.load(f).items()} diffs = [(p, rp.counter.get(p, 0), n) for p, n in node.items() if rp.counter.get(p, 0) != n] - print("\ncounter cross-check against the node: %d members compared, %d differ" % (len(node), len(diffs))) + live = sum(1 for n in node.values() if n) + print("\ncounter cross-check against the node: %d members compared, %d differ; node counters non-zero: %d" + % (len(node), len(diffs), live)) + if live == 0: + print(" NOTE: every node counter is 0 -- an all-zero replay passes this too; the check discriminates " + "only on a snapshot taken while streaks are live (during an outage run)") for p, mine, theirs in diffs[:10]: print(" %s replay=%d node=%d" % (p[:12], mine, theirs)) rc |= 0 if not diffs else 2 diff --git a/ops/tests/fixtures/dsl-enforcement-replay-synthetic.json b/ops/tests/fixtures/dsl-enforcement-replay-synthetic.json index 787b970..4cdaf2f 100644 --- a/ops/tests/fixtures/dsl-enforcement-replay-synthetic.json +++ b/ops/tests/fixtures/dsl-enforcement-replay-synthetic.json @@ -16,6 +16,10 @@ "Epoch 7 is absent (freeze). Epoch 8 is a mass outage (3 of 14 = 21.4%): nothing changes.", "Epoch 9: A online -> recovered; B missed -> counter 1. Epoch 10 has a wrong listSize and", "must be skipped, so C (index 11) is never counted.", + "Epoch 11: N (index 0) is missed, but N left the list during the epoch -- the test builds", + "the base list of epoch 12 (the list at boundary 288) without N -- so the node skips it", + "(deterministicmns.cpp:423) and the counter must not move. Without that list the tool", + "counts N and says it could not tell.", "Epoch 3's stored missedProTxHashes is DELIBERATELY the display-order resolution (indices", "13 and 12 read in display order are N and M), so the cross-check warning must fire there", "and nowhere else." @@ -46,6 +50,7 @@ {"epoch": 7, "boundaryHeight": 192, "status": "absent", "commitmentVersion": null, "listSize": null, "missedCount": null, "missedIndices": [], "unobservedIndices": [], "missedProTxHashes": []}, {"epoch": 8, "boundaryHeight": 216, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 3, "missedIndices": [13, 12, 11], "unobservedIndices": [], "missedProTxHashes": ["01000000000000000000000000000000000000000000000000000000000000ff", "02000000000000000000000000000000000000000000000000000000000000ee", "03000000000000000000000000000000000000000000000000000000000000dd"]}, {"epoch": 9, "boundaryHeight": 240, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 1, "missedIndices": [12], "unobservedIndices": [], "missedProTxHashes": ["02000000000000000000000000000000000000000000000000000000000000ee"]}, - {"epoch": 10, "boundaryHeight": 264, "status": "committed", "commitmentVersion": 2, "listSize": 15, "missedCount": 1, "missedIndices": [11], "unobservedIndices": [], "missedProTxHashes": []} + {"epoch": 10, "boundaryHeight": 264, "status": "committed", "commitmentVersion": 2, "listSize": 15, "missedCount": 1, "missedIndices": [11], "unobservedIndices": [], "missedProTxHashes": []}, + {"epoch": 11, "boundaryHeight": 288, "status": "committed", "commitmentVersion": 2, "listSize": 14, "missedCount": 1, "missedIndices": [0], "unobservedIndices": [], "missedProTxHashes": ["0e00000000000000000000000000000000000000000000000000000000000022"]} ] } diff --git a/ops/tests/test_dsl_enforcement_replay.py b/ops/tests/test_dsl_enforcement_replay.py index 282dc2e..4131fb7 100644 --- a/ops/tests/test_dsl_enforcement_replay.py +++ b/ops/tests/test_dsl_enforcement_replay.py @@ -2,8 +2,10 @@ ops/dsl-enforcement-replay.py against a synthetic fourteen-masternode chain (fixtures/dsl-enforcement-replay-synthetic.json) built so that every branch of CDeterministicMNList::ApplyServiceCommitment runs at least once -- including the -one no recorded devnet epoch has exercised: an UNOBSERVED verdict on a member -whose missed streak is live, which must hold the counter rather than heal it. +two no recorded devnet epoch has exercised: an UNOBSERVED verdict on a member +whose missed streak is live, which must hold the counter rather than heal it, and +a member removed from the list between the epoch base and the boundary, which the +node skips (deterministicmns.cpp:423) and so must the tool. Fourteen members, not fewer: the first draft had six, where one missed masternode is already 16.7% of the list and the 15% mass-outage guard swallowed @@ -19,8 +21,10 @@ Real-data validation is not repeated here (the base lists are 13 MB): on 2026-09-20 the tool reproduced the chain's own E1b events (suspended at 8496, banned at 8520, recovered at 8568, the five host-1 masternodes), the 2026-09-03 -shadow outage, the E2 guard at 23 of 152, the 100-epoch baseline (nobody) and -the node's live counters (152 of 152 equal). +shadow outage, the E2 guard at 23 of 152 and the 100-epoch baseline (nobody). +The node's live counters agreed too (152 of 152), but every one of them was 0 +at that tip, so that comparison carried no weight -- it discriminates only on +a snapshot taken while streaks are live. python3 -m unittest discover -s ops/tests -p 'test_*.py' -v """ @@ -44,13 +48,23 @@ NAMES = "ABCDEFGHIJKLMN" A, B, C, N = M["A"], M["B"], M["C"], M["N"] DISPLAY_ORDER = [M[k] for k in NAMES] -BASE = {"mnList": [{"proRegTxHash": h, "confirmedHash": "00" * 32, "isValid": True} for h in DISPLAY_ORDER]} -BASES = {r["epoch"]: BASE for r in FIX["epochs"]} -def replay(enforce_from=1, upto=None, **kw): +def base_for(epoch, members): + # base(k) = 24k, so base(k+1) is the list at epoch k's boundary block 24(k+1). + return {"epoch": epoch, "base": 24 * epoch, + "mnList": [{"proRegTxHash": h, "confirmedHash": "00" * 32, "isValid": True} for h in members]} + + +BASES = {e: base_for(e, DISPLAY_ORDER) for e in range(1, 12)} +# N's collateral is spent during epoch 11: the list at boundary 288 (= base 12) lacks it. +BASES[12] = base_for(12, [h for h in DISPLAY_ORDER if h != N]) +BASE = BASES[1] + + +def replay(enforce_from=1, upto=None, bases=None, enforce_from_height=None, **kw): rows = [r for r in FIX["epochs"] if upto is None or r["epoch"] <= upto] - rp, skipped = mod.run(rows, BASES, enforce_from, **kw) + rp, skipped = mod.run(rows, BASES if bases is None else bases, enforce_from, enforce_from_height, **kw) return rp, skipped @@ -76,9 +90,11 @@ def test_the_full_story_with_enforcement(self): [(5, "suspended", A), (6, "banned", A), (9, "recovered", A)]) # The ban height is the boundary block of the epoch that reached the count. self.assertEqual([h for _, h, k, _ in rp.events if k == "banned"], [168]) - # Final state: A recovered, B missed once at epoch 9, nobody else touched. + # Final state: A recovered; B missed once at epoch 9 and was online again at 11; + # nobody else touched (N's miss at 11 does not count: it had left the list). self.assertEqual(rp.counter[A], 0) - self.assertEqual(rp.counter[B], 1) + self.assertEqual(rp.counter[B], 0) + self.assertEqual(replay(enforce_from=1, upto=9)[0].counter[B], 1) self.assertEqual(rp.suspended, set()) self.assertEqual(rp.banned, {}) for k in NAMES[2:]: @@ -123,13 +139,64 @@ def test_size_mismatch_is_skipped_with_a_warning(self): # Nothing from epoch 10 leaked into the state: C (index 11) was never counted # (epoch 8, its other miss, was a mass outage). self.assertEqual(rp.counter[C], 0) - self.assertEqual(rp.table[-1][2], "size-mismatch") + self.assertEqual([t[2] for t in rp.table if t[0] == 10], ["size-mismatch"]) def test_branch_coverage_is_reported(self): rp, _ = replay(enforce_from=1) self.assertEqual(rp.hold_protected, 1) # B at epoch 4 self.assertEqual(rp.mass_protected, 1) # A at epoch 8 (B and C had no streak there) self.assertEqual(rp.absent_frozen, 1) # epoch 7 with A at 5 + self.assertEqual(rp.removed_skipped, 1) # N at epoch 11 + + +class RemovedMember(unittest.TestCase): + """deterministicmns.cpp:423 -- a member of the base list that GetMN() no longer + finds at the boundary is skipped: no counter, no last-epoch stamp, no event.""" + + def test_a_member_removed_before_the_boundary_is_skipped(self): + rp, _ = replay(enforce_from=1) + self.assertEqual(rp.counter[N], 0) # missed at 11, but gone from the list at 288 + self.assertEqual(rp.last_epoch.get(N), 9) # 10 was skipped whole; 11 never touched N + self.assertEqual(rp.removed_skipped, 1) + self.assertFalse(any("no list at the boundary" in w for w in rp.warnings)) + + def test_without_the_boundary_list_the_member_is_counted_and_the_run_says_so(self): + # NEGATIVE CONTROL: drop base 12 and the skip cannot happen; the tool must count N + # (the old behaviour) and say that it could not tell. + bases = {e: b for e, b in BASES.items() if e != 12} + rp, _ = replay(enforce_from=1, bases=bases) + self.assertEqual(rp.counter[N], 1) + self.assertEqual(rp.removed_skipped, 0) + self.assertEqual([w for w in rp.warnings if "no list at the boundary" in w], + ["epoch 11: no list at the boundary 288 (the next epoch's base): " + "a member removed during the epoch would be counted"]) + + def test_a_next_list_at_the_wrong_height_is_not_used(self): + bases = dict(BASES) + bases[12] = dict(BASES[12], base=289) + rp, _ = replay(enforce_from=1, bases=bases) + self.assertEqual(rp.counter[N], 1) + self.assertTrue(any(w.startswith("epoch 11: no list at the boundary 288") for w in rp.warnings)) + + +class EnforceByHeight(unittest.TestCase): + """The node enforces when the boundary block's height >= nDSLEnforcementHeight + (deterministicmns.cpp:419); the height flag must follow that rule exactly.""" + + def test_the_boundary_block_itself_enforces(self): + by_height, _ = replay(enforce_from=None, enforce_from_height=168) # epoch 6's boundary + by_epoch, _ = replay(enforce_from=6) + self.assertEqual(by_height.events, by_epoch.events) + self.assertEqual([(e, k) for e, _, k, p in by_height.events if p == A], + [(6, "suspended"), (6, "banned"), (9, "recovered")]) + + def test_one_block_later_misses_that_epoch(self): + # From 169: epoch 7 is absent, 8 a mass outage, and at 9 A is online without + # ever having been suspended -- so no event at all, and the counters unchanged. + rp, _ = replay(enforce_from=None, enforce_from_height=169) + self.assertEqual(rp.events, []) + # The counters do not depend on where enforcement starts. + self.assertEqual(dict(rp.counter), dict(replay(enforce_from=1)[0].counter)) class CrossCheck(unittest.TestCase):