From 1bee26395e3d16907a1cfcd7806d37bf45007016 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:16:53 +0000 Subject: [PATCH 1/8] fix(consensus): re-offer same-round votes the peer never acknowledged Vote gossip marks a vote as held by the peer the moment it is handed to the transport, and nothing inside a round puts it back on offer: a copy dropped after enqueue (router queue overflow, receiver over its allowance) is never sent again. The only in-round repair, the VoteSetMaj23 exchange, needs one side to already hold a +2/3 majority, so when the lost votes are exactly what both sides lack to reach one, the round stalls with no timeout to fall back on (prevote-wait starts only on HasTwoThirdsAny). PeerState now tracks current-round prevotes and precommits that were sent but not confirmed by the peer (HasVote, VoteSetBits, or the peer's own copy), in two redelivery windows rotated once per interval. A vote still unconfirmed after a full window is unmarked so regular gossip offers it again; the window scheme means nothing is resent while its acknowledgement is merely in flight, and bookkeeping of a round the peer has left is dropped. The interval follows PeerQueryMaj23SleepDuration, the cadence of the existing periodic repair. Co-Authored-By: Claude Fable 5.1 --- internal/consensus/gossiper.go | 5 +- internal/consensus/gossiper_test.go | 119 ++++++++++++++++++++ internal/consensus/peer_state.go | 155 +++++++++++++++++++++++++- internal/consensus/peer_state_test.go | 63 +++++++++++ internal/consensus/reactor.go | 3 +- 5 files changed, 341 insertions(+), 4 deletions(-) diff --git a/internal/consensus/gossiper.go b/internal/consensus/gossiper.go index 1e2ed80fe2..3ecd30c732 100644 --- a/internal/consensus/gossiper.go +++ b/internal/consensus/gossiper.go @@ -518,6 +518,9 @@ func (g *msgGossiper) GossipCommit(ctx context.Context, rs cstypes.RoundState, p // GossipVote sends a vote message to the peer func (g *msgGossiper) GossipVote(ctx context.Context, rs cstypes.RoundState, prs *cstypes.PeerRoundState) { + // Sent votes are marked as the peer's on the spot; votes it never confirmed + // are periodically put back on offer so a copy lost in transit is resent. + g.ps.ReofferUnackedVotes() vote, found := g.pickVoteForGossip(rs, prs) if !found { return @@ -694,7 +697,7 @@ func updatePeerCommit(ps *PeerState, commit *types.Commit) func() error { func updatePeerVote(ps *PeerState, vote *types.Vote) func() error { return func() error { - return ps.SetHasVote(vote) + return ps.SetVoteSent(vote) } } diff --git a/internal/consensus/gossiper_test.go b/internal/consensus/gossiper_test.go index 1b0238707e..d574b68f60 100644 --- a/internal/consensus/gossiper_test.go +++ b/internal/consensus/gossiper_test.go @@ -1547,3 +1547,122 @@ func (suite *GossiperSuiteTest) signVote(vote *types.Vote) { err = vote.VoteExtensions.CopySignsFromProto(protoVote.VoteExtensions) suite.Require().NoError(err) } + +// TestGossipVoteRedeliversUnacknowledged verifies that a current-round vote +// marked delivered on send is offered again, at a bounded cadence, while the +// peer stays in that round without acknowledging it — and not again once it +// does. Neither side holds a majority, so no VoteSetMaj23 reconciliation can +// restore the vote to the offer; without redelivery a lost copy would leave +// the round stalled. +func (suite *GossiperSuiteTest) TestGossipVoteRedeliversUnacknowledged() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + const voteResendInterval = time.Second + + for _, tc := range []struct { + name string + voteType tmproto.SignedMsgType + step cstypes.RoundStepType + }{ + {"prevote", tmproto.PrevoteType, cstypes.RoundStepPrevote}, + {"precommit", tmproto.PrecommitType, cstypes.RoundStepPrecommit}, + } { + suite.Run(tc.name, func() { + ps := NewPeerState(suite.logger, suite.ps.peerID, + WithPeerStateClock(suite.clock), + WithVoteResendInterval(voteResendInterval)) + suite.ps, suite.sender.ps, suite.gossiper.ps = ps, ps, ps + + vote := suite.makeSignedVote(100, 0, tc.voteType) + votes := cstypes.NewHeightVoteSet(factory.DefaultTestChainID, 100, suite.valSet) + _, _ = votes.AddVote(vote) + rs := cstypes.RoundState{Height: 100, Round: 0, Votes: votes} + ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: tc.step, ProposalPOLRound: -1} + + sends := 0 + suite.voteCh. + On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: vote.ToProto()}}). + Run(func(_ mock.Arguments) { sends++ }). + Return(nil) + + // The vote goes out once and is then held back as delivered. + for range 10 { + suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState()) + } + suite.Require().Equal(1, sends, "a vote just sent must not be repeated within the interval") + + // Still unacknowledged after the interval: offered again, once. + suite.clock.Advance(voteResendInterval) + for range 10 { + suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState()) + } + suite.clock.Advance(voteResendInterval) + for range 10 { + suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState()) + } + suite.Require().Equal(2, sends, "an unacknowledged vote must be resent once the interval elapses") + + // The peer reports it has the vote: redelivery stops. + suite.Require().NoError(ps.ApplyHasVoteMessage(&HasVoteMessage{ + Height: 100, Round: 0, Type: tc.voteType, Index: vote.ValidatorIndex, + })) + for range 3 { + suite.clock.Advance(voteResendInterval) + for range 10 { + suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState()) + } + } + suite.Require().Equal(2, sends, "an acknowledged vote must not be resent") + }) + } +} + +// TestGossipVoteRedeliveryStopsWhenPeerMovesOn verifies that an unacknowledged +// vote is no longer pushed once the peer has left the round it belongs to: +// the peer's new round keeps the regular offer rules, so stale bookkeeping +// from the old round must not clear bits in it. +func (suite *GossiperSuiteTest) TestGossipVoteRedeliveryStopsWhenPeerMovesOn() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + const voteResendInterval = time.Second + + ps := NewPeerState(suite.logger, suite.ps.peerID, + WithPeerStateClock(suite.clock), + WithVoteResendInterval(voteResendInterval)) + suite.ps, suite.sender.ps, suite.gossiper.ps = ps, ps, ps + + prevoteR0 := suite.makeSignedVote(100, 0, tmproto.PrevoteType) + prevoteR1 := suite.makeSignedVote(100, 1, tmproto.PrevoteType) + votes := cstypes.NewHeightVoteSet(factory.DefaultTestChainID, 100, suite.valSet) + _, _ = votes.AddVote(prevoteR0) + _, _ = votes.AddVote(prevoteR1) + ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: cstypes.RoundStepPrevote, ProposalPOLRound: -1} + + suite.voteCh. + On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: prevoteR0.ToProto()}}). + Once(). + Return(nil) + suite.gossiper.GossipVote(ctx, cstypes.RoundState{Height: 100, Round: 0, Votes: votes}, ps.GetRoundState()) + + // Peer advances to round 1 before acknowledging; the round-1 prevote is + // sent and acknowledged normally. + ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 100, Round: 1, Step: cstypes.RoundStepPrevote, LastCommitRound: -1}) + rsR1 := cstypes.RoundState{Height: 100, Round: 1, Votes: votes} + suite.voteCh. + On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: prevoteR1.ToProto()}}). + Once(). + Return(nil) + suite.gossiper.GossipVote(ctx, rsR1, ps.GetRoundState()) + suite.Require().NoError(ps.ApplyHasVoteMessage(&HasVoteMessage{ + Height: 100, Round: 1, Type: tmproto.PrevoteType, Index: prevoteR1.ValidatorIndex, + })) + + // Nothing from round 0 may be resent, and the acknowledged round-1 vote + // must not be cleared by the stale round-0 bookkeeping. + for range 3 { + suite.clock.Advance(voteResendInterval) + for range 10 { + suite.gossiper.GossipVote(ctx, rsR1, ps.GetRoundState()) + } + } +} diff --git a/internal/consensus/peer_state.go b/internal/consensus/peer_state.go index 3aa668f596..3f4e7f9fdd 100644 --- a/internal/consensus/peer_state.go +++ b/internal/consensus/peer_state.go @@ -46,12 +46,16 @@ func (pss peerStateStats) String() string { type PeerState struct { peerID types.NodeID logger log.Logger - // clock is the time source the majority-claim answer memory is dated - // against; the wall clock unless a test overrides it. + // clock is the time source the majority-claim answer memory and the vote + // redelivery windows are dated against; the wall clock unless a test + // overrides it. clock clockwork.Clock // maj23AnswerTTL is how long an answer keeps the same majority claim from // being answered again. maj23AnswerTTL time.Duration + // voteResendInterval is how long a vote handed to the transport stays off + // the offer before, still unacknowledged, it is offered again. + voteResendInterval time.Duration // NOTE: Modify below using setters, never directly. mtx sync.RWMutex @@ -67,6 +71,11 @@ type PeerState struct { answeredMaj23 [maj23AnswerMemory]maj23Answer answeredMaj23Next int + // unacked are the current-round votes gossiped to the peer that it has not + // acknowledged yet; voteResendAt is when they are next due for re-offer. + unacked unackedVotes + voteResendAt time.Time + // ProTxHash is accessible only for the validator ProTxHash types.ProTxHash @@ -101,6 +110,15 @@ func WithMaj23AnswerTTL(ttl time.Duration) PeerStateOptionFunc { } } +// WithVoteResendInterval sets how long a vote gossiped to the peer stays off the +// offer before, still unacknowledged, it is offered again. Sized against the +// majority-claim gossip cadence by default (see ReofferUnackedVotes). +func WithVoteResendInterval(interval time.Duration) PeerStateOptionFunc { + return func(ps *PeerState) { + ps.voteResendInterval = interval + } +} + // NewPeerState returns a new PeerState for the given node ID. func NewPeerState(logger log.Logger, peerID types.NodeID, opts ...PeerStateOptionFunc) *PeerState { ps := &PeerState{ @@ -121,6 +139,9 @@ func NewPeerState(logger log.Logger, peerID types.NodeID, opts ...PeerStateOptio if ps.maj23AnswerTTL <= 0 { ps.maj23AnswerTTL = maj23AnswerTTLFor(0) } + if ps.voteResendInterval <= 0 { + ps.voteResendInterval = config.DefaultConsensusConfig().PeerQueryMaj23SleepDuration + } return ps } @@ -536,9 +557,138 @@ func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.Sign return ErrPeerStateInvalidVoteIndex } } + ps.ackVote(height, round, voteType, index) return nil } +// unackedVotes are the votes of one height/round that were handed to the +// transport for the peer and that the peer has not confirmed having, split by +// the redelivery window they were sent in: recent since the last rotation, due +// before it. Slots follow unackedVoteTypes. +type unackedVotes struct { + height int64 + round int32 + due [len(unackedVoteTypes)]*bits.BitArray + recent [len(unackedVoteTypes)]*bits.BitArray +} + +var unackedVoteTypes = [...]tmproto.SignedMsgType{tmproto.PrevoteType, tmproto.PrecommitType} + +func unackedVoteSlot(voteType tmproto.SignedMsgType) (int, bool) { + for slot, t := range unackedVoteTypes { + if t == voteType { + return slot, true + } + } + return 0, false +} + +// SetVoteSent records that the vote was handed to the transport for the peer. +// The peer is treated as having it, so it is not offered again — until a +// redelivery interval passes without the peer confirming it, which puts it +// back on offer (see ReofferUnackedVotes). Only the peer's current round is +// tracked: votes for any other round are marked exactly like SetHasVote. +func (ps *PeerState) SetVoteSent(vote *types.Vote) error { + if vote == nil { + return ErrPeerStateSetNilVote + } + ps.mtx.Lock() + defer ps.mtx.Unlock() + + if err := ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex); err != nil { + return err + } + slot, ok := unackedVoteSlot(vote.Type) + if !ok || vote.Height != ps.PRS.Height || vote.Round != ps.PRS.Round { + return nil + } + psVotes := ps.getVoteBitArray(vote.Height, vote.Round, vote.Type) + if psVotes == nil { + return nil + } + if ps.unacked.height != vote.Height || ps.unacked.round != vote.Round { + ps.unacked = unackedVotes{height: vote.Height, round: vote.Round} + } + if ps.unacked.recent[slot] == nil { + ps.unacked.recent[slot] = bits.NewBitArray(psVotes.Size()) + } + ps.unacked.recent[slot].SetIndex(int(vote.ValidatorIndex), true) + return nil +} + +// ackVote drops a vote from redelivery tracking: the peer has shown it has it. +func (ps *PeerState) ackVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) { + slot, ok := unackedVoteSlot(voteType) + if !ok || ps.unacked.height != height || ps.unacked.round != round { + return + } + ps.unacked.due[slot].SetIndex(int(index), false) + ps.unacked.recent[slot].SetIndex(int(index), false) +} + +// ackVoteBits drops every vote set in acked from redelivery tracking. +func (ps *PeerState) ackVoteBits(height int64, round int32, voteType tmproto.SignedMsgType, acked *bits.BitArray) { + slot, ok := unackedVoteSlot(voteType) + if !ok || acked == nil || ps.unacked.height != height || ps.unacked.round != round { + return + } + if ps.unacked.due[slot] != nil { + ps.unacked.due[slot] = ps.unacked.due[slot].Sub(acked) + } + if ps.unacked.recent[slot] != nil { + ps.unacked.recent[slot] = ps.unacked.recent[slot].Sub(acked) + } +} + +// ReofferUnackedVotes puts current-round votes the peer has not acknowledged +// back on offer, once per voteResendInterval. +// +// Gossip marks a vote as the peer's the moment it is handed to the transport; +// a copy dropped after that (router queue overflow, receiver over its +// allowance) is otherwise never sent again, and the only in-round repair — a +// VoteSetMaj23 exchange — needs a +2/3 majority on one side. When the lost +// votes are exactly what both sides lack to reach one, the round stalls with +// no timeout to fall back on. +// +// Each rotation clears the peer's bits for votes sent before the previous +// rotation and promotes those sent since, so an unacknowledged vote is offered +// again between one and two intervals after it went out, while one whose +// HasVote is merely in flight is not: the healthy path sends nothing twice. +// Bookkeeping of a round the peer has left is dropped, never applied to the +// new round's bits. +func (ps *PeerState) ReofferUnackedVotes() { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + now := ps.clock.Now() + if now.Before(ps.voteResendAt) { + return + } + ps.voteResendAt = now.Add(ps.voteResendInterval) + + if ps.unacked.height != ps.PRS.Height || ps.unacked.round != ps.PRS.Round { + ps.unacked = unackedVotes{} + return + } + reoffered := 0 + for slot, due := range ps.unacked.due { + if due == nil { + continue + } + psVotes := ps.getVoteBitArray(ps.unacked.height, ps.unacked.round, unackedVoteTypes[slot]) + if psVotes == nil { + continue + } + reoffered += due.CountTrueBits() + psVotes.Update(psVotes.Sub(due)) + } + if reoffered > 0 { + ps.logger.Debug("re-offering votes the peer has not acknowledged", + "height", ps.unacked.height, "round", ps.unacked.round, "votes", reoffered) + } + ps.unacked.due, ps.unacked.recent = ps.unacked.recent, [len(unackedVoteTypes)]*bits.BitArray{} +} + // SetHasCommit sets the given vote as known by the peer func (ps *PeerState) SetHasCommit(commit *types.Commit) { ps.mtx.Lock() @@ -842,6 +992,7 @@ func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes * votes.Update(hasVotes) } } + ps.ackVoteBits(msg.Height, msg.Round, msg.Type, msg.Votes) } // String returns a string representation of the PeerState diff --git a/internal/consensus/peer_state_test.go b/internal/consensus/peer_state_test.go index 9c4b6fdcbc..af720cc01a 100644 --- a/internal/consensus/peer_state_test.go +++ b/internal/consensus/peer_state_test.go @@ -2,7 +2,9 @@ package consensus import ( "testing" + "time" + "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" "github.com/dashpay/tenderdash/libs/log" @@ -99,3 +101,64 @@ func TestApplyHasVoteMessage(t *testing.T) { // skip test cases like v & v3 in TestSetHasVote due to the same path } + +// TestVoteRedeliveryWindows pins the cadence contract of vote redelivery: a vote +// recorded as sent is unmarked only once a full interval has elapsed since the +// rotation that followed its send — never before one interval, always by two — +// and any acknowledgement in between cancels it. +func TestVoteRedeliveryWindows(t *testing.T) { + const interval = time.Second + clock := clockwork.NewFakeClock() + ps := NewPeerState(log.NewNopLogger(), "testPeerState", + WithPeerStateClock(clock), WithVoteResendInterval(interval)) + ps.PRS.Height, ps.PRS.Round = 1, 1 + ps.ensureVoteBitArrays(1, 3) + vote := func(idx int32) *types.Vote { + return &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: idx} + } + + // First rotation arms the clock; a vote sent right after it sits in the + // current window. + ps.ReofferUnackedVotes() + require.NoError(t, ps.SetVoteSent(vote(0))) + require.True(t, ps.PRS.Prevotes.GetIndex(0)) + + // One interval later it is promoted but not yet re-offered ... + clock.Advance(interval) + ps.ReofferUnackedVotes() + require.True(t, ps.PRS.Prevotes.GetIndex(0), "a vote sent within the last interval stays marked") + + // ... a vote sent now joins the next window ... + require.NoError(t, ps.SetVoteSent(vote(1))) + // ... and an acknowledgement withdraws vote 2 before its window is due. + require.NoError(t, ps.SetVoteSent(vote(2))) + require.NoError(t, ps.ApplyHasVoteMessage(&HasVoteMessage{Height: 1, Round: 1, Type: tmproto.PrevoteType, Index: 2})) + + // Before the interval nothing changes, however often it is asked. + clock.Advance(interval - time.Nanosecond) + for range 5 { + ps.ReofferUnackedVotes() + } + require.True(t, ps.PRS.Prevotes.GetIndex(0)) + require.True(t, ps.PRS.Prevotes.GetIndex(1)) + + // At the interval, vote 0 is re-offered; vote 1 waits one more rotation. + clock.Advance(time.Nanosecond) + ps.ReofferUnackedVotes() + require.False(t, ps.PRS.Prevotes.GetIndex(0), "unacknowledged vote must be re-offered after a full window") + require.True(t, ps.PRS.Prevotes.GetIndex(1)) + require.True(t, ps.PRS.Prevotes.GetIndex(2), "acknowledged vote must stay marked") + + clock.Advance(interval) + ps.ReofferUnackedVotes() + require.False(t, ps.PRS.Prevotes.GetIndex(1)) + require.True(t, ps.PRS.Prevotes.GetIndex(2), "acknowledged vote must stay marked") + + // A vote the peer already reports having is not tracked at all. + require.NoError(t, ps.SetHasVote(vote(0))) + clock.Advance(2 * interval) + ps.ReofferUnackedVotes() + clock.Advance(2 * interval) + ps.ReofferUnackedVotes() + require.True(t, ps.PRS.Prevotes.GetIndex(0), "SetHasVote records an acknowledged vote") +} diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index d9131003f3..9011868a3c 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -644,7 +644,8 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries if !ok { ps = NewPeerState(r.logger, peerUpdate.NodeID, WithPeerStateClock(r.clock), - WithMaj23AnswerTTL(maj23AnswerTTLFor(r.state.config.PeerQueryMaj23SleepDuration))) + WithMaj23AnswerTTL(maj23AnswerTTLFor(r.state.config.PeerQueryMaj23SleepDuration)), + WithVoteResendInterval(r.state.config.PeerQueryMaj23SleepDuration)) ps.SetProTxHash(peerUpdate.ProTxHash) r.peers[peerUpdate.NodeID] = ps } else if len(peerUpdate.ProTxHash) > 0 { From 54349368258212783816e93240a9ef19c540ac1f Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:31:49 +0000 Subject: [PATCH 2/8] fix(consensus): keep vote redelivery from looping on acknowledged votes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An acknowledgement applied between picking a vote and recording it as sent was discarded: SetVoteSent tracked the vote regardless, so two rotations later it was resent — and a resend can never be acknowledged, since the receiver drops the duplicate before any HasVote is published. The vote then cycled for the rest of the round. The same loop followed a HasVote lost in transit. SetVoteSent now tracks a vote only when the peer's bit was still clear at the time of the call; the per-peer gossip goroutine picks votes only while their bit is clear, so a set bit means an ack landed in that window. Marking the bit and acknowledging are split (setVoteBit / setHasVote) so the sent path cannot ack itself. Tracking is per vote, with the send time and a re-offer count: each re-offer doubles the wait before the next, capped at voteResendMaxBackoff doublings, so a vote whose ack was lost is resent at a falling rate rather than every interval, and is never given up on. Tests: cover the ack-before-sent interleaving, the backoff schedule, the VoteSetBits acknowledgement path, and make the moved-on test reach the stale-round reset (round-0 bookkeeping must not clear the peer's POL bits that now stand for round 0). Co-Authored-By: Claude Fable 5.1 --- internal/consensus/gossiper_test.go | 30 +++--- internal/consensus/peer_state.go | 113 +++++++++++++++-------- internal/consensus/peer_state_test.go | 128 ++++++++++++++++++++++---- 3 files changed, 206 insertions(+), 65 deletions(-) diff --git a/internal/consensus/gossiper_test.go b/internal/consensus/gossiper_test.go index d574b68f60..b8ba367dd4 100644 --- a/internal/consensus/gossiper_test.go +++ b/internal/consensus/gossiper_test.go @@ -1577,7 +1577,7 @@ func (suite *GossiperSuiteTest) TestGossipVoteRedeliversUnacknowledged() { votes := cstypes.NewHeightVoteSet(factory.DefaultTestChainID, 100, suite.valSet) _, _ = votes.AddVote(vote) rs := cstypes.RoundState{Height: 100, Round: 0, Votes: votes} - ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: tc.step, ProposalPOLRound: -1} + ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: tc.step, ProposalPOLRound: -1, CatchupCommitRound: -1} sends := 0 suite.voteCh. @@ -1617,10 +1617,10 @@ func (suite *GossiperSuiteTest) TestGossipVoteRedeliversUnacknowledged() { } } -// TestGossipVoteRedeliveryStopsWhenPeerMovesOn verifies that an unacknowledged -// vote is no longer pushed once the peer has left the round it belongs to: -// the peer's new round keeps the regular offer rules, so stale bookkeeping -// from the old round must not clear bits in it. +// TestGossipVoteRedeliveryStopsWhenPeerMovesOn verifies that bookkeeping of a +// round the peer has left is dropped at the next rotation rather than applied +// to the bits the old round number now stands for: here the peer's proposal +// POL bits, which name that same round under its new round state. func (suite *GossiperSuiteTest) TestGossipVoteRedeliveryStopsWhenPeerMovesOn() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1636,7 +1636,7 @@ func (suite *GossiperSuiteTest) TestGossipVoteRedeliveryStopsWhenPeerMovesOn() { votes := cstypes.NewHeightVoteSet(factory.DefaultTestChainID, 100, suite.valSet) _, _ = votes.AddVote(prevoteR0) _, _ = votes.AddVote(prevoteR1) - ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: cstypes.RoundStepPrevote, ProposalPOLRound: -1} + ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: cstypes.RoundStepPrevote, ProposalPOLRound: -1, CatchupCommitRound: -1} suite.voteCh. On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: prevoteR0.ToProto()}}). @@ -1644,10 +1644,19 @@ func (suite *GossiperSuiteTest) TestGossipVoteRedeliveryStopsWhenPeerMovesOn() { Return(nil) suite.gossiper.GossipVote(ctx, cstypes.RoundState{Height: 100, Round: 0, Votes: votes}, ps.GetRoundState()) - // Peer advances to round 1 before acknowledging; the round-1 prevote is - // sent and acknowledged normally. + // Peer moves to round 1 without acknowledging, and reports (via its + // proposal's POL) that it holds the round-0 prevote after all. ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 100, Round: 1, Step: cstypes.RoundStepPrevote, LastCommitRound: -1}) + ps.PRS.ProposalPOLRound = 0 + ps.PRS.ProposalPOL = bits.NewBitArray(suite.valSet.Size()) + ps.PRS.ProposalPOL.SetIndex(int(prevoteR0.ValidatorIndex), true) + + // The first rotation after the move sees round-0 bookkeeping against a + // round-1 peer and must drop it; the round-1 prevote then goes out once and + // is acknowledged. The round-0 prevote must never be offered again, which + // is what clearing POL bit 0 from the stale bookkeeping would cause. rsR1 := cstypes.RoundState{Height: 100, Round: 1, Votes: votes} + suite.clock.Advance(voteResendInterval) suite.voteCh. On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: prevoteR1.ToProto()}}). Once(). @@ -1656,13 +1665,12 @@ func (suite *GossiperSuiteTest) TestGossipVoteRedeliveryStopsWhenPeerMovesOn() { suite.Require().NoError(ps.ApplyHasVoteMessage(&HasVoteMessage{ Height: 100, Round: 1, Type: tmproto.PrevoteType, Index: prevoteR1.ValidatorIndex, })) - - // Nothing from round 0 may be resent, and the acknowledged round-1 vote - // must not be cleared by the stale round-0 bookkeeping. for range 3 { suite.clock.Advance(voteResendInterval) for range 10 { suite.gossiper.GossipVote(ctx, rsR1, ps.GetRoundState()) } } + suite.Require().True(ps.PRS.ProposalPOL.GetIndex(int(prevoteR0.ValidatorIndex)), + "stale round-0 bookkeeping must not clear the peer's POL bits for round 0") } diff --git a/internal/consensus/peer_state.go b/internal/consensus/peer_state.go index 3f4e7f9fdd..dfc67ff320 100644 --- a/internal/consensus/peer_state.go +++ b/internal/consensus/peer_state.go @@ -72,7 +72,7 @@ type PeerState struct { answeredMaj23Next int // unacked are the current-round votes gossiped to the peer that it has not - // acknowledged yet; voteResendAt is when they are next due for re-offer. + // acknowledged yet; voteResendAt is when they are next checked for re-offer. unacked unackedVotes voteResendAt time.Time @@ -535,8 +535,18 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) error { return ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex) } -// setHasVote will return an error when the index exceeds the bitArray length +// setHasVote records a vote the peer has shown it holds. It returns an error +// when the index exceeds the bitArray length. func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) error { + if err := ps.setVoteBit(height, round, voteType, index); err != nil { + return err + } + ps.ackVote(height, round, voteType, index) + return nil +} + +// setVoteBit marks the vote as the peer's without touching redelivery tracking. +func (ps *PeerState) setVoteBit(height int64, round int32, voteType tmproto.SignedMsgType, index int32) error { ps.logger.Trace( "peerState setHasVote", "peer", ps.peerID, @@ -557,19 +567,31 @@ func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.Sign return ErrPeerStateInvalidVoteIndex } } - ps.ackVote(height, round, voteType, index) return nil } +// voteResendMaxBackoff caps how many times the wait before re-offering one +// unacknowledged vote doubles. A vote whose HasVote was lost can never be +// acknowledged by resending it — the receiver drops the duplicate before any +// HasVote is published — so without backoff it would be resent every interval +// for the rest of the round; without a cap it would effectively be given up +// on, which is the permanent loss redelivery exists to prevent. +const voteResendMaxBackoff = 3 + +// unackedVote is one vote handed to the transport for the peer and not yet +// confirmed by it. A zero sentAt means the vote is not tracked. +type unackedVote struct { + sentAt time.Time + reoffers uint8 +} + // unackedVotes are the votes of one height/round that were handed to the -// transport for the peer and that the peer has not confirmed having, split by -// the redelivery window they were sent in: recent since the last rotation, due -// before it. Slots follow unackedVoteTypes. +// transport for the peer and that the peer has not confirmed having, indexed +// by validator index; slots follow unackedVoteTypes. type unackedVotes struct { height int64 round int32 - due [len(unackedVoteTypes)]*bits.BitArray - recent [len(unackedVoteTypes)]*bits.BitArray + votes [len(unackedVoteTypes)][]unackedVote } var unackedVoteTypes = [...]tmproto.SignedMsgType{tmproto.PrevoteType, tmproto.PrecommitType} @@ -588,6 +610,11 @@ func unackedVoteSlot(voteType tmproto.SignedMsgType) (int, bool) { // redelivery interval passes without the peer confirming it, which puts it // back on offer (see ReofferUnackedVotes). Only the peer's current round is // tracked: votes for any other round are marked exactly like SetHasVote. +// +// Gossip picks a vote only while the peer's bit for it is clear, and the send +// runs between that pick and this call, unlocked. A bit found already set here +// therefore means the peer confirmed the vote in that window, and the vote is +// not tracked: a resend could never be acknowledged. func (ps *PeerState) SetVoteSent(vote *types.Vote) error { if vote == nil { return ErrPeerStateSetNilVote @@ -595,24 +622,27 @@ func (ps *PeerState) SetVoteSent(vote *types.Vote) error { ps.mtx.Lock() defer ps.mtx.Unlock() - if err := ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex); err != nil { - return err - } slot, ok := unackedVoteSlot(vote.Type) - if !ok || vote.Height != ps.PRS.Height || vote.Round != ps.PRS.Round { - return nil + track := ok && vote.Height == ps.PRS.Height && vote.Round == ps.PRS.Round + var psVotes *bits.BitArray + if track { + psVotes = ps.getVoteBitArray(vote.Height, vote.Round, vote.Type) + track = psVotes != nil && !psVotes.GetIndex(int(vote.ValidatorIndex)) } - psVotes := ps.getVoteBitArray(vote.Height, vote.Round, vote.Type) - if psVotes == nil { + if err := ps.setVoteBit(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex); err != nil { + return err + } + if !track { return nil } if ps.unacked.height != vote.Height || ps.unacked.round != vote.Round { ps.unacked = unackedVotes{height: vote.Height, round: vote.Round} } - if ps.unacked.recent[slot] == nil { - ps.unacked.recent[slot] = bits.NewBitArray(psVotes.Size()) + if len(ps.unacked.votes[slot]) != psVotes.Size() { + ps.unacked.votes[slot] = make([]unackedVote, psVotes.Size()) } - ps.unacked.recent[slot].SetIndex(int(vote.ValidatorIndex), true) + // A re-offered vote keeps its backoff; setVoteBit validated the index. + ps.unacked.votes[slot][vote.ValidatorIndex].sentAt = ps.clock.Now() return nil } @@ -622,8 +652,9 @@ func (ps *PeerState) ackVote(height int64, round int32, voteType tmproto.SignedM if !ok || ps.unacked.height != height || ps.unacked.round != round { return } - ps.unacked.due[slot].SetIndex(int(index), false) - ps.unacked.recent[slot].SetIndex(int(index), false) + if entries := ps.unacked.votes[slot]; 0 <= index && int(index) < len(entries) { + entries[index] = unackedVote{} + } } // ackVoteBits drops every vote set in acked from redelivery tracking. @@ -632,16 +663,15 @@ func (ps *PeerState) ackVoteBits(height int64, round int32, voteType tmproto.Sig if !ok || acked == nil || ps.unacked.height != height || ps.unacked.round != round { return } - if ps.unacked.due[slot] != nil { - ps.unacked.due[slot] = ps.unacked.due[slot].Sub(acked) - } - if ps.unacked.recent[slot] != nil { - ps.unacked.recent[slot] = ps.unacked.recent[slot].Sub(acked) + for index := range ps.unacked.votes[slot] { + if acked.GetIndex(index) { + ps.unacked.votes[slot][index] = unackedVote{} + } } } // ReofferUnackedVotes puts current-round votes the peer has not acknowledged -// back on offer, once per voteResendInterval. +// back on offer, checking once per voteResendInterval. // // Gossip marks a vote as the peer's the moment it is handed to the transport; // a copy dropped after that (router queue overflow, receiver over its @@ -650,12 +680,12 @@ func (ps *PeerState) ackVoteBits(height int64, round int32, voteType tmproto.Sig // votes are exactly what both sides lack to reach one, the round stalls with // no timeout to fall back on. // -// Each rotation clears the peer's bits for votes sent before the previous -// rotation and promotes those sent since, so an unacknowledged vote is offered -// again between one and two intervals after it went out, while one whose -// HasVote is merely in flight is not: the healthy path sends nothing twice. -// Bookkeeping of a round the peer has left is dropped, never applied to the -// new round's bits. +// A vote is re-offered once a full interval has passed since it was sent — so +// one whose HasVote is merely in flight is not, and the healthy path sends +// nothing twice — and at most two intervals after, given the once-per-interval +// check. Each re-offer doubles the wait before the next, up to +// voteResendMaxBackoff doublings. Bookkeeping of a round the peer has left is +// dropped, never applied to the bits that round number now stands for. func (ps *PeerState) ReofferUnackedVotes() { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -671,22 +701,31 @@ func (ps *PeerState) ReofferUnackedVotes() { return } reoffered := 0 - for slot, due := range ps.unacked.due { - if due == nil { + for slot, entries := range ps.unacked.votes { + if len(entries) == 0 { continue } psVotes := ps.getVoteBitArray(ps.unacked.height, ps.unacked.round, unackedVoteTypes[slot]) if psVotes == nil { continue } - reoffered += due.CountTrueBits() - psVotes.Update(psVotes.Sub(due)) + for index := range entries { + entry := &entries[index] + if entry.sentAt.IsZero() || now.Sub(entry.sentAt) < ps.voteResendInterval< 0 { ps.logger.Debug("re-offering votes the peer has not acknowledged", "height", ps.unacked.height, "round", ps.unacked.round, "votes", reoffered) } - ps.unacked.due, ps.unacked.recent = ps.unacked.recent, [len(unackedVoteTypes)]*bits.BitArray{} } // SetHasCommit sets the given vote as known by the peer diff --git a/internal/consensus/peer_state_test.go b/internal/consensus/peer_state_test.go index af720cc01a..cd3cf90354 100644 --- a/internal/consensus/peer_state_test.go +++ b/internal/consensus/peer_state_test.go @@ -7,6 +7,7 @@ import ( "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" + "github.com/dashpay/tenderdash/libs/bits" "github.com/dashpay/tenderdash/libs/log" tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" @@ -103,9 +104,8 @@ func TestApplyHasVoteMessage(t *testing.T) { } // TestVoteRedeliveryWindows pins the cadence contract of vote redelivery: a vote -// recorded as sent is unmarked only once a full interval has elapsed since the -// rotation that followed its send — never before one interval, always by two — -// and any acknowledgement in between cancels it. +// recorded as sent is unmarked no sooner than one interval after it went out +// and no later than two, and any acknowledgement in between cancels it. func TestVoteRedeliveryWindows(t *testing.T) { const interval = time.Second clock := clockwork.NewFakeClock() @@ -117,48 +117,142 @@ func TestVoteRedeliveryWindows(t *testing.T) { return &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: idx} } - // First rotation arms the clock; a vote sent right after it sits in the - // current window. + // t=0: the first check arms the clock; t=0.5: vote 0 goes out. ps.ReofferUnackedVotes() + clock.Advance(interval / 2) require.NoError(t, ps.SetVoteSent(vote(0))) require.True(t, ps.PRS.Prevotes.GetIndex(0)) - // One interval later it is promoted but not yet re-offered ... - clock.Advance(interval) + // t=1: half an interval old, vote 0 stays marked. + clock.Advance(interval / 2) ps.ReofferUnackedVotes() require.True(t, ps.PRS.Prevotes.GetIndex(0), "a vote sent within the last interval stays marked") - // ... a vote sent now joins the next window ... + // t=1.5: votes 1 and 2 go out; vote 2 is acknowledged at once. + clock.Advance(interval / 2) require.NoError(t, ps.SetVoteSent(vote(1))) - // ... and an acknowledgement withdraws vote 2 before its window is due. require.NoError(t, ps.SetVoteSent(vote(2))) require.NoError(t, ps.ApplyHasVoteMessage(&HasVoteMessage{Height: 1, Round: 1, Type: tmproto.PrevoteType, Index: 2})) - // Before the interval nothing changes, however often it is asked. - clock.Advance(interval - time.Nanosecond) + // Until t=2 nothing changes, however often it is asked. + clock.Advance(interval/2 - time.Nanosecond) for range 5 { ps.ReofferUnackedVotes() } require.True(t, ps.PRS.Prevotes.GetIndex(0)) require.True(t, ps.PRS.Prevotes.GetIndex(1)) - // At the interval, vote 0 is re-offered; vote 1 waits one more rotation. + // t=2: vote 0 (1.5 intervals old) is re-offered; vote 1 (0.5) waits. clock.Advance(time.Nanosecond) ps.ReofferUnackedVotes() - require.False(t, ps.PRS.Prevotes.GetIndex(0), "unacknowledged vote must be re-offered after a full window") + require.False(t, ps.PRS.Prevotes.GetIndex(0), "unacknowledged vote must be re-offered after a full interval") require.True(t, ps.PRS.Prevotes.GetIndex(1)) require.True(t, ps.PRS.Prevotes.GetIndex(2), "acknowledged vote must stay marked") + // t=3: vote 1 is re-offered. clock.Advance(interval) ps.ReofferUnackedVotes() require.False(t, ps.PRS.Prevotes.GetIndex(1)) require.True(t, ps.PRS.Prevotes.GetIndex(2), "acknowledged vote must stay marked") - // A vote the peer already reports having is not tracked at all. + // The peer's own copy of vote 0 acknowledges it for good. require.NoError(t, ps.SetHasVote(vote(0))) - clock.Advance(2 * interval) + for range 4 { + clock.Advance(interval) + ps.ReofferUnackedVotes() + } + require.True(t, ps.PRS.Prevotes.GetIndex(0), "SetHasVote records an acknowledged vote") +} + +// TestVoteRedeliveryIgnoresAckBeforeSent covers the gossip interleaving in +// which the peer's HasVote is applied between picking a vote and recording it +// as sent (the send sits between the two, unlocked). The ack must win: a +// resend could never be acknowledged, since the receiver's duplicate check +// drops it before any HasVote is published. +func TestVoteRedeliveryIgnoresAckBeforeSent(t *testing.T) { + const interval = time.Second + clock := clockwork.NewFakeClock() + ps := NewPeerState(log.NewNopLogger(), "testPeerState", + WithPeerStateClock(clock), WithVoteResendInterval(interval)) + ps.PRS.Height, ps.PRS.Round = 1, 1 + ps.ensureVoteBitArrays(1, 1) + vote := &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: 0} + ps.ReofferUnackedVotes() - clock.Advance(2 * interval) + require.NoError(t, ps.ApplyHasVoteMessage(&HasVoteMessage{Height: 1, Round: 1, Type: tmproto.PrevoteType, Index: 0})) + require.NoError(t, ps.SetVoteSent(vote)) + for range 3 { + clock.Advance(interval) + ps.ReofferUnackedVotes() + } + require.True(t, ps.PRS.Prevotes.GetIndex(0), "a vote acknowledged before it was recorded as sent must not be re-offered") +} + +// TestVoteRedeliveryBacksOff pins the bound on a vote the peer never +// acknowledges — a lost HasVote cannot be repaired by resending, since the +// receiver drops the duplicate silently: each re-offer doubles the wait before +// the next, up to voteResendMaxBackoff, so the vote keeps being offered at a +// rate that falls off instead of every interval. +func TestVoteRedeliveryBacksOff(t *testing.T) { + const interval = time.Second + clock := clockwork.NewFakeClock() + ps := NewPeerState(log.NewNopLogger(), "testPeerState", + WithPeerStateClock(clock), WithVoteResendInterval(interval)) + ps.PRS.Height, ps.PRS.Round = 1, 1 + ps.ensureVoteBitArrays(1, 1) + vote := &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: 0} + + // rotationsUntilReoffer resends the vote, then counts the rotations (one + // per interval) until it is put back on offer. + rotationsUntilReoffer := func() int { + require.NoError(t, ps.SetVoteSent(vote)) + for n := 1; n <= 64; n++ { + clock.Advance(interval) + ps.ReofferUnackedVotes() + if !ps.PRS.Prevotes.GetIndex(0) { + return n + } + } + t.Fatal("vote was never re-offered") + return 0 + } ps.ReofferUnackedVotes() - require.True(t, ps.PRS.Prevotes.GetIndex(0), "SetHasVote records an acknowledged vote") + require.Equal(t, 1, rotationsUntilReoffer(), "first re-offer after one interval") + require.Equal(t, 2, rotationsUntilReoffer(), "second re-offer doubles the wait") + require.Equal(t, 4, rotationsUntilReoffer()) + require.Equal(t, 1< Date: Mon, 9 Nov 2026 14:09:51 +0000 Subject: [PATCH 3/8] fix(consensus): report a redelivered rejected vote once A precommit the application refuses, or a vote conflicting with one already held, is never stored, so no HasVote acknowledges it and its sender re-offers it for the rest of the round. Each copy logged at Error and, for a conflict, was appended to the evidence pool's buffer again. BlockExecutor.VerifyVoteExtension now returns sm.ErrVoteExtensionRejected, classified as peer-floodable so copies log at Debug; the first rejection of a vote at its height is logged at Warn with whether the voted block is processed. A conflicting vote is reported to the evidence pool and logged at Error once per height; copies log at Debug. Co-Authored-By: Claude Fable 5.1 --- internal/consensus/log_amplification_test.go | 59 ++++++++++++++++++++ internal/consensus/msg_handlers.go | 8 ++- internal/consensus/state_add_vote.go | 55 +++++++++++++++++- internal/consensus/state_add_vote_test.go | 54 ++++++++++++++++++ internal/state/errors.go | 1 + internal/state/execution.go | 6 +- 6 files changed, 178 insertions(+), 5 deletions(-) diff --git a/internal/consensus/log_amplification_test.go b/internal/consensus/log_amplification_test.go index d3a0b38a87..2aed736b0f 100644 --- a/internal/consensus/log_amplification_test.go +++ b/internal/consensus/log_amplification_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "strings" "testing" "time" @@ -12,6 +13,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/dashpay/tenderdash/abci/example/kvstore" + abci "github.com/dashpay/tenderdash/abci/types" "github.com/dashpay/tenderdash/dash" sm "github.com/dashpay/tenderdash/internal/state" "github.com/dashpay/tenderdash/libs/log" @@ -42,6 +45,9 @@ func TestIsPeerFloodableError(t *testing.T) { assert.True(t, isPeerFloodableError(fmt.Errorf("%w: AB12, expected CD34", types.ErrRoundSkipCertificateWrongQuorum)), "a certificate naming another quorum is refused before any signature is read") + assert.True(t, isPeerFloodableError(fmt.Errorf("verifying precommit: %w", sm.ErrVoteExtensionRejected)), + "a precommit the application refuses is never stored, so it is replayed and re-sent for the rest of the round") + assert.False(t, isPeerFloodableError(ErrPrivValidatorNotSet), "an internal fault must not be treated as floodable") assert.False(t, isPeerFloodableError(errors.New("some other internal error"))) @@ -324,3 +330,56 @@ func TestLoggingMiddlewareSeparatesLocalFaultsFromReplay(t *testing.T) { assert.Contains(t, out, `"level":"debug"`) }) } + +// voteExtensionRejectingApp refuses every vote extension, as an application +// does for a precommit of a block it has not processed. +type voteExtensionRejectingApp struct{ abci.Application } + +func (voteExtensionRejectingApp) VerifyVoteExtension( + context.Context, *abci.RequestVerifyVoteExtension, +) (*abci.ResponseVerifyVoteExtension, error) { + return &abci.ResponseVerifyVoteExtension{Status: abci.ResponseVerifyVoteExtension_REJECT}, nil +} + +// A precommit the application refuses is never stored, so no HasVote answers it +// and its sender re-offers it for the rest of the round (see +// PeerState.ReofferUnackedVotes); any peer can also replay it for free. The +// first copy is worth one Warn line to an operator; every further copy of it +// logs at Debug, and none logs at Error. +func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + inner, err := kvstore.NewMemoryApp() + require.NoError(t, err) + h := newFloodHarness(ctx, t, floodHarnessArgs{validators: 4, application: voteExtensionRejectingApp{inner}}) + stateData := h.stateData() + vote := signPrecommitWithExtensions(ctx, t, h.vss[1], &stateData, 1) + + var buf bytes.Buffer + logger, err := log.NewLogger("debug", &buf) + require.NoError(t, err) + handler := loggingMiddleware(logger)(func(ctx context.Context, sd *StateData, env msgEnvelope) error { + return h.cs.ctrl.Dispatch(ctx, &AddVoteEvent{Vote: env.Msg.(*VoteMessage).Vote, PeerID: env.PeerID}, sd) + }) + deliver := func() string { + buf.Reset() + sd := h.stateData() + env := msgEnvelope{msgInfo: msgInfo{Msg: &VoteMessage{Vote: vote}, PeerID: "peer"}} + require.NoError(t, handler(h.runCtx(ctx), &sd, env)) + return buf.String() + } + + first := deliver() + assert.NotContains(t, first, `"level":"error"`, "a rejected peer precommit must not log at Error") + assert.Equal(t, 1, strings.Count(first, `"level":"warn"`), "the first rejection is reported once:\n%s", first) + + for range 3 { + again := deliver() + assert.NotContains(t, again, `"level":"error"`, "a redelivered rejected precommit must not log at Error") + assert.NotContains(t, again, `"level":"warn"`, "a redelivered rejected precommit is not news") + assert.Contains(t, again, `"level":"debug"`) + } + stateData = h.stateData() + require.Nil(t, storedVote(&stateData, vote), "a rejected precommit is not stored") +} diff --git a/internal/consensus/msg_handlers.go b/internal/consensus/msg_handlers.go index 51659c3be7..741c82b584 100644 --- a/internal/consensus/msg_handlers.go +++ b/internal/consensus/msg_handlers.go @@ -74,7 +74,13 @@ func isPeerFloodableError(err error) bool { // block signature, and the vector is covered by no signature a peer must // forge. The rejection of the commit this node assembled from its own // precommits is not a message error: tryFinalizeCommit logs it at Error. - errors.Is(err, sm.ErrCommitExtensionsRejected) + errors.Is(err, sm.ErrCommitExtensionsRejected) || + // A precommit whose extensions the application refuses — as it does for + // a block this node has not processed — is never stored, so any peer can + // replay it for free and its sender re-offers it for the rest of the + // round (see PeerState.ReofferUnackedVotes). Its first copy is reported + // by addVoteVerifyVoteExtensionMw. + errors.Is(err, sm.ErrVoteExtensionRejected) } type msgInfoDispatcher struct { diff --git a/internal/consensus/state_add_vote.go b/internal/consensus/state_add_vote.go index e7d10c3345..f54619f904 100644 --- a/internal/consensus/state_add_vote.go +++ b/internal/consensus/state_add_vote.go @@ -9,6 +9,7 @@ import ( cstypes "github.com/dashpay/tenderdash/internal/consensus/types" tmstrings "github.com/dashpay/tenderdash/internal/libs/strings" + sm "github.com/dashpay/tenderdash/internal/state" "github.com/dashpay/tenderdash/libs/eventemitter" "github.com/dashpay/tenderdash/libs/log" tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" @@ -275,6 +276,8 @@ func addVoteVerifyVoteExtensionMw( privVal = data.(privValidator) return nil }) + // Touched only by the consensus goroutine, like every vote middleware. + var rejected rejectedVotes return func(next AddVoteFunc) AddVoteFunc { return func(ctx context.Context, stateData *StateData, vote *types.Vote) (bool, error) { // Verify VoteExtension if precommit and not nil @@ -333,6 +336,15 @@ func addVoteVerifyVoteExtensionMw( err = blockExec.blockExec.VerifyVoteExtension(ctx, vote) metrics.MarkVoteExtensionReceived(err == nil) if err != nil { + // Every copy logs at Debug (see isPeerFloodableError); the first + // is worth more. A refusal of a block not processed for the round + // is transient — the application has no context for it yet — where + // one of a processed block is a disagreement over the extensions. + if errors.Is(err, sm.ErrVoteExtensionRejected) && rejected.remember(vote) { + rs := &stateData.RoundState + log.FromCtxOrNop(ctx).Warn("application rejected a precommit's vote extensions; the precommit is not counted", + "voted_block_processed", processedFor(rs, vote.Round) && rs.ProcessedBlockID.Equals(vote.BlockID)) + } return false, err } return next(ctxWithVoteVerification(ctx, vote, verified), stateData, vote) @@ -379,6 +391,8 @@ func addVoteErrorMw(evpool evidencePool, logger log.Logger, privVal privValidato privVal = data.(privValidator) return nil }) + // Touched only by the consensus goroutine, like every vote middleware. + var reported rejectedVotes return func(next AddVoteFunc) AddVoteFunc { return func(ctx context.Context, stateData *StateData, vote *types.Vote) (bool, error) { added, err := next(ctx, stateData, vote) @@ -408,7 +422,15 @@ func addVoteErrorMw(evpool evidencePool, logger log.Logger, privVal privValidato return added, err } - // report conflicting votes to the evidence pool + // A conflicting vote is never stored, so its sender re-offers it + // for the rest of the round (see PeerState.ReofferUnackedVotes). + // The pool drops a duplicate when it forms the evidence; reporting + // it once spares the pool's buffer and the log the copies. + if !reported.remember(vote) { + logger.Debug("conflicting vote delivered again; already reported to the evidence pool", + "height", vote.Height, "round", vote.Round, "type", vote.Type, "val_index", vote.ValidatorIndex) + return added, err + } evpool.ReportConflictingVotes(voteErr.VoteA, voteErr.VoteB) logger.Error("found and sent conflicting votes to the evidence pool", "vote_a", voteErr.VoteA, @@ -418,6 +440,37 @@ func addVoteErrorMw(evpool evidencePool, logger log.Logger, privVal privValidato } } +// rejectedVotes remembers the peer votes of one height that were rejected for +// a reason no resend can change. A rejected vote is never stored, so no HasVote +// acknowledges it and its sender re-offers it for the rest of the round (see +// PeerState.ReofferUnackedVotes); remembering it lets only the first copy be +// reported. The memory is reset when the height changes. +type rejectedVotes struct { + height int64 + seen map[rejectedVoteKey]struct{} +} + +type rejectedVoteKey struct { + round int32 + voteType tmproto.SignedMsgType + validatorIndex int32 + blockKey string +} + +// remember records the vote's rejection and reports whether it is the first of +// that vote at its height. +func (r *rejectedVotes) remember(vote *types.Vote) (first bool) { + if r.seen == nil || r.height != vote.Height { + r.height, r.seen = vote.Height, map[rejectedVoteKey]struct{}{} + } + key := rejectedVoteKey{vote.Round, vote.Type, vote.ValidatorIndex, vote.BlockID.Key()} + if _, seen := r.seen[key]; seen { + return false + } + r.seen[key] = struct{}{} + return true +} + func addVoteLoggingMw() AddVoteMiddlewareFunc { return func(next AddVoteFunc) AddVoteFunc { return func(ctx context.Context, stateData *StateData, vote *types.Vote) (bool, error) { diff --git a/internal/consensus/state_add_vote_test.go b/internal/consensus/state_add_vote_test.go index 54468ecd86..35eb42d8ce 100644 --- a/internal/consensus/state_add_vote_test.go +++ b/internal/consensus/state_add_vote_test.go @@ -1,6 +1,7 @@ package consensus import ( + "bytes" "context" "errors" "fmt" @@ -696,3 +697,56 @@ func TestPolkaRetargetKeepsProposalForTheChosenBlock(t *testing.T) { assert.Equal(t, receiveTime, stateData.ProposalReceiveTime, "dropping the receive time of a surviving proposal loses its timeliness") } + +// recordingEvidencePool counts the conflicting-vote pairs reported to it. +type recordingEvidencePool struct{ reports int } + +func (p *recordingEvidencePool) ReportConflictingVotes(_, _ *types.Vote) { p.reports++ } + +// A conflicting vote is never stored, so its sender re-offers it for the rest +// of the round (see PeerState.ReofferUnackedVotes). The first copy is reported +// to the evidence pool and logged at Error; a redelivered copy is neither, as +// the pool holds that equivocation already. Another block from the same +// validator, or the same pair at a later height, is a new report. +func TestAddVoteErrorMwReportsEachConflictOnce(t *testing.T) { + valSet, privVals := factory.MockValidatorSet() + privVal := privValidator{PrivValidator: privVals[0], ProTxHash: valSet.Validators[0].ProTxHash} + pool := &recordingEvidencePool{} + var buf bytes.Buffer + logger, err := log.NewLogger("debug", &buf) + require.NoError(t, err) + + voteFor := func(height int64, block string) *types.Vote { + return &types.Vote{ + Type: tmproto.PrecommitType, Height: height, Round: 0, + ValidatorIndex: 1, ValidatorProTxHash: valSet.Validators[1].ProTxHash, + BlockID: types.BlockID{Hash: []byte(block)}, + } + } + existing := voteFor(1, "block A") + conflicting := func(_ context.Context, _ *StateData, vote *types.Vote) (bool, error) { + return false, types.NewConflictingVoteError(existing, vote) + } + mw := addVoteErrorMw(pool, logger, privVal, eventemitter.New())(conflicting) + deliver := func(vote *types.Vote) string { + buf.Reset() + _, err := mw(context.Background(), &StateData{}, vote) + require.ErrorAs(t, err, new(*types.ErrVoteConflictingVotes), "the conflict is still reported to the caller") + return buf.String() + } + + out := deliver(voteFor(1, "block B")) + assert.Equal(t, 1, pool.reports) + assert.Contains(t, out, `"level":"error"`, "the first copy of an equivocation is news") + + out = deliver(voteFor(1, "block B")) + assert.Equal(t, 1, pool.reports, "a redelivered conflicting vote is in the pool already") + assert.NotContains(t, out, `"level":"error"`) + assert.Contains(t, out, `"level":"debug"`) + + deliver(voteFor(1, "block C")) + assert.Equal(t, 2, pool.reports, "a third block from the same validator is a new equivocation") + + deliver(voteFor(2, "block B")) + assert.Equal(t, 3, pool.reports, "a new height starts afresh") +} diff --git a/internal/state/errors.go b/internal/state/errors.go index a7e84130e0..206e8ec1f5 100644 --- a/internal/state/errors.go +++ b/internal/state/errors.go @@ -8,6 +8,7 @@ import ( var ( ErrBlockRejected = errors.New("block not accepted by abci app") ErrCommitExtensionsRejected = errors.New("commit vote extensions not accepted by abci app") + ErrVoteExtensionRejected = errors.New("vote extensions not accepted by abci app") ) type ( diff --git a/internal/state/execution.go b/internal/state/execution.go index 6f90f77b91..cc001c9475 100644 --- a/internal/state/execution.go +++ b/internal/state/execution.go @@ -105,8 +105,8 @@ type Executor interface { // VerifyVoteExtension asks the application to verify vote extensions: those // of one validator's precommit when vote.ValidatorProTxHash is set, or a // commit's threshold-recovered vector, after ProcessProposal of its block, - // when it is empty. It returns an error on REJECT and panics if the ABCI - // call itself fails. + // when it is empty. It returns ErrVoteExtensionRejected on REJECT and + // panics if the ABCI call itself fails. VerifyVoteExtension(ctx context.Context, vote *types.Vote) error } @@ -718,7 +718,7 @@ func (blockExec *BlockExecutor) VerifyVoteExtension(ctx context.Context, vote *t } if !resp.IsOK() { - return errors.New("invalid vote extension") + return ErrVoteExtensionRejected } return nil From e4074b6bf6b18a9469ea266ce9e32cb536de307d Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 13:50:03 +0000 Subject: [PATCH 4/8] test(consensus): cover redelivery reset on height change at the same round Co-Authored-By: Claude Opus 5.5 --- .../peer_state_redelivery_height_test.go | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 internal/consensus/peer_state_redelivery_height_test.go diff --git a/internal/consensus/peer_state_redelivery_height_test.go b/internal/consensus/peer_state_redelivery_height_test.go new file mode 100644 index 0000000000..a0a261c6db --- /dev/null +++ b/internal/consensus/peer_state_redelivery_height_test.go @@ -0,0 +1,43 @@ +package consensus + +import ( + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" + "github.com/dashpay/tenderdash/types" +) + +// TestVoteRedeliveryDropsBookkeepingOnHeightChange verifies that redelivery +// bookkeeping of a height the peer has left is dropped even when the new +// height starts at the same round number (round 0, the common case), rather +// than being applied to the peer's new-height vote bits. +func TestVoteRedeliveryDropsBookkeepingOnHeightChange(t *testing.T) { + const interval = time.Second + clock := clockwork.NewFakeClock() + ps := NewPeerState(log.NewNopLogger(), "testPeerState", + WithPeerStateClock(clock), WithVoteResendInterval(interval)) + ps.PRS.Height, ps.PRS.Round, ps.PRS.Step = 1, 0, 0 + ps.PRS.CatchupCommitRound, ps.PRS.ProposalPOLRound, ps.PRS.LastCommitRound = -1, -1, -1 + ps.ensureVoteBitArrays(1, 2) + + ps.ReofferUnackedVotes() + require.NoError(t, ps.SetVoteSent(&types.Vote{Height: 1, Round: 0, Type: tmproto.PrecommitType, ValidatorIndex: 0})) + + // Peer commits height 1 in round 0 and enters height 2, round 0, without + // acknowledging; it then reports holding precommit 0 of height 2. + ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 2, Round: 0, Step: 1, LastCommitRound: 0}) + ps.ensureVoteBitArrays(2, 2) + require.NoError(t, ps.ApplyHasVoteMessage(&HasVoteMessage{Height: 2, Round: 0, Type: tmproto.PrecommitType, Index: 0})) + + for range 3 { + clock.Advance(interval) + ps.ReofferUnackedVotes() + } + require.True(t, ps.PRS.Precommits.GetIndex(0), + "stale height-1 bookkeeping must not re-offer the peer's height-2 precommit") +} From 1d7ba7cb7c7a87eaca7ef3d81ac0d5787b60f90a Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:12:48 +0000 Subject: [PATCH 5/8] docs(consensus): state what vote redelivery changes for the round vote resend The round vote resend handler's comments said a vote lost after the channel took it is never sent again; it is re-offered after a redelivery interval, with backoff. peer-query-maj23-sleep-duration also paces that redelivery, which its config comment, TOML template and docs now say, and the WithVoteResendInterval comment names where the interval comes from. Co-Authored-By: Claude Fable 5.1 --- config/config.go | 7 ++++++- config/toml.go | 4 +++- docs/nodes/configuration.md | 7 ++++++- internal/consensus/gossip_handlers.go | 15 +++++++++------ internal/consensus/gossip_peer_worker.go | 8 ++++---- internal/consensus/peer_state.go | 6 ++++-- 6 files changed, 32 insertions(+), 15 deletions(-) diff --git a/config/config.go b/config/config.go index 9662eff831..a8d5b480e6 100644 --- a/config/config.go +++ b/config/config.go @@ -1167,7 +1167,12 @@ type ConsensusConfig struct { // tick then already exceeds the gap it is meant to enforce. Lowering it // does not shrink the gap — it only fits more ticks (and so more sends of // a multi-part block) into that same fixed window before it applies. - PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"` + PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"` + // PeerQueryMaj23SleepDuration is the cadence of the per-peer majority-claim + // gossip and of the repairs paced against it: round skip certificate and + // round vote resends, and the redelivery of votes a peer has not + // acknowledged (one interval after a vote went out, then doubling up to 8 + // intervals). Raising it to cut majority-claim chatter slows stall recovery. PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer-query-maj23-sleep-duration"` // PeerVoteRateLimit bounds the per-second token budget a single peer may diff --git a/config/toml.go b/config/toml.go index 904b26bb26..78159d26b3 100644 --- a/config/toml.go +++ b/config/toml.go @@ -566,7 +566,9 @@ create-empty-blocks-interval = "{{ .Consensus.CreateEmptyBlocksInterval }}" # Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the # tick rate a lagging peer's catch-up block-part replay is metered against # (a fixed 500ms internal throttle): raising this above 500ms disables that -# throttle. +# throttle. peer-query-maj23-sleep-duration also paces the redelivery of votes a +# peer has not acknowledged: one interval after a vote went out, then doubling +# up to 8 intervals. Raising it slows stall recovery. peer-gossip-sleep-duration = "{{ .Consensus.PeerGossipSleepDuration }}" peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" diff --git a/docs/nodes/configuration.md b/docs/nodes/configuration.md index 8c50909394..09b168ae2b 100644 --- a/docs/nodes/configuration.md +++ b/docs/nodes/configuration.md @@ -407,7 +407,12 @@ double-sign-check-height = 0 create-empty-blocks = true create-empty-blocks-interval = "0s" -# Reactor sleep duration parameters +# Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the +# tick rate a lagging peer's catch-up block-part replay is metered against +# (a fixed 500ms internal throttle): raising this above 500ms disables that +# throttle. peer-query-maj23-sleep-duration also paces the redelivery of votes a +# peer has not acknowledged: one interval after a vote went out, then doubling +# up to 8 intervals. Raising it slows stall recovery. peer-gossip-sleep-duration = "100ms" peer-query-maj23-sleep-duration = "2s" diff --git a/internal/consensus/gossip_handlers.go b/internal/consensus/gossip_handlers.go index 122de500fd..92d1c5bf5c 100644 --- a/internal/consensus/gossip_handlers.go +++ b/internal/consensus/gossip_handlers.go @@ -68,12 +68,15 @@ func roundSkipGossipHandler( // transactions in this node's round, which a skip certificate began, or is in // a round this node has left. // -// Vote gossip takes a vote as the peer's once the channel takes it, so one -// lost after that is not sent again. On a chain that skips rounds, a peer -// that lost the votes of its round can be left in it for good: a waiting -// round has no timeout and its proposer's heartbeats keep arriving, and the -// prevote and precommit steps start their timeouts only on votes. Votes are -// picked at random, so over the passes the peer gets each one this node has. +// Vote gossip takes a vote as the peer's once the channel takes it; one lost +// after that is offered again only after a redelivery interval, then with +// backoff (see PeerState.ReofferUnackedVotes), and a later round's votes are +// not offered to a peer in an earlier round at all. On a chain that skips +// rounds, a peer that lost the votes of its round waits all that time: a +// waiting round has no timeout and its proposer's heartbeats keep arriving, +// and the prevote and precommit steps start their timeouts only on votes. +// Votes are picked at random, so over the passes the peer gets each one this +// node has. // // A peer waiting in this node's round, after round 0, is sent the round's // prevotes: one ends the wait (see StateData.waitsAfterRoundSkip). A peer in a diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index 22cdcd5029..c3f405aae4 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -78,10 +78,10 @@ func newPeerGossipWorker( ), // So is a vote of its round, or of a later round +2/3 voted in, to // a peer that waits in this node's round, or is in a round this - // node left: a vote lost after the channel took it is not sent - // again, and the peer may have no timeout to fall back on. It - // stops once the peer announces the propose step, or this node's - // round. + // node left: ordinary gossip re-offers a lost vote only after a + // redelivery interval, never one of a later round, and the peer + // may have no timeout to fall back on. It stops once the peer + // announces the propose step, or this node's round. newGossipHandler( roundVoteResendGossipHandler(ps, logger, msgSender.send), state.config.PeerQueryMaj23SleepDuration, diff --git a/internal/consensus/peer_state.go b/internal/consensus/peer_state.go index 6bba7e2c26..817d6e0c43 100644 --- a/internal/consensus/peer_state.go +++ b/internal/consensus/peer_state.go @@ -116,8 +116,10 @@ func WithMaj23AnswerTTL(ttl time.Duration) PeerStateOptionFunc { } // WithVoteResendInterval sets how long a vote gossiped to the peer stays off the -// offer before, still unacknowledged, it is offered again. Sized against the -// majority-claim gossip cadence by default (see ReofferUnackedVotes). +// offer before, still unacknowledged, it is offered again (see +// ReofferUnackedVotes). The reactor passes PeerQueryMaj23SleepDuration, the +// cadence of the in-round repairs the redelivery backs up; NewPeerState falls +// back to its default. func WithVoteResendInterval(interval time.Duration) PeerStateOptionFunc { return func(ps *PeerState) { ps.voteResendInterval = interval From c1794d84f6e91f123104e04fa50ff3a91c6fc5eb Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:12:48 +0000 Subject: [PATCH 6/8] test(consensus): share the PeerState vote redelivery fixture Co-Authored-By: Claude Fable 5.1 --- internal/consensus/peer_state_test.go | 36 ++++++++++++--------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/internal/consensus/peer_state_test.go b/internal/consensus/peer_state_test.go index cd3cf90354..d08ff2d033 100644 --- a/internal/consensus/peer_state_test.go +++ b/internal/consensus/peer_state_test.go @@ -103,16 +103,24 @@ func TestApplyHasVoteMessage(t *testing.T) { // skip test cases like v & v3 in TestSetHasVote due to the same path } +// newRedeliveryPeerState returns a PeerState re-offering unacknowledged votes +// after interval on a fake clock, with its peer at height 1, round 1, and vote +// bit arrays for n validators. +func newRedeliveryPeerState(interval time.Duration, n int) (*PeerState, *clockwork.FakeClock) { + clock := clockwork.NewFakeClock() + ps := NewPeerState(log.NewNopLogger(), "testPeerState", + WithPeerStateClock(clock), WithVoteResendInterval(interval)) + ps.PRS.Height, ps.PRS.Round = 1, 1 + ps.ensureVoteBitArrays(1, n) + return ps, clock +} + // TestVoteRedeliveryWindows pins the cadence contract of vote redelivery: a vote // recorded as sent is unmarked no sooner than one interval after it went out // and no later than two, and any acknowledgement in between cancels it. func TestVoteRedeliveryWindows(t *testing.T) { const interval = time.Second - clock := clockwork.NewFakeClock() - ps := NewPeerState(log.NewNopLogger(), "testPeerState", - WithPeerStateClock(clock), WithVoteResendInterval(interval)) - ps.PRS.Height, ps.PRS.Round = 1, 1 - ps.ensureVoteBitArrays(1, 3) + ps, clock := newRedeliveryPeerState(interval, 3) vote := func(idx int32) *types.Vote { return &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: idx} } @@ -171,11 +179,7 @@ func TestVoteRedeliveryWindows(t *testing.T) { // drops it before any HasVote is published. func TestVoteRedeliveryIgnoresAckBeforeSent(t *testing.T) { const interval = time.Second - clock := clockwork.NewFakeClock() - ps := NewPeerState(log.NewNopLogger(), "testPeerState", - WithPeerStateClock(clock), WithVoteResendInterval(interval)) - ps.PRS.Height, ps.PRS.Round = 1, 1 - ps.ensureVoteBitArrays(1, 1) + ps, clock := newRedeliveryPeerState(interval, 1) vote := &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: 0} ps.ReofferUnackedVotes() @@ -195,11 +199,7 @@ func TestVoteRedeliveryIgnoresAckBeforeSent(t *testing.T) { // rate that falls off instead of every interval. func TestVoteRedeliveryBacksOff(t *testing.T) { const interval = time.Second - clock := clockwork.NewFakeClock() - ps := NewPeerState(log.NewNopLogger(), "testPeerState", - WithPeerStateClock(clock), WithVoteResendInterval(interval)) - ps.PRS.Height, ps.PRS.Round = 1, 1 - ps.ensureVoteBitArrays(1, 1) + ps, clock := newRedeliveryPeerState(interval, 1) vote := &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: 0} // rotationsUntilReoffer resends the vote, then counts the rotations (one @@ -228,11 +228,7 @@ func TestVoteRedeliveryBacksOff(t *testing.T) { // as acknowledgement for the votes it covers and for those only. func TestVoteRedeliveryVoteSetBitsAck(t *testing.T) { const interval = time.Second - clock := clockwork.NewFakeClock() - ps := NewPeerState(log.NewNopLogger(), "testPeerState", - WithPeerStateClock(clock), WithVoteResendInterval(interval)) - ps.PRS.Height, ps.PRS.Round = 1, 1 - ps.ensureVoteBitArrays(1, 2) + ps, clock := newRedeliveryPeerState(interval, 2) blockID := types.BlockID{Hash: []byte("block")} ps.ReofferUnackedVotes() From a4829e9ff0e4410eeb96c58d8d8089a048f2aa79 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:23:01 +0000 Subject: [PATCH 7/8] fix(consensus): warn only for extensions refused of a processed block A precommit the application refuses for a block this node has not processed is expected and now logs at Debug only; the first refusal of a processed block logs at Warn, counted from when the block was processed, so an early refusal cannot hide a later disagreement over the extensions. The rejected-vote memory is capped: nothing stored bounds a rejected vote and a validator key signs one for any round and block, so past the cap a vote is reported on every copy rather than remembered. Co-Authored-By: Claude Fable 5.1 --- internal/consensus/log_amplification_test.go | 39 +++++++++++++++----- internal/consensus/state_add_vote.go | 31 +++++++++++----- internal/consensus/state_add_vote_test.go | 25 +++++++++++++ 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/internal/consensus/log_amplification_test.go b/internal/consensus/log_amplification_test.go index 2aed736b0f..f00f9f5b4b 100644 --- a/internal/consensus/log_amplification_test.go +++ b/internal/consensus/log_amplification_test.go @@ -343,9 +343,11 @@ func (voteExtensionRejectingApp) VerifyVoteExtension( // A precommit the application refuses is never stored, so no HasVote answers it // and its sender re-offers it for the rest of the round (see -// PeerState.ReofferUnackedVotes); any peer can also replay it for free. The -// first copy is worth one Warn line to an operator; every further copy of it -// logs at Debug, and none logs at Error. +// PeerState.ReofferUnackedVotes); any peer can also replay it for free. A +// refusal of a block this node has not processed is expected and logs at Debug +// only, on every copy. Once the block is processed, the same refusal is a +// disagreement over the extensions: its first copy is worth one Warn line to an +// operator, every further copy logs at Debug, and none logs at Error. func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -353,8 +355,16 @@ func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { inner, err := kvstore.NewMemoryApp() require.NoError(t, err) h := newFloodHarness(ctx, t, floodHarnessArgs{validators: 4, application: voteExtensionRejectingApp{inner}}) - stateData := h.stateData() - vote := signPrecommitWithExtensions(ctx, t, h.vss[1], &stateData, 1) + ctx = h.runCtx(ctx) + sd := h.stateData() + _, block := decideProposal(ctx, t, h.cs, h.vss[0], sd.Height, 0) + parts, err := block.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + blockID := block.BlockID(parts) + exts := testPrecommitVote(1).VoteExtensions + vote, err := h.vss[1].signVote(ctx, tmproto.PrecommitType, sd.state.ChainID, blockID, + sd.Validators.QuorumType, sd.Validators.QuorumHash, exts) + require.NoError(t, err) var buf bytes.Buffer logger, err := log.NewLogger("debug", &buf) @@ -364,15 +374,26 @@ func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { }) deliver := func() string { buf.Reset() - sd := h.stateData() env := msgEnvelope{msgInfo: msgInfo{Msg: &VoteMessage{Vote: vote}, PeerID: "peer"}} - require.NoError(t, handler(h.runCtx(ctx), &sd, env)) + require.NoError(t, handler(ctx, &sd, env)) + require.Nil(t, storedVote(&sd, vote), "a rejected precommit is not stored") return buf.String() } + for range 2 { + unprocessed := deliver() + assert.NotContains(t, unprocessed, `"level":"error"`, "a rejected peer precommit must not log at Error") + assert.NotContains(t, unprocessed, `"level":"warn"`, "refusing a block not processed yet is expected:\n%s", unprocessed) + assert.Contains(t, unprocessed, `"level":"debug"`) + } + + sd.ProposalBlock, sd.ProposalBlockParts = block, parts + require.NoError(t, h.cs.blockExecutor.ensureProcess(ctx, &sd.RoundState, sd.Round)) + require.True(t, processedFor(&sd.RoundState, sd.Round)) + first := deliver() assert.NotContains(t, first, `"level":"error"`, "a rejected peer precommit must not log at Error") - assert.Equal(t, 1, strings.Count(first, `"level":"warn"`), "the first rejection is reported once:\n%s", first) + assert.Equal(t, 1, strings.Count(first, `"level":"warn"`), "the first refusal of a processed block is reported once:\n%s", first) for range 3 { again := deliver() @@ -380,6 +401,4 @@ func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { assert.NotContains(t, again, `"level":"warn"`, "a redelivered rejected precommit is not news") assert.Contains(t, again, `"level":"debug"`) } - stateData = h.stateData() - require.Nil(t, storedVote(&stateData, vote), "a rejected precommit is not stored") } diff --git a/internal/consensus/state_add_vote.go b/internal/consensus/state_add_vote.go index f54619f904..9bf5b3075f 100644 --- a/internal/consensus/state_add_vote.go +++ b/internal/consensus/state_add_vote.go @@ -336,14 +336,17 @@ func addVoteVerifyVoteExtensionMw( err = blockExec.blockExec.VerifyVoteExtension(ctx, vote) metrics.MarkVoteExtensionReceived(err == nil) if err != nil { - // Every copy logs at Debug (see isPeerFloodableError); the first - // is worth more. A refusal of a block not processed for the round - // is transient — the application has no context for it yet — where - // one of a processed block is a disagreement over the extensions. - if errors.Is(err, sm.ErrVoteExtensionRejected) && rejected.remember(vote) { - rs := &stateData.RoundState - log.FromCtxOrNop(ctx).Warn("application rejected a precommit's vote extensions; the precommit is not counted", - "voted_block_processed", processedFor(rs, vote.Round) && rs.ProcessedBlockID.Equals(vote.BlockID)) + // Every copy logs at Debug (see isPeerFloodableError). A refusal + // of a block not processed for the round is expected — the + // application has no context for it yet — where one of a + // processed block is a disagreement over the extensions, worth + // one Warn: the first copy's, counted from when the block was + // processed, so a refusal before that does not hide it. + rs := &stateData.RoundState + if errors.Is(err, sm.ErrVoteExtensionRejected) && + processedFor(rs, vote.Round) && rs.ProcessedBlockID.Equals(vote.BlockID) && + rejected.remember(vote) { + log.FromCtxOrNop(ctx).Warn("application rejected a precommit's vote extensions for a block it processed; the precommit is not counted") } return false, err } @@ -440,6 +443,11 @@ func addVoteErrorMw(evpool evidencePool, logger log.Logger, privVal privValidato } } +// rejectedVotesCap bounds a rejectedVotes memory. Nothing stored bounds a +// rejected vote, and a validator key signs one for any round and block, so +// past the cap a new vote is reported on every copy rather than remembered. +const rejectedVotesCap = 4096 + // rejectedVotes remembers the peer votes of one height that were rejected for // a reason no resend can change. A rejected vote is never stored, so no HasVote // acknowledges it and its sender re-offers it for the rest of the round (see @@ -458,7 +466,8 @@ type rejectedVoteKey struct { } // remember records the vote's rejection and reports whether it is the first of -// that vote at its height. +// that vote at its height. A vote the full memory cannot take is not recorded, +// so every copy of it counts as the first. func (r *rejectedVotes) remember(vote *types.Vote) (first bool) { if r.seen == nil || r.height != vote.Height { r.height, r.seen = vote.Height, map[rejectedVoteKey]struct{}{} @@ -467,7 +476,9 @@ func (r *rejectedVotes) remember(vote *types.Vote) (first bool) { if _, seen := r.seen[key]; seen { return false } - r.seen[key] = struct{}{} + if len(r.seen) < rejectedVotesCap { + r.seen[key] = struct{}{} + } return true } diff --git a/internal/consensus/state_add_vote_test.go b/internal/consensus/state_add_vote_test.go index 35eb42d8ce..282a99e074 100644 --- a/internal/consensus/state_add_vote_test.go +++ b/internal/consensus/state_add_vote_test.go @@ -750,3 +750,28 @@ func TestAddVoteErrorMwReportsEachConflictOnce(t *testing.T) { deliver(voteFor(2, "block B")) assert.Equal(t, 3, pool.reports, "a new height starts afresh") } + +// rejectedVotes is fed by precommits a validator key holder can mint at will +// for a stalled height, so it is bounded. Past the bound a new vote is not +// remembered and every copy of it is reported, as before the memory existed, +// rather than any report being dropped; votes remembered before stay +// remembered, and a new height starts afresh. +func TestRejectedVotesBounded(t *testing.T) { + var r rejectedVotes + voteFor := func(height int64, round int32) *types.Vote { + return &types.Vote{Type: tmproto.PrecommitType, Height: height, Round: round, ValidatorIndex: 0} + } + for round := range int32(rejectedVotesCap) { + require.True(t, r.remember(voteFor(1, round))) + } + require.False(t, r.remember(voteFor(1, 0)), "a remembered vote stays remembered") + + overflow := voteFor(1, int32(rejectedVotesCap)) + require.True(t, r.remember(overflow)) + require.True(t, r.remember(overflow), "past the bound every copy is reported") + require.Len(t, r.seen, rejectedVotesCap) + + require.True(t, r.remember(voteFor(2, 0))) + require.False(t, r.remember(voteFor(2, 0))) + require.Len(t, r.seen, 1, "a new height starts afresh") +} From 9856707a1282bc2708a4ed7fd031934e0bdf2c70 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:27:25 +0000 Subject: [PATCH 8/8] test(consensus): cover both guards of the processed-block extension warning A refused precommit for another block of the processed round, or for the processed block in another round, must not warn: each case fails if its guard is dropped. Co-Authored-By: Claude Fable 5.1 --- internal/consensus/log_amplification_test.go | 29 ++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/internal/consensus/log_amplification_test.go b/internal/consensus/log_amplification_test.go index f00f9f5b4b..fadfca50e1 100644 --- a/internal/consensus/log_amplification_test.go +++ b/internal/consensus/log_amplification_test.go @@ -17,6 +17,7 @@ import ( abci "github.com/dashpay/tenderdash/abci/types" "github.com/dashpay/tenderdash/dash" sm "github.com/dashpay/tenderdash/internal/state" + "github.com/dashpay/tenderdash/internal/test/factory" "github.com/dashpay/tenderdash/libs/log" tmtime "github.com/dashpay/tenderdash/libs/time" tmcons "github.com/dashpay/tenderdash/proto/tendermint/consensus" @@ -361,10 +362,17 @@ func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { parts, err := block.MakePartSet(types.BlockPartSizeBytes) require.NoError(t, err) blockID := block.BlockID(parts) - exts := testPrecommitVote(1).VoteExtensions - vote, err := h.vss[1].signVote(ctx, tmproto.PrecommitType, sd.state.ChainID, blockID, - sd.Validators.QuorumType, sd.Validators.QuorumHash, exts) - require.NoError(t, err) + precommit := func(blockID types.BlockID) *types.Vote { + // Signing writes into the extensions, so each vote gets its own. + vote, err := h.vss[1].signVote(ctx, tmproto.PrecommitType, sd.state.ChainID, blockID, + sd.Validators.QuorumType, sd.Validators.QuorumHash, testPrecommitVote(1).VoteExtensions) + require.NoError(t, err) + return vote + } + vote := precommit(blockID) + otherBlock := precommit(factory.MakeBlockID()) + incrementRound(h.vss[1]) + laterRound := precommit(blockID) var buf bytes.Buffer logger, err := log.NewLogger("debug", &buf) @@ -372,13 +380,14 @@ func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { handler := loggingMiddleware(logger)(func(ctx context.Context, sd *StateData, env msgEnvelope) error { return h.cs.ctrl.Dispatch(ctx, &AddVoteEvent{Vote: env.Msg.(*VoteMessage).Vote, PeerID: env.PeerID}, sd) }) - deliver := func() string { + deliverVote := func(vote *types.Vote) string { buf.Reset() env := msgEnvelope{msgInfo: msgInfo{Msg: &VoteMessage{Vote: vote}, PeerID: "peer"}} require.NoError(t, handler(ctx, &sd, env)) require.Nil(t, storedVote(&sd, vote), "a rejected precommit is not stored") return buf.String() } + deliver := func() string { return deliverVote(vote) } for range 2 { unprocessed := deliver() @@ -401,4 +410,14 @@ func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { assert.NotContains(t, again, `"level":"warn"`, "a redelivered rejected precommit is not news") assert.Contains(t, again, `"level":"debug"`) } + + // Only the processed block, for the round it was processed for, is in + // context: a refusal of another block, or of the block for another round, + // is still the expected kind. + for name, vote := range map[string]*types.Vote{"another block": otherBlock, "another round": laterRound} { + out := deliverVote(vote) + assert.NotContains(t, out, `"level":"error"`, name) + assert.NotContains(t, out, `"level":"warn"`, "%s: refusing what is not processed is expected:\n%s", name, out) + assert.Contains(t, out, `"level":"debug"`, name) + } }