diff --git a/config/config.go b/config/config.go index 9662eff83..a8d5b480e 100644 --- a/config/config.go +++ b/config/config.go @@ -1167,7 +1167,12 @@ type ConsensusConfig struct { // tick then already exceeds the gap it is meant to enforce. Lowering it // does not shrink the gap — it only fits more ticks (and so more sends of // a multi-part block) into that same fixed window before it applies. - PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"` + PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"` + // PeerQueryMaj23SleepDuration is the cadence of the per-peer majority-claim + // gossip and of the repairs paced against it: round skip certificate and + // round vote resends, and the redelivery of votes a peer has not + // acknowledged (one interval after a vote went out, then doubling up to 8 + // intervals). Raising it to cut majority-claim chatter slows stall recovery. PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer-query-maj23-sleep-duration"` // PeerVoteRateLimit bounds the per-second token budget a single peer may diff --git a/config/toml.go b/config/toml.go index 904b26bb2..78159d26b 100644 --- a/config/toml.go +++ b/config/toml.go @@ -566,7 +566,9 @@ create-empty-blocks-interval = "{{ .Consensus.CreateEmptyBlocksInterval }}" # Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the # tick rate a lagging peer's catch-up block-part replay is metered against # (a fixed 500ms internal throttle): raising this above 500ms disables that -# throttle. +# throttle. peer-query-maj23-sleep-duration also paces the redelivery of votes a +# peer has not acknowledged: one interval after a vote went out, then doubling +# up to 8 intervals. Raising it slows stall recovery. peer-gossip-sleep-duration = "{{ .Consensus.PeerGossipSleepDuration }}" peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" diff --git a/docs/nodes/configuration.md b/docs/nodes/configuration.md index 8c5090939..09b168ae2 100644 --- a/docs/nodes/configuration.md +++ b/docs/nodes/configuration.md @@ -407,7 +407,12 @@ double-sign-check-height = 0 create-empty-blocks = true create-empty-blocks-interval = "0s" -# Reactor sleep duration parameters +# Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the +# tick rate a lagging peer's catch-up block-part replay is metered against +# (a fixed 500ms internal throttle): raising this above 500ms disables that +# throttle. peer-query-maj23-sleep-duration also paces the redelivery of votes a +# peer has not acknowledged: one interval after a vote went out, then doubling +# up to 8 intervals. Raising it slows stall recovery. peer-gossip-sleep-duration = "100ms" peer-query-maj23-sleep-duration = "2s" diff --git a/internal/consensus/gossip_handlers.go b/internal/consensus/gossip_handlers.go index 122de500f..92d1c5bf5 100644 --- a/internal/consensus/gossip_handlers.go +++ b/internal/consensus/gossip_handlers.go @@ -68,12 +68,15 @@ func roundSkipGossipHandler( // transactions in this node's round, which a skip certificate began, or is in // a round this node has left. // -// Vote gossip takes a vote as the peer's once the channel takes it, so one -// lost after that is not sent again. On a chain that skips rounds, a peer -// that lost the votes of its round can be left in it for good: a waiting -// round has no timeout and its proposer's heartbeats keep arriving, and the -// prevote and precommit steps start their timeouts only on votes. Votes are -// picked at random, so over the passes the peer gets each one this node has. +// Vote gossip takes a vote as the peer's once the channel takes it; one lost +// after that is offered again only after a redelivery interval, then with +// backoff (see PeerState.ReofferUnackedVotes), and a later round's votes are +// not offered to a peer in an earlier round at all. On a chain that skips +// rounds, a peer that lost the votes of its round waits all that time: a +// waiting round has no timeout and its proposer's heartbeats keep arriving, +// and the prevote and precommit steps start their timeouts only on votes. +// Votes are picked at random, so over the passes the peer gets each one this +// node has. // // A peer waiting in this node's round, after round 0, is sent the round's // prevotes: one ends the wait (see StateData.waitsAfterRoundSkip). A peer in a diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index 22cdcd502..c3f405aae 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -78,10 +78,10 @@ func newPeerGossipWorker( ), // So is a vote of its round, or of a later round +2/3 voted in, to // a peer that waits in this node's round, or is in a round this - // node left: a vote lost after the channel took it is not sent - // again, and the peer may have no timeout to fall back on. It - // stops once the peer announces the propose step, or this node's - // round. + // node left: ordinary gossip re-offers a lost vote only after a + // redelivery interval, never one of a later round, and the peer + // may have no timeout to fall back on. It stops once the peer + // announces the propose step, or this node's round. newGossipHandler( roundVoteResendGossipHandler(ps, logger, msgSender.send), state.config.PeerQueryMaj23SleepDuration, diff --git a/internal/consensus/gossiper.go b/internal/consensus/gossiper.go index 1e2ed80fe..3ecd30c73 100644 --- a/internal/consensus/gossiper.go +++ b/internal/consensus/gossiper.go @@ -518,6 +518,9 @@ func (g *msgGossiper) GossipCommit(ctx context.Context, rs cstypes.RoundState, p // GossipVote sends a vote message to the peer func (g *msgGossiper) GossipVote(ctx context.Context, rs cstypes.RoundState, prs *cstypes.PeerRoundState) { + // Sent votes are marked as the peer's on the spot; votes it never confirmed + // are periodically put back on offer so a copy lost in transit is resent. + g.ps.ReofferUnackedVotes() vote, found := g.pickVoteForGossip(rs, prs) if !found { return @@ -694,7 +697,7 @@ func updatePeerCommit(ps *PeerState, commit *types.Commit) func() error { func updatePeerVote(ps *PeerState, vote *types.Vote) func() error { return func() error { - return ps.SetHasVote(vote) + return ps.SetVoteSent(vote) } } diff --git a/internal/consensus/gossiper_test.go b/internal/consensus/gossiper_test.go index 1b0238707..b8ba367dd 100644 --- a/internal/consensus/gossiper_test.go +++ b/internal/consensus/gossiper_test.go @@ -1547,3 +1547,130 @@ func (suite *GossiperSuiteTest) signVote(vote *types.Vote) { err = vote.VoteExtensions.CopySignsFromProto(protoVote.VoteExtensions) suite.Require().NoError(err) } + +// TestGossipVoteRedeliversUnacknowledged verifies that a current-round vote +// marked delivered on send is offered again, at a bounded cadence, while the +// peer stays in that round without acknowledging it — and not again once it +// does. Neither side holds a majority, so no VoteSetMaj23 reconciliation can +// restore the vote to the offer; without redelivery a lost copy would leave +// the round stalled. +func (suite *GossiperSuiteTest) TestGossipVoteRedeliversUnacknowledged() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + const voteResendInterval = time.Second + + for _, tc := range []struct { + name string + voteType tmproto.SignedMsgType + step cstypes.RoundStepType + }{ + {"prevote", tmproto.PrevoteType, cstypes.RoundStepPrevote}, + {"precommit", tmproto.PrecommitType, cstypes.RoundStepPrecommit}, + } { + suite.Run(tc.name, func() { + ps := NewPeerState(suite.logger, suite.ps.peerID, + WithPeerStateClock(suite.clock), + WithVoteResendInterval(voteResendInterval)) + suite.ps, suite.sender.ps, suite.gossiper.ps = ps, ps, ps + + vote := suite.makeSignedVote(100, 0, tc.voteType) + votes := cstypes.NewHeightVoteSet(factory.DefaultTestChainID, 100, suite.valSet) + _, _ = votes.AddVote(vote) + rs := cstypes.RoundState{Height: 100, Round: 0, Votes: votes} + ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: tc.step, ProposalPOLRound: -1, CatchupCommitRound: -1} + + sends := 0 + suite.voteCh. + On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: vote.ToProto()}}). + Run(func(_ mock.Arguments) { sends++ }). + Return(nil) + + // The vote goes out once and is then held back as delivered. + for range 10 { + suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState()) + } + suite.Require().Equal(1, sends, "a vote just sent must not be repeated within the interval") + + // Still unacknowledged after the interval: offered again, once. + suite.clock.Advance(voteResendInterval) + for range 10 { + suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState()) + } + suite.clock.Advance(voteResendInterval) + for range 10 { + suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState()) + } + suite.Require().Equal(2, sends, "an unacknowledged vote must be resent once the interval elapses") + + // The peer reports it has the vote: redelivery stops. + suite.Require().NoError(ps.ApplyHasVoteMessage(&HasVoteMessage{ + Height: 100, Round: 0, Type: tc.voteType, Index: vote.ValidatorIndex, + })) + for range 3 { + suite.clock.Advance(voteResendInterval) + for range 10 { + suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState()) + } + } + suite.Require().Equal(2, sends, "an acknowledged vote must not be resent") + }) + } +} + +// TestGossipVoteRedeliveryStopsWhenPeerMovesOn verifies that bookkeeping of a +// round the peer has left is dropped at the next rotation rather than applied +// to the bits the old round number now stands for: here the peer's proposal +// POL bits, which name that same round under its new round state. +func (suite *GossiperSuiteTest) TestGossipVoteRedeliveryStopsWhenPeerMovesOn() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + const voteResendInterval = time.Second + + ps := NewPeerState(suite.logger, suite.ps.peerID, + WithPeerStateClock(suite.clock), + WithVoteResendInterval(voteResendInterval)) + suite.ps, suite.sender.ps, suite.gossiper.ps = ps, ps, ps + + prevoteR0 := suite.makeSignedVote(100, 0, tmproto.PrevoteType) + prevoteR1 := suite.makeSignedVote(100, 1, tmproto.PrevoteType) + votes := cstypes.NewHeightVoteSet(factory.DefaultTestChainID, 100, suite.valSet) + _, _ = votes.AddVote(prevoteR0) + _, _ = votes.AddVote(prevoteR1) + ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: cstypes.RoundStepPrevote, ProposalPOLRound: -1, CatchupCommitRound: -1} + + suite.voteCh. + On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: prevoteR0.ToProto()}}). + Once(). + Return(nil) + suite.gossiper.GossipVote(ctx, cstypes.RoundState{Height: 100, Round: 0, Votes: votes}, ps.GetRoundState()) + + // Peer moves to round 1 without acknowledging, and reports (via its + // proposal's POL) that it holds the round-0 prevote after all. + ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 100, Round: 1, Step: cstypes.RoundStepPrevote, LastCommitRound: -1}) + ps.PRS.ProposalPOLRound = 0 + ps.PRS.ProposalPOL = bits.NewBitArray(suite.valSet.Size()) + ps.PRS.ProposalPOL.SetIndex(int(prevoteR0.ValidatorIndex), true) + + // The first rotation after the move sees round-0 bookkeeping against a + // round-1 peer and must drop it; the round-1 prevote then goes out once and + // is acknowledged. The round-0 prevote must never be offered again, which + // is what clearing POL bit 0 from the stale bookkeeping would cause. + rsR1 := cstypes.RoundState{Height: 100, Round: 1, Votes: votes} + suite.clock.Advance(voteResendInterval) + suite.voteCh. + On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: prevoteR1.ToProto()}}). + Once(). + Return(nil) + suite.gossiper.GossipVote(ctx, rsR1, ps.GetRoundState()) + suite.Require().NoError(ps.ApplyHasVoteMessage(&HasVoteMessage{ + Height: 100, Round: 1, Type: tmproto.PrevoteType, Index: prevoteR1.ValidatorIndex, + })) + for range 3 { + suite.clock.Advance(voteResendInterval) + for range 10 { + suite.gossiper.GossipVote(ctx, rsR1, ps.GetRoundState()) + } + } + suite.Require().True(ps.PRS.ProposalPOL.GetIndex(int(prevoteR0.ValidatorIndex)), + "stale round-0 bookkeeping must not clear the peer's POL bits for round 0") +} diff --git a/internal/consensus/log_amplification_test.go b/internal/consensus/log_amplification_test.go index d3a0b38a8..fadfca50e 100644 --- a/internal/consensus/log_amplification_test.go +++ b/internal/consensus/log_amplification_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "strings" "testing" "time" @@ -12,8 +13,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/dashpay/tenderdash/abci/example/kvstore" + abci "github.com/dashpay/tenderdash/abci/types" "github.com/dashpay/tenderdash/dash" sm "github.com/dashpay/tenderdash/internal/state" + "github.com/dashpay/tenderdash/internal/test/factory" "github.com/dashpay/tenderdash/libs/log" tmtime "github.com/dashpay/tenderdash/libs/time" tmcons "github.com/dashpay/tenderdash/proto/tendermint/consensus" @@ -42,6 +46,9 @@ func TestIsPeerFloodableError(t *testing.T) { assert.True(t, isPeerFloodableError(fmt.Errorf("%w: AB12, expected CD34", types.ErrRoundSkipCertificateWrongQuorum)), "a certificate naming another quorum is refused before any signature is read") + assert.True(t, isPeerFloodableError(fmt.Errorf("verifying precommit: %w", sm.ErrVoteExtensionRejected)), + "a precommit the application refuses is never stored, so it is replayed and re-sent for the rest of the round") + assert.False(t, isPeerFloodableError(ErrPrivValidatorNotSet), "an internal fault must not be treated as floodable") assert.False(t, isPeerFloodableError(errors.New("some other internal error"))) @@ -324,3 +331,93 @@ func TestLoggingMiddlewareSeparatesLocalFaultsFromReplay(t *testing.T) { assert.Contains(t, out, `"level":"debug"`) }) } + +// voteExtensionRejectingApp refuses every vote extension, as an application +// does for a precommit of a block it has not processed. +type voteExtensionRejectingApp struct{ abci.Application } + +func (voteExtensionRejectingApp) VerifyVoteExtension( + context.Context, *abci.RequestVerifyVoteExtension, +) (*abci.ResponseVerifyVoteExtension, error) { + return &abci.ResponseVerifyVoteExtension{Status: abci.ResponseVerifyVoteExtension_REJECT}, nil +} + +// A precommit the application refuses is never stored, so no HasVote answers it +// and its sender re-offers it for the rest of the round (see +// PeerState.ReofferUnackedVotes); any peer can also replay it for free. A +// refusal of a block this node has not processed is expected and logs at Debug +// only, on every copy. Once the block is processed, the same refusal is a +// disagreement over the extensions: its first copy is worth one Warn line to an +// operator, every further copy logs at Debug, and none logs at Error. +func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + inner, err := kvstore.NewMemoryApp() + require.NoError(t, err) + h := newFloodHarness(ctx, t, floodHarnessArgs{validators: 4, application: voteExtensionRejectingApp{inner}}) + ctx = h.runCtx(ctx) + sd := h.stateData() + _, block := decideProposal(ctx, t, h.cs, h.vss[0], sd.Height, 0) + parts, err := block.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + blockID := block.BlockID(parts) + precommit := func(blockID types.BlockID) *types.Vote { + // Signing writes into the extensions, so each vote gets its own. + vote, err := h.vss[1].signVote(ctx, tmproto.PrecommitType, sd.state.ChainID, blockID, + sd.Validators.QuorumType, sd.Validators.QuorumHash, testPrecommitVote(1).VoteExtensions) + require.NoError(t, err) + return vote + } + vote := precommit(blockID) + otherBlock := precommit(factory.MakeBlockID()) + incrementRound(h.vss[1]) + laterRound := precommit(blockID) + + var buf bytes.Buffer + logger, err := log.NewLogger("debug", &buf) + require.NoError(t, err) + handler := loggingMiddleware(logger)(func(ctx context.Context, sd *StateData, env msgEnvelope) error { + return h.cs.ctrl.Dispatch(ctx, &AddVoteEvent{Vote: env.Msg.(*VoteMessage).Vote, PeerID: env.PeerID}, sd) + }) + deliverVote := func(vote *types.Vote) string { + buf.Reset() + env := msgEnvelope{msgInfo: msgInfo{Msg: &VoteMessage{Vote: vote}, PeerID: "peer"}} + require.NoError(t, handler(ctx, &sd, env)) + require.Nil(t, storedVote(&sd, vote), "a rejected precommit is not stored") + return buf.String() + } + deliver := func() string { return deliverVote(vote) } + + for range 2 { + unprocessed := deliver() + assert.NotContains(t, unprocessed, `"level":"error"`, "a rejected peer precommit must not log at Error") + assert.NotContains(t, unprocessed, `"level":"warn"`, "refusing a block not processed yet is expected:\n%s", unprocessed) + assert.Contains(t, unprocessed, `"level":"debug"`) + } + + sd.ProposalBlock, sd.ProposalBlockParts = block, parts + require.NoError(t, h.cs.blockExecutor.ensureProcess(ctx, &sd.RoundState, sd.Round)) + require.True(t, processedFor(&sd.RoundState, sd.Round)) + + first := deliver() + assert.NotContains(t, first, `"level":"error"`, "a rejected peer precommit must not log at Error") + assert.Equal(t, 1, strings.Count(first, `"level":"warn"`), "the first refusal of a processed block is reported once:\n%s", first) + + for range 3 { + again := deliver() + assert.NotContains(t, again, `"level":"error"`, "a redelivered rejected precommit must not log at Error") + assert.NotContains(t, again, `"level":"warn"`, "a redelivered rejected precommit is not news") + assert.Contains(t, again, `"level":"debug"`) + } + + // Only the processed block, for the round it was processed for, is in + // context: a refusal of another block, or of the block for another round, + // is still the expected kind. + for name, vote := range map[string]*types.Vote{"another block": otherBlock, "another round": laterRound} { + out := deliverVote(vote) + assert.NotContains(t, out, `"level":"error"`, name) + assert.NotContains(t, out, `"level":"warn"`, "%s: refusing what is not processed is expected:\n%s", name, out) + assert.Contains(t, out, `"level":"debug"`, name) + } +} diff --git a/internal/consensus/msg_handlers.go b/internal/consensus/msg_handlers.go index 51659c3be..741c82b58 100644 --- a/internal/consensus/msg_handlers.go +++ b/internal/consensus/msg_handlers.go @@ -74,7 +74,13 @@ func isPeerFloodableError(err error) bool { // block signature, and the vector is covered by no signature a peer must // forge. The rejection of the commit this node assembled from its own // precommits is not a message error: tryFinalizeCommit logs it at Error. - errors.Is(err, sm.ErrCommitExtensionsRejected) + errors.Is(err, sm.ErrCommitExtensionsRejected) || + // A precommit whose extensions the application refuses — as it does for + // a block this node has not processed — is never stored, so any peer can + // replay it for free and its sender re-offers it for the rest of the + // round (see PeerState.ReofferUnackedVotes). Its first copy is reported + // by addVoteVerifyVoteExtensionMw. + errors.Is(err, sm.ErrVoteExtensionRejected) } type msgInfoDispatcher struct { diff --git a/internal/consensus/peer_state.go b/internal/consensus/peer_state.go index 25befbe19..817d6e0c4 100644 --- a/internal/consensus/peer_state.go +++ b/internal/consensus/peer_state.go @@ -46,12 +46,16 @@ func (pss peerStateStats) String() string { type PeerState struct { peerID types.NodeID logger log.Logger - // clock is the time source the majority-claim answer memory is dated - // against; the wall clock unless a test overrides it. + // clock is the time source the majority-claim answer memory and the vote + // redelivery windows are dated against; the wall clock unless a test + // overrides it. clock clockwork.Clock // maj23AnswerTTL is how long an answer keeps the same majority claim from // being answered again. maj23AnswerTTL time.Duration + // voteResendInterval is how long a vote handed to the transport stays off + // the offer before, still unacknowledged, it is offered again. + voteResendInterval time.Duration // NOTE: Modify below using setters, never directly. mtx sync.RWMutex @@ -67,6 +71,11 @@ type PeerState struct { answeredMaj23 [maj23AnswerMemory]maj23Answer answeredMaj23Next int + // unacked are the current-round votes gossiped to the peer that it has not + // acknowledged yet; voteResendAt is when they are next checked for re-offer. + unacked unackedVotes + voteResendAt time.Time + // ProTxHash is accessible only for the validator ProTxHash types.ProTxHash @@ -106,6 +115,17 @@ func WithMaj23AnswerTTL(ttl time.Duration) PeerStateOptionFunc { } } +// WithVoteResendInterval sets how long a vote gossiped to the peer stays off the +// offer before, still unacknowledged, it is offered again (see +// ReofferUnackedVotes). The reactor passes PeerQueryMaj23SleepDuration, the +// cadence of the in-round repairs the redelivery backs up; NewPeerState falls +// back to its default. +func WithVoteResendInterval(interval time.Duration) PeerStateOptionFunc { + return func(ps *PeerState) { + ps.voteResendInterval = interval + } +} + // NewPeerState returns a new PeerState for the given node ID. func NewPeerState(logger log.Logger, peerID types.NodeID, opts ...PeerStateOptionFunc) *PeerState { ps := &PeerState{ @@ -126,6 +146,9 @@ func NewPeerState(logger log.Logger, peerID types.NodeID, opts ...PeerStateOptio if ps.maj23AnswerTTL <= 0 { ps.maj23AnswerTTL = maj23AnswerTTLFor(0) } + if ps.voteResendInterval <= 0 { + ps.voteResendInterval = config.DefaultConsensusConfig().PeerQueryMaj23SleepDuration + } return ps } @@ -519,8 +542,18 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) error { return ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex) } -// setHasVote will return an error when the index exceeds the bitArray length +// setHasVote records a vote the peer has shown it holds. It returns an error +// when the index exceeds the bitArray length. func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) error { + if err := ps.setVoteBit(height, round, voteType, index); err != nil { + return err + } + ps.ackVote(height, round, voteType, index) + return nil +} + +// setVoteBit marks the vote as the peer's without touching redelivery tracking. +func (ps *PeerState) setVoteBit(height int64, round int32, voteType tmproto.SignedMsgType, index int32) error { ps.logger.Trace( "peerState setHasVote", "peer", ps.peerID, @@ -544,6 +577,164 @@ func (ps *PeerState) setHasVote(height int64, round int32, voteType tmproto.Sign return nil } +// voteResendMaxBackoff caps how many times the wait before re-offering one +// unacknowledged vote doubles. A vote whose HasVote was lost can never be +// acknowledged by resending it — the receiver drops the duplicate before any +// HasVote is published — so without backoff it would be resent every interval +// for the rest of the round; without a cap it would effectively be given up +// on, which is the permanent loss redelivery exists to prevent. +const voteResendMaxBackoff = 3 + +// unackedVote is one vote handed to the transport for the peer and not yet +// confirmed by it. A zero sentAt means the vote is not tracked. +type unackedVote struct { + sentAt time.Time + reoffers uint8 +} + +// unackedVotes are the votes of one height/round that were handed to the +// transport for the peer and that the peer has not confirmed having, indexed +// by validator index; slots follow unackedVoteTypes. +type unackedVotes struct { + height int64 + round int32 + votes [len(unackedVoteTypes)][]unackedVote +} + +var unackedVoteTypes = [...]tmproto.SignedMsgType{tmproto.PrevoteType, tmproto.PrecommitType} + +func unackedVoteSlot(voteType tmproto.SignedMsgType) (int, bool) { + for slot, t := range unackedVoteTypes { + if t == voteType { + return slot, true + } + } + return 0, false +} + +// SetVoteSent records that the vote was handed to the transport for the peer. +// The peer is treated as having it, so it is not offered again — until a +// redelivery interval passes without the peer confirming it, which puts it +// back on offer (see ReofferUnackedVotes). Only the peer's current round is +// tracked: votes for any other round are marked exactly like SetHasVote. +// +// Gossip picks a vote only while the peer's bit for it is clear, and the send +// runs between that pick and this call, unlocked. A bit found already set here +// therefore means the peer confirmed the vote in that window, and the vote is +// not tracked: a resend could never be acknowledged. +func (ps *PeerState) SetVoteSent(vote *types.Vote) error { + if vote == nil { + return ErrPeerStateSetNilVote + } + ps.mtx.Lock() + defer ps.mtx.Unlock() + + slot, ok := unackedVoteSlot(vote.Type) + track := ok && vote.Height == ps.PRS.Height && vote.Round == ps.PRS.Round + var psVotes *bits.BitArray + if track { + psVotes = ps.getVoteBitArray(vote.Height, vote.Round, vote.Type) + track = psVotes != nil && !psVotes.GetIndex(int(vote.ValidatorIndex)) + } + if err := ps.setVoteBit(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex); err != nil { + return err + } + if !track { + return nil + } + if ps.unacked.height != vote.Height || ps.unacked.round != vote.Round { + ps.unacked = unackedVotes{height: vote.Height, round: vote.Round} + } + if len(ps.unacked.votes[slot]) != psVotes.Size() { + ps.unacked.votes[slot] = make([]unackedVote, psVotes.Size()) + } + // A re-offered vote keeps its backoff; setVoteBit validated the index. + ps.unacked.votes[slot][vote.ValidatorIndex].sentAt = ps.clock.Now() + return nil +} + +// ackVote drops a vote from redelivery tracking: the peer has shown it has it. +func (ps *PeerState) ackVote(height int64, round int32, voteType tmproto.SignedMsgType, index int32) { + slot, ok := unackedVoteSlot(voteType) + if !ok || ps.unacked.height != height || ps.unacked.round != round { + return + } + if entries := ps.unacked.votes[slot]; 0 <= index && int(index) < len(entries) { + entries[index] = unackedVote{} + } +} + +// ackVoteBits drops every vote set in acked from redelivery tracking. +func (ps *PeerState) ackVoteBits(height int64, round int32, voteType tmproto.SignedMsgType, acked *bits.BitArray) { + slot, ok := unackedVoteSlot(voteType) + if !ok || acked == nil || ps.unacked.height != height || ps.unacked.round != round { + return + } + for index := range ps.unacked.votes[slot] { + if acked.GetIndex(index) { + ps.unacked.votes[slot][index] = unackedVote{} + } + } +} + +// ReofferUnackedVotes puts current-round votes the peer has not acknowledged +// back on offer, checking once per voteResendInterval. +// +// Gossip marks a vote as the peer's the moment it is handed to the transport; +// a copy dropped after that (router queue overflow, receiver over its +// allowance) is otherwise never sent again, and the only in-round repair — a +// VoteSetMaj23 exchange — needs a +2/3 majority on one side. When the lost +// votes are exactly what both sides lack to reach one, the round stalls with +// no timeout to fall back on. +// +// A vote is re-offered once a full interval has passed since it was sent — so +// one whose HasVote is merely in flight is not, and the healthy path sends +// nothing twice — and at most two intervals after, given the once-per-interval +// check. Each re-offer doubles the wait before the next, up to +// voteResendMaxBackoff doublings. Bookkeeping of a round the peer has left is +// dropped, never applied to the bits that round number now stands for. +func (ps *PeerState) ReofferUnackedVotes() { + ps.mtx.Lock() + defer ps.mtx.Unlock() + + now := ps.clock.Now() + if now.Before(ps.voteResendAt) { + return + } + ps.voteResendAt = now.Add(ps.voteResendInterval) + + if ps.unacked.height != ps.PRS.Height || ps.unacked.round != ps.PRS.Round { + ps.unacked = unackedVotes{} + return + } + reoffered := 0 + for slot, entries := range ps.unacked.votes { + if len(entries) == 0 { + continue + } + psVotes := ps.getVoteBitArray(ps.unacked.height, ps.unacked.round, unackedVoteTypes[slot]) + if psVotes == nil { + continue + } + for index := range entries { + entry := &entries[index] + if entry.sentAt.IsZero() || now.Sub(entry.sentAt) < ps.voteResendInterval< 0 { + ps.logger.Debug("re-offering votes the peer has not acknowledged", + "height", ps.unacked.height, "round", ps.unacked.round, "votes", reoffered) + } +} + // SetHasRoundSkip records that the peer holds a round skip certificate for // round at height: it sent one. func (ps *PeerState) SetHasRoundSkip(height int64, round int32) { @@ -868,6 +1059,7 @@ func (ps *PeerState) ApplyVoteSetBitsMessage(msg *VoteSetBitsMessage, ourVotes * votes.Update(hasVotes) } } + ps.ackVoteBits(msg.Height, msg.Round, msg.Type, msg.Votes) } // String returns a string representation of the PeerState diff --git a/internal/consensus/peer_state_redelivery_height_test.go b/internal/consensus/peer_state_redelivery_height_test.go new file mode 100644 index 000000000..a0a261c6d --- /dev/null +++ b/internal/consensus/peer_state_redelivery_height_test.go @@ -0,0 +1,43 @@ +package consensus + +import ( + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/libs/log" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" + "github.com/dashpay/tenderdash/types" +) + +// TestVoteRedeliveryDropsBookkeepingOnHeightChange verifies that redelivery +// bookkeeping of a height the peer has left is dropped even when the new +// height starts at the same round number (round 0, the common case), rather +// than being applied to the peer's new-height vote bits. +func TestVoteRedeliveryDropsBookkeepingOnHeightChange(t *testing.T) { + const interval = time.Second + clock := clockwork.NewFakeClock() + ps := NewPeerState(log.NewNopLogger(), "testPeerState", + WithPeerStateClock(clock), WithVoteResendInterval(interval)) + ps.PRS.Height, ps.PRS.Round, ps.PRS.Step = 1, 0, 0 + ps.PRS.CatchupCommitRound, ps.PRS.ProposalPOLRound, ps.PRS.LastCommitRound = -1, -1, -1 + ps.ensureVoteBitArrays(1, 2) + + ps.ReofferUnackedVotes() + require.NoError(t, ps.SetVoteSent(&types.Vote{Height: 1, Round: 0, Type: tmproto.PrecommitType, ValidatorIndex: 0})) + + // Peer commits height 1 in round 0 and enters height 2, round 0, without + // acknowledging; it then reports holding precommit 0 of height 2. + ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 2, Round: 0, Step: 1, LastCommitRound: 0}) + ps.ensureVoteBitArrays(2, 2) + require.NoError(t, ps.ApplyHasVoteMessage(&HasVoteMessage{Height: 2, Round: 0, Type: tmproto.PrecommitType, Index: 0})) + + for range 3 { + clock.Advance(interval) + ps.ReofferUnackedVotes() + } + require.True(t, ps.PRS.Precommits.GetIndex(0), + "stale height-1 bookkeeping must not re-offer the peer's height-2 precommit") +} diff --git a/internal/consensus/peer_state_test.go b/internal/consensus/peer_state_test.go index 9c4b6fdcb..d08ff2d03 100644 --- a/internal/consensus/peer_state_test.go +++ b/internal/consensus/peer_state_test.go @@ -2,9 +2,12 @@ package consensus import ( "testing" + "time" + "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" + "github.com/dashpay/tenderdash/libs/bits" "github.com/dashpay/tenderdash/libs/log" tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" @@ -99,3 +102,153 @@ func TestApplyHasVoteMessage(t *testing.T) { // skip test cases like v & v3 in TestSetHasVote due to the same path } + +// newRedeliveryPeerState returns a PeerState re-offering unacknowledged votes +// after interval on a fake clock, with its peer at height 1, round 1, and vote +// bit arrays for n validators. +func newRedeliveryPeerState(interval time.Duration, n int) (*PeerState, *clockwork.FakeClock) { + clock := clockwork.NewFakeClock() + ps := NewPeerState(log.NewNopLogger(), "testPeerState", + WithPeerStateClock(clock), WithVoteResendInterval(interval)) + ps.PRS.Height, ps.PRS.Round = 1, 1 + ps.ensureVoteBitArrays(1, n) + return ps, clock +} + +// TestVoteRedeliveryWindows pins the cadence contract of vote redelivery: a vote +// recorded as sent is unmarked no sooner than one interval after it went out +// and no later than two, and any acknowledgement in between cancels it. +func TestVoteRedeliveryWindows(t *testing.T) { + const interval = time.Second + ps, clock := newRedeliveryPeerState(interval, 3) + vote := func(idx int32) *types.Vote { + return &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: idx} + } + + // t=0: the first check arms the clock; t=0.5: vote 0 goes out. + ps.ReofferUnackedVotes() + clock.Advance(interval / 2) + require.NoError(t, ps.SetVoteSent(vote(0))) + require.True(t, ps.PRS.Prevotes.GetIndex(0)) + + // t=1: half an interval old, vote 0 stays marked. + clock.Advance(interval / 2) + ps.ReofferUnackedVotes() + require.True(t, ps.PRS.Prevotes.GetIndex(0), "a vote sent within the last interval stays marked") + + // t=1.5: votes 1 and 2 go out; vote 2 is acknowledged at once. + clock.Advance(interval / 2) + require.NoError(t, ps.SetVoteSent(vote(1))) + require.NoError(t, ps.SetVoteSent(vote(2))) + require.NoError(t, ps.ApplyHasVoteMessage(&HasVoteMessage{Height: 1, Round: 1, Type: tmproto.PrevoteType, Index: 2})) + + // Until t=2 nothing changes, however often it is asked. + clock.Advance(interval/2 - time.Nanosecond) + for range 5 { + ps.ReofferUnackedVotes() + } + require.True(t, ps.PRS.Prevotes.GetIndex(0)) + require.True(t, ps.PRS.Prevotes.GetIndex(1)) + + // t=2: vote 0 (1.5 intervals old) is re-offered; vote 1 (0.5) waits. + clock.Advance(time.Nanosecond) + ps.ReofferUnackedVotes() + require.False(t, ps.PRS.Prevotes.GetIndex(0), "unacknowledged vote must be re-offered after a full interval") + require.True(t, ps.PRS.Prevotes.GetIndex(1)) + require.True(t, ps.PRS.Prevotes.GetIndex(2), "acknowledged vote must stay marked") + + // t=3: vote 1 is re-offered. + clock.Advance(interval) + ps.ReofferUnackedVotes() + require.False(t, ps.PRS.Prevotes.GetIndex(1)) + require.True(t, ps.PRS.Prevotes.GetIndex(2), "acknowledged vote must stay marked") + + // The peer's own copy of vote 0 acknowledges it for good. + require.NoError(t, ps.SetHasVote(vote(0))) + for range 4 { + clock.Advance(interval) + ps.ReofferUnackedVotes() + } + require.True(t, ps.PRS.Prevotes.GetIndex(0), "SetHasVote records an acknowledged vote") +} + +// TestVoteRedeliveryIgnoresAckBeforeSent covers the gossip interleaving in +// which the peer's HasVote is applied between picking a vote and recording it +// as sent (the send sits between the two, unlocked). The ack must win: a +// resend could never be acknowledged, since the receiver's duplicate check +// drops it before any HasVote is published. +func TestVoteRedeliveryIgnoresAckBeforeSent(t *testing.T) { + const interval = time.Second + ps, clock := newRedeliveryPeerState(interval, 1) + vote := &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: 0} + + ps.ReofferUnackedVotes() + require.NoError(t, ps.ApplyHasVoteMessage(&HasVoteMessage{Height: 1, Round: 1, Type: tmproto.PrevoteType, Index: 0})) + require.NoError(t, ps.SetVoteSent(vote)) + for range 3 { + clock.Advance(interval) + ps.ReofferUnackedVotes() + } + require.True(t, ps.PRS.Prevotes.GetIndex(0), "a vote acknowledged before it was recorded as sent must not be re-offered") +} + +// TestVoteRedeliveryBacksOff pins the bound on a vote the peer never +// acknowledges — a lost HasVote cannot be repaired by resending, since the +// receiver drops the duplicate silently: each re-offer doubles the wait before +// the next, up to voteResendMaxBackoff, so the vote keeps being offered at a +// rate that falls off instead of every interval. +func TestVoteRedeliveryBacksOff(t *testing.T) { + const interval = time.Second + ps, clock := newRedeliveryPeerState(interval, 1) + vote := &types.Vote{Height: 1, Round: 1, Type: tmproto.PrevoteType, ValidatorIndex: 0} + + // rotationsUntilReoffer resends the vote, then counts the rotations (one + // per interval) until it is put back on offer. + rotationsUntilReoffer := func() int { + require.NoError(t, ps.SetVoteSent(vote)) + for n := 1; n <= 64; n++ { + clock.Advance(interval) + ps.ReofferUnackedVotes() + if !ps.PRS.Prevotes.GetIndex(0) { + return n + } + } + t.Fatal("vote was never re-offered") + return 0 + } + ps.ReofferUnackedVotes() + require.Equal(t, 1, rotationsUntilReoffer(), "first re-offer after one interval") + require.Equal(t, 2, rotationsUntilReoffer(), "second re-offer doubles the wait") + require.Equal(t, 4, rotationsUntilReoffer()) + require.Equal(t, 1< 0 { diff --git a/internal/consensus/state_add_vote.go b/internal/consensus/state_add_vote.go index e7d10c334..9bf5b3075 100644 --- a/internal/consensus/state_add_vote.go +++ b/internal/consensus/state_add_vote.go @@ -9,6 +9,7 @@ import ( cstypes "github.com/dashpay/tenderdash/internal/consensus/types" tmstrings "github.com/dashpay/tenderdash/internal/libs/strings" + sm "github.com/dashpay/tenderdash/internal/state" "github.com/dashpay/tenderdash/libs/eventemitter" "github.com/dashpay/tenderdash/libs/log" tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" @@ -275,6 +276,8 @@ func addVoteVerifyVoteExtensionMw( privVal = data.(privValidator) return nil }) + // Touched only by the consensus goroutine, like every vote middleware. + var rejected rejectedVotes return func(next AddVoteFunc) AddVoteFunc { return func(ctx context.Context, stateData *StateData, vote *types.Vote) (bool, error) { // Verify VoteExtension if precommit and not nil @@ -333,6 +336,18 @@ func addVoteVerifyVoteExtensionMw( err = blockExec.blockExec.VerifyVoteExtension(ctx, vote) metrics.MarkVoteExtensionReceived(err == nil) if err != nil { + // Every copy logs at Debug (see isPeerFloodableError). A refusal + // of a block not processed for the round is expected — the + // application has no context for it yet — where one of a + // processed block is a disagreement over the extensions, worth + // one Warn: the first copy's, counted from when the block was + // processed, so a refusal before that does not hide it. + rs := &stateData.RoundState + if errors.Is(err, sm.ErrVoteExtensionRejected) && + processedFor(rs, vote.Round) && rs.ProcessedBlockID.Equals(vote.BlockID) && + rejected.remember(vote) { + log.FromCtxOrNop(ctx).Warn("application rejected a precommit's vote extensions for a block it processed; the precommit is not counted") + } return false, err } return next(ctxWithVoteVerification(ctx, vote, verified), stateData, vote) @@ -379,6 +394,8 @@ func addVoteErrorMw(evpool evidencePool, logger log.Logger, privVal privValidato privVal = data.(privValidator) return nil }) + // Touched only by the consensus goroutine, like every vote middleware. + var reported rejectedVotes return func(next AddVoteFunc) AddVoteFunc { return func(ctx context.Context, stateData *StateData, vote *types.Vote) (bool, error) { added, err := next(ctx, stateData, vote) @@ -408,7 +425,15 @@ func addVoteErrorMw(evpool evidencePool, logger log.Logger, privVal privValidato return added, err } - // report conflicting votes to the evidence pool + // A conflicting vote is never stored, so its sender re-offers it + // for the rest of the round (see PeerState.ReofferUnackedVotes). + // The pool drops a duplicate when it forms the evidence; reporting + // it once spares the pool's buffer and the log the copies. + if !reported.remember(vote) { + logger.Debug("conflicting vote delivered again; already reported to the evidence pool", + "height", vote.Height, "round", vote.Round, "type", vote.Type, "val_index", vote.ValidatorIndex) + return added, err + } evpool.ReportConflictingVotes(voteErr.VoteA, voteErr.VoteB) logger.Error("found and sent conflicting votes to the evidence pool", "vote_a", voteErr.VoteA, @@ -418,6 +443,45 @@ func addVoteErrorMw(evpool evidencePool, logger log.Logger, privVal privValidato } } +// rejectedVotesCap bounds a rejectedVotes memory. Nothing stored bounds a +// rejected vote, and a validator key signs one for any round and block, so +// past the cap a new vote is reported on every copy rather than remembered. +const rejectedVotesCap = 4096 + +// rejectedVotes remembers the peer votes of one height that were rejected for +// a reason no resend can change. A rejected vote is never stored, so no HasVote +// acknowledges it and its sender re-offers it for the rest of the round (see +// PeerState.ReofferUnackedVotes); remembering it lets only the first copy be +// reported. The memory is reset when the height changes. +type rejectedVotes struct { + height int64 + seen map[rejectedVoteKey]struct{} +} + +type rejectedVoteKey struct { + round int32 + voteType tmproto.SignedMsgType + validatorIndex int32 + blockKey string +} + +// remember records the vote's rejection and reports whether it is the first of +// that vote at its height. A vote the full memory cannot take is not recorded, +// so every copy of it counts as the first. +func (r *rejectedVotes) remember(vote *types.Vote) (first bool) { + if r.seen == nil || r.height != vote.Height { + r.height, r.seen = vote.Height, map[rejectedVoteKey]struct{}{} + } + key := rejectedVoteKey{vote.Round, vote.Type, vote.ValidatorIndex, vote.BlockID.Key()} + if _, seen := r.seen[key]; seen { + return false + } + if len(r.seen) < rejectedVotesCap { + r.seen[key] = struct{}{} + } + return true +} + func addVoteLoggingMw() AddVoteMiddlewareFunc { return func(next AddVoteFunc) AddVoteFunc { return func(ctx context.Context, stateData *StateData, vote *types.Vote) (bool, error) { diff --git a/internal/consensus/state_add_vote_test.go b/internal/consensus/state_add_vote_test.go index 54468ecd8..282a99e07 100644 --- a/internal/consensus/state_add_vote_test.go +++ b/internal/consensus/state_add_vote_test.go @@ -1,6 +1,7 @@ package consensus import ( + "bytes" "context" "errors" "fmt" @@ -696,3 +697,81 @@ func TestPolkaRetargetKeepsProposalForTheChosenBlock(t *testing.T) { assert.Equal(t, receiveTime, stateData.ProposalReceiveTime, "dropping the receive time of a surviving proposal loses its timeliness") } + +// recordingEvidencePool counts the conflicting-vote pairs reported to it. +type recordingEvidencePool struct{ reports int } + +func (p *recordingEvidencePool) ReportConflictingVotes(_, _ *types.Vote) { p.reports++ } + +// A conflicting vote is never stored, so its sender re-offers it for the rest +// of the round (see PeerState.ReofferUnackedVotes). The first copy is reported +// to the evidence pool and logged at Error; a redelivered copy is neither, as +// the pool holds that equivocation already. Another block from the same +// validator, or the same pair at a later height, is a new report. +func TestAddVoteErrorMwReportsEachConflictOnce(t *testing.T) { + valSet, privVals := factory.MockValidatorSet() + privVal := privValidator{PrivValidator: privVals[0], ProTxHash: valSet.Validators[0].ProTxHash} + pool := &recordingEvidencePool{} + var buf bytes.Buffer + logger, err := log.NewLogger("debug", &buf) + require.NoError(t, err) + + voteFor := func(height int64, block string) *types.Vote { + return &types.Vote{ + Type: tmproto.PrecommitType, Height: height, Round: 0, + ValidatorIndex: 1, ValidatorProTxHash: valSet.Validators[1].ProTxHash, + BlockID: types.BlockID{Hash: []byte(block)}, + } + } + existing := voteFor(1, "block A") + conflicting := func(_ context.Context, _ *StateData, vote *types.Vote) (bool, error) { + return false, types.NewConflictingVoteError(existing, vote) + } + mw := addVoteErrorMw(pool, logger, privVal, eventemitter.New())(conflicting) + deliver := func(vote *types.Vote) string { + buf.Reset() + _, err := mw(context.Background(), &StateData{}, vote) + require.ErrorAs(t, err, new(*types.ErrVoteConflictingVotes), "the conflict is still reported to the caller") + return buf.String() + } + + out := deliver(voteFor(1, "block B")) + assert.Equal(t, 1, pool.reports) + assert.Contains(t, out, `"level":"error"`, "the first copy of an equivocation is news") + + out = deliver(voteFor(1, "block B")) + assert.Equal(t, 1, pool.reports, "a redelivered conflicting vote is in the pool already") + assert.NotContains(t, out, `"level":"error"`) + assert.Contains(t, out, `"level":"debug"`) + + deliver(voteFor(1, "block C")) + assert.Equal(t, 2, pool.reports, "a third block from the same validator is a new equivocation") + + deliver(voteFor(2, "block B")) + assert.Equal(t, 3, pool.reports, "a new height starts afresh") +} + +// rejectedVotes is fed by precommits a validator key holder can mint at will +// for a stalled height, so it is bounded. Past the bound a new vote is not +// remembered and every copy of it is reported, as before the memory existed, +// rather than any report being dropped; votes remembered before stay +// remembered, and a new height starts afresh. +func TestRejectedVotesBounded(t *testing.T) { + var r rejectedVotes + voteFor := func(height int64, round int32) *types.Vote { + return &types.Vote{Type: tmproto.PrecommitType, Height: height, Round: round, ValidatorIndex: 0} + } + for round := range int32(rejectedVotesCap) { + require.True(t, r.remember(voteFor(1, round))) + } + require.False(t, r.remember(voteFor(1, 0)), "a remembered vote stays remembered") + + overflow := voteFor(1, int32(rejectedVotesCap)) + require.True(t, r.remember(overflow)) + require.True(t, r.remember(overflow), "past the bound every copy is reported") + require.Len(t, r.seen, rejectedVotesCap) + + require.True(t, r.remember(voteFor(2, 0))) + require.False(t, r.remember(voteFor(2, 0))) + require.Len(t, r.seen, 1, "a new height starts afresh") +} diff --git a/internal/state/errors.go b/internal/state/errors.go index a7e84130e..206e8ec1f 100644 --- a/internal/state/errors.go +++ b/internal/state/errors.go @@ -8,6 +8,7 @@ import ( var ( ErrBlockRejected = errors.New("block not accepted by abci app") ErrCommitExtensionsRejected = errors.New("commit vote extensions not accepted by abci app") + ErrVoteExtensionRejected = errors.New("vote extensions not accepted by abci app") ) type ( diff --git a/internal/state/execution.go b/internal/state/execution.go index 6f90f77b9..cc001c947 100644 --- a/internal/state/execution.go +++ b/internal/state/execution.go @@ -105,8 +105,8 @@ type Executor interface { // VerifyVoteExtension asks the application to verify vote extensions: those // of one validator's precommit when vote.ValidatorProTxHash is set, or a // commit's threshold-recovered vector, after ProcessProposal of its block, - // when it is empty. It returns an error on REJECT and panics if the ABCI - // call itself fails. + // when it is empty. It returns ErrVoteExtensionRejected on REJECT and + // panics if the ABCI call itself fails. VerifyVoteExtension(ctx context.Context, vote *types.Vote) error } @@ -718,7 +718,7 @@ func (blockExec *BlockExecutor) VerifyVoteExtension(ctx context.Context, vote *t } if !resp.IsOK() { - return errors.New("invalid vote extension") + return ErrVoteExtensionRejected } return nil