Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1167,7 +1167,12 @@ type ConsensusConfig struct {
// tick then already exceeds the gap it is meant to enforce. Lowering it
// does not shrink the gap — it only fits more ticks (and so more sends of
// a multi-part block) into that same fixed window before it applies.
PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"`
PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"`
// PeerQueryMaj23SleepDuration is the cadence of the per-peer majority-claim
// gossip and of the repairs paced against it: round skip certificate and
// round vote resends, and the redelivery of votes a peer has not
// acknowledged (one interval after a vote went out, then doubling up to 8
// intervals). Raising it to cut majority-claim chatter slows stall recovery.
PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer-query-maj23-sleep-duration"`

// PeerVoteRateLimit bounds the per-second token budget a single peer may
Expand Down
4 changes: 3 additions & 1 deletion config/toml.go
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,9 @@ create-empty-blocks-interval = "{{ .Consensus.CreateEmptyBlocksInterval }}"
# Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the
# tick rate a lagging peer's catch-up block-part replay is metered against
# (a fixed 500ms internal throttle): raising this above 500ms disables that
# throttle.
# throttle. peer-query-maj23-sleep-duration also paces the redelivery of votes a
# peer has not acknowledged: one interval after a vote went out, then doubling
# up to 8 intervals. Raising it slows stall recovery.
peer-gossip-sleep-duration = "{{ .Consensus.PeerGossipSleepDuration }}"
peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}"

Expand Down
7 changes: 6 additions & 1 deletion docs/nodes/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,12 @@ double-sign-check-height = 0
create-empty-blocks = true
create-empty-blocks-interval = "0s"

# Reactor sleep duration parameters
# Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the
# tick rate a lagging peer's catch-up block-part replay is metered against
# (a fixed 500ms internal throttle): raising this above 500ms disables that
# throttle. peer-query-maj23-sleep-duration also paces the redelivery of votes a
# peer has not acknowledged: one interval after a vote went out, then doubling
# up to 8 intervals. Raising it slows stall recovery.
peer-gossip-sleep-duration = "100ms"
peer-query-maj23-sleep-duration = "2s"

Expand Down
15 changes: 9 additions & 6 deletions internal/consensus/gossip_handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,15 @@ func roundSkipGossipHandler(
// transactions in this node's round, which a skip certificate began, or is in
// a round this node has left.
//
// Vote gossip takes a vote as the peer's once the channel takes it, so one
// lost after that is not sent again. On a chain that skips rounds, a peer
// that lost the votes of its round can be left in it for good: a waiting
// round has no timeout and its proposer's heartbeats keep arriving, and the
// prevote and precommit steps start their timeouts only on votes. Votes are
// picked at random, so over the passes the peer gets each one this node has.
// Vote gossip takes a vote as the peer's once the channel takes it; one lost
// after that is offered again only after a redelivery interval, then with
// backoff (see PeerState.ReofferUnackedVotes), and a later round's votes are
// not offered to a peer in an earlier round at all. On a chain that skips
// rounds, a peer that lost the votes of its round waits all that time: a
// waiting round has no timeout and its proposer's heartbeats keep arriving,
// and the prevote and precommit steps start their timeouts only on votes.
// Votes are picked at random, so over the passes the peer gets each one this
// node has.
//
// A peer waiting in this node's round, after round 0, is sent the round's
// prevotes: one ends the wait (see StateData.waitsAfterRoundSkip). A peer in a
Expand Down
8 changes: 4 additions & 4 deletions internal/consensus/gossip_peer_worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,10 @@ func newPeerGossipWorker(
),
// So is a vote of its round, or of a later round +2/3 voted in, to
// a peer that waits in this node's round, or is in a round this
// node left: a vote lost after the channel took it is not sent
// again, and the peer may have no timeout to fall back on. It
// stops once the peer announces the propose step, or this node's
// round.
// node left: ordinary gossip re-offers a lost vote only after a
// redelivery interval, never one of a later round, and the peer
// may have no timeout to fall back on. It stops once the peer
// announces the propose step, or this node's round.
newGossipHandler(
roundVoteResendGossipHandler(ps, logger, msgSender.send),
state.config.PeerQueryMaj23SleepDuration,
Expand Down
5 changes: 4 additions & 1 deletion internal/consensus/gossiper.go
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,9 @@ func (g *msgGossiper) GossipCommit(ctx context.Context, rs cstypes.RoundState, p

// GossipVote sends a vote message to the peer
func (g *msgGossiper) GossipVote(ctx context.Context, rs cstypes.RoundState, prs *cstypes.PeerRoundState) {
// Sent votes are marked as the peer's on the spot; votes it never confirmed
// are periodically put back on offer so a copy lost in transit is resent.
g.ps.ReofferUnackedVotes()
vote, found := g.pickVoteForGossip(rs, prs)
if !found {
return
Expand Down Expand Up @@ -694,7 +697,7 @@ func updatePeerCommit(ps *PeerState, commit *types.Commit) func() error {

func updatePeerVote(ps *PeerState, vote *types.Vote) func() error {
return func() error {
return ps.SetHasVote(vote)
return ps.SetVoteSent(vote)
}
}

Expand Down
127 changes: 127 additions & 0 deletions internal/consensus/gossiper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1547,3 +1547,130 @@ func (suite *GossiperSuiteTest) signVote(vote *types.Vote) {
err = vote.VoteExtensions.CopySignsFromProto(protoVote.VoteExtensions)
suite.Require().NoError(err)
}

// TestGossipVoteRedeliversUnacknowledged verifies that a current-round vote
// marked delivered on send is offered again, at a bounded cadence, while the
// peer stays in that round without acknowledging it — and not again once it
// does. Neither side holds a majority, so no VoteSetMaj23 reconciliation can
// restore the vote to the offer; without redelivery a lost copy would leave
// the round stalled.
func (suite *GossiperSuiteTest) TestGossipVoteRedeliversUnacknowledged() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
const voteResendInterval = time.Second

for _, tc := range []struct {
name string
voteType tmproto.SignedMsgType
step cstypes.RoundStepType
}{
{"prevote", tmproto.PrevoteType, cstypes.RoundStepPrevote},
{"precommit", tmproto.PrecommitType, cstypes.RoundStepPrecommit},
} {
suite.Run(tc.name, func() {
ps := NewPeerState(suite.logger, suite.ps.peerID,
WithPeerStateClock(suite.clock),
WithVoteResendInterval(voteResendInterval))
suite.ps, suite.sender.ps, suite.gossiper.ps = ps, ps, ps

vote := suite.makeSignedVote(100, 0, tc.voteType)
votes := cstypes.NewHeightVoteSet(factory.DefaultTestChainID, 100, suite.valSet)
_, _ = votes.AddVote(vote)
rs := cstypes.RoundState{Height: 100, Round: 0, Votes: votes}
ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: tc.step, ProposalPOLRound: -1, CatchupCommitRound: -1}

sends := 0
suite.voteCh.
On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: vote.ToProto()}}).
Run(func(_ mock.Arguments) { sends++ }).
Return(nil)

// The vote goes out once and is then held back as delivered.
for range 10 {
suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState())
}
suite.Require().Equal(1, sends, "a vote just sent must not be repeated within the interval")

// Still unacknowledged after the interval: offered again, once.
suite.clock.Advance(voteResendInterval)
for range 10 {
suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState())
}
suite.clock.Advance(voteResendInterval)
for range 10 {
suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState())
}
suite.Require().Equal(2, sends, "an unacknowledged vote must be resent once the interval elapses")

// The peer reports it has the vote: redelivery stops.
suite.Require().NoError(ps.ApplyHasVoteMessage(&HasVoteMessage{
Height: 100, Round: 0, Type: tc.voteType, Index: vote.ValidatorIndex,
}))
for range 3 {
suite.clock.Advance(voteResendInterval)
for range 10 {
suite.gossiper.GossipVote(ctx, rs, ps.GetRoundState())
}
}
suite.Require().Equal(2, sends, "an acknowledged vote must not be resent")
})
}
}

// TestGossipVoteRedeliveryStopsWhenPeerMovesOn verifies that bookkeeping of a
// round the peer has left is dropped at the next rotation rather than applied
// to the bits the old round number now stands for: here the peer's proposal
// POL bits, which name that same round under its new round state.
func (suite *GossiperSuiteTest) TestGossipVoteRedeliveryStopsWhenPeerMovesOn() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
const voteResendInterval = time.Second

ps := NewPeerState(suite.logger, suite.ps.peerID,
WithPeerStateClock(suite.clock),
WithVoteResendInterval(voteResendInterval))
suite.ps, suite.sender.ps, suite.gossiper.ps = ps, ps, ps

prevoteR0 := suite.makeSignedVote(100, 0, tmproto.PrevoteType)
prevoteR1 := suite.makeSignedVote(100, 1, tmproto.PrevoteType)
votes := cstypes.NewHeightVoteSet(factory.DefaultTestChainID, 100, suite.valSet)
_, _ = votes.AddVote(prevoteR0)
_, _ = votes.AddVote(prevoteR1)
ps.PRS = cstypes.PeerRoundState{Height: 100, Round: 0, Step: cstypes.RoundStepPrevote, ProposalPOLRound: -1, CatchupCommitRound: -1}

suite.voteCh.
On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: prevoteR0.ToProto()}}).
Once().
Return(nil)
suite.gossiper.GossipVote(ctx, cstypes.RoundState{Height: 100, Round: 0, Votes: votes}, ps.GetRoundState())

// Peer moves to round 1 without acknowledging, and reports (via its
// proposal's POL) that it holds the round-0 prevote after all.
ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 100, Round: 1, Step: cstypes.RoundStepPrevote, LastCommitRound: -1})
ps.PRS.ProposalPOLRound = 0
ps.PRS.ProposalPOL = bits.NewBitArray(suite.valSet.Size())
ps.PRS.ProposalPOL.SetIndex(int(prevoteR0.ValidatorIndex), true)

// The first rotation after the move sees round-0 bookkeeping against a
// round-1 peer and must drop it; the round-1 prevote then goes out once and
// is acknowledged. The round-0 prevote must never be offered again, which
// is what clearing POL bit 0 from the stale bookkeeping would cause.
rsR1 := cstypes.RoundState{Height: 100, Round: 1, Votes: votes}
suite.clock.Advance(voteResendInterval)
suite.voteCh.
On("Send", ctx, p2p.Envelope{To: ps.peerID, Message: &tmcons.Vote{Vote: prevoteR1.ToProto()}}).
Once().
Return(nil)
suite.gossiper.GossipVote(ctx, rsR1, ps.GetRoundState())
suite.Require().NoError(ps.ApplyHasVoteMessage(&HasVoteMessage{
Height: 100, Round: 1, Type: tmproto.PrevoteType, Index: prevoteR1.ValidatorIndex,
}))
for range 3 {
suite.clock.Advance(voteResendInterval)
for range 10 {
suite.gossiper.GossipVote(ctx, rsR1, ps.GetRoundState())
}
}
suite.Require().True(ps.PRS.ProposalPOL.GetIndex(int(prevoteR0.ValidatorIndex)),
"stale round-0 bookkeeping must not clear the peer's POL bits for round 0")
}
97 changes: 97 additions & 0 deletions internal/consensus/log_amplification_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,19 @@ import (
"context"
"errors"
"fmt"
"strings"
"testing"
"time"

"github.com/cosmos/gogoproto/proto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/dashpay/tenderdash/abci/example/kvstore"
abci "github.com/dashpay/tenderdash/abci/types"
"github.com/dashpay/tenderdash/dash"
sm "github.com/dashpay/tenderdash/internal/state"
"github.com/dashpay/tenderdash/internal/test/factory"
"github.com/dashpay/tenderdash/libs/log"
tmtime "github.com/dashpay/tenderdash/libs/time"
tmcons "github.com/dashpay/tenderdash/proto/tendermint/consensus"
Expand Down Expand Up @@ -42,6 +46,9 @@ func TestIsPeerFloodableError(t *testing.T) {
assert.True(t, isPeerFloodableError(fmt.Errorf("%w: AB12, expected CD34", types.ErrRoundSkipCertificateWrongQuorum)),
"a certificate naming another quorum is refused before any signature is read")

assert.True(t, isPeerFloodableError(fmt.Errorf("verifying precommit: %w", sm.ErrVoteExtensionRejected)),
"a precommit the application refuses is never stored, so it is replayed and re-sent for the rest of the round")

assert.False(t, isPeerFloodableError(ErrPrivValidatorNotSet),
"an internal fault must not be treated as floodable")
assert.False(t, isPeerFloodableError(errors.New("some other internal error")))
Expand Down Expand Up @@ -324,3 +331,93 @@ func TestLoggingMiddlewareSeparatesLocalFaultsFromReplay(t *testing.T) {
assert.Contains(t, out, `"level":"debug"`)
})
}

// voteExtensionRejectingApp refuses every vote extension, as an application
// does for a precommit of a block it has not processed.
type voteExtensionRejectingApp struct{ abci.Application }

func (voteExtensionRejectingApp) VerifyVoteExtension(
context.Context, *abci.RequestVerifyVoteExtension,
) (*abci.ResponseVerifyVoteExtension, error) {
return &abci.ResponseVerifyVoteExtension{Status: abci.ResponseVerifyVoteExtension_REJECT}, nil
}

// A precommit the application refuses is never stored, so no HasVote answers it
// and its sender re-offers it for the rest of the round (see
// PeerState.ReofferUnackedVotes); any peer can also replay it for free. A
// refusal of a block this node has not processed is expected and logs at Debug
// only, on every copy. Once the block is processed, the same refusal is a
// disagreement over the extensions: its first copy is worth one Warn line to an
// operator, every further copy logs at Debug, and none logs at Error.
func TestRejectedVoteExtensionLogsOnceThenDebug(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()

inner, err := kvstore.NewMemoryApp()
require.NoError(t, err)
h := newFloodHarness(ctx, t, floodHarnessArgs{validators: 4, application: voteExtensionRejectingApp{inner}})
ctx = h.runCtx(ctx)
sd := h.stateData()
_, block := decideProposal(ctx, t, h.cs, h.vss[0], sd.Height, 0)
parts, err := block.MakePartSet(types.BlockPartSizeBytes)
require.NoError(t, err)
blockID := block.BlockID(parts)
precommit := func(blockID types.BlockID) *types.Vote {
// Signing writes into the extensions, so each vote gets its own.
vote, err := h.vss[1].signVote(ctx, tmproto.PrecommitType, sd.state.ChainID, blockID,
sd.Validators.QuorumType, sd.Validators.QuorumHash, testPrecommitVote(1).VoteExtensions)
require.NoError(t, err)
return vote
}
vote := precommit(blockID)
otherBlock := precommit(factory.MakeBlockID())
incrementRound(h.vss[1])
laterRound := precommit(blockID)

var buf bytes.Buffer
logger, err := log.NewLogger("debug", &buf)
require.NoError(t, err)
handler := loggingMiddleware(logger)(func(ctx context.Context, sd *StateData, env msgEnvelope) error {
return h.cs.ctrl.Dispatch(ctx, &AddVoteEvent{Vote: env.Msg.(*VoteMessage).Vote, PeerID: env.PeerID}, sd)
})
deliverVote := func(vote *types.Vote) string {
buf.Reset()
env := msgEnvelope{msgInfo: msgInfo{Msg: &VoteMessage{Vote: vote}, PeerID: "peer"}}
require.NoError(t, handler(ctx, &sd, env))
require.Nil(t, storedVote(&sd, vote), "a rejected precommit is not stored")
return buf.String()
}
deliver := func() string { return deliverVote(vote) }

for range 2 {
unprocessed := deliver()
assert.NotContains(t, unprocessed, `"level":"error"`, "a rejected peer precommit must not log at Error")
assert.NotContains(t, unprocessed, `"level":"warn"`, "refusing a block not processed yet is expected:\n%s", unprocessed)
assert.Contains(t, unprocessed, `"level":"debug"`)
}

sd.ProposalBlock, sd.ProposalBlockParts = block, parts
require.NoError(t, h.cs.blockExecutor.ensureProcess(ctx, &sd.RoundState, sd.Round))
require.True(t, processedFor(&sd.RoundState, sd.Round))

first := deliver()
assert.NotContains(t, first, `"level":"error"`, "a rejected peer precommit must not log at Error")
assert.Equal(t, 1, strings.Count(first, `"level":"warn"`), "the first refusal of a processed block is reported once:\n%s", first)

for range 3 {
again := deliver()
assert.NotContains(t, again, `"level":"error"`, "a redelivered rejected precommit must not log at Error")
assert.NotContains(t, again, `"level":"warn"`, "a redelivered rejected precommit is not news")
assert.Contains(t, again, `"level":"debug"`)
}

// Only the processed block, for the round it was processed for, is in
// context: a refusal of another block, or of the block for another round,
// is still the expected kind.
for name, vote := range map[string]*types.Vote{"another block": otherBlock, "another round": laterRound} {
out := deliverVote(vote)
assert.NotContains(t, out, `"level":"error"`, name)
assert.NotContains(t, out, `"level":"warn"`, "%s: refusing what is not processed is expected:\n%s", name, out)
assert.Contains(t, out, `"level":"debug"`, name)
}
}
8 changes: 7 additions & 1 deletion internal/consensus/msg_handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,13 @@ func isPeerFloodableError(err error) bool {
// block signature, and the vector is covered by no signature a peer must
// forge. The rejection of the commit this node assembled from its own
// precommits is not a message error: tryFinalizeCommit logs it at Error.
errors.Is(err, sm.ErrCommitExtensionsRejected)
errors.Is(err, sm.ErrCommitExtensionsRejected) ||
// A precommit whose extensions the application refuses — as it does for
// a block this node has not processed — is never stored, so any peer can
// replay it for free and its sender re-offers it for the rest of the
// round (see PeerState.ReofferUnackedVotes). Its first copy is reported
// by addVoteVerifyVoteExtensionMw.
errors.Is(err, sm.ErrVoteExtensionRejected)
}

type msgInfoDispatcher struct {
Expand Down
Loading
Loading