Skip to content
Open
6 changes: 6 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -1167,6 +1167,12 @@ type ConsensusConfig struct {
// tick then already exceeds the gap it is meant to enforce. Lowering it
// does not shrink the gap — it only fits more ticks (and so more sends of
// a multi-part block) into that same fixed window before it applies.
//
// PeerQueryMaj23SleepDuration also sets how often the node re-announces its
// round state to each peer, which bounds how long a peer's view of this
// node stays stale after a lost announcement. Either duration under 1ms is
// run at 1ms, which only prevents a busy loop: well below the defaults,
// peers drop this node's state updates over their per-peer allowance.
PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"`
PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer-query-maj23-sleep-duration"`

Expand Down
6 changes: 5 additions & 1 deletion config/toml.go
Original file line number Diff line number Diff line change
Expand Up @@ -566,7 +566,11 @@ create-empty-blocks-interval = "{{ .Consensus.CreateEmptyBlocksInterval }}"
# Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the
# tick rate a lagging peer's catch-up block-part replay is metered against
# (a fixed 500ms internal throttle): raising this above 500ms disables that
# throttle.
# throttle. peer-query-maj23-sleep-duration also sets how often the node
# re-announces its round state to each peer (recovery latency after a lost
# announcement). Either duration under 1ms is run at 1ms, which only prevents a
# busy loop: well below the defaults, peers drop this node's state updates over
# their per-peer allowance.
peer-gossip-sleep-duration = "{{ .Consensus.PeerGossipSleepDuration }}"
peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}"

Expand Down
9 changes: 8 additions & 1 deletion docs/nodes/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,14 @@ double-sign-check-height = 0
create-empty-blocks = true
create-empty-blocks-interval = "0s"

# Reactor sleep duration parameters
# Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the
# tick rate a lagging peer's catch-up block-part replay is metered against
# (a fixed 500ms internal throttle): raising this above 500ms disables that
# throttle. peer-query-maj23-sleep-duration also sets how often the node
# re-announces its round state to each peer (recovery latency after a lost
# announcement). Either duration under 1ms is run at 1ms, which only prevents a
# busy loop: well below the defaults, peers drop this node's state updates over
# their per-peer allowance.
peer-gossip-sleep-duration = "100ms"
peer-query-maj23-sleep-duration = "2s"

Expand Down
35 changes: 35 additions & 0 deletions internal/consensus/gossip_handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package consensus

import (
"context"
"fmt"

"github.com/cosmos/gogoproto/proto"

Expand All @@ -13,6 +14,40 @@ import (
"github.com/dashpay/tenderdash/types"
)

// roundStateGossipHandler re-announces this node's round state to the peer:
// its height, round and step and, when the round has block parts, which of
// them it holds. Those announcements are otherwise sent once, on a state
// change, and never acknowledged, so one lost after the channel accepted it —
// to a full router queue or the peer's state-channel allowance — would leave
// the peer gossiping for the round it last heard of until the next change, and
// the step this node is in may be waiting on exactly the votes, proposal or
// block parts the peer is holding back. A repeated round step is ignored by a
// peer whose picture is current; a repeated NewValidBlock re-applies it, which
// also corrects, within a pass, a stale copy that landed after a newer one
// (see PeerState.ApplyNewValidBlockMessage). A failed send is left to the next
// pass.
func roundStateGossipHandler(logger log.Logger, send func(context.Context, proto.Message) error) gossipHandlerFunc {
return func(ctx context.Context, stateData StateData) {
for _, msg := range roundStateMessages(stateData.RoundState) {
if err := send(ctx, msg); err != nil {
logger.Debug("refreshing round state for peer failed", "msg_type", fmt.Sprintf("%T", msg), "err", err)
return
}
}
}
}

// roundStateMessages returns what a peer needs to rebuild this node's round
// picture: the round step and, if the round has block parts, which of them it
// holds.
func roundStateMessages(rs cstypes.RoundState) []proto.Message {
msgs := []proto.Message{rs.NewRoundStepMessage()}
if rs.ProposalBlockParts != nil {
msgs = append(msgs, rs.NewValidBlockMessage())
}
return msgs
}

func queryMaj23GossipHandler(ps *PeerState, gossiper Gossiper) gossipHandlerFunc {
return func(ctx context.Context, stateData StateData) {
// If peer is not a validator, we do nothing
Expand Down
61 changes: 61 additions & 0 deletions internal/consensus/gossip_handlers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package consensus

import (
"context"
"errors"
"fmt"
"testing"

Expand Down Expand Up @@ -496,6 +497,66 @@ func TestShouldVoteBeGossiped(t *testing.T) {
}
}

// A peer rebuilds this node's round picture from the round step and, when the
// round has block parts, which of them this node holds. Both are sent at every
// pass; a failed send is left to the next pass.
func TestRoundStateGossipHandler(t *testing.T) {
ctx := context.Background()
parts := types.NewPartSetFromData(tmrand.Bytes(3*int(types.BlockPartSizeBytes)), types.BlockPartSizeBytes)
withParts := StateData{RoundState: cstypes.RoundState{
Height: 7, Round: 2, Step: cstypes.RoundStepPrevote, ProposalBlockParts: parts,
}}
withoutParts := StateData{RoundState: cstypes.RoundState{Height: 7, Round: 2, Step: cstypes.RoundStepPropose}}
errSend := errors.New("channel full")

testCases := map[string]struct {
stateData StateData
// failAt is the 1-based send that fails, 0 for none.
failAt int
wantSent []proto.Message
}{
"should announce the round step alone when the round has no block parts": {
stateData: withoutParts,
wantSent: []proto.Message{withoutParts.NewRoundStepMessage()},
},
"should announce the round step and the block parts it holds": {
stateData: withParts,
wantSent: []proto.Message{withParts.NewRoundStepMessage(), withParts.NewValidBlockMessage()},
},
"should stop the pass at a failed send and return": {
stateData: withParts,
failAt: 1,
wantSent: []proto.Message{withParts.NewRoundStepMessage()},
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
var sent []proto.Message
send := func(_ context.Context, msg proto.Message) error {
sent = append(sent, msg)
if len(sent) == tc.failAt {
return errSend
}
return nil
}
roundStateGossipHandler(log.NewNopLogger(), send)(ctx, tc.stateData)
require.Len(t, sent, len(tc.wantSent))
for i, want := range tc.wantSent {
switch want := want.(type) {
case *tmcons.NewRoundStep:
got, ok := sent[i].(*tmcons.NewRoundStep)
require.True(t, ok, "sent[%d] is %T", i, sent[i])
require.Equal(t, want.Height, got.Height)
require.Equal(t, want.Round, got.Round)
require.Equal(t, want.Step, got.Step)
default:
require.Equal(t, want, sent[i])
}
}
})
}
}

func TestRoundSkipGossipHandler(t *testing.T) {
ctx := context.Background()
certificate := &types.RoundSkipCertificate{
Expand Down
2 changes: 1 addition & 1 deletion internal/consensus/gossip_msg_sender.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (c *p2pMsgSender) send(ctx context.Context, protoMsg proto.Message) error {
return c.sendTo(ctx, c.chans.data, protoMsg)
case *tmproto.Proposal:
return c.sendTo(ctx, c.chans.data, &tmcons.Proposal{Proposal: *pm})
case *tmcons.VoteSetMaj23, *tmcons.NewRoundStep, *tmcons.RoundSkipCertificate:
case *tmcons.VoteSetMaj23, *tmcons.NewRoundStep, *tmcons.NewValidBlock, *tmcons.RoundSkipCertificate:
return c.sendTo(ctx, c.chans.state, protoMsg)
}
return fmt.Errorf("given unsupported p2p message %T", protoMsg)
Expand Down
5 changes: 5 additions & 0 deletions internal/consensus/gossip_msg_sender_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,11 @@ func TestP2PMsgSender_Send(t *testing.T) {
msg: &tmcons.VoteSetMaj23{},
want: &tmcons.VoteSetMaj23{},
},
{
mockChannel: mockStateChannelFn,
msg: &tmcons.NewValidBlock{},
want: &tmcons.NewValidBlock{},
},
}
for i, tc := range testCases {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
Expand Down
55 changes: 50 additions & 5 deletions internal/consensus/gossip_peer_worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,49 @@ import (

"github.com/jonboulle/clockwork"

"github.com/dashpay/tenderdash/config"
"github.com/dashpay/tenderdash/libs/log"
)

// minGossipSleepDuration is the shortest cadence a gossip handler runs at,
// whatever the config says. A handler sleeping for zero runs back to back,
// spinning a goroutine per handler per peer, and config validation runs only
// on the CLI path, so the reactor holds a floor itself. It stops the spin and
// nothing more: well below the defaults the handlers still offer peers more
// than their state-channel allowance admits (see peerStateRateLimit), and the
// excess is dropped there. 1ms is the floor incoming-connection-window uses
// too; the test config runs at 5ms.
const minGossipSleepDuration = time.Millisecond

// gossipCadence is how often each peer's gossip handlers run, floored at
// minGossipSleepDuration. The reactor derives it from the config once and
// every consumer of these settings reads it from there.
type gossipCadence struct {
// gossip is PeerGossipSleepDuration: votes, commits and block parts.
gossip time.Duration
// queryMaj23 is PeerQueryMaj23SleepDuration: majority claims, round state,
// skip certificates and vote resends.
queryMaj23 time.Duration
}

// flooredGossipCadence returns the configured cadences, each raised to
// minGossipSleepDuration if shorter, warning once per raised setting.
func flooredGossipCadence(logger log.Logger, cfg *config.ConsensusConfig) gossipCadence {
return gossipCadence{
gossip: flooredGossipSleep(logger, "peer-gossip-sleep-duration", cfg.PeerGossipSleepDuration),
queryMaj23: flooredGossipSleep(logger, "peer-query-maj23-sleep-duration", cfg.PeerQueryMaj23SleepDuration),
}
}

func flooredGossipSleep(logger log.Logger, key string, configured time.Duration) time.Duration {
if configured >= minGossipSleepDuration {
return configured
}
logger.Warn("gossip sleep duration below the minimum, gossiping at the minimum instead",
"key", key, "configured", configured, "minimum", minGossipSleepDuration)
return minGossipSleepDuration
}

type gossipHandlerFunc func(ctx context.Context, appState StateData)

type gossipHandler struct {
Expand Down Expand Up @@ -41,6 +81,7 @@ func newPeerGossipWorker(
ps *PeerState,
state *State,
msgSender *p2pMsgSender,
cadence gossipCadence,
) *peerGossipWorker {
clock := clockwork.NewRealClock()
gossiper := msgGossiper{
Expand All @@ -59,22 +100,26 @@ func newPeerGossipWorker(
handlers: []gossipHandler{
newGossipHandler(
votesAndCommitGossipHandler(ps, state.blockStore, &gossiper),
state.config.PeerGossipSleepDuration,
cadence.gossip,
),
newGossipHandler(
dataGossipHandler(ps, logger, state.blockStore, &gossiper),
state.config.PeerGossipSleepDuration,
cadence.gossip,
),
newGossipHandler(
queryMaj23GossipHandler(ps, &gossiper),
state.config.PeerQueryMaj23SleepDuration,
cadence.queryMaj23,
),
newGossipHandler(
roundStateGossipHandler(logger, msgSender.send),
cadence.queryMaj23,
),
// A certificate is resent at the majority-claim cadence: a peer
// behind waits for nothing else, and one that applied it stops
// being sent it as soon as it announces its new round.
newGossipHandler(
roundSkipGossipHandler(ps, logger, msgSender.send),
state.config.PeerQueryMaj23SleepDuration,
cadence.queryMaj23,
),
// So is a vote of its round, or of a later round +2/3 voted in, to
// a peer that waits in this node's round, or is in a round this
Expand All @@ -84,7 +129,7 @@ func newPeerGossipWorker(
// round.
newGossipHandler(
roundVoteResendGossipHandler(ps, logger, msgSender.send),
state.config.PeerQueryMaj23SleepDuration,
cadence.queryMaj23,
),
},
}
Expand Down
42 changes: 42 additions & 0 deletions internal/consensus/gossip_peer_worker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,10 @@ import (
"github.com/jonboulle/clockwork"
"github.com/stretchr/testify/require"

"github.com/dashpay/tenderdash/config"
"github.com/dashpay/tenderdash/libs/eventemitter"
"github.com/dashpay/tenderdash/libs/log"
"github.com/dashpay/tenderdash/types"
)

func TestPeerGossipWorker(t *testing.T) {
Expand Down Expand Up @@ -49,3 +51,43 @@ func TestPeerGossipWorker(t *testing.T) {
require.False(t, pg.IsRunning())
close(handlerCalledCh)
}

// A handler whose sleep is zero runs back to back, spinning a goroutine per
// handler per peer, and a config of 0s — or a stray "2" read as 2ns — is not
// caught where validation is skipped. The reactor floors the cadence once, for
// every consumer of the two settings.
func TestFlooredGossipCadence(t *testing.T) {
for name, tc := range map[string]struct {
gossip, queryMaj23 time.Duration
want gossipCadence
}{
"zero": {0, 0, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}},
"negative": {-time.Second, -time.Second, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}},
"below the floor": {minGossipSleepDuration / 2, minGossipSleepDuration / 2, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}},
"at the floor": {minGossipSleepDuration, minGossipSleepDuration, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}},
"each on its own": {0, 3 * minGossipSleepDuration, gossipCadence{minGossipSleepDuration, 3 * minGossipSleepDuration}},
"defaults kept": {100 * time.Millisecond, 2 * time.Second, gossipCadence{100 * time.Millisecond, 2 * time.Second}},
} {
t.Run(name, func(t *testing.T) {
cfg := config.DefaultConsensusConfig()
cfg.PeerGossipSleepDuration, cfg.PeerQueryMaj23SleepDuration = tc.gossip, tc.queryMaj23
require.Equal(t, tc.want, flooredGossipCadence(log.NewNopLogger(), cfg))
})
}
}

func TestReactorFloorsGossipCadence(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cs, _ := makeState(ctx, t, makeStateArgs{})
cs.config.PeerGossipSleepDuration = 0
cs.config.PeerQueryMaj23SleepDuration = 0
r := NewReactor(log.NewNopLogger(), cs, nil, nil, cs.eventBus, false, NopMetrics())
require.Equal(t, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}, r.cadence)

ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer"))
pgw := newPeerGossipWorker(log.NewNopLogger(), ps, cs, &p2pMsgSender{logger: log.NewNopLogger(), ps: ps}, r.cadence)
for i, hd := range pgw.handlers {
require.Equal(t, minGossipSleepDuration, hd.sleepDuration, "handler %d", i)
}
}
19 changes: 13 additions & 6 deletions internal/consensus/peer_state.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,16 +237,17 @@ func (ps *PeerState) SetHasProposal(proposal *types.Proposal) {
}

ps.PRS.Proposal = true
ps.PRS.ProposalPOLRound = proposal.POLRound
ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received.

// ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage
// A NewValidBlockMessage may already have told us which parts the peer
// holds; the proposal's POL round is recorded above regardless.
if ps.PRS.ProposalBlockParts != nil {
return
}

ps.PRS.ProposalBlockPartSetHeader = proposal.BlockID.PartSetHeader
ps.PRS.ProposalBlockParts = bits.NewBitArray(int(proposal.BlockID.PartSetHeader.Total))
ps.PRS.ProposalPOLRound = proposal.POLRound
ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received.
}

// InitProposalBlockParts initializes the peer's proposal block parts header
Expand Down Expand Up @@ -658,6 +659,12 @@ func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) {
}

// ApplyNewValidBlockMessage updates the peer state for the new valid block.
//
// The header and holdings replace what is held, with no ordering guard: the
// message carries no sequence, and the peer repeats it every gossip pass, so a
// stale copy that lands after a newer one is corrected within one pass. A
// guard that only let holdings grow would instead make a wrong bit permanent,
// and a part the peer is marked as holding is never sent to it.
func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) {
ps.mtx.Lock()
defer ps.mtx.Unlock()
Expand Down Expand Up @@ -746,9 +753,9 @@ const maj23AnswerTicks = 2
// the cadence at which a gossip loop repeats it.
func maj23AnswerTTLFor(sleep time.Duration) time.Duration {
if sleep <= 0 {
// Nothing throttles the gossip loop, so there is no cadence to count
// passes of. The default is what the rest of these ceilings are sized
// against.
// The reactor passes its floored cadence (see gossipCadence); zero is a
// peer state built without one, as NewPeerState does. The default is
// what the rest of these ceilings are sized against.
sleep = config.DefaultConsensusConfig().PeerQueryMaj23SleepDuration
}
return maj23AnswerTicks * sleep
Expand Down
Loading
Loading