diff --git a/config/config.go b/config/config.go index 9662eff83..10df67e82 100644 --- a/config/config.go +++ b/config/config.go @@ -1167,6 +1167,12 @@ type ConsensusConfig struct { // tick then already exceeds the gap it is meant to enforce. Lowering it // does not shrink the gap — it only fits more ticks (and so more sends of // a multi-part block) into that same fixed window before it applies. + // + // PeerQueryMaj23SleepDuration also sets how often the node re-announces its + // round state to each peer, which bounds how long a peer's view of this + // node stays stale after a lost announcement. Either duration under 1ms is + // run at 1ms, which only prevents a busy loop: well below the defaults, + // peers drop this node's state updates over their per-peer allowance. PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"` PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer-query-maj23-sleep-duration"` diff --git a/config/toml.go b/config/toml.go index 904b26bb2..7a995090e 100644 --- a/config/toml.go +++ b/config/toml.go @@ -566,7 +566,11 @@ create-empty-blocks-interval = "{{ .Consensus.CreateEmptyBlocksInterval }}" # Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the # tick rate a lagging peer's catch-up block-part replay is metered against # (a fixed 500ms internal throttle): raising this above 500ms disables that -# throttle. +# throttle. peer-query-maj23-sleep-duration also sets how often the node +# re-announces its round state to each peer (recovery latency after a lost +# announcement). Either duration under 1ms is run at 1ms, which only prevents a +# busy loop: well below the defaults, peers drop this node's state updates over +# their per-peer allowance. peer-gossip-sleep-duration = "{{ .Consensus.PeerGossipSleepDuration }}" peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" diff --git a/docs/nodes/configuration.md b/docs/nodes/configuration.md index 8c5090939..86809cc47 100644 --- a/docs/nodes/configuration.md +++ b/docs/nodes/configuration.md @@ -407,7 +407,14 @@ double-sign-check-height = 0 create-empty-blocks = true create-empty-blocks-interval = "0s" -# Reactor sleep duration parameters +# Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the +# tick rate a lagging peer's catch-up block-part replay is metered against +# (a fixed 500ms internal throttle): raising this above 500ms disables that +# throttle. peer-query-maj23-sleep-duration also sets how often the node +# re-announces its round state to each peer (recovery latency after a lost +# announcement). Either duration under 1ms is run at 1ms, which only prevents a +# busy loop: well below the defaults, peers drop this node's state updates over +# their per-peer allowance. peer-gossip-sleep-duration = "100ms" peer-query-maj23-sleep-duration = "2s" diff --git a/internal/consensus/gossip_handlers.go b/internal/consensus/gossip_handlers.go index 122de500f..59e933dfe 100644 --- a/internal/consensus/gossip_handlers.go +++ b/internal/consensus/gossip_handlers.go @@ -2,6 +2,7 @@ package consensus import ( "context" + "fmt" "github.com/cosmos/gogoproto/proto" @@ -13,6 +14,40 @@ import ( "github.com/dashpay/tenderdash/types" ) +// roundStateGossipHandler re-announces this node's round state to the peer: +// its height, round and step and, when the round has block parts, which of +// them it holds. Those announcements are otherwise sent once, on a state +// change, and never acknowledged, so one lost after the channel accepted it — +// to a full router queue or the peer's state-channel allowance — would leave +// the peer gossiping for the round it last heard of until the next change, and +// the step this node is in may be waiting on exactly the votes, proposal or +// block parts the peer is holding back. A repeated round step is ignored by a +// peer whose picture is current; a repeated NewValidBlock re-applies it, which +// also corrects, within a pass, a stale copy that landed after a newer one +// (see PeerState.ApplyNewValidBlockMessage). A failed send is left to the next +// pass. +func roundStateGossipHandler(logger log.Logger, send func(context.Context, proto.Message) error) gossipHandlerFunc { + return func(ctx context.Context, stateData StateData) { + for _, msg := range roundStateMessages(stateData.RoundState) { + if err := send(ctx, msg); err != nil { + logger.Debug("refreshing round state for peer failed", "msg_type", fmt.Sprintf("%T", msg), "err", err) + return + } + } + } +} + +// roundStateMessages returns what a peer needs to rebuild this node's round +// picture: the round step and, if the round has block parts, which of them it +// holds. +func roundStateMessages(rs cstypes.RoundState) []proto.Message { + msgs := []proto.Message{rs.NewRoundStepMessage()} + if rs.ProposalBlockParts != nil { + msgs = append(msgs, rs.NewValidBlockMessage()) + } + return msgs +} + func queryMaj23GossipHandler(ps *PeerState, gossiper Gossiper) gossipHandlerFunc { return func(ctx context.Context, stateData StateData) { // If peer is not a validator, we do nothing diff --git a/internal/consensus/gossip_handlers_test.go b/internal/consensus/gossip_handlers_test.go index 589636c2b..f614992a6 100644 --- a/internal/consensus/gossip_handlers_test.go +++ b/internal/consensus/gossip_handlers_test.go @@ -2,6 +2,7 @@ package consensus import ( "context" + "errors" "fmt" "testing" @@ -496,6 +497,66 @@ func TestShouldVoteBeGossiped(t *testing.T) { } } +// A peer rebuilds this node's round picture from the round step and, when the +// round has block parts, which of them this node holds. Both are sent at every +// pass; a failed send is left to the next pass. +func TestRoundStateGossipHandler(t *testing.T) { + ctx := context.Background() + parts := types.NewPartSetFromData(tmrand.Bytes(3*int(types.BlockPartSizeBytes)), types.BlockPartSizeBytes) + withParts := StateData{RoundState: cstypes.RoundState{ + Height: 7, Round: 2, Step: cstypes.RoundStepPrevote, ProposalBlockParts: parts, + }} + withoutParts := StateData{RoundState: cstypes.RoundState{Height: 7, Round: 2, Step: cstypes.RoundStepPropose}} + errSend := errors.New("channel full") + + testCases := map[string]struct { + stateData StateData + // failAt is the 1-based send that fails, 0 for none. + failAt int + wantSent []proto.Message + }{ + "should announce the round step alone when the round has no block parts": { + stateData: withoutParts, + wantSent: []proto.Message{withoutParts.NewRoundStepMessage()}, + }, + "should announce the round step and the block parts it holds": { + stateData: withParts, + wantSent: []proto.Message{withParts.NewRoundStepMessage(), withParts.NewValidBlockMessage()}, + }, + "should stop the pass at a failed send and return": { + stateData: withParts, + failAt: 1, + wantSent: []proto.Message{withParts.NewRoundStepMessage()}, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + var sent []proto.Message + send := func(_ context.Context, msg proto.Message) error { + sent = append(sent, msg) + if len(sent) == tc.failAt { + return errSend + } + return nil + } + roundStateGossipHandler(log.NewNopLogger(), send)(ctx, tc.stateData) + require.Len(t, sent, len(tc.wantSent)) + for i, want := range tc.wantSent { + switch want := want.(type) { + case *tmcons.NewRoundStep: + got, ok := sent[i].(*tmcons.NewRoundStep) + require.True(t, ok, "sent[%d] is %T", i, sent[i]) + require.Equal(t, want.Height, got.Height) + require.Equal(t, want.Round, got.Round) + require.Equal(t, want.Step, got.Step) + default: + require.Equal(t, want, sent[i]) + } + } + }) + } +} + func TestRoundSkipGossipHandler(t *testing.T) { ctx := context.Background() certificate := &types.RoundSkipCertificate{ diff --git a/internal/consensus/gossip_msg_sender.go b/internal/consensus/gossip_msg_sender.go index 8c6045ce7..cd313d0e0 100644 --- a/internal/consensus/gossip_msg_sender.go +++ b/internal/consensus/gossip_msg_sender.go @@ -34,7 +34,7 @@ func (c *p2pMsgSender) send(ctx context.Context, protoMsg proto.Message) error { return c.sendTo(ctx, c.chans.data, protoMsg) case *tmproto.Proposal: return c.sendTo(ctx, c.chans.data, &tmcons.Proposal{Proposal: *pm}) - case *tmcons.VoteSetMaj23, *tmcons.NewRoundStep, *tmcons.RoundSkipCertificate: + case *tmcons.VoteSetMaj23, *tmcons.NewRoundStep, *tmcons.NewValidBlock, *tmcons.RoundSkipCertificate: return c.sendTo(ctx, c.chans.state, protoMsg) } return fmt.Errorf("given unsupported p2p message %T", protoMsg) diff --git a/internal/consensus/gossip_msg_sender_test.go b/internal/consensus/gossip_msg_sender_test.go index 25e68cc43..4753e3ab0 100644 --- a/internal/consensus/gossip_msg_sender_test.go +++ b/internal/consensus/gossip_msg_sender_test.go @@ -76,6 +76,11 @@ func TestP2PMsgSender_Send(t *testing.T) { msg: &tmcons.VoteSetMaj23{}, want: &tmcons.VoteSetMaj23{}, }, + { + mockChannel: mockStateChannelFn, + msg: &tmcons.NewValidBlock{}, + want: &tmcons.NewValidBlock{}, + }, } for i, tc := range testCases { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index 22cdcd502..e12c298aa 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -8,9 +8,49 @@ import ( "github.com/jonboulle/clockwork" + "github.com/dashpay/tenderdash/config" "github.com/dashpay/tenderdash/libs/log" ) +// minGossipSleepDuration is the shortest cadence a gossip handler runs at, +// whatever the config says. A handler sleeping for zero runs back to back, +// spinning a goroutine per handler per peer, and config validation runs only +// on the CLI path, so the reactor holds a floor itself. It stops the spin and +// nothing more: well below the defaults the handlers still offer peers more +// than their state-channel allowance admits (see peerStateRateLimit), and the +// excess is dropped there. 1ms is the floor incoming-connection-window uses +// too; the test config runs at 5ms. +const minGossipSleepDuration = time.Millisecond + +// gossipCadence is how often each peer's gossip handlers run, floored at +// minGossipSleepDuration. The reactor derives it from the config once and +// every consumer of these settings reads it from there. +type gossipCadence struct { + // gossip is PeerGossipSleepDuration: votes, commits and block parts. + gossip time.Duration + // queryMaj23 is PeerQueryMaj23SleepDuration: majority claims, round state, + // skip certificates and vote resends. + queryMaj23 time.Duration +} + +// flooredGossipCadence returns the configured cadences, each raised to +// minGossipSleepDuration if shorter, warning once per raised setting. +func flooredGossipCadence(logger log.Logger, cfg *config.ConsensusConfig) gossipCadence { + return gossipCadence{ + gossip: flooredGossipSleep(logger, "peer-gossip-sleep-duration", cfg.PeerGossipSleepDuration), + queryMaj23: flooredGossipSleep(logger, "peer-query-maj23-sleep-duration", cfg.PeerQueryMaj23SleepDuration), + } +} + +func flooredGossipSleep(logger log.Logger, key string, configured time.Duration) time.Duration { + if configured >= minGossipSleepDuration { + return configured + } + logger.Warn("gossip sleep duration below the minimum, gossiping at the minimum instead", + "key", key, "configured", configured, "minimum", minGossipSleepDuration) + return minGossipSleepDuration +} + type gossipHandlerFunc func(ctx context.Context, appState StateData) type gossipHandler struct { @@ -41,6 +81,7 @@ func newPeerGossipWorker( ps *PeerState, state *State, msgSender *p2pMsgSender, + cadence gossipCadence, ) *peerGossipWorker { clock := clockwork.NewRealClock() gossiper := msgGossiper{ @@ -59,22 +100,26 @@ func newPeerGossipWorker( handlers: []gossipHandler{ newGossipHandler( votesAndCommitGossipHandler(ps, state.blockStore, &gossiper), - state.config.PeerGossipSleepDuration, + cadence.gossip, ), newGossipHandler( dataGossipHandler(ps, logger, state.blockStore, &gossiper), - state.config.PeerGossipSleepDuration, + cadence.gossip, ), newGossipHandler( queryMaj23GossipHandler(ps, &gossiper), - state.config.PeerQueryMaj23SleepDuration, + cadence.queryMaj23, + ), + newGossipHandler( + roundStateGossipHandler(logger, msgSender.send), + cadence.queryMaj23, ), // A certificate is resent at the majority-claim cadence: a peer // behind waits for nothing else, and one that applied it stops // being sent it as soon as it announces its new round. newGossipHandler( roundSkipGossipHandler(ps, logger, msgSender.send), - state.config.PeerQueryMaj23SleepDuration, + cadence.queryMaj23, ), // So is a vote of its round, or of a later round +2/3 voted in, to // a peer that waits in this node's round, or is in a round this @@ -84,7 +129,7 @@ func newPeerGossipWorker( // round. newGossipHandler( roundVoteResendGossipHandler(ps, logger, msgSender.send), - state.config.PeerQueryMaj23SleepDuration, + cadence.queryMaj23, ), }, } diff --git a/internal/consensus/gossip_peer_worker_test.go b/internal/consensus/gossip_peer_worker_test.go index dc9a16031..93b6d2e32 100644 --- a/internal/consensus/gossip_peer_worker_test.go +++ b/internal/consensus/gossip_peer_worker_test.go @@ -9,8 +9,10 @@ import ( "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" + "github.com/dashpay/tenderdash/config" "github.com/dashpay/tenderdash/libs/eventemitter" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/types" ) func TestPeerGossipWorker(t *testing.T) { @@ -49,3 +51,43 @@ func TestPeerGossipWorker(t *testing.T) { require.False(t, pg.IsRunning()) close(handlerCalledCh) } + +// A handler whose sleep is zero runs back to back, spinning a goroutine per +// handler per peer, and a config of 0s — or a stray "2" read as 2ns — is not +// caught where validation is skipped. The reactor floors the cadence once, for +// every consumer of the two settings. +func TestFlooredGossipCadence(t *testing.T) { + for name, tc := range map[string]struct { + gossip, queryMaj23 time.Duration + want gossipCadence + }{ + "zero": {0, 0, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}}, + "negative": {-time.Second, -time.Second, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}}, + "below the floor": {minGossipSleepDuration / 2, minGossipSleepDuration / 2, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}}, + "at the floor": {minGossipSleepDuration, minGossipSleepDuration, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}}, + "each on its own": {0, 3 * minGossipSleepDuration, gossipCadence{minGossipSleepDuration, 3 * minGossipSleepDuration}}, + "defaults kept": {100 * time.Millisecond, 2 * time.Second, gossipCadence{100 * time.Millisecond, 2 * time.Second}}, + } { + t.Run(name, func(t *testing.T) { + cfg := config.DefaultConsensusConfig() + cfg.PeerGossipSleepDuration, cfg.PeerQueryMaj23SleepDuration = tc.gossip, tc.queryMaj23 + require.Equal(t, tc.want, flooredGossipCadence(log.NewNopLogger(), cfg)) + }) + } +} + +func TestReactorFloorsGossipCadence(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cs, _ := makeState(ctx, t, makeStateArgs{}) + cs.config.PeerGossipSleepDuration = 0 + cs.config.PeerQueryMaj23SleepDuration = 0 + r := NewReactor(log.NewNopLogger(), cs, nil, nil, cs.eventBus, false, NopMetrics()) + require.Equal(t, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}, r.cadence) + + ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer")) + pgw := newPeerGossipWorker(log.NewNopLogger(), ps, cs, &p2pMsgSender{logger: log.NewNopLogger(), ps: ps}, r.cadence) + for i, hd := range pgw.handlers { + require.Equal(t, minGossipSleepDuration, hd.sleepDuration, "handler %d", i) + } +} diff --git a/internal/consensus/peer_state.go b/internal/consensus/peer_state.go index 25befbe19..289a789e1 100644 --- a/internal/consensus/peer_state.go +++ b/internal/consensus/peer_state.go @@ -237,16 +237,17 @@ func (ps *PeerState) SetHasProposal(proposal *types.Proposal) { } ps.PRS.Proposal = true + ps.PRS.ProposalPOLRound = proposal.POLRound + ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. - // ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage + // A NewValidBlockMessage may already have told us which parts the peer + // holds; the proposal's POL round is recorded above regardless. if ps.PRS.ProposalBlockParts != nil { return } ps.PRS.ProposalBlockPartSetHeader = proposal.BlockID.PartSetHeader ps.PRS.ProposalBlockParts = bits.NewBitArray(int(proposal.BlockID.PartSetHeader.Total)) - ps.PRS.ProposalPOLRound = proposal.POLRound - ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. } // InitProposalBlockParts initializes the peer's proposal block parts header @@ -658,6 +659,12 @@ func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { } // ApplyNewValidBlockMessage updates the peer state for the new valid block. +// +// The header and holdings replace what is held, with no ordering guard: the +// message carries no sequence, and the peer repeats it every gossip pass, so a +// stale copy that lands after a newer one is corrected within one pass. A +// guard that only let holdings grow would instead make a wrong bit permanent, +// and a part the peer is marked as holding is never sent to it. func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) { ps.mtx.Lock() defer ps.mtx.Unlock() @@ -746,9 +753,9 @@ const maj23AnswerTicks = 2 // the cadence at which a gossip loop repeats it. func maj23AnswerTTLFor(sleep time.Duration) time.Duration { if sleep <= 0 { - // Nothing throttles the gossip loop, so there is no cadence to count - // passes of. The default is what the rest of these ceilings are sized - // against. + // The reactor passes its floored cadence (see gossipCadence); zero is a + // peer state built without one, as NewPeerState does. The default is + // what the rest of these ceilings are sized against. sleep = config.DefaultConsensusConfig().PeerQueryMaj23SleepDuration } return maj23AnswerTicks * sleep diff --git a/internal/consensus/peer_state_test.go b/internal/consensus/peer_state_test.go index 9c4b6fdcb..094adafe9 100644 --- a/internal/consensus/peer_state_test.go +++ b/internal/consensus/peer_state_test.go @@ -5,7 +5,9 @@ import ( "github.com/stretchr/testify/require" + cstypes "github.com/dashpay/tenderdash/internal/consensus/types" "github.com/dashpay/tenderdash/libs/log" + tmrand "github.com/dashpay/tenderdash/libs/rand" tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -99,3 +101,27 @@ func TestApplyHasVoteMessage(t *testing.T) { // skip test cases like v & v3 in TestSetHasVote due to the same path } + +// A NewValidBlock from the peer that arrives before this node sends it the +// round's proposal records the peer's holdings; the proposal must still record +// its POL round, or the peer is never sent the POL prevotes. +func TestPeerStateSetHasProposalKeepsPOLRoundAfterNewValidBlock(t *testing.T) { + ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer")) + ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 5, Round: 2, Step: cstypes.RoundStepPropose}) + + parts := types.NewPartSetFromData(tmrand.Bytes(100), 64) + // The peer's periodic NewValidBlock: it holds the proposal's parts. + ps.ApplyNewValidBlockMessage(&NewValidBlockMessage{ + Height: 5, Round: 2, BlockPartSetHeader: parts.Header(), BlockParts: parts.BitArray(), + }) + // Then our data gossip sends it the proposal, a re-proposal with POL round 1. + proposal := &types.Proposal{ + Height: 5, Round: 2, POLRound: 1, + BlockID: types.BlockID{Hash: tmrand.Bytes(32), PartSetHeader: parts.Header()}, + } + ps.SetHasProposal(proposal) + + prs := ps.GetRoundState() + require.True(t, prs.Proposal) + require.EqualValues(t, 1, prs.ProposalPOLRound, "POL round lost to the periodic NewValidBlock") +} diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 8452081a5..cdac5dca0 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -147,8 +147,10 @@ const ( // ceiling over the cheap state updates too would be one several identities // could fill from inside their own legal allowances, and every honest peer's // round-step and vote announcements would then be dropped indiscriminately — - // those are sent once, on a state change, and never repeated, so dropping - // them leaves our picture of a peer stale and the gossip we send it wrong. + // vote announcements are sent once, on a state change, and never repeated, + // and round steps are repeated only once per PeerQueryMaj23SleepDuration, so + // dropping them leaves our picture of a peer stale and the gossip we send it + // wrong. // They cost a bit-set each and are already bounded per peer; the answer that // costs a bit array over every validator is what needs bounding across peers. maj23NodeRateLimit = 400 @@ -234,6 +236,10 @@ type Reactor struct { eventBus *eventbus.EventBus Metrics *Metrics + // cadence is how often each peer's gossip handlers run, from the config + // with the floor applied once. + cadence gossipCadence + mtx sync.RWMutex peers map[types.NodeID]*PeerState waitSync bool @@ -308,6 +314,7 @@ func NewReactor( r := &Reactor{ logger: logger, state: cs, + cadence: flooredGossipCadence(logger, cs.config), waitSync: waitSync, peers: make(map[types.NodeID]*PeerState), eventBus: eventBus, @@ -734,13 +741,11 @@ func (r *Reactor) sendState(ctx context.Context, stateCh p2p.Channel) { // has block parts, which of them it holds. func (r *Reactor) resendRoundState(ctx context.Context, stateCh p2p.Channel) { rs := r.state.GetStateData().RoundState - err := r.broadcast(ctx, stateCh, rs.NewRoundStepMessage()) - r.logResult(err, r.logger, "broadcasting round step message again", "height", rs.Height, "round", rs.Round) - if rs.ProposalBlockParts == nil { - return + for _, msg := range roundStateMessages(rs) { + err := r.broadcast(ctx, stateCh, msg) + r.logResult(err, r.logger, "broadcasting round state message again", + "msg_type", fmt.Sprintf("%T", msg), "height", rs.Height, "round", rs.Round) } - err = r.broadcast(ctx, stateCh, rs.NewValidBlockMessage()) - r.logResult(err, r.logger, "broadcasting new valid block message again", "height", rs.Height, "round", rs.Round) } // broadcast sends a broadcast message to all peers connected to the `channel`. @@ -921,7 +926,7 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries if !ok { ps = NewPeerState(r.logger, peerUpdate.NodeID, WithPeerStateClock(r.clock), - WithMaj23AnswerTTL(maj23AnswerTTLFor(r.state.config.PeerQueryMaj23SleepDuration))) + WithMaj23AnswerTTL(maj23AnswerTTLFor(r.cadence.queryMaj23))) ps.SetProTxHash(peerUpdate.ProTxHash) r.peers[peerUpdate.NodeID] = ps } else if len(peerUpdate.ProTxHash) > 0 { @@ -941,7 +946,7 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries ) // TODO needs to register this gossip worker, to be able to stop it once a peer will be down msgSender := p2pMsgSender{logger: logger, ps: ps, chans: chans} - pgw := newPeerGossipWorker(logger, ps, r.state, &msgSender) + pgw := newPeerGossipWorker(logger, ps, r.state, &msgSender, r.cadence) select { case <-ctx.Done(): @@ -974,18 +979,9 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries if !ps.IsRunning() { return } - // start goroutines for this peer + // start goroutines for this peer; the round-state handler's first + // pass announces our height, round and step to the peer at once _ = pgw.Start(ctx) - - // Send our state to the peer. If we're block-syncing, broadcast a - // RoundStepMessage later upon SwitchToConsensus(). - if !r.WaitSync() { - go func() { - rs := r.state.GetRoundState() - err := msgSender.send(ctx, rs.NewRoundStepMessage()) - r.logResult(err, r.logger, "sending round step msg", "height", rs.Height, "round", rs.Round) - }() - } }() } } diff --git a/internal/consensus/round_step_refresh_test.go b/internal/consensus/round_step_refresh_test.go new file mode 100644 index 000000000..093742e3c --- /dev/null +++ b/internal/consensus/round_step_refresh_test.go @@ -0,0 +1,91 @@ +package consensus + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/crypto" + "github.com/dashpay/tenderdash/internal/p2p" + p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" + "github.com/dashpay/tenderdash/libs/log" + tmcons "github.com/dashpay/tenderdash/proto/tendermint/consensus" + "github.com/dashpay/tenderdash/types" +) + +// A peer learns this node's height, round and step from NewRoundStep +// announcements, none of which is acknowledged. One lost after the channel +// accepted it — to a full router queue or the peer's state-channel allowance — +// leaves the peer gossiping for the round it last heard of. That picture must +// be corrected without a step change on this node: the step it is in may be +// waiting on exactly the votes or proposal the peer is holding back. +func TestReactorRefreshesRoundStepAfterLostAnnouncement(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const refreshInterval = 20 * time.Millisecond + cs, _ := makeState(ctx, t, makeStateArgs{}) + cs.config.PeerQueryMaj23SleepDuration = refreshInterval + r := NewReactor(log.NewNopLogger(), cs, nil, nil, cs.eventBus, false, NopMetrics()) + + announcements := make(chan *tmcons.NewRoundStep, 64) + stateCh := p2pmocks.NewChannel(t) + stateCh.On("Send", mock.Anything, mock.Anything).Maybe(). + Run(func(args mock.Arguments) { + msg, ok := args.Get(1).(p2p.Envelope).Message.(*tmcons.NewRoundStep) + if !ok { + return + } + select { + case announcements <- msg: + default: + } + }). + Return(nil) + + peerUpdate := p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusUp, ProTxHash: crypto.RandProTxHash()} + r.peerUp(ctx, peerUpdate, 3, channelBundle{state: stateCh}) + defer r.peerDown(ctx, p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusDown}, channelBundle{}) + + want := cs.GetRoundState() + + // Loss window: everything announced on connect and shortly after is lost. + lossWindow := time.After(3 * refreshInterval) + lost := 0 +drain: + for { + select { + case <-announcements: + lost++ + case <-lossWindow: + break drain + } + } + require.NotZero(t, lost, "the round step must be announced during the loss window") + + // The peer's picture of this node, built only from what arrives after the loss. + peerView := NewPeerState(log.NewNopLogger(), types.NodeID("this-node")) + deadline := time.After(ensureTimeout) + for { + select { + case msg := <-announcements: + m, err := MsgFromProto(msg) + require.NoError(t, err) + peerView.ApplyNewRoundStepMessage(m.(*NewRoundStepMessage)) + prs := peerView.GetRoundState() + if prs.Height == want.Height && prs.Round == want.Round && prs.Step == want.Step { + got := cs.GetRoundState() + require.Equal(t, want.Height, got.Height, "sender height changed during the test") + require.Equal(t, want.Round, got.Round, "sender round changed during the test") + require.Equal(t, want.Step, got.Step, "sender step changed during the test") + return + } + case <-deadline: + t.Fatalf("round step %d/%d/%v was never re-announced after the lost announcement; peer still sees %d/%d/%v", + want.Height, want.Round, want.Step, peerView.PRS.Height, peerView.PRS.Round, peerView.PRS.Step) + } + } +} diff --git a/spec/p2p/messages/consensus.md b/spec/p2p/messages/consensus.md index 40590d548..d0df52ff7 100644 --- a/spec/p2p/messages/consensus.md +++ b/spec/p2p/messages/consensus.md @@ -56,7 +56,11 @@ and the block part. NewRoundStep is sent for every step transition during the core consensus algorithm execution. It is used in the gossip part of the Tenderdash protocol to inform peers about a current -height/round/step a process is in. +height/round/step a process is in. It is also sent to each peer when the peer connects and +then periodically, at the `peer-query-maj23-sleep-duration` cadence, so that a peer whose +copy was lost in transit learns the current step without waiting for the next transition. +A receiver ignores a copy that does not advance the height, round or step it already holds +for the sender. | Name | Type | Description | Field Number | |--------------------------|--------|----------------------------------------|--------------| @@ -72,7 +76,9 @@ NewValidBlock advertises a block-part download target: its height, round, part-set header, and the parts the node already holds. `is_commit` identifies a commit target. Consensus emits this event for a current-round polka or a verified commit even when the proposal or complete block is not held. It is a gossip -announcement, not proof that the sender has processed a valid proposal. +announcement, not proof that the sender has processed a valid proposal. While the +node holds block parts for its round, the periodic per-peer NewRoundStep described +above is followed by a NewValidBlock for the same height and round. See [valid-block bookkeeping](../../consensus/consensus-alternative-flows.md#valid-block-bookkeeping) and [commit handling](../../consensus/consensus-commit.md).