From 730f4bb8ac3982c3ad1035e60fe5840ff5b6ea45 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:16:10 +0000 Subject: [PATCH 1/8] fix(consensus): periodically re-announce round step to peers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A peer learns this node's height, round and step only from NewRoundStep, sent once on connect and once per step change, never acknowledged. One lost after the channel accepted it (router queue overflow, the peer's state-channel allowance) leaves the peer gossiping votes and proposals for the round it last heard of until this node changes step — which may never happen when that step is waiting on exactly the votes or proposal the peer is holding back, stalling the round until a reconnect. Add a per-peer gossip handler that re-sends the current round step on the PeerQueryMaj23SleepDuration cadence (one cost-1 state message per peer every 2s at the default). A repeat is a no-op for a peer whose picture is current, so the healthy path is unchanged beyond the message. Co-Authored-By: Claude Fable 5.1 --- internal/consensus/gossip_handlers.go | 15 +++ internal/consensus/gossip_peer_worker.go | 4 + internal/consensus/reactor.go | 6 +- internal/consensus/round_step_refresh_test.go | 107 ++++++++++++++++++ 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 internal/consensus/round_step_refresh_test.go diff --git a/internal/consensus/gossip_handlers.go b/internal/consensus/gossip_handlers.go index 640ad67ef..c4e87be75 100644 --- a/internal/consensus/gossip_handlers.go +++ b/internal/consensus/gossip_handlers.go @@ -8,6 +8,21 @@ import ( "github.com/dashpay/tenderdash/libs/log" ) +// roundStepGossipHandler re-announces this node's height, round and step to +// the peer. The announcement is otherwise sent once per step change and never +// acknowledged, so one lost after the channel accepted it — to a full router +// queue or the peer's state-channel allowance — would leave the peer gossiping +// for the round it last heard of until the next step change, and the step this +// node is in may be waiting on exactly the votes or proposal the peer is +// holding back. A repeat is a no-op for a peer whose picture is current. +func roundStepGossipHandler(logger log.Logger, msgSender *p2pMsgSender) gossipHandlerFunc { + return func(ctx context.Context, stateData StateData) { + if err := msgSender.send(ctx, stateData.RoundState.NewRoundStepMessage()); err != nil { + logger.Debug("refreshing round step for peer failed", "err", err) + } + } +} + func queryMaj23GossipHandler(ps *PeerState, gossiper Gossiper) gossipHandlerFunc { return func(ctx context.Context, stateData StateData) { // If peer is not a validator, we do nothing diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index 06667c796..d56c16551 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -69,6 +69,10 @@ func newPeerGossipWorker( queryMaj23GossipHandler(ps, &gossiper), state.config.PeerQueryMaj23SleepDuration, ), + newGossipHandler( + roundStepGossipHandler(logger, msgSender), + state.config.PeerQueryMaj23SleepDuration, + ), }, } } diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index d9131003f..63354fea5 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -131,8 +131,10 @@ const ( // ceiling over the cheap state updates too would be one several identities // could fill from inside their own legal allowances, and every honest peer's // round-step and vote announcements would then be dropped indiscriminately — - // those are sent once, on a state change, and never repeated, so dropping - // them leaves our picture of a peer stale and the gossip we send it wrong. + // vote announcements are sent once, on a state change, and never repeated, + // and round steps are repeated only once per PeerQueryMaj23SleepDuration, so + // dropping them leaves our picture of a peer stale and the gossip we send it + // wrong. // They cost a bit-set each and are already bounded per peer; the answer that // costs a bit array over every validator is what needs bounding across peers. maj23NodeRateLimit = 400 diff --git a/internal/consensus/round_step_refresh_test.go b/internal/consensus/round_step_refresh_test.go new file mode 100644 index 000000000..b5770945f --- /dev/null +++ b/internal/consensus/round_step_refresh_test.go @@ -0,0 +1,107 @@ +package consensus + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/crypto" + cstypes "github.com/dashpay/tenderdash/internal/consensus/types" + "github.com/dashpay/tenderdash/internal/p2p" + p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" + "github.com/dashpay/tenderdash/libs/log" + tmcons "github.com/dashpay/tenderdash/proto/tendermint/consensus" + "github.com/dashpay/tenderdash/types" +) + +// A peer learns this node's height, round and step only from NewRoundStep +// announcements, none of which is acknowledged. One lost after the channel +// accepted it — to a full router queue or the peer's state-channel allowance — +// leaves the peer gossiping for the round it last heard of. That picture must +// be corrected without a step change on this node: the step it is in may be +// waiting on exactly the votes or proposal the peer is holding back. +func TestReactorRefreshesRoundStepAfterLostAnnouncement(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + const refreshInterval = 20 * time.Millisecond + cs, _ := makeState(ctx, t, makeStateArgs{}) + cs.config.PeerQueryMaj23SleepDuration = refreshInterval + r := NewReactor(log.NewNopLogger(), cs, nil, nil, cs.eventBus, false, NopMetrics()) + + announcements := make(chan *tmcons.NewRoundStep, 64) + stateCh := p2pmocks.NewChannel(t) + stateCh.On("Send", mock.Anything, mock.Anything).Maybe(). + Run(func(args mock.Arguments) { + msg, ok := args.Get(1).(p2p.Envelope).Message.(*tmcons.NewRoundStep) + if !ok { + return + } + select { + case announcements <- msg: + default: + } + }). + Return(nil) + + peerUpdate := p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusUp, ProTxHash: crypto.RandProTxHash()} + r.peerUp(ctx, peerUpdate, 3, channelBundle{state: stateCh}) + defer r.peerDown(ctx, p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusDown}, channelBundle{}) + + want := cs.GetRoundState() + + // Loss window: everything announced on connect and shortly after is lost. + lossWindow := time.After(3 * refreshInterval) + lost := 0 +drain: + for { + select { + case <-announcements: + lost++ + case <-lossWindow: + break drain + } + } + require.NotZero(t, lost, "peer up must announce the round step") + + // The peer's picture of this node, built only from what arrives after the loss. + peerView := NewPeerState(log.NewNopLogger(), types.NodeID("this-node")) + deadline := time.After(ensureTimeout) + for { + select { + case msg := <-announcements: + m, err := MsgFromProto(msg) + require.NoError(t, err) + peerView.ApplyNewRoundStepMessage(m.(*NewRoundStepMessage)) + prs := peerView.GetRoundState() + if prs.Height == want.Height && prs.Round == want.Round && prs.Step == want.Step { + got := cs.GetRoundState() + require.Equal(t, want.Height, got.Height, "sender height changed during the test") + require.Equal(t, want.Round, got.Round, "sender round changed during the test") + require.Equal(t, want.Step, got.Step, "sender step changed during the test") + return + } + case <-deadline: + t.Fatalf("round step %d/%d/%v was never re-announced after the lost announcement; peer still sees %d/%d/%v", + want.Height, want.Round, want.Step, peerView.PRS.Height, peerView.PRS.Round, peerView.PRS.Step) + } + } +} + +func TestRoundStepGossipHandler(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ps := NewPeerState(log.NewNopLogger(), "peer") + stateData := StateData{RoundState: cstypes.RoundState{Height: 7, Round: 2, Step: cstypes.RoundStepPrevote}} + stateCh := p2pmocks.NewChannel(t) + stateCh.On("Send", ctx, p2p.Envelope{To: "peer", Message: stateData.RoundState.NewRoundStepMessage()}). + Once(). + Return(nil) + sender := &p2pMsgSender{logger: log.NewNopLogger(), ps: ps, chans: channelBundle{state: stateCh}} + + roundStepGossipHandler(log.NewNopLogger(), sender)(ctx, stateData) +} From 3419c13b73412a4ba48f31ca28300875cc226f9b Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:17:10 +0000 Subject: [PATCH 2/8] refactor(consensus): take a send func in roundStepGossipHandler Decouples the handler from p2pMsgSender and matches the shape the pending round-skip gossip handlers use. Co-Authored-By: Claude Fable 5.1 --- internal/consensus/gossip_handlers.go | 6 ++++-- internal/consensus/gossip_peer_worker.go | 2 +- internal/consensus/round_step_refresh_test.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/consensus/gossip_handlers.go b/internal/consensus/gossip_handlers.go index c4e87be75..473f0f1c7 100644 --- a/internal/consensus/gossip_handlers.go +++ b/internal/consensus/gossip_handlers.go @@ -3,6 +3,8 @@ package consensus import ( "context" + "github.com/cosmos/gogoproto/proto" + cstypes "github.com/dashpay/tenderdash/internal/consensus/types" sm "github.com/dashpay/tenderdash/internal/state" "github.com/dashpay/tenderdash/libs/log" @@ -15,9 +17,9 @@ import ( // for the round it last heard of until the next step change, and the step this // node is in may be waiting on exactly the votes or proposal the peer is // holding back. A repeat is a no-op for a peer whose picture is current. -func roundStepGossipHandler(logger log.Logger, msgSender *p2pMsgSender) gossipHandlerFunc { +func roundStepGossipHandler(logger log.Logger, send func(context.Context, proto.Message) error) gossipHandlerFunc { return func(ctx context.Context, stateData StateData) { - if err := msgSender.send(ctx, stateData.RoundState.NewRoundStepMessage()); err != nil { + if err := send(ctx, stateData.RoundState.NewRoundStepMessage()); err != nil { logger.Debug("refreshing round step for peer failed", "err", err) } } diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index d56c16551..cede568c5 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -70,7 +70,7 @@ func newPeerGossipWorker( state.config.PeerQueryMaj23SleepDuration, ), newGossipHandler( - roundStepGossipHandler(logger, msgSender), + roundStepGossipHandler(logger, msgSender.send), state.config.PeerQueryMaj23SleepDuration, ), }, diff --git a/internal/consensus/round_step_refresh_test.go b/internal/consensus/round_step_refresh_test.go index b5770945f..b283a208e 100644 --- a/internal/consensus/round_step_refresh_test.go +++ b/internal/consensus/round_step_refresh_test.go @@ -103,5 +103,5 @@ func TestRoundStepGossipHandler(t *testing.T) { Return(nil) sender := &p2pMsgSender{logger: log.NewNopLogger(), ps: ps, chans: channelBundle{state: stateCh}} - roundStepGossipHandler(log.NewNopLogger(), sender)(ctx, stateData) + roundStepGossipHandler(log.NewNopLogger(), sender.send)(ctx, stateData) } From ed1bff8892bc2b840a356151fba5342f72419f91 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 11:25:29 +0000 Subject: [PATCH 3/8] fix(config): require a positive peer-query-maj23-sleep-duration The per-peer gossip handlers registered at this cadence run back to back when it is zero; the round-step refresh sends unconditionally, so a zero value would flood every peer's state channel and burn this node's allowance there. Reject zero in ValidateBasic and document that the setting also bounds round-step re-announcement latency. Co-Authored-By: Claude Fable 5.1 --- config/config.go | 8 ++++++-- config/config_test.go | 1 + config/toml.go | 4 +++- docs/nodes/configuration.md | 7 ++++++- internal/consensus/gossip_handlers.go | 2 +- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index 86471f614..675a7ef64 100644 --- a/config/config.go +++ b/config/config.go @@ -1166,6 +1166,10 @@ type ConsensusConfig struct { // tick then already exceeds the gap it is meant to enforce. Lowering it // does not shrink the gap — it only fits more ticks (and so more sends of // a multi-part block) into that same fixed window before it applies. + // + // PeerQueryMaj23SleepDuration also sets how often the node re-announces its + // round step to each peer, which bounds how long a peer's view of this + // node stays stale after a lost announcement. It must be positive. PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"` PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer-query-maj23-sleep-duration"` @@ -1315,8 +1319,8 @@ func (cfg *ConsensusConfig) ValidateBasic() error { if cfg.PeerGossipSleepDuration < 0 { return errors.New("peer-gossip-sleep-duration can't be negative") } - if cfg.PeerQueryMaj23SleepDuration < 0 { - return errors.New("peer-query-maj23-sleep-duration can't be negative") + if cfg.PeerQueryMaj23SleepDuration <= 0 { + return errors.New("peer-query-maj23-sleep-duration must be positive") } if cfg.DoubleSignCheckHeight < 0 { return errors.New("double-sign-check-height can't be negative") diff --git a/config/config_test.go b/config/config_test.go index d7c2d91d3..edd065799 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -182,6 +182,7 @@ func TestConsensusConfig_ValidateBasic(t *testing.T) { "PeerGossipSleepDuration negative": {func(c *ConsensusConfig) { c.PeerGossipSleepDuration = -1 }, true}, "PeerQueryMaj23SleepDuration": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = time.Second }, false}, "PeerQueryMaj23SleepDuration negative": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = -1 }, true}, + "PeerQueryMaj23SleepDuration zero": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = 0 }, true}, "DoubleSignCheckHeight negative": {func(c *ConsensusConfig) { c.DoubleSignCheckHeight = -1 }, true}, } for desc, tc := range testcases { diff --git a/config/toml.go b/config/toml.go index 6eb9ac923..a7fff1866 100644 --- a/config/toml.go +++ b/config/toml.go @@ -565,7 +565,9 @@ create-empty-blocks-interval = "{{ .Consensus.CreateEmptyBlocksInterval }}" # Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the # tick rate a lagging peer's catch-up block-part replay is metered against # (a fixed 500ms internal throttle): raising this above 500ms disables that -# throttle. +# throttle. peer-query-maj23-sleep-duration also sets how often the node +# re-announces its round step to each peer (recovery latency after a lost +# announcement); it must be positive. peer-gossip-sleep-duration = "{{ .Consensus.PeerGossipSleepDuration }}" peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" diff --git a/docs/nodes/configuration.md b/docs/nodes/configuration.md index 8c5090939..b5e62e7eb 100644 --- a/docs/nodes/configuration.md +++ b/docs/nodes/configuration.md @@ -407,7 +407,12 @@ double-sign-check-height = 0 create-empty-blocks = true create-empty-blocks-interval = "0s" -# Reactor sleep duration parameters +# Reactor sleep duration parameters. peer-gossip-sleep-duration also sets the +# tick rate a lagging peer's catch-up block-part replay is metered against +# (a fixed 500ms internal throttle): raising this above 500ms disables that +# throttle. peer-query-maj23-sleep-duration also sets how often the node +# re-announces its round step to each peer (recovery latency after a lost +# announcement); it must be positive. peer-gossip-sleep-duration = "100ms" peer-query-maj23-sleep-duration = "2s" diff --git a/internal/consensus/gossip_handlers.go b/internal/consensus/gossip_handlers.go index 473f0f1c7..6f7f415d8 100644 --- a/internal/consensus/gossip_handlers.go +++ b/internal/consensus/gossip_handlers.go @@ -19,7 +19,7 @@ import ( // holding back. A repeat is a no-op for a peer whose picture is current. func roundStepGossipHandler(logger log.Logger, send func(context.Context, proto.Message) error) gossipHandlerFunc { return func(ctx context.Context, stateData StateData) { - if err := send(ctx, stateData.RoundState.NewRoundStepMessage()); err != nil { + if err := send(ctx, stateData.NewRoundStepMessage()); err != nil { logger.Debug("refreshing round step for peer failed", "err", err) } } From 96bc9b0744361d6674ae52053a0b50c1d398bcd4 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:06:14 +0000 Subject: [PATCH 4/8] fix(consensus): refresh the round's block parts with the round step The per-peer refresh re-sent only NewRoundStep, while resendRoundState treats a round picture as NewRoundStep plus NewValidBlock whenever the round has block parts. A lost NewValidBlock left the peer without this node's part-set header and holdings, and nothing repeated it. The handler now sends both from a shared roundStateMessages helper, also used by resendRoundState; a failed send ends the pass and the next one retries. The explicit NewRoundStep send in peerUp is removed: the worker starts after readySignal, where WaitSync is already false, and its first pass sends the announcement at once, so the two paths were redundant. The handler test moves next to its sibling handler tests, uses the same capturing-send pattern, and covers the send-failure branch; the P2P spec notes the periodic re-announcement. Co-Authored-By: Claude Fable 5.1 --- internal/consensus/gossip_handlers.go | 37 ++++++++--- internal/consensus/gossip_handlers_test.go | 61 +++++++++++++++++++ internal/consensus/gossip_msg_sender.go | 2 +- internal/consensus/gossip_msg_sender_test.go | 5 ++ internal/consensus/gossip_peer_worker.go | 2 +- internal/consensus/reactor.go | 23 ++----- internal/consensus/round_step_refresh_test.go | 20 +----- spec/p2p/messages/consensus.md | 10 ++- 8 files changed, 111 insertions(+), 49 deletions(-) diff --git a/internal/consensus/gossip_handlers.go b/internal/consensus/gossip_handlers.go index ee512418b..486d9aed4 100644 --- a/internal/consensus/gossip_handlers.go +++ b/internal/consensus/gossip_handlers.go @@ -2,6 +2,7 @@ package consensus import ( "context" + "fmt" "github.com/cosmos/gogoproto/proto" @@ -13,21 +14,37 @@ import ( "github.com/dashpay/tenderdash/types" ) -// roundStepGossipHandler re-announces this node's height, round and step to -// the peer. The announcement is otherwise sent once per step change and never -// acknowledged, so one lost after the channel accepted it — to a full router -// queue or the peer's state-channel allowance — would leave the peer gossiping -// for the round it last heard of until the next step change, and the step this -// node is in may be waiting on exactly the votes or proposal the peer is -// holding back. A repeat is a no-op for a peer whose picture is current. -func roundStepGossipHandler(logger log.Logger, send func(context.Context, proto.Message) error) gossipHandlerFunc { +// roundStateGossipHandler re-announces this node's round state to the peer: +// its height, round and step and, when the round has block parts, which of +// them it holds. Those announcements are otherwise sent once, on a state +// change, and never acknowledged, so one lost after the channel accepted it — +// to a full router queue or the peer's state-channel allowance — would leave +// the peer gossiping for the round it last heard of until the next change, and +// the step this node is in may be waiting on exactly the votes, proposal or +// block parts the peer is holding back. A repeat is a no-op for a peer whose +// picture is current. A failed send is left to the next pass. +func roundStateGossipHandler(logger log.Logger, send func(context.Context, proto.Message) error) gossipHandlerFunc { return func(ctx context.Context, stateData StateData) { - if err := send(ctx, stateData.NewRoundStepMessage()); err != nil { - logger.Debug("refreshing round step for peer failed", "err", err) + for _, msg := range roundStateMessages(stateData.RoundState) { + if err := send(ctx, msg); err != nil { + logger.Debug("refreshing round state for peer failed", "msg_type", fmt.Sprintf("%T", msg), "err", err) + return + } } } } +// roundStateMessages returns what a peer needs to rebuild this node's round +// picture: the round step and, if the round has block parts, which of them it +// holds. +func roundStateMessages(rs cstypes.RoundState) []proto.Message { + msgs := []proto.Message{rs.NewRoundStepMessage()} + if rs.ProposalBlockParts != nil { + msgs = append(msgs, rs.NewValidBlockMessage()) + } + return msgs +} + func queryMaj23GossipHandler(ps *PeerState, gossiper Gossiper) gossipHandlerFunc { return func(ctx context.Context, stateData StateData) { // If peer is not a validator, we do nothing diff --git a/internal/consensus/gossip_handlers_test.go b/internal/consensus/gossip_handlers_test.go index 589636c2b..f614992a6 100644 --- a/internal/consensus/gossip_handlers_test.go +++ b/internal/consensus/gossip_handlers_test.go @@ -2,6 +2,7 @@ package consensus import ( "context" + "errors" "fmt" "testing" @@ -496,6 +497,66 @@ func TestShouldVoteBeGossiped(t *testing.T) { } } +// A peer rebuilds this node's round picture from the round step and, when the +// round has block parts, which of them this node holds. Both are sent at every +// pass; a failed send is left to the next pass. +func TestRoundStateGossipHandler(t *testing.T) { + ctx := context.Background() + parts := types.NewPartSetFromData(tmrand.Bytes(3*int(types.BlockPartSizeBytes)), types.BlockPartSizeBytes) + withParts := StateData{RoundState: cstypes.RoundState{ + Height: 7, Round: 2, Step: cstypes.RoundStepPrevote, ProposalBlockParts: parts, + }} + withoutParts := StateData{RoundState: cstypes.RoundState{Height: 7, Round: 2, Step: cstypes.RoundStepPropose}} + errSend := errors.New("channel full") + + testCases := map[string]struct { + stateData StateData + // failAt is the 1-based send that fails, 0 for none. + failAt int + wantSent []proto.Message + }{ + "should announce the round step alone when the round has no block parts": { + stateData: withoutParts, + wantSent: []proto.Message{withoutParts.NewRoundStepMessage()}, + }, + "should announce the round step and the block parts it holds": { + stateData: withParts, + wantSent: []proto.Message{withParts.NewRoundStepMessage(), withParts.NewValidBlockMessage()}, + }, + "should stop the pass at a failed send and return": { + stateData: withParts, + failAt: 1, + wantSent: []proto.Message{withParts.NewRoundStepMessage()}, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + var sent []proto.Message + send := func(_ context.Context, msg proto.Message) error { + sent = append(sent, msg) + if len(sent) == tc.failAt { + return errSend + } + return nil + } + roundStateGossipHandler(log.NewNopLogger(), send)(ctx, tc.stateData) + require.Len(t, sent, len(tc.wantSent)) + for i, want := range tc.wantSent { + switch want := want.(type) { + case *tmcons.NewRoundStep: + got, ok := sent[i].(*tmcons.NewRoundStep) + require.True(t, ok, "sent[%d] is %T", i, sent[i]) + require.Equal(t, want.Height, got.Height) + require.Equal(t, want.Round, got.Round) + require.Equal(t, want.Step, got.Step) + default: + require.Equal(t, want, sent[i]) + } + } + }) + } +} + func TestRoundSkipGossipHandler(t *testing.T) { ctx := context.Background() certificate := &types.RoundSkipCertificate{ diff --git a/internal/consensus/gossip_msg_sender.go b/internal/consensus/gossip_msg_sender.go index 8c6045ce7..cd313d0e0 100644 --- a/internal/consensus/gossip_msg_sender.go +++ b/internal/consensus/gossip_msg_sender.go @@ -34,7 +34,7 @@ func (c *p2pMsgSender) send(ctx context.Context, protoMsg proto.Message) error { return c.sendTo(ctx, c.chans.data, protoMsg) case *tmproto.Proposal: return c.sendTo(ctx, c.chans.data, &tmcons.Proposal{Proposal: *pm}) - case *tmcons.VoteSetMaj23, *tmcons.NewRoundStep, *tmcons.RoundSkipCertificate: + case *tmcons.VoteSetMaj23, *tmcons.NewRoundStep, *tmcons.NewValidBlock, *tmcons.RoundSkipCertificate: return c.sendTo(ctx, c.chans.state, protoMsg) } return fmt.Errorf("given unsupported p2p message %T", protoMsg) diff --git a/internal/consensus/gossip_msg_sender_test.go b/internal/consensus/gossip_msg_sender_test.go index 25e68cc43..4753e3ab0 100644 --- a/internal/consensus/gossip_msg_sender_test.go +++ b/internal/consensus/gossip_msg_sender_test.go @@ -76,6 +76,11 @@ func TestP2PMsgSender_Send(t *testing.T) { msg: &tmcons.VoteSetMaj23{}, want: &tmcons.VoteSetMaj23{}, }, + { + mockChannel: mockStateChannelFn, + msg: &tmcons.NewValidBlock{}, + want: &tmcons.NewValidBlock{}, + }, } for i, tc := range testCases { t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index 597f3f525..a1712f633 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -70,7 +70,7 @@ func newPeerGossipWorker( state.config.PeerQueryMaj23SleepDuration, ), newGossipHandler( - roundStepGossipHandler(logger, msgSender.send), + roundStateGossipHandler(logger, msgSender.send), state.config.PeerQueryMaj23SleepDuration, ), // A certificate is resent at the majority-claim cadence: a peer diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 3ebfc4d15..552c6ecc7 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -736,13 +736,11 @@ func (r *Reactor) sendState(ctx context.Context, stateCh p2p.Channel) { // has block parts, which of them it holds. func (r *Reactor) resendRoundState(ctx context.Context, stateCh p2p.Channel) { rs := r.state.GetStateData().RoundState - err := r.broadcast(ctx, stateCh, rs.NewRoundStepMessage()) - r.logResult(err, r.logger, "broadcasting round step message again", "height", rs.Height, "round", rs.Round) - if rs.ProposalBlockParts == nil { - return + for _, msg := range roundStateMessages(rs) { + err := r.broadcast(ctx, stateCh, msg) + r.logResult(err, r.logger, "broadcasting round state message again", + "msg_type", fmt.Sprintf("%T", msg), "height", rs.Height, "round", rs.Round) } - err = r.broadcast(ctx, stateCh, rs.NewValidBlockMessage()) - r.logResult(err, r.logger, "broadcasting new valid block message again", "height", rs.Height, "round", rs.Round) } // broadcast sends a broadcast message to all peers connected to the `channel`. @@ -976,18 +974,9 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries if !ps.IsRunning() { return } - // start goroutines for this peer + // start goroutines for this peer; the round-state handler's first + // pass announces our height, round and step to the peer at once _ = pgw.Start(ctx) - - // Send our state to the peer. If we're block-syncing, broadcast a - // RoundStepMessage later upon SwitchToConsensus(). - if !r.WaitSync() { - go func() { - rs := r.state.GetRoundState() - err := msgSender.send(ctx, rs.NewRoundStepMessage()) - r.logResult(err, r.logger, "sending round step msg", "height", rs.Height, "round", rs.Round) - }() - } }() } } diff --git a/internal/consensus/round_step_refresh_test.go b/internal/consensus/round_step_refresh_test.go index b283a208e..093742e3c 100644 --- a/internal/consensus/round_step_refresh_test.go +++ b/internal/consensus/round_step_refresh_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/require" "github.com/dashpay/tenderdash/crypto" - cstypes "github.com/dashpay/tenderdash/internal/consensus/types" "github.com/dashpay/tenderdash/internal/p2p" p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" "github.com/dashpay/tenderdash/libs/log" @@ -17,7 +16,7 @@ import ( "github.com/dashpay/tenderdash/types" ) -// A peer learns this node's height, round and step only from NewRoundStep +// A peer learns this node's height, round and step from NewRoundStep // announcements, none of which is acknowledged. One lost after the channel // accepted it — to a full router queue or the peer's state-channel allowance — // leaves the peer gossiping for the round it last heard of. That picture must @@ -65,7 +64,7 @@ drain: break drain } } - require.NotZero(t, lost, "peer up must announce the round step") + require.NotZero(t, lost, "the round step must be announced during the loss window") // The peer's picture of this node, built only from what arrives after the loss. peerView := NewPeerState(log.NewNopLogger(), types.NodeID("this-node")) @@ -90,18 +89,3 @@ drain: } } } - -func TestRoundStepGossipHandler(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - ps := NewPeerState(log.NewNopLogger(), "peer") - stateData := StateData{RoundState: cstypes.RoundState{Height: 7, Round: 2, Step: cstypes.RoundStepPrevote}} - stateCh := p2pmocks.NewChannel(t) - stateCh.On("Send", ctx, p2p.Envelope{To: "peer", Message: stateData.RoundState.NewRoundStepMessage()}). - Once(). - Return(nil) - sender := &p2pMsgSender{logger: log.NewNopLogger(), ps: ps, chans: channelBundle{state: stateCh}} - - roundStepGossipHandler(log.NewNopLogger(), sender.send)(ctx, stateData) -} diff --git a/spec/p2p/messages/consensus.md b/spec/p2p/messages/consensus.md index 40590d548..d0df52ff7 100644 --- a/spec/p2p/messages/consensus.md +++ b/spec/p2p/messages/consensus.md @@ -56,7 +56,11 @@ and the block part. NewRoundStep is sent for every step transition during the core consensus algorithm execution. It is used in the gossip part of the Tenderdash protocol to inform peers about a current -height/round/step a process is in. +height/round/step a process is in. It is also sent to each peer when the peer connects and +then periodically, at the `peer-query-maj23-sleep-duration` cadence, so that a peer whose +copy was lost in transit learns the current step without waiting for the next transition. +A receiver ignores a copy that does not advance the height, round or step it already holds +for the sender. | Name | Type | Description | Field Number | |--------------------------|--------|----------------------------------------|--------------| @@ -72,7 +76,9 @@ NewValidBlock advertises a block-part download target: its height, round, part-set header, and the parts the node already holds. `is_commit` identifies a commit target. Consensus emits this event for a current-round polka or a verified commit even when the proposal or complete block is not held. It is a gossip -announcement, not proof that the sender has processed a valid proposal. +announcement, not proof that the sender has processed a valid proposal. While the +node holds block parts for its round, the periodic per-peer NewRoundStep described +above is followed by a NewValidBlock for the same height and round. See [valid-block bookkeeping](../../consensus/consensus-alternative-flows.md#valid-block-bookkeeping) and [commit handling](../../consensus/consensus-commit.md). From adbc82201eca00038402cb4a556fc3a9a51cbbfa Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:06:27 +0000 Subject: [PATCH 5/8] fix(consensus): floor gossip sleep durations instead of rejecting zero Rejecting peer-query-maj23-sleep-duration <= 0 in ValidateBasic was a breaking config change that still left the loop unguarded: any tiny positive value spun the handlers the same way, peer-gossip-sleep-duration = 0 was still accepted, and ValidateBasic runs only on the CLI path, so an embedder calling node.New got no check at all. The per-peer gossip worker now runs no handler faster than minGossipSleepDuration (1ms, the same floor as incoming-connection-window) whatever either setting says, warning once per worker when it clamps. The <= 0 rejection and its test are reverted, so a 0s config starts again, and the config comments and the maj23AnswerTTLFor zero branch describe the floor rather than a validation rule. Co-Authored-By: Claude Fable 5.1 --- config/config.go | 9 ++--- config/config_test.go | 1 - config/toml.go | 4 +-- docs/nodes/configuration.md | 4 +-- internal/consensus/gossip_peer_worker.go | 34 +++++++++++++++---- internal/consensus/gossip_peer_worker_test.go | 32 +++++++++++++++++ internal/consensus/peer_state.go | 6 ++-- 7 files changed, 72 insertions(+), 18 deletions(-) diff --git a/config/config.go b/config/config.go index b265a2cd3..a70aa0c62 100644 --- a/config/config.go +++ b/config/config.go @@ -1169,8 +1169,9 @@ type ConsensusConfig struct { // a multi-part block) into that same fixed window before it applies. // // PeerQueryMaj23SleepDuration also sets how often the node re-announces its - // round step to each peer, which bounds how long a peer's view of this - // node stays stale after a lost announcement. It must be positive. + // round state to each peer, which bounds how long a peer's view of this + // node stays stale after a lost announcement. Either duration under 1ms is + // run at 1ms. PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"` PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer-query-maj23-sleep-duration"` @@ -1320,8 +1321,8 @@ func (cfg *ConsensusConfig) ValidateBasic() error { if cfg.PeerGossipSleepDuration < 0 { return errors.New("peer-gossip-sleep-duration can't be negative") } - if cfg.PeerQueryMaj23SleepDuration <= 0 { - return errors.New("peer-query-maj23-sleep-duration must be positive") + if cfg.PeerQueryMaj23SleepDuration < 0 { + return errors.New("peer-query-maj23-sleep-duration can't be negative") } if cfg.DoubleSignCheckHeight < 0 { return errors.New("double-sign-check-height can't be negative") diff --git a/config/config_test.go b/config/config_test.go index edd065799..d7c2d91d3 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -182,7 +182,6 @@ func TestConsensusConfig_ValidateBasic(t *testing.T) { "PeerGossipSleepDuration negative": {func(c *ConsensusConfig) { c.PeerGossipSleepDuration = -1 }, true}, "PeerQueryMaj23SleepDuration": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = time.Second }, false}, "PeerQueryMaj23SleepDuration negative": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = -1 }, true}, - "PeerQueryMaj23SleepDuration zero": {func(c *ConsensusConfig) { c.PeerQueryMaj23SleepDuration = 0 }, true}, "DoubleSignCheckHeight negative": {func(c *ConsensusConfig) { c.DoubleSignCheckHeight = -1 }, true}, } for desc, tc := range testcases { diff --git a/config/toml.go b/config/toml.go index 3aab4edce..ebb3f0ee0 100644 --- a/config/toml.go +++ b/config/toml.go @@ -567,8 +567,8 @@ create-empty-blocks-interval = "{{ .Consensus.CreateEmptyBlocksInterval }}" # tick rate a lagging peer's catch-up block-part replay is metered against # (a fixed 500ms internal throttle): raising this above 500ms disables that # throttle. peer-query-maj23-sleep-duration also sets how often the node -# re-announces its round step to each peer (recovery latency after a lost -# announcement); it must be positive. +# re-announces its round state to each peer (recovery latency after a lost +# announcement). Either duration under 1ms is run at 1ms. peer-gossip-sleep-duration = "{{ .Consensus.PeerGossipSleepDuration }}" peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" diff --git a/docs/nodes/configuration.md b/docs/nodes/configuration.md index b5e62e7eb..6dee8c0f7 100644 --- a/docs/nodes/configuration.md +++ b/docs/nodes/configuration.md @@ -411,8 +411,8 @@ create-empty-blocks-interval = "0s" # tick rate a lagging peer's catch-up block-part replay is metered against # (a fixed 500ms internal throttle): raising this above 500ms disables that # throttle. peer-query-maj23-sleep-duration also sets how often the node -# re-announces its round step to each peer (recovery latency after a lost -# announcement); it must be positive. +# re-announces its round state to each peer (recovery latency after a lost +# announcement). Either duration under 1ms is run at 1ms. peer-gossip-sleep-duration = "100ms" peer-query-maj23-sleep-duration = "2s" diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index a1712f633..05fd3ee85 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -11,6 +11,15 @@ import ( "github.com/dashpay/tenderdash/libs/log" ) +// minGossipSleepDuration is the shortest cadence a gossip handler runs at, +// whatever the config says. A handler sleeping for zero runs back to back, and +// the round-state handler sends on every pass, so a config of 0s — or a stray +// "2" read as 2ns — would spin a goroutine per handler per peer and flood the +// peer's state channel. Config validation runs only on the CLI path, so the +// worker holds the floor itself. 1ms is the same floor as +// incoming-connection-window; the test config runs at 5ms. +const minGossipSleepDuration = time.Millisecond + type gossipHandlerFunc func(ctx context.Context, appState StateData) type gossipHandler struct { @@ -43,6 +52,8 @@ func newPeerGossipWorker( msgSender *p2pMsgSender, ) *peerGossipWorker { clock := clockwork.NewRealClock() + gossipSleep := flooredGossipSleep(logger, "peer-gossip-sleep-duration", state.config.PeerGossipSleepDuration) + queryMaj23Sleep := flooredGossipSleep(logger, "peer-query-maj23-sleep-duration", state.config.PeerQueryMaj23SleepDuration) gossiper := msgGossiper{ ps: ps, blockStore: &blockRepository{BlockStore: state.blockStore, logger: logger}, @@ -59,26 +70,26 @@ func newPeerGossipWorker( handlers: []gossipHandler{ newGossipHandler( votesAndCommitGossipHandler(ps, state.blockStore, &gossiper), - state.config.PeerGossipSleepDuration, + gossipSleep, ), newGossipHandler( dataGossipHandler(ps, logger, state.blockStore, &gossiper), - state.config.PeerGossipSleepDuration, + gossipSleep, ), newGossipHandler( queryMaj23GossipHandler(ps, &gossiper), - state.config.PeerQueryMaj23SleepDuration, + queryMaj23Sleep, ), newGossipHandler( roundStateGossipHandler(logger, msgSender.send), - state.config.PeerQueryMaj23SleepDuration, + queryMaj23Sleep, ), // A certificate is resent at the majority-claim cadence: a peer // behind waits for nothing else, and one that applied it stops // being sent it as soon as it announces its new round. newGossipHandler( roundSkipGossipHandler(ps, logger, msgSender.send), - state.config.PeerQueryMaj23SleepDuration, + queryMaj23Sleep, ), // So is a vote of its round, or of a later round +2/3 voted in, to // a peer that waits in this node's round, or is in a round this @@ -88,12 +99,23 @@ func newPeerGossipWorker( // round. newGossipHandler( roundVoteResendGossipHandler(ps, logger, msgSender.send), - state.config.PeerQueryMaj23SleepDuration, + queryMaj23Sleep, ), }, } } +// flooredGossipSleep returns the configured sleep, or minGossipSleepDuration +// if it is shorter. It is logged once per worker, not per pass. +func flooredGossipSleep(logger log.Logger, key string, configured time.Duration) time.Duration { + if configured >= minGossipSleepDuration { + return configured + } + logger.Warn("gossip sleep duration below the minimum, gossiping at the minimum instead", + "key", key, "configured", configured, "minimum", minGossipSleepDuration) + return minGossipSleepDuration +} + func (g *peerGossipWorker) IsRunning() bool { return g.running.Load() } diff --git a/internal/consensus/gossip_peer_worker_test.go b/internal/consensus/gossip_peer_worker_test.go index dc9a16031..81e702ed4 100644 --- a/internal/consensus/gossip_peer_worker_test.go +++ b/internal/consensus/gossip_peer_worker_test.go @@ -11,6 +11,7 @@ import ( "github.com/dashpay/tenderdash/libs/eventemitter" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/types" ) func TestPeerGossipWorker(t *testing.T) { @@ -49,3 +50,34 @@ func TestPeerGossipWorker(t *testing.T) { require.False(t, pg.IsRunning()) close(handlerCalledCh) } + +// A handler whose sleep is zero runs back to back, and the round-state handler +// sends on every pass, so a config of 0s — or a stray "2" read as 2ns — would +// flood every peer. The worker runs no handler faster than the floor, whatever +// the config says and whether or not it was validated. +func TestPeerGossipWorkerFloorsSleepDurations(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cs, _ := makeState(ctx, t, makeStateArgs{}) + ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer")) + sleepsFor := func(gossip, queryMaj23 time.Duration) []time.Duration { + cs.config.PeerGossipSleepDuration = gossip + cs.config.PeerQueryMaj23SleepDuration = queryMaj23 + pgw := newPeerGossipWorker(log.NewNopLogger(), ps, cs, &p2pMsgSender{logger: log.NewNopLogger(), ps: ps}) + sleeps := make([]time.Duration, 0, len(pgw.handlers)) + for _, hd := range pgw.handlers { + sleeps = append(sleeps, hd.sleepDuration) + } + return sleeps + } + + for _, sleep := range []time.Duration{0, -time.Second, minGossipSleepDuration / 2} { + for i, got := range sleepsFor(sleep, sleep) { + require.Equal(t, minGossipSleepDuration, got, "handler %d configured at %s", i, sleep) + } + } + + kept := sleepsFor(minGossipSleepDuration, 3*minGossipSleepDuration) + require.Contains(t, kept, minGossipSleepDuration) + require.Contains(t, kept, 3*minGossipSleepDuration) +} diff --git a/internal/consensus/peer_state.go b/internal/consensus/peer_state.go index 25befbe19..72e42319d 100644 --- a/internal/consensus/peer_state.go +++ b/internal/consensus/peer_state.go @@ -746,9 +746,9 @@ const maj23AnswerTicks = 2 // the cadence at which a gossip loop repeats it. func maj23AnswerTTLFor(sleep time.Duration) time.Duration { if sleep <= 0 { - // Nothing throttles the gossip loop, so there is no cadence to count - // passes of. The default is what the rest of these ceilings are sized - // against. + // The gossip loop runs such a cadence at its floor (see + // minGossipSleepDuration), too fast for a pass count to mean anything. + // The default is what the rest of these ceilings are sized against. sleep = config.DefaultConsensusConfig().PeerQueryMaj23SleepDuration } return maj23AnswerTicks * sleep From 466ea1a07a27464dceccb7d67e73280cd540f7ca Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:12:06 +0000 Subject: [PATCH 6/8] test(consensus): reproduce POL round lost to a periodic NewValidBlock A peer's periodic NewValidBlock landing before we send it the round's proposal makes PeerState.SetHasProposal return early, so the proposal's POL round is never recorded for the peer. The test fails on adbc82201. Co-Authored-By: Claude Opus 5.5 --- internal/consensus/review_nvb_pol_test.go | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 internal/consensus/review_nvb_pol_test.go diff --git a/internal/consensus/review_nvb_pol_test.go b/internal/consensus/review_nvb_pol_test.go new file mode 100644 index 000000000..7ac85445d --- /dev/null +++ b/internal/consensus/review_nvb_pol_test.go @@ -0,0 +1,35 @@ +package consensus + +import ( + "testing" + + "github.com/stretchr/testify/require" + + cstypes "github.com/dashpay/tenderdash/internal/consensus/types" + "github.com/dashpay/tenderdash/libs/log" + tmrand "github.com/dashpay/tenderdash/libs/rand" + "github.com/dashpay/tenderdash/types" +) + +// A periodic NewValidBlock that reaches us before we send the peer the round's +// proposal must not stop us from learning the proposal's POL round. +func TestReviewPeriodicNewValidBlockKeepsProposalPOLRound(t *testing.T) { + ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer")) + ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 5, Round: 2, Step: cstypes.RoundStepPropose}) + + parts := types.NewPartSetFromData(tmrand.Bytes(100), 64) + // The peer's periodic NewValidBlock: it holds the proposal's parts. + ps.ApplyNewValidBlockMessage(&NewValidBlockMessage{ + Height: 5, Round: 2, BlockPartSetHeader: parts.Header(), BlockParts: parts.BitArray(), + }) + // Then our data gossip sends it the proposal, a re-proposal with POL round 1. + proposal := &types.Proposal{ + Height: 5, Round: 2, POLRound: 1, + BlockID: types.BlockID{Hash: tmrand.Bytes(32), PartSetHeader: parts.Header()}, + } + ps.SetHasProposal(proposal) + + prs := ps.GetRoundState() + require.True(t, prs.Proposal) + require.EqualValues(t, 1, prs.ProposalPOLRound, "POL round lost to the periodic NewValidBlock") +} From 39dcbc02138019ea1240b12dee7296d175cae415 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:18:48 +0000 Subject: [PATCH 7/8] fix(consensus): record a proposal's POL round after a NewValidBlock SetHasProposal returned before recording ProposalPOLRound when the peer's NewValidBlock had already set its block parts, so a re-proposal's POL round stayed at -1 and the peer was never sent the POL prevotes or the POL majority claim. The periodic round-state refresh makes that ordering routine. The POL fields are now recorded before the block-parts early return; the reviewer's repro test moves next to the other peer-state tests. ApplyNewValidBlockMessage keeps replacing the header and holdings with no ordering guard, now documented: a stale copy is corrected within one pass, whereas a grow-only guard would make a wrong bit permanent. Co-Authored-By: Claude Fable 5.1 --- internal/consensus/gossip_handlers.go | 7 +++-- internal/consensus/peer_state.go | 13 +++++++-- internal/consensus/peer_state_test.go | 26 +++++++++++++++++ internal/consensus/review_nvb_pol_test.go | 35 ----------------------- 4 files changed, 41 insertions(+), 40 deletions(-) delete mode 100644 internal/consensus/review_nvb_pol_test.go diff --git a/internal/consensus/gossip_handlers.go b/internal/consensus/gossip_handlers.go index 486d9aed4..59e933dfe 100644 --- a/internal/consensus/gossip_handlers.go +++ b/internal/consensus/gossip_handlers.go @@ -21,8 +21,11 @@ import ( // to a full router queue or the peer's state-channel allowance — would leave // the peer gossiping for the round it last heard of until the next change, and // the step this node is in may be waiting on exactly the votes, proposal or -// block parts the peer is holding back. A repeat is a no-op for a peer whose -// picture is current. A failed send is left to the next pass. +// block parts the peer is holding back. A repeated round step is ignored by a +// peer whose picture is current; a repeated NewValidBlock re-applies it, which +// also corrects, within a pass, a stale copy that landed after a newer one +// (see PeerState.ApplyNewValidBlockMessage). A failed send is left to the next +// pass. func roundStateGossipHandler(logger log.Logger, send func(context.Context, proto.Message) error) gossipHandlerFunc { return func(ctx context.Context, stateData StateData) { for _, msg := range roundStateMessages(stateData.RoundState) { diff --git a/internal/consensus/peer_state.go b/internal/consensus/peer_state.go index 72e42319d..1ab633472 100644 --- a/internal/consensus/peer_state.go +++ b/internal/consensus/peer_state.go @@ -237,16 +237,17 @@ func (ps *PeerState) SetHasProposal(proposal *types.Proposal) { } ps.PRS.Proposal = true + ps.PRS.ProposalPOLRound = proposal.POLRound + ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. - // ps.PRS.ProposalBlockParts is set due to NewValidBlockMessage + // A NewValidBlockMessage may already have told us which parts the peer + // holds; the proposal's POL round is recorded above regardless. if ps.PRS.ProposalBlockParts != nil { return } ps.PRS.ProposalBlockPartSetHeader = proposal.BlockID.PartSetHeader ps.PRS.ProposalBlockParts = bits.NewBitArray(int(proposal.BlockID.PartSetHeader.Total)) - ps.PRS.ProposalPOLRound = proposal.POLRound - ps.PRS.ProposalPOL = nil // Nil until ProposalPOLMessage received. } // InitProposalBlockParts initializes the peer's proposal block parts header @@ -658,6 +659,12 @@ func (ps *PeerState) ApplyNewRoundStepMessage(msg *NewRoundStepMessage) { } // ApplyNewValidBlockMessage updates the peer state for the new valid block. +// +// The header and holdings replace what is held, with no ordering guard: the +// message carries no sequence, and the peer repeats it every gossip pass, so a +// stale copy that lands after a newer one is corrected within one pass. A +// guard that only let holdings grow would instead make a wrong bit permanent, +// and a part the peer is marked as holding is never sent to it. func (ps *PeerState) ApplyNewValidBlockMessage(msg *NewValidBlockMessage) { ps.mtx.Lock() defer ps.mtx.Unlock() diff --git a/internal/consensus/peer_state_test.go b/internal/consensus/peer_state_test.go index 9c4b6fdcb..094adafe9 100644 --- a/internal/consensus/peer_state_test.go +++ b/internal/consensus/peer_state_test.go @@ -5,7 +5,9 @@ import ( "github.com/stretchr/testify/require" + cstypes "github.com/dashpay/tenderdash/internal/consensus/types" "github.com/dashpay/tenderdash/libs/log" + tmrand "github.com/dashpay/tenderdash/libs/rand" tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -99,3 +101,27 @@ func TestApplyHasVoteMessage(t *testing.T) { // skip test cases like v & v3 in TestSetHasVote due to the same path } + +// A NewValidBlock from the peer that arrives before this node sends it the +// round's proposal records the peer's holdings; the proposal must still record +// its POL round, or the peer is never sent the POL prevotes. +func TestPeerStateSetHasProposalKeepsPOLRoundAfterNewValidBlock(t *testing.T) { + ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer")) + ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 5, Round: 2, Step: cstypes.RoundStepPropose}) + + parts := types.NewPartSetFromData(tmrand.Bytes(100), 64) + // The peer's periodic NewValidBlock: it holds the proposal's parts. + ps.ApplyNewValidBlockMessage(&NewValidBlockMessage{ + Height: 5, Round: 2, BlockPartSetHeader: parts.Header(), BlockParts: parts.BitArray(), + }) + // Then our data gossip sends it the proposal, a re-proposal with POL round 1. + proposal := &types.Proposal{ + Height: 5, Round: 2, POLRound: 1, + BlockID: types.BlockID{Hash: tmrand.Bytes(32), PartSetHeader: parts.Header()}, + } + ps.SetHasProposal(proposal) + + prs := ps.GetRoundState() + require.True(t, prs.Proposal) + require.EqualValues(t, 1, prs.ProposalPOLRound, "POL round lost to the periodic NewValidBlock") +} diff --git a/internal/consensus/review_nvb_pol_test.go b/internal/consensus/review_nvb_pol_test.go deleted file mode 100644 index 7ac85445d..000000000 --- a/internal/consensus/review_nvb_pol_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package consensus - -import ( - "testing" - - "github.com/stretchr/testify/require" - - cstypes "github.com/dashpay/tenderdash/internal/consensus/types" - "github.com/dashpay/tenderdash/libs/log" - tmrand "github.com/dashpay/tenderdash/libs/rand" - "github.com/dashpay/tenderdash/types" -) - -// A periodic NewValidBlock that reaches us before we send the peer the round's -// proposal must not stop us from learning the proposal's POL round. -func TestReviewPeriodicNewValidBlockKeepsProposalPOLRound(t *testing.T) { - ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer")) - ps.ApplyNewRoundStepMessage(&NewRoundStepMessage{Height: 5, Round: 2, Step: cstypes.RoundStepPropose}) - - parts := types.NewPartSetFromData(tmrand.Bytes(100), 64) - // The peer's periodic NewValidBlock: it holds the proposal's parts. - ps.ApplyNewValidBlockMessage(&NewValidBlockMessage{ - Height: 5, Round: 2, BlockPartSetHeader: parts.Header(), BlockParts: parts.BitArray(), - }) - // Then our data gossip sends it the proposal, a re-proposal with POL round 1. - proposal := &types.Proposal{ - Height: 5, Round: 2, POLRound: 1, - BlockID: types.BlockID{Hash: tmrand.Bytes(32), PartSetHeader: parts.Header()}, - } - ps.SetHasProposal(proposal) - - prs := ps.GetRoundState() - require.True(t, prs.Proposal) - require.EqualValues(t, 1, prs.ProposalPOLRound, "POL round lost to the periodic NewValidBlock") -} From 68dba53da7d41893744a187d1db90af509894015 Mon Sep 17 00:00:00 2001 From: Lukasz Klimek <842586+lklimek@users.noreply.github.com> Date: Mon, 9 Nov 2026 14:19:12 +0000 Subject: [PATCH 8/8] refactor(consensus): floor the gossip cadence once at the reactor The floor was applied inside newPeerGossipWorker, so the maj23 answer TTL still read the raw config value and the warning repeated on every peerUp. NewReactor now derives a gossipCadence from the config once, warning once per raised setting, and both the per-peer workers and maj23AnswerTTLFor read it from there. The floor's comment and the config docs now say what it does: it stops a busy loop and nothing more; well below the defaults the handlers still offer peers more than their state-channel allowance admits. Co-Authored-By: Claude Fable 5.1 --- config/config.go | 3 +- config/toml.go | 4 +- docs/nodes/configuration.md | 4 +- internal/consensus/gossip_peer_worker.go | 69 ++++++++++++------- internal/consensus/gossip_peer_worker_test.go | 58 +++++++++------- internal/consensus/peer_state.go | 6 +- internal/consensus/reactor.go | 9 ++- 7 files changed, 96 insertions(+), 57 deletions(-) diff --git a/config/config.go b/config/config.go index a70aa0c62..10df67e82 100644 --- a/config/config.go +++ b/config/config.go @@ -1171,7 +1171,8 @@ type ConsensusConfig struct { // PeerQueryMaj23SleepDuration also sets how often the node re-announces its // round state to each peer, which bounds how long a peer's view of this // node stays stale after a lost announcement. Either duration under 1ms is - // run at 1ms. + // run at 1ms, which only prevents a busy loop: well below the defaults, + // peers drop this node's state updates over their per-peer allowance. PeerGossipSleepDuration time.Duration `mapstructure:"peer-gossip-sleep-duration"` PeerQueryMaj23SleepDuration time.Duration `mapstructure:"peer-query-maj23-sleep-duration"` diff --git a/config/toml.go b/config/toml.go index ebb3f0ee0..7a995090e 100644 --- a/config/toml.go +++ b/config/toml.go @@ -568,7 +568,9 @@ create-empty-blocks-interval = "{{ .Consensus.CreateEmptyBlocksInterval }}" # (a fixed 500ms internal throttle): raising this above 500ms disables that # throttle. peer-query-maj23-sleep-duration also sets how often the node # re-announces its round state to each peer (recovery latency after a lost -# announcement). Either duration under 1ms is run at 1ms. +# announcement). Either duration under 1ms is run at 1ms, which only prevents a +# busy loop: well below the defaults, peers drop this node's state updates over +# their per-peer allowance. peer-gossip-sleep-duration = "{{ .Consensus.PeerGossipSleepDuration }}" peer-query-maj23-sleep-duration = "{{ .Consensus.PeerQueryMaj23SleepDuration }}" diff --git a/docs/nodes/configuration.md b/docs/nodes/configuration.md index 6dee8c0f7..86809cc47 100644 --- a/docs/nodes/configuration.md +++ b/docs/nodes/configuration.md @@ -412,7 +412,9 @@ create-empty-blocks-interval = "0s" # (a fixed 500ms internal throttle): raising this above 500ms disables that # throttle. peer-query-maj23-sleep-duration also sets how often the node # re-announces its round state to each peer (recovery latency after a lost -# announcement). Either duration under 1ms is run at 1ms. +# announcement). Either duration under 1ms is run at 1ms, which only prevents a +# busy loop: well below the defaults, peers drop this node's state updates over +# their per-peer allowance. peer-gossip-sleep-duration = "100ms" peer-query-maj23-sleep-duration = "2s" diff --git a/internal/consensus/gossip_peer_worker.go b/internal/consensus/gossip_peer_worker.go index 05fd3ee85..e12c298aa 100644 --- a/internal/consensus/gossip_peer_worker.go +++ b/internal/consensus/gossip_peer_worker.go @@ -8,18 +8,49 @@ import ( "github.com/jonboulle/clockwork" + "github.com/dashpay/tenderdash/config" "github.com/dashpay/tenderdash/libs/log" ) // minGossipSleepDuration is the shortest cadence a gossip handler runs at, -// whatever the config says. A handler sleeping for zero runs back to back, and -// the round-state handler sends on every pass, so a config of 0s — or a stray -// "2" read as 2ns — would spin a goroutine per handler per peer and flood the -// peer's state channel. Config validation runs only on the CLI path, so the -// worker holds the floor itself. 1ms is the same floor as -// incoming-connection-window; the test config runs at 5ms. +// whatever the config says. A handler sleeping for zero runs back to back, +// spinning a goroutine per handler per peer, and config validation runs only +// on the CLI path, so the reactor holds a floor itself. It stops the spin and +// nothing more: well below the defaults the handlers still offer peers more +// than their state-channel allowance admits (see peerStateRateLimit), and the +// excess is dropped there. 1ms is the floor incoming-connection-window uses +// too; the test config runs at 5ms. const minGossipSleepDuration = time.Millisecond +// gossipCadence is how often each peer's gossip handlers run, floored at +// minGossipSleepDuration. The reactor derives it from the config once and +// every consumer of these settings reads it from there. +type gossipCadence struct { + // gossip is PeerGossipSleepDuration: votes, commits and block parts. + gossip time.Duration + // queryMaj23 is PeerQueryMaj23SleepDuration: majority claims, round state, + // skip certificates and vote resends. + queryMaj23 time.Duration +} + +// flooredGossipCadence returns the configured cadences, each raised to +// minGossipSleepDuration if shorter, warning once per raised setting. +func flooredGossipCadence(logger log.Logger, cfg *config.ConsensusConfig) gossipCadence { + return gossipCadence{ + gossip: flooredGossipSleep(logger, "peer-gossip-sleep-duration", cfg.PeerGossipSleepDuration), + queryMaj23: flooredGossipSleep(logger, "peer-query-maj23-sleep-duration", cfg.PeerQueryMaj23SleepDuration), + } +} + +func flooredGossipSleep(logger log.Logger, key string, configured time.Duration) time.Duration { + if configured >= minGossipSleepDuration { + return configured + } + logger.Warn("gossip sleep duration below the minimum, gossiping at the minimum instead", + "key", key, "configured", configured, "minimum", minGossipSleepDuration) + return minGossipSleepDuration +} + type gossipHandlerFunc func(ctx context.Context, appState StateData) type gossipHandler struct { @@ -50,10 +81,9 @@ func newPeerGossipWorker( ps *PeerState, state *State, msgSender *p2pMsgSender, + cadence gossipCadence, ) *peerGossipWorker { clock := clockwork.NewRealClock() - gossipSleep := flooredGossipSleep(logger, "peer-gossip-sleep-duration", state.config.PeerGossipSleepDuration) - queryMaj23Sleep := flooredGossipSleep(logger, "peer-query-maj23-sleep-duration", state.config.PeerQueryMaj23SleepDuration) gossiper := msgGossiper{ ps: ps, blockStore: &blockRepository{BlockStore: state.blockStore, logger: logger}, @@ -70,26 +100,26 @@ func newPeerGossipWorker( handlers: []gossipHandler{ newGossipHandler( votesAndCommitGossipHandler(ps, state.blockStore, &gossiper), - gossipSleep, + cadence.gossip, ), newGossipHandler( dataGossipHandler(ps, logger, state.blockStore, &gossiper), - gossipSleep, + cadence.gossip, ), newGossipHandler( queryMaj23GossipHandler(ps, &gossiper), - queryMaj23Sleep, + cadence.queryMaj23, ), newGossipHandler( roundStateGossipHandler(logger, msgSender.send), - queryMaj23Sleep, + cadence.queryMaj23, ), // A certificate is resent at the majority-claim cadence: a peer // behind waits for nothing else, and one that applied it stops // being sent it as soon as it announces its new round. newGossipHandler( roundSkipGossipHandler(ps, logger, msgSender.send), - queryMaj23Sleep, + cadence.queryMaj23, ), // So is a vote of its round, or of a later round +2/3 voted in, to // a peer that waits in this node's round, or is in a round this @@ -99,23 +129,12 @@ func newPeerGossipWorker( // round. newGossipHandler( roundVoteResendGossipHandler(ps, logger, msgSender.send), - queryMaj23Sleep, + cadence.queryMaj23, ), }, } } -// flooredGossipSleep returns the configured sleep, or minGossipSleepDuration -// if it is shorter. It is logged once per worker, not per pass. -func flooredGossipSleep(logger log.Logger, key string, configured time.Duration) time.Duration { - if configured >= minGossipSleepDuration { - return configured - } - logger.Warn("gossip sleep duration below the minimum, gossiping at the minimum instead", - "key", key, "configured", configured, "minimum", minGossipSleepDuration) - return minGossipSleepDuration -} - func (g *peerGossipWorker) IsRunning() bool { return g.running.Load() } diff --git a/internal/consensus/gossip_peer_worker_test.go b/internal/consensus/gossip_peer_worker_test.go index 81e702ed4..93b6d2e32 100644 --- a/internal/consensus/gossip_peer_worker_test.go +++ b/internal/consensus/gossip_peer_worker_test.go @@ -9,6 +9,7 @@ import ( "github.com/jonboulle/clockwork" "github.com/stretchr/testify/require" + "github.com/dashpay/tenderdash/config" "github.com/dashpay/tenderdash/libs/eventemitter" "github.com/dashpay/tenderdash/libs/log" "github.com/dashpay/tenderdash/types" @@ -51,33 +52,42 @@ func TestPeerGossipWorker(t *testing.T) { close(handlerCalledCh) } -// A handler whose sleep is zero runs back to back, and the round-state handler -// sends on every pass, so a config of 0s — or a stray "2" read as 2ns — would -// flood every peer. The worker runs no handler faster than the floor, whatever -// the config says and whether or not it was validated. -func TestPeerGossipWorkerFloorsSleepDurations(t *testing.T) { +// A handler whose sleep is zero runs back to back, spinning a goroutine per +// handler per peer, and a config of 0s — or a stray "2" read as 2ns — is not +// caught where validation is skipped. The reactor floors the cadence once, for +// every consumer of the two settings. +func TestFlooredGossipCadence(t *testing.T) { + for name, tc := range map[string]struct { + gossip, queryMaj23 time.Duration + want gossipCadence + }{ + "zero": {0, 0, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}}, + "negative": {-time.Second, -time.Second, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}}, + "below the floor": {minGossipSleepDuration / 2, minGossipSleepDuration / 2, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}}, + "at the floor": {minGossipSleepDuration, minGossipSleepDuration, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}}, + "each on its own": {0, 3 * minGossipSleepDuration, gossipCadence{minGossipSleepDuration, 3 * minGossipSleepDuration}}, + "defaults kept": {100 * time.Millisecond, 2 * time.Second, gossipCadence{100 * time.Millisecond, 2 * time.Second}}, + } { + t.Run(name, func(t *testing.T) { + cfg := config.DefaultConsensusConfig() + cfg.PeerGossipSleepDuration, cfg.PeerQueryMaj23SleepDuration = tc.gossip, tc.queryMaj23 + require.Equal(t, tc.want, flooredGossipCadence(log.NewNopLogger(), cfg)) + }) + } +} + +func TestReactorFloorsGossipCadence(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cs, _ := makeState(ctx, t, makeStateArgs{}) - ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer")) - sleepsFor := func(gossip, queryMaj23 time.Duration) []time.Duration { - cs.config.PeerGossipSleepDuration = gossip - cs.config.PeerQueryMaj23SleepDuration = queryMaj23 - pgw := newPeerGossipWorker(log.NewNopLogger(), ps, cs, &p2pMsgSender{logger: log.NewNopLogger(), ps: ps}) - sleeps := make([]time.Duration, 0, len(pgw.handlers)) - for _, hd := range pgw.handlers { - sleeps = append(sleeps, hd.sleepDuration) - } - return sleeps - } + cs.config.PeerGossipSleepDuration = 0 + cs.config.PeerQueryMaj23SleepDuration = 0 + r := NewReactor(log.NewNopLogger(), cs, nil, nil, cs.eventBus, false, NopMetrics()) + require.Equal(t, gossipCadence{minGossipSleepDuration, minGossipSleepDuration}, r.cadence) - for _, sleep := range []time.Duration{0, -time.Second, minGossipSleepDuration / 2} { - for i, got := range sleepsFor(sleep, sleep) { - require.Equal(t, minGossipSleepDuration, got, "handler %d configured at %s", i, sleep) - } + ps := NewPeerState(log.NewNopLogger(), types.NodeID("peer")) + pgw := newPeerGossipWorker(log.NewNopLogger(), ps, cs, &p2pMsgSender{logger: log.NewNopLogger(), ps: ps}, r.cadence) + for i, hd := range pgw.handlers { + require.Equal(t, minGossipSleepDuration, hd.sleepDuration, "handler %d", i) } - - kept := sleepsFor(minGossipSleepDuration, 3*minGossipSleepDuration) - require.Contains(t, kept, minGossipSleepDuration) - require.Contains(t, kept, 3*minGossipSleepDuration) } diff --git a/internal/consensus/peer_state.go b/internal/consensus/peer_state.go index 1ab633472..289a789e1 100644 --- a/internal/consensus/peer_state.go +++ b/internal/consensus/peer_state.go @@ -753,9 +753,9 @@ const maj23AnswerTicks = 2 // the cadence at which a gossip loop repeats it. func maj23AnswerTTLFor(sleep time.Duration) time.Duration { if sleep <= 0 { - // The gossip loop runs such a cadence at its floor (see - // minGossipSleepDuration), too fast for a pass count to mean anything. - // The default is what the rest of these ceilings are sized against. + // The reactor passes its floored cadence (see gossipCadence); zero is a + // peer state built without one, as NewPeerState does. The default is + // what the rest of these ceilings are sized against. sleep = config.DefaultConsensusConfig().PeerQueryMaj23SleepDuration } return maj23AnswerTicks * sleep diff --git a/internal/consensus/reactor.go b/internal/consensus/reactor.go index 552c6ecc7..cdac5dca0 100644 --- a/internal/consensus/reactor.go +++ b/internal/consensus/reactor.go @@ -236,6 +236,10 @@ type Reactor struct { eventBus *eventbus.EventBus Metrics *Metrics + // cadence is how often each peer's gossip handlers run, from the config + // with the floor applied once. + cadence gossipCadence + mtx sync.RWMutex peers map[types.NodeID]*PeerState waitSync bool @@ -310,6 +314,7 @@ func NewReactor( r := &Reactor{ logger: logger, state: cs, + cadence: flooredGossipCadence(logger, cs.config), waitSync: waitSync, peers: make(map[types.NodeID]*PeerState), eventBus: eventBus, @@ -921,7 +926,7 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries if !ok { ps = NewPeerState(r.logger, peerUpdate.NodeID, WithPeerStateClock(r.clock), - WithMaj23AnswerTTL(maj23AnswerTTLFor(r.state.config.PeerQueryMaj23SleepDuration))) + WithMaj23AnswerTTL(maj23AnswerTTLFor(r.cadence.queryMaj23))) ps.SetProTxHash(peerUpdate.ProTxHash) r.peers[peerUpdate.NodeID] = ps } else if len(peerUpdate.ProTxHash) > 0 { @@ -941,7 +946,7 @@ func (r *Reactor) peerUp(ctx context.Context, peerUpdate p2p.PeerUpdate, retries ) // TODO needs to register this gossip worker, to be able to stop it once a peer will be down msgSender := p2pMsgSender{logger: logger, ps: ps, chans: chans} - pgw := newPeerGossipWorker(logger, ps, r.state, &msgSender) + pgw := newPeerGossipWorker(logger, ps, r.state, &msgSender, r.cadence) select { case <-ctx.Done():