Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
f0bf55d
fix(blocksync): hand over to consensus at the tip despite unserved pe…
lklimek Nov 11, 2026
3a77810
fix(blocksync): judge the tip by block time alone and require backed …
lklimek Nov 11, 2026
304db7b
test(blocksync): reproduce block sync held at the tip by far claims u…
lklimek Nov 11, 2026
42ea395
Merge remote-tracking branch 'origin/dev' into fix/blocksync-handover…
lklimek Nov 11, 2026
7451f76
fix(blocksync): estimate the chain's pace from the blocks applied so far
lklimek Nov 11, 2026
f557388
test(blocksync): reproduce stall backstop and producer wait issues fo…
lklimek Nov 11, 2026
34a9ade
fix(blocksync): measure the stall backstop's distance from backed cla…
lklimek Nov 11, 2026
624641e
fix(blocksync): request a lower retry height before waiting on a high…
lklimek Nov 11, 2026
38cfc34
fix(blocksync): ask peers that backed their claims before those that …
lklimek Nov 11, 2026
050fe87
fix(blocksync): keep the handover loop off the applier lock
lklimek Nov 11, 2026
4c57d66
docs(blocksync): state when the at-tip handover bound holds
lklimek Nov 11, 2026
3584fe7
fix(blocksync): ask every peer for the height to apply once a request…
lklimek Nov 11, 2026
7a346db
fix(blocksync): credit the supplier of an applied block removed meanw…
lklimek Nov 11, 2026
79c4d07
fix(blocksync): use one consensus catch-up target for every handover …
lklimek Nov 11, 2026
bebe0ca
test(blocksync): keep syncing behind a sole source that fails now and…
lklimek Nov 11, 2026
b92bc63
docs(blocksync): describe asking every peer for the next block and th…
lklimek Nov 11, 2026
83e73ef
Merge remote-tracking branch 'origin/dev' into fix/blocksync-handover…
lklimek Nov 11, 2026
85a905c
test(blocksync): reproduce handover gaps behind late suppliers and fa…
lklimek Nov 11, 2026
f23a65f
fix(blocksync): count every claim in the stall backstop when the last…
lklimek Nov 11, 2026
55b5c9e
test(blocksync): drive the stall backstop tests tick by tick from an …
lklimek Nov 11, 2026
6331866
fix(blocksync): stop asking more peers for the height to apply within…
lklimek Nov 11, 2026
c1e3a20
fix(blocksync): learn the chain's pace from the block store at start
lklimek Nov 11, 2026
6b25314
docs(blocksync): describe the current stall backstop rules and tie ti…
lklimek Nov 11, 2026
1ac0aca
docs(blocksync): state the handover behaviour operators can expect
lklimek Nov 11, 2026
9aef425
test(blocksync): check the handover behind a peer answering inside th…
lklimek Nov 11, 2026
ed01f13
fix(blocksync): judge the old-block rule by a day, at the start of th…
lklimek Nov 11, 2026
3b2eb25
fix(blocksync): ask every peer for the height to apply once it goes u…
lklimek Nov 11, 2026
ea2e7ea
fix(blocksync): narrow the at-tip lag to the wait before every peer i…
lklimek Nov 11, 2026
4280100
docs(blocksync): match the operator notes to the current handover rules
lklimek Nov 11, 2026
5d5d674
test(blocksync): check the fetch volume far behind and the handover o…
lklimek Nov 11, 2026
aaaf2df
fix(blocksync): keep the at-tip window floor above the nothing-to-fet…
lklimek Nov 11, 2026
2e78156
fix(blocksync): count the wait for the height to apply from when it b…
lklimek Nov 11, 2026
b09fcad
docs(blocksync): state when the next block is asked of every peer
lklimek Nov 11, 2026
dc12b76
refactor(blocksync): judge the handover by block time and ask every p…
lklimek Nov 11, 2026
6e2726b
docs(blocksync): describe the handover rules and what operators can e…
lklimek Nov 11, 2026
9961ff1
fix(blocksync): go on to the tip while closing the gap and ask every …
lklimek Nov 11, 2026
554703c
docs(blocksync): state when the at-tip wait starts over and which han…
lklimek Nov 11, 2026
eaed8f0
docs(blocksync): note the slower catch-up with few serving peers and …
lklimek Nov 11, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions docs/tenderdash-core/block-sync/reactor.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,74 @@ and hard failure policy are unchanged. The 60-second request deadline can still
reject a large response on a slow connection even if its eventual service rate
would meet the scheduling allowance.

Block sync hands over to consensus when one of these holds:

- the node reaches the highest height any peer reports;
- no peer reports the next height and no block has been applied for 60
seconds;
- the node's last applied block has been recent, at most five minutes old by
the local clock, for two minutes without a break and without getting more
than 30 seconds younger meanwhile;
- no block has been applied for 10 minutes, and either that wait began on a
block less than a day old, or the node is within 10 blocks of every peer
that reports the next height.

The heights peers report are not verified, so the last two rules rest on the
time of the last applied block instead, which the validators commit to: a node
far behind the network applies old blocks. Five minutes covers the three-minute
empty-block interval Dash Platform runs with and a margin for extra rounds and
clock skew. A last block that keeps getting younger means the node is still
closing the gap, so the two minutes start over each time it has gained 30
seconds: a node applying blocks at least a quarter faster than the chain
commits them goes on until its last block is within 30 seconds of the chain's.
A block time further ahead of the local clock than the synchrony precision does
not count as recent. When a node with a recent last block hands over, the
consensus catch-up target is the highest height reported within 10 blocks of
it. Only a node whose last block was more than a day old when it stopped
receiving blocks keeps waiting for as long as a peer reports the next height
and a height more than 10 blocks above the node, and resumes as soon as a peer
serves again.

Blocks are applied in order, so the next block is the one that matters. When
no block has been applied for 60 seconds and the next one has not arrived, it
is requested from every peer that reports it, once each, starting a second
after the peer first reports it. Sixty seconds is also how long a request may
take, so a block that arrives within its request deadline is fetched once,
however slow the link.

What operators can expect:

- A node that has caught up hands over within two to four minutes, whatever
heights its peers report, provided a connected peer serves each new block
when asked. Peers that delay blocks can stretch this to about 20 minutes,
and a single block by about a minute.
- On a chain that has stopped producing blocks, a restarted node hands over
after 10 minutes without a block. If the chain had been stopped for more
than a day when the node started waiting, the node keeps waiting while a peer
reports the next height and a height more than 10 blocks above the node.
- A node may hand over behind the network by up to five minutes of blocks:
one or two blocks on a chain committing every three minutes, a few hundred on
one committing every second. Consensus catch-up closes the rest. On a chain
that keeps committing a block every second or two this can take long, and it
happens only when the node received blocks less than a quarter faster than
the chain produced them.
- A node less than a day behind the network whose peers all fail to deliver
the next block for 10 minutes hands over where it is. Restart it once its
peers serve blocks again if consensus catch-up does not close the gap.
- With only a few peers serving blocks, a node restarting far behind a chain
that commits a block every second or two may need much longer to reach the
tip. More peers that serve blocks help.
- A node whose clock runs minutes ahead of the validators', or
behind them by more than the synchrony precision, does not see its last
block as recent and leaves block sync only by the other rules. Keep the
clock synchronised.
- After a handover below a height a peer reported, the node may hold back its
own proposals for up to 10 minutes while consensus catches up. With a recent
last block only heights reported within 10 blocks of the node count.
- A node that stays in block sync while its blocks are recent, or hands over
far behind, is worth reporting with its logs: a handover below the highest
height a peer reports is logged at warning or error level with its reason.

The historical Blocksync reactor is organised as a set of concurrent tasks:

- Receive routine of Blocksync Reactor
Expand Down
131 changes: 131 additions & 0 deletions internal/blocksync/ask_every_peer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package blocksync

import (
"context"
"sync/atomic"
"testing"
"time"

"github.com/jonboulle/clockwork"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"

"github.com/dashpay/tenderdash/internal/p2p/client"
"github.com/dashpay/tenderdash/internal/p2p/client/mocks"
"github.com/dashpay/tenderdash/libs/promise"
"github.com/dashpay/tenderdash/libs/workerpool"
bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync"
"github.com/dashpay/tenderdash/types"
)

func reservedIDs(peers []PeerData) []types.NodeID {
ids := make([]types.NodeID, 0, len(peers))
for _, peer := range peers {
ids = append(ids, peer.peerID)
}
return ids
}

// TestReserveNextBlockPeers checks who is asked for the height block sync waits
// to apply, and when: nobody until its block is syncTimeout overdue, then every
// peer that advertises it, each once and a second after it is first seen
// advertising it, until the block arrives or the height is applied.
func TestReserveNextBlockPeers(t *testing.T) {
const height = int64(100)
clock := clockwork.NewFakeClock()
sync := NewSynchronizer(height, nil, nil, WithClock(clock), WithWorkerPool(workerpool.New(0)))
sync.lastAdvance = clock.Now()
sync.AddPeer(newPeerData("first", 1, height))
sync.AddPeer(newPeerData("second", 1, height+1000))
sync.AddPeer(newPeerData("above", height+1, height+1000))
busy := newPeerData("busy", 1, height)
busy.numPending = maxPendingRequestsPerPeer
sync.AddPeer(busy)
next := func() []types.NodeID {
clock.Advance(time.Second)
reservedFor, peers := sync.reserveNextBlockPeers()
applying, _ := sync.GetStatus()
require.Equal(t, applying, reservedFor)
return reservedIDs(peers)
}

clock.Advance(syncTimeout - 2*time.Second)
require.Empty(t, next(), "the block is not overdue yet")
require.Empty(t, next(), "the block is not overdue yet")
require.Empty(t, next(), "peers advertising the height are given a second first")
require.ElementsMatch(t, []types.NodeID{"first", "second"}, next())
for _, peerID := range []types.NodeID{"first", "second"} {
peer, _ := sync.peerStore.Get(peerID)
require.EqualValues(t, 1, peer.numPending, "the request counts against the peer's limit")
}
require.Empty(t, next(), "each peer is asked once")

// a peer that left is not kept on record
sync.RemovePeer("first")
require.NotContains(t, sync.asked, types.NodeID("first"))
sync.AddPeer(newPeerData("first", 1, height))
require.Empty(t, next())
require.ElementsMatch(t, []types.NodeID{"first"}, next())

// a peer that starts advertising the height, and one that gets free
sync.AddPeer(newPeerData("above", 1, height+1000))
sync.peerStore.Update("busy", AddNumPending(-1))
require.ElementsMatch(t, []types.NodeID{"busy"}, next())
require.ElementsMatch(t, []types.NodeID{"above"}, next())

// nobody else is asked once the block is here
sync.AddPeer(newPeerData("late", 1, height))
require.NoError(t, sync.addBlock(BlockResponse{
PeerID: "first",
Block: &types.Block{Header: types.Header{Height: height}},
}))
require.Empty(t, next())
require.Empty(t, next())

// the next height starts over
sync.advance(clock.Now(), defaultPrecision)
clock.Advance(syncTimeout - time.Second)
require.Empty(t, next(), "the next block is not overdue yet")
require.Empty(t, next())
require.ElementsMatch(t, []types.NodeID{"second", "above"}, next())
}

// TestNextBlockIsFetchedWhileTheProducerWaits checks that the height block sync
// waits to apply is requested again after its only request failed, although the
// job producer is waiting for a peer to serve a higher height that only a peer
// with no room for another request reports.
func TestNextBlockIsFetchedWhileTheProducerWaits(t *testing.T) {
const height = int64(100)
var asked atomic.Int32
// fail is closed once the producer has moved on to the next height
fail := make(chan struct{})
blockClient := mocks.NewBlockClient(t)
blockClient.
On("GetBlock", mock.Anything, height, types.NodeID("idle peer")).
Return(func(context.Context, int64, types.NodeID) *promise.Promise[*bcproto.BlockResponse] {
asked.Add(1)
return promise.New(func(_ func(*bcproto.BlockResponse), reject func(error)) {
<-fail
reject(client.ErrPeerNotResponded)
})
}, nil)
clock := clockwork.NewFakeClock()
sync := NewSynchronizer(height, blockClient, nil, WithClock(clock), WithWorkerPool(workerpool.New(2)))
sync.AddPeer(newPeerData("idle peer", 1, height))
busy := newPeerData("busy claimant", height+1, maxPlausiblePeerHeight)
busy.numPending = maxPendingRequestsPerPeer
sync.AddPeer(busy)

require.NoError(t, sync.Start(context.Background()))
defer sync.Stop()
require.Eventually(t, func() bool { return asked.Load() == 1 }, 5*time.Second, time.Millisecond,
"the producer requests the height first")
time.Sleep(50 * time.Millisecond)
close(fail)

require.Eventually(t, func() bool {
clock.Advance(time.Second)
return asked.Load() >= 2
}, 10*time.Second, 5*time.Millisecond,
"height %d was not requested again while the producer waits for a peer for height %d", height, height+1)
}
75 changes: 75 additions & 0 deletions internal/blocksync/handover_bursts_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package blocksync

import (
"fmt"
"time"

"github.com/stretchr/testify/mock"

"github.com/dashpay/tenderdash/types"
)

// TestWaitForSyncHandsOverAtTheTipOfAChainCommittingInBursts checks the handover
// of a node level with a chain that commits blocks only while there are
// transactions: a run of blocks three seconds apart, then a pause until the
// next run. Three peers hold every block and answer at once, so the node
// applies each block within a status round of its commit, and a far range is
// reported by a peer nobody asks. A pause in block production is not the node
// falling behind, so it must hand over as it does on a chain committing a block
// every three seconds without pauses.
func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverAtTheTipOfAChainCommittingInBursts() {
const (
height = int64(100)
interval = 3 * time.Second
limit = time.Hour
)
for _, tc := range []struct {
name string
// run is the number of blocks committed interval apart before each pause
run int64
pause time.Duration
}{
{name: "no pauses"},
{name: "12 blocks, then 20 seconds without one", run: 12, pause: 20 * time.Second},
{name: "12 blocks, then 50 seconds without one", run: 12, pause: 50 * time.Second},
{name: "12 blocks, then 80 seconds without one", run: 12, pause: 80 * time.Second},
} {
suite.Run(tc.name, func() {
suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil)
h := suite.newWaitForSyncHarness(height)
defer h.cancel()
start := h.clock.Now()
r := &tipRun{
h: h,
interval: interval,
blockTimes: map[int64]time.Time{},
suppliedBy: map[types.NodeID]int{},
tip: height - 1,
nextCommit: start,
}
for i := 0; i < 3; i++ {
r.reporting = append(r.reporting, types.NodeID(fmt.Sprintf("holder %d", i)))
}
h.sync.AddPeer(newPeerData("far range", 1_000_000_000, maxPlausiblePeerHeight))

breaks := 0
elapsed, caughtUp, ended := h.run(limit, func(elapsed time.Duration) {
// the gap after the block about to be committed
r.interval = interval
if next := r.tip + 1 - height; tc.run > 0 && next%tc.run == tc.run-1 {
r.interval = tc.pause
}
r.step(elapsed)
st := h.sync.stallSnapshot()
if !isRecent(st.lastBlockTime, tipRecency, st.precision, h.clock.Now().Add(time.Second)) {
breaks++
}
})
suite.Require().True(ended,
"still in block sync after %s: %d blocks applied, each within a status round of its commit; the node counted as away from the tip at %d ticks",
elapsed, r.applied, breaks)
suite.Require().False(caughtUp)
suite.T().Logf("handed over after %s: %d blocks applied, away from the tip at %d ticks", elapsed, r.applied, breaks)
})
}
}
Loading
Loading