diff --git a/docs/tenderdash-core/block-sync/reactor.md b/docs/tenderdash-core/block-sync/reactor.md index 5a31d7545..384533064 100644 --- a/docs/tenderdash-core/block-sync/reactor.md +++ b/docs/tenderdash-core/block-sync/reactor.md @@ -53,6 +53,74 @@ and hard failure policy are unchanged. The 60-second request deadline can still reject a large response on a slow connection even if its eventual service rate would meet the scheduling allowance. +Block sync hands over to consensus when one of these holds: + +- the node reaches the highest height any peer reports; +- no peer reports the next height and no block has been applied for 60 + seconds; +- the node's last applied block has been recent, at most five minutes old by + the local clock, for two minutes without a break and without getting more + than 30 seconds younger meanwhile; +- no block has been applied for 10 minutes, and either that wait began on a + block less than a day old, or the node is within 10 blocks of every peer + that reports the next height. + +The heights peers report are not verified, so the last two rules rest on the +time of the last applied block instead, which the validators commit to: a node +far behind the network applies old blocks. Five minutes covers the three-minute +empty-block interval Dash Platform runs with and a margin for extra rounds and +clock skew. A last block that keeps getting younger means the node is still +closing the gap, so the two minutes start over each time it has gained 30 +seconds: a node applying blocks at least a quarter faster than the chain +commits them goes on until its last block is within 30 seconds of the chain's. +A block time further ahead of the local clock than the synchrony precision does +not count as recent. When a node with a recent last block hands over, the +consensus catch-up target is the highest height reported within 10 blocks of +it. Only a node whose last block was more than a day old when it stopped +receiving blocks keeps waiting for as long as a peer reports the next height +and a height more than 10 blocks above the node, and resumes as soon as a peer +serves again. + +Blocks are applied in order, so the next block is the one that matters. When +no block has been applied for 60 seconds and the next one has not arrived, it +is requested from every peer that reports it, once each, starting a second +after the peer first reports it. Sixty seconds is also how long a request may +take, so a block that arrives within its request deadline is fetched once, +however slow the link. + +What operators can expect: + +- A node that has caught up hands over within two to four minutes, whatever + heights its peers report, provided a connected peer serves each new block + when asked. Peers that delay blocks can stretch this to about 20 minutes, + and a single block by about a minute. +- On a chain that has stopped producing blocks, a restarted node hands over + after 10 minutes without a block. If the chain had been stopped for more + than a day when the node started waiting, the node keeps waiting while a peer + reports the next height and a height more than 10 blocks above the node. +- A node may hand over behind the network by up to five minutes of blocks: + one or two blocks on a chain committing every three minutes, a few hundred on + one committing every second. Consensus catch-up closes the rest. On a chain + that keeps committing a block every second or two this can take long, and it + happens only when the node received blocks less than a quarter faster than + the chain produced them. +- A node less than a day behind the network whose peers all fail to deliver + the next block for 10 minutes hands over where it is. Restart it once its + peers serve blocks again if consensus catch-up does not close the gap. +- With only a few peers serving blocks, a node restarting far behind a chain + that commits a block every second or two may need much longer to reach the + tip. More peers that serve blocks help. +- A node whose clock runs minutes ahead of the validators', or + behind them by more than the synchrony precision, does not see its last + block as recent and leaves block sync only by the other rules. Keep the + clock synchronised. +- After a handover below a height a peer reported, the node may hold back its + own proposals for up to 10 minutes while consensus catches up. With a recent + last block only heights reported within 10 blocks of the node count. +- A node that stays in block sync while its blocks are recent, or hands over + far behind, is worth reporting with its logs: a handover below the highest + height a peer reports is logged at warning or error level with its reason. + The historical Blocksync reactor is organised as a set of concurrent tasks: - Receive routine of Blocksync Reactor diff --git a/internal/blocksync/ask_every_peer_test.go b/internal/blocksync/ask_every_peer_test.go new file mode 100644 index 000000000..76f984169 --- /dev/null +++ b/internal/blocksync/ask_every_peer_test.go @@ -0,0 +1,131 @@ +package blocksync + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/dashpay/tenderdash/internal/p2p/client" + "github.com/dashpay/tenderdash/internal/p2p/client/mocks" + "github.com/dashpay/tenderdash/libs/promise" + "github.com/dashpay/tenderdash/libs/workerpool" + bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" + "github.com/dashpay/tenderdash/types" +) + +func reservedIDs(peers []PeerData) []types.NodeID { + ids := make([]types.NodeID, 0, len(peers)) + for _, peer := range peers { + ids = append(ids, peer.peerID) + } + return ids +} + +// TestReserveNextBlockPeers checks who is asked for the height block sync waits +// to apply, and when: nobody until its block is syncTimeout overdue, then every +// peer that advertises it, each once and a second after it is first seen +// advertising it, until the block arrives or the height is applied. +func TestReserveNextBlockPeers(t *testing.T) { + const height = int64(100) + clock := clockwork.NewFakeClock() + sync := NewSynchronizer(height, nil, nil, WithClock(clock), WithWorkerPool(workerpool.New(0))) + sync.lastAdvance = clock.Now() + sync.AddPeer(newPeerData("first", 1, height)) + sync.AddPeer(newPeerData("second", 1, height+1000)) + sync.AddPeer(newPeerData("above", height+1, height+1000)) + busy := newPeerData("busy", 1, height) + busy.numPending = maxPendingRequestsPerPeer + sync.AddPeer(busy) + next := func() []types.NodeID { + clock.Advance(time.Second) + reservedFor, peers := sync.reserveNextBlockPeers() + applying, _ := sync.GetStatus() + require.Equal(t, applying, reservedFor) + return reservedIDs(peers) + } + + clock.Advance(syncTimeout - 2*time.Second) + require.Empty(t, next(), "the block is not overdue yet") + require.Empty(t, next(), "the block is not overdue yet") + require.Empty(t, next(), "peers advertising the height are given a second first") + require.ElementsMatch(t, []types.NodeID{"first", "second"}, next()) + for _, peerID := range []types.NodeID{"first", "second"} { + peer, _ := sync.peerStore.Get(peerID) + require.EqualValues(t, 1, peer.numPending, "the request counts against the peer's limit") + } + require.Empty(t, next(), "each peer is asked once") + + // a peer that left is not kept on record + sync.RemovePeer("first") + require.NotContains(t, sync.asked, types.NodeID("first")) + sync.AddPeer(newPeerData("first", 1, height)) + require.Empty(t, next()) + require.ElementsMatch(t, []types.NodeID{"first"}, next()) + + // a peer that starts advertising the height, and one that gets free + sync.AddPeer(newPeerData("above", 1, height+1000)) + sync.peerStore.Update("busy", AddNumPending(-1)) + require.ElementsMatch(t, []types.NodeID{"busy"}, next()) + require.ElementsMatch(t, []types.NodeID{"above"}, next()) + + // nobody else is asked once the block is here + sync.AddPeer(newPeerData("late", 1, height)) + require.NoError(t, sync.addBlock(BlockResponse{ + PeerID: "first", + Block: &types.Block{Header: types.Header{Height: height}}, + })) + require.Empty(t, next()) + require.Empty(t, next()) + + // the next height starts over + sync.advance(clock.Now(), defaultPrecision) + clock.Advance(syncTimeout - time.Second) + require.Empty(t, next(), "the next block is not overdue yet") + require.Empty(t, next()) + require.ElementsMatch(t, []types.NodeID{"second", "above"}, next()) +} + +// TestNextBlockIsFetchedWhileTheProducerWaits checks that the height block sync +// waits to apply is requested again after its only request failed, although the +// job producer is waiting for a peer to serve a higher height that only a peer +// with no room for another request reports. +func TestNextBlockIsFetchedWhileTheProducerWaits(t *testing.T) { + const height = int64(100) + var asked atomic.Int32 + // fail is closed once the producer has moved on to the next height + fail := make(chan struct{}) + blockClient := mocks.NewBlockClient(t) + blockClient. + On("GetBlock", mock.Anything, height, types.NodeID("idle peer")). + Return(func(context.Context, int64, types.NodeID) *promise.Promise[*bcproto.BlockResponse] { + asked.Add(1) + return promise.New(func(_ func(*bcproto.BlockResponse), reject func(error)) { + <-fail + reject(client.ErrPeerNotResponded) + }) + }, nil) + clock := clockwork.NewFakeClock() + sync := NewSynchronizer(height, blockClient, nil, WithClock(clock), WithWorkerPool(workerpool.New(2))) + sync.AddPeer(newPeerData("idle peer", 1, height)) + busy := newPeerData("busy claimant", height+1, maxPlausiblePeerHeight) + busy.numPending = maxPendingRequestsPerPeer + sync.AddPeer(busy) + + require.NoError(t, sync.Start(context.Background())) + defer sync.Stop() + require.Eventually(t, func() bool { return asked.Load() == 1 }, 5*time.Second, time.Millisecond, + "the producer requests the height first") + time.Sleep(50 * time.Millisecond) + close(fail) + + require.Eventually(t, func() bool { + clock.Advance(time.Second) + return asked.Load() >= 2 + }, 10*time.Second, 5*time.Millisecond, + "height %d was not requested again while the producer waits for a peer for height %d", height, height+1) +} diff --git a/internal/blocksync/handover_bursts_test.go b/internal/blocksync/handover_bursts_test.go new file mode 100644 index 000000000..553f7c419 --- /dev/null +++ b/internal/blocksync/handover_bursts_test.go @@ -0,0 +1,75 @@ +package blocksync + +import ( + "fmt" + "time" + + "github.com/stretchr/testify/mock" + + "github.com/dashpay/tenderdash/types" +) + +// TestWaitForSyncHandsOverAtTheTipOfAChainCommittingInBursts checks the handover +// of a node level with a chain that commits blocks only while there are +// transactions: a run of blocks three seconds apart, then a pause until the +// next run. Three peers hold every block and answer at once, so the node +// applies each block within a status round of its commit, and a far range is +// reported by a peer nobody asks. A pause in block production is not the node +// falling behind, so it must hand over as it does on a chain committing a block +// every three seconds without pauses. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverAtTheTipOfAChainCommittingInBursts() { + const ( + height = int64(100) + interval = 3 * time.Second + limit = time.Hour + ) + for _, tc := range []struct { + name string + // run is the number of blocks committed interval apart before each pause + run int64 + pause time.Duration + }{ + {name: "no pauses"}, + {name: "12 blocks, then 20 seconds without one", run: 12, pause: 20 * time.Second}, + {name: "12 blocks, then 50 seconds without one", run: 12, pause: 50 * time.Second}, + {name: "12 blocks, then 80 seconds without one", run: 12, pause: 80 * time.Second}, + } { + suite.Run(tc.name, func() { + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height) + defer h.cancel() + start := h.clock.Now() + r := &tipRun{ + h: h, + interval: interval, + blockTimes: map[int64]time.Time{}, + suppliedBy: map[types.NodeID]int{}, + tip: height - 1, + nextCommit: start, + } + for i := 0; i < 3; i++ { + r.reporting = append(r.reporting, types.NodeID(fmt.Sprintf("holder %d", i))) + } + h.sync.AddPeer(newPeerData("far range", 1_000_000_000, maxPlausiblePeerHeight)) + + breaks := 0 + elapsed, caughtUp, ended := h.run(limit, func(elapsed time.Duration) { + // the gap after the block about to be committed + r.interval = interval + if next := r.tip + 1 - height; tc.run > 0 && next%tc.run == tc.run-1 { + r.interval = tc.pause + } + r.step(elapsed) + st := h.sync.stallSnapshot() + if !isRecent(st.lastBlockTime, tipRecency, st.precision, h.clock.Now().Add(time.Second)) { + breaks++ + } + }) + suite.Require().True(ended, + "still in block sync after %s: %d blocks applied, each within a status round of its commit; the node counted as away from the tip at %d ticks", + elapsed, r.applied, breaks) + suite.Require().False(caughtUp) + suite.T().Logf("handed over after %s: %d blocks applied, away from the tip at %d ticks", elapsed, r.applied, breaks) + }) + } +} diff --git a/internal/blocksync/handover_deadline_test.go b/internal/blocksync/handover_deadline_test.go new file mode 100644 index 000000000..32f674a51 --- /dev/null +++ b/internal/blocksync/handover_deadline_test.go @@ -0,0 +1,298 @@ +package blocksync + +import ( + "errors" + "fmt" + "time" + + "github.com/stretchr/testify/mock" + + "github.com/dashpay/tenderdash/internal/p2p/client" + "github.com/dashpay/tenderdash/types" +) + +// requestDeadline is how long the block client waits for a peer's response. +const requestDeadline = 60 * time.Second + +// tipRun drives the producer and the consumer of a node following a chain that +// commits a block every interval. Jobs are handed out as produceJob and nextJob +// do: shouldJobBeGenerated admits them, nextHeight picks the height and +// reservePeer the peer; a height no peer advertises yet is held until one does, +// as getPeer holds it. Once a second, as askEveryPeer does, the peers +// reserveNextBlockPeers returns are asked for the height to apply. Responses are +// queued and applied in height order as consumeJobResult and applyBlock do. A +// peer listed in declining answers at once that it has no such block; any other +// answers with the block, a peer listed in hold that long after it is asked and +// the rest at once. The peers in reporting report the chain's height every +// status round. +type tipRun struct { + h *waitForSyncHarness + interval time.Duration + hold map[types.NodeID]time.Duration + declining map[types.NodeID]bool + // reporting are the peers that report the chain's height, in the order + // their reports arrive. + reporting []types.NodeID + + blockTimes map[int64]time.Time + tip int64 + nextCommit time.Time + inFlight []*tipRequest + // waiting is the height the producer holds until a peer advertises it. + waiting int64 + + applied int + suppliedBy map[types.NodeID]int + // blocksSent counts the requests answered with a block. + blocksSent int + // oldest is the greatest age the last applied block reached. + oldest time.Duration +} + +type tipRequest struct { + peer PeerData + request *serviceRequest + height int64 + endAt time.Time +} + +func (r *tipRun) step(elapsed time.Duration) { + now := r.h.clock.Now() + for !now.Before(r.nextCommit) { + r.tip++ + r.blockTimes[r.tip] = r.nextCommit + r.nextCommit = r.nextCommit.Add(r.interval) + } + if elapsed%(statusUpdateIntervalSeconds*time.Second) == 0 { + for _, id := range r.reporting { + r.h.sync.AddPeer(newPeerData(id, 1, r.tip)) + } + } + for r.consume(now) || r.produce(now) { + } + height, peers := r.h.sync.reserveNextBlockPeers() + for _, peer := range peers { + r.request(peer, height, now) + } + for r.consume(now) || r.produce(now) { + } + st := r.h.sync.stallSnapshot() + // the age the next tick reads + r.oldest = max(r.oldest, now.Add(time.Second).Sub(st.lastBlockTime)) +} + +// consume hands the consumer every response due by now and applies what can be +// applied, and reports whether there was any. +func (r *tipRun) consume(now time.Time) bool { + sync := r.h.sync + var waiting []*tipRequest + consumed := false + for _, req := range r.inFlight { + if now.Before(req.endAt) { + waiting = append(waiting, req) + continue + } + consumed = true + r.blocksSent++ + req.request.Finished() + req.request.complete(1000) + sync.peerStore.Update(req.peer.peerID, AddNumPending(-1), ResetFailures()) + err := sync.addBlock(BlockResponse{ + PeerID: req.peer.peerID, + Block: &types.Block{Header: types.Header{Height: req.height}}, + }) + if err != nil && !errors.Is(err, errDuplicateBlock) { + r.h.suite.Require().NoError(err) + } + for { + resp, ok := sync.getPendingResponse() + if !ok { + break + } + sync.advance(r.blockTimes[resp.Block.Height], defaultPrecision) + r.applied++ + r.suppliedBy[resp.PeerID]++ + } + } + r.inFlight = waiting + return consumed +} + +// produce hands out every job due now and reports whether there was any. +func (r *tipRun) produce(now time.Time) bool { + gen := r.h.sync.jobGen + produced := false + for { + height := r.waiting + if height == 0 { + if !gen.shouldJobBeGenerated(r.h.sync.backlog()) { + return produced + } + height = gen.nextHeight() + } + peer, found := r.h.sync.peerStore.reservePeer(height) + if !found { + r.waiting = height + return produced + } + r.waiting = 0 + r.request(peer, height, now) + produced = true + } +} + +// request has peer answer a request for height as its kind does. +func (r *tipRun) request(peer PeerData, height int64, now time.Time) { + request := peer.service.request() + request.Started() + if r.declining[peer.peerID] { + request.Finished() + request.complete(0) + r.h.fail(peer.peerID, height, &client.ErrBlockNotFound{Height: height}) + return + } + r.inFlight = append(r.inFlight, &tipRequest{ + peer: peer, + request: request, + height: height, + endAt: now.Add(r.hold[peer.peerID]), + }) +} + +// TestWaitForSyncHandsOverBehindAPeerAnsweringJustInsideTheDeadline checks the +// handover of a node that restarts three blocks behind the network, with a +// prompt peer connected, while a far range is reported by a peer nobody asks. +// One more peer answers every request with the block, one second inside the +// request deadline, and never fails one. +// +// On a chain committing a block every three minutes the node hands over within +// minutes whichever peer reports first. On one committing a block every few +// seconds it must as well. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverBehindAPeerAnsweringJustInsideTheDeadline() { + const ( + height = int64(100) + prompt = types.NodeID("prompt peer") + slow = types.NodeID("peer answering inside the deadline") + behind = 3 + limit = time.Hour + ) + for _, tc := range []struct { + name string + interval time.Duration + reporting []types.NodeID + }{ + {name: "a block every 3m, the prompt peer reports first", interval: 3 * time.Minute, reporting: []types.NodeID{prompt, slow}}, + {name: "a block every 3m, the other peer reports first", interval: 3 * time.Minute, reporting: []types.NodeID{slow, prompt}}, + {name: "a block every 5s, the prompt peer reports first", interval: 5 * time.Second, reporting: []types.NodeID{prompt, slow}}, + {name: "a block every 5s, the other peer reports first", interval: 5 * time.Second, reporting: []types.NodeID{slow, prompt}}, + {name: "a block every 2s, the other peer reports first", interval: 2 * time.Second, reporting: []types.NodeID{slow, prompt}}, + {name: "a block every second, the other peer reports first", interval: time.Second, reporting: []types.NodeID{slow, prompt}}, + } { + suite.Run(tc.name, func() { + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height) + defer h.cancel() + start := h.clock.Now() + r := &tipRun{ + h: h, + interval: tc.interval, + hold: map[types.NodeID]time.Duration{slow: requestDeadline - time.Second}, + reporting: tc.reporting, + blockTimes: map[int64]time.Time{}, + suppliedBy: map[types.NodeID]int{}, + tip: height - 1 + behind, + } + // the last committed block is two seconds old + for i := int64(0); i < behind; i++ { + r.blockTimes[r.tip-i] = start.Add(-2*time.Second - time.Duration(i)*tc.interval) + } + r.nextCommit = r.blockTimes[r.tip].Add(tc.interval) + h.sync.AddPeer(newPeerData("far range", 1_000_000_000, maxPlausiblePeerHeight)) + + elapsed, caughtUp, ended := h.run(limit, r.step) + suite.Require().True(ended, + "still in block sync after %s: %d blocks applied (%s), the last applied block never older than %s", + elapsed, r.applied, fmt.Sprint(r.suppliedBy), r.oldest) + suite.Require().False(caughtUp) + suite.T().Logf("handed over after %s: %d blocks applied (%s), the last applied block never older than %s", + elapsed, r.applied, fmt.Sprint(r.suppliedBy), r.oldest) + }) + } +} + +// TestWaitForSyncFetchesEachBlockAtTheTipFromOnePeer checks how many times a +// node following the network fetches each new block while it is still in block +// sync. Five peers hold every block and answer at once. One more peer reports a +// far height and answers every request that it has no such block; it is the +// only peer advertising the next height until the others report it, so it is +// asked for it first. Its answer must not have every block fetched from all +// five peers: the request the producer sends once a peer reports the block +// brings it before the others are asked. +func (suite *SynchronizerTestSuite) TestWaitForSyncFetchesEachBlockAtTheTipFromOnePeer() { + const ( + height = int64(100) + interval = 5 * time.Second + claimant = types.NodeID("claimant") + holders = 5 + ) + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height) + defer h.cancel() + start := h.clock.Now() + r := &tipRun{ + h: h, + interval: interval, + declining: map[types.NodeID]bool{claimant: true}, + blockTimes: map[int64]time.Time{height: start.Add(-2 * time.Second)}, + suppliedBy: map[types.NodeID]int{}, + tip: height, + nextCommit: start.Add(interval - 2*time.Second), + } + for i := 0; i < holders; i++ { + r.reporting = append(r.reporting, types.NodeID(fmt.Sprintf("holder %d", i))) + } + h.sync.AddPeer(newPeerData(claimant, 1, maxPlausiblePeerHeight)) + + elapsed, _, ended := h.run(tipGrace+time.Minute, r.step) + suite.Require().True(ended) + suite.T().Logf("%d blocks applied in %s, %d block responses fetched (%s)", r.applied, elapsed, r.blocksSent, fmt.Sprint(r.suppliedBy)) + suite.Require().Greater(r.applied, 10) + suite.Require().LessOrEqual(r.blocksSent, 2*r.applied, + "%d blocks applied in %s, %d block responses fetched from %d peers", + r.applied, elapsed, r.blocksSent, holders) +} + +// TestWaitForSyncHandsOverPastAFarClaimFromNewPeersOnAChainHaltedForLong checks +// that a far claim made by a new peer every minute does not keep a node in block +// sync on a chain that stopped producing blocks, up to staleBlockAge ago. +// Validators restarted to bring such a chain back reach consensus only through +// this handover. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverPastAFarClaimFromNewPeersOnAChainHaltedForLong() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + ) + for _, tc := range []struct { + name string + // haltedFor is the age of the last block when the node starts + haltedFor time.Duration + }{ + {name: "half an hour", haltedFor: 30 * time.Minute}, + {name: "55 minutes", haltedFor: 55 * time.Minute}, + {name: "two hours", haltedFor: 2 * time.Hour}, + {name: "just under the stale age", haltedFor: staleBlockAge - time.Minute}, + } { + suite.Run(tc.name, func() { + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height, newPeerData(tipPeer, 1, height-1)) + defer h.cancel() + h.setLastBlockTime(h.clock.Now().Add(-tc.haltedFor)) + + elapsed, caughtUp, ended := h.run(maxSyncStall+time.Minute, h.newPeerEveryMinute()) + suite.Require().True(ended, + "still in block sync after %s on a chain halted for %s, behind a claim moved to a new peer every minute", + elapsed, tc.haltedFor) + suite.Require().False(caughtUp) + }) + } +} diff --git a/internal/blocksync/handover_fetch_volume_test.go b/internal/blocksync/handover_fetch_volume_test.go new file mode 100644 index 000000000..bb79946f2 --- /dev/null +++ b/internal/blocksync/handover_fetch_volume_test.go @@ -0,0 +1,379 @@ +package blocksync + +import ( + "fmt" + "time" + + "github.com/stretchr/testify/mock" + + "github.com/dashpay/tenderdash/internal/p2p/client" + "github.com/dashpay/tenderdash/types" +) + +// linkRun drives tipRun for a node far behind whose peers each send one block +// at a time over their connection, taking perBlock for each, as a peer serving +// its response queue over one rate-limited connection does. Every request is +// answered with the block, in the order the peer was asked. +// +// With downlink set, the node receives at most that many blocks a second over +// all its connections, shared between the peers in turn, each peer sending at +// most one a second, and a request not answered within the request deadline +// fails as it does in the block client. With every set, the downlink carries +// one block every that long instead. +// +// Once a second, before the link moves, the peers reserveNextBlockPeers returns +// are asked for the height to apply, as askEveryPeer does. +type linkRun struct { + *tipRun + perBlock time.Duration + downlink int + every time.Duration + // asked counts the requests made as askEveryPeer makes them. + asked int + // busyUntil is when a peer finishes sending what it was asked for so far. + busyUntil map[types.NodeID]time.Time + scheduled map[*tipRequest]bool + // slowest is the longest a request took to be answered. + slowest time.Duration + + // peers is the order the downlink is shared in; queues and askedAt hold + // what each peer has still to send and when it was asked. + peers []types.NodeID + queues map[types.NodeID][]*tipRequest + askedAt map[*tipRequest]time.Time + turn int + timeouts int +} + +// never is when a response waiting for the downlink is due. +var never = time.Unix(1<<40, 0) + +func (l *linkRun) step(elapsed time.Duration) { + now := l.h.clock.Now() + height, peers := l.h.sync.reserveNextBlockPeers() + for _, peer := range peers { + l.asked++ + l.request(peer, height, now) + } + if l.every > 0 { + l.downlink = 0 + if elapsed%l.every == 0 { + l.downlink = 1 + } + } + if l.downlink > 0 || l.every > 0 { + l.sharedStep() + return + } + for { + consumed := l.consume(now) + produced := l.produce(now) + for _, req := range l.inFlight { + if l.scheduled[req] { + continue + } + l.scheduled[req] = true + start := now + if busy := l.busyUntil[req.peer.peerID]; busy.After(start) { + start = busy + } + req.endAt = start.Add(l.perBlock) + l.busyUntil[req.peer.peerID] = req.endAt + l.slowest = max(l.slowest, req.endAt.Sub(now)) + } + if !consumed && !produced { + return + } + } +} + +// TestBlockSyncFarBehindFetchesEachBlockOnce checks how many block responses a +// node far behind downloads for the blocks it applies. Twelve peers hold every +// block and none fails or delays a request beyond its place in the peer's +// queue. When a peer needs a second to send a block, a full queue of twenty +// takes twenty seconds, well inside the 60-second request deadline; the node +// must still fetch each block about once, as it does when blocks arrive at +// once. +func (suite *SynchronizerTestSuite) TestBlockSyncFarBehindFetchesEachBlockOnce() { + const ( + height = int64(100) + blocks = int64(600) + peers = 12 + ) + for _, tc := range []struct { + name string + perBlock time.Duration + }{ + {name: "blocks arrive at once"}, + {name: "a peer sends a block a second", perBlock: time.Second}, + } { + suite.Run(tc.name, func() { + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height) + defer h.cancel() + // the node is re-syncing history committed long ago + h.setLastBlockTime(h.clock.Now().Add(-365 * 24 * time.Hour)) + l := &linkRun{ + tipRun: &tipRun{ + h: h, + blockTimes: map[int64]time.Time{}, + suppliedBy: map[types.NodeID]int{}, + }, + perBlock: tc.perBlock, + busyUntil: map[types.NodeID]time.Time{}, + scheduled: map[*tipRequest]bool{}, + } + for i := 0; i < peers; i++ { + h.sync.AddPeer(newPeerData(types.NodeID(fmt.Sprintf("peer %d", i)), 1, height+blocks-1)) + } + + elapsed, caughtUp, ended := h.run(30*time.Minute, l.step) + suite.Require().True(ended, "not caught up after %s: %d blocks applied", elapsed, l.applied) + suite.Require().True(caughtUp) + // the handover comes with the last block still to apply + suite.Require().GreaterOrEqual(int64(l.applied), blocks-1) + suite.T().Logf("%d blocks applied in %s, %d block responses downloaded, slowest answer %s", + l.applied, elapsed, l.blocksSent, l.slowest) + suite.Require().LessOrEqual(l.blocksSent, 2*l.applied, + "%d blocks applied in %s, %d block responses downloaded from %d peers, slowest answer %s", + l.applied, elapsed, l.blocksSent, peers, l.slowest) + }) + } +} + +// sharedStep is step for a node with a limited downlink. +func (l *linkRun) sharedStep() { + now := l.h.clock.Now() + // requests past the deadline fail + for _, peerID := range l.peers { + var kept []*tipRequest + for _, req := range l.queues[peerID] { + if now.Sub(l.askedAt[req]) < requestDeadline { + kept = append(kept, req) + continue + } + l.timeouts++ + l.drop(req) + req.request.Finished() + req.request.complete(0) + l.h.fail(peerID, req.height, client.ErrPeerNotResponded) + } + l.queues[peerID] = kept + } + // the downlink carries the next block of this many peers, in turn + for sent, tried := 0, 0; sent < l.downlink && tried < len(l.peers); tried++ { + peerID := l.peers[l.turn%len(l.peers)] + l.turn++ + if queue := l.queues[peerID]; len(queue) > 0 { + queue[0].endAt = now + l.slowest = max(l.slowest, now.Sub(l.askedAt[queue[0]])) + l.queues[peerID] = queue[1:] + sent++ + } + } + for { + consumed := l.consume(now) + produced := l.produce(now) + for _, req := range l.inFlight { + if _, queued := l.askedAt[req]; queued { + continue + } + l.askedAt[req] = now + req.endAt = never + l.queues[req.peer.peerID] = append(l.queues[req.peer.peerID], req) + } + if !consumed && !produced { + return + } + } +} + +// drop takes req out of the requests in flight. +func (l *linkRun) drop(req *tipRequest) { + for i, other := range l.inFlight { + if other == req { + l.inFlight = append(l.inFlight[:i], l.inFlight[i+1:]...) + return + } + } +} + +// TestBlockSyncFarBehindOnASharedDownlink checks the same for a node whose +// downlink carries four blocks a second in all, as much as the four peers a +// window of requests is spread over can send. Each block must still be fetched +// about once, and no request may run into the request deadline. +func (suite *SynchronizerTestSuite) TestBlockSyncFarBehindOnASharedDownlink() { + const ( + height = int64(100) + blocks = int64(600) + peers = 12 + ) + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height) + defer h.cancel() + // the node is re-syncing history committed long ago + h.setLastBlockTime(h.clock.Now().Add(-365 * 24 * time.Hour)) + l := &linkRun{ + tipRun: &tipRun{ + h: h, + blockTimes: map[int64]time.Time{}, + suppliedBy: map[types.NodeID]int{}, + }, + downlink: 4, + queues: map[types.NodeID][]*tipRequest{}, + askedAt: map[*tipRequest]time.Time{}, + } + for i := 0; i < peers; i++ { + peerID := types.NodeID(fmt.Sprintf("peer %d", i)) + l.peers = append(l.peers, peerID) + h.sync.AddPeer(newPeerData(peerID, 1, height+blocks-1)) + } + + elapsed, _, ended := h.run(2*time.Hour, l.step) + suite.T().Logf("%d blocks applied in %s, %d block responses downloaded, %d requests timed out, %d of %d peers left, slowest answer %s", + l.applied, elapsed, l.blocksSent, l.timeouts, h.sync.peerStore.Len(), peers, l.slowest) + suite.Require().True(ended, "not caught up after %s: %d blocks applied, %d requests timed out, %d of %d peers left", + elapsed, l.applied, l.timeouts, h.sync.peerStore.Len(), peers) + suite.Require().LessOrEqual(elapsed, 5*time.Minute, + "%d blocks applied in %s, %d block responses downloaded, slowest answer %s; four peers alone send them in under four minutes", + l.applied, elapsed, l.blocksSent, l.slowest) + suite.Require().Zero(l.timeouts, + "%d blocks applied in %s, %d block responses downloaded, %d requests timed out, %d of %d peers left", + l.applied, elapsed, l.blocksSent, l.timeouts, h.sync.peerStore.Len(), peers) + suite.Require().LessOrEqual(l.blocksSent, 2*l.applied, + "%d blocks applied in %s, %d block responses downloaded from %d peers", l.applied, elapsed, l.blocksSent, peers) +} + +// TestBlockSyncFarBehindOnASlowDownlink checks a node behind the network by +// hours over a downlink that carries one block in 12 or 20 seconds. +// Twelve peers hold every block and never fail a request on their own. The node +// must reach the tip and fetch each block about once: a block that takes its +// time within the request deadline is not a reason to ask every peer for it. +func (suite *SynchronizerTestSuite) TestBlockSyncFarBehindOnASlowDownlink() { + const ( + height = int64(100) + blocks = int64(200) + peers = 12 + ) + for _, tc := range []struct { + name string + every time.Duration + // reconnect has every peer report its range every status round, so a + // dropped peer comes back as a reconnecting one would. + reconnect bool + }{ + {name: "a block every 12s", every: 12 * time.Second}, + {name: "a block every 20s", every: 20 * time.Second}, + {name: "a block every 12s, dropped peers reconnect", every: 12 * time.Second, reconnect: true}, + } { + suite.Run(tc.name, func() { + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height) + defer h.cancel() + blockTime := h.clock.Now().Add(-staleBlockAge / 2) + h.setLastBlockTime(blockTime) + l := &linkRun{ + tipRun: &tipRun{ + h: h, + blockTimes: map[int64]time.Time{}, + suppliedBy: map[types.NodeID]int{}, + }, + every: tc.every, + queues: map[types.NodeID][]*tipRequest{}, + askedAt: map[*tipRequest]time.Time{}, + } + top := height + blocks - 1 + for i := 0; i < peers; i++ { + l.peers = append(l.peers, types.NodeID(fmt.Sprintf("peer %d", i))) + } + for applying := height; applying <= top; applying++ { + l.blockTimes[applying] = blockTime + } + report := func() { + for _, id := range l.peers { + h.sync.AddPeer(newPeerData(id, 1, top)) + } + } + report() + + elapsed, caughtUp, ended := h.run(6*time.Hour, func(elapsed time.Duration) { + if tc.reconnect && elapsed%(statusUpdateIntervalSeconds*time.Second) == 0 { + report() + } + l.step(elapsed) + }) + summary := fmt.Sprintf("caught up %v after %s: %d of %d blocks applied, %d responses downloaded, "+ + "%d asked of every peer, %d requests timed out, %d of %d peers left", + caughtUp, elapsed, l.applied, blocks, l.blocksSent, l.asked, l.timeouts, h.sync.peerStore.Len(), peers) + suite.T().Log(summary) + suite.Require().True(ended, summary) + suite.Require().True(caughtUp, "handed over behind: "+summary) + suite.Require().LessOrEqual(l.blocksSent, 2*l.applied, summary) + }) + } +} + +// TestWaitForSyncGoesOnToTheTipBehindPeersFasterThanTheChain checks the at-tip +// rule on a chain committing a block every second, with every peer serving what +// it reports. The node restarts ten minutes behind and its peers together +// deliver more blocks a second than the chain commits, so block sync closes the +// gap. The node must not hand over on its way: by the time it does, its last +// block is within tipGain of the chain's. +func (suite *SynchronizerTestSuite) TestWaitForSyncGoesOnToTheTipBehindPeersFasterThanTheChain() { + const ( + height = int64(1000) + behind = int64(600) + peers = 5 + ) + for _, downlink := range []int{2, 3, 4} { + suite.Run(fmt.Sprintf("peers deliver %d blocks a second", downlink), func() { + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height) + defer h.cancel() + start := h.clock.Now() + l := &linkRun{ + tipRun: &tipRun{ + h: h, + interval: time.Second, + blockTimes: map[int64]time.Time{}, + suppliedBy: map[types.NodeID]int{}, + tip: height - 1 + behind, + nextCommit: start.Add(time.Second), + }, + downlink: downlink, + queues: map[types.NodeID][]*tipRequest{}, + askedAt: map[*tipRequest]time.Time{}, + } + for i := 0; i < peers; i++ { + l.peers = append(l.peers, types.NodeID(fmt.Sprintf("peer %d", i))) + } + for committed := int64(1); committed <= l.tip; committed++ { + l.blockTimes[committed] = start.Add(-time.Duration(l.tip-committed) * time.Second) + } + h.setLastBlockTime(l.blockTimes[height-1]) + + elapsed, caughtUp, ended := h.run(time.Hour, func(elapsed time.Duration) { + now := h.clock.Now() + for !now.Before(l.nextCommit) { + l.tip++ + l.blockTimes[l.tip] = l.nextCommit + l.nextCommit = l.nextCommit.Add(l.interval) + } + if elapsed%(statusUpdateIntervalSeconds*time.Second) == 0 { + for _, id := range l.peers { + h.sync.AddPeer(newPeerData(id, 1, l.tip)) + } + } + l.step(elapsed) + }) + applying, _ := h.sync.GetStatus() + summary := fmt.Sprintf("caught up %v after %s: applying %d, chain tip %d, %d responses, %d timeouts", + caughtUp, elapsed, applying, l.tip, l.blocksSent, l.timeouts) + suite.T().Log(summary) + suite.Require().True(ended, summary) + suite.Require().LessOrEqual(l.tip-applying+1, int64(tipGain/time.Second), + "handed over %d blocks behind peers delivering %d blocks a second on a chain committing one: %s", + l.tip-applying+1, downlink, summary) + }) + } +} diff --git a/internal/blocksync/handover_late_supply_test.go b/internal/blocksync/handover_late_supply_test.go new file mode 100644 index 000000000..326d9368e --- /dev/null +++ b/internal/blocksync/handover_late_supply_test.go @@ -0,0 +1,240 @@ +package blocksync + +import ( + "fmt" + "time" + + "github.com/stretchr/testify/mock" + + "github.com/dashpay/tenderdash/internal/p2p/client" + "github.com/dashpay/tenderdash/types" +) + +// lateSupplyRun drives a node that restarted a few blocks behind a chain +// committing a block every three minutes. Requests for the height the node waits +// for are handed out as block sync does: one at a time by the peer store, and +// once a second to the peers reserveNextBlockPeers returns. The peers named +// late answer a request only once the block is lateBy old, or answer just +// inside the request deadline that they have no such block when that is further +// off; the peer named prompt answers at once. Every peer reports each block as +// soon as it is committed, and one more peer reports a far range it is never +// asked for. +type lateSupplyRun struct { + h *waitForSyncHarness + prompt types.NodeID + lateBy time.Duration + interval time.Duration + // blockTimes maps a committed height to its block time. + blockTimes map[int64]time.Time + tip int64 + nextCommit time.Time + // outstanding are the requests for the height the node waits for. + outstanding []*lateRequest + // servedByPrompt counts the blocks the prompt peer was asked for. + servedByPrompt int + served int +} + +type lateRequest struct { + peer PeerData + request *serviceRequest + height int64 + endAt time.Time + deliver bool +} + +func (r *lateSupplyRun) ids() []types.NodeID { + var ids []types.NodeID + for _, peer := range r.h.sync.peerStore.All() { + if peer.base == 1 { + ids = append(ids, peer.peerID) + } + } + return ids +} + +func (r *lateSupplyRun) commit(blockTime time.Time) { + r.tip++ + r.blockTimes[r.tip] = blockTime + for _, id := range r.ids() { + r.h.sync.AddPeer(newPeerData(id, 1, r.tip)) + } +} + +func (r *lateSupplyRun) apply(peer PeerData, request *serviceRequest, height int64) { + sync := r.h.sync + request.Finished() + request.complete(1000) + sync.peerStore.Update(peer.peerID, AddNumPending(-1), ResetFailures()) + if applied, _ := sync.GetStatus(); height < applied { + return + } + r.h.suite.Require().NoError(sync.addBlock(BlockResponse{ + PeerID: peer.peerID, + Block: &types.Block{Header: types.Header{Height: height}}, + })) + sync.advance(r.blockTimes[height], defaultPrecision) + r.served++ +} + +func (r *lateSupplyRun) step(time.Duration) { + sync := r.h.sync + now := r.h.clock.Now() + for !now.Before(r.nextCommit) { + r.commit(r.nextCommit) + r.nextCommit = r.nextCommit.Add(r.interval) + } + var waiting []*lateRequest + for _, out := range r.outstanding { + switch { + case now.Before(out.endAt): + waiting = append(waiting, out) + case out.deliver: + r.apply(out.peer, out.request, out.height) + default: + out.request.Finished() + out.request.complete(0) + r.h.fail(out.peer.peerID, out.height, &client.ErrBlockNotFound{Height: out.height}) + } + } + r.outstanding = waiting + for len(r.outstanding) == 0 { + height, _ := sync.GetStatus() + peer, found := sync.peerStore.reservePeer(height) + if height > r.tip || !found { + break + } + r.ask(peer, height, now) + } + height, peers := sync.reserveNextBlockPeers() + for _, peer := range peers { + if applying, _ := sync.GetStatus(); applying != height { + // applied meanwhile; the request would bring a duplicate + sync.peerStore.Update(peer.peerID, AddNumPending(-1)) + continue + } + r.ask(peer, height, now) + } +} + +// ask has peer answer a request for height as its kind does. +func (r *lateSupplyRun) ask(peer PeerData, height int64, now time.Time) { + request := peer.service.request() + request.Started() + if peer.peerID == r.prompt { + r.servedByPrompt++ + r.apply(peer, request, height) + return + } + deliverAt := r.blockTimes[height].Add(r.lateBy) + if !deliverAt.After(now) { + r.apply(peer, request, height) + return + } + out := &lateRequest{peer: peer, request: request, height: height} + if deliverAt.Sub(now) < syncTimeout { + out.endAt, out.deliver = deliverAt, true + } else { + out.endAt = now.Add(syncTimeout - time.Second) + } + r.outstanding = append(r.outstanding, out) +} + +// TestWaitForSyncHandsOverBehindPeersSupplyingEachBlockLate checks the handover +// of a node that restarts three blocks behind a chain committing a block every +// three minutes, with a prompt peer connected, while a far range is reported by +// a peer nobody asks. The node receives every block within its at-tip window. +// +// When the prompt peer reports first it supplies every block and the node hands +// over two minutes after reaching the tip. When a few peers that supply each +// block about three minutes late report first, they are asked for every block +// first. The prompt peer must be asked as well, and the node must hand over. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverBehindPeersSupplyingEachBlockLate() { + const ( + height = int64(100) + prompt = types.NodeID("prompt peer") + interval = 3 * time.Minute + // late enough that less than tipGrace of the at-tip window is left + lateBy = tipRecency - tipGrace + 5*time.Second + limit = time.Hour + ) + for _, tc := range []struct { + name string + promptFirst bool + }{ + {name: "prompt peer reports first", promptFirst: true}, + {name: "late peers report first"}, + } { + suite.Run(tc.name, func() { + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height) + defer h.cancel() + start := h.clock.Now() + // the node was down for ten minutes + h.setLastBlockTime(start.Add(-10 * time.Minute)) + + r := &lateSupplyRun{ + h: h, + prompt: prompt, + lateBy: lateBy, + interval: interval, + blockTimes: map[int64]time.Time{}, + tip: height - 1, + } + if tc.promptFirst { + h.sync.AddPeer(newPeerData(prompt, 1, r.tip)) + } + for i := 0; i < 6; i++ { + h.sync.AddPeer(newPeerData(types.NodeID(fmt.Sprintf("late peer %d", i)), 1, r.tip)) + } + if !tc.promptFirst { + h.sync.AddPeer(newPeerData(prompt, 1, r.tip)) + } + h.sync.AddPeer(newPeerData("far range", 1_000_000_000, maxPlausiblePeerHeight)) + // the three blocks committed while the node was down + for _, age := range []time.Duration{7 * time.Minute, 4 * time.Minute, time.Minute} { + r.commit(start.Add(-age)) + } + r.nextCommit = start.Add(2 * time.Minute) + + elapsed, caughtUp, ended := h.run(limit, r.step) + suite.Require().True(ended, + "still in block sync after %s: %d blocks applied, each within the at-tip window, %d of them from the prompt peer", + elapsed, r.served, r.servedByPrompt) + suite.Require().False(caughtUp) + }) + } +} + +// TestWaitForSyncKeepsSyncingFarBehindPeersThatFailEveryRequest checks that a +// node far behind the network, by the time of its last block and by every +// claim, is not handed over to consensus because its peers, which earlier +// supplied blocks, fail every request for longer than the stall backstop. +// Consensus catch-up does not close such a gap, and block sync resumes as soon +// as a peer serves again (dashpay/tenderdash#1413). +func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsSyncingFarBehindPeersThatFailEveryRequest() { + const height = int64(24175) + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + var peers []PeerData + for i := 0; i < 5; i++ { + peer := newPeerData(types.NodeID(fmt.Sprintf("peer %d", i)), 1, 300_000) + peers = append(peers, peer) + } + h := suite.newWaitForSyncHarness(height, peers...) + defer h.cancel() + // the node is re-syncing history committed long ago + h.setLastBlockTime(h.clock.Now().Add(-365 * 24 * time.Hour)) + + elapsed, _, ended := h.run(maxSyncStall+2*time.Minute, func(elapsed time.Duration) { + if elapsed == 0 || elapsed%syncTimeout != 0 { + return + } + // every peer times out the request it holds and reports its range again + for _, peer := range peers { + h.fail(peer.peerID, height, client.ErrPeerNotResponded) + h.sync.AddPeer(newPeerData(peer.peerID, 1, 300_000)) + } + }) + suite.Require().False(ended, + "handed over after %s, 275825 blocks behind, although five peers still report the next height", elapsed) +} diff --git a/internal/blocksync/handover_review_test.go b/internal/blocksync/handover_review_test.go new file mode 100644 index 000000000..6dd692484 --- /dev/null +++ b/internal/blocksync/handover_review_test.go @@ -0,0 +1,22 @@ +package blocksync + +import ( + "time" +) + +// TestWaitForSyncStallBackstopIgnoresAFarClaimOnARecentBlock checks the stall +// backstop against a far claim made while another peer is within maxCatchupGap +// and the node's last block is less than staleBlockAge old. The claim is not +// verified, so it must not keep the stalled node in block sync. +func (suite *SynchronizerTestSuite) TestWaitForSyncStallBackstopIgnoresAFarClaimOnARecentBlock() { + const height = int64(100) + near := newPeerData("near peer", 1, height+maxCatchupGap/2) + h := suite.newWaitForSyncHarness(height, near, newPeerData("claimant", 1, maxPlausiblePeerHeight)) + defer h.cancel() + h.setLastBlockTime(h.clock.Now().Add(-staleBlockAge / 2)) + + elapsed, caughtUp, ended := h.run(maxSyncStall+time.Minute, nil) + suite.Require().True(ended, "a far claim kept a node stalled on a recent block in block sync") + suite.Require().False(caughtUp) + suite.Require().Greater(elapsed, maxSyncStall) +} diff --git a/internal/blocksync/peer_service_test.go b/internal/blocksync/peer_service_test.go index 0dfd42812..9834d91cf 100644 --- a/internal/blocksync/peer_service_test.go +++ b/internal/blocksync/peer_service_test.go @@ -215,10 +215,10 @@ func TestServiceProbationRemainsServableBelowTip(t *testing.T) { endServiceRequest(syncer.peerStore, peer, request, 100) _, found := syncer.peerStore.FindPeer(1) require.False(t, found, "peer is cooling") - _, _, servable, _, hasPeers, maxServable := syncer.stallSnapshot() - require.True(t, servable) - require.True(t, hasPeers) - require.EqualValues(t, 1000, maxServable) + st := syncer.stallSnapshot() + require.True(t, st.servable) + require.True(t, st.hasPeers) + require.EqualValues(t, 1000, st.maxServableHeight) } func TestServiceSavedCreditCannotHideNewSlowService(t *testing.T) { diff --git a/internal/blocksync/peer_store.go b/internal/blocksync/peer_store.go index 3c3d2d000..64bc1a1ae 100644 --- a/internal/blocksync/peer_store.go +++ b/internal/blocksync/peer_store.go @@ -146,6 +146,11 @@ func (p *InMemPeerStore) reservePeer(height int64) (PeerData, bool) { if !found { return PeerData{}, false } + return p.reserve(peer, height) +} + +// reserve is reservePeer for a peer the caller selected. +func (p *InMemPeerStore) reserve(peer PeerData, height int64) (PeerData, bool) { reserved := false p.store.Update(peer.peerID, func(_ types.NodeID, current *PeerData) { if current.service != peer.service || current.numPending >= maxPendingRequestsPerPeer || diff --git a/internal/blocksync/reactor.go b/internal/blocksync/reactor.go index 43fb6501d..3da6aa586 100644 --- a/internal/blocksync/reactor.go +++ b/internal/blocksync/reactor.go @@ -34,11 +34,14 @@ const ( // hand over to consensus after this duration of inactivity, even when peers // still report higher blocks, so that a wedged synchronizer cannot keep the - // node out of consensus forever - but only within maxCatchupGap of the tip + // node out of consensus forever - but only within maxCatchupGap of the tip, + // or when the inactivity began on a block less than staleBlockAge old maxSyncStall = 10 * time.Minute // maxCatchupGap is how far behind the highest height any peer claims the node - // may be and still hand over to consensus on the stall backstop above. + // may be and still hand over to consensus on the stall backstop above when + // its last block is older than staleBlockAge. It also bounds the claims a + // node with a recent last block passes on as the consensus catch-up target. // // What is left after the handover is covered by consensus catch-up, which // moves about one block per gossip cycle: it closes a gap of a few blocks in @@ -47,6 +50,26 @@ const ( // up on it puts a validator at heights the network committed long ago // (dashpay/tenderdash#1413). maxCatchupGap int64 = 10 + + // tipRecency is the oldest the last applied block may be for the node to + // count as at the tip. It covers the three-minute empty-block interval Dash + // Platform runs with and a margin for extra rounds and clock skew. + tipRecency = 5 * time.Minute + + // tipGrace is how long the node must stay at the tip before higher height + // claims stop keeping it in block sync. + tipGrace = 2 * syncTimeout + + // tipGain is how much younger the last applied block must get while the + // node counts as at the tip for tipGrace to start over: the node is still + // closing the gap, as it does when it applies blocks a quarter faster than + // the chain commits them. One block of a slow chain does that too, once. + tipGain = 30 * time.Second + + // staleBlockAge is how old the last applied block must be, when a stall + // begins, for peer height claims to count in the stall backstop. It is + // longer than any halt a restarted node is expected to help end. + staleBlockAge = 24 * time.Hour ) type ReactorOption func(*Reactor) diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index 1c214d051..32886ac27 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -131,6 +131,13 @@ type ( logger log.Logger lastAdvance time.Time + // lastBlockTime is the time of the last applied block and precision the + // synchrony precision of the state it left. Guarded by mtx. + lastBlockTime time.Time + precision time.Duration + // asked holds, for the height to apply, the peers askEveryPeer has seen + // advertise it, and whether it asked them for it yet. Guarded by mtx. + asked map[types.NodeID]bool mtx sync.RWMutex @@ -206,6 +213,8 @@ func NewSynchronizer(start int64, client client.BlockClient, blockExec *blockApp monitorInterval: defaultSyncRateIntervalBlocks, workerPool: workerpool.New(poolWorkerSize, workerpool.WithLogger(logger)), pendingToApply: map[int64]BlockResponse{}, + precision: types.DefaultSynchronyParams().Precision, + asked: map[types.NodeID]bool{}, } for _, opt := range opts { opt(bp) @@ -244,6 +253,11 @@ func (s *Synchronizer) OnStart(ctx context.Context) error { } s.lastAdvance = s.clock.Now() s.lastMonitorUpdate = s.lastAdvance + if s.applier != nil { + state := s.applier.State() + s.lastBlockTime = state.LastBlockTime + s.precision = state.ConsensusParams.Synchrony.SynchronyParamsOrDefaults().Precision + } s.ctx, s.cancel = context.WithCancel(ctx) applicationCtx, ok := ctx.Value(applicationContextKey{}).(context.Context) if !ok { @@ -252,6 +266,7 @@ func (s *Synchronizer) OnStart(ctx context.Context) error { s.consumerDone = make(chan struct{}) s.workerPool.Run(s.ctx) go s.runHandler(s.ctx, s.produceJob) + go s.runHandler(s.ctx, s.askEveryPeer) go func() { defer close(s.consumerDone) s.runHandler(s.ctx, func(handlerCtx context.Context) error { @@ -324,13 +339,21 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { if !errors.As(res.Err, &bfErr) { return nil } + // A block applied meanwhile, typically from another peer asked for it + // as well, is not fetched again. + applyHeight, _ := s.GetStatus() + pending := bfErr.height >= applyHeight var missing *client.ErrBlockNotFound if errors.As(bfErr.err, &missing) && missing.Height == bfErr.height { s.peerStore.RecordMissingBlock(bfErr.peerID, bfErr.height) - s.jobGen.pushBack(bfErr.height) + if pending { + s.jobGen.pushBack(bfErr.height) + } return nil } - s.jobGen.pushBack(bfErr.height) + if pending { + s.jobGen.pushBack(bfErr.height) + } // One failed request is usually a timeout under load, not a bad peer. // Dropping the peer also fails its other in-flight requests, each of // which drops another peer in turn, so react only to a sustained run of @@ -412,6 +435,60 @@ func (s *Synchronizer) backlog() (applyHeight int64, pendingBytes int) { return s.height, pendingBytes } +// askEveryPeer asks, once a second, every peer reserveNextBlockPeers returns for +// the height block sync waits to apply. +func (s *Synchronizer) askEveryPeer(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-s.clock.After(time.Second): + } + height, peers := s.reserveNextBlockPeers() + for _, peer := range peers { + s.jobProgressCounter.Add(1) + if err := s.workerPool.Send(ctx, workerpool.NewJob(blockFetchJobHandler(s.client, peer, height))); err != nil { + s.jobProgressCounter.Add(-1) + return err + } + } + return nil +} + +// reserveNextBlockPeers returns the height block sync waits to apply and, once +// block sync has stalled on it for syncTimeout, every peer to ask for it: each +// peer that advertises it, once. Blocks are applied in order, so whichever peers +// hold the requests made for that height so far must not decide how long the +// others, which may serve it at once, go unasked. The wait is as long as a +// request may take, so a link that carries blocks slowly but within their +// request deadline does not have each of them fetched from every peer. +// +// A peer is asked on the second call that finds it advertising the height, so +// the request the job producer sends when a peer first reports a new block gets +// a second to bring it, and a block served promptly is fetched once. +func (s *Synchronizer) reserveNextBlockPeers() (int64, []PeerData) { + s.mtx.Lock() + defer s.mtx.Unlock() + if _, received := s.pendingToApply[s.height]; received || s.clock.Since(s.lastAdvance) <= syncTimeout { + return s.height, nil + } + var peers []PeerData + for _, peer := range s.peerStore.Query(heightBetweenPeerHeightRange(s.height), 0) { + asked, seen := s.asked[peer.peerID] + if !seen { + s.asked[peer.peerID] = false + continue + } + if asked { + continue + } + if reserved, ok := s.peerStore.reserve(peer, s.height); ok { + s.asked[peer.peerID] = true + peers = append(peers, reserved) + } + } + return s.height, peers +} + // GetStatus returns synchronizer's height, count of in progress requests func (s *Synchronizer) GetStatus() (int64, int32) { cnt := s.jobProgressCounter.Load() @@ -433,7 +510,8 @@ func (s *Synchronizer) IsCaughtUp() bool { } // WaitForSync blocks until block sync is finished, and reports whether the node -// actually caught up and the highest peer target observed while waiting. +// actually caught up and the highest peer target observed while waiting; for a +// node at the tip, the highest within maxCatchupGap of it. // // A stall on its own is not a reason to stop. Handing over to consensus is a // one-way door - only the state sync path ever switches back to block sync - and @@ -444,65 +522,138 @@ func (s *Synchronizer) IsCaughtUp() bool { // block, or once the stall has outlasted maxSyncStall within maxCatchupGap of // the tip, so that a wedged synchronizer can still hand over rather than // blocking forever without handing over a node consensus cannot catch up. +// +// What peers report about themselves is not verified, so it cannot be what +// keeps the node in block sync once it has caught up. The time of the last +// applied block is: the validators commit to it, and a node far behind applies +// old blocks. Once that time has been within tipRecency of the local clock for +// tipGrace, the node hands over whatever peers report, even though blocks +// arriving meanwhile keep it from stalling. The wait starts over each time the +// last block has got tipGain younger: the node is still closing the gap. A +// stall of maxSyncStall that began on a block less than staleBlockAge old ends +// block sync the same way. func (s *Synchronizer) WaitForSync(ctx context.Context) (caughtUp bool, targetHeight int64) { ticker := s.clock.NewTicker(switchToConsensusIntervalSeconds * time.Second) defer ticker.Stop() + // tipAge is the age of the last block at atTipSince. + atTipSince := s.clock.Now() + tipAge := atTipSince.Sub(s.stallSnapshot().lastBlockTime) + // nearTarget is the highest claim seen within maxCatchupGap of the node. + var nearTarget int64 for { select { case <-ctx.Done(): - height, _, _, peerHeight, hasPeers, _ := s.stallSnapshot() - return hasPeers && height >= peerHeight, max(targetHeight, peerHeight) + st := s.stallSnapshot() + return st.hasPeers && st.height >= st.maxPeerHeight, max(targetHeight, st.maxPeerHeight) case <-ticker.Chan(): - height, stalledFor, servable, maxPeerHeight, hasPeers, maxServableHeight := s.stallSnapshot() - targetHeight = max(targetHeight, maxPeerHeight) - if hasPeers && height >= maxPeerHeight { - return true, targetHeight + st := s.stallSnapshot() + targetHeight = max(targetHeight, st.maxPeerHeight) + nearTarget = max(nearTarget, st.maxNearHeight) + now := s.clock.Now() + // At the tip, claims out of reach of it must not hold back our own + // proposals, whichever way block sync ends. + target := targetHeight + recent := isRecent(st.lastBlockTime, tipRecency, st.precision, now) + if recent { + target = nearTarget + } + if age := now.Sub(st.lastBlockTime); !recent || tipAge-age > tipGain { + atTipSince, tipAge = now, age + } + if st.hasPeers && st.height >= st.maxPeerHeight { + return true, target } - switch stallVerdictFor(servable, maxServableHeight-height, stalledFor) { + // Only far back in the chain's history does the distance to what peers + // report keep a stalled node in block sync. The age is taken when the + // stall began, so a stall does not age a stopped chain into it. + behindBy := st.maxServableHeight - st.height + if isRecent(st.lastBlockTime, staleBlockAge, st.precision, now.Add(-st.stalledFor)) { + behindBy = 0 + } + verdict := stallVerdictFor(st.servable, behindBy, st.stalledFor, now.Sub(atTipSince)) + switch verdict { + case stopAtTip: + s.logger.Warn( + "at the tip by block time, handing over to consensus below the highest peer claim", + "height", st.height, + "max_peer_height", st.maxPeerHeight, + "last_block_time", st.lastBlockTime, + ) + return false, target case stopNothingToFetch: - if maxPeerHeight > height { + if st.maxPeerHeight > st.height { // Peers claim to be ahead yet none of them holds the block we // need. Worth saying: the node stops while apparently behind, // which otherwise looks like it gave up for no reason. s.logger.Error( "no peer can serve the next block, handing over to consensus", - "height", height, - "max_peer_height", maxPeerHeight, - "stalled_for", stalledFor, + "height", st.height, + "max_peer_height", st.maxPeerHeight, + "stalled_for", st.stalledFor, ) } - return false, targetHeight + return false, target case stopStalledTooLong: s.logger.Error( "block sync stalled for too long, handing over to consensus while still behind", - "height", height, - "max_peer_height", maxPeerHeight, - "max_servable_height", maxServableHeight, - "stalled_for", stalledFor, + "height", st.height, + "max_peer_height", st.maxPeerHeight, + "max_servable_height", st.maxServableHeight, + "last_block_time", st.lastBlockTime, + "stalled_for", st.stalledFor, ) - return false, targetHeight + return false, target } - if stalledFor > syncTimeout { + if st.stalledFor > syncTimeout { s.logger.Error( "block sync has stalled but a peer still holds the block we need, still trying", - "height", height, - "max_peer_height", maxPeerHeight, - "stalled_for", stalledFor, - "max_servable_height", maxServableHeight, - "behind_by", maxServableHeight-height, + "height", st.height, + "max_peer_height", st.maxPeerHeight, + "stalled_for", st.stalledFor, + "max_servable_height", st.maxServableHeight, + "behind_by", st.maxServableHeight-st.height, ) continue } s.logger.Info( "not caught up yet", - "height", height, - "max_peer_height", maxPeerHeight, - "timeout_in", syncTimeout-stalledFor, + "height", st.height, + "max_peer_height", st.maxPeerHeight, + "timeout_in", syncTimeout-st.stalledFor, ) } } } +// isRecent reports whether blockTime, the time of the last applied block, is +// at most window old at now, and no further ahead of it than precision, the +// bound validators hold block times to. Further ahead, the local clock is behind +// the validators' and cannot tell how old the block is. +func isRecent(blockTime time.Time, window, precision time.Duration, now time.Time) bool { + age := now.Sub(blockTime) + return age <= window && age >= -precision +} + +// stallState is one observation of what the stall verdict is formed from. +type stallState struct { + // height is the next height to apply, waiting for stalledFor. + height int64 + stalledFor time.Duration + hasPeers bool + // servable says some peer advertises height; maxServableHeight is the + // highest such claim and maxPeerHeight the highest claim of any peer. + servable bool + maxServableHeight int64 + maxPeerHeight int64 + // maxNearHeight is the highest claim within maxCatchupGap of the last + // applied block. + maxNearHeight int64 + // lastBlockTime is the time of the last applied block and precision the + // synchrony precision of the state it left. + lastBlockTime time.Time + precision time.Duration +} + // stallSnapshot reads everything the stall verdict is formed from as one // observation: the height block sync is waiting for, how long it has been // waiting for it, and the highest advertised and servable peer heights. @@ -514,30 +665,38 @@ func (s *Synchronizer) WaitForSync(ctx context.Context) (caughtUp bool, targetHe // claims to be. Only servable peers determine the stall gap; the overall maximum // remains the handover target so pruning cannot erase evidence of a higher tip. // -// They are read under one lock because advance() stamps the height and the -// advance time together under that same lock. A block applied concurrently -// therefore lands either wholly inside the snapshot or wholly outside it, and -// the verdict can never pair a height with a staleness or a servability -// measured against a different one. Read separately, it could: a height read -// before an advance, checked for servability after one, reports nothing to -// fetch while the height we had by then moved on to is served. Ending block -// sync is a one-way door, so a stop assembled from two inconsistent readings -// leaves the node in consensus catch-up it cannot leave. -func (s *Synchronizer) stallSnapshot() ( - height int64, stalledFor time.Duration, servable bool, maxPeerHeight int64, hasPeers bool, maxServableHeight int64, -) { +// They are read under one lock because advance() stamps the height, the advance +// time and the block time together under that same lock. A block applied +// concurrently therefore lands either wholly inside the snapshot or wholly +// outside it, and the verdict can never pair a height with a staleness or a +// servability measured against a different one. Read separately, it could: a +// height read before an advance, checked for servability after one, reports +// nothing to fetch while the height we had by then moved on to is served. Ending +// block sync is a one-way door, so a stop assembled from two inconsistent +// readings leaves the node in consensus catch-up it cannot leave. +func (s *Synchronizer) stallSnapshot() stallState { s.mtx.RLock() defer s.mtx.RUnlock() peers := s.peerStore.All() + st := stallState{ + height: s.height, + stalledFor: s.clock.Since(s.lastAdvance), + hasPeers: len(peers) > 0, + lastBlockTime: s.lastBlockTime, + precision: s.precision, + } hasHeight := heightBetweenPeerHeightRange(s.height) for _, peer := range peers { - maxPeerHeight = max(maxPeerHeight, peer.height) + st.maxPeerHeight = max(st.maxPeerHeight, peer.height) + if peer.height < s.height+maxCatchupGap { + st.maxNearHeight = max(st.maxNearHeight, peer.height) + } if hasHeight(peer.peerID, peer) { - servable = true - maxServableHeight = max(maxServableHeight, peer.height) + st.servable = true + st.maxServableHeight = max(st.maxServableHeight, peer.height) } } - return s.height, s.clock.Since(s.lastAdvance), servable, maxPeerHeight, len(peers) > 0, maxServableHeight + return st } // stallVerdict says what a lack of progress in block sync means. @@ -550,6 +709,8 @@ const ( stopNothingToFetch // stopStalledTooLong means a peer still holds it but we have to hand over anyway stopStalledTooLong + // stopAtTip means the last applied block has stayed recent for tipGrace + stopAtTip ) // stallVerdictFor decides what to do when block sync has made no progress for @@ -569,8 +730,18 @@ const ( // block arrives anyway - but only within maxCatchupGap of the tip. Further back // the backstop buys nothing and costs what dashpay/tenderdash#1413 describes, so // block sync keeps retrying and says so every interval. -func stallVerdictFor(servable bool, behindBy int64, stalledFor time.Duration) stallVerdict { +// +// The caller passes no distance for a stall that began on a recent block, so +// such a stall ends at maxSyncStall whatever peers report: askEveryPeer has by +// then asked every peer advertising the block for it. +// +// atTipFor is how long the last applied block has been recent. Unlike +// stalledFor it is not reset by applying blocks, so a node following the +// network is not kept in block sync by a height nobody serves. +func stallVerdictFor(servable bool, behindBy int64, stalledFor, atTipFor time.Duration) stallVerdict { switch { + case atTipFor > tipGrace: + return stopAtTip case stalledFor <= syncTimeout: return keepSyncing case !servable: @@ -625,6 +796,7 @@ func (s *Synchronizer) dropPeer(peerID types.NodeID) []int64 { } } s.peerStore.Delete(peerID) + delete(s.asked, peerID) return heights } @@ -641,7 +813,8 @@ func (s *Synchronizer) applyBlock(ctx context.Context) (BlockResponse, error) { if err != nil { return resp, fmt.Errorf("cannot apply block: %w", err) } - s.advance() + state := s.applier.State() + s.advance(state.LastBlockTime, state.ConsensusParams.Synchrony.SynchronyParamsOrDefaults().Precision) s.updateMonitor() } } @@ -653,9 +826,14 @@ func (s *Synchronizer) getPendingResponse() (BlockResponse, bool) { return resp, ok } -func (s *Synchronizer) advance() { +// advance moves past the block just applied, which carries blockTime and left +// the chain with synchrony precision. +func (s *Synchronizer) advance(blockTime time.Time, precision time.Duration) { s.mtx.Lock() defer s.mtx.Unlock() + s.lastBlockTime = blockTime + s.precision = precision + clear(s.asked) delete(s.pendingToApply, s.height) s.height++ s.lastAdvance = s.clock.Now() diff --git a/internal/blocksync/synchronizer_test.go b/internal/blocksync/synchronizer_test.go index fb0a92363..4b084b2f9 100644 --- a/internal/blocksync/synchronizer_test.go +++ b/internal/blocksync/synchronizer_test.go @@ -1326,6 +1326,59 @@ func (suite *SynchronizerTestSuite) TestNoBlockResponseForWrongHeightPreservesRa suite.EqualValues(1, peer.numFailures) } +// defaultPrecision is the synchrony precision of a chain on default parameters. +var defaultPrecision = types.DefaultSynchronyParams().Precision + +// TestStartSeedsTheLastBlockTime checks that block sync judges the tip from the +// last block the node already has, and the synchrony precision of its stored +// state, until it applies another block, so a node restarting after long +// downtime is not taken for one at the tip. +func (suite *SynchronizerTestSuite) TestStartSeedsTheLastBlockTime() { + state := suite.initialState.Copy() + state.LastBlockTime = time.Unix(1_700_000_000, 0) + state.ConsensusParams.Synchrony.Precision = 2 * time.Second + applier := newBlockApplier(suite.blockExec, suite.store, applierWithState(state)) + pool := NewSynchronizer(1, suite.client, applier, WithWorkerPool(workerpool.New(0))) + suite.Require().NoError(pool.Start(context.Background())) + defer pool.Stop() + st := pool.stallSnapshot() + suite.Equal(state.LastBlockTime, st.lastBlockTime) + suite.Equal(2*time.Second, st.precision) +} + +// TestApplyingABlockTakesThePrecisionOfTheNewState checks that the synchrony +// precision follows the chain's parameters as blocks are applied, not those it +// had when block sync started. +func (suite *SynchronizerTestSuite) TestApplyingABlockTakesThePrecisionOfTheNewState() { + ctx := context.Background() + b := suite.newBacklogHarness() + suite.Equal(defaultPrecision, b.pool.stallSnapshot().precision) + state := suite.initialState.Copy() + state.ConsensusParams.Synchrony.Precision = 3 * time.Second + b.pool.applier.UpdateState(state) + resp := b.produce(ctx) + suite.Require().NotNil(resp) + b.deliver(ctx, resp) + st := b.pool.stallSnapshot() + suite.EqualValues(2, st.height) + suite.Equal(3*time.Second, st.precision) +} + +// TestWaitForSyncDoesNotWaitForABlockBeingApplied checks that the handover +// loop does not wait on the applier, which is held for the whole time a block +// is executed, so it notices cancellation while a slow block is applied. +func (suite *SynchronizerTestSuite) TestWaitForSyncDoesNotWaitForABlockBeingApplied() { + const height = int64(100) + h := suite.newWaitForSyncHarness(height, newPeerData("peer", 1, height+1000)) + h.sync.applier.mtx.Lock() + defer h.sync.applier.mtx.Unlock() + h.clock.Advance(time.Second) + time.Sleep(10 * time.Millisecond) + h.cancel() + _, ended := h.result(time.Second) + suite.Require().True(ended, "the handover loop waited for the block being applied") +} + // TestStatusRefreshUpdatesAdvertisedRange checks that a status response still does // the one thing it is for: moving the range of blocks the peer claims to serve. It // is the guard on the tests above, which a synchronizer that ignored status @@ -1409,6 +1462,7 @@ func TestStallVerdictFor(t *testing.T) { servable bool behindBy int64 stalledFor time.Duration + atTipFor time.Duration want stallVerdict }{ { @@ -1461,10 +1515,32 @@ func TestStallVerdictFor(t *testing.T) { stalledFor: maxSyncStall + time.Second, want: stopNothingToFetch, }, + { + name: "stalled past the wedge limit with no claim backed", + servable: true, + behindBy: -100, + stalledFor: maxSyncStall + time.Second, + want: stopStalledTooLong, + }, + { + name: "at the tip past the grace, behind a far claim", + servable: true, + behindBy: maxCatchupGap + 1, + atTipFor: tipGrace + time.Second, + want: stopAtTip, + }, + { + name: "at the tip within the grace", + servable: true, + behindBy: maxCatchupGap + 1, + atTipFor: tipGrace, + want: keepSyncing, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.want, stallVerdictFor(tc.servable, tc.behindBy, tc.stalledFor)) + require.Equal(t, tc.want, + stallVerdictFor(tc.servable, tc.behindBy, tc.stalledFor, tc.atTipFor)) }) } } @@ -1473,9 +1549,14 @@ func TestStallVerdictFor(t *testing.T) { // The synchronizer never advances a height, so the only thing that changes the // verdict is how far the clock is moved. type waitForSyncHarness struct { - clock *clockwork.FakeClock - done chan bool - cancel context.CancelFunc + suite *SynchronizerTestSuite + sync *Synchronizer + resultCh chan workerpool.Result + clock *clockwork.FakeClock + done chan bool + cancel context.CancelFunc + // target is the handover target WaitForSync returned; read it only after done. + target int64 } // newWaitForSyncHarness starts WaitForSync at the given height with the given @@ -1484,22 +1565,35 @@ type waitForSyncHarness struct { func (suite *SynchronizerTestSuite) newWaitForSyncHarness(height int64, peers ...PeerData) *waitForSyncHarness { clock := clockwork.NewFakeClock() applier := newBlockApplier(suite.blockExec, suite.store, applierWithState(suite.initialState)) - sync := NewSynchronizer(height, suite.client, applier, WithClock(clock)) + resultCh := make(chan workerpool.Result, 1) + sync := NewSynchronizer(height, suite.client, applier, WithClock(clock), + WithWorkerPool(workerpool.New(0, workerpool.WithResultCh(resultCh)))) // Deliberately not started: WaitForSync reads only the height, the peer store // and lastAdvance, and leaving the job pipeline out keeps the ticker the sole // waiter on the fake clock. sync.lastAdvance = clock.Now() + // The node starts at the tip: its last block is recent. + sync.lastBlockTime = clock.Now() for _, peer := range peers { sync.AddPeer(peer) } + h := suite.startWaitForSync(sync, clock) + h.resultCh = resultCh + return h +} + +// startWaitForSync runs WaitForSync for sync on its own goroutine and returns once +// its ticker is registered with clock. +func (suite *SynchronizerTestSuite) startWaitForSync(sync *Synchronizer, clock *clockwork.FakeClock) *waitForSyncHarness { ctx, cancel := context.WithCancel(context.Background()) - done := make(chan bool, 1) + h := &waitForSyncHarness{suite: suite, sync: sync, clock: clock, done: make(chan bool, 1), cancel: cancel} go func() { - caughtUp, _ := sync.WaitForSync(ctx) - done <- caughtUp + caughtUp, target := sync.WaitForSync(ctx) + h.target = target + h.done <- caughtUp }() suite.Require().NoError(clock.BlockUntilContext(ctx, 1)) - return &waitForSyncHarness{clock: clock, done: done, cancel: cancel} + return h } // result waits up to timeout of real time for WaitForSync to return. ended @@ -1554,18 +1648,19 @@ func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsGoingWhileAPeerCanServeU // that claims our height and never answers keeps the node in block sync // indefinitely unless a stall long enough to look like a wedge ends it. Within // reach of the tip consensus catch-up can cover what is left, so the backstop -// applies. +// applies. The last block is old, so the node is not at the tip. func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverAfterMaxSyncStall() { const height = int64(100) peer := newPeerData("peer in range", 1, height+maxCatchupGap) h := suite.newWaitForSyncHarness(height, peer) defer h.cancel() + h.setLastBlockTime(h.clock.Now().Add(-2 * staleBlockAge)) - h.clock.Advance(maxSyncStall + time.Second) - - caughtUp, ended := h.result(2 * time.Second) + elapsed, caughtUp, ended := h.run(maxSyncStall+time.Minute, nil) suite.Require().True(ended, "no progress for %s did not hand over to consensus", maxSyncStall) suite.Require().False(caughtUp, "handing over after a stall is not catching up") + suite.Require().Greater(elapsed, maxSyncStall) + suite.Require().LessOrEqual(elapsed, maxSyncStall+unservedHandoverSlack) } // TestWaitForSyncKeepsSyncingWhileFarBehind is the counterpart: the wall-clock @@ -1578,11 +1673,10 @@ func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsSyncingWhileFarBehind() peer := newPeerData("peer far ahead", 1, height+maxCatchupGap+1) h := suite.newWaitForSyncHarness(height, peer) defer h.cancel() + h.setLastBlockTime(h.clock.Now().Add(-2 * staleBlockAge)) - h.clock.Advance(maxSyncStall + time.Second) - - _, ended := h.result(200 * time.Millisecond) - suite.Require().False(ended, "block sync gave up while still %d blocks behind", maxCatchupGap+1) + elapsed, _, ended := h.run(maxSyncStall+time.Minute, nil) + suite.Require().False(ended, "block sync gave up after %s while still %d blocks behind", elapsed, maxCatchupGap+1) } func (suite *SynchronizerTestSuite) TestStallSnapshotServableHeight() { @@ -1594,6 +1688,7 @@ func (suite *SynchronizerTestSuite) TestStallSnapshotServableHeight() { peers []PeerData wantMax int64 wantUsable int64 + wantNear int64 }{ {name: "no peers"}, { @@ -1609,9 +1704,10 @@ func (suite *SynchronizerTestSuite) TestStallSnapshotServableHeight() { wantUsable: 20, }, { - name: "no overlapping range", - peers: []PeerData{newPeerData("older", 1, 9), newPeerData("recent", 25, 30)}, - wantMax: 30, + name: "no overlapping range", + peers: []PeerData{newPeerData("older", 1, 9), newPeerData("recent", 25, 30)}, + wantMax: 30, + wantNear: 9, }, { name: "busy peer remains servable", @@ -1619,6 +1715,13 @@ func (suite *SynchronizerTestSuite) TestStallSnapshotServableHeight() { wantMax: 30, wantUsable: 30, }, + { + name: "claims within reach of the node", + peers: []PeerData{newPeerData("near", 1, height+maxCatchupGap-1), newPeerData("far", 1, 30)}, + wantMax: 30, + wantUsable: 30, + wantNear: height + maxCatchupGap - 1, + }, } for _, tc := range testCases { suite.Run(tc.name, func() { @@ -1627,11 +1730,12 @@ func (suite *SynchronizerTestSuite) TestStallSnapshotServableHeight() { for _, peer := range tc.peers { sync.AddPeer(peer) } - _, _, servable, maxHeight, hasPeers, maxServableHeight := sync.stallSnapshot() - suite.Require().Equal(tc.wantMax, maxHeight) - suite.Require().Equal(tc.wantUsable, maxServableHeight) - suite.Require().Equal(tc.wantUsable > 0, servable) - suite.Require().Equal(len(tc.peers) > 0, hasPeers) + st := sync.stallSnapshot() + suite.Require().Equal(tc.wantMax, st.maxPeerHeight) + suite.Require().Equal(tc.wantUsable, st.maxServableHeight) + suite.Require().Equal(tc.wantUsable > 0, st.servable) + suite.Require().Equal(tc.wantNear, st.maxNearHeight) + suite.Require().Equal(len(tc.peers) > 0, st.hasPeers) }) } } @@ -1656,12 +1760,14 @@ func (suite *SynchronizerTestSuite) TestStallSnapshotIsOneObservation() { applied := make(chan struct{}) go func() { <-ready - sync.advance() + sync.advance(time.Time{}, defaultPrecision) close(applied) }() runtime.Gosched() close(ready) - height, stalled, servable, maxPeerHeight, hasPeers, maxServableHeight := sync.stallSnapshot() + st := sync.stallSnapshot() + height, stalled, servable, maxPeerHeight, hasPeers, maxServableHeight := + st.height, st.stalledFor, st.servable, st.maxPeerHeight, st.hasPeers, st.maxServableHeight <-applied suite.Require().True(hasPeers) suite.Require().Equal(int64(1000), maxPeerHeight, "the retained handover target") @@ -1697,6 +1803,758 @@ func (suite *SynchronizerTestSuite) TestWaitForSyncStopsWhenThereIsNothingToFetc suite.Require().False(caughtUp) } +// fail hands the consumer a request to peerID for height that failed with err, as +// the worker pool would. +func (h *waitForSyncHarness) fail(peerID types.NodeID, height int64, err error) { + h.resultCh <- errorResult(PeerData{peerID: peerID}, height, err) + h.suite.Require().NoError(h.sync.consumeJobResult(context.Background(), context.Background())) +} + +// run moves the fake clock a second at a time for up to limit, calling each +// before every step. It returns how long WaitForSync took to hand over, its +// verdict, and whether it did by limit. +func (h *waitForSyncHarness) run(limit time.Duration, each func(elapsed time.Duration)) ( + elapsed time.Duration, caughtUp bool, ended bool, +) { + for ; elapsed <= limit; elapsed += time.Second { + select { + case caughtUp = <-h.done: + return elapsed, caughtUp, true + default: + } + if each != nil { + each(elapsed) + } + h.clock.Advance(time.Second) + // let WaitForSync read this tick before the next one moves the clock again + time.Sleep(time.Millisecond) + } + caughtUp, ended = h.result(2 * time.Second) + return elapsed, caughtUp, ended +} + +// tipBlocks returns a step for run that, every interval, lets the tip peer +// advertise the height the node needs and applies it, as for a node level with +// the network when it commits a block. Blocks arrive well within syncTimeout, so +// block sync never sees a stall. +func (h *waitForSyncHarness) tipBlocks(tipPeer types.NodeID, interval time.Duration) func(time.Duration) { + return func(elapsed time.Duration) { + if elapsed == 0 || elapsed%interval != 0 { + return + } + height, _ := h.sync.GetStatus() + h.sync.AddPeer(newPeerData(tipPeer, 1, height)) + h.suite.Require().NoError(h.sync.addBlock(BlockResponse{ + PeerID: tipPeer, + Block: &types.Block{Header: types.Header{Height: height}}, + })) + h.sync.advance(h.clock.Now(), defaultPrecision) + } +} + +// setLastBlockTime makes blockTime the time of the last block the node applied. +func (h *waitForSyncHarness) setLastBlockTime(blockTime time.Time) { + h.sync.mtx.Lock() + defer h.sync.mtx.Unlock() + h.sync.lastBlockTime = blockTime +} + +const ( + // tipBlockInterval is shorter than syncTimeout, so chain growth alone + // keeps every stall-based handover from firing. + tipBlockInterval = 30 * time.Second + // unservedHandoverSlack covers ticks WaitForSync reads late. + unservedHandoverSlack = 10 * time.Second +) + +// TestWaitForSyncHandsOverPastUnservedClaims checks that a node level with the +// other peers hands over to consensus even while another peer claims heights +// far above and never serves them, and while new blocks keep arriving faster +// than a stall can be declared. Each variant costs the peer nothing to send. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverPastUnservedClaims() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + unserving = types.NodeID("unserving peer") + ) + testCases := []struct { + name string + claimBase int64 + evidence func(h *waitForSyncHarness) + }{ + { + name: "claims our height and never answers", + claimBase: 1, + evidence: func(h *waitForSyncHarness) { h.fail(unserving, height, client.ErrPeerNotResponded) }, + }, + { + name: "claims our height and answers it has no block", + claimBase: 1, + evidence: func(h *waitForSyncHarness) { h.fail(unserving, height, &client.ErrBlockNotFound{Height: height}) }, + }, + { + // The base stays above every height new blocks take us to here, + // so the peer is never asked for anything and never fails. + name: "claims a range starting above ours", + claimBase: height + 1000, + evidence: func(*waitForSyncHarness) {}, + }, + } + for _, tc := range testCases { + suite.Run(tc.name, func() { + h := suite.newWaitForSyncHarness(height, + newPeerData(tipPeer, 1, height-1), + newPeerData(unserving, tc.claimBase, maxPlausiblePeerHeight)) + defer h.cancel() + tc.evidence(h) + + elapsed, caughtUp, ended := h.run(tipGrace+time.Minute, h.tipBlocks(tipPeer, tipBlockInterval)) + suite.Require().True(ended, "block sync kept following the tip behind a claim nobody serves") + suite.Require().False(caughtUp, "an unserved claim was not reached, so this is not catching up") + suite.Require().LessOrEqual(elapsed, tipGrace+unservedHandoverSlack) + }) + } +} + +// TestWaitForSyncHandsOverPastRotatingUnservedClaims checks that unserved far +// claims arriving under a fresh peer ID before the previous one has failed do +// not keep a node at the tip in block sync for good. A node ID costs a peer +// nothing, so a claim record kept per ID must not be the only way out. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverPastRotatingUnservedClaims() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + // a new ID joins every rotateEvery; each fails syncTimeout after joining + rotateEvery = 30 * time.Second + ) + h := suite.newWaitForSyncHarness(height, newPeerData(tipPeer, 1, height-1)) + defer h.cancel() + + traffic := h.tipBlocks(tipPeer, tipBlockInterval) + limit := maxSyncStall + tipGrace + elapsed, caughtUp, ended := h.run(limit, func(elapsed time.Duration) { + traffic(elapsed) + if elapsed%rotateEvery == 0 { + id := types.NodeID(fmt.Sprintf("rotating peer %d", elapsed/rotateEvery)) + h.sync.AddPeer(newPeerData(id, 1, maxPlausiblePeerHeight)) + } + if elapsed >= syncTimeout && (elapsed-syncTimeout)%rotateEvery == 0 { + id := types.NodeID(fmt.Sprintf("rotating peer %d", (elapsed-syncTimeout)/rotateEvery)) + next, _ := h.sync.GetStatus() + h.fail(id, next, client.ErrPeerNotResponded) + } + }) + suite.Require().True(ended, + "block sync still following the tip after %s behind far claims under rotating peer IDs", elapsed) + suite.Require().False(caughtUp) +} + +// TestWaitForSyncKeepsSyncingWhenASlowPeerRecovers checks that a peer far ahead +// which once failed the height we need, and then serves a run of blocks from it, +// keeps block sync going: one timeout from a slow peer does not make its claim +// false, and handing over while thousands of blocks behind leaves the node in +// consensus catch-up it cannot finish (dashpay/tenderdash#1413). +func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsSyncingWhenASlowPeerRecovers() { + ctx := context.Background() + b := suite.newBacklogHarness() + clock := clockwork.NewFakeClock() + b.pool.clock = clock + b.pool.lastAdvance = clock.Now() + for _, peer := range b.pool.peerStore.All() { + b.pool.peerStore.Delete(peer.peerID) + } + const slow = types.NodeID("slow peer") + b.pool.AddPeer(newPeerData(slow, 1, 10000)) + b.pool.AddPeer(newPeerData("level", 0, 0)) + + const run = maxCatchupGap + 1 + first := b.produce(ctx) + suite.Require().NotNil(first) + suite.Require().EqualValues(1, first.Block.Height) + b.timeout(ctx, slow, 1) + responses := map[int64]*BlockResponse{} + for len(responses) < int(run) { + resp := b.produce(ctx) + suite.Require().NotNil(resp) + responses[resp.Block.Height] = resp + } + + h := suite.startWaitForSync(b.pool, clock) + defer h.cancel() + _, _, ended := h.run(3*tipGrace, func(elapsed time.Duration) { + if elapsed == tipGrace/2 { + for height := int64(1); height <= run; height++ { + b.deliver(ctx, responses[height]) + } + b.pool.AddPeer(newPeerData("level", 0, run)) + } + }) + suite.Require().False(ended, "block sync gave up on a peer that served the heights it had failed") + height, _ := b.pool.GetStatus() + suite.Require().EqualValues(run+1, height, "the recovered blocks must have been applied") +} + +// TestWaitForSyncKeepsSyncingBehindASoleSlowPeerThatFailsTwice checks that a sole +// peer thousands of blocks ahead, which times out the height we need and then +// times out the recovery probe for it too, does not make block sync hand over +// while the node is still far behind. Two timeouts are well short of the +// failures that drop a peer, and the peer serves the height afterwards. +func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsSyncingBehindASoleSlowPeerThatFailsTwice() { + ctx := context.Background() + b := suite.newBacklogHarness() + fake := clockwork.NewFakeClock() + b.pool.clock = fake + b.pool.lastAdvance = fake.Now() + for _, peer := range b.pool.peerStore.All() { + b.pool.peerStore.Delete(peer.peerID) + } + const slow = types.NodeID("slow peer") + b.pool.AddPeer(newPeerData(slow, 1, 10000)) + + resp := b.produce(ctx) + suite.Require().NotNil(resp) + suite.Require().EqualValues(1, resp.Block.Height) + b.timeout(ctx, slow, 1) + + h := suite.startWaitForSync(b.pool, fake) + defer h.cancel() + // first timeout at 0; probe 30s after it times out 60s later; served afterwards + const ( + probeTimedOut = 90 * time.Second + served = tipGrace + 30*time.Second + ) + elapsed, _, ended := h.run(3*tipGrace, func(elapsed time.Duration) { + switch elapsed { + case probeTimedOut: + b.timeout(ctx, slow, 1) + case served: + b.deliver(ctx, resp) + } + }) + suite.Require().False(ended, + "block sync handed over after %s while 9999 blocks behind a peer that failed only twice", elapsed) +} + +// TestWaitForSyncKeepsSyncingBehindASlowPeerWithASyncingPeerAlongside is the sole +// slow peer case with one more peer that is itself still syncing at our height, +// as when several nodes start together. That peer has nothing to offer, so it +// must not make the node give up on the slow peer thousands of blocks ahead. +func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsSyncingBehindASlowPeerWithASyncingPeerAlongside() { + ctx := context.Background() + b := suite.newBacklogHarness() + fake := clockwork.NewFakeClock() + b.pool.clock = fake + b.pool.lastAdvance = fake.Now() + for _, peer := range b.pool.peerStore.All() { + b.pool.peerStore.Delete(peer.peerID) + } + const slow = types.NodeID("slow peer") + b.pool.AddPeer(newPeerData(slow, 1, 10000)) + b.pool.AddPeer(newPeerData("syncing peer", 0, 0)) + + resp := b.produce(ctx) + suite.Require().NotNil(resp) + suite.Require().EqualValues(1, resp.Block.Height) + b.timeout(ctx, slow, 1) + + h := suite.startWaitForSync(b.pool, fake) + defer h.cancel() + const ( + probeTimedOut = 90 * time.Second + served = tipGrace + 30*time.Second + ) + elapsed, _, ended := h.run(3*tipGrace, func(elapsed time.Duration) { + switch elapsed { + case probeTimedOut: + b.timeout(ctx, slow, 1) + case served: + b.deliver(ctx, resp) + } + }) + suite.Require().False(ended, + "block sync handed over after %s while 9999 blocks behind, because a syncing peer sat at our height", elapsed) +} + +// TestWaitForSyncHandsOverPastAnUnservedClaimOnAFastChain checks the handover bound +// when the chain commits more than maxCatchupGap blocks between two status rounds, +// so each status briefly shows the other peers more than maxCatchupGap ahead +// until the node applies what they report a second later. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverPastAnUnservedClaimOnAFastChain() { + const ( + height = int64(100) + tip = types.NodeID("tip peer") + unserving = types.NodeID("unserving peer") + perRound = maxCatchupGap + 2 + ) + h := suite.newWaitForSyncHarness(height, + newPeerData(tip, 1, height-1), + newPeerData(unserving, 1, maxPlausiblePeerHeight)) + defer h.cancel() + h.fail(unserving, height, client.ErrPeerNotResponded) + + limit := 10 * time.Minute + elapsed, caughtUp, ended := h.run(limit, func(elapsed time.Duration) { + if elapsed == 0 { + return + } + next, _ := h.sync.GetStatus() + switch elapsed % (10 * time.Second) { + case 0: + // status round: the tip peer reports perRound new blocks + h.sync.AddPeer(newPeerData(tip, 1, next-1+perRound)) + case 2 * time.Second: + for i := int64(0); i < perRound; i++ { + h.sync.advance(h.clock.Now(), defaultPrecision) + } + } + }) + suite.Require().True(ended, "block sync still following the tip after %s behind a claim nobody serves", limit) + suite.Require().False(caughtUp) + suite.Require().LessOrEqual(elapsed, syncTimeout+tipGrace+unservedHandoverSlack) +} + +// TestWaitForSyncHandsOverPastAnUnservedClaimOnAHaltedChain checks that a chain +// that stopped producing blocks, as one waiting for this node's vote may, does not +// leave the node in block sync behind a claim nobody serves. Its last block is +// too old to say the node is at the tip, so the wedge limit decides. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverPastAnUnservedClaimOnAHaltedChain() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + unserving = types.NodeID("unserving peer") + ) + h := suite.newWaitForSyncHarness(height, + newPeerData(tipPeer, 1, height-1), + newPeerData(unserving, 1, maxPlausiblePeerHeight)) + defer h.cancel() + h.setLastBlockTime(h.clock.Now().Add(-tipRecency)) + + // the peer fails the probes it is sent for the height, too seldom to + // be dropped for it, and reports its status again after each + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + const probeEvery = 150 * time.Second + elapsed, caughtUp, ended := h.run(maxSyncStall+time.Minute, func(elapsed time.Duration) { + if elapsed > 0 && elapsed%probeEvery == 0 { + h.fail(unserving, height, client.ErrPeerNotResponded) + h.sync.AddPeer(newPeerData(unserving, 1, maxPlausiblePeerHeight)) + } + }) + suite.Require().True(ended, "block sync kept waiting on a halted chain behind a claim nobody serves") + suite.Require().False(caughtUp) + suite.Require().Greater(elapsed, maxSyncStall, "handed over with no recent block before the wedge limit") + suite.Require().LessOrEqual(elapsed, maxSyncStall+unservedHandoverSlack) +} + +// flakySource returns a step for run in which source, the only peer far ahead, +// supplies the next block every deliverEvery and times out one request every +// failEvery, the way a loaded peer does. The node keeps making progress. +func (h *waitForSyncHarness) flakySource(source types.NodeID, deliverEvery, failEvery time.Duration) func(time.Duration) { + return func(elapsed time.Duration) { + if elapsed == 0 { + return + } + next, _ := h.sync.GetStatus() + if elapsed%failEvery == 0 { + h.fail(source, next, client.ErrPeerNotResponded) + return + } + if elapsed%deliverEvery == 0 { + h.suite.Require().NoError(h.sync.addBlock(BlockResponse{ + PeerID: source, + Block: &types.Block{Header: types.Header{Height: next}}, + })) + // a block the network committed long ago + h.sync.advance(time.Time{}, defaultPrecision) + h.sync.peerStore.Update(source, ResetFailures()) + } + } +} + +// TestWaitForSyncKeepsSyncingFromAFlakySourceWithAPeerSyncingBehind checks that a +// node far behind, making steady progress from its only source ahead, does not +// hand over because that source times out a request now and then while another +// node syncs behind it. +func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsSyncingFromAFlakySourceWithAPeerSyncingBehind() { + const ( + height = int64(100) + source = types.NodeID("source") + behind = types.NodeID("peer syncing behind") + limit = 3 * tipGrace + deliver = 5 * time.Second + failure = 50 * time.Second + ) + h := suite.newWaitForSyncHarness(height, + newPeerData(source, 1, height+10000), + newPeerData(behind, 1, height-5)) + defer h.cancel() + h.fail(source, height, client.ErrPeerNotResponded) + + flaky := h.flakySource(source, deliver, failure) + elapsed, _, ended := h.run(limit, func(elapsed time.Duration) { + flaky(elapsed) + if elapsed%deliver == 0 { + next, _ := h.sync.GetStatus() + h.sync.AddPeer(newPeerData(behind, 1, next-3)) + } + }) + applied, _ := h.sync.GetStatus() + suite.Require().False(ended, + "handed over after %s at height %d, about 10000 blocks behind a source still delivering", elapsed, applied) +} + +// TestWaitForSyncKeepsSyncingFromAFlakySourceThroughADropAndReAdd checks the same +// for a sole source that once fails enough requests in a row to be dropped and +// comes back with its next status. +func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsSyncingFromAFlakySourceThroughADropAndReAdd() { + const ( + height = int64(100) + source = types.NodeID("source") + limit = 3 * tipGrace + deliver = 5 * time.Second + failure = 50 * time.Second + dropAt = tipGrace + 7*time.Second + ) + h := suite.newWaitForSyncHarness(height, newPeerData(source, 1, height+10000)) + defer h.cancel() + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h.fail(source, height, client.ErrPeerNotResponded) + + flaky := h.flakySource(source, deliver, failure) + elapsed, _, ended := h.run(limit, func(elapsed time.Duration) { + switch elapsed { + case dropAt: + next, _ := h.sync.GetStatus() + for i := int64(0); i < int64(maxConsecutiveFailures); i++ { + h.fail(source, next+i, client.ErrPeerNotResponded) + } + return + case dropAt + 5*time.Second: + h.sync.AddPeer(newPeerData(source, 1, height+10000)) + return + } + if _, present := h.sync.peerStore.Get(source); present { + flaky(elapsed) + } + }) + applied, _ := h.sync.GetStatus() + suite.Require().False(ended, + "handed over after %s at height %d, about 10000 blocks behind a source still delivering", elapsed, applied) +} + +// TestWaitForSyncKeepsSyncingWhenTheLastBlockLooksNewerThanTheLocalClock checks +// that a node whose clock runs behind the validators', so that a block it applied +// long ago carries a time ahead of its clock, does not take that block for one at +// the tip and give up on a slow peer thousands of blocks ahead. +func (suite *SynchronizerTestSuite) TestWaitForSyncKeepsSyncingWhenTheLastBlockLooksNewerThanTheLocalClock() { + const ( + height = int64(100) + slow = types.NodeID("slow peer") + ) + h := suite.newWaitForSyncHarness(height, newPeerData(slow, 1, height+10000)) + defer h.cancel() + h.setLastBlockTime(h.clock.Now().Add(30 * time.Minute)) + h.fail(slow, height, client.ErrPeerNotResponded) + + elapsed, _, ended := h.run(3*tipGrace, nil) + suite.Require().False(ended, + "handed over after %s, 10000 blocks behind, on a block time ahead of the local clock", elapsed) +} + +// TestIsRecent checks which last block times count as recent: none older than +// the window, and none further ahead of the local clock than the precision +// validators hold block times to. +func TestIsRecent(t *testing.T) { + const ( + window = time.Minute + precision = time.Second + ) + now := time.Unix(1_700_000_000, 0) + for _, tc := range []struct { + name string + age time.Duration + want bool + }{ + {name: "just applied", age: 0, want: true}, + {name: "as old as the window", age: window, want: true}, + {name: "older than the window", age: window + time.Millisecond}, + {name: "ahead within the precision", age: -precision, want: true}, + {name: "ahead beyond the precision", age: -precision - time.Millisecond}, + } { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, isRecent(now.Add(-tc.age), window, precision, now)) + }) + } + require.False(t, isRecent(time.Time{}, window, precision, now), "no block time") +} + +// TestWaitForSyncAtTipHandoverTargetsNearClaims checks that handing over at the +// tip does not pass on a claim out of reach of the tip as the consensus catch-up +// target, which would hold back the node's own proposals for that long again. +func (suite *SynchronizerTestSuite) TestWaitForSyncAtTipHandoverTargetsNearClaims() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + unserving = types.NodeID("unserving peer") + ) + h := suite.newWaitForSyncHarness(height, + newPeerData(tipPeer, 1, height-1), + newPeerData(unserving, 1, maxPlausiblePeerHeight)) + defer h.cancel() + + _, caughtUp, ended := h.run(tipGrace+time.Minute, h.tipBlocks(tipPeer, tipBlockInterval)) + suite.Require().True(ended) + suite.Require().False(caughtUp) + applied, _ := h.sync.GetStatus() + suite.Require().Equal(applied-1, h.target, "a claim out of reach of the tip must not become the catch-up target") +} + +// TestWaitForSyncHandoverTargetAtTheTip checks that whichever way a node at the +// tip hands over, the consensus catch-up target holds the claims within reach +// of the tip that were seen while waiting, including those of peers gone since, +// and leaves out claims beyond it. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandoverTargetAtTheTip() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + claimant = types.NodeID("claimant") + nearPeer = types.NodeID("near peer") + nearClaim = height + 3 + ) + testCases := []struct { + name string + peers []PeerData + setup func(h *waitForSyncHarness) + step func(h *waitForSyncHarness, elapsed time.Duration) + want int64 + // caughtUp is set where no claim above the node remains + caughtUp bool + }{ + { + name: "the far claimant has no block for the next height", + peers: []PeerData{newPeerData(tipPeer, 1, height-1), newPeerData(claimant, 1, maxPlausiblePeerHeight)}, + setup: func(h *waitForSyncHarness) { + h.fail(claimant, height, &client.ErrBlockNotFound{Height: height}) + }, + want: height - 1, + }, + { + name: "the far claimant is removed for failing", + peers: []PeerData{newPeerData(tipPeer, 1, height-1), newPeerData(claimant, 1, maxPlausiblePeerHeight)}, + step: func(h *waitForSyncHarness, elapsed time.Duration) { + if elapsed == 5*time.Second { + for i := int64(0); i < int64(maxConsecutiveFailures); i++ { + h.fail(claimant, height+i, client.ErrPeerNotResponded) + } + } + }, + want: height - 1, + caughtUp: true, + }, + { + name: "a near peer leaves without failing", + peers: []PeerData{ + newPeerData(nearPeer, 1, nearClaim), + newPeerData(claimant, 1, maxPlausiblePeerHeight), + }, + step: func(h *waitForSyncHarness, elapsed time.Duration) { + if elapsed == 5*time.Second { + h.sync.RemovePeer(nearPeer) + } + }, + want: nearClaim, + }, + } + for _, tc := range testCases { + suite.Run(tc.name, func() { + suite.client.On("Send", mock.Anything, mock.AnythingOfType("p2p.PeerError")).Maybe().Return(nil) + h := suite.newWaitForSyncHarness(height, tc.peers...) + defer h.cancel() + if tc.setup != nil { + tc.setup(h) + } + _, caughtUp, ended := h.run(tipGrace+time.Minute, func(elapsed time.Duration) { + if tc.step != nil { + tc.step(h, elapsed) + } + }) + suite.Require().True(ended) + suite.Require().Equal(tc.caughtUp, caughtUp) + suite.Require().Equal(tc.want, h.target) + }) + } +} + +// newPeerEveryMinute returns a step for run that, every minute, drops the peer +// claiming a far height and connects a new one making the same claim. Node IDs +// cost nothing, so no new peer has a history to judge it by. +func (h *waitForSyncHarness) newPeerEveryMinute() func(time.Duration) { + var current types.NodeID + return func(elapsed time.Duration) { + if elapsed%time.Minute != 0 { + return + } + if current != "" { + h.sync.RemovePeer(current) + } + current = types.NodeID(fmt.Sprintf("claimant %d", elapsed/time.Minute)) + h.sync.AddPeer(newPeerData(current, 1, maxPlausiblePeerHeight)) + } +} + +// TestWaitForSyncHandsOverPastAFarClaimFromNewPeers checks that a far claim made +// by a new peer every minute does not keep a node level with the network in +// block sync while the chain keeps committing blocks. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverPastAFarClaimFromNewPeers() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + ) + h := suite.newWaitForSyncHarness(height, newPeerData(tipPeer, 1, height-1)) + defer h.cancel() + + traffic := h.tipBlocks(tipPeer, tipBlockInterval) + claims := h.newPeerEveryMinute() + elapsed, caughtUp, ended := h.run(tipGrace+time.Minute, func(elapsed time.Duration) { + claims(elapsed) + traffic(elapsed) + }) + suite.Require().True(ended, "block sync kept following the tip behind a claim moved to a new peer every minute") + suite.Require().False(caughtUp) + suite.Require().LessOrEqual(elapsed, tipGrace+unservedHandoverSlack) +} + +// TestWaitForSyncHandsOverPastAFarClaimFromNewPeersOnAHaltedChain checks the same +// on a chain that stopped producing blocks: the stall backstop still fires. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverPastAFarClaimFromNewPeersOnAHaltedChain() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + ) + h := suite.newWaitForSyncHarness(height, newPeerData(tipPeer, 1, height-1)) + defer h.cancel() + h.setLastBlockTime(h.clock.Now().Add(-staleBlockAge / 2)) + + elapsed, caughtUp, ended := h.run(maxSyncStall+time.Minute, h.newPeerEveryMinute()) + suite.Require().True(ended, "block sync kept waiting on a halted chain behind a claim moved to a new peer every minute") + suite.Require().False(caughtUp) + suite.Require().Greater(elapsed, maxSyncStall) + suite.Require().LessOrEqual(elapsed, maxSyncStall+unservedHandoverSlack) +} + +// TestWaitForSyncHandsOverPastAFarClaimFromAPeerSupplyingTheTip checks that a peer +// claiming a far height does not keep the node in block sync by supplying each +// new block as the network commits it. It never fails a request, yet its blocks +// only ever take the node to the network's head. +func (suite *SynchronizerTestSuite) TestWaitForSyncHandsOverPastAFarClaimFromAPeerSupplyingTheTip() { + const ( + height = int64(100) + tipPeer = types.NodeID("tip peer") + relay = types.NodeID("relay") + ) + h := suite.newWaitForSyncHarness(height, + newPeerData(tipPeer, 1, height-1), + newPeerData(relay, 1, maxPlausiblePeerHeight)) + defer h.cancel() + + elapsed, caughtUp, ended := h.run(tipGrace+time.Minute, func(elapsed time.Duration) { + if elapsed == 0 || elapsed%tipBlockInterval != 0 { + return + } + next, _ := h.sync.GetStatus() + h.sync.AddPeer(newPeerData(tipPeer, 1, next)) + h.suite.Require().NoError(h.sync.addBlock(BlockResponse{ + PeerID: relay, + Block: &types.Block{Header: types.Header{Height: next}}, + })) + h.sync.advance(h.clock.Now(), defaultPrecision) + h.sync.peerStore.Update(relay, ResetFailures()) + }) + suite.Require().True(ended, "block sync kept following the tip behind a claim of a peer supplying the tip") + suite.Require().False(caughtUp) + suite.Require().LessOrEqual(elapsed, tipGrace+unservedHandoverSlack) +} + +// TestWaitForSyncOnAFastChainBehindTheTip checks where the at-tip rule draws +// the line on a chain committing two blocks a second, for a node behind a +// source that supplies blocks no faster than the chain commits them, whatever +// another peer claims. While its last block stays older than tipRecency the +// node keeps syncing. Once that block has been within tipRecency for tipGrace +// the node hands over, up to tipRecency of blocks behind: no peer serves it +// faster than the chain moves, so block sync would not close the gap either. +func (suite *SynchronizerTestSuite) TestWaitForSyncOnAFastChainBehindTheTip() { + const ( + height = int64(100) + source = types.NodeID("source") + interval = 500 * time.Millisecond + ) + testCases := []struct { + name string + // behind is how old the node's last block is when block sync starts + behind time.Duration + // supplyEvery is how often the source supplies the next block + supplyEvery time.Duration + // failEvery, when set, is how often the source, then the only peer, + // fails a request + failEvery time.Duration + wantHandover bool + }{ + {name: "at the chain's pace, out of the window", behind: tipRecency + time.Minute, supplyEvery: interval}, + {name: "falling behind a slow source", behind: tipRecency - time.Minute, supplyEvery: 15 * time.Second}, + { + name: "a sole source that fails now and then, out of the window", + behind: tipRecency + time.Minute, + supplyEvery: interval, + failEvery: 50 * time.Second, + }, + {name: "at the chain's pace, within the window", behind: 100 * time.Second, supplyEvery: interval, wantHandover: true}, + } + for _, tc := range testCases { + suite.Run(tc.name, func() { + peers := []PeerData{newPeerData(source, 1, height+10000)} + if tc.failEvery == 0 { + peers = append(peers, newPeerData("claimant", 1, maxPlausiblePeerHeight)) + } + h := suite.newWaitForSyncHarness(height, peers...) + defer h.cancel() + last := h.clock.Now().Add(-tc.behind) + h.setLastBlockTime(last) + supply := func() { + next, _ := h.sync.GetStatus() + h.suite.Require().NoError(h.sync.addBlock(BlockResponse{ + PeerID: source, + Block: &types.Block{Header: types.Header{Height: next}}, + })) + h.sync.advance(last.Add(time.Duration(next-height+1)*interval), defaultPrecision) + h.sync.peerStore.Update(source, ResetFailures()) + } + elapsed, caughtUp, ended := h.run(maxSyncStall/2, func(elapsed time.Duration) { + if tc.failEvery > 0 && elapsed > 0 && elapsed%tc.failEvery == 0 { + next, _ := h.sync.GetStatus() + h.fail(source, next, client.ErrPeerNotResponded) + return + } + if tc.supplyEvery < time.Second { + for i := 0; i < int(time.Second/tc.supplyEvery); i++ { + supply() + } + } else if elapsed%tc.supplyEvery == 0 { + supply() + } + }) + applied, _ := h.sync.GetStatus() + if !tc.wantHandover { + suite.Require().False(ended, "handed over after %s at height %d, more than %s of blocks behind", + elapsed, applied, tipRecency) + return + } + suite.Require().True(ended, "still in block sync after %s with a last block %s old", elapsed, tc.behind) + suite.Require().False(caughtUp) + suite.Require().Greater(elapsed, tipGrace) + suite.Require().LessOrEqual(elapsed, tipGrace+unservedHandoverSlack) + }) + } +} + // backlogChainLen is how many heights the backlog tests have blocks for. It is // wider than the pending window so that a run which ignores the window keeps // going well past it instead of stopping for want of a block to serve. @@ -2320,9 +3178,11 @@ func (suite *SynchronizerTestSuite) TestSmallBlocksAfterSlowApplyKeepPeersUsable } fakeClock.Advance(syncTimeout + time.Second) - height, stalled, servable, tip, _, servableTip := h.pool.stallSnapshot() - suite.Equal(keepSyncing, stallVerdictFor(servable, servableTip-height, stalled), - "healthy peers must not trigger consensus handoff at height %d below tip %d", height, tip) + st := h.pool.stallSnapshot() + height := st.height + suite.Equal(keepSyncing, + stallVerdictFor(st.servable, st.maxServableHeight-height, st.stalledFor, 0), + "healthy peers must not trigger consensus handoff at height %d below tip %d", height, st.maxPeerHeight) _, found := h.pool.peerStore.FindPeer(height) suite.True(found, "healthy peers must accept the next block request after the batch drains") if found {