diff --git a/docs/nodes/metrics.md b/docs/nodes/metrics.md index 564ed762a..bd2da1823 100644 --- a/docs/nodes/metrics.md +++ b/docs/nodes/metrics.md @@ -43,7 +43,7 @@ The following metrics are available: | consensus_step_duration | Histogram | step | Histogram of durations for each step in the consensus protocol | | consensus_block_gossip_receive_latency | Histogram | | Histogram of time taken to receive a block in seconds, measure between when a new block is first discovered to when the block is completed | | consensus_block_gossip_parts_received | Counter | matches_current | Number of block parts received by the node | -| consensus_block_sync_apply_stage_duration | Histogram | stage | Time a block spent in each stage of the block sync applier, in ms: partset, verify_commit, verify_block, save, exec, and wait (idle time between blocks) | +| consensus_block_sync_apply_stage_duration | Histogram | stage | Time a block spent in each stage of the block sync applier, in ms: decode, partset, verify_commit, verify_block, save, exec, and wait (idle time between blocks) | | consensus_quorum_prevote_delay | Gauge | | Interval in seconds between the proposal timestamp and the timestamp of the earliest prevote that achieved a quorum | | consensus_full_prevote_delay | Gauge | | Interval in seconds between the proposal timestamp and the timestamp of the latest prevote in a round where all validators voted | | consensus_proposal_timestamp_difference | Histogram | | Difference between the timestamp in the proposal message and the local time of the validator at the time it received the message | diff --git a/internal/blocksync/applier.go b/internal/blocksync/applier.go index 66e7c61fa..ace4593a2 100644 --- a/internal/blocksync/applier.go +++ b/internal/blocksync/applier.go @@ -69,29 +69,71 @@ func newBlockApplier(blockExec sm.Executor, store sm.BlockStore, opts ...applier return applier } -// Apply safely verifies, saves to the store and executes a block with commit -func (e *blockApplier) Apply(ctx context.Context, block *types.Block, commit *types.Commit) error { +// Apply safely verifies, saves to the store and executes a block with commit. +// blockBytes is the block as it was committed, the bytes block was decoded +// from. +func (e *blockApplier) Apply( + ctx context.Context, + block *types.Block, + blockBytes []byte, + commit *types.Commit, +) error { e.mtx.Lock() defer e.mtx.Unlock() + e.observeWait() defer func() { e.lastDone = time.Now() }() + return e.apply(ctx, block, blockBytes, commit, 0) +} - // Time between the end of the previous apply and the start of this one. With - // a fast application this is what the sync rate is actually limited by, and - // it is block fetching, not block execution. +// ApplyBytes is Apply for a block held only as the bytes it was committed with: +// it decodes the block from them first. Block sync keeps pending blocks in that +// form, so that a response waiting for a lower height does not hold a decoded +// copy beside its bytes. +func (e *blockApplier) ApplyBytes(ctx context.Context, blockBytes []byte, commit *types.Commit) error { + e.mtx.Lock() + defer e.mtx.Unlock() + e.observeWait() + defer func() { e.lastDone = time.Now() }() + + start := time.Now() + block, err := decodeBlock(blockBytes) + decodeTime := e.observeSince("decode", start) + if err != nil { + return err + } + return e.apply(ctx, block, blockBytes, commit, decodeTime) +} + +// observeWait records the time between the end of the previous apply and the +// start of this one. With a fast application this is what the sync rate is +// actually limited by, and it is block fetching, not block execution. +func (e *blockApplier) observeWait() { if !e.lastDone.IsZero() { e.observeSince("wait", e.lastDone) } +} +// apply is Apply with the lock held. decodeTime is how long decoding block +// took, counted with building its part set in the per-block timings. +func (e *blockApplier) apply( + ctx context.Context, + block *types.Block, + blockBytes []byte, + commit *types.Commit, + decodeTime time.Duration, +) error { // The part set is needed twice: to derive the block ID and to persist the - // block. Building it serializes the whole block and builds a Merkle tree over - // the parts, so build it once and pass it to both. + // block. Building it builds a Merkle tree over the parts, so build it once + // and pass it to both. It is cut from the bytes the block was committed + // with, not from block: the commit's part set header is computed over those + // bytes, and a re-encoding of block drops unknown fields and non-canonical + // encoding. Only the standard cut is rebuilt, the one honest proposers use. + // Consensus does not require it yet, and a block committed with another cut + // fails the commit check whichever peer serves it. start := time.Now() - blockParts, err := block.MakePartSet(types.BlockPartSizeBytes) - if err != nil { - return err - } + blockParts := types.NewPartSetFromData(blockBytes, types.BlockPartSizeBytes) blockID := block.BlockID(blockParts) - partSetTime := e.observeSince("partset", start) + partSetTime := decodeTime + e.observeSince("partset", start) start = time.Now() verified, err := e.verify(ctx, blockID, block, commit) @@ -137,8 +179,8 @@ func (e *blockApplier) Apply(ctx context.Context, block *types.Block, commit *ty e.metrics.ObserveBlockSyncStage("exec", execTime) e.stats.add(partSetTime, verifyTime, saveTime, execTime) - // ByteSize is the size of the serialized block we just built, so the metric - // costs nothing extra here + // ByteSize is the length of the bytes the part set was cut from, so the + // metric costs nothing extra here e.metrics.RecordConsMetrics(block, blockParts.ByteSize()) return nil } @@ -219,7 +261,8 @@ func (e *blockApplier) observeSince(stage string, start time.Time) time.Duration // applyTimings is the average time a single block spends in each stage of the // apply pipeline. type applyTimings struct { - // PartSet is block serialization plus the Merkle tree over its parts + // PartSet is decoding the block, when it is held as bytes, and the Merkle + // tree over its parts PartSet time.Duration // Verify is commit signature verification and block validation Verify time.Duration @@ -231,8 +274,8 @@ type applyTimings struct { // applyStats accumulates per-stage timings of the block apply pipeline. Block // sync applies blocks one at a time, so this is where a slow sync shows up; the -// breakdown attributes it to serialization, signature verification, disk or the -// ABCI application without needing a profiler. +// breakdown attributes it to building the part set, signature verification, +// disk or the ABCI application without needing a profiler. type applyStats struct { mtx sync.Mutex blocks int64 diff --git a/internal/blocksync/applier_test.go b/internal/blocksync/applier_test.go index 31333ac05..fea9fa172 100644 --- a/internal/blocksync/applier_test.go +++ b/internal/blocksync/applier_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/cosmos/gogoproto/proto" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -27,6 +28,7 @@ import ( "github.com/dashpay/tenderdash/internal/test/metricspy" tmrequire "github.com/dashpay/tenderdash/internal/test/require" "github.com/dashpay/tenderdash/libs/log" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -47,7 +49,7 @@ func TestBlockApplierChecksAppResponseBeforeSave(t *testing.T) { executor := sm.NewBlockExecutor(stateStore, client, nil, sm.EmptyEvidencePool{}, blockStore, nil) applier := newBlockApplier(executor, blockStore, applierWithState(initialState)) - require.Panics(t, func() { _ = applier.Apply(ctx, block, commit) }) + require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), commit) }) require.Zero(t, blockStore.Height(), "an inconsistent app response must not advance the block store") require.Equal(t, initialState.LastBlockHeight, applier.State().LastBlockHeight) loaded, err := stateStore.Load() @@ -174,7 +176,7 @@ func TestBlockApplierApply(t *testing.T) { tc.mockFn() } fn := func() { - err := applier.Apply(ctx, tc.block, tc.commit) + err := applier.Apply(ctx, tc.block, blockBytes(t, tc.block), tc.commit) tmrequire.Error(t, tc.wantErr, err) } if tc.wantPanic { @@ -267,7 +269,7 @@ func TestBlockApplierDoesNotSaveBlockRejectedByApp(t *testing.T) { Return(sm.CurrentRoundState{}, errors.New("app rejected the block")) applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState)) - require.Panics(t, func() { _ = applier.Apply(ctx, block, commit) }) + require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), commit) }) } // TestBlockApplierSavesBlockBeforeFinalize pins the order the handshake relies @@ -313,7 +315,7 @@ func TestBlockApplierSavesBlockBeforeFinalize(t *testing.T) { Return(state, nil, nil) applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState)) - require.NoError(t, applier.Apply(ctx, block, commit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), commit)) require.Equal(t, []string{"process", "verify", "save", "finalize"}, calls) } @@ -349,7 +351,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState), applierWithMetrics(m)) - require.NoError(t, applier.Apply(ctx, blockH1, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1)) for _, stage := range []string{"partset", "verify_commit", "verify_block", "save", "exec"} { require.Len(t, hist.Samples[stage], 1, "stage %q after the first apply", stage) @@ -360,7 +362,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { // A failed attempt must advance the idle-time boundary as well, so a retry // does not count the failed attempt's wait and verification time again. failureStart := time.Now() - require.Error(t, applier.Apply(ctx, blockH1, new(types.Commit))) + require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), new(types.Commit))) failureEnd := time.Now() failureDone := applier.lastDone require.False(t, failureDone.Before(failureStart), "failed apply must advance lastDone") @@ -369,7 +371,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { // The retry measures only the idle time after the failed attempt. retryStart := time.Now() - require.NoError(t, applier.Apply(ctx, blockH1, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1)) retryEnd := time.Now() require.Len(t, hist.Samples["wait"], 2) wait := hist.Samples["wait"][1] @@ -400,7 +402,7 @@ func TestBlockApplierVerifyFailureTimesOnlyTheCheckThatRan(t *testing.T) { // A rejected commit must stop before ValidateBlock is reached. mockBlockExec.On("VerifyCommit", initialState, blockH1.BlockID(nil), blockH1.Height, new(types.Commit)). Once().Return(types.VerifiedCommit{}, errors.New("bad signature")) - require.Error(t, applier.Apply(ctx, blockH1, new(types.Commit))) + require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), new(types.Commit))) require.Len(t, hist.Samples["partset"], 1) require.Len(t, hist.Samples["verify_commit"], 1) @@ -444,7 +446,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { blockExec.On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, blockH1ID, blockH1, commitH1, none). Once().Return(initialState, nil, nil) applier := newBlockApplier(blockExec, blockStore, applierWithState(initialState)) - require.NoError(t, applier.Apply(ctx, blockH1, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1)) return applier, blockExec } // expectH2 expects block 2 to be validated and applied offering lastCommit @@ -460,7 +462,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { t.Run("the next block is offered the previous commit's verification", func(t *testing.T) { applier, blockExec := applyH1(t) expectH2(blockExec, verifiedH1) - require.NoError(t, applier.Apply(ctx, blockH2, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2)) }) t.Run("a rejected commit does not replace the verification", func(t *testing.T) { @@ -468,17 +470,17 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { bad := new(types.Commit) blockExec.On("VerifyCommit", mock.Anything, blockH2ID, blockH2.Height, bad). Once().Return(none, errors.New("bad signature")) - require.Error(t, applier.Apply(ctx, blockH2, bad)) + require.Error(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), bad)) expectH2(blockExec, verifiedH1) - require.NoError(t, applier.Apply(ctx, blockH2, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2)) }) t.Run("replacing the state discards the verification", func(t *testing.T) { applier, blockExec := applyH1(t) applier.UpdateState(initialState) expectH2(blockExec, none) - require.NoError(t, applier.Apply(ctx, blockH2, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2)) }) } @@ -532,9 +534,91 @@ func TestBlockApplierSkipsTheLastCommitItVerified(t *testing.T) { wantSkipped float64 }{{1, 0}, {2, 3}, {3, 6}} { block, _, _, seenCommit := makeNextBlock(ctx, t, applier.State(), privVals[0], step.height, commit) - require.NoError(t, applier.Apply(ctx, block, seenCommit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), seenCommit)) require.Equal(t, step.wantSkipped, skipped.Value(), "LastCommit threshold verifications skipped after applying height %d", step.height) commit = seenCommit } } + +// blockBytes returns the bytes block is committed with when its proposer +// encodes it the canonical way, the bytes a peer serves it as. +func blockBytes(t *testing.T, block *types.Block) []byte { + t.Helper() + pbb, err := block.ToProto() + require.NoError(t, err) + bz, err := proto.Marshal(pbb) + require.NoError(t, err) + return bz +} + +// finalizeBlockIDApp records the block IDs FinalizeBlock is called with. +type finalizeBlockIDApp struct { + abci.BaseApplication + blockIDs []*tmproto.BlockID +} + +func (app *finalizeBlockIDApp) FinalizeBlock( + ctx context.Context, + req *abci.RequestFinalizeBlock, +) (*abci.ResponseFinalizeBlock, error) { + app.blockIDs = append(app.blockIDs, req.BlockID) + return app.BaseApplication.FinalizeBlock(ctx, req) +} + +// TestFinalizeBlockGetsCommittedBlockID checks that the application is given +// the block ID that was committed, when applying a block and replaying it, for +// a block committed with bytes that a re-encoding does not reproduce. +func TestFinalizeBlockGetsCommittedBlockID(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := log.NewNopLogger() + genDoc, privVals := factory.RandGenesisDoc(1, factory.ConsensusParams()) + state, err := sm.MakeGenesisState(genDoc) + require.NoError(t, err) + stateStore := sm.NewStore(dbm.NewMemDB()) + require.NoError(t, stateStore.Save(state)) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + + finalizeApp := &finalizeBlockIDApp{} + localClient := abciclient.NewLocalClient(logger, finalizeApp) + app := proxy.New(localClient, logger, proxy.NopMetrics()) + require.NoError(t, app.Start(ctx)) + eventBus := eventbus.NewDefault(logger) + require.NoError(t, eventBus.Start(ctx)) + mp := &mpmocks.Mempool{} + mp.On("Lock").Return() + mp.On("Unlock").Return() + mp.On("FlushAppConn", mock.Anything).Return(nil) + mp.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything, mock.Anything).Return(nil) + blockExec := sm.NewBlockExecutor(stateStore, app, mp, sm.EmptyEvidencePool{}, blockStore, eventBus) + applier := newBlockApplier(blockExec, blockStore, applierWithState(state)) + + block, blockID, partSet, seenCommit := makeNextBlockWithBytes(ctx, t, applier.State(), privVals[0], 1, + types.NewCommit(0, 0, types.BlockID{}, nil, nil), withUnknownField) + require.False(t, blockID.Equals(block.BlockID(nil)), "the block must not be one a re-encoding reproduces") + require.NoError(t, applier.Apply(ctx, block, loadBlockBytes(storedParts(t, partSet), 1), seenCommit)) + + want := seenCommit.BlockID.ToProto() + require.Equal(t, []*tmproto.BlockID{&want}, finalizeApp.blockIDs, "applying the block") + + _, err = sm.ExecReplayedCommitBlock(ctx, localClient, block, seenCommit, logger) + require.NoError(t, err) + require.Equal(t, []*tmproto.BlockID{&want, &want}, finalizeApp.blockIDs, "replaying the block") +} + +// storedParts returns a store that holds partSet as the parts of the block at +// height 1. +func storedParts(t *testing.T, partSet *types.PartSet) sm.BlockStore { + t.Helper() + blockStore := mocks.NewBlockStore(t) + blockStore.On("LoadBlockMeta", int64(1)).Return(&types.BlockMeta{ + BlockID: types.BlockID{PartSetHeader: partSet.Header()}, + BlockSize: int(partSet.ByteSize()), + }) + for i := range int(partSet.Total()) { + blockStore.On("LoadBlockPart", int64(1), i).Return(partSet.GetPart(i)) + } + return blockStore +} diff --git a/internal/blocksync/block_fetch_job.go b/internal/blocksync/block_fetch_job.go index 7c58f5255..1a88fb4d1 100644 --- a/internal/blocksync/block_fetch_job.go +++ b/internal/blocksync/block_fetch_job.go @@ -54,9 +54,9 @@ func blockFetchJobHandler(blockClient client.BlockClient, peer PeerData, height if err != nil { return errorResult(responder, height, err) } - if resp.Block.Height != height { + if resp.Height != height { return errorResult(responder, height, - fmt.Errorf("peer sent block at height %d, requested height %d", resp.Block.Height, height)) + fmt.Errorf("peer sent block at height %d, requested height %d", resp.Height, height)) } resp.ConnID = responder.connID size = protoResp.Size() diff --git a/internal/blocksync/block_fetch_job_test.go b/internal/blocksync/block_fetch_job_test.go index 17bbb2385..d1aa6ba94 100644 --- a/internal/blocksync/block_fetch_job_test.go +++ b/internal/blocksync/block_fetch_job_test.go @@ -159,8 +159,13 @@ func TestAttributedConn(t *testing.T) { } func TestBlockResponseValidate(t *testing.T) { - block := func(h int64) *types.Block { - return &types.Block{Header: types.Header{Height: h}} + // a block at height h, as a response holds it + block := func(h int64) BlockResponse { + return BlockResponse{Height: h, BlockBytes: blockBytes(t, &types.Block{Header: types.Header{Height: h}})} + } + withCommit := func(resp BlockResponse, commit *types.Commit) BlockResponse { + resp.Commit = commit + return resp } commit := func(h int64) *types.Commit { return &types.Commit{Height: h} @@ -182,17 +187,17 @@ func TestBlockResponseValidate(t *testing.T) { }, { name: "block, nil commit", - resp: BlockResponse{Block: block(10)}, + resp: block(10), wantErr: "a block without a commit at height 10", }, { name: "height mismatch", - resp: BlockResponse{Block: block(10), Commit: commit(11)}, + resp: withCommit(block(10), commit(11)), wantErr: "heights don't match", }, { name: "valid", - resp: BlockResponse{Block: block(10), Commit: commit(10)}, + resp: withCommit(block(10), commit(10)), }, } for _, tc := range testCases { @@ -269,8 +274,8 @@ func (suite *BlockFetchJobTestSuite) TestGeneratorNextJobWaitForPeerAndPushBackH resp2 := res2.Value.(*BlockResponse) return suite.Equal(suite.peer.peerID, resp1.PeerID) && suite.Equal(suite.peer.peerID, resp2.PeerID) && - suite.Contains([]int64{5, 9}, resp1.Block.Height) && - suite.Contains([]int64{5, 9}, resp2.Block.Height) + suite.Contains([]int64{5, 9}, resp1.Height) && + suite.Contains([]int64{5, 9}, resp2.Height) }, 10*time.Millisecond, 5*time.Millisecond) } diff --git a/internal/blocksync/commit_extensions_test.go b/internal/blocksync/commit_extensions_test.go index 724751c98..788dfbf02 100644 --- a/internal/blocksync/commit_extensions_test.go +++ b/internal/blocksync/commit_extensions_test.go @@ -52,11 +52,11 @@ func TestBlockApplierRejectsCommitExtensionsAndRetries(t *testing.T) { metrics := consensus.NopMetrics() metrics.CommitVerifyFailures = counter applier := newBlockApplier(exec, store, applierWithState(initial), applierWithMetrics(metrics)) - require.Error(t, applier.Apply(ctx, block, commit)) + require.Error(t, applier.Apply(ctx, block, blockBytes(t, block), commit)) require.Equal(t, float64(1), counter.Value()) require.Equal(t, []string{"process", "verify"}, calls) require.Equal(t, initial.LastBlockHeight, applier.State().LastBlockHeight) - require.NoError(t, applier.Apply(ctx, block, commit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), commit)) require.Equal(t, float64(1), counter.Value()) require.Equal(t, []string{"process", "verify", "verify", "save", "finalize"}, calls) } @@ -81,7 +81,7 @@ func TestBlockApplierInvalidatesRejectedProposal(t *testing.T) { exec.On("VerifyVoteExtension", mock.Anything, mock.Anything). Twice().Return(errors.New("invalid vote extension")) applier := newBlockApplier(exec, store, applierWithState(initial)) - require.ErrorIs(t, applier.Apply(ctx, block, commit), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), commit), sm.ErrCommitExtensionsRejected) switch change { case "block": block.AppHash = append(block.AppHash.Copy(), 1) @@ -90,7 +90,7 @@ func TestBlockApplierInvalidatesRejectedProposal(t *testing.T) { case "state": applier.UpdateState(initial) } - require.ErrorIs(t, applier.Apply(ctx, block, commit), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), commit), sm.ErrCommitExtensionsRejected) }) } } @@ -135,9 +135,9 @@ func TestBlockApplierReprocessesRoundAfterAnother(t *testing.T) { applier := newBlockApplier(exec, store, applierWithState(initial)) for _, round := range []int32{1, 1, 2} { - require.ErrorIs(t, applier.Apply(ctx, block, commitAt(round)), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), commitAt(round)), sm.ErrCommitExtensionsRejected) } - require.NotPanics(t, func() { require.NoError(t, applier.Apply(ctx, block, commitAt(1))) }) + require.NotPanics(t, func() { require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), commitAt(1))) }) require.Equal(t, []int32{1, 2, 1}, rounds, "a round is processed again only after another one") } diff --git a/internal/blocksync/p2p_msg_handler.go b/internal/blocksync/p2p_msg_handler.go index 9c24512a0..5e8b6c024 100644 --- a/internal/blocksync/p2p_msg_handler.go +++ b/internal/blocksync/p2p_msg_handler.go @@ -155,11 +155,11 @@ func (h *blockP2PMessageHandler) Handle(ctx context.Context, p2pClient *client.C func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (proto.Message, bool, error) { envelope := &job.envelope msg := envelope.Message.(*bcproto.BlockRequest) - block := h.store.LoadBlock(msg.Height) + blockBytes := loadBlockBytes(h.store, msg.Height) if !h.connectionIsCurrent(envelope.From, envelope.ConnID) { return nil, false, nil } - if block == nil { + if blockBytes == nil { h.logger.Debug("peer requesting a block we do not have", "peer", envelope.From, "height", msg.Height) return &bcproto.NoBlockResponse{Height: msg.Height}, false, nil } @@ -167,16 +167,39 @@ func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (pro if commit == nil { return nil, false, fmt.Errorf("%w at height %d", errStoredBlockMissingCommit, msg.Height) } - blockProto, err := block.ToProto() - if err != nil { - return nil, false, fmt.Errorf("failed to convert block to protobuf: %w", err) - } return &bcproto.BlockResponse{ - Block: blockProto, + Block: blockBytes, Commit: commit.ToProto(), }, true, nil } +// loadBlockBytes returns the bytes of the stored parts of the block at height, +// in order, or nil when the store does not hold every part of it. The commit's +// part set header is computed over those bytes, and a re-encoding of the +// decoded block need not reproduce them: it drops unknown fields and +// non-canonical encoding. +func loadBlockBytes(store sm.BlockStore, height int64) []byte { + meta := store.LoadBlockMeta(height) + if meta == nil { + return nil + } + // BlockSize is -1 for a header saved without its block + blockBytes := make([]byte, 0, max(meta.BlockSize, 0)) + for i := range int(meta.BlockID.PartSetHeader.Total) { + part := store.LoadBlockPart(height, i) + // a header saved without its block, or parts pruned after the meta was + // loaded + if part == nil { + return nil + } + blockBytes = append(blockBytes, part.Bytes...) + } + if len(blockBytes) == 0 { + return nil + } + return blockBytes +} + func (h *blockP2PMessageHandler) enqueueBlockResponse(job blockResponseJob) { peerID := job.envelope.From h.responseMtx.Lock() diff --git a/internal/blocksync/p2p_msg_handler_test.go b/internal/blocksync/p2p_msg_handler_test.go index 365c74627..2bd98dcb4 100644 --- a/internal/blocksync/p2p_msg_handler_test.go +++ b/internal/blocksync/p2p_msg_handler_test.go @@ -2,13 +2,17 @@ package blocksync import ( "context" + "encoding/binary" "fmt" + "runtime" + "slices" "testing" "time" "github.com/cosmos/gogoproto/proto" "github.com/google/uuid" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/dashpay/tenderdash/config" @@ -16,6 +20,10 @@ import ( "github.com/dashpay/tenderdash/internal/p2p/client" p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" "github.com/dashpay/tenderdash/internal/state/mocks" + statefactory "github.com/dashpay/tenderdash/internal/state/test/factory" + "github.com/dashpay/tenderdash/internal/store" + "github.com/dashpay/tenderdash/internal/test/factory" + dbm "github.com/dashpay/tenderdash/libs/db" "github.com/dashpay/tenderdash/libs/log" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" "github.com/dashpay/tenderdash/types" @@ -62,9 +70,6 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { ctx := context.Background() peerID := types.NodeID("peer") const H1001 = int64(1001) - blockH1001 := types.Block{Header: types.Header{Height: 1001}} - protoBlockH1001, err := blockH1001.ToProto() - suite.Require().NoError(err) commit1001 := types.Commit{Height: 1001} protoCommit := commit1001.ToProto() blockRequestH1001 := &bcproto.BlockRequest{Height: H1001} @@ -75,7 +80,7 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { { mockFn: func() { suite.fakeStore. - On("LoadBlock", H1001). + On("LoadBlockMeta", H1001). Once(). Return(nil) }, @@ -83,10 +88,7 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { }, { mockFn: func() { - suite.fakeStore. - On("LoadBlock", H1001). - Once(). - Return(&blockH1001) + suite.expectStoredBlock(H1001, 1) suite.fakeStore. On("LoadSeenCommitAt", H1001). Once(). @@ -95,17 +97,14 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { }, { mockFn: func() { - suite.fakeStore. - On("LoadBlock", H1001). - Once(). - Return(&blockH1001) + suite.expectStoredBlock(H1001, 1) suite.fakeStore. On("LoadSeenCommitAt", H1001). Once(). Return(&commit1001) }, wantResp: &bcproto.BlockResponse{ - Block: protoBlockH1001, + Block: suite.storedBlockBytes(H1001), Commit: protoCommit, }, }, @@ -128,10 +127,27 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { } } +// expectStoredBlock makes the store serve the block at height, in one part, +// times times. It returns the call that loads the block's meta. +func (suite *BlockP2PMessageHandlerTestSuite) expectStoredBlock(height int64, times int) *mock.Call { + blockBytes := suite.storedBlockBytes(height) + meta := &types.BlockMeta{ + BlockID: types.BlockID{PartSetHeader: types.PartSetHeader{Total: 1}}, + BlockSize: len(blockBytes), + } + suite.fakeStore.On("LoadBlockPart", height, 0).Times(times).Return(&types.Part{Bytes: blockBytes}) + return suite.fakeStore.On("LoadBlockMeta", height).Times(times).Return(meta) +} + +// storedBlockBytes returns the bytes of the block expectStoredBlock stores at +// height. +func (suite *BlockP2PMessageHandlerTestSuite) storedBlockBytes(height int64) []byte { + return blockBytes(suite.T(), &types.Block{Header: types.Header{Height: height}}) +} + func (suite *BlockP2PMessageHandlerTestSuite) TestServeBlockRequestReportsMissingCommit() { suite.handler.handlePeerUpdate(p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusUp, ConnID: 1}) - block := &types.Block{Header: types.Header{Height: 1001}} - suite.fakeStore.On("LoadBlock", int64(1001)).Once().Return(block) + suite.expectStoredBlock(1001, 1) suite.fakeStore.On("LoadSeenCommitAt", int64(1001)).Once().Return(nil) _, _, err := suite.handler.prepareBlockResponse(blockResponseJob{ @@ -150,9 +166,8 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestBlockResponseBackpressureAppli defer cancel() peerID := types.NodeID("slow-peer") - block := &types.Block{Header: types.Header{Height: 1001}} commit := &types.Commit{Height: 1001} - suite.fakeStore.On("LoadBlock", int64(1001)).Times(3).Return(block) + suite.expectStoredBlock(1001, 3) suite.fakeStore.On("LoadSeenCommitAt", int64(1001)).Times(3).Return(commit) delivery := make(chan p2p.Envelope, 3) @@ -183,10 +198,9 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestStalledPeerDeliveryDoesNotBloc ctx, cancel := context.WithCancel(context.Background()) defer cancel() - block := &types.Block{Header: types.Header{Height: 1001}} commit := &types.Commit{Height: 1001} totalRequests := maxInFlightBlockResponses + 1 - suite.fakeStore.On("LoadBlock", int64(1001)).Times(totalRequests).Return(block) + suite.expectStoredBlock(1001, totalRequests) suite.fakeStore.On("LoadSeenCommitAt", int64(1001)).Times(totalRequests).Return(commit) delivery := make(chan p2p.Envelope, totalRequests) @@ -243,9 +257,8 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestDisconnectedPeerQueueIsPurgedB peerID := types.NodeID("disconnecting-peer") const connID = uint64(7) - block := &types.Block{Header: types.Header{Height: 1001}} commit := &types.Commit{Height: 1001} - suite.fakeStore.On("LoadBlock", int64(1001)).Once().Return(block) + suite.expectStoredBlock(1001, 1) suite.fakeStore.On("LoadSeenCommitAt", int64(1001)).Once().Return(commit) delivery := make(chan p2p.Envelope, 1) @@ -291,14 +304,11 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestDisconnectDuringLoadSkipsRespo const connID = uint64(8) started := make(chan struct{}) release := make(chan struct{}) - suite.fakeStore. - On("LoadBlock", int64(1001)). - Once(). + suite.expectStoredBlock(1001, 1). Run(func(mock.Arguments) { close(started) <-release - }). - Return(&types.Block{Header: types.Header{Height: 1001}}) + }) suite.handler.handlePeerUpdate(p2p.PeerUpdate{ NodeID: peerID, @@ -466,7 +476,7 @@ func newMockPeerAdder(t *testing.T) *mockPeerAdder { func (suite *BlockP2PMessageHandlerTestSuite) TestRequestWaitsForConnectionUpdate() { peerID := types.NodeID("early-peer") - suite.fakeStore.On("LoadBlock", int64(1001)).Once().Return(nil) + suite.fakeStore.On("LoadBlockMeta", int64(1001)).Once().Return(nil) delivered := make(chan struct{}, 1) suite.fakeP2PChannel.On("Send", mock.Anything, mock.Anything).Once().Run(func(mock.Arguments) { delivered <- struct{}{} }).Return(nil) suite.Require().NoError(suite.handler.Handle(context.Background(), suite.fakeClient, &p2p.Envelope{ @@ -512,3 +522,188 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestReconnectRequestSurvivesEarlie suite.Equal(uint64(11), job.envelope.ConnID) suite.Zero(h.queuedResponses) } + +// TestServeBlockAsCommitted checks that a block is served as the bytes of the +// parts it was stored with, which are the parts it was committed with, and that +// cutting them the standard way rebuilds the stored part set. +func TestServeBlockAsCommitted(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 5, &state, privVals, 1) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + + const smallPartSize = 100 + standard := blockBytes(t, blocks[0]) + unknownField := withUnknownField(t, blocks[1]) + smallParts := blockBytes(t, blocks[2]) + testCases := []struct { + name string + parts *types.PartSet + bytes []byte + // whether the standard cut of the bytes is the stored part set + standardCut bool + }{ + {"standard", types.NewPartSetFromData(standard, types.BlockPartSizeBytes), standard, true}, + {"unknown field", types.NewPartSetFromData(unknownField, types.BlockPartSizeBytes), unknownField, true}, + // served as stored; block sync does not rebuild a cut other than the + // standard one, so a syncing node rejects it against the commit + {"small parts", types.NewPartSetFromData(smallParts, smallPartSize), smallParts, false}, + } + for i, tc := range testCases { + blockStore.SaveBlock(blocks[i], tc.parts, blocks[i+1].LastCommit) + } + // a header stored without its block, as state sync stores the one it + // restores the state at + header := blocks[3] + require.NoError(t, blockStore.SaveSignedHeader( + &types.SignedHeader{Header: &header.Header, Commit: blocks[4].LastCommit}, + blocks[4].LastCommit.BlockID, + )) + + handler := newBlockP2PMessageHandler(ctx, log.NewNopLogger(), blockStore, newMockPeerAdder(t)) + t.Cleanup(handler.stop) + handler.handlePeerUpdate(p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusUp, ConnID: 1}) + request := func(height int64) proto.Message { + msg, _, err := handler.prepareBlockResponse(blockResponseJob{ + ctx: ctx, + envelope: p2p.Envelope{From: "peer", ConnID: 1, Message: &bcproto.BlockRequest{Height: height}}, + }) + require.NoError(t, err) + return msg + } + + for i, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + msg, ok := request(blocks[i].Height).(*bcproto.BlockResponse) + require.True(t, ok) + require.Equal(t, tc.bytes, msg.Block) + + resp, err := BlockResponseFromProto(msg, "peer") + require.NoError(t, err) + require.Equal(t, tc.bytes, resp.BlockBytes) + require.Equal(t, blocks[i].Height, resp.Height) + require.Equal(t, blocks[i+1].LastCommit.BlockID, resp.Commit.BlockID) + cut := types.NewPartSetFromData(resp.BlockBytes, types.BlockPartSizeBytes) + require.Equal(t, tc.standardCut, tc.parts.Header().Equals(cut.Header())) + }) + } + require.Equal(t, &bcproto.NoBlockResponse{Height: header.Height}, request(header.Height)) +} + +// TestBlockResponseWireCompatible checks that a BlockResponse encodes as it did +// when its block field was an embedded tendermint.types.Block, so peers on +// either side of the change read what the other sends. +func TestBlockResponseWireCompatible(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) + block, commit := blocks[0], blocks[1].LastCommit.ToProto() + pbb, err := block.ToProto() + require.NoError(t, err) + commitBytes, err := proto.Marshal(commit) + require.NoError(t, err) + + embedded, err := proto.Marshal(pbb) + require.NoError(t, err) + // an embedded message is its encoding, length-delimited + older := appendBytesField(appendBytesField(nil, 1, embedded), 2, commitBytes) + + newer, err := proto.Marshal(&bcproto.BlockResponse{Block: blockBytes(t, block), Commit: commit}) + require.NoError(t, err) + require.Equal(t, older, newer) + + var decoded bcproto.BlockResponse + require.NoError(t, proto.Unmarshal(older, &decoded)) + resp, err := BlockResponseFromProto(&decoded, "peer") + require.NoError(t, err) + require.Equal(t, embedded, resp.BlockBytes) + decodedBlock, err := decodeBlock(resp.BlockBytes) + require.NoError(t, err) + require.Equal(t, block.Hash(), decodedBlock.Hash()) +} + +// appendBytesField appends a length-delimited field to serialized protobuf. +func appendBytesField(bz []byte, number byte, value []byte) []byte { + bz = append(bz, number<<3|2) + bz = binary.AppendUvarint(bz, uint64(len(value))) + return append(bz, value...) +} + +// TestBlockResponseDoesNotRetainOverwrittenBlockField checks that a repeated +// block field does not leave the allocation of an earlier, larger occurrence +// held behind the small one that is decoded and charged to the backlog. +func TestBlockResponseDoesNotRetainOverwrittenBlockField(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) + bz := blockBytes(t, blocks[0]) + commitBytes, err := proto.Marshal(blocks[1].LastCommit.ToProto()) + require.NoError(t, err) + const overwritten = 1 << 20 + raw := appendBytesField(appendBytesField(appendBytesField(nil, 1, make([]byte, overwritten)), 1, bz), 2, commitBytes) + + var decoded bcproto.BlockResponse + require.NoError(t, proto.Unmarshal(raw, &decoded)) + require.GreaterOrEqual(t, cap(decoded.Block), overwritten, "the decoder is expected to reuse the larger allocation") + + resp, err := BlockResponseFromProto(&decoded, "peer") + require.NoError(t, err) + require.Equal(t, bz, resp.BlockBytes) + require.Equal(t, len(bz), resp.Size) + require.LessOrEqual(t, cap(resp.BlockBytes), 2*len(bz)) +} + +// TestBlockResponseRetainsOnlyItsBytes checks that a block response, as it +// waits in the backlog for a lower height, holds its block only as the bytes it +// is charged for there, and not also decoded. A block of many small +// transactions costs more decoded than its bytes, so holding both would retain +// more than twice the charge. +func TestBlockResponseRetainsOnlyItsBytes(t *testing.T) { + const ( + txs = 1 << 14 + txSize = 64 + responses = 16 + ) + blockTxs := make([]types.Tx, txs) + for i := range blockTxs { + blockTxs[i] = make(types.Tx, txSize) + binary.BigEndian.PutUint64(blockTxs[i], uint64(i)) + } + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + block := statefactory.MakeBlocks(context.Background(), t, 1, &state, privVals, 1)[0] + block.Data = types.Data{Txs: blockTxs} + block.DataHash = block.Data.Hash() + bz := blockBytes(t, block) + + heapAlloc := func() uint64 { + runtime.GC() + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + return stats.HeapAlloc + } + inputs := make([][]byte, responses) + for i := range inputs { + inputs[i] = slices.Clone(bz) + } + before := heapAlloc() + held := make([]*BlockResponse, responses) + charged := 0 + for i := range held { + resp, err := BlockResponseFromProto(&bcproto.BlockResponse{Block: inputs[i]}, "peer") + require.NoError(t, err) + held[i] = resp + charged += resp.Size + } + // the inputs, allocated before the first reading, are the bytes the + // responses hold, so what is retained here is only what they keep beyond + // those bytes + retained := int64(heapAlloc()) - int64(before) + runtime.KeepAlive(held) + t.Logf("charged %d bytes, retained %d more", charged, retained) + require.Less(t, retained, int64(charged)/2, + "a pending response must not keep a decoded copy of its block") +} diff --git a/internal/blocksync/reactor_test.go b/internal/blocksync/reactor_test.go index 5197cfd83..9141e4546 100644 --- a/internal/blocksync/reactor_test.go +++ b/internal/blocksync/reactor_test.go @@ -43,8 +43,16 @@ type reactorTestSuite struct { blockSyncChannels map[types.NodeID]p2p.Channel peerChans map[types.NodeID]chan p2p.PeerUpdate peerUpdates map[types.NodeID]*p2p.PeerUpdates + + // blockBytes returns the bytes the nodes commit their blocks with + blockBytes blockBytesFunc } +// blockBytesFunc returns the bytes a block is committed with, cut into parts of +// types.BlockPartSizeBytes. Nil commits the block as types.Block.MakePartSet +// encodes it. +type blockBytesFunc func(t *testing.T, block *types.Block) []byte + func setup( ctx context.Context, t *testing.T, @@ -55,6 +63,22 @@ func setup( opts ...ReactorOption, ) *reactorTestSuite { t.Helper() + return setupWithBlockBytes(ctx, t, conf, genDoc, privVal, maxBlockHeights, nil, opts...) +} + +// setupWithBlockBytes is setup with the nodes committing their blocks with the +// bytes blockBytes returns. +func setupWithBlockBytes( + ctx context.Context, + t *testing.T, + conf *config.Config, + genDoc *types.GenesisDoc, + privVal types.PrivValidator, + maxBlockHeights []int64, + blockBytes blockBytesFunc, + opts ...ReactorOption, +) *reactorTestSuite { + t.Helper() var cancel context.CancelFunc ctx, cancel = context.WithCancel(ctx) @@ -72,6 +96,7 @@ func setup( blockSyncChannels: make(map[types.NodeID]p2p.Channel, numNodes), peerChans: make(map[types.NodeID]chan p2p.PeerUpdate, numNodes), peerUpdates: make(map[types.NodeID]*p2p.PeerUpdates, numNodes), + blockBytes: blockBytes, } chDesc := &p2p.ChannelDescriptor{ID: p2p.BlockSyncChannel} @@ -199,7 +224,8 @@ func (rts *reactorTestSuite) addNode( state, err := reactor.stateStore.Load() require.NoError(t, err) for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ { - block, blockID, partSet, seenCommit := makeNextBlock(ctx, t, state, privVal, blockHeight, commit) + block, blockID, partSet, seenCommit := makeNextBlockWithBytes(ctx, t, state, privVal, blockHeight, commit, + rts.blockBytes) state, err = reactor.blockExec.ApplyBlock(ctx, state, blockID, block, seenCommit, types.VerifiedCommit{}) require.NoError(t, err) reactor.store.SaveBlock(block, partSet, seenCommit) @@ -217,13 +243,29 @@ func makeNextBlock(ctx context.Context, signer types.PrivValidator, height int64, commit *types.Commit) (*types.Block, types.BlockID, *types.PartSet, *types.Commit) { + return makeNextBlockWithBytes(ctx, t, state, signer, height, commit, nil) +} + +// makeNextBlockWithBytes is makeNextBlock with the block committed with the +// bytes blockBytes returns. +func makeNextBlockWithBytes(ctx context.Context, + t *testing.T, + state sm.State, + signer types.PrivValidator, + height int64, + commit *types.Commit, + blockBytes blockBytesFunc) (*types.Block, types.BlockID, *types.PartSet, *types.Commit) { block, err := sf.MakeBlock(state, height, commit, 0) require.NoError(t, err) block.CoreChainLockedHeight = state.LastCoreChainLockedBlockHeight - partSet, err := block.MakePartSet(types.BlockPartSizeBytes) - require.NoError(t, err) + var partSet *types.PartSet + if blockBytes != nil { + partSet = types.NewPartSetFromData(blockBytes(t, block), types.BlockPartSizeBytes) + } else { + partSet, err = block.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + } blockID := block.BlockID(partSet) - require.NoError(t, err) // Simulate a commit for the current height vote, err := factory.MakeVote( @@ -416,6 +458,62 @@ func TestReactor_NoBlockResponse(t *testing.T) { } } +// TestReactor_SyncBlocksAsCommitted syncs a block committed with bytes that +// types.Block.MakePartSet does not reproduce from the decoded block. The +// commit's part set header is computed over the committed bytes, so a peer can +// only serve such a block from the parts it stored, and the syncing node must +// cut the part set from the bytes it received. +func TestReactor_SyncBlocksAsCommitted(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) + require.NoError(t, err) + defer os.RemoveAll(cfg.RootDir) + + genDoc, privVals := factory.RandGenesisDoc(1, factory.ConsensusParams()) + const ( + maxBlockHeight = int64(6) + // a field that tendermint.types.Block does not define, which decoding + // skips, appended to the block's bytes + unknownFieldHeight = int64(3) + ) + committedBytes := func(t *testing.T, block *types.Block) []byte { + if block.Height == unknownFieldHeight { + return withUnknownField(t, block) + } + return blockBytes(t, block) + } + rts := setupWithBlockBytes(ctx, t, cfg, genDoc, privVals[0], []int64{maxBlockHeight, 0}, committedBytes) + source, syncing := rts.reactors[rts.nodes[0]], rts.reactors[rts.nodes[1]] + require.Equal(t, maxBlockHeight, source.store.Height()) + require.NotEqual(t, + source.store.LoadBlockMeta(unknownFieldHeight).BlockID.PartSetHeader, + source.store.LoadBlock(unknownFieldHeight).BlockID(nil).PartSetHeader, + "the block must be committed with bytes a re-encoding does not reproduce") + + rts.start(ctx, t) + + require.Eventually(t, + func() bool { return syncing.store.Height() > unknownFieldHeight }, + 30*time.Second, + 10*time.Millisecond, + "expected node to sync the block committed with other bytes") + require.Equal(t, + source.store.LoadBlockMeta(unknownFieldHeight).BlockID, + syncing.store.LoadBlockMeta(unknownFieldHeight).BlockID) + require.Equal(t, + loadBlockBytes(source.store, unknownFieldHeight), + loadBlockBytes(syncing.store, unknownFieldHeight)) +} + +// withUnknownField returns the bytes of block with field 15 appended, a field +// tendermint.types.Block does not define and decoding skips. +func withUnknownField(t *testing.T, block *types.Block) []byte { + // tag (15 << 3 | varint wire type), then the value 1 + return append(blockBytes(t, block), 15<<3, 1) +} + func TestReactor_BadBlockStopsPeer(t *testing.T) { // Ultimately, this should be refactored to be less integration test oriented // and more unit test oriented by simply testing channel sends and receives. diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index 1c214d051..473b37c0c 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -4,9 +4,11 @@ import ( "context" "errors" "fmt" + "slices" "sync/atomic" "time" + "github.com/cosmos/gogoproto/proto" "github.com/jonboulle/clockwork" sync "github.com/sasha-s/go-deadlock" @@ -16,6 +18,7 @@ import ( "github.com/dashpay/tenderdash/libs/service" "github.com/dashpay/tenderdash/libs/workerpool" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -43,11 +46,12 @@ const ( // costs once blocks are large, because a count of blocks says nothing about // that when one block may be tens of megabytes. // - // Serialized bytes understate the heap actually retained. The size covers the - // block alone, not the commit held alongside it, and a decoded block costs - // more than its wire form - several times more for blocks small enough that - // fixed per-response overhead dominates. Read this as a budget for block - // data, not as the memory the backlog occupies. + // A pending response holds its block only as those bytes, the form it is + // applied from, and not decoded as well: the block is decoded again when it + // is applied. The size still understates the heap retained, since it covers + // the block alone, not the commit held alongside it or the per-response + // overhead. Read this as a budget for block data, not as the memory the + // backlog occupies. maxPendingApplyBytes = 256 << 20 // maxOutstandingHeights is how many heights may be requested and not yet @@ -357,7 +361,7 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { if err != nil { if !errors.Is(err, errDuplicateBlock) { s.logger.Error("cannot add a block to the pending list", - "height", resp.Block.Height, + "height", resp.Height, "error", err.Error()) _ = s.client.Send(ctx, p2p.PeerError{NodeID: resp.PeerID, Err: err, ConnID: resp.ConnID}) return nil @@ -366,7 +370,7 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { // this height. Info level: a healthy sync produces none, so this is the only // signal an operator gets that heights are being re-requested. s.logger.Info("dropping duplicate block response", - "height", resp.Block.Height, + "height", resp.Height, "peer", resp.PeerID, "reason", err.Error()) } @@ -380,7 +384,7 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { // Charge the failure to the peer that supplied the failing block: removing it // also drops its pending entry, so the height is fetched from someone else. s.logger.Error("cannot apply a block", - "height", failed.Block.Height, + "height", failed.Height, "peer", failed.PeerID, "error", err.Error()) s.RemovePeer(failed.PeerID) @@ -391,8 +395,7 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { // backlog reports what block sync is holding: the height it is waiting to // apply, which is the lowest height not applied yet, and the serialized size of -// the responses held above it. That size is what the blocks measured on the -// wire, which is less than what holding them decoded costs; see +// the responses held above it, which hold their blocks as those bytes; see // maxPendingApplyBytes. Both are read under one lock, so the pair describes a // single state and a job cannot be admitted against a height from one moment // and a backlog from another. @@ -637,7 +640,7 @@ func (s *Synchronizer) applyBlock(ctx context.Context) (BlockResponse, error) { if !ok { return BlockResponse{}, nil } - err := s.applier.Apply(ctx, resp.Block, resp.Commit) + err := s.applier.ApplyBytes(ctx, resp.BlockBytes, resp.Commit) if err != nil { return resp, fmt.Errorf("cannot apply block: %w", err) } @@ -675,8 +678,8 @@ func (s *Synchronizer) updateMonitor() { "blocks/s", syncRate, } // blocks are applied one at a time, so the per-stage averages say whether the - // rate above is limited by serialization, signature verification, disk or the - // ABCI application + // rate above is limited by building the part set, signature verification, + // disk or the ABCI application if timings, measured := s.applier.Timings(); measured { // logged as durations rather than whole milliseconds: part set building // and commit verification are often sub-millisecond, and truncating them @@ -731,16 +734,15 @@ func (s *Synchronizer) addBlock(resp BlockResponse) error { s.mtx.Lock() defer s.mtx.Unlock() - block := resp.Block - if block.Height < s.height { + if resp.Height < s.height { return fmt.Errorf("%w (peer: %s, block height: %d already applied)", - errDuplicateBlock, resp.PeerID, block.Height) + errDuplicateBlock, resp.PeerID, resp.Height) } - _, ok := s.pendingToApply[block.Height] + _, ok := s.pendingToApply[resp.Height] if ok { - return fmt.Errorf("%w (peer: %s, block height: %d)", errDuplicateBlock, resp.PeerID, block.Height) + return fmt.Errorf("%w (peer: %s, block height: %d)", errDuplicateBlock, resp.PeerID, resp.Height) } - s.pendingToApply[block.Height] = resp + s.pendingToApply[resp.Height] = resp return nil } @@ -775,25 +777,32 @@ type BlockResponse struct { // ConnID is the generation of the connection that delivered this response, which // its failures are fenced to. ConnID uint64 - Block *types.Block - Commit *types.Commit - // Size is the serialized size of Block in bytes, measured while decoding the - // response. Deriving it from Block again means re-serializing the whole - // block, which is too expensive to do on the block apply path. + // Height is the height of the block in BlockBytes. + Height int64 + // BlockBytes is the block as it was committed. It is applied with the part + // set cut from these bytes, which is what the commit's block ID describes: + // re-encoding the decoded block need not give the same bytes. A response + // holds the block only in this form. It is decoded when it arrives, to be + // checked, and again when it is applied, so a response waiting for a lower + // height holds one copy of its block, the one it is charged for. + BlockBytes []byte + Commit *types.Commit + // Size is the length of BlockBytes, what the response is charged in the + // backlog. Size int } func (r *BlockResponse) Validate() error { - if r.Block == nil { + if len(r.BlockBytes) == 0 { return errors.New("block response without a block") } if r.Commit == nil { // See https://github.com/tendermint/tendermint/pull/8433#discussion_r866790631 - return fmt.Errorf("a block without a commit at height %d - possible node store corruption", r.Block.Height) + return fmt.Errorf("a block without a commit at height %d - possible node store corruption", r.Height) } - if r.Block.Height != r.Commit.Height { + if r.Height != r.Commit.Height { return fmt.Errorf("heights don't match, not adding block (block height: %d, commit height: %d)", - r.Block.Height, + r.Height, r.Commit.Height) } return nil @@ -803,7 +812,18 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* if resp == nil { return nil, errors.New("invalid") } - block, err := types.BlockFromProto(resp.Block) + if len(resp.Block) == 0 { + return nil, errors.New("block response without a block") + } + blockBytes := resp.Block + // A repeated block field leaves the allocation of its longest occurrence + // behind the last one, and only the length is charged to the backlog. + if cap(blockBytes) > 2*len(blockBytes) { + blockBytes = slices.Clone(blockBytes) + } + // Decoded here so that a block that does not decode is refused before it is + // queued, and then dropped: see BlockResponse.BlockBytes. + block, err := decodeBlock(blockBytes) if err != nil { return nil, err } @@ -815,11 +835,19 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* } } return &BlockResponse{ - PeerID: peerID, - Block: block, - Commit: commit, - // measured here, on a worker goroutine, rather than on the single - // goroutine that applies blocks - Size: resp.Block.Size(), + PeerID: peerID, + Height: block.Height, + BlockBytes: blockBytes, + Commit: commit, + Size: len(blockBytes), }, nil } + +// decodeBlock decodes a serialized tendermint.types.Block. +func decodeBlock(blockBytes []byte) (*types.Block, error) { + pbb := new(tmproto.Block) + if err := proto.Unmarshal(blockBytes, pbb); err != nil { + return nil, fmt.Errorf("cannot decode block: %w", err) + } + return types.BlockFromProto(pbb) +} diff --git a/internal/blocksync/synchronizer_test.go b/internal/blocksync/synchronizer_test.go index fb0a92363..ad7803c98 100644 --- a/internal/blocksync/synchronizer_test.go +++ b/internal/blocksync/synchronizer_test.go @@ -141,7 +141,7 @@ func (suite *SynchronizerTestSuite) assertHandoverWaitsForConsumer(beforeApply b paused := make(chan struct{}) release := make(chan struct{}) if beforeApply { - pool.pendingToApply[resp.Block.Height] = *resp + pool.pendingToApply[resp.Height] = *resp pool.logger = &handoverPauseLogger{Logger: log.NewNopLogger(), paused: paused, release: release} } suite.store.On("SaveBlock", mock.Anything, mock.Anything, mock.Anything).Once() @@ -154,7 +154,7 @@ func (suite *SynchronizerTestSuite) assertHandoverWaitsForConsumer(beforeApply b finalState := suite.initialState.Copy() finalState.LastBlockHeight = 1 suite.blockExec.On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, mock.Anything, - resp.Block, resp.Commit, mock.Anything). + sameBlock(suite.T(), resp), resp.Commit, mock.Anything). Run(func(args mock.Arguments) { finalizing <- args.Get(0).(context.Context) if !beforeApply { @@ -353,7 +353,7 @@ func (suite *SynchronizerTestSuite) TestConsumeJobResult() { Once(). Return(nil) suite.blockExec. - On("ValidateBlock", mock.Anything, mock.Anything, respH1.Block, mock.Anything). + On("ValidateBlock", mock.Anything, mock.Anything, sameBlock(suite.T(), respH1), mock.Anything). Once(). Return(nil) suite.blockExec. @@ -361,11 +361,11 @@ func (suite *SynchronizerTestSuite) TestConsumeJobResult() { Maybe(). Return(types.VerifiedCommit{}, nil) suite.blockExec. - On("ProcessProposal", mock.Anything, respH1.Block, mock.Anything, mock.Anything, true, mock.Anything). + On("ProcessProposal", mock.Anything, sameBlock(suite.T(), respH1), mock.Anything, mock.Anything, true, mock.Anything). Once(). Return(sm.CurrentRoundState{}, nil) suite.blockExec. - On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, mock.Anything, respH1.Block, respH1.Commit, mock.Anything). + On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, mock.Anything, sameBlock(suite.T(), respH1), respH1.Commit, mock.Anything). Once(). Return(sm.State{}, nil, nil) }, @@ -399,11 +399,11 @@ func (suite *SynchronizerTestSuite) TestConsumeJobResult() { result: workerpool.Result{Value: respH1}, wantPushBack: []int64{1, 2}, mockFn: func(pool *Synchronizer) { - pool.pendingToApply[2] = BlockResponse{PeerID: "peer 1", Block: respH2.Block} + pool.pendingToApply[2] = BlockResponse{PeerID: "peer 1", Height: respH2.Height, BlockBytes: respH2.BlockBytes} // VerifyCommit is covered by the Maybe expectation the earlier case // registered on this shared mock suite.blockExec. - On("ValidateBlock", mock.Anything, mock.Anything, respH1.Block, mock.Anything). + On("ValidateBlock", mock.Anything, mock.Anything, sameBlock(suite.T(), respH1), mock.Anything). Once(). Return(errors.New("invalid error")) suite.client. @@ -424,7 +424,7 @@ func (suite *SynchronizerTestSuite) TestConsumeJobResult() { // so a peer report fails this case. result: workerpool.Result{Value: duplicateH2}, mockFn: func(pool *Synchronizer) { - pool.pendingToApply[respH2.Block.Height] = *respH2 + pool.pendingToApply[respH2.Height] = *respH2 }, }, } @@ -519,7 +519,7 @@ func (suite *SynchronizerTestSuite) TestRemovePeer() { pool.AddPeer(peer) } for _, resp := range tc.responses { - pool.pendingToApply[resp.Block.Height] = *resp + pool.pendingToApply[resp.Height] = *resp } for _, height := range tc.initialPushBack { pool.jobGen.pushBack(height) @@ -566,11 +566,11 @@ func (suite *SynchronizerTestSuite) TestConsumeDuplicateThenDrain() { applier := newBlockApplier(suite.blockExec, suite.store, applierWithState(suite.initialState)) pool := NewSynchronizer(1, suite.client, applier, WithWorkerPool(wp)) // height 2 is already in hand and waiting on height 1 - pool.pendingToApply[respH2.Block.Height] = *respH2 + pool.pendingToApply[respH2.Height] = *respH2 resultCh <- workerpool.Result{Value: duplicateH2} suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) - suite.Require().Equal(peerID1, pool.pendingToApply[respH2.Block.Height].PeerID, + suite.Require().Equal(peerID1, pool.pendingToApply[respH2.Height].PeerID, "the response already held must be kept") suite.Require().Empty(pool.jobGen.pushedBack, "a duplicate must not re-queue the height") suite.client.AssertNotCalled(suite.T(), "Send", mock.Anything, mock.Anything) @@ -619,7 +619,7 @@ func (suite *SynchronizerTestSuite) TestDuplicateBlockIsVisibleAtDefaultLogLevel // Start above the duplicated height so nothing is applied and the duplicate is // all that is under test. pool := NewSynchronizer(2, suite.client, applier, WithWorkerPool(wp), WithLogger(logger)) - pool.pendingToApply[respH1.Block.Height] = *respH1 + pool.pendingToApply[respH1.Height] = *respH1 resultCh <- workerpool.Result{Value: duplicateH1} suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) @@ -675,16 +675,16 @@ func (suite *SynchronizerTestSuite) TestPeerErrorsCarrySupplyingConnection() { poisoned, err := BlockResponseFromProto(suite.responses[0], peerID) suite.Require().NoError(err) poisoned.ConnID = connID - pool.pendingToApply[poisoned.Block.Height] = *poisoned + pool.pendingToApply[poisoned.Height] = *poisoned honest, err := BlockResponseFromProto(suite.responses[1], "honest peer") suite.Require().NoError(err) suite.blockExec. - On("VerifyCommit", mock.Anything, mock.Anything, poisoned.Block.Height, poisoned.Commit). + On("VerifyCommit", mock.Anything, mock.Anything, poisoned.Height, poisoned.Commit). Once(). Return(types.VerifiedCommit{}, nil) suite.blockExec. - On("ValidateBlock", mock.Anything, mock.Anything, poisoned.Block, mock.Anything). + On("ValidateBlock", mock.Anything, mock.Anything, sameBlock(suite.T(), poisoned), mock.Anything). Once(). Return(errors.New("invalid block")) expectReport() @@ -956,14 +956,14 @@ func (suite *SynchronizerTestSuite) TestApplyFailurePunishesSupplyingPeer() { pool := NewSynchronizer(1, suite.client, applier, WithWorkerPool(wp)) pool.AddPeer(newPeerData(poisonPeerID, 1, 100)) pool.AddPeer(newPeerData(honestPeerID, 1, 100)) - pool.pendingToApply[poisonH1.Block.Height] = *poisonH1 + pool.pendingToApply[poisonH1.Height] = *poisonH1 suite.blockExec. - On("VerifyCommit", mock.Anything, mock.Anything, poisonH1.Block.Height, poisonH1.Commit). + On("VerifyCommit", mock.Anything, mock.Anything, poisonH1.Height, poisonH1.Commit). Once(). Return(types.VerifiedCommit{}, nil) suite.blockExec. - On("ValidateBlock", mock.Anything, mock.Anything, poisonH1.Block, mock.Anything). + On("ValidateBlock", mock.Anything, mock.Anything, sameBlock(suite.T(), poisonH1), mock.Anything). Once(). Return(errors.New("invalid block")) suite.client. @@ -978,13 +978,13 @@ func (suite *SynchronizerTestSuite) TestApplyFailurePunishesSupplyingPeer() { // The poisoned height is dropped and re-requested from someone else, so the // synchronizer can still advance. - suite.Require().Equal([]int64{poisonH1.Block.Height}, pool.jobGen.pushedBack) - suite.Require().NotContains(pool.pendingToApply, poisonH1.Block.Height) + suite.Require().Equal([]int64{poisonH1.Height}, pool.jobGen.pushedBack) + suite.Require().NotContains(pool.pendingToApply, poisonH1.Height) _, found := pool.peerStore.Get(poisonPeerID) suite.Require().False(found) // The honest peer keeps both its response and its place in the peer store. - suite.Require().Contains(pool.pendingToApply, honestH2.Block.Height) + suite.Require().Contains(pool.pendingToApply, honestH2.Height) _, found = pool.peerStore.Get(honestPeerID) suite.Require().True(found) } @@ -999,7 +999,7 @@ func (suite *SynchronizerTestSuite) TestAddBlockDropsAlreadyAppliedHeight() { const currentHeight = int64(5) peerID := types.NodeID("peer 1") staleH1, _ := BlockResponseFromProto(suite.responses[0], peerID) - suite.Require().Less(staleH1.Block.Height, currentHeight) + suite.Require().Less(staleH1.Height, currentHeight) resultCh := make(chan workerpool.Result, 1) wp := workerpool.New(1, workerpool.WithResultCh(resultCh)) @@ -1024,7 +1024,7 @@ func (suite *SynchronizerTestSuite) TestRemovePeerReleasesLockBeforePushBack() { applier := newBlockApplier(suite.blockExec, suite.store, applierWithState(suite.initialState)) pool := NewSynchronizer(1, suite.client, applier) pool.AddPeer(newPeerData(peerID, 1, 100)) - pool.pendingToApply[respH1.Block.Height] = *respH1 + pool.pendingToApply[respH1.Height] = *respH1 // Block the job generator so the re-queue cannot complete. pool.jobGen.mtx.Lock() @@ -1053,7 +1053,7 @@ func (suite *SynchronizerTestSuite) TestRemovePeerReleasesLockBeforePushBack() { case <-time.After(2 * time.Second): suite.Require().Fail("RemovePeer did not finish after the job generator lock was released") } - suite.Require().Equal([]int64{respH1.Block.Height}, pool.jobGen.pushedBack) + suite.Require().Equal([]int64{respH1.Height}, pool.jobGen.pushedBack) } func (suite *SynchronizerTestSuite) TestUpdateMonitor() { @@ -1361,10 +1361,8 @@ func (suite *SynchronizerTestSuite) TestStatusRefreshUpdatesAdvertisedRange() { func generateBlockResponses(t *testing.T, blocks []*types.Block) []*blocksync.BlockResponse { responses := make([]*blocksync.BlockResponse, 0, len(blocks)-1) for i := 0; i < len(blocks)-1; i++ { - protoBlock, err := blocks[i].ToProto() - require.NoError(t, err) responses = append(responses, &blocksync.BlockResponse{ - Block: protoBlock, + Block: blockBytes(t, blocks[i]), Commit: blocks[i+1].LastCommit.ToProto(), }) } @@ -1825,7 +1823,7 @@ func (h *backlogHarness) fillWindow(ctx context.Context, blocking int64) types.N if resp == nil { return blockingPeer } - if resp.Block.Height == blocking { + if resp.Height == blocking { blockingPeer = resp.PeerID continue } @@ -1865,7 +1863,7 @@ func (suite *SynchronizerTestSuite) TestBacklogIsBoundedWhileTheBlockingHeightIs continue } requested++ - if resp.Block.Height == blocking { + if resp.Height == blocking { continue } h.deliver(ctx, resp) @@ -1903,7 +1901,7 @@ func (suite *SynchronizerTestSuite) TestBacklogIsBoundedByBytesWhenBlocksAreLarg refused++ continue } - if resp.Block.Height == blocking { + if resp.Height == blocking { continue } h.deliverSized(ctx, resp, blockSize) @@ -1938,7 +1936,7 @@ func (suite *SynchronizerTestSuite) TestBlockingHeightIsStillFetchableAtTheByteB if resp == nil { break } - if resp.Block.Height == blocking { + if resp.Height == blocking { blockingPeer = resp.PeerID continue } @@ -1956,7 +1954,7 @@ func (suite *SynchronizerTestSuite) TestBlockingHeightIsStillFetchableAtTheByteB resp := h.produce(ctx) suite.Require().NotNil(resp, "the byte budget is spent and the height it is waiting for can never be requested again") - suite.Require().EqualValues(blocking, resp.Block.Height) + suite.Require().EqualValues(blocking, resp.Height) h.deliver(ctx, resp) @@ -1986,7 +1984,7 @@ func (suite *SynchronizerTestSuite) TestBlockingHeightIsStillFetchableAtTheBound resp := h.produce(ctx) suite.Require().NotNil(resp, "the window is full and the height it is waiting for can never be requested again") - suite.Require().EqualValues(blocking, resp.Block.Height, + suite.Require().EqualValues(blocking, resp.Height, "the retry of the blocking height must be what goes out first") h.deliver(ctx, resp) @@ -2022,7 +2020,7 @@ func (suite *SynchronizerTestSuite) TestDroppingTheBlockingPeerLeavesTheHeightTo for range requeued { resp := h.produce(ctx) suite.Require().NotNil(resp, "a re-queued height was starved by the window") - suite.Require().NotEqualValues(blocking, resp.Block.Height) + suite.Require().NotEqualValues(blocking, resp.Height) } suite.Require().Nil(h.produce(ctx), "the window must still refuse new heights") @@ -2030,7 +2028,7 @@ func (suite *SynchronizerTestSuite) TestDroppingTheBlockingPeerLeavesTheHeightTo h.timeout(ctx, blockingPeer, blocking) resp := h.produce(ctx) suite.Require().NotNil(resp, "the blocking height was stranded by dropping its peer") - suite.Require().EqualValues(blocking, resp.Block.Height) + suite.Require().EqualValues(blocking, resp.Height) } // TestInOrderSyncIsNotThrottledByTheBound checks that the window does not slow a @@ -2092,7 +2090,7 @@ func (suite *SynchronizerTestSuite) TestRequeuedHeightsAreNotStarvedByTheBound() for range requeued { resp := h.produce(ctx) suite.Require().NotNil(resp, "a re-queued height was starved by the window") - reissued = append(reissued, resp.Block.Height) + reissued = append(reissued, resp.Height) } suite.Require().Equal(requeued, reissued, "re-queued heights must go out lowest first") @@ -2214,7 +2212,7 @@ func (suite *SynchronizerTestSuite) TestClientTimeoutUnwedgesAFullWindow() { // Everything above the dropped height fills the window and stops there. wantBytes := 0 for height := blocking + 1; height < blocking+maxOutstandingHeights; height++ { - wantBytes += chain[height-1].Block.Size() + wantBytes += len(chain[height-1].Block) } suite.Require().Eventually(func() bool { _, pendingBytes := pool.backlog() @@ -2328,7 +2326,7 @@ func (suite *SynchronizerTestSuite) TestSmallBlocksAfterSlowApplyKeepPeersUsable if found { response := h.produce(ctx) suite.Require().NotNil(response) - suite.Equal(height, response.Block.Height) + suite.Equal(height, response.Height) h.deliver(ctx, response) suite.Equal(height+1, h.pool.height) } @@ -2344,3 +2342,13 @@ func (suite *SynchronizerTestSuite) TestSmallBlocksAfterSlowApplyKeepPeersUsable suite.True(stopped, "WaitForSync must still stop on cancellation") } } + +// sameBlock matches the block resp holds, which is decoded again from its +// bytes when it is applied. +func sameBlock(t *testing.T, resp *BlockResponse) interface{} { + t.Helper() + block, err := decodeBlock(resp.BlockBytes) + require.NoError(t, err) + hash := block.Hash() + return mock.MatchedBy(func(b *types.Block) bool { return b.HashesTo(hash) }) +} diff --git a/internal/consensus/metrics.go b/internal/consensus/metrics.go index 3c72d5951..3919172b1 100644 --- a/internal/consensus/metrics.go +++ b/internal/consensus/metrics.go @@ -229,8 +229,8 @@ func (m *Metrics) ObserveBlockSyncStage(stage string, d time.Duration) { // RecordConsMetrics uses for recording the block related metrics during fast-sync. // // blockSize is the size of the serialized block in bytes. It is passed in rather -// than derived from block, because callers on the block sync path have already -// serialized the block and recomputing it there is expensive. +// than derived from block, because callers on the block sync path already hold +// the serialized block and serializing it again there is expensive. func (m *Metrics) RecordConsMetrics(block *types.Block, blockSize int64) { m.NumTxs.Set(float64(len(block.Data.Txs))) m.TotalTxs.Add(float64(len(block.Data.Txs))) diff --git a/internal/consensus/replayer.go b/internal/consensus/replayer.go index 8c334bb62..d263250d1 100644 --- a/internal/consensus/replayer.go +++ b/internal/consensus/replayer.go @@ -247,7 +247,7 @@ func (r *BlockReplayer) replayBlocks( } } if !mutateState { - err = r.publishEvents(block, ucState) + err = r.publishEvents(block, commit, ucState) if err != nil { return nil, err } @@ -420,10 +420,10 @@ func (r *BlockReplayer) execInitChain(ctx context.Context, rs *replayState, stat func (r *BlockReplayer) publishEvents( block *types.Block, + commit *types.Commit, ucState sm.CurrentRoundState, ) error { - blockID := block.BlockID(nil) - es := sm.NewFullEventSet(block, blockID, ucState, ucState.NextValidators) + es := sm.NewFullEventSet(block, commit.BlockID, ucState, ucState.NextValidators) err := es.Publish(r.publisher) if err != nil { r.logger.Error("failed publishing event", "err", err) diff --git a/internal/state/execution.go b/internal/state/execution.go index c196489fe..46abe6505 100644 --- a/internal/state/execution.go +++ b/internal/state/execution.go @@ -835,8 +835,9 @@ func execBlock( if err != nil { return nil, err } - blockID := block.BlockID(nil) - protoBlockID := blockID.ToProto() + // The commit's block ID, not block.BlockID(nil): the block may have been + // committed with parts that re-encoding it does not reproduce. + protoBlockID := commit.BlockID.ToProto() responseFinalizeBlock, err := appConn.FinalizeBlock( ctx, diff --git a/proto/tendermint/blocksync/types.pb.go b/proto/tendermint/blocksync/types.pb.go index 5bb19dc5c..91538985f 100644 --- a/proto/tendermint/blocksync/types.pb.go +++ b/proto/tendermint/blocksync/types.pb.go @@ -116,7 +116,18 @@ func (m *NoBlockResponse) GetHeight() int64 { // BlockResponse returns block to the requested type BlockResponse struct { - Block *types.Block `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` + // block is a serialized tendermint.types.Block: the bytes of the parts it + // was committed with, in order. The receiver cuts them into parts of 64kB, + // the last one shorter, and checks the part set header it computes against + // the commit's. Decoding and re-encoding the block can change its bytes + // (unknown fields, non-canonical encoding), so it is served as stored. Only + // that standard cut, the one honest proposers use, is rebuilt. Consensus + // does not require it yet: it accepts parts of other sizes and proofs of a + // tree larger than the header counts, and a block committed that way fails + // the commit check whichever peer serves it. On the wire this is the same + // as an embedded tendermint.types.Block, the type of this field in older + // releases. + Block []byte `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` Commit *types.Commit `protobuf:"bytes,2,opt,name=commit,proto3" json:"commit,omitempty"` } @@ -153,7 +164,7 @@ func (m *BlockResponse) XXX_DiscardUnknown() { var xxx_messageInfo_BlockResponse proto.InternalMessageInfo -func (m *BlockResponse) GetBlock() *types.Block { +func (m *BlockResponse) GetBlock() []byte { if m != nil { return m.Block } @@ -268,24 +279,24 @@ func init() { func init() { proto.RegisterFile("tendermint/blocksync/types.proto", fileDescriptor_19b397c236e0fa07) } var fileDescriptor_19b397c236e0fa07 = []byte{ - // 272 bytes of a gzipped FileDescriptorProto + // 264 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x28, 0x49, 0xcd, 0x4b, 0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x4f, 0xca, 0xc9, 0x4f, 0xce, 0x2e, 0xae, 0xcc, 0x4b, 0xd6, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x41, 0xa8, - 0xd0, 0x83, 0xab, 0x90, 0x92, 0x41, 0xd2, 0x07, 0x56, 0x0d, 0xd1, 0x0d, 0xd1, 0x83, 0x45, 0x16, - 0xc9, 0x44, 0x25, 0x35, 0x2e, 0x1e, 0x27, 0x90, 0xe2, 0xa0, 0xd4, 0xc2, 0xd2, 0xd4, 0xe2, 0x12, - 0x21, 0x31, 0x2e, 0xb6, 0x8c, 0xd4, 0xcc, 0xf4, 0x8c, 0x12, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xe6, - 0x20, 0x28, 0x4f, 0x49, 0x93, 0x8b, 0xdf, 0x2f, 0x1f, 0xaa, 0xb2, 0xb8, 0x20, 0x3f, 0xaf, 0x38, - 0x15, 0xa7, 0xd2, 0x02, 0x2e, 0x5e, 0x54, 0x85, 0xba, 0x5c, 0xac, 0x60, 0x07, 0x81, 0xd5, 0x71, - 0x1b, 0x89, 0xeb, 0x21, 0xf9, 0x02, 0xe2, 0x16, 0x88, 0x7a, 0x88, 0x2a, 0x21, 0x03, 0x2e, 0xb6, - 0xe4, 0xfc, 0xdc, 0xdc, 0xcc, 0x12, 0x09, 0x26, 0xb0, 0x7a, 0x09, 0x4c, 0xf5, 0xce, 0x60, 0xf9, - 0x20, 0xa8, 0x3a, 0x25, 0x7e, 0x2e, 0xde, 0xe0, 0x92, 0xc4, 0x92, 0xd2, 0x62, 0xa8, 0x2f, 0x94, - 0x6c, 0xb8, 0xf8, 0x60, 0x02, 0xf8, 0x1d, 0x2b, 0x24, 0xc4, 0xc5, 0x92, 0x94, 0x58, 0x9c, 0x0a, - 0xb6, 0x8a, 0x39, 0x08, 0xcc, 0x76, 0x0a, 0x3a, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, - 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, 0xc6, 0x63, 0x39, - 0x86, 0x28, 0x8b, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0xfd, 0x94, 0xc4, - 0xe2, 0x8c, 0x82, 0xc4, 0x4a, 0x7d, 0x88, 0xe3, 0x40, 0x3c, 0x7d, 0x70, 0x90, 0xea, 0x63, 0x8b, - 0xc5, 0x24, 0x36, 0xb0, 0x9c, 0x31, 0x60, 0x00, 0x3d, 0x3d, 0xc3, 0xea, 0xe4, 0x01, 0x00, 0x00, + 0xd0, 0x83, 0xab, 0x90, 0x92, 0x41, 0xd2, 0x07, 0x56, 0x8d, 0xac, 0x47, 0x49, 0x8d, 0x8b, 0xc7, + 0x09, 0xa4, 0x34, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x8c, 0x8b, 0x2d, 0x23, 0x35, + 0x33, 0x3d, 0xa3, 0x44, 0x82, 0x51, 0x81, 0x51, 0x83, 0x39, 0x08, 0xca, 0x53, 0xd2, 0xe4, 0xe2, + 0xf7, 0xcb, 0x87, 0xaa, 0x2c, 0x2e, 0xc8, 0xcf, 0x2b, 0x4e, 0xc5, 0xa9, 0x34, 0x9c, 0x8b, 0x17, + 0x55, 0xa1, 0x08, 0x17, 0x2b, 0xd8, 0x39, 0x60, 0x75, 0x3c, 0x41, 0x10, 0x8e, 0x90, 0x01, 0x17, + 0x5b, 0x72, 0x7e, 0x6e, 0x6e, 0x66, 0x89, 0x04, 0x93, 0x02, 0xa3, 0x06, 0xb7, 0x91, 0x84, 0x1e, + 0x92, 0xf3, 0x21, 0x4e, 0x74, 0x06, 0xcb, 0x07, 0x41, 0xd5, 0x29, 0xf1, 0x73, 0xf1, 0x06, 0x97, + 0x24, 0x96, 0x94, 0x16, 0x43, 0x1d, 0xab, 0x64, 0xc3, 0xc5, 0x07, 0x13, 0xc0, 0xef, 0x26, 0x21, + 0x21, 0x2e, 0x96, 0xa4, 0xc4, 0xe2, 0x54, 0xb0, 0x55, 0xcc, 0x41, 0x60, 0xb6, 0x53, 0xd0, 0x89, + 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, + 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x44, 0x59, 0xa4, 0x67, 0x96, 0x64, 0x94, 0x26, + 0xe9, 0x25, 0xe7, 0xe7, 0xea, 0xa7, 0x24, 0x16, 0x67, 0x14, 0x24, 0x56, 0xea, 0x43, 0x1c, 0x07, + 0xe2, 0xe9, 0x83, 0x43, 0x4e, 0x1f, 0x5b, 0x74, 0x24, 0xb1, 0x81, 0xe5, 0x8c, 0x01, 0x03, 0x00, + 0x64, 0x5c, 0x54, 0x1e, 0xad, 0x01, 0x00, 0x00, } func (m *BlockRequest) Marshal() (dAtA []byte, err error) { @@ -385,15 +396,10 @@ func (m *BlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x12 } - if m.Block != nil { - { - size, err := m.Block.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintTypes(dAtA, i, uint64(size)) - } + if len(m.Block) > 0 { + i -= len(m.Block) + copy(dAtA[i:], m.Block) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Block))) i-- dAtA[i] = 0xa } @@ -503,8 +509,8 @@ func (m *BlockResponse) Size() (n int) { } var l int _ = l - if m.Block != nil { - l = m.Block.Size() + l = len(m.Block) + if l > 0 { n += 1 + l + sovTypes(uint64(l)) } if m.Commit != nil { @@ -715,7 +721,7 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType) } - var msglen int + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowTypes @@ -725,26 +731,24 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + if byteLen < 0 { return ErrInvalidLengthTypes } - postIndex := iNdEx + msglen + postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthTypes } if postIndex > l { return io.ErrUnexpectedEOF } + m.Block = append(m.Block[:0], dAtA[iNdEx:postIndex]...) if m.Block == nil { - m.Block = &types.Block{} - } - if err := m.Block.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err + m.Block = []byte{} } iNdEx = postIndex case 2: diff --git a/proto/tendermint/blocksync/types.proto b/proto/tendermint/blocksync/types.proto index 3b229350d..bc8d05a6c 100644 --- a/proto/tendermint/blocksync/types.proto +++ b/proto/tendermint/blocksync/types.proto @@ -3,7 +3,6 @@ package tendermint.blocksync; option go_package = "github.com/dashpay/tenderdash/proto/tendermint/blocksync"; -import "tendermint/types/block.proto"; import "tendermint/types/types.proto"; // BlockRequest requests a block for a specific height @@ -19,7 +18,18 @@ message NoBlockResponse { // BlockResponse returns block to the requested message BlockResponse { - tendermint.types.Block block = 1; + // block is a serialized tendermint.types.Block: the bytes of the parts it + // was committed with, in order. The receiver cuts them into parts of 64kB, + // the last one shorter, and checks the part set header it computes against + // the commit's. Decoding and re-encoding the block can change its bytes + // (unknown fields, non-canonical encoding), so it is served as stored. Only + // that standard cut, the one honest proposers use, is rebuilt. Consensus + // does not require it yet: it accepts parts of other sizes and proofs of a + // tree larger than the header counts, and a block committed that way fails + // the commit check whichever peer serves it. On the wire this is the same + // as an embedded tendermint.types.Block, the type of this field in older + // releases. + bytes block = 1; tendermint.types.Commit commit = 2; } diff --git a/spec/p2p/messages/block-sync.md b/spec/p2p/messages/block-sync.md index 48aa6155f..3faeb5307 100644 --- a/spec/p2p/messages/block-sync.md +++ b/spec/p2p/messages/block-sync.md @@ -34,11 +34,23 @@ NoBlockResponse notifies the peer requesting a block that the node does not cont ### BlockResponse -BlockResponse contains the block requested. - -| Name | Type | Description | Field Number | -|-------|----------------------------------------------|-----------------|--------------| -| Block | [Block](../../core/data_structures.md#block) | Requested Block | 1 | +BlockResponse contains the block requested and its commit. + +| Name | Type | Description | Field Number | +|--------|------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| +| Block | bytes | Requested [Block](../../core/data_structures.md#block), serialized as it was committed: the bytes of its parts, in order. On the wire this is the same as an embedded Block. | 1 | +| Commit | [Commit](../../core/data_structures.md#commit) | Commit for the requested block | 2 | + +The receiver cuts `Block` into parts of 64kB, the last one shorter, and checks +the part set header it computes against the one in the commit's block ID. +`Block` is served as stored, because decoding and re-encoding the block can +change its bytes. + +Block sync rebuilds only this standard cut, which honest proposers use. +Consensus does not require it yet: it accepts parts of other sizes, and proofs +of a tree larger than the header counts. A block committed that way fails the +commit check (`wrong block ID`) whichever peer serves it, and block sync does +not get past its height. ### StatusRequest