From cf9ef9230b7208d4bd3f584b36773367933d8033 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Thu, 5 Nov 2026 18:08:09 +0700 Subject: [PATCH 1/8] fix(blocksync): serve blocks as the parts they were committed with Block sync served LoadBlock(h).ToProto(), a re-encoding of the decoded block, and the syncing node rebuilt the part set from that re-encoding. When the committed bytes carry an unknown field or a non-canonical encoding, or the proposer cut them into parts shorter than 64kB, the rebuilt part set header differs from the one in the commit's block ID. The commit check then fails with "wrong block ID", so no honest peer can serve the block. BlockResponse.block is now bytes, the stored parts concatenated, which is the same wire format as the embedded Block it replaces. A new part_sizes field carries the part sizes when they are not the standard 64kB cut. The syncing node decodes the block from those bytes and builds the part set it verifies and stores from them, with the new types.NewPartSetFromPartSizes for a non-standard cut. Co-Authored-By: Claude Opus 5.5 --- internal/blocksync/applier.go | 34 +++- internal/blocksync/applier_test.go | 56 ++++-- internal/blocksync/commit_extensions_test.go | 12 +- internal/blocksync/p2p_msg_handler.go | 68 ++++++- internal/blocksync/p2p_msg_handler_test.go | 179 ++++++++++++++--- internal/blocksync/reactor_test.go | 119 +++++++++++- internal/blocksync/synchronizer.go | 47 +++-- internal/blocksync/synchronizer_test.go | 6 +- proto/tendermint/blocksync/types.pb.go | 190 +++++++++++++++---- proto/tendermint/blocksync/types.proto | 13 +- spec/p2p/messages/block-sync.md | 14 +- types/part_set.go | 50 ++++- types/part_set_test.go | 51 +++++ 13 files changed, 703 insertions(+), 136 deletions(-) diff --git a/internal/blocksync/applier.go b/internal/blocksync/applier.go index 66e7c61fa5..9a5651f575 100644 --- a/internal/blocksync/applier.go +++ b/internal/blocksync/applier.go @@ -69,8 +69,17 @@ func newBlockApplier(blockExec sm.Executor, store sm.BlockStore, opts ...applier return applier } -// Apply safely verifies, saves to the store and executes a block with commit -func (e *blockApplier) Apply(ctx context.Context, block *types.Block, commit *types.Commit) error { +// Apply safely verifies, saves to the store and executes a block with commit. +// blockBytes are the bytes block was decoded from, the parts it was committed +// with in order, and partSizes the size of each part, or nil for parts of +// types.BlockPartSizeBytes. +func (e *blockApplier) Apply( + ctx context.Context, + block *types.Block, + blockBytes []byte, + partSizes []uint32, + commit *types.Commit, +) error { e.mtx.Lock() defer e.mtx.Unlock() defer func() { e.lastDone = time.Now() }() @@ -83,10 +92,13 @@ func (e *blockApplier) Apply(ctx context.Context, block *types.Block, commit *ty } // The part set is needed twice: to derive the block ID and to persist the - // block. Building it serializes the whole block and builds a Merkle tree over - // the parts, so build it once and pass it to both. + // block. Building it builds a Merkle tree over the parts, so build it once + // and pass it to both. It is rebuilt from the bytes the block was committed + // with, not from block: the commit's part set header is computed over + // those, and a re-encoding of block drops unknown fields and non-canonical + // encoding. start := time.Now() - blockParts, err := block.MakePartSet(types.BlockPartSizeBytes) + blockParts, err := committedPartSet(blockBytes, partSizes) if err != nil { return err } @@ -208,6 +220,16 @@ func (e *blockApplier) verify( return verified, nil } +// committedPartSet returns the part set a block was committed with, from its +// bytes and the size of each part, or nil sizes for parts of +// types.BlockPartSizeBytes. +func committedPartSet(blockBytes []byte, partSizes []uint32) (*types.PartSet, error) { + if len(partSizes) == 0 { + return types.NewPartSetFromData(blockBytes, types.BlockPartSizeBytes), nil + } + return types.NewPartSetFromPartSizes(blockBytes, partSizes) +} + // observeSince records the time since start under stage and returns it, so the // same measurement can also feed the per-block averages in applyStats. func (e *blockApplier) observeSince(stage string, start time.Time) time.Duration { @@ -219,7 +241,7 @@ func (e *blockApplier) observeSince(stage string, start time.Time) time.Duration // applyTimings is the average time a single block spends in each stage of the // apply pipeline. type applyTimings struct { - // PartSet is block serialization plus the Merkle tree over its parts + // PartSet is the Merkle tree over the block's parts PartSet time.Duration // Verify is commit signature verification and block validation Verify time.Duration diff --git a/internal/blocksync/applier_test.go b/internal/blocksync/applier_test.go index 31333ac05f..1507e89790 100644 --- a/internal/blocksync/applier_test.go +++ b/internal/blocksync/applier_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/cosmos/gogoproto/proto" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" @@ -47,7 +48,7 @@ func TestBlockApplierChecksAppResponseBeforeSave(t *testing.T) { executor := sm.NewBlockExecutor(stateStore, client, nil, sm.EmptyEvidencePool{}, blockStore, nil) applier := newBlockApplier(executor, blockStore, applierWithState(initialState)) - require.Panics(t, func() { _ = applier.Apply(ctx, block, commit) }) + require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), nil, commit) }) require.Zero(t, blockStore.Height(), "an inconsistent app response must not advance the block store") require.Equal(t, initialState.LastBlockHeight, applier.State().LastBlockHeight) loaded, err := stateStore.Load() @@ -174,7 +175,7 @@ func TestBlockApplierApply(t *testing.T) { tc.mockFn() } fn := func() { - err := applier.Apply(ctx, tc.block, tc.commit) + err := applier.Apply(ctx, tc.block, blockBytes(t, tc.block), nil, tc.commit) tmrequire.Error(t, tc.wantErr, err) } if tc.wantPanic { @@ -267,7 +268,7 @@ func TestBlockApplierDoesNotSaveBlockRejectedByApp(t *testing.T) { Return(sm.CurrentRoundState{}, errors.New("app rejected the block")) applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState)) - require.Panics(t, func() { _ = applier.Apply(ctx, block, commit) }) + require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), nil, commit) }) } // TestBlockApplierSavesBlockBeforeFinalize pins the order the handshake relies @@ -313,7 +314,7 @@ func TestBlockApplierSavesBlockBeforeFinalize(t *testing.T) { Return(state, nil, nil) applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState)) - require.NoError(t, applier.Apply(ctx, block, commit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit)) require.Equal(t, []string{"process", "verify", "save", "finalize"}, calls) } @@ -349,7 +350,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState), applierWithMetrics(m)) - require.NoError(t, applier.Apply(ctx, blockH1, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, commitH1)) for _, stage := range []string{"partset", "verify_commit", "verify_block", "save", "exec"} { require.Len(t, hist.Samples[stage], 1, "stage %q after the first apply", stage) @@ -360,7 +361,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { // A failed attempt must advance the idle-time boundary as well, so a retry // does not count the failed attempt's wait and verification time again. failureStart := time.Now() - require.Error(t, applier.Apply(ctx, blockH1, new(types.Commit))) + require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, new(types.Commit))) failureEnd := time.Now() failureDone := applier.lastDone require.False(t, failureDone.Before(failureStart), "failed apply must advance lastDone") @@ -369,7 +370,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { // The retry measures only the idle time after the failed attempt. retryStart := time.Now() - require.NoError(t, applier.Apply(ctx, blockH1, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, commitH1)) retryEnd := time.Now() require.Len(t, hist.Samples["wait"], 2) wait := hist.Samples["wait"][1] @@ -400,7 +401,7 @@ func TestBlockApplierVerifyFailureTimesOnlyTheCheckThatRan(t *testing.T) { // A rejected commit must stop before ValidateBlock is reached. mockBlockExec.On("VerifyCommit", initialState, blockH1.BlockID(nil), blockH1.Height, new(types.Commit)). Once().Return(types.VerifiedCommit{}, errors.New("bad signature")) - require.Error(t, applier.Apply(ctx, blockH1, new(types.Commit))) + require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, new(types.Commit))) require.Len(t, hist.Samples["partset"], 1) require.Len(t, hist.Samples["verify_commit"], 1) @@ -444,7 +445,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { blockExec.On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, blockH1ID, blockH1, commitH1, none). Once().Return(initialState, nil, nil) applier := newBlockApplier(blockExec, blockStore, applierWithState(initialState)) - require.NoError(t, applier.Apply(ctx, blockH1, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, commitH1)) return applier, blockExec } // expectH2 expects block 2 to be validated and applied offering lastCommit @@ -460,7 +461,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { t.Run("the next block is offered the previous commit's verification", func(t *testing.T) { applier, blockExec := applyH1(t) expectH2(blockExec, verifiedH1) - require.NoError(t, applier.Apply(ctx, blockH2, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), nil, commitH2)) }) t.Run("a rejected commit does not replace the verification", func(t *testing.T) { @@ -468,17 +469,17 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { bad := new(types.Commit) blockExec.On("VerifyCommit", mock.Anything, blockH2ID, blockH2.Height, bad). Once().Return(none, errors.New("bad signature")) - require.Error(t, applier.Apply(ctx, blockH2, bad)) + require.Error(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), nil, bad)) expectH2(blockExec, verifiedH1) - require.NoError(t, applier.Apply(ctx, blockH2, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), nil, commitH2)) }) t.Run("replacing the state discards the verification", func(t *testing.T) { applier, blockExec := applyH1(t) applier.UpdateState(initialState) expectH2(blockExec, none) - require.NoError(t, applier.Apply(ctx, blockH2, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), nil, commitH2)) }) } @@ -532,9 +533,36 @@ func TestBlockApplierSkipsTheLastCommitItVerified(t *testing.T) { wantSkipped float64 }{{1, 0}, {2, 3}, {3, 6}} { block, _, _, seenCommit := makeNextBlock(ctx, t, applier.State(), privVals[0], step.height, commit) - require.NoError(t, applier.Apply(ctx, block, seenCommit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), nil, seenCommit)) require.Equal(t, step.wantSkipped, skipped.Value(), "LastCommit threshold verifications skipped after applying height %d", step.height) commit = seenCommit } } + +// blockBytes returns the bytes block is committed with when its proposer +// encodes it the canonical way, the bytes a peer serves it as. +func blockBytes(t *testing.T, block *types.Block) []byte { + t.Helper() + pbb, err := block.ToProto() + require.NoError(t, err) + bz, err := proto.Marshal(pbb) + require.NoError(t, err) + return bz +} + +// TestBlockApplierRejectsPartSizesNotCoveringTheBlock checks that part sizes a +// peer sent that do not cut the block's bytes are refused before the block is +// verified or saved. +func TestBlockApplierRejectsPartSizesNotCoveringTheBlock(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) + block, commit := blocks[0], blocks[1].LastCommit + bz := blockBytes(t, block) + applier := newBlockApplier(mocks.NewExecutor(t), mocks.NewBlockStore(t), applierWithState(state)) + + err := applier.Apply(ctx, block, bz, []uint32{uint32(len(bz) - 1)}, commit) + require.ErrorContains(t, err, "part sizes cover") +} diff --git a/internal/blocksync/commit_extensions_test.go b/internal/blocksync/commit_extensions_test.go index 724751c986..082a79f681 100644 --- a/internal/blocksync/commit_extensions_test.go +++ b/internal/blocksync/commit_extensions_test.go @@ -52,11 +52,11 @@ func TestBlockApplierRejectsCommitExtensionsAndRetries(t *testing.T) { metrics := consensus.NopMetrics() metrics.CommitVerifyFailures = counter applier := newBlockApplier(exec, store, applierWithState(initial), applierWithMetrics(metrics)) - require.Error(t, applier.Apply(ctx, block, commit)) + require.Error(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit)) require.Equal(t, float64(1), counter.Value()) require.Equal(t, []string{"process", "verify"}, calls) require.Equal(t, initial.LastBlockHeight, applier.State().LastBlockHeight) - require.NoError(t, applier.Apply(ctx, block, commit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit)) require.Equal(t, float64(1), counter.Value()) require.Equal(t, []string{"process", "verify", "verify", "save", "finalize"}, calls) } @@ -81,7 +81,7 @@ func TestBlockApplierInvalidatesRejectedProposal(t *testing.T) { exec.On("VerifyVoteExtension", mock.Anything, mock.Anything). Twice().Return(errors.New("invalid vote extension")) applier := newBlockApplier(exec, store, applierWithState(initial)) - require.ErrorIs(t, applier.Apply(ctx, block, commit), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit), sm.ErrCommitExtensionsRejected) switch change { case "block": block.AppHash = append(block.AppHash.Copy(), 1) @@ -90,7 +90,7 @@ func TestBlockApplierInvalidatesRejectedProposal(t *testing.T) { case "state": applier.UpdateState(initial) } - require.ErrorIs(t, applier.Apply(ctx, block, commit), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit), sm.ErrCommitExtensionsRejected) }) } } @@ -135,9 +135,9 @@ func TestBlockApplierReprocessesRoundAfterAnother(t *testing.T) { applier := newBlockApplier(exec, store, applierWithState(initial)) for _, round := range []int32{1, 1, 2} { - require.ErrorIs(t, applier.Apply(ctx, block, commitAt(round)), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commitAt(round)), sm.ErrCommitExtensionsRejected) } - require.NotPanics(t, func() { require.NoError(t, applier.Apply(ctx, block, commitAt(1))) }) + require.NotPanics(t, func() { require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commitAt(1))) }) require.Equal(t, []int32{1, 2, 1}, rounds, "a round is processed again only after another one") } diff --git a/internal/blocksync/p2p_msg_handler.go b/internal/blocksync/p2p_msg_handler.go index 9c24512a0b..72ba45ad1d 100644 --- a/internal/blocksync/p2p_msg_handler.go +++ b/internal/blocksync/p2p_msg_handler.go @@ -14,6 +14,7 @@ import ( "github.com/dashpay/tenderdash/internal/p2p/client" sm "github.com/dashpay/tenderdash/internal/state" "github.com/dashpay/tenderdash/libs/log" + tmmath "github.com/dashpay/tenderdash/libs/math" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" "github.com/dashpay/tenderdash/types" ) @@ -155,11 +156,11 @@ func (h *blockP2PMessageHandler) Handle(ctx context.Context, p2pClient *client.C func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (proto.Message, bool, error) { envelope := &job.envelope msg := envelope.Message.(*bcproto.BlockRequest) - block := h.store.LoadBlock(msg.Height) + blockBytes, partSizes := loadCommittedBlock(h.store, msg.Height) if !h.connectionIsCurrent(envelope.From, envelope.ConnID) { return nil, false, nil } - if block == nil { + if blockBytes == nil { h.logger.Debug("peer requesting a block we do not have", "peer", envelope.From, "height", msg.Height) return &bcproto.NoBlockResponse{Height: msg.Height}, false, nil } @@ -167,16 +168,67 @@ func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (pro if commit == nil { return nil, false, fmt.Errorf("%w at height %d", errStoredBlockMissingCommit, msg.Height) } - blockProto, err := block.ToProto() - if err != nil { - return nil, false, fmt.Errorf("failed to convert block to protobuf: %w", err) - } return &bcproto.BlockResponse{ - Block: blockProto, - Commit: commit.ToProto(), + Block: blockBytes, + Commit: commit.ToProto(), + PartSizes: partSizes, }, true, nil } +// loadCommittedBlock returns the block at height as it was committed: the +// bytes of its stored parts, in order, and the size of each part, or nil sizes +// when the parts are cut the way types.NewPartSetFromData cuts them. The +// commit's part set header is computed over those parts, and the peer can only +// rebuild it from them; a re-encoding of the decoded block drops unknown +// fields and non-canonical encoding, and a proposer may cut a block into parts +// of any size. It returns nil bytes when the store does not hold every part. +func loadCommittedBlock(store sm.BlockStore, height int64) ([]byte, []uint32) { + meta := store.LoadBlockMeta(height) + if meta == nil { + return nil, nil + } + total := int(meta.BlockID.PartSetHeader.Total) + // BlockSize is -1 for a header saved without its block + blockBytes := make([]byte, 0, max(meta.BlockSize, 0)) + partSizes := make([]uint32, total) + for i := range total { + part := store.LoadBlockPart(height, i) + // a header saved without its block, or parts pruned after the meta was + // loaded + if part == nil { + return nil, nil + } + blockBytes = append(blockBytes, part.Bytes...) + partSizes[i] = tmmath.MustConvertUint32(len(part.Bytes)) + } + if len(blockBytes) == 0 { + return nil, nil + } + if standardPartSizes(partSizes, len(blockBytes)) { + partSizes = nil + } + return blockBytes, partSizes +} + +// standardPartSizes reports whether parts of the given sizes are how +// types.NewPartSetFromData cuts size bytes into parts of +// types.BlockPartSizeBytes. +func standardPartSizes(partSizes []uint32, size int) bool { + partSize := int(types.BlockPartSizeBytes) + total := (size + partSize - 1) / partSize + if total == 0 || len(partSizes) != total { + return false + } + // the last part takes what is left, and with total parts that is between 1 + // and partSize bytes + for _, s := range partSizes[:total-1] { + if s != types.BlockPartSizeBytes { + return false + } + } + return true +} + func (h *blockP2PMessageHandler) enqueueBlockResponse(job blockResponseJob) { peerID := job.envelope.From h.responseMtx.Lock() diff --git a/internal/blocksync/p2p_msg_handler_test.go b/internal/blocksync/p2p_msg_handler_test.go index 365c746272..4cae5cccf6 100644 --- a/internal/blocksync/p2p_msg_handler_test.go +++ b/internal/blocksync/p2p_msg_handler_test.go @@ -2,6 +2,7 @@ package blocksync import ( "context" + "encoding/binary" "fmt" "testing" "time" @@ -9,6 +10,7 @@ import ( "github.com/cosmos/gogoproto/proto" "github.com/google/uuid" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/dashpay/tenderdash/config" @@ -16,6 +18,10 @@ import ( "github.com/dashpay/tenderdash/internal/p2p/client" p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" "github.com/dashpay/tenderdash/internal/state/mocks" + statefactory "github.com/dashpay/tenderdash/internal/state/test/factory" + "github.com/dashpay/tenderdash/internal/store" + "github.com/dashpay/tenderdash/internal/test/factory" + dbm "github.com/dashpay/tenderdash/libs/db" "github.com/dashpay/tenderdash/libs/log" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" "github.com/dashpay/tenderdash/types" @@ -62,9 +68,6 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { ctx := context.Background() peerID := types.NodeID("peer") const H1001 = int64(1001) - blockH1001 := types.Block{Header: types.Header{Height: 1001}} - protoBlockH1001, err := blockH1001.ToProto() - suite.Require().NoError(err) commit1001 := types.Commit{Height: 1001} protoCommit := commit1001.ToProto() blockRequestH1001 := &bcproto.BlockRequest{Height: H1001} @@ -75,7 +78,7 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { { mockFn: func() { suite.fakeStore. - On("LoadBlock", H1001). + On("LoadBlockMeta", H1001). Once(). Return(nil) }, @@ -83,10 +86,7 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { }, { mockFn: func() { - suite.fakeStore. - On("LoadBlock", H1001). - Once(). - Return(&blockH1001) + suite.expectStoredBlock(H1001, 1) suite.fakeStore. On("LoadSeenCommitAt", H1001). Once(). @@ -95,17 +95,14 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { }, { mockFn: func() { - suite.fakeStore. - On("LoadBlock", H1001). - Once(). - Return(&blockH1001) + suite.expectStoredBlock(H1001, 1) suite.fakeStore. On("LoadSeenCommitAt", H1001). Once(). Return(&commit1001) }, wantResp: &bcproto.BlockResponse{ - Block: protoBlockH1001, + Block: suite.storedBlockBytes(H1001), Commit: protoCommit, }, }, @@ -128,10 +125,27 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestHandleBlockRequest() { } } +// expectStoredBlock makes the store serve the block at height, in one part, +// times times. It returns the call that loads the block's meta. +func (suite *BlockP2PMessageHandlerTestSuite) expectStoredBlock(height int64, times int) *mock.Call { + blockBytes := suite.storedBlockBytes(height) + meta := &types.BlockMeta{ + BlockID: types.BlockID{PartSetHeader: types.PartSetHeader{Total: 1}}, + BlockSize: len(blockBytes), + } + suite.fakeStore.On("LoadBlockPart", height, 0).Times(times).Return(&types.Part{Bytes: blockBytes}) + return suite.fakeStore.On("LoadBlockMeta", height).Times(times).Return(meta) +} + +// storedBlockBytes returns the bytes of the block expectStoredBlock stores at +// height. +func (suite *BlockP2PMessageHandlerTestSuite) storedBlockBytes(height int64) []byte { + return blockBytes(suite.T(), &types.Block{Header: types.Header{Height: height}}) +} + func (suite *BlockP2PMessageHandlerTestSuite) TestServeBlockRequestReportsMissingCommit() { suite.handler.handlePeerUpdate(p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusUp, ConnID: 1}) - block := &types.Block{Header: types.Header{Height: 1001}} - suite.fakeStore.On("LoadBlock", int64(1001)).Once().Return(block) + suite.expectStoredBlock(1001, 1) suite.fakeStore.On("LoadSeenCommitAt", int64(1001)).Once().Return(nil) _, _, err := suite.handler.prepareBlockResponse(blockResponseJob{ @@ -150,9 +164,8 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestBlockResponseBackpressureAppli defer cancel() peerID := types.NodeID("slow-peer") - block := &types.Block{Header: types.Header{Height: 1001}} commit := &types.Commit{Height: 1001} - suite.fakeStore.On("LoadBlock", int64(1001)).Times(3).Return(block) + suite.expectStoredBlock(1001, 3) suite.fakeStore.On("LoadSeenCommitAt", int64(1001)).Times(3).Return(commit) delivery := make(chan p2p.Envelope, 3) @@ -183,10 +196,9 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestStalledPeerDeliveryDoesNotBloc ctx, cancel := context.WithCancel(context.Background()) defer cancel() - block := &types.Block{Header: types.Header{Height: 1001}} commit := &types.Commit{Height: 1001} totalRequests := maxInFlightBlockResponses + 1 - suite.fakeStore.On("LoadBlock", int64(1001)).Times(totalRequests).Return(block) + suite.expectStoredBlock(1001, totalRequests) suite.fakeStore.On("LoadSeenCommitAt", int64(1001)).Times(totalRequests).Return(commit) delivery := make(chan p2p.Envelope, totalRequests) @@ -243,9 +255,8 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestDisconnectedPeerQueueIsPurgedB peerID := types.NodeID("disconnecting-peer") const connID = uint64(7) - block := &types.Block{Header: types.Header{Height: 1001}} commit := &types.Commit{Height: 1001} - suite.fakeStore.On("LoadBlock", int64(1001)).Once().Return(block) + suite.expectStoredBlock(1001, 1) suite.fakeStore.On("LoadSeenCommitAt", int64(1001)).Once().Return(commit) delivery := make(chan p2p.Envelope, 1) @@ -291,14 +302,11 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestDisconnectDuringLoadSkipsRespo const connID = uint64(8) started := make(chan struct{}) release := make(chan struct{}) - suite.fakeStore. - On("LoadBlock", int64(1001)). - Once(). + suite.expectStoredBlock(1001, 1). Run(func(mock.Arguments) { close(started) <-release - }). - Return(&types.Block{Header: types.Header{Height: 1001}}) + }) suite.handler.handlePeerUpdate(p2p.PeerUpdate{ NodeID: peerID, @@ -466,7 +474,7 @@ func newMockPeerAdder(t *testing.T) *mockPeerAdder { func (suite *BlockP2PMessageHandlerTestSuite) TestRequestWaitsForConnectionUpdate() { peerID := types.NodeID("early-peer") - suite.fakeStore.On("LoadBlock", int64(1001)).Once().Return(nil) + suite.fakeStore.On("LoadBlockMeta", int64(1001)).Once().Return(nil) delivered := make(chan struct{}, 1) suite.fakeP2PChannel.On("Send", mock.Anything, mock.Anything).Once().Run(func(mock.Arguments) { delivered <- struct{}{} }).Return(nil) suite.Require().NoError(suite.handler.Handle(context.Background(), suite.fakeClient, &p2p.Envelope{ @@ -512,3 +520,120 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestReconnectRequestSurvivesEarlie suite.Equal(uint64(11), job.envelope.ConnID) suite.Zero(h.queuedResponses) } + +// TestServeBlockAsCommitted checks that a block is served as the parts it was +// stored with, which are the parts it was committed with, and that the part set +// rebuilt from the response is the stored one. +func TestServeBlockAsCommitted(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 5, &state, privVals, 1) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + + const smallPartSize = 100 + standard := blockBytes(t, blocks[0]) + unknownField := withUnknownField(t, blocks[1]) + smallParts := blockBytes(t, blocks[2]) + testCases := []struct { + parts *types.PartSet + wantBytes []byte + wantPartSizes []uint32 + }{ + {types.NewPartSetFromData(standard, types.BlockPartSizeBytes), standard, nil}, + {types.NewPartSetFromData(unknownField, types.BlockPartSizeBytes), unknownField, nil}, + { + types.NewPartSetFromData(smallParts, smallPartSize), + smallParts, + smallPartSizes(len(smallParts), smallPartSize), + }, + } + for i, tc := range testCases { + blockStore.SaveBlock(blocks[i], tc.parts, blocks[i+1].LastCommit) + } + // a header stored without its block, as state sync stores the one it + // restores the state at + header := blocks[3] + require.NoError(t, blockStore.SaveSignedHeader( + &types.SignedHeader{Header: &header.Header, Commit: blocks[4].LastCommit}, + blocks[4].LastCommit.BlockID, + )) + + handler := newBlockP2PMessageHandler(ctx, log.NewNopLogger(), blockStore, newMockPeerAdder(t)) + t.Cleanup(handler.stop) + handler.handlePeerUpdate(p2p.PeerUpdate{NodeID: "peer", Status: p2p.PeerStatusUp, ConnID: 1}) + request := func(height int64) proto.Message { + msg, _, err := handler.prepareBlockResponse(blockResponseJob{ + ctx: ctx, + envelope: p2p.Envelope{From: "peer", ConnID: 1, Message: &bcproto.BlockRequest{Height: height}}, + }) + require.NoError(t, err) + return msg + } + + for i, tc := range testCases { + height := blocks[i].Height + t.Run(fmt.Sprintf("height %d", height), func(t *testing.T) { + msg, ok := request(height).(*bcproto.BlockResponse) + require.True(t, ok) + require.Equal(t, tc.wantBytes, msg.Block) + require.Equal(t, tc.wantPartSizes, msg.PartSizes) + + resp, err := BlockResponseFromProto(msg, "peer") + require.NoError(t, err) + require.Equal(t, blocks[i].Hash(), resp.Block.Hash()) + require.Equal(t, blocks[i+1].LastCommit.BlockID, resp.Commit.BlockID) + parts, err := committedPartSet(resp.BlockBytes, resp.PartSizes) + require.NoError(t, err) + require.Equal(t, tc.parts.Header(), parts.Header()) + }) + } + require.Equal(t, &bcproto.NoBlockResponse{Height: header.Height}, request(header.Height)) +} + +// smallPartSizes returns the sizes of the parts types.NewPartSetFromData cuts +// size bytes into with parts of partSize. +func smallPartSizes(size, partSize int) []uint32 { + var sizes []uint32 + for ; size > partSize; size -= partSize { + sizes = append(sizes, uint32(partSize)) + } + return append(sizes, uint32(size)) +} + +// TestBlockResponseWireCompatible checks that a BlockResponse encodes as it did +// when its block field was an embedded tendermint.types.Block, so peers on +// either side of the change read what the other sends. +func TestBlockResponseWireCompatible(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) + block, commit := blocks[0], blocks[1].LastCommit.ToProto() + pbb, err := block.ToProto() + require.NoError(t, err) + commitBytes, err := proto.Marshal(commit) + require.NoError(t, err) + + // an embedded message is its encoding, length-delimited + field := func(bz []byte, number byte, value []byte) []byte { + bz = append(bz, number<<3|2) + bz = binary.AppendUvarint(bz, uint64(len(value))) + return append(bz, value...) + } + embedded, err := proto.Marshal(pbb) + require.NoError(t, err) + older := field(field(nil, 1, embedded), 2, commitBytes) + + newer, err := proto.Marshal(&bcproto.BlockResponse{Block: blockBytes(t, block), Commit: commit}) + require.NoError(t, err) + require.Equal(t, older, newer) + + var decoded bcproto.BlockResponse + require.NoError(t, proto.Unmarshal(older, &decoded)) + resp, err := BlockResponseFromProto(&decoded, "peer") + require.NoError(t, err) + require.Equal(t, block.Hash(), resp.Block.Hash()) + require.Equal(t, embedded, resp.BlockBytes) + require.Nil(t, resp.PartSizes) +} diff --git a/internal/blocksync/reactor_test.go b/internal/blocksync/reactor_test.go index 5197cfd830..bed4681b80 100644 --- a/internal/blocksync/reactor_test.go +++ b/internal/blocksync/reactor_test.go @@ -43,8 +43,15 @@ type reactorTestSuite struct { blockSyncChannels map[types.NodeID]p2p.Channel peerChans map[types.NodeID]chan p2p.PeerUpdate peerUpdates map[types.NodeID]*p2p.PeerUpdates + + // blockParts returns the parts the nodes commit their blocks with + blockParts blockPartsFunc } +// blockPartsFunc returns the parts a block at height is committed with, or nil +// for the parts types.Block.MakePartSet cuts it into. +type blockPartsFunc func(t *testing.T, height int64, block *types.Block) *types.PartSet + func setup( ctx context.Context, t *testing.T, @@ -55,6 +62,22 @@ func setup( opts ...ReactorOption, ) *reactorTestSuite { t.Helper() + return setupWithBlockParts(ctx, t, conf, genDoc, privVal, maxBlockHeights, nil, opts...) +} + +// setupWithBlockParts is setup with the nodes committing their blocks with the +// parts blockParts returns. +func setupWithBlockParts( + ctx context.Context, + t *testing.T, + conf *config.Config, + genDoc *types.GenesisDoc, + privVal types.PrivValidator, + maxBlockHeights []int64, + blockParts blockPartsFunc, + opts ...ReactorOption, +) *reactorTestSuite { + t.Helper() var cancel context.CancelFunc ctx, cancel = context.WithCancel(ctx) @@ -72,6 +95,7 @@ func setup( blockSyncChannels: make(map[types.NodeID]p2p.Channel, numNodes), peerChans: make(map[types.NodeID]chan p2p.PeerUpdate, numNodes), peerUpdates: make(map[types.NodeID]*p2p.PeerUpdates, numNodes), + blockParts: blockParts, } chDesc := &p2p.ChannelDescriptor{ID: p2p.BlockSyncChannel} @@ -199,7 +223,8 @@ func (rts *reactorTestSuite) addNode( state, err := reactor.stateStore.Load() require.NoError(t, err) for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ { - block, blockID, partSet, seenCommit := makeNextBlock(ctx, t, state, privVal, blockHeight, commit) + block, blockID, partSet, seenCommit := makeNextBlockWithParts(ctx, t, state, privVal, blockHeight, commit, + rts.blockParts) state, err = reactor.blockExec.ApplyBlock(ctx, state, blockID, block, seenCommit, types.VerifiedCommit{}) require.NoError(t, err) reactor.store.SaveBlock(block, partSet, seenCommit) @@ -217,13 +242,30 @@ func makeNextBlock(ctx context.Context, signer types.PrivValidator, height int64, commit *types.Commit) (*types.Block, types.BlockID, *types.PartSet, *types.Commit) { + return makeNextBlockWithParts(ctx, t, state, signer, height, commit, nil) +} + +// makeNextBlockWithParts is makeNextBlock with the block committed with the +// parts blockParts returns. +func makeNextBlockWithParts(ctx context.Context, + t *testing.T, + state sm.State, + signer types.PrivValidator, + height int64, + commit *types.Commit, + blockParts blockPartsFunc) (*types.Block, types.BlockID, *types.PartSet, *types.Commit) { block, err := sf.MakeBlock(state, height, commit, 0) require.NoError(t, err) block.CoreChainLockedHeight = state.LastCoreChainLockedBlockHeight - partSet, err := block.MakePartSet(types.BlockPartSizeBytes) - require.NoError(t, err) + var partSet *types.PartSet + if blockParts != nil { + partSet = blockParts(t, height, block) + } + if partSet == nil { + partSet, err = block.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + } blockID := block.BlockID(partSet) - require.NoError(t, err) // Simulate a commit for the current height vote, err := factory.MakeVote( @@ -416,6 +458,75 @@ func TestReactor_NoBlockResponse(t *testing.T) { } } +// TestReactor_SyncBlocksAsCommitted syncs blocks committed with bytes or parts +// that types.Block.MakePartSet does not reproduce from the decoded block. The +// commit's part set header is computed over the committed parts, so a peer can +// only serve such a block from the parts it stored, and the syncing node must +// rebuild the part set from the bytes it received. +func TestReactor_SyncBlocksAsCommitted(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cfg, err := config.ResetTestRoot(t.TempDir(), t.Name()) + require.NoError(t, err) + defer os.RemoveAll(cfg.RootDir) + + genDoc, privVals := factory.RandGenesisDoc(1, factory.ConsensusParams()) + const ( + maxBlockHeight = int64(10) + // a field that tendermint.types.Block does not define, which decoding + // skips, appended to the block's bytes + unknownFieldHeight = int64(3) + // the block's bytes cut into parts shorter than types.BlockPartSizeBytes + smallPartsHeight = int64(5) + bothHeight = int64(7) + smallPartSize = 100 + ) + blockParts := func(t *testing.T, height int64, block *types.Block) *types.PartSet { + switch height { + case unknownFieldHeight: + return types.NewPartSetFromData(withUnknownField(t, block), types.BlockPartSizeBytes) + case smallPartsHeight: + return types.NewPartSetFromData(blockBytes(t, block), smallPartSize) + case bothHeight: + return types.NewPartSetFromData(withUnknownField(t, block), smallPartSize) + } + return nil + } + rts := setupWithBlockParts(ctx, t, cfg, genDoc, privVals[0], []int64{maxBlockHeight, 0}, blockParts) + source, syncing := rts.reactors[rts.nodes[0]], rts.reactors[rts.nodes[1]] + require.Equal(t, maxBlockHeight, source.store.Height()) + for _, height := range []int64{unknownFieldHeight, smallPartsHeight, bothHeight} { + require.NotEqual(t, + source.store.LoadBlockMeta(height).BlockID.PartSetHeader, + source.store.LoadBlock(height).BlockID(nil).PartSetHeader, + "height %d must be committed with parts a re-encoding does not reproduce", height) + } + + rts.start(ctx, t) + + require.Eventually(t, + func() bool { return syncing.store.Height() > bothHeight }, + 30*time.Second, + 10*time.Millisecond, + "expected node to sync the blocks committed with other bytes") + for _, height := range []int64{unknownFieldHeight, smallPartsHeight, bothHeight} { + require.Equal(t, source.store.LoadBlockMeta(height).BlockID, syncing.store.LoadBlockMeta(height).BlockID, + "block ID at height %d", height) + wantBytes, wantSizes := loadCommittedBlock(source.store, height) + gotBytes, gotSizes := loadCommittedBlock(syncing.store, height) + require.Equal(t, wantBytes, gotBytes, "stored block bytes at height %d", height) + require.Equal(t, wantSizes, gotSizes, "stored part sizes at height %d", height) + } +} + +// withUnknownField returns the bytes of block with field 15 appended, a field +// tendermint.types.Block does not define and decoding skips. +func withUnknownField(t *testing.T, block *types.Block) []byte { + // tag (15 << 3 | varint wire type), then the value 1 + return append(blockBytes(t, block), 15<<3, 1) +} + func TestReactor_BadBlockStopsPeer(t *testing.T) { // Ultimately, this should be refactored to be less integration test oriented // and more unit test oriented by simply testing channel sends and receives. diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index 1c214d0511..6802e05040 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "time" + "github.com/cosmos/gogoproto/proto" "github.com/jonboulle/clockwork" sync "github.com/sasha-s/go-deadlock" @@ -16,6 +17,7 @@ import ( "github.com/dashpay/tenderdash/libs/service" "github.com/dashpay/tenderdash/libs/workerpool" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -44,10 +46,11 @@ const ( // that when one block may be tens of megabytes. // // Serialized bytes understate the heap actually retained. The size covers the - // block alone, not the commit held alongside it, and a decoded block costs - // more than its wire form - several times more for blocks small enough that - // fixed per-response overhead dominates. Read this as a budget for block - // data, not as the memory the backlog occupies. + // block alone, not the commit held alongside it, and a response holds the + // decoded block next to its wire form, the bytes it is applied from. A + // decoded block costs more than its wire form - several times more for + // blocks small enough that fixed per-response overhead dominates. Read this + // as a budget for block data, not as the memory the backlog occupies. maxPendingApplyBytes = 256 << 20 // maxOutstandingHeights is how many heights may be requested and not yet @@ -637,7 +640,7 @@ func (s *Synchronizer) applyBlock(ctx context.Context) (BlockResponse, error) { if !ok { return BlockResponse{}, nil } - err := s.applier.Apply(ctx, resp.Block, resp.Commit) + err := s.applier.Apply(ctx, resp.Block, resp.BlockBytes, resp.PartSizes, resp.Commit) if err != nil { return resp, fmt.Errorf("cannot apply block: %w", err) } @@ -776,10 +779,15 @@ type BlockResponse struct { // its failures are fenced to. ConnID uint64 Block *types.Block - Commit *types.Commit - // Size is the serialized size of Block in bytes, measured while decoding the - // response. Deriving it from Block again means re-serializing the whole - // block, which is too expensive to do on the block apply path. + // BlockBytes is Block as it was committed, the bytes of its parts in order, + // and PartSizes the size of each part, or nil for parts of + // types.BlockPartSizeBytes. Block is decoded from BlockBytes, and applied + // with the part set rebuilt from them: that is what the commit's block ID + // describes, and re-encoding Block need not give the same bytes. + BlockBytes []byte + PartSizes []uint32 + Commit *types.Commit + // Size is the serialized size of Block in bytes, the length of BlockBytes. Size int } @@ -803,7 +811,14 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* if resp == nil { return nil, errors.New("invalid") } - block, err := types.BlockFromProto(resp.Block) + if len(resp.Block) == 0 { + return nil, errors.New("block response without a block") + } + pbb := new(tmproto.Block) + if err := proto.Unmarshal(resp.Block, pbb); err != nil { + return nil, fmt.Errorf("cannot decode block: %w", err) + } + block, err := types.BlockFromProto(pbb) if err != nil { return nil, err } @@ -815,11 +830,11 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* } } return &BlockResponse{ - PeerID: peerID, - Block: block, - Commit: commit, - // measured here, on a worker goroutine, rather than on the single - // goroutine that applies blocks - Size: resp.Block.Size(), + PeerID: peerID, + Block: block, + BlockBytes: resp.Block, + PartSizes: resp.PartSizes, + Commit: commit, + Size: len(resp.Block), }, nil } diff --git a/internal/blocksync/synchronizer_test.go b/internal/blocksync/synchronizer_test.go index fb0a92363e..348379f547 100644 --- a/internal/blocksync/synchronizer_test.go +++ b/internal/blocksync/synchronizer_test.go @@ -1361,10 +1361,8 @@ func (suite *SynchronizerTestSuite) TestStatusRefreshUpdatesAdvertisedRange() { func generateBlockResponses(t *testing.T, blocks []*types.Block) []*blocksync.BlockResponse { responses := make([]*blocksync.BlockResponse, 0, len(blocks)-1) for i := 0; i < len(blocks)-1; i++ { - protoBlock, err := blocks[i].ToProto() - require.NoError(t, err) responses = append(responses, &blocksync.BlockResponse{ - Block: protoBlock, + Block: blockBytes(t, blocks[i]), Commit: blocks[i+1].LastCommit.ToProto(), }) } @@ -2214,7 +2212,7 @@ func (suite *SynchronizerTestSuite) TestClientTimeoutUnwedgesAFullWindow() { // Everything above the dropped height fills the window and stops there. wantBytes := 0 for height := blocking + 1; height < blocking+maxOutstandingHeights; height++ { - wantBytes += chain[height-1].Block.Size() + wantBytes += len(chain[height-1].Block) } suite.Require().Eventually(func() bool { _, pendingBytes := pool.backlog() diff --git a/proto/tendermint/blocksync/types.pb.go b/proto/tendermint/blocksync/types.pb.go index 5bb19dc5cb..7a95249606 100644 --- a/proto/tendermint/blocksync/types.pb.go +++ b/proto/tendermint/blocksync/types.pb.go @@ -116,8 +116,18 @@ func (m *NoBlockResponse) GetHeight() int64 { // BlockResponse returns block to the requested type BlockResponse struct { - Block *types.Block `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` + // block is a serialized tendermint.types.Block: the bytes of the parts it + // was committed with, in order. The commit's part set header is computed + // over those bytes, and decoding and re-encoding the block can change them + // (unknown fields, non-canonical encoding). On the wire this is the same as + // an embedded tendermint.types.Block, the type of this field in older + // releases. + Block []byte `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` Commit *types.Commit `protobuf:"bytes,2,opt,name=commit,proto3" json:"commit,omitempty"` + // part_sizes is the size of each part the block was committed with, in + // order. It is empty when the block was cut into parts of 64kB, the last + // one shorter, the way honest proposers cut it. + PartSizes []uint32 `protobuf:"varint,3,rep,packed,name=part_sizes,json=partSizes,proto3" json:"part_sizes,omitempty"` } func (m *BlockResponse) Reset() { *m = BlockResponse{} } @@ -153,7 +163,7 @@ func (m *BlockResponse) XXX_DiscardUnknown() { var xxx_messageInfo_BlockResponse proto.InternalMessageInfo -func (m *BlockResponse) GetBlock() *types.Block { +func (m *BlockResponse) GetBlock() []byte { if m != nil { return m.Block } @@ -167,6 +177,13 @@ func (m *BlockResponse) GetCommit() *types.Commit { return nil } +func (m *BlockResponse) GetPartSizes() []uint32 { + if m != nil { + return m.PartSizes + } + return nil +} + // StatusRequest requests the status of a peer. type StatusRequest struct { } @@ -268,24 +285,25 @@ func init() { func init() { proto.RegisterFile("tendermint/blocksync/types.proto", fileDescriptor_19b397c236e0fa07) } var fileDescriptor_19b397c236e0fa07 = []byte{ - // 272 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x28, 0x49, 0xcd, 0x4b, - 0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x4f, 0xca, 0xc9, 0x4f, 0xce, 0x2e, 0xae, 0xcc, 0x4b, - 0xd6, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x41, 0xa8, - 0xd0, 0x83, 0xab, 0x90, 0x92, 0x41, 0xd2, 0x07, 0x56, 0x0d, 0xd1, 0x0d, 0xd1, 0x83, 0x45, 0x16, - 0xc9, 0x44, 0x25, 0x35, 0x2e, 0x1e, 0x27, 0x90, 0xe2, 0xa0, 0xd4, 0xc2, 0xd2, 0xd4, 0xe2, 0x12, - 0x21, 0x31, 0x2e, 0xb6, 0x8c, 0xd4, 0xcc, 0xf4, 0x8c, 0x12, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xe6, - 0x20, 0x28, 0x4f, 0x49, 0x93, 0x8b, 0xdf, 0x2f, 0x1f, 0xaa, 0xb2, 0xb8, 0x20, 0x3f, 0xaf, 0x38, - 0x15, 0xa7, 0xd2, 0x02, 0x2e, 0x5e, 0x54, 0x85, 0xba, 0x5c, 0xac, 0x60, 0x07, 0x81, 0xd5, 0x71, - 0x1b, 0x89, 0xeb, 0x21, 0xf9, 0x02, 0xe2, 0x16, 0x88, 0x7a, 0x88, 0x2a, 0x21, 0x03, 0x2e, 0xb6, - 0xe4, 0xfc, 0xdc, 0xdc, 0xcc, 0x12, 0x09, 0x26, 0xb0, 0x7a, 0x09, 0x4c, 0xf5, 0xce, 0x60, 0xf9, - 0x20, 0xa8, 0x3a, 0x25, 0x7e, 0x2e, 0xde, 0xe0, 0x92, 0xc4, 0x92, 0xd2, 0x62, 0xa8, 0x2f, 0x94, - 0x6c, 0xb8, 0xf8, 0x60, 0x02, 0xf8, 0x1d, 0x2b, 0x24, 0xc4, 0xc5, 0x92, 0x94, 0x58, 0x9c, 0x0a, - 0xb6, 0x8a, 0x39, 0x08, 0xcc, 0x76, 0x0a, 0x3a, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 0xc6, - 0x07, 0x8f, 0xe4, 0x18, 0x27, 0x3c, 0x96, 0x63, 0xb8, 0xf0, 0x58, 0x8e, 0xe1, 0xc6, 0x63, 0x39, - 0x86, 0x28, 0x8b, 0xf4, 0xcc, 0x92, 0x8c, 0xd2, 0x24, 0xbd, 0xe4, 0xfc, 0x5c, 0xfd, 0x94, 0xc4, - 0xe2, 0x8c, 0x82, 0xc4, 0x4a, 0x7d, 0x88, 0xe3, 0x40, 0x3c, 0x7d, 0x70, 0x90, 0xea, 0x63, 0x8b, - 0xc5, 0x24, 0x36, 0xb0, 0x9c, 0x31, 0x60, 0x00, 0x3d, 0x3d, 0xc3, 0xea, 0xe4, 0x01, 0x00, 0x00, + // 287 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0x31, 0x4b, 0xc4, 0x30, + 0x14, 0xc7, 0x2f, 0x56, 0x0f, 0x7c, 0xde, 0x79, 0x10, 0x0e, 0x29, 0xa2, 0xa1, 0x74, 0x90, 0xba, + 0x34, 0xa2, 0x8b, 0x83, 0xd3, 0xb9, 0x3b, 0xe4, 0x36, 0x17, 0x69, 0x7b, 0xe1, 0x5a, 0xb4, 0x4d, + 0x6d, 0x5e, 0x85, 0xfa, 0x29, 0xfc, 0x58, 0x8e, 0x37, 0x3a, 0x4a, 0xfb, 0x45, 0xa4, 0x69, 0xd5, + 0x0a, 0xea, 0x96, 0xff, 0xff, 0xfd, 0x1e, 0xfc, 0x92, 0x80, 0x83, 0x32, 0x5b, 0xc9, 0x22, 0x4d, + 0x32, 0xe4, 0xe1, 0x83, 0x8a, 0xee, 0x75, 0x95, 0x45, 0x1c, 0xab, 0x5c, 0x6a, 0x3f, 0x2f, 0x14, + 0x2a, 0x3a, 0xff, 0x26, 0xfc, 0x2f, 0xe2, 0xf0, 0x68, 0xb0, 0x67, 0xe8, 0xe1, 0x8e, 0x7b, 0x02, + 0x93, 0x45, 0x8b, 0x0a, 0xf9, 0x58, 0x4a, 0x8d, 0xf4, 0x00, 0xc6, 0xb1, 0x4c, 0xd6, 0x31, 0xda, + 0xc4, 0x21, 0x9e, 0x25, 0xfa, 0xe4, 0x9e, 0xc2, 0xec, 0x46, 0xf5, 0xa4, 0xce, 0x55, 0xa6, 0xe5, + 0x9f, 0xe8, 0x13, 0x4c, 0x7f, 0x82, 0x73, 0xd8, 0x31, 0x3a, 0x86, 0x9b, 0x88, 0x2e, 0xd0, 0x33, + 0x18, 0x47, 0x2a, 0x4d, 0x13, 0xb4, 0xb7, 0x1c, 0xe2, 0xed, 0x9d, 0xdb, 0xfe, 0x40, 0xbf, 0x53, + 0xbc, 0x36, 0x73, 0xd1, 0x73, 0xf4, 0x18, 0x20, 0x0f, 0x0a, 0xbc, 0xd3, 0xc9, 0xb3, 0xd4, 0xb6, + 0xe5, 0x58, 0xde, 0x54, 0xec, 0xb6, 0xcd, 0xb2, 0x2d, 0xdc, 0x19, 0x4c, 0x97, 0x18, 0x60, 0xa9, + 0xfb, 0xbb, 0xb8, 0x57, 0xb0, 0xff, 0x59, 0xfc, 0xaf, 0x4c, 0x29, 0x6c, 0x87, 0x81, 0x96, 0xc6, + 0xc4, 0x12, 0xe6, 0xbc, 0x10, 0xaf, 0x35, 0x23, 0x9b, 0x9a, 0x91, 0xf7, 0x9a, 0x91, 0x97, 0x86, + 0x8d, 0x36, 0x0d, 0x1b, 0xbd, 0x35, 0x6c, 0x74, 0x7b, 0xb9, 0x4e, 0x30, 0x2e, 0x43, 0x3f, 0x52, + 0x29, 0x5f, 0x05, 0x3a, 0xce, 0x83, 0x8a, 0x77, 0xee, 0x6d, 0xe2, 0xe6, 0x61, 0xf9, 0x6f, 0xbf, + 0x15, 0x8e, 0xcd, 0xec, 0xe2, 0x63, 0x00, 0xd3, 0xa1, 0x58, 0x18, 0xcc, 0x01, 0x00, 0x00, } func (m *BlockRequest) Marshal() (dAtA []byte, err error) { @@ -373,6 +391,24 @@ func (m *BlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.PartSizes) > 0 { + dAtA2 := make([]byte, len(m.PartSizes)*10) + var j1 int + for _, num := range m.PartSizes { + for num >= 1<<7 { + dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j1++ + } + dAtA2[j1] = uint8(num) + j1++ + } + i -= j1 + copy(dAtA[i:], dAtA2[:j1]) + i = encodeVarintTypes(dAtA, i, uint64(j1)) + i-- + dAtA[i] = 0x1a + } if m.Commit != nil { { size, err := m.Commit.MarshalToSizedBuffer(dAtA[:i]) @@ -385,15 +421,10 @@ func (m *BlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x12 } - if m.Block != nil { - { - size, err := m.Block.MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintTypes(dAtA, i, uint64(size)) - } + if len(m.Block) > 0 { + i -= len(m.Block) + copy(dAtA[i:], m.Block) + i = encodeVarintTypes(dAtA, i, uint64(len(m.Block))) i-- dAtA[i] = 0xa } @@ -503,14 +534,21 @@ func (m *BlockResponse) Size() (n int) { } var l int _ = l - if m.Block != nil { - l = m.Block.Size() + l = len(m.Block) + if l > 0 { n += 1 + l + sovTypes(uint64(l)) } if m.Commit != nil { l = m.Commit.Size() n += 1 + l + sovTypes(uint64(l)) } + if len(m.PartSizes) > 0 { + l = 0 + for _, e := range m.PartSizes { + l += sovTypes(uint64(e)) + } + n += 1 + sovTypes(uint64(l)) + l + } return n } @@ -715,7 +753,7 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType) } - var msglen int + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowTypes @@ -725,26 +763,24 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + byteLen |= int(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + if byteLen < 0 { return ErrInvalidLengthTypes } - postIndex := iNdEx + msglen + postIndex := iNdEx + byteLen if postIndex < 0 { return ErrInvalidLengthTypes } if postIndex > l { return io.ErrUnexpectedEOF } + m.Block = append(m.Block[:0], dAtA[iNdEx:postIndex]...) if m.Block == nil { - m.Block = &types.Block{} - } - if err := m.Block.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err + m.Block = []byte{} } iNdEx = postIndex case 2: @@ -783,6 +819,82 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType == 0 { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.PartSizes = append(m.PartSizes, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.PartSizes) == 0 { + m.PartSizes = make([]uint32, 0, elementCount) + } + for iNdEx < postIndex { + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.PartSizes = append(m.PartSizes, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field PartSizes", wireType) + } default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) diff --git a/proto/tendermint/blocksync/types.proto b/proto/tendermint/blocksync/types.proto index 3b229350d0..dde27069ce 100644 --- a/proto/tendermint/blocksync/types.proto +++ b/proto/tendermint/blocksync/types.proto @@ -3,7 +3,6 @@ package tendermint.blocksync; option go_package = "github.com/dashpay/tenderdash/proto/tendermint/blocksync"; -import "tendermint/types/block.proto"; import "tendermint/types/types.proto"; // BlockRequest requests a block for a specific height @@ -19,8 +18,18 @@ message NoBlockResponse { // BlockResponse returns block to the requested message BlockResponse { - tendermint.types.Block block = 1; + // block is a serialized tendermint.types.Block: the bytes of the parts it + // was committed with, in order. The commit's part set header is computed + // over those bytes, and decoding and re-encoding the block can change them + // (unknown fields, non-canonical encoding). On the wire this is the same as + // an embedded tendermint.types.Block, the type of this field in older + // releases. + bytes block = 1; tendermint.types.Commit commit = 2; + // part_sizes is the size of each part the block was committed with, in + // order. It is empty when the block was cut into parts of 64kB, the last + // one shorter, the way honest proposers cut it. + repeated uint32 part_sizes = 3; } // StatusRequest requests the status of a peer. diff --git a/spec/p2p/messages/block-sync.md b/spec/p2p/messages/block-sync.md index 48aa6155fd..d83d4705d9 100644 --- a/spec/p2p/messages/block-sync.md +++ b/spec/p2p/messages/block-sync.md @@ -34,11 +34,17 @@ NoBlockResponse notifies the peer requesting a block that the node does not cont ### BlockResponse -BlockResponse contains the block requested. +BlockResponse contains the block requested and its commit. -| Name | Type | Description | Field Number | -|-------|----------------------------------------------|-----------------|--------------| -| Block | [Block](../../core/data_structures.md#block) | Requested Block | 1 | +| Name | Type | Description | Field Number | +|------------|------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| +| Block | bytes | Requested [Block](../../core/data_structures.md#block), serialized as it was committed: the bytes of its parts, in order. On the wire this is the same as an embedded Block. | 1 | +| Commit | [Commit](../../core/data_structures.md#commit) | Commit for the requested block | 2 | +| PartSizes | repeated uint32 | Size of each part the block was committed with, in order. Empty when the block was cut into parts of 64kB, the last one shorter | 3 | + +The receiver rebuilds the block's part set from `Block` and `PartSizes`: the +part set header in the commit's block ID is computed over the committed parts, +and decoding and re-encoding the block can change its bytes. ### StatusRequest diff --git a/types/part_set.go b/types/part_set.go index 74e504c3fc..6ad15886c5 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -181,16 +181,54 @@ type PartSet struct { func NewPartSetFromData(data []byte, partSize uint32) *PartSet { // divide data into 4kb parts. total := (uint32(len(data)) + partSize - 1) / partSize - parts := make([]*Part, total) partsBytes := make([][]byte, total) + for i := uint32(0); i < total; i++ { + partsBytes[i] = data[i*partSize : tmmath.MinInt(len(data), int((i+1)*partSize))] + } + return newPartSetFromParts(partsBytes, len(data)) +} + +// NewPartSetFromPartSizes returns an immutable, full PartSet from the data +// bytes, cut into consecutive parts of the given sizes. The part set header +// depends on where the data is cut, and a proposer may cut a block other than +// NewPartSetFromData would, so this rebuilds the part set a block was +// committed with from its bytes and the sizes of its parts. +func NewPartSetFromPartSizes(data []byte, sizes []uint32) (*PartSet, error) { + if len(sizes) == 0 { + return nil, errors.New("no part sizes") + } + if len(sizes) > int(MaxBlockPartsCount) { + return nil, fmt.Errorf("too many parts: %d, max: %d", len(sizes), MaxBlockPartsCount) + } + partsBytes := make([][]byte, len(sizes)) + offset := 0 + for i, size := range sizes { + if size > BlockPartSizeBytes { + return nil, fmt.Errorf("part %d too big: %d bytes, max: %d", i, size, BlockPartSizeBytes) + } + if int(size) > len(data)-offset { + return nil, fmt.Errorf("part sizes exceed the %d bytes of data", len(data)) + } + partsBytes[i] = data[offset : offset+int(size)] + offset += int(size) + } + if offset != len(data) { + return nil, fmt.Errorf("part sizes cover %d of the %d bytes of data", offset, len(data)) + } + return newPartSetFromParts(partsBytes, len(data)), nil +} + +// newPartSetFromParts returns a full PartSet of the given parts, which must be +// consecutive slices of data byteSize bytes long. +func newPartSetFromParts(partsBytes [][]byte, byteSize int) *PartSet { + total := tmmath.MustConvertUint32(len(partsBytes)) + parts := make([]*Part, total) partsBitArray := bits.NewBitArray(int(total)) for i := uint32(0); i < total; i++ { - part := &Part{ + parts[i] = &Part{ Index: i, - Bytes: data[i*partSize : tmmath.MinInt(len(data), int((i+1)*partSize))], + Bytes: partsBytes[i], } - parts[i] = part - partsBytes[i] = part.Bytes partsBitArray.SetIndex(int(i), true) } // Compute merkle proofs @@ -204,7 +242,7 @@ func NewPartSetFromData(data []byte, partSize uint32) *PartSet { parts: parts, partsBitArray: partsBitArray, count: total, - byteSize: int64(len(data)), + byteSize: int64(byteSize), } } diff --git a/types/part_set_test.go b/types/part_set_test.go index 0e3ad11f11..f05bf58e7b 100644 --- a/types/part_set_test.go +++ b/types/part_set_test.go @@ -301,3 +301,54 @@ func TestPartIndexProofBinding(t *testing.T) { require.False(t, added) require.ErrorIs(t, err, ErrPartSetUnexpectedIndex) } + +func TestNewPartSetFromPartSizes(t *testing.T) { + data := tmrand.Bytes(2*int(BlockPartSizeBytes) + 100) + + t.Run("the standard cut", func(t *testing.T) { + partSet, err := NewPartSetFromPartSizes(data, []uint32{BlockPartSizeBytes, BlockPartSizeBytes, 100}) + require.NoError(t, err) + assert.Equal(t, NewPartSetFromData(data, BlockPartSizeBytes).Header(), partSet.Header()) + }) + + t.Run("another cut", func(t *testing.T) { + sizes := []uint32{100, BlockPartSizeBytes, 0, BlockPartSizeBytes} + partSet, err := NewPartSetFromPartSizes(data, sizes) + require.NoError(t, err) + assert.True(t, partSet.IsComplete()) + assert.EqualValues(t, len(sizes), partSet.Total()) + assert.EqualValues(t, len(data), partSet.ByteSize()) + assert.NotEqual(t, NewPartSetFromData(data, BlockPartSizeBytes).Header(), partSet.Header()) + + // the parts verify against the header, as a part set gossiped by a + // proposer that cut the data this way would + received := NewPartSetFromHeader(partSet.Header()) + for i, size := range sizes { + part := partSet.GetPart(i) + assert.Len(t, part.Bytes, int(size)) + added, err := received.AddPart(part) + require.NoError(t, err) + require.True(t, added) + } + got, err := io.ReadAll(received.GetReader()) + require.NoError(t, err) + assert.Equal(t, data, got) + }) + + testCases := map[string]struct { + sizes []uint32 + wantErr string + }{ + "no sizes": {nil, "no part sizes"}, + "too many parts": {make([]uint32, MaxBlockPartsCount+1), "too many parts"}, + "part too big": {[]uint32{BlockPartSizeBytes + 1, BlockPartSizeBytes - 1, 100}, "too big"}, + "sizes beyond end": {[]uint32{BlockPartSizeBytes, BlockPartSizeBytes, 101}, "exceed"}, + "sizes short": {[]uint32{BlockPartSizeBytes, BlockPartSizeBytes, 99}, "cover"}, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + _, err := NewPartSetFromPartSizes(data, tc.sizes) + assert.ErrorContains(t, err, tc.wantErr) + }) + } +} From 9c80931e0c1d54fb6c8af66f78b809c783b0bf36 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Mon, 9 Nov 2026 19:38:04 +0700 Subject: [PATCH 2/8] fix(blocksync): cover every committed part set and bound its metadata Follow-up to review of the committed-parts change: - PartSet.AddPart checks a proof against the root but not the header's total, so a part set can be committed whose root cannot be recomputed from its parts. BlockResponse now carries the stored proofs in that case, and the receiver verifies them against the commit's header. - FinalizeBlock and replay events used block.BlockID(nil), a re-encoding that differs from the committed ID for these blocks. They now use the commit's block ID. - Part sizes and proofs from a peer were held unchecked while waiting for a lower height. They are validated when the response is decoded and charged to the backlog. - The block sync channel had no room for part metadata or the commit on top of a maximum size block. Its capacity now covers both. Co-Authored-By: Claude Opus 5.5 --- internal/blocksync/applier.go | 21 +-- internal/blocksync/applier_test.go | 127 ++++++++++++-- internal/blocksync/block_fetch_job_test.go | 17 ++ internal/blocksync/commit_extensions_test.go | 12 +- internal/blocksync/committed_parts.go | 174 +++++++++++++++++++ internal/blocksync/msgs.go | 7 +- internal/blocksync/p2p_msg_handler.go | 66 +------ internal/blocksync/p2p_msg_handler_test.go | 140 +++++++++++++-- internal/blocksync/reactor_test.go | 34 +++- internal/blocksync/synchronizer.go | 34 ++-- internal/consensus/replayer.go | 6 +- internal/p2p/channel_params.go | 35 +++- internal/state/execution.go | 5 +- proto/tendermint/blocksync/types.pb.go | 110 ++++++++++-- proto/tendermint/blocksync/types.proto | 7 + spec/p2p/messages/block-sync.md | 7 +- types/part_set.go | 39 +++-- 17 files changed, 656 insertions(+), 185 deletions(-) create mode 100644 internal/blocksync/committed_parts.go diff --git a/internal/blocksync/applier.go b/internal/blocksync/applier.go index 9a5651f575..a9d2c53580 100644 --- a/internal/blocksync/applier.go +++ b/internal/blocksync/applier.go @@ -70,14 +70,11 @@ func newBlockApplier(blockExec sm.Executor, store sm.BlockStore, opts ...applier } // Apply safely verifies, saves to the store and executes a block with commit. -// blockBytes are the bytes block was decoded from, the parts it was committed -// with in order, and partSizes the size of each part, or nil for parts of -// types.BlockPartSizeBytes. +// parts is the block as it was committed, whose bytes block was decoded from. func (e *blockApplier) Apply( ctx context.Context, block *types.Block, - blockBytes []byte, - partSizes []uint32, + parts CommittedParts, commit *types.Commit, ) error { e.mtx.Lock() @@ -93,12 +90,12 @@ func (e *blockApplier) Apply( // The part set is needed twice: to derive the block ID and to persist the // block. Building it builds a Merkle tree over the parts, so build it once - // and pass it to both. It is rebuilt from the bytes the block was committed + // and pass it to both. It is rebuilt from the parts the block was committed // with, not from block: the commit's part set header is computed over // those, and a re-encoding of block drops unknown fields and non-canonical // encoding. start := time.Now() - blockParts, err := committedPartSet(blockBytes, partSizes) + blockParts, err := parts.partSet(commit.BlockID.PartSetHeader) if err != nil { return err } @@ -220,16 +217,6 @@ func (e *blockApplier) verify( return verified, nil } -// committedPartSet returns the part set a block was committed with, from its -// bytes and the size of each part, or nil sizes for parts of -// types.BlockPartSizeBytes. -func committedPartSet(blockBytes []byte, partSizes []uint32) (*types.PartSet, error) { - if len(partSizes) == 0 { - return types.NewPartSetFromData(blockBytes, types.BlockPartSizeBytes), nil - } - return types.NewPartSetFromPartSizes(blockBytes, partSizes) -} - // observeSince records the time since start under stage and returns it, so the // same measurement can also feed the per-block averages in applyStats. func (e *blockApplier) observeSince(stage string, start time.Time) time.Duration { diff --git a/internal/blocksync/applier_test.go b/internal/blocksync/applier_test.go index 1507e89790..30e49f3454 100644 --- a/internal/blocksync/applier_test.go +++ b/internal/blocksync/applier_test.go @@ -28,6 +28,7 @@ import ( "github.com/dashpay/tenderdash/internal/test/metricspy" tmrequire "github.com/dashpay/tenderdash/internal/test/require" "github.com/dashpay/tenderdash/libs/log" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -48,7 +49,7 @@ func TestBlockApplierChecksAppResponseBeforeSave(t *testing.T) { executor := sm.NewBlockExecutor(stateStore, client, nil, sm.EmptyEvidencePool{}, blockStore, nil) applier := newBlockApplier(executor, blockStore, applierWithState(initialState)) - require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), nil, commit) }) + require.Panics(t, func() { _ = applier.Apply(ctx, block, standardParts(t, block), commit) }) require.Zero(t, blockStore.Height(), "an inconsistent app response must not advance the block store") require.Equal(t, initialState.LastBlockHeight, applier.State().LastBlockHeight) loaded, err := stateStore.Load() @@ -175,7 +176,7 @@ func TestBlockApplierApply(t *testing.T) { tc.mockFn() } fn := func() { - err := applier.Apply(ctx, tc.block, blockBytes(t, tc.block), nil, tc.commit) + err := applier.Apply(ctx, tc.block, standardParts(t, tc.block), tc.commit) tmrequire.Error(t, tc.wantErr, err) } if tc.wantPanic { @@ -268,7 +269,7 @@ func TestBlockApplierDoesNotSaveBlockRejectedByApp(t *testing.T) { Return(sm.CurrentRoundState{}, errors.New("app rejected the block")) applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState)) - require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), nil, commit) }) + require.Panics(t, func() { _ = applier.Apply(ctx, block, standardParts(t, block), commit) }) } // TestBlockApplierSavesBlockBeforeFinalize pins the order the handshake relies @@ -314,7 +315,7 @@ func TestBlockApplierSavesBlockBeforeFinalize(t *testing.T) { Return(state, nil, nil) applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState)) - require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit)) + require.NoError(t, applier.Apply(ctx, block, standardParts(t, block), commit)) require.Equal(t, []string{"process", "verify", "save", "finalize"}, calls) } @@ -350,7 +351,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState), applierWithMetrics(m)) - require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), commitH1)) for _, stage := range []string{"partset", "verify_commit", "verify_block", "save", "exec"} { require.Len(t, hist.Samples[stage], 1, "stage %q after the first apply", stage) @@ -361,7 +362,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { // A failed attempt must advance the idle-time boundary as well, so a retry // does not count the failed attempt's wait and verification time again. failureStart := time.Now() - require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, new(types.Commit))) + require.Error(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), new(types.Commit))) failureEnd := time.Now() failureDone := applier.lastDone require.False(t, failureDone.Before(failureStart), "failed apply must advance lastDone") @@ -370,7 +371,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { // The retry measures only the idle time after the failed attempt. retryStart := time.Now() - require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), commitH1)) retryEnd := time.Now() require.Len(t, hist.Samples["wait"], 2) wait := hist.Samples["wait"][1] @@ -401,7 +402,7 @@ func TestBlockApplierVerifyFailureTimesOnlyTheCheckThatRan(t *testing.T) { // A rejected commit must stop before ValidateBlock is reached. mockBlockExec.On("VerifyCommit", initialState, blockH1.BlockID(nil), blockH1.Height, new(types.Commit)). Once().Return(types.VerifiedCommit{}, errors.New("bad signature")) - require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, new(types.Commit))) + require.Error(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), new(types.Commit))) require.Len(t, hist.Samples["partset"], 1) require.Len(t, hist.Samples["verify_commit"], 1) @@ -445,7 +446,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { blockExec.On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, blockH1ID, blockH1, commitH1, none). Once().Return(initialState, nil, nil) applier := newBlockApplier(blockExec, blockStore, applierWithState(initialState)) - require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), nil, commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), commitH1)) return applier, blockExec } // expectH2 expects block 2 to be validated and applied offering lastCommit @@ -461,7 +462,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { t.Run("the next block is offered the previous commit's verification", func(t *testing.T) { applier, blockExec := applyH1(t) expectH2(blockExec, verifiedH1) - require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), nil, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, standardParts(t, blockH2), commitH2)) }) t.Run("a rejected commit does not replace the verification", func(t *testing.T) { @@ -469,17 +470,17 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { bad := new(types.Commit) blockExec.On("VerifyCommit", mock.Anything, blockH2ID, blockH2.Height, bad). Once().Return(none, errors.New("bad signature")) - require.Error(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), nil, bad)) + require.Error(t, applier.Apply(ctx, blockH2, standardParts(t, blockH2), bad)) expectH2(blockExec, verifiedH1) - require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), nil, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, standardParts(t, blockH2), commitH2)) }) t.Run("replacing the state discards the verification", func(t *testing.T) { applier, blockExec := applyH1(t) applier.UpdateState(initialState) expectH2(blockExec, none) - require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), nil, commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, standardParts(t, blockH2), commitH2)) }) } @@ -533,7 +534,7 @@ func TestBlockApplierSkipsTheLastCommitItVerified(t *testing.T) { wantSkipped float64 }{{1, 0}, {2, 3}, {3, 6}} { block, _, _, seenCommit := makeNextBlock(ctx, t, applier.State(), privVals[0], step.height, commit) - require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), nil, seenCommit)) + require.NoError(t, applier.Apply(ctx, block, standardParts(t, block), seenCommit)) require.Equal(t, step.wantSkipped, skipped.Value(), "LastCommit threshold verifications skipped after applying height %d", step.height) commit = seenCommit @@ -551,6 +552,13 @@ func blockBytes(t *testing.T, block *types.Block) []byte { return bz } +// standardParts returns block as its proposer commits it when it encodes and +// cuts it the canonical way. +func standardParts(t *testing.T, block *types.Block) CommittedParts { + t.Helper() + return CommittedParts{Bytes: blockBytes(t, block)} +} + // TestBlockApplierRejectsPartSizesNotCoveringTheBlock checks that part sizes a // peer sent that do not cut the block's bytes are refused before the block is // verified or saved. @@ -563,6 +571,95 @@ func TestBlockApplierRejectsPartSizesNotCoveringTheBlock(t *testing.T) { bz := blockBytes(t, block) applier := newBlockApplier(mocks.NewExecutor(t), mocks.NewBlockStore(t), applierWithState(state)) - err := applier.Apply(ctx, block, bz, []uint32{uint32(len(bz) - 1)}, commit) + err := applier.Apply(ctx, block, CommittedParts{Bytes: bz, Sizes: []uint32{uint32(len(bz) - 1)}}, commit) require.ErrorContains(t, err, "part sizes cover") } + +// finalizeBlockIDApp records the block IDs FinalizeBlock is called with. +type finalizeBlockIDApp struct { + abci.BaseApplication + blockIDs []*tmproto.BlockID +} + +func (app *finalizeBlockIDApp) FinalizeBlock( + ctx context.Context, + req *abci.RequestFinalizeBlock, +) (*abci.ResponseFinalizeBlock, error) { + app.blockIDs = append(app.blockIDs, req.BlockID) + return app.BaseApplication.FinalizeBlock(ctx, req) +} + +// TestFinalizeBlockGetsCommittedBlockID checks that the application is given +// the block ID that was committed, when applying a block and replaying it, for +// blocks committed with parts that a re-encoding does not reproduce. +func TestFinalizeBlockGetsCommittedBlockID(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := log.NewNopLogger() + genDoc, privVals := factory.RandGenesisDoc(1, factory.ConsensusParams()) + state, err := sm.MakeGenesisState(genDoc) + require.NoError(t, err) + stateStore := sm.NewStore(dbm.NewMemDB()) + require.NoError(t, stateStore.Save(state)) + blockStore := store.NewBlockStore(dbm.NewMemDB()) + + finalizeApp := &finalizeBlockIDApp{} + localClient := abciclient.NewLocalClient(logger, finalizeApp) + app := proxy.New(localClient, logger, proxy.NopMetrics()) + require.NoError(t, app.Start(ctx)) + eventBus := eventbus.NewDefault(logger) + require.NoError(t, eventBus.Start(ctx)) + mp := &mpmocks.Mempool{} + mp.On("Lock").Return() + mp.On("Unlock").Return() + mp.On("FlushAppConn", mock.Anything).Return(nil) + mp.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything, + mock.Anything, mock.Anything, mock.Anything).Return(nil) + blockExec := sm.NewBlockExecutor(stateStore, app, mp, sm.EmptyEvidencePool{}, blockStore, eventBus) + applier := newBlockApplier(blockExec, blockStore, applierWithState(state)) + + blockParts := func(t *testing.T, height int64, block *types.Block) *types.PartSet { + switch height { + case 1: + return types.NewPartSetFromData(withUnknownField(t, block), types.BlockPartSizeBytes) + case 2: + return types.NewPartSetFromData(blockBytes(t, block), 100) + case 3: + return withProofOfLargerTree(t, block) + } + return nil + } + commit := types.NewCommit(0, 0, types.BlockID{}, nil, nil) + for height := int64(1); height <= 3; height++ { + block, blockID, partSet, seenCommit := makeNextBlockWithParts(ctx, t, applier.State(), privVals[0], height, + commit, blockParts) + require.False(t, blockID.Equals(block.BlockID(nil)), "height %d must not be a block a re-encoding reproduces", height) + parts := committedPartsOf(t, partSet) + require.NoError(t, applier.Apply(ctx, block, parts, seenCommit)) + + want := seenCommit.BlockID.ToProto() + require.Equal(t, &want, finalizeApp.blockIDs[len(finalizeApp.blockIDs)-1], "applying height %d", height) + + _, err := sm.ExecReplayedCommitBlock(ctx, localClient, block, seenCommit, logger) + require.NoError(t, err) + require.Equal(t, &want, finalizeApp.blockIDs[len(finalizeApp.blockIDs)-1], "replaying height %d", height) + commit = seenCommit + } +} + +// committedPartsOf returns parts in the form a peer serves them, by storing them. +func committedPartsOf(t *testing.T, partSet *types.PartSet) CommittedParts { + t.Helper() + total := int(partSet.Total()) + blockStore := mocks.NewBlockStore(t) + blockStore.On("LoadBlockMeta", int64(1)).Return(&types.BlockMeta{ + BlockID: types.BlockID{PartSetHeader: partSet.Header()}, + BlockSize: int(partSet.ByteSize()), + }) + for i := range total { + blockStore.On("LoadBlockPart", int64(1), i).Return(partSet.GetPart(i)) + } + parts := loadCommittedParts(blockStore, 1) + require.NotNil(t, parts) + return *parts +} diff --git a/internal/blocksync/block_fetch_job_test.go b/internal/blocksync/block_fetch_job_test.go index 17bbb23859..47b0889336 100644 --- a/internal/blocksync/block_fetch_job_test.go +++ b/internal/blocksync/block_fetch_job_test.go @@ -107,6 +107,23 @@ func (suite *BlockFetchJobTestSuite) TestExecuteHeightMismatch() { suite.requireError("peer sent block at height 6, requested height 5", res.Err) } +// TestExecuteRejectsOversizedPartMetadata pins that a response with more part +// sizes than a block can have fails its fetch, so it is never held waiting for +// the lower heights still missing. +func (suite *BlockFetchJobTestSuite) TestExecuteRejectsOversizedPartMetadata() { + ctx := context.Background() + const requested = int64(9) + oversized := *suite.responses[requested-1] + oversized.PartSizes = make([]uint32, types.MaxBlockPartsCount+1) + suite.client. + On("GetBlock", mock.Anything, requested, suite.peer.peerID). + Once(). + Return(suite.getBlockReturnFunc(suite.promiseResolveResponse(&oversized)), nil) + res := blockFetchJobHandler(suite.client, suite.peer, requested)(ctx) + suite.requireError("too many parts", res.Err) + suite.Nil(res.Value) +} + // TestExecuteCarriesPeerConnection pins that a job's outcome stays attributed to the // connection generation the peer was reserved under, including a timeout, which has // no response envelope to take a generation from. diff --git a/internal/blocksync/commit_extensions_test.go b/internal/blocksync/commit_extensions_test.go index 082a79f681..7e84c48971 100644 --- a/internal/blocksync/commit_extensions_test.go +++ b/internal/blocksync/commit_extensions_test.go @@ -52,11 +52,11 @@ func TestBlockApplierRejectsCommitExtensionsAndRetries(t *testing.T) { metrics := consensus.NopMetrics() metrics.CommitVerifyFailures = counter applier := newBlockApplier(exec, store, applierWithState(initial), applierWithMetrics(metrics)) - require.Error(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit)) + require.Error(t, applier.Apply(ctx, block, standardParts(t, block), commit)) require.Equal(t, float64(1), counter.Value()) require.Equal(t, []string{"process", "verify"}, calls) require.Equal(t, initial.LastBlockHeight, applier.State().LastBlockHeight) - require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit)) + require.NoError(t, applier.Apply(ctx, block, standardParts(t, block), commit)) require.Equal(t, float64(1), counter.Value()) require.Equal(t, []string{"process", "verify", "verify", "save", "finalize"}, calls) } @@ -81,7 +81,7 @@ func TestBlockApplierInvalidatesRejectedProposal(t *testing.T) { exec.On("VerifyVoteExtension", mock.Anything, mock.Anything). Twice().Return(errors.New("invalid vote extension")) applier := newBlockApplier(exec, store, applierWithState(initial)) - require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, standardParts(t, block), commit), sm.ErrCommitExtensionsRejected) switch change { case "block": block.AppHash = append(block.AppHash.Copy(), 1) @@ -90,7 +90,7 @@ func TestBlockApplierInvalidatesRejectedProposal(t *testing.T) { case "state": applier.UpdateState(initial) } - require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commit), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, standardParts(t, block), commit), sm.ErrCommitExtensionsRejected) }) } } @@ -135,9 +135,9 @@ func TestBlockApplierReprocessesRoundAfterAnother(t *testing.T) { applier := newBlockApplier(exec, store, applierWithState(initial)) for _, round := range []int32{1, 1, 2} { - require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commitAt(round)), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, standardParts(t, block), commitAt(round)), sm.ErrCommitExtensionsRejected) } - require.NotPanics(t, func() { require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), nil, commitAt(1))) }) + require.NotPanics(t, func() { require.NoError(t, applier.Apply(ctx, block, standardParts(t, block), commitAt(1))) }) require.Equal(t, []int32{1, 2, 1}, rounds, "a round is processed again only after another one") } diff --git a/internal/blocksync/committed_parts.go b/internal/blocksync/committed_parts.go new file mode 100644 index 0000000000..0eee4194e2 --- /dev/null +++ b/internal/blocksync/committed_parts.go @@ -0,0 +1,174 @@ +package blocksync + +import ( + "errors" + "fmt" + + "github.com/dashpay/tenderdash/crypto/merkle" + sm "github.com/dashpay/tenderdash/internal/state" + tmmath "github.com/dashpay/tenderdash/libs/math" + bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" + tmcrypto "github.com/dashpay/tenderdash/proto/tendermint/crypto" + "github.com/dashpay/tenderdash/types" +) + +// CommittedParts is a block as it was committed. The commit's part set header +// is computed over the parts the proposer gossiped, and a peer can only rebuild +// it from those: a re-encoding of the decoded block drops unknown fields and +// non-canonical encoding, and consensus accepts any parts that verify against +// the header, however the proposer cut the block. +type CommittedParts struct { + // Bytes are the bytes of the parts, in order. + Bytes []byte + // Sizes is the size of each part, nil for parts of types.BlockPartSizeBytes. + Sizes []uint32 + // Proofs is the Merkle proof of each part. It is nil when the part set header + // can be computed from Bytes and Sizes. + Proofs []merkle.Proof +} + +// loadCommittedParts returns the block at height as it was committed, or nil +// when the store does not hold every part of it. +func loadCommittedParts(store sm.BlockStore, height int64) *CommittedParts { + meta := store.LoadBlockMeta(height) + if meta == nil { + return nil + } + total := int(meta.BlockID.PartSetHeader.Total) + // BlockSize is -1 for a header saved without its block + blockBytes := make([]byte, 0, max(meta.BlockSize, 0)) + sizes := make([]uint32, total) + proofs := make([]merkle.Proof, total) + computable := true + for i := range total { + part := store.LoadBlockPart(height, i) + // a header saved without its block, or parts pruned after the meta was + // loaded + if part == nil { + return nil + } + blockBytes = append(blockBytes, part.Bytes...) + sizes[i] = tmmath.MustConvertUint32(len(part.Bytes)) + proofs[i] = part.Proof + // PartSet.AddPart checks a proof against the header's root but not its + // total, so a proposer can commit to a tree of more leaves than the + // header has parts. Every part verified at the header's total means the + // tree over the parts is the header's. + computable = computable && part.Proof.Total == int64(total) + } + if len(blockBytes) == 0 { + return nil + } + parts := &CommittedParts{Bytes: blockBytes, Sizes: sizes, Proofs: proofs} + if computable { + parts.Proofs = nil + if standardPartSizes(sizes, len(blockBytes)) { + parts.Sizes = nil + } + } + return parts +} + +// standardPartSizes reports whether parts of the given sizes are how +// types.NewPartSetFromData cuts size bytes into parts of +// types.BlockPartSizeBytes. +func standardPartSizes(partSizes []uint32, size int) bool { + partSize := int(types.BlockPartSizeBytes) + total := (size + partSize - 1) / partSize + if total == 0 || len(partSizes) != total { + return false + } + // the last part takes what is left, and with total parts that is between 1 + // and partSize bytes + for _, s := range partSizes[:total-1] { + if s != types.BlockPartSizeBytes { + return false + } + } + return true +} + +// committedPartsFromProto returns the parts in a block response, after checking +// that they are well formed. It does not check them against the commit. +func committedPartsFromProto(resp *bcproto.BlockResponse) (CommittedParts, error) { + parts := CommittedParts{Bytes: resp.Block, Sizes: resp.PartSizes} + if len(resp.PartSizes) > 0 { + if err := types.ValidatePartSizes(len(resp.Block), resp.PartSizes); err != nil { + return CommittedParts{}, err + } + } + if len(resp.PartProofs) == 0 { + return parts, nil + } + if len(resp.PartProofs) != len(resp.PartSizes) { + return CommittedParts{}, fmt.Errorf("%d proofs for %d parts", len(resp.PartProofs), len(resp.PartSizes)) + } + parts.Proofs = make([]merkle.Proof, len(resp.PartProofs)) + for i := range resp.PartProofs { + proof, err := merkle.ProofFromProto(&resp.PartProofs[i]) + if err != nil { + return CommittedParts{}, fmt.Errorf("proof of part %d: %w", i, err) + } + if proof.Index != int64(i) { + return CommittedParts{}, fmt.Errorf("proof of part %d has index %d", i, proof.Index) + } + parts.Proofs[i] = *proof + } + return parts, nil +} + +// committedPartsSize returns the serialized size of the parts in a block +// response. +func committedPartsSize(resp *bcproto.BlockResponse) int { + size := len(resp.Block) + 4*len(resp.PartSizes) + for i := range resp.PartProofs { + size += resp.PartProofs[i].Size() + } + return size +} + +// protoProofs returns Proofs in their protobuf form. +func (p CommittedParts) protoProofs() []tmcrypto.Proof { + if p.Proofs == nil { + return nil + } + proofs := make([]tmcrypto.Proof, len(p.Proofs)) + for i := range p.Proofs { + proofs[i] = *p.Proofs[i].ToProto() + } + return proofs +} + +// partSet returns the part set the block was committed with. header is the +// part set header of the commit; the caller still has to verify that commit. +func (p CommittedParts) partSet(header types.PartSetHeader) (*types.PartSet, error) { + if len(p.Proofs) == 0 { + if len(p.Sizes) == 0 { + return types.NewPartSetFromData(p.Bytes, types.BlockPartSizeBytes), nil + } + return types.NewPartSetFromPartSizes(p.Bytes, p.Sizes) + } + if len(p.Proofs) != len(p.Sizes) || int64(len(p.Proofs)) != int64(header.Total) { + return nil, fmt.Errorf("%d proofs and %d part sizes for %d parts", len(p.Proofs), len(p.Sizes), header.Total) + } + if err := types.ValidatePartSizes(len(p.Bytes), p.Sizes); err != nil { + return nil, err + } + partSet := types.NewPartSetFromHeader(header) + offset := 0 + for i, size := range p.Sizes { + part := &types.Part{ + Index: tmmath.MustConvertUint32(i), + Bytes: p.Bytes[offset : offset+int(size)], + Proof: p.Proofs[i], + } + offset += int(size) + if _, err := partSet.AddPart(part); err != nil { + return nil, fmt.Errorf("part %d: %w", i, err) + } + } + if !partSet.IsComplete() { + return nil, errors.New("incomplete part set") + } + return partSet, nil +} diff --git a/internal/blocksync/msgs.go b/internal/blocksync/msgs.go index d54174cc9e..db41da0342 100644 --- a/internal/blocksync/msgs.go +++ b/internal/blocksync/msgs.go @@ -1,12 +1,9 @@ package blocksync import ( - p2pproto "github.com/dashpay/tenderdash/proto/tendermint/p2p" - "github.com/dashpay/tenderdash/types" + "github.com/dashpay/tenderdash/internal/p2p" ) const ( - MaxMsgSize = types.MaxBlockSizeBytes + - p2pproto.BlockResponseMessagePrefixSize + - p2pproto.BlockResponseMessageFieldKeySize + MaxMsgSize = p2p.BlockSyncMaxMsgSize ) diff --git a/internal/blocksync/p2p_msg_handler.go b/internal/blocksync/p2p_msg_handler.go index 72ba45ad1d..0a76d5da42 100644 --- a/internal/blocksync/p2p_msg_handler.go +++ b/internal/blocksync/p2p_msg_handler.go @@ -14,7 +14,6 @@ import ( "github.com/dashpay/tenderdash/internal/p2p/client" sm "github.com/dashpay/tenderdash/internal/state" "github.com/dashpay/tenderdash/libs/log" - tmmath "github.com/dashpay/tenderdash/libs/math" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" "github.com/dashpay/tenderdash/types" ) @@ -156,11 +155,11 @@ func (h *blockP2PMessageHandler) Handle(ctx context.Context, p2pClient *client.C func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (proto.Message, bool, error) { envelope := &job.envelope msg := envelope.Message.(*bcproto.BlockRequest) - blockBytes, partSizes := loadCommittedBlock(h.store, msg.Height) + parts := loadCommittedParts(h.store, msg.Height) if !h.connectionIsCurrent(envelope.From, envelope.ConnID) { return nil, false, nil } - if blockBytes == nil { + if parts == nil { h.logger.Debug("peer requesting a block we do not have", "peer", envelope.From, "height", msg.Height) return &bcproto.NoBlockResponse{Height: msg.Height}, false, nil } @@ -169,66 +168,13 @@ func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (pro return nil, false, fmt.Errorf("%w at height %d", errStoredBlockMissingCommit, msg.Height) } return &bcproto.BlockResponse{ - Block: blockBytes, - Commit: commit.ToProto(), - PartSizes: partSizes, + Block: parts.Bytes, + Commit: commit.ToProto(), + PartSizes: parts.Sizes, + PartProofs: parts.protoProofs(), }, true, nil } -// loadCommittedBlock returns the block at height as it was committed: the -// bytes of its stored parts, in order, and the size of each part, or nil sizes -// when the parts are cut the way types.NewPartSetFromData cuts them. The -// commit's part set header is computed over those parts, and the peer can only -// rebuild it from them; a re-encoding of the decoded block drops unknown -// fields and non-canonical encoding, and a proposer may cut a block into parts -// of any size. It returns nil bytes when the store does not hold every part. -func loadCommittedBlock(store sm.BlockStore, height int64) ([]byte, []uint32) { - meta := store.LoadBlockMeta(height) - if meta == nil { - return nil, nil - } - total := int(meta.BlockID.PartSetHeader.Total) - // BlockSize is -1 for a header saved without its block - blockBytes := make([]byte, 0, max(meta.BlockSize, 0)) - partSizes := make([]uint32, total) - for i := range total { - part := store.LoadBlockPart(height, i) - // a header saved without its block, or parts pruned after the meta was - // loaded - if part == nil { - return nil, nil - } - blockBytes = append(blockBytes, part.Bytes...) - partSizes[i] = tmmath.MustConvertUint32(len(part.Bytes)) - } - if len(blockBytes) == 0 { - return nil, nil - } - if standardPartSizes(partSizes, len(blockBytes)) { - partSizes = nil - } - return blockBytes, partSizes -} - -// standardPartSizes reports whether parts of the given sizes are how -// types.NewPartSetFromData cuts size bytes into parts of -// types.BlockPartSizeBytes. -func standardPartSizes(partSizes []uint32, size int) bool { - partSize := int(types.BlockPartSizeBytes) - total := (size + partSize - 1) / partSize - if total == 0 || len(partSizes) != total { - return false - } - // the last part takes what is left, and with total parts that is between 1 - // and partSize bytes - for _, s := range partSizes[:total-1] { - if s != types.BlockPartSizeBytes { - return false - } - } - return true -} - func (h *blockP2PMessageHandler) enqueueBlockResponse(job blockResponseJob) { peerID := job.envelope.From h.responseMtx.Lock() diff --git a/internal/blocksync/p2p_msg_handler_test.go b/internal/blocksync/p2p_msg_handler_test.go index 4cae5cccf6..9d86b33002 100644 --- a/internal/blocksync/p2p_msg_handler_test.go +++ b/internal/blocksync/p2p_msg_handler_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/binary" "fmt" + "math" "testing" "time" @@ -14,6 +15,8 @@ import ( "github.com/stretchr/testify/suite" "github.com/dashpay/tenderdash/config" + tmcrypto "github.com/dashpay/tenderdash/crypto" + "github.com/dashpay/tenderdash/crypto/merkle" "github.com/dashpay/tenderdash/internal/p2p" "github.com/dashpay/tenderdash/internal/p2p/client" p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" @@ -24,6 +27,9 @@ import ( dbm "github.com/dashpay/tenderdash/libs/db" "github.com/dashpay/tenderdash/libs/log" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" + "github.com/dashpay/tenderdash/proto/tendermint/crypto" + p2pproto "github.com/dashpay/tenderdash/proto/tendermint/p2p" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -133,7 +139,7 @@ func (suite *BlockP2PMessageHandlerTestSuite) expectStoredBlock(height int64, ti BlockID: types.BlockID{PartSetHeader: types.PartSetHeader{Total: 1}}, BlockSize: len(blockBytes), } - suite.fakeStore.On("LoadBlockPart", height, 0).Times(times).Return(&types.Part{Bytes: blockBytes}) + suite.fakeStore.On("LoadBlockPart", height, 0).Times(times).Return(&types.Part{Bytes: blockBytes, Proof: merkle.Proof{Total: 1}}) return suite.fakeStore.On("LoadBlockMeta", height).Times(times).Return(meta) } @@ -528,35 +534,39 @@ func TestServeBlockAsCommitted(t *testing.T) { ctx := context.Background() valSet, privVals := factory.MockValidatorSet() state := fakeInitialState(valSet) - blocks := statefactory.MakeBlocks(ctx, t, 5, &state, privVals, 1) + blocks := statefactory.MakeBlocks(ctx, t, 6, &state, privVals, 1) blockStore := store.NewBlockStore(dbm.NewMemDB()) const smallPartSize = 100 standard := blockBytes(t, blocks[0]) unknownField := withUnknownField(t, blocks[1]) smallParts := blockBytes(t, blocks[2]) + largerTree := withProofOfLargerTree(t, blocks[3]) testCases := []struct { parts *types.PartSet wantBytes []byte wantPartSizes []uint32 + wantProofs bool }{ - {types.NewPartSetFromData(standard, types.BlockPartSizeBytes), standard, nil}, - {types.NewPartSetFromData(unknownField, types.BlockPartSizeBytes), unknownField, nil}, + {types.NewPartSetFromData(standard, types.BlockPartSizeBytes), standard, nil, false}, + {types.NewPartSetFromData(unknownField, types.BlockPartSizeBytes), unknownField, nil, false}, { types.NewPartSetFromData(smallParts, smallPartSize), smallParts, smallPartSizes(len(smallParts), smallPartSize), + false, }, + {largerTree, blockBytes(t, blocks[3]), []uint32{uint32(largerTree.ByteSize())}, true}, } for i, tc := range testCases { blockStore.SaveBlock(blocks[i], tc.parts, blocks[i+1].LastCommit) } // a header stored without its block, as state sync stores the one it // restores the state at - header := blocks[3] + header := blocks[4] require.NoError(t, blockStore.SaveSignedHeader( - &types.SignedHeader{Header: &header.Header, Commit: blocks[4].LastCommit}, - blocks[4].LastCommit.BlockID, + &types.SignedHeader{Header: &header.Header, Commit: blocks[5].LastCommit}, + blocks[5].LastCommit.BlockID, )) handler := newBlockP2PMessageHandler(ctx, log.NewNopLogger(), blockStore, newMockPeerAdder(t)) @@ -578,12 +588,13 @@ func TestServeBlockAsCommitted(t *testing.T) { require.True(t, ok) require.Equal(t, tc.wantBytes, msg.Block) require.Equal(t, tc.wantPartSizes, msg.PartSizes) + require.Equal(t, tc.wantProofs, len(msg.PartProofs) > 0) resp, err := BlockResponseFromProto(msg, "peer") require.NoError(t, err) require.Equal(t, blocks[i].Hash(), resp.Block.Hash()) require.Equal(t, blocks[i+1].LastCommit.BlockID, resp.Commit.BlockID) - parts, err := committedPartSet(resp.BlockBytes, resp.PartSizes) + parts, err := resp.Parts.partSet(tc.parts.Header()) require.NoError(t, err) require.Equal(t, tc.parts.Header(), parts.Header()) }) @@ -634,6 +645,115 @@ func TestBlockResponseWireCompatible(t *testing.T) { resp, err := BlockResponseFromProto(&decoded, "peer") require.NoError(t, err) require.Equal(t, block.Hash(), resp.Block.Hash()) - require.Equal(t, embedded, resp.BlockBytes) - require.Nil(t, resp.PartSizes) + require.Equal(t, CommittedParts{Bytes: embedded}, resp.Parts) +} + +// TestBlockResponseFromProtoRejectsMalformedParts checks that part metadata a +// peer sent that cannot describe its block is refused while decoding the +// response, before it can be held waiting for a lower height. +func TestBlockResponseFromProtoRejectsMalformedParts(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) + bz := blockBytes(t, blocks[0]) + size := uint32(len(bz)) + proof := crypto.Proof{Total: 1, LeafHash: make([]byte, tmcrypto.HashSize)} + tooManyAunts := proof + tooManyAunts.Aunts = make([][]byte, merkle.MaxAunts+1) + for i := range tooManyAunts.Aunts { + tooManyAunts.Aunts[i] = make([]byte, tmcrypto.HashSize) + } + testCases := map[string]struct { + sizes []uint32 + proofs []crypto.Proof + wantErr string + }{ + "too many parts": {make([]uint32, types.MaxBlockPartsCount+1), nil, "too many parts"}, + "part too big": {[]uint32{types.BlockPartSizeBytes + 1}, nil, "too big"}, + "sizes short": {[]uint32{size - 1}, nil, "part sizes cover"}, + "sizes beyond block": {[]uint32{size + 1}, nil, "exceed"}, + "proofs without sizes": { + nil, []crypto.Proof{proof}, "1 proofs for 0 parts", + }, + "too few proofs": { + []uint32{size - 1, 1}, []crypto.Proof{proof}, "1 proofs for 2 parts", + }, + "proof of another part": { + []uint32{size}, []crypto.Proof{{Total: 1, Index: 1, LeafHash: proof.LeafHash}}, "has index 1", + }, + "proof too long": {[]uint32{size}, []crypto.Proof{tooManyAunts}, "aunts"}, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + _, err := BlockResponseFromProto(&bcproto.BlockResponse{ + Block: bz, + Commit: blocks[1].LastCommit.ToProto(), + PartSizes: tc.sizes, + PartProofs: tc.proofs, + }, "peer") + require.ErrorContains(t, err, tc.wantErr) + }) + } +} + +// TestBlockResponseSizeChargesPartMetadata checks that a response is charged to +// the backlog for its part metadata, and not only for its block. +func TestBlockResponseSizeChargesPartMetadata(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) + bz := blockBytes(t, blocks[0]) + proof := crypto.Proof{Total: 2, LeafHash: make([]byte, tmcrypto.HashSize)} + resp, err := BlockResponseFromProto(&bcproto.BlockResponse{ + Block: bz, + Commit: blocks[1].LastCommit.ToProto(), + PartSizes: []uint32{uint32(len(bz))}, + PartProofs: []crypto.Proof{proof}, + }, "peer") + require.NoError(t, err) + require.Equal(t, len(bz)+4+proof.Size(), resp.Size) +} + +// TestBlockResponseFitsBlockSyncChannel checks that the largest block response +// a node can have to serve fits the receive capacity of the block sync channel. +func TestBlockResponseFitsBlockSyncChannel(t *testing.T) { + const parts = int(types.MaxBlockPartsCount) + aunts := make([][]byte, merkle.MaxAunts) + for i := range aunts { + aunts[i] = make([]byte, tmcrypto.HashSize) + } + sizes := make([]uint32, parts) + proofs := make([]crypto.Proof, parts) + for i := range parts { + sizes[i] = types.BlockPartSizeBytes + proofs[i] = crypto.Proof{Total: math.MaxInt64, Index: int64(parts), LeafHash: aunts[0], Aunts: aunts} + } + blockID := tmproto.BlockID{ + Hash: make([]byte, tmcrypto.HashSize), + PartSetHeader: tmproto.PartSetHeader{Total: uint32(parts), Hash: make([]byte, tmcrypto.HashSize)}, + StateID: make([]byte, tmcrypto.HashSize), + } + envelope := &p2pproto.Envelope{Attributes: map[string]string{client.RequestIDAttribute: uuid.NewString()}} + require.NoError(t, envelope.Wrap(&bcproto.BlockResponse{ + Block: make([]byte, types.MaxBlockSizeBytes), + Commit: &tmproto.Commit{ + Height: math.MaxInt64, + Round: math.MaxInt32, + BlockID: blockID, + QuorumHash: make([]byte, tmcrypto.HashSize), + ThresholdBlockSignature: make([]byte, 96), + // everything a vote on the consensus vote channel can carry + ThresholdVoteExtensions: []*tmproto.VoteExtension{{ + Extension: make([]byte, 1<<20-1024), + Signature: make([]byte, 96), + }}, + }, + PartSizes: sizes, + PartProofs: proofs, + })) + + descriptor := p2p.ChannelDescriptors(config.TestConfig())[p2p.BlockSyncChannel] + require.LessOrEqual(t, envelope.Size(), descriptor.RecvMessageCapacity) } diff --git a/internal/blocksync/reactor_test.go b/internal/blocksync/reactor_test.go index bed4681b80..15312ad89a 100644 --- a/internal/blocksync/reactor_test.go +++ b/internal/blocksync/reactor_test.go @@ -16,6 +16,7 @@ import ( abciclient "github.com/dashpay/tenderdash/abci/client" abci "github.com/dashpay/tenderdash/abci/types" "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/crypto/merkle" "github.com/dashpay/tenderdash/internal/consensus" "github.com/dashpay/tenderdash/internal/eventbus" mpmocks "github.com/dashpay/tenderdash/internal/mempool/mocks" @@ -473,15 +474,19 @@ func TestReactor_SyncBlocksAsCommitted(t *testing.T) { genDoc, privVals := factory.RandGenesisDoc(1, factory.ConsensusParams()) const ( - maxBlockHeight = int64(10) + maxBlockHeight = int64(12) // a field that tendermint.types.Block does not define, which decoding // skips, appended to the block's bytes unknownFieldHeight = int64(3) // the block's bytes cut into parts shorter than types.BlockPartSizeBytes smallPartsHeight = int64(5) bothHeight = int64(7) + // one part, whose proof is of a tree of more leaves than the header + // counts, so the header cannot be computed from the part + largerTreeHeight = int64(9) smallPartSize = 100 ) + committedHeights := []int64{unknownFieldHeight, smallPartsHeight, bothHeight, largerTreeHeight} blockParts := func(t *testing.T, height int64, block *types.Block) *types.PartSet { switch height { case unknownFieldHeight: @@ -490,13 +495,15 @@ func TestReactor_SyncBlocksAsCommitted(t *testing.T) { return types.NewPartSetFromData(blockBytes(t, block), smallPartSize) case bothHeight: return types.NewPartSetFromData(withUnknownField(t, block), smallPartSize) + case largerTreeHeight: + return withProofOfLargerTree(t, block) } return nil } rts := setupWithBlockParts(ctx, t, cfg, genDoc, privVals[0], []int64{maxBlockHeight, 0}, blockParts) source, syncing := rts.reactors[rts.nodes[0]], rts.reactors[rts.nodes[1]] require.Equal(t, maxBlockHeight, source.store.Height()) - for _, height := range []int64{unknownFieldHeight, smallPartsHeight, bothHeight} { + for _, height := range committedHeights { require.NotEqual(t, source.store.LoadBlockMeta(height).BlockID.PartSetHeader, source.store.LoadBlock(height).BlockID(nil).PartSetHeader, @@ -506,17 +513,15 @@ func TestReactor_SyncBlocksAsCommitted(t *testing.T) { rts.start(ctx, t) require.Eventually(t, - func() bool { return syncing.store.Height() > bothHeight }, + func() bool { return syncing.store.Height() > largerTreeHeight }, 30*time.Second, 10*time.Millisecond, "expected node to sync the blocks committed with other bytes") - for _, height := range []int64{unknownFieldHeight, smallPartsHeight, bothHeight} { + for _, height := range committedHeights { require.Equal(t, source.store.LoadBlockMeta(height).BlockID, syncing.store.LoadBlockMeta(height).BlockID, "block ID at height %d", height) - wantBytes, wantSizes := loadCommittedBlock(source.store, height) - gotBytes, gotSizes := loadCommittedBlock(syncing.store, height) - require.Equal(t, wantBytes, gotBytes, "stored block bytes at height %d", height) - require.Equal(t, wantSizes, gotSizes, "stored part sizes at height %d", height) + require.Equal(t, loadCommittedParts(source.store, height), loadCommittedParts(syncing.store, height), + "stored parts at height %d", height) } } @@ -527,6 +532,19 @@ func withUnknownField(t *testing.T, block *types.Block) []byte { return append(blockBytes(t, block), 15<<3, 1) } +// withProofOfLargerTree returns a part set of one part with the bytes of block, +// whose root is that of a tree of two leaves. PartSet.AddPart checks a proof +// against the root and not against the header's total, so consensus accepts it. +func withProofOfLargerTree(t *testing.T, block *types.Block) *types.PartSet { + bz := blockBytes(t, block) + root, proofs := merkle.ProofsFromByteSlices([][]byte{bz, []byte("leaf the header does not count")}) + partSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 1, Hash: root}) + added, err := partSet.AddPart(&types.Part{Index: 0, Bytes: bz, Proof: *proofs[0]}) + require.NoError(t, err) + require.True(t, added) + return partSet +} + func TestReactor_BadBlockStopsPeer(t *testing.T) { // Ultimately, this should be refactored to be less integration test oriented // and more unit test oriented by simply testing channel sends and receives. diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index 6802e05040..41dba30593 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -640,7 +640,7 @@ func (s *Synchronizer) applyBlock(ctx context.Context) (BlockResponse, error) { if !ok { return BlockResponse{}, nil } - err := s.applier.Apply(ctx, resp.Block, resp.BlockBytes, resp.PartSizes, resp.Commit) + err := s.applier.Apply(ctx, resp.Block, resp.Parts, resp.Commit) if err != nil { return resp, fmt.Errorf("cannot apply block: %w", err) } @@ -779,15 +779,12 @@ type BlockResponse struct { // its failures are fenced to. ConnID uint64 Block *types.Block - // BlockBytes is Block as it was committed, the bytes of its parts in order, - // and PartSizes the size of each part, or nil for parts of - // types.BlockPartSizeBytes. Block is decoded from BlockBytes, and applied - // with the part set rebuilt from them: that is what the commit's block ID - // describes, and re-encoding Block need not give the same bytes. - BlockBytes []byte - PartSizes []uint32 - Commit *types.Commit - // Size is the serialized size of Block in bytes, the length of BlockBytes. + // Parts is Block as it was committed. Block is decoded from Parts.Bytes, and + // applied with the part set rebuilt from Parts: that is what the commit's + // block ID describes, and re-encoding Block need not give the same bytes. + Parts CommittedParts + Commit *types.Commit + // Size is the serialized size of Parts in bytes. Size int } @@ -814,6 +811,12 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* if len(resp.Block) == 0 { return nil, errors.New("block response without a block") } + // Checked before anything is decoded or queued: the response can wait behind + // a missing lower height, and only its size is charged to the backlog. + parts, err := committedPartsFromProto(resp) + if err != nil { + return nil, fmt.Errorf("invalid block parts: %w", err) + } pbb := new(tmproto.Block) if err := proto.Unmarshal(resp.Block, pbb); err != nil { return nil, fmt.Errorf("cannot decode block: %w", err) @@ -830,11 +833,10 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* } } return &BlockResponse{ - PeerID: peerID, - Block: block, - BlockBytes: resp.Block, - PartSizes: resp.PartSizes, - Commit: commit, - Size: len(resp.Block), + PeerID: peerID, + Block: block, + Parts: parts, + Commit: commit, + Size: committedPartsSize(resp), }, nil } diff --git a/internal/consensus/replayer.go b/internal/consensus/replayer.go index 8c334bb624..d263250d10 100644 --- a/internal/consensus/replayer.go +++ b/internal/consensus/replayer.go @@ -247,7 +247,7 @@ func (r *BlockReplayer) replayBlocks( } } if !mutateState { - err = r.publishEvents(block, ucState) + err = r.publishEvents(block, commit, ucState) if err != nil { return nil, err } @@ -420,10 +420,10 @@ func (r *BlockReplayer) execInitChain(ctx context.Context, rs *replayState, stat func (r *BlockReplayer) publishEvents( block *types.Block, + commit *types.Commit, ucState sm.CurrentRoundState, ) error { - blockID := block.BlockID(nil) - es := sm.NewFullEventSet(block, blockID, ucState, ucState.NextValidators) + es := sm.NewFullEventSet(block, commit.BlockID, ucState, ucState.NextValidators) err := es.Publish(r.publisher) if err != nil { r.logger.Error("failed publishing event", "err", err) diff --git a/internal/p2p/channel_params.go b/internal/p2p/channel_params.go index b08e1d9059..c603942730 100644 --- a/internal/p2p/channel_params.go +++ b/internal/p2p/channel_params.go @@ -6,6 +6,8 @@ import ( "github.com/cosmos/gogoproto/proto" "github.com/dashpay/tenderdash/config" + "github.com/dashpay/tenderdash/crypto" + "github.com/dashpay/tenderdash/crypto/merkle" "github.com/dashpay/tenderdash/proto/tendermint/blocksync" "github.com/dashpay/tenderdash/proto/tendermint/consensus" "github.com/dashpay/tenderdash/proto/tendermint/mempool" @@ -51,6 +53,25 @@ const ( // consensusMsgSize is the maximum size of a consensus message maxMsgSize = 1048576 // 1MB; NOTE: keep in sync with types.PartSet sizes. + // blockPartSizeMaxBytes bounds one encoded part_sizes entry of a block + // response: a varint of at most types.BlockPartSizeBytes. + blockPartSizeMaxBytes = 3 + // blockPartProofMaxBytes bounds one encoded part_proofs entry of a block + // response: its key and length, the two int64 varints with their keys, and + // the leaf hash and aunts with their keys and lengths. + blockPartProofMaxBytes = 4 + 2*(1+10) + (merkle.MaxAunts+1)*(2+crypto.HashSize) + // blockSyncMaxOverheadBytes bounds what a block response carries besides + // the block: part metadata for the most parts a block can have, a commit, + // whose vote extensions came in one vote on a maxMsgSize channel, and the + // envelope with its attributes. + blockSyncMaxOverheadBytes = int(types.MaxBlockPartsCount)*(blockPartSizeMaxBytes+blockPartProofMaxBytes) + + maxMsgSize + 1024 + + // BlockSyncMaxMsgSize is the maximum size of a block sync message + BlockSyncMaxMsgSize = types.MaxBlockSizeBytes + + p2pproto.BlockResponseMessagePrefixSize + + p2pproto.BlockResponseMessageFieldKeySize + + blockSyncMaxOverheadBytes ) // ChannelDescriptors returns a map of all supported descriptors @@ -64,14 +85,12 @@ func ChannelDescriptors(cfg *config.Config) map[ChannelID]*ChannelDescriptor { Name: "error", }, BlockSyncChannel: { - ID: BlockSyncChannel, - Priority: 6, - SendQueueCapacity: 4, - RecvBufferCapacity: 1024, - RecvMessageCapacity: types.MaxBlockSizeBytes + - p2pproto.BlockResponseMessagePrefixSize + - p2pproto.BlockResponseMessageFieldKeySize, - Name: "blockSync", + ID: BlockSyncChannel, + Priority: 6, + SendQueueCapacity: 4, + RecvBufferCapacity: 1024, + RecvMessageCapacity: BlockSyncMaxMsgSize, + Name: "blockSync", }, MempoolChannel: { ID: MempoolChannel, diff --git a/internal/state/execution.go b/internal/state/execution.go index c196489fe6..46abe65052 100644 --- a/internal/state/execution.go +++ b/internal/state/execution.go @@ -835,8 +835,9 @@ func execBlock( if err != nil { return nil, err } - blockID := block.BlockID(nil) - protoBlockID := blockID.ToProto() + // The commit's block ID, not block.BlockID(nil): the block may have been + // committed with parts that re-encoding it does not reproduce. + protoBlockID := commit.BlockID.ToProto() responseFinalizeBlock, err := appConn.FinalizeBlock( ctx, diff --git a/proto/tendermint/blocksync/types.pb.go b/proto/tendermint/blocksync/types.pb.go index 7a95249606..0b45a2d86f 100644 --- a/proto/tendermint/blocksync/types.pb.go +++ b/proto/tendermint/blocksync/types.pb.go @@ -5,7 +5,9 @@ package blocksync import ( fmt "fmt" + _ "github.com/cosmos/gogoproto/gogoproto" proto "github.com/cosmos/gogoproto/proto" + crypto "github.com/dashpay/tenderdash/proto/tendermint/crypto" types "github.com/dashpay/tenderdash/proto/tendermint/types" io "io" math "math" @@ -128,6 +130,11 @@ type BlockResponse struct { // order. It is empty when the block was cut into parts of 64kB, the last // one shorter, the way honest proposers cut it. PartSizes []uint32 `protobuf:"varint,3,rep,packed,name=part_sizes,json=partSizes,proto3" json:"part_sizes,omitempty"` + // part_proofs is the Merkle proof of each part the block was committed with, + // in order, alongside part_sizes. It is empty when the part set header can + // be computed from the parts, which it cannot when a proof's total differs + // from the header's. + PartProofs []crypto.Proof `protobuf:"bytes,4,rep,name=part_proofs,json=partProofs,proto3" json:"part_proofs"` } func (m *BlockResponse) Reset() { *m = BlockResponse{} } @@ -184,6 +191,13 @@ func (m *BlockResponse) GetPartSizes() []uint32 { return nil } +func (m *BlockResponse) GetPartProofs() []crypto.Proof { + if m != nil { + return m.PartProofs + } + return nil +} + // StatusRequest requests the status of a peer. type StatusRequest struct { } @@ -285,25 +299,29 @@ func init() { func init() { proto.RegisterFile("tendermint/blocksync/types.proto", fileDescriptor_19b397c236e0fa07) } var fileDescriptor_19b397c236e0fa07 = []byte{ - // 287 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0x31, 0x4b, 0xc4, 0x30, - 0x14, 0xc7, 0x2f, 0x56, 0x0f, 0x7c, 0xde, 0x79, 0x10, 0x0e, 0x29, 0xa2, 0xa1, 0x74, 0x90, 0xba, - 0x34, 0xa2, 0x8b, 0x83, 0xd3, 0xb9, 0x3b, 0xe4, 0x36, 0x17, 0x69, 0x7b, 0xe1, 0x5a, 0xb4, 0x4d, - 0x6d, 0x5e, 0x85, 0xfa, 0x29, 0xfc, 0x58, 0x8e, 0x37, 0x3a, 0x4a, 0xfb, 0x45, 0xa4, 0x69, 0xd5, - 0x0a, 0xea, 0x96, 0xff, 0xff, 0xfd, 0x1e, 0xfc, 0x92, 0x80, 0x83, 0x32, 0x5b, 0xc9, 0x22, 0x4d, - 0x32, 0xe4, 0xe1, 0x83, 0x8a, 0xee, 0x75, 0x95, 0x45, 0x1c, 0xab, 0x5c, 0x6a, 0x3f, 0x2f, 0x14, - 0x2a, 0x3a, 0xff, 0x26, 0xfc, 0x2f, 0xe2, 0xf0, 0x68, 0xb0, 0x67, 0xe8, 0xe1, 0x8e, 0x7b, 0x02, - 0x93, 0x45, 0x8b, 0x0a, 0xf9, 0x58, 0x4a, 0x8d, 0xf4, 0x00, 0xc6, 0xb1, 0x4c, 0xd6, 0x31, 0xda, - 0xc4, 0x21, 0x9e, 0x25, 0xfa, 0xe4, 0x9e, 0xc2, 0xec, 0x46, 0xf5, 0xa4, 0xce, 0x55, 0xa6, 0xe5, - 0x9f, 0xe8, 0x13, 0x4c, 0x7f, 0x82, 0x73, 0xd8, 0x31, 0x3a, 0x86, 0x9b, 0x88, 0x2e, 0xd0, 0x33, - 0x18, 0x47, 0x2a, 0x4d, 0x13, 0xb4, 0xb7, 0x1c, 0xe2, 0xed, 0x9d, 0xdb, 0xfe, 0x40, 0xbf, 0x53, - 0xbc, 0x36, 0x73, 0xd1, 0x73, 0xf4, 0x18, 0x20, 0x0f, 0x0a, 0xbc, 0xd3, 0xc9, 0xb3, 0xd4, 0xb6, - 0xe5, 0x58, 0xde, 0x54, 0xec, 0xb6, 0xcd, 0xb2, 0x2d, 0xdc, 0x19, 0x4c, 0x97, 0x18, 0x60, 0xa9, - 0xfb, 0xbb, 0xb8, 0x57, 0xb0, 0xff, 0x59, 0xfc, 0xaf, 0x4c, 0x29, 0x6c, 0x87, 0x81, 0x96, 0xc6, - 0xc4, 0x12, 0xe6, 0xbc, 0x10, 0xaf, 0x35, 0x23, 0x9b, 0x9a, 0x91, 0xf7, 0x9a, 0x91, 0x97, 0x86, - 0x8d, 0x36, 0x0d, 0x1b, 0xbd, 0x35, 0x6c, 0x74, 0x7b, 0xb9, 0x4e, 0x30, 0x2e, 0x43, 0x3f, 0x52, - 0x29, 0x5f, 0x05, 0x3a, 0xce, 0x83, 0x8a, 0x77, 0xee, 0x6d, 0xe2, 0xe6, 0x61, 0xf9, 0x6f, 0xbf, - 0x15, 0x8e, 0xcd, 0xec, 0xe2, 0x63, 0x00, 0xd3, 0xa1, 0x58, 0x18, 0xcc, 0x01, 0x00, 0x00, + // 346 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xc1, 0x4a, 0xf3, 0x40, + 0x14, 0x85, 0x33, 0x7f, 0xfa, 0x17, 0xbc, 0x6d, 0x2d, 0x0c, 0x45, 0x42, 0xb1, 0x31, 0x64, 0x21, + 0x75, 0x93, 0x11, 0xdd, 0xb8, 0x10, 0x84, 0xba, 0x17, 0x99, 0xee, 0xdc, 0x48, 0x92, 0x8e, 0x49, + 0xd0, 0x64, 0xc6, 0xcc, 0x74, 0x11, 0x9f, 0xc2, 0xb7, 0xf1, 0x15, 0xba, 0xec, 0xd2, 0x95, 0x48, + 0xf3, 0x22, 0x92, 0x49, 0xd4, 0x28, 0xea, 0x6e, 0xee, 0x39, 0x1f, 0x67, 0xe6, 0xdc, 0x04, 0x1c, + 0xc5, 0xb2, 0x05, 0xcb, 0xd3, 0x24, 0x53, 0x24, 0xb8, 0xe3, 0xe1, 0xad, 0x2c, 0xb2, 0x90, 0xa8, + 0x42, 0x30, 0xe9, 0x89, 0x9c, 0x2b, 0x8e, 0x47, 0x9f, 0x84, 0xf7, 0x41, 0x8c, 0x47, 0x11, 0x8f, + 0xb8, 0x06, 0x48, 0x75, 0xaa, 0xd9, 0xf1, 0xa4, 0x95, 0x16, 0xe6, 0x85, 0x50, 0x9c, 0x88, 0x9c, + 0xf3, 0x9b, 0xc6, 0xde, 0x6d, 0xd9, 0xfa, 0x8a, 0xf6, 0x45, 0xee, 0x3e, 0xf4, 0x67, 0x55, 0x3e, + 0x65, 0xf7, 0x4b, 0x26, 0x15, 0xde, 0x81, 0x6e, 0xcc, 0x92, 0x28, 0x56, 0x16, 0x72, 0xd0, 0xd4, + 0xa4, 0xcd, 0xe4, 0x1e, 0xc0, 0xf0, 0x82, 0x37, 0xa4, 0x14, 0x3c, 0x93, 0xec, 0x57, 0xf4, 0x09, + 0xc1, 0xe0, 0x2b, 0x39, 0x82, 0xff, 0xba, 0x84, 0x06, 0xfb, 0xb4, 0x1e, 0xf0, 0x21, 0x74, 0x43, + 0x9e, 0xa6, 0x89, 0xb2, 0xfe, 0x39, 0x68, 0xda, 0x3b, 0xb2, 0xbc, 0x56, 0xe9, 0xfa, 0x8d, 0xe7, + 0xda, 0xa7, 0x0d, 0x87, 0x27, 0x00, 0xc2, 0xcf, 0xd5, 0xb5, 0x4c, 0x1e, 0x98, 0xb4, 0x4c, 0xc7, + 0x9c, 0x0e, 0xe8, 0x56, 0xa5, 0xcc, 0x2b, 0x01, 0x9f, 0x41, 0x4f, 0xdb, 0xba, 0xbd, 0xb4, 0x3a, + 0x8e, 0xf9, 0x3d, 0xb5, 0x5e, 0x8f, 0x77, 0x59, 0x01, 0xb3, 0xce, 0xea, 0x65, 0xcf, 0xa0, 0x3a, + 0x51, 0x0b, 0xd2, 0x1d, 0xc2, 0x60, 0xae, 0x7c, 0xb5, 0x94, 0xcd, 0x36, 0xdc, 0x53, 0xd8, 0x7e, + 0x17, 0xfe, 0x2e, 0x8d, 0x31, 0x74, 0x02, 0x5f, 0x32, 0x5d, 0xc5, 0xa4, 0xfa, 0x3c, 0xa3, 0xab, + 0x8d, 0x8d, 0xd6, 0x1b, 0x1b, 0xbd, 0x6e, 0x6c, 0xf4, 0x58, 0xda, 0xc6, 0xba, 0xb4, 0x8d, 0xe7, + 0xd2, 0x36, 0xae, 0x4e, 0xa2, 0x44, 0xc5, 0xcb, 0xc0, 0x0b, 0x79, 0x4a, 0x16, 0xbe, 0x8c, 0x85, + 0x5f, 0x90, 0xfa, 0x99, 0xd5, 0x44, 0xea, 0x4f, 0xfc, 0xd3, 0x4f, 0x12, 0x74, 0xb5, 0x77, 0xfc, + 0x36, 0x00, 0x9f, 0xdb, 0x4b, 0x3e, 0x43, 0x02, 0x00, 0x00, } func (m *BlockRequest) Marshal() (dAtA []byte, err error) { @@ -391,6 +409,20 @@ func (m *BlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.PartProofs) > 0 { + for iNdEx := len(m.PartProofs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.PartProofs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } + } if len(m.PartSizes) > 0 { dAtA2 := make([]byte, len(m.PartSizes)*10) var j1 int @@ -549,6 +581,12 @@ func (m *BlockResponse) Size() (n int) { } n += 1 + sovTypes(uint64(l)) + l } + if len(m.PartProofs) > 0 { + for _, e := range m.PartProofs { + l = e.Size() + n += 1 + l + sovTypes(uint64(l)) + } + } return n } @@ -895,6 +933,40 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field PartSizes", wireType) } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PartProofs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PartProofs = append(m.PartProofs, crypto.Proof{}) + if err := m.PartProofs[len(m.PartProofs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) diff --git a/proto/tendermint/blocksync/types.proto b/proto/tendermint/blocksync/types.proto index dde27069ce..c62343969b 100644 --- a/proto/tendermint/blocksync/types.proto +++ b/proto/tendermint/blocksync/types.proto @@ -3,6 +3,8 @@ package tendermint.blocksync; option go_package = "github.com/dashpay/tenderdash/proto/tendermint/blocksync"; +import "gogoproto/gogo.proto"; +import "tendermint/crypto/proof.proto"; import "tendermint/types/types.proto"; // BlockRequest requests a block for a specific height @@ -30,6 +32,11 @@ message BlockResponse { // order. It is empty when the block was cut into parts of 64kB, the last // one shorter, the way honest proposers cut it. repeated uint32 part_sizes = 3; + // part_proofs is the Merkle proof of each part the block was committed with, + // in order, alongside part_sizes. It is empty when the part set header can + // be computed from the parts, which it cannot when a proof's total differs + // from the header's. + repeated tendermint.crypto.Proof part_proofs = 4 [(gogoproto.nullable) = false]; } // StatusRequest requests the status of a peer. diff --git a/spec/p2p/messages/block-sync.md b/spec/p2p/messages/block-sync.md index d83d4705d9..af7113c663 100644 --- a/spec/p2p/messages/block-sync.md +++ b/spec/p2p/messages/block-sync.md @@ -41,10 +41,11 @@ BlockResponse contains the block requested and its commit. | Block | bytes | Requested [Block](../../core/data_structures.md#block), serialized as it was committed: the bytes of its parts, in order. On the wire this is the same as an embedded Block. | 1 | | Commit | [Commit](../../core/data_structures.md#commit) | Commit for the requested block | 2 | | PartSizes | repeated uint32 | Size of each part the block was committed with, in order. Empty when the block was cut into parts of 64kB, the last one shorter | 3 | +| PartProofs | repeated [Proof](../../core/data_structures.md#proof) | Merkle proof of each part the block was committed with, in order, alongside PartSizes. Empty when the part set header can be computed from the parts, which it cannot when a proof's total differs from the header's | 4 | -The receiver rebuilds the block's part set from `Block` and `PartSizes`: the -part set header in the commit's block ID is computed over the committed parts, -and decoding and re-encoding the block can change its bytes. +The receiver rebuilds the block's part set from `Block`, `PartSizes` and +`PartProofs`: the part set header in the commit's block ID is computed over the +committed parts, and decoding and re-encoding the block can change its bytes. ### StatusRequest diff --git a/types/part_set.go b/types/part_set.go index 6ad15886c5..2f999f8e40 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -194,28 +194,41 @@ func NewPartSetFromData(data []byte, partSize uint32) *PartSet { // NewPartSetFromData would, so this rebuilds the part set a block was // committed with from its bytes and the sizes of its parts. func NewPartSetFromPartSizes(data []byte, sizes []uint32) (*PartSet, error) { + if err := ValidatePartSizes(len(data), sizes); err != nil { + return nil, err + } + partsBytes := make([][]byte, len(sizes)) + offset := 0 + for i, size := range sizes { + partsBytes[i] = data[offset : offset+int(size)] + offset += int(size) + } + return newPartSetFromParts(partsBytes, len(data)), nil +} + +// ValidatePartSizes checks that parts of the given sizes are valid block parts +// that together cut exactly size bytes. +func ValidatePartSizes(size int, sizes []uint32) error { if len(sizes) == 0 { - return nil, errors.New("no part sizes") + return errors.New("no part sizes") } if len(sizes) > int(MaxBlockPartsCount) { - return nil, fmt.Errorf("too many parts: %d, max: %d", len(sizes), MaxBlockPartsCount) + return fmt.Errorf("too many parts: %d, max: %d", len(sizes), MaxBlockPartsCount) } - partsBytes := make([][]byte, len(sizes)) offset := 0 - for i, size := range sizes { - if size > BlockPartSizeBytes { - return nil, fmt.Errorf("part %d too big: %d bytes, max: %d", i, size, BlockPartSizeBytes) + for i, partSize := range sizes { + if partSize > BlockPartSizeBytes { + return fmt.Errorf("part %d too big: %d bytes, max: %d", i, partSize, BlockPartSizeBytes) } - if int(size) > len(data)-offset { - return nil, fmt.Errorf("part sizes exceed the %d bytes of data", len(data)) + if int(partSize) > size-offset { + return fmt.Errorf("part sizes exceed the %d bytes of data", size) } - partsBytes[i] = data[offset : offset+int(size)] - offset += int(size) + offset += int(partSize) } - if offset != len(data) { - return nil, fmt.Errorf("part sizes cover %d of the %d bytes of data", offset, len(data)) + if offset != size { + return fmt.Errorf("part sizes cover %d of the %d bytes of data", offset, size) } - return newPartSetFromParts(partsBytes, len(data)), nil + return nil } // newPartSetFromParts returns a full PartSet of the given parts, which must be From bf7abf6b730750b3cc57fec0700a4b318afc80ce Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Mon, 9 Nov 2026 19:53:47 +0700 Subject: [PATCH 3/8] test(blocksync): keep malformed part metadata out of the backlog Covers a later response with oversized part metadata while a lower height is missing, through the synchronizer rather than the fetch job alone. The same response without the metadata is held. Co-Authored-By: Claude Opus 5.5 --- internal/blocksync/block_fetch_job_test.go | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/internal/blocksync/block_fetch_job_test.go b/internal/blocksync/block_fetch_job_test.go index 47b0889336..00c7f63b03 100644 --- a/internal/blocksync/block_fetch_job_test.go +++ b/internal/blocksync/block_fetch_job_test.go @@ -16,6 +16,7 @@ import ( "github.com/dashpay/tenderdash/internal/p2p/client" "github.com/dashpay/tenderdash/internal/p2p/client/mocks" p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" + statemocks "github.com/dashpay/tenderdash/internal/state/mocks" statefactory "github.com/dashpay/tenderdash/internal/state/test/factory" "github.com/dashpay/tenderdash/internal/test/factory" tmrequire "github.com/dashpay/tenderdash/internal/test/require" @@ -124,6 +125,41 @@ func (suite *BlockFetchJobTestSuite) TestExecuteRejectsOversizedPartMetadata() { suite.Nil(res.Value) } +// TestOversizedPartMetadataIsNotHeldBehindMissingHeight pins the same through +// the synchronizer: while a lower height is missing, a later response with too +// many part sizes stays out of the backlog, where the same response without +// them is held. +func (suite *BlockFetchJobTestSuite) TestOversizedPartMetadataIsNotHeldBehindMissingHeight() { + ctx := context.Background() + const later = int64(9) + fetch := func(resp *bcproto.BlockResponse) workerpool.Result { + suite.client. + On("GetBlock", mock.Anything, later, suite.peer.peerID). + Once(). + Return(suite.getBlockReturnFunc(suite.promiseResolveResponse(resp)), nil) + return blockFetchJobHandler(suite.client, suite.peer, later)(ctx) + } + resultCh := make(chan workerpool.Result, 1) + wp := workerpool.New(1, workerpool.WithResultCh(resultCh)) + applier := newBlockApplier(statemocks.NewExecutor(suite.T()), statemocks.NewBlockStore(suite.T())) + pool := NewSynchronizer(1, suite.client, applier, WithWorkerPool(wp)) + pool.AddPeer(suite.peer) + + oversized := *suite.responses[later-1] + oversized.PartSizes = make([]uint32, types.MaxBlockPartsCount+1) + resultCh <- fetch(&oversized) + suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) + height, pendingBytes := pool.backlog() + suite.EqualValues(1, height, "the lower height must still be the one being waited for") + suite.Zero(pendingBytes) + + resultCh <- fetch(suite.responses[later-1]) + suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) + height, pendingBytes = pool.backlog() + suite.EqualValues(1, height, "the lower height must still be the one being waited for") + suite.Equal(len(suite.responses[later-1].Block), pendingBytes) +} + // TestExecuteCarriesPeerConnection pins that a job's outcome stays attributed to the // connection generation the peer was reserved under, including a timeout, which has // no response envelope to take a generation from. From 0f343d410998653d036e23a0c54135f00bdaadd9 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Mon, 9 Nov 2026 20:54:15 +0700 Subject: [PATCH 4/8] fix(blocksync): bound part metadata before decoding it Every received message is decoded before it is matched to a request, and protobuf decoding allocates for each occurrence of a repeated field. A peer could send an unsolicited response of repeated empty proofs that expanded about thirty times in memory. The part sizes and proofs move into one opaque part_layout bytes field, a serialized BlockPartLayout, decoded only for a response to a request and after its encoding passes bounds checks on the number of sizes, proofs and aunts and the length of hashes. A repeated block field also left the allocation of a larger occurrence held behind the small last value, uncharged to the backlog. Oversized backing storage is now copied into a right-sized allocation. Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- internal/blocksync/block_fetch_job_test.go | 4 +- internal/blocksync/committed_parts.go | 155 ++++++++-- internal/blocksync/p2p_msg_handler.go | 7 +- internal/blocksync/p2p_msg_handler_test.go | 128 ++++++-- internal/blocksync/synchronizer.go | 2 +- internal/p2p/channel_params.go | 11 +- proto/tendermint/blocksync/types.pb.go | 340 ++++++++++++++++----- proto/tendermint/blocksync/types.proto | 26 +- spec/p2p/messages/block-sync.md | 20 +- 10 files changed, 540 insertions(+), 155 deletions(-) diff --git a/go.mod b/go.mod index 4f17668d1c..7347ffa421 100644 --- a/go.mod +++ b/go.mod @@ -276,7 +276,6 @@ require ( golang.org/x/tools v0.52.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20261012220534-4341edf3f941 // indirect - google.golang.org/protobuf v1.36.13-0.20261016172727-e9ee2a76ab8e // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.8.2 // indirect mvdan.cc/gofumpt v0.12.0 // indirect @@ -337,6 +336,7 @@ require ( golang.org/x/term v0.47.0 golang.org/x/time v0.16.0 google.golang.org/grpc v1.85.0 + google.golang.org/protobuf v1.36.13-0.20261016172727-e9ee2a76ab8e pgregory.net/rapid v1.3.0 ) diff --git a/internal/blocksync/block_fetch_job_test.go b/internal/blocksync/block_fetch_job_test.go index 00c7f63b03..cac3df332c 100644 --- a/internal/blocksync/block_fetch_job_test.go +++ b/internal/blocksync/block_fetch_job_test.go @@ -115,7 +115,7 @@ func (suite *BlockFetchJobTestSuite) TestExecuteRejectsOversizedPartMetadata() { ctx := context.Background() const requested = int64(9) oversized := *suite.responses[requested-1] - oversized.PartSizes = make([]uint32, types.MaxBlockPartsCount+1) + oversized.PartLayout = partLayout(suite.T(), make([]uint32, types.MaxBlockPartsCount+1), nil) suite.client. On("GetBlock", mock.Anything, requested, suite.peer.peerID). Once(). @@ -146,7 +146,7 @@ func (suite *BlockFetchJobTestSuite) TestOversizedPartMetadataIsNotHeldBehindMis pool.AddPeer(suite.peer) oversized := *suite.responses[later-1] - oversized.PartSizes = make([]uint32, types.MaxBlockPartsCount+1) + oversized.PartLayout = partLayout(suite.T(), make([]uint32, types.MaxBlockPartsCount+1), nil) resultCh <- fetch(&oversized) suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) height, pendingBytes := pool.backlog() diff --git a/internal/blocksync/committed_parts.go b/internal/blocksync/committed_parts.go index 0eee4194e2..a224040c11 100644 --- a/internal/blocksync/committed_parts.go +++ b/internal/blocksync/committed_parts.go @@ -3,8 +3,14 @@ package blocksync import ( "errors" "fmt" + "slices" + "github.com/cosmos/gogoproto/proto" + "google.golang.org/protobuf/encoding/protowire" + + "github.com/dashpay/tenderdash/crypto" "github.com/dashpay/tenderdash/crypto/merkle" + "github.com/dashpay/tenderdash/internal/p2p" sm "github.com/dashpay/tenderdash/internal/state" tmmath "github.com/dashpay/tenderdash/libs/math" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" @@ -91,21 +97,42 @@ func standardPartSizes(partSizes []uint32, size int) bool { // committedPartsFromProto returns the parts in a block response, after checking // that they are well formed. It does not check them against the commit. func committedPartsFromProto(resp *bcproto.BlockResponse) (CommittedParts, error) { - parts := CommittedParts{Bytes: resp.Block, Sizes: resp.PartSizes} - if len(resp.PartSizes) > 0 { - if err := types.ValidatePartSizes(len(resp.Block), resp.PartSizes); err != nil { + blockBytes := resp.Block + // A repeated block field leaves the allocation of its longest occurrence + // behind the last one, and only the length is charged to the backlog. + if cap(blockBytes) > 2*len(blockBytes) { + blockBytes = slices.Clone(blockBytes) + } + parts := CommittedParts{Bytes: blockBytes} + if len(resp.PartLayout) == 0 { + return parts, nil + } + if len(resp.PartLayout) > p2p.BlockPartLayoutMaxBytes { + return CommittedParts{}, fmt.Errorf("part layout too big: %d bytes, max: %d", + len(resp.PartLayout), p2p.BlockPartLayoutMaxBytes) + } + if err := checkPartLayoutBounds(resp.PartLayout); err != nil { + return CommittedParts{}, err + } + var layout bcproto.BlockPartLayout + if err := proto.Unmarshal(resp.PartLayout, &layout); err != nil { + return CommittedParts{}, fmt.Errorf("cannot decode part layout: %w", err) + } + parts.Sizes = layout.Sizes + if len(layout.Sizes) > 0 { + if err := types.ValidatePartSizes(len(blockBytes), layout.Sizes); err != nil { return CommittedParts{}, err } } - if len(resp.PartProofs) == 0 { + if len(layout.Proofs) == 0 { return parts, nil } - if len(resp.PartProofs) != len(resp.PartSizes) { - return CommittedParts{}, fmt.Errorf("%d proofs for %d parts", len(resp.PartProofs), len(resp.PartSizes)) + if len(layout.Proofs) != len(layout.Sizes) { + return CommittedParts{}, fmt.Errorf("%d proofs for %d parts", len(layout.Proofs), len(layout.Sizes)) } - parts.Proofs = make([]merkle.Proof, len(resp.PartProofs)) - for i := range resp.PartProofs { - proof, err := merkle.ProofFromProto(&resp.PartProofs[i]) + parts.Proofs = make([]merkle.Proof, len(layout.Proofs)) + for i := range layout.Proofs { + proof, err := merkle.ProofFromProto(&layout.Proofs[i]) if err != nil { return CommittedParts{}, fmt.Errorf("proof of part %d: %w", i, err) } @@ -117,26 +144,106 @@ func committedPartsFromProto(resp *bcproto.BlockResponse) (CommittedParts, error return parts, nil } -// committedPartsSize returns the serialized size of the parts in a block -// response. -func committedPartsSize(resp *bcproto.BlockResponse) int { - size := len(resp.Block) + 4*len(resp.PartSizes) - for i := range resp.PartProofs { - size += resp.PartProofs[i].Size() - } - return size +// Field numbers of BlockPartLayout and tendermint.crypto.Proof. +const ( + layoutSizesField protowire.Number = 1 + layoutProofsField protowire.Number = 2 + proofLeafHashField protowire.Number = 3 + proofAuntsField protowire.Number = 4 +) + +// checkPartLayoutBounds refuses a serialized BlockPartLayout with more entries +// than a block can have, without decoding it. Decoding allocates tens of bytes +// for every occurrence of a repeated field, and an occurrence can take two +// bytes, so the limits have to hold before the layout is decoded. +func checkPartLayoutBounds(layout []byte) error { + sizes, proofs := 0, 0 + return walkFields(layout, func(number protowire.Number, wireType protowire.Type, value []byte) error { + switch { + case number == layoutSizesField && wireType == protowire.VarintType: + sizes++ + case number == layoutSizesField && wireType == protowire.BytesType: + // a packed list has one varint per byte without the continuation bit + for _, b := range value { + if b < 0x80 { + sizes++ + } + } + case number == layoutProofsField && wireType == protowire.BytesType: + proofs++ + if err := checkProofBounds(value); err != nil { + return fmt.Errorf("proof %d: %w", proofs-1, err) + } + } + if sizes > int(types.MaxBlockPartsCount) { + return fmt.Errorf("too many parts: more than %d", types.MaxBlockPartsCount) + } + if proofs > int(types.MaxBlockPartsCount) { + return fmt.Errorf("too many proofs: more than %d", types.MaxBlockPartsCount) + } + return nil + }) } -// protoProofs returns Proofs in their protobuf form. -func (p CommittedParts) protoProofs() []tmcrypto.Proof { - if p.Proofs == nil { +// checkProofBounds is checkPartLayoutBounds for a serialized proof. +func checkProofBounds(proof []byte) error { + aunts := 0 + return walkFields(proof, func(number protowire.Number, wireType protowire.Type, value []byte) error { + if wireType != protowire.BytesType || (number != proofLeafHashField && number != proofAuntsField) { + return nil + } + if len(value) > crypto.HashSize { + return fmt.Errorf("hash too long: %d bytes, max: %d", len(value), crypto.HashSize) + } + if number == proofAuntsField { + aunts++ + } + if aunts > merkle.MaxAunts { + return fmt.Errorf("too many aunts: more than %d", merkle.MaxAunts) + } return nil + }) +} + +// walkFields calls visit for every field of a serialized message, with the +// contents of the field when it is length-delimited. +func walkFields(bz []byte, visit func(protowire.Number, protowire.Type, []byte) error) error { + for len(bz) > 0 { + number, wireType, n := protowire.ConsumeTag(bz) + if n < 0 { + return protowire.ParseError(n) + } + bz = bz[n:] + n = protowire.ConsumeFieldValue(number, wireType, bz) + if n < 0 { + return protowire.ParseError(n) + } + value := bz[:n] + if wireType == protowire.BytesType { + value, _ = protowire.ConsumeBytes(value) + } + if err := visit(number, wireType, value); err != nil { + return err + } + bz = bz[n:] } - proofs := make([]tmcrypto.Proof, len(p.Proofs)) - for i := range p.Proofs { - proofs[i] = *p.Proofs[i].ToProto() + return nil +} + +// partLayout returns the serialized BlockPartLayout of the parts, nil when the +// parts need none. +func (p CommittedParts) partLayout() ([]byte, error) { + if len(p.Sizes) == 0 && len(p.Proofs) == 0 { + return nil, nil + } + layout := bcproto.BlockPartLayout{Sizes: p.Sizes} + if len(p.Proofs) > 0 { + layout.Proofs = make([]tmcrypto.Proof, len(p.Proofs)) + for i := range p.Proofs { + layout.Proofs[i] = *p.Proofs[i].ToProto() + } } - return proofs + return proto.Marshal(&layout) } // partSet returns the part set the block was committed with. header is the diff --git a/internal/blocksync/p2p_msg_handler.go b/internal/blocksync/p2p_msg_handler.go index 0a76d5da42..5471e0302e 100644 --- a/internal/blocksync/p2p_msg_handler.go +++ b/internal/blocksync/p2p_msg_handler.go @@ -167,11 +167,14 @@ func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (pro if commit == nil { return nil, false, fmt.Errorf("%w at height %d", errStoredBlockMissingCommit, msg.Height) } + layout, err := parts.partLayout() + if err != nil { + return nil, false, fmt.Errorf("failed to encode part layout: %w", err) + } return &bcproto.BlockResponse{ Block: parts.Bytes, Commit: commit.ToProto(), - PartSizes: parts.Sizes, - PartProofs: parts.protoProofs(), + PartLayout: layout, }, true, nil } diff --git a/internal/blocksync/p2p_msg_handler_test.go b/internal/blocksync/p2p_msg_handler_test.go index 9d86b33002..e247ae4628 100644 --- a/internal/blocksync/p2p_msg_handler_test.go +++ b/internal/blocksync/p2p_msg_handler_test.go @@ -1,6 +1,7 @@ package blocksync import ( + "bytes" "context" "encoding/binary" "fmt" @@ -587,11 +588,12 @@ func TestServeBlockAsCommitted(t *testing.T) { msg, ok := request(height).(*bcproto.BlockResponse) require.True(t, ok) require.Equal(t, tc.wantBytes, msg.Block) - require.Equal(t, tc.wantPartSizes, msg.PartSizes) - require.Equal(t, tc.wantProofs, len(msg.PartProofs) > 0) + require.Equal(t, tc.wantPartSizes == nil, len(msg.PartLayout) == 0) resp, err := BlockResponseFromProto(msg, "peer") require.NoError(t, err) + require.Equal(t, tc.wantPartSizes, resp.Parts.Sizes) + require.Equal(t, tc.wantProofs, len(resp.Parts.Proofs) > 0) require.Equal(t, blocks[i].Hash(), resp.Block.Hash()) require.Equal(t, blocks[i+1].LastCommit.BlockID, resp.Commit.BlockID) parts, err := resp.Parts.partSet(tc.parts.Header()) @@ -626,15 +628,10 @@ func TestBlockResponseWireCompatible(t *testing.T) { commitBytes, err := proto.Marshal(commit) require.NoError(t, err) - // an embedded message is its encoding, length-delimited - field := func(bz []byte, number byte, value []byte) []byte { - bz = append(bz, number<<3|2) - bz = binary.AppendUvarint(bz, uint64(len(value))) - return append(bz, value...) - } embedded, err := proto.Marshal(pbb) require.NoError(t, err) - older := field(field(nil, 1, embedded), 2, commitBytes) + // an embedded message is its encoding, length-delimited + older := appendBytesField(appendBytesField(nil, 1, embedded), 2, commitBytes) newer, err := proto.Marshal(&bcproto.BlockResponse{Block: blockBytes(t, block), Commit: commit}) require.NoError(t, err) @@ -648,6 +645,21 @@ func TestBlockResponseWireCompatible(t *testing.T) { require.Equal(t, CommittedParts{Bytes: embedded}, resp.Parts) } +// appendBytesField appends a length-delimited field to serialized protobuf. +func appendBytesField(bz []byte, number byte, value []byte) []byte { + bz = append(bz, number<<3|2) + bz = binary.AppendUvarint(bz, uint64(len(value))) + return append(bz, value...) +} + +// partLayout returns a serialized BlockPartLayout. +func partLayout(t *testing.T, sizes []uint32, proofs []crypto.Proof) []byte { + t.Helper() + bz, err := proto.Marshal(&bcproto.BlockPartLayout{Sizes: sizes, Proofs: proofs}) + require.NoError(t, err) + return bz +} + // TestBlockResponseFromProtoRejectsMalformedParts checks that part metadata a // peer sent that cannot describe its block is refused while decoding the // response, before it can be held waiting for a lower height. @@ -664,33 +676,39 @@ func TestBlockResponseFromProtoRejectsMalformedParts(t *testing.T) { for i := range tooManyAunts.Aunts { tooManyAunts.Aunts[i] = make([]byte, tmcrypto.HashSize) } + longAunt := proof + longAunt.Aunts = [][]byte{make([]byte, tmcrypto.HashSize+1)} + tooManyParts := int(types.MaxBlockPartsCount) + 1 testCases := map[string]struct { - sizes []uint32 - proofs []crypto.Proof + layout []byte wantErr string }{ - "too many parts": {make([]uint32, types.MaxBlockPartsCount+1), nil, "too many parts"}, - "part too big": {[]uint32{types.BlockPartSizeBytes + 1}, nil, "too big"}, - "sizes short": {[]uint32{size - 1}, nil, "part sizes cover"}, - "sizes beyond block": {[]uint32{size + 1}, nil, "exceed"}, + "layout too big": {make([]byte, p2p.BlockPartLayoutMaxBytes+1), "part layout too big"}, + "not a layout": {[]byte{0x0a, 0x05}, "unexpected EOF"}, + "too many parts": {partLayout(t, make([]uint32, tooManyParts), nil), "too many parts"}, + "unpacked parts": {bytes.Repeat([]byte{1 << 3, 0}, tooManyParts), "too many parts"}, + "too many proofs": {bytes.Repeat([]byte{2<<3 | 2, 0}, tooManyParts), "too many proofs"}, + "too many aunts": {partLayout(t, []uint32{size}, []crypto.Proof{tooManyAunts}), "too many aunts"}, + "aunt too long": {partLayout(t, []uint32{size}, []crypto.Proof{longAunt}), "hash too long"}, + "part too big": {partLayout(t, []uint32{types.BlockPartSizeBytes + 1}, nil), "too big"}, + "sizes short": {partLayout(t, []uint32{size - 1}, nil), "part sizes cover"}, + "sizes beyond block": {partLayout(t, []uint32{size + 1}, nil), "exceed"}, "proofs without sizes": { - nil, []crypto.Proof{proof}, "1 proofs for 0 parts", + partLayout(t, nil, []crypto.Proof{proof}), "1 proofs for 0 parts", }, "too few proofs": { - []uint32{size - 1, 1}, []crypto.Proof{proof}, "1 proofs for 2 parts", + partLayout(t, []uint32{size - 1, 1}, []crypto.Proof{proof}), "1 proofs for 2 parts", }, "proof of another part": { - []uint32{size}, []crypto.Proof{{Total: 1, Index: 1, LeafHash: proof.LeafHash}}, "has index 1", + partLayout(t, []uint32{size}, []crypto.Proof{{Total: 1, Index: 1, LeafHash: proof.LeafHash}}), "has index 1", }, - "proof too long": {[]uint32{size}, []crypto.Proof{tooManyAunts}, "aunts"}, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { _, err := BlockResponseFromProto(&bcproto.BlockResponse{ Block: bz, Commit: blocks[1].LastCommit.ToProto(), - PartSizes: tc.sizes, - PartProofs: tc.proofs, + PartLayout: tc.layout, }, "peer") require.ErrorContains(t, err, tc.wantErr) }) @@ -706,14 +724,70 @@ func TestBlockResponseSizeChargesPartMetadata(t *testing.T) { blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) bz := blockBytes(t, blocks[0]) proof := crypto.Proof{Total: 2, LeafHash: make([]byte, tmcrypto.HashSize)} + layout := partLayout(t, []uint32{uint32(len(bz))}, []crypto.Proof{proof}) resp, err := BlockResponseFromProto(&bcproto.BlockResponse{ Block: bz, Commit: blocks[1].LastCommit.ToProto(), - PartSizes: []uint32{uint32(len(bz))}, - PartProofs: []crypto.Proof{proof}, + PartLayout: layout, }, "peer") require.NoError(t, err) - require.Equal(t, len(bz)+4+proof.Size(), resp.Size) + require.Equal(t, len(bz)+len(layout), resp.Size) +} + +// TestPartLayoutIsOpaqueAtEnvelopeBoundary checks that part metadata costs no +// per-entry allocation when a message is received, which happens before it is +// matched to a request, nor when the layout is refused. A layout of empty +// proofs is the smallest encoding of the most entries. +func TestPartLayoutIsOpaqueAtEnvelopeBoundary(t *testing.T) { + const allowedAllocs = 32 + layout := bytes.Repeat([]byte{2<<3 | 2, 0}, int(types.MaxBlockPartsCount)+1) + // block_response is field 9 of the envelope, part_layout field 3 of the response + raw := appendBytesField(nil, 9, appendBytesField(nil, 3, layout)) + + require.Less(t, testing.AllocsPerRun(10, func() { + var envelope p2pproto.Envelope + _ = proto.Unmarshal(raw, &envelope) + }), float64(allowedAllocs), "decoding the envelope must not allocate per layout entry") + + var envelope p2pproto.Envelope + require.NoError(t, proto.Unmarshal(raw, &envelope)) + msg, err := envelope.Unwrap() + require.NoError(t, err) + resp, ok := msg.(*bcproto.BlockResponse) + require.True(t, ok) + require.Equal(t, layout, resp.PartLayout) + + resp.Block = []byte{1} + require.Less(t, testing.AllocsPerRun(10, func() { + _, _ = committedPartsFromProto(resp) + }), float64(allowedAllocs), "refusing the layout must not allocate per layout entry") + _, err = committedPartsFromProto(resp) + require.ErrorContains(t, err, "too many proofs") +} + +// TestBlockResponseDoesNotRetainOverwrittenBlockField checks that a repeated +// block field does not leave the allocation of an earlier, larger occurrence +// held behind the small one that is decoded and charged to the backlog. +func TestBlockResponseDoesNotRetainOverwrittenBlockField(t *testing.T) { + ctx := context.Background() + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) + bz := blockBytes(t, blocks[0]) + commitBytes, err := proto.Marshal(blocks[1].LastCommit.ToProto()) + require.NoError(t, err) + const overwritten = 1 << 20 + raw := appendBytesField(appendBytesField(appendBytesField(nil, 1, make([]byte, overwritten)), 1, bz), 2, commitBytes) + + var decoded bcproto.BlockResponse + require.NoError(t, proto.Unmarshal(raw, &decoded)) + require.GreaterOrEqual(t, cap(decoded.Block), overwritten, "the decoder is expected to reuse the larger allocation") + + resp, err := BlockResponseFromProto(&decoded, "peer") + require.NoError(t, err) + require.Equal(t, bz, resp.Parts.Bytes) + require.Equal(t, len(bz), resp.Size) + require.LessOrEqual(t, cap(resp.Parts.Bytes), 2*len(bz)) } // TestBlockResponseFitsBlockSyncChannel checks that the largest block response @@ -735,6 +809,9 @@ func TestBlockResponseFitsBlockSyncChannel(t *testing.T) { PartSetHeader: tmproto.PartSetHeader{Total: uint32(parts), Hash: make([]byte, tmcrypto.HashSize)}, StateID: make([]byte, tmcrypto.HashSize), } + layout := partLayout(t, sizes, proofs) + require.LessOrEqual(t, len(layout), p2p.BlockPartLayoutMaxBytes) + require.NoError(t, checkPartLayoutBounds(layout)) envelope := &p2pproto.Envelope{Attributes: map[string]string{client.RequestIDAttribute: uuid.NewString()}} require.NoError(t, envelope.Wrap(&bcproto.BlockResponse{ Block: make([]byte, types.MaxBlockSizeBytes), @@ -750,8 +827,7 @@ func TestBlockResponseFitsBlockSyncChannel(t *testing.T) { Signature: make([]byte, 96), }}, }, - PartSizes: sizes, - PartProofs: proofs, + PartLayout: layout, })) descriptor := p2p.ChannelDescriptors(config.TestConfig())[p2p.BlockSyncChannel] diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index 41dba30593..3358644793 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -837,6 +837,6 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* Block: block, Parts: parts, Commit: commit, - Size: committedPartsSize(resp), + Size: len(resp.Block) + len(resp.PartLayout), }, nil } diff --git a/internal/p2p/channel_params.go b/internal/p2p/channel_params.go index c603942730..befce7340f 100644 --- a/internal/p2p/channel_params.go +++ b/internal/p2p/channel_params.go @@ -60,12 +60,13 @@ const ( // response: its key and length, the two int64 varints with their keys, and // the leaf hash and aunts with their keys and lengths. blockPartProofMaxBytes = 4 + 2*(1+10) + (merkle.MaxAunts+1)*(2+crypto.HashSize) + // BlockPartLayoutMaxBytes bounds a serialized BlockPartLayout: a size and a + // proof for the most parts a block can have, and the keys of the two fields. + BlockPartLayoutMaxBytes = int(types.MaxBlockPartsCount)*(blockPartSizeMaxBytes+blockPartProofMaxBytes) + 16 // blockSyncMaxOverheadBytes bounds what a block response carries besides - // the block: part metadata for the most parts a block can have, a commit, - // whose vote extensions came in one vote on a maxMsgSize channel, and the - // envelope with its attributes. - blockSyncMaxOverheadBytes = int(types.MaxBlockPartsCount)*(blockPartSizeMaxBytes+blockPartProofMaxBytes) + - maxMsgSize + 1024 + // the block: its part layout, a commit, whose vote extensions came in one + // vote on a maxMsgSize channel, and the envelope with its attributes. + blockSyncMaxOverheadBytes = BlockPartLayoutMaxBytes + maxMsgSize + 1024 // BlockSyncMaxMsgSize is the maximum size of a block sync message BlockSyncMaxMsgSize = types.MaxBlockSizeBytes + diff --git a/proto/tendermint/blocksync/types.pb.go b/proto/tendermint/blocksync/types.pb.go index 0b45a2d86f..ba34a2242d 100644 --- a/proto/tendermint/blocksync/types.pb.go +++ b/proto/tendermint/blocksync/types.pb.go @@ -126,15 +126,12 @@ type BlockResponse struct { // releases. Block []byte `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` Commit *types.Commit `protobuf:"bytes,2,opt,name=commit,proto3" json:"commit,omitempty"` - // part_sizes is the size of each part the block was committed with, in - // order. It is empty when the block was cut into parts of 64kB, the last - // one shorter, the way honest proposers cut it. - PartSizes []uint32 `protobuf:"varint,3,rep,packed,name=part_sizes,json=partSizes,proto3" json:"part_sizes,omitempty"` - // part_proofs is the Merkle proof of each part the block was committed with, - // in order, alongside part_sizes. It is empty when the part set header can - // be computed from the parts, which it cannot when a proof's total differs - // from the header's. - PartProofs []crypto.Proof `protobuf:"bytes,4,rep,name=part_proofs,json=partProofs,proto3" json:"part_proofs"` + // part_layout is a serialized BlockPartLayout. It is empty when the block was + // cut into parts of 64kB, the last one shorter, the way honest proposers cut + // it. It is opaque here because every received message is decoded before it + // is matched to a request: the layout is only decoded once it is known to be + // the response to one, and within bounds. + PartLayout []byte `protobuf:"bytes,3,opt,name=part_layout,json=partLayout,proto3" json:"part_layout,omitempty"` } func (m *BlockResponse) Reset() { *m = BlockResponse{} } @@ -184,16 +181,67 @@ func (m *BlockResponse) GetCommit() *types.Commit { return nil } -func (m *BlockResponse) GetPartSizes() []uint32 { +func (m *BlockResponse) GetPartLayout() []byte { if m != nil { - return m.PartSizes + return m.PartLayout } return nil } -func (m *BlockResponse) GetPartProofs() []crypto.Proof { +// BlockPartLayout describes the parts a block was committed with, when they +// cannot be derived from the block's bytes. +type BlockPartLayout struct { + // sizes is the size of each part, in order. + Sizes []uint32 `protobuf:"varint,1,rep,packed,name=sizes,proto3" json:"sizes,omitempty"` + // proofs is the Merkle proof of each part, in order. It is empty when the + // part set header can be computed from the parts, which it cannot when a + // proof's total differs from the header's. + Proofs []crypto.Proof `protobuf:"bytes,2,rep,name=proofs,proto3" json:"proofs"` +} + +func (m *BlockPartLayout) Reset() { *m = BlockPartLayout{} } +func (m *BlockPartLayout) String() string { return proto.CompactTextString(m) } +func (*BlockPartLayout) ProtoMessage() {} +func (*BlockPartLayout) Descriptor() ([]byte, []int) { + return fileDescriptor_19b397c236e0fa07, []int{3} +} +func (m *BlockPartLayout) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *BlockPartLayout) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_BlockPartLayout.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *BlockPartLayout) XXX_Merge(src proto.Message) { + xxx_messageInfo_BlockPartLayout.Merge(m, src) +} +func (m *BlockPartLayout) XXX_Size() int { + return m.Size() +} +func (m *BlockPartLayout) XXX_DiscardUnknown() { + xxx_messageInfo_BlockPartLayout.DiscardUnknown(m) +} + +var xxx_messageInfo_BlockPartLayout proto.InternalMessageInfo + +func (m *BlockPartLayout) GetSizes() []uint32 { if m != nil { - return m.PartProofs + return m.Sizes + } + return nil +} + +func (m *BlockPartLayout) GetProofs() []crypto.Proof { + if m != nil { + return m.Proofs } return nil } @@ -206,7 +254,7 @@ func (m *StatusRequest) Reset() { *m = StatusRequest{} } func (m *StatusRequest) String() string { return proto.CompactTextString(m) } func (*StatusRequest) ProtoMessage() {} func (*StatusRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_19b397c236e0fa07, []int{3} + return fileDescriptor_19b397c236e0fa07, []int{4} } func (m *StatusRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -245,7 +293,7 @@ func (m *StatusResponse) Reset() { *m = StatusResponse{} } func (m *StatusResponse) String() string { return proto.CompactTextString(m) } func (*StatusResponse) ProtoMessage() {} func (*StatusResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_19b397c236e0fa07, []int{4} + return fileDescriptor_19b397c236e0fa07, []int{5} } func (m *StatusResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -292,6 +340,7 @@ func init() { proto.RegisterType((*BlockRequest)(nil), "tendermint.blocksync.BlockRequest") proto.RegisterType((*NoBlockResponse)(nil), "tendermint.blocksync.NoBlockResponse") proto.RegisterType((*BlockResponse)(nil), "tendermint.blocksync.BlockResponse") + proto.RegisterType((*BlockPartLayout)(nil), "tendermint.blocksync.BlockPartLayout") proto.RegisterType((*StatusRequest)(nil), "tendermint.blocksync.StatusRequest") proto.RegisterType((*StatusResponse)(nil), "tendermint.blocksync.StatusResponse") } @@ -299,29 +348,30 @@ func init() { func init() { proto.RegisterFile("tendermint/blocksync/types.proto", fileDescriptor_19b397c236e0fa07) } var fileDescriptor_19b397c236e0fa07 = []byte{ - // 346 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xc1, 0x4a, 0xf3, 0x40, - 0x14, 0x85, 0x33, 0x7f, 0xfa, 0x17, 0xbc, 0x6d, 0x2d, 0x0c, 0x45, 0x42, 0xb1, 0x31, 0x64, 0x21, - 0x75, 0x93, 0x11, 0xdd, 0xb8, 0x10, 0x84, 0xba, 0x17, 0x99, 0xee, 0xdc, 0x48, 0x92, 0x8e, 0x49, - 0xd0, 0x64, 0xc6, 0xcc, 0x74, 0x11, 0x9f, 0xc2, 0xb7, 0xf1, 0x15, 0xba, 0xec, 0xd2, 0x95, 0x48, - 0xf3, 0x22, 0x92, 0x49, 0xd4, 0x28, 0xea, 0x6e, 0xee, 0x39, 0x1f, 0x67, 0xe6, 0xdc, 0x04, 0x1c, - 0xc5, 0xb2, 0x05, 0xcb, 0xd3, 0x24, 0x53, 0x24, 0xb8, 0xe3, 0xe1, 0xad, 0x2c, 0xb2, 0x90, 0xa8, - 0x42, 0x30, 0xe9, 0x89, 0x9c, 0x2b, 0x8e, 0x47, 0x9f, 0x84, 0xf7, 0x41, 0x8c, 0x47, 0x11, 0x8f, - 0xb8, 0x06, 0x48, 0x75, 0xaa, 0xd9, 0xf1, 0xa4, 0x95, 0x16, 0xe6, 0x85, 0x50, 0x9c, 0x88, 0x9c, - 0xf3, 0x9b, 0xc6, 0xde, 0x6d, 0xd9, 0xfa, 0x8a, 0xf6, 0x45, 0xee, 0x3e, 0xf4, 0x67, 0x55, 0x3e, - 0x65, 0xf7, 0x4b, 0x26, 0x15, 0xde, 0x81, 0x6e, 0xcc, 0x92, 0x28, 0x56, 0x16, 0x72, 0xd0, 0xd4, - 0xa4, 0xcd, 0xe4, 0x1e, 0xc0, 0xf0, 0x82, 0x37, 0xa4, 0x14, 0x3c, 0x93, 0xec, 0x57, 0xf4, 0x09, - 0xc1, 0xe0, 0x2b, 0x39, 0x82, 0xff, 0xba, 0x84, 0x06, 0xfb, 0xb4, 0x1e, 0xf0, 0x21, 0x74, 0x43, - 0x9e, 0xa6, 0x89, 0xb2, 0xfe, 0x39, 0x68, 0xda, 0x3b, 0xb2, 0xbc, 0x56, 0xe9, 0xfa, 0x8d, 0xe7, - 0xda, 0xa7, 0x0d, 0x87, 0x27, 0x00, 0xc2, 0xcf, 0xd5, 0xb5, 0x4c, 0x1e, 0x98, 0xb4, 0x4c, 0xc7, - 0x9c, 0x0e, 0xe8, 0x56, 0xa5, 0xcc, 0x2b, 0x01, 0x9f, 0x41, 0x4f, 0xdb, 0xba, 0xbd, 0xb4, 0x3a, - 0x8e, 0xf9, 0x3d, 0xb5, 0x5e, 0x8f, 0x77, 0x59, 0x01, 0xb3, 0xce, 0xea, 0x65, 0xcf, 0xa0, 0x3a, - 0x51, 0x0b, 0xd2, 0x1d, 0xc2, 0x60, 0xae, 0x7c, 0xb5, 0x94, 0xcd, 0x36, 0xdc, 0x53, 0xd8, 0x7e, - 0x17, 0xfe, 0x2e, 0x8d, 0x31, 0x74, 0x02, 0x5f, 0x32, 0x5d, 0xc5, 0xa4, 0xfa, 0x3c, 0xa3, 0xab, - 0x8d, 0x8d, 0xd6, 0x1b, 0x1b, 0xbd, 0x6e, 0x6c, 0xf4, 0x58, 0xda, 0xc6, 0xba, 0xb4, 0x8d, 0xe7, - 0xd2, 0x36, 0xae, 0x4e, 0xa2, 0x44, 0xc5, 0xcb, 0xc0, 0x0b, 0x79, 0x4a, 0x16, 0xbe, 0x8c, 0x85, - 0x5f, 0x90, 0xfa, 0x99, 0xd5, 0x44, 0xea, 0x4f, 0xfc, 0xd3, 0x4f, 0x12, 0x74, 0xb5, 0x77, 0xfc, - 0x36, 0x00, 0x9f, 0xdb, 0x4b, 0x3e, 0x43, 0x02, 0x00, 0x00, + // 361 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xc1, 0x4a, 0xeb, 0x40, + 0x14, 0x86, 0x93, 0xa6, 0x37, 0x8b, 0xd3, 0xf6, 0x16, 0x42, 0xb9, 0x84, 0x72, 0x6f, 0x1a, 0xb2, + 0xb8, 0xd4, 0x4d, 0x46, 0x14, 0xc4, 0x85, 0xab, 0xba, 0x15, 0x29, 0xe3, 0xce, 0x4d, 0x49, 0xd2, + 0x31, 0x09, 0x36, 0x99, 0x31, 0x33, 0x01, 0xe3, 0x53, 0xf8, 0x58, 0x5d, 0x76, 0xe9, 0x4a, 0xa4, + 0x7d, 0x11, 0xc9, 0x4c, 0xac, 0x51, 0xd4, 0xdd, 0x9c, 0xf3, 0x7f, 0x9c, 0x7f, 0xce, 0xcf, 0x01, + 0x57, 0x90, 0x7c, 0x49, 0x8a, 0x2c, 0xcd, 0x05, 0x0a, 0x57, 0x34, 0xba, 0xe5, 0x55, 0x1e, 0x21, + 0x51, 0x31, 0xc2, 0x7d, 0x56, 0x50, 0x41, 0xad, 0xd1, 0x3b, 0xe1, 0xef, 0x89, 0xf1, 0x28, 0xa6, + 0x31, 0x95, 0x00, 0xaa, 0x5f, 0x8a, 0x1d, 0xff, 0x6b, 0x4d, 0x8b, 0x8a, 0x8a, 0x09, 0x8a, 0x58, + 0x41, 0xe9, 0x4d, 0x23, 0xff, 0x6d, 0xc9, 0xd2, 0xa2, 0x6d, 0xe4, 0xfd, 0x87, 0xfe, 0xac, 0x9e, + 0x8f, 0xc9, 0x5d, 0x49, 0xb8, 0xb0, 0xfe, 0x80, 0x99, 0x90, 0x34, 0x4e, 0x84, 0xad, 0xbb, 0xfa, + 0xd4, 0xc0, 0x4d, 0xe5, 0x1d, 0xc0, 0xf0, 0x92, 0x36, 0x24, 0x67, 0x34, 0xe7, 0xe4, 0x5b, 0xf4, + 0x1e, 0x06, 0x1f, 0xc1, 0x11, 0xfc, 0x92, 0x3b, 0x48, 0xae, 0x8f, 0x55, 0x61, 0x1d, 0x82, 0x19, + 0xd1, 0x2c, 0x4b, 0x85, 0xdd, 0x71, 0xf5, 0x69, 0xef, 0xc8, 0xf6, 0x5b, 0x3b, 0xab, 0x2f, 0x9e, + 0x4b, 0x1d, 0x37, 0x9c, 0x35, 0x81, 0x1e, 0x0b, 0x0a, 0xb1, 0x58, 0x05, 0x15, 0x2d, 0x85, 0x6d, + 0xc8, 0x69, 0x50, 0xb7, 0x2e, 0x64, 0xc7, 0x5b, 0xc0, 0x50, 0x3a, 0xcf, 0xf7, 0xad, 0xda, 0x9b, + 0xa7, 0x0f, 0x84, 0xdb, 0xba, 0x6b, 0x4c, 0x07, 0x58, 0x15, 0xd6, 0x09, 0x98, 0x32, 0x22, 0x6e, + 0x77, 0x5c, 0xe3, 0xb3, 0xb7, 0xca, 0xd0, 0x9f, 0xd7, 0xc0, 0xac, 0xbb, 0x7e, 0x9e, 0x68, 0xb8, + 0xa1, 0xbd, 0x21, 0x0c, 0xae, 0x44, 0x20, 0x4a, 0xde, 0xc4, 0xe5, 0x9d, 0xc1, 0xef, 0xb7, 0xc6, + 0xcf, 0xa9, 0x58, 0x16, 0x74, 0xc3, 0x80, 0x13, 0xb9, 0xac, 0x81, 0xe5, 0x7b, 0x86, 0xd7, 0x5b, + 0x47, 0xdf, 0x6c, 0x1d, 0xfd, 0x65, 0xeb, 0xe8, 0x8f, 0x3b, 0x47, 0xdb, 0xec, 0x1c, 0xed, 0x69, + 0xe7, 0x68, 0xd7, 0xa7, 0x71, 0x2a, 0x92, 0x32, 0xf4, 0x23, 0x9a, 0xa1, 0x65, 0xc0, 0x13, 0x16, + 0x54, 0x48, 0x7d, 0xb1, 0xae, 0x90, 0xba, 0x81, 0xaf, 0xae, 0x28, 0x34, 0xa5, 0x76, 0xfc, 0x3a, + 0x00, 0xfa, 0x6f, 0x07, 0x87, 0x64, 0x02, 0x00, 0x00, } func (m *BlockRequest) Marshal() (dAtA []byte, err error) { @@ -409,35 +459,10 @@ func (m *BlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.PartProofs) > 0 { - for iNdEx := len(m.PartProofs) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.PartProofs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintTypes(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x22 - } - } - if len(m.PartSizes) > 0 { - dAtA2 := make([]byte, len(m.PartSizes)*10) - var j1 int - for _, num := range m.PartSizes { - for num >= 1<<7 { - dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j1++ - } - dAtA2[j1] = uint8(num) - j1++ - } - i -= j1 - copy(dAtA[i:], dAtA2[:j1]) - i = encodeVarintTypes(dAtA, i, uint64(j1)) + if len(m.PartLayout) > 0 { + i -= len(m.PartLayout) + copy(dAtA[i:], m.PartLayout) + i = encodeVarintTypes(dAtA, i, uint64(len(m.PartLayout))) i-- dAtA[i] = 0x1a } @@ -463,6 +488,64 @@ func (m *BlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *BlockPartLayout) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockPartLayout) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *BlockPartLayout) MarshalToSizedBuffer(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Proofs) > 0 { + for iNdEx := len(m.Proofs) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Proofs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintTypes(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + } + if len(m.Sizes) > 0 { + dAtA3 := make([]byte, len(m.Sizes)*10) + var j2 int + for _, num := range m.Sizes { + for num >= 1<<7 { + dAtA3[j2] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j2++ + } + dAtA3[j2] = uint8(num) + j2++ + } + i -= j2 + copy(dAtA[i:], dAtA3[:j2]) + i = encodeVarintTypes(dAtA, i, uint64(j2)) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *StatusRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -574,15 +657,28 @@ func (m *BlockResponse) Size() (n int) { l = m.Commit.Size() n += 1 + l + sovTypes(uint64(l)) } - if len(m.PartSizes) > 0 { + l = len(m.PartLayout) + if l > 0 { + n += 1 + l + sovTypes(uint64(l)) + } + return n +} + +func (m *BlockPartLayout) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Sizes) > 0 { l = 0 - for _, e := range m.PartSizes { + for _, e := range m.Sizes { l += sovTypes(uint64(e)) } n += 1 + sovTypes(uint64(l)) + l } - if len(m.PartProofs) > 0 { - for _, e := range m.PartProofs { + if len(m.Proofs) > 0 { + for _, e := range m.Proofs { l = e.Size() n += 1 + l + sovTypes(uint64(l)) } @@ -858,6 +954,90 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { } iNdEx = postIndex case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PartLayout", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthTypes + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PartLayout = append(m.PartLayout[:0], dAtA[iNdEx:postIndex]...) + if m.PartLayout == nil { + m.PartLayout = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipTypes(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthTypes + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlockPartLayout) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockPartLayout: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockPartLayout: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: if wireType == 0 { var v uint32 for shift := uint(0); ; shift += 7 { @@ -874,7 +1054,7 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { break } } - m.PartSizes = append(m.PartSizes, v) + m.Sizes = append(m.Sizes, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -909,8 +1089,8 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { } } elementCount = count - if elementCount != 0 && len(m.PartSizes) == 0 { - m.PartSizes = make([]uint32, 0, elementCount) + if elementCount != 0 && len(m.Sizes) == 0 { + m.Sizes = make([]uint32, 0, elementCount) } for iNdEx < postIndex { var v uint32 @@ -928,14 +1108,14 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { break } } - m.PartSizes = append(m.PartSizes, v) + m.Sizes = append(m.Sizes, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field PartSizes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Sizes", wireType) } - case 4: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PartProofs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Proofs", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -962,8 +1142,8 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.PartProofs = append(m.PartProofs, crypto.Proof{}) - if err := m.PartProofs[len(m.PartProofs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Proofs = append(m.Proofs, crypto.Proof{}) + if err := m.Proofs[len(m.Proofs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex diff --git a/proto/tendermint/blocksync/types.proto b/proto/tendermint/blocksync/types.proto index c62343969b..9055f81af1 100644 --- a/proto/tendermint/blocksync/types.proto +++ b/proto/tendermint/blocksync/types.proto @@ -28,15 +28,23 @@ message BlockResponse { // releases. bytes block = 1; tendermint.types.Commit commit = 2; - // part_sizes is the size of each part the block was committed with, in - // order. It is empty when the block was cut into parts of 64kB, the last - // one shorter, the way honest proposers cut it. - repeated uint32 part_sizes = 3; - // part_proofs is the Merkle proof of each part the block was committed with, - // in order, alongside part_sizes. It is empty when the part set header can - // be computed from the parts, which it cannot when a proof's total differs - // from the header's. - repeated tendermint.crypto.Proof part_proofs = 4 [(gogoproto.nullable) = false]; + // part_layout is a serialized BlockPartLayout. It is empty when the block was + // cut into parts of 64kB, the last one shorter, the way honest proposers cut + // it. It is opaque here because every received message is decoded before it + // is matched to a request: the layout is only decoded once it is known to be + // the response to one, and within bounds. + bytes part_layout = 3; +} + +// BlockPartLayout describes the parts a block was committed with, when they +// cannot be derived from the block's bytes. +message BlockPartLayout { + // sizes is the size of each part, in order. + repeated uint32 sizes = 1; + // proofs is the Merkle proof of each part, in order. It is empty when the + // part set header can be computed from the parts, which it cannot when a + // proof's total differs from the header's. + repeated tendermint.crypto.Proof proofs = 2 [(gogoproto.nullable) = false]; } // StatusRequest requests the status of a peer. diff --git a/spec/p2p/messages/block-sync.md b/spec/p2p/messages/block-sync.md index af7113c663..83977549fd 100644 --- a/spec/p2p/messages/block-sync.md +++ b/spec/p2p/messages/block-sync.md @@ -40,12 +40,22 @@ BlockResponse contains the block requested and its commit. |------------|------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| | Block | bytes | Requested [Block](../../core/data_structures.md#block), serialized as it was committed: the bytes of its parts, in order. On the wire this is the same as an embedded Block. | 1 | | Commit | [Commit](../../core/data_structures.md#commit) | Commit for the requested block | 2 | -| PartSizes | repeated uint32 | Size of each part the block was committed with, in order. Empty when the block was cut into parts of 64kB, the last one shorter | 3 | -| PartProofs | repeated [Proof](../../core/data_structures.md#proof) | Merkle proof of each part the block was committed with, in order, alongside PartSizes. Empty when the part set header can be computed from the parts, which it cannot when a proof's total differs from the header's | 4 | +| PartLayout | bytes | Serialized BlockPartLayout. Empty when the block was cut into parts of 64kB, the last one shorter | 3 | -The receiver rebuilds the block's part set from `Block`, `PartSizes` and -`PartProofs`: the part set header in the commit's block ID is computed over the -committed parts, and decoding and re-encoding the block can change its bytes. +The receiver rebuilds the block's part set from `Block` and `PartLayout`: the +part set header in the commit's block ID is computed over the committed parts, +and decoding and re-encoding the block can change its bytes. + +### BlockPartLayout + +BlockPartLayout describes the parts a block was committed with, when they cannot +be derived from the block's bytes. It is carried as opaque bytes so that it is +only decoded for a response to a request, after bounds checks on its encoding. + +| Name | Type | Description | Field Number | +|--------|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| +| Sizes | repeated uint32 | Size of each part, in order | 1 | +| Proofs | repeated [Proof](../../core/data_structures.md#proof) | Merkle proof of each part, in order. Empty when the part set header can be computed from the parts, which it cannot when a proof's total differs from the header's | 2 | ### StatusRequest From eeff79648a1e11779220f04196ed5e09064541c3 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Mon, 9 Nov 2026 21:51:13 +0700 Subject: [PATCH 5/8] fix(blocksync): make the part layout scan agree with the decoder The bounds check counted packed sizes by terminating bytes, but the generated decoder lets a varint in a packed field run past the end of that field. A truncated varint made the two split the layout into different fields, so proofs and aunts could be hidden from the check inside what it took for an unknown field. The check now decodes each packed varint within its field and accepts only the fields of a layout, well formed, which also removes the decoder's own rules for skipping unknown fields from the comparison. Co-Authored-By: Claude Opus 5.5 --- internal/blocksync/committed_parts.go | 24 ++++- internal/blocksync/p2p_msg_handler_test.go | 109 +++++++++++++++------ 2 files changed, 97 insertions(+), 36 deletions(-) diff --git a/internal/blocksync/committed_parts.go b/internal/blocksync/committed_parts.go index a224040c11..01d8db7add 100644 --- a/internal/blocksync/committed_parts.go +++ b/internal/blocksync/committed_parts.go @@ -148,6 +148,8 @@ func committedPartsFromProto(resp *bcproto.BlockResponse) (CommittedParts, error const ( layoutSizesField protowire.Number = 1 layoutProofsField protowire.Number = 2 + proofTotalField protowire.Number = 1 + proofIndexField protowire.Number = 2 proofLeafHashField protowire.Number = 3 proofAuntsField protowire.Number = 4 ) @@ -156,6 +158,11 @@ const ( // than a block can have, without decoding it. Decoding allocates tens of bytes // for every occurrence of a repeated field, and an occurrence can take two // bytes, so the limits have to hold before the layout is decoded. +// +// The limits only hold if this walk and the decoder split the bytes into the +// same fields, so it accepts nothing but the fields of a layout, each of them +// well formed. The generated decoder lets a varint in a packed field run past +// the end of that field, and skips unknown fields by rules of its own. func checkPartLayoutBounds(layout []byte) error { sizes, proofs := 0, 0 return walkFields(layout, func(number protowire.Number, wireType protowire.Type, value []byte) error { @@ -163,17 +170,21 @@ func checkPartLayoutBounds(layout []byte) error { case number == layoutSizesField && wireType == protowire.VarintType: sizes++ case number == layoutSizesField && wireType == protowire.BytesType: - // a packed list has one varint per byte without the continuation bit - for _, b := range value { - if b < 0x80 { - sizes++ + for len(value) > 0 { + _, n := protowire.ConsumeVarint(value) + if n < 0 { + return protowire.ParseError(n) } + sizes++ + value = value[n:] } case number == layoutProofsField && wireType == protowire.BytesType: proofs++ if err := checkProofBounds(value); err != nil { return fmt.Errorf("proof %d: %w", proofs-1, err) } + default: + return fmt.Errorf("unexpected field %d of wire type %d", number, wireType) } if sizes > int(types.MaxBlockPartsCount) { return fmt.Errorf("too many parts: more than %d", types.MaxBlockPartsCount) @@ -189,9 +200,12 @@ func checkPartLayoutBounds(layout []byte) error { func checkProofBounds(proof []byte) error { aunts := 0 return walkFields(proof, func(number protowire.Number, wireType protowire.Type, value []byte) error { - if wireType != protowire.BytesType || (number != proofLeafHashField && number != proofAuntsField) { + if wireType == protowire.VarintType && (number == proofTotalField || number == proofIndexField) { return nil } + if wireType != protowire.BytesType || (number != proofLeafHashField && number != proofAuntsField) { + return fmt.Errorf("unexpected field %d of wire type %d", number, wireType) + } if len(value) > crypto.HashSize { return fmt.Errorf("hash too long: %d bytes, max: %d", len(value), crypto.HashSize) } diff --git a/internal/blocksync/p2p_msg_handler_test.go b/internal/blocksync/p2p_msg_handler_test.go index e247ae4628..70ffa88460 100644 --- a/internal/blocksync/p2p_msg_handler_test.go +++ b/internal/blocksync/p2p_msg_handler_test.go @@ -660,6 +660,25 @@ func partLayout(t *testing.T, sizes []uint32, proofs []crypto.Proof) []byte { return bz } +// hiddenProofsBehindPackedSize returns a layout that opens with a packed size +// whose varint does not end inside its field. The generated decoder reads on +// past the field, so a scan that took the field's length at face value would +// see one unknown field where the decoder finds that many empty proofs. +func hiddenProofsBehindPackedSize(proofs int) []byte { + return bytes.Join([][]byte{ + {0x0a, 0x01, 0x80, 0x1a, 0x82, 0x80, 0x80, 0x01, 0x00, 0x18, 0x80, 0x00}, + bytes.Repeat([]byte{2<<3 | 2, 0}, proofs+1), + }, nil) +} + +// hiddenAuntsBehindPackedSize is hiddenProofsBehindPackedSize with chunks that +// each hide a proof of seven empty aunts. +func hiddenAuntsBehindPackedSize(chunks int) []byte { + chunk := append([]byte{0x0a, 0x01, 0x80, 0x1a, 0x12, 0x11, 0x08, 0x80, 0x01}, + bytes.Repeat([]byte{4<<3 | 2, 0}, 7)...) + return bytes.Repeat(chunk, chunks+1) +} + // TestBlockResponseFromProtoRejectsMalformedParts checks that part metadata a // peer sent that cannot describe its block is refused while decoding the // response, before it can be held waiting for a lower height. @@ -683,11 +702,24 @@ func TestBlockResponseFromProtoRejectsMalformedParts(t *testing.T) { layout []byte wantErr string }{ - "layout too big": {make([]byte, p2p.BlockPartLayoutMaxBytes+1), "part layout too big"}, - "not a layout": {[]byte{0x0a, 0x05}, "unexpected EOF"}, - "too many parts": {partLayout(t, make([]uint32, tooManyParts), nil), "too many parts"}, - "unpacked parts": {bytes.Repeat([]byte{1 << 3, 0}, tooManyParts), "too many parts"}, - "too many proofs": {bytes.Repeat([]byte{2<<3 | 2, 0}, tooManyParts), "too many proofs"}, + "layout too big": {make([]byte, p2p.BlockPartLayoutMaxBytes+1), "part layout too big"}, + "not a layout": {[]byte{0x0a, 0x05}, "unexpected EOF"}, + "too many parts": {partLayout(t, make([]uint32, tooManyParts), nil), "too many parts"}, + "unpacked parts": {bytes.Repeat([]byte{1 << 3, 0}, tooManyParts), "too many parts"}, + "too many proofs": {bytes.Repeat([]byte{2<<3 | 2, 0}, tooManyParts), "too many proofs"}, + "truncated packed size hiding proofs": { + hiddenProofsBehindPackedSize(int(types.MaxBlockPartsCount)), "unexpected EOF", + }, + "truncated packed size hiding aunts": { + hiddenAuntsBehindPackedSize(int(types.MaxBlockPartsCount)), "unexpected EOF", + }, + "unknown field": {[]byte{3 << 3, 0}, "unexpected field 3"}, + "group": {[]byte{3<<3 | 3, 3<<3 | 4}, "unexpected field 3"}, + "proofs of wrong type": {[]byte{2 << 3, 0}, "unexpected field 2"}, + "unknown proof field": {[]byte{2<<3 | 2, 2, 5 << 3, 0}, "unexpected field 5"}, + "proof total of wrong type": { + []byte{2<<3 | 2, 2, 1<<3 | 2, 0}, "unexpected field 1", + }, "too many aunts": {partLayout(t, []uint32{size}, []crypto.Proof{tooManyAunts}), "too many aunts"}, "aunt too long": {partLayout(t, []uint32{size}, []crypto.Proof{longAunt}), "hash too long"}, "part too big": {partLayout(t, []uint32{types.BlockPartSizeBytes + 1}, nil), "too big"}, @@ -736,33 +768,48 @@ func TestBlockResponseSizeChargesPartMetadata(t *testing.T) { // TestPartLayoutIsOpaqueAtEnvelopeBoundary checks that part metadata costs no // per-entry allocation when a message is received, which happens before it is -// matched to a request, nor when the layout is refused. A layout of empty -// proofs is the smallest encoding of the most entries. +// matched to a request, nor when the layout is refused. Empty proofs are the +// smallest encoding of the most entries. func TestPartLayoutIsOpaqueAtEnvelopeBoundary(t *testing.T) { - const allowedAllocs = 32 - layout := bytes.Repeat([]byte{2<<3 | 2, 0}, int(types.MaxBlockPartsCount)+1) - // block_response is field 9 of the envelope, part_layout field 3 of the response - raw := appendBytesField(nil, 9, appendBytesField(nil, 3, layout)) - - require.Less(t, testing.AllocsPerRun(10, func() { - var envelope p2pproto.Envelope - _ = proto.Unmarshal(raw, &envelope) - }), float64(allowedAllocs), "decoding the envelope must not allocate per layout entry") - - var envelope p2pproto.Envelope - require.NoError(t, proto.Unmarshal(raw, &envelope)) - msg, err := envelope.Unwrap() - require.NoError(t, err) - resp, ok := msg.(*bcproto.BlockResponse) - require.True(t, ok) - require.Equal(t, layout, resp.PartLayout) - - resp.Block = []byte{1} - require.Less(t, testing.AllocsPerRun(10, func() { - _, _ = committedPartsFromProto(resp) - }), float64(allowedAllocs), "refusing the layout must not allocate per layout entry") - _, err = committedPartsFromProto(resp) - require.ErrorContains(t, err, "too many proofs") + const ( + allowedAllocs = 32 + entries = int(types.MaxBlockPartsCount) + ) + testCases := map[string]struct { + layout []byte + wantErr string + }{ + "empty proofs": {bytes.Repeat([]byte{2<<3 | 2, 0}, entries+1), "too many proofs"}, + "proofs behind a packed size": {hiddenProofsBehindPackedSize(1 << 20), "unexpected EOF"}, + "aunts behind a packed size": {hiddenAuntsBehindPackedSize(entries), "unexpected EOF"}, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require.LessOrEqual(t, len(tc.layout), p2p.BlockPartLayoutMaxBytes) + // block_response is field 9 of the envelope, part_layout field 3 of the response + raw := appendBytesField(nil, 9, appendBytesField(nil, 3, tc.layout)) + + require.Less(t, testing.AllocsPerRun(10, func() { + var envelope p2pproto.Envelope + _ = proto.Unmarshal(raw, &envelope) + }), float64(allowedAllocs), "decoding the envelope must not allocate per layout entry") + + var envelope p2pproto.Envelope + require.NoError(t, proto.Unmarshal(raw, &envelope)) + msg, err := envelope.Unwrap() + require.NoError(t, err) + resp, ok := msg.(*bcproto.BlockResponse) + require.True(t, ok) + require.Equal(t, tc.layout, resp.PartLayout) + + resp.Block = []byte{1} + require.Less(t, testing.AllocsPerRun(10, func() { + _, _ = committedPartsFromProto(resp) + }), float64(allowedAllocs), "refusing the layout must not allocate per layout entry") + _, err = committedPartsFromProto(resp) + require.ErrorContains(t, err, tc.wantErr) + }) + } } // TestBlockResponseDoesNotRetainOverwrittenBlockField checks that a repeated From 847dce54d0ab07d39f6caadea4983f63a411237b Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Tue, 10 Nov 2026 22:43:43 +0700 Subject: [PATCH 6/8] refactor(blocksync): rebuild only the standard part cut Block sync now has a single way to rebuild a committed part set. The serving node sends the stored parts' bytes as BlockResponse.block, and the receiver cuts them into parts of 64kB and computes the part set root itself, as the proposer did. Honest Tenderdash proposers have always used that cut, so every block they produced syncs, including blocks whose bytes a re-encoding does not reproduce. A block committed with a different cut, or with proofs of a larger tree, is no longer rebuilt: consensus should reject those, gated by consensus version. This removes BlockResponse.part_layout and BlockPartLayout, the wire-level bounds checks they needed, types.NewPartSetFromPartSizes and ValidatePartSizes, and the larger block sync channel capacity. The commit's block ID still goes to FinalizeBlock and the replay events, and a block buffer left oversized by a repeated block field is still copied before it is kept. Co-Authored-By: Claude Opus 5.5 --- go.mod | 2 +- internal/blocksync/applier.go | 18 +- internal/blocksync/applier_test.go | 103 ++--- internal/blocksync/block_fetch_job_test.go | 53 --- internal/blocksync/commit_extensions_test.go | 12 +- internal/blocksync/committed_parts.go | 295 ------------- internal/blocksync/msgs.go | 7 +- internal/blocksync/p2p_msg_handler.go | 40 +- internal/blocksync/p2p_msg_handler_test.go | 283 ++----------- internal/blocksync/reactor_test.go | 119 ++---- internal/blocksync/synchronizer.go | 38 +- internal/p2p/channel_params.go | 36 +- proto/tendermint/blocksync/types.pb.go | 416 ++----------------- proto/tendermint/blocksync/types.proto | 30 +- spec/p2p/messages/block-sync.md | 29 +- types/part_set.go | 63 +-- types/part_set_test.go | 51 --- 17 files changed, 228 insertions(+), 1367 deletions(-) delete mode 100644 internal/blocksync/committed_parts.go diff --git a/go.mod b/go.mod index 7347ffa421..4f17668d1c 100644 --- a/go.mod +++ b/go.mod @@ -276,6 +276,7 @@ require ( golang.org/x/tools v0.52.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20261012220534-4341edf3f941 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20261012220534-4341edf3f941 // indirect + google.golang.org/protobuf v1.36.13-0.20261016172727-e9ee2a76ab8e // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.8.2 // indirect mvdan.cc/gofumpt v0.12.0 // indirect @@ -336,7 +337,6 @@ require ( golang.org/x/term v0.47.0 golang.org/x/time v0.16.0 google.golang.org/grpc v1.85.0 - google.golang.org/protobuf v1.36.13-0.20261016172727-e9ee2a76ab8e pgregory.net/rapid v1.3.0 ) diff --git a/internal/blocksync/applier.go b/internal/blocksync/applier.go index a9d2c53580..2e07e78910 100644 --- a/internal/blocksync/applier.go +++ b/internal/blocksync/applier.go @@ -70,11 +70,12 @@ func newBlockApplier(blockExec sm.Executor, store sm.BlockStore, opts ...applier } // Apply safely verifies, saves to the store and executes a block with commit. -// parts is the block as it was committed, whose bytes block was decoded from. +// blockBytes is the block as it was committed, the bytes block was decoded +// from. func (e *blockApplier) Apply( ctx context.Context, block *types.Block, - parts CommittedParts, + blockBytes []byte, commit *types.Commit, ) error { e.mtx.Lock() @@ -90,15 +91,12 @@ func (e *blockApplier) Apply( // The part set is needed twice: to derive the block ID and to persist the // block. Building it builds a Merkle tree over the parts, so build it once - // and pass it to both. It is rebuilt from the parts the block was committed - // with, not from block: the commit's part set header is computed over - // those, and a re-encoding of block drops unknown fields and non-canonical - // encoding. + // and pass it to both. It is cut from the bytes the block was committed + // with, the way the proposer cut them, not from block: the commit's part + // set header is computed over those bytes, and a re-encoding of block drops + // unknown fields and non-canonical encoding. start := time.Now() - blockParts, err := parts.partSet(commit.BlockID.PartSetHeader) - if err != nil { - return err - } + blockParts := types.NewPartSetFromData(blockBytes, types.BlockPartSizeBytes) blockID := block.BlockID(blockParts) partSetTime := e.observeSince("partset", start) diff --git a/internal/blocksync/applier_test.go b/internal/blocksync/applier_test.go index 30e49f3454..fea9fa172e 100644 --- a/internal/blocksync/applier_test.go +++ b/internal/blocksync/applier_test.go @@ -49,7 +49,7 @@ func TestBlockApplierChecksAppResponseBeforeSave(t *testing.T) { executor := sm.NewBlockExecutor(stateStore, client, nil, sm.EmptyEvidencePool{}, blockStore, nil) applier := newBlockApplier(executor, blockStore, applierWithState(initialState)) - require.Panics(t, func() { _ = applier.Apply(ctx, block, standardParts(t, block), commit) }) + require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), commit) }) require.Zero(t, blockStore.Height(), "an inconsistent app response must not advance the block store") require.Equal(t, initialState.LastBlockHeight, applier.State().LastBlockHeight) loaded, err := stateStore.Load() @@ -176,7 +176,7 @@ func TestBlockApplierApply(t *testing.T) { tc.mockFn() } fn := func() { - err := applier.Apply(ctx, tc.block, standardParts(t, tc.block), tc.commit) + err := applier.Apply(ctx, tc.block, blockBytes(t, tc.block), tc.commit) tmrequire.Error(t, tc.wantErr, err) } if tc.wantPanic { @@ -269,7 +269,7 @@ func TestBlockApplierDoesNotSaveBlockRejectedByApp(t *testing.T) { Return(sm.CurrentRoundState{}, errors.New("app rejected the block")) applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState)) - require.Panics(t, func() { _ = applier.Apply(ctx, block, standardParts(t, block), commit) }) + require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), commit) }) } // TestBlockApplierSavesBlockBeforeFinalize pins the order the handshake relies @@ -315,7 +315,7 @@ func TestBlockApplierSavesBlockBeforeFinalize(t *testing.T) { Return(state, nil, nil) applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState)) - require.NoError(t, applier.Apply(ctx, block, standardParts(t, block), commit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), commit)) require.Equal(t, []string{"process", "verify", "save", "finalize"}, calls) } @@ -351,7 +351,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState), applierWithMetrics(m)) - require.NoError(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1)) for _, stage := range []string{"partset", "verify_commit", "verify_block", "save", "exec"} { require.Len(t, hist.Samples[stage], 1, "stage %q after the first apply", stage) @@ -362,7 +362,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { // A failed attempt must advance the idle-time boundary as well, so a retry // does not count the failed attempt's wait and verification time again. failureStart := time.Now() - require.Error(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), new(types.Commit))) + require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), new(types.Commit))) failureEnd := time.Now() failureDone := applier.lastDone require.False(t, failureDone.Before(failureStart), "failed apply must advance lastDone") @@ -371,7 +371,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) { // The retry measures only the idle time after the failed attempt. retryStart := time.Now() - require.NoError(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1)) retryEnd := time.Now() require.Len(t, hist.Samples["wait"], 2) wait := hist.Samples["wait"][1] @@ -402,7 +402,7 @@ func TestBlockApplierVerifyFailureTimesOnlyTheCheckThatRan(t *testing.T) { // A rejected commit must stop before ValidateBlock is reached. mockBlockExec.On("VerifyCommit", initialState, blockH1.BlockID(nil), blockH1.Height, new(types.Commit)). Once().Return(types.VerifiedCommit{}, errors.New("bad signature")) - require.Error(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), new(types.Commit))) + require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), new(types.Commit))) require.Len(t, hist.Samples["partset"], 1) require.Len(t, hist.Samples["verify_commit"], 1) @@ -446,7 +446,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { blockExec.On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, blockH1ID, blockH1, commitH1, none). Once().Return(initialState, nil, nil) applier := newBlockApplier(blockExec, blockStore, applierWithState(initialState)) - require.NoError(t, applier.Apply(ctx, blockH1, standardParts(t, blockH1), commitH1)) + require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1)) return applier, blockExec } // expectH2 expects block 2 to be validated and applied offering lastCommit @@ -462,7 +462,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { t.Run("the next block is offered the previous commit's verification", func(t *testing.T) { applier, blockExec := applyH1(t) expectH2(blockExec, verifiedH1) - require.NoError(t, applier.Apply(ctx, blockH2, standardParts(t, blockH2), commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2)) }) t.Run("a rejected commit does not replace the verification", func(t *testing.T) { @@ -470,17 +470,17 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) { bad := new(types.Commit) blockExec.On("VerifyCommit", mock.Anything, blockH2ID, blockH2.Height, bad). Once().Return(none, errors.New("bad signature")) - require.Error(t, applier.Apply(ctx, blockH2, standardParts(t, blockH2), bad)) + require.Error(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), bad)) expectH2(blockExec, verifiedH1) - require.NoError(t, applier.Apply(ctx, blockH2, standardParts(t, blockH2), commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2)) }) t.Run("replacing the state discards the verification", func(t *testing.T) { applier, blockExec := applyH1(t) applier.UpdateState(initialState) expectH2(blockExec, none) - require.NoError(t, applier.Apply(ctx, blockH2, standardParts(t, blockH2), commitH2)) + require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2)) }) } @@ -534,7 +534,7 @@ func TestBlockApplierSkipsTheLastCommitItVerified(t *testing.T) { wantSkipped float64 }{{1, 0}, {2, 3}, {3, 6}} { block, _, _, seenCommit := makeNextBlock(ctx, t, applier.State(), privVals[0], step.height, commit) - require.NoError(t, applier.Apply(ctx, block, standardParts(t, block), seenCommit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), seenCommit)) require.Equal(t, step.wantSkipped, skipped.Value(), "LastCommit threshold verifications skipped after applying height %d", step.height) commit = seenCommit @@ -552,29 +552,6 @@ func blockBytes(t *testing.T, block *types.Block) []byte { return bz } -// standardParts returns block as its proposer commits it when it encodes and -// cuts it the canonical way. -func standardParts(t *testing.T, block *types.Block) CommittedParts { - t.Helper() - return CommittedParts{Bytes: blockBytes(t, block)} -} - -// TestBlockApplierRejectsPartSizesNotCoveringTheBlock checks that part sizes a -// peer sent that do not cut the block's bytes are refused before the block is -// verified or saved. -func TestBlockApplierRejectsPartSizesNotCoveringTheBlock(t *testing.T) { - ctx := context.Background() - valSet, privVals := factory.MockValidatorSet() - state := fakeInitialState(valSet) - blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) - block, commit := blocks[0], blocks[1].LastCommit - bz := blockBytes(t, block) - applier := newBlockApplier(mocks.NewExecutor(t), mocks.NewBlockStore(t), applierWithState(state)) - - err := applier.Apply(ctx, block, CommittedParts{Bytes: bz, Sizes: []uint32{uint32(len(bz) - 1)}}, commit) - require.ErrorContains(t, err, "part sizes cover") -} - // finalizeBlockIDApp records the block IDs FinalizeBlock is called with. type finalizeBlockIDApp struct { abci.BaseApplication @@ -591,7 +568,7 @@ func (app *finalizeBlockIDApp) FinalizeBlock( // TestFinalizeBlockGetsCommittedBlockID checks that the application is given // the block ID that was committed, when applying a block and replaying it, for -// blocks committed with parts that a re-encoding does not reproduce. +// a block committed with bytes that a re-encoding does not reproduce. func TestFinalizeBlockGetsCommittedBlockID(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -618,48 +595,30 @@ func TestFinalizeBlockGetsCommittedBlockID(t *testing.T) { blockExec := sm.NewBlockExecutor(stateStore, app, mp, sm.EmptyEvidencePool{}, blockStore, eventBus) applier := newBlockApplier(blockExec, blockStore, applierWithState(state)) - blockParts := func(t *testing.T, height int64, block *types.Block) *types.PartSet { - switch height { - case 1: - return types.NewPartSetFromData(withUnknownField(t, block), types.BlockPartSizeBytes) - case 2: - return types.NewPartSetFromData(blockBytes(t, block), 100) - case 3: - return withProofOfLargerTree(t, block) - } - return nil - } - commit := types.NewCommit(0, 0, types.BlockID{}, nil, nil) - for height := int64(1); height <= 3; height++ { - block, blockID, partSet, seenCommit := makeNextBlockWithParts(ctx, t, applier.State(), privVals[0], height, - commit, blockParts) - require.False(t, blockID.Equals(block.BlockID(nil)), "height %d must not be a block a re-encoding reproduces", height) - parts := committedPartsOf(t, partSet) - require.NoError(t, applier.Apply(ctx, block, parts, seenCommit)) - - want := seenCommit.BlockID.ToProto() - require.Equal(t, &want, finalizeApp.blockIDs[len(finalizeApp.blockIDs)-1], "applying height %d", height) - - _, err := sm.ExecReplayedCommitBlock(ctx, localClient, block, seenCommit, logger) - require.NoError(t, err) - require.Equal(t, &want, finalizeApp.blockIDs[len(finalizeApp.blockIDs)-1], "replaying height %d", height) - commit = seenCommit - } + block, blockID, partSet, seenCommit := makeNextBlockWithBytes(ctx, t, applier.State(), privVals[0], 1, + types.NewCommit(0, 0, types.BlockID{}, nil, nil), withUnknownField) + require.False(t, blockID.Equals(block.BlockID(nil)), "the block must not be one a re-encoding reproduces") + require.NoError(t, applier.Apply(ctx, block, loadBlockBytes(storedParts(t, partSet), 1), seenCommit)) + + want := seenCommit.BlockID.ToProto() + require.Equal(t, []*tmproto.BlockID{&want}, finalizeApp.blockIDs, "applying the block") + + _, err = sm.ExecReplayedCommitBlock(ctx, localClient, block, seenCommit, logger) + require.NoError(t, err) + require.Equal(t, []*tmproto.BlockID{&want, &want}, finalizeApp.blockIDs, "replaying the block") } -// committedPartsOf returns parts in the form a peer serves them, by storing them. -func committedPartsOf(t *testing.T, partSet *types.PartSet) CommittedParts { +// storedParts returns a store that holds partSet as the parts of the block at +// height 1. +func storedParts(t *testing.T, partSet *types.PartSet) sm.BlockStore { t.Helper() - total := int(partSet.Total()) blockStore := mocks.NewBlockStore(t) blockStore.On("LoadBlockMeta", int64(1)).Return(&types.BlockMeta{ BlockID: types.BlockID{PartSetHeader: partSet.Header()}, BlockSize: int(partSet.ByteSize()), }) - for i := range total { + for i := range int(partSet.Total()) { blockStore.On("LoadBlockPart", int64(1), i).Return(partSet.GetPart(i)) } - parts := loadCommittedParts(blockStore, 1) - require.NotNil(t, parts) - return *parts + return blockStore } diff --git a/internal/blocksync/block_fetch_job_test.go b/internal/blocksync/block_fetch_job_test.go index cac3df332c..17bbb23859 100644 --- a/internal/blocksync/block_fetch_job_test.go +++ b/internal/blocksync/block_fetch_job_test.go @@ -16,7 +16,6 @@ import ( "github.com/dashpay/tenderdash/internal/p2p/client" "github.com/dashpay/tenderdash/internal/p2p/client/mocks" p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" - statemocks "github.com/dashpay/tenderdash/internal/state/mocks" statefactory "github.com/dashpay/tenderdash/internal/state/test/factory" "github.com/dashpay/tenderdash/internal/test/factory" tmrequire "github.com/dashpay/tenderdash/internal/test/require" @@ -108,58 +107,6 @@ func (suite *BlockFetchJobTestSuite) TestExecuteHeightMismatch() { suite.requireError("peer sent block at height 6, requested height 5", res.Err) } -// TestExecuteRejectsOversizedPartMetadata pins that a response with more part -// sizes than a block can have fails its fetch, so it is never held waiting for -// the lower heights still missing. -func (suite *BlockFetchJobTestSuite) TestExecuteRejectsOversizedPartMetadata() { - ctx := context.Background() - const requested = int64(9) - oversized := *suite.responses[requested-1] - oversized.PartLayout = partLayout(suite.T(), make([]uint32, types.MaxBlockPartsCount+1), nil) - suite.client. - On("GetBlock", mock.Anything, requested, suite.peer.peerID). - Once(). - Return(suite.getBlockReturnFunc(suite.promiseResolveResponse(&oversized)), nil) - res := blockFetchJobHandler(suite.client, suite.peer, requested)(ctx) - suite.requireError("too many parts", res.Err) - suite.Nil(res.Value) -} - -// TestOversizedPartMetadataIsNotHeldBehindMissingHeight pins the same through -// the synchronizer: while a lower height is missing, a later response with too -// many part sizes stays out of the backlog, where the same response without -// them is held. -func (suite *BlockFetchJobTestSuite) TestOversizedPartMetadataIsNotHeldBehindMissingHeight() { - ctx := context.Background() - const later = int64(9) - fetch := func(resp *bcproto.BlockResponse) workerpool.Result { - suite.client. - On("GetBlock", mock.Anything, later, suite.peer.peerID). - Once(). - Return(suite.getBlockReturnFunc(suite.promiseResolveResponse(resp)), nil) - return blockFetchJobHandler(suite.client, suite.peer, later)(ctx) - } - resultCh := make(chan workerpool.Result, 1) - wp := workerpool.New(1, workerpool.WithResultCh(resultCh)) - applier := newBlockApplier(statemocks.NewExecutor(suite.T()), statemocks.NewBlockStore(suite.T())) - pool := NewSynchronizer(1, suite.client, applier, WithWorkerPool(wp)) - pool.AddPeer(suite.peer) - - oversized := *suite.responses[later-1] - oversized.PartLayout = partLayout(suite.T(), make([]uint32, types.MaxBlockPartsCount+1), nil) - resultCh <- fetch(&oversized) - suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) - height, pendingBytes := pool.backlog() - suite.EqualValues(1, height, "the lower height must still be the one being waited for") - suite.Zero(pendingBytes) - - resultCh <- fetch(suite.responses[later-1]) - suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) - height, pendingBytes = pool.backlog() - suite.EqualValues(1, height, "the lower height must still be the one being waited for") - suite.Equal(len(suite.responses[later-1].Block), pendingBytes) -} - // TestExecuteCarriesPeerConnection pins that a job's outcome stays attributed to the // connection generation the peer was reserved under, including a timeout, which has // no response envelope to take a generation from. diff --git a/internal/blocksync/commit_extensions_test.go b/internal/blocksync/commit_extensions_test.go index 7e84c48971..788dfbf029 100644 --- a/internal/blocksync/commit_extensions_test.go +++ b/internal/blocksync/commit_extensions_test.go @@ -52,11 +52,11 @@ func TestBlockApplierRejectsCommitExtensionsAndRetries(t *testing.T) { metrics := consensus.NopMetrics() metrics.CommitVerifyFailures = counter applier := newBlockApplier(exec, store, applierWithState(initial), applierWithMetrics(metrics)) - require.Error(t, applier.Apply(ctx, block, standardParts(t, block), commit)) + require.Error(t, applier.Apply(ctx, block, blockBytes(t, block), commit)) require.Equal(t, float64(1), counter.Value()) require.Equal(t, []string{"process", "verify"}, calls) require.Equal(t, initial.LastBlockHeight, applier.State().LastBlockHeight) - require.NoError(t, applier.Apply(ctx, block, standardParts(t, block), commit)) + require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), commit)) require.Equal(t, float64(1), counter.Value()) require.Equal(t, []string{"process", "verify", "verify", "save", "finalize"}, calls) } @@ -81,7 +81,7 @@ func TestBlockApplierInvalidatesRejectedProposal(t *testing.T) { exec.On("VerifyVoteExtension", mock.Anything, mock.Anything). Twice().Return(errors.New("invalid vote extension")) applier := newBlockApplier(exec, store, applierWithState(initial)) - require.ErrorIs(t, applier.Apply(ctx, block, standardParts(t, block), commit), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), commit), sm.ErrCommitExtensionsRejected) switch change { case "block": block.AppHash = append(block.AppHash.Copy(), 1) @@ -90,7 +90,7 @@ func TestBlockApplierInvalidatesRejectedProposal(t *testing.T) { case "state": applier.UpdateState(initial) } - require.ErrorIs(t, applier.Apply(ctx, block, standardParts(t, block), commit), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), commit), sm.ErrCommitExtensionsRejected) }) } } @@ -135,9 +135,9 @@ func TestBlockApplierReprocessesRoundAfterAnother(t *testing.T) { applier := newBlockApplier(exec, store, applierWithState(initial)) for _, round := range []int32{1, 1, 2} { - require.ErrorIs(t, applier.Apply(ctx, block, standardParts(t, block), commitAt(round)), sm.ErrCommitExtensionsRejected) + require.ErrorIs(t, applier.Apply(ctx, block, blockBytes(t, block), commitAt(round)), sm.ErrCommitExtensionsRejected) } - require.NotPanics(t, func() { require.NoError(t, applier.Apply(ctx, block, standardParts(t, block), commitAt(1))) }) + require.NotPanics(t, func() { require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), commitAt(1))) }) require.Equal(t, []int32{1, 2, 1}, rounds, "a round is processed again only after another one") } diff --git a/internal/blocksync/committed_parts.go b/internal/blocksync/committed_parts.go deleted file mode 100644 index 01d8db7add..0000000000 --- a/internal/blocksync/committed_parts.go +++ /dev/null @@ -1,295 +0,0 @@ -package blocksync - -import ( - "errors" - "fmt" - "slices" - - "github.com/cosmos/gogoproto/proto" - "google.golang.org/protobuf/encoding/protowire" - - "github.com/dashpay/tenderdash/crypto" - "github.com/dashpay/tenderdash/crypto/merkle" - "github.com/dashpay/tenderdash/internal/p2p" - sm "github.com/dashpay/tenderdash/internal/state" - tmmath "github.com/dashpay/tenderdash/libs/math" - bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" - tmcrypto "github.com/dashpay/tenderdash/proto/tendermint/crypto" - "github.com/dashpay/tenderdash/types" -) - -// CommittedParts is a block as it was committed. The commit's part set header -// is computed over the parts the proposer gossiped, and a peer can only rebuild -// it from those: a re-encoding of the decoded block drops unknown fields and -// non-canonical encoding, and consensus accepts any parts that verify against -// the header, however the proposer cut the block. -type CommittedParts struct { - // Bytes are the bytes of the parts, in order. - Bytes []byte - // Sizes is the size of each part, nil for parts of types.BlockPartSizeBytes. - Sizes []uint32 - // Proofs is the Merkle proof of each part. It is nil when the part set header - // can be computed from Bytes and Sizes. - Proofs []merkle.Proof -} - -// loadCommittedParts returns the block at height as it was committed, or nil -// when the store does not hold every part of it. -func loadCommittedParts(store sm.BlockStore, height int64) *CommittedParts { - meta := store.LoadBlockMeta(height) - if meta == nil { - return nil - } - total := int(meta.BlockID.PartSetHeader.Total) - // BlockSize is -1 for a header saved without its block - blockBytes := make([]byte, 0, max(meta.BlockSize, 0)) - sizes := make([]uint32, total) - proofs := make([]merkle.Proof, total) - computable := true - for i := range total { - part := store.LoadBlockPart(height, i) - // a header saved without its block, or parts pruned after the meta was - // loaded - if part == nil { - return nil - } - blockBytes = append(blockBytes, part.Bytes...) - sizes[i] = tmmath.MustConvertUint32(len(part.Bytes)) - proofs[i] = part.Proof - // PartSet.AddPart checks a proof against the header's root but not its - // total, so a proposer can commit to a tree of more leaves than the - // header has parts. Every part verified at the header's total means the - // tree over the parts is the header's. - computable = computable && part.Proof.Total == int64(total) - } - if len(blockBytes) == 0 { - return nil - } - parts := &CommittedParts{Bytes: blockBytes, Sizes: sizes, Proofs: proofs} - if computable { - parts.Proofs = nil - if standardPartSizes(sizes, len(blockBytes)) { - parts.Sizes = nil - } - } - return parts -} - -// standardPartSizes reports whether parts of the given sizes are how -// types.NewPartSetFromData cuts size bytes into parts of -// types.BlockPartSizeBytes. -func standardPartSizes(partSizes []uint32, size int) bool { - partSize := int(types.BlockPartSizeBytes) - total := (size + partSize - 1) / partSize - if total == 0 || len(partSizes) != total { - return false - } - // the last part takes what is left, and with total parts that is between 1 - // and partSize bytes - for _, s := range partSizes[:total-1] { - if s != types.BlockPartSizeBytes { - return false - } - } - return true -} - -// committedPartsFromProto returns the parts in a block response, after checking -// that they are well formed. It does not check them against the commit. -func committedPartsFromProto(resp *bcproto.BlockResponse) (CommittedParts, error) { - blockBytes := resp.Block - // A repeated block field leaves the allocation of its longest occurrence - // behind the last one, and only the length is charged to the backlog. - if cap(blockBytes) > 2*len(blockBytes) { - blockBytes = slices.Clone(blockBytes) - } - parts := CommittedParts{Bytes: blockBytes} - if len(resp.PartLayout) == 0 { - return parts, nil - } - if len(resp.PartLayout) > p2p.BlockPartLayoutMaxBytes { - return CommittedParts{}, fmt.Errorf("part layout too big: %d bytes, max: %d", - len(resp.PartLayout), p2p.BlockPartLayoutMaxBytes) - } - if err := checkPartLayoutBounds(resp.PartLayout); err != nil { - return CommittedParts{}, err - } - var layout bcproto.BlockPartLayout - if err := proto.Unmarshal(resp.PartLayout, &layout); err != nil { - return CommittedParts{}, fmt.Errorf("cannot decode part layout: %w", err) - } - parts.Sizes = layout.Sizes - if len(layout.Sizes) > 0 { - if err := types.ValidatePartSizes(len(blockBytes), layout.Sizes); err != nil { - return CommittedParts{}, err - } - } - if len(layout.Proofs) == 0 { - return parts, nil - } - if len(layout.Proofs) != len(layout.Sizes) { - return CommittedParts{}, fmt.Errorf("%d proofs for %d parts", len(layout.Proofs), len(layout.Sizes)) - } - parts.Proofs = make([]merkle.Proof, len(layout.Proofs)) - for i := range layout.Proofs { - proof, err := merkle.ProofFromProto(&layout.Proofs[i]) - if err != nil { - return CommittedParts{}, fmt.Errorf("proof of part %d: %w", i, err) - } - if proof.Index != int64(i) { - return CommittedParts{}, fmt.Errorf("proof of part %d has index %d", i, proof.Index) - } - parts.Proofs[i] = *proof - } - return parts, nil -} - -// Field numbers of BlockPartLayout and tendermint.crypto.Proof. -const ( - layoutSizesField protowire.Number = 1 - layoutProofsField protowire.Number = 2 - proofTotalField protowire.Number = 1 - proofIndexField protowire.Number = 2 - proofLeafHashField protowire.Number = 3 - proofAuntsField protowire.Number = 4 -) - -// checkPartLayoutBounds refuses a serialized BlockPartLayout with more entries -// than a block can have, without decoding it. Decoding allocates tens of bytes -// for every occurrence of a repeated field, and an occurrence can take two -// bytes, so the limits have to hold before the layout is decoded. -// -// The limits only hold if this walk and the decoder split the bytes into the -// same fields, so it accepts nothing but the fields of a layout, each of them -// well formed. The generated decoder lets a varint in a packed field run past -// the end of that field, and skips unknown fields by rules of its own. -func checkPartLayoutBounds(layout []byte) error { - sizes, proofs := 0, 0 - return walkFields(layout, func(number protowire.Number, wireType protowire.Type, value []byte) error { - switch { - case number == layoutSizesField && wireType == protowire.VarintType: - sizes++ - case number == layoutSizesField && wireType == protowire.BytesType: - for len(value) > 0 { - _, n := protowire.ConsumeVarint(value) - if n < 0 { - return protowire.ParseError(n) - } - sizes++ - value = value[n:] - } - case number == layoutProofsField && wireType == protowire.BytesType: - proofs++ - if err := checkProofBounds(value); err != nil { - return fmt.Errorf("proof %d: %w", proofs-1, err) - } - default: - return fmt.Errorf("unexpected field %d of wire type %d", number, wireType) - } - if sizes > int(types.MaxBlockPartsCount) { - return fmt.Errorf("too many parts: more than %d", types.MaxBlockPartsCount) - } - if proofs > int(types.MaxBlockPartsCount) { - return fmt.Errorf("too many proofs: more than %d", types.MaxBlockPartsCount) - } - return nil - }) -} - -// checkProofBounds is checkPartLayoutBounds for a serialized proof. -func checkProofBounds(proof []byte) error { - aunts := 0 - return walkFields(proof, func(number protowire.Number, wireType protowire.Type, value []byte) error { - if wireType == protowire.VarintType && (number == proofTotalField || number == proofIndexField) { - return nil - } - if wireType != protowire.BytesType || (number != proofLeafHashField && number != proofAuntsField) { - return fmt.Errorf("unexpected field %d of wire type %d", number, wireType) - } - if len(value) > crypto.HashSize { - return fmt.Errorf("hash too long: %d bytes, max: %d", len(value), crypto.HashSize) - } - if number == proofAuntsField { - aunts++ - } - if aunts > merkle.MaxAunts { - return fmt.Errorf("too many aunts: more than %d", merkle.MaxAunts) - } - return nil - }) -} - -// walkFields calls visit for every field of a serialized message, with the -// contents of the field when it is length-delimited. -func walkFields(bz []byte, visit func(protowire.Number, protowire.Type, []byte) error) error { - for len(bz) > 0 { - number, wireType, n := protowire.ConsumeTag(bz) - if n < 0 { - return protowire.ParseError(n) - } - bz = bz[n:] - n = protowire.ConsumeFieldValue(number, wireType, bz) - if n < 0 { - return protowire.ParseError(n) - } - value := bz[:n] - if wireType == protowire.BytesType { - value, _ = protowire.ConsumeBytes(value) - } - if err := visit(number, wireType, value); err != nil { - return err - } - bz = bz[n:] - } - return nil -} - -// partLayout returns the serialized BlockPartLayout of the parts, nil when the -// parts need none. -func (p CommittedParts) partLayout() ([]byte, error) { - if len(p.Sizes) == 0 && len(p.Proofs) == 0 { - return nil, nil - } - layout := bcproto.BlockPartLayout{Sizes: p.Sizes} - if len(p.Proofs) > 0 { - layout.Proofs = make([]tmcrypto.Proof, len(p.Proofs)) - for i := range p.Proofs { - layout.Proofs[i] = *p.Proofs[i].ToProto() - } - } - return proto.Marshal(&layout) -} - -// partSet returns the part set the block was committed with. header is the -// part set header of the commit; the caller still has to verify that commit. -func (p CommittedParts) partSet(header types.PartSetHeader) (*types.PartSet, error) { - if len(p.Proofs) == 0 { - if len(p.Sizes) == 0 { - return types.NewPartSetFromData(p.Bytes, types.BlockPartSizeBytes), nil - } - return types.NewPartSetFromPartSizes(p.Bytes, p.Sizes) - } - if len(p.Proofs) != len(p.Sizes) || int64(len(p.Proofs)) != int64(header.Total) { - return nil, fmt.Errorf("%d proofs and %d part sizes for %d parts", len(p.Proofs), len(p.Sizes), header.Total) - } - if err := types.ValidatePartSizes(len(p.Bytes), p.Sizes); err != nil { - return nil, err - } - partSet := types.NewPartSetFromHeader(header) - offset := 0 - for i, size := range p.Sizes { - part := &types.Part{ - Index: tmmath.MustConvertUint32(i), - Bytes: p.Bytes[offset : offset+int(size)], - Proof: p.Proofs[i], - } - offset += int(size) - if _, err := partSet.AddPart(part); err != nil { - return nil, fmt.Errorf("part %d: %w", i, err) - } - } - if !partSet.IsComplete() { - return nil, errors.New("incomplete part set") - } - return partSet, nil -} diff --git a/internal/blocksync/msgs.go b/internal/blocksync/msgs.go index db41da0342..d54174cc9e 100644 --- a/internal/blocksync/msgs.go +++ b/internal/blocksync/msgs.go @@ -1,9 +1,12 @@ package blocksync import ( - "github.com/dashpay/tenderdash/internal/p2p" + p2pproto "github.com/dashpay/tenderdash/proto/tendermint/p2p" + "github.com/dashpay/tenderdash/types" ) const ( - MaxMsgSize = p2p.BlockSyncMaxMsgSize + MaxMsgSize = types.MaxBlockSizeBytes + + p2pproto.BlockResponseMessagePrefixSize + + p2pproto.BlockResponseMessageFieldKeySize ) diff --git a/internal/blocksync/p2p_msg_handler.go b/internal/blocksync/p2p_msg_handler.go index 5471e0302e..5e8b6c024c 100644 --- a/internal/blocksync/p2p_msg_handler.go +++ b/internal/blocksync/p2p_msg_handler.go @@ -155,11 +155,11 @@ func (h *blockP2PMessageHandler) Handle(ctx context.Context, p2pClient *client.C func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (proto.Message, bool, error) { envelope := &job.envelope msg := envelope.Message.(*bcproto.BlockRequest) - parts := loadCommittedParts(h.store, msg.Height) + blockBytes := loadBlockBytes(h.store, msg.Height) if !h.connectionIsCurrent(envelope.From, envelope.ConnID) { return nil, false, nil } - if parts == nil { + if blockBytes == nil { h.logger.Debug("peer requesting a block we do not have", "peer", envelope.From, "height", msg.Height) return &bcproto.NoBlockResponse{Height: msg.Height}, false, nil } @@ -167,17 +167,39 @@ func (h *blockP2PMessageHandler) prepareBlockResponse(job blockResponseJob) (pro if commit == nil { return nil, false, fmt.Errorf("%w at height %d", errStoredBlockMissingCommit, msg.Height) } - layout, err := parts.partLayout() - if err != nil { - return nil, false, fmt.Errorf("failed to encode part layout: %w", err) - } return &bcproto.BlockResponse{ - Block: parts.Bytes, - Commit: commit.ToProto(), - PartLayout: layout, + Block: blockBytes, + Commit: commit.ToProto(), }, true, nil } +// loadBlockBytes returns the bytes of the stored parts of the block at height, +// in order, or nil when the store does not hold every part of it. The commit's +// part set header is computed over those bytes, and a re-encoding of the +// decoded block need not reproduce them: it drops unknown fields and +// non-canonical encoding. +func loadBlockBytes(store sm.BlockStore, height int64) []byte { + meta := store.LoadBlockMeta(height) + if meta == nil { + return nil + } + // BlockSize is -1 for a header saved without its block + blockBytes := make([]byte, 0, max(meta.BlockSize, 0)) + for i := range int(meta.BlockID.PartSetHeader.Total) { + part := store.LoadBlockPart(height, i) + // a header saved without its block, or parts pruned after the meta was + // loaded + if part == nil { + return nil + } + blockBytes = append(blockBytes, part.Bytes...) + } + if len(blockBytes) == 0 { + return nil + } + return blockBytes +} + func (h *blockP2PMessageHandler) enqueueBlockResponse(job blockResponseJob) { peerID := job.envelope.From h.responseMtx.Lock() diff --git a/internal/blocksync/p2p_msg_handler_test.go b/internal/blocksync/p2p_msg_handler_test.go index 70ffa88460..547cdf771a 100644 --- a/internal/blocksync/p2p_msg_handler_test.go +++ b/internal/blocksync/p2p_msg_handler_test.go @@ -1,11 +1,9 @@ package blocksync import ( - "bytes" "context" "encoding/binary" "fmt" - "math" "testing" "time" @@ -16,8 +14,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/dashpay/tenderdash/config" - tmcrypto "github.com/dashpay/tenderdash/crypto" - "github.com/dashpay/tenderdash/crypto/merkle" "github.com/dashpay/tenderdash/internal/p2p" "github.com/dashpay/tenderdash/internal/p2p/client" p2pmocks "github.com/dashpay/tenderdash/internal/p2p/mocks" @@ -28,9 +24,6 @@ import ( dbm "github.com/dashpay/tenderdash/libs/db" "github.com/dashpay/tenderdash/libs/log" bcproto "github.com/dashpay/tenderdash/proto/tendermint/blocksync" - "github.com/dashpay/tenderdash/proto/tendermint/crypto" - p2pproto "github.com/dashpay/tenderdash/proto/tendermint/p2p" - tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -140,7 +133,7 @@ func (suite *BlockP2PMessageHandlerTestSuite) expectStoredBlock(height int64, ti BlockID: types.BlockID{PartSetHeader: types.PartSetHeader{Total: 1}}, BlockSize: len(blockBytes), } - suite.fakeStore.On("LoadBlockPart", height, 0).Times(times).Return(&types.Part{Bytes: blockBytes, Proof: merkle.Proof{Total: 1}}) + suite.fakeStore.On("LoadBlockPart", height, 0).Times(times).Return(&types.Part{Bytes: blockBytes}) return suite.fakeStore.On("LoadBlockMeta", height).Times(times).Return(meta) } @@ -528,46 +521,42 @@ func (suite *BlockP2PMessageHandlerTestSuite) TestReconnectRequestSurvivesEarlie suite.Zero(h.queuedResponses) } -// TestServeBlockAsCommitted checks that a block is served as the parts it was -// stored with, which are the parts it was committed with, and that the part set -// rebuilt from the response is the stored one. +// TestServeBlockAsCommitted checks that a block is served as the bytes of the +// parts it was stored with, which are the parts it was committed with, and that +// cutting them the standard way rebuilds the stored part set. func TestServeBlockAsCommitted(t *testing.T) { ctx := context.Background() valSet, privVals := factory.MockValidatorSet() state := fakeInitialState(valSet) - blocks := statefactory.MakeBlocks(ctx, t, 6, &state, privVals, 1) + blocks := statefactory.MakeBlocks(ctx, t, 5, &state, privVals, 1) blockStore := store.NewBlockStore(dbm.NewMemDB()) const smallPartSize = 100 standard := blockBytes(t, blocks[0]) unknownField := withUnknownField(t, blocks[1]) smallParts := blockBytes(t, blocks[2]) - largerTree := withProofOfLargerTree(t, blocks[3]) testCases := []struct { - parts *types.PartSet - wantBytes []byte - wantPartSizes []uint32 - wantProofs bool + name string + parts *types.PartSet + bytes []byte + // whether the standard cut of the bytes is the stored part set + standardCut bool }{ - {types.NewPartSetFromData(standard, types.BlockPartSizeBytes), standard, nil, false}, - {types.NewPartSetFromData(unknownField, types.BlockPartSizeBytes), unknownField, nil, false}, - { - types.NewPartSetFromData(smallParts, smallPartSize), - smallParts, - smallPartSizes(len(smallParts), smallPartSize), - false, - }, - {largerTree, blockBytes(t, blocks[3]), []uint32{uint32(largerTree.ByteSize())}, true}, + {"standard", types.NewPartSetFromData(standard, types.BlockPartSizeBytes), standard, true}, + {"unknown field", types.NewPartSetFromData(unknownField, types.BlockPartSizeBytes), unknownField, true}, + // served as stored; block sync does not rebuild a cut other than the + // standard one, so a syncing node rejects it against the commit + {"small parts", types.NewPartSetFromData(smallParts, smallPartSize), smallParts, false}, } for i, tc := range testCases { blockStore.SaveBlock(blocks[i], tc.parts, blocks[i+1].LastCommit) } // a header stored without its block, as state sync stores the one it // restores the state at - header := blocks[4] + header := blocks[3] require.NoError(t, blockStore.SaveSignedHeader( - &types.SignedHeader{Header: &header.Header, Commit: blocks[5].LastCommit}, - blocks[5].LastCommit.BlockID, + &types.SignedHeader{Header: &header.Header, Commit: blocks[4].LastCommit}, + blocks[4].LastCommit.BlockID, )) handler := newBlockP2PMessageHandler(ctx, log.NewNopLogger(), blockStore, newMockPeerAdder(t)) @@ -583,37 +572,23 @@ func TestServeBlockAsCommitted(t *testing.T) { } for i, tc := range testCases { - height := blocks[i].Height - t.Run(fmt.Sprintf("height %d", height), func(t *testing.T) { - msg, ok := request(height).(*bcproto.BlockResponse) + t.Run(tc.name, func(t *testing.T) { + msg, ok := request(blocks[i].Height).(*bcproto.BlockResponse) require.True(t, ok) - require.Equal(t, tc.wantBytes, msg.Block) - require.Equal(t, tc.wantPartSizes == nil, len(msg.PartLayout) == 0) + require.Equal(t, tc.bytes, msg.Block) resp, err := BlockResponseFromProto(msg, "peer") require.NoError(t, err) - require.Equal(t, tc.wantPartSizes, resp.Parts.Sizes) - require.Equal(t, tc.wantProofs, len(resp.Parts.Proofs) > 0) + require.Equal(t, tc.bytes, resp.BlockBytes) require.Equal(t, blocks[i].Hash(), resp.Block.Hash()) require.Equal(t, blocks[i+1].LastCommit.BlockID, resp.Commit.BlockID) - parts, err := resp.Parts.partSet(tc.parts.Header()) - require.NoError(t, err) - require.Equal(t, tc.parts.Header(), parts.Header()) + cut := types.NewPartSetFromData(resp.BlockBytes, types.BlockPartSizeBytes) + require.Equal(t, tc.standardCut, tc.parts.Header().Equals(cut.Header())) }) } require.Equal(t, &bcproto.NoBlockResponse{Height: header.Height}, request(header.Height)) } -// smallPartSizes returns the sizes of the parts types.NewPartSetFromData cuts -// size bytes into with parts of partSize. -func smallPartSizes(size, partSize int) []uint32 { - var sizes []uint32 - for ; size > partSize; size -= partSize { - sizes = append(sizes, uint32(partSize)) - } - return append(sizes, uint32(size)) -} - // TestBlockResponseWireCompatible checks that a BlockResponse encodes as it did // when its block field was an embedded tendermint.types.Block, so peers on // either side of the change read what the other sends. @@ -642,7 +617,7 @@ func TestBlockResponseWireCompatible(t *testing.T) { resp, err := BlockResponseFromProto(&decoded, "peer") require.NoError(t, err) require.Equal(t, block.Hash(), resp.Block.Hash()) - require.Equal(t, CommittedParts{Bytes: embedded}, resp.Parts) + require.Equal(t, embedded, resp.BlockBytes) } // appendBytesField appends a length-delimited field to serialized protobuf. @@ -652,166 +627,6 @@ func appendBytesField(bz []byte, number byte, value []byte) []byte { return append(bz, value...) } -// partLayout returns a serialized BlockPartLayout. -func partLayout(t *testing.T, sizes []uint32, proofs []crypto.Proof) []byte { - t.Helper() - bz, err := proto.Marshal(&bcproto.BlockPartLayout{Sizes: sizes, Proofs: proofs}) - require.NoError(t, err) - return bz -} - -// hiddenProofsBehindPackedSize returns a layout that opens with a packed size -// whose varint does not end inside its field. The generated decoder reads on -// past the field, so a scan that took the field's length at face value would -// see one unknown field where the decoder finds that many empty proofs. -func hiddenProofsBehindPackedSize(proofs int) []byte { - return bytes.Join([][]byte{ - {0x0a, 0x01, 0x80, 0x1a, 0x82, 0x80, 0x80, 0x01, 0x00, 0x18, 0x80, 0x00}, - bytes.Repeat([]byte{2<<3 | 2, 0}, proofs+1), - }, nil) -} - -// hiddenAuntsBehindPackedSize is hiddenProofsBehindPackedSize with chunks that -// each hide a proof of seven empty aunts. -func hiddenAuntsBehindPackedSize(chunks int) []byte { - chunk := append([]byte{0x0a, 0x01, 0x80, 0x1a, 0x12, 0x11, 0x08, 0x80, 0x01}, - bytes.Repeat([]byte{4<<3 | 2, 0}, 7)...) - return bytes.Repeat(chunk, chunks+1) -} - -// TestBlockResponseFromProtoRejectsMalformedParts checks that part metadata a -// peer sent that cannot describe its block is refused while decoding the -// response, before it can be held waiting for a lower height. -func TestBlockResponseFromProtoRejectsMalformedParts(t *testing.T) { - ctx := context.Background() - valSet, privVals := factory.MockValidatorSet() - state := fakeInitialState(valSet) - blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) - bz := blockBytes(t, blocks[0]) - size := uint32(len(bz)) - proof := crypto.Proof{Total: 1, LeafHash: make([]byte, tmcrypto.HashSize)} - tooManyAunts := proof - tooManyAunts.Aunts = make([][]byte, merkle.MaxAunts+1) - for i := range tooManyAunts.Aunts { - tooManyAunts.Aunts[i] = make([]byte, tmcrypto.HashSize) - } - longAunt := proof - longAunt.Aunts = [][]byte{make([]byte, tmcrypto.HashSize+1)} - tooManyParts := int(types.MaxBlockPartsCount) + 1 - testCases := map[string]struct { - layout []byte - wantErr string - }{ - "layout too big": {make([]byte, p2p.BlockPartLayoutMaxBytes+1), "part layout too big"}, - "not a layout": {[]byte{0x0a, 0x05}, "unexpected EOF"}, - "too many parts": {partLayout(t, make([]uint32, tooManyParts), nil), "too many parts"}, - "unpacked parts": {bytes.Repeat([]byte{1 << 3, 0}, tooManyParts), "too many parts"}, - "too many proofs": {bytes.Repeat([]byte{2<<3 | 2, 0}, tooManyParts), "too many proofs"}, - "truncated packed size hiding proofs": { - hiddenProofsBehindPackedSize(int(types.MaxBlockPartsCount)), "unexpected EOF", - }, - "truncated packed size hiding aunts": { - hiddenAuntsBehindPackedSize(int(types.MaxBlockPartsCount)), "unexpected EOF", - }, - "unknown field": {[]byte{3 << 3, 0}, "unexpected field 3"}, - "group": {[]byte{3<<3 | 3, 3<<3 | 4}, "unexpected field 3"}, - "proofs of wrong type": {[]byte{2 << 3, 0}, "unexpected field 2"}, - "unknown proof field": {[]byte{2<<3 | 2, 2, 5 << 3, 0}, "unexpected field 5"}, - "proof total of wrong type": { - []byte{2<<3 | 2, 2, 1<<3 | 2, 0}, "unexpected field 1", - }, - "too many aunts": {partLayout(t, []uint32{size}, []crypto.Proof{tooManyAunts}), "too many aunts"}, - "aunt too long": {partLayout(t, []uint32{size}, []crypto.Proof{longAunt}), "hash too long"}, - "part too big": {partLayout(t, []uint32{types.BlockPartSizeBytes + 1}, nil), "too big"}, - "sizes short": {partLayout(t, []uint32{size - 1}, nil), "part sizes cover"}, - "sizes beyond block": {partLayout(t, []uint32{size + 1}, nil), "exceed"}, - "proofs without sizes": { - partLayout(t, nil, []crypto.Proof{proof}), "1 proofs for 0 parts", - }, - "too few proofs": { - partLayout(t, []uint32{size - 1, 1}, []crypto.Proof{proof}), "1 proofs for 2 parts", - }, - "proof of another part": { - partLayout(t, []uint32{size}, []crypto.Proof{{Total: 1, Index: 1, LeafHash: proof.LeafHash}}), "has index 1", - }, - } - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - _, err := BlockResponseFromProto(&bcproto.BlockResponse{ - Block: bz, - Commit: blocks[1].LastCommit.ToProto(), - PartLayout: tc.layout, - }, "peer") - require.ErrorContains(t, err, tc.wantErr) - }) - } -} - -// TestBlockResponseSizeChargesPartMetadata checks that a response is charged to -// the backlog for its part metadata, and not only for its block. -func TestBlockResponseSizeChargesPartMetadata(t *testing.T) { - ctx := context.Background() - valSet, privVals := factory.MockValidatorSet() - state := fakeInitialState(valSet) - blocks := statefactory.MakeBlocks(ctx, t, 2, &state, privVals, 1) - bz := blockBytes(t, blocks[0]) - proof := crypto.Proof{Total: 2, LeafHash: make([]byte, tmcrypto.HashSize)} - layout := partLayout(t, []uint32{uint32(len(bz))}, []crypto.Proof{proof}) - resp, err := BlockResponseFromProto(&bcproto.BlockResponse{ - Block: bz, - Commit: blocks[1].LastCommit.ToProto(), - PartLayout: layout, - }, "peer") - require.NoError(t, err) - require.Equal(t, len(bz)+len(layout), resp.Size) -} - -// TestPartLayoutIsOpaqueAtEnvelopeBoundary checks that part metadata costs no -// per-entry allocation when a message is received, which happens before it is -// matched to a request, nor when the layout is refused. Empty proofs are the -// smallest encoding of the most entries. -func TestPartLayoutIsOpaqueAtEnvelopeBoundary(t *testing.T) { - const ( - allowedAllocs = 32 - entries = int(types.MaxBlockPartsCount) - ) - testCases := map[string]struct { - layout []byte - wantErr string - }{ - "empty proofs": {bytes.Repeat([]byte{2<<3 | 2, 0}, entries+1), "too many proofs"}, - "proofs behind a packed size": {hiddenProofsBehindPackedSize(1 << 20), "unexpected EOF"}, - "aunts behind a packed size": {hiddenAuntsBehindPackedSize(entries), "unexpected EOF"}, - } - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - require.LessOrEqual(t, len(tc.layout), p2p.BlockPartLayoutMaxBytes) - // block_response is field 9 of the envelope, part_layout field 3 of the response - raw := appendBytesField(nil, 9, appendBytesField(nil, 3, tc.layout)) - - require.Less(t, testing.AllocsPerRun(10, func() { - var envelope p2pproto.Envelope - _ = proto.Unmarshal(raw, &envelope) - }), float64(allowedAllocs), "decoding the envelope must not allocate per layout entry") - - var envelope p2pproto.Envelope - require.NoError(t, proto.Unmarshal(raw, &envelope)) - msg, err := envelope.Unwrap() - require.NoError(t, err) - resp, ok := msg.(*bcproto.BlockResponse) - require.True(t, ok) - require.Equal(t, tc.layout, resp.PartLayout) - - resp.Block = []byte{1} - require.Less(t, testing.AllocsPerRun(10, func() { - _, _ = committedPartsFromProto(resp) - }), float64(allowedAllocs), "refusing the layout must not allocate per layout entry") - _, err = committedPartsFromProto(resp) - require.ErrorContains(t, err, tc.wantErr) - }) - } -} - // TestBlockResponseDoesNotRetainOverwrittenBlockField checks that a repeated // block field does not leave the allocation of an earlier, larger occurrence // held behind the small one that is decoded and charged to the backlog. @@ -832,51 +647,7 @@ func TestBlockResponseDoesNotRetainOverwrittenBlockField(t *testing.T) { resp, err := BlockResponseFromProto(&decoded, "peer") require.NoError(t, err) - require.Equal(t, bz, resp.Parts.Bytes) + require.Equal(t, bz, resp.BlockBytes) require.Equal(t, len(bz), resp.Size) - require.LessOrEqual(t, cap(resp.Parts.Bytes), 2*len(bz)) -} - -// TestBlockResponseFitsBlockSyncChannel checks that the largest block response -// a node can have to serve fits the receive capacity of the block sync channel. -func TestBlockResponseFitsBlockSyncChannel(t *testing.T) { - const parts = int(types.MaxBlockPartsCount) - aunts := make([][]byte, merkle.MaxAunts) - for i := range aunts { - aunts[i] = make([]byte, tmcrypto.HashSize) - } - sizes := make([]uint32, parts) - proofs := make([]crypto.Proof, parts) - for i := range parts { - sizes[i] = types.BlockPartSizeBytes - proofs[i] = crypto.Proof{Total: math.MaxInt64, Index: int64(parts), LeafHash: aunts[0], Aunts: aunts} - } - blockID := tmproto.BlockID{ - Hash: make([]byte, tmcrypto.HashSize), - PartSetHeader: tmproto.PartSetHeader{Total: uint32(parts), Hash: make([]byte, tmcrypto.HashSize)}, - StateID: make([]byte, tmcrypto.HashSize), - } - layout := partLayout(t, sizes, proofs) - require.LessOrEqual(t, len(layout), p2p.BlockPartLayoutMaxBytes) - require.NoError(t, checkPartLayoutBounds(layout)) - envelope := &p2pproto.Envelope{Attributes: map[string]string{client.RequestIDAttribute: uuid.NewString()}} - require.NoError(t, envelope.Wrap(&bcproto.BlockResponse{ - Block: make([]byte, types.MaxBlockSizeBytes), - Commit: &tmproto.Commit{ - Height: math.MaxInt64, - Round: math.MaxInt32, - BlockID: blockID, - QuorumHash: make([]byte, tmcrypto.HashSize), - ThresholdBlockSignature: make([]byte, 96), - // everything a vote on the consensus vote channel can carry - ThresholdVoteExtensions: []*tmproto.VoteExtension{{ - Extension: make([]byte, 1<<20-1024), - Signature: make([]byte, 96), - }}, - }, - PartLayout: layout, - })) - - descriptor := p2p.ChannelDescriptors(config.TestConfig())[p2p.BlockSyncChannel] - require.LessOrEqual(t, envelope.Size(), descriptor.RecvMessageCapacity) + require.LessOrEqual(t, cap(resp.BlockBytes), 2*len(bz)) } diff --git a/internal/blocksync/reactor_test.go b/internal/blocksync/reactor_test.go index 15312ad89a..9141e4546b 100644 --- a/internal/blocksync/reactor_test.go +++ b/internal/blocksync/reactor_test.go @@ -16,7 +16,6 @@ import ( abciclient "github.com/dashpay/tenderdash/abci/client" abci "github.com/dashpay/tenderdash/abci/types" "github.com/dashpay/tenderdash/config" - "github.com/dashpay/tenderdash/crypto/merkle" "github.com/dashpay/tenderdash/internal/consensus" "github.com/dashpay/tenderdash/internal/eventbus" mpmocks "github.com/dashpay/tenderdash/internal/mempool/mocks" @@ -45,13 +44,14 @@ type reactorTestSuite struct { peerChans map[types.NodeID]chan p2p.PeerUpdate peerUpdates map[types.NodeID]*p2p.PeerUpdates - // blockParts returns the parts the nodes commit their blocks with - blockParts blockPartsFunc + // blockBytes returns the bytes the nodes commit their blocks with + blockBytes blockBytesFunc } -// blockPartsFunc returns the parts a block at height is committed with, or nil -// for the parts types.Block.MakePartSet cuts it into. -type blockPartsFunc func(t *testing.T, height int64, block *types.Block) *types.PartSet +// blockBytesFunc returns the bytes a block is committed with, cut into parts of +// types.BlockPartSizeBytes. Nil commits the block as types.Block.MakePartSet +// encodes it. +type blockBytesFunc func(t *testing.T, block *types.Block) []byte func setup( ctx context.Context, @@ -63,19 +63,19 @@ func setup( opts ...ReactorOption, ) *reactorTestSuite { t.Helper() - return setupWithBlockParts(ctx, t, conf, genDoc, privVal, maxBlockHeights, nil, opts...) + return setupWithBlockBytes(ctx, t, conf, genDoc, privVal, maxBlockHeights, nil, opts...) } -// setupWithBlockParts is setup with the nodes committing their blocks with the -// parts blockParts returns. -func setupWithBlockParts( +// setupWithBlockBytes is setup with the nodes committing their blocks with the +// bytes blockBytes returns. +func setupWithBlockBytes( ctx context.Context, t *testing.T, conf *config.Config, genDoc *types.GenesisDoc, privVal types.PrivValidator, maxBlockHeights []int64, - blockParts blockPartsFunc, + blockBytes blockBytesFunc, opts ...ReactorOption, ) *reactorTestSuite { t.Helper() @@ -96,7 +96,7 @@ func setupWithBlockParts( blockSyncChannels: make(map[types.NodeID]p2p.Channel, numNodes), peerChans: make(map[types.NodeID]chan p2p.PeerUpdate, numNodes), peerUpdates: make(map[types.NodeID]*p2p.PeerUpdates, numNodes), - blockParts: blockParts, + blockBytes: blockBytes, } chDesc := &p2p.ChannelDescriptor{ID: p2p.BlockSyncChannel} @@ -224,8 +224,8 @@ func (rts *reactorTestSuite) addNode( state, err := reactor.stateStore.Load() require.NoError(t, err) for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ { - block, blockID, partSet, seenCommit := makeNextBlockWithParts(ctx, t, state, privVal, blockHeight, commit, - rts.blockParts) + block, blockID, partSet, seenCommit := makeNextBlockWithBytes(ctx, t, state, privVal, blockHeight, commit, + rts.blockBytes) state, err = reactor.blockExec.ApplyBlock(ctx, state, blockID, block, seenCommit, types.VerifiedCommit{}) require.NoError(t, err) reactor.store.SaveBlock(block, partSet, seenCommit) @@ -243,26 +243,25 @@ func makeNextBlock(ctx context.Context, signer types.PrivValidator, height int64, commit *types.Commit) (*types.Block, types.BlockID, *types.PartSet, *types.Commit) { - return makeNextBlockWithParts(ctx, t, state, signer, height, commit, nil) + return makeNextBlockWithBytes(ctx, t, state, signer, height, commit, nil) } -// makeNextBlockWithParts is makeNextBlock with the block committed with the -// parts blockParts returns. -func makeNextBlockWithParts(ctx context.Context, +// makeNextBlockWithBytes is makeNextBlock with the block committed with the +// bytes blockBytes returns. +func makeNextBlockWithBytes(ctx context.Context, t *testing.T, state sm.State, signer types.PrivValidator, height int64, commit *types.Commit, - blockParts blockPartsFunc) (*types.Block, types.BlockID, *types.PartSet, *types.Commit) { + blockBytes blockBytesFunc) (*types.Block, types.BlockID, *types.PartSet, *types.Commit) { block, err := sf.MakeBlock(state, height, commit, 0) require.NoError(t, err) block.CoreChainLockedHeight = state.LastCoreChainLockedBlockHeight var partSet *types.PartSet - if blockParts != nil { - partSet = blockParts(t, height, block) - } - if partSet == nil { + if blockBytes != nil { + partSet = types.NewPartSetFromData(blockBytes(t, block), types.BlockPartSizeBytes) + } else { partSet, err = block.MakePartSet(types.BlockPartSizeBytes) require.NoError(t, err) } @@ -459,11 +458,11 @@ func TestReactor_NoBlockResponse(t *testing.T) { } } -// TestReactor_SyncBlocksAsCommitted syncs blocks committed with bytes or parts -// that types.Block.MakePartSet does not reproduce from the decoded block. The -// commit's part set header is computed over the committed parts, so a peer can +// TestReactor_SyncBlocksAsCommitted syncs a block committed with bytes that +// types.Block.MakePartSet does not reproduce from the decoded block. The +// commit's part set header is computed over the committed bytes, so a peer can // only serve such a block from the parts it stored, and the syncing node must -// rebuild the part set from the bytes it received. +// cut the part set from the bytes it received. func TestReactor_SyncBlocksAsCommitted(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -474,55 +473,38 @@ func TestReactor_SyncBlocksAsCommitted(t *testing.T) { genDoc, privVals := factory.RandGenesisDoc(1, factory.ConsensusParams()) const ( - maxBlockHeight = int64(12) + maxBlockHeight = int64(6) // a field that tendermint.types.Block does not define, which decoding // skips, appended to the block's bytes unknownFieldHeight = int64(3) - // the block's bytes cut into parts shorter than types.BlockPartSizeBytes - smallPartsHeight = int64(5) - bothHeight = int64(7) - // one part, whose proof is of a tree of more leaves than the header - // counts, so the header cannot be computed from the part - largerTreeHeight = int64(9) - smallPartSize = 100 ) - committedHeights := []int64{unknownFieldHeight, smallPartsHeight, bothHeight, largerTreeHeight} - blockParts := func(t *testing.T, height int64, block *types.Block) *types.PartSet { - switch height { - case unknownFieldHeight: - return types.NewPartSetFromData(withUnknownField(t, block), types.BlockPartSizeBytes) - case smallPartsHeight: - return types.NewPartSetFromData(blockBytes(t, block), smallPartSize) - case bothHeight: - return types.NewPartSetFromData(withUnknownField(t, block), smallPartSize) - case largerTreeHeight: - return withProofOfLargerTree(t, block) + committedBytes := func(t *testing.T, block *types.Block) []byte { + if block.Height == unknownFieldHeight { + return withUnknownField(t, block) } - return nil + return blockBytes(t, block) } - rts := setupWithBlockParts(ctx, t, cfg, genDoc, privVals[0], []int64{maxBlockHeight, 0}, blockParts) + rts := setupWithBlockBytes(ctx, t, cfg, genDoc, privVals[0], []int64{maxBlockHeight, 0}, committedBytes) source, syncing := rts.reactors[rts.nodes[0]], rts.reactors[rts.nodes[1]] require.Equal(t, maxBlockHeight, source.store.Height()) - for _, height := range committedHeights { - require.NotEqual(t, - source.store.LoadBlockMeta(height).BlockID.PartSetHeader, - source.store.LoadBlock(height).BlockID(nil).PartSetHeader, - "height %d must be committed with parts a re-encoding does not reproduce", height) - } + require.NotEqual(t, + source.store.LoadBlockMeta(unknownFieldHeight).BlockID.PartSetHeader, + source.store.LoadBlock(unknownFieldHeight).BlockID(nil).PartSetHeader, + "the block must be committed with bytes a re-encoding does not reproduce") rts.start(ctx, t) require.Eventually(t, - func() bool { return syncing.store.Height() > largerTreeHeight }, + func() bool { return syncing.store.Height() > unknownFieldHeight }, 30*time.Second, 10*time.Millisecond, - "expected node to sync the blocks committed with other bytes") - for _, height := range committedHeights { - require.Equal(t, source.store.LoadBlockMeta(height).BlockID, syncing.store.LoadBlockMeta(height).BlockID, - "block ID at height %d", height) - require.Equal(t, loadCommittedParts(source.store, height), loadCommittedParts(syncing.store, height), - "stored parts at height %d", height) - } + "expected node to sync the block committed with other bytes") + require.Equal(t, + source.store.LoadBlockMeta(unknownFieldHeight).BlockID, + syncing.store.LoadBlockMeta(unknownFieldHeight).BlockID) + require.Equal(t, + loadBlockBytes(source.store, unknownFieldHeight), + loadBlockBytes(syncing.store, unknownFieldHeight)) } // withUnknownField returns the bytes of block with field 15 appended, a field @@ -532,19 +514,6 @@ func withUnknownField(t *testing.T, block *types.Block) []byte { return append(blockBytes(t, block), 15<<3, 1) } -// withProofOfLargerTree returns a part set of one part with the bytes of block, -// whose root is that of a tree of two leaves. PartSet.AddPart checks a proof -// against the root and not against the header's total, so consensus accepts it. -func withProofOfLargerTree(t *testing.T, block *types.Block) *types.PartSet { - bz := blockBytes(t, block) - root, proofs := merkle.ProofsFromByteSlices([][]byte{bz, []byte("leaf the header does not count")}) - partSet := types.NewPartSetFromHeader(types.PartSetHeader{Total: 1, Hash: root}) - added, err := partSet.AddPart(&types.Part{Index: 0, Bytes: bz, Proof: *proofs[0]}) - require.NoError(t, err) - require.True(t, added) - return partSet -} - func TestReactor_BadBlockStopsPeer(t *testing.T) { // Ultimately, this should be refactored to be less integration test oriented // and more unit test oriented by simply testing channel sends and receives. diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index 3358644793..07f6c2ef01 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "sync/atomic" "time" @@ -640,7 +641,7 @@ func (s *Synchronizer) applyBlock(ctx context.Context) (BlockResponse, error) { if !ok { return BlockResponse{}, nil } - err := s.applier.Apply(ctx, resp.Block, resp.Parts, resp.Commit) + err := s.applier.Apply(ctx, resp.Block, resp.BlockBytes, resp.Commit) if err != nil { return resp, fmt.Errorf("cannot apply block: %w", err) } @@ -779,12 +780,13 @@ type BlockResponse struct { // its failures are fenced to. ConnID uint64 Block *types.Block - // Parts is Block as it was committed. Block is decoded from Parts.Bytes, and - // applied with the part set rebuilt from Parts: that is what the commit's - // block ID describes, and re-encoding Block need not give the same bytes. - Parts CommittedParts - Commit *types.Commit - // Size is the serialized size of Parts in bytes. + // BlockBytes is Block as it was committed, the bytes Block is decoded from. + // Block is applied with the part set cut from these: that is what the + // commit's block ID describes, and re-encoding Block need not give the same + // bytes. + BlockBytes []byte + Commit *types.Commit + // Size is the serialized size of Block in bytes, the length of BlockBytes. Size int } @@ -811,14 +813,14 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* if len(resp.Block) == 0 { return nil, errors.New("block response without a block") } - // Checked before anything is decoded or queued: the response can wait behind - // a missing lower height, and only its size is charged to the backlog. - parts, err := committedPartsFromProto(resp) - if err != nil { - return nil, fmt.Errorf("invalid block parts: %w", err) + blockBytes := resp.Block + // A repeated block field leaves the allocation of its longest occurrence + // behind the last one, and only the length is charged to the backlog. + if cap(blockBytes) > 2*len(blockBytes) { + blockBytes = slices.Clone(blockBytes) } pbb := new(tmproto.Block) - if err := proto.Unmarshal(resp.Block, pbb); err != nil { + if err := proto.Unmarshal(blockBytes, pbb); err != nil { return nil, fmt.Errorf("cannot decode block: %w", err) } block, err := types.BlockFromProto(pbb) @@ -833,10 +835,10 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* } } return &BlockResponse{ - PeerID: peerID, - Block: block, - Parts: parts, - Commit: commit, - Size: len(resp.Block) + len(resp.PartLayout), + PeerID: peerID, + Block: block, + BlockBytes: blockBytes, + Commit: commit, + Size: len(blockBytes), }, nil } diff --git a/internal/p2p/channel_params.go b/internal/p2p/channel_params.go index befce7340f..b08e1d9059 100644 --- a/internal/p2p/channel_params.go +++ b/internal/p2p/channel_params.go @@ -6,8 +6,6 @@ import ( "github.com/cosmos/gogoproto/proto" "github.com/dashpay/tenderdash/config" - "github.com/dashpay/tenderdash/crypto" - "github.com/dashpay/tenderdash/crypto/merkle" "github.com/dashpay/tenderdash/proto/tendermint/blocksync" "github.com/dashpay/tenderdash/proto/tendermint/consensus" "github.com/dashpay/tenderdash/proto/tendermint/mempool" @@ -53,26 +51,6 @@ const ( // consensusMsgSize is the maximum size of a consensus message maxMsgSize = 1048576 // 1MB; NOTE: keep in sync with types.PartSet sizes. - // blockPartSizeMaxBytes bounds one encoded part_sizes entry of a block - // response: a varint of at most types.BlockPartSizeBytes. - blockPartSizeMaxBytes = 3 - // blockPartProofMaxBytes bounds one encoded part_proofs entry of a block - // response: its key and length, the two int64 varints with their keys, and - // the leaf hash and aunts with their keys and lengths. - blockPartProofMaxBytes = 4 + 2*(1+10) + (merkle.MaxAunts+1)*(2+crypto.HashSize) - // BlockPartLayoutMaxBytes bounds a serialized BlockPartLayout: a size and a - // proof for the most parts a block can have, and the keys of the two fields. - BlockPartLayoutMaxBytes = int(types.MaxBlockPartsCount)*(blockPartSizeMaxBytes+blockPartProofMaxBytes) + 16 - // blockSyncMaxOverheadBytes bounds what a block response carries besides - // the block: its part layout, a commit, whose vote extensions came in one - // vote on a maxMsgSize channel, and the envelope with its attributes. - blockSyncMaxOverheadBytes = BlockPartLayoutMaxBytes + maxMsgSize + 1024 - - // BlockSyncMaxMsgSize is the maximum size of a block sync message - BlockSyncMaxMsgSize = types.MaxBlockSizeBytes + - p2pproto.BlockResponseMessagePrefixSize + - p2pproto.BlockResponseMessageFieldKeySize + - blockSyncMaxOverheadBytes ) // ChannelDescriptors returns a map of all supported descriptors @@ -86,12 +64,14 @@ func ChannelDescriptors(cfg *config.Config) map[ChannelID]*ChannelDescriptor { Name: "error", }, BlockSyncChannel: { - ID: BlockSyncChannel, - Priority: 6, - SendQueueCapacity: 4, - RecvBufferCapacity: 1024, - RecvMessageCapacity: BlockSyncMaxMsgSize, - Name: "blockSync", + ID: BlockSyncChannel, + Priority: 6, + SendQueueCapacity: 4, + RecvBufferCapacity: 1024, + RecvMessageCapacity: types.MaxBlockSizeBytes + + p2pproto.BlockResponseMessagePrefixSize + + p2pproto.BlockResponseMessageFieldKeySize, + Name: "blockSync", }, MempoolChannel: { ID: MempoolChannel, diff --git a/proto/tendermint/blocksync/types.pb.go b/proto/tendermint/blocksync/types.pb.go index ba34a2242d..9e174eb89e 100644 --- a/proto/tendermint/blocksync/types.pb.go +++ b/proto/tendermint/blocksync/types.pb.go @@ -5,9 +5,7 @@ package blocksync import ( fmt "fmt" - _ "github.com/cosmos/gogoproto/gogoproto" proto "github.com/cosmos/gogoproto/proto" - crypto "github.com/dashpay/tenderdash/proto/tendermint/crypto" types "github.com/dashpay/tenderdash/proto/tendermint/types" io "io" math "math" @@ -119,19 +117,14 @@ func (m *NoBlockResponse) GetHeight() int64 { // BlockResponse returns block to the requested type BlockResponse struct { // block is a serialized tendermint.types.Block: the bytes of the parts it - // was committed with, in order. The commit's part set header is computed - // over those bytes, and decoding and re-encoding the block can change them - // (unknown fields, non-canonical encoding). On the wire this is the same as - // an embedded tendermint.types.Block, the type of this field in older - // releases. + // was committed with, in order. The receiver cuts them into parts of 64kB, + // as the proposer did, and checks the part set header it computes against + // the commit's. Decoding and re-encoding the block can change its bytes + // (unknown fields, non-canonical encoding), so it is served as stored. On + // the wire this is the same as an embedded tendermint.types.Block, the type + // of this field in older releases. Block []byte `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` Commit *types.Commit `protobuf:"bytes,2,opt,name=commit,proto3" json:"commit,omitempty"` - // part_layout is a serialized BlockPartLayout. It is empty when the block was - // cut into parts of 64kB, the last one shorter, the way honest proposers cut - // it. It is opaque here because every received message is decoded before it - // is matched to a request: the layout is only decoded once it is known to be - // the response to one, and within bounds. - PartLayout []byte `protobuf:"bytes,3,opt,name=part_layout,json=partLayout,proto3" json:"part_layout,omitempty"` } func (m *BlockResponse) Reset() { *m = BlockResponse{} } @@ -181,71 +174,6 @@ func (m *BlockResponse) GetCommit() *types.Commit { return nil } -func (m *BlockResponse) GetPartLayout() []byte { - if m != nil { - return m.PartLayout - } - return nil -} - -// BlockPartLayout describes the parts a block was committed with, when they -// cannot be derived from the block's bytes. -type BlockPartLayout struct { - // sizes is the size of each part, in order. - Sizes []uint32 `protobuf:"varint,1,rep,packed,name=sizes,proto3" json:"sizes,omitempty"` - // proofs is the Merkle proof of each part, in order. It is empty when the - // part set header can be computed from the parts, which it cannot when a - // proof's total differs from the header's. - Proofs []crypto.Proof `protobuf:"bytes,2,rep,name=proofs,proto3" json:"proofs"` -} - -func (m *BlockPartLayout) Reset() { *m = BlockPartLayout{} } -func (m *BlockPartLayout) String() string { return proto.CompactTextString(m) } -func (*BlockPartLayout) ProtoMessage() {} -func (*BlockPartLayout) Descriptor() ([]byte, []int) { - return fileDescriptor_19b397c236e0fa07, []int{3} -} -func (m *BlockPartLayout) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *BlockPartLayout) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_BlockPartLayout.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *BlockPartLayout) XXX_Merge(src proto.Message) { - xxx_messageInfo_BlockPartLayout.Merge(m, src) -} -func (m *BlockPartLayout) XXX_Size() int { - return m.Size() -} -func (m *BlockPartLayout) XXX_DiscardUnknown() { - xxx_messageInfo_BlockPartLayout.DiscardUnknown(m) -} - -var xxx_messageInfo_BlockPartLayout proto.InternalMessageInfo - -func (m *BlockPartLayout) GetSizes() []uint32 { - if m != nil { - return m.Sizes - } - return nil -} - -func (m *BlockPartLayout) GetProofs() []crypto.Proof { - if m != nil { - return m.Proofs - } - return nil -} - // StatusRequest requests the status of a peer. type StatusRequest struct { } @@ -254,7 +182,7 @@ func (m *StatusRequest) Reset() { *m = StatusRequest{} } func (m *StatusRequest) String() string { return proto.CompactTextString(m) } func (*StatusRequest) ProtoMessage() {} func (*StatusRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_19b397c236e0fa07, []int{4} + return fileDescriptor_19b397c236e0fa07, []int{3} } func (m *StatusRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -293,7 +221,7 @@ func (m *StatusResponse) Reset() { *m = StatusResponse{} } func (m *StatusResponse) String() string { return proto.CompactTextString(m) } func (*StatusResponse) ProtoMessage() {} func (*StatusResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_19b397c236e0fa07, []int{5} + return fileDescriptor_19b397c236e0fa07, []int{4} } func (m *StatusResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -340,7 +268,6 @@ func init() { proto.RegisterType((*BlockRequest)(nil), "tendermint.blocksync.BlockRequest") proto.RegisterType((*NoBlockResponse)(nil), "tendermint.blocksync.NoBlockResponse") proto.RegisterType((*BlockResponse)(nil), "tendermint.blocksync.BlockResponse") - proto.RegisterType((*BlockPartLayout)(nil), "tendermint.blocksync.BlockPartLayout") proto.RegisterType((*StatusRequest)(nil), "tendermint.blocksync.StatusRequest") proto.RegisterType((*StatusResponse)(nil), "tendermint.blocksync.StatusResponse") } @@ -348,30 +275,24 @@ func init() { func init() { proto.RegisterFile("tendermint/blocksync/types.proto", fileDescriptor_19b397c236e0fa07) } var fileDescriptor_19b397c236e0fa07 = []byte{ - // 361 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x92, 0xc1, 0x4a, 0xeb, 0x40, - 0x14, 0x86, 0x93, 0xa6, 0x37, 0x8b, 0xd3, 0xf6, 0x16, 0x42, 0xb9, 0x84, 0x72, 0x6f, 0x1a, 0xb2, - 0xb8, 0xd4, 0x4d, 0x46, 0x14, 0xc4, 0x85, 0xab, 0xba, 0x15, 0x29, 0xe3, 0xce, 0x4d, 0x49, 0xd2, - 0x31, 0x09, 0x36, 0x99, 0x31, 0x33, 0x01, 0xe3, 0x53, 0xf8, 0x58, 0x5d, 0x76, 0xe9, 0x4a, 0xa4, - 0x7d, 0x11, 0xc9, 0x4c, 0xac, 0x51, 0xd4, 0xdd, 0x9c, 0xf3, 0x7f, 0x9c, 0x7f, 0xce, 0xcf, 0x01, - 0x57, 0x90, 0x7c, 0x49, 0x8a, 0x2c, 0xcd, 0x05, 0x0a, 0x57, 0x34, 0xba, 0xe5, 0x55, 0x1e, 0x21, - 0x51, 0x31, 0xc2, 0x7d, 0x56, 0x50, 0x41, 0xad, 0xd1, 0x3b, 0xe1, 0xef, 0x89, 0xf1, 0x28, 0xa6, - 0x31, 0x95, 0x00, 0xaa, 0x5f, 0x8a, 0x1d, 0xff, 0x6b, 0x4d, 0x8b, 0x8a, 0x8a, 0x09, 0x8a, 0x58, - 0x41, 0xe9, 0x4d, 0x23, 0xff, 0x6d, 0xc9, 0xd2, 0xa2, 0x6d, 0xe4, 0xfd, 0x87, 0xfe, 0xac, 0x9e, - 0x8f, 0xc9, 0x5d, 0x49, 0xb8, 0xb0, 0xfe, 0x80, 0x99, 0x90, 0x34, 0x4e, 0x84, 0xad, 0xbb, 0xfa, - 0xd4, 0xc0, 0x4d, 0xe5, 0x1d, 0xc0, 0xf0, 0x92, 0x36, 0x24, 0x67, 0x34, 0xe7, 0xe4, 0x5b, 0xf4, - 0x1e, 0x06, 0x1f, 0xc1, 0x11, 0xfc, 0x92, 0x3b, 0x48, 0xae, 0x8f, 0x55, 0x61, 0x1d, 0x82, 0x19, - 0xd1, 0x2c, 0x4b, 0x85, 0xdd, 0x71, 0xf5, 0x69, 0xef, 0xc8, 0xf6, 0x5b, 0x3b, 0xab, 0x2f, 0x9e, - 0x4b, 0x1d, 0x37, 0x9c, 0x35, 0x81, 0x1e, 0x0b, 0x0a, 0xb1, 0x58, 0x05, 0x15, 0x2d, 0x85, 0x6d, - 0xc8, 0x69, 0x50, 0xb7, 0x2e, 0x64, 0xc7, 0x5b, 0xc0, 0x50, 0x3a, 0xcf, 0xf7, 0xad, 0xda, 0x9b, - 0xa7, 0x0f, 0x84, 0xdb, 0xba, 0x6b, 0x4c, 0x07, 0x58, 0x15, 0xd6, 0x09, 0x98, 0x32, 0x22, 0x6e, - 0x77, 0x5c, 0xe3, 0xb3, 0xb7, 0xca, 0xd0, 0x9f, 0xd7, 0xc0, 0xac, 0xbb, 0x7e, 0x9e, 0x68, 0xb8, - 0xa1, 0xbd, 0x21, 0x0c, 0xae, 0x44, 0x20, 0x4a, 0xde, 0xc4, 0xe5, 0x9d, 0xc1, 0xef, 0xb7, 0xc6, - 0xcf, 0xa9, 0x58, 0x16, 0x74, 0xc3, 0x80, 0x13, 0xb9, 0xac, 0x81, 0xe5, 0x7b, 0x86, 0xd7, 0x5b, - 0x47, 0xdf, 0x6c, 0x1d, 0xfd, 0x65, 0xeb, 0xe8, 0x8f, 0x3b, 0x47, 0xdb, 0xec, 0x1c, 0xed, 0x69, - 0xe7, 0x68, 0xd7, 0xa7, 0x71, 0x2a, 0x92, 0x32, 0xf4, 0x23, 0x9a, 0xa1, 0x65, 0xc0, 0x13, 0x16, - 0x54, 0x48, 0x7d, 0xb1, 0xae, 0x90, 0xba, 0x81, 0xaf, 0xae, 0x28, 0x34, 0xa5, 0x76, 0xfc, 0x3a, - 0x00, 0xfa, 0x6f, 0x07, 0x87, 0x64, 0x02, 0x00, 0x00, + // 264 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x28, 0x49, 0xcd, 0x4b, + 0x49, 0x2d, 0xca, 0xcd, 0xcc, 0x2b, 0xd1, 0x4f, 0xca, 0xc9, 0x4f, 0xce, 0x2e, 0xae, 0xcc, 0x4b, + 0xd6, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x12, 0x41, 0xa8, + 0xd0, 0x83, 0xab, 0x90, 0x92, 0x41, 0xd2, 0x07, 0x56, 0x8d, 0xac, 0x47, 0x49, 0x8d, 0x8b, 0xc7, + 0x09, 0xa4, 0x34, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x48, 0x8c, 0x8b, 0x2d, 0x23, 0x35, + 0x33, 0x3d, 0xa3, 0x44, 0x82, 0x51, 0x81, 0x51, 0x83, 0x39, 0x08, 0xca, 0x53, 0xd2, 0xe4, 0xe2, + 0xf7, 0xcb, 0x87, 0xaa, 0x2c, 0x2e, 0xc8, 0xcf, 0x2b, 0x4e, 0xc5, 0xa9, 0x34, 0x9c, 0x8b, 0x17, + 0x55, 0xa1, 0x08, 0x17, 0x2b, 0xd8, 0x39, 0x60, 0x75, 0x3c, 0x41, 0x10, 0x8e, 0x90, 0x01, 0x17, + 0x5b, 0x72, 0x7e, 0x6e, 0x6e, 0x66, 0x89, 0x04, 0x93, 0x02, 0xa3, 0x06, 0xb7, 0x91, 0x84, 0x1e, + 0x92, 0xf3, 0x21, 0x4e, 0x74, 0x06, 0xcb, 0x07, 0x41, 0xd5, 0x29, 0xf1, 0x73, 0xf1, 0x06, 0x97, + 0x24, 0x96, 0x94, 0x16, 0x43, 0x1d, 0xab, 0x64, 0xc3, 0xc5, 0x07, 0x13, 0xc0, 0xef, 0x26, 0x21, + 0x21, 0x2e, 0x96, 0xa4, 0xc4, 0xe2, 0x54, 0xb0, 0x55, 0xcc, 0x41, 0x60, 0xb6, 0x53, 0xd0, 0x89, + 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, + 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e, 0xcb, 0x31, 0x44, 0x59, 0xa4, 0x67, 0x96, 0x64, 0x94, 0x26, + 0xe9, 0x25, 0xe7, 0xe7, 0xea, 0xa7, 0x24, 0x16, 0x67, 0x14, 0x24, 0x56, 0xea, 0x43, 0x1c, 0x07, + 0xe2, 0xe9, 0x83, 0x43, 0x4e, 0x1f, 0x5b, 0x74, 0x24, 0xb1, 0x81, 0xe5, 0x8c, 0x01, 0x03, 0x00, + 0x64, 0x5c, 0x54, 0x1e, 0xad, 0x01, 0x00, 0x00, } func (m *BlockRequest) Marshal() (dAtA []byte, err error) { @@ -459,13 +380,6 @@ func (m *BlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.PartLayout) > 0 { - i -= len(m.PartLayout) - copy(dAtA[i:], m.PartLayout) - i = encodeVarintTypes(dAtA, i, uint64(len(m.PartLayout))) - i-- - dAtA[i] = 0x1a - } if m.Commit != nil { { size, err := m.Commit.MarshalToSizedBuffer(dAtA[:i]) @@ -488,64 +402,6 @@ func (m *BlockResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *BlockPartLayout) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *BlockPartLayout) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *BlockPartLayout) MarshalToSizedBuffer(dAtA []byte) (int, error) { - if m == nil { - return 0, nil - } - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Proofs) > 0 { - for iNdEx := len(m.Proofs) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Proofs[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintTypes(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } - if len(m.Sizes) > 0 { - dAtA3 := make([]byte, len(m.Sizes)*10) - var j2 int - for _, num := range m.Sizes { - for num >= 1<<7 { - dAtA3[j2] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j2++ - } - dAtA3[j2] = uint8(num) - j2++ - } - i -= j2 - copy(dAtA[i:], dAtA3[:j2]) - i = encodeVarintTypes(dAtA, i, uint64(j2)) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - func (m *StatusRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -657,32 +513,6 @@ func (m *BlockResponse) Size() (n int) { l = m.Commit.Size() n += 1 + l + sovTypes(uint64(l)) } - l = len(m.PartLayout) - if l > 0 { - n += 1 + l + sovTypes(uint64(l)) - } - return n -} - -func (m *BlockPartLayout) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if len(m.Sizes) > 0 { - l = 0 - for _, e := range m.Sizes { - l += sovTypes(uint64(e)) - } - n += 1 + sovTypes(uint64(l)) + l - } - if len(m.Proofs) > 0 { - for _, e := range m.Proofs { - l = e.Size() - n += 1 + l + sovTypes(uint64(l)) - } - } return n } @@ -953,200 +783,6 @@ func (m *BlockResponse) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field PartLayout", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthTypes - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthTypes - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.PartLayout = append(m.PartLayout[:0], dAtA[iNdEx:postIndex]...) - if m.PartLayout == nil { - m.PartLayout = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipTypes(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthTypes - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *BlockPartLayout) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: BlockPartLayout: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: BlockPartLayout: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType == 0 { - var v uint32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Sizes = append(m.Sizes, v) - } else if wireType == 2 { - var packedLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - packedLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if packedLen < 0 { - return ErrInvalidLengthTypes - } - postIndex := iNdEx + packedLen - if postIndex < 0 { - return ErrInvalidLengthTypes - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - var elementCount int - var count int - for _, integer := range dAtA[iNdEx:postIndex] { - if integer < 128 { - count++ - } - } - elementCount = count - if elementCount != 0 && len(m.Sizes) == 0 { - m.Sizes = make([]uint32, 0, elementCount) - } - for iNdEx < postIndex { - var v uint32 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - m.Sizes = append(m.Sizes, v) - } - } else { - return fmt.Errorf("proto: wrong wireType = %d for field Sizes", wireType) - } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Proofs", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTypes - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthTypes - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthTypes - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Proofs = append(m.Proofs, crypto.Proof{}) - if err := m.Proofs[len(m.Proofs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) diff --git a/proto/tendermint/blocksync/types.proto b/proto/tendermint/blocksync/types.proto index 9055f81af1..5dc218bfd2 100644 --- a/proto/tendermint/blocksync/types.proto +++ b/proto/tendermint/blocksync/types.proto @@ -3,8 +3,6 @@ package tendermint.blocksync; option go_package = "github.com/dashpay/tenderdash/proto/tendermint/blocksync"; -import "gogoproto/gogo.proto"; -import "tendermint/crypto/proof.proto"; import "tendermint/types/types.proto"; // BlockRequest requests a block for a specific height @@ -21,30 +19,14 @@ message NoBlockResponse { // BlockResponse returns block to the requested message BlockResponse { // block is a serialized tendermint.types.Block: the bytes of the parts it - // was committed with, in order. The commit's part set header is computed - // over those bytes, and decoding and re-encoding the block can change them - // (unknown fields, non-canonical encoding). On the wire this is the same as - // an embedded tendermint.types.Block, the type of this field in older - // releases. + // was committed with, in order. The receiver cuts them into parts of 64kB, + // as the proposer did, and checks the part set header it computes against + // the commit's. Decoding and re-encoding the block can change its bytes + // (unknown fields, non-canonical encoding), so it is served as stored. On + // the wire this is the same as an embedded tendermint.types.Block, the type + // of this field in older releases. bytes block = 1; tendermint.types.Commit commit = 2; - // part_layout is a serialized BlockPartLayout. It is empty when the block was - // cut into parts of 64kB, the last one shorter, the way honest proposers cut - // it. It is opaque here because every received message is decoded before it - // is matched to a request: the layout is only decoded once it is known to be - // the response to one, and within bounds. - bytes part_layout = 3; -} - -// BlockPartLayout describes the parts a block was committed with, when they -// cannot be derived from the block's bytes. -message BlockPartLayout { - // sizes is the size of each part, in order. - repeated uint32 sizes = 1; - // proofs is the Merkle proof of each part, in order. It is empty when the - // part set header can be computed from the parts, which it cannot when a - // proof's total differs from the header's. - repeated tendermint.crypto.Proof proofs = 2 [(gogoproto.nullable) = false]; } // StatusRequest requests the status of a peer. diff --git a/spec/p2p/messages/block-sync.md b/spec/p2p/messages/block-sync.md index 83977549fd..7d9dded873 100644 --- a/spec/p2p/messages/block-sync.md +++ b/spec/p2p/messages/block-sync.md @@ -36,26 +36,15 @@ NoBlockResponse notifies the peer requesting a block that the node does not cont BlockResponse contains the block requested and its commit. -| Name | Type | Description | Field Number | -|------------|------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| -| Block | bytes | Requested [Block](../../core/data_structures.md#block), serialized as it was committed: the bytes of its parts, in order. On the wire this is the same as an embedded Block. | 1 | -| Commit | [Commit](../../core/data_structures.md#commit) | Commit for the requested block | 2 | -| PartLayout | bytes | Serialized BlockPartLayout. Empty when the block was cut into parts of 64kB, the last one shorter | 3 | - -The receiver rebuilds the block's part set from `Block` and `PartLayout`: the -part set header in the commit's block ID is computed over the committed parts, -and decoding and re-encoding the block can change its bytes. - -### BlockPartLayout - -BlockPartLayout describes the parts a block was committed with, when they cannot -be derived from the block's bytes. It is carried as opaque bytes so that it is -only decoded for a response to a request, after bounds checks on its encoding. - -| Name | Type | Description | Field Number | -|--------|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| -| Sizes | repeated uint32 | Size of each part, in order | 1 | -| Proofs | repeated [Proof](../../core/data_structures.md#proof) | Merkle proof of each part, in order. Empty when the part set header can be computed from the parts, which it cannot when a proof's total differs from the header's | 2 | +| Name | Type | Description | Field Number | +|--------|------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------| +| Block | bytes | Requested [Block](../../core/data_structures.md#block), serialized as it was committed: the bytes of its parts, in order. On the wire this is the same as an embedded Block. | 1 | +| Commit | [Commit](../../core/data_structures.md#commit) | Commit for the requested block | 2 | + +The receiver cuts `Block` into parts of 64kB, the last one shorter, as the +proposer did, and checks the part set header it computes against the one in the +commit's block ID. `Block` is served as stored, because decoding and re-encoding +the block can change its bytes. ### StatusRequest diff --git a/types/part_set.go b/types/part_set.go index 2f999f8e40..74e504c3fc 100644 --- a/types/part_set.go +++ b/types/part_set.go @@ -181,67 +181,16 @@ type PartSet struct { func NewPartSetFromData(data []byte, partSize uint32) *PartSet { // divide data into 4kb parts. total := (uint32(len(data)) + partSize - 1) / partSize - partsBytes := make([][]byte, total) - for i := uint32(0); i < total; i++ { - partsBytes[i] = data[i*partSize : tmmath.MinInt(len(data), int((i+1)*partSize))] - } - return newPartSetFromParts(partsBytes, len(data)) -} - -// NewPartSetFromPartSizes returns an immutable, full PartSet from the data -// bytes, cut into consecutive parts of the given sizes. The part set header -// depends on where the data is cut, and a proposer may cut a block other than -// NewPartSetFromData would, so this rebuilds the part set a block was -// committed with from its bytes and the sizes of its parts. -func NewPartSetFromPartSizes(data []byte, sizes []uint32) (*PartSet, error) { - if err := ValidatePartSizes(len(data), sizes); err != nil { - return nil, err - } - partsBytes := make([][]byte, len(sizes)) - offset := 0 - for i, size := range sizes { - partsBytes[i] = data[offset : offset+int(size)] - offset += int(size) - } - return newPartSetFromParts(partsBytes, len(data)), nil -} - -// ValidatePartSizes checks that parts of the given sizes are valid block parts -// that together cut exactly size bytes. -func ValidatePartSizes(size int, sizes []uint32) error { - if len(sizes) == 0 { - return errors.New("no part sizes") - } - if len(sizes) > int(MaxBlockPartsCount) { - return fmt.Errorf("too many parts: %d, max: %d", len(sizes), MaxBlockPartsCount) - } - offset := 0 - for i, partSize := range sizes { - if partSize > BlockPartSizeBytes { - return fmt.Errorf("part %d too big: %d bytes, max: %d", i, partSize, BlockPartSizeBytes) - } - if int(partSize) > size-offset { - return fmt.Errorf("part sizes exceed the %d bytes of data", size) - } - offset += int(partSize) - } - if offset != size { - return fmt.Errorf("part sizes cover %d of the %d bytes of data", offset, size) - } - return nil -} - -// newPartSetFromParts returns a full PartSet of the given parts, which must be -// consecutive slices of data byteSize bytes long. -func newPartSetFromParts(partsBytes [][]byte, byteSize int) *PartSet { - total := tmmath.MustConvertUint32(len(partsBytes)) parts := make([]*Part, total) + partsBytes := make([][]byte, total) partsBitArray := bits.NewBitArray(int(total)) for i := uint32(0); i < total; i++ { - parts[i] = &Part{ + part := &Part{ Index: i, - Bytes: partsBytes[i], + Bytes: data[i*partSize : tmmath.MinInt(len(data), int((i+1)*partSize))], } + parts[i] = part + partsBytes[i] = part.Bytes partsBitArray.SetIndex(int(i), true) } // Compute merkle proofs @@ -255,7 +204,7 @@ func newPartSetFromParts(partsBytes [][]byte, byteSize int) *PartSet { parts: parts, partsBitArray: partsBitArray, count: total, - byteSize: int64(byteSize), + byteSize: int64(len(data)), } } diff --git a/types/part_set_test.go b/types/part_set_test.go index f05bf58e7b..0e3ad11f11 100644 --- a/types/part_set_test.go +++ b/types/part_set_test.go @@ -301,54 +301,3 @@ func TestPartIndexProofBinding(t *testing.T) { require.False(t, added) require.ErrorIs(t, err, ErrPartSetUnexpectedIndex) } - -func TestNewPartSetFromPartSizes(t *testing.T) { - data := tmrand.Bytes(2*int(BlockPartSizeBytes) + 100) - - t.Run("the standard cut", func(t *testing.T) { - partSet, err := NewPartSetFromPartSizes(data, []uint32{BlockPartSizeBytes, BlockPartSizeBytes, 100}) - require.NoError(t, err) - assert.Equal(t, NewPartSetFromData(data, BlockPartSizeBytes).Header(), partSet.Header()) - }) - - t.Run("another cut", func(t *testing.T) { - sizes := []uint32{100, BlockPartSizeBytes, 0, BlockPartSizeBytes} - partSet, err := NewPartSetFromPartSizes(data, sizes) - require.NoError(t, err) - assert.True(t, partSet.IsComplete()) - assert.EqualValues(t, len(sizes), partSet.Total()) - assert.EqualValues(t, len(data), partSet.ByteSize()) - assert.NotEqual(t, NewPartSetFromData(data, BlockPartSizeBytes).Header(), partSet.Header()) - - // the parts verify against the header, as a part set gossiped by a - // proposer that cut the data this way would - received := NewPartSetFromHeader(partSet.Header()) - for i, size := range sizes { - part := partSet.GetPart(i) - assert.Len(t, part.Bytes, int(size)) - added, err := received.AddPart(part) - require.NoError(t, err) - require.True(t, added) - } - got, err := io.ReadAll(received.GetReader()) - require.NoError(t, err) - assert.Equal(t, data, got) - }) - - testCases := map[string]struct { - sizes []uint32 - wantErr string - }{ - "no sizes": {nil, "no part sizes"}, - "too many parts": {make([]uint32, MaxBlockPartsCount+1), "too many parts"}, - "part too big": {[]uint32{BlockPartSizeBytes + 1, BlockPartSizeBytes - 1, 100}, "too big"}, - "sizes beyond end": {[]uint32{BlockPartSizeBytes, BlockPartSizeBytes, 101}, "exceed"}, - "sizes short": {[]uint32{BlockPartSizeBytes, BlockPartSizeBytes, 99}, "cover"}, - } - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - _, err := NewPartSetFromPartSizes(data, tc.sizes) - assert.ErrorContains(t, err, tc.wantErr) - }) - } -} From b73adb8429b5ab3f2aa17087eab322dd0cb695fe Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Tue, 10 Nov 2026 23:34:14 +0700 Subject: [PATCH 7/8] docs(blocksync): state the standard-cut limit and the backlog's heap cost Block sync rebuilds only the standard 64kB cut, while consensus still accepts other part sizes and proofs of a larger tree. The spec, the proto comment and the applier now say so: a block committed with another cut fails the commit check whichever peer serves it. The backlog ceiling is in serialized bytes, and a pending response holds its block twice, as received bytes and decoded. The comment now says the heap is at least twice the charged bytes, about 14 GB at the ceiling. Charging both copies would only change the budget term, not the in-flight term that dominates. The part set stage no longer serializes the block, so three comments that said it did are reworded. Co-Authored-By: Claude Opus 5.5 --- internal/blocksync/applier.go | 16 +++++++++------- internal/blocksync/synchronizer.go | 11 +++++++++-- internal/consensus/metrics.go | 4 ++-- proto/tendermint/blocksync/types.pb.go | 12 ++++++++---- proto/tendermint/blocksync/types.proto | 12 ++++++++---- spec/p2p/messages/block-sync.md | 14 ++++++++++---- 6 files changed, 46 insertions(+), 23 deletions(-) diff --git a/internal/blocksync/applier.go b/internal/blocksync/applier.go index 2e07e78910..b9abcabe8d 100644 --- a/internal/blocksync/applier.go +++ b/internal/blocksync/applier.go @@ -92,9 +92,11 @@ func (e *blockApplier) Apply( // The part set is needed twice: to derive the block ID and to persist the // block. Building it builds a Merkle tree over the parts, so build it once // and pass it to both. It is cut from the bytes the block was committed - // with, the way the proposer cut them, not from block: the commit's part - // set header is computed over those bytes, and a re-encoding of block drops - // unknown fields and non-canonical encoding. + // with, not from block: the commit's part set header is computed over those + // bytes, and a re-encoding of block drops unknown fields and non-canonical + // encoding. Only the standard cut is rebuilt, the one honest proposers use. + // Consensus does not require it yet, and a block committed with another cut + // fails the commit check whichever peer serves it. start := time.Now() blockParts := types.NewPartSetFromData(blockBytes, types.BlockPartSizeBytes) blockID := block.BlockID(blockParts) @@ -144,8 +146,8 @@ func (e *blockApplier) Apply( e.metrics.ObserveBlockSyncStage("exec", execTime) e.stats.add(partSetTime, verifyTime, saveTime, execTime) - // ByteSize is the size of the serialized block we just built, so the metric - // costs nothing extra here + // ByteSize is the length of the bytes the part set was cut from, so the + // metric costs nothing extra here e.metrics.RecordConsMetrics(block, blockParts.ByteSize()) return nil } @@ -238,8 +240,8 @@ type applyTimings struct { // applyStats accumulates per-stage timings of the block apply pipeline. Block // sync applies blocks one at a time, so this is where a slow sync shows up; the -// breakdown attributes it to serialization, signature verification, disk or the -// ABCI application without needing a profiler. +// breakdown attributes it to building the part set, signature verification, +// disk or the ABCI application without needing a profiler. type applyStats struct { mtx sync.Mutex blocks int64 diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index 07f6c2ef01..c2492f564c 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -82,6 +82,13 @@ const ( // unconditional, about 6.98 GB, and configuring a chain for smaller blocks // does not lower it. // + // That figure, like the budget, is in serialized bytes, the size charged + // for each response. A response holds its block twice, as the bytes it was + // committed with and decoded, so the heap it occupies is at least twice + // what it is charged: about 14 GB at this ceiling. Charging both copies + // would only change the first term, since the second is in flight before + // the budget is consulted. + // // Lowering maxOutstandingHeights does, in proportion, and it is the only // lever on it here. 64 is the balance struck: holding that term to the // budget itself would mean a window of two, and even a 1 GiB allowance for @@ -679,8 +686,8 @@ func (s *Synchronizer) updateMonitor() { "blocks/s", syncRate, } // blocks are applied one at a time, so the per-stage averages say whether the - // rate above is limited by serialization, signature verification, disk or the - // ABCI application + // rate above is limited by building the part set, signature verification, + // disk or the ABCI application if timings, measured := s.applier.Timings(); measured { // logged as durations rather than whole milliseconds: part set building // and commit verification are often sub-millisecond, and truncating them diff --git a/internal/consensus/metrics.go b/internal/consensus/metrics.go index 3c72d5951c..3919172b1c 100644 --- a/internal/consensus/metrics.go +++ b/internal/consensus/metrics.go @@ -229,8 +229,8 @@ func (m *Metrics) ObserveBlockSyncStage(stage string, d time.Duration) { // RecordConsMetrics uses for recording the block related metrics during fast-sync. // // blockSize is the size of the serialized block in bytes. It is passed in rather -// than derived from block, because callers on the block sync path have already -// serialized the block and recomputing it there is expensive. +// than derived from block, because callers on the block sync path already hold +// the serialized block and serializing it again there is expensive. func (m *Metrics) RecordConsMetrics(block *types.Block, blockSize int64) { m.NumTxs.Set(float64(len(block.Data.Txs))) m.TotalTxs.Add(float64(len(block.Data.Txs))) diff --git a/proto/tendermint/blocksync/types.pb.go b/proto/tendermint/blocksync/types.pb.go index 9e174eb89e..91538985f6 100644 --- a/proto/tendermint/blocksync/types.pb.go +++ b/proto/tendermint/blocksync/types.pb.go @@ -118,11 +118,15 @@ func (m *NoBlockResponse) GetHeight() int64 { type BlockResponse struct { // block is a serialized tendermint.types.Block: the bytes of the parts it // was committed with, in order. The receiver cuts them into parts of 64kB, - // as the proposer did, and checks the part set header it computes against + // the last one shorter, and checks the part set header it computes against // the commit's. Decoding and re-encoding the block can change its bytes - // (unknown fields, non-canonical encoding), so it is served as stored. On - // the wire this is the same as an embedded tendermint.types.Block, the type - // of this field in older releases. + // (unknown fields, non-canonical encoding), so it is served as stored. Only + // that standard cut, the one honest proposers use, is rebuilt. Consensus + // does not require it yet: it accepts parts of other sizes and proofs of a + // tree larger than the header counts, and a block committed that way fails + // the commit check whichever peer serves it. On the wire this is the same + // as an embedded tendermint.types.Block, the type of this field in older + // releases. Block []byte `protobuf:"bytes,1,opt,name=block,proto3" json:"block,omitempty"` Commit *types.Commit `protobuf:"bytes,2,opt,name=commit,proto3" json:"commit,omitempty"` } diff --git a/proto/tendermint/blocksync/types.proto b/proto/tendermint/blocksync/types.proto index 5dc218bfd2..bc8d05a6ca 100644 --- a/proto/tendermint/blocksync/types.proto +++ b/proto/tendermint/blocksync/types.proto @@ -20,11 +20,15 @@ message NoBlockResponse { message BlockResponse { // block is a serialized tendermint.types.Block: the bytes of the parts it // was committed with, in order. The receiver cuts them into parts of 64kB, - // as the proposer did, and checks the part set header it computes against + // the last one shorter, and checks the part set header it computes against // the commit's. Decoding and re-encoding the block can change its bytes - // (unknown fields, non-canonical encoding), so it is served as stored. On - // the wire this is the same as an embedded tendermint.types.Block, the type - // of this field in older releases. + // (unknown fields, non-canonical encoding), so it is served as stored. Only + // that standard cut, the one honest proposers use, is rebuilt. Consensus + // does not require it yet: it accepts parts of other sizes and proofs of a + // tree larger than the header counts, and a block committed that way fails + // the commit check whichever peer serves it. On the wire this is the same + // as an embedded tendermint.types.Block, the type of this field in older + // releases. bytes block = 1; tendermint.types.Commit commit = 2; } diff --git a/spec/p2p/messages/block-sync.md b/spec/p2p/messages/block-sync.md index 7d9dded873..3faeb53072 100644 --- a/spec/p2p/messages/block-sync.md +++ b/spec/p2p/messages/block-sync.md @@ -41,10 +41,16 @@ BlockResponse contains the block requested and its commit. | Block | bytes | Requested [Block](../../core/data_structures.md#block), serialized as it was committed: the bytes of its parts, in order. On the wire this is the same as an embedded Block. | 1 | | Commit | [Commit](../../core/data_structures.md#commit) | Commit for the requested block | 2 | -The receiver cuts `Block` into parts of 64kB, the last one shorter, as the -proposer did, and checks the part set header it computes against the one in the -commit's block ID. `Block` is served as stored, because decoding and re-encoding -the block can change its bytes. +The receiver cuts `Block` into parts of 64kB, the last one shorter, and checks +the part set header it computes against the one in the commit's block ID. +`Block` is served as stored, because decoding and re-encoding the block can +change its bytes. + +Block sync rebuilds only this standard cut, which honest proposers use. +Consensus does not require it yet: it accepts parts of other sizes, and proofs +of a tree larger than the header counts. A block committed that way fails the +commit check (`wrong block ID`) whichever peer serves it, and block sync does +not get past its height. ### StatusRequest From abfa42be14c81787071ca37a835b48155f509f98 Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Wed, 11 Nov 2026 00:40:20 +0700 Subject: [PATCH 8/8] fix(blocksync): hold pending blocks only as their committed bytes A block response kept both the committed bytes and the decoded block while it waited behind a missing lower height, but was charged to the backlog for its bytes only. That doubled what the in-order backlog can retain, to about 14 GB at its ceiling, against what dev held. A response now keeps the block's height and bytes. The block is still decoded on arrival, so a block that does not decode is refused before it is queued, and is decoded again when it is applied: the new blockApplier.ApplyBytes decodes it under the applier lock and times it as the "decode" stage, so decoding does not count as waiting. A pending response so holds one copy of its block, the one it is charged for. TestBlockResponseRetainsOnlyItsBytes holds 16 responses of a block of small transactions and checks what they retain beyond their bytes. With the decoded block kept as well it retains 23 MB more, against about 10 KB without. Co-Authored-By: Claude Opus 5.5 --- docs/nodes/metrics.md | 2 +- internal/blocksync/applier.go | 44 ++++++++++-- internal/blocksync/block_fetch_job.go | 4 +- internal/blocksync/block_fetch_job_test.go | 19 +++-- internal/blocksync/p2p_msg_handler_test.go | 60 +++++++++++++++- internal/blocksync/synchronizer.go | 84 +++++++++++----------- internal/blocksync/synchronizer_test.go | 78 +++++++++++--------- 7 files changed, 199 insertions(+), 92 deletions(-) diff --git a/docs/nodes/metrics.md b/docs/nodes/metrics.md index 564ed762a2..bd2da18230 100644 --- a/docs/nodes/metrics.md +++ b/docs/nodes/metrics.md @@ -43,7 +43,7 @@ The following metrics are available: | consensus_step_duration | Histogram | step | Histogram of durations for each step in the consensus protocol | | consensus_block_gossip_receive_latency | Histogram | | Histogram of time taken to receive a block in seconds, measure between when a new block is first discovered to when the block is completed | | consensus_block_gossip_parts_received | Counter | matches_current | Number of block parts received by the node | -| consensus_block_sync_apply_stage_duration | Histogram | stage | Time a block spent in each stage of the block sync applier, in ms: partset, verify_commit, verify_block, save, exec, and wait (idle time between blocks) | +| consensus_block_sync_apply_stage_duration | Histogram | stage | Time a block spent in each stage of the block sync applier, in ms: decode, partset, verify_commit, verify_block, save, exec, and wait (idle time between blocks) | | consensus_quorum_prevote_delay | Gauge | | Interval in seconds between the proposal timestamp and the timestamp of the earliest prevote that achieved a quorum | | consensus_full_prevote_delay | Gauge | | Interval in seconds between the proposal timestamp and the timestamp of the latest prevote in a round where all validators voted | | consensus_proposal_timestamp_difference | Histogram | | Difference between the timestamp in the proposal message and the local time of the validator at the time it received the message | diff --git a/internal/blocksync/applier.go b/internal/blocksync/applier.go index b9abcabe8d..ace4593a27 100644 --- a/internal/blocksync/applier.go +++ b/internal/blocksync/applier.go @@ -80,15 +80,48 @@ func (e *blockApplier) Apply( ) error { e.mtx.Lock() defer e.mtx.Unlock() + e.observeWait() + defer func() { e.lastDone = time.Now() }() + return e.apply(ctx, block, blockBytes, commit, 0) +} + +// ApplyBytes is Apply for a block held only as the bytes it was committed with: +// it decodes the block from them first. Block sync keeps pending blocks in that +// form, so that a response waiting for a lower height does not hold a decoded +// copy beside its bytes. +func (e *blockApplier) ApplyBytes(ctx context.Context, blockBytes []byte, commit *types.Commit) error { + e.mtx.Lock() + defer e.mtx.Unlock() + e.observeWait() defer func() { e.lastDone = time.Now() }() - // Time between the end of the previous apply and the start of this one. With - // a fast application this is what the sync rate is actually limited by, and - // it is block fetching, not block execution. + start := time.Now() + block, err := decodeBlock(blockBytes) + decodeTime := e.observeSince("decode", start) + if err != nil { + return err + } + return e.apply(ctx, block, blockBytes, commit, decodeTime) +} + +// observeWait records the time between the end of the previous apply and the +// start of this one. With a fast application this is what the sync rate is +// actually limited by, and it is block fetching, not block execution. +func (e *blockApplier) observeWait() { if !e.lastDone.IsZero() { e.observeSince("wait", e.lastDone) } +} +// apply is Apply with the lock held. decodeTime is how long decoding block +// took, counted with building its part set in the per-block timings. +func (e *blockApplier) apply( + ctx context.Context, + block *types.Block, + blockBytes []byte, + commit *types.Commit, + decodeTime time.Duration, +) error { // The part set is needed twice: to derive the block ID and to persist the // block. Building it builds a Merkle tree over the parts, so build it once // and pass it to both. It is cut from the bytes the block was committed @@ -100,7 +133,7 @@ func (e *blockApplier) Apply( start := time.Now() blockParts := types.NewPartSetFromData(blockBytes, types.BlockPartSizeBytes) blockID := block.BlockID(blockParts) - partSetTime := e.observeSince("partset", start) + partSetTime := decodeTime + e.observeSince("partset", start) start = time.Now() verified, err := e.verify(ctx, blockID, block, commit) @@ -228,7 +261,8 @@ func (e *blockApplier) observeSince(stage string, start time.Time) time.Duration // applyTimings is the average time a single block spends in each stage of the // apply pipeline. type applyTimings struct { - // PartSet is the Merkle tree over the block's parts + // PartSet is decoding the block, when it is held as bytes, and the Merkle + // tree over its parts PartSet time.Duration // Verify is commit signature verification and block validation Verify time.Duration diff --git a/internal/blocksync/block_fetch_job.go b/internal/blocksync/block_fetch_job.go index 7c58f52555..1a88fb4d13 100644 --- a/internal/blocksync/block_fetch_job.go +++ b/internal/blocksync/block_fetch_job.go @@ -54,9 +54,9 @@ func blockFetchJobHandler(blockClient client.BlockClient, peer PeerData, height if err != nil { return errorResult(responder, height, err) } - if resp.Block.Height != height { + if resp.Height != height { return errorResult(responder, height, - fmt.Errorf("peer sent block at height %d, requested height %d", resp.Block.Height, height)) + fmt.Errorf("peer sent block at height %d, requested height %d", resp.Height, height)) } resp.ConnID = responder.connID size = protoResp.Size() diff --git a/internal/blocksync/block_fetch_job_test.go b/internal/blocksync/block_fetch_job_test.go index 17bbb23859..d1aa6ba94e 100644 --- a/internal/blocksync/block_fetch_job_test.go +++ b/internal/blocksync/block_fetch_job_test.go @@ -159,8 +159,13 @@ func TestAttributedConn(t *testing.T) { } func TestBlockResponseValidate(t *testing.T) { - block := func(h int64) *types.Block { - return &types.Block{Header: types.Header{Height: h}} + // a block at height h, as a response holds it + block := func(h int64) BlockResponse { + return BlockResponse{Height: h, BlockBytes: blockBytes(t, &types.Block{Header: types.Header{Height: h}})} + } + withCommit := func(resp BlockResponse, commit *types.Commit) BlockResponse { + resp.Commit = commit + return resp } commit := func(h int64) *types.Commit { return &types.Commit{Height: h} @@ -182,17 +187,17 @@ func TestBlockResponseValidate(t *testing.T) { }, { name: "block, nil commit", - resp: BlockResponse{Block: block(10)}, + resp: block(10), wantErr: "a block without a commit at height 10", }, { name: "height mismatch", - resp: BlockResponse{Block: block(10), Commit: commit(11)}, + resp: withCommit(block(10), commit(11)), wantErr: "heights don't match", }, { name: "valid", - resp: BlockResponse{Block: block(10), Commit: commit(10)}, + resp: withCommit(block(10), commit(10)), }, } for _, tc := range testCases { @@ -269,8 +274,8 @@ func (suite *BlockFetchJobTestSuite) TestGeneratorNextJobWaitForPeerAndPushBackH resp2 := res2.Value.(*BlockResponse) return suite.Equal(suite.peer.peerID, resp1.PeerID) && suite.Equal(suite.peer.peerID, resp2.PeerID) && - suite.Contains([]int64{5, 9}, resp1.Block.Height) && - suite.Contains([]int64{5, 9}, resp2.Block.Height) + suite.Contains([]int64{5, 9}, resp1.Height) && + suite.Contains([]int64{5, 9}, resp2.Height) }, 10*time.Millisecond, 5*time.Millisecond) } diff --git a/internal/blocksync/p2p_msg_handler_test.go b/internal/blocksync/p2p_msg_handler_test.go index 547cdf771a..2bd98dcb44 100644 --- a/internal/blocksync/p2p_msg_handler_test.go +++ b/internal/blocksync/p2p_msg_handler_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/binary" "fmt" + "runtime" + "slices" "testing" "time" @@ -580,7 +582,7 @@ func TestServeBlockAsCommitted(t *testing.T) { resp, err := BlockResponseFromProto(msg, "peer") require.NoError(t, err) require.Equal(t, tc.bytes, resp.BlockBytes) - require.Equal(t, blocks[i].Hash(), resp.Block.Hash()) + require.Equal(t, blocks[i].Height, resp.Height) require.Equal(t, blocks[i+1].LastCommit.BlockID, resp.Commit.BlockID) cut := types.NewPartSetFromData(resp.BlockBytes, types.BlockPartSizeBytes) require.Equal(t, tc.standardCut, tc.parts.Header().Equals(cut.Header())) @@ -616,8 +618,10 @@ func TestBlockResponseWireCompatible(t *testing.T) { require.NoError(t, proto.Unmarshal(older, &decoded)) resp, err := BlockResponseFromProto(&decoded, "peer") require.NoError(t, err) - require.Equal(t, block.Hash(), resp.Block.Hash()) require.Equal(t, embedded, resp.BlockBytes) + decodedBlock, err := decodeBlock(resp.BlockBytes) + require.NoError(t, err) + require.Equal(t, block.Hash(), decodedBlock.Hash()) } // appendBytesField appends a length-delimited field to serialized protobuf. @@ -651,3 +655,55 @@ func TestBlockResponseDoesNotRetainOverwrittenBlockField(t *testing.T) { require.Equal(t, len(bz), resp.Size) require.LessOrEqual(t, cap(resp.BlockBytes), 2*len(bz)) } + +// TestBlockResponseRetainsOnlyItsBytes checks that a block response, as it +// waits in the backlog for a lower height, holds its block only as the bytes it +// is charged for there, and not also decoded. A block of many small +// transactions costs more decoded than its bytes, so holding both would retain +// more than twice the charge. +func TestBlockResponseRetainsOnlyItsBytes(t *testing.T) { + const ( + txs = 1 << 14 + txSize = 64 + responses = 16 + ) + blockTxs := make([]types.Tx, txs) + for i := range blockTxs { + blockTxs[i] = make(types.Tx, txSize) + binary.BigEndian.PutUint64(blockTxs[i], uint64(i)) + } + valSet, privVals := factory.MockValidatorSet() + state := fakeInitialState(valSet) + block := statefactory.MakeBlocks(context.Background(), t, 1, &state, privVals, 1)[0] + block.Data = types.Data{Txs: blockTxs} + block.DataHash = block.Data.Hash() + bz := blockBytes(t, block) + + heapAlloc := func() uint64 { + runtime.GC() + var stats runtime.MemStats + runtime.ReadMemStats(&stats) + return stats.HeapAlloc + } + inputs := make([][]byte, responses) + for i := range inputs { + inputs[i] = slices.Clone(bz) + } + before := heapAlloc() + held := make([]*BlockResponse, responses) + charged := 0 + for i := range held { + resp, err := BlockResponseFromProto(&bcproto.BlockResponse{Block: inputs[i]}, "peer") + require.NoError(t, err) + held[i] = resp + charged += resp.Size + } + // the inputs, allocated before the first reading, are the bytes the + // responses hold, so what is retained here is only what they keep beyond + // those bytes + retained := int64(heapAlloc()) - int64(before) + runtime.KeepAlive(held) + t.Logf("charged %d bytes, retained %d more", charged, retained) + require.Less(t, retained, int64(charged)/2, + "a pending response must not keep a decoded copy of its block") +} diff --git a/internal/blocksync/synchronizer.go b/internal/blocksync/synchronizer.go index c2492f564c..473b37c0cc 100644 --- a/internal/blocksync/synchronizer.go +++ b/internal/blocksync/synchronizer.go @@ -46,12 +46,12 @@ const ( // costs once blocks are large, because a count of blocks says nothing about // that when one block may be tens of megabytes. // - // Serialized bytes understate the heap actually retained. The size covers the - // block alone, not the commit held alongside it, and a response holds the - // decoded block next to its wire form, the bytes it is applied from. A - // decoded block costs more than its wire form - several times more for - // blocks small enough that fixed per-response overhead dominates. Read this - // as a budget for block data, not as the memory the backlog occupies. + // A pending response holds its block only as those bytes, the form it is + // applied from, and not decoded as well: the block is decoded again when it + // is applied. The size still understates the heap retained, since it covers + // the block alone, not the commit held alongside it or the per-response + // overhead. Read this as a budget for block data, not as the memory the + // backlog occupies. maxPendingApplyBytes = 256 << 20 // maxOutstandingHeights is how many heights may be requested and not yet @@ -82,13 +82,6 @@ const ( // unconditional, about 6.98 GB, and configuring a chain for smaller blocks // does not lower it. // - // That figure, like the budget, is in serialized bytes, the size charged - // for each response. A response holds its block twice, as the bytes it was - // committed with and decoded, so the heap it occupies is at least twice - // what it is charged: about 14 GB at this ceiling. Charging both copies - // would only change the first term, since the second is in flight before - // the budget is consulted. - // // Lowering maxOutstandingHeights does, in proportion, and it is the only // lever on it here. 64 is the balance struck: holding that term to the // budget itself would mean a window of two, and even a 1 GiB allowance for @@ -368,7 +361,7 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { if err != nil { if !errors.Is(err, errDuplicateBlock) { s.logger.Error("cannot add a block to the pending list", - "height", resp.Block.Height, + "height", resp.Height, "error", err.Error()) _ = s.client.Send(ctx, p2p.PeerError{NodeID: resp.PeerID, Err: err, ConnID: resp.ConnID}) return nil @@ -377,7 +370,7 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { // this height. Info level: a healthy sync produces none, so this is the only // signal an operator gets that heights are being re-requested. s.logger.Info("dropping duplicate block response", - "height", resp.Block.Height, + "height", resp.Height, "peer", resp.PeerID, "reason", err.Error()) } @@ -391,7 +384,7 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { // Charge the failure to the peer that supplied the failing block: removing it // also drops its pending entry, so the height is fetched from someone else. s.logger.Error("cannot apply a block", - "height", failed.Block.Height, + "height", failed.Height, "peer", failed.PeerID, "error", err.Error()) s.RemovePeer(failed.PeerID) @@ -402,8 +395,7 @@ func (s *Synchronizer) consumeJobResult(ctx, applyCtx context.Context) error { // backlog reports what block sync is holding: the height it is waiting to // apply, which is the lowest height not applied yet, and the serialized size of -// the responses held above it. That size is what the blocks measured on the -// wire, which is less than what holding them decoded costs; see +// the responses held above it, which hold their blocks as those bytes; see // maxPendingApplyBytes. Both are read under one lock, so the pair describes a // single state and a job cannot be admitted against a height from one moment // and a backlog from another. @@ -648,7 +640,7 @@ func (s *Synchronizer) applyBlock(ctx context.Context) (BlockResponse, error) { if !ok { return BlockResponse{}, nil } - err := s.applier.Apply(ctx, resp.Block, resp.BlockBytes, resp.Commit) + err := s.applier.ApplyBytes(ctx, resp.BlockBytes, resp.Commit) if err != nil { return resp, fmt.Errorf("cannot apply block: %w", err) } @@ -742,16 +734,15 @@ func (s *Synchronizer) addBlock(resp BlockResponse) error { s.mtx.Lock() defer s.mtx.Unlock() - block := resp.Block - if block.Height < s.height { + if resp.Height < s.height { return fmt.Errorf("%w (peer: %s, block height: %d already applied)", - errDuplicateBlock, resp.PeerID, block.Height) + errDuplicateBlock, resp.PeerID, resp.Height) } - _, ok := s.pendingToApply[block.Height] + _, ok := s.pendingToApply[resp.Height] if ok { - return fmt.Errorf("%w (peer: %s, block height: %d)", errDuplicateBlock, resp.PeerID, block.Height) + return fmt.Errorf("%w (peer: %s, block height: %d)", errDuplicateBlock, resp.PeerID, resp.Height) } - s.pendingToApply[block.Height] = resp + s.pendingToApply[resp.Height] = resp return nil } @@ -786,28 +777,32 @@ type BlockResponse struct { // ConnID is the generation of the connection that delivered this response, which // its failures are fenced to. ConnID uint64 - Block *types.Block - // BlockBytes is Block as it was committed, the bytes Block is decoded from. - // Block is applied with the part set cut from these: that is what the - // commit's block ID describes, and re-encoding Block need not give the same - // bytes. + // Height is the height of the block in BlockBytes. + Height int64 + // BlockBytes is the block as it was committed. It is applied with the part + // set cut from these bytes, which is what the commit's block ID describes: + // re-encoding the decoded block need not give the same bytes. A response + // holds the block only in this form. It is decoded when it arrives, to be + // checked, and again when it is applied, so a response waiting for a lower + // height holds one copy of its block, the one it is charged for. BlockBytes []byte Commit *types.Commit - // Size is the serialized size of Block in bytes, the length of BlockBytes. + // Size is the length of BlockBytes, what the response is charged in the + // backlog. Size int } func (r *BlockResponse) Validate() error { - if r.Block == nil { + if len(r.BlockBytes) == 0 { return errors.New("block response without a block") } if r.Commit == nil { // See https://github.com/tendermint/tendermint/pull/8433#discussion_r866790631 - return fmt.Errorf("a block without a commit at height %d - possible node store corruption", r.Block.Height) + return fmt.Errorf("a block without a commit at height %d - possible node store corruption", r.Height) } - if r.Block.Height != r.Commit.Height { + if r.Height != r.Commit.Height { return fmt.Errorf("heights don't match, not adding block (block height: %d, commit height: %d)", - r.Block.Height, + r.Height, r.Commit.Height) } return nil @@ -826,11 +821,9 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* if cap(blockBytes) > 2*len(blockBytes) { blockBytes = slices.Clone(blockBytes) } - pbb := new(tmproto.Block) - if err := proto.Unmarshal(blockBytes, pbb); err != nil { - return nil, fmt.Errorf("cannot decode block: %w", err) - } - block, err := types.BlockFromProto(pbb) + // Decoded here so that a block that does not decode is refused before it is + // queued, and then dropped: see BlockResponse.BlockBytes. + block, err := decodeBlock(blockBytes) if err != nil { return nil, err } @@ -843,9 +836,18 @@ func BlockResponseFromProto(resp *bcproto.BlockResponse, peerID types.NodeID) (* } return &BlockResponse{ PeerID: peerID, - Block: block, + Height: block.Height, BlockBytes: blockBytes, Commit: commit, Size: len(blockBytes), }, nil } + +// decodeBlock decodes a serialized tendermint.types.Block. +func decodeBlock(blockBytes []byte) (*types.Block, error) { + pbb := new(tmproto.Block) + if err := proto.Unmarshal(blockBytes, pbb); err != nil { + return nil, fmt.Errorf("cannot decode block: %w", err) + } + return types.BlockFromProto(pbb) +} diff --git a/internal/blocksync/synchronizer_test.go b/internal/blocksync/synchronizer_test.go index 348379f547..ad7803c987 100644 --- a/internal/blocksync/synchronizer_test.go +++ b/internal/blocksync/synchronizer_test.go @@ -141,7 +141,7 @@ func (suite *SynchronizerTestSuite) assertHandoverWaitsForConsumer(beforeApply b paused := make(chan struct{}) release := make(chan struct{}) if beforeApply { - pool.pendingToApply[resp.Block.Height] = *resp + pool.pendingToApply[resp.Height] = *resp pool.logger = &handoverPauseLogger{Logger: log.NewNopLogger(), paused: paused, release: release} } suite.store.On("SaveBlock", mock.Anything, mock.Anything, mock.Anything).Once() @@ -154,7 +154,7 @@ func (suite *SynchronizerTestSuite) assertHandoverWaitsForConsumer(beforeApply b finalState := suite.initialState.Copy() finalState.LastBlockHeight = 1 suite.blockExec.On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, mock.Anything, - resp.Block, resp.Commit, mock.Anything). + sameBlock(suite.T(), resp), resp.Commit, mock.Anything). Run(func(args mock.Arguments) { finalizing <- args.Get(0).(context.Context) if !beforeApply { @@ -353,7 +353,7 @@ func (suite *SynchronizerTestSuite) TestConsumeJobResult() { Once(). Return(nil) suite.blockExec. - On("ValidateBlock", mock.Anything, mock.Anything, respH1.Block, mock.Anything). + On("ValidateBlock", mock.Anything, mock.Anything, sameBlock(suite.T(), respH1), mock.Anything). Once(). Return(nil) suite.blockExec. @@ -361,11 +361,11 @@ func (suite *SynchronizerTestSuite) TestConsumeJobResult() { Maybe(). Return(types.VerifiedCommit{}, nil) suite.blockExec. - On("ProcessProposal", mock.Anything, respH1.Block, mock.Anything, mock.Anything, true, mock.Anything). + On("ProcessProposal", mock.Anything, sameBlock(suite.T(), respH1), mock.Anything, mock.Anything, true, mock.Anything). Once(). Return(sm.CurrentRoundState{}, nil) suite.blockExec. - On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, mock.Anything, respH1.Block, respH1.Commit, mock.Anything). + On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, mock.Anything, sameBlock(suite.T(), respH1), respH1.Commit, mock.Anything). Once(). Return(sm.State{}, nil, nil) }, @@ -399,11 +399,11 @@ func (suite *SynchronizerTestSuite) TestConsumeJobResult() { result: workerpool.Result{Value: respH1}, wantPushBack: []int64{1, 2}, mockFn: func(pool *Synchronizer) { - pool.pendingToApply[2] = BlockResponse{PeerID: "peer 1", Block: respH2.Block} + pool.pendingToApply[2] = BlockResponse{PeerID: "peer 1", Height: respH2.Height, BlockBytes: respH2.BlockBytes} // VerifyCommit is covered by the Maybe expectation the earlier case // registered on this shared mock suite.blockExec. - On("ValidateBlock", mock.Anything, mock.Anything, respH1.Block, mock.Anything). + On("ValidateBlock", mock.Anything, mock.Anything, sameBlock(suite.T(), respH1), mock.Anything). Once(). Return(errors.New("invalid error")) suite.client. @@ -424,7 +424,7 @@ func (suite *SynchronizerTestSuite) TestConsumeJobResult() { // so a peer report fails this case. result: workerpool.Result{Value: duplicateH2}, mockFn: func(pool *Synchronizer) { - pool.pendingToApply[respH2.Block.Height] = *respH2 + pool.pendingToApply[respH2.Height] = *respH2 }, }, } @@ -519,7 +519,7 @@ func (suite *SynchronizerTestSuite) TestRemovePeer() { pool.AddPeer(peer) } for _, resp := range tc.responses { - pool.pendingToApply[resp.Block.Height] = *resp + pool.pendingToApply[resp.Height] = *resp } for _, height := range tc.initialPushBack { pool.jobGen.pushBack(height) @@ -566,11 +566,11 @@ func (suite *SynchronizerTestSuite) TestConsumeDuplicateThenDrain() { applier := newBlockApplier(suite.blockExec, suite.store, applierWithState(suite.initialState)) pool := NewSynchronizer(1, suite.client, applier, WithWorkerPool(wp)) // height 2 is already in hand and waiting on height 1 - pool.pendingToApply[respH2.Block.Height] = *respH2 + pool.pendingToApply[respH2.Height] = *respH2 resultCh <- workerpool.Result{Value: duplicateH2} suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) - suite.Require().Equal(peerID1, pool.pendingToApply[respH2.Block.Height].PeerID, + suite.Require().Equal(peerID1, pool.pendingToApply[respH2.Height].PeerID, "the response already held must be kept") suite.Require().Empty(pool.jobGen.pushedBack, "a duplicate must not re-queue the height") suite.client.AssertNotCalled(suite.T(), "Send", mock.Anything, mock.Anything) @@ -619,7 +619,7 @@ func (suite *SynchronizerTestSuite) TestDuplicateBlockIsVisibleAtDefaultLogLevel // Start above the duplicated height so nothing is applied and the duplicate is // all that is under test. pool := NewSynchronizer(2, suite.client, applier, WithWorkerPool(wp), WithLogger(logger)) - pool.pendingToApply[respH1.Block.Height] = *respH1 + pool.pendingToApply[respH1.Height] = *respH1 resultCh <- workerpool.Result{Value: duplicateH1} suite.Require().NoError(pool.consumeJobResult(ctx, ctx)) @@ -675,16 +675,16 @@ func (suite *SynchronizerTestSuite) TestPeerErrorsCarrySupplyingConnection() { poisoned, err := BlockResponseFromProto(suite.responses[0], peerID) suite.Require().NoError(err) poisoned.ConnID = connID - pool.pendingToApply[poisoned.Block.Height] = *poisoned + pool.pendingToApply[poisoned.Height] = *poisoned honest, err := BlockResponseFromProto(suite.responses[1], "honest peer") suite.Require().NoError(err) suite.blockExec. - On("VerifyCommit", mock.Anything, mock.Anything, poisoned.Block.Height, poisoned.Commit). + On("VerifyCommit", mock.Anything, mock.Anything, poisoned.Height, poisoned.Commit). Once(). Return(types.VerifiedCommit{}, nil) suite.blockExec. - On("ValidateBlock", mock.Anything, mock.Anything, poisoned.Block, mock.Anything). + On("ValidateBlock", mock.Anything, mock.Anything, sameBlock(suite.T(), poisoned), mock.Anything). Once(). Return(errors.New("invalid block")) expectReport() @@ -956,14 +956,14 @@ func (suite *SynchronizerTestSuite) TestApplyFailurePunishesSupplyingPeer() { pool := NewSynchronizer(1, suite.client, applier, WithWorkerPool(wp)) pool.AddPeer(newPeerData(poisonPeerID, 1, 100)) pool.AddPeer(newPeerData(honestPeerID, 1, 100)) - pool.pendingToApply[poisonH1.Block.Height] = *poisonH1 + pool.pendingToApply[poisonH1.Height] = *poisonH1 suite.blockExec. - On("VerifyCommit", mock.Anything, mock.Anything, poisonH1.Block.Height, poisonH1.Commit). + On("VerifyCommit", mock.Anything, mock.Anything, poisonH1.Height, poisonH1.Commit). Once(). Return(types.VerifiedCommit{}, nil) suite.blockExec. - On("ValidateBlock", mock.Anything, mock.Anything, poisonH1.Block, mock.Anything). + On("ValidateBlock", mock.Anything, mock.Anything, sameBlock(suite.T(), poisonH1), mock.Anything). Once(). Return(errors.New("invalid block")) suite.client. @@ -978,13 +978,13 @@ func (suite *SynchronizerTestSuite) TestApplyFailurePunishesSupplyingPeer() { // The poisoned height is dropped and re-requested from someone else, so the // synchronizer can still advance. - suite.Require().Equal([]int64{poisonH1.Block.Height}, pool.jobGen.pushedBack) - suite.Require().NotContains(pool.pendingToApply, poisonH1.Block.Height) + suite.Require().Equal([]int64{poisonH1.Height}, pool.jobGen.pushedBack) + suite.Require().NotContains(pool.pendingToApply, poisonH1.Height) _, found := pool.peerStore.Get(poisonPeerID) suite.Require().False(found) // The honest peer keeps both its response and its place in the peer store. - suite.Require().Contains(pool.pendingToApply, honestH2.Block.Height) + suite.Require().Contains(pool.pendingToApply, honestH2.Height) _, found = pool.peerStore.Get(honestPeerID) suite.Require().True(found) } @@ -999,7 +999,7 @@ func (suite *SynchronizerTestSuite) TestAddBlockDropsAlreadyAppliedHeight() { const currentHeight = int64(5) peerID := types.NodeID("peer 1") staleH1, _ := BlockResponseFromProto(suite.responses[0], peerID) - suite.Require().Less(staleH1.Block.Height, currentHeight) + suite.Require().Less(staleH1.Height, currentHeight) resultCh := make(chan workerpool.Result, 1) wp := workerpool.New(1, workerpool.WithResultCh(resultCh)) @@ -1024,7 +1024,7 @@ func (suite *SynchronizerTestSuite) TestRemovePeerReleasesLockBeforePushBack() { applier := newBlockApplier(suite.blockExec, suite.store, applierWithState(suite.initialState)) pool := NewSynchronizer(1, suite.client, applier) pool.AddPeer(newPeerData(peerID, 1, 100)) - pool.pendingToApply[respH1.Block.Height] = *respH1 + pool.pendingToApply[respH1.Height] = *respH1 // Block the job generator so the re-queue cannot complete. pool.jobGen.mtx.Lock() @@ -1053,7 +1053,7 @@ func (suite *SynchronizerTestSuite) TestRemovePeerReleasesLockBeforePushBack() { case <-time.After(2 * time.Second): suite.Require().Fail("RemovePeer did not finish after the job generator lock was released") } - suite.Require().Equal([]int64{respH1.Block.Height}, pool.jobGen.pushedBack) + suite.Require().Equal([]int64{respH1.Height}, pool.jobGen.pushedBack) } func (suite *SynchronizerTestSuite) TestUpdateMonitor() { @@ -1823,7 +1823,7 @@ func (h *backlogHarness) fillWindow(ctx context.Context, blocking int64) types.N if resp == nil { return blockingPeer } - if resp.Block.Height == blocking { + if resp.Height == blocking { blockingPeer = resp.PeerID continue } @@ -1863,7 +1863,7 @@ func (suite *SynchronizerTestSuite) TestBacklogIsBoundedWhileTheBlockingHeightIs continue } requested++ - if resp.Block.Height == blocking { + if resp.Height == blocking { continue } h.deliver(ctx, resp) @@ -1901,7 +1901,7 @@ func (suite *SynchronizerTestSuite) TestBacklogIsBoundedByBytesWhenBlocksAreLarg refused++ continue } - if resp.Block.Height == blocking { + if resp.Height == blocking { continue } h.deliverSized(ctx, resp, blockSize) @@ -1936,7 +1936,7 @@ func (suite *SynchronizerTestSuite) TestBlockingHeightIsStillFetchableAtTheByteB if resp == nil { break } - if resp.Block.Height == blocking { + if resp.Height == blocking { blockingPeer = resp.PeerID continue } @@ -1954,7 +1954,7 @@ func (suite *SynchronizerTestSuite) TestBlockingHeightIsStillFetchableAtTheByteB resp := h.produce(ctx) suite.Require().NotNil(resp, "the byte budget is spent and the height it is waiting for can never be requested again") - suite.Require().EqualValues(blocking, resp.Block.Height) + suite.Require().EqualValues(blocking, resp.Height) h.deliver(ctx, resp) @@ -1984,7 +1984,7 @@ func (suite *SynchronizerTestSuite) TestBlockingHeightIsStillFetchableAtTheBound resp := h.produce(ctx) suite.Require().NotNil(resp, "the window is full and the height it is waiting for can never be requested again") - suite.Require().EqualValues(blocking, resp.Block.Height, + suite.Require().EqualValues(blocking, resp.Height, "the retry of the blocking height must be what goes out first") h.deliver(ctx, resp) @@ -2020,7 +2020,7 @@ func (suite *SynchronizerTestSuite) TestDroppingTheBlockingPeerLeavesTheHeightTo for range requeued { resp := h.produce(ctx) suite.Require().NotNil(resp, "a re-queued height was starved by the window") - suite.Require().NotEqualValues(blocking, resp.Block.Height) + suite.Require().NotEqualValues(blocking, resp.Height) } suite.Require().Nil(h.produce(ctx), "the window must still refuse new heights") @@ -2028,7 +2028,7 @@ func (suite *SynchronizerTestSuite) TestDroppingTheBlockingPeerLeavesTheHeightTo h.timeout(ctx, blockingPeer, blocking) resp := h.produce(ctx) suite.Require().NotNil(resp, "the blocking height was stranded by dropping its peer") - suite.Require().EqualValues(blocking, resp.Block.Height) + suite.Require().EqualValues(blocking, resp.Height) } // TestInOrderSyncIsNotThrottledByTheBound checks that the window does not slow a @@ -2090,7 +2090,7 @@ func (suite *SynchronizerTestSuite) TestRequeuedHeightsAreNotStarvedByTheBound() for range requeued { resp := h.produce(ctx) suite.Require().NotNil(resp, "a re-queued height was starved by the window") - reissued = append(reissued, resp.Block.Height) + reissued = append(reissued, resp.Height) } suite.Require().Equal(requeued, reissued, "re-queued heights must go out lowest first") @@ -2326,7 +2326,7 @@ func (suite *SynchronizerTestSuite) TestSmallBlocksAfterSlowApplyKeepPeersUsable if found { response := h.produce(ctx) suite.Require().NotNil(response) - suite.Equal(height, response.Block.Height) + suite.Equal(height, response.Height) h.deliver(ctx, response) suite.Equal(height+1, h.pool.height) } @@ -2342,3 +2342,13 @@ func (suite *SynchronizerTestSuite) TestSmallBlocksAfterSlowApplyKeepPeersUsable suite.True(stopped, "WaitForSync must still stop on cancellation") } } + +// sameBlock matches the block resp holds, which is decoded again from its +// bytes when it is applied. +func sameBlock(t *testing.T, resp *BlockResponse) interface{} { + t.Helper() + block, err := decodeBlock(resp.BlockBytes) + require.NoError(t, err) + hash := block.Hash() + return mock.MatchedBy(func(b *types.Block) bool { return b.HashesTo(hash) }) +}