Skip to content
Merged
2 changes: 1 addition & 1 deletion docs/nodes/metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ The following metrics are available:
| consensus_step_duration | Histogram | step | Histogram of durations for each step in the consensus protocol |
| consensus_block_gossip_receive_latency | Histogram | | Histogram of time taken to receive a block in seconds, measure between when a new block is first discovered to when the block is completed |
| consensus_block_gossip_parts_received | Counter | matches_current | Number of block parts received by the node |
| consensus_block_sync_apply_stage_duration | Histogram | stage | Time a block spent in each stage of the block sync applier, in ms: partset, verify_commit, verify_block, save, exec, and wait (idle time between blocks) |
| consensus_block_sync_apply_stage_duration | Histogram | stage | Time a block spent in each stage of the block sync applier, in ms: decode, partset, verify_commit, verify_block, save, exec, and wait (idle time between blocks) |
| consensus_quorum_prevote_delay | Gauge | | Interval in seconds between the proposal timestamp and the timestamp of the earliest prevote that achieved a quorum |
| consensus_full_prevote_delay | Gauge | | Interval in seconds between the proposal timestamp and the timestamp of the latest prevote in a round where all validators voted |
| consensus_proposal_timestamp_difference | Histogram | | Difference between the timestamp in the proposal message and the local time of the validator at the time it received the message |
Expand Down
77 changes: 60 additions & 17 deletions internal/blocksync/applier.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,29 +69,71 @@ func newBlockApplier(blockExec sm.Executor, store sm.BlockStore, opts ...applier
return applier
}

// Apply safely verifies, saves to the store and executes a block with commit
func (e *blockApplier) Apply(ctx context.Context, block *types.Block, commit *types.Commit) error {
// Apply safely verifies, saves to the store and executes a block with commit.
// blockBytes is the block as it was committed, the bytes block was decoded
// from.
func (e *blockApplier) Apply(
ctx context.Context,
block *types.Block,
blockBytes []byte,
commit *types.Commit,
) error {
e.mtx.Lock()
defer e.mtx.Unlock()
e.observeWait()
defer func() { e.lastDone = time.Now() }()
return e.apply(ctx, block, blockBytes, commit, 0)
}

// Time between the end of the previous apply and the start of this one. With
// a fast application this is what the sync rate is actually limited by, and
// it is block fetching, not block execution.
// ApplyBytes is Apply for a block held only as the bytes it was committed with:
// it decodes the block from them first. Block sync keeps pending blocks in that
// form, so that a response waiting for a lower height does not hold a decoded
// copy beside its bytes.
func (e *blockApplier) ApplyBytes(ctx context.Context, blockBytes []byte, commit *types.Commit) error {
e.mtx.Lock()
defer e.mtx.Unlock()
e.observeWait()
defer func() { e.lastDone = time.Now() }()

start := time.Now()
block, err := decodeBlock(blockBytes)
decodeTime := e.observeSince("decode", start)
if err != nil {
return err
}
return e.apply(ctx, block, blockBytes, commit, decodeTime)
}

// observeWait records the time between the end of the previous apply and the
// start of this one. With a fast application this is what the sync rate is
// actually limited by, and it is block fetching, not block execution.
func (e *blockApplier) observeWait() {
if !e.lastDone.IsZero() {
e.observeSince("wait", e.lastDone)
}
}

// apply is Apply with the lock held. decodeTime is how long decoding block
// took, counted with building its part set in the per-block timings.
func (e *blockApplier) apply(
ctx context.Context,
block *types.Block,
blockBytes []byte,
commit *types.Commit,
decodeTime time.Duration,
) error {
// The part set is needed twice: to derive the block ID and to persist the
// block. Building it serializes the whole block and builds a Merkle tree over
// the parts, so build it once and pass it to both.
// block. Building it builds a Merkle tree over the parts, so build it once
// and pass it to both. It is cut from the bytes the block was committed
// with, not from block: the commit's part set header is computed over those
// bytes, and a re-encoding of block drops unknown fields and non-canonical
// encoding. Only the standard cut is rebuilt, the one honest proposers use.
// Consensus does not require it yet, and a block committed with another cut
// fails the commit check whichever peer serves it.
start := time.Now()
blockParts, err := block.MakePartSet(types.BlockPartSizeBytes)
if err != nil {
return err
}
blockParts := types.NewPartSetFromData(blockBytes, types.BlockPartSizeBytes)
blockID := block.BlockID(blockParts)
partSetTime := e.observeSince("partset", start)
partSetTime := decodeTime + e.observeSince("partset", start)

start = time.Now()
verified, err := e.verify(ctx, blockID, block, commit)
Expand Down Expand Up @@ -137,8 +179,8 @@ func (e *blockApplier) Apply(ctx context.Context, block *types.Block, commit *ty
e.metrics.ObserveBlockSyncStage("exec", execTime)

e.stats.add(partSetTime, verifyTime, saveTime, execTime)
// ByteSize is the size of the serialized block we just built, so the metric
// costs nothing extra here
// ByteSize is the length of the bytes the part set was cut from, so the
// metric costs nothing extra here
e.metrics.RecordConsMetrics(block, blockParts.ByteSize())
return nil
}
Expand Down Expand Up @@ -219,7 +261,8 @@ func (e *blockApplier) observeSince(stage string, start time.Time) time.Duration
// applyTimings is the average time a single block spends in each stage of the
// apply pipeline.
type applyTimings struct {
// PartSet is block serialization plus the Merkle tree over its parts
// PartSet is decoding the block, when it is held as bytes, and the Merkle
// tree over its parts
PartSet time.Duration
// Verify is commit signature verification and block validation
Verify time.Duration
Expand All @@ -231,8 +274,8 @@ type applyTimings struct {

// applyStats accumulates per-stage timings of the block apply pipeline. Block
// sync applies blocks one at a time, so this is where a slow sync shows up; the
// breakdown attributes it to serialization, signature verification, disk or the
// ABCI application without needing a profiler.
// breakdown attributes it to building the part set, signature verification,
// disk or the ABCI application without needing a profiler.
type applyStats struct {
mtx sync.Mutex
blocks int64
Expand Down
112 changes: 98 additions & 14 deletions internal/blocksync/applier_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"testing"
"time"

"github.com/cosmos/gogoproto/proto"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"

Expand All @@ -27,6 +28,7 @@ import (
"github.com/dashpay/tenderdash/internal/test/metricspy"
tmrequire "github.com/dashpay/tenderdash/internal/test/require"
"github.com/dashpay/tenderdash/libs/log"
tmproto "github.com/dashpay/tenderdash/proto/tendermint/types"
"github.com/dashpay/tenderdash/types"
)

Expand All @@ -47,7 +49,7 @@ func TestBlockApplierChecksAppResponseBeforeSave(t *testing.T) {
executor := sm.NewBlockExecutor(stateStore, client, nil, sm.EmptyEvidencePool{}, blockStore, nil)
applier := newBlockApplier(executor, blockStore, applierWithState(initialState))

require.Panics(t, func() { _ = applier.Apply(ctx, block, commit) })
require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), commit) })
require.Zero(t, blockStore.Height(), "an inconsistent app response must not advance the block store")
require.Equal(t, initialState.LastBlockHeight, applier.State().LastBlockHeight)
loaded, err := stateStore.Load()
Expand Down Expand Up @@ -174,7 +176,7 @@ func TestBlockApplierApply(t *testing.T) {
tc.mockFn()
}
fn := func() {
err := applier.Apply(ctx, tc.block, tc.commit)
err := applier.Apply(ctx, tc.block, blockBytes(t, tc.block), tc.commit)
tmrequire.Error(t, tc.wantErr, err)
}
if tc.wantPanic {
Expand Down Expand Up @@ -267,7 +269,7 @@ func TestBlockApplierDoesNotSaveBlockRejectedByApp(t *testing.T) {
Return(sm.CurrentRoundState{}, errors.New("app rejected the block"))

applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState))
require.Panics(t, func() { _ = applier.Apply(ctx, block, commit) })
require.Panics(t, func() { _ = applier.Apply(ctx, block, blockBytes(t, block), commit) })
}

// TestBlockApplierSavesBlockBeforeFinalize pins the order the handshake relies
Expand Down Expand Up @@ -313,7 +315,7 @@ func TestBlockApplierSavesBlockBeforeFinalize(t *testing.T) {
Return(state, nil, nil)

applier := newBlockApplier(mockBlockExec, mockBlockStore, applierWithState(initialState))
require.NoError(t, applier.Apply(ctx, block, commit))
require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), commit))
require.Equal(t, []string{"process", "verify", "save", "finalize"}, calls)
}

Expand Down Expand Up @@ -349,7 +351,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) {
applier := newBlockApplier(mockBlockExec, mockBlockStore,
applierWithState(initialState), applierWithMetrics(m))

require.NoError(t, applier.Apply(ctx, blockH1, commitH1))
require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1))

for _, stage := range []string{"partset", "verify_commit", "verify_block", "save", "exec"} {
require.Len(t, hist.Samples[stage], 1, "stage %q after the first apply", stage)
Expand All @@ -360,7 +362,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) {
// A failed attempt must advance the idle-time boundary as well, so a retry
// does not count the failed attempt's wait and verification time again.
failureStart := time.Now()
require.Error(t, applier.Apply(ctx, blockH1, new(types.Commit)))
require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), new(types.Commit)))
failureEnd := time.Now()
failureDone := applier.lastDone
require.False(t, failureDone.Before(failureStart), "failed apply must advance lastDone")
Expand All @@ -369,7 +371,7 @@ func TestBlockApplierRecordsStageMetrics(t *testing.T) {

// The retry measures only the idle time after the failed attempt.
retryStart := time.Now()
require.NoError(t, applier.Apply(ctx, blockH1, commitH1))
require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1))
retryEnd := time.Now()
require.Len(t, hist.Samples["wait"], 2)
wait := hist.Samples["wait"][1]
Expand Down Expand Up @@ -400,7 +402,7 @@ func TestBlockApplierVerifyFailureTimesOnlyTheCheckThatRan(t *testing.T) {
// A rejected commit must stop before ValidateBlock is reached.
mockBlockExec.On("VerifyCommit", initialState, blockH1.BlockID(nil), blockH1.Height, new(types.Commit)).
Once().Return(types.VerifiedCommit{}, errors.New("bad signature"))
require.Error(t, applier.Apply(ctx, blockH1, new(types.Commit)))
require.Error(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), new(types.Commit)))

require.Len(t, hist.Samples["partset"], 1)
require.Len(t, hist.Samples["verify_commit"], 1)
Expand Down Expand Up @@ -444,7 +446,7 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) {
blockExec.On("FinalizeBlock", mock.Anything, mock.Anything, mock.Anything, blockH1ID, blockH1, commitH1, none).
Once().Return(initialState, nil, nil)
applier := newBlockApplier(blockExec, blockStore, applierWithState(initialState))
require.NoError(t, applier.Apply(ctx, blockH1, commitH1))
require.NoError(t, applier.Apply(ctx, blockH1, blockBytes(t, blockH1), commitH1))
return applier, blockExec
}
// expectH2 expects block 2 to be validated and applied offering lastCommit
Expand All @@ -460,25 +462,25 @@ func TestBlockApplierOffersTheVerifiedCommitForward(t *testing.T) {
t.Run("the next block is offered the previous commit's verification", func(t *testing.T) {
applier, blockExec := applyH1(t)
expectH2(blockExec, verifiedH1)
require.NoError(t, applier.Apply(ctx, blockH2, commitH2))
require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2))
})

t.Run("a rejected commit does not replace the verification", func(t *testing.T) {
applier, blockExec := applyH1(t)
bad := new(types.Commit)
blockExec.On("VerifyCommit", mock.Anything, blockH2ID, blockH2.Height, bad).
Once().Return(none, errors.New("bad signature"))
require.Error(t, applier.Apply(ctx, blockH2, bad))
require.Error(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), bad))

expectH2(blockExec, verifiedH1)
require.NoError(t, applier.Apply(ctx, blockH2, commitH2))
require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2))
})

t.Run("replacing the state discards the verification", func(t *testing.T) {
applier, blockExec := applyH1(t)
applier.UpdateState(initialState)
expectH2(blockExec, none)
require.NoError(t, applier.Apply(ctx, blockH2, commitH2))
require.NoError(t, applier.Apply(ctx, blockH2, blockBytes(t, blockH2), commitH2))
})
}

Expand Down Expand Up @@ -532,9 +534,91 @@ func TestBlockApplierSkipsTheLastCommitItVerified(t *testing.T) {
wantSkipped float64
}{{1, 0}, {2, 3}, {3, 6}} {
block, _, _, seenCommit := makeNextBlock(ctx, t, applier.State(), privVals[0], step.height, commit)
require.NoError(t, applier.Apply(ctx, block, seenCommit))
require.NoError(t, applier.Apply(ctx, block, blockBytes(t, block), seenCommit))
require.Equal(t, step.wantSkipped, skipped.Value(),
"LastCommit threshold verifications skipped after applying height %d", step.height)
commit = seenCommit
}
}

// blockBytes returns the bytes block is committed with when its proposer
// encodes it the canonical way, the bytes a peer serves it as.
func blockBytes(t *testing.T, block *types.Block) []byte {
t.Helper()
pbb, err := block.ToProto()
require.NoError(t, err)
bz, err := proto.Marshal(pbb)
require.NoError(t, err)
return bz
}

// finalizeBlockIDApp records the block IDs FinalizeBlock is called with.
type finalizeBlockIDApp struct {
abci.BaseApplication
blockIDs []*tmproto.BlockID
}

func (app *finalizeBlockIDApp) FinalizeBlock(
ctx context.Context,
req *abci.RequestFinalizeBlock,
) (*abci.ResponseFinalizeBlock, error) {
app.blockIDs = append(app.blockIDs, req.BlockID)
return app.BaseApplication.FinalizeBlock(ctx, req)
}

// TestFinalizeBlockGetsCommittedBlockID checks that the application is given
// the block ID that was committed, when applying a block and replaying it, for
// a block committed with bytes that a re-encoding does not reproduce.
func TestFinalizeBlockGetsCommittedBlockID(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
logger := log.NewNopLogger()
genDoc, privVals := factory.RandGenesisDoc(1, factory.ConsensusParams())
state, err := sm.MakeGenesisState(genDoc)
require.NoError(t, err)
stateStore := sm.NewStore(dbm.NewMemDB())
require.NoError(t, stateStore.Save(state))
blockStore := store.NewBlockStore(dbm.NewMemDB())

finalizeApp := &finalizeBlockIDApp{}
localClient := abciclient.NewLocalClient(logger, finalizeApp)
app := proxy.New(localClient, logger, proxy.NopMetrics())
require.NoError(t, app.Start(ctx))
eventBus := eventbus.NewDefault(logger)
require.NoError(t, eventBus.Start(ctx))
mp := &mpmocks.Mempool{}
mp.On("Lock").Return()
mp.On("Unlock").Return()
mp.On("FlushAppConn", mock.Anything).Return(nil)
mp.On("Update", mock.Anything, mock.Anything, mock.Anything, mock.Anything,
mock.Anything, mock.Anything, mock.Anything).Return(nil)
blockExec := sm.NewBlockExecutor(stateStore, app, mp, sm.EmptyEvidencePool{}, blockStore, eventBus)
applier := newBlockApplier(blockExec, blockStore, applierWithState(state))

block, blockID, partSet, seenCommit := makeNextBlockWithBytes(ctx, t, applier.State(), privVals[0], 1,
types.NewCommit(0, 0, types.BlockID{}, nil, nil), withUnknownField)
require.False(t, blockID.Equals(block.BlockID(nil)), "the block must not be one a re-encoding reproduces")
require.NoError(t, applier.Apply(ctx, block, loadBlockBytes(storedParts(t, partSet), 1), seenCommit))

want := seenCommit.BlockID.ToProto()
require.Equal(t, []*tmproto.BlockID{&want}, finalizeApp.blockIDs, "applying the block")

_, err = sm.ExecReplayedCommitBlock(ctx, localClient, block, seenCommit, logger)
require.NoError(t, err)
require.Equal(t, []*tmproto.BlockID{&want, &want}, finalizeApp.blockIDs, "replaying the block")
}

// storedParts returns a store that holds partSet as the parts of the block at
// height 1.
func storedParts(t *testing.T, partSet *types.PartSet) sm.BlockStore {
t.Helper()
blockStore := mocks.NewBlockStore(t)
blockStore.On("LoadBlockMeta", int64(1)).Return(&types.BlockMeta{
BlockID: types.BlockID{PartSetHeader: partSet.Header()},
BlockSize: int(partSet.ByteSize()),
})
for i := range int(partSet.Total()) {
blockStore.On("LoadBlockPart", int64(1), i).Return(partSet.GetPart(i))
}
return blockStore
}
4 changes: 2 additions & 2 deletions internal/blocksync/block_fetch_job.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ func blockFetchJobHandler(blockClient client.BlockClient, peer PeerData, height
if err != nil {
return errorResult(responder, height, err)
}
if resp.Block.Height != height {
if resp.Height != height {
return errorResult(responder, height,
fmt.Errorf("peer sent block at height %d, requested height %d", resp.Block.Height, height))
fmt.Errorf("peer sent block at height %d, requested height %d", resp.Height, height))
}
resp.ConnID = responder.connID
size = protoResp.Size()
Expand Down
Loading
Loading