diff --git a/internal/consensus/block_executor.go b/internal/consensus/block_executor.go index d8f55d815..d5fe16dd1 100644 --- a/internal/consensus/block_executor.go +++ b/internal/consensus/block_executor.go @@ -96,7 +96,7 @@ func (c *blockExecutor) ensureProcess(ctx context.Context, rs *cstypes.RoundStat if err != nil { rs.CurrentRoundState = sm.CurrentRoundState{} rs.ProcessedBlockID = types.BlockID{} - return fmt.Errorf("ProcessProposal abci method: %w", err) + return fmt.Errorf("process proposal: %w", err) } rs.CurrentRoundState = uncommittedState rs.ProcessedBlockID = block.BlockID(rs.ProposalBlockParts).Copy() diff --git a/internal/consensus/proposal_state_validation_test.go b/internal/consensus/proposal_state_validation_test.go new file mode 100644 index 000000000..5a09e83d6 --- /dev/null +++ b/internal/consensus/proposal_state_validation_test.go @@ -0,0 +1,246 @@ +package consensus + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + abci "github.com/dashpay/tenderdash/abci/types" + abcimocks "github.com/dashpay/tenderdash/abci/types/mocks" + "github.com/dashpay/tenderdash/dash" + cstypes "github.com/dashpay/tenderdash/internal/consensus/types" + sm "github.com/dashpay/tenderdash/internal/state" + "github.com/dashpay/tenderdash/internal/test/factory" + "github.com/dashpay/tenderdash/libs/log" + tmtime "github.com/dashpay/tenderdash/libs/time" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" + "github.com/dashpay/tenderdash/types" +) + +// blocksNotFollowingState corrupts, one per case, a header field that only the +// committed state can tell is wrong. +var blocksNotFollowingState = []struct { + name string + mutate func(*types.Block) +}{ + {"height", func(b *types.Block) { b.Height++ }}, + {"chain ID", func(b *types.Block) { b.ChainID = "other-chain" }}, + {"last block ID", func(b *types.Block) { b.LastBlockID = factory.MakeBlockID() }}, +} + +// newNodeNotProposing returns a validator node at the initial height that is +// not round 0's proposer, the proposer's key, and the node's application, which +// fails ProcessProposal as an application may for a block it cannot process. +func newNodeNotProposing(ctx context.Context, t *testing.T) (*State, types.PrivValidator, *abcimocks.Application) { + t.Helper() + app := abcimocks.NewApplication(t) + app.On("ProcessProposal", mock.Anything, mock.Anything). + Return(nil, errors.New("application cannot process this block")).Maybe() + + genDoc, privVals := factory.RandGenesisDoc(2, factory.ConsensusParams()) + state, err := sm.MakeGenesisState(genDoc) + require.NoError(t, err) + cs := newStateWithConfig(ctx, t, consensusLogger(t), configSetup(t), state, privVals[0], app) + + sd := cs.GetStateData() + proposer, err := sd.ProposerSelector.GetProposer(sd.Height, 0) + require.NoError(t, err) + if proposer.ProTxHash.Equal(cs.privValidator.ProTxHash) { + cs.SetPrivValidator(ctx, privVals[1]) + return cs, privVals[0], app + } + return cs, privVals[1], app +} + +// proposalOfMutatedBlock returns round 0's proposal, signed by proposer, of the +// block the node expects, altered by mutate. +func proposalOfMutatedBlock( + ctx context.Context, + t *testing.T, + cs *State, + proposer types.PrivValidator, + mutate func(*types.Block), +) (*types.Proposal, *types.PartSet) { + t.Helper() + sd := cs.GetStateData() + proTxHash, err := proposer.GetProTxHash(ctx) + require.NoError(t, err) + + lastCommit := sd.LastCommit + if lastCommit == nil { + lastCommit = types.NewCommit(0, 0, types.BlockID{}, nil, nil) + } + block := sd.state.MakeBlock(sd.Height, nil, lastCommit, nil, proTxHash, 1) + block.CoreChainLockedHeight = sd.state.LastCoreChainLockedBlockHeight + mutate(block) + parts, err := block.MakePartSet(types.BlockPartSizeBytes) + require.NoError(t, err) + + proposal := types.NewProposal(sd.Height, block.CoreChainLockedHeight, 0, -1, block.BlockID(parts), block.Time) + p := proposal.ToProto() + _, err = proposer.SignProposal(ctx, sd.state.ChainID, sd.Validators.QuorumType, sd.Validators.QuorumHash, p) + require.NoError(t, err) + proposal.Signature = p.Signature + return proposal, parts +} + +// TestStatePrevotesNilOnBlockNotFollowingState checks that a validator prevotes +// nil, without asking the application, on a proposed block whose header does +// not follow its committed state. +func TestStatePrevotesNilOnBlockNotFollowingState(t *testing.T) { + for _, tc := range blocksNotFollowingState { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cs, proposer, app := newNodeNotProposing(ctx, t) + proposal, parts := proposalOfMutatedBlock(ctx, t, cs, proposer, tc.mutate) + voteCh := subscribeToVoter(ctx, t, cs, cs.privValidator.ProTxHash) + + require.NoError(t, cs.SetProposalAndBlock(ctx, proposal, parts, "peer")) + startTestRound(ctx, cs, proposal.Height, 0) + + ensurePrevoteMatch(t, voteCh, proposal.Height, 0, nil) + app.AssertNotCalled(t, "ProcessProposal", mock.Anything, mock.Anything) + }) + } +} + +// TestReplayOfBlockNotFollowingStatePrevotesNil checks that replaying the WAL +// of a node that received a proposed block not following its committed state +// prevotes nil again without asking the application, and the node keeps running. +func TestReplayOfBlockNotFollowingStatePrevotesNil(t *testing.T) { + for _, tc := range blocksNotFollowingState { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cs, proposer, app := newNodeNotProposing(ctx, t) + proposal, parts := proposalOfMutatedBlock(ctx, t, cs, proposer, tc.mutate) + replayProposal(ctx, t, cs, proposal, parts, tmtime.Now()) + + requireOwnNilPrevote(ctx, t, cs) + app.AssertNotCalled(t, "ProcessProposal", mock.Anything, mock.Anything) + }) + } +} + +// TestReplayOfFirstBlockWithoutGenesisTimePrevotesNil checks that replaying the +// WAL of a node that received a first block not dated at genesis time prevotes +// nil without asking the application, as the node did when it received it. +func TestReplayOfFirstBlockWithoutGenesisTimePrevotesNil(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cs, proposer, app := newNodeNotProposing(ctx, t) + proposal, parts := proposalOfMutatedBlock(ctx, t, cs, proposer, func(b *types.Block) { + b.Time = b.Time.AddDate(100, 0, 0) + }) + replayProposal(ctx, t, cs, proposal, parts, tmtime.Now()) + + requireOwnNilPrevote(ctx, t, cs) + app.AssertNotCalled(t, "ProcessProposal", mock.Anything, mock.Anything) +} + +// TestReplayJudgesProposalTimelinessByWALTime checks, above the first height, +// that replay judges a proposal by the time of its WAL entry: one written too +// early for the proposal's timestamp, and so received too early, gets a nil +// prevote without an application call, and a timely one reaches the +// application. +func TestReplayJudgesProposalTimelinessByWALTime(t *testing.T) { + for _, tc := range []struct { + name string + walDelay time.Duration + reachesApp bool + }{ + {"received too early", -time.Hour, false}, + {"received in time", 0, true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + live := newRoundSkipTest(ctx, t, "consensus_replay_proposal_timeliness_test", nil) + config, err := ResetConfig(t, "consensus_replay_proposal_timeliness_test") + require.NoError(t, err) + config.Consensus.CreateEmptyBlocks = false + app := abcimocks.NewApplication(t) + app.On("ProcessProposal", mock.Anything, mock.Anything). + Return(&abci.ResponseProcessProposal{Status: abci.ResponseProcessProposal_REJECT}, nil).Maybe() + cs := newStateWithConfigAndBlockStore(ctx, t, log.NewNopLogger(), config, live.cs.GetStateData().state, + live.cs.privValidator.PrivValidator, app, live.cs.blockStore) + + sd := cs.GetStateData() + require.Equal(t, sd.state.InitialHeight+1, sd.Height) + proposer, err := sd.ProposerSelector.GetProposer(sd.Height, 0) + require.NoError(t, err) + var proposerKey types.PrivValidator + for _, vs := range live.vss { + if proTxHash, _ := vs.GetProTxHash(ctx); proposer.ProTxHash.Equal(proTxHash) { + proposerKey = vs.PrivValidator + } + } + require.NotNil(t, proposerKey) + + proposal, parts := proposalOfMutatedBlock(ctx, t, cs, proposerKey, func(*types.Block) {}) + replayProposal(ctx, t, cs, proposal, parts, proposal.Timestamp.Add(tc.walDelay)) + + requireOwnNilPrevote(ctx, t, cs) + if tc.reachesApp { + app.AssertNumberOfCalls(t, "ProcessProposal", 1) + } else { + app.AssertNotCalled(t, "ProcessProposal", mock.Anything, mock.Anything) + } + }) + } +} + +// requireOwnNilPrevote checks that cs, not running, is at the prevote step and +// has queued its own prevote, for nil. +func requireOwnNilPrevote(ctx context.Context, t *testing.T, cs *State) { + t.Helper() + require.Equal(t, cstypes.RoundStepPrevote, cs.GetStateData().Step) + ctx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + for { + mi, ok := cs.msgInfoQueue.sender.internalQueue.recv(ctx) + require.True(t, ok, "the node cast no prevote") + if msg, isVote := mi.Msg.(*VoteMessage); isVote && msg.Vote.Type == tmproto.PrevoteType { + assert.True(t, msg.Vote.BlockID.IsNil(), "the node prevoted for block %v", msg.Vote.BlockID) + return + } + } +} + +// replayProposal replays, in replay mode, the WAL entries of proposal and its +// parts as written at walTime by a node that received them from a peer. +func replayProposal( + ctx context.Context, + t *testing.T, + cs *State, + proposal *types.Proposal, + parts *types.PartSet, + walTime time.Time, +) { + t.Helper() + wal := []msgInfo{{Msg: &ProposalMessage{Proposal: proposal}, PeerID: "peer"}} + for i := 0; i < int(parts.Total()); i++ { + wal = append(wal, msgInfo{ + Msg: &BlockPartMessage{Height: proposal.Height, Round: 0, Part: parts.GetPart(i)}, + PeerID: "peer", + }) + } + + cs.emitter.Emit(setReplayModeEventName, true) + ctx = dash.ContextWithProTxHash(ctx, cs.privValidator.ProTxHash) + require.NotPanics(t, func() { + for _, msg := range wal { + assert.NoError(t, cs.readReplayMessage(ctx, &TimedWALMessage{Time: walTime, Msg: msg}, nil)) + } + }) +} diff --git a/internal/consensus/replay.go b/internal/consensus/replay.go index 166c0af48..fccde2e2c 100644 --- a/internal/consensus/replay.go +++ b/internal/consensus/replay.go @@ -82,6 +82,8 @@ func (cs *State) readReplayMessage(ctx context.Context, msg *TimedWALMessage, ne } } case msgInfo: + // The WAL keeps no receive time; the entry's time is the closest, never earlier. + m.ReceiveTime = msg.Time peerID := m.PeerID if peerID == "" { peerID = "local" diff --git a/internal/consensus/state_data.go b/internal/consensus/state_data.go index 9efd97e23..7e6f7775a 100644 --- a/internal/consensus/state_data.go +++ b/internal/consensus/state_data.go @@ -370,6 +370,10 @@ func (s *StateData) proposalIsTimely() error { s.ProposalReceiveTime.Sub(s.Proposal.Timestamp).String(), ) case 1: // too late + if s.replayMode { + // In replay, the receive time is the WAL time, which can lag the receipt. + return nil + } return fmt.Errorf( "%w: received too late: height %d, round %d, delay %s", errPrevoteProposalNotTimely, s.Height, s.Round, @@ -726,7 +730,7 @@ func (s *StateData) isValidForPrevote() error { } // if this block was not validated yet, we check if it's timely - if !s.replayMode && !s.ProposalBlock.HashesTo(s.ValidBlock.Hash()) { + if !s.ProposalBlock.HashesTo(s.ValidBlock.Hash()) { if err := s.proposalIsTimely(); err != nil { return err } diff --git a/internal/consensus/state_data_test.go b/internal/consensus/state_data_test.go index 5a7a5fcf1..d1c3a1e75 100644 --- a/internal/consensus/state_data_test.go +++ b/internal/consensus/state_data_test.go @@ -27,9 +27,10 @@ func TestIsValidForPrevote(t *testing.T) { Validators: valSet, } testCases := []struct { - state sm.State - rs cstypes.RoundState - wantErr string + state sm.State + rs cstypes.RoundState + replayMode bool + wantErr string }{ { // invalid proposal-block @@ -100,12 +101,42 @@ func TestIsValidForPrevote(t *testing.T) { }, wantErr: "", }, + { + // in replay, the WAL time shows the proposal was received too early + state: sm.State{InitialHeight: 1}, + rs: cstypes.RoundState{ + Height: 1000, + ProposalBlock: &types.Block{Header: types.Header{Time: now.Add(time.Hour)}}, + Proposal: &types.Proposal{Timestamp: now.Add(time.Hour), POLRound: -1}, + ProposalReceiveTime: now, + LockedRound: -1, + Validators: valSet, + }, + replayMode: true, + wantErr: "proposal is not timely", + }, + { + // in replay, a WAL time later than the window is no proof of a late + // receipt: the WAL dates a message by when it was handled + state: sm.State{InitialHeight: 1}, + rs: cstypes.RoundState{ + Height: 1000, + ProposalBlock: &types.Block{Header: types.Header{Time: now}}, + Proposal: &types.Proposal{Timestamp: now, POLRound: -1}, + ProposalReceiveTime: now.Add(time.Hour), + LockedRound: -1, + Validators: valSet, + }, + replayMode: true, + wantErr: "", + }, } for i, tc := range testCases { t.Run(fmt.Sprintf("test-case #%d", i), func(t *testing.T) { stateData := StateData{ state: tc.state, RoundState: tc.rs, + replayMode: tc.replayMode, } selector, err := selectproposer.NewHeightRoundProposerSelector(valSet.Copy(), tc.rs.Height, tc.rs.Round, nil, nil) require.NoError(t, err) diff --git a/internal/consensus/state_prevoter.go b/internal/consensus/state_prevoter.go index e9f4ae2c7..68db46401 100644 --- a/internal/consensus/state_prevoter.go +++ b/internal/consensus/state_prevoter.go @@ -3,7 +3,6 @@ package consensus import ( "context" "errors" - "fmt" "time" cstypes "github.com/dashpay/tenderdash/internal/consensus/types" @@ -72,7 +71,7 @@ func (p *prevoter) handleError(err error, args ...any) { return } // Unknown error, so we panic - panic(fmt.Sprintf("ProcessProposal: %v", err)) + panic(err.Error()) } func (p *prevoter) signAndAddNilVote(ctx context.Context, stateData *StateData) { diff --git a/internal/consensus/state_prevoter_test.go b/internal/consensus/state_prevoter_test.go index 8723b615e..f01a97b51 100644 --- a/internal/consensus/state_prevoter_test.go +++ b/internal/consensus/state_prevoter_test.go @@ -2,6 +2,7 @@ package consensus import ( "context" + "errors" "fmt" "regexp" "testing" @@ -152,6 +153,21 @@ func (suite *PrevoterTestSuite) TestDo() { } } +// TestDoPanicsOnProcessProposalFailure pins the panic text for a ProcessProposal +// error that is neither a rejection nor an invalid block. +func (suite *PrevoterTestSuite) TestDoPanicsOnProcessProposalFailure() { + ctx := context.Background() + validStateData := suite.makeValidStateData() + stateData := StateData{state: validStateData.state, RoundState: validStateData.RoundState} + suite.mockExecutor. + On("ProcessProposal", ctx, mock.Anything, int32(0), mock.Anything, true, types.VerifiedCommit{}). + Once(). + Return(sm.CurrentRoundState{}, errors.New("ProcessProposal abci method: client has stopped")) + suite.PanicsWithValue("process proposal: ProcessProposal abci method: client has stopped", func() { + _ = suite.prevoter.Do(ctx, &stateData) + }) +} + func (suite *PrevoterTestSuite) TestCheckProposalBlock() { testCases := []struct { modifier func(stateData *StateData) diff --git a/internal/consensus/state_test.go b/internal/consensus/state_test.go index cea535df6..985c1aec4 100644 --- a/internal/consensus/state_test.go +++ b/internal/consensus/state_test.go @@ -3453,7 +3453,7 @@ func TestStateTryAddCommitPanicsOnClientError(t *testing.T) { ReceiveTime: time.Time{}, }) assert.PanicsWithError(t, - "ABCI client stopped, Tenderdash needs to be restarted: ProcessProposal abci method: client has stopped", + "ABCI client stopped, Tenderdash needs to be restarted: process proposal: ProcessProposal abci method: client has stopped", func() { _ = consensusState.ctrl.Dispatch(ctx, &TryAddCommitEvent{Commit: commit, PeerID: peerID}, &stateData) }) diff --git a/internal/consensus/state_try_add_commit.go b/internal/consensus/state_try_add_commit.go index b98941c86..33c25b94b 100644 --- a/internal/consensus/state_try_add_commit.go +++ b/internal/consensus/state_try_add_commit.go @@ -335,7 +335,7 @@ func verifyProcessedCommit(ctx context.Context, blockExec *blockExecutor, stateD // this is a non-recoverable error in current architecture panic(fmt.Errorf("ABCI client stopped, Tenderdash needs to be restarted: %w", err)) } - return fmt.Errorf("unable to process proposal: %w", err) + return err } if err := blockExec.validate(ctx, stateData); err != nil { return fmt.Errorf("+2/3 committed an invalid block: %w", err) diff --git a/internal/state/execution.go b/internal/state/execution.go index 6f90f77b9..a248b0b6d 100644 --- a/internal/state/execution.go +++ b/internal/state/execution.go @@ -17,6 +17,7 @@ import ( "github.com/dashpay/tenderdash/internal/mempool" "github.com/dashpay/tenderdash/libs/log" tmstate "github.com/dashpay/tenderdash/proto/tendermint/state" + tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" "github.com/dashpay/tenderdash/types" ) @@ -359,7 +360,9 @@ func deterministicPrepareProposalHashes(request *abci.RequestPrepareProposal, re return crypto.Checksum(reqBytes), crypto.Checksum(respBytes) } -// ProcessProposal sends the proposal to ABCI App and verifies the response +// ProcessProposal sends the proposal to ABCI App and verifies the response. +// With verify, the block is validated against state before the app sees it, +// and against the app's response afterwards; a failure returns ErrInvalidBlock. func (blockExec *BlockExecutor) ProcessProposal( ctx context.Context, block *types.Block, @@ -368,8 +371,16 @@ func (blockExec *BlockExecutor) ProcessProposal( verify bool, lastCommit types.VerifiedCommit, ) (CurrentRoundState, error) { - version := block.Version.ToProto() stages := blockExec.metrics.startStages() + // The app may fail, rather than reject, on a block that does not continue + // state, and consensus treats such a failure as fatal. + if verify { + if err := blockExec.ValidateBlock(ctx, state, block, lastCommit); err != nil { + return CurrentRoundState{}, ErrInvalidBlock{fmt.Errorf("validate block before ProcessProposal: %w", err)} + } + stages.done("process_proposal_prevalidate") + } + version := block.Version.ToProto() resp, err := blockExec.appClient.ProcessProposal(ctx, &abci.RequestProcessProposal{ Hash: block.Header.Hash(), Height: block.Header.Height, @@ -389,7 +400,7 @@ func (blockExec *BlockExecutor) ProcessProposal( QuorumHash: state.Validators.QuorumHash, }) if err != nil { - return CurrentRoundState{}, err + return CurrentRoundState{}, fmt.Errorf("ProcessProposal abci method: %w", err) } if resp.IsStatusUnknown() { return CurrentRoundState{}, fmt.Errorf("ProcessProposal responded with status %s", resp.Status.String()) @@ -447,12 +458,17 @@ func (blockExec *BlockExecutor) ValidateBlock( block *types.Block, lastCommit types.VerifiedCommit, ) error { - hash := block.Hash() - if _, ok := blockExec.cache[hash.String()]; ok { + key, err := validationCacheKey(block) + if err != nil { + return err + } + // The key does not cover state, so a hit counts only for a block that still + // continues state. + if _, ok := blockExec.cache[key]; ok && validateBlockPosition(state, block) == nil { return nil } - err := validateBlock(state, block, lastCommit, blockExec.metrics) + err = validateBlock(state, block, lastCommit, blockExec.metrics) if err != nil { return err } @@ -462,10 +478,26 @@ func (blockExec *BlockExecutor) ValidateBlock( return err } - blockExec.cache[hash.String()] = struct{}{} + blockExec.cache[key] = struct{}{} return nil } +// validationCacheKey identifies the block content ValidateBlock checks: the +// block hash, which covers the data and evidence, and in full the header, the +// last commit and the chain lock, which the hash covers only in part. +func validationCacheKey(block *types.Block) (string, error) { + hash := block.Hash() // fills the header's data, evidence and commit hashes + uncovered, err := (&tmproto.Block{ + Header: *block.Header.ToProto(), + LastCommit: block.LastCommit.ToProto(), + CoreChainLock: block.CoreChainLock.ToProto(), + }).Marshal() + if err != nil { + return "", fmt.Errorf("encode block for validation cache: %w", err) + } + return hash.String() + hex.EncodeToString(crypto.Checksum(uncovered)), nil +} + // VerifyCommit verifies commit against state.Validators as the commit for the // block blockID at height. On success it returns commit with the proof of that // verification, which the caller offers back as lastCommit when the same commit @@ -508,11 +540,11 @@ func (blockExec *BlockExecutor) ValidateBlockWithRoundState( ) } - // ValidateBlock above normally verifies block.LastCommit, but it short-circuits - // on its per-height cache, which is keyed on the block hash alone and so says - // nothing about the state the earlier validation ran against. That is why this - // second verification exists. lastCommit's proof is safe to skip on instead: - // it names the verified data itself, not a block hash. + // ValidateBlock above normally verifies block.LastCommit, but a hit on its + // cache re-checks only the block's position against state, not whether + // LastCommit verifies against it. That is why this second verification + // exists. lastCommit's proof is safe to skip on instead: it names + // the verified data itself, not a block hash. if block.Height > state.InitialHeight { if err := verifyLastCommit(state, block, lastCommit, blockExec.metrics); err != nil { return err @@ -530,7 +562,7 @@ func (blockExec *BlockExecutor) ValidateBlockWithRoundState( uncommittedState.NextConsensusParams.HashConsensusParams(), block.NextConsensusHash, ) } - return validateCoreChainLock(block, state) + return nil } // FinalizeBlock validates the block against the state, fires the relevant events, diff --git a/internal/state/execution_test.go b/internal/state/execution_test.go index c277409bf..e6e63885e 100644 --- a/internal/state/execution_test.go +++ b/internal/state/execution_test.go @@ -285,6 +285,60 @@ func TestProcessProposal(t *testing.T) { app.AssertCalled(t, "ProcessProposal", ctx, expectedRpp) } +// TestProcessProposalValidatesBlockBeforeApp checks that a block whose header +// does not follow the committed state is refused as invalid without reaching the +// application, which may fail on it rather than reject it. +func TestProcessProposalValidatesBlockBeforeApp(t *testing.T) { + state, stateDB, _ := makeState(t, 1, 1) + // the state after the genesis block, for a proposal that repeats that block + advanced := state.Copy() + advanced.LastBlockHeight = state.InitialHeight + advanced.LastBlockID = factory.MakeBlockID() + + testCases := []struct { + name string + mutate func(*types.Block) + state sm.State + // validate the block against state first, caching its hash + validateFirst bool + }{ + {name: "height", mutate: func(b *types.Block) { b.Height++ }, state: state}, + {name: "chain ID", mutate: func(b *types.Block) { b.ChainID = "other-chain" }, state: state}, + {name: "last block ID", mutate: func(b *types.Block) { b.LastBlockID = factory.MakeBlockID() }, state: state}, + {name: "genesis block at the next height", state: advanced, validateFirst: true}, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + app := abcimocks.NewApplication(t) + app.On("ProcessProposal", mock.Anything, mock.Anything). + Return(nil, errors.New("application cannot process this block")).Maybe() + logger := log.NewNopLogger() + proxyApp := proxy.New(abciclient.NewLocalClient(logger, app), logger, proxy.NopMetrics()) + require.NoError(t, proxyApp.Start(ctx)) + eventBus := eventbus.NewDefault(logger) + require.NoError(t, eventBus.Start(ctx)) + blockExec := sm.NewBlockExecutor(sm.NewStore(stateDB), proxyApp, new(mpmocks.Mempool), + sm.EmptyEvidencePool{}, store.NewBlockStore(dbm.NewMemDB()), eventBus) + + block, err := sf.MakeBlock(state, state.InitialHeight, new(types.Commit), 1) + require.NoError(t, err) + if tc.validateFirst { + require.NoError(t, blockExec.ValidateBlock(ctx, state, block, types.VerifiedCommit{})) + } + if tc.mutate != nil { + tc.mutate(block) + } + + _, err = blockExec.ProcessProposal(ctx, block, 0, tc.state, true, types.VerifiedCommit{}) + require.ErrorAs(t, err, &sm.ErrInvalidBlock{}) + app.AssertNotCalled(t, "ProcessProposal", mock.Anything, mock.Anything) + }) + } +} + func TestUpdateConsensusParams(t *testing.T) { const ( height = 1 @@ -1447,6 +1501,7 @@ const badCommitSignature = "invalid commit signatures for quorum" // commit verification flow needs. type verifiedCommitFixture struct { ctx context.Context + app *testApp blockExec *sm.BlockExecutor // skipped counts the LastCommit verifications the executor skipped skipped *metricspy.Counter @@ -1500,6 +1555,7 @@ func newVerifiedCommitFixture(t *testing.T) verifiedCommitFixture { f := verifiedCommitFixture{ ctx: ctx, + app: app, blockExec: blockExec, skipped: skipped, state: state, @@ -1804,3 +1860,79 @@ func TestAppReadyProbesTheProposalBackends(t *testing.T) { assert.NoError(t, ready(backend(true), backend(false)), "a stalled default backend does not hold back a healthy proposal backend") } + +// TestProcessProposalValidatesChainLockHeightBeforeApp checks that a block whose +// header core chain locked height does not follow the committed state, a check +// that needs nothing from the application, is refused as invalid without +// reaching the application. Consensus checks this before prevoting +// (ValidateBlockChainLock); the other callers that verify rely on +// ProcessProposal. +func TestProcessProposalValidatesChainLockHeightBeforeApp(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + state, stateDB, _ := makeState(t, 1, 1) + app := abcimocks.NewApplication(t) + app.On("ProcessProposal", mock.Anything, mock.Anything). + Return(nil, errors.New("application cannot process this block")).Maybe() + logger := log.NewNopLogger() + proxyApp := proxy.New(abciclient.NewLocalClient(logger, app), logger, proxy.NopMetrics()) + require.NoError(t, proxyApp.Start(ctx)) + eventBus := eventbus.NewDefault(logger) + require.NoError(t, eventBus.Start(ctx)) + blockExec := sm.NewBlockExecutor(sm.NewStore(stateDB), proxyApp, new(mpmocks.Mempool), + sm.EmptyEvidencePool{}, store.NewBlockStore(dbm.NewMemDB()), eventBus) + + block, err := sf.MakeBlock(state, state.InitialHeight, new(types.Commit), 1) + require.NoError(t, err) + // no chain lock in the block, yet the header moves the locked height + require.Nil(t, block.CoreChainLock) + block.CoreChainLockedHeight = state.LastCoreChainLockedBlockHeight + 1000 + + _, err = blockExec.ProcessProposal(ctx, block, 0, state, true, types.VerifiedCommit{}) + require.ErrorAs(t, err, &sm.ErrInvalidBlock{}) + app.AssertNotCalled(t, "ProcessProposal", mock.Anything, mock.Anything) +} + +// TestProcessProposalRevalidatesUncoveredPartsOfCachedBlock checks that a block +// whose header was validated before, but whose LastCommit or chain lock +// differs where the header hash does not cover it, is refused as invalid +// without reaching the application. +func TestProcessProposalRevalidatesUncoveredPartsOfCachedBlock(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*types.Block) + }{ + {"last commit quorum hash", func(b *types.Block) { b.LastCommit.QuorumHash = crypto.RandQuorumHash() }}, + {"last commit round", func(b *types.Block) { b.LastCommit.Round++ }}, + {"last commit vote extensions", func(b *types.Block) { + b.LastCommit.ThresholdVoteExtensions = append(b.LastCommit.ThresholdVoteExtensions, + &tmtypes.VoteExtension{Type: tmtypes.VoteExtensionType_THRESHOLD_RECOVER, Signature: []byte{1}}) + }}, + {"chain lock", func(b *types.Block) { + b.CoreChainLock = &types.CoreChainLock{CoreBlockHeight: b.CoreChainLockedHeight} + }}, + {"header chain locked height", func(b *types.Block) { b.CoreChainLockedHeight++ }}, + {"malformed chain lock agreeing with the header", func(b *types.Block) { + b.CoreChainLockedHeight++ + b.CoreChainLock = &types.CoreChainLock{CoreBlockHeight: b.CoreChainLockedHeight} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newVerifiedCommitFixture(t) + require.NoError(t, f.blockExec.ValidateBlock(f.ctx, f.state, f.block, types.VerifiedCommit{})) + + pb, err := f.block.ToProto() + require.NoError(t, err) + altered, err := types.BlockFromProto(pb) + require.NoError(t, err) + tc.mutate(altered) + require.Equal(t, f.block.Hash(), altered.Hash(), "the header must not change") + + calls := f.app.ProcessProposalCalls.Load() + _, err = f.blockExec.ProcessProposal(f.ctx, altered, 0, f.state, true, types.VerifiedCommit{}) + require.ErrorAs(t, err, &sm.ErrInvalidBlock{}) + assert.Equal(t, calls, f.app.ProcessProposalCalls.Load(), "the application was asked") + }) + } +} diff --git a/internal/state/helpers_test.go b/internal/state/helpers_test.go index e883011b3..fb85e70d9 100644 --- a/internal/state/helpers_test.go +++ b/internal/state/helpers_test.go @@ -3,6 +3,7 @@ package state_test import ( "context" "fmt" + "sync/atomic" "testing" "github.com/dashpay/dashd-go/btcjson" @@ -242,6 +243,8 @@ type testApp struct { ValidatorSetUpdate *abci.ValidatorSetUpdate // ProposeNextBlockImmediately is returned from FinalizeBlock. ProposeNextBlockImmediately bool + // ProcessProposalCalls counts ProcessProposal requests. + ProcessProposalCalls atomic.Int32 } var _ abci.Application = (*testApp)(nil) @@ -283,6 +286,7 @@ func (app *testApp) PrepareProposal(_ context.Context, req *abci.RequestPrepareP } func (app *testApp) ProcessProposal(_ context.Context, req *abci.RequestProcessProposal) (*abci.ResponseProcessProposal, error) { + app.ProcessProposalCalls.Add(1) resTxs := factory.ExecTxResults(types.NewTxs(req.Txs)) if resTxs == nil { return &abci.ResponseProcessProposal{Status: abci.ResponseProcessProposal_REJECT}, nil diff --git a/internal/state/validation.go b/internal/state/validation.go index a9bb3bc70..4bfad3917 100644 --- a/internal/state/validation.go +++ b/internal/state/validation.go @@ -29,37 +29,11 @@ func validateBlock(state State, block *types.Block, lastCommit types.VerifiedCom } } - // Validate basic info. - // We don't validate app version because proposer can override it on a block-by-block basis. - if block.Version.Block != state.Version.Consensus.Block { - return fmt.Errorf("wrong Block.Header.Version.Block. Expected %v, got %v", - state.Version.Consensus, - block.Version, - ) - } - if block.ChainID != state.ChainID { - return fmt.Errorf("wrong Block.Header.ChainID. Expected %v, got %v", - state.ChainID, - block.ChainID, - ) - } - if state.LastBlockHeight == 0 && block.Height != state.InitialHeight { - return fmt.Errorf("wrong Block.Header.Height. Expected %v for initial block, got %v", - block.Height, state.InitialHeight) - } - if state.LastBlockHeight > 0 && block.Height != state.LastBlockHeight+1 { - return fmt.Errorf("wrong Block.Header.Height. Expected %v, got %v", - state.LastBlockHeight+1, - block.Height, - ) + if err := validateBlockPosition(state, block); err != nil { + return err } - - // Validate prev block info. - if !block.LastBlockID.Equals(state.LastBlockID) { - return fmt.Errorf("wrong Block.Header.LastBlockID. Expected %v, got %v", - state.LastBlockID, - block.LastBlockID, - ) + if err := validateCoreChainLock(block, state); err != nil { + return err } // Validate app info @@ -133,6 +107,44 @@ func validateBlock(state State, block *types.Block, lastCommit types.VerifiedCom return nil } +// validateBlockPosition checks that block's header continues the chain where +// state ends: same chain and block version, next height, and last block ID. +func validateBlockPosition(state State, block *types.Block) error { + // Validate basic info. + // We don't validate app version because proposer can override it on a block-by-block basis. + if block.Version.Block != state.Version.Consensus.Block { + return fmt.Errorf("wrong Block.Header.Version.Block. Expected %v, got %v", + state.Version.Consensus, + block.Version, + ) + } + if block.ChainID != state.ChainID { + return fmt.Errorf("wrong Block.Header.ChainID. Expected %v, got %v", + state.ChainID, + block.ChainID, + ) + } + if state.LastBlockHeight == 0 && block.Height != state.InitialHeight { + return fmt.Errorf("wrong Block.Header.Height. Expected %v for initial block, got %v", + state.InitialHeight, block.Height) + } + if state.LastBlockHeight > 0 && block.Height != state.LastBlockHeight+1 { + return fmt.Errorf("wrong Block.Header.Height. Expected %v, got %v", + state.LastBlockHeight+1, + block.Height, + ) + } + + // Validate prev block info. + if !block.LastBlockID.Equals(state.LastBlockID) { + return fmt.Errorf("wrong Block.Header.LastBlockID. Expected %v, got %v", + state.LastBlockID, + block.LastBlockID, + ) + } + return nil +} + // verifyLastCommit verifies block.LastCommit against state.LastValidators as // the commit for state.LastBlockID. lastCommit's proof spares that BLS // threshold verification only when it covers exactly this verification — same diff --git a/spec/abci/methods.md b/spec/abci/methods.md index 880c016ac..b65c51c30 100644 --- a/spec/abci/methods.md +++ b/spec/abci/methods.md @@ -406,8 +406,11 @@ validation and reuse rules are in `ensureProcess` processes the held block for the round being checked unless the same block/round result is already current. It runs before the prevote lock rules, -so a locked validator can process another block and still prevote nil. A -well-formed application rejection or block/header mismatch produces a nil prevote; +so a locked validator can process another block and still prevote nil. A block +that does not validate against Tenderdash's committed state (for example a wrong +height, chain ID or last block ID) is never sent to the application and produces +a nil prevote, as do a well-formed application rejection and a block/header +mismatch with the application's answer; unknown status, malformed results and transport errors are fatal on this path. See the exact [prevote decision](../consensus/consensus-transitions.md#prevote-decision-prevoterdo). diff --git a/spec/consensus/consensus-alternative-flows.md b/spec/consensus/consensus-alternative-flows.md index 53408e7a7..7c89e902d 100644 --- a/spec/consensus/consensus-alternative-flows.md +++ b/spec/consensus/consensus-alternative-flows.md @@ -102,12 +102,14 @@ undecodable block → error (`addProposalBlockPart`). Parts with invalid Merkle proofs are charged to the sender's `blockPartProofBudget` and dropped silently once it is exhausted (`internal/consensus/block_part_budget.go`). -### Application rejects the block, or the header does not match - -`ProcessProposal` returning `REJECT` (`sm.ErrBlockRejected`), or a -well-formed `ACCEPT` whose contents disagree with the block — header -mismatch / invalid `LastCommit` / invalid chain lock (`sm.ErrInvalidBlock`) -— → nil prevote (`prevoter.handleError`). A failed `ProcessProposal` clears +### Block is invalid, application rejects it, or the header does not match + +`BlockExecutor.ProcessProposal` first validates the block against the +committed state (`ValidateBlock`); a block that fails is never sent to the +application. That failure (`sm.ErrInvalidBlock`), the application returning +`REJECT` (`sm.ErrBlockRejected`), or a well-formed `ACCEPT` whose contents +disagree with the block — header mismatch / invalid `LastCommit` +(`sm.ErrInvalidBlock`) — → nil prevote (`prevoter.handleError`). A failed `ProcessProposal` clears `CurrentRoundState`, so no round counts as processed until a later call succeeds (`blockExecutor.ensureProcess`). Any **other** error **panics** the node: transport failure, `UNKNOWN` status, or a malformed response — an @@ -289,9 +291,15 @@ and `addVoteErrorMw` (`internal/consensus/state_add_vote.go`). - Timeliness is checked **only at prevote time**, against the proposal's receive time (`ProposalReceiveTime` = enqueue time of the `ProposalMessage`, `chanMsgSender.send`), with the window of - `checkTimely` (rule P-Timely). It is skipped in replay mode and for a - block that is the node's `ValidBlock`, so re-proposals of an already - polka'd block are not re-judged. + `checkTimely` (rule P-Timely). It is skipped for a block that is the + node's `ValidBlock`, so re-proposals of an already polka'd block are not + re-judged. In replay mode the receive time is the time of the proposal's + WAL entry, and only a proposal received too early fails. The entry is never + earlier than the receipt, so replay refuses only what the node refused when + it received the proposal, not always all of it. That time also becomes + `ValidBlockRecvTime` for a block made valid during replay, which decides + whether the node re-proposes it after a restart + (`Proposaler.checkValidBlock`), and feeds `ProposalTimestampDifference`. - Timeliness governs re-proposal (`Proposaler.checkValidBlock`). - Metric `ProposalTimestampDifference{is_timely}` is recorded for proposals with `POLRound = -1` (`Proposaler.proposalTimestampDifferenceMetric`). @@ -345,7 +353,7 @@ counts from `StartTime`. Other rounds `≥ 1` never wait. On - Header fields: `CoreChainLockedHeight`; optional `CoreChainLock` in the block (and `CoreChainLockUpdate` in the proposal, not signed). - `ValidateBlockChainLock` (prevote) and `validateCoreChainLock` - (`ValidateBlockWithRoundState`): with a lock, the header height equals the + (`validateBlock`, before the application call): with a lock, the header height equals the lock's and exceeds `state.LastCoreChainLockedBlockHeight`; without one, it equals it. - `Proposaler.Set` rejects `proposal.CoreChainLockedHeight` below the diff --git a/spec/consensus/consensus-happy-path.md b/spec/consensus/consensus-happy-path.md index 6290ef32d..0a1c9ae15 100644 --- a/spec/consensus/consensus-happy-path.md +++ b/spec/consensus/consensus-happy-path.md @@ -32,6 +32,7 @@ sequenceDiagram P-->>V: ProposalMessage, BlockPartMessages (gossip) V->>V: Proposaler.Set (verify proposer signature), AddProposalBlockPart V->>V: ProposalCompleted → EnterPrevote(h,0) + V->>V: ValidateBlock against committed state V->>A: ProcessProposal(block, round=0) A-->>V: ACCEPT + AppHash/TxResults/... V->>V: ValidateBlockWithRoundState, sign PREVOTE(B) @@ -202,26 +203,30 @@ POL round's prevotes already have a majority) → `EnterPrevote(h, Round)`. → `prevoter.Do` (`internal/consensus/state_prevoter.go`): 1. `StateData.isValidForPrevote`: block and proposal held; proposal timestamp - equals block time; the proposal is timely (unless replaying or the block is - already `ValidBlock`); the core chain lock is consistent + equals block time; the proposal is timely (unless the block is already + `ValidBlock`; in replay, only a proposal received too early fails); the + core chain lock is consistent (`sm.ValidateBlockChainLock`); for `POLRound == -1` the block's `ProposerProTxHash` is this round's proposer. -2. `blockExecutor.ensureProcess(rs, Round)` → ABCI `ProcessProposal` - (`verify = true`): the response must be `ACCEPT`, well-formed, and +2. `blockExecutor.ensureProcess(rs, Round)` → `ValidateBlock` against the + committed state, core chain lock included, then ABCI `ProcessProposal` + (`verify = true`); a block + failing `ValidateBlock` never reaches the application. The response must be + `ACCEPT`, well-formed, and `ValidateBlockWithRoundState` must pass — header fields `AppHash`, `ResultsHash`, `NextValidatorsHash`, `NextConsensusHash` must equal the - application's answer, `LastCommit` must verify against `LastValidators`, - and the core chain lock must validate. The result becomes + application's answer, and `LastCommit` must verify against + `LastValidators`. The result becomes `CurrentRoundState` tagged with round 0. 3. `blockExecutor.mustValidate` (panics on failure, cannot fail after step 2). 4. `prevoter.signVote`: with no lock and `POLRound == -1`, `checkProposalBlock` is true → `voteSigner.signAddVote(PrevoteType, BlockID)`. Step `Prevote` is set on return. A failure in step 1, or `ErrBlockRejected` -(application `REJECT`) or `ErrInvalidBlock` (`ValidateBlockWithRoundState`) in -step 2, yields a **nil prevote** instead. Any other step-2 error — transport -failure, `UNKNOWN` status or a malformed response — **panics** the node -(`prevoter.handleError`). **⇢** [Prevote rule](./consensus-transitions.md#prevote-decision-prevoterdo). +(application `REJECT`) or `ErrInvalidBlock` (`ValidateBlock` before the call, +`ValidateBlockWithRoundState` after it) in step 2, yields a **nil prevote** +instead. Any other step-2 error — transport failure, `UNKNOWN` status or a +malformed response — **panics** the node (`prevoter.handleError`). **⇢** [Prevote rule](./consensus-transitions.md#prevote-decision-prevoterdo). ### 6. Collecting prevotes diff --git a/spec/consensus/consensus-transitions.md b/spec/consensus/consensus-transitions.md index ba7971c43..2b04ec008 100644 --- a/spec/consensus/consensus-transitions.md +++ b/spec/consensus/consensus-transitions.md @@ -349,16 +349,18 @@ flowchart TD B -- no --> NIL B -- yes --> C{Proposal.Timestamp == block.Time?} C -- no --> NIL - C -- yes --> D{replay mode or block == ValidBlock?} + C -- yes --> D{block == ValidBlock?} D -- yes --> F - D -- no --> E{timely? InitialHeight: timestamp == genesis time; else receive time within window for round r} + D -- no --> E{timely? InitialHeight: timestamp == genesis time; else receive time within window for round r, in replay only too early fails} E -- no --> NIL E -- yes --> F{ValidateBlockChainLock ok?} F -- no --> NIL F -- yes --> G{POLRound == -1 implies block.ProposerProTxHash == proposer h,r?} G -- no --> NIL - G -- yes --> H[ensureProcess: ABCI ProcessProposal for round r unless already processed] - H --> I{ACCEPT and header matches app result?} + G -- yes --> H[ensureProcess for round r unless already processed] + H --> V{ValidateBlock against committed state ok?} + V -- no --> NIL + V -- yes --> I{ABCI ProcessProposal: ACCEPT and header matches app result?} I -- REJECT or mismatch --> NIL I -- other error --> PANIC[panic] I -- yes --> J[mustValidate] @@ -379,12 +381,14 @@ Rules, in evaluation order: (`errPrevoteProposalBlockNil`, `errPrevoteProposalNil`). This is the path taken on `timeoutPropose`. No ABCI call is made. 2. **P-Timestamp.** `Proposal.Timestamp ≠ ProposalBlock.Time` → nil. -3. **P-Timely.** Skipped in replay mode and when the block is already - `ValidBlock`. At `InitialHeight` the timestamp must equal the genesis time; +3. **P-Timely.** Skipped when the block is already `ValidBlock`. At + `InitialHeight` the timestamp must equal the genesis time; otherwise `Proposal.CheckTimely(ProposalReceiveTime, SynchronyParams, r)` must be 0 (`checkTimely` in `types/validation.go`): `ts − Precision ≤ recv ≤ ts + MessageDelay·2^⌊r/10⌋ + Precision` - (the shift is capped to avoid overflow). + (the shift is capped to avoid overflow). In replay mode `recv` is the time + of the proposal's WAL entry, which can lag its receipt, so only + `recv < ts − Precision` (too early) fails. 4. **P-ChainLock.** `sm.ValidateBlockChainLock(state, block)` (`internal/state/validation.go`): with a `CoreChainLock` in the block, `Header.CoreChainLockedHeight == CoreChainLock.CoreBlockHeight` and @@ -395,10 +399,13 @@ Rules, in evaluation order: (`POLRound ≥ 0`) keep the original header's proposer and are exempt. 6. **P-Process.** `blockExecutor.ensureProcess(rs, r)`: ABCI `ProcessProposal` for round `r` unless `CurrentRoundState` already is the - result for this block and round. `ErrBlockRejected` (application - `REJECT`) or `ErrInvalidBlock` (a well-formed `ACCEPT` whose contents - disagree with the block: header fields differ from the application's - answer, `LastCommit` invalid, chain lock invalid — + result for this block and round. The block is first validated against the + committed state (`BlockExecutor.ValidateBlock`: version, chain ID, height, + `LastBlockID`, core chain lock, `LastCommit`, proposer, evidence, …); a block that fails is + never sent to the application and yields `ErrInvalidBlock`. + `ErrBlockRejected` (application `REJECT`) or `ErrInvalidBlock` (also a + well-formed `ACCEPT` whose contents disagree with the block: header fields + differ from the application's answer, `LastCommit` invalid — `ValidateBlockWithRoundState`) → nil prevote; any other error → **panic** (`prevoter.handleError`). A malformed application response is in the panic class, not the nil-prevote class: `UNKNOWN` status, an @@ -491,8 +498,8 @@ that checks each is given. | Proposal `BlockID` hash and part-set header equal the assembled block's | `AddProposalBlockPartAction.addProposalBlockPart` | | Parts' Merkle proofs valid; total size `≤ Block.MaxBytes` | `PartSet.AddPart`, `addProposalBlockPart` | | `Proposal.Timestamp == Header.Time` | `isValidForPrevote`, `EnterPrecommitAction.Execute` | -| Timely (PBTS window), unless `ValidBlock` or replay | `StateData.proposalIsTimely` | -| Core chain lock consistent with state | `sm.ValidateBlockChainLock` | +| Timely (PBTS window; in replay, not too early), unless `ValidBlock` | `StateData.proposalIsTimely` | +| Core chain lock consistent with state | `sm.ValidateBlockChainLock`, `validateBlock` | | `Header.ProposerProTxHash == proposer(h, r)` when `POLRound == -1` | `isValidForPrevote` | | `Block.ValidateBasic`; version, chain id, height, `LastBlockID`, `ConsensusHash`, `ValidatorsHash`, `LastCommit` threshold signature, proposer is a validator, `Time > LastBlockTime` (`≥ genesis` at initial height), evidence size | `validateBlock` in `internal/state/validation.go` | | ABCI `ProcessProposal` = `ACCEPT`; `AppHash`, `ResultsHash`, `NextValidatorsHash`, `NextConsensusHash` equal the application's; app hash size | `BlockExecutor.ProcessProposal`, `ValidateBlockWithRoundState` |