diff --git a/cmd/tenderdash/commands/rollback_test.go b/cmd/tenderdash/commands/rollback_test.go index e05f51589..86ad2c96c 100644 --- a/cmd/tenderdash/commands/rollback_test.go +++ b/cmd/tenderdash/commands/rollback_test.go @@ -3,6 +3,7 @@ package commands_test import ( "context" "os" + "strings" "testing" "time" @@ -21,11 +22,21 @@ func TestRollbackIntegration(t *testing.T) { t.Skip("skipping test in short mode") } - var height int64 + t.Run("keep blocks", func(t *testing.T) { testRollbackIntegration(t, false) }) + // With the block store rolled back too, the restarted node finds the app, the state and the store at the + // same height, so the handshake compares the app's app hash with the rolled back state's. + t.Run("roll back blocks", func(t *testing.T) { testRollbackIntegration(t, true) }) +} + +func testRollbackIntegration(t *testing.T, rollbackStore bool) { + var ( + height int64 + appHash []byte + ) dir := t.TempDir() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cfg, err := rpctest.CreateConfig(t, t.Name()) + cfg, err := rpctest.CreateConfig(t, strings.ReplaceAll(t.Name(), "/", "_")) require.NoError(t, err) cfg.BaseConfig.DBBackend = "goleveldb" @@ -72,7 +83,7 @@ func TestRollbackIntegration(t *testing.T) { t.Run("Rollback", func(t *testing.T) { time.Sleep(time.Second) require.NoError(t, app.Rollback()) - height, _, err = commands.RollbackState(cfg, false) + height, appHash, err = commands.RollbackState(cfg, rollbackStore) require.NoError(t, err, "%d", height) }) t.Run("Restart", func(t *testing.T) { @@ -89,6 +100,16 @@ func TestRollbackIntegration(t *testing.T) { client, err := local.New(logger, node2.(local.NodeService)) require.NoError(t, err) + if rollbackStore { + // The handshake passed. The only validator signed the next heights before the rollback and refuses + // to sign them again, so check where the node starts instead of waiting for blocks. + status, err := client.Status(ctx) + require.NoError(t, err) + require.Equal(t, height, status.SyncInfo.LatestBlockHeight) + require.EqualValues(t, appHash, status.SyncInfo.LatestAppHash) + return + } + ticker := time.NewTicker(200 * time.Millisecond) for { select { diff --git a/internal/state/rollback.go b/internal/state/rollback.go index 569077862..dee8cb20f 100644 --- a/internal/state/rollback.go +++ b/internal/state/rollback.go @@ -42,11 +42,6 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { if rollbackBlock == nil { return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight) } - // we also need to retrieve the latest block because the app hash and last results hash is only agreed upon in the following block - latestBlock := bs.LoadBlockMeta(invalidState.LastBlockHeight) - if latestBlock == nil { - return -1, nil, fmt.Errorf("block at height %d not found", invalidState.LastBlockHeight) - } previousLastValidatorSet, err := ss.LoadValidators(rollbackHeight, bs) if err != nil { @@ -97,8 +92,9 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { ConsensusParams: previousParams, LastHeightConsensusParamsChanged: paramsChangeHeight, - LastResultsHash: latestBlock.Header.ResultsHash, - LastAppHash: latestBlock.Header.AppHash, + // Execution is same-block: the header carries the app and results hashes after its own block. + LastResultsHash: rollbackBlock.Header.ResultsHash, + LastAppHash: rollbackBlock.Header.AppHash, } // persist the new state. This overrides the invalid one. NOTE: this will also diff --git a/internal/state/rollback_test.go b/internal/state/rollback_test.go index 1681a6002..5f8f52b68 100644 --- a/internal/state/rollback_test.go +++ b/internal/state/rollback_test.go @@ -34,6 +34,7 @@ func TestRollback(t *testing.T) { nextState.Version.Consensus.App = 11 nextState.LastBlockID = factory.MakeBlockID() nextState.LastAppHash = factory.RandomHash() + nextState.LastResultsHash = factory.RandomHash() nextState.LastValidators = initialState.Validators nextState.ConsensusParams = *newParams nextState.LastHeightConsensusParamsChanged = nextHeight + 1 @@ -42,11 +43,12 @@ func TestRollback(t *testing.T) { // update the state require.NoError(t, stateStore.Save(nextState)) + // With same-block execution, each header carries the app and results hashes after its own block. block := &types.BlockMeta{ BlockID: initialState.LastBlockID, Header: types.Header{ Height: initialState.LastBlockHeight, - AppHash: factory.RandomHash(), + AppHash: initialState.LastAppHash, LastBlockID: factory.MakeBlockID(), ResultsHash: initialState.LastResultsHash, ProposerProTxHash: initialState.Validators.Validators[0].ProTxHash, @@ -55,10 +57,10 @@ func TestRollback(t *testing.T) { }, } nextBlock := &types.BlockMeta{ - BlockID: initialState.LastBlockID, + BlockID: nextState.LastBlockID, Header: types.Header{ Height: nextState.LastBlockHeight, - AppHash: initialState.LastAppHash, + AppHash: nextState.LastAppHash, LastBlockID: block.BlockID, ResultsHash: nextState.LastResultsHash, ProposerProTxHash: nextState.Validators.Validators[1].ProTxHash, @@ -66,7 +68,8 @@ func TestRollback(t *testing.T) { }, } blockStore.On("LoadBlockMeta", height).Return(block) - blockStore.On("LoadBlockMeta", nextHeight).Return(nextBlock) + // The rolled back state needs nothing from the block being rolled back. + blockStore.On("LoadBlockMeta", nextHeight).Return(nextBlock).Maybe() blockStore.On("Height").Return(nextHeight) blockStore.On("Base").Return(height) diff --git a/internal/statesync/reactor_test.go b/internal/statesync/reactor_test.go index 303c95fbf..5002fae68 100644 --- a/internal/statesync/reactor_test.go +++ b/internal/statesync/reactor_test.go @@ -2,7 +2,10 @@ package statesync import ( "context" + "encoding/json" "fmt" + "net/http" + "net/http/httptest" "strconv" "strings" "testing" @@ -32,9 +35,13 @@ import ( "github.com/dashpay/tenderdash/internal/store" "github.com/dashpay/tenderdash/internal/test/factory" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/light" "github.com/dashpay/tenderdash/light/provider" + lightdb "github.com/dashpay/tenderdash/light/store/db" ssproto "github.com/dashpay/tenderdash/proto/tendermint/statesync" tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" + "github.com/dashpay/tenderdash/rpc/coretypes" + rpctypes "github.com/dashpay/tenderdash/rpc/jsonrpc/types" "github.com/dashpay/tenderdash/types" "github.com/dashpay/tenderdash/version" ) @@ -262,7 +269,12 @@ func testReactorSync(t *testing.T, before, after int64) { require.NoError(t, pool.Start(local)) defer pool.Close() hookCalled := false + var chain map[int64]*types.LightBlock rts.reactor.postSyncHook = func(ctx context.Context, restored sm.State) error { + // The restored state is at the snapshot height, so it carries that block's hashes. The handshake after + // a restart compares LastAppHash with the app hash the application reports. + require.Equal(t, chain[snapshotHeight].AppHash, restored.LastAppHash) + require.Equal(t, chain[snapshotHeight].ResultsHash, restored.LastResultsHash) pool.Update(ctx, restored, nil) require.Equal(t, after, pool.State().ConsensusParams.Evidence.MaxBytes) require.Equal(t, restored.LastBlockHeight, pool.State().LastBlockHeight) @@ -270,7 +282,7 @@ func testReactorSync(t *testing.T, before, after int64) { return nil } - chain := buildLightBlockChain(ctx, t, 1, 10, time.Now(), rts.privVal) + chain = buildLightBlockChain(ctx, t, 1, 10, time.Now(), rts.privVal) registerMockCoreQuorums(t, rts.dashcoreClient, chain) // app accepts any snapshot rts.conn. @@ -288,7 +300,7 @@ func testReactorSync(t *testing.T, before, after int64) { Return(&abci.ResponseInfo{ AppVersion: testAppVersion, LastBlockHeight: snapshotHeight, - LastBlockAppHash: chain[snapshotHeight+1].AppHash, + LastBlockAppHash: chain[snapshotHeight].AppHash, }, nil) // store accepts state and validator sets @@ -700,6 +712,8 @@ func TestReactor_StateProviderP2P(t *testing.T) { state, err := rts.reactor.stateProvider.State(actx, 5) require.NoError(t, err) require.Equal(t, appHash, state.LastAppHash) + require.Equal(t, chain[5].AppHash, state.LastAppHash) + require.Equal(t, chain[5].ResultsHash, state.LastResultsHash) require.Equal(t, types.DefaultConsensusParams(), &state.ConsensusParams) commit, err := rts.reactor.stateProvider.Commit(actx, 5) @@ -713,6 +727,70 @@ func TestReactor_StateProviderP2P(t *testing.T) { require.True(t, added) } +// TestReactor_StateProviderRPC checks the state the RPC state provider restores. Its light client gets +// light blocks from peers, and a JSON-RPC server answers the consensus params request. +func TestReactor_StateProviderRPC(t *testing.T) { + const snapshotHeight = 5 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rts := setup(ctx, t, nil, nil, nil, 2) + closeCh := make(chan struct{}) + defer close(closeCh) + + chain := buildLightBlockChain(ctx, t, 1, 10, time.Now(), rts.privVal) + registerMockCoreQuorums(t, rts.dashcoreClient, chain) + go handleLightBlockRequests(ctx, t, chain, rts.blockOutCh, rts.blockInCh, closeCh, 0, 0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ( + req rpctypes.RPCRequest + args coretypes.RequestConsensusParams + ) + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Method != "consensus_params" || + json.Unmarshal(req.Params, &args) != nil || args.Height == nil { + http.Error(w, "unexpected request", http.StatusBadRequest) + return + } + resp := req.MakeResponse(&coretypes.ResultConsensusParams{ + BlockHeight: int64(*args.Height), + ConsensusParams: *types.DefaultConsensusParams(), + }) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + + providers := []provider.Provider{ + NewBlockProvider(types.NodeID(strings.Repeat("a", 2*types.NodeIDByteLength)), factory.DefaultTestChainID, rts.reactor.dispatcher), + NewBlockProvider(types.NodeID(strings.Repeat("b", 2*types.NodeIDByteLength)), factory.DefaultTestChainID, rts.reactor.dispatcher), + } + lc, err := light.NewClient(ctx, factory.DefaultTestChainID, providers[0], providers[1:], + lightdb.New(dbm.NewMemDB()), rts.dashcoreClient, light.Logger(log.NewNopLogger())) + require.NoError(t, err) + sp := &stateProviderRPC{ + lc: lc, + initialHeight: 1, + providers: map[provider.Provider]string{providers[0]: server.URL, providers[1]: server.URL}, + logger: log.NewNopLogger(), + dashCoreClient: rts.dashcoreClient, + } + + actx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + appHash, err := sp.AppHash(actx, snapshotHeight) + require.NoError(t, err) + require.Equal(t, chain[snapshotHeight].AppHash, appHash) + + state, err := sp.State(actx, snapshotHeight) + require.NoError(t, err) + require.EqualValues(t, snapshotHeight, state.LastBlockHeight) + require.Equal(t, appHash, state.LastAppHash) + require.Equal(t, chain[snapshotHeight].ResultsHash, state.LastResultsHash) + require.Equal(t, types.DefaultConsensusParams(), &state.ConsensusParams) +} + func TestReactor_Backfill(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") @@ -975,11 +1053,11 @@ func mockLB(ctx context.Context, t *testing.T, height int64, time time.Time, las currentVals *types.ValidatorSet, currentPrivVals []types.PrivValidator, ) (*types.ValidatorSet, []types.PrivValidator, *types.LightBlock) { t.Helper() + // MakeHeader gives each height random app and results hashes, so a hash taken from the wrong height shows. header := factory.MakeHeader(t, &types.Header{ Height: height, LastBlockID: lastBlockID, Time: time, - AppHash: make([]byte, crypto.DefaultHashSize), ProposerProTxHash: currentVals.Proposer().ProTxHash, }) header.Version.App = testAppVersion diff --git a/internal/statesync/stateprovider.go b/internal/statesync/stateprovider.go index 7de84607d..e75310055 100644 --- a/internal/statesync/stateprovider.go +++ b/internal/statesync/stateprovider.go @@ -120,9 +120,8 @@ func (s *stateProviderRPC) verifyLightBlockAtHeight(ctx context.Context, height return s.lc.VerifyLightBlockAtHeight(ctx, int64(height), ts) } -// AppHash implements part of StateProvider. It calls the application to verify the -// light blocks at heights h+1 and h+2 and, if verification succeeds, reports the app -// hash for the block at height h+1 which correlates to the state at height h. +// AppHash implements part of StateProvider. It verifies the light block at height h and, +// if verification succeeds, reports its app hash, the app state after block h. func (s *stateProviderRPC) AppHash(ctx context.Context, height uint64) (tmbytes.HexBytes, error) { s.Lock() defer s.Unlock() @@ -182,8 +181,9 @@ func (s *stateProviderRPC) State(ctx context.Context, height uint64) (sm.State, state.LastBlockTime = lastLightBlock.Time state.LastBlockID = lastLightBlock.Commit.BlockID state.LastCoreChainLockedBlockHeight = lastLightBlock.Header.CoreChainLockedHeight - state.LastAppHash = currentLightBlock.AppHash - state.LastResultsHash = currentLightBlock.ResultsHash + // Execution is same-block: header H carries the app and results hashes after block H. + state.LastAppHash = lastLightBlock.AppHash + state.LastResultsHash = lastLightBlock.ResultsHash state.LastValidators, err = authenticateStateSyncValidatorSet(lastLightBlock, s.dashCoreClient) if err != nil { return sm.State{}, fmt.Errorf("authenticating last validator set: %w", err) @@ -339,8 +339,9 @@ func (s *stateProviderP2P) State(ctx context.Context, height uint64) (sm.State, state.LastBlockTime = lastLightBlock.Time state.LastBlockID = lastLightBlock.Commit.BlockID state.LastCoreChainLockedBlockHeight = lastLightBlock.Header.CoreChainLockedHeight - state.LastAppHash = currentLightBlock.AppHash - state.LastResultsHash = currentLightBlock.ResultsHash + // Execution is same-block: header H carries the app and results hashes after block H. + state.LastAppHash = lastLightBlock.AppHash + state.LastResultsHash = lastLightBlock.ResultsHash state.LastValidators, err = authenticateStateSyncValidatorSet(lastLightBlock, s.dashCoreClient) if err != nil { return sm.State{}, fmt.Errorf("authenticating last validator set: %w", err)