From a08b6a3974aab90fa1670467f75a3b1968a0596e Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Wed, 4 Nov 2026 02:33:04 +0700 Subject: [PATCH 1/2] fix(statesync): restore LastAppHash of the snapshot height, not the next one State sync and Rollback took LastAppHash and LastResultsHash from the header one height above the state they build. That comes from Tendermint's next-block execution, where header H+1 carried the results of block H. Tenderdash executes same-block: header H already carries them, and header H+1 holds the app state after block H+1. Nothing reads LastAppHash before the next block commits, except the handshake after a restart at exactly the restored height, which then fails with "state.AppHash does not match AppHash after replay". Both state providers and Rollback now take the hashes from the header at the state's height. Rollback no longer loads the block it rolls back. The mock light blocks in the statesync tests get a random app hash per height, so the tests see a hash taken from the wrong height. A new test covers the RPC state provider's State. Refs dashpay/platform#4682 Co-Authored-By: Claude Opus 5.5 --- internal/state/rollback.go | 10 ++-- internal/state/rollback_test.go | 11 ++-- internal/statesync/reactor_test.go | 84 +++++++++++++++++++++++++++-- internal/statesync/stateprovider.go | 15 +++--- 4 files changed, 99 insertions(+), 21 deletions(-) diff --git a/internal/state/rollback.go b/internal/state/rollback.go index 5690778629..dee8cb20f2 100644 --- a/internal/state/rollback.go +++ b/internal/state/rollback.go @@ -42,11 +42,6 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { if rollbackBlock == nil { return -1, nil, fmt.Errorf("block at height %d not found", rollbackHeight) } - // we also need to retrieve the latest block because the app hash and last results hash is only agreed upon in the following block - latestBlock := bs.LoadBlockMeta(invalidState.LastBlockHeight) - if latestBlock == nil { - return -1, nil, fmt.Errorf("block at height %d not found", invalidState.LastBlockHeight) - } previousLastValidatorSet, err := ss.LoadValidators(rollbackHeight, bs) if err != nil { @@ -97,8 +92,9 @@ func Rollback(bs BlockStore, ss Store) (int64, []byte, error) { ConsensusParams: previousParams, LastHeightConsensusParamsChanged: paramsChangeHeight, - LastResultsHash: latestBlock.Header.ResultsHash, - LastAppHash: latestBlock.Header.AppHash, + // Execution is same-block: the header carries the app and results hashes after its own block. + LastResultsHash: rollbackBlock.Header.ResultsHash, + LastAppHash: rollbackBlock.Header.AppHash, } // persist the new state. This overrides the invalid one. NOTE: this will also diff --git a/internal/state/rollback_test.go b/internal/state/rollback_test.go index 1681a6002f..5f8f52b687 100644 --- a/internal/state/rollback_test.go +++ b/internal/state/rollback_test.go @@ -34,6 +34,7 @@ func TestRollback(t *testing.T) { nextState.Version.Consensus.App = 11 nextState.LastBlockID = factory.MakeBlockID() nextState.LastAppHash = factory.RandomHash() + nextState.LastResultsHash = factory.RandomHash() nextState.LastValidators = initialState.Validators nextState.ConsensusParams = *newParams nextState.LastHeightConsensusParamsChanged = nextHeight + 1 @@ -42,11 +43,12 @@ func TestRollback(t *testing.T) { // update the state require.NoError(t, stateStore.Save(nextState)) + // With same-block execution, each header carries the app and results hashes after its own block. block := &types.BlockMeta{ BlockID: initialState.LastBlockID, Header: types.Header{ Height: initialState.LastBlockHeight, - AppHash: factory.RandomHash(), + AppHash: initialState.LastAppHash, LastBlockID: factory.MakeBlockID(), ResultsHash: initialState.LastResultsHash, ProposerProTxHash: initialState.Validators.Validators[0].ProTxHash, @@ -55,10 +57,10 @@ func TestRollback(t *testing.T) { }, } nextBlock := &types.BlockMeta{ - BlockID: initialState.LastBlockID, + BlockID: nextState.LastBlockID, Header: types.Header{ Height: nextState.LastBlockHeight, - AppHash: initialState.LastAppHash, + AppHash: nextState.LastAppHash, LastBlockID: block.BlockID, ResultsHash: nextState.LastResultsHash, ProposerProTxHash: nextState.Validators.Validators[1].ProTxHash, @@ -66,7 +68,8 @@ func TestRollback(t *testing.T) { }, } blockStore.On("LoadBlockMeta", height).Return(block) - blockStore.On("LoadBlockMeta", nextHeight).Return(nextBlock) + // The rolled back state needs nothing from the block being rolled back. + blockStore.On("LoadBlockMeta", nextHeight).Return(nextBlock).Maybe() blockStore.On("Height").Return(nextHeight) blockStore.On("Base").Return(height) diff --git a/internal/statesync/reactor_test.go b/internal/statesync/reactor_test.go index 303c95fbf5..5002fae685 100644 --- a/internal/statesync/reactor_test.go +++ b/internal/statesync/reactor_test.go @@ -2,7 +2,10 @@ package statesync import ( "context" + "encoding/json" "fmt" + "net/http" + "net/http/httptest" "strconv" "strings" "testing" @@ -32,9 +35,13 @@ import ( "github.com/dashpay/tenderdash/internal/store" "github.com/dashpay/tenderdash/internal/test/factory" "github.com/dashpay/tenderdash/libs/log" + "github.com/dashpay/tenderdash/light" "github.com/dashpay/tenderdash/light/provider" + lightdb "github.com/dashpay/tenderdash/light/store/db" ssproto "github.com/dashpay/tenderdash/proto/tendermint/statesync" tmproto "github.com/dashpay/tenderdash/proto/tendermint/types" + "github.com/dashpay/tenderdash/rpc/coretypes" + rpctypes "github.com/dashpay/tenderdash/rpc/jsonrpc/types" "github.com/dashpay/tenderdash/types" "github.com/dashpay/tenderdash/version" ) @@ -262,7 +269,12 @@ func testReactorSync(t *testing.T, before, after int64) { require.NoError(t, pool.Start(local)) defer pool.Close() hookCalled := false + var chain map[int64]*types.LightBlock rts.reactor.postSyncHook = func(ctx context.Context, restored sm.State) error { + // The restored state is at the snapshot height, so it carries that block's hashes. The handshake after + // a restart compares LastAppHash with the app hash the application reports. + require.Equal(t, chain[snapshotHeight].AppHash, restored.LastAppHash) + require.Equal(t, chain[snapshotHeight].ResultsHash, restored.LastResultsHash) pool.Update(ctx, restored, nil) require.Equal(t, after, pool.State().ConsensusParams.Evidence.MaxBytes) require.Equal(t, restored.LastBlockHeight, pool.State().LastBlockHeight) @@ -270,7 +282,7 @@ func testReactorSync(t *testing.T, before, after int64) { return nil } - chain := buildLightBlockChain(ctx, t, 1, 10, time.Now(), rts.privVal) + chain = buildLightBlockChain(ctx, t, 1, 10, time.Now(), rts.privVal) registerMockCoreQuorums(t, rts.dashcoreClient, chain) // app accepts any snapshot rts.conn. @@ -288,7 +300,7 @@ func testReactorSync(t *testing.T, before, after int64) { Return(&abci.ResponseInfo{ AppVersion: testAppVersion, LastBlockHeight: snapshotHeight, - LastBlockAppHash: chain[snapshotHeight+1].AppHash, + LastBlockAppHash: chain[snapshotHeight].AppHash, }, nil) // store accepts state and validator sets @@ -700,6 +712,8 @@ func TestReactor_StateProviderP2P(t *testing.T) { state, err := rts.reactor.stateProvider.State(actx, 5) require.NoError(t, err) require.Equal(t, appHash, state.LastAppHash) + require.Equal(t, chain[5].AppHash, state.LastAppHash) + require.Equal(t, chain[5].ResultsHash, state.LastResultsHash) require.Equal(t, types.DefaultConsensusParams(), &state.ConsensusParams) commit, err := rts.reactor.stateProvider.Commit(actx, 5) @@ -713,6 +727,70 @@ func TestReactor_StateProviderP2P(t *testing.T) { require.True(t, added) } +// TestReactor_StateProviderRPC checks the state the RPC state provider restores. Its light client gets +// light blocks from peers, and a JSON-RPC server answers the consensus params request. +func TestReactor_StateProviderRPC(t *testing.T) { + const snapshotHeight = 5 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rts := setup(ctx, t, nil, nil, nil, 2) + closeCh := make(chan struct{}) + defer close(closeCh) + + chain := buildLightBlockChain(ctx, t, 1, 10, time.Now(), rts.privVal) + registerMockCoreQuorums(t, rts.dashcoreClient, chain) + go handleLightBlockRequests(ctx, t, chain, rts.blockOutCh, rts.blockInCh, closeCh, 0, 0) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var ( + req rpctypes.RPCRequest + args coretypes.RequestConsensusParams + ) + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Method != "consensus_params" || + json.Unmarshal(req.Params, &args) != nil || args.Height == nil { + http.Error(w, "unexpected request", http.StatusBadRequest) + return + } + resp := req.MakeResponse(&coretypes.ResultConsensusParams{ + BlockHeight: int64(*args.Height), + ConsensusParams: *types.DefaultConsensusParams(), + }) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + })) + defer server.Close() + + providers := []provider.Provider{ + NewBlockProvider(types.NodeID(strings.Repeat("a", 2*types.NodeIDByteLength)), factory.DefaultTestChainID, rts.reactor.dispatcher), + NewBlockProvider(types.NodeID(strings.Repeat("b", 2*types.NodeIDByteLength)), factory.DefaultTestChainID, rts.reactor.dispatcher), + } + lc, err := light.NewClient(ctx, factory.DefaultTestChainID, providers[0], providers[1:], + lightdb.New(dbm.NewMemDB()), rts.dashcoreClient, light.Logger(log.NewNopLogger())) + require.NoError(t, err) + sp := &stateProviderRPC{ + lc: lc, + initialHeight: 1, + providers: map[provider.Provider]string{providers[0]: server.URL, providers[1]: server.URL}, + logger: log.NewNopLogger(), + dashCoreClient: rts.dashcoreClient, + } + + actx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + appHash, err := sp.AppHash(actx, snapshotHeight) + require.NoError(t, err) + require.Equal(t, chain[snapshotHeight].AppHash, appHash) + + state, err := sp.State(actx, snapshotHeight) + require.NoError(t, err) + require.EqualValues(t, snapshotHeight, state.LastBlockHeight) + require.Equal(t, appHash, state.LastAppHash) + require.Equal(t, chain[snapshotHeight].ResultsHash, state.LastResultsHash) + require.Equal(t, types.DefaultConsensusParams(), &state.ConsensusParams) +} + func TestReactor_Backfill(t *testing.T) { if testing.Short() { t.Skip("skipping test in short mode") @@ -975,11 +1053,11 @@ func mockLB(ctx context.Context, t *testing.T, height int64, time time.Time, las currentVals *types.ValidatorSet, currentPrivVals []types.PrivValidator, ) (*types.ValidatorSet, []types.PrivValidator, *types.LightBlock) { t.Helper() + // MakeHeader gives each height random app and results hashes, so a hash taken from the wrong height shows. header := factory.MakeHeader(t, &types.Header{ Height: height, LastBlockID: lastBlockID, Time: time, - AppHash: make([]byte, crypto.DefaultHashSize), ProposerProTxHash: currentVals.Proposer().ProTxHash, }) header.Version.App = testAppVersion diff --git a/internal/statesync/stateprovider.go b/internal/statesync/stateprovider.go index 7de84607d1..e753100552 100644 --- a/internal/statesync/stateprovider.go +++ b/internal/statesync/stateprovider.go @@ -120,9 +120,8 @@ func (s *stateProviderRPC) verifyLightBlockAtHeight(ctx context.Context, height return s.lc.VerifyLightBlockAtHeight(ctx, int64(height), ts) } -// AppHash implements part of StateProvider. It calls the application to verify the -// light blocks at heights h+1 and h+2 and, if verification succeeds, reports the app -// hash for the block at height h+1 which correlates to the state at height h. +// AppHash implements part of StateProvider. It verifies the light block at height h and, +// if verification succeeds, reports its app hash, the app state after block h. func (s *stateProviderRPC) AppHash(ctx context.Context, height uint64) (tmbytes.HexBytes, error) { s.Lock() defer s.Unlock() @@ -182,8 +181,9 @@ func (s *stateProviderRPC) State(ctx context.Context, height uint64) (sm.State, state.LastBlockTime = lastLightBlock.Time state.LastBlockID = lastLightBlock.Commit.BlockID state.LastCoreChainLockedBlockHeight = lastLightBlock.Header.CoreChainLockedHeight - state.LastAppHash = currentLightBlock.AppHash - state.LastResultsHash = currentLightBlock.ResultsHash + // Execution is same-block: header H carries the app and results hashes after block H. + state.LastAppHash = lastLightBlock.AppHash + state.LastResultsHash = lastLightBlock.ResultsHash state.LastValidators, err = authenticateStateSyncValidatorSet(lastLightBlock, s.dashCoreClient) if err != nil { return sm.State{}, fmt.Errorf("authenticating last validator set: %w", err) @@ -339,8 +339,9 @@ func (s *stateProviderP2P) State(ctx context.Context, height uint64) (sm.State, state.LastBlockTime = lastLightBlock.Time state.LastBlockID = lastLightBlock.Commit.BlockID state.LastCoreChainLockedBlockHeight = lastLightBlock.Header.CoreChainLockedHeight - state.LastAppHash = currentLightBlock.AppHash - state.LastResultsHash = currentLightBlock.ResultsHash + // Execution is same-block: header H carries the app and results hashes after block H. + state.LastAppHash = lastLightBlock.AppHash + state.LastResultsHash = lastLightBlock.ResultsHash state.LastValidators, err = authenticateStateSyncValidatorSet(lastLightBlock, s.dashCoreClient) if err != nil { return sm.State{}, fmt.Errorf("authenticating last validator set: %w", err) From 23c1443e4077c7c7dfedf00f9446c762e669660f Mon Sep 17 00:00:00 2001 From: QuantumExplorer Date: Wed, 4 Nov 2026 02:38:28 +0700 Subject: [PATCH 2/2] test(rollback): restart after rolling back the block store too With `tenderdash rollback --store`, the restarted node finds the app, the state and the block store at the same height, so the handshake compares the app's app hash with the rolled back state's LastAppHash. Before the previous commit it stopped with "state.AppHash does not match AppHash after replay". The only validator already signed the next heights and refuses to sign them again, so this case checks the height and app hash the node starts at instead of waiting for new blocks. Refs dashpay/platform#4682 Co-Authored-By: Claude Opus 5.5 --- cmd/tenderdash/commands/rollback_test.go | 27 +++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/cmd/tenderdash/commands/rollback_test.go b/cmd/tenderdash/commands/rollback_test.go index e05f515895..86ad2c96cb 100644 --- a/cmd/tenderdash/commands/rollback_test.go +++ b/cmd/tenderdash/commands/rollback_test.go @@ -3,6 +3,7 @@ package commands_test import ( "context" "os" + "strings" "testing" "time" @@ -21,11 +22,21 @@ func TestRollbackIntegration(t *testing.T) { t.Skip("skipping test in short mode") } - var height int64 + t.Run("keep blocks", func(t *testing.T) { testRollbackIntegration(t, false) }) + // With the block store rolled back too, the restarted node finds the app, the state and the store at the + // same height, so the handshake compares the app's app hash with the rolled back state's. + t.Run("roll back blocks", func(t *testing.T) { testRollbackIntegration(t, true) }) +} + +func testRollbackIntegration(t *testing.T, rollbackStore bool) { + var ( + height int64 + appHash []byte + ) dir := t.TempDir() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cfg, err := rpctest.CreateConfig(t, t.Name()) + cfg, err := rpctest.CreateConfig(t, strings.ReplaceAll(t.Name(), "/", "_")) require.NoError(t, err) cfg.BaseConfig.DBBackend = "goleveldb" @@ -72,7 +83,7 @@ func TestRollbackIntegration(t *testing.T) { t.Run("Rollback", func(t *testing.T) { time.Sleep(time.Second) require.NoError(t, app.Rollback()) - height, _, err = commands.RollbackState(cfg, false) + height, appHash, err = commands.RollbackState(cfg, rollbackStore) require.NoError(t, err, "%d", height) }) t.Run("Restart", func(t *testing.T) { @@ -89,6 +100,16 @@ func TestRollbackIntegration(t *testing.T) { client, err := local.New(logger, node2.(local.NodeService)) require.NoError(t, err) + if rollbackStore { + // The handshake passed. The only validator signed the next heights before the rollback and refuses + // to sign them again, so check where the node starts instead of waiting for blocks. + status, err := client.Status(ctx) + require.NoError(t, err) + require.Equal(t, height, status.SyncInfo.LatestBlockHeight) + require.EqualValues(t, appHash, status.SyncInfo.LatestAppHash) + return + } + ticker := time.NewTicker(200 * time.Millisecond) for { select {