From 4d21752bb107d14a1e0e7db174b0aadc51f264c0 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 5 Feb 2026 18:08:47 +0100 Subject: [PATCH 1/3] claude replay fix --- block/internal/common/replay.go | 72 ++++++----- block/internal/common/replay_test.go | 181 ++++++++++++++++++--------- 2 files changed, 167 insertions(+), 86 deletions(-) diff --git a/block/internal/common/replay.go b/block/internal/common/replay.go index ac3b9fea7..e6b6266db 100644 --- a/block/internal/common/replay.go +++ b/block/internal/common/replay.go @@ -150,27 +150,26 @@ func (s *Replayer) replayBlock(ctx context.Context, height uint64) error { // Get the previous state var prevState types.State if height == s.genesis.InitialHeight { - // For the first block, use genesis state + // For the first block, use genesis state. + // The header.AppHash contains the previous state's app hash (i.e., the genesis app hash). + // This is what ExecuteTxs needs as input. prevState = types.State{ ChainID: s.genesis.ChainID, InitialHeight: s.genesis.InitialHeight, LastBlockHeight: s.genesis.InitialHeight - 1, LastBlockTime: s.genesis.StartTime, - AppHash: header.AppHash, // This will be updated by InitChain + AppHash: header.AppHash, // Genesis app hash (input to first block execution) } } else { - // Get previous state from store + // Get previous state from store. + // GetStateAtHeight(height-1) returns the state AFTER block height-1 was executed, + // which contains the correct AppHash to use as input for executing block at 'height'. prevState, err = s.store.GetStateAtHeight(ctx, height-1) if err != nil { return fmt.Errorf("failed to get previous state: %w", err) } - // We need the state at height-1, so load that block's app hash - prevHeader, _, err := s.store.GetBlockData(ctx, height-1) - if err != nil { - return fmt.Errorf("failed to get previous block header: %w", err) - } - prevState.AppHash = prevHeader.AppHash - prevState.LastBlockHeight = height - 1 + // Note: prevState.AppHash is already correct - it's the result of executing block height-1, + // which is what we need as input for executing block at 'height'. } // Prepare transactions @@ -190,27 +189,40 @@ func (s *Replayer) replayBlock(ctx context.Context, height uint64) error { return fmt.Errorf("failed to execute transactions: %w", err) } - // DEBUG: Log comparison of expected vs actual app hash - s.logger.Debug(). - Uint64("height", height). - Str("expected_app_hash", hex.EncodeToString(header.AppHash)). - Str("actual_app_hash", hex.EncodeToString(newAppHash)). - Bool("hashes_match", bytes.Equal(newAppHash, header.AppHash)). - Msg("replayBlock: ExecuteTxs completed") - - // Verify the app hash matches - if !bytes.Equal(newAppHash, header.AppHash) { - err := fmt.Errorf("app hash mismatch: expected %s got %s", - hex.EncodeToString(header.AppHash), - hex.EncodeToString(newAppHash), - ) - s.logger.Error(). - Str("expected", hex.EncodeToString(header.AppHash)). - Str("got", hex.EncodeToString(newAppHash)). + // The result of ExecuteTxs (newAppHash) should match the stored state at this height. + // Note: header.AppHash is the PREVIOUS state's app hash (input), not the expected output. + // The expected output is stored in state[height].AppHash or equivalently in header[height+1].AppHash. + + // For verification, we need to get the expected app hash from the stored state at this height. + // If this state doesn't exist (which would be unusual since we fetched the block), we skip verification. + expectedState, err := s.store.GetStateAtHeight(ctx, height) + if err == nil { + // State exists, verify the app hash matches + if !bytes.Equal(newAppHash, expectedState.AppHash) { + err := fmt.Errorf("app hash mismatch at height %d: expected %s got %s", + height, + hex.EncodeToString(expectedState.AppHash), + hex.EncodeToString(newAppHash), + ) + s.logger.Error(). + Str("expected", hex.EncodeToString(expectedState.AppHash)). + Str("got", hex.EncodeToString(newAppHash)). + Uint64("height", height). + Err(err). + Msg("app hash mismatch during replay") + return err + } + s.logger.Debug(). + Uint64("height", height). + Str("app_hash", hex.EncodeToString(newAppHash)). + Msg("replayBlock: app hash verified against stored state") + } else { + // State doesn't exist yet - this is expected during replay. + // We trust the execution result since we're replaying validated blocks. + s.logger.Debug(). Uint64("height", height). - Err(err). - Msg("app hash mismatch during replay") - return err + Str("app_hash", hex.EncodeToString(newAppHash)). + Msg("replayBlock: ExecuteTxs completed (no stored state to verify against)") } // Calculate new state diff --git a/block/internal/common/replay_test.go b/block/internal/common/replay_test.go index d55247bf7..f7398af9c 100644 --- a/block/internal/common/replay_test.go +++ b/block/internal/common/replay_test.go @@ -2,6 +2,7 @@ package common import ( "context" + "errors" "testing" "time" @@ -37,6 +38,7 @@ func TestReplayer_SyncToHeight_ExecutorBehind(t *testing.T) { now := uint64(time.Now().UnixNano()) // Setup store to return block data for height 100 + // header.AppHash is the PREVIOUS state's app hash (input to execution) mockStore.EXPECT().GetBlockData(mock.Anything, uint64(100)).Return( &types.SignedHeader{ Header: types.Header{ @@ -45,7 +47,7 @@ func TestReplayer_SyncToHeight_ExecutorBehind(t *testing.T) { Time: now, ChainID: "test-chain", }, - AppHash: []byte("app-hash-100"), + AppHash: []byte("app-hash-99"), // This is the input app hash (from state 99) }, }, &types.Data{ @@ -54,37 +56,33 @@ func TestReplayer_SyncToHeight_ExecutorBehind(t *testing.T) { nil, ) - // Setup store to return previous block for state - mockStore.EXPECT().GetBlockData(mock.Anything, uint64(99)).Return( - &types.SignedHeader{ - Header: types.Header{ - BaseHeader: types.BaseHeader{ - Height: 99, - Time: now - 1000000000, - ChainID: "test-chain", - }, - AppHash: []byte("app-hash-99"), - }, - }, - &types.Data{}, - nil, - ) - - // Setup state at height 99 + // Setup state at height 99 - this provides the input app hash for ExecuteTxs mockStore.EXPECT().GetStateAtHeight(mock.Anything, uint64(99)).Return( types.State{ ChainID: gen.ChainID, InitialHeight: gen.InitialHeight, LastBlockHeight: 99, - AppHash: []byte("app-hash-99"), + AppHash: []byte("app-hash-99"), // Result of executing block 99 }, nil, ) - // Expect ExecuteTxs to be called for height 100 + // Expect ExecuteTxs to be called with the previous state's app hash + // and return the new app hash (result of executing block 100) mockExec.On("ExecuteTxs", mock.Anything, mock.Anything, uint64(100), mock.Anything, []byte("app-hash-99")). Return([]byte("app-hash-100"), nil) + // Setup state at height 100 for verification (optional - may not exist during replay) + mockStore.EXPECT().GetStateAtHeight(mock.Anything, uint64(100)).Return( + types.State{ + ChainID: gen.ChainID, + InitialHeight: gen.InitialHeight, + LastBlockHeight: 100, + AppHash: []byte("app-hash-100"), // Expected result + }, + nil, + ) + // Setup batch for state persistence mockBatch := mocks.NewMockBatch(t) mockStore.EXPECT().NewBatch(mock.Anything).Return(mockBatch, nil) @@ -227,7 +225,7 @@ func TestReplayer_SyncToHeight_MultipleBlocks(t *testing.T) { now := uint64(time.Now().UnixNano()) - // First, the sync checks that the target block exists in the store (line 100 in replay.go) + // First, the sync checks that the target block exists in the store mockStore.EXPECT().GetBlockData(mock.Anything, targetHeight).Return( &types.SignedHeader{ Header: types.Header{ @@ -236,7 +234,7 @@ func TestReplayer_SyncToHeight_MultipleBlocks(t *testing.T) { Time: now + (100 * 1000000000), ChainID: "test-chain", }, - AppHash: []byte("app-hash-100"), + AppHash: []byte("app-hash-99"), // Input app hash }, }, &types.Data{ @@ -258,7 +256,7 @@ func TestReplayer_SyncToHeight_MultipleBlocks(t *testing.T) { Time: now + (height * 1000000000), ChainID: "test-chain", }, - AppHash: []byte("app-hash-" + string(rune('0'+height))), + AppHash: []byte("app-hash-" + string(rune('0'+prevHeight))), // Input = previous state's result }, }, &types.Data{ @@ -267,23 +265,7 @@ func TestReplayer_SyncToHeight_MultipleBlocks(t *testing.T) { nil, ).Once() - // Previous block data (for getting previous app hash) - mockStore.EXPECT().GetBlockData(mock.Anything, prevHeight).Return( - &types.SignedHeader{ - Header: types.Header{ - BaseHeader: types.BaseHeader{ - Height: prevHeight, - Time: now + (prevHeight * 1000000000), - ChainID: "test-chain", - }, - AppHash: []byte("app-hash-" + string(rune('0'+prevHeight))), - }, - }, - &types.Data{}, - nil, - ).Once() - - // State at previous height + // State at previous height (provides input app hash) mockStore.EXPECT().GetStateAtHeight(mock.Anything, prevHeight).Return( types.State{ ChainID: gen.ChainID, @@ -295,9 +277,21 @@ func TestReplayer_SyncToHeight_MultipleBlocks(t *testing.T) { ).Once() // ExecuteTxs for current block - mockExec.On("ExecuteTxs", mock.Anything, mock.Anything, height, mock.Anything, mock.Anything). + mockExec.On("ExecuteTxs", mock.Anything, mock.Anything, height, mock.Anything, + []byte("app-hash-"+string(rune('0'+prevHeight)))). Return([]byte("app-hash-"+string(rune('0'+height))), nil).Once() + // State at current height for verification (may or may not exist) + mockStore.EXPECT().GetStateAtHeight(mock.Anything, height).Return( + types.State{ + ChainID: gen.ChainID, + InitialHeight: gen.InitialHeight, + LastBlockHeight: height, + AppHash: []byte("app-hash-" + string(rune('0'+height))), + }, + nil, + ).Once() + // Setup batch for state persistence mockBatch := mocks.NewMockBatch(t) mockStore.EXPECT().NewBatch(mock.Anything).Return(mockBatch, nil).Once() @@ -332,7 +326,10 @@ func TestReplayer_ReplayBlock_FirstBlock(t *testing.T) { now := uint64(time.Now().UnixNano()) + mockExec.On("GetLatestHeight", mock.Anything).Return(uint64(0), nil) + // Setup store to return first block (at initial height) + // For genesis block, header.AppHash is the genesis app hash mockStore.EXPECT().GetBlockData(mock.Anything, uint64(1)).Return( &types.SignedHeader{ Header: types.Header{ @@ -341,7 +338,7 @@ func TestReplayer_ReplayBlock_FirstBlock(t *testing.T) { Time: now, ChainID: "test-chain", }, - AppHash: []byte("app-hash-1"), + AppHash: []byte("genesis-app-hash"), // Genesis app hash (input) }, }, &types.Data{ @@ -350,12 +347,15 @@ func TestReplayer_ReplayBlock_FirstBlock(t *testing.T) { nil, ) - // For first block, ExecuteTxs should be called with genesis app hash - mockExec.On("ExecuteTxs", mock.Anything, mock.Anything, uint64(1), mock.Anything, []byte("app-hash-1")). + // For first block, ExecuteTxs should be called with genesis app hash (from header.AppHash) + mockExec.On("ExecuteTxs", mock.Anything, mock.Anything, uint64(1), mock.Anything, []byte("genesis-app-hash")). Return([]byte("app-hash-1"), nil) - // Call replayBlock directly (this is a private method, so we test it through SyncToHeight) - mockExec.On("GetLatestHeight", mock.Anything).Return(uint64(0), nil) + // State at height 1 for verification (may not exist during fresh replay) + mockStore.EXPECT().GetStateAtHeight(mock.Anything, uint64(1)).Return( + types.State{}, + errors.New("not found"), + ) // Setup batch for state persistence mockBatch := mocks.NewMockBatch(t) @@ -399,7 +399,7 @@ func TestReplayer_AppHashMismatch(t *testing.T) { Time: now, ChainID: "test-chain", }, - AppHash: []byte("expected-app-hash"), + AppHash: []byte("app-hash-99"), // Input app hash }, }, &types.Data{ @@ -408,18 +408,77 @@ func TestReplayer_AppHashMismatch(t *testing.T) { nil, ) - mockStore.EXPECT().GetBlockData(mock.Anything, uint64(99)).Return( + mockStore.EXPECT().GetStateAtHeight(mock.Anything, uint64(99)).Return( + types.State{ + ChainID: gen.ChainID, + InitialHeight: gen.InitialHeight, + LastBlockHeight: 99, + AppHash: []byte("app-hash-99"), + }, + nil, + ) + + // ExecuteTxs returns a different app hash than expected + mockExec.On("ExecuteTxs", mock.Anything, mock.Anything, uint64(100), mock.Anything, []byte("app-hash-99")). + Return([]byte("different-app-hash"), nil) + + // State at height 100 exists and has the expected app hash + mockStore.EXPECT().GetStateAtHeight(mock.Anything, uint64(100)).Return( + types.State{ + ChainID: gen.ChainID, + InitialHeight: gen.InitialHeight, + LastBlockHeight: 100, + AppHash: []byte("expected-app-hash-100"), // Expected result doesn't match + }, + nil, + ) + + // Should fail with mismatch error + err := syncer.SyncToHeight(ctx, targetHeight) + require.Error(t, err) + require.Contains(t, err.Error(), "app hash mismatch") + + mockExec.AssertExpectations(t) + mockStore.AssertExpectations(t) +} + +func TestReplayer_AppHashMismatch_NoStoredState(t *testing.T) { + // When there's no stored state at the target height, we can't verify + // and should proceed without error (trust the execution) + ctx := context.Background() + mockExec := mocks.NewMockHeightAwareExecutor(t) + mockStore := mocks.NewMockStore(t) + logger := zerolog.Nop() + + gen := genesis.Genesis{ + ChainID: "test-chain", + InitialHeight: 1, + StartTime: time.Now().UTC(), + } + + syncer := NewReplayer(mockStore, mockExec, gen, logger) + + targetHeight := uint64(100) + execHeight := uint64(99) + + mockExec.On("GetLatestHeight", mock.Anything).Return(execHeight, nil) + + now := uint64(time.Now().UnixNano()) + + mockStore.EXPECT().GetBlockData(mock.Anything, uint64(100)).Return( &types.SignedHeader{ Header: types.Header{ BaseHeader: types.BaseHeader{ - Height: 99, - Time: now - 1000000000, + Height: 100, + Time: now, ChainID: "test-chain", }, AppHash: []byte("app-hash-99"), }, }, - &types.Data{}, + &types.Data{ + Txs: []types.Tx{[]byte("tx1")}, + }, nil, ) @@ -433,14 +492,24 @@ func TestReplayer_AppHashMismatch(t *testing.T) { nil, ) - // ExecuteTxs returns a different app hash than expected mockExec.On("ExecuteTxs", mock.Anything, mock.Anything, uint64(100), mock.Anything, []byte("app-hash-99")). - Return([]byte("different-app-hash"), nil) + Return([]byte("app-hash-100"), nil) - // Should fail with mismatch error + // State at height 100 doesn't exist - skip verification + mockStore.EXPECT().GetStateAtHeight(mock.Anything, uint64(100)).Return( + types.State{}, + errors.New("not found"), + ) + + // Setup batch for state persistence + mockBatch := mocks.NewMockBatch(t) + mockStore.EXPECT().NewBatch(mock.Anything).Return(mockBatch, nil) + mockBatch.EXPECT().UpdateState(mock.Anything).Return(nil) + mockBatch.EXPECT().Commit().Return(nil) + + // Should succeed even without verification err := syncer.SyncToHeight(ctx, targetHeight) - require.Error(t, err) - require.Contains(t, err.Error(), "app hash mismatch") + require.NoError(t, err) mockExec.AssertExpectations(t) mockStore.AssertExpectations(t) From c44210b2406f39fc6c0ef45347f03066ea8ea691 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 5 Feb 2026 20:34:24 +0100 Subject: [PATCH 2/3] reduce comments --- block/internal/common/replay.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/block/internal/common/replay.go b/block/internal/common/replay.go index e6b6266db..ba13a5a4b 100644 --- a/block/internal/common/replay.go +++ b/block/internal/common/replay.go @@ -151,8 +151,6 @@ func (s *Replayer) replayBlock(ctx context.Context, height uint64) error { var prevState types.State if height == s.genesis.InitialHeight { // For the first block, use genesis state. - // The header.AppHash contains the previous state's app hash (i.e., the genesis app hash). - // This is what ExecuteTxs needs as input. prevState = types.State{ ChainID: s.genesis.ChainID, InitialHeight: s.genesis.InitialHeight, @@ -161,15 +159,12 @@ func (s *Replayer) replayBlock(ctx context.Context, height uint64) error { AppHash: header.AppHash, // Genesis app hash (input to first block execution) } } else { - // Get previous state from store. // GetStateAtHeight(height-1) returns the state AFTER block height-1 was executed, // which contains the correct AppHash to use as input for executing block at 'height'. prevState, err = s.store.GetStateAtHeight(ctx, height-1) if err != nil { return fmt.Errorf("failed to get previous state: %w", err) } - // Note: prevState.AppHash is already correct - it's the result of executing block height-1, - // which is what we need as input for executing block at 'height'. } // Prepare transactions @@ -217,8 +212,7 @@ func (s *Replayer) replayBlock(ctx context.Context, height uint64) error { Str("app_hash", hex.EncodeToString(newAppHash)). Msg("replayBlock: app hash verified against stored state") } else { - // State doesn't exist yet - this is expected during replay. - // We trust the execution result since we're replaying validated blocks. + // State doesn't exist yet, we trust the execution result since we're replaying validated blocks. s.logger.Debug(). Uint64("height", height). Str("app_hash", hex.EncodeToString(newAppHash)). From e6895fc33f27564d62788ec8f50e3a033c1447a7 Mon Sep 17 00:00:00 2001 From: Julien Robert Date: Thu, 5 Feb 2026 20:36:04 +0100 Subject: [PATCH 3/3] prep rc.3 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b762086a2..fbbd62911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## v1.0.0-rc.3 + ### Added - Add DA Hints for P2P transactions. This allows a catching up node to be on sync with both DA and P2P. ([#2891](https://github.com/evstack/ev-node/pull/2891)) @@ -18,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Improve `cache.NumPendingData` to not return empty data. Automatically bumps `LastSubmittedHeight` to reflect that. ([#3046](https://github.com/evstack/ev-node/pull/3046)) - **BREAKING** Make pending events cache and tx cache fully ephemeral. Those will be re-fetched on restart. DA Inclusion cache persists until cleared up after DA inclusion has been processed. Persist accross restart using store metadata. ([#3047](https://github.com/evstack/ev-node/pull/3047)) - Replace LRU cache by standard mem cache with manual eviction in `store_adapter`. When P2P blocks were fetched too fast, they would be evicted before being executed [#3051](https://github.com/evstack/ev-node/pull/3051) +- Fix replay logic leading to app hashes by verifying against the wrong block [#3053](https://github.com/evstack/ev-node/pull/3053). ## v1.0.0-rc.2