Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 25 additions & 8 deletions pkg/logpoller/log_poller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -677,8 +677,9 @@ func TestLogPoller_BlockTimestamps(t *testing.T) {
func TestLogPoller_SynchronizedWithGeth(t *testing.T) {
t.Parallel()
testCases := []struct {
name string
skipEmptyBlocks bool
name string
skipEmptyBlocks bool
batchUnfinalizedBlocks bool
}{
{
name: "with empty blocks",
Expand All @@ -688,6 +689,16 @@ func TestLogPoller_SynchronizedWithGeth(t *testing.T) {
name: "without empty blocks",
skipEmptyBlocks: true,
},
{
name: "with empty blocks and batched unfinalized walk",
skipEmptyBlocks: false,
batchUnfinalizedBlocks: true,
},
{
name: "without empty blocks and batched unfinalized walk",
skipEmptyBlocks: true,
batchUnfinalizedBlocks: true,
},
}

for _, tc := range testCases {
Expand All @@ -696,6 +707,10 @@ func TestLogPoller_SynchronizedWithGeth(t *testing.T) {
// with the canonical chain of geth's despite arbitrary mixes of mining and reorgs.
testParams := gopter.DefaultTestParameters()
testParams.MinSuccessfulTests = 100
if tc.batchUnfinalizedBlocks {
// The batch path doubles the property matrix; keep CI time bounded.
testParams.MinSuccessfulTests = 25
}
p := gopter.NewProperties(testParams)
numChainInserts := 3
finalityDepth := 5
Expand Down Expand Up @@ -723,12 +738,14 @@ func TestLogPoller_SynchronizedWithGeth(t *testing.T) {
require.NoError(t, err)

lpOpts := logpoller.Opts{
PollPeriod: 15 * time.Second,
FinalityDepth: int64(finalityDepth),
BackfillBatchSize: 3,
RPCBatchSize: 2,
KeepFinalizedBlocksDepth: 1000,
SkipEmptyBlocks: tc.skipEmptyBlocks,
PollPeriod: 15 * time.Second,
FinalityDepth: int64(finalityDepth),
BackfillBatchSize: 3,
RPCBatchSize: 2,
KeepFinalizedBlocksDepth: 1000,
SkipEmptyBlocks: tc.skipEmptyBlocks,
BatchUnfinalizedBlocks: tc.batchUnfinalizedBlocks,
BatchUnfinalizedBlocksChunkSize: 3,
Comment on lines +747 to +748
}
simulatedClient := client.NewSimulatedBackendClient(t, backend, chainID)
ht := headstest.NewSimulatedHeadTracker(simulatedClient, lpOpts.UseFinalityTag, lpOpts.FinalityDepth)
Expand Down
15 changes: 15 additions & 0 deletions pkg/logpoller/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,21 @@ var (
Name: "evm_log_poller_last_processed_block",
Help: "The last block that the log poller has processed. Main purpose is to signal if the log poller is stuck and not processing new blocks. May be reported with a delay.",
}, []string{"chainFamily", "chainID"})

promLpUnfinalizedBatchChunks = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "evm_log_poller_unfinalized_batch_chunks_total",
Help: "The number of opt-in batched unfinalized-block chunks processed by the log poller, by outcome.",
}, []string{"chainID", "outcome"})

promLpUnfinalizedBatchChunkDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "evm_log_poller_unfinalized_batch_duration_seconds",
Help: "The duration of an opt-in batched unfinalized-block chunk, including header fetch and log queries.",
}, []string{"chainID"})

promLpUnfinalizedBatchHeaders = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "evm_log_poller_unfinalized_batch_headers_total",
Help: "The number of unfinalized headers walked by the opt-in batched log poller path.",
}, []string{"chainID"})
)

type Metrics interface {
Expand Down
30 changes: 27 additions & 3 deletions pkg/logpoller/unfinalized_batch.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"fmt"
"math/big"
"sync"
"time"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
Expand All @@ -16,6 +17,13 @@ import (
evmtypes "github.com/smartcontractkit/chainlink-evm/pkg/types"
)

func (lp *logPoller) recordUnfinalizedBatchChunk(outcome string, headers int, start time.Time) {
chainID := lp.ec.ConfiguredChainID().String()
promLpUnfinalizedBatchChunks.WithLabelValues(chainID, outcome).Inc()
promLpUnfinalizedBatchChunkDuration.WithLabelValues(chainID).Observe(time.Since(start).Seconds())
promLpUnfinalizedBatchHeaders.WithLabelValues(chainID).Add(float64(headers))
}

func (lp *logPoller) unfinalizedBlock(head *evmtypes.Head, safe, finalized int64) Block {
return Block{
BlockHash: head.Hash,
Expand Down Expand Up @@ -46,7 +54,7 @@ func (lp *logPoller) batchFetchHeaders(ctx context.Context, from, to int64) ([]*
return nil, nil
}

reqs := make([]rpc.BatchElem, 0, int(to-from+1))
reqs := make([]rpc.BatchElem, 0, int(to-from+1)) //nolint:gosec // G115: bounded by unfinalizedBatchChunkSize (<=50)
for n := from; n <= to; n++ {
reqs = append(reqs, newBlockReq(hexutil.EncodeBig(big.NewInt(n))))
}
Expand All @@ -57,7 +65,7 @@ func (lp *logPoller) batchFetchHeaders(ctx context.Context, from, to int64) ([]*

headers := make([]*evmtypes.Head, 0, len(reqs))
for i, req := range reqs {
expected := from + int64(i)
expected := from + int64(i) //nolint:gosec // G115: batch index is bounded by the chunk size
if req.Error != nil {
return headers, fmt.Errorf("failed to fetch unfinalized header %d: %w", expected, req.Error)
}
Expand Down Expand Up @@ -99,7 +107,7 @@ func (lp *logPoller) batchFilterLogsByHash(ctx context.Context, headers []*evmty
ctx, cancel := context.WithCancel(parentCtx)
defer cancel()

concurrency := int(lp.unfinalizedBatchLogConcurrency)
concurrency := int(lp.unfinalizedBatchLogConcurrency) //nolint:gosec // G115: bounded by the number of headers in the chunk
if concurrency < 1 {
concurrency = 1
}
Expand Down Expand Up @@ -207,8 +215,14 @@ func (lp *logPoller) getUnfinalizedLogsBatched(ctx context.Context, currentBlock
}
to := min(latest, currentBlock.Number+chunkSize)

chunkStart := time.Now()
headers, headerErr := lp.batchFetchHeaders(ctx, currentBlock.Number+1, to)
if len(headers) == 0 {
outcome := "partial"
if headerErr != nil {
outcome = "error"
}
lp.recordUnfinalizedBatchChunk(outcome, 0, chunkStart)
return blocks, logs, headerErr
}

Expand All @@ -218,15 +232,18 @@ func (lp *logPoller) getUnfinalizedLogsBatched(ctx context.Context, currentBlock
prev := currentBlock
for _, header := range headers {
if header.ParentHash != prev.Hash {
lp.recordUnfinalizedBatchChunk("reorg", 0, chunkStart)
return blocks, logs, newReorgError(header)
}
if isReplay {
dbBlock, dbErr := lp.orm.SelectBlockByNumber(ctx, header.Number)
if dbErr != nil && !errors.Is(dbErr, sql.ErrNoRows) {
lp.lggr.Warnw("Unable to get next block from DB during replay, retrying on next poll", "err", dbErr, "block", header.Number)
lp.recordUnfinalizedBatchChunk("error", 0, chunkStart)
return blocks, logs, fmt.Errorf("failed to get next block from DB during replay: %w", dbErr)
}
if dbBlock != nil && header.Hash != dbBlock.BlockHash {
lp.recordUnfinalizedBatchChunk("reorg", 0, chunkStart)
return blocks, logs, newReorgError(header)
}
}
Expand All @@ -242,15 +259,22 @@ func (lp *logPoller) getUnfinalizedLogsBatched(ctx context.Context, currentBlock
walked++
}
if logErr != nil {
lp.recordUnfinalizedBatchChunk("error", completed, chunkStart)
return blocks, logs, logErr
}

currentBlock = headers[len(headers)-1]
if currentBlock.Number < to {
// The endpoint was behind or a header failed. Persist the prefix and
// retry the missing height on the next poll.
outcome := "partial"
if headerErr != nil {
outcome = "error"
}
lp.recordUnfinalizedBatchChunk(outcome, completed, chunkStart)
return blocks, logs, headerErr
}
lp.recordUnfinalizedBatchChunk("success", len(headers), chunkStart)
}

return blocks, logs, nil
Expand Down
Loading